diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml new file mode 100644 index 0000000..e15b307 --- /dev/null +++ b/.github/workflows/develop.yml @@ -0,0 +1,32 @@ +name: Develop CI + +on: + push: + branches: [develop] + pull_request: + branches: [develop] + +jobs: + fast-tests: + name: Lint & Fast Tests + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pandas numpy scipy pyarrow PyYAML omegaconf antlr4-python3-runtime datasets pytest ruff + + - name: Run linter + run: ruff check . + + - name: Run fast-tier tests + run: PYTHONPATH=. pytest tests/ diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..cb964ed --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,32 @@ +name: Main CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + full-tests: + name: Lint & Full Test Suite + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pandas numpy scipy pyarrow PyYAML omegaconf antlr4-python3-runtime datasets pytest ruff + + - name: Run linter + run: ruff check . + + - name: Run full test suite + run: PYTHONPATH=. pytest tests/ diff --git a/config/training_vast_qlora_grpo.yaml b/config/training_vast_qlora_grpo.yaml index c0520b3..82cdd1e 100644 --- a/config/training_vast_qlora_grpo.yaml +++ b/config/training_vast_qlora_grpo.yaml @@ -4,9 +4,9 @@ model: adapter: qlora grpo: - script: factor_lab.training.train_grpo_qlora + script: src.training.train_grpo_qlora objective: executable_factor_reward - reward_source: factor_lab.training.grpo_reward.make_grpo_reward_func + reward_source: src.training.grpo_reward.make_grpo_reward_func reward_components: - DSL validity - daily cross-sectional RankIC @@ -23,6 +23,6 @@ grpo: loss_type: dapo data: - local_crypto_panel: ClassProject/data/crypto_panel_clean.pkl + local_crypto_panel: data/crypto/crypto_panel_clean.pkl wrds_crsp_target: daily_cross_sectional_rankic taq_target: intraday_microstructure_direction diff --git a/config/verl_qwen25_7b_grpo.yaml b/config/verl_qwen25_7b_grpo.yaml index fbc0b36..e1ec8f2 100644 --- a/config/verl_qwen25_7b_grpo.yaml +++ b/config/verl_qwen25_7b_grpo.yaml @@ -26,7 +26,7 @@ model: data: train_parquet: factor_lab/outputs/verl/crypto_grpo_tasks.parquet crypto_panel: data/crypto_panel_clean.pkl - tickers: BTC-USD,ETH-USD,XRP-USD + tickers: ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD reward: function: factor_lab.verl.reward_function.reward_fn diff --git a/config/verl_qwen3_14b_fullft_a100.yaml b/config/verl_qwen3_14b_fullft_a100.yaml index c0ebd10..4b25103 100644 --- a/config/verl_qwen3_14b_fullft_a100.yaml +++ b/config/verl_qwen3_14b_fullft_a100.yaml @@ -17,7 +17,7 @@ data: train_parquet: outputs/verl/crypto_grpo_tasks.parquet val_parquet: outputs/verl/crypto_grpo_val_tasks.parquet crypto_panel: data/crypto/crypto_panel_clean.pkl - tickers: BTC-USD,ETH-USD,XRP-USD + tickers: ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD max_prompt_length: 1536 max_response_length: 256 @@ -58,4 +58,6 @@ verl: nnodes: 1 n_gpus_per_node: 8 save_freq: 20 + test_freq: 20 + val_before_train: true total_epochs: 1 diff --git a/examples/baseline_rollout.py b/examples/baseline_rollout.py index 224eb9a..634cd01 100644 --- a/examples/baseline_rollout.py +++ b/examples/baseline_rollout.py @@ -12,6 +12,7 @@ from examples.build_seed_bank import crypto_frames_from_panel from examples.dsl_smoke_test import DEFAULT_CRYPTO_PANEL, load_dotenv +from src.data import verify_crypto_panel from src.rft import ( DatabaseSelectionConfig, MinerConfig, @@ -33,11 +34,11 @@ def run_crypto_baseline_rollout( *, - oracle_model: str, - miner_model: str, + oracle_model: str = "deepseek/deepseek-chat-v3.1", + miner_model: str = "qwen/qwen3-235b-a22b-2507", panel_path: Path = DEFAULT_CRYPTO_PANEL, tickers: tuple[str, ...] | None = None, - oracle_count: int = 12, + oracle_count: int = 24, miner_count: int = 4, top_k_seeds: int = 3, output_dir: Path = DEFAULT_OUTPUT_DIR, @@ -45,7 +46,7 @@ def run_crypto_baseline_rollout( """Run oracle seed generation plus miner baseline rollouts on crypto data.""" load_dotenv() - panel = pd.read_pickle(panel_path) + panel = verify_crypto_panel(panel_path, tickers=tickers) frames = crypto_frames_from_panel(panel, tickers=tickers) close = panel["close"] scenario = FactorScenario.from_benchmark( @@ -70,7 +71,7 @@ def run_crypto_baseline_rollout( windows=windows, raw_candidates=raw_candidates, min_history=30, - min_assets=max(3, min(5, len(frames))), + min_assets=8, pool_config=SeedPoolConfig(top_k=top_k_seeds, quality_threshold=-1.0), ) @@ -84,7 +85,7 @@ def run_crypto_baseline_rollout( price_col="close", count=miner_count, min_history=30, - min_assets=max(3, min(5, len(frames))), + min_assets=8, miner_config=MinerConfig(model=miner_model), archive=archive, selection_config=DatabaseSelectionConfig(min_score=-1.0, min_reward=-1.0), @@ -143,8 +144,8 @@ def main() -> int: parser.add_argument("--oracle-model", default="deepseek/deepseek-chat-v3.1") parser.add_argument("--miner-model", default="qwen/qwen3-235b-a22b-2507") parser.add_argument("--crypto-panel", type=Path, default=DEFAULT_CRYPTO_PANEL) - parser.add_argument("--tickers", default="BTC-USD,ETH-USD,XRP-USD") - parser.add_argument("--oracle-count", type=int, default=12) + parser.add_argument("--tickers", default="ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD") + parser.add_argument("--oracle-count", type=int, default=24) parser.add_argument("--miner-count", type=int, default=4) parser.add_argument("--top-k-seeds", type=int, default=3) parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) diff --git a/examples/build_seed_bank.py b/examples/build_seed_bank.py index af684da..e03a9b3 100644 --- a/examples/build_seed_bank.py +++ b/examples/build_seed_bank.py @@ -7,7 +7,7 @@ import pandas as pd -from src.data import adapt_crypto_ohlcv, adapt_crsp_dsf_v2 +from src.data import adapt_crsp_dsf_v2, adapt_crypto_ohlcv, verify_crypto_panel from examples.dsl_smoke_test import ( DEFAULT_CRYPTO_PANEL, fetch_wrds_crsp_dsf_v2_sample, @@ -25,11 +25,8 @@ def crypto_frames_from_panel(panel: dict, tickers: tuple[str, ...] | None = None) -> dict[str, pd.DataFrame]: """Convert saved crypto panel dict into per-asset DSL-ready frames.""" + verify_crypto_panel(panel, tickers=tickers) required = ["open", "high", "low", "close", "volume"] - missing = [key for key in required if key not in panel] - if missing: - raise ValueError(f"crypto panel missing keys: {missing}") - available = tuple(str(col) for col in panel["close"].columns) selected = tickers or available frames = {} @@ -51,7 +48,7 @@ def build_crypto_cross_sectional_seed_bank( ) -> dict: """Build a cross-sectional crypto seed bank from the saved project panel.""" - panel = pd.read_pickle(panel_path) + panel = verify_crypto_panel(panel_path, tickers=tickers) frames = crypto_frames_from_panel(panel, tickers=tickers) close = panel["close"] windows = [ @@ -71,7 +68,7 @@ def build_crypto_cross_sectional_seed_bank( windows=windows, raw_candidates=raw_candidates, min_history=30, - min_assets=max(3, min(5, len(frames))), + min_assets=8, pool_config=SeedPoolConfig(top_k=top_k, quality_threshold=quality_threshold), ) return {"result": result, "asset_count": len(frames)} @@ -128,7 +125,7 @@ def main() -> int: parser.add_argument("--candidate-source", choices=("template", "openrouter", "file"), default="template") parser.add_argument("--oracle-output-file", type=Path) parser.add_argument("--oracle-count", type=int, default=24) - parser.add_argument("--openrouter-model", default="deepseek/deepseek-chat") + parser.add_argument("--openrouter-model", default="deepseek/deepseek-chat-v3.1") parser.add_argument("--wrds-crsp", action="store_true", help="also build a tiny live WRDS CRSP seed bank") parser.add_argument("--wrds-permno", type=int, default=14593) parser.add_argument("--wrds-start", default="2023-01-01") diff --git a/examples/check_openrouter_tokens.py b/examples/check_openrouter_tokens.py index 01c7596..55cee02 100644 --- a/examples/check_openrouter_tokens.py +++ b/examples/check_openrouter_tokens.py @@ -42,7 +42,7 @@ def main() -> int: parser.add_argument("--namespace") parser.add_argument("--count", type=int, default=12) parser.add_argument("--seed-expr", default="div(ts_mean(crypto.volume(10)), ts_std(crypto.returns(30)))") - parser.add_argument("--seed-score", type=float, default=0.655671862964597) + parser.add_argument("--seed-score", type=float, default=0.38385972330719526) parser.add_argument("--max-tokens", type=int, default=1) args = parser.parse_args() diff --git a/examples/dsl_smoke_test.py b/examples/dsl_smoke_test.py index 527a138..2de72f7 100644 --- a/examples/dsl_smoke_test.py +++ b/examples/dsl_smoke_test.py @@ -13,7 +13,7 @@ import pandas as pd -from src.data import adapt_crypto_ohlcv, adapt_crsp_dsf_v2 +from src.data import adapt_crsp_dsf_v2, adapt_crypto_ohlcv, verify_crypto_panel from src.dsl import PointInTimeContext, evaluate_expr @@ -71,11 +71,8 @@ def load_dotenv(path: Path | None = None) -> None: def crypto_asset_frame_from_panel(panel: dict, ticker: str) -> pd.DataFrame: """Build one asset OHLCV frame from the saved crypto panel dictionary.""" + verify_crypto_panel(panel, tickers=(ticker,)) required = ["open", "high", "low", "close", "volume"] - missing = [key for key in required if key not in panel] - if missing: - raise ValueError(f"crypto panel missing keys: {missing}") - frame = pd.DataFrame({key: panel[key][ticker] for key in required}) if "returns" in panel: frame["returns"] = panel["returns"][ticker] @@ -106,7 +103,7 @@ def evaluate_last_available( def run_crypto_smoke(panel_path: Path = DEFAULT_CRYPTO_PANEL, ticker: str = "BTC-USD") -> dict[str, float]: """Run the crypto namespace smoke test on the saved project panel.""" - panel = pd.read_pickle(panel_path) + panel = verify_crypto_panel(panel_path, tickers=(ticker,)) frame = crypto_asset_frame_from_panel(panel, ticker) adapted = adapt_crypto_ohlcv(frame) value = evaluate_last_available("crypto", adapted, CRYPTO_EXPR, min_history=25) diff --git a/examples/evaluate_rollout_library.py b/examples/evaluate_rollout_library.py index c420b4f..494ff4d 100644 --- a/examples/evaluate_rollout_library.py +++ b/examples/evaluate_rollout_library.py @@ -8,11 +8,12 @@ import pandas as pd -from src.evaluation import PostSelectionConfig, evaluate_factor_library -from src.evaluation.post_selection import write_evaluation_report from examples.baseline_rollout import DEFAULT_OUTPUT_DIR from examples.build_seed_bank import crypto_frames_from_panel from examples.dsl_smoke_test import DEFAULT_CRYPTO_PANEL +from src.data import verify_crypto_panel +from src.evaluation import PostSelectionConfig, evaluate_factor_library +from src.evaluation.post_selection import write_evaluation_report def load_valid_rollout_exprs(path: Path) -> list[str]: @@ -46,7 +47,7 @@ def run_crypto_rollout_library_evaluation( """Run validation-guided decorrelated selection and equal-weight fusion.""" exprs = load_valid_rollout_exprs(rollout_json) - panel = pd.read_pickle(panel_path) + panel = verify_crypto_panel(panel_path, tickers=tickers) frames = crypto_frames_from_panel(panel, tickers=tickers) if validation_start is None or validation_end is None or test_start is None or test_end is None: dates = pd.DatetimeIndex(panel["close"].index).sort_values() @@ -69,7 +70,7 @@ def run_crypto_rollout_library_evaluation( correlation_threshold=correlation_threshold, top_k=top_k, min_history=30, - min_assets=max(3, min(5, len(frames))), + min_assets=8, horizon=1, ), ) @@ -82,7 +83,7 @@ def main() -> int: parser.add_argument("--rollout-json", type=Path, default=DEFAULT_OUTPUT_DIR / "baseline_rollout.json") parser.add_argument("--crypto-panel", type=Path, default=DEFAULT_CRYPTO_PANEL) parser.add_argument("--output-path", type=Path, default=DEFAULT_OUTPUT_DIR / "paper_style_evaluation.json") - parser.add_argument("--tickers", default="BTC-USD,ETH-USD,XRP-USD") + parser.add_argument("--tickers", default="ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD") parser.add_argument("--validation-start") parser.add_argument("--validation-end") parser.add_argument("--test-start") diff --git a/examples/launch_verl_qwen3_14b_a100.sh b/examples/launch_verl_qwen3_14b_a100.sh index 7c40b7d..995f9b9 100755 --- a/examples/launch_verl_qwen3_14b_a100.sh +++ b/examples/launch_verl_qwen3_14b_a100.sh @@ -10,7 +10,7 @@ set -euo pipefail # with a fallback default, so build_dataset below and the eventual Verl # reward_fn agree on the same crypto data/tickers/output locations). export FACTOR_LAB_CRYPTO_PANEL="${FACTOR_LAB_CRYPTO_PANEL:-data/crypto/crypto_panel_clean.pkl}" -export FACTOR_LAB_TICKERS="${FACTOR_LAB_TICKERS:-BTC-USD,ETH-USD,XRP-USD}" +export FACTOR_LAB_TICKERS="${FACTOR_LAB_TICKERS:-ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD}" export FACTOR_LAB_ARCHIVE_JSONL="${FACTOR_LAB_ARCHIVE_JSONL:-outputs/verl/qwen3_14b_fullft/mined_factors.jsonl}" export FACTOR_LAB_REWARD_LOG_JSONL="${FACTOR_LAB_REWARD_LOG_JSONL:-outputs/verl/qwen3_14b_fullft/reward_rollouts.jsonl}" @@ -19,7 +19,7 @@ export FACTOR_LAB_REWARD_LOG_JSONL="${FACTOR_LAB_REWARD_LOG_JSONL:-outputs/verl/ python -m src.verl_integration.build_dataset \ --output outputs/verl/crypto_grpo_tasks.parquet \ --seed-expr "div(ts_mean(crypto.volume(10)), ts_std(crypto.returns(30)))" \ - --seed-score 0.655671862964597 \ + --seed-score 0.38385972330719526 \ --crypto-panel "${FACTOR_LAB_CRYPTO_PANEL}" \ --tickers "${FACTOR_LAB_TICKERS}" \ --repeats 400 @@ -29,7 +29,7 @@ python -m src.verl_integration.build_dataset \ python -m src.verl_integration.build_dataset \ --output outputs/verl/crypto_grpo_val_tasks.parquet \ --seed-expr "div(ts_mean(crypto.volume(10)), ts_std(crypto.returns(30)))" \ - --seed-score 0.655671862964597 \ + --seed-score 0.38385972330719526 \ --crypto-panel "${FACTOR_LAB_CRYPTO_PANEL}" \ --tickers "${FACTOR_LAB_TICKERS}" \ --repeats 32 diff --git a/examples/run_compared_approaches.py b/examples/run_compared_approaches.py index de322a9..28903ef 100644 --- a/examples/run_compared_approaches.py +++ b/examples/run_compared_approaches.py @@ -14,7 +14,7 @@ def main() -> int: parser = argparse.ArgumentParser(description="Run compared alpha-discovery benchmark approaches.") parser.add_argument("--model", default="qwen/qwen3-235b-a22b-2507") parser.add_argument("--crypto-panel", type=Path, default=DEFAULT_CRYPTO_PANEL) - parser.add_argument("--tickers", default="BTC-USD,ETH-USD,XRP-USD") + parser.add_argument("--tickers", default="ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD") parser.add_argument( "--approaches", default="alphabench,quantaalpha,rd_agent,alpha_jungle,factor_lab", diff --git a/examples/run_tier1_gate.py b/examples/run_tier1_gate.py new file mode 100644 index 0000000..5be40d9 --- /dev/null +++ b/examples/run_tier1_gate.py @@ -0,0 +1,519 @@ +"""Tier 1 acceptance gate (coding-plan.md Sections 10.4 / 10.5 / 10.6). + +Runs the synthetic factor-quality sweep (10.4), the within-rollout-group +reward variance diagnostic (10.5) against the real crypto panel via the real +OpenRouter miner model, and writes the locked Tier 1 JSON report (10.6). + +The crypto panel is a hard prerequisite (Section 9.1): if the file is missing, +the gate hard-stops naming the missing path and does not fall back to a +synthetic or empty panel. + +Usage (repository root as working directory): + + python -m examples.run_tier1_gate \ + --crypto-panel data/crypto/crypto_panel_clean.pkl \ + --tickers ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD \ + --output outputs/tier1/tier1_report.json +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +from examples.build_seed_bank import crypto_frames_from_panel +from examples.dsl_smoke_test import load_dotenv +from src.data import adapt_crypto_ohlcv, verify_crypto_panel +from src.dsl import ValidationError, validate_expr +from src.llm.openrouter import call_openrouter_chat, openrouter_content +from src.rft import ( + DatabaseSelectionConfig, + DiCoRewardConfig, + MinedFactorDatabase, + build_miner_messages, + reward_completions, +) +from src.scoring import score_cross_sectional_rankic +from src.seeds import EvaluationWindow, FactorScenario, SeedTask + + +MIN_HISTORY = 30 +LOCKED_CRYPTO_TICKERS = ( + "ADA-USD", + "BNB-USD", + "BTC-USD", + "DOGE-USD", + "ETH-USD", + "LINK-USD", + "XLM-USD", + "XRP-USD", +) +LOCKED_CRYPTO_TICKERS_ARG = ",".join(LOCKED_CRYPTO_TICKERS) +MIN_ASSETS = 8 +MIN_ASSETS_DERIVATION = ( + "Locked minimum universe size of 8 for cross-sectional RankIC; timestamps " + "with fewer than 8 finite factor/forward-return pairs are skipped, and a " + "fully undersized evaluation returns score=-1.0." +) +BOOTSTRAP_SEED_EXPR = "div(ts_mean(crypto.volume(10)), ts_std(crypto.returns(30)))" +BOOTSTRAP_SEED_SCORE = 0.38385972330719526 +BOOTSTRAP_SEED_DERIVATION = ( + "Section 5.1 cross-sectional RankIC formula against the full locked 8-asset crypto panel history " + "(ADA-USD, BNB-USD, BTC-USD, DOGE-USD, ETH-USD, LINK-USD, XLM-USD, XRP-USD) with min_assets=8; " + "recomputed from the locked panel, matching the code-review.md cited value." +) +ORACLE_MODEL = "deepseek/deepseek-chat-v3.1" +MINER_MODEL = "qwen/qwen3-235b-a22b-2507" +ROLLOUT_GROUP_SIZE = 8 +SAMPLING = {"temperature": 1.0, "top_p": 0.95, "top_k": 50} + +SWEEP_EXPRESSIONS = ( + "div(ts_sum(crypto.returns(30)), ts_std(crypto.returns(30)))", + "neg(ts_mean(crypto.returns(5)))", + "div(ts_mean(crypto.volume(7)), ts_mean(crypto.volume(30)))", + "ts_mean(crypto.close(1))", + "div(crypto.close(1), sub(crypto.close(1), crypto.close(1)))", +) +INVALID_EXPRESSION = "ts_mean(ts_mean(crypto.close(1)))" + + +def run_synthetic_sweep( + frames: dict[str, pd.DataFrame], + *, + min_history: int = MIN_HISTORY, + min_assets: int = MIN_ASSETS, +) -> dict: + """Run the Section 10.4 synthetic factor-quality sweep. + + Scores the 5 locked expressions with the Section 5.1 formula directly and + confirms a genuinely grammar-invalid expression is rejected at validation. + Returns the ``synthetic_sweep`` section of the locked JSON report. + """ + + validated = [] + for expr in SWEEP_EXPRESSIONS: + try: + validate_expr(expr) + validated.append(True) + except ValidationError: + validated.append(False) + + scores: list[float | None] = [] + score_errors: list[str] = [] + for expr in SWEEP_EXPRESSIONS: + try: + result = score_cross_sectional_rankic( + expr, + "crypto", + frames, + price_col="close", + min_history=min_history, + min_assets=min_assets, + ) + scores.append(float(result.score)) + score_errors.append("") + except Exception as exc: # noqa: BLE001 - report any scoring failure verbatim + scores.append(None) + score_errors.append(str(exc)) + + correctly_rejected = False + rejection_error = "" + try: + validate_expr(INVALID_EXPRESSION) + except ValidationError as exc: + correctly_rejected = True + rejection_error = str(exc) + + top_three = [scores[0], scores[1], scores[2]] + top_three_finite_in_range = bool( + all(s is not None and math.isfinite(s) and -1.0 <= s <= 1.0 for s in top_three) + ) + top_three_not_identical = bool( + all(s is not None for s in top_three) and len(set(top_three)) > 1 # type: ignore[arg-type] + ) + expression_4_finite = scores[3] is not None and math.isfinite(scores[3]) # type: ignore[arg-type] + expression_4_distinct_from_1_3 = bool( + expression_4_finite and all(scores[3] != s for s in top_three if s is not None) + ) + expression_5_degenerate = scores[4] == -1.0 + + undersized_frames = {k: v for i, (k, v) in enumerate(frames.items()) if i < max(1, min_assets - 1)} + undersized_score: float | None = None + try: + undersized_result = score_cross_sectional_rankic( + SWEEP_EXPRESSIONS[0], + "crypto", + undersized_frames, + price_col="close", + min_history=min_history, + min_assets=min_assets, + ) + undersized_score = float(undersized_result.score) + except Exception: # noqa: BLE001 + undersized_score = None + undersized_guard_passed = bool(undersized_score == -1.0) + + criteria = { + "all_sweep_expressions_validate": bool(all(validated)), + "invalid_expression_rejected_at_validation": bool(correctly_rejected), + "expressions_1_3_finite_in_range": top_three_finite_in_range, + "expressions_1_3_not_identical": top_three_not_identical, + "expression_4_finite": expression_4_finite, + "expression_4_distinct_from_1_3": expression_4_distinct_from_1_3, + "expression_5_degenerate_score_minus_one": expression_5_degenerate, + "undersized_universe_guard_returns_minus_one": undersized_guard_passed, + } + + for name, holds in criteria.items(): + print(f"[tier1-gate] sweep criterion {name}: {'PASS' if holds else 'FAIL'}") + if any(score_errors): + for expr, err in zip(SWEEP_EXPRESSIONS, score_errors): + if err: + print(f"[tier1-gate] sweep scoring error for {expr}: {err}") + + return { + "expressions": list(SWEEP_EXPRESSIONS), + "scores": [s if s is not None else None for s in scores], + "invalid_expression_rejected": { + "expr": INVALID_EXPRESSION, + "correctly_rejected": bool(correctly_rejected), + "error": rejection_error, + }, + "undersized_universe_guard": { + "tested_asset_count": len(undersized_frames), + "min_assets": min_assets, + "score": undersized_score, + "correctly_rejected": undersized_guard_passed, + }, + "passed": bool(all(criteria.values())), + } + + +def _build_windows(panel: dict, *, n: int, min_history: int) -> tuple[EvaluationWindow, ...]: + """Split the panel date range into ``n`` contiguous, distinct windows. + + Each window is required to hold at least ``min_history + 1`` bars so the + Section 5.1 scorers always have at least one scored timestamp. + """ + + index = panel["close"].index + if len(index) < n: + raise RuntimeError( + f"crypto panel has {len(index)} bars; cannot build {n} distinct evaluation windows" + ) + edges = [int(round(i * len(index) / n)) for i in range(n + 1)] + windows = [] + for i in range(n): + chunk = index[edges[i]:edges[i + 1]] + if len(chunk) < min_history + 1: + raise RuntimeError( + f"evaluation window {i} spans only {len(chunk)} bars " + f"(needs at least {min_history + 1} to score)" + ) + windows.append(EvaluationWindow(str(chunk[0].date()), str(chunk[-1].date()))) + return tuple(windows) + + +def _frames_for_window( + panel: dict, + tickers: tuple[str, ...], + window: EvaluationWindow, +) -> dict[str, pd.DataFrame]: + """Slice the crypto panel to one evaluation window into DSL-ready frames.""" + + start, end = pd.Timestamp(window.start), pd.Timestamp(window.end) + frames = {} + for ticker in tickers: + frame = pd.DataFrame( + {key: panel[key].loc[start:end][ticker] for key in ("open", "high", "low", "close", "volume")} + ) + if "returns" in panel: + frame["returns"] = panel["returns"].loc[start:end][ticker] + frames[str(ticker)] = adapt_crypto_ohlcv(frame) + return frames + + +def build_tier1_tasks( + panel: dict, + tickers: tuple[str, ...], + *, + min_history: int = MIN_HISTORY, +) -> tuple[tuple[SeedTask, object, str], ...]: + """Build the fixed 5-task set: 3 cross-sectional + 2 directional (Section 10.5). + + Every task carries the locked bootstrap seed against a distinct evaluation + window. The second element is the window-sliced data the scorer runs on; + the third is the price column. + """ + + windows = _build_windows(panel, n=5, min_history=min_history) + cross_scenario = FactorScenario.from_benchmark( + "daily_cross_sectional_rankic", + market="crypto", + horizon=1, + name="tier1_crypto_cross_sectional_rankic", + ) + direction_scenario = FactorScenario.from_benchmark( + "single_asset_direction", + market="crypto", + horizon=1, + name="tier1_crypto_single_asset_direction", + ) + + tasks = [] + for i, window in enumerate(windows): + if i < 3: + scenario = cross_scenario + data = _frames_for_window(panel, tickers, window) + else: + scenario = direction_scenario + data = _frames_for_window(panel, tickers, window)[tickers[0]] + tasks.append( + ( + SeedTask( + seed_expr=BOOTSTRAP_SEED_EXPR, + seed_score=BOOTSTRAP_SEED_SCORE, + scenario=scenario, + window=window, + objective=scenario.objective, + ), + data, + "close", + ) + ) + return tuple(tasks) + + +def _rollout_record(result) -> dict: + """Convert one RewardBridgeResult into the locked per-rollout record.""" + + score = float(result.score) if math.isfinite(result.score) else -1.0 + reward = float(result.reward) if math.isfinite(result.reward) else -1.0 + return { + "completion": result.completion, + "expr": result.expr, + "valid": bool(result.valid), + "score": score, + "reward": reward, + "reason": result.reason, + } + + +def run_reward_variance_diagnostic( + tasks: tuple[tuple[SeedTask, object, str], ...], + *, + miner_model: str = MINER_MODEL, + min_history: int = MIN_HISTORY, + min_assets: int = MIN_ASSETS, + k: int = ROLLOUT_GROUP_SIZE, +) -> dict: + """Run the Section 10.5 within-rollout-group reward variance diagnostic. + + K = ``k`` independent completions per task from the real OpenRouter-backed + miner model at the locked sampling parameters; each completion is scored + and rewarded with the real reward pipeline end to end (reward bridge). + """ + + archive = MinedFactorDatabase() + selection_config = DatabaseSelectionConfig() + reward_config = DiCoRewardConfig() + + task_records = [] + for task_id, (task, data, price_col) in enumerate(tasks): + messages = build_miner_messages(task, namespace="crypto", count=1) + completions = [] + for _ in range(k): + raw = call_openrouter_chat( + model=miner_model, + messages=messages, + temperature=SAMPLING["temperature"], + max_tokens=1600, + top_p=SAMPLING["top_p"], + top_k=SAMPLING["top_k"], + ) + completions.append(openrouter_content(raw)) + + results = reward_completions( + completions, + task=task, + namespace="crypto", + data=data, + price_col=price_col, + archive=archive, + min_history=min_history, + min_assets=min_assets, + reward_config=reward_config, + selection_config=selection_config, + ) + + rollouts = [_rollout_record(result) for result in results] + rewards = np.asarray([r["reward"] for r in rollouts], dtype=float) + reward_mean = float(np.mean(rewards)) + reward_std = float(np.std(rewards)) + + task_records.append( + { + "task_id": task_id, + "benchmark": task.scenario.benchmark, + "seed_expr": task.seed_expr, + "rollouts": rollouts, + "reward_mean": reward_mean, + "reward_std": reward_std, + } + ) + print( + f"[tier1-gate] task {task_id} {task.scenario.benchmark} " + f"window={task.window.start}..{task.window.end} " + f"reward_mean={reward_mean:.4f} reward_std={reward_std:.4f}" + ) + + tasks_with_std_above_0_05 = sum(1 for record in task_records if record["reward_std"] > 0.05) + tasks_all_invalid = sum( + 1 for record in task_records if all(r["reward"] == -1.0 for r in record["rollouts"]) + ) + at_least_one_valid_and_scored = any( + r["valid"] and math.isfinite(r["score"]) + for record in task_records + for r in record["rollouts"] + ) + + passed = bool( + at_least_one_valid_and_scored and tasks_with_std_above_0_05 >= 3 and tasks_all_invalid == 0 + ) + + return { + "oracle_model": ORACLE_MODEL, + "miner_model": miner_model, + "rollout_group_size_k": k, + "temperature": SAMPLING["temperature"], + "top_p": SAMPLING["top_p"], + "top_k": SAMPLING["top_k"], + "tasks": task_records, + "tasks_with_std_above_0_05": int(tasks_with_std_above_0_05), + "tasks_all_invalid": int(tasks_all_invalid), + "at_least_one_valid_and_scored": bool(at_least_one_valid_and_scored), + "passed": passed, + } + + +def build_tier1_report( + *, + panel: dict, + panel_path: str, + tickers: tuple[str, ...], + min_history: int = MIN_HISTORY, + miner_model: str = MINER_MODEL, + k: int = ROLLOUT_GROUP_SIZE, +) -> dict: + """Run both gate sections against the real panel and assemble the report.""" + + min_assets = MIN_ASSETS + frames = crypto_frames_from_panel(panel, tickers=tickers) + + sweep = run_synthetic_sweep(frames, min_history=min_history, min_assets=min_assets) + tasks = build_tier1_tasks(panel, tickers, min_history=min_history) + diagnostic = run_reward_variance_diagnostic( + tasks, + miner_model=miner_model, + min_history=min_history, + min_assets=min_assets, + k=k, + ) + + report = { + "tier": 1, + "crypto_panel_path": panel_path, + "tickers": list(tickers), + "min_history": min_history, + "min_assets": min_assets, + "min_assets_derivation": MIN_ASSETS_DERIVATION, + "bootstrap_seed": { + "expr": BOOTSTRAP_SEED_EXPR, + "score": BOOTSTRAP_SEED_SCORE, + "score_derivation": BOOTSTRAP_SEED_DERIVATION, + }, + "synthetic_sweep": sweep, + "reward_variance_diagnostic": diagnostic, + "overall_passed": bool(sweep["passed"] and diagnostic["passed"]), + } + return report + + +def _hard_stop(message: str) -> None: + print(f"[tier1-gate] HARD STOP: {message}", file=sys.stderr) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run the Tier 1 acceptance gate (coding-plan.md 10.4/10.5/10.6)." + ) + parser.add_argument("--crypto-panel", default="data/crypto/crypto_panel_clean.pkl") + parser.add_argument("--tickers", default=LOCKED_CRYPTO_TICKERS_ARG) + parser.add_argument("--miner-model", default=MINER_MODEL) + parser.add_argument("--rollout-group-size", type=int, default=ROLLOUT_GROUP_SIZE) + parser.add_argument("--min-history", type=int, default=MIN_HISTORY) + parser.add_argument("--output", type=Path, default=Path("outputs") / "tier1" / "tier1_report.json") + args = parser.parse_args() + + load_dotenv() + + tickers = tuple(str(item) for item in args.tickers.split(",") if item.strip()) or None + try: + panel = verify_crypto_panel(str(args.crypto_panel), tickers=tickers) + except (FileNotFoundError, ValueError) as exc: + _hard_stop( + f"crypto panel prerequisite failed (Section 9.1): {exc}" + ) + return 1 + + if not os.getenv("OPENROUTER_API_KEY"): + _hard_stop( + "OPENROUTER_API_KEY is not set in the environment or .env " + "(Section 9.3 requires it for the Section 10.5 miner rollouts)." + ) + return 1 + + try: + report = build_tier1_report( + panel=panel, + panel_path=str(args.crypto_panel), + tickers=tickers or tuple(panel["close"].columns), + min_history=args.min_history, + miner_model=args.miner_model, + k=args.rollout_group_size, + ) + except (FileNotFoundError, ValueError, RuntimeError) as exc: + _hard_stop(str(exc)) + return 1 + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + + sweep = report["synthetic_sweep"] + diag = report["reward_variance_diagnostic"] + print( + "[tier1-gate] synthetic sweep passed={} | " + "tasks_with_std_above_0_05={}/{} tasks_all_invalid={} " + "at_least_one_valid_and_scored={} | overall_passed={}".format( + sweep["passed"], + diag["tasks_with_std_above_0_05"], + len(diag["tasks"]), + diag["tasks_all_invalid"], + diag["at_least_one_valid_and_scored"], + report["overall_passed"], + ) + ) + print(f"[tier1-gate] report written to {args.output}") + print(json.dumps(report, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..789b609 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +[tool.pytest.ini_options] +pythonpath = ["."] +testpaths = ["tests"] + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +select = ["E", "F"] +ignore = ["E402", "F401", "F841", "E501"] diff --git a/runbook.md b/runbook.md index cb7ee6f..71aa8be 100644 --- a/runbook.md +++ b/runbook.md @@ -127,8 +127,8 @@ PY Navigate to the factor_lab folder and run the following commnad: -``` -.venv/bin/python src/data/wrds_crsp.py \ +```bash +python src/data/wrds_crsp.py \ --start-date 2019-01-01 \ --end-date 2024-12-31 \ --universe sp500 \ @@ -139,8 +139,8 @@ Navigate to the factor_lab folder and run the following commnad: The TAQ dataset takes longer than the CRSP dataset. First, try with a smaller smoke test to see if the pipeline is working. -``` -.venv/bin/python -m factor_lab/src/data/wrds_taq \ +```bash +python src/data/wrds_taq.py \ --symbols AAPL,MSFT \ --start-date 2024-01-02 \ --end-date 2024-01-04 \ @@ -165,6 +165,30 @@ Expected: passed ``` +### 5.5 Tier 1 Acceptance Gate (coding-plan.md Sections 10.4/10.5/10.6) + +Run the full local no-GPU acceptance gate before spending any GPU budget: the +synthetic factor-quality sweep (10.4) against the real crypto panel, the +within-rollout-group reward variance diagnostic (10.5) via the real OpenRouter +miner model (`qwen/qwen3-235b-a22b-2507`, 8 rollouts/task at +`temperature=1.0, top_p=0.95, top_k=50`), and the locked JSON report (10.6). + +Requires `data/crypto/crypto_panel_clean.pkl` (Section 9.1 hard prerequisite) +and `OPENROUTER_API_KEY` in the environment or `.env`. Run from the repository +root: + +```bash +python -m examples.run_tier1_gate \ + --crypto-panel data/crypto/crypto_panel_clean.pkl \ + --tickers ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD \ + --output outputs/tier1/tier1_report.json +``` + +The report contains a computed `overall_passed` that is `true` only when every +Section 10.4 and 10.5 criterion holds. If the crypto panel file is missing, the +gate hard-stops with a message naming the missing path (it never falls back to +a synthetic or empty panel). + ## 6. Download Models from Huggingface Syntax for downloading models individually @@ -211,7 +235,7 @@ div(ts_mean(crypto.volume(10)), ts_std(crypto.returns(30))) Seed score: ```text -0.655671862964597 +0.38385972330719526 ``` ## 8. Build Verl Task Dataset @@ -219,19 +243,19 @@ Seed score: This creates the full-scale task bank rows: seed expression, factor scenario, time window, objective, prompt, and rule-reward metadata. ```bash -python -m factor_lab.verl.build_dataset \ - --output factor_lab/outputs/verl/crypto_grpo_tasks.parquet \ +python -m src.verl_integration.build_dataset \ + --output outputs/verl/crypto_grpo_tasks.parquet \ --seed-expr "div(ts_mean(crypto.volume(10)), ts_std(crypto.returns(30)))" \ - --seed-score 0.655671862964597 \ - --crypto-panel data/crypto_panel_clean.pkl \ - --tickers BTC-USD,ETH-USD,XRP-USD \ + --seed-score 0.38385972330719526 \ + --crypto-panel data/crypto/crypto_panel_clean.pkl \ + --tickers ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD \ --repeats 400 ``` Expected: ```text -wrote 400 rows to factor_lab/outputs/verl/crypto_grpo_tasks.parquet +wrote 400 rows to outputs/verl/crypto_grpo_tasks.parquet ``` ## 9. Configure Verl Reward Workers @@ -239,16 +263,16 @@ wrote 400 rows to factor_lab/outputs/verl/crypto_grpo_tasks.parquet Set these before launching Verl: ```bash -export FACTOR_LAB_CRYPTO_PANEL=data/crypto_panel_clean.pkl -export FACTOR_LAB_TICKERS=BTC-USD,ETH-USD,XRP-USD -export FACTOR_LAB_ARCHIVE_JSONL=factor_lab/outputs/verl/mined_factors.jsonl -export FACTOR_LAB_REWARD_LOG_JSONL=factor_lab/outputs/verl/reward_rollouts.jsonl +export FACTOR_LAB_CRYPTO_PANEL=data/crypto/crypto_panel_clean.pkl +export FACTOR_LAB_TICKERS=ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD +export FACTOR_LAB_ARCHIVE_JSONL=outputs/verl/qwen3_14b_fullft/mined_factors.jsonl +export FACTOR_LAB_REWARD_LOG_JSONL=outputs/verl/qwen3_14b_fullft/reward_rollouts.jsonl ``` Reward function import path: ```text -factor_lab.verl.reward_function.reward_fn +src.verl_integration.reward_function.reward_fn ``` This is the executable reward bridge: completions become Factor-DSL expressions, expressions are validated and backtested, RankIC/IC/ICIR-style metrics are converted into DiCo reward, and the reward function returns the scalar tensor used by GRPO. @@ -258,9 +282,9 @@ This is the executable reward bridge: completions become Factor-DSL expressions, Use the installed Verl commit's GRPO/PPO launcher and map these values into its config: ```text -train parquet: factor_lab/outputs/verl/crypto_grpo_tasks.parquet -reward bridge: factor_lab.verl.reward_bridge.FactorLabVerlRewardBridge -scalar reward fn: factor_lab.verl.reward_function.reward_fn +train parquet: outputs/verl/crypto_grpo_tasks.parquet +reward bridge: src.verl_integration.reward_bridge.FactorLabVerlRewardBridge +scalar reward fn: src.verl_integration.reward_function.reward_fn model: /workspace/models/Qwen3-14B GPUs: 8 lora_rank: 0 @@ -278,7 +302,7 @@ save_steps: 20 The project-side config lives at: ```text -factor_lab/config/verl_qwen3_14b_fullft_a100.yaml +config/verl_qwen3_14b_fullft_a100.yaml ``` Helper script: @@ -292,8 +316,8 @@ That script builds the train/validation parquet files and prints the exact Facto Main launch command: ```bash -python -m factor_lab.verl.verl_main \ - --config factor_lab/config/verl_qwen3_14b_fullft_a100.yaml \ +python -m src.verl_integration.verl_main \ + --config config/verl_qwen3_14b_fullft_a100.yaml \ --base-config config/verl_ppo_trainer_base.yaml ``` @@ -304,14 +328,15 @@ Use this only to verify the reward loop before spending on the 8-A100 run. Conservative A100 40GB run: ```bash -python training/train_grpo_qlora \ +python -m src.training.train_grpo_qlora \ --model /workspace/models/Qwen2.5-0.5B-Instruct \ --seed-expr "div(ts_mean(crypto.volume(10)), ts_std(crypto.returns(30)))" \ - --seed-score 0.655671862964597 \ - --crypto-panel data/crypto_panel_clean.pkl \ - --tickers BTC-USD,ETH-USD,XRP-USD \ - --output-dir factor_lab/outputs/grpo/Qwen2.5-0.5B-Instruct_crypto_grpo \ - --archive-jsonl factor_lab/outputs/grpo/Qwen2.5-0.5B-Instruct/mined_factors.jsonl \ + --seed-score 0.38385972330719526 \ + --crypto-panel data/crypto/crypto_panel_clean.pkl \ + --tickers ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD \ + --output-dir outputs/grpo/Qwen2.5-0.5B-Instruct_crypto_grpo \ + --archive-jsonl outputs/grpo/Qwen2.5-0.5B-Instruct/mined_factors.jsonl \ + --reward-log-jsonl outputs/grpo/Qwen2.5-0.5B-Instruct/reward_rollouts.jsonl \ --batch-size 1 \ --grad-accum 8 \ --generations 8 \ @@ -324,7 +349,6 @@ python training/train_grpo_qlora \ --steps 50 \ --save-steps 20 \ --dataset-repeat 400 \ - --reward-log-jsonl factor_lab/outputs/grpo/Qwen2.5-7B-Instruct/reward_rollouts.jsonl \ --lr 0.000005 \ --beta 0.02 \ --loss-type dapo @@ -346,12 +370,12 @@ Reduce in this order: After training, save these: ```bash -factor_lab/outputs/verl/crypto_grpo_tasks.parquet -factor_lab/outputs/verl/mined_factors.jsonl -factor_lab/outputs/verl/reward_rollouts.jsonl -factor_lab/outputs/grpo/Qwen2.5-7B-Instruct_crypto_grpo/ -factor_lab/outputs/grpo/Qwen2.5-7B-Instruct/mined_factors.jsonl -factor_lab/outputs/grpo/Qwen2.5-7B-Instruct/reward_rollouts.jsonl +outputs/verl/crypto_grpo_tasks.parquet +outputs/verl/qwen3_14b_fullft/mined_factors.jsonl +outputs/verl/qwen3_14b_fullft/reward_rollouts.jsonl +outputs/grpo/Qwen2.5-7B-Instruct_crypto_grpo/ +outputs/grpo/Qwen2.5-7B-Instruct/mined_factors.jsonl +outputs/grpo/Qwen2.5-7B-Instruct/reward_rollouts.jsonl ``` The model output/checkpoint directory is the trained model artifact. The mined factor JSONL is the accepted factor database. The reward rollout JSONL stores every completion, extracted expression, reward, validity flag, metrics, and rejection reason. diff --git a/src/benchmarks/compared_approaches.py b/src/benchmarks/compared_approaches.py index cf35f27..f76bf33 100644 --- a/src/benchmarks/compared_approaches.py +++ b/src/benchmarks/compared_approaches.py @@ -14,10 +14,11 @@ import pandas as pd -from src.evaluation import PostSelectionConfig, evaluate_factor_library -from src.evaluation.post_selection import FusedEvaluation, write_evaluation_report from examples.build_seed_bank import crypto_frames_from_panel from examples.dsl_smoke_test import DEFAULT_CRYPTO_PANEL +from src.data import verify_crypto_panel +from src.evaluation import PostSelectionConfig, evaluate_factor_library +from src.evaluation.post_selection import FusedEvaluation, write_evaluation_report from src.rft import MinerConfig from src.rft.miner import generate_miner_candidates from src.seeds import EvaluationWindow, FactorScenario, SeedPoolConfig, build_scenario_seed_bank @@ -88,7 +89,7 @@ def run_compared_approaches( ) -> dict: """Run compared approaches under one DSL/evaluation protocol.""" - panel = pd.read_pickle(panel_path) + panel = verify_crypto_panel(panel_path, tickers=tickers) frames = crypto_frames_from_panel(panel, tickers=tickers) dates = pd.DatetimeIndex(panel["close"].index).sort_values() split_idx = int(len(dates) * 0.7) @@ -112,7 +113,7 @@ def run_compared_approaches( correlation_threshold=correlation_threshold, top_k=top_k, min_history=30, - min_assets=max(3, min(5, len(frames))), + min_assets=8, horizon=1, ) @@ -131,7 +132,7 @@ def run_compared_approaches( windows=windows, raw_candidates=candidates, min_history=30, - min_assets=max(3, min(5, len(frames))), + min_assets=8, pool_config=SeedPoolConfig(top_k=max(top_k, 1), quality_threshold=-1.0), ) exprs = tuple(seed.expr for seed in seed_bank.seeds) @@ -187,7 +188,7 @@ def _generate_candidates_for_approach( windows=[EvaluationWindow("", "")], raw_candidates=seeds, min_history=30, - min_assets=max(3, min(5, len(frames))), + min_assets=8, pool_config=SeedPoolConfig(top_k=3, quality_threshold=-1.0), ) candidates = [] diff --git a/src/data/__init__.py b/src/data/__init__.py index 11132b3..5ff1cc5 100644 --- a/src/data/__init__.py +++ b/src/data/__init__.py @@ -1,15 +1,15 @@ """Data adapters for Factor Lab namespaces.""" -from src.data.adapters import adapt_crypto_ohlcv, adapt_crsp_dsf_v2, adapt_taq_features - -# wrds_crsp.py / wrds_taq.py are intentionally not re-exported here: they use -# bare same-directory imports (e.g. `from adapters import ...`) so they stay -# runnable directly as `python src/data/wrds_crsp.py`. Importing them through -# this package would execute those bare imports under a different sys.path -# and fail. Import them directly from their file if you need their functions. +from src.data.adapters import ( + adapt_crsp_dsf_v2, + adapt_crypto_ohlcv, + adapt_taq_features, + verify_crypto_panel, +) __all__ = [ "adapt_crypto_ohlcv", "adapt_crsp_dsf_v2", "adapt_taq_features", + "verify_crypto_panel", ] diff --git a/src/data/adapters.py b/src/data/adapters.py index 5b2bf84..686ccac 100644 --- a/src/data/adapters.py +++ b/src/data/adapters.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path +from typing import Iterable import pandas as pd @@ -63,3 +65,44 @@ def adapt_taq_features(frame: pd.DataFrame, date_col: str | None = None) -> pd.D _require_columns(out, required) return out[required].apply(pd.to_numeric, errors="coerce") + +def verify_crypto_panel( + panel_or_path: str | Path | dict, + tickers: Iterable[str] | None = None, +) -> dict: + """Verify crypto panel existence and schema non-degeneracy per ยง9.1. + + Raises FileNotFoundError naming the missing path if a path was passed and doesn't exist. + Raises ValueError naming any missing required keys or any missing tickers. + """ + + if isinstance(panel_or_path, (str, Path)): + path = Path(panel_or_path) + if not path.exists(): + raise FileNotFoundError(f"crypto panel file not found: '{path}'") + panel = pd.read_pickle(path) + elif isinstance(panel_or_path, dict): + panel = panel_or_path + else: + raise ValueError(f"expected dict or path for crypto panel, got {type(panel_or_path).__name__}") + + required = ["open", "high", "low", "close", "volume"] + missing = [key for key in required if key not in panel] + if missing: + raise ValueError(f"crypto panel missing required key(s): {missing}") + + if tickers is not None: + ticker_list = tuple(str(t) for t in tickers) + if hasattr(panel["close"], "columns"): + available = set(panel["close"].columns) + elif isinstance(panel["close"], dict): + available = set(panel["close"].keys()) + else: + available = set() + missing_tickers = [t for t in ticker_list if t not in available] + if missing_tickers: + raise ValueError(f"crypto panel missing requested ticker(s): {missing_tickers}") + + return panel + + diff --git a/src/data/wrds_crsp.py b/src/data/wrds_crsp.py index 4cd4910..bc52b67 100644 --- a/src/data/wrds_crsp.py +++ b/src/data/wrds_crsp.py @@ -4,13 +4,18 @@ import argparse import os +import sys import tempfile from pathlib import Path from typing import Any import pandas as pd -from adapters import adapt_crsp_dsf_v2 +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from src.data.adapters import adapt_crsp_dsf_v2 CRSP_DSF_V2_COLUMNS = [ "permno", @@ -24,8 +29,6 @@ "dlycap", ] -REPO_ROOT = Path(__file__).resolve().parents[2] - def normalize_wrds_password(value: str | None) -> str: """Return password from a raw password or pgpass-style entry.""" diff --git a/src/data/wrds_taq.py b/src/data/wrds_taq.py index 717145e..f26c525 100644 --- a/src/data/wrds_taq.py +++ b/src/data/wrds_taq.py @@ -3,13 +3,18 @@ from __future__ import annotations import argparse +import sys from datetime import datetime from pathlib import Path from typing import Any, Iterable import pandas as pd -from wrds_crsp import connect_wrds +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from src.data.wrds_crsp import connect_wrds TAQ_1M_COLUMNS = ["open", "high", "low", "close", "volume", "spread", "midret", "imbalance", "trade_size", "trade_count"] diff --git a/src/evaluation/post_selection.py b/src/evaluation/post_selection.py index f1fd0a4..48646b1 100644 --- a/src/evaluation/post_selection.py +++ b/src/evaluation/post_selection.py @@ -29,7 +29,7 @@ class PostSelectionConfig: correlation_threshold: float = 0.7 top_k: int = 5 min_history: int = 30 - min_assets: int = 3 + min_assets: int = 8 horizon: int = 1 @@ -76,13 +76,37 @@ def evaluate_factor_library( """Evaluate, select, decorrelate, and fuse a candidate factor library.""" cfg = config or PostSelectionConfig() + min_assets = max(8, int(cfg.min_assets)) + if len(frames_by_asset) < min_assets: + empty_eval = FactorEvaluation( + expr="undersized_universe", + split="undersized", + dir_acc=float("nan"), + ic_mean=float("nan"), + rank_ic_mean=float("nan"), + icir=float("nan"), + long_short_mean=float("nan"), + long_short_sharpe=float("nan"), + valid_times=0, + mean_assets_per_time=0.0, + ) + return FusedEvaluation((), empty_eval, empty_eval, float(cfg.correlation_threshold)) + factor_matrices = { expr: _factor_matrix(expr, namespace, frames_by_asset, min_history=cfg.min_history) for expr in exprs } fwd = _forward_return_matrix(frames_by_asset, price_col=price_col, horizon=cfg.horizon) validation_evals = [ - evaluate_factor_matrix(expr, matrix, fwd, split="validation", start=cfg.validation_start, end=cfg.validation_end) + evaluate_factor_matrix( + expr, + matrix, + fwd, + split="validation", + start=cfg.validation_start, + end=cfg.validation_end, + min_assets=min_assets, + ) for expr, matrix in factor_matrices.items() ] selected = select_decorrelated_factors( @@ -101,6 +125,7 @@ def evaluate_factor_library( split="validation_fused", start=cfg.validation_start, end=cfg.validation_end, + min_assets=min_assets, ) test = evaluate_fused_signal( selected_matrices, @@ -108,6 +133,7 @@ def evaluate_factor_library( split="test_fused", start=cfg.test_start, end=cfg.test_end, + min_assets=min_assets, ) return FusedEvaluation(tuple(selected), validation, test, float(cfg.correlation_threshold)) @@ -120,10 +146,26 @@ def evaluate_factor_matrix( split: str, start: str | None = None, end: str | None = None, + min_assets: int = 8, ) -> FactorEvaluation: """Compute DirAcc, IC, RankIC, ICIR, and long-short spread for a factor matrix.""" + min_assets = max(8, int(min_assets)) factor, fwd = _align_split(factor, fwd, start=start, end=end) + if factor.shape[1] < min_assets: + return FactorEvaluation( + expr=expr, + split=split, + dir_acc=float("nan"), + ic_mean=float("nan"), + rank_ic_mean=float("nan"), + icir=float("nan"), + long_short_mean=float("nan"), + long_short_sharpe=float("nan"), + valid_times=0, + mean_assets_per_time=0.0, + ) + dir_hits: list[float] = [] ics: list[float] = [] rank_ics: list[float] = [] @@ -131,8 +173,8 @@ def evaluate_factor_matrix( assets_per_time: list[int] = [] for timestamp in factor.index: - pair = pd.concat({"factor": factor.loc[timestamp], "fwd": fwd.loc[timestamp]}, axis=1).dropna() - if pair.shape[0] < 2: + pair = pd.concat({"factor": factor.loc[timestamp], "fwd": fwd.loc[timestamp]}, axis=1, sort=False).dropna() + if pair.shape[0] < int(min_assets): continue assets_per_time.append(int(pair.shape[0])) signs = np.sign(pair["factor"].to_numpy(dtype=float)) @@ -202,15 +244,16 @@ def evaluate_fused_signal( split: str, start: str | None = None, end: str | None = None, + min_assets: int = 8, ) -> FactorEvaluation: """Equal-weight normalized-rank fusion of selected factor matrices.""" if not factor_matrices: empty = pd.DataFrame(index=fwd.index, columns=fwd.columns, dtype=float) - return evaluate_factor_matrix("equal_weight_fusion", empty, fwd, split=split, start=start, end=end) + return evaluate_factor_matrix("equal_weight_fusion", empty, fwd, split=split, start=start, end=end, min_assets=min_assets) aligned = [_rank_normalize(matrix) for matrix in factor_matrices] fused = sum(aligned) / float(len(aligned)) - return evaluate_factor_matrix("equal_weight_fusion", fused, fwd, split=split, start=start, end=end) + return evaluate_factor_matrix("equal_weight_fusion", fused, fwd, split=split, start=start, end=end, min_assets=min_assets) def write_evaluation_report(path: Path, evaluation: FusedEvaluation) -> None: diff --git a/src/llm/openrouter.py b/src/llm/openrouter.py index 35fc9ad..8f34842 100644 --- a/src/llm/openrouter.py +++ b/src/llm/openrouter.py @@ -32,6 +32,8 @@ def call_openrouter_chat( max_tokens: int = 1, timeout_seconds: int = 60, api_key: str | None = None, + top_p: float | None = None, + top_k: int | None = None, ) -> dict[str, Any]: """Call OpenRouter chat completions and return parsed JSON.""" @@ -46,6 +48,10 @@ def call_openrouter_chat( "temperature": temperature, "max_tokens": max_tokens, } + if top_p is not None: + payload["top_p"] = float(top_p) + if top_k is not None: + payload["top_k"] = int(top_k) req = urllib.request.Request( OPENROUTER_URL, data=json.dumps(payload).encode("utf-8"), diff --git a/src/rft/discovery_loop.py b/src/rft/discovery_loop.py index ea342d7..1866dae 100644 --- a/src/rft/discovery_loop.py +++ b/src/rft/discovery_loop.py @@ -39,7 +39,7 @@ def run_discovery_for_task( price_col: str, count: int = 4, min_history: int = 1, - min_assets: int = 3, + min_assets: int = 8, generator: CandidateGenerator | None = None, miner_config: MinerConfig | None = None, archive=None, diff --git a/src/rft/miner.py b/src/rft/miner.py index 31f5a0e..7f83539 100644 --- a/src/rft/miner.py +++ b/src/rft/miner.py @@ -14,7 +14,7 @@ class MinerConfig: """OpenRouter request settings for the local miner baseline.""" - model: str = "qwen/qwen-2.5-7b-instruct" + model: str = "qwen/qwen3-235b-a22b-2507" temperature: float = 0.8 max_tokens: int = 1600 timeout_seconds: int = 60 diff --git a/src/rft/reward_bridge.py b/src/rft/reward_bridge.py index 5f25117..03ec33b 100644 --- a/src/rft/reward_bridge.py +++ b/src/rft/reward_bridge.py @@ -35,7 +35,7 @@ def reward_completion( price_col: str, archive: MinedFactorDatabase | None = None, min_history: int = 1, - min_assets: int = 3, + min_assets: int = 8, reward_config: DiCoRewardConfig | None = None, selection_config: DatabaseSelectionConfig | None = None, ) -> RewardBridgeResult: @@ -101,7 +101,7 @@ def reward_completions( price_col: str, archive: MinedFactorDatabase | None = None, min_history: int = 1, - min_assets: int = 3, + min_assets: int = 8, reward_config: DiCoRewardConfig | None = None, selection_config: DatabaseSelectionConfig | None = None, ) -> tuple[RewardBridgeResult, ...]: diff --git a/src/scoring/cross_sectional.py b/src/scoring/cross_sectional.py index a34c6a7..3a9d2db 100644 --- a/src/scoring/cross_sectional.py +++ b/src/scoring/cross_sectional.py @@ -11,6 +11,9 @@ from src.scoring.series import evaluate_factor_series, forward_returns +MIN_CROSS_SECTIONAL_ASSETS = 8 + + @dataclass(frozen=True) class CrossSectionalScore: """Cross-sectional factor ranking metrics.""" @@ -33,10 +36,14 @@ def score_cross_sectional_rankic( price_col: str, horizon: int = 1, min_history: int = 1, - min_assets: int = 3, + min_assets: int = MIN_CROSS_SECTIONAL_ASSETS, ) -> CrossSectionalScore: """Score a factor by average cross-sectional RankIC over time.""" + min_assets = max(MIN_CROSS_SECTIONAL_ASSETS, int(min_assets)) + if len(frames_by_asset) < min_assets: + return CrossSectionalScore(-1.0, float("nan"), float("nan"), float("nan"), float("nan"), float("nan"), 0, 0.0) + factor_by_asset = {} fwd_by_asset = {} for asset, frame in frames_by_asset.items(): diff --git a/src/seeds/build.py b/src/seeds/build.py index bfa8e4c..7dfee66 100644 --- a/src/seeds/build.py +++ b/src/seeds/build.py @@ -36,7 +36,7 @@ def build_scenario_seed_bank( windows: tuple[EvaluationWindow, ...] | list[EvaluationWindow], raw_candidates: tuple[str, ...] | list[str] | None = None, min_history: int = 1, - min_assets: int = 3, + min_assets: int = 8, pool_config: SeedPoolConfig | None = None, ) -> ScenarioSeedBuild: """Build empirically scored seeds and seed-window tasks for a scenario.""" diff --git a/src/seeds/oracle.py b/src/seeds/oracle.py index e01245b..45029a0 100644 --- a/src/seeds/oracle.py +++ b/src/seeds/oracle.py @@ -21,7 +21,7 @@ class OpenRouterConfig: """OpenRouter request settings for oracle seed generation.""" - model: str = "deepseek/deepseek-chat" + model: str = "deepseek/deepseek-chat-v3.1" temperature: float = 0.7 max_tokens: int = 1600 timeout_seconds: int = 60 diff --git a/src/seeds/refine.py b/src/seeds/refine.py index 576c3e2..f9434af 100644 --- a/src/seeds/refine.py +++ b/src/seeds/refine.py @@ -37,7 +37,7 @@ def refine_scenario(raw: str) -> RefinedScenario: constraints=("use only point-in-time TAQ-derived fields",), ), namespace="taq", - price_col="midret", + price_col="close", ) if "crypto" in text or any(token in text for token in ("btc", "eth", "coin")): diff --git a/src/training/grpo_reward.py b/src/training/grpo_reward.py index 6fd8335..66140c8 100644 --- a/src/training/grpo_reward.py +++ b/src/training/grpo_reward.py @@ -22,7 +22,7 @@ class FactorRewardRuntime: price_col: str archive: MinedFactorDatabase min_history: int = 30 - min_assets: int = 3 + min_assets: int = 8 reward_log_jsonl: Path | None = None diff --git a/src/training/train_grpo_qlora.py b/src/training/train_grpo_qlora.py index a3b245b..4f2ef4b 100644 --- a/src/training/train_grpo_qlora.py +++ b/src/training/train_grpo_qlora.py @@ -25,7 +25,7 @@ def main() -> int: parser.add_argument("--seed-expr", required=True) parser.add_argument("--seed-score", type=float, default=0.0) parser.add_argument("--crypto-panel", type=Path, default=DEFAULT_CRYPTO_PANEL) - parser.add_argument("--tickers", default="BTC-USD,ETH-USD,XRP-USD") + parser.add_argument("--tickers", default="ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD") parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--archive-jsonl", type=Path, default=Path("factor_lab/outputs/grpo/mined_factors.jsonl")) parser.add_argument("--max-prompt-length", type=int, default=1536) @@ -85,7 +85,7 @@ def main() -> int: price_col="close", archive=archive, min_history=30, - min_assets=max(3, min(5, len(frames))), + min_assets=8, reward_log_jsonl=args.reward_log_jsonl, ) ) diff --git a/src/verl_integration/build_dataset.py b/src/verl_integration/build_dataset.py index 6f943f1..e3e1184 100644 --- a/src/verl_integration/build_dataset.py +++ b/src/verl_integration/build_dataset.py @@ -9,6 +9,7 @@ from examples.build_seed_bank import crypto_frames_from_panel from examples.dsl_smoke_test import DEFAULT_CRYPTO_PANEL +from src.data import verify_crypto_panel from src.seeds import EvaluationWindow, FactorScenario, SeedCandidate, build_task_bank from src.verl_integration.dataset import build_verl_prompt_rows, write_verl_prompt_dataset @@ -25,15 +26,15 @@ def main() -> int: parser.add_argument("--window-start") parser.add_argument("--window-end") parser.add_argument("--crypto-panel", type=Path, default=DEFAULT_CRYPTO_PANEL) - parser.add_argument("--tickers", default="BTC-USD,ETH-USD,XRP-USD") + parser.add_argument("--tickers", default="ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD") parser.add_argument("--repeats", type=int, default=400) args = parser.parse_args() start = args.window_start end = args.window_end if args.namespace == "crypto" and (start is None or end is None): - panel = pd.read_pickle(args.crypto_panel) tickers = tuple(item.strip() for item in args.tickers.split(",") if item.strip()) or None + panel = verify_crypto_panel(args.crypto_panel, tickers=tickers) frames = crypto_frames_from_panel(panel, tickers=tickers) if not frames: raise ValueError("no crypto frames were loaded") diff --git a/src/verl_integration/reward_function.py b/src/verl_integration/reward_function.py index 6c8256d..162f4ba 100644 --- a/src/verl_integration/reward_function.py +++ b/src/verl_integration/reward_function.py @@ -63,7 +63,7 @@ def _runtime_from_metadata(metadata: dict[str, Any]) -> FactorRewardRuntime: panel_path = Path(os.getenv("FACTOR_LAB_CRYPTO_PANEL", "data/crypto/crypto_panel_clean.pkl")) tickers = tuple( item.strip() - for item in os.getenv("FACTOR_LAB_TICKERS", "BTC-USD,ETH-USD,XRP-USD").split(",") + for item in os.getenv("FACTOR_LAB_TICKERS", "ADA-USD,BNB-USD,BTC-USD,DOGE-USD,ETH-USD,LINK-USD,XLM-USD,XRP-USD").split(",") if item.strip() ) frames = _load_crypto_frames(str(panel_path), tickers) @@ -77,7 +77,7 @@ def _runtime_from_metadata(metadata: dict[str, Any]) -> FactorRewardRuntime: price_col="close", archive=archive, min_history=int(os.getenv("FACTOR_LAB_MIN_HISTORY", "30")), - min_assets=max(3, min(5, len(frames))), + min_assets=8, reward_log_jsonl=Path(reward_log) if reward_log else None, ) @@ -100,8 +100,8 @@ def _task_from_extra(extra: dict[str, Any]): ) start = str(window_payload.get("start") or os.getenv("FACTOR_LAB_WINDOW_START", "2019-01-01")) end = str(window_payload.get("end") or os.getenv("FACTOR_LAB_WINDOW_END", "2026-06-16")) - seed_expr = str(extra.get("seed_expr") or "ts_mean(crypto.returns(5))") - seed_score = float(extra.get("seed_score") or 0.0) + seed_expr = str(extra.get("seed_expr") or "div(ts_mean(crypto.volume(10)), ts_std(crypto.returns(30)))") + seed_score = float(extra.get("seed_score") if extra.get("seed_score") is not None else 0.38385972330719526) return build_task_bank( [SeedCandidate(seed_expr, seed_score, "")], scenario, diff --git a/src/verl_integration/verl_main.py b/src/verl_integration/verl_main.py index 3bf9173..73a4555 100644 --- a/src/verl_integration/verl_main.py +++ b/src/verl_integration/verl_main.py @@ -122,8 +122,8 @@ def build_verl_overrides(config: dict[str, Any]) -> dict[str, Any]: "n_gpus_per_node": int(trainer.get("n_gpus_per_node", 8)), "total_epochs": int(trainer.get("total_epochs", 1)), "save_freq": int(trainer.get("save_freq", 20)), - "test_freq": int(trainer.get("test_freq", -1)), - "val_before_train": False, + "test_freq": int(trainer.get("test_freq", trainer.get("save_freq", 20))), + "val_before_train": bool(trainer.get("val_before_train", True)), "critic_warmup": 0, "default_local_dir": str(Path(output_root) / "checkpoints" / experiment_name), "validation_data_dir": str(Path(output_root) / "validation_log"), diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 0c5756d..24725aa 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -1,7 +1,12 @@ import pandas as pd import pytest -from src.data.adapters import adapt_crypto_ohlcv, adapt_crsp_dsf_v2, adapt_taq_features +from src.data.adapters import ( + adapt_crsp_dsf_v2, + adapt_crypto_ohlcv, + adapt_taq_features, + verify_crypto_panel, +) def test_adapt_crsp_dsf_v2_maps_expected_columns(): @@ -57,3 +62,47 @@ def test_adapter_missing_columns_fail(): with pytest.raises(ValueError, match="missing required columns"): adapt_crypto_ohlcv(pd.DataFrame({"close": [1.0]})) + +def test_verify_crypto_panel_missing_file_raises_file_not_found(tmp_path): + missing = tmp_path / "nonexistent.pkl" + with pytest.raises(FileNotFoundError, match="crypto panel file not found"): + verify_crypto_panel(missing) + + +def test_verify_crypto_panel_missing_keys_raises_value_error(): + incomplete = { + "open": pd.DataFrame({"BTC": [10.0]}), + "close": pd.DataFrame({"BTC": [10.0]}), + } + with pytest.raises(ValueError, match="crypto panel missing required key"): + verify_crypto_panel(incomplete) + + +def test_verify_crypto_panel_missing_tickers_raises_value_error(): + panel = { + "open": pd.DataFrame({"BTC": [10.0]}), + "high": pd.DataFrame({"BTC": [11.0]}), + "low": pd.DataFrame({"BTC": [9.0]}), + "close": pd.DataFrame({"BTC": [10.0]}), + "volume": pd.DataFrame({"BTC": [100.0]}), + } + with pytest.raises(ValueError, match=r"crypto panel missing requested ticker\(s\): \['ETH'\]"): + verify_crypto_panel(panel, tickers=["BTC", "ETH"]) + + +def test_verify_crypto_panel_valid_panel(tmp_path): + panel = { + "open": pd.DataFrame({"BTC": [10.0], "ETH": [20.0]}), + "high": pd.DataFrame({"BTC": [11.0], "ETH": [21.0]}), + "low": pd.DataFrame({"BTC": [9.0], "ETH": [19.0]}), + "close": pd.DataFrame({"BTC": [10.0], "ETH": [20.0]}), + "volume": pd.DataFrame({"BTC": [100.0], "ETH": [200.0]}), + } + path = tmp_path / "panel.pkl" + pd.to_pickle(panel, path) + + loaded = verify_crypto_panel(path, tickers=["BTC", "ETH"]) + assert set(loaded.keys()) >= {"open", "high", "low", "close", "volume"} + assert set(loaded["close"].columns) == {"BTC", "ETH"} + + diff --git a/tests/test_baseline_rollout.py b/tests/test_baseline_rollout.py index 89eb939..63cdfde 100644 --- a/tests/test_baseline_rollout.py +++ b/tests/test_baseline_rollout.py @@ -1,10 +1,15 @@ from unittest.mock import patch +import pytest + from examples.baseline_rollout import run_crypto_baseline_rollout from examples.dsl_smoke_test import DEFAULT_CRYPTO_PANEL def test_run_crypto_baseline_rollout_with_mocked_llms(tmp_path): + if not DEFAULT_CRYPTO_PANEL.exists(): + pytest.skip("saved crypto project panel is not available") + raw_candidates = ( "div(ts_sum(crypto.returns(30)), ts_std(crypto.returns(30)))", "div(ts_mean(crypto.volume(7)), ts_mean(crypto.volume(30)))", diff --git a/tests/test_compared_approaches.py b/tests/test_compared_approaches.py index acf1887..a0fe63c 100644 --- a/tests/test_compared_approaches.py +++ b/tests/test_compared_approaches.py @@ -5,13 +5,10 @@ def test_run_compared_approaches_with_fake_generator(tmp_path): dates = pd.date_range("2024-01-01", periods=40, freq="D") + tickers = ["ADA-USD", "BNB-USD", "BTC-USD", "DOGE-USD", "ETH-USD", "LINK-USD", "XLM-USD", "XRP-USD"] panel = { "open": pd.DataFrame( - { - "BTC-USD": range(100, 140), - "ETH-USD": range(80, 120), - "XRP-USD": range(40, 80), - }, + {t: range(100 + i * 10, 140 + i * 10) for i, t in enumerate(tickers)}, index=dates, dtype=float, ) @@ -20,11 +17,7 @@ def test_run_compared_approaches_with_fake_generator(tmp_path): panel["low"] = panel["open"] - 1 panel["close"] = panel["open"] panel["volume"] = pd.DataFrame( - { - "BTC-USD": range(1000, 1040), - "ETH-USD": range(900, 940), - "XRP-USD": range(800, 840), - }, + {t: range(1000 + i * 50, 1040 + i * 50) for i, t in enumerate(tickers)}, index=dates, dtype=float, ) diff --git a/tests/test_oracle_seed_generation.py b/tests/test_oracle_seed_generation.py index d81432d..bb7004c 100644 --- a/tests/test_oracle_seed_generation.py +++ b/tests/test_oracle_seed_generation.py @@ -95,3 +95,15 @@ def test_generate_oracle_seed_candidates_calls_openrouter(monkeypatch): body = json.loads(req.data.decode("utf-8")) assert body["model"] == "test/model" assert out == ("sign(diff(crypto.close(2)))",) + + +def test_model_role_defaults_match_roles_config(): + import yaml + from src.rft import MinerConfig + + roles_path = Path("config/openrouter_llm_roles.yaml") + if roles_path.exists(): + roles = yaml.safe_load(roles_path.read_text()) + assert OpenRouterConfig().model == roles["oracle_llm"]["default_model"] + assert MinerConfig().model == roles["miner_llm"]["default_model"] + diff --git a/tests/test_paper_style_evaluation.py b/tests/test_paper_style_evaluation.py index 7d621b4..cd6e4ee 100644 --- a/tests/test_paper_style_evaluation.py +++ b/tests/test_paper_style_evaluation.py @@ -38,6 +38,33 @@ def test_select_decorrelated_factors_applies_threshold(): def test_evaluate_factor_library_returns_fused_validation_and_test_metrics(): + frames = { + f"asset_{i}": _crypto_frame([10 + i * 0.1 + j * 0.2 for j in range(8)]) + for i in range(8) + } + + result = evaluate_factor_library( + ["ts_mean(crypto.returns(2))", "neg(ts_mean(crypto.returns(2)))"], + "crypto", + frames, + price_col="close", + config=PostSelectionConfig( + validation_start="2024-01-01", + validation_end="2024-01-05", + test_start="2024-01-06", + test_end="2024-01-08", + top_k=1, + min_history=3, + min_assets=8, + ), + ) + + assert len(result.selected_exprs) == 1 + assert result.validation.valid_times > 0 + assert result.test.valid_times > 0 + + +def test_evaluate_factor_library_rejects_undersized_universe(): frames = { "winner": _crypto_frame([10, 11, 12, 13, 14, 15, 16, 17]), "middle": _crypto_frame([10, 10.5, 11, 11.5, 12, 12.5, 13, 13.5]), @@ -60,6 +87,6 @@ def test_evaluate_factor_library_returns_fused_validation_and_test_metrics(): ), ) - assert len(result.selected_exprs) == 1 - assert result.validation.valid_times > 0 - assert result.test.valid_times > 0 + assert len(result.selected_exprs) == 0 + assert result.validation.valid_times == 0 + assert result.test.valid_times == 0 diff --git a/tests/test_scoring.py b/tests/test_scoring.py index 2862ab0..20b656c 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -60,7 +60,7 @@ def test_directional_prediction_scores_known_signal(): assert result.score == result.directional_accuracy -def test_cross_sectional_rankic_scores_ranked_assets(): +def test_cross_sectional_rankic_rejects_undersized_universe(): dates = pd.date_range("2024-01-01", periods=6, freq="D") frames = { "winner": _crypto_frame([10, 11, 12, 13, 14, 15]).loc[dates], @@ -77,6 +77,25 @@ def test_cross_sectional_rankic_scores_ranked_assets(): min_assets=3, ) + assert result.valid_times == 0 + assert result.score == -1.0 + + +def test_cross_sectional_rankic_scores_ranked_assets_with_locked_minimum(): + dates = pd.date_range("2024-01-01", periods=6, freq="D") + frames = { + f"asset_{i}": _crypto_frame([10, 10 + i * 0.1 + 1, 10 + i * 0.2 + 2, 10 + i * 0.3 + 3, 10 + i * 0.4 + 4, 10 + i * 0.5 + 5]).loc[dates] + for i in range(8) + } + + result = score_cross_sectional_rankic( + "ts_mean(crypto.returns(2))", + "crypto", + frames, + price_col="close", + min_history=3, + ) + assert result.valid_times > 0 assert result.rank_ic_mean > 0.0 assert result.score > 0.0 diff --git a/tests/test_seed_construction.py b/tests/test_seed_construction.py index d9e9885..d526bd5 100644 --- a/tests/test_seed_construction.py +++ b/tests/test_seed_construction.py @@ -54,6 +54,7 @@ def test_refine_scenario_maps_taq_text_to_intraday_surface(): assert refined.scenario.benchmark == "intraday_microstructure_direction" assert refined.namespace == "taq" + assert refined.price_col == "close" def test_make_time_windows_splits_range(): @@ -185,3 +186,40 @@ def test_build_scenario_seed_bank_handles_single_asset_direction(): assert len(result.seeds) > 0 assert len(result.tasks) == len(result.seeds) * 2 + + +def test_taq_scenario_scoring_uses_close_price(): + refined = refine_scenario("TAQ microstructure intraday direction") + idx = pd.date_range("2024-01-01 09:30", periods=10, freq="1min") + taq_frame = pd.DataFrame( + { + "open": [100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0], + "high": [100.5, 101.5, 102.5, 103.5, 104.5, 105.5, 106.5, 107.5, 108.5, 109.5], + "low": [99.5, 100.5, 101.5, 102.5, 103.5, 104.5, 105.5, 106.5, 107.5, 108.5], + "close": [100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0], + "volume": [1000.0] * 10, + "spread": [0.02] * 10, + "midret": [0.001] * 10, + "imbalance": [0.1] * 10, + "trade_size": [100.0] * 10, + "trade_count": [10.0] * 10, + }, + index=idx, + ) + windows = [EvaluationWindow("2024-01-01", "2024-01-01")] + + result = build_scenario_seed_bank( + refined.scenario, + namespace=refined.namespace, + data=taq_frame, + price_col=refined.price_col, + windows=windows, + raw_candidates=("taq.spread(1)", "taq.imbalance(1)"), + min_history=2, + pool_config=SeedPoolConfig(top_k=2, quality_threshold=-1.0), + ) + assert refined.price_col == "close" + assert len(result.seeds) == 2 + for seed in result.seeds: + assert seed.score > -1.0 + diff --git a/tests/test_verl_integration.py b/tests/test_verl_integration.py index e99aa4d..6c212a7 100644 --- a/tests/test_verl_integration.py +++ b/tests/test_verl_integration.py @@ -7,6 +7,11 @@ from src.verl_integration.dataset import build_verl_prompt_rows from src.verl_integration.reward_bridge import FactorLabVerlRewardBridge from src.verl_integration.reward_function import reward_fn +from src.verl_integration.verl_main import ( + build_verl_config, + build_verl_overrides, + load_factor_verl_config, +) def _task(): @@ -56,9 +61,8 @@ def frame(vals): ) frames = { - "a": frame([10, 11, 12, 13, 14, 15, 16, 17]), - "b": frame([10, 10.5, 11, 11.5, 12, 12.5, 13, 13.5]), - "c": frame([10, 9.8, 9.6, 9.4, 9.2, 9.0, 8.8, 8.6]), + f"asset_{i}": frame([10, 10 + i * 0.1 + 1, 10 + i * 0.2 + 2, 10 + i * 0.3 + 3, 10 + i * 0.4 + 4, 10 + i * 0.5 + 5, 10 + i * 0.6 + 6, 10 + i * 0.7 + 7]) + for i in range(8) } panel = { "open": pd.concat({k: v["open"] for k, v in frames.items()}, axis=1), @@ -71,7 +75,7 @@ def frame(vals): panel_path = tmp_path / "panel.pkl" pd.to_pickle(panel, panel_path) monkeypatch.setenv("FACTOR_LAB_CRYPTO_PANEL", str(panel_path)) - monkeypatch.setenv("FACTOR_LAB_TICKERS", "a,b,c") + monkeypatch.setenv("FACTOR_LAB_TICKERS", ",".join(frames)) monkeypatch.setenv("FACTOR_LAB_ARCHIVE_JSONL", str(tmp_path / "archive.jsonl")) extra_info = build_verl_prompt_rows([_task()], namespace="crypto")[0].extra_info @@ -119,3 +123,17 @@ class Batch: assert reward_tensor[1, 3] == pytest.approx(-1.0) assert reward_tensor[0, 0] == 0.0 assert out["reward_extra_info"]["expr"][0] == "ts_mean(crypto.returns(2))" + + +def test_verl_config_merge_enables_validation(): + raw_config = load_factor_verl_config("config/verl_qwen3_14b_fullft_a100.yaml") + overrides = build_verl_overrides(raw_config) + + assert overrides["trainer"]["val_before_train"] is True + assert overrides["trainer"]["test_freq"] == 20 + assert overrides["trainer"]["save_freq"] == 20 + + merged = build_verl_config(raw_config, base_config="config/verl_ppo_trainer_base.yaml") + assert merged.trainer.val_before_train is True + assert merged.trainer.test_freq == 20 + assert merged.trainer.save_freq == 20