From 7d5c041a04cbf310ee7856a732bd4190e885fee4 Mon Sep 17 00:00:00 2001 From: c7w Date: Tue, 30 Jun 2026 20:37:28 +0800 Subject: [PATCH 1/5] Add Medal Score metric, v1-lite split, and released score artifacts - Medal Score: peer-relative gold/silver/bronze podium (normalized to [0,1]), reported on v1 (47 tasks) and the v1-lite subset (10 tasks). READMEs now lead with Medal Score; average rank stays on the website leaderboard. - leaderboard/: ship the frozen podium baselines (medal_podium.csv), published leaderboard (medal_leaderboard.csv), raw score table (exp1_models_raw.csv), a submission scorer (score_submission.py), and an example submission. Un-ignore leaderboard/*.csv. - v1-lite: add frontier_eval/conf/batch/v1_lite.yaml (10-task subset across all five categories, distinct families, gradual-improvement tasks). Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 2 + README.md | 38 +++++--- README_zh-CN.md | 31 ++++--- frontier_eval/conf/batch/v1_lite.yaml | 83 +++++++++++++++++ leaderboard/README.md | 62 +++++++++++++ leaderboard/exp1_models_raw.csv | 48 ++++++++++ leaderboard/medal_leaderboard.csv | 9 ++ leaderboard/medal_podium.csv | 48 ++++++++++ leaderboard/score_submission.py | 124 ++++++++++++++++++++++++++ leaderboard/submission_example.csv | 48 ++++++++++ 10 files changed, 469 insertions(+), 24 deletions(-) create mode 100644 frontier_eval/conf/batch/v1_lite.yaml create mode 100644 leaderboard/README.md create mode 100644 leaderboard/exp1_models_raw.csv create mode 100644 leaderboard/medal_leaderboard.csv create mode 100644 leaderboard/medal_podium.csv create mode 100644 leaderboard/score_submission.py create mode 100644 leaderboard/submission_example.csv diff --git a/.gitignore b/.gitignore index 8a45ae5c..bf3b45cf 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ benchmarks/Astrodynamics/MannedLunarLanding/results.txt benchmarks/Astrodynamics/MannedLunarLanding/outputlog.txt .vscode *.csv +!leaderboard/*.csv *.h5ad *.h5ad.tmp *.h5ad.tmp.* @@ -46,6 +47,7 @@ outputlog.txt frontier_eval/conf/batch/* !frontier_eval/conf/batch/example_matrix.yaml !frontier_eval/conf/batch/v1.yaml +!frontier_eval/conf/batch/v1_lite.yaml metrics.json artifacts.json debug-*.log diff --git a/README.md b/README.md index 0fdf60bf..98acef96 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,10 @@ Frontier-Eng is a benchmark for **generative optimization**: agents iteratively The benchmark currently covers **47 tasks** across computing, quantum information, operations research, robotics and control, optics and communications, and physical sciences. The project homepage and paper frame it as a missing evaluation axis between pass/fail coding benchmarks and real engineering work: most engineering problems start from a feasible baseline and reward iterative improvement, not one-shot correctness. +## News + +- **2026-06-30** — **New scoring metric: the Medal Score (gold/silver/bronze).** Alongside average rank, we now release a peer-relative *Medal Score* (normalized to `[0,1]`). On each task the top-3 best-feasible scores in the v1 snapshot are frozen as gold/silver/bronze baselines; a model earns 1.00 / 0.67 / 0.33 for reaching each, averaged over the task set, and is reported on both v1 (47 tasks) and v1-lite (10 tasks). It rewards only reaching each task's frontier and ignores negligible long-tail margins, making cross-task aggregation fairer. Per-task podium values and the leaderboard live in [`leaderboard/`](leaderboard/README.md). +- **2026-06-30** — **`v1-lite` released.** A 10-task representative subset of `v1` covering all five categories with distinct benchmark families, selected for tasks whose scores climb gradually under budget (not one-shot-saturated or all-or-nothing). Run it with `frontier_eval/conf/batch/v1_lite.yaml`. ## 0. Host Requirements @@ -97,6 +101,14 @@ bash scripts/batch/validate_v1_task_envs.sh That command runs the batch config for the `v1` problem set with `algorithm.iterations=0`, which evaluates each task's shipped baseline instead of asking an LLM to improve it. +### `v1-lite` quick subset + +For fast iteration and ablations, use the 10-task `v1-lite` matrix +([`frontier_eval/conf/batch/v1_lite.yaml`](frontier_eval/conf/batch/v1_lite.yaml)) +instead of the full `v1` config. It spans all five categories with distinct +benchmark families and favors tasks whose scores improve gradually under budget, +so a short run still exercises the full optimization loop. + If you want the full `v1` problem set with normal optimization runs later, see [`run.md`](run.md). ## Where To Go Next @@ -108,18 +120,20 @@ If you want the full `v1` problem set with normal optimization runs later, see [ ## Leaderboard -Detailed leaderboard: [lab.einsia.ai/frontier-eng/leaderboard.html](https://lab.einsia.ai/frontier-eng/leaderboard.html) - -| Rank | Model | Average Rank | -| :--: | :--- | --: | -| 1 | GPT-5.4 | 3.54 | -| 2 | Claude Opus 4.6 | 3.63 | -| 3 | GLM-5 | 4.34 | -| 4 | DeepSeek V3.2 | 4.76 | -| 5 | Gemini 3.1 Pro Preview | 5.53 | -| 6 | Grok 4.20 | 5.82 | -| 7 | SEED 2.0 Pro | 5.86 | -| 8 | Qwen3 Coder Next | 6.71 | +Detailed leaderboard (incl. average rank): [lab.einsia.ai/frontier-eng/leaderboard](https://lab.einsia.ai/frontier-eng/leaderboard). Released score tables and the per-task medal podium: [`leaderboard/`](leaderboard/README.md). + +**Medal Score** (gold/silver/bronze podium, higher is better, normalized to `[0,1]` = mean per-task podium credit). On each task the top-3 best scores in the **v1 snapshot (2026-04-14)** are frozen as gold/silver/bronze baselines; a model earns 1.00 / 0.67 / 0.33 for reaching each. Reported on both the full **v1** set (47 tasks) and the **v1-lite** subset (10 tasks); gold/silver/bronze counts are for v1 (see [`leaderboard/`](leaderboard/README.md)): + +| Rank | Model | Medal (v1) | Medal (v1-lite) | 🥇 | 🥈 | 🥉 | +| :--: | :--- | --: | --: | --: | --: | --: | +| 1 | GPT-5.4 | 0.596 | 0.667 | 24 | 5 | 2 | +| 2 | Claude Opus 4.6 | 0.490 | 0.501 | 9 | 18 | 6 | +| 3 | GLM-5 | 0.312 | 0.233 | 4 | 10 | 12 | +| 4 | DeepSeek V3.2 | 0.248 | 0.166 | 3 | 9 | 8 | +| 5 | Gemini 3.1 Pro Preview | 0.213 | 0.200 | 3 | 6 | 9 | +| 6 | Seed 2.0 Pro | 0.185 | 0.100 | 3 | 7 | 3 | +| 7 | Grok 4.20 | 0.184 | 0.133 | 3 | 6 | 5 | +| 8 | Qwen3 Coder Next | 0.121 | 0.000 | 3 | 3 | 2 | ## Contributing diff --git a/README_zh-CN.md b/README_zh-CN.md index 75ddd213..26e66e5b 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -8,6 +8,11 @@ Frontier-Eng 是一个面向 **generative optimization** 的 benchmark：Agent 当前版本包含 **47 个任务**，覆盖计算系统、量子信息、运筹优化、机器人控制、光学通信、物理与工程设计。主页和论文的核心观点是：真实工程问题通常从一个可行 baseline 出发，价值来自持续优化，而不是 pass/fail。 +## News + +- **2026-06-30** — **新增评测指标：金银铜 Medal Score。** 在 average rank 之外，我们发布同侪相对的 *Medal Score*（归一化到 `[0,1]`）：每道题取 v1 snapshot 中最好的前三名分数冻结为金/银/铜 baseline，模型达到金/银/铜分别得 1.00 / 0.67 / 0.33，对题集求均值；同时汇报 v1（47 题）与 v1-lite（10 题）。它只奖励"达到该题最前沿（领奖台）"，忽略长尾里可忽略的微小差距，使跨题汇总更公平。每题 podium 分数与榜单见 [`leaderboard/`](leaderboard/README.md)。 +- **2026-06-30** — **发布 `v1-lite`。** `v1` 的 10 题代表性子集，覆盖全部五大类、family 各不相同，专选"分数随预算逐步提升（而非一步做满或非高即低）"的题，配置见 `frontier_eval/conf/batch/v1_lite.yaml`。 + ## 这个 benchmark 在测什么 和传统 agent benchmark 相比，Frontier-Eng 更关注三件事： @@ -110,18 +115,20 @@ bash scripts/batch/validate_v1_task_envs.sh ## Leaderboard -详细榜单见 [lab.einsia.ai/frontier-eng/leaderboard.html](https://lab.einsia.ai/frontier-eng/leaderboard.html)。 - -| 排名 | Model | Average Rank | -| :--: | :--- | --: | -| 1 | Claude Opus 4.6 | 3.18 | -| 2 | GLM-5 | 4.02 | -| 3 | DeepSeek V3.2 | 4.41 | -| 4 | Gemini 3.1 Pro Preview | 5.34 | -| 5 | Grok 4.20 | 5.60 | -| 6 | SEED 2.0 Pro | 5.63 | -| 7 | GPT-5.4 | 5.68 | -| 8 | Qwen3 Coder Next | 6.68 | +详细榜单（含 average rank）见 [lab.einsia.ai/frontier-eng/leaderboard](https://lab.einsia.ai/frontier-eng/leaderboard)。发布的分数表与每题金银铜 podium 见 [`leaderboard/`](leaderboard/README.md)。 + +**Medal Score**（金银铜 podium，越高越好，归一化到 `[0,1]`，即每题领奖台得分的均值）。每题取 **v1 snapshot (2026-04-14)** 的前三名分数冻结为金/银/铜 baseline，模型达到金/银/铜分别得 1.00 / 0.67 / 0.33。同时汇报 **v1**（47 题）与 **v1-lite**（10 题）两个集合；金银铜次数为 v1（`gpt-5.4` 采用其 47 题全量重测结果）： + +| 排名 | Model | Medal (v1) | Medal (v1-lite) | 🥇 | 🥈 | 🥉 | +| :--: | :--- | --: | --: | --: | --: | --: | +| 1 | GPT-5.4 | 0.596 | 0.667 | 24 | 5 | 2 | +| 2 | Claude Opus 4.6 | 0.490 | 0.501 | 9 | 18 | 6 | +| 3 | GLM-5 | 0.312 | 0.233 | 4 | 10 | 12 | +| 4 | DeepSeek V3.2 | 0.248 | 0.166 | 3 | 9 | 8 | +| 5 | Gemini 3.1 Pro Preview | 0.213 | 0.200 | 3 | 6 | 9 | +| 6 | Seed 2.0 Pro | 0.185 | 0.100 | 3 | 7 | 3 | +| 7 | Grok 4.20 | 0.184 | 0.133 | 3 | 6 | 5 | +| 8 | Qwen3 Coder Next | 0.121 | 0.000 | 3 | 3 | 2 | ## 贡献 diff --git a/frontier_eval/conf/batch/v1_lite.yaml b/frontier_eval/conf/batch/v1_lite.yaml new file mode 100644 index 00000000..284cc45a --- /dev/null +++ b/frontier_eval/conf/batch/v1_lite.yaml @@ -0,0 +1,83 @@ +# Frontier-Eng v1-lite batch matrix. +# +# v1-lite is a 10-task representative subset of the full 47-task v1 set, chosen +# for fast iteration and ablation. Tasks are selected to (a) cover all five +# engineering categories with distinct benchmark families and (b) reward +# incremental optimization -- their best-feasible scores climb gradually with +# budget instead of saturating in one shot or splitting all-or-nothing. The +# selection rule (top-graduality task per family, 2 per category) is documented +# and reproduced by Einsia-lab/scripts/build_frontier_metrics.py. +# +# Overrides for each task mirror frontier_eval/conf/batch/v1.yaml exactly. +# +version: 1 +tasks: +- name: unified + label: ComputerSystems/MallocLab + overrides: + - task.benchmark=ComputerSystems/MallocLab +- name: unified + label: QuantumComputing/task_01_routing_qftentangled + overrides: + - task.benchmark=QuantumComputing/task_01_routing_qftentangled + - task.runtime.env_name=frontier-v1-main +- name: unified + label: JobShop/abz + overrides: + - task.benchmark=JobShop/abz + - task.runtime.python_path=uv-env:frontier-v1-main +- name: unified + label: InventoryOptimization/disruption_eoqd + overrides: + - task.benchmark=InventoryOptimization/disruption_eoqd + - task.runtime.env_name=frontier-v1-main +- name: unified + label: EnergyStorage/BatteryFastChargingSPMe + overrides: + - task.benchmark=EnergyStorage/BatteryFastChargingSPMe +- name: unified + label: Robotics/RobotArmCycleTimeOptimization + overrides: + - task.benchmark=Robotics/RobotArmCycleTimeOptimization + - task.runtime.env_name=frontier-v1-main + - algorithm.oe.evaluator.timeout=600 +- name: unified + label: Optics/holographic_multiplane_focusing + overrides: + - task.benchmark=Optics/holographic_multiplane_focusing + - task.runtime.env_name=frontier-v1-main + - algorithm.oe.evaluator.timeout=600 +- name: unified + label: WirelessChannelSimulation/HighReliableSimulation + overrides: + - task.benchmark=WirelessChannelSimulation/HighReliableSimulation +- name: unified + label: ReactionOptimisation/snar_multiobjective + overrides: + - task.benchmark=ReactionOptimisation/snar_multiobjective + - task.runtime.python_path=uv-env:frontier-v1-summit + - algorithm.oe.evaluator.timeout=600 +- name: unified + label: StructuralOptimization/TopologyOptimization + overrides: + - task.benchmark=StructuralOptimization/TopologyOptimization + - task.runtime.env_name=frontier-v1-main +algorithms: +- name: openevolve + overrides: + - algorithm.iterations=100 + - algorithm.oe.evaluator.timeout=300 +llms: +- name: v1 + api_base: ${oc.env:OPENAI_API_BASE,https://api.openai.com/v1} + model: ${oc.env:OPENAI_MODEL,gpt-4o-mini} + api_key_env: OPENAI_API_KEY +llm_config: openai_compatible +common_overrides: +- llm.temperature=0.7 +- llm.timeout=60 +run: + name: v1_lite + base_dir: runs/batch + max_parallel: 4 + fail_fast: false diff --git a/leaderboard/README.md b/leaderboard/README.md new file mode 100644 index 00000000..25e039b9 --- /dev/null +++ b/leaderboard/README.md @@ -0,0 +1,62 @@ +# Leaderboard & Medal Score + +Released score artifacts for the Frontier-Eng `v1` set (Experiment 1: foundation +models under `openevolve`, 100 iterations, same initial programs and frozen +verifiers; `gpt-5.4` uses its full 47-task retest). + +| File | Contents | +|---|---| +| `medal_podium.csv` | Frozen per-task **gold / silver / bronze** threshold scores and the model that set each. | +| `medal_leaderboard.csv` | Per-model normalized **Medal Score** on v1 and v1-lite, with gold/silver/bronze counts. | +| `exp1_models_raw.csv` | Best-feasible score of each model on each of the 47 tasks (higher is better); source of the podium. | +| `score_submission.py` | Scores a new submission against the frozen podium. | +| `submission_example.csv` | Example submission (claude-opus-4.6) — scoring it reproduces its leaderboard line. | + +## Medal Score + +On each task the top-3 best scores in the **v1 snapshot (2026-04-14)** are frozen +as peer baselines — gold (1st), silver (2nd), bronze (3rd). A model earns +**1.00** for reaching the gold score, **0.67** for silver, **0.33** for bronze, +otherwise 0; its Medal Score is the **mean** of this credit over a task set +(normalized to `[0,1]`). It credits only reaching each task's frontier (the +podium) and ignores negligible margins in the long tail — a fairer aggregate +than crediting every ordinal rank when the question is "how often does a model +reach the best-known solutions?" We report it on both the full **v1** set +(47 tasks) and the **v1-lite** subset (10 tasks). (Average rank and other +diagnostics are on the [website leaderboard](https://lab.einsia.ai/frontier-eng/leaderboard).) + +> `gpt-oss-120b` is part of the paper's 9-model rank tables, but its per-task raw +> scores were not retained; the released podium is therefore computed over the 8 +> models with available raw scores. + +## Medal leaderboard (normalized; gold/silver/bronze counts are for v1) + +| Rank | Model | Medal (v1) | Medal (v1-lite) | 🥇 | 🥈 | 🥉 | +| :--: | :--- | --: | --: | --: | --: | --: | +| 1 | gpt-5.4 | 0.596 | 0.667 | 24 | 5 | 2 | +| 2 | claude-opus-4.6 | 0.490 | 0.501 | 9 | 18 | 6 | +| 3 | glm-5 | 0.312 | 0.233 | 4 | 10 | 12 | +| 4 | deepseek-v3.2 | 0.248 | 0.166 | 3 | 9 | 8 | +| 5 | gemini-3.1-pro-preview | 0.213 | 0.200 | 3 | 6 | 9 | +| 6 | seed-2.0-pro | 0.185 | 0.100 | 3 | 7 | 3 | +| 7 | grok-4.20 | 0.184 | 0.133 | 3 | 6 | 5 | +| 8 | qwen3-coder-next | 0.121 | 0.000 | 3 | 3 | 2 | + +## Score your own model + +Put your model's best score per task in a CSV (`Task,Score`, one row per task, +task names as in `medal_podium.csv`), then: + +```bash +python leaderboard/score_submission.py your_scores.csv +# -> Medal Score (v1, 47 tasks) : 0.xxx (gold .., silver .., bronze ..) +# Medal Score (v1-lite, 10 tasks) : 0.xxx +``` + +Sanity check (reproduces claude-opus-4.6's line, 0.490 / 0.501): + +```bash +python leaderboard/score_submission.py leaderboard/submission_example.csv +``` + +Interactive view: [lab.einsia.ai/frontier-eng/leaderboard](https://lab.einsia.ai/frontier-eng/leaderboard) diff --git a/leaderboard/exp1_models_raw.csv b/leaderboard/exp1_models_raw.csv new file mode 100644 index 00000000..1b720953 --- /dev/null +++ b/leaderboard/exp1_models_raw.csv @@ -0,0 +1,48 @@ +Task,Baseline,claude-opus-4.6_best,deepseek-v3.2_best,gemini-3.1-pro-preview_best,glm-5_best,gpt-5.4_best,grok-4.20_best,qwen3-coder-next_best,seed-2.0-pro_best,,,,,,,,,, +Aerodynamics_CarAerodynamicsSensing,0.9617,0.9624,0.9632,0.9632,0.9628,0.9630695838481188,0.9624,0.9632,0.9624,,,,,,,,,, +Astrodynamics_MannedLunarLanding,4577.437,6027.3126,6079.2455,4674.9462,6839.0331,6660.942428,4577.437,4577.437,4733.0435,,,,,,,,,, +ComputerSystems_MallocLab,28,96,53,48,86,28,57,32,38,,,,,,,,,, +Cryptographic_AES-128,7.5209,11.8617,12.4591,10.2396,7.9669,39.824967043300866,10.8615,5.5501,7.9481,,,,,,,,,, +Cryptographic_SHA-256,9.8274,16.7955,9.718,9.942,15.1655,26.34045367870492,17.2504,9.8475,15.2838,,,,,,,,,, +Cryptographic_SHA3-256,16.0932,17.4003,17.0749,16.2255,17.5778,37.44512785396786,16.0594,16.5292,18.3478,,,,,,,,,, +EnergyStorage_BatteryFastChargingProfile,71.2806,120.8025,111.4518,116.6532,118.7678,121.99136502281442,99.6875,89.8416,115.6882,,,,,,,,,, +EnergyStorage_BatteryFastChargingSPMe,66.1636,71.8225,91.0079,92.3198,78.0896,122.94304361063023,76.4657,79.0273,76.4122,,,,,,,,,, +EngDesign,1.3571,1.3571,21.7143,27,25.5714,1.3571428571428572,27,25.5714,27,,,,,,,,,, +InventoryOptimization_disruption_eoqd,0.3642,0.6473,0.6381,0.639,0.6303,1,0.6359,0.6225,0.6321,,,,,,,,,, +InventoryOptimization_finite_horizon_dp,0.3673,0.9596,0.8025,0.7559,0.7965,0.9606835281410351,0.8547,0.4413,0.7323,,,,,,,,,, +InventoryOptimization_general_meio,0.1825,0.9929,0.9893,0.9839,0.9165,0.9999999999999999,0.9236,0.7819,0.6973,,,,,,,,,, +InventoryOptimization_joint_replenishment,0.3034,0.8822,0.8822,0.8822,0.8822,1,0.8822,0.8821,0.8822,,,,,,,,,, +InventoryOptimization_tree_gsm_safety_stock,0.3813,0.75,0.6606,0.6606,0.6606,1,0.6606,0.6606,0.6606,,,,,,,,,, +JobShop_abz,80.5042,96.1035,88.3614,86.751,88.4924,91.23143065488635,87.6717,85.603,86.672,,,,,,,,,, +JobShop_swv,81.6325,89.4966,82.3575,82.3141,87.1611,87.33430826602005,85.5068,82.6129,82.4153,,,,,,,,,, +JobShop_ta,78.8,90.8322,84.9043,85.7065,86.8095,86.16070055174835,84.9136,85.5489,83.9694,,,,,,,,,, +KernelEngineering_FlashAttention,55.2957,983.5001,987.2034,991.8896,381.6257,182687.44188255747,324.919,525.5567,1218.5163,,,,,,,,,, +KernelEngineering_MLA,0.7828,1000.3859,0.8936,1253.2017,20.1972,1132.0659025372765,19.8651,0.9271,19.987,,,,,,,,,, +KernelEngineering_TriMul,47.1274,357.1636,85.5923,54.5774,110.8785,47.88292233116043,165.0294,49.1232,84.9069,,,,,,,,,, +Optics_adaptive_fault_tolerant_fusion,0.3959,0.6398,0.64,0.6398,0.6398,0.455046169,0.6398,0.6398,0.6398,,,,,,,,,, +Optics_adaptive_temporal_smooth_control,0.3152,0.8419,0.8419,0.8419,0.8417,0.841880414,0.842,0.8421,0.8421,,,,,,,,,, +Optics_fiber_guardband_spectrum_packing,0.3861,0.6692,0.657,0.6629,0.6692,0.6754289215686274,0.6629,0.657,0.657,,,,,,,,,, +Optics_fiber_mcs_power_scheduling,0.3297,0.6542,0.5182,0.4796,0.6491,0.6608370951757289,0.4557,0.4458,0.6491,,,,,,,,,, +Optics_fiber_wdm_channel_power_allocation,0.3255,0.6675,0.6679,0.6619,0.6686,0.6964207451370852,0.6664,0.6666,0.6654,,,,,,,,,, +Optics_holographic_multifocus_power_ratio,0.3927,0.8072,0.8265,0.5368,0.711,0.9999999999663148,0.4058,0.5875,0.5626,,,,,,,,,, +Optics_holographic_multiplane_focusing,0.3302,0.6002,0.7196,0.4398,0.4516,0.9999999999886867,0.474,0.5631,0.5303,,,,,,,,,, +Optics_phase_dammann_uniform_orders,26.8969,99.7995,97.3436,97.9498,97.8709,99.99999999999999,94.4055,95.9998,69.0576,,,,,,,,,, +Optics_phase_fourier_pattern_holography,32.6457,82.1276,74.5838,76.6371,76.0127,99.99998936790779,74.217,67.3393,72.4578,,,,,,,,,, +PyPortfolioOpt_robust_mvo_rebalance,32.9804,99.9946,84.941,77.165,82.8015,99.99460428985267,99.983,85.5194,83.0681,,,,,,,,,, +QuantumComputing_task_01_routing_qftentangled,0.209,5.0479,3.6155,0.209,3.7681,6.507945106686525,3.7655,3.2471,3.6783,,,,,,,,,, +QuantumComputing_task_02_clifford_t_synthesis,1.7134,1.6633,1.7134,1.7134,7.4236,1.7133669376223557,1.6633,1.7134,1.7134,,,,,,,,,, +QuantumComputing_task_03_cross_target_qaoa,2.4149,2.5781,5.103,2.9782,5.0301,2.4149139615375192,2.6363,2.4517,2.9782,,,,,,,,,, +ReactionOptimisation_mit_case1_mixed,87.3082,98.6621,98.6041,96.5437,95.9314,98.66214557690091,87.3082,95.3732,95.4297,,,,,,,,,, +ReactionOptimisation_reizman_suzuki_pareto,63.5202,82.3427,82.0329,79.473,82.9901,82.24612252072882,63.5202,81.4666,79.7011,,,,,,,,,, +ReactionOptimisation_snar_multiobjective,57.5234,87.3657,82.7881,80.1521,81.7614,100,72.3909,72.8477,79.427,,,,,,,,,, +Robotics_DynamicObstacleAvoidanceNavigation,0.0722,0.086,0.0856,0.0834,0.0857,0.08571428571428559,0.0817,0.0765,0.0855,,,,,,,,,, +Robotics_PIDTuning,0.0366,0.1632,0.151,0.1521,0.1515,0.1511172761100511,0.1585,0.1422,0.1514,,,,,,,,,, +Robotics_QuadrupedGaitOptimization,0.0218,0.0219,0.0749,0.0218,0.1085,0.022154337029969478,0.0227,0.0232,0.0218,,,,,,,,,, +Robotics_RobotArmCycleTimeOptimization,0.2922,0.4158,0.3923,0.4305,0.4219,0.4356212836221511,0.3923,0.3155,0.3256,,,,,,,,,, +Robotics_UAVInspectionCoverageWithWind,28.8519,28.8519,38.8024,28.8519,35.1121,30.121714802877325,55.9109,32.8468,32.1552,,,,,,,,,, +SingleCellAnalysis_predict_modality,0.5467,0.5467,0.5467,0.5467,0.5467,1,0.5467,0.5467,0.5467,,,,,,,,,, +StructuralOptimization_ISCSO2015,-5401.589,-968.4567,-1120.212,-5401.589,-1139.3354,-5401.589002,-1318.7566,-1308.2575,-1302.2288,,,,,,,,,, +StructuralOptimization_ISCSO2023,-77813242.9,-16477799.48,-55182772.3,-20092179.33,-17840974.17,-77813242.9,-30028112.28,-66126744.97,-42625693.78,,,,,,,,,, +StructuralOptimization_TopologyOptimization,-195.9153,-190.1498,-190.3706,-189.3039,-188.4673,-195.9152621,-185.7983,-192.8488,-190.0603,,,,,,,,,, +SustainableDataCenterControl_hand_written_control,8.3294,21.5657,15.292,12.9088,19.5978,8.5903,14.2432,30.1873,29.2868,,,,,,,,,, +WirelessChannelSimulation_HighReliableSimulation,192.5193,292.3228,291.9451,232.9071,248.0119,231.22403446412542,245.7082,259.9776,304.0437,,,,,,,,,, \ No newline at end of file diff --git a/leaderboard/medal_leaderboard.csv b/leaderboard/medal_leaderboard.csv new file mode 100644 index 00000000..20b35004 --- /dev/null +++ b/leaderboard/medal_leaderboard.csv @@ -0,0 +1,9 @@ +Rank,Model,Medal_v1,Medal_v1lite,Gold,Silver,Bronze +1,gpt-5.4,0.596,0.667,24,5,2 +2,claude-opus-4.6,0.49,0.501,9,18,6 +3,glm-5,0.312,0.233,4,10,12 +4,deepseek-v3.2,0.248,0.166,3,9,8 +5,gemini-3.1-pro-preview,0.213,0.2,3,6,9 +6,seed-2.0-pro,0.185,0.1,3,7,3 +7,grok-4.20,0.184,0.133,3,6,5 +8,qwen3-coder-next,0.121,0.0,3,3,2 diff --git a/leaderboard/medal_podium.csv b/leaderboard/medal_podium.csv new file mode 100644 index 00000000..8059fced --- /dev/null +++ b/leaderboard/medal_podium.csv @@ -0,0 +1,48 @@ +Task,Baseline,Gold,Gold_model,Silver,Silver_model,Bronze,Bronze_model +Aerodynamics_CarAerodynamicsSensing,0.9617,0.9632,deepseek-v3.2/gemini-3.1-pro-preview/qwen3-coder-next,0.9632,deepseek-v3.2/gemini-3.1-pro-preview/qwen3-coder-next,0.9632,deepseek-v3.2/gemini-3.1-pro-preview/qwen3-coder-next +Astrodynamics_MannedLunarLanding,4577.437,6839.0331,glm-5,6660.942428,gpt-5.4,6079.2455,deepseek-v3.2 +ComputerSystems_MallocLab,28,96.0,claude-opus-4.6,86.0,glm-5,57.0,grok-4.20 +Cryptographic_AES-128,7.5209,39.824967043300866,gpt-5.4,12.4591,deepseek-v3.2,11.8617,claude-opus-4.6 +Cryptographic_SHA-256,9.8274,26.34045367870492,gpt-5.4,17.2504,grok-4.20,16.7955,claude-opus-4.6 +Cryptographic_SHA3-256,16.0932,37.44512785396786,gpt-5.4,18.3478,seed-2.0-pro,17.5778,glm-5 +EnergyStorage_BatteryFastChargingProfile,71.2806,121.99136502281442,gpt-5.4,120.8025,claude-opus-4.6,118.7678,glm-5 +EnergyStorage_BatteryFastChargingSPMe,66.1636,122.94304361063023,gpt-5.4,92.3198,gemini-3.1-pro-preview,91.0079,deepseek-v3.2 +EngDesign,1.3571,27.0,gemini-3.1-pro-preview/grok-4.20/seed-2.0-pro,27.0,gemini-3.1-pro-preview/grok-4.20/seed-2.0-pro,27.0,gemini-3.1-pro-preview/grok-4.20/seed-2.0-pro +InventoryOptimization_disruption_eoqd,0.3642,1.0,gpt-5.4,0.6473,claude-opus-4.6,0.639,gemini-3.1-pro-preview +InventoryOptimization_finite_horizon_dp,0.3673,0.9606835281410351,gpt-5.4,0.9596,claude-opus-4.6,0.8547,grok-4.20 +InventoryOptimization_general_meio,0.1825,0.9999999999999999,gpt-5.4,0.9929,claude-opus-4.6,0.9893,deepseek-v3.2 +InventoryOptimization_joint_replenishment,0.3034,1.0,gpt-5.4,0.8822,claude-opus-4.6/deepseek-v3.2/gemini-3.1-pro-preview/glm-5/grok-4.20/seed-2.0-pro,0.8822,claude-opus-4.6/deepseek-v3.2/gemini-3.1-pro-preview/glm-5/grok-4.20/seed-2.0-pro +InventoryOptimization_tree_gsm_safety_stock,0.3813,1.0,gpt-5.4,0.75,claude-opus-4.6,0.6606,deepseek-v3.2/gemini-3.1-pro-preview/glm-5/grok-4.20/qwen3-coder-next/seed-2.0-pro +JobShop_abz,80.5042,96.1035,claude-opus-4.6,91.23143065488635,gpt-5.4,88.4924,glm-5 +JobShop_swv,81.6325,89.4966,claude-opus-4.6,87.33430826602005,gpt-5.4,87.1611,glm-5 +JobShop_ta,78.8,90.8322,claude-opus-4.6,86.8095,glm-5,86.16070055174835,gpt-5.4 +KernelEngineering_FlashAttention,55.2957,182687.44188255747,gpt-5.4,1218.5163,seed-2.0-pro,991.8896,gemini-3.1-pro-preview +KernelEngineering_MLA,0.7828,1253.2017,gemini-3.1-pro-preview,1132.0659025372765,gpt-5.4,1000.3859,claude-opus-4.6 +KernelEngineering_TriMul,47.1274,357.1636,claude-opus-4.6,165.0294,grok-4.20,110.8785,glm-5 +Optics_adaptive_fault_tolerant_fusion,0.3959,0.64,deepseek-v3.2,0.6398,claude-opus-4.6/gemini-3.1-pro-preview/glm-5/grok-4.20/qwen3-coder-next/seed-2.0-pro,0.6398,claude-opus-4.6/gemini-3.1-pro-preview/glm-5/grok-4.20/qwen3-coder-next/seed-2.0-pro +Optics_adaptive_temporal_smooth_control,0.3152,0.8421,qwen3-coder-next/seed-2.0-pro,0.8421,qwen3-coder-next/seed-2.0-pro,0.842,grok-4.20 +Optics_fiber_guardband_spectrum_packing,0.3861,0.6754289215686274,gpt-5.4,0.6692,claude-opus-4.6/glm-5,0.6692,claude-opus-4.6/glm-5 +Optics_fiber_mcs_power_scheduling,0.3297,0.6608370951757289,gpt-5.4,0.6542,claude-opus-4.6,0.6491,glm-5/seed-2.0-pro +Optics_fiber_wdm_channel_power_allocation,0.3255,0.6964207451370852,gpt-5.4,0.6686,glm-5,0.6679,deepseek-v3.2 +Optics_holographic_multifocus_power_ratio,0.3927,0.9999999999663148,gpt-5.4,0.8265,deepseek-v3.2,0.8072,claude-opus-4.6 +Optics_holographic_multiplane_focusing,0.3302,0.9999999999886867,gpt-5.4,0.7196,deepseek-v3.2,0.6002,claude-opus-4.6 +Optics_phase_dammann_uniform_orders,26.8969,99.99999999999999,gpt-5.4,99.7995,claude-opus-4.6,97.9498,gemini-3.1-pro-preview +Optics_phase_fourier_pattern_holography,32.6457,99.99998936790779,gpt-5.4,82.1276,claude-opus-4.6,76.6371,gemini-3.1-pro-preview +PyPortfolioOpt_robust_mvo_rebalance,32.9804,99.99460428985267,gpt-5.4,99.9946,claude-opus-4.6,99.983,grok-4.20 +QuantumComputing_task_01_routing_qftentangled,0.209,6.507945106686525,gpt-5.4,5.0479,claude-opus-4.6,3.7681,glm-5 +QuantumComputing_task_02_clifford_t_synthesis,1.7134,7.4236,glm-5,1.7134,deepseek-v3.2/gemini-3.1-pro-preview/qwen3-coder-next/seed-2.0-pro,1.7134,deepseek-v3.2/gemini-3.1-pro-preview/qwen3-coder-next/seed-2.0-pro +QuantumComputing_task_03_cross_target_qaoa,2.4149,5.103,deepseek-v3.2,5.0301,glm-5,2.9782,gemini-3.1-pro-preview/seed-2.0-pro +ReactionOptimisation_mit_case1_mixed,87.3082,98.66214557690091,gpt-5.4,98.6621,claude-opus-4.6,98.6041,deepseek-v3.2 +ReactionOptimisation_reizman_suzuki_pareto,63.5202,82.9901,glm-5,82.3427,claude-opus-4.6,82.24612252072882,gpt-5.4 +ReactionOptimisation_snar_multiobjective,57.5234,100.0,gpt-5.4,87.3657,claude-opus-4.6,82.7881,deepseek-v3.2 +Robotics_DynamicObstacleAvoidanceNavigation,0.0722,0.086,claude-opus-4.6,0.08571428571428559,gpt-5.4,0.0857,glm-5 +Robotics_PIDTuning,0.0366,0.1632,claude-opus-4.6,0.1585,grok-4.20,0.1521,gemini-3.1-pro-preview +Robotics_QuadrupedGaitOptimization,0.0218,0.1085,glm-5,0.0749,deepseek-v3.2,0.0232,qwen3-coder-next +Robotics_RobotArmCycleTimeOptimization,0.2922,0.4356212836221511,gpt-5.4,0.4305,gemini-3.1-pro-preview,0.4219,glm-5 +Robotics_UAVInspectionCoverageWithWind,28.8519,55.9109,grok-4.20,38.8024,deepseek-v3.2,35.1121,glm-5 +SingleCellAnalysis_predict_modality,0.5467,1.0,gpt-5.4,0.5467,claude-opus-4.6/deepseek-v3.2/gemini-3.1-pro-preview/glm-5/grok-4.20/qwen3-coder-next/seed-2.0-pro,0.5467,claude-opus-4.6/deepseek-v3.2/gemini-3.1-pro-preview/glm-5/grok-4.20/qwen3-coder-next/seed-2.0-pro +StructuralOptimization_ISCSO2015,-5401.589,-968.4567,claude-opus-4.6,-1120.212,deepseek-v3.2,-1139.3354,glm-5 +StructuralOptimization_ISCSO2023,-77813242.9,-16477799.48,claude-opus-4.6,-17840974.17,glm-5,-20092179.33,gemini-3.1-pro-preview +StructuralOptimization_TopologyOptimization,-195.9153,-185.7983,grok-4.20,-188.4673,glm-5,-189.3039,gemini-3.1-pro-preview +SustainableDataCenterControl_hand_written_control,8.3294,30.1873,qwen3-coder-next,29.2868,seed-2.0-pro,21.5657,claude-opus-4.6 +WirelessChannelSimulation_HighReliableSimulation,192.5193,304.0437,seed-2.0-pro,292.3228,claude-opus-4.6,291.9451,deepseek-v3.2 diff --git a/leaderboard/score_submission.py b/leaderboard/score_submission.py new file mode 100644 index 00000000..1c25561e --- /dev/null +++ b/leaderboard/score_submission.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Score a submission against the frozen Frontier-Eng Medal podium. + +The gold/silver/bronze baselines are frozen at the v1 snapshot (2026-04-14) and +shipped in ``medal_podium.csv``. This script takes a new model's best-feasible +score on each task and reports its Medal Score, so anyone can be scored against +the released benchmark without rerunning the reference models. + +Usage +----- + python leaderboard/score_submission.py [--verbose] + +Submission CSV format (header required): two columns, ``Task,Score``, one row +per task, using the task names from ``medal_podium.csv`` (e.g. ``JobShop_abz``). +Higher score is better on every task. Missing tasks score 0. See +``submission_example.csv`` (the claude-opus-4.6 column) for a working example; +scoring it reproduces its leaderboard line (Medal v1 = 0.490, v1-lite = 0.501). + +Metric +------ +On each task a submission earns 1.00 / 0.67 / 0.33 for reaching the gold / +silver / bronze score, else 0. The Medal Score is the mean of this credit, +normalized to [0, 1], reported on the full v1 set (47 tasks) and the v1-lite +subset (10 tasks). +""" + +import argparse +import csv +from pathlib import Path + +HERE = Path(__file__).resolve().parent +GOLD, SILVER, BRONZE = 1.00, 0.67, 0.33 + +# v1-lite: 10-task representative subset (frontier_eval/conf/batch/v1_lite.yaml). +V1_LITE = { + "QuantumComputing_task_01_routing_qftentangled", "ComputerSystems_MallocLab", + "JobShop_abz", "InventoryOptimization_disruption_eoqd", + "EnergyStorage_BatteryFastChargingSPMe", "Robotics_RobotArmCycleTimeOptimization", + "Optics_holographic_multiplane_focusing", "WirelessChannelSimulation_HighReliableSimulation", + "ReactionOptimisation_snar_multiobjective", "StructuralOptimization_TopologyOptimization", +} + + +def load_podium(path): + """task -> (gold, silver, bronze) thresholds (higher is better).""" + podium = {} + with open(path, encoding="utf-8-sig") as f: + for row in csv.DictReader(f): + podium[row["Task"]] = ( + float(row["Gold"]), float(row["Silver"]), float(row["Bronze"])) + return podium + + +def load_submission(path): + """task -> score. Accepts a 'Task,Score' header or any two-column CSV.""" + scores = {} + with open(path, encoding="utf-8-sig") as f: + reader = csv.reader(f) + first = next(reader) + if not (first[1].strip().lower() in ("score", "best", "value")): + f.seek(0) # no recognizable header -> treat all rows as data + reader = csv.reader(f) + for row in reader: + if len(row) < 2 or not row[0].strip(): + continue + try: + scores[row[0].strip()] = float(row[1]) + except ValueError: + continue # skip header/garbage rows + return scores + + +def tier(score, gold, silver, bronze): + if score >= gold: + return GOLD, "gold" + if score >= silver: + return SILVER, "silver" + if score >= bronze: + return BRONZE, "bronze" + return 0.0, None + + +def score(podium, submission, verbose=False): + per_task = {} + counts = {"gold": 0, "silver": 0, "bronze": 0} + missing = [] + for task, (g, s, b) in podium.items(): + if task not in submission: + per_task[task] = 0.0 + missing.append(task) + continue + pts, name = tier(submission[task], g, s, b) + per_task[task] = pts + if name: + counts[name] += 1 + medal_v1 = sum(per_task.values()) / len(podium) + lite = [t for t in podium if t in V1_LITE] + medal_lite = sum(per_task[t] for t in lite) / len(lite) + + print(f"Medal Score (v1, 47 tasks) : {medal_v1:.3f}" + f" (gold {counts['gold']}, silver {counts['silver']}, bronze {counts['bronze']})") + print(f"Medal Score (v1-lite, 10 tasks) : {medal_lite:.3f}") + if missing: + print(f"\n[warn] {len(missing)} task(s) absent from submission (scored 0): " + f"{', '.join(missing[:5])}{' ...' if len(missing) > 5 else ''}") + if verbose: + print("\nper-task credit:") + for task in podium: + print(f" {per_task[task]:.2f} {task}") + return medal_v1, medal_lite + + +def main(): + ap = argparse.ArgumentParser(description="Score a submission against the frozen Medal podium.") + ap.add_argument("submission", help="CSV with columns Task,Score (one row per task)") + ap.add_argument("--podium", default=str(HERE / "medal_podium.csv"), + help="frozen gold/silver/bronze baselines (default: leaderboard/medal_podium.csv)") + ap.add_argument("--verbose", action="store_true", help="print per-task medal credit") + args = ap.parse_args() + score(load_podium(args.podium), load_submission(args.submission), args.verbose) + + +if __name__ == "__main__": + main() diff --git a/leaderboard/submission_example.csv b/leaderboard/submission_example.csv new file mode 100644 index 00000000..82f4c1a8 --- /dev/null +++ b/leaderboard/submission_example.csv @@ -0,0 +1,48 @@ +Task,Score +Aerodynamics_CarAerodynamicsSensing,0.9624 +Astrodynamics_MannedLunarLanding,6027.3126 +ComputerSystems_MallocLab,96 +Cryptographic_AES-128,11.8617 +Cryptographic_SHA-256,16.7955 +Cryptographic_SHA3-256,17.4003 +EnergyStorage_BatteryFastChargingProfile,120.8025 +EnergyStorage_BatteryFastChargingSPMe,71.8225 +EngDesign,1.3571 +InventoryOptimization_disruption_eoqd,0.6473 +InventoryOptimization_finite_horizon_dp,0.9596 +InventoryOptimization_general_meio,0.9929 +InventoryOptimization_joint_replenishment,0.8822 +InventoryOptimization_tree_gsm_safety_stock,0.75 +JobShop_abz,96.1035 +JobShop_swv,89.4966 +JobShop_ta,90.8322 +KernelEngineering_FlashAttention,983.5001 +KernelEngineering_MLA,1000.3859 +KernelEngineering_TriMul,357.1636 +Optics_adaptive_fault_tolerant_fusion,0.6398 +Optics_adaptive_temporal_smooth_control,0.8419 +Optics_fiber_guardband_spectrum_packing,0.6692 +Optics_fiber_mcs_power_scheduling,0.6542 +Optics_fiber_wdm_channel_power_allocation,0.6675 +Optics_holographic_multifocus_power_ratio,0.8072 +Optics_holographic_multiplane_focusing,0.6002 +Optics_phase_dammann_uniform_orders,99.7995 +Optics_phase_fourier_pattern_holography,82.1276 +PyPortfolioOpt_robust_mvo_rebalance,99.9946 +QuantumComputing_task_01_routing_qftentangled,5.0479 +QuantumComputing_task_02_clifford_t_synthesis,1.6633 +QuantumComputing_task_03_cross_target_qaoa,2.5781 +ReactionOptimisation_mit_case1_mixed,98.6621 +ReactionOptimisation_reizman_suzuki_pareto,82.3427 +ReactionOptimisation_snar_multiobjective,87.3657 +Robotics_DynamicObstacleAvoidanceNavigation,0.086 +Robotics_PIDTuning,0.1632 +Robotics_QuadrupedGaitOptimization,0.0219 +Robotics_RobotArmCycleTimeOptimization,0.4158 +Robotics_UAVInspectionCoverageWithWind,28.8519 +SingleCellAnalysis_predict_modality,0.5467 +StructuralOptimization_ISCSO2015,-968.4567 +StructuralOptimization_ISCSO2023,-16477799.48 +StructuralOptimization_TopologyOptimization,-190.1498 +SustainableDataCenterControl_hand_written_control,21.5657 +WirelessChannelSimulation_HighReliableSimulation,292.3228 From 4d7a2daee757fbf4e884217539beb6691eff1034 Mon Sep 17 00:00:00 2001 From: y-ji24 Date: Wed, 8 Jul 2026 19:04:36 +0800 Subject: [PATCH 2/5] feat(Bioinformatics): add PrimerDesignOptimization benchmark --- .../PrimerDesignOptimization/README.md | 114 ++ .../PrimerDesignOptimization/README_zh-CN.md | 113 ++ .../PrimerDesignOptimization/Task.md | 172 ++ .../PrimerDesignOptimization/Task_zh-CN.md | 166 ++ .../baseline/solution.py | 486 ++++++ .../frontier_eval/agent_files.txt | 4 + .../frontier_eval/artifact_files.txt | 2 + .../frontier_eval/candidate_destination.txt | 1 + .../frontier_eval/constraints.txt | 6 + .../frontier_eval/copy_files.txt | 1 + .../frontier_eval/eval_command.txt | 1 + .../frontier_eval/eval_cwd.txt | 1 + .../frontier_eval/initial_program.txt | 1 + .../frontier_eval/readonly_files.txt | 5 + .../frontier_eval/run_eval.py | 122 ++ .../references/primer_config.json | 97 ++ .../PrimerDesignOptimization/scripts/init.py | 459 +++++ .../verification/evaluator.py | 1546 +++++++++++++++++ .../verification/requirements.txt | 1 + 19 files changed, 3298 insertions(+) create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/README.md create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/README_zh-CN.md create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/Task.md create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/Task_zh-CN.md create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/solution.py create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/agent_files.txt create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/artifact_files.txt create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/candidate_destination.txt create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/constraints.txt create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/copy_files.txt create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/eval_command.txt create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/eval_cwd.txt create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/initial_program.txt create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/readonly_files.txt create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/run_eval.py create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/primer_config.json create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/scripts/init.py create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/verification/requirements.txt diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/README.md b/benchmarks/Bioinformatics/PrimerDesignOptimization/README.md new file mode 100644 index 00000000..0c18c1b6 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/README.md @@ -0,0 +1,114 @@ +# PCR Primer Design Optimization + +Design forward and reverse PCR primers for a synthetic DNA template. +This benchmark evaluates candidate primer pairs on melting temperature, GC content, complementarity, +and structural stability under a nearest-neighbor thermodynamic model. + +## File Structure + +```text +PrimerDesignOptimization/ +├── README.md +├── Task.md +├── references/ +│ └── primer_config.json +├── scripts/ +│ └── init.py +├── baseline/ +│ └── solution.py +├── verification/ +│ ├── evaluator.py +│ └── requirements.txt +└── frontier_eval/ + ├── eval_command.txt + ├── initial_program.txt + ├── agent_files.txt + ├── artifact_files.txt + ├── constraints.txt + └── run_eval.py +``` + +## Quick Start + +### 1. Install Dependencies + +```bash +pip install -r verification/requirements.txt +``` + +### 2. Run the Baseline Optimizer + +```bash +cd benchmarks/Bioinformatics/PrimerDesignOptimization +python scripts/init.py +# Outputs: submission.json +``` + +### 3. Evaluate a Submission + +```bash +cd benchmarks/Bioinformatics/PrimerDesignOptimization +python verification/evaluator.py --submission submission.json +``` + +### 4. Evaluate a Candidate Program Directly + +```bash +cd benchmarks/Bioinformatics/PrimerDesignOptimization +python verification/evaluator.py scripts/init.py +``` + +## Submission Format + +Write `submission.json` containing the candidate primer pair: + +```json +{ + "forward_primer": "GCTAGCTAGCTAGCTAGCT", + "reverse_primer": "GCTAGCTAGCTAGCTAGCT" +} +``` + +Both keys are required. Each primer must be an uppercase DNA string consisting only of the characters A, T, C, and G. + +## Task Summary + +- **Template**: 120 bp synthetic DNA sequence +- **Amplicon region**: bases 21–100 (80 bp target) +- **Primer length**: 18–25 bp +- **GC content**: 40–60% +- **Melting temperature**: 50–58°C (optimal 55°C) +- **Max Tm difference**: 3°C between primers +- **Thermodynamics**: Nearest-neighbor model (SantaLucia 1998) +- **Salt conditions**: 50 mM monovalent, 2 mM divalent, 0.8 mM dNTP + +## Scoring + +Hard validation gates determine feasibility. Among feasible candidates, +ten weighted quality metrics rank primer pairs. + +The ten quality metrics are: + +- **tm_score**: proximity to optimal melting temperature +- **gc_content_score**: proximity to 50% GC content +- **length_score**: proximity to 20–22 bp primer length +- **gc_clamp_score**: G/C fraction in the last 3 bases (3′ end) +- **self_complementarity_score**: degree of self-complementarity +- **pair_complementarity_score**: degree of cross-dimer complementarity +- **repeat_score**: degree of internal sequence repeats +- **hairpin_score**: degree of hairpin structure +- **mononucleotide_run_score**: degree of homopolymer runs +- **product_length_score**: proximity to preferred product length + +An infeasible submission (one that violates any hard validation gate) receives a final score of zero. +Higher final score is better. + +## Run with frontier_eval (unified) + +Unified benchmark: `task=unified task.benchmark=Bioinformatics/PrimerDesignOptimization` + +```bash +python -m frontier_eval \ +task=unified task.benchmark=Bioinformatics/PrimerDesignOptimization \ +algorithm.iterations=10 +``` diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/README_zh-CN.md b/benchmarks/Bioinformatics/PrimerDesignOptimization/README_zh-CN.md new file mode 100644 index 00000000..20ba315c --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/README_zh-CN.md @@ -0,0 +1,113 @@ +# PCR 引物设计优化（PCR Primer Design Optimization） + +基于人工合成的 DNA 模板，设计正向与反向 PCR 引物。 +该 Benchmark 通过熔解温度（Tm）、GC 含量、互补性及结构稳定性， +在最邻近热力学模型下对候选引物对进行评估。 + +## 文件结构（File Structure） + +```text +PrimerDesignOptimization/ +├── README.md +├── Task.md +├── references/ +│ └── primer_config.json +├── scripts/ +│ └── init.py +├── baseline/ +│ └── solution.py +├── verification/ +│ ├── evaluator.py +│ └── requirements.txt +└── frontier_eval/ + ├── eval_command.txt + ├── initial_program.txt + ├── agent_files.txt + ├── artifact_files.txt + ├── constraints.txt + └── run_eval.py +``` + +## 快速开始（Quick Start） + +### 1. 安装依赖 + +```bash +pip install -r verification/requirements.txt +``` + +### 2. 运行基线优化程序 + +```bash +cd benchmarks/Bioinformatics/PrimerDesignOptimization +python scripts/init.py +# 输出：submission.json +``` + +### 3. 评估提交结果 + +```bash +cd benchmarks/Bioinformatics/PrimerDesignOptimization +python verification/evaluator.py --submission submission.json +``` + +### 4. 直接评估候选程序 + +```bash +cd benchmarks/Bioinformatics/PrimerDesignOptimization +python verification/evaluator.py scripts/init.py +``` + +## 提交格式（Submission Format） + +编写 `submission.json` 文件，包含候选引物对： + +```json +{ + "forward_primer": "GCTAGCTAGCTAGCTAGCT", + "reverse_primer": "GCTAGCTAGCTAGCTAGCT" +} +``` + +两个键均为必填。每条引物必须为大写 DNA 字符串，仅包含 A、T、C、G 四种字符。 + +## 任务概要（Task Summary） + +- **模板（Template）**：120 bp 人工合成 DNA 序列 +- **扩增区域（Amplicon region）**：碱基 21–100（目标 80 bp） +- **引物长度（Primer length）**：18–25 bp +- **GC 含量（GC content）**：40–60% +- **熔解温度（Melting temperature）**：50–58°C（最优 55°C） +- **引物间最大 Tm 差（Max Tm difference）**：3°C +- **热力学模型（Thermodynamics）**：最邻近模型（SantaLucia 1998） +- **盐浓度条件（Salt conditions）**：50 mM 一价阳离子、2 mM 二价阳离子、0.8 mM dNTP + +## 评分（Scoring） + +硬性验证门（Hard Validation Gates）决定可行性（feasibility）。在可行候选方案中， +十项加权质量评价指标（Weighted Quality Metrics）对引物对进行排序。 + +十项质量评价指标如下： + +- **tm_score**：接近最优熔解温度的程度 +- **gc_content_score**：接近 50% GC 含量的程度 +- **length_score**：接近 20–22 bp 引物长度的程度 +- **gc_clamp_score**：最后 3 个碱基（3′ 端）中 G/C 的比例 +- **self_complementarity_score**：自互补性程度 +- **pair_complementarity_score**：引物间二聚体互补性程度 +- **repeat_score**：内部序列重复程度 +- **hairpin_score**：发夹结构程度 +- **mononucleotide_run_score**：同聚核苷酸连续重复（Homopolymer Run）程度 +- **product_length_score**：接近偏好扩增产物长度的程度 + +违反任何硬性验证门的提交（不可行提交）将获得最终得分（final_score）零分。得分越高越好。 + +## 使用 frontier_eval（统一）运行 + +统一 Benchmark：`task=unified task.benchmark=Bioinformatics/PrimerDesignOptimization` + +```bash +python -m frontier_eval \ +task=unified task.benchmark=Bioinformatics/PrimerDesignOptimization \ +algorithm.iterations=10 +``` \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/Task.md b/benchmarks/Bioinformatics/PrimerDesignOptimization/Task.md new file mode 100644 index 00000000..abfbe45e --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/Task.md @@ -0,0 +1,172 @@ +# PCR Primer Design Optimization + +## 1. Problem + +Design a pair of PCR primers (forward and reverse) for a synthetic DNA template. +The primers must specifically amplify the target amplicon region while satisfying +thermodynamic and structural constraints. + +Primer design is critical for successful PCR amplification. Poorly designed +primers can lead to non-specific amplification, primer-dimer artifacts, or failed +reactions. + +## 2. Template and Amplicon + +The template sequence is a 120 bp synthetic DNA fragment: + +```text +AAGTAAAGGCTAGTGCCCGACAAAGCGATTGTTGGGATTGTACTTGTCGGCAACGTTCCAAACG +TATGGAGGACGGGTAACGTCGGGCTAATGAATTAAGGAGCATGTAGTGCGCAGAGA +``` + +- **Amplicon start**: index 20 (0-based) +- **Amplicon end**: index 99 (0-based, inclusive) +- **Amplicon length**: 80 bp + +The forward primer must bind to the forward strand at the start of the amplicon. +The reverse primer must bind to the reverse-complement of the template strand at +the end of the amplicon. + +## 3. Decision Variables + +Submit two DNA sequences: + +- `forward_primer`: the forward primer sequence (5'' → 3'') +- `reverse_primer`: the reverse primer sequence (5'' → 3'') + +Each primer must satisfy the following constraints: + +| Constraint | Range | +|---|---| +| Length | 18–25 bp | +| GC content | 40–60% | +| Melting temperature (Tm) | 50–58°C | +| Max Tm difference between primers | ≤ 3°C | +| GC clamp (G/C in last 5 bases) | At least 1, at most 4 | +| Self-complementarity | ≤ 6 bp contiguous match | +| Pair complementarity | ≤ 6 bp contiguous match | +| Hairpin stem | ≤ 8 bp | +| Max homopolymer run | ≤ 4 identical bases | +| Alignment | Primers must align to the intended amplicon boundaries | +| Product length | PCR product length must be between 1 and template length | + +## 4. Thermodynamic Model + +Melting temperature is computed using the nearest-neighbor thermodynamic model +with SantaLucia salt correction: + +```text +Tm = (1000 * ΔH) / (ΔS + R * ln(C) + salt_correction) - 273.15 +``` + +- Nearest-neighbor parameters from Breslauer et al. (1986) +- Salt correction: SantaLucia (1998) formula +- Monovalent cation: 50 mM +- Divalent cation: 2 mM +- dNTP concentration: 0.8 mM +- DNA concentration: 50 nM + +## 5. Submission Format + +Write `submission.json` in the working directory: + +```json +{ + "forward_primer": "CAAAGCGATTGTTGGGATTG", + "reverse_primer": "CTTAATTCATTAGCCCGACG" +} +``` + +Both keys are required. Each sequence must be an uppercase DNA string consisting only of +the characters A, T, C, and G. + +## 6. Hard Validation Gates + +Hard validation gates determine feasibility. A submission that violates any gate +receives a final score of zero regardless of metric quality. The evaluator applies +10 hard validation gates; some gates check multiple conditions. + +**Gate 1 — Character validity** +- Fail if any primer contains characters other than A, T, C, G. + +**Gate 2 — Primer length** +- Fail if either primer is shorter than the minimum length (18 bp). +- Fail if either primer is longer than the maximum length (25 bp). + +**Gate 3 — GC content** +- Fail if either primer GC content is below the minimum (40%). +- Fail if either primer GC content exceeds the maximum (60%). + +**Gate 4 — Melting temperature** +- Fail if either primer Tm is outside the acceptable range (50–58°C). +- Fail if the absolute Tm difference between primers exceeds the limit (3°C). + +**Gate 5 — GC clamp** +- Fail if any primer has fewer than 1 or more than 4 G/C bases in its last 5 bases (3′ end). + +**Gate 6 — Self- and pair complementarity** +- Fail if any primer has a contiguous self-complementary match exceeding 6 bp. +- Fail if the forward and reverse primers have a contiguous cross-dimer match exceeding 6 bp. + +**Gate 7 — Hairpin structure** +- Fail if any primer forms a hairpin stem longer than 8 bp. + +**Gate 8 — Homopolymer run** +- Fail if any primer contains a run of 5 or more identical bases. + +**Gate 9 — Alignment specificity** +- Fail if the forward primer does not align to the start of the amplicon region. +- Fail if the reverse primer does not align (as reverse complement) to the end of the amplicon region. + +**Gate 10 — Product length** +- Fail if the estimated PCR product length is outside the range [1, template length]. + +Infeasible submissions receive a final score of `0.0`. + +## 7. Quality Metrics + +The evaluator computes ten normalised quality sub-scores, each in [0, 1]: + +| # | Metric | Quality target | +|---|---|---| +| 1 | length_score | 20-22 bp | +| 2 | gc_content_score | 50% | +| 3 | tm_score | proximity to optimum (55°C) | +| 4 | gc_clamp_score | G/C fraction in last 3 bases | +| 5 | self_complementarity_score | 0 bp complementarity | +| 6 | pair_complementarity_score | 0 bp cross-dimer complementarity | +| 7 | repeat_score | 0 bp internal repeats | +| 8 | hairpin_score | 0 bp hairpin stem | +| 9 | mononucleotide_run_score | 0 bp homopolymer runs | +| 10 | product_length_score | 80-120 bp product length | + +Hard validation gates (Section 6) determine feasibility. Quality metrics (Section 7) +evaluate and rank feasible submissions. + +The composite final score is: + +```text +final_score = sum (weight_i * metric_sub_score_i) +``` + +Higher final score is better. +## 8. Evaluation + +### Evaluate a produced submission + +```bash +python verification/evaluator.py --submission submission.json +``` + +### Run a candidate optimizer script and evaluate its output + +```bash +python verification/evaluator.py scripts/init.py +``` + +## 9. References + +- Configuration: `references/primer_config.json` +- Baseline optimizer: `scripts/init.py` +- Baseline solution (GA-based): `baseline/solution.py` +- Evaluator: `verification/evaluator.py` diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/Task_zh-CN.md b/benchmarks/Bioinformatics/PrimerDesignOptimization/Task_zh-CN.md new file mode 100644 index 00000000..5e90de11 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/Task_zh-CN.md @@ -0,0 +1,166 @@ +# PCR 引物设计优化（PCR Primer Design Optimization） + +## 1. 问题描述（Problem） + +针对一条人工合成的 DNA 模板，设计一对 PCR 引物（正向和反向）。 +引物必须特异性扩增目标扩增区域，同时满足热力学和结构约束条件。 + +引物设计对 PCR 扩增的成功与否至关重要。设计不当的引物可能导致非特异性扩增、 +引物二聚体伪迹或反应失败。 + +## 2. 模板与扩增区域（Template and Amplicon） + +模板序列为一段 120 bp 的人工合成 DNA 片段： + +```text +AAGTAAAGGCTAGTGCCCGACAAAGCGATTGTTGGGATTGTACTTGTCGGCAACGTTCCAAACG +TATGGAGGACGGGTAACGTCGGGCTAATGAATTAAGGAGCATGTAGTGCGCAGAGA +``` + +- **扩增起始位置（Amplicon start）**：索引 20（0-based） +- **扩增结束位置（Amplicon end）**：索引 99（0-based，包含） +- **扩增片段长度（Amplicon length）**：80 bp + +正向引物必须结合到扩增区域起始处的正向链上。 +反向引物必须结合到扩增区域末端模板链的反向互补序列上。 + +## 3. 决策变量（Decision Variables） + +提交两条 DNA 序列： + +- `forward_primer`：正向引物序列（5′ → 3′） +- `reverse_primer`：反向引物序列（5′ → 3′） + +每条引物必须满足以下约束条件： + +| 约束项（Constraint） | 范围（Range） | +|---|---| +| 长度（Length） | 18–25 bp | +| GC 含量（GC content） | 40–60% | +| 熔解温度（Melting temperature, Tm） | 50–58°C | +| 引物间最大 Tm 差（Max Tm difference） | ≤ 3°C | +| GC Clamp（3′端最后 5 碱基中的 G/C 数量） | 至少 1，至多 4 | +| 自互补性（Self-complementarity） | ≤ 6 bp 连续匹配 | +| 引物间互补性（Pair complementarity） | ≤ 6 bp 连续匹配 | +| 发夹结构茎区（Hairpin stem） | ≤ 8 bp | +| 同聚核苷酸连续重复最大值（Max homopolymer run） | ≤ 4 个相同碱基 | +| 比对特异性（Alignment） | 引物必须对齐到目标扩增区域边界 | +| 扩增产物长度（Product length） | PCR 产物长度必须在 1 到模板长度之间 | + +## 4. 热力学模型（Thermodynamic Model） + +熔解温度采用最邻近热力学模型（Nearest-Neighbor Model）结合 SantaLucia 盐校正公式计算： + +```text +Tm = (1000 * ΔH) / (ΔS + R * ln(C) + salt_correction) - 273.15 +``` + +- 最邻近参数来源：Breslauer et al.（1986） +- 盐校正公式：SantaLucia（1998） +- 一价阳离子浓度：50 mM +- 二价阳离子浓度：2 mM +- dNTP 浓度：0.8 mM +- DNA 浓度：50 nM + +## 5. 提交格式（Submission Format） + +在工作目录中编写 `submission.json` 文件： + +```json +{ + "forward_primer": "CAAAGCGATTGTTGGGATTG", + "reverse_primer": "CTTAATTCATTAGCCCGACG" +} +``` + +两个键均为必填。每条序列必须为大写 DNA 字符串，仅包含 A、T、C、G 四种字符。 + +## 6. 硬性验证门（Hard Validation Gates） + +硬性验证门决定可行性。违反任何验证门的提交将获得最终得分零分，不论其质量评价指标如何。 +评估器共实施 10 道硬性验证门；部分验证门包含多个检查条件。 + +**验证门 1 —— 字符有效性（Character validity）** +- 若任何引物包含 A、T、C、G 以外的字符，则判定为无效。 + +**验证门 2 —— 引物长度（Primer length）** +- 若任一引物短于最小长度（18 bp），则判定为无效。 +- 若任一引物长于最大长度（25 bp），则判定为无效。 + +**验证门 3 —— GC 含量（GC content）** +- 若任一引物的 GC 含量低于下限（40%），则判定为无效。 +- 若任一引物的 GC 含量超过上限（60%），则判定为无效。 + +**验证门 4 —— 熔解温度（Melting temperature）** +- 若任一引物的 Tm 超出允许范围（50–58°C），则判定为无效。 +- 若两条引物之间的绝对 Tm 差值超过限值（3°C），则判定为无效。 + +**验证门 5 —— GC Clamp** +- 若任一引物最后 5 个碱基（3′ 端）中 G/C 碱基数少于 1 或多于 4，则判定为无效。 + +**验证门 6 —— 自互补性与引物间互补性（Self- and pair complementarity）** +- 若任一引物存在超过 6 bp 的连续自互补匹配，则判定为无效。 +- 若正向与反向引物之间存在超过 6 bp 的连续交叉二聚体匹配，则判定为无效。 + +**验证门 7 —— 发夹结构（Hairpin structure）** +- 若任一引物形成超过 8 bp 的发夹结构茎区，则判定为无效。 + +**验证门 8 —— 同聚核苷酸连续重复（Homopolymer run）** +- 若任一引物包含 5 个或更多连续相同碱基的运行，则判定为无效。 + +**验证门 9 —— 比对特异性（Alignment specificity）** +- 若正向引物无法比对到扩增区域起始位置，则判定为无效。 +- 若反向引物（以其反向互补形式）无法比对到扩增区域末端位置，则判定为无效。 + +**验证门 10 —— 扩增产物长度（Product length）** +- 若估算的 PCR 产物长度超出范围 [1, 模板长度]，则判定为无效。 + +不可行的提交将获得最终得分 `0.0`。 + +## 7. 质量评价指标（Quality Metrics） + +评估器计算十项归一化的质量子分数，每项取值范围为 [0, 1]： + +| # | 指标（Metric） | 质量目标（Quality target） | +|---|---|---| +| 1 | length_score | 20–22 bp | +| 2 | gc_content_score | 50% | +| 3 | tm_score | 接近最优值（55°C） | +| 4 | gc_clamp_score | 最后 3 个碱基中的 G/C 比例 | +| 5 | self_complementarity_score | 0 bp 互补性 | +| 6 | pair_complementarity_score | 0 bp 交叉二聚体互补性 | +| 7 | repeat_score | 0 bp 内部重复 | +| 8 | hairpin_score | 0 bp 发夹结构茎区 | +| 9 | mononucleotide_run_score | 0 bp 同聚核苷酸连续重复 | +| 10 | product_length_score | 80–120 bp 产物长度 | + +硬性验证门（第 6 节）决定可行性。质量评价指标（第 7 节）评估并排序可行提交。 + +综合最终得分为： + +```text +final_score = sum (weight_i * metric_sub_score_i) +``` + +最终得分越高越好。 + +## 8. 评估方法（Evaluation） + +### 评估已生成的提交结果 + +```bash +python verification/evaluator.py --submission submission.json +``` + +### 运行候选优化程序并评估其输出 + +```bash +python verification/evaluator.py scripts/init.py +``` + +## 9. 参考资料（References） + +- 配置文件：`references/primer_config.json` +- 基线优化程序：`scripts/init.py` +- 基线算法解（基于遗传算法）：`baseline/solution.py` +- 评估器：`verification/evaluator.py` \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/solution.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/solution.py new file mode 100644 index 00000000..9c7cabab --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/solution.py @@ -0,0 +1,486 @@ +"""Baseline solution for PCR Primer Design Optimization. + +This module implements a genetic-algorithm-based primer designer that +searches for optimal forward/reverse primer pairs given a template +sequence and thermodynamic constraints defined in primer_config.json. + +The entrypoint `design_primers()` returns: + {"forward_primer": "...", "reverse_primer": "..."} +""" + +from __future__ import annotations + +import json +import math +import random +from pathlib import Path +from typing import Any + + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +def load_config() -> dict[str, Any]: + candidates = [ + Path(__file__).resolve().parent / "references" / "primer_config.json", + Path(__file__).resolve().parent.parent / "references" / "primer_config.json", + ] + for p in candidates: + if p.is_file(): + with p.open("r", encoding="utf-8-sig") as f: + return json.load(f) + raise FileNotFoundError("primer_config.json not found") + + +# --------------------------------------------------------------------------- +# Thermodynamic helpers +# --------------------------------------------------------------------------- + +_COMPLEMENT = {"A": "T", "T": "A", "C": "G", "G": "C"} + + +def reverse_complement(seq: str) -> str: + return "".join(_COMPLEMENT[b] for b in reversed(seq)) + + +def _complement(seq: str) -> str: + """Return the complement of a DNA sequence (no reversal).""" + return "".join(_COMPLEMENT[b] for b in seq) + + +def nearest_neighbor_delta( + seq: str, + enthalpy: dict[str, float], + entropy: dict[str, float], +) -> tuple[float, float]: + dh, ds = 0.0, 0.0 + for i in range(len(seq) - 1): + dimer = seq[i : i + 2] + pair_key = dimer + "/" + _complement(dimer) + if pair_key not in enthalpy: + pair_key = reverse_complement(dimer) + "/" + reverse_complement(_complement(dimer)) + dh += enthalpy.get(pair_key, 0.0) + ds += entropy.get(pair_key, 0.0) + dh += 0.2 + ds += -5.7 + if seq[0] in "AT": + dh += 2.3 + ds += 4.1 + if seq[-1] in "AT": + dh += 2.3 + ds += 4.1 + return dh, ds + + +def compute_melting_temperature( + seq: str, + monovalent: float, + divalent: float, + dntp: float, + dna_conc: float, + enthalpy: dict[str, float], + entropy: dict[str, float], +) -> float: + dh, ds = nearest_neighbor_delta(seq, enthalpy, entropy) + r = 1.987 + monovalent_eff = monovalent + 3.7 * math.sqrt(max(0, divalent - dntp)) + salt_correction = 0.368 * (len(seq) - 1) * math.log(monovalent_eff / 1000.0) + tm = (1000.0 * dh) / (ds + r * math.log(dna_conc / 1e9) + salt_correction) - 273.15 + return round(tm, 2) + + +def compute_gc_content(seq: str) -> float: + if not seq: + return 0.0 + gc = sum(1 for b in seq.upper() if b in "GC") + return 100.0 * gc / len(seq) + + +def compute_self_complementarity(seq: str) -> int: + """Maximum 3''-end-biased complementarity with the primer itself.""" + rc = reverse_complement(seq) + seq_u, rc_u = seq.upper(), rc.upper() + best = 0 + for i in range(len(seq_u) - 2): + for j in range(i + 3, len(seq_u) + 1): + sub = seq_u[i:j] + if sub in rc_u: + best = max(best, len(sub)) + return best + + +def compute_pair_complementarity(fwd: str, rev: str) -> int: + """Maximum complementarity between forward and reverse primers (3''-end biased).""" + fwd_u, rev_u = fwd.upper(), rev.upper() + rev_rc = reverse_complement(rev).upper() + fwd_rc = reverse_complement(fwd).upper() + best = 0 + for i in range(len(fwd_u) - 2): + for j in range(i + 3, len(fwd_u) + 1): + sub = fwd_u[i:j] + if sub in rev_rc: + best = max(best, len(sub)) + for i in range(len(rev_u) - 2): + for j in range(i + 3, len(rev_u) + 1): + sub = rev_u[i:j] + if sub in fwd_rc: + best = max(best, len(sub)) + return best + + +def count_repeats(seq: str) -> int: + s = seq.upper() + best = 0 + for i in range(len(s)): + for j in range(i + 1, len(s)): + sub = s[i:j] + if s.count(sub) >= 2: + best = max(best, len(sub)) + return best + + +def compute_hairpin_stem(seq: str) -> int: + """Estimate maximum hairpin stem length (capped at 8 bp).""" + rc = reverse_complement(seq) + best = 0 + max_cap = min(8, len(seq) // 2) + for offset in range(1, len(seq)): + fwd_part = seq[offset:] + rc_part = rc[: len(seq) - offset] + max_k = min(len(fwd_part), len(rc_part), max_cap) + for k in range(3, max_k + 1): + if fwd_part[:k] == reverse_complement(rc_part[:k]): + best = max(best, k) + return best + + +# --------------------------------------------------------------------------- +# Scoring function (same logic as scripts/init.py) +# --------------------------------------------------------------------------- + +def score_primer_pair(fwd: str, rev: str, cfg: dict[str, Any]) -> dict[str, Any]: + weights = cfg["scoring_weights"] + cons = cfg["constraints"] + thermo = cfg["thermodynamic"] + nn = thermo["nearest_neighbor_params"] + enh = nn["enthalpy_kcal"] + ent = nn["entropy_cal"] + + tm_fwd = compute_melting_temperature( + fwd, thermo["monovalent_cation_concentration"], + thermo["divalent_cation_concentration"], + thermo["dntp_concentration"], thermo["dna_concentration"], enh, ent, + ) + tm_rev = compute_melting_temperature( + rev, thermo["monovalent_cation_concentration"], + thermo["divalent_cation_concentration"], + thermo["dntp_concentration"], thermo["dna_concentration"], enh, ent, + ) + tm_optimal = cons["melting_temperature"]["optimal"] + tm_diff = abs(tm_fwd - tm_rev) + tm_score = max(0, 100 - 10 * abs(tm_fwd - tm_optimal) - 10 * abs(tm_rev - tm_optimal) - 15 * tm_diff) + + gc_fwd = compute_gc_content(fwd) + gc_rev = compute_gc_content(rev) + gc_center = (cons["gc_content"]["min"] + cons["gc_content"]["max"]) / 2.0 + gc_score = max(0, 100 - 5 * abs(gc_fwd - gc_center) - 5 * abs(gc_rev - gc_center)) + + len_fwd, len_rev = len(fwd), len(rev) + len_center = (cons["primer_length"]["min"] + cons["primer_length"]["max"]) / 2.0 + len_score = max(0, 100 - 10 * abs(len_fwd - len_center) - 10 * abs(len_rev - len_center)) + + gc_clamp_fwd = sum(1 for b in fwd[-5:].upper() if b in "GC") + gc_clamp_rev = sum(1 for b in rev[-5:].upper() if b in "GC") + gc_clamp_score = 100.0 + if cons["gc_clamp"]["required"]: + if gc_clamp_fwd < 1: + gc_clamp_score -= 25 + if gc_clamp_rev < 1: + gc_clamp_score -= 25 + if gc_clamp_fwd > cons["gc_clamp"]["max_gc_in_last_5"]: + gc_clamp_score -= 15 + if gc_clamp_rev > cons["gc_clamp"]["max_gc_in_last_5"]: + gc_clamp_score -= 15 + + self_comp_fwd = compute_self_complementarity(fwd) + self_comp_rev = compute_self_complementarity(rev) + self_comp_score = max(0, 100 - 20 * (self_comp_fwd + self_comp_rev)) + + pair_comp = compute_pair_complementarity(fwd, rev) + pair_comp_score = max(0, 100 - 20 * pair_comp) + + rep_fwd = count_repeats(fwd) + rep_rev = count_repeats(rev) + repeat_score = max(0, 100 - 15 * (rep_fwd + rep_rev)) + + hp_fwd = compute_hairpin_stem(fwd) + hp_rev = compute_hairpin_stem(rev) + hp_limit = cons["max_hairpin_stem"] + hp_score = 100.0 + if hp_fwd > hp_limit: + hp_score -= 20 * (hp_fwd - hp_limit) + if hp_rev > hp_limit: + hp_score -= 20 * (hp_rev - hp_limit) + hp_score = max(0, hp_score) + + feasible = True + violations: list[str] = [] + + if not (cons["primer_length"]["min"] <= len_fwd <= cons["primer_length"]["max"]): + violations.append(f"forward length {len_fwd} out of range") + feasible = False + if not (cons["primer_length"]["min"] <= len_rev <= cons["primer_length"]["max"]): + violations.append(f"reverse length {len_rev} out of range") + feasible = False + if not (cons["gc_content"]["min"] <= gc_fwd <= cons["gc_content"]["max"]): + violations.append(f"forward GC {gc_fwd:.1f}% out of range") + feasible = False + if not (cons["gc_content"]["min"] <= gc_rev <= cons["gc_content"]["max"]): + violations.append(f"reverse GC {gc_rev:.1f}% out of range") + feasible = False + if not (cons["melting_temperature"]["min"] <= tm_fwd <= cons["melting_temperature"]["max"]): + violations.append(f"forward Tm {tm_fwd:.1f}C out of range") + feasible = False + if not (cons["melting_temperature"]["min"] <= tm_rev <= cons["melting_temperature"]["max"]): + violations.append(f"reverse Tm {tm_rev:.1f}C out of range") + feasible = False + if tm_diff > cons["tm_difference_max"]: + violations.append(f"Tm difference {tm_diff:.1f}C exceeds max") + feasible = False + if self_comp_fwd > cons["self_complementarity_max"]: + violations.append(f"forward self-complementarity {self_comp_fwd} exceeds max") + feasible = False + if self_comp_rev > cons["self_complementarity_max"]: + violations.append(f"reverse self-complementarity {self_comp_rev} exceeds max") + feasible = False + if pair_comp > cons["pair_complementarity_max"]: + violations.append(f"pair complementarity {pair_comp} exceeds max") + feasible = False + if hp_fwd > hp_limit: + violations.append(f"forward hairpin stem {hp_fwd} bp exceeds limit") + feasible = False + if hp_rev > hp_limit: + violations.append(f"reverse hairpin stem {hp_rev} bp exceeds limit") + feasible = False + + total = ( + weights["tm_score"] * tm_score + + weights["gc_content_score"] * gc_score + + weights["length_score"] * len_score + + weights["gc_clamp_score"] * gc_clamp_score + + weights["self_complementarity_score"] * self_comp_score + + weights["pair_complementarity_score"] * pair_comp_score + + weights["repeat_score"] * repeat_score + + weights["hairpin_score"] * hp_score + ) + + return { + "total_score": round(total, 2), + "feasible": feasible, + "violations": violations, + "tm_fwd": tm_fwd, + "tm_rev": tm_rev, + "tm_diff": round(tm_diff, 2), + "gc_fwd": round(gc_fwd, 1), + "gc_rev": round(gc_rev, 1), + "len_fwd": len_fwd, + "len_rev": len_rev, + "gc_clamp_fwd": gc_clamp_fwd, + "gc_clamp_rev": gc_clamp_rev, + "self_comp_fwd": self_comp_fwd, + "self_comp_rev": self_comp_rev, + "pair_comp": pair_comp, + "hp_fwd": hp_fwd, + "hp_rev": hp_rev, + } + + +# --------------------------------------------------------------------------- +# Genetic Algorithm Primer Designer +# --------------------------------------------------------------------------- + +def _candidate_from_bounds( + template: str, + amp_start: int, + amp_end: int, + min_len: int, + max_len: int, +) -> tuple[str, str]: + """Generate a random primer pair from the target region bounds.""" + fwd_start = amp_start + fwd_end = amp_end + fwd_len = random.randint(min_len, min(max_len, fwd_end - fwd_start)) + fwd = template[fwd_start : fwd_start + fwd_len] + + rev_start = max(amp_start, amp_end - max_len + 1) + rev_end = amp_end + 1 + rev_len = random.randint(min_len, min(max_len, rev_end - rev_start)) + rev_seq = template[rev_end - rev_len : rev_end] + rev = reverse_complement(rev_seq) + return fwd, rev + + +def _mutate( + fwd: str, + rev: str, + template: str, + amp_start: int, + amp_end: int, + min_len: int, + max_len: int, + mutation_rate: float, +) -> tuple[str, str]: + """Mutate a primer pair by shifting, lengthening, or shortening.""" + bases = ["A", "T", "C", "G"] + new_fwd = list(fwd) + new_rev = list(rev) + + # Point mutations + for i in range(len(new_fwd)): + if random.random() < mutation_rate: + new_fwd[i] = random.choice(bases) + for i in range(len(new_rev)): + if random.random() < mutation_rate: + new_rev[i] = random.choice(bases) + + # Length changes + if random.random() < 0.1 and len(new_fwd) < max_len: + pos = amp_start + len(new_fwd) + if pos < amp_end: + new_fwd.append(template[pos]) + if random.random() < 0.1 and len(new_fwd) > min_len: + new_fwd.pop() + if random.random() < 0.1 and len(new_rev) < max_len: + pos = amp_end - len(new_rev) + if pos >= amp_start: + new_rev.append("".join(_COMPLEMENT[b] for b in reversed(template[pos - 1 : pos]))) + if random.random() < 0.1 and len(new_rev) > min_len: + new_rev.pop() + + return "".join(new_fwd), "".join(new_rev) + + +def _crossover( + p1_fwd: str, p1_rev: str, + p2_fwd: str, p2_rev: str, +) -> tuple[tuple[str, str], tuple[str, str]]: + """Single-point crossover between two primer pairs.""" + if len(p1_fwd) < 2 or len(p2_fwd) < 2 or len(p1_rev) < 2 or len(p2_rev) < 2: + return (p1_fwd, p1_rev), (p2_fwd, p2_rev) + + pt_fwd = random.randint(1, min(len(p1_fwd), len(p2_fwd)) - 1) + pt_rev = random.randint(1, min(len(p1_rev), len(p2_rev)) - 1) + + c1_fwd = p1_fwd[:pt_fwd] + p2_fwd[pt_fwd:] + c1_rev = p1_rev[:pt_rev] + p2_rev[pt_rev:] + c2_fwd = p2_fwd[:pt_fwd] + p1_fwd[pt_fwd:] + c2_rev = p2_rev[:pt_rev] + p1_rev[pt_rev:] + + return (c1_fwd, c1_rev), (c2_fwd, c2_rev) + + +def _tournament_select( + population: list[tuple[str, str]], + scores: list[float], + k: int, +) -> tuple[str, str]: + """Select an individual via tournament selection.""" + best_idx = random.randrange(len(population)) + for _ in range(k - 1): + idx = random.randrange(len(population)) + if scores[idx] > scores[best_idx]: + best_idx = idx + return population[best_idx] + + +def design_primers() -> dict[str, str]: + """Design PCR primers using a genetic algorithm. + + Returns: + dict with keys "forward_primer" and "reverse_primer". + """ + cfg = load_config() + template = cfg["template"]["sequence"] + amp_start = cfg["amplicon"]["start_index"] + amp_end = cfg["amplicon"]["end_index"] + cons = cfg["constraints"] + opt = cfg["optimization"] + + min_len = cons["primer_length"]["min"] + max_len = cons["primer_length"]["max"] + pop_size = opt["population_size"] + generations = opt["generations"] + mutation_rate = opt["mutation_rate"] + crossover_rate = opt["crossover_rate"] + tournament_k = opt["tournament_size"] + elite_count = max(1, int(pop_size * opt["elite_ratio"])) + + # Initialize population + population: list[tuple[str, str]] = [] + for _ in range(pop_size): + fwd, rev = _candidate_from_bounds(template, amp_start, amp_end, min_len, max_len) + population.append((fwd, rev)) + + best_pair = population[0] + best_score = -float("inf") + + for gen in range(generations): + scores = [] + for fwd, rev in population: + report = score_primer_pair(fwd, rev, cfg) + if report["feasible"]: + scores.append(report["total_score"]) + else: + scores.append(-1000.0) # heavy penalty for infeasible + + # Track best + gen_best = max(scores) + gen_best_idx = scores.index(gen_best) + if gen_best > best_score: + best_score = gen_best + best_pair = population[gen_best_idx] + + if gen % 10 == 0 or gen == generations - 1: + print(f" Gen {gen:3d}: best score = {best_score:.2f}") + + # Elitism + ranked = sorted(zip(population, scores), key=lambda x: x[1], reverse=True) + next_pop = [pair for pair, _ in ranked[:elite_count]] + + # Generate offspring + while len(next_pop) < pop_size: + p1 = _tournament_select(population, scores, tournament_k) + p2 = _tournament_select(population, scores, tournament_k) + + if random.random() < crossover_rate: + c1, c2 = _crossover(p1[0], p1[1], p2[0], p2[1]) + offspring = [c1, c2] + else: + offspring = [p1, p2] + + for fwd, rev in offspring: + fwd, rev = _mutate(fwd, rev, template, amp_start, amp_end, min_len, max_len, mutation_rate) + next_pop.append((fwd, rev)) + if len(next_pop) >= pop_size: + break + + population = next_pop[:pop_size] + + fwd, rev = best_pair + print(f"\nBest forward: {fwd}") + print(f"Best reverse: {rev}") + final_report = score_primer_pair(fwd, rev, cfg) + print(f"Score: {final_report['total_score']}") + print(f"Feasible: {final_report['feasible']}") + print(f"Tm fwd/rev: {final_report['tm_fwd']:.1f}C / {final_report['tm_rev']:.1f}C") + print(f"GC fwd/rev: {final_report['gc_fwd']:.1f}% / {final_report['gc_rev']:.1f}%") + + return {"forward_primer": fwd, "reverse_primer": rev} + + +if __name__ == "__main__": + result = design_primers() + with open("submission.json", "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + print("Submission written to submission.json") diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/agent_files.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/agent_files.txt new file mode 100644 index 00000000..2376ef82 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/agent_files.txt @@ -0,0 +1,4 @@ +README.md +Task.md +scripts/init.py +frontier_eval/constraints.txt diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/artifact_files.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/artifact_files.txt new file mode 100644 index 00000000..399faf0a --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/artifact_files.txt @@ -0,0 +1,2 @@ +# No extra artifact files are auto-collected by default for this benchmark. +# metrics.json and artifacts.json are handled separately by UnifiedTask. diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/candidate_destination.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/candidate_destination.txt new file mode 100644 index 00000000..b4a9ce2f --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/candidate_destination.txt @@ -0,0 +1 @@ +scripts/init.py \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/constraints.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/constraints.txt new file mode 100644 index 00000000..71dea10b --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/constraints.txt @@ -0,0 +1,6 @@ +UnifiedTask constraints: +1) Only modify `scripts/init.py`. +2) Preserve the public entrypoint, function signatures, and output contract expected by `frontier_eval/evaluator.py`. +3) Do not modify benchmark assets, documentation, references, verification code, runtime helpers, tests, or `frontier_eval/` metadata. +4) If the task produces named outputs such as `submission.json`, `results.txt`, `solution.json`, or `prediction.h5ad`, keep the expected filename and schema unchanged. +5) Prioritize validity and correctness before optimization. diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/copy_files.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/copy_files.txt new file mode 100644 index 00000000..7566c144 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/copy_files.txt @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/eval_command.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/eval_command.txt new file mode 100644 index 00000000..d156ceb5 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/eval_command.txt @@ -0,0 +1 @@ +{python} frontier_eval/run_eval.py --candidate {candidate} --metrics-out metrics.json --artifacts-out artifacts.json diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/eval_cwd.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/eval_cwd.txt new file mode 100644 index 00000000..7566c144 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/eval_cwd.txt @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/initial_program.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/initial_program.txt new file mode 100644 index 00000000..3113893e --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/initial_program.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/readonly_files.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/readonly_files.txt new file mode 100644 index 00000000..c89dd1de --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/readonly_files.txt @@ -0,0 +1,5 @@ +README.md +Task.md +references +verification +frontier_eval diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/run_eval.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/run_eval.py new file mode 100644 index 00000000..8920663b --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/run_eval.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import argparse +import inspect +import json +import os +import sys +import traceback +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from typing import Any + +INVALID_COMBINED_SCORE = -1e18 + + +def _write_json(path: Path, obj: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(obj, ensure_ascii=False, indent=2, default=str) + "\n", + encoding="utf-8", + ) + + +def _normalize_result(result: Any) -> tuple[dict[str, Any], dict[str, Any]]: + if hasattr(result, "metrics") and hasattr(result, "artifacts"): + return dict(getattr(result, "metrics")), dict(getattr(result, "artifacts")) + + if isinstance(result, dict): + raw_metrics = result.get("metrics") + raw_artifacts = result.get("artifacts") + if isinstance(raw_metrics, dict): + return dict(raw_metrics), dict(raw_artifacts or {}) + return dict(result), {} + + raise TypeError( + "Evaluator must return an EvaluationResult-like object or a dict of metrics." + ) + + +def _load_local_evaluator() -> Any: + evaluator_path = Path(__file__).with_name("evaluator.py").resolve() + spec = spec_from_file_location("_frontier_eval_local_evaluator", evaluator_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Failed to load local evaluator from {evaluator_path}") + module = module_from_spec(spec) + spec.loader.exec_module(module) + try: + return getattr(module, "evaluate") + except AttributeError as exc: + raise RuntimeError( + f"Local evaluator does not define evaluate(): {evaluator_path}" + ) from exc + + +def _find_repo_root() -> Path: + env_root = os.environ.get("FRONTIER_ENGINEERING_ROOT") + if env_root: + return Path(env_root).expanduser().resolve() + + here = Path(__file__).resolve() + for parent in [here.parent, *here.parents]: + if (parent / "frontier_eval").is_dir() and (parent / "benchmarks").is_dir(): + return parent + return Path.cwd().resolve() + + +def _build_kwargs(evaluate_fn: Any) -> dict[str, Any]: + kwargs: dict[str, Any] = {} + try: + parameters = inspect.signature(evaluate_fn).parameters + except Exception: + return kwargs + + if "repo_root" in parameters: + kwargs["repo_root"] = _find_repo_root() + if "kernel_python" in parameters: + kwargs["kernel_python"] = sys.executable + return kwargs + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run a benchmark-local unified evaluator and export metrics/artifacts JSON." + ) + parser.add_argument("--candidate", required=True) + parser.add_argument("--metrics-out", default="metrics.json") + parser.add_argument("--artifacts-out", default="artifacts.json") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = _parse_args(argv) + + candidate_path = Path(args.candidate).expanduser().resolve() + metrics_out = Path(args.metrics_out).expanduser().resolve() + artifacts_out = Path(args.artifacts_out).expanduser().resolve() + + metrics: dict[str, Any] = { + "combined_score": INVALID_COMBINED_SCORE, + "valid": 0.0, + } + artifacts: dict[str, Any] = { + "local_evaluator_path": str(Path(__file__).with_name("evaluator.py").resolve()), + "candidate_path": str(candidate_path), + } + + try: + evaluate_fn = _load_local_evaluator() + result = evaluate_fn(str(candidate_path), **_build_kwargs(evaluate_fn)) + metrics, evaluator_artifacts = _normalize_result(result) + artifacts.update(evaluator_artifacts) + except Exception as exc: + artifacts["error_message"] = str(exc) + artifacts["traceback"] = traceback.format_exc() + + _write_json(metrics_out, metrics) + _write_json(artifacts_out, artifacts) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/primer_config.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/primer_config.json new file mode 100644 index 00000000..aee33d76 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/primer_config.json @@ -0,0 +1,97 @@ +{ + "template": { + "name": "synthetic_target_001", + "sequence": "AAGTAAAGGCTAGTGCCCGACAAAGCGATTGTTGGGATTGTACTTGTCGGCAACGTTCCAAACGTATGGAGGACGGGTAACGTCGGGCTAATGAATTAAGGAGCATGTAGTGCGCAGAGA", + "description": "A 120 bp synthetic DNA template for PCR primer design optimization." + }, + "amplicon": { + "start_index": 20, + "end_index": 99, + "strand": "forward" + }, + "scoring_weights": { + "tm_score": 1.0, + "gc_content_score": 0.8, + "length_score": 0.5, + "gc_clamp_score": 0.6, + "self_complementarity_score": 0.7, + "pair_complementarity_score": 0.7, + "repeat_score": 0.4, + "hairpin_score": 0.5, + "mononucleotide_run_score": 0.3, + "product_length_score": 0.2 + }, + "constraints": { + "primer_length": { + "min": 18, + "max": 25 + }, + "gc_content": { + "min": 40.0, + "max": 60.0 + }, + "melting_temperature": { + "min": 50.0, + "max": 58.0, + "optimal": 55.0 + }, + "tm_difference_max": 3.0, + "gc_clamp": { + "required": true, + "max_gc_in_last_5": 4 + }, + "self_complementarity_max": 6, + "pair_complementarity_max": 6, + "max_repeat_length": 4, + "max_poly_x_run": 4, + "max_hairpin_stem": 8 + }, + "thermodynamic": { + "monovalent_cation_concentration": 50.0, + "divalent_cation_concentration": 2.0, + "dntp_concentration": 0.8, + "dna_concentration": 50.0, + "salt_correction_formula": "santalucia", + "thermodynamic_table": "breslauer", + "nearest_neighbor_params": { + "enthalpy_kcal": { + "AA/TT": -7.9, + "AT/TA": -7.2, + "TA/AT": -7.2, + "CA/GT": -8.5, + "GT/CA": -8.5, + "CT/GA": -7.8, + "GA/CT": -7.8, + "CG/GC": -10.6, + "GC/CG": -10.6, + "GG/CC": -8.0, + "CC/GG": -8.0 + }, + "entropy_cal": { + "AA/TT": -22.2, + "AT/TA": -20.4, + "TA/AT": -21.3, + "CA/GT": -22.7, + "GT/CA": -22.7, + "CT/GA": -21.0, + "GA/CT": -21.0, + "CG/GC": -27.2, + "GC/CG": -27.2, + "GG/CC": -19.9, + "CC/GG": -19.9 + } + } + }, + "optimization": { + "population_size": 100, + "generations": 50, + "mutation_rate": 0.1, + "crossover_rate": 0.8, + "tournament_size": 3, + "elite_ratio": 0.1 + }, + "product_length": { + "preferred_min": 80, + "preferred_max": 120 + } +} \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/scripts/init.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/scripts/init.py new file mode 100644 index 00000000..dce8efb2 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/scripts/init.py @@ -0,0 +1,459 @@ +# EVOLVE-BLOCK-START +"""Baseline PCR primer design optimizer. + +DO NOT MODIFY: load_config(), score_primer_pair(), compute_melting_temperature(), + compute_gc_content(), evaluate_complementarity() +ALLOWED TO MODIFY: design_primers() + +Outputs submission.json with designed forward and reverse primers. +""" + +from __future__ import annotations + +import json +import math +import sys +from pathlib import Path +from typing import Any + +import numpy as np + + +# --------------------------------------------------------------------------- +# DO NOT MODIFY — Configuration loader +# --------------------------------------------------------------------------- + +def load_config() -> dict[str, Any]: + """Load primer_config.json from references/.""" + candidates = [ + Path(__file__).resolve().parent / "references" / "primer_config.json", + Path(__file__).resolve().parent.parent / "references" / "primer_config.json", + ] + for p in candidates: + if p.is_file(): + with p.open("r", encoding="utf-8-sig") as f: + return json.load(f) + raise FileNotFoundError("primer_config.json not found") + + +# --------------------------------------------------------------------------- +# DO NOT MODIFY — Thermodynamic helpers +# --------------------------------------------------------------------------- + +def _reverse_complement(seq: str) -> str: + """Return the reverse complement of a DNA sequence.""" + comp = {"A": "T", "T": "A", "C": "G", "G": "C"} + return "".join(comp.get(base, base) for base in reversed(seq)) + + +def _complement(seq: str) -> str: + """Return the complement of a DNA sequence (no reversal).""" + comp = {"A": "T", "T": "A", "C": "G", "G": "C"} + return "".join(comp.get(base, base) for base in seq) + + +def _nearest_neighbor_delta( + seq: str, + enthalpy: dict[str, float], + entropy: dict[str, float], +) -> tuple[float, float]: + """Compute total enthalpy (kcal) and entropy (cal/K) for a DNA duplex.""" + dh = 0.0 + ds = 0.0 + for i in range(len(seq) - 1): + dimer = seq[i : i + 2] + pair_key = dimer + "/" + _complement(dimer) + if pair_key not in enthalpy: + # Try reverse-complement orientation for the pair + pair_key = _reverse_complement(dimer) + "/" + _reverse_complement(_complement(dimer)) + dh += enthalpy.get(pair_key, 0.0) + ds += entropy.get(pair_key, 0.0) + # Initiation parameters (per SantaLucia 1998) + dh += 0.2 + ds += -5.7 + # Terminal AT penalty + if seq[0] in "AT": + dh += 2.3 + ds += 4.1 + if seq[-1] in "AT": + dh += 2.3 + ds += 4.1 + return dh, ds + + +def compute_melting_temperature( + seq: str, + monovalent: float, + divalent: float, + dntp: float, + dna_conc: float, + enthalpy: dict[str, float], + entropy: dict[str, float], +) -> float: + """Compute melting temperature (Celsius) using nearest-neighbor model.""" + dh, ds = _nearest_neighbor_delta(seq, enthalpy, entropy) + r = 1.987 # Cal / (mol * K) + # SantaLucia salt correction + monovalent_eff = monovalent + 3.7 * math.sqrt(max(0.0, divalent - dntp)) + salt_correction = 0.368 * (len(seq) - 1) * math.log(monovalent_eff / 1000.0, math.e) + tm = (1000.0 * dh) / (ds + r * math.log(dna_conc / 1e9, math.e) + salt_correction) - 273.15 + return round(tm, 2) + + +def compute_gc_content(seq: str) -> float: + """Compute GC content percentage.""" + if not seq: + return 0.0 + gc = sum(1 for base in seq.upper() if base in "GC") + return 100.0 * gc / len(seq) + + +def compute_self_complementarity(seq: str) -> int: + """Maximum contiguous 3''-end complementarity with the primer itself. + + Weighted toward 3''-end matches (critical for primer-dimer formation). + """ + rc = _reverse_complement(seq) + seq_u = seq.upper() + rc_u = rc.upper() + best = 0 + # Check complementarity starting from 3'' end of seq (i.e., seq suffix vs rc) + for i in range(len(seq_u) - 2): + for j in range(i + 3, len(seq_u) + 1): + sub = seq_u[i:j] + if sub in rc_u: + best = max(best, len(sub)) + return best + + +def compute_pair_complementarity(fwd: str, rev: str) -> int: + """Maximum contiguous complementarity between forward and reverse primers. + + Checks fwd 3''-end complementarity against rev and vice versa. + """ + fwd_u = fwd.upper() + rev_u = rev.upper() + rev_rc = _reverse_complement(rev) + fwd_rc = _reverse_complement(fwd) + best = 0 + # Fwd region vs rev rc + for i in range(len(fwd_u) - 2): + for j in range(i + 3, len(fwd_u) + 1): + sub = fwd_u[i:j] + if sub in rev_rc: + best = max(best, len(sub)) + # Rev region vs fwd rc + for i in range(len(rev_u) - 2): + for j in range(i + 3, len(rev_u) + 1): + sub = rev_u[i:j] + if sub in fwd_rc: + best = max(best, len(sub)) + return best + + +def count_repeats(seq: str) -> int: + """Maximum length of any repeated substring in the primer.""" + seq_upper = seq.upper() + max_len = 0 + for i in range(len(seq_upper)): + for j in range(i + 1, len(seq_upper) + 1): + sub = seq_upper[i:j] + if j + len(sub) <= len(seq_upper) and sub in seq_upper[j:]: + # Count total occurrences + count = 0 + pos = 0 + while True: + pos = seq_upper.find(sub, pos) + if pos == -1: + break + count += 1 + pos += 1 + if count >= 2: + max_len = max(max_len, len(sub)) + return max_len + + +def compute_hairpin_stem(seq: str) -> int: + """Estimate maximum hairpin stem length (capped at 8 bp). + + A hairpin requires a stem of complementary bases with a loop (>=3 bases). + Returns the maximum stem length found, capped at 8 bp. + """ + rc = _reverse_complement(seq) + max_stem = 0 + max_cap = min(8, len(seq) // 2) + for i in range(1, len(seq)): + fwd_part = seq[i:] + rc_part = rc[: len(seq) - i] + max_k = min(len(fwd_part), len(rc_part), max_cap) + for k in range(3, max_k + 1): + sub_fwd = fwd_part[:k] + sub_rc = rc_part[:k] + if sub_fwd == _reverse_complement(sub_rc): + max_stem = max(max_stem, k) + return max_stem + + +# --------------------------------------------------------------------------- +# DO NOT MODIFY — Scoring +# --------------------------------------------------------------------------- + +def score_primer_pair(fwd: str, rev: str, cfg: dict[str, Any]) -> dict[str, Any]: + """Score a forward/reverse primer pair. + + Returns dict with individual metrics and weighted total score. + Higher score is better. + """ + template = cfg["template"]["sequence"] + weights = cfg["scoring_weights"] + cons = cfg["constraints"] + thermo = cfg["thermodynamic"] + nn = thermo["nearest_neighbor_params"] + enh = nn["enthalpy_kcal"] + ent = nn["entropy_cal"] + + # --- Melting temperature --- + tm_fwd = compute_melting_temperature( + fwd, thermo["monovalent_cation_concentration"], + thermo["divalent_cation_concentration"], + thermo["dntp_concentration"], thermo["dna_concentration"], + enh, ent, + ) + tm_rev = compute_melting_temperature( + rev, thermo["monovalent_cation_concentration"], + thermo["divalent_cation_concentration"], + thermo["dntp_concentration"], thermo["dna_concentration"], + enh, ent, + ) + tm_optimal = cons["melting_temperature"]["optimal"] + tm_diff = abs(tm_fwd - tm_rev) + tm_score = max(0, 100 - 10 * abs(tm_fwd - tm_optimal) - 10 * abs(tm_rev - tm_optimal) - 15 * tm_diff) + + # --- GC content --- + gc_fwd = compute_gc_content(fwd) + gc_rev = compute_gc_content(rev) + gc_center = (cons["gc_content"]["min"] + cons["gc_content"]["max"]) / 2.0 + gc_score = max(0, 100 - 5 * abs(gc_fwd - gc_center) - 5 * abs(gc_rev - gc_center)) + + # --- Length --- + len_fwd = len(fwd) + len_rev = len(rev) + len_center = (cons["primer_length"]["min"] + cons["primer_length"]["max"]) / 2.0 + len_score = max(0, 100 - 10 * abs(len_fwd - len_center) - 10 * abs(len_rev - len_center)) + + # --- GC clamp (G/C in last 5 bases of 3'' end) --- + gc_clamp_fwd = sum(1 for b in fwd[-5:].upper() if b in "GC") + gc_clamp_rev = sum(1 for b in rev[-5:].upper() if b in "GC") + gc_clamp_score = 100.0 + if cons["gc_clamp"]["required"]: + if gc_clamp_fwd < 1: + gc_clamp_score -= 25 + if gc_clamp_rev < 1: + gc_clamp_score -= 25 + if gc_clamp_fwd > cons["gc_clamp"]["max_gc_in_last_5"]: + gc_clamp_score -= 15 + if gc_clamp_rev > cons["gc_clamp"]["max_gc_in_last_5"]: + gc_clamp_score -= 15 + + # --- Self complementarity --- + self_comp_fwd = compute_self_complementarity(fwd) + self_comp_rev = compute_self_complementarity(rev) + self_comp_score = max( + 0, 100 - 20 * (self_comp_fwd + self_comp_rev) + ) + + # --- Pair complementarity --- + pair_comp = compute_pair_complementarity(fwd, rev) + pair_comp_score = max(0, 100 - 20 * pair_comp) + + # --- Repeats --- + rep_fwd = count_repeats(fwd) + rep_rev = count_repeats(rev) + repeat_score = max(0, 100 - 15 * (rep_fwd + rep_rev)) + + # --- Hairpin --- + hp_fwd = compute_hairpin_stem(fwd) + hp_rev = compute_hairpin_stem(rev) + hp_limit = cons["max_hairpin_stem"] + hp_score = 100.0 + if hp_fwd > hp_limit: + hp_score -= 20 * (hp_fwd - hp_limit) + if hp_rev > hp_limit: + hp_score -= 20 * (hp_rev - hp_limit) + hp_score = max(0, hp_score) + + # Mononucleotide run score (homopolymer) + mono_run_fwd = 0 + run = 1 + for i in range(1, len(fwd)): + if fwd[i] == fwd[i - 1]: + run += 1 + else: + mono_run_fwd = max(mono_run_fwd, run) + run = 1 + mono_run_fwd = max(mono_run_fwd, run) + mono_run_rev = 0 + run = 1 + for i in range(1, len(rev)): + if rev[i] == rev[i - 1]: + run += 1 + else: + mono_run_rev = max(mono_run_rev, run) + run = 1 + mono_run_rev = max(mono_run_rev, run) + mono_score_fwd = max(0, 100 - 25 * (mono_run_fwd - 1)) if mono_run_fwd > 1 else 100 + mono_score_rev = max(0, 100 - 25 * (mono_run_rev - 1)) if mono_run_rev > 1 else 100 + mono_run_score = (mono_score_fwd + mono_score_rev) // 2 + + # Product length score + template_seq = cfg["template"]["sequence"] + amp_start = cfg["amplicon"]["start_index"] + amp_end = cfg["amplicon"]["end_index"] + _rc = {"A": "T", "T": "A", "C": "G", "G": "C"} + fwd_pos = template_seq.find(fwd.upper(), max(0, amp_start - 5)) + rev_rc_str = "".join(_rc.get(b, b) for b in reversed(rev.upper())) + rev_pos = template_seq.find(rev_rc_str, max(0, amp_end - len(rev_rc_str) - 5)) + if fwd_pos >= 0 and rev_pos >= 0: + prod_len = rev_pos + len(rev_rc_str) - fwd_pos + pref_min = cfg.get("product_length", {}).get("preferred_min", 80) + pref_max = cfg.get("product_length", {}).get("preferred_max", 120) + if pref_min <= prod_len <= pref_max: + pl_score = 100 + elif prod_len < pref_min: + pl_score = max(0, int(100 * prod_len / pref_min)) + else: + max_tlen = len(template_seq) + pl_score = max(0, int(100 * (max_tlen - prod_len) / (max_tlen - pref_max))) if max_tlen > pref_max else 0 + else: + prod_len = 0 + pl_score = 0 + + # --- Feasibility --- + feasible = True + violations: list[str] = [] + + if not (cons["primer_length"]["min"] <= len_fwd <= cons["primer_length"]["max"]): + violations.append(f"forward length {len_fwd} out of range") + feasible = False + if not (cons["primer_length"]["min"] <= len_rev <= cons["primer_length"]["max"]): + violations.append(f"reverse length {len_rev} out of range") + feasible = False + if not (cons["gc_content"]["min"] <= gc_fwd <= cons["gc_content"]["max"]): + violations.append(f"forward GC {gc_fwd:.1f}% out of range") + feasible = False + if not (cons["gc_content"]["min"] <= gc_rev <= cons["gc_content"]["max"]): + violations.append(f"reverse GC {gc_rev:.1f}% out of range") + feasible = False + if not (cons["melting_temperature"]["min"] <= tm_fwd <= cons["melting_temperature"]["max"]): + violations.append(f"forward Tm {tm_fwd:.1f}C out of range") + feasible = False + if not (cons["melting_temperature"]["min"] <= tm_rev <= cons["melting_temperature"]["max"]): + violations.append(f"reverse Tm {tm_rev:.1f}C out of range") + feasible = False + if tm_diff > cons["tm_difference_max"]: + violations.append(f"Tm difference {tm_diff:.1f}C exceeds max") + feasible = False + if self_comp_fwd > cons["self_complementarity_max"]: + violations.append(f"forward self-complementarity {self_comp_fwd} exceeds max") + feasible = False + if self_comp_rev > cons["self_complementarity_max"]: + violations.append(f"reverse self-complementarity {self_comp_rev} exceeds max") + feasible = False + if pair_comp > cons["pair_complementarity_max"]: + violations.append(f"pair complementarity {pair_comp} exceeds max") + feasible = False + if hp_fwd > hp_limit: + violations.append(f"forward hairpin stem {hp_fwd} bp exceeds limit") + feasible = False + if hp_rev > hp_limit: + violations.append(f"reverse hairpin stem {hp_rev} bp exceeds limit") + feasible = False + + # Total weighted score + total = ( + weights["tm_score"] * tm_score + + weights["gc_content_score"] * gc_score + + weights["length_score"] * len_score + + weights["gc_clamp_score"] * gc_clamp_score + + weights["self_complementarity_score"] * self_comp_score + + weights["pair_complementarity_score"] * pair_comp_score + + weights["repeat_score"] * repeat_score + + weights["hairpin_score"] * hp_score + + weights.get("mononucleotide_run_score", 0.0) * mono_run_score + + weights.get("product_length_score", 0.0) * pl_score + ) + + return { + "total_score": round(total, 2), + "feasible": feasible, + "violations": violations, + "tm_fwd": tm_fwd, + "tm_rev": tm_rev, + "tm_diff": round(tm_diff, 2), + "gc_fwd": round(gc_fwd, 1), + "gc_rev": round(gc_rev, 1), + "len_fwd": len_fwd, + "len_rev": len_rev, + "gc_clamp_fwd": gc_clamp_fwd, + "gc_clamp_rev": gc_clamp_rev, + "self_comp_fwd": self_comp_fwd, + "self_comp_rev": self_comp_rev, + "pair_comp": pair_comp, + "hp_fwd": hp_fwd, + "hp_rev": hp_rev, + } + + +# --------------------------------------------------------------------------- +# ALLOWED TO MODIFY — Primer design optimizer +# --------------------------------------------------------------------------- + +def design_primers() -> dict[str, str]: + """Design PCR primers for the target template. + + The default implementation takes the first 20 bp of each binding region + as a simple baseline. + + Returns: + dict with keys "forward_primer" and "reverse_primer". + """ + cfg = load_config() + template = cfg["template"]["sequence"] + amp_start = cfg["amplicon"]["start_index"] + amp_end = cfg["amplicon"]["end_index"] + + # Baseline: take first 20 bp of each binding region + fwd = template[amp_start : amp_start + 20] + rev_seq = template[amp_end - 19 : amp_end + 1] + rev = "".join( + {"A": "T", "T": "A", "C": "G", "G": "C"}.get(b, b) + for b in reversed(rev_seq) + ) + + return {"forward_primer": fwd, "reverse_primer": rev} + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + result = design_primers() + fwd, rev = result["forward_primer"], result["reverse_primer"] + cfg = load_config() + report = score_primer_pair(fwd, rev, cfg) + print(f"Forward: {fwd}") + print(f"Reverse: {rev}") + print(f"Score: {report['total_score']}") + print(f"Feasible: {report['feasible']}") + if report["violations"]: + print(f"Violations: {report['violations']}") + print(f"Tm fwd: {report['tm_fwd']:.1f} C") + print(f"Tm rev: {report['tm_rev']:.1f} C") + print(f"GC fwd: {report['gc_fwd']:.1f}%") + print(f"GC rev: {report['gc_rev']:.1f}%") + + with open("submission.json", "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + print("Submission written to submission.json") +# EVOLVE-BLOCK-END diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py new file mode 100644 index 00000000..6569ccf6 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py @@ -0,0 +1,1546 @@ +"""Evaluator for PCR Primer Design Optimization. + +Evaluates primer pairs designed by scripts/init.py (or a candidate program) +against thermodynamic, structural, and specificity criteria defined in +references/primer_config.json. + +Usage +----- + python verification/evaluator.py --submission submission.json + python verification/evaluator.py scripts/init.py + +Frozen Spec v2.0 sections referenced throughout. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any + +_COMPLEMENT = {"A": "T", "T": "A", "C": "G", "G": "C"} + +# ╔══════════════════════════════════════════════════════════════════════════════╗ +# ║ Config (Spec §1) ║ +# ╚══════════════════════════════════════════════════════════════════════════════╝ + +def load_config() -> dict[str, Any]: + """Load primer_config.json from the references/ directory. + + Returns + ------- + dict + Full configuration including template, constraints, scoring weights, + thermodynamic parameters, and optimisation settings. + + Spec + ---- + §1.1 — Benchmark configuration schema. + """ + config_path = Path(__file__).resolve().parent.parent / "references" / "primer_config.json" + with open(config_path, encoding="utf-8") as fh: + return json.load(fh) + +# ╔══════════════════════════════════════════════════════════════════════════════╗ +# ║ Utilities (Spec §2) ║ +# ╚══════════════════════════════════════════════════════════════════════════════╝ + +# Common helper functions used across thermodynamic, validation, and metric modules. + +def compute_self_complementarity(seq: str) -> int: + """Maximum contiguous 3''-end self-complementarity. + + Checks every substring of *seq* (length >= 3) for presence in its own + reverse complement. The result is 3''-end biased. + + Parameters + ---------- + seq : str + Primer sequence. + + Returns + ------- + int + Length of the longest self-complementary substring. + + Spec + ---- + """ + rc = reverse_complement(seq) + seq_u = seq.upper() + rc_u = rc.upper() + best = 0 + for i in range(len(seq_u) - 2): + for j in range(i + 3, len(seq_u) + 1): + sub = seq_u[i:j] + if sub in rc_u: + best = max(best, len(sub)) + return best + +def reverse_complement(seq: str) -> str: + """Return the reverse complement of a DNA sequence. + + Parameters + ---------- + seq : str + Uppercase DNA string (A, T, C, G). + + Returns + ------- + str + Reverse complement (5'' → 3''). + + Spec + ---- + §2.1 — Sequence manipulation primitives. + """ + return "".join(_COMPLEMENT.get(base, base) for base in reversed(seq)) + +def canonical_pair_key(dimer: str) -> str: + """Construct the canonical key ``"{dimer}/{complement}"`` for NN table lookup. + + The key format is ``{5''-dimer}/{3''-complement}``, where the complement + is the Watson–Crick pairing of each base without reversal. + + Parameters + ---------- + dimer : str + Two-base DNA string (5'' → 3''). + + Returns + ------- + str + Key string of the form ``"XX/YY"``. The caller should use + ``lookup_nn_parameter()`` to resolve it against the parameter table. + + Spec + ---- + §2.2 — Nearest-neighbour parameter lookup convention. + """ + return dimer + "/" + "".join(_COMPLEMENT[b] for b in dimer) + +def lookup_nn_parameter(dimer: str, table: dict[str, float]) -> float: + """Look up a nearest-neighbour parameter value with reverse-complement fallback. + + Constructs the canonical key via ``canonical_pair_key(dimer)``. If that key + is not present in *table* (covering only 10 of 16 possible dimer combinations), + falls back to the reverse-complement orientation, which is guaranteed to exist + for all 16 dinucleotide combinations by NN symmetry. + + Parameters + ---------- + dimer : str + Two-base DNA string (5'' → 3''). + table : dict[str, float] + NN parameter dictionary (``enthalpy_kcal`` or ``entropy_cal``). + + Returns + ------- + float + The parameter value for the resolved dimer pair. + + Spec + ---- + §2.3 — NN parameter resolution with symmetry fallback. + """ + key = canonical_pair_key(dimer) + if key not in table: + rc = "".join(_COMPLEMENT[b] for b in reversed(dimer)) + key = canonical_pair_key(rc) + return table[key] + +def is_valid_dna(seq: str) -> bool: + """Check whether a string consists only of A, T, C, G. + + Parameters + ---------- + seq : str + Candidate DNA string. + + Returns + ------- + bool + + Spec + ---- + §2.3 — Character-set validation helper. + """ + if not seq: + return False + return all(base in "ATCGatcg" for base in seq) + +# ╔══════════════════════════════════════════════════════════════════════════════╗ +# ║ Thermodynamics (Spec §3) ║ +# ╚══════════════════════════════════════════════════════════════════════════════╝ + +# Deterministic nearest-neighbour thermodynamics (SantaLucia 1998 / Breslauer 1986). + +def compute_nn_enthalpy(seq: str, cfg: dict[str, Any]) -> float: + """Total nearest-neighbour enthalpy ΔH (kcal/mol) for a DNA duplex. + + Parameters + ---------- + seq : str + Primer-length DNA sequence (18–25 bp). + cfg : dict + Benchmark configuration (provides NN enthalpy table). + + Returns + ------- + float + Sum of nearest-neighbour enthalpy contributions plus initiation + and terminal penalties. + + Spec + ---- + §3.1 — Nearest-neighbour enthalpy. + """ + seq = seq.upper() + enthalpy = cfg["thermodynamic"]["nearest_neighbor_params"]["enthalpy_kcal"] + dh = 0.0 + for i in range(len(seq) - 1): + dimer = seq[i : i + 2] + dh += lookup_nn_parameter(dimer, enthalpy) + dh += 0.2 + if seq[0] in "AT": + dh += 2.3 + if seq[-1] in "AT": + dh += 2.3 + return dh + +def compute_nn_entropy(seq: str, cfg: dict[str, Any]) -> float: + """Total nearest-neighbour entropy ΔS (cal/mol·K) for a DNA duplex. + + Parameters + ---------- + seq : str + Primer-length DNA sequence (18–25 bp). + cfg : dict + Benchmark configuration (provides NN entropy table). + + Returns + ------- + float + Sum of nearest-neighbour entropy contributions plus initiation + and terminal penalties. + + Spec + ---- + §3.2 — Nearest-neighbour entropy. + """ + seq = seq.upper() + entropy = cfg["thermodynamic"]["nearest_neighbor_params"]["entropy_cal"] + ds = 0.0 + for i in range(len(seq) - 1): + dimer = seq[i : i + 2] + ds += lookup_nn_parameter(dimer, entropy) + ds += -5.7 + if seq[0] in "AT": + ds += 4.1 + if seq[-1] in "AT": + ds += 4.1 + return ds + +def salt_correction(seq_len: int, cfg: dict[str, Any]) -> float: + """SantaLucia salt correction term for melting temperature. + + Parameters + ---------- + seq_len : int + Length of the primer in bases. + cfg : dict + Benchmark configuration (ion concentrations). + + Returns + ------- + float + Correction term added to the denominator before the Tm formula. + + Spec + ---- + §3.3 — Salt correction (SantaLucia 1998). + """ + thermo = cfg["thermodynamic"] + na = thermo["monovalent_cation_concentration"] + mg = thermo["divalent_cation_concentration"] + dntp = thermo["dntp_concentration"] + na_eq = na + 3.7 * math.sqrt(max(0.0, mg - dntp)) + return 0.368 * (seq_len - 1) * math.log(na_eq / 1000.0) + +def compute_tm(seq: str, cfg: dict[str, Any]) -> float: + """Melting temperature Tm (°C) using the nearest-neighbour model. + + Tm = (1000·ΔH) / (ΔS + R·ln(C) + salt_correction) − 273.15 + + Parameters + ---------- + seq : str + Primer-length DNA sequence. + cfg : dict + Benchmark configuration. + + Returns + ------- + float + Tm in degrees Celsius, rounded to two decimal places. + + Spec + ---- + §3.4 — Melting temperature (primary thermodynamic metric). + """ + seq = seq.upper() + R = 1.987 + dh = compute_nn_enthalpy(seq, cfg) + ds = compute_nn_entropy(seq, cfg) + sc = salt_correction(len(seq), cfg) + dna_conc = cfg["thermodynamic"]["dna_concentration"] + ct = dna_conc * 1e-9 + tm = (1000.0 * dh) / (ds + R * math.log(ct) + sc) - 273.15 + return round(tm, 2) + +def compute_gc_content(seq: str) -> float: + """GC content as a percentage of total bases. + + Parameters + ---------- + seq : str + DNA string. + + Returns + ------- + float + GC percentage in [0, 100]. Returns 0.0 for empty input. + + Spec + ---- + §3.5 — GC content (simple composition metric). + """ + if not seq: + return 0.0 + seq = seq.upper() + gc_count = sum(1 for base in seq if base in "GC") + return 100.0 * gc_count / len(seq) + +# ╔══════════════════════════════════════════════════════════════════════════════╗ +# ║ Template Alignment (Spec §4) ║ +# ╚══════════════════════════════════════════════════════════════════════════════╝ + +# Primer-template alignment and off-target binding (Spec v2.0 §4). + +def align_primers( + fwd: str, + rev: str, + template: str, + amp_start: int, + amp_end: int, +) -> dict[str, Any]: + """Align forward and reverse primers to the template. + + The forward primer aligns to the forward strand at the amplicon start. + The reverse primer aligns (as reverse complement) to the forward strand + at the amplicon end. + + Parameters + ---------- + fwd : str + Forward primer sequence. + rev : str + Reverse primer sequence. + template : str + Full template sequence. + amp_start : int + 0-based start index of the amplicon. + amp_end : int + 0-based inclusive end index of the amplicon. + + Returns + ------- + dict + ``{"fwd_start", "fwd_end", "rev_start", "rev_end", "perfect_match"}`` + describing the aligned positions. + + Spec + ---- + §4.1 — Primer-template alignment. + """ + fwd_u = fwd.upper() + rev_u = rev.upper() + fwd_start = template.upper().find(fwd_u, amp_start) + if fwd_start == -1: + fwd_start = template.upper().find(fwd_u) + fwd_end = fwd_start + len(fwd_u) if fwd_start != -1 else -1 + + rev_rc = reverse_complement(rev_u) + rev_end = template.upper().find(rev_rc, 0, amp_end + len(rev_u)) + if rev_end == -1: + rev_end = template.upper().find(rev_rc) + rev_start = rev_end - len(rev_u) + 1 if rev_end != -1 else -1 + if rev_start < 0 and rev_end != -1: + rev_start = 0 + + perfect_match = ( + fwd_start != -1 + and template[fwd_start:fwd_start + len(fwd_u)].upper() == fwd_u + and rev_end != -1 + and template[rev_end:rev_end + len(rev_rc)].upper() == rev_rc + ) + + return { + "fwd_start": fwd_start, + "fwd_end": fwd_end, + "rev_start": rev_start, + "rev_end": rev_end, + "perfect_match": perfect_match, + } + +def count_offtarget_binding( + primer: str, + template: str, + min_homology: int = 7, +) -> int: + """Count non-target binding sites for a primer on the template. + + Slides the primer along the template and identifies sub-sequences + with complementarity ≥ ``min_homology`` bases outside the intended + binding region. + + Parameters + ---------- + primer : str + Primer sequence to test. + template : str + Full template sequence. + min_homology : int + Minimum contiguous match to count as a potential off-target site. + + Returns + ------- + int + Number of off-target binding sites found. + + Spec + ---- + §4.2 — In-silico PCR specificity. + """ + raise NotImplementedError + +# ╔══════════════════════════════════════════════════════════════════════════════╗ +# ║ Validation Gates (Spec §5) ║ +# ╚══════════════════════════════════════════════════════════════════════════════╝ + +# Hard validation gates (Spec v2.0 §5). +# Any violation sets valid = 0 and combined_score = 0. + +def validate_charset(fwd: str, rev: str) -> bool: + """Check that both primers contain only A, T, C, G. + + Parameters + ---------- + fwd : str + rev : str + + Returns + ------- + bool + True if both primers contain only valid DNA bases. + + Spec + ---- + §5.1 — Character set. + """ + return is_valid_dna(fwd) and is_valid_dna(rev) + +def validate_length(fwd: str, rev: str, cfg: dict[str, Any]) -> bool: + """Check that each primer length is within [min, max]. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + bool + True if both primer lengths are within bounds. + + Spec + ---- + §5.2 — Primer length. + """ + lo = cfg["constraints"]["primer_length"]["min"] + hi = cfg["constraints"]["primer_length"]["max"] + return lo <= len(fwd) <= hi and lo <= len(rev) <= hi + +def validate_gc_content(fwd: str, rev: str, cfg: dict[str, Any]) -> bool: + """Check that each primer GC content is within [min, max]. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + bool + True if both primer GC percentages are within bounds. + + Spec + ---- + §5.3 — GC content range. + """ + lo = cfg["constraints"]["gc_content"]["min"] + hi = cfg["constraints"]["gc_content"]["max"] + gc_fwd = compute_gc_content(fwd) + gc_rev = compute_gc_content(rev) + return lo <= gc_fwd <= hi and lo <= gc_rev <= hi + +def validate_tm(fwd: str, rev: str, cfg: dict[str, Any]) -> bool: + """Check that both Tm values and their difference are within bounds. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + bool + True if both Tm values and |Tm_fwd − Tm_rev| pass. + + Spec + ---- + §5.4 — Melting temperature bounds. + """ + tm_min = cfg["constraints"]["melting_temperature"]["min"] + tm_max = cfg["constraints"]["melting_temperature"]["max"] + tm_diff_max = cfg["constraints"]["tm_difference_max"] + tm_fwd = compute_tm(fwd, cfg) + tm_rev = compute_tm(rev, cfg) + if not (tm_min <= tm_fwd <= tm_max): + return False + if not (tm_min <= tm_rev <= tm_max): + return False + return abs(tm_fwd - tm_rev) <= tm_diff_max + +def validate_gc_clamp(fwd: str, rev: str, cfg: dict[str, Any]) -> bool: + """Check GC-clamp (G or C in the last 5 bases at the 3' end). + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + bool + True if both primers satisfy the GC-clamp requirement. + + Spec + ---- + §5.5 — GC clamp requirement. + """ + required = cfg["constraints"]["gc_clamp"]["required"] + if not required: + return True + for primer in (fwd, rev): + last5 = primer[-5:].upper() + gc_count = sum(1 for base in last5 if base in "GC") + if gc_count < 1: + return False + return True + +def validate_self_complementarity( + fwd: str, rev: str, cfg: dict[str, Any], +) -> bool: + """Check self- and cross-dimer complementarity limits. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + bool + True if all self- and cross-complementarity values are within bounds. + + Spec + ---- + §5.6 — Self- and cross-complementarity. + """ + max_self = cfg["constraints"]["self_complementarity_max"] + max_pair = cfg["constraints"]["pair_complementarity_max"] + if compute_self_complementarity(fwd) > max_self: + return False + if compute_self_complementarity(rev) > max_self: + return False + fwd_rc = reverse_complement(fwd) + rev_rc = reverse_complement(rev) + fwd_u = fwd.upper() + rev_u = rev.upper() + fwd_rc_u = fwd_rc.upper() + rev_rc_u = rev_rc.upper() + best_cross = 0 + for i in range(len(fwd_u) - 2): + for j in range(i + 3, len(fwd_u) + 1): + sub = fwd_u[i:j] + if sub in rev_rc_u: + best_cross = max(best_cross, len(sub)) + if best_cross > max_pair: + return False + best_cross_rev = 0 + for i in range(len(rev_u) - 2): + for j in range(i + 3, len(rev_u) + 1): + sub = rev_u[i:j] + if sub in fwd_rc_u: + best_cross_rev = max(best_cross_rev, len(sub)) + return best_cross_rev <= max_pair + +def validate_hairpin(fwd: str, rev: str, cfg: dict[str, Any]) -> bool: + """Check maximum hairpin stem length for both primers. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + bool + True if both primers have hairpin stem length within limits. + + Spec + ---- + §5.7 — Hairpin structure. + """ + max_stem = cfg["constraints"]["max_hairpin_stem"] + for primer in (fwd, rev): + seq = primer.upper() + best = 0 + for i in range(len(seq) - 4): + for j in range(i + 3, len(seq) - 2): + sub = seq[i:j] + sub_rc = "".join(_COMPLEMENT[b] for b in reversed(sub)) + if len(sub) < 3: + continue + for k in range(j + 3, len(seq) - len(sub) + 1): + candidate = seq[k:k + len(sub)] + if candidate == sub_rc: + best = max(best, len(sub)) + if best > max_stem: + return False + return True + +def validate_mononucleotide_run( + fwd: str, rev: str, cfg: dict[str, Any], +) -> bool: + """Check that no homopolymer run exceeds max_poly_x_run. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + bool + True if neither primer contains an excessive mononucleotide run. + + Spec + ---- + §5.8 — Homopolymer / mononucleotide runs. + """ + max_run = cfg["constraints"]["max_poly_x_run"] + for primer in (fwd, rev): + best = 1 + run = 1 + seq = primer.upper() + for i in range(1, len(seq)): + if seq[i] == seq[i - 1]: + run += 1 + best = max(best, run) + else: + run = 1 + if best > max_run: + return False + return True + +def validate_alignment( + fwd: str, rev: str, template: str, amp_start: int, amp_end: int, +) -> bool: + """Check that primers align specifically to the intended amplicon region. + + Parameters + ---------- + fwd : str + rev : str + template : str + amp_start : int + amp_end : int + + Returns + ------- + bool + True if both primers align to the expected amplicon boundaries. + + Spec + ---- + §5.9 — Alignment specificity. + """ + fwd_u = fwd.upper() + rev_u = rev.upper() + tpl_u = template.upper() + fwd_pos = tpl_u.find(fwd_u, max(0, amp_start - 5)) + if fwd_pos == -1 or fwd_pos > amp_start + 5: + return False + rev_rc = reverse_complement(rev_u) + rev_rc_pos = tpl_u.find(rev_rc, max(0, amp_end - len(rev_rc) - 5)) + if rev_rc_pos == -1: + return False + return True + +def validate_product_length( + fwd: str, rev: str, template: str, +) -> bool: + """Estimate PCR product length from aligned primer positions. + + Parameters + ---------- + fwd : str + rev : str + template : str + + Returns + ------- + bool + True if the estimated product length is positive and plausible. + + Spec + ---- + §5.10 — Product length sanity check. + """ + fwd_u = fwd.upper() + rev_u = rev.upper() + tpl_u = template.upper() + fwd_pos = tpl_u.find(fwd_u) + if fwd_pos == -1: + return False + rev_rc = reverse_complement(rev_u) + rev_pos = tpl_u.rfind(rev_rc) + if rev_pos == -1: + return False + prod_len = rev_pos + len(rev_rc) - fwd_pos + return 1 <= prod_len <= len(template) + +def run_hard_gates( + fwd: str, rev: str, cfg: dict[str, Any], + template: str, amp_start: int, amp_end: int, +) -> bool: + """Run every hard validation gate, including template-dependent gates. + + This is the single aggregation point for all hard gates defined in + Frozen Spec §5. Short-circuits on the first violation. + + Parameters + ---------- + fwd : str + Forward primer sequence. + rev : str + Reverse primer sequence. + cfg : dict + Benchmark configuration. + template : str + Full template sequence (required for §5.9, §5.10). + amp_start : int + 0-based start index of the amplicon region. + amp_end : int + 0-based inclusive end index of the amplicon region. + + Returns + ------- + bool + True if every hard gate passes; False on the first failure. + + Spec + ---- + §5.11 — Hard-gate aggregation. + """ + if not validate_charset(fwd, rev): + return False + if not validate_length(fwd, rev, cfg): + return False + if not validate_gc_content(fwd, rev, cfg): + return False + if not validate_tm(fwd, rev, cfg): + return False + if not validate_gc_clamp(fwd, rev, cfg): + return False + if not validate_self_complementarity(fwd, rev, cfg): + return False + if not validate_hairpin(fwd, rev, cfg): + return False + if not validate_mononucleotide_run(fwd, rev, cfg): + return False + if not validate_alignment(fwd, rev, template, amp_start, amp_end): + return False + if not validate_product_length(fwd, rev, template): + return False + return True + +# ╔══════════════════════════════════════════════════════════════════════════════╗ +# ║ Metrics (Spec §6) ║ +# ╚══════════════════════════════════════════════════════════════════════════════╝ + +def _linear_score(value: float, threshold: float) -> float: + """Normalised linear penalty in [0, 1]. + + Score = 1.0 at 0, decreases linearly to 0.0 at *threshold*. + Values above *threshold* return 0.0. + + Parameters + ---------- + value : float + Raw measurement to penalise. + threshold : float + Maximum acceptable value (score reaches 0 here). + + Returns + ------- + float + Normalised score in [0, 1]. + """ + return max(0.0, 1.0 - value / threshold) + +# Metric sub-scores (Spec v2.0 §6). +# Each returns a value in [0, 100] (higher is better). Combined via weighted sum. + +def length_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: + """Primer-length quality score in [0, 1]. + + Piecewise-linear function: + - Score = 1.0 inside the ideal interval [20, 22]. + - Linear ramp from the valid boundary to the ideal boundary. + - Clamped to [0, 1]. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.1 — Length metric. + """ + lo = cfg["constraints"]["primer_length"]["min"] + hi = cfg["constraints"]["primer_length"]["max"] + ideal_lo, ideal_hi = 20, 22 + + def _score_one(seq: str) -> float: + n = len(seq) + if n < lo or n > hi: + return 0.0 + if ideal_lo <= n <= ideal_hi: + return 1.0 + if n < ideal_lo: + return (n - lo) / (ideal_lo - lo) + return (hi - n) / (hi - ideal_hi) + + return (_score_one(fwd) + _score_one(rev)) / 2.0 + +def gc_content_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: + """GC-content quality score in [0, 1]. + + Symmetric triangular function using constraint bounds from config. + Score = 1.0 at centre = (gc_min + gc_max) / 2, + drops linearly to 0.0 at gc_min and gc_max. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.2 — GC content metric. + """ + gc_min = cfg["constraints"]["gc_content"]["min"] + gc_max = cfg["constraints"]["gc_content"]["max"] + centre = (gc_min + gc_max) / 2.0 + radius = (gc_max - gc_min) / 2.0 + + gc_fwd = compute_gc_content(fwd) + gc_rev = compute_gc_content(rev) + + def _triangular(gc_pct: float) -> float: + return max(0.0, 1.0 - abs(gc_pct - centre) / radius) + + return (_triangular(gc_fwd) + _triangular(gc_rev)) / 2.0 + +def tm_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: + """Melting-temperature quality score in [0, 1]. + + Two multiplicative components: + 1. Tm quality — proximity to optimal Tm (per-primer, then averaged). + 2. Consistency — small |Tm_fwd − Tm_rev|. + + Combined as: score = quality_avg × consistency. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.3 — Melting temperature metric. + """ + tm_min = cfg["constraints"]["melting_temperature"]["min"] + tm_max = cfg["constraints"]["melting_temperature"]["max"] + tm_opt = cfg["constraints"]["melting_temperature"]["optimal"] + tm_diff_max = cfg["constraints"]["tm_difference_max"] + + tm_fwd = compute_tm(fwd, cfg) + tm_rev = compute_tm(rev, cfg) + + # Per-primer quality: linear peak at optimal + spread = max(tm_opt - tm_min, tm_max - tm_opt) + def _quality(tm: float) -> float: + return max(0.0, 1.0 - abs(tm - tm_opt) / spread) + + quality_avg = (_quality(tm_fwd) + _quality(tm_rev)) / 2.0 + + # Pair consistency + delta = abs(tm_fwd - tm_rev) + consistency = max(0.0, 1.0 - delta / tm_diff_max) + + return quality_avg * consistency + +def gc_clamp_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: + """GC-clamp quality score in [0, 1]. + + Considers only the last 3 nt of each primer (3'' end). + Score = fraction of G/C within those 3 nt, averaged across both primers. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.4 — GC clamp metric. + """ + def _clamp_frac(primer: str) -> float: + last3 = primer[-3:].upper() + if len(last3) < 3: + return 0.0 + gc_count = sum(1 for b in last3 if b in "GC") + return gc_count / 3.0 + + return (_clamp_frac(fwd) + _clamp_frac(rev)) / 2.0 + +def self_complementarity_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: + """Self-complementarity quality score in [0, 1]. + + Score = 1.0 at 0 bp complementarity, decreases linearly to 0.0 + at self_complementarity_max (from config). + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.5 — Self-complementarity metric. + """ + max_val = cfg["constraints"]["self_complementarity_max"] + comp_fwd = compute_self_complementarity(fwd) + comp_rev = compute_self_complementarity(rev) + + return (_linear_score(comp_fwd, max_val) + _linear_score(comp_rev, max_val)) / 2.0 + +def pair_complementarity_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: + """Pair-complementarity (cross-dimer) quality score in [0, 1]. + + Score = 1.0 at 0 bp cross-dimer complementarity, decreases linearly + to 0.0 at pair_complementarity_max (from config). + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.6 — Pair-complementarity metric. + """ + max_val = cfg["constraints"]["pair_complementarity_max"] + fwd_rc = reverse_complement(fwd) + rev_rc = reverse_complement(rev) + fwd_u = fwd.upper() + rev_u = rev.upper() + fwd_rc_u = fwd_rc.upper() + rev_rc_u = rev_rc.upper() + + best = 0 + for i in range(len(fwd_u) - 2): + for j in range(i + 3, len(fwd_u) + 1): + sub = fwd_u[i:j] + if sub in rev_rc_u: + best = max(best, len(sub)) + for i in range(len(rev_u) - 2): + for j in range(i + 3, len(rev_u) + 1): + sub = rev_u[i:j] + if sub in fwd_rc_u: + best = max(best, len(sub)) + + return max(0.0, 1.0 - best / max_val) + +def hairpin_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: + """Hairpin-stem quality score in [0, 1]. + + Score = 1.0 at 0 bp stem, decreases linearly to 0.0 at + max_hairpin_stem (from config). + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.7 — Hairpin stability metric. + """ + max_stem = cfg["constraints"]["max_hairpin_stem"] + + def _max_stem(seq: str) -> int: + s = seq.upper() + best = 0 + for i in range(len(s) - 4): + for j in range(i + 3, len(s) - 2): + sub = s[i:j] + sub_rc = "".join(_COMPLEMENT[b] for b in reversed(sub)) + if len(sub) < 3: + continue + for k in range(j + 3, len(s) - len(sub) + 1): + candidate = s[k:k + len(sub)] + if candidate == sub_rc: + best = max(best, len(sub)) + return best + + stem_fwd = _max_stem(fwd) + stem_rev = _max_stem(rev) + + return (_linear_score(stem_fwd, max_stem) + _linear_score(stem_rev, max_stem)) / 2.0 + +def repeat_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: + """Internal-repeat quality score in [0, 1]. + + Score = 1.0 at 0 bp repeats, decreases linearly to 0.0 at + max_repeat_length (from config). A repeat is a substring that + appears at least twice in a primer sequence. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.8 — Internal repeat metric. + """ + max_val = cfg["constraints"]["max_repeat_length"] + + def _max_repeat(seq: str) -> int: + s = seq.upper() + best = 0 + for i in range(len(s)): + for j in range(i + 1, len(s)): + sub = s[i:j] + if s.count(sub) >= 2: + best = max(best, len(sub)) + return best + + rep_fwd = _max_repeat(fwd) + rep_rev = _max_repeat(rev) + + return (_linear_score(rep_fwd, max_val) + _linear_score(rep_rev, max_val)) / 2.0 + +def mononucleotide_run_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: + """Homopolymer-run quality score in [0, 1]. + + Score = 1.0 at 0 bp run, decreases linearly to 0.0 at + max_poly_x_run (from config). + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.9 — Homopolymer run metric. + """ + max_run = cfg["constraints"]["max_poly_x_run"] + + def _max_run(seq: str) -> int: + s = seq.upper() + best = 1 + run = 1 + for i in range(1, len(s)): + if s[i] == s[i - 1]: + run += 1 + if run > best: + best = run + else: + run = 1 + return best + + run_fwd = _max_run(fwd) + run_rev = _max_run(rev) + + # Shift baseline: a single base is not a run + # v=1 -> 1.0, v=max_run -> 0.0, linear in between + def _linear(v: int) -> float: + if v <= 1: + return 1.0 + return max(0.0, 1.0 - (v - 1) / (max_run - 1)) + + return (_linear(run_fwd) + _linear(run_rev)) / 2.0 + +def product_length_score( + fwd: str, rev: str, cfg: dict[str, Any], product_length: int, +) -> float: + """PCR product-length quality score in [0, 1]. + + Plateau + linear-decay function configured by product_length + preferences in the benchmark config. + + Score = 1.0 inside [preferred_min, preferred_max], + then linear decay to 0 at the extremes (0 and len(template)). + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + product_length : int + PCR product length in base pairs (computed upstream). + + Returns + ------- + float + Normalised score in [0, 1]. + + Spec + ---- + §6.10 — Product length metric. + """ + product_cfg = cfg.get("product_length", {}) + pref_lo = product_cfg.get("preferred_min", 60) + pref_hi = product_cfg.get("preferred_max", 150) + max_len = len(cfg.get("template", {}).get("sequence", "")) + + if product_length <= 0: + return 0.0 + if pref_lo <= product_length <= pref_hi: + return 1.0 + if product_length < pref_lo: + return max(0.0, float(product_length) / pref_lo) + # product_length > pref_hi + if max_len > pref_hi: + return max(0.0, (max_len - product_length) / (max_len - pref_hi)) + return 0.0 + +# Composite scoring (Spec v2.0 §7). + +# 1. Run hard gates. +# 2. If violations exist → combined_score = 0.0, valid = 0. +# 3. Otherwise compute weighted sum of metric sub-scores. +# 4. Return both combined_score and a full score_breakdown. + +def compute_score_breakdown( + fwd: str, rev: str, cfg: dict[str, Any], product_length: int, +) -> dict[str, Any]: + """Compute all 10 quality metrics and aggregate into a composite score. + + Responsibilities + ---------------- + 1. Compute every metric sub-score by calling each metric function. + 2. Read weights from ``cfg["scoring_weights"]``. + 3. Compute weighted sum and normalise to [0, 1]. + 4. Return a detailed breakdown dict. + + This function does NOT perform hard validation, template alignment, + product-length calculation, or I/O. It is called by ``evaluate()`` + after validation has passed. + + Parameters + ---------- + fwd : str + rev : str + cfg : dict + product_length : int + PCR product length in base pairs (computed upstream by evaluate()). + + Returns + ------- + dict + ``{"final_score": float, "sub_scores": dict, "weighted_scores": dict, + "total_weight": float}`` + + Spec + ---- + §7.1 — Score breakdown computation. + """ + # --- 1. Compute all 10 metric sub-scores --- + sub = {} + sub["length_score"] = length_score(fwd, rev, cfg) + sub["gc_content_score"] = gc_content_score(fwd, rev, cfg) + sub["tm_score"] = tm_score(fwd, rev, cfg) + sub["gc_clamp_score"] = gc_clamp_score(fwd, rev, cfg) + sub["self_complementarity_score"] = self_complementarity_score(fwd, rev, cfg) + sub["pair_complementarity_score"] = pair_complementarity_score(fwd, rev, cfg) + sub["repeat_score"] = repeat_score(fwd, rev, cfg) + sub["hairpin_score"] = hairpin_score(fwd, rev, cfg) + sub["mononucleotide_run_score"] = mononucleotide_run_score(fwd, rev, cfg) + sub["product_length_score"] = product_length_score(fwd, rev, cfg, product_length) + + # --- 2. Read weights from configuration --- + weights: dict[str, float] = cfg["scoring_weights"] + + # --- 3. Compute weighted scores --- + weighted: dict[str, float] = {} + weighted_sum = 0.0 + for key in sub: + w = weights.get(key, 0.0) + ws = sub[key] * w + weighted[key] = round(ws, 8) + weighted_sum += ws + + total_weight = sum(weights.values()) + + # --- 4. Normalise to [0, 1] --- + if total_weight > 0.0: + final_score = round(weighted_sum / total_weight, 6) + else: + final_score = 0.0 + + # --- 5. Clamp for numerical safety --- + final_score = max(0.0, min(1.0, final_score)) + + return { + "final_score": final_score, + "sub_scores": sub, + "weighted_scores": weighted, + "total_weight": round(total_weight, 4), + } + +def evaluate( + program_path: str, + *, + repo_root: Path | None = None, + kernel_python: str | None = None, +) -> dict[str, Any]: + """Full evaluation entrypoint called by frontier_eval/run_eval.py. + + This function is the bridge between the unified benchmark harness and + the local evaluator. It: + + 1. Locates the submission (either a ``submission.json`` or a candidate + Python script). + 2. If a script is given, runs it in a temporary directory and captures + its ``submission.json`` output. + 3. Loads the submission and runs hard gates + metric scoring. + 4. Returns a dict with scores and validation status. + + Parameters + ---------- + program_path : str + Path to submission.json or a candidate ``.py`` script. + repo_root : Path or None + Repository root (auto-detected if None). + kernel_python : str or None + Python interpreter path for running candidate scripts. + + Returns + ------- + dict + ``{"final_score": float, "valid": bool, "failure_reason": str, + "sub_scores": dict, "weighted_scores": dict, "total_weight": float}`` + + Spec + ---- + §7.2 — Evaluation entrypoint (unified harness bridge). + """ + # --- 1. Resolve paths --- + if repo_root is None: + repo_root = Path(__file__).resolve().parent.parent.parent.parent.parent + # Resolve candidate path relative to benchmark root first (for scripts), + # then fall back to repo_root (for harness-provided JSON). + benchmark_root = Path(__file__).resolve().parent.parent + program_path_obj = Path(program_path) + if not program_path_obj.is_absolute(): + candidate = benchmark_root / program_path_obj + if candidate.exists(): + program_path_obj = candidate + else: + program_path_obj = repo_root / program_path_obj + + # --- 2. Load configuration --- + try: + cfg = load_config() + except Exception as exc: + return { + "final_score": 0.0, + "valid": False, + "failure_reason": f"Config load failed: {exc}", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + + template_seq: str = cfg["template"]["sequence"] + amp_start: int = cfg["amplicon"]["start_index"] + amp_end: int = cfg["amplicon"]["end_index"] + + # --- 3. Load submission --- + if program_path_obj.suffix == ".json": + with open(program_path_obj) as sf: + submission = json.load(sf) + else: + # Run candidate script and capture submission.json + import subprocess + import tempfile + python = kernel_python or sys.executable + try: + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + script_result = subprocess.run( + [python, str(program_path_obj)], + capture_output=True, + text=True, + timeout=120, + cwd=tmpdir, + ) + if script_result.returncode != 0: + return { + "final_score": 0.0, + "valid": False, + "failure_reason": f"Candidate script failed: {script_result.stderr.strip()}", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + # Parse JSON from stdout + try: + submission = json.loads(script_result.stdout) + except json.JSONDecodeError: + submission_path_candidate = tmp_path / "submission.json" + if submission_path_candidate.exists(): + with open(submission_path_candidate) as sf: + submission = json.load(sf) + else: + return { + "final_score": 0.0, + "valid": False, + "failure_reason": "Candidate produced no parseable submission JSON", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + except subprocess.TimeoutExpired: + return { + "final_score": 0.0, + "valid": False, + "failure_reason": "Candidate script timed out (120s)", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + except Exception as exc: + return { + "final_score": 0.0, + "valid": False, + "failure_reason": f"Candidate execution error: {exc}", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + + # --- 4. Parse submission --- + fwd: str = submission.get("forward_primer", submission.get("fwd", "")) + rev: str = submission.get("reverse_primer", submission.get("rev", "")) + if not fwd or not rev: + return { + "final_score": 0.0, + "valid": False, + "failure_reason": "Submission missing forward_primer or reverse_primer", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + + # --- 5. Compute product length (once) --- + alignment = align_primers(fwd, rev, template_seq, amp_start, amp_end) + if not alignment["perfect_match"]: + pass # alignment failed; hard gate will catch it + rev_rc_len = len(reverse_complement(rev)) + if alignment["fwd_start"] != -1 and alignment["rev_end"] != -1: + product_length = alignment["rev_end"] + rev_rc_len - alignment["fwd_start"] + else: + product_length = 0 + + # --- 6. Run hard validation gates --- + gates_ok = run_hard_gates( + fwd, rev, cfg, + template_seq, amp_start, amp_end, + ) + if not gates_ok: + return { + "final_score": 0.0, + "valid": False, + "failure_reason": "Hard validation gate(s) failed", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + + # --- 7. Compute score breakdown --- + breakdown = compute_score_breakdown(fwd, rev, cfg, product_length) + + return { + "final_score": breakdown["final_score"], + "valid": True, + "failure_reason": "", + "sub_scores": breakdown["sub_scores"], + "weighted_scores": breakdown["weighted_scores"], + "total_weight": breakdown["total_weight"], + } + +# ╔══════════════════════════════════════════════════════════════════════════════╗ +# ║ CLI (Spec §7.2) ║ +# ╚══════════════════════════════════════════════════════════════════════════════╝ + +def main() -> None: + """Command-line entry point for standalone evaluation. + + Supports two invocation modes: + + - ``python verification/evaluator.py --submission submission.json`` + - ``python verification/evaluator.py scripts/init.py`` + + Spec + ---- + §7.2 — CLI invocation. + """ + parser = argparse.ArgumentParser( + description="PCR Primer Design Optimization — Evaluator", + ) + parser.add_argument( + "candidate", + nargs="?", + help="Path to a candidate Python script (e.g. scripts/init.py).", + ) + parser.add_argument( + "--submission", + type=str, + default=None, + help="Path to an existing submission.json.", + ) + parser.add_argument( + "--metrics-out", + type=str, + default=None, + help="Optional path to write metrics JSON (used by frontier_eval).", + ) + args = parser.parse_args() + + # Determine the input: --submission takes priority, then positional arg. + if args.submission is not None: + target: str = args.submission + elif args.candidate is not None: + target = args.candidate + else: + parser.print_help() + sys.exit(1) + + result = evaluate(target) + + # Pretty-print to stdout. + print(json.dumps(result, indent=2)) + + # Write metrics file if requested. + if args.metrics_out is not None: + metrics_path = Path(args.metrics_out) + metrics_path.parent.mkdir(parents=True, exist_ok=True) + with open(metrics_path, "w", encoding="utf-8") as mf: + json.dump(result, mf, indent=2) + +if __name__ == "__main__": + main() diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/requirements.txt b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/requirements.txt new file mode 100644 index 00000000..ee3a51aa --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/requirements.txt @@ -0,0 +1 @@ +numpy>=1.20 From 077f3a47b29ddc1dd047fa9a4aa4959719f40719 Mon Sep 17 00:00:00 2001 From: y-ji24 Date: Tue, 21 Jul 2026 16:30:15 +0800 Subject: [PATCH 3/5] fix(Bioinformatics/PrimerDesignOptimization): address PR #84 review issues - Move EVOLVE-BLOCK to wrap only design_primers() in scripts/init.py - Add random.seed(42) for reproducibility in baseline/solution.py - Fix validate_gc_clamp() to enforce upper bound (max_gc_in_last_5) - Add hidden templates (AT-rich, GC-rich, medium) for anti-cheating - Add frontier_eval/evaluator.py delegation wrapper - Add baseline/result_log.json with reference score - Add verification/test_evaluator.py with 45 comprehensive tests - Fix UTF-8 BOM encoding in baseline/solution.py --- .../baseline/result_log.json | 30 +++ .../baseline/solution.py | 3 +- .../frontier_eval/evaluator.py | 34 +++ .../hidden_templates/hidden_at_rich_001.json | 12 + .../hidden_templates/hidden_gc_rich_001.json | 12 + .../hidden_templates/hidden_medium_001.json | 12 + .../PrimerDesignOptimization/scripts/init.py | 7 +- .../verification/evaluator.py | 61 ++++- .../verification/test_evaluator.py | 246 ++++++++++++++++++ 9 files changed, 413 insertions(+), 4 deletions(-) create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/result_log.json create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/evaluator.py create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_at_rich_001.json create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_gc_rich_001.json create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_medium_001.json create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/verification/test_evaluator.py diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/result_log.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/result_log.json new file mode 100644 index 00000000..0dd33874 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/result_log.json @@ -0,0 +1,30 @@ +{ + "final_score": 0.462081, + "valid": true, + "failure_reason": "", + "sub_scores": { + "length_score": 1.0, + "gc_content_score": 0.5, + "tm_score": 0.12969333333333324, + "gc_clamp_score": 0.5, + "self_complementarity_score": 0.4166666666666667, + "pair_complementarity_score": 0.5, + "repeat_score": 0.125, + "hairpin_score": 0.625, + "mononucleotide_run_score": 0.33333333333333337, + "product_length_score": 1.0 + }, + "weighted_scores": { + "length_score": 0.5, + "gc_content_score": 0.4, + "tm_score": 0.12969333, + "gc_clamp_score": 0.3, + "self_complementarity_score": 0.29166667, + "pair_complementarity_score": 0.35, + "repeat_score": 0.05, + "hairpin_score": 0.3125, + "mononucleotide_run_score": 0.1, + "product_length_score": 0.2 + }, + "total_weight": 5.7 +} \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/solution.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/solution.py index 9c7cabab..b84ea3f5 100644 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/solution.py +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/baseline/solution.py @@ -1,4 +1,4 @@ -"""Baseline solution for PCR Primer Design Optimization. +"""Baseline solution for PCR Primer Design Optimization. This module implements a genetic-algorithm-based primer designer that searches for optimal forward/reverse primer pairs given a template @@ -401,6 +401,7 @@ def design_primers() -> dict[str, str]: dict with keys "forward_primer" and "reverse_primer". """ cfg = load_config() + random.seed(42) template = cfg["template"]["sequence"] amp_start = cfg["amplicon"]["start_index"] amp_end = cfg["amplicon"]["end_index"] diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/evaluator.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/evaluator.py new file mode 100644 index 00000000..3788cae3 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/frontier_eval/evaluator.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import inspect +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from typing import Any + + +def _load_verification_module() -> Any: + evaluator_path = ( + Path(__file__).resolve().parents[1] / "verification" / "evaluator.py" + ).resolve() + spec = spec_from_file_location("_frontier_eval_verification_evaluator", evaluator_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Failed to load verification evaluator from {evaluator_path}") + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def evaluate( + program_path: str, + *, + repo_root: Path | None = None, + kernel_python: str | None = None, +) -> Any: + module = _load_verification_module() + evaluate_fn = getattr(module, "evaluate") + kwargs: dict[str, Any] = {} + if "repo_root" in inspect.signature(evaluate_fn).parameters and repo_root is not None: + kwargs["repo_root"] = repo_root + if "kernel_python" in inspect.signature(evaluate_fn).parameters and kernel_python is not None: + kwargs["kernel_python"] = kernel_python + return evaluate_fn(program_path, **kwargs) diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_at_rich_001.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_at_rich_001.json new file mode 100644 index 00000000..387a4a57 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_at_rich_001.json @@ -0,0 +1,12 @@ +{ + "template": { + "name": "hidden_at_rich_001", + "sequence": "TTTGCGTCCACAAGAAAGCGTTATACTCTATCAAGACCTGACATTGCTATATGATAGTCTTTTCAAGGAAGTCGTAGGATAGTTGTCCTATAACAGACTA", + "description": "Hidden validation template with AT-rich content (~38%), 100 bp." + }, + "amplicon": { + "start_index": 20, + "end_index": 80, + "strand": "forward" + } +} diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_gc_rich_001.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_gc_rich_001.json new file mode 100644 index 00000000..34726dbc --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_gc_rich_001.json @@ -0,0 +1,12 @@ +{ + "template": { + "name": "hidden_gc_rich_001", + "sequence": "CTTACCACGGCCGTTGTGCGTAGTAACCTAGCGATAAAAGCTTGAACTAGAAGGTGCCCCGGGACTGGTGCACCGTTCAAAAAATTTCGTAAGCGCGCTGTCGGTCTCCGACGGCCTACCACGTGTTCCAGTGATCCCCGAGTGCCGGGG", + "description": "Hidden validation template with GC-rich content (~57%), 150 bp." + }, + "amplicon": { + "start_index": 30, + "end_index": 120, + "strand": "forward" + } +} diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_medium_001.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_medium_001.json new file mode 100644 index 00000000..d5437aca --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_medium_001.json @@ -0,0 +1,12 @@ +{ + "template": { + "name": "hidden_medium_001", + "sequence": "CATTTCTCACGATGGGATAACATCGCCACCCAATTTTAAGCGGATGTCCTTCAGCATACTTACGGCGACTGGTTCTGTCTGTAGTCGTGTCGGTAGTGATACTCCATGGGTGGATTCCGCTTGGGGCAGAGAAACCGAAGTAATAGTTGCCACAAACACTGTTTAGTAAGGCCAAGAATCAACGACATGCCCGATCCTTA", + "description": "Hidden validation template with medium GC content (~48%), 200 bp." + }, + "amplicon": { + "start_index": 40, + "end_index": 160, + "strand": "forward" + } +} diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/scripts/init.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/scripts/init.py index dce8efb2..613477e7 100644 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/scripts/init.py +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/scripts/init.py @@ -1,4 +1,3 @@ -# EVOLVE-BLOCK-START """Baseline PCR primer design optimizer. DO NOT MODIFY: load_config(), score_primer_pair(), compute_melting_temperature(), @@ -408,6 +407,9 @@ def score_primer_pair(fwd: str, rev: str, cfg: dict[str, Any]) -> dict[str, Any] # ALLOWED TO MODIFY — Primer design optimizer # --------------------------------------------------------------------------- + +# EVOLVE-BLOCK-START + def design_primers() -> dict[str, str]: """Design PCR primers for the target template. @@ -432,6 +434,8 @@ def design_primers() -> dict[str, str]: return {"forward_primer": fwd, "reverse_primer": rev} +# EVOLVE-BLOCK-END + # --------------------------------------------------------------------------- # Main @@ -456,4 +460,3 @@ def design_primers() -> dict[str, str]: with open("submission.json", "w", encoding="utf-8") as f: json.dump(result, f, indent=2) print("Submission written to submission.json") -# EVOLVE-BLOCK-END diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py index 6569ccf6..0466e00b 100644 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py @@ -549,10 +549,11 @@ def validate_gc_clamp(fwd: str, rev: str, cfg: dict[str, Any]) -> bool: required = cfg["constraints"]["gc_clamp"]["required"] if not required: return True + max_gc = cfg["constraints"]["gc_clamp"]["max_gc_in_last_5"] for primer in (fwd, rev): last5 = primer[-5:].upper() gc_count = sum(1 for base in last5 if base in "GC") - if gc_count < 1: + if gc_count < 1 or gc_count > max_gc: return False return True @@ -1542,5 +1543,63 @@ def main() -> None: with open(metrics_path, "w", encoding="utf-8") as mf: json.dump(result, mf, indent=2) + + +def _load_hidden_templates() -> list[dict[str, Any]]: + """Load all hidden template configurations. + + Scans ``references/hidden_templates/`` for JSON files, each containing + a template sequence and amplicon definition. + + Returns + ------- + list[dict] + Each entry has ``{"template": ..., "amplicon": ...}``. + + Spec + ---- + з§7.3 з§ Hidden template validation. + """ + ht_dir = Path(__file__).resolve().parent.parent / "references" / "hidden_templates" + if not ht_dir.exists(): + return [] + templates: list[dict[str, Any]] = [] + for fpath in sorted(ht_dir.glob("*.json")): + try: + with open(fpath, encoding="utf-8") as f: + templates.append(json.load(f)) + except (json.JSONDecodeError, OSError): + continue + return templates + + +def _make_hidden_config( + hidden_template: dict[str, Any], + base_cfg: dict[str, Any], +) -> dict[str, Any]: + """Build a full config dict from a hidden template and the base config. + + The returned config uses the hidden template's sequence and amplicon, + but inherits all other parameters (constraints, weights, thermodynamics) + from the base config. + + Parameters + ---------- + hidden_template : dict + Dict with ``"template"`` and ``"amplicon"`` keys. + base_cfg : dict + The full benchmark config (:func:`load_config`). + + Returns + ------- + dict + A new config dict ready for evaluation. + """ + cfg = dict(base_cfg) + cfg["template"] = dict(hidden_template["template"]) + cfg["amplicon"] = dict(hidden_template["amplicon"]) + return cfg + + if __name__ == "__main__": main() diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/test_evaluator.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/test_evaluator.py new file mode 100644 index 00000000..79253ba2 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/test_evaluator.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Comprehensive test suite for PrimerDesignOptimization evaluator.""" +from __future__ import annotations +import json, math, os, sys, unittest +from pathlib import Path +BENCHMARK_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(BENCHMARK_ROOT)) +from verification import evaluator as ev +def _load_cfg(): + return ev.load_config() + +class TestHardGates(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.cfg = _load_cfg() + cls.template = cls.cfg["template"]["sequence"] + cls.amp_start = cls.cfg["amplicon"]["start_index"] + cls.amp_end = cls.cfg["amplicon"]["end_index"] + cls.valid_fwd = "CAAAGCGATTGTTGGGATTGTACT" + cls.valid_rev = "TTAATTCATTAGCCCGACGTTACC" + + def test_charset_valid(self): + self.assertTrue(ev.validate_charset("ATCG","ATCG")) + self.assertTrue(ev.validate_charset("atcg","atcg")) + + def test_charset_invalid(self): + self.assertFalse(ev.validate_charset("ATCGX","ATCG")) + self.assertFalse(ev.validate_charset("","ATCG")) + + def test_length_valid(self): + self.assertTrue(ev.validate_length("A"*20,"T"*20,self.cfg)) + self.assertTrue(ev.validate_length("A"*25,"T"*25,self.cfg)) + + def test_length_too_short(self): + self.assertFalse(ev.validate_length("A"*17,"T"*20,self.cfg)) + self.assertFalse(ev.validate_length("A"*20,"T"*17,self.cfg)) + + def test_length_too_long(self): + self.assertFalse(ev.validate_length("A"*26,"T"*20,self.cfg)) + self.assertFalse(ev.validate_length("A"*20,"T"*26,self.cfg)) + + def test_gc_valid(self): + self.assertTrue(ev.validate_gc_content("ACACACACACACACACACAC","ACACACACACACACACACAC",self.cfg)) + + def test_gc_too_low(self): + self.assertFalse(ev.validate_gc_content("A"*20,"ACACACACACACACACACAC",self.cfg)) + + def test_gc_too_high(self): + self.assertFalse(ev.validate_gc_content("G"*20,"ACACACACACACACACACAC",self.cfg)) + + def test_tm_valid(self): + self.assertTrue(ev.validate_tm(self.valid_fwd,self.valid_rev,self.cfg)) + + def test_tm_too_low(self): + self.assertFalse(ev.validate_tm("A"*20,"A"*20,self.cfg)) + + def test_gc_clamp_valid(self): + self.assertTrue(ev.validate_gc_clamp("AAAAAAAAAAAAAAACCC","TTTTTTTTTTTTTTTGGG",self.cfg)) + + def test_gc_clamp_invalid(self): + self.assertFalse(ev.validate_gc_clamp("A"*19,"T"*19,self.cfg)) + + def test_self_comp_valid(self): + self.assertTrue(ev.validate_self_complementarity("AAAACCCCTTTTGGGG","TTTTTTTTTTTTTTTT",self.cfg)) + + def test_self_comp_invalid(self): + self.assertFalse(ev.validate_self_complementarity("A"*12,"T"*12,self.cfg)) + + def test_hairpin_valid(self): + self.assertTrue(ev.validate_hairpin("ACGTACGTACGT","TTTTTTTTTTTT",self.cfg)) + + def test_hairpin_invalid(self): + self.assertFalse(ev.validate_hairpin("AAAAAAAAACCCGGGTTTTTTTTT","A"*24,self.cfg)) + + def test_mono_run_valid(self): + self.assertTrue(ev.validate_mononucleotide_run("ACGTACGTACGT","ACGTACGTACGT",self.cfg)) + + def test_mono_run_invalid(self): + self.assertFalse(ev.validate_mononucleotide_run("A"*14,"ACGTACGTACGT",self.cfg)) + + def test_alignment_valid(self): + self.assertTrue(ev.validate_alignment(self.valid_fwd,self.valid_rev,self.template,self.amp_start,self.amp_end)) + + def test_alignment_invalid(self): + self.assertFalse(ev.validate_alignment("A"*20,"T"*20,self.template,self.amp_start,self.amp_end)) + + def test_prod_len_valid(self): + self.assertTrue(ev.validate_product_length(self.valid_fwd,self.valid_rev,self.template)) + + def test_prod_len_invalid(self): + self.assertFalse(ev.validate_product_length("A"*20,"A"*20,self.template)) + + def test_run_hard_gates_valid(self): + self.assertTrue(ev.run_hard_gates(self.valid_fwd,self.valid_rev,self.cfg,self.template,self.amp_start,self.amp_end)) + + def test_run_hard_gates_invalid(self): + self.assertFalse(ev.run_hard_gates("ATCGX","ATCG",self.cfg,self.template,self.amp_start,self.amp_end)) + +class TestMetrics(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.cfg = _load_cfg() + cls.valid_fwd = "CAAAGCGATTGTTGGGATTGTACT" + cls.valid_rev = "TTAATTCATTAGCCCGACGTTACC" + + def _check(self, name, func, *args, **kw): + r = func(*args, **kw) + self.assertIsInstance(r, float) + self.assertFalse(math.isnan(r), f"{name} NaN") + self.assertFalse(math.isinf(r), f"{name} Inf") + self.assertGreaterEqual(r, 0.0, f"{name} < 0") + self.assertLessEqual(r, 1.0, f"{name} > 1") + return r + + def test_length_score(self): + self._check("length_score", ev.length_score, "A"*20, "T"*20, self.cfg) + + def test_gc_content_score(self): + self._check("gc_content_score", ev.gc_content_score, "ACACACACACACACACACAC","ACACACACACACACACACAC",self.cfg) + + def test_tm_score(self): + self._check("tm_score", ev.tm_score, self.valid_fwd, self.valid_rev, self.cfg) + + def test_gc_clamp_score(self): + self._check("gc_clamp_score", ev.gc_clamp_score, self.valid_fwd, self.valid_rev, self.cfg) + + def test_self_comp_score(self): + self._check("self_complementarity_score", ev.self_complementarity_score, self.valid_fwd, self.valid_rev, self.cfg) + + def test_pair_comp_score(self): + self._check("pair_complementarity_score", ev.pair_complementarity_score, self.valid_fwd, self.valid_rev, self.cfg) + + def test_repeat_score(self): + self._check("repeat_score", ev.repeat_score, self.valid_fwd, self.valid_rev, self.cfg) + + def test_hairpin_score(self): + self._check("hairpin_score", ev.hairpin_score, self.valid_fwd, self.valid_rev, self.cfg) + + def test_mono_run_score(self): + self._check("mononucleotide_run_score", ev.mononucleotide_run_score, self.valid_fwd, self.valid_rev, self.cfg) + + def test_prod_len_score(self): + self._check("product_length_score", ev.product_length_score, self.valid_fwd, self.valid_rev, self.cfg, 80) + +class TestBaseline(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.cfg = _load_cfg() + cls.benchmark_root = Path(__file__).resolve().parent.parent + + def test_baseline_eval(self): + r = ev.evaluate(str(self.benchmark_root/"scripts"/"init.py")) + self.assertIn("final_score", r) + self.assertIn("valid", r) + self.assertIn("failure_reason", r) + self.assertIn("sub_scores", r) + self.assertIn("weighted_scores", r) + self.assertIn("total_weight", r) + self.assertTrue(r["valid"], f"Baseline failed: {r['failure_reason']}") + self.assertGreaterEqual(r["final_score"], 0.0) + self.assertLessEqual(r["final_score"], 1.0) + expected = ["length_score","gc_content_score","tm_score","gc_clamp_score","self_complementarity_score","pair_complementarity_score","repeat_score","hairpin_score","mononucleotide_run_score","product_length_score"] + for m in expected: + self.assertIn(m, r["sub_scores"]) + self.assertIn(m, r["weighted_scores"]) + self.assertGreaterEqual(r["sub_scores"][m], 0.0) + self.assertLessEqual(r["sub_scores"][m], 1.0) + self.assertAlmostEqual(r["total_weight"], sum(self.cfg["scoring_weights"].values()), places=4) + + def test_determinism(self): + candidate = str(self.benchmark_root/"scripts"/"init.py") + r1 = ev.evaluate(candidate) + r2 = ev.evaluate(candidate) + r3 = ev.evaluate(candidate) + self.assertEqual(r1["final_score"], r2["final_score"]) + self.assertEqual(r1["final_score"], r3["final_score"]) + self.assertEqual(r1["sub_scores"], r2["sub_scores"]) + +class TestHiddenTemplates(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.benchmark_root = Path(__file__).resolve().parent.parent + + def test_hidden_loaded(self): + hidden = ev._load_hidden_templates() + self.assertGreater(len(hidden), 0) + for ht in hidden: + self.assertIn("template", ht) + self.assertIn("amplicon", ht) + self.assertGreater(len(ht["template"]["sequence"]), 0) + + def test_hidden_config(self): + cfg = _load_cfg() + hidden = ev._load_hidden_templates() + if not hidden: + self.skipTest("no hidden templates") + for ht in hidden: + hcfg = ev._make_hidden_config(ht, cfg) + self.assertEqual(hcfg["template"]["sequence"], ht["template"]["sequence"]) + +class TestUtilities(unittest.TestCase): + def test_rc(self): + self.assertEqual(ev.reverse_complement("ATCG"), "CGAT") + self.assertEqual(ev.reverse_complement(""), "") + + def test_canonical(self): + self.assertEqual(ev.canonical_pair_key("AA"), "AA/TT") + self.assertEqual(ev.canonical_pair_key("TT"), "TT/AA") + + def test_valid_dna(self): + self.assertTrue(ev.is_valid_dna("ATCG")) + self.assertFalse(ev.is_valid_dna("ATCGX")) + + def test_gc_content(self): + self.assertAlmostEqual(ev.compute_gc_content("ATCG"), 50.0) + self.assertAlmostEqual(ev.compute_gc_content("AAAA"), 0.0) + self.assertAlmostEqual(ev.compute_gc_content("GGGG"), 100.0) + + def test_tm_deterministic(self): + cfg = _load_cfg() + self.assertAlmostEqual(ev.compute_tm("CGTCCACAAGAAAGCTTTCC",cfg), ev.compute_tm("CGTCCACAAGAAAGCTTTCC",cfg), places=6) + +class TestScoreBreakdown(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.cfg = _load_cfg() + cls.valid_fwd = "CAAAGCGATTGTTGGGATTGTACT" + cls.valid_rev = "TTAATTCATTAGCCCGACGTTACC" + + def test_structure(self): + bd = ev.compute_score_breakdown(self.valid_fwd, self.valid_rev, self.cfg, 80) + self.assertIn("final_score", bd) + self.assertIn("sub_scores", bd) + self.assertIn("weighted_scores", bd) + self.assertIn("total_weight", bd) + self.assertNotIn("valid", bd) + self.assertEqual(len(bd["sub_scores"]), 10) + self.assertEqual(len(bd["weighted_scores"]), 10) + + def test_deterministic(self): + bd1 = ev.compute_score_breakdown(self.valid_fwd, self.valid_rev, self.cfg, 80) + bd2 = ev.compute_score_breakdown(self.valid_fwd, self.valid_rev, self.cfg, 80) + self.assertEqual(bd1["final_score"], bd2["final_score"]) + +if __name__ == "__main__": + unittest.main(verbosity=2) From b2966732453251c068063836321263a4bad32448 Mon Sep 17 00:00:00 2001 From: y-ji24 Date: Tue, 21 Jul 2026 16:37:28 +0800 Subject: [PATCH 4/5] docs(Bioinformatics/PrimerDesignOptimization): fix remaining cosmetic issues before PR merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove Cyrillic з corruption in evaluator.py line 1561 (з§ → §) - Replace 'Frozen Spec v2.0' references with 'See Task.md for the full benchmark specification' - Remove 'Frozen Spec' prefix from line 753 (Frozen Spec §5 → §5) - Remove 'Spec v2.0' → 'Spec' in section header comments - Update README.md file tree to include: README_zh-CN.md, Task_zh-CN.md, test_evaluator.py, hidden_templates/, result_log.json, frontier_eval/evaluator.py - Update README_zh-CN.md file tree to match --- .../PrimerDesignOptimization/README.md | 13 +++++++++++-- .../PrimerDesignOptimization/README_zh-CN.md | 13 +++++++++++-- .../verification/evaluator.py | 14 +++++++------- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/README.md b/benchmarks/Bioinformatics/PrimerDesignOptimization/README.md index 0c18c1b6..8cc98142 100644 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/README.md +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/README.md @@ -9,15 +9,23 @@ and structural stability under a nearest-neighbor thermodynamic model. ```text PrimerDesignOptimization/ ├── README.md +├── README_zh-CN.md ├── Task.md +├── Task_zh-CN.md ├── references/ -│ └── primer_config.json +│ ├── primer_config.json +│ └── hidden_templates/ +│ ├── hidden_at_rich_001.json +│ ├── hidden_gc_rich_001.json +│ └── hidden_medium_001.json ├── scripts/ │ └── init.py ├── baseline/ -│ └── solution.py +│ ├── solution.py +│ └── result_log.json ├── verification/ │ ├── evaluator.py +│ ├── test_evaluator.py │ └── requirements.txt └── frontier_eval/ ├── eval_command.txt @@ -25,6 +33,7 @@ PrimerDesignOptimization/ ├── agent_files.txt ├── artifact_files.txt ├── constraints.txt + ├── evaluator.py └── run_eval.py ``` diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/README_zh-CN.md b/benchmarks/Bioinformatics/PrimerDesignOptimization/README_zh-CN.md index 20ba315c..b767ba5d 100644 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/README_zh-CN.md +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/README_zh-CN.md @@ -9,15 +9,23 @@ ```text PrimerDesignOptimization/ ├── README.md +├── README_zh-CN.md ├── Task.md +├── Task_zh-CN.md ├── references/ -│ └── primer_config.json +│ ├── primer_config.json +│ └── hidden_templates/ +│ ├── hidden_at_rich_001.json +│ ├── hidden_gc_rich_001.json +│ └── hidden_medium_001.json ├── scripts/ │ └── init.py ├── baseline/ -│ └── solution.py +│ ├── solution.py +│ └── result_log.json ├── verification/ │ ├── evaluator.py +│ ├── test_evaluator.py │ └── requirements.txt └── frontier_eval/ ├── eval_command.txt @@ -25,6 +33,7 @@ PrimerDesignOptimization/ ├── agent_files.txt ├── artifact_files.txt ├── constraints.txt + ├── evaluator.py └── run_eval.py ``` diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py index 0466e00b..0df2bbc0 100644 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py @@ -9,7 +9,7 @@ python verification/evaluator.py --submission submission.json python verification/evaluator.py scripts/init.py -Frozen Spec v2.0 sections referenced throughout. +See Task.md for the full benchmark specification. """ from __future__ import annotations @@ -328,7 +328,7 @@ def compute_gc_content(seq: str) -> float: # ║ Template Alignment (Spec §4) ║ # ╚══════════════════════════════════════════════════════════════════════════════╝ -# Primer-template alignment and off-target binding (Spec v2.0 §4). +# Primer-template alignment and off-target binding (Spec §4). def align_primers( fwd: str, @@ -431,7 +431,7 @@ def count_offtarget_binding( # ║ Validation Gates (Spec §5) ║ # ╚══════════════════════════════════════════════════════════════════════════════╝ -# Hard validation gates (Spec v2.0 §5). +# Hard validation gates (Spec §5). # Any violation sets valid = 0 and combined_score = 0. def validate_charset(fwd: str, rev: str) -> bool: @@ -750,7 +750,7 @@ def run_hard_gates( """Run every hard validation gate, including template-dependent gates. This is the single aggregation point for all hard gates defined in - Frozen Spec §5. Short-circuits on the first violation. + §5. Short-circuits on the first violation. Parameters ---------- @@ -822,7 +822,7 @@ def _linear_score(value: float, threshold: float) -> float: """ return max(0.0, 1.0 - value / threshold) -# Metric sub-scores (Spec v2.0 §6). +# Metric sub-scores (Spec §6). # Each returns a value in [0, 100] (higher is better). Combined via weighted sum. def length_score(fwd: str, rev: str, cfg: dict[str, Any]) -> float: @@ -1218,7 +1218,7 @@ def product_length_score( return max(0.0, (max_len - product_length) / (max_len - pref_hi)) return 0.0 -# Composite scoring (Spec v2.0 §7). +# Composite scoring (Spec §7). # 1. Run hard gates. # 2. If violations exist → combined_score = 0.0, valid = 0. @@ -1558,7 +1558,7 @@ def _load_hidden_templates() -> list[dict[str, Any]]: Spec ---- - з§7.3 з§ Hidden template validation. + §7.3 § Hidden template validation. """ ht_dir = Path(__file__).resolve().parent.parent / "references" / "hidden_templates" if not ht_dir.exists(): From 977d73156f890f456a346f0cba90c75e73ef36a8 Mon Sep 17 00:00:00 2001 From: y-ji24 Date: Wed, 22 Jul 2026 14:10:43 +0800 Subject: [PATCH 5/5] fix(Bioinformatics/PrimerDesignOptimization): integrate hidden templates into evaluation pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace old non-functional hidden templates with 3 validated templates (AT-rich ~45% GC, balanced ~50% GC, GC-rich ~55-60% GC) - Add _run_candidate_on_config() to execute candidate against any config - Integrate hidden template evaluation into evaluate() — iterates all 3 hidden templates after default template evaluation; failure on any hidden template returns valid=false, final_score=0.0 - Verify baseline passes all 3 hidden templates - Add test_hidden_template_evaluation and test_hidden_template_failure_rejection to test_evaluator.py Closes the final critical issue from PR #84 review. --- .../hidden_templates/hidden_at_rich_001.json | 12 -- .../hidden_templates/hidden_gc_rich_001.json | 12 -- .../hidden_templates/hidden_medium_001.json | 12 -- .../hidden_templates/hidden_template_001.json | 12 ++ .../hidden_templates/hidden_template_002.json | 12 ++ .../hidden_templates/hidden_template_003.json | 12 ++ .../verification/evaluator.py | 133 +++++++++++++++++- .../verification/test_evaluator.py | 28 +++- 8 files changed, 191 insertions(+), 42 deletions(-) delete mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_at_rich_001.json delete mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_gc_rich_001.json delete mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_medium_001.json create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_001.json create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_002.json create mode 100644 benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_003.json diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_at_rich_001.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_at_rich_001.json deleted file mode 100644 index 387a4a57..00000000 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_at_rich_001.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "name": "hidden_at_rich_001", - "sequence": "TTTGCGTCCACAAGAAAGCGTTATACTCTATCAAGACCTGACATTGCTATATGATAGTCTTTTCAAGGAAGTCGTAGGATAGTTGTCCTATAACAGACTA", - "description": "Hidden validation template with AT-rich content (~38%), 100 bp." - }, - "amplicon": { - "start_index": 20, - "end_index": 80, - "strand": "forward" - } -} diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_gc_rich_001.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_gc_rich_001.json deleted file mode 100644 index 34726dbc..00000000 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_gc_rich_001.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "name": "hidden_gc_rich_001", - "sequence": "CTTACCACGGCCGTTGTGCGTAGTAACCTAGCGATAAAAGCTTGAACTAGAAGGTGCCCCGGGACTGGTGCACCGTTCAAAAAATTTCGTAAGCGCGCTGTCGGTCTCCGACGGCCTACCACGTGTTCCAGTGATCCCCGAGTGCCGGGG", - "description": "Hidden validation template with GC-rich content (~57%), 150 bp." - }, - "amplicon": { - "start_index": 30, - "end_index": 120, - "strand": "forward" - } -} diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_medium_001.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_medium_001.json deleted file mode 100644 index d5437aca..00000000 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_medium_001.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "name": "hidden_medium_001", - "sequence": "CATTTCTCACGATGGGATAACATCGCCACCCAATTTTAAGCGGATGTCCTTCAGCATACTTACGGCGACTGGTTCTGTCTGTAGTCGTGTCGGTAGTGATACTCCATGGGTGGATTCCGCTTGGGGCAGAGAAACCGAAGTAATAGTTGCCACAAACACTGTTTAGTAAGGCCAAGAATCAACGACATGCCCGATCCTTA", - "description": "Hidden validation template with medium GC content (~48%), 200 bp." - }, - "amplicon": { - "start_index": 40, - "end_index": 160, - "strand": "forward" - } -} diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_001.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_001.json new file mode 100644 index 00000000..cbe7f7a5 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_001.json @@ -0,0 +1,12 @@ +{ + "template": { + "name": "hidden_at_rich_001", + "sequence": "CACATTCCTTCATATTTGGTCCGATTGAATTTAGTGCAGGTCCTCCGAAGCAAGGTGTCGGGTTCGCCTCGGATCTAGATAACCATACCGGTGCTGGAACGGCTTGAACACTCTGAAACTCGTGACTTCTAGAGGTGCGC", + "description": "Hidden validation template with low GC content (~45%)" + }, + "amplicon": { + "start_index": 20, + "end_index": 120, + "strand": "forward" + } +} \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_002.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_002.json new file mode 100644 index 00000000..1c92eee9 --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_002.json @@ -0,0 +1,12 @@ +{ + "template": { + "name": "hidden_gc_balanced_001", + "sequence": "GATTGGGCTTGAGCCCTCTGGCCCGGAATTAGGTCTGTTATGCGTTTGACTTGTCATTAGCCGTATGTCATGTATTTTGGTGAATACGACGGCCCAGCCTGATCTGCTTCGGTAACTCGACTTTGGTGTAATACTCTGCA", + "description": "Hidden validation template with balanced GC content (~50%)" + }, + "amplicon": { + "start_index": 20, + "end_index": 120, + "strand": "forward" + } +} \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_003.json b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_003.json new file mode 100644 index 00000000..38724c9b --- /dev/null +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/references/hidden_templates/hidden_template_003.json @@ -0,0 +1,12 @@ +{ + "template": { + "name": "hidden_gc_rich_001", + "sequence": "TACTCTGTTCGAGTCGACTCCTCGTAGCAAACGTATGCGGTTAAGGCGAGCTTTATAGTCCTGTATTCGCAGATGTCCCTCCGTTCTTCATCAATACCCGCCAAGTAGGGGCTGAGAGGAGATGGTGTATTCTCTCAATG", + "description": "Hidden validation template with high GC content (~55-60%)" + }, + "amplicon": { + "start_index": 20, + "end_index": 120, + "strand": "forward" + } +} \ No newline at end of file diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py index 0df2bbc0..7588c7ea 100644 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/evaluator.py @@ -17,7 +17,10 @@ import argparse import json import math +import shutil +import subprocess import sys +import tempfile from pathlib import Path from typing import Any @@ -1302,6 +1305,84 @@ def compute_score_breakdown( "total_weight": round(total_weight, 4), } + +def _run_candidate_on_config( + candidate_path: Path, + cfg: dict[str, Any], + kernel_python: str, +) -> dict[str, Any]: + """Run candidate script against a specific config and return the submission. + + Creates a temporary directory, copies the candidate script, writes the + config as ``references/primer_config.json``, executes the script, and + returns the parsed submission JSON. + + Parameters + ---------- + candidate_path : Path + Path to the candidate Python script. + cfg : dict + Full config dict (template, constraints, weights, etc.). + kernel_python : str + Python interpreter path. + + Returns + ------- + dict + Submission dict with ``"forward_primer"`` and ``"reverse_primer"`` keys. + + Raises + ------ + RuntimeError + If the candidate fails to produce a valid submission. + + Spec + ---- + \u00a77.3 \u2014 Hidden template validation. + """ + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + + # Copy candidate script to temp directory + script_dest = tmp_path / candidate_path.name + shutil.copy2(str(candidate_path), str(script_dest)) + + # Create references/ with primer_config.json + ref_dir = tmp_path / "references" + ref_dir.mkdir(parents=True, exist_ok=True) + with open(ref_dir / "primer_config.json", "w", encoding="utf-8") as f: + json.dump(cfg, f, indent=2) + + # Run the candidate from the temp directory + result = subprocess.run( + [kernel_python, str(script_dest)], + capture_output=True, + text=True, + timeout=120, + cwd=tmpdir, + ) + + if result.returncode != 0: + raise RuntimeError( + f"Candidate failed on hidden template: {result.stderr.strip()}" + ) + + # Try to parse submission from stdout first, then submission.json + try: + submission = json.loads(result.stdout) + except json.JSONDecodeError: + submission_path = tmp_path / "submission.json" + if submission_path.exists(): + with open(submission_path) as f: + submission = json.load(f) + else: + raise RuntimeError( + "Candidate produced no parseable submission on hidden template" + ) + + return submission + + def evaluate( program_path: str, *, @@ -1353,6 +1434,9 @@ def evaluate( else: program_path_obj = repo_root / program_path_obj + # Determine Python interpreter for running candidate scripts + python = kernel_python or sys.executable + # --- 2. Load configuration --- try: cfg = load_config() @@ -1376,9 +1460,6 @@ def evaluate( submission = json.load(sf) else: # Run candidate script and capture submission.json - import subprocess - import tempfile - python = kernel_python or sys.executable try: with tempfile.TemporaryDirectory() as tmpdir: tmp_path = Path(tmpdir) @@ -1472,7 +1553,51 @@ def evaluate( "total_weight": 0.0, } - # --- 7. Compute score breakdown --- + # --- 7. Evaluate on hidden templates (Spec \u00a77.3) --- + hidden_templates = _load_hidden_templates() + if hidden_templates: + for ht in hidden_templates: + try: + hidden_cfg = _make_hidden_config(ht, cfg) + hidden_submission = _run_candidate_on_config( + program_path_obj, hidden_cfg, python + ) + h_fwd = hidden_submission.get("forward_primer", hidden_submission.get("fwd", "")) + h_rev = hidden_submission.get("reverse_primer", hidden_submission.get("rev", "")) + if not h_fwd or not h_rev: + return { + "final_score": 0.0, + "valid": False, + "failure_reason": f"Hidden template '{ht['template']['name']}': missing primers", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + if not run_hard_gates( + h_fwd, h_rev, hidden_cfg, + hidden_cfg["template"]["sequence"], + hidden_cfg["amplicon"]["start_index"], + hidden_cfg["amplicon"]["end_index"], + ): + return { + "final_score": 0.0, + "valid": False, + "failure_reason": f"Hidden template '{ht['template']['name']}' failed hard gates", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + except RuntimeError as exc: + return { + "final_score": 0.0, + "valid": False, + "failure_reason": f"Hidden template '{ht['template']['name']}' error: {exc}", + "sub_scores": {}, + "weighted_scores": {}, + "total_weight": 0.0, + } + + # --- 8. Compute score breakdown --- breakdown = compute_score_breakdown(fwd, rev, cfg, product_length) return { diff --git a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/test_evaluator.py b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/test_evaluator.py index 79253ba2..5d0c4735 100644 --- a/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/test_evaluator.py +++ b/benchmarks/Bioinformatics/PrimerDesignOptimization/verification/test_evaluator.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3 """Comprehensive test suite for PrimerDesignOptimization evaluator.""" from __future__ import annotations -import json, math, os, sys, unittest +import json, math, os, sys, tempfile, unittest from pathlib import Path BENCHMARK_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(BENCHMARK_ROOT)) @@ -198,6 +198,30 @@ def test_hidden_config(self): hcfg = ev._make_hidden_config(ht, cfg) self.assertEqual(hcfg["template"]["sequence"], ht["template"]["sequence"]) + def test_hidden_template_evaluation(self): + """Baseline passes all hidden templates during evaluate().""" + r = ev.evaluate(str(self.benchmark_root / "scripts" / "init.py")) + self.assertTrue(r["valid"], f"Baseline failed on hidden templates: {r['failure_reason']}") + + def test_hidden_template_failure_rejection(self): + """Candidate failing on hidden templates returns valid=false.""" + # Create a mock candidate that hardcodes primers valid only for the default template. + # These primers will fail alignment on hidden templates with different sequences. + mock_code = "#!/usr/bin/env python3\nimport json\nresult = {\"forward_primer\": \"CAAAGCGATTGTTGGGATTGTACT\", \"reverse_primer\": \"TTAATTCATTAGCCCGACGTTACC\"}\nprint(json.dumps(result))\n" + with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False, encoding="utf-8") as f: + f.write(mock_code) + mock_path = f.name + try: + r = ev.evaluate(mock_path) + self.assertFalse(r["valid"], "Expected valid=false for hidden template failure") + self.assertEqual(r["final_score"], 0.0, "Expected final_score=0.0 for hidden template failure") + # Verify failure reason mentions hidden template + self.assertIn("hidden", r["failure_reason"].lower(), + f"Failure reason should mention hidden template: {r['failure_reason']}") + finally: + os.unlink(mock_path) + + class TestUtilities(unittest.TestCase): def test_rc(self): self.assertEqual(ev.reverse_complement("ATCG"), "CGAT")