diff --git a/TASK_DETAILS.md b/TASK_DETAILS.md index 7475c4cc..5e07bb64 100644 --- a/TASK_DETAILS.md +++ b/TASK_DETAILS.md @@ -342,10 +342,14 @@ We welcome new engineering problem ideas — even without complete verification Importance-sampling BER estimator for Hamming(127,120) - PowerSystems + PowerSystems EV2GymSmartCharging Upstream-aligned EV smart charging scheduling + + TelecomBackup + Time-sequenced backup-power scheduling for telecom sites under a coverage constraint + AdditiveManufacturing DiffSimThermalControl diff --git a/TASK_DETAILS_zh-CN.md b/TASK_DETAILS_zh-CN.md index e2a070a2..5347d43a 100644 --- a/TASK_DETAILS_zh-CN.md +++ b/TASK_DETAILS_zh-CN.md @@ -342,10 +342,14 @@ Frontier-Eng 目前已覆盖以下领域的任务。每个任务均配有可运 使用 importance sampling 估计 Hamming(127,120) 的 BER - PowerSystems + PowerSystems EV2GymSmartCharging 上游对齐的电动车智能充电调度 + + TelecomBackup + 覆盖约束下的通信基站备电时序调度 + AdditiveManufacturing DiffSimThermalControl diff --git a/benchmarks/PowerSystems/README.md b/benchmarks/PowerSystems/README.md index 6a3f9d2e..d0bd6a70 100644 --- a/benchmarks/PowerSystems/README.md +++ b/benchmarks/PowerSystems/README.md @@ -9,3 +9,7 @@ Current tasks emphasize realistic operational constraints, economic objectives, - Unified benchmark: `task=unified task.benchmark=PowerSystems/EV2GymSmartCharging` - Quick run: `python -m frontier_eval task=unified task.benchmark=PowerSystems/EV2GymSmartCharging task.runtime.env_name=frontier-eval-driver algorithm.iterations=0` - Description: upstream-aligned EV smart charging with transformer constraints in the real `EV2Gym` simulator +- `TelecomBackup` + - Unified benchmark: `task=unified task.benchmark=PowerSystems/TelecomBackup` + - Quick run (official scoring uses generated instances and requires a non-public seed; the timeout is raised so the full per-instance budget fits): `TELECOM_EVAL_GENERATE_SEED= python -m frontier_eval task=unified task.benchmark=PowerSystems/TelecomBackup algorithm.iterations=0 algorithm.evaluator.timeout=1200` + - Description: time-sequenced on/off scheduling of telecom backup power supplies to maximize outage backup time while keeping LTE coverage >= 80% diff --git a/benchmarks/PowerSystems/README_zh-CN.md b/benchmarks/PowerSystems/README_zh-CN.md index 30fcfd26..b2600cb0 100644 --- a/benchmarks/PowerSystems/README_zh-CN.md +++ b/benchmarks/PowerSystems/README_zh-CN.md @@ -9,3 +9,7 @@ - `frontier_eval` 任务：`task=unified task.benchmark=PowerSystems/EV2GymSmartCharging` - 快速运行：`python -m frontier_eval task=unified task.benchmark=PowerSystems/EV2GymSmartCharging task.runtime.env_name=frontier-eval-driver algorithm.iterations=0` - 简介：在真实上游 `EV2Gym` 模拟器中进行、与上游数据对齐的 EV 智能充电与变压器约束优化 +- `TelecomBackup` + - `frontier_eval` 任务：`task=unified task.benchmark=PowerSystems/TelecomBackup` + - 快速运行（官方评测用运行时生成实例，必须提供非公开种子；已抬高超时以容纳完整每实例预算）：`TELECOM_EVAL_GENERATE_SEED= python -m frontier_eval task=unified task.benchmark=PowerSystems/TelecomBackup algorithm.iterations=0 algorithm.evaluator.timeout=1200` + - 简介：电信备电电源的时序开关调度，在停电时最大化备电时长同时保持 LTE 覆盖 ≥ 80% diff --git a/benchmarks/PowerSystems/TelecomBackup/.gitignore b/benchmarks/PowerSystems/TelecomBackup/.gitignore new file mode 100644 index 00000000..7a60b85e --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/benchmarks/PowerSystems/TelecomBackup/README.md b/benchmarks/PowerSystems/TelecomBackup/README.md new file mode 100644 index 00000000..cdb703d1 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/README.md @@ -0,0 +1,256 @@ +# TelecomBackup: Power-Backup Scheduling for Telecom Sites (Frontier-Eng Benchmark) + +An **original** Frontier-Engineering benchmark: given a region of telecom sites powered by +batteries, a solver must produce a **time-sequenced on/off schedule for every power supply** that +maximizes the region's total backup time while keeping good LTE coverage (RSRP > -105 dBm) above +80% at every moment. The power-consumption parameters live in each instance and are calibrated so +that stagger/rotation scheduling has clear, reproducible headroom over the naive always-on strategy +(see "Scoring" and "Provenance / design rationale"). + +The full game rules and evaluation semantics are in [Task.md](./Task.md). + +## Layout + +``` +benchmarks/PowerSystems/TelecomBackup/ +├── baseline/solver.py # Candidate solver (EVOLVE-BLOCK region is the only editable part) +├── verification/ +│ ├── generator.py # Fixed-seed instance generator +│ ├── simulator.py # Scoring simulator (coverage/power/battery simulation) +│ ├── evaluate.py # Evaluation entry (local / official modes, scoring) +│ ├── validator.py # Integrity checks (static + env stripping + determinism) +│ ├── ref_solver.py # Reference heuristic (rest-rotation) — documented "good" score +│ ├── test_simulator.py # Unit tests: simulator correctness +│ ├── test_validator.py # Unit tests: integrity checks / env stripping / determinism +│ ├── test_evaluator.py # Unit tests: evaluation behavior, modes, reference path +│ ├── test_frontier_eval_evaluator.py # Unit tests: sandbox path / official mode wiring +│ ├── data/instances/ # 8 fixed instances (seed-fixed, reproducible; local mode) +│ └── requirements.txt +├── frontier_eval/ # UnifiedTask metadata (ConnectFour/AntGame pattern) +├── Task.md # Task rules, interface, scoring, reference scores +└── README.md +``` + +## Requirements + +- Python >= 3.10, standard library only (no third-party dependencies). +- Runtime is pure-Python simulation; each instance evaluation takes well under a second for + the baseline solver. + +## Evaluation modes + +| Mode | Instance pool | Used by | +|---|---|---| +| `local` (default) | the 8 committed fixed instances | development / smoke tests | +| `official` (`TELECOM_EVAL_MODE=official`) | **only** freshly generated instances | the unified (`frontier_eval`) path | + +In **official** mode the committed fixed instances are never scored, so a candidate cannot get +credit by memorising them. A generation seed (`TELECOM_EVAL_GENERATE_SEED`) is **mandatory**: if it +is missing the evaluator raises instead of silently falling back to the public instances. + +## Run + +```powershell +# local (development): the fixed 8 instances, default 60s budget per instance +python verification/evaluate.py baseline/solver.py --local + +# local + runtime-generated instances (anti-hardcoding) +python verification/evaluate.py baseline/solver.py --local --generate-seed + +# official: generated instances only; the seed is mandatory +TELECOM_EVAL_MODE=official TELECOM_EVAL_GENERATE_SEED= \ + python verification/evaluate.py baseline/solver.py + +# tighter budget (challenge tier: 10s) +python verification/evaluate.py baseline/solver.py --local --time-budget 10 +``` + +### Reproducing the reference score + +The bundled reference heuristic is rejected by the candidate integrity checks (it has no +EVOLVE-BLOCK marker and contains a token the validator forbids), so scoring it uses the explicit +`--reference` bypass — it skips the candidate checks and exists **only** for reproducing the +documented reference score: + +```powershell +python verification/evaluate.py verification/ref_solver.py --reference --local +# -> combined_score = 271.25, valid = 1.0 (the documented reference score) +``` + +### Docker + +The evaluator is pure stdlib, so a minimal `python` image suffices. Build it and +use the unified runtime's `isolation_mode=docker`: + +```bash +# Build (inside the TelecomBackup directory) +docker build -t telecombackup-benchmark -f verification/docker/Dockerfile . + +# From the repo root +python -m frontier_eval task=unified task.benchmark=PowerSystems/TelecomBackup algorithm.iterations=0 \ + algorithm.evaluator.timeout=1200 \ + task.runtime.isolation_mode=docker task.runtime.docker_image=telecombackup-benchmark +``` + +> Docker isolation is validated on Linux / WSL. `frontier_eval/eval_command.txt` +> injects the host-benchmark path via the `{benchmark_source}` placeholder, so +> scoring works without framework changes; if the container user cannot write +> the evaluation sandbox, set `task.runtime.docker_user=:` +> (e.g. `1000:1000`). On Windows hosts the unified docker path is blocked by a +> framework path bug (`Path.resolve()` rewrites container paths to drive +> paths) — run docker mode under WSL instead. + +## Tests + +```powershell +# From the TelecomBackup task directory (stdlib unittest, no dependencies) +python -m unittest discover -s verification -p "test_*.py" +``` + +44 tests across four modules (simulator / validator / evaluator / sandbox wiring): simulator +correctness (manual golden cases, interval normalization, battery depletion, coverage constraint, +determinism); validator integrity (EVOLVE-BLOCK / forbidden references / absolute paths / +per-instance hardcoding / env stripping / determinism probe); evaluator behavior (scoring, +malformed/timeout/preflight handling, runtime generation, reproducibility, **official mode requires +a seed and excludes the public instances**, **`--reference` bypass**, **per-instance budget +self-limits to the framework timeout**); and the sandbox wiring (host-loaded generation, official +mode selected by `eval_command.txt`, the reference-solver cheat still rejected). + +## Running inside the Frontier-Eng framework (official path) + +`frontier_eval/eval_command.txt` sets `TELECOM_EVAL_MODE=official`, so the framework path always +scores freshly generated instances and **requires** the seed: + +```powershell +# Windows: point the unified runtime at the venv python (WSL bash can't run Windows exes, +# so also use an MSYS2/Git bash instead of the default `bash`). Set PYTHONUTF8=1 to avoid +# GBK decoding crashes in some framework libs, and raise the LLM timeout for thinking models. +$env:PYTHONUTF8 = "1" +$env:FRONTIER_EVAL_UNIFIED_PYTHON = "\.venvs\frontier-eval-driver\Scripts\python.exe" +$env:TELECOM_EVAL_GENERATE_SEED = "" # mandatory in official mode +python -m frontier_eval task=unified task.benchmark=PowerSystems/TelecomBackup algorithm.iterations=0 ` + algorithm.evaluator.timeout=1200 "task.runtime.shell=" llm.timeout=300 +``` + +**Pick a seed that is not committed to the repository.** The generator lives in this repo, so a +public, fixed seed would let someone with repo access precompute the generated instances and +hardcode their solutions — exactly the hole the official mode closes for the evolving agent. Use a +fresh seed per official run; the seed actually used is recorded in the metrics +(`generate_seed`) so any run can be replayed. + +**Timeout.** The framework caps the whole evaluation with `FRONTIER_EVAL_EVALUATOR_TIMEOUT_S` +(the algorithm side defaults it to 300 s). An official run scores 8 generated instances plus a +cross-size determinism probe (3 instances run twice) = 14 solver runs, so the worst case is +`14 × 60 s = 840 s`, which exceeds 300 s — the official run must raise the cap (e.g. +`algorithm.evaluator.timeout=1200`, as above). If the observed cap is too small the evaluator +**shrinks the per-instance budget so the run still fits** (recording `budget_shrunk` and +`effective_time_budget_s`) rather than being killed mid-run. This bounds the *solver* time the +evaluator controls; fixed overhead (instance generation, interpreter start-up) is not shrinkable, +so with a cap far below that overhead the framework's own deadline remains the backstop. + +## Time Budget Tiers (from the original problem) + +| Tier | Budget | Notes | +|---|---|---| +| Base | 300 s | T+1: 10-min solve | +| Advanced (default) | 60 s | T+2: 1-min solve | +| Challenge | 10 s | T+3: 10-s solve | + +Select a tier with `--time-budget ` (CLI) or `TELECOM_EVAL_TIME_BUDGET=` (the framework +path reads this variable; the default is the 60 s advanced tier). + +## Integrity / threat model + +- **Official vs local scoring**: the official (framework) path scores *generated* instances only, + so memorising the committed instances buys nothing; `local` mode exists for development and is + explicitly not the official score. +- **Runtime-generated instances**: in official mode the evaluator generates fresh instances at + evaluation time (into a temp dir, never written to the repo or the sandbox). A missing generation + seed is a hard error, not a silent fall-back to the public instances. +- **Candidate env stripping**: candidate subprocesses get `FRONTIER_*` / `TELECOM_EVAL_*` + variables stripped (see `verification/validator.py`), closing the host-env side channel. +- **Static checks**: EVOLVE-BLOCK markers + fixed-region byte diff vs the initial baseline, + forbidden imports of evaluation/generation modules, absolute paths, per-instance hardcoding, + plus a determinism probe (two runs must match). Any violation scores 0. `--reference` is the only + way to skip these, and it is for the bundled reference solver, not for candidates. +- Honest note: in process mode the candidate has host filesystem access (framework-wide + limitation); this benchmark relies on the layered defenses above. `verification/simulator.py` + is intentionally exposed as a white-box scorer for candidate-side search. +- **Sandbox scope** (design trade-off): the 8 fixed instances and the evaluator/validator + sources are visible to the candidate during evolution (they are needed for scoring and the + simulator is intentionally usable); they are simply not scored in official mode. The name-keyed + hardcoding check is best-effort (array-index dispatch can evade it, as in any static check). The + reference solver (`ref_solver.py`) and the generator are **not** copied into the sandbox and are + additionally forbidden by the validator. +- **Residual limitation**: this benchmark has no strong process-level isolation (a framework-wide + limitation, see above) and no secret store, so it cannot defend against a repository-reading + attacker who can run the generator offline for a *known* seed. The mitigation is procedural: + the official seed is supplied per run and not committed (see "Running inside the framework"). + +## Scoring + +- Instances: `local` mode = the 8 committed fixed instances (N = 20..40 sites, K = 6..12 power + supplies, each instance carries its power-consumption params `p_silent` / `p_work_base` / + `p_work_coef`); `official` mode = freshly generated instances only. Score = mean backup time + (minutes). +- Malformed output / out-of-range intervals / crash / timeout ⇒ 0 points for that instance. +- **Power calibration**: instances are generated with `p_silent=0.05`, `p_work_base=3.0`, + `p_work_coef=3.0` (silent is cheap, working is expensive), and the generator accepts only + instances where a multi-rest stagger (rest 1..3 supplies at a time) beats "always-on" by ≥ 25% + per instance, so the scheduling problem has large, reproducible headroom. Measured on the 8 + committed instances the acceptance headroom is **+27.9% .. +119.0%, mean +57.5%**. + (This is *not* the same number as the reference solver's **+54%** over the baseline: + 271.25 / 176.25 − 1 = +53.9%. The two are easy to conflate; they measure different things.) +- Reference scores (deepseek-v4-flash agents; agent scores are **verified by directly evaluating + the saved programs** from the task directory — candidate solvers resolve `verification/simulator.py` + relative to their own location, so re-evaluating a saved program from an arbitrary path silently + degrades it to the always-on fallback). + - baseline (always-on, no scheduling): **176.2** minutes + - reference heuristic (`verification/ref_solver.py`, multi-rest rotation): **271.2** minutes (+54%) + - agent (openevolve, 25 iterations, best saved program): **414.4** minutes (+135%; run `20260816_130700`) + - agent (ShinkaEvolve, 15 generations, best generation program): **312.5** minutes (+77%; run `20260816_214014`, gen 3) + - agent (AB-MCTS, 15 iterations, best saved program): **266.9** minutes (+51%; run `20260816_220646`) + - **multi-run statistics** (3 runs per framework, best valid saved program per run): + - openevolve (25 iterations): 414.4 / 298.1 / 357.5 → **mean 356.7 ± 47.5** + - ShinkaEvolve (15 generations): 312.5 / 357.5 / 325.6 → **mean 331.9 ± 18.9** + - AB-MCTS (15 iterations): 266.9 / 208.8 / 227.5 → **mean 234.4 ± 24.2** + - The numbers above are `local`-mode scores (the fixed 8 instances). Under **official** scoring + the same saved programs also generalise to freshly generated instances, e.g. at seed 42 + (fixed 8 + 8 generated): openevolve **410.9**, ShinkaEvolve **319.4**, AB-MCTS **280.3**. + Because official scoring uses a per-run (non-committed) seed, official scores are + seed-dependent; the fixed-instance numbers are kept as the stable, reproducible reference. + - horizon: 480 minutes (upper bound if coverage never fails) + - note: with 5-15 iterations agents mostly plateau at the baseline; more iterations let all + three frameworks discover stagger/coverage-driven schedules that beat it (openevolve even + found a near-horizon minimal-covering-subset schedule, surpassing the reference heuristic). + ShinkaEvolve's 25-generation run produced a higher-scoring but **non-deterministic** program + (time-budgeted simulated annealing) that fails the determinism probe — only deterministic + programs count (312.5 from the 15-generation run is the valid best). + +## Provenance / design rationale + +This is an original benchmark (no external dataset), so the parameter values are documented here +rather than cited. The region models a small urban district and the parameters are +engineering-plausible magnitudes chosen to give a **clear, reproducible** gap between naive and +scheduled operation: + +- **Region 200 m × 200 m, 20 × 20 grid**: a neighbourhood-sized area, small enough that the + pure-Python simulator stays fast, large enough that several sites are needed for coverage. +- **Path loss `P = pt_dbm − 10·n_exp·log10(d + 1)`**, `pt_dbm=20`, `n_exp=6`: a log-distance model; + with these values a site's "good coverage" radius is roughly 120 m, so a 200 m district requires + multiple cooperating sites. +- **RSRP threshold −105 dBm**: LTE RSRP above ≈ −105 dBm is commonly treated as "good" coverage + (cell edge is usually around −110 dBm); we adopt −105 dBm as the "good cell" bar. +- **`coverage_ratio = 0.8`**: 80% of grid cells must be good at every instant. Deliberately below + 100% so the coverage constraint is *redundant* — supplies can take turns resting without + breaching it. This redundancy is what creates the scheduling problem. +- **Power model `p_silent` vs `p_work_base + p_work_coef·load`**: `p_silent=0.05` kW (idling a + supply is cheap) versus `p_work_base=3.0`, `p_work_coef=3.0` kW (working is expensive). This + ratio is the whole point: it makes resting some supplies at any moment worth more than running + everything. It was calibrated empirically — an earlier setting with a high silent drain made + always-on close to optimal and the task had no headroom. +- **Battery 15–30 kWh, `site_cap=60`, `delta_min=5 min`, `horizon=96`** (8 h planning period): + engineering-plausible sizes for a district backup system over an extended outage. + +The generator (`verification/generator.py`) additionally rejects any instance where the best +stagger does not beat always-on by ≥ 25%, so every scored instance provably rewards scheduling. diff --git a/benchmarks/PowerSystems/TelecomBackup/Task.md b/benchmarks/PowerSystems/TelecomBackup/Task.md new file mode 100644 index 00000000..455ee45a --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/Task.md @@ -0,0 +1,167 @@ +# Regional Backup-Power Scheduling (TelecomBackup) + +## Task overview + +When an outage hits a region, you are given each base station's (site's) backup energy, power +draw, and coverage relationships. You must set a **time-sequenced on/off schedule for every power +supply** that **maximizes the region's total backup time** subject to the constraint that the +fraction of well-covered grid cells stays **≥ 80% at every moment**. + +This is a constrained scheduling/optimization problem: the coverage constraint makes the solution +space enormous, so the goal is to produce the best schedule you can within a fixed solve-time +budget. + +## Instance input + +The evaluation passes one instance JSON (its path is a command-line argument) containing: + +| Field | Meaning | +| --- | --- | +| `seed` | The instance's generation seed (identifies the instance). | +| `grid` | Region 200 m × 200 m, rasterised into nx × ny cells (default 20 × 20 = 400). | +| `sites` | Site coordinates (N sites). | +| `groups` | Power-supply grouping: each supply manages a group of sites (K supplies; each group shares one battery). | +| `battery` | Energy per battery (kWh). | +| `demand` | Per-cell demand (relative load unit, 0–1). | +| `pt_dbm / n_exp / threshold` | Coverage parameters: Pt = 20 dBm, path-loss exponent n = 6, threshold −105 dBm. | +| `coverage_ratio` | Coverage-constraint ratio, 0.8. | +| `delta_min` | Slot length, 5 minutes. | +| `horizon` | Total number of slots, 96 (= 8-hour planning period). | +| `site_cap` | Site rated capacity (used to normalise load). | + +## Rules (scoring-simulator semantics) + +- **Coverage**: the level site *s* provides to cell *g* is + `P(g,s) = pt_dbm - 10*n_exp*log10(d(g,s)+1)` (dBm, *d* in metres). A cell attaches to the + **strongest live site** (live = its supply is on and its battery is not depleted); a cell is + "well covered" iff its level is > −105 dBm. +- **Power draw**: a site's working draw is `p_work_base + p_work_coef*min(load_s/site_cap, 1.0)` kW + (load_s is the sum of the demands of the cells it covers, so it is load-dependent; the current + instances use `p_work_base=3.0, p_work_coef=3.0`). A site's silent draw is **`p_silent`** kW + (current instances use `p_silent=0.05`; when a supply is off its sites go silent — they provide + no coverage but still draw silent power). The power parameters come with each instance (see the + instance JSON and the README). +- **Load migration**: turning a supply off → its sites go silent → the cells they covered + **immediately** attach to the strongest live site → that site's load and power draw rise. +- **Energy**: supply *k* drains `Δt × Σ(site draws)` per slot; when its energy ≤ 0 the supply + stops (no draw, no coverage). +- **Backup time**: simulate forward from slot 0; the backup endpoint is the moment just before the + first slot in which the fraction of well-covered cells drops below 80%. If it survives the whole + horizon, backup time = horizon × 5 minutes. + +## Decision output (the format your program must print) + +Run as `python baseline/solver.py ` and print a single JSON object to **stdout**: + +```json +{"on": [ + [[0, 96]], // supply 0: on the whole time + [[0, 40], [60, 96]], // supply 1: on during 0..39 and 60..95 + [] // supply 2: never on +]} +``` + +- `on[k]` is the list of **on-interval**s for supply *k*, each a half-open interval `[a, b)` + (0-indexed, 0 ≤ a ≤ b ≤ horizon). +- Supply *k* is on in slot *s* iff some interval satisfies `a ≤ s < b`. Adjacent/overlapping + intervals are merged, so ordering mistakes are harmless. +- `[[0, horizon]]` = on the whole time (the simplest legal solution); `[]` = never on. +- **Malformed output, out-of-range intervals, crashes, or timeouts → 0 points for that instance** + (the simulator validates every schedule). + +## Evaluation and scoring + +- Instance pool: **official** evaluation scores **only runtime-generated instances** + (anti-hardcoding); a `local` development mode scores the 8 committed fixed instances + (N = 20..40, K = 6..12). +- Score = the **mean** backup time (minutes) over the scored instances. +- Each instance gets a fixed solve-time budget (default **60 s**); a timeout scores 0 for it. +- The budget is configurable: `--time-budget 300` (10 min) / `60` (1 min) / `10` (10 s) — a tighter + budget usually means a lower score. + +Local runs: + +```powershell +python verification/evaluate.py baseline/solver.py --local # fixed 8 instances +python verification/evaluate.py baseline/solver.py --local --time-budget 10 +python verification/evaluate.py baseline/solver.py --local --generate-seed 42 # fixed 8 + 8 generated +``` + +For the framework (unified) evaluation, `frontier_eval/eval_command.txt` selects +`TELECOM_EVAL_MODE=official`, so the score is computed on freshly generated instances and the +generation seed is **mandatory** (missing → explicit error, never a silent fall-back to the public +instances). Supply a **non-public** seed per run and raise the framework timeout so the full +per-instance budget fits: + +```powershell +$env:TELECOM_EVAL_GENERATE_SEED = "" +python -m frontier_eval task=unified task.benchmark=PowerSystems/TelecomBackup algorithm.iterations=0 ` + algorithm.evaluator.timeout=1200 "task.runtime.shell=" +``` + +To reproduce the documented reference score, score the bundled reference solver with the explicit +bypass (the integrity checks reject it: it has no EVOLVE-BLOCK region and it contains a token the +validator forbids): + +```powershell +python verification/evaluate.py verification/ref_solver.py --reference --local # -> 271.25 +``` + +### Integrity / anti-cheating (threat model) + +- **Runtime-generated instances**: in official mode the evaluator generates fresh instances at + evaluation time (they exist only in a temp dir, never in the repo or the sandbox), so a candidate + cannot pre-position solutions for them. Public fixed instances are not scored officially. +- **Candidate env stripping**: the candidate subprocess environment strips `FRONTIER_*` and + `TELECOM_EVAL_*`, closing the side channel that would let it locate the evaluation baseline. +- **Static checks**: before scoring, the candidate is checked for the EVOLVE-BLOCK markers and for + byte-identity of the code outside them (vs the initial baseline), forbidden imports of the + evaluation/generation modules, absolute paths, and per-instance hardcoding; the same instance + must produce identical output across two runs (determinism probe). Any violation scores 0. +- **Honest note**: in process mode the candidate has host filesystem access (a framework + limitation); this benchmark relies on the layered defenses above, and `verification/simulator.py` + is intentionally exposed as a white-box scorer for candidate-side search. + +## Reference scores (measured) + +| Strategy | Mean backup time | +| --- | --- | +| baseline (naive always-on, no scheduling) | **176.2** min | +| agent (AB-MCTS, 15 iterations, best saved program) | **266.9** min (+51%) | +| reference heuristic (`verification/ref_solver.py`, multi-rest rotation) | **271.2** min (+54%) | +| agent (ShinkaEvolve, 15 generations, best generation program) | **312.5** min (+77%) | +| agent (openevolve, 25 iterations, best saved program) | **414.4** min (+135%) | +| horizon (upper bound if coverage never drops below 80%) | 480 min | + +The baseline only runs "always on" (highest coverage but the batteries drain in parallel); +`ref_solver.py`'s multi-rest rotation staggers the discharge and substantially extends backup time; +the three agent frameworks all discover schedules that beat always-on given enough iterations, and +openevolve at 25 iterations finds a near-horizon **minimum-covering-subset rotation** (surpassing +the simple reference heuristic by 53%). Instances are calibrated (silent power is cheap, working +power is expensive) and the generator guarantees a reproducible stagger-over-always-on gain of +≥ 25% per instance (measured **+27.9% .. +119.0%, mean +57.5%** — note this is the generator's +*acceptance* headroom, a different quantity from the reference solver's +54% over the baseline). + +> Note: agent scores are for the **saved programs run directly**, and must be re-evaluated **from +> inside the task directory** — candidate solvers resolve `verification/simulator.py` relative to +> their own location, so running a saved program from an arbitrary path silently degrades it to the +> always-on fallback (an artificially low score). ShinkaEvolve's "saved best" understates its best +> generation (a framework tracking issue); its highest-scoring generation is used instead. +> ShinkaEvolve's 25-generation run once produced a higher score (~409) but as a **time-budgeted +> simulated annealer that is non-deterministic**, which fails the determinism probe, so it does not +> count (the valid best is 312.5 from the 15-generation run). +> At 5–15 iterations agents mostly plateau near the baseline; more iterations let them search more +> thoroughly — this task rewards exhaustive in-simulator search. + +## Optimization hints + +1. The backup endpoint is almost always caused by a supply depleting and coverage dropping below + the bar — the strategy is essentially to **stagger the batteries' discharge**. +2. Coverage is **redundant** (the 80% constraint is far below full-on coverage), so you can rotate + some supplies to rest (saving silent power) as long as coverage stays ≥ 80% at all times. +3. Load migrates when a supply is turned off, raising the remaining sites' draw — so rotation + groups should be **geographically close** (same cluster in one group), otherwise one group may + fail to cover its area. +4. You may `import verification/simulator.py` (read-only) inside your solver to evaluate candidate + schedules' backup time and run a "generate–simulate–improve" search within the budget. +5. Make sure the output is **always legal** first (better fully-on than malformed), then optimize. diff --git a/benchmarks/PowerSystems/TelecomBackup/baseline/result_log.txt b/benchmarks/PowerSystems/TelecomBackup/baseline/result_log.txt new file mode 100644 index 00000000..54f74d19 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/baseline/result_log.txt @@ -0,0 +1,52 @@ +TelecomBackup evaluation log +============================= +All scores are mean backup time (minutes) over the fixed 8-instance set, +verified by directly evaluating the saved programs from the task directory +(candidates resolve verification/simulator.py relative to their own location, +so re-evaluation must run from the task root). Only deterministic programs +(passing the cross-size determinism probe) count. + +Date : 2026-08-17 +Evaluator : python verification/evaluate.py (stdlib only, time budget 60s) + +Solvers +------- +baseline (always-on, no scheduling) : 176.25 (valid 1.0) +reference heuristic (verification/ref_solver.py): 271.2 (multi-rest rotation, + direct simulate) + +Agents (deepseek-v4-flash, run IDs under runs/unified__TelecomBackup/...) +----------------------------------------------------------------------- +openevolve 25 iterations 414.38 run 20260816_130700 (openevolve/best/best_program.py) +ShinkaEvolve 15 generations 312.50 run 20260816_214014 (shinkaevolve/gen_3/main.py) +AB-MCTS 15 iterations 266.88 run 20260816_220646 (abmcts/best/solver.py) + +Notes +----- +- A ShinkaEvolve 25-generation run (20260817_035543) reached ~409 in a single + evaluation, but that program is a time-budgeted simulated-annealing solver + (non-deterministic) and fails the cross-size determinism probe — excluded. +- openevolve's 25-iteration schedule (414.38) is near the 480-min horizon. +- AB-MCTS stays ~250-270 regardless of reasoning mode. + +Note: the agent scores above are on the fixed 8 instances only (no +TELECOM_EVAL_GENERATE_SEED was set for those runs). Set the seed to +additionally score fresh (anti-hardcoding) instances. + +Generalization (fixed 8 + 8 generated, TELECOM_EVAL_GENERATE_SEED=42) +------------------------------------------------------------------- +openevolve best program 410.94 (valid 1.0, 16 instances) +ShinkaEvolve gen_3 319.38 (valid 1.0, 16 instances) +AB-MCTS best solver 280.31 (valid 1.0, 16 instances) +The saved programs generalize to freshly generated instances (not overfit +to the fixed 8); generated instances pass the same _stagger_ok acceptance. + +Multi-run statistics (3 runs per framework; best valid saved program per run) +------------------------------------------------------------------------------ +openevolve 25 iters: 414.38 / 298.12 / 357.50 -> mean 356.67, std 47.47 +ShinkaEvolve 15 gens: 312.50 / 357.50 / 325.62 -> mean 331.87, std 18.90 +AB-MCTS 15 iters: 266.88 / 208.75 / 227.50 -> mean 234.38, std 24.22 +Run IDs (runs/unified__PowerSystems__TelecomBackup/...): + openevolve: 20260816_130700, 20260819_123913, 20260819_135819 + ShinkaEvolve: 20260816_214014 (gen 3), 20260819_203149 (gen 2), 20260819_221722 (gen 10) + AB-MCTS: 20260816_220646, 20260819_151209, 20260819_171751 diff --git a/benchmarks/PowerSystems/TelecomBackup/baseline/solver.py b/benchmarks/PowerSystems/TelecomBackup/baseline/solver.py new file mode 100644 index 00000000..6c47cf8d --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/baseline/solver.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""区域备电 baseline 求解器（朴素基线：全程开启，不做时序调度）。 + +用法：python baseline/solver.py +输出：stdout 打印 {"on": [[[a,b),...], ...]}，on[k] 为电源 k 的开启时隙区间列表 +（0-index，半开区间；[[0, horizon]] = 全程开启，[] = 全程关闭）。 + +只允许修改 EVOLVE-BLOCK 区域内的代码；接口契约（main/stdin-json/stdout-json）必须保留。 + +基线取"全程开启"（朴素、无调度）：覆盖最高但电池并行耗尽。校准后的实例保证 +"错峰调度"（多路轮流休息）能显著超越它（参考启发式 +54%），因此基线是一个 +"正常且可被明显超越"的起点。 +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +def solve(inst: dict) -> list[list[list[int]]]: + """返回 on_intervals 列表（每电源的开启时隙区间）。""" + # EVOLVE-BLOCK-START + # 朴素策略：所有电源全程开启（不调度）。这是合法的基线解—— + # 覆盖最高但电池并行耗尽；通过"时序开关调度"（错峰/轮换供电， + # 让电池错峰放电、利用覆盖冗余）可以显著延长备电时长。 + k = len(inst["groups"]) + horizon = int(inst["horizon"]) + return [[[0, horizon]] for _ in range(k)] + # EVOLVE-BLOCK-END + + +def main() -> int: + if len(sys.argv) < 2: + print("usage: python solver.py ", file=sys.stderr) + return 2 + inst_path = Path(sys.argv[1]) + inst = json.loads(inst_path.read_text(encoding="utf-8")) + on = solve(inst) + print(json.dumps({"on": on})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/agent_files.txt b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/agent_files.txt new file mode 100644 index 00000000..640607a1 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/agent_files.txt @@ -0,0 +1,4 @@ +README.md +Task.md +baseline/solver.py +frontier_eval/constraints.txt diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/artifact_files.txt b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/artifact_files.txt new file mode 100644 index 00000000..a52b8b16 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/artifact_files.txt @@ -0,0 +1,2 @@ +# No extra artifact files are auto-collected by default for this benchmark. +# metrics.json and artifacts.json are handled separately by UnifiedTask. diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/constraints.txt b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/constraints.txt new file mode 100644 index 00000000..dd6a7cc9 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/constraints.txt @@ -0,0 +1,12 @@ +UnifiedTask constraints: +1) Only modify `baseline/solver.py`, and only inside the EVOLVE-BLOCK-START / EVOLVE-BLOCK-END region. +2) Preserve the public contract: run as `python baseline/solver.py `, print a single JSON object + `{"on": [[[a,b),...], ...]}` to stdout -- one list of half-open slot intervals [a, b) per power supply, + each interval within [0, horizon] and a <= b ([[0, horizon]] means "always on", [] means "never on"). +3) Do not modify benchmark assets, documentation, verification code, instance data, or `frontier_eval/` metadata. +4) Your output must always be a valid schedule. Malformed output, out-of-range intervals, crashes, or timeouts + score 0 for that instance (the simulator validates every schedule). +5) You may import `verification/simulator.py` (read-only) to evaluate candidate schedules while searching, + but the schedule you finally print must come from your own algorithm. +6) Prioritize validity before optimization. The solver is given a fixed time budget per instance (default 60s); + keep it fast enough, or your schedule is discarded. diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/copy_files.txt b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/copy_files.txt new file mode 100644 index 00000000..79f8b1a5 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/copy_files.txt @@ -0,0 +1,6 @@ +baseline +verification/evaluate.py +verification/simulator.py +verification/validator.py +verification/data/instances +frontier_eval diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/eval_command.txt b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/eval_command.txt new file mode 100644 index 00000000..b17876ec --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/eval_command.txt @@ -0,0 +1 @@ +TELECOM_EVAL_MODE=official FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR={benchmark_source} {python} frontier_eval/run_eval.py --candidate {candidate} --metrics-out metrics.json --artifacts-out artifacts.json diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/eval_cwd.txt b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/eval_cwd.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/eval_cwd.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/evaluator.py b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/evaluator.py new file mode 100644 index 00000000..924fd606 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/evaluator.py @@ -0,0 +1,53 @@ +"""Unified evaluator entry point for the TelecomBackup benchmark. + +This module is loaded by `frontier_eval/run_eval.py` and must expose a +top-level `evaluate(program_path, **kwargs)` callable. The real +implementation lives in `verification/evaluate.py`. + +The per-instance solve budget is deliberately *not* hard-coded here: it is +resolved from ``TELECOM_EVAL_TIME_BUDGET`` (the 300 / 60 / 10 s budget tiers +documented in the README); the default is the 60 s "advanced" tier. +""" + +from __future__ import annotations + +import importlib.util +import os +from pathlib import Path +from typing import Any + +DEFAULT_TIME_BUDGET_S = 60.0 + + +def _time_budget_s() -> float: + """Per-instance solve budget (s); override with TELECOM_EVAL_TIME_BUDGET.""" + raw = os.environ.get("TELECOM_EVAL_TIME_BUDGET", "").strip() + if not raw: + return DEFAULT_TIME_BUDGET_S + try: + value = float(raw) + except ValueError: + return DEFAULT_TIME_BUDGET_S + return value if value > 0 else DEFAULT_TIME_BUDGET_S + + +def _load_verification_evaluator() -> Any: + evaluator_path = ( + Path(__file__).resolve().parent.parent / "verification" / "evaluate.py" + ) + spec = importlib.util.spec_from_file_location( + "_telecombackup_verification_evaluator", evaluator_path + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"Failed to load verification evaluator from {evaluator_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def evaluate(program_path: str, **kwargs: Any) -> Any: + module = _load_verification_evaluator() + result = module.evaluate(program_path, time_budget=_time_budget_s(), **kwargs) + if isinstance(result, dict) and "metrics" in result: + return result + return {"metrics": result, "artifacts": {}} diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/initial_program.txt b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/initial_program.txt new file mode 100644 index 00000000..6645b02f --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/initial_program.txt @@ -0,0 +1 @@ +baseline/solver.py diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/readonly_files.txt b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/readonly_files.txt new file mode 100644 index 00000000..1205de75 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/readonly_files.txt @@ -0,0 +1,4 @@ +README.md +Task.md +verification +frontier_eval diff --git a/benchmarks/PowerSystems/TelecomBackup/frontier_eval/run_eval.py b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/run_eval.py new file mode 100644 index 00000000..720d0e09 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/frontier_eval/run_eval.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import argparse +import json +import sys +import traceback +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from typing import Any + +INVALID_COMBINED_SCORE = -1e18 + + +def _write_json(path: Path, obj: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(obj, ensure_ascii=False, indent=2, default=str) + "\n", + encoding="utf-8", + ) + + +def _normalize_result(result: Any) -> tuple[dict[str, Any], dict[str, Any]]: + if hasattr(result, "metrics") and hasattr(result, "artifacts"): + return dict(getattr(result, "metrics")), dict(getattr(result, "artifacts")) + + if isinstance(result, dict): + raw_metrics = result.get("metrics") + raw_artifacts = result.get("artifacts") + if isinstance(raw_metrics, dict): + return dict(raw_metrics), dict(raw_artifacts or {}) + return dict(result), {} + + raise TypeError( + "Evaluator must return an EvaluationResult-like object or a dict of metrics." + ) + + +def _load_local_evaluator() -> Any: + evaluator_path = Path(__file__).with_name("evaluator.py").resolve() + spec = spec_from_file_location("_frontier_eval_local_evaluator", evaluator_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Failed to load local evaluator from {evaluator_path}") + module = module_from_spec(spec) + spec.loader.exec_module(module) + try: + return getattr(module, "evaluate") + except AttributeError as exc: + raise RuntimeError( + f"Local evaluator does not define evaluate(): {evaluator_path}" + ) from exc + + +def _find_repo_root() -> Path: + import os + + env_root = os.environ.get("FRONTIER_ENGINEERING_ROOT") + if env_root: + return Path(env_root).expanduser().resolve() + + here = Path(__file__).resolve() + for parent in [here.parent, *here.parents]: + if (parent / "frontier_eval").is_dir() and (parent / "benchmarks").is_dir(): + return parent + return Path.cwd().resolve() + + +def _build_kwargs(evaluate_fn: Any) -> dict[str, Any]: + kwargs: dict[str, Any] = {} + try: + parameters = inspect_signature(evaluate_fn) + except Exception: + return kwargs + + if "repo_root" in parameters: + kwargs["repo_root"] = _find_repo_root() + return kwargs + + +def inspect_signature(fn: Any) -> set[str]: + import inspect + + return set(inspect.signature(fn).parameters) + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run a benchmark-local unified evaluator and export metrics/artifacts JSON." + ) + parser.add_argument("--candidate", required=True) + parser.add_argument("--metrics-out", default="metrics.json") + parser.add_argument("--artifacts-out", default="artifacts.json") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = _parse_args(argv) + + candidate_path = Path(args.candidate).expanduser().resolve() + metrics_out = Path(args.metrics_out).expanduser().resolve() + artifacts_out = Path(args.artifacts_out).expanduser().resolve() + + metrics: dict[str, Any] = { + "combined_score": INVALID_COMBINED_SCORE, + "valid": 0.0, + } + artifacts: dict[str, Any] = { + "local_evaluator_path": str(Path(__file__).with_name("evaluator.py").resolve()), + "candidate_path": str(candidate_path), + } + + try: + evaluate_fn = _load_local_evaluator() + result = evaluate_fn(str(candidate_path), **_build_kwargs(evaluate_fn)) + metrics, evaluator_artifacts = _normalize_result(result) + artifacts.update(evaluator_artifacts) + except Exception as exc: + artifacts["error_message"] = str(exc) + artifacts["traceback"] = traceback.format_exc() + + _write_json(metrics_out, metrics) + _write_json(artifacts_out, artifacts) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_1.json b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_1.json new file mode 100644 index 00000000..7aa0a680 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_1.json @@ -0,0 +1,545 @@ +{ + "seed": 1, + "grid": { + "width": 200.0, + "height": 200.0, + "nx": 20, + "ny": 20 + }, + "sites": [ + [ + 128.2, + 50.6 + ], + [ + 152.1, + 141.4 + ], + [ + 165.6, + 37.7 + ], + [ + 144.7, + 35.2 + ], + [ + 42.2, + 145.0 + ], + [ + 54.5, + 50.9 + ], + [ + 168.2, + 42.7 + ], + [ + 81.6, + 143.8 + ], + [ + 54.1, + 102.8 + ], + [ + 33.0, + 174.8 + ], + [ + 193.7, + 136.5 + ], + [ + 163.3, + 128.3 + ], + [ + 174.5, + 107.6 + ], + [ + 63.0, + 62.4 + ], + [ + 106.1, + 147.9 + ], + [ + 41.2, + 175.1 + ], + [ + 163.5, + 130.5 + ], + [ + 117.3, + 54.1 + ], + [ + 139.4, + 156.3 + ], + [ + 75.3, + 65.3 + ] + ], + "groups": [ + [ + 0, + 2, + 3, + 6, + 17 + ], + [ + 5, + 8, + 13, + 19 + ], + [ + 1, + 14, + 18 + ], + [ + 12 + ], + [ + 10, + 11, + 16 + ], + [ + 4, + 7, + 9, + 15 + ] + ], + "battery": [ + 26.0, + 22.0, + 19.6, + 27.7, + 24.2, + 23.7 + ], + "demand": [ + 0.647, + 0.169, + 0.227, + 0.012, + 0.2, + 0.92, + 0.548, + 0.404, + 0.344, + 0.847, + 0.353, + 0.91, + 0.659, + 0.609, + 0.729, + 0.384, + 0.857, + 0.955, + 0.938, + 0.512, + 0.129, + 0.777, + 0.205, + 0.95, + 0.481, + 0.365, + 0.554, + 0.941, + 0.413, + 0.813, + 0.414, + 0.002, + 0.54, + 0.786, + 0.331, + 0.6, + 0.805, + 0.635, + 0.551, + 0.181, + 0.092, + 0.551, + 0.851, + 0.931, + 0.032, + 0.944, + 0.07, + 0.868, + 0.453, + 0.754, + 0.281, + 0.269, + 0.797, + 0.185, + 0.29, + 0.167, + 0.255, + 0.952, + 0.657, + 0.648, + 0.294, + 0.703, + 0.496, + 0.114, + 0.312, + 0.343, + 0.796, + 0.258, + 0.253, + 0.73, + 0.977, + 0.966, + 0.432, + 0.976, + 0.225, + 0.397, + 0.035, + 0.96, + 0.446, + 0.506, + 0.427, + 0.832, + 0.977, + 0.631, + 0.695, + 0.451, + 0.524, + 0.031, + 0.675, + 0.803, + 0.66, + 0.426, + 0.737, + 0.126, + 0.212, + 0.047, + 0.071, + 0.076, + 0.917, + 0.298, + 0.158, + 0.565, + 0.13, + 0.561, + 0.851, + 0.591, + 0.218, + 0.901, + 0.461, + 0.828, + 0.87, + 0.78, + 0.623, + 0.037, + 0.2, + 0.099, + 0.573, + 0.897, + 0.591, + 0.492, + 0.938, + 0.39, + 0.504, + 0.017, + 0.612, + 0.402, + 0.281, + 0.157, + 0.858, + 0.811, + 0.563, + 0.135, + 0.429, + 0.267, + 0.096, + 0.379, + 0.548, + 0.914, + 0.838, + 0.534, + 0.768, + 0.533, + 0.065, + 0.04, + 0.133, + 0.167, + 0.538, + 0.268, + 0.332, + 0.506, + 0.255, + 0.339, + 0.114, + 0.235, + 0.944, + 0.78, + 0.715, + 0.489, + 0.58, + 0.77, + 0.321, + 0.407, + 0.38, + 0.991, + 0.147, + 0.125, + 0.115, + 0.587, + 0.926, + 0.077, + 0.55, + 0.566, + 0.952, + 0.365, + 0.296, + 0.534, + 0.114, + 0.897, + 0.108, + 0.046, + 0.296, + 0.614, + 0.015, + 0.414, + 0.826, + 0.79, + 0.188, + 0.786, + 0.587, + 0.162, + 0.451, + 0.681, + 0.159, + 0.845, + 0.435, + 0.965, + 0.806, + 0.543, + 0.818, + 0.55, + 0.712, + 0.314, + 0.208, + 0.317, + 0.027, + 0.787, + 0.926, + 0.726, + 0.32, + 0.391, + 0.399, + 0.064, + 0.317, + 0.601, + 0.456, + 0.25, + 0.785, + 0.778, + 0.891, + 0.868, + 0.469, + 0.356, + 0.183, + 0.208, + 0.199, + 0.36, + 0.82, + 0.089, + 0.753, + 0.09, + 0.574, + 0.339, + 0.227, + 0.967, + 0.041, + 0.187, + 0.793, + 0.579, + 0.921, + 0.246, + 0.101, + 0.611, + 0.808, + 0.092, + 0.22, + 0.808, + 0.402, + 0.268, + 0.868, + 0.729, + 0.022, + 0.01, + 0.751, + 0.359, + 0.469, + 0.859, + 0.101, + 0.778, + 0.328, + 0.509, + 0.665, + 0.18, + 0.15, + 0.142, + 0.866, + 0.306, + 0.709, + 0.835, + 0.602, + 0.126, + 0.207, + 0.545, + 0.723, + 0.78, + 0.821, + 0.624, + 0.672, + 0.553, + 0.943, + 0.987, + 0.205, + 0.299, + 0.537, + 0.049, + 0.862, + 0.247, + 0.778, + 0.682, + 0.447, + 0.43, + 0.25, + 0.439, + 0.538, + 0.011, + 0.836, + 0.172, + 0.486, + 0.793, + 0.933, + 0.976, + 0.019, + 0.692, + 0.58, + 0.594, + 0.138, + 0.983, + 0.277, + 0.564, + 0.172, + 0.089, + 0.486, + 0.178, + 0.317, + 0.893, + 0.92, + 0.93, + 0.639, + 0.226, + 0.313, + 0.687, + 0.957, + 0.713, + 0.337, + 0.611, + 0.728, + 0.653, + 0.972, + 0.219, + 0.922, + 0.763, + 0.645, + 0.369, + 0.512, + 0.792, + 0.204, + 0.299, + 0.3, + 0.552, + 0.165, + 0.701, + 0.465, + 0.085, + 0.123, + 0.606, + 0.514, + 0.377, + 0.156, + 0.427, + 0.942, + 0.72, + 0.782, + 0.495, + 0.394, + 0.637, + 0.384, + 0.845, + 0.544, + 0.994, + 0.524, + 0.09, + 0.255, + 0.101, + 0.737, + 0.084, + 0.975, + 0.969, + 0.617, + 0.967, + 0.687, + 0.082, + 0.851, + 0.241, + 0.851, + 0.94, + 0.903, + 0.397, + 0.91, + 0.438, + 0.622, + 0.488, + 0.212, + 0.431, + 0.534, + 0.909, + 0.661, + 0.278, + 0.379, + 0.559, + 0.96, + 0.528, + 0.579, + 0.031, + 0.973, + 0.242, + 0.26, + 0.173, + 0.148, + 0.2, + 0.311, + 0.757 + ], + "pt_dbm": 20.0, + "n_exp": 6.0, + "threshold": -105.0, + "coverage_ratio": 0.8, + "delta_min": 5.0, + "horizon": 96, + "site_cap": 60.0, + "p_silent": 0.05, + "p_work_base": 3.0, + "p_work_coef": 3.0 +} diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_2.json b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_2.json new file mode 100644 index 00000000..9723b2dc --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_2.json @@ -0,0 +1,568 @@ +{ + "seed": 2, + "grid": { + "width": 200.0, + "height": 200.0, + "nx": 20, + "ny": 20 + }, + "sites": [ + [ + 147.4, + 120.3 + ], + [ + 49.9, + 163.8 + ], + [ + 39.8, + 44.3 + ], + [ + 126.7, + 22.4 + ], + [ + 29.1, + 144.1 + ], + [ + 82.9, + 126.5 + ], + [ + 32.9, + 153.8 + ], + [ + 113.2, + 69.2 + ], + [ + 136.4, + 144.6 + ], + [ + 55.5, + 61.1 + ], + [ + 80.0, + 149.8 + ], + [ + 171.7, + 149.3 + ], + [ + 145.7, + 6.7 + ], + [ + 54.6, + 131.9 + ], + [ + 167.7, + 78.4 + ], + [ + 141.5, + 37.5 + ], + [ + 71.0, + 139.3 + ], + [ + 38.4, + 131.3 + ], + [ + 131.0, + 75.9 + ], + [ + 135.9, + 51.2 + ], + [ + 65.0, + 115.1 + ], + [ + 53.3, + 127.4 + ], + [ + 80.2, + 141.7 + ], + [ + 144.8, + 62.6 + ] + ], + "groups": [ + [ + 1, + 4, + 6 + ], + [ + 11 + ], + [ + 13, + 17, + 20, + 21 + ], + [ + 3, + 7, + 12, + 14, + 15, + 18, + 19, + 23 + ], + [ + 2, + 9 + ], + [ + 5, + 10, + 16, + 22 + ], + [ + 0, + 8 + ] + ], + "battery": [ + 28.6, + 17.9, + 26.2, + 15.9, + 24.8, + 19.1, + 18.4 + ], + "demand": [ + 0.875, + 0.106, + 0.522, + 0.854, + 0.245, + 0.21, + 0.881, + 0.423, + 0.717, + 0.032, + 0.362, + 0.172, + 0.673, + 0.083, + 0.955, + 0.025, + 0.729, + 0.021, + 0.256, + 0.813, + 0.157, + 0.184, + 0.691, + 0.386, + 0.043, + 0.99, + 0.151, + 0.036, + 0.344, + 0.615, + 0.742, + 0.113, + 0.337, + 0.031, + 0.449, + 0.766, + 0.74, + 0.902, + 0.756, + 0.862, + 0.705, + 0.473, + 0.226, + 0.661, + 0.316, + 0.102, + 0.448, + 0.875, + 0.128, + 0.585, + 0.393, + 0.515, + 0.144, + 0.96, + 0.259, + 0.606, + 0.42, + 0.018, + 0.558, + 0.141, + 0.057, + 0.034, + 0.161, + 0.096, + 0.635, + 0.508, + 0.983, + 0.934, + 0.995, + 0.232, + 0.445, + 0.251, + 0.591, + 0.624, + 0.8, + 0.709, + 0.257, + 0.423, + 0.526, + 0.005, + 0.035, + 0.409, + 0.111, + 0.724, + 0.241, + 0.1, + 0.182, + 0.232, + 0.217, + 0.521, + 0.464, + 0.31, + 0.642, + 0.212, + 0.907, + 0.963, + 0.729, + 0.434, + 0.512, + 0.581, + 0.051, + 0.418, + 0.525, + 0.181, + 0.094, + 0.803, + 0.366, + 0.519, + 0.921, + 0.611, + 0.29, + 0.984, + 0.372, + 0.019, + 0.685, + 0.101, + 0.306, + 0.841, + 0.673, + 0.016, + 0.451, + 0.411, + 0.486, + 0.208, + 0.589, + 0.074, + 0.284, + 0.373, + 0.935, + 0.077, + 0.755, + 0.192, + 0.572, + 0.392, + 0.463, + 0.754, + 0.395, + 0.122, + 0.122, + 0.081, + 0.85, + 0.641, + 0.96, + 0.693, + 0.025, + 0.659, + 0.777, + 0.724, + 0.498, + 0.358, + 0.457, + 0.799, + 0.269, + 0.526, + 0.478, + 0.955, + 0.804, + 0.932, + 0.836, + 0.297, + 0.232, + 0.489, + 0.259, + 0.428, + 0.679, + 0.919, + 0.586, + 0.818, + 0.096, + 0.356, + 0.998, + 0.147, + 0.417, + 0.067, + 0.086, + 0.896, + 0.989, + 0.648, + 0.129, + 0.296, + 0.232, + 0.671, + 0.681, + 0.439, + 0.524, + 0.112, + 0.541, + 0.95, + 0.756, + 0.096, + 0.517, + 0.715, + 0.257, + 0.895, + 0.461, + 0.703, + 0.404, + 0.995, + 0.783, + 0.573, + 0.145, + 0.441, + 0.029, + 0.595, + 0.882, + 0.18, + 0.51, + 0.482, + 0.405, + 0.71, + 0.937, + 0.705, + 0.472, + 0.962, + 0.331, + 0.746, + 0.658, + 0.762, + 0.852, + 0.225, + 0.621, + 0.403, + 0.667, + 0.977, + 0.635, + 0.012, + 0.465, + 0.712, + 0.883, + 0.65, + 0.816, + 0.017, + 0.943, + 0.729, + 0.606, + 0.905, + 0.885, + 0.1, + 0.816, + 0.767, + 0.2, + 0.744, + 0.586, + 0.191, + 0.804, + 0.138, + 0.612, + 0.434, + 0.254, + 0.566, + 0.467, + 0.205, + 0.967, + 0.073, + 0.003, + 0.485, + 0.837, + 0.658, + 0.755, + 0.485, + 0.675, + 0.335, + 0.267, + 0.503, + 0.028, + 0.08, + 0.754, + 0.174, + 0.75, + 0.784, + 0.404, + 0.675, + 0.787, + 0.864, + 0.135, + 0.163, + 0.382, + 0.465, + 0.295, + 0.01, + 0.557, + 0.967, + 0.366, + 0.538, + 0.382, + 0.443, + 0.87, + 0.308, + 0.649, + 0.484, + 0.539, + 0.915, + 0.077, + 0.824, + 0.304, + 0.646, + 0.796, + 0.653, + 0.393, + 0.841, + 0.093, + 0.633, + 0.391, + 0.53, + 0.851, + 0.798, + 0.629, + 0.308, + 0.233, + 0.458, + 0.232, + 0.277, + 0.958, + 0.112, + 0.819, + 0.379, + 0.365, + 0.318, + 0.077, + 0.457, + 0.166, + 0.442, + 0.292, + 0.895, + 0.922, + 0.442, + 0.64, + 0.93, + 0.326, + 0.1, + 0.238, + 0.19, + 0.678, + 0.374, + 0.356, + 0.795, + 0.233, + 0.809, + 0.633, + 0.4, + 0.824, + 0.342, + 0.879, + 0.926, + 0.503, + 0.69, + 0.949, + 0.743, + 0.751, + 0.869, + 0.936, + 0.754, + 0.979, + 0.292, + 0.622, + 0.671, + 0.367, + 0.395, + 0.175, + 0.958, + 0.354, + 0.477, + 0.894, + 0.186, + 0.961, + 0.127, + 0.028, + 0.351, + 0.359, + 0.918, + 0.883, + 0.762, + 0.436, + 0.543, + 0.237, + 0.834, + 0.39, + 0.285, + 0.638, + 0.151, + 0.316, + 0.926, + 0.095, + 0.142, + 0.204, + 0.251, + 0.42, + 0.25, + 0.343, + 0.246, + 0.24, + 0.611, + 0.336, + 0.373, + 0.768, + 0.062, + 0.144, + 0.851, + 0.43, + 0.779 + ], + "pt_dbm": 20.0, + "n_exp": 6.0, + "threshold": -105.0, + "coverage_ratio": 0.8, + "delta_min": 5.0, + "horizon": 96, + "site_cap": 60.0, + "p_silent": 0.05, + "p_work_base": 3.0, + "p_work_coef": 3.0 +} diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_3.json b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_3.json new file mode 100644 index 00000000..58b056bc --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_3.json @@ -0,0 +1,591 @@ +{ + "seed": 104, + "grid": { + "width": 200.0, + "height": 200.0, + "nx": 20, + "ny": 20 + }, + "sites": [ + [ + 60.3, + 150.4 + ], + [ + 58.0, + 45.1 + ], + [ + 171.5, + 169.7 + ], + [ + 45.2, + 155.2 + ], + [ + 132.6, + 140.2 + ], + [ + 61.6, + 146.0 + ], + [ + 31.1, + 162.1 + ], + [ + 149.3, + 162.6 + ], + [ + 71.8, + 65.9 + ], + [ + 57.2, + 36.5 + ], + [ + 145.8, + 56.5 + ], + [ + 153.5, + 141.8 + ], + [ + 158.6, + 26.3 + ], + [ + 195.0, + 115.5 + ], + [ + 158.1, + 140.2 + ], + [ + 171.8, + 54.4 + ], + [ + 144.1, + 144.1 + ], + [ + 159.5, + 154.6 + ], + [ + 166.1, + 165.8 + ], + [ + 132.8, + 31.6 + ], + [ + 146.7, + 133.1 + ], + [ + 76.4, + 17.3 + ], + [ + 125.9, + 47.1 + ], + [ + 45.8, + 47.7 + ], + [ + 57.2, + 71.9 + ], + [ + 159.3, + 59.2 + ], + [ + 119.1, + 130.3 + ], + [ + 78.7, + 171.5 + ] + ], + "groups": [ + [ + 1, + 8, + 9, + 23, + 24 + ], + [ + 4, + 11, + 13, + 14, + 16, + 20, + 26 + ], + [ + 2, + 7, + 17, + 18 + ], + [ + 19, + 21, + 22 + ], + [ + 0, + 5 + ], + [ + 10, + 12, + 15, + 25 + ], + [ + 3, + 6 + ], + [ + 27 + ] + ], + "battery": [ + 20.8, + 23.1, + 18.6, + 17.8, + 27.8, + 27.9, + 23.9, + 25.3 + ], + "demand": [ + 0.734, + 0.625, + 0.386, + 0.706, + 0.509, + 0.914, + 0.48, + 0.709, + 0.008, + 0.867, + 0.29, + 0.087, + 0.226, + 0.071, + 0.032, + 0.056, + 0.78, + 0.927, + 0.613, + 0.915, + 0.442, + 0.072, + 0.513, + 0.504, + 0.04, + 0.578, + 0.157, + 0.981, + 0.175, + 0.27, + 0.886, + 0.262, + 0.528, + 0.454, + 0.695, + 0.164, + 0.487, + 0.124, + 0.729, + 0.061, + 0.309, + 0.064, + 0.334, + 0.244, + 0.542, + 0.108, + 0.241, + 0.694, + 0.86, + 0.273, + 0.755, + 0.804, + 0.472, + 0.88, + 0.715, + 0.5, + 0.821, + 0.207, + 0.733, + 0.151, + 0.372, + 0.106, + 0.322, + 0.66, + 0.422, + 0.583, + 0.891, + 0.813, + 0.954, + 0.87, + 0.1, + 0.522, + 0.362, + 0.822, + 0.143, + 0.735, + 0.806, + 0.331, + 0.216, + 0.719, + 0.866, + 0.678, + 0.463, + 0.21, + 0.555, + 0.906, + 0.172, + 0.93, + 0.295, + 0.764, + 0.616, + 0.146, + 0.124, + 0.543, + 0.09, + 0.371, + 0.245, + 0.65, + 0.82, + 0.162, + 0.201, + 0.811, + 0.971, + 0.907, + 0.323, + 0.852, + 0.262, + 0.685, + 0.322, + 0.702, + 0.639, + 0.771, + 0.618, + 0.592, + 0.777, + 0.398, + 0.672, + 0.503, + 0.674, + 0.165, + 0.106, + 0.33, + 0.258, + 0.682, + 0.995, + 0.713, + 0.621, + 0.305, + 0.49, + 0.554, + 0.194, + 0.275, + 0.118, + 0.875, + 0.879, + 0.037, + 0.115, + 0.081, + 0.078, + 0.682, + 0.599, + 0.586, + 0.767, + 0.327, + 0.574, + 0.878, + 0.36, + 0.121, + 0.259, + 0.809, + 0.084, + 0.121, + 0.23, + 0.752, + 0.506, + 0.529, + 0.098, + 0.943, + 0.624, + 0.076, + 0.501, + 0.825, + 0.697, + 0.904, + 0.218, + 0.607, + 0.216, + 0.977, + 0.395, + 0.652, + 0.314, + 0.652, + 0.795, + 0.689, + 0.118, + 0.959, + 0.105, + 0.965, + 0.588, + 0.179, + 0.879, + 0.978, + 0.041, + 0.91, + 0.547, + 0.969, + 0.784, + 0.565, + 0.823, + 0.553, + 0.136, + 0.845, + 0.688, + 0.041, + 0.4, + 0.722, + 0.152, + 0.008, + 0.873, + 0.346, + 0.145, + 0.726, + 0.578, + 0.849, + 0.235, + 0.214, + 0.536, + 0.497, + 0.6, + 0.854, + 0.655, + 0.52, + 0.197, + 0.572, + 0.445, + 0.679, + 0.085, + 0.483, + 0.041, + 0.402, + 0.666, + 0.821, + 0.575, + 0.283, + 0.824, + 0.359, + 0.021, + 0.502, + 0.128, + 0.335, + 0.433, + 0.861, + 0.335, + 0.074, + 0.688, + 0.36, + 0.575, + 0.606, + 0.546, + 0.472, + 0.487, + 0.796, + 0.352, + 0.462, + 0.04, + 0.789, + 0.138, + 0.777, + 0.327, + 0.205, + 0.814, + 0.607, + 0.387, + 0.424, + 0.015, + 0.244, + 0.936, + 0.763, + 0.626, + 0.922, + 0.485, + 0.937, + 0.481, + 0.56, + 0.036, + 0.773, + 0.773, + 0.303, + 0.724, + 0.621, + 0.257, + 0.452, + 0.869, + 0.923, + 0.063, + 0.223, + 0.395, + 0.779, + 0.068, + 0.502, + 0.204, + 0.68, + 0.635, + 0.069, + 0.899, + 0.694, + 0.906, + 0.479, + 0.459, + 0.815, + 0.845, + 0.037, + 0.664, + 0.805, + 0.168, + 0.75, + 0.618, + 0.265, + 0.911, + 0.983, + 0.562, + 0.67, + 0.075, + 0.565, + 0.245, + 0.27, + 0.83, + 0.488, + 0.011, + 0.472, + 0.117, + 0.155, + 0.778, + 0.178, + 0.644, + 0.5, + 0.59, + 0.486, + 0.669, + 0.122, + 0.476, + 0.109, + 0.717, + 0.203, + 0.679, + 0.482, + 0.241, + 0.368, + 0.101, + 0.384, + 0.337, + 0.265, + 0.981, + 0.813, + 0.356, + 0.076, + 0.177, + 0.795, + 0.967, + 0.136, + 0.147, + 0.809, + 0.558, + 0.477, + 0.345, + 0.989, + 0.902, + 0.676, + 0.51, + 0.52, + 0.157, + 0.923, + 0.963, + 0.482, + 0.277, + 0.274, + 0.873, + 0.645, + 0.353, + 0.011, + 0.719, + 0.481, + 0.525, + 0.943, + 0.558, + 0.059, + 0.151, + 0.149, + 0.531, + 0.265, + 0.449, + 0.017, + 0.96, + 0.914, + 0.103, + 0.057, + 0.505, + 0.894, + 0.605, + 0.501, + 0.169, + 0.211, + 0.611, + 0.9, + 0.815, + 0.767, + 0.957, + 0.455, + 0.888, + 0.731, + 0.382, + 0.553, + 0.878, + 0.131, + 0.789, + 0.43, + 0.477, + 0.718, + 0.815, + 0.571 + ], + "pt_dbm": 20.0, + "n_exp": 6.0, + "threshold": -105.0, + "coverage_ratio": 0.8, + "delta_min": 5.0, + "horizon": 96, + "site_cap": 60.0, + "p_silent": 0.05, + "p_work_base": 3.0, + "p_work_coef": 3.0 +} diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_4.json b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_4.json new file mode 100644 index 00000000..ed831ec5 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_4.json @@ -0,0 +1,617 @@ +{ + "seed": 4, + "grid": { + "width": 200.0, + "height": 200.0, + "nx": 20, + "ny": 20 + }, + "sites": [ + [ + 54.2, + 156.1 + ], + [ + 34.3, + 145.0 + ], + [ + 154.4, + 74.4 + ], + [ + 57.0, + 13.2 + ], + [ + 60.2, + 94.7 + ], + [ + 54.1, + 37.5 + ], + [ + 40.1, + 89.0 + ], + [ + 73.3, + 64.1 + ], + [ + 37.5, + 149.5 + ], + [ + 41.0, + 195.0 + ], + [ + 66.6, + 50.8 + ], + [ + 63.8, + 30.5 + ], + [ + 35.4, + 157.6 + ], + [ + 59.0, + 164.5 + ], + [ + 169.5, + 60.9 + ], + [ + 21.3, + 46.6 + ], + [ + 68.3, + 73.7 + ], + [ + 138.5, + 57.0 + ], + [ + 46.9, + 168.5 + ], + [ + 41.2, + 40.0 + ], + [ + 55.7, + 54.2 + ], + [ + 145.5, + 176.0 + ], + [ + 183.6, + 182.4 + ], + [ + 44.3, + 51.5 + ], + [ + 189.5, + 19.7 + ], + [ + 66.4, + 51.9 + ], + [ + 153.2, + 129.7 + ], + [ + 146.5, + 62.6 + ], + [ + 46.0, + 164.6 + ], + [ + 148.7, + 152.4 + ], + [ + 131.7, + 55.3 + ], + [ + 76.0, + 49.5 + ] + ], + "groups": [ + [ + 9 + ], + [ + 4, + 6 + ], + [ + 0, + 13, + 18, + 28 + ], + [ + 2, + 14, + 17, + 24, + 27, + 30 + ], + [ + 3 + ], + [ + 1, + 8, + 12 + ], + [ + 21, + 22, + 26, + 29 + ], + [ + 7, + 10, + 16, + 20, + 25, + 31 + ], + [ + 15, + 19, + 23 + ], + [ + 5, + 11 + ] + ], + "battery": [ + 23.9, + 29.6, + 24.1, + 23.2, + 23.7, + 19.8, + 15.5, + 27.5, + 19.4, + 30.0 + ], + "demand": [ + 0.652, + 0.2, + 0.202, + 0.439, + 0.492, + 0.219, + 0.444, + 0.647, + 0.428, + 0.216, + 0.188, + 0.037, + 0.253, + 0.526, + 0.772, + 0.417, + 0.262, + 0.325, + 0.891, + 0.315, + 0.117, + 0.57, + 0.963, + 0.653, + 0.871, + 0.717, + 0.041, + 0.387, + 0.43, + 0.93, + 0.572, + 0.944, + 0.165, + 0.296, + 0.471, + 0.901, + 0.315, + 0.42, + 0.215, + 0.801, + 0.801, + 0.339, + 0.392, + 0.496, + 0.859, + 0.28, + 0.628, + 0.191, + 0.395, + 0.62, + 0.767, + 0.269, + 0.856, + 0.87, + 0.688, + 0.465, + 0.472, + 0.404, + 0.39, + 0.798, + 0.211, + 0.157, + 0.61, + 0.257, + 0.396, + 0.77, + 0.381, + 0.222, + 0.053, + 0.951, + 0.162, + 0.608, + 0.818, + 0.562, + 0.784, + 0.915, + 0.44, + 0.079, + 0.695, + 0.113, + 0.561, + 0.257, + 0.774, + 0.041, + 0.08, + 0.879, + 0.905, + 0.297, + 0.347, + 0.075, + 0.96, + 0.454, + 0.205, + 0.977, + 0.389, + 0.759, + 0.855, + 0.4, + 0.077, + 0.916, + 0.798, + 0.366, + 1.0, + 0.416, + 0.711, + 0.444, + 0.629, + 0.941, + 0.637, + 0.93, + 0.479, + 0.118, + 0.558, + 0.372, + 0.163, + 0.707, + 0.327, + 0.86, + 0.259, + 0.974, + 0.005, + 0.169, + 0.648, + 0.121, + 0.111, + 0.786, + 0.601, + 0.526, + 0.521, + 0.411, + 0.293, + 0.229, + 0.18, + 0.864, + 0.002, + 0.053, + 0.908, + 0.544, + 0.909, + 0.782, + 0.568, + 0.937, + 0.507, + 0.441, + 0.617, + 0.392, + 0.252, + 0.597, + 0.986, + 0.34, + 0.433, + 0.606, + 0.675, + 0.622, + 0.285, + 0.94, + 0.197, + 0.794, + 0.675, + 0.883, + 0.077, + 0.646, + 0.904, + 0.33, + 0.372, + 0.175, + 0.883, + 0.604, + 0.524, + 0.755, + 0.089, + 0.359, + 0.813, + 0.098, + 0.182, + 0.987, + 0.661, + 0.993, + 0.077, + 0.926, + 0.986, + 0.172, + 0.479, + 0.886, + 0.222, + 0.787, + 0.66, + 0.853, + 0.688, + 0.881, + 0.237, + 0.49, + 0.221, + 0.368, + 0.746, + 0.537, + 0.53, + 0.452, + 0.803, + 0.506, + 0.97, + 0.313, + 0.438, + 0.588, + 0.251, + 0.897, + 0.459, + 0.561, + 0.826, + 0.404, + 0.389, + 0.028, + 0.511, + 0.458, + 0.965, + 0.631, + 0.347, + 0.796, + 0.91, + 0.756, + 0.483, + 0.64, + 0.477, + 0.604, + 0.066, + 0.924, + 0.069, + 0.133, + 0.047, + 0.398, + 0.582, + 0.621, + 0.713, + 0.538, + 0.019, + 0.245, + 0.751, + 0.205, + 0.949, + 0.082, + 0.271, + 0.879, + 0.643, + 0.61, + 0.632, + 0.172, + 0.149, + 0.415, + 0.13, + 0.709, + 0.081, + 0.364, + 0.003, + 0.881, + 0.156, + 0.412, + 0.215, + 0.298, + 0.508, + 0.376, + 0.163, + 0.89, + 0.515, + 0.537, + 0.303, + 0.401, + 0.17, + 0.062, + 0.98, + 0.279, + 0.676, + 0.304, + 0.789, + 0.086, + 0.116, + 0.609, + 0.015, + 0.231, + 0.009, + 0.911, + 0.532, + 0.885, + 0.452, + 0.732, + 0.87, + 0.684, + 0.697, + 0.336, + 0.173, + 0.53, + 0.188, + 0.475, + 0.903, + 0.357, + 0.93, + 0.832, + 0.413, + 0.738, + 0.821, + 0.212, + 0.579, + 0.802, + 0.591, + 0.784, + 0.477, + 0.834, + 0.807, + 0.083, + 0.236, + 0.316, + 0.104, + 0.976, + 0.038, + 0.322, + 0.476, + 0.955, + 0.928, + 0.177, + 0.043, + 0.519, + 0.827, + 0.711, + 0.039, + 0.676, + 0.336, + 0.602, + 0.138, + 0.309, + 0.508, + 0.889, + 0.825, + 0.839, + 0.071, + 0.226, + 0.274, + 0.954, + 0.891, + 0.973, + 0.011, + 0.698, + 0.213, + 0.249, + 0.738, + 0.345, + 0.983, + 0.858, + 0.632, + 0.403, + 0.173, + 0.494, + 0.064, + 0.511, + 0.509, + 0.56, + 0.283, + 0.208, + 0.703, + 0.322, + 0.215, + 0.479, + 0.437, + 0.972, + 0.331, + 0.909, + 0.95, + 0.928, + 0.41, + 0.75, + 0.21, + 0.28, + 0.087, + 0.403, + 0.609, + 0.676, + 0.642, + 0.613, + 0.313, + 0.935, + 0.782, + 0.62, + 0.701, + 0.934, + 0.254, + 0.216, + 0.642, + 0.764, + 0.645, + 0.008, + 0.57, + 0.204, + 0.094, + 0.519, + 0.095, + 0.798, + 0.316, + 0.869, + 0.582, + 0.151, + 0.746, + 0.662 + ], + "pt_dbm": 20.0, + "n_exp": 6.0, + "threshold": -105.0, + "coverage_ratio": 0.8, + "delta_min": 5.0, + "horizon": 96, + "site_cap": 60.0, + "p_silent": 0.05, + "p_work_base": 3.0, + "p_work_coef": 3.0 +} diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_5.json b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_5.json new file mode 100644 index 00000000..1504665c --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_5.json @@ -0,0 +1,640 @@ +{ + "seed": 5, + "grid": { + "width": 200.0, + "height": 200.0, + "nx": 20, + "ny": 20 + }, + "sites": [ + [ + 52.5, + 35.1 + ], + [ + 58.0, + 144.3 + ], + [ + 63.3, + 35.6 + ], + [ + 47.1, + 148.5 + ], + [ + 139.4, + 156.9 + ], + [ + 69.2, + 80.1 + ], + [ + 134.0, + 135.4 + ], + [ + 190.5, + 150.5 + ], + [ + 157.5, + 99.8 + ], + [ + 161.5, + 138.1 + ], + [ + 120.8, + 142.7 + ], + [ + 171.9, + 195.0 + ], + [ + 191.4, + 141.2 + ], + [ + 170.5, + 52.8 + ], + [ + 148.4, + 167.9 + ], + [ + 24.2, + 132.2 + ], + [ + 36.4, + 184.4 + ], + [ + 45.8, + 37.9 + ], + [ + 48.1, + 43.4 + ], + [ + 67.9, + 144.1 + ], + [ + 85.0, + 54.0 + ], + [ + 122.6, + 76.9 + ], + [ + 62.1, + 53.7 + ], + [ + 161.0, + 160.2 + ], + [ + 39.5, + 65.5 + ], + [ + 151.5, + 40.6 + ], + [ + 153.5, + 45.8 + ], + [ + 56.2, + 173.2 + ], + [ + 165.9, + 41.0 + ], + [ + 43.3, + 166.1 + ], + [ + 158.5, + 123.2 + ], + [ + 161.9, + 49.8 + ], + [ + 73.7, + 163.2 + ], + [ + 82.2, + 56.5 + ], + [ + 160.2, + 30.4 + ], + [ + 83.9, + 94.1 + ] + ], + "groups": [ + [ + 0, + 2, + 17, + 18 + ], + [ + 28, + 31 + ], + [ + 21 + ], + [ + 13 + ], + [ + 8, + 30 + ], + [ + 7, + 9, + 11, + 12, + 23 + ], + [ + 22, + 24 + ], + [ + 4, + 6, + 10, + 14 + ], + [ + 5, + 20, + 33, + 35 + ], + [ + 25, + 26, + 34 + ], + [ + 1, + 3, + 15, + 16, + 19, + 27, + 29, + 32 + ] + ], + "battery": [ + 16.6, + 21.5, + 17.2, + 27.7, + 19.4, + 21.8, + 30.0, + 27.8, + 29.6, + 21.8, + 22.3 + ], + "demand": [ + 0.73, + 0.479, + 0.291, + 0.404, + 0.147, + 0.377, + 0.988, + 0.96, + 0.627, + 0.499, + 0.338, + 0.089, + 0.272, + 0.782, + 0.867, + 0.361, + 0.786, + 0.775, + 0.695, + 0.664, + 0.76, + 0.363, + 0.704, + 0.281, + 0.486, + 0.77, + 0.691, + 0.294, + 0.946, + 0.65, + 0.581, + 0.012, + 0.547, + 0.251, + 0.672, + 0.463, + 0.817, + 0.647, + 0.798, + 0.348, + 0.644, + 0.738, + 0.828, + 0.35, + 0.843, + 0.87, + 0.688, + 0.976, + 0.957, + 0.518, + 0.529, + 0.166, + 0.837, + 0.937, + 0.477, + 0.691, + 0.72, + 0.73, + 0.172, + 0.78, + 0.581, + 0.666, + 0.421, + 0.624, + 0.775, + 0.637, + 0.72, + 0.028, + 0.16, + 0.441, + 0.65, + 0.219, + 0.686, + 0.631, + 0.042, + 0.472, + 0.226, + 0.054, + 0.134, + 0.317, + 0.182, + 0.193, + 0.036, + 0.465, + 0.38, + 0.612, + 0.59, + 0.238, + 0.903, + 0.001, + 0.405, + 0.279, + 0.41, + 0.115, + 0.831, + 0.374, + 0.036, + 0.614, + 0.095, + 0.545, + 0.339, + 0.581, + 0.958, + 0.819, + 0.419, + 0.813, + 0.642, + 0.369, + 0.142, + 0.596, + 0.564, + 0.957, + 0.968, + 0.609, + 0.351, + 0.893, + 0.001, + 0.108, + 0.566, + 0.615, + 0.141, + 0.629, + 0.891, + 0.376, + 0.432, + 0.226, + 0.291, + 0.972, + 0.38, + 0.961, + 0.914, + 0.596, + 0.26, + 0.981, + 0.496, + 0.415, + 0.319, + 0.984, + 0.492, + 0.286, + 0.477, + 0.122, + 0.622, + 0.443, + 0.293, + 0.782, + 0.827, + 0.013, + 0.533, + 0.274, + 0.935, + 0.782, + 0.246, + 0.268, + 0.155, + 0.989, + 0.293, + 0.608, + 0.475, + 0.645, + 0.604, + 0.743, + 0.118, + 0.76, + 0.301, + 0.533, + 0.336, + 0.297, + 0.53, + 0.464, + 0.361, + 0.745, + 0.591, + 0.036, + 0.252, + 0.456, + 0.917, + 0.888, + 0.546, + 0.015, + 0.778, + 0.428, + 0.576, + 0.708, + 0.632, + 0.482, + 0.912, + 0.385, + 0.392, + 0.852, + 0.196, + 0.296, + 0.83, + 0.066, + 0.836, + 0.695, + 0.433, + 0.286, + 0.781, + 0.911, + 0.143, + 0.478, + 0.549, + 0.498, + 0.331, + 0.154, + 0.586, + 0.812, + 0.068, + 0.23, + 0.82, + 0.792, + 0.664, + 0.026, + 0.723, + 0.979, + 0.998, + 0.701, + 0.049, + 0.842, + 0.219, + 0.646, + 0.952, + 0.712, + 0.135, + 0.292, + 0.918, + 0.15, + 0.611, + 0.414, + 0.161, + 0.622, + 0.044, + 0.108, + 0.379, + 0.072, + 0.058, + 0.575, + 0.742, + 0.878, + 0.134, + 0.432, + 0.315, + 0.6, + 0.49, + 0.939, + 0.374, + 0.056, + 0.697, + 0.151, + 0.631, + 0.506, + 0.91, + 0.555, + 0.621, + 0.263, + 0.552, + 0.254, + 0.751, + 0.517, + 0.134, + 0.234, + 0.371, + 0.737, + 0.179, + 0.713, + 0.655, + 0.085, + 0.668, + 0.091, + 0.125, + 0.594, + 0.239, + 0.877, + 0.48, + 0.323, + 0.796, + 0.029, + 0.725, + 0.054, + 0.151, + 0.952, + 0.681, + 0.223, + 0.116, + 0.973, + 0.665, + 0.821, + 0.14, + 0.625, + 0.354, + 0.235, + 0.333, + 0.614, + 0.349, + 0.386, + 0.136, + 0.831, + 0.648, + 0.805, + 0.433, + 0.852, + 0.517, + 0.593, + 0.573, + 0.74, + 0.396, + 0.097, + 0.033, + 0.202, + 0.039, + 0.889, + 0.481, + 0.76, + 0.0, + 0.47, + 0.89, + 0.619, + 0.429, + 0.466, + 0.1, + 0.155, + 0.159, + 0.375, + 0.386, + 0.88, + 0.152, + 0.254, + 0.277, + 0.162, + 0.287, + 0.235, + 0.482, + 0.032, + 0.925, + 0.369, + 0.938, + 0.688, + 0.674, + 0.472, + 0.945, + 0.118, + 0.669, + 0.291, + 0.674, + 0.729, + 0.163, + 0.201, + 0.025, + 0.23, + 0.078, + 0.401, + 0.974, + 0.364, + 0.312, + 0.468, + 0.283, + 0.732, + 0.718, + 0.163, + 0.241, + 0.672, + 0.94, + 0.646, + 0.431, + 0.975, + 0.006, + 0.061, + 0.779, + 0.41, + 0.045, + 0.548, + 0.99, + 0.519, + 0.35, + 0.094, + 0.071, + 0.899, + 0.491, + 0.936, + 0.054, + 0.243, + 0.05, + 0.397, + 0.06, + 0.255, + 0.407, + 0.306, + 0.051, + 0.038, + 0.972, + 0.179, + 0.509, + 0.402, + 0.531, + 0.084, + 0.314, + 0.108, + 0.542, + 0.921 + ], + "pt_dbm": 20.0, + "n_exp": 6.0, + "threshold": -105.0, + "coverage_ratio": 0.8, + "delta_min": 5.0, + "horizon": 96, + "site_cap": 60.0, + "p_silent": 0.05, + "p_work_base": 3.0, + "p_work_coef": 3.0 +} diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_6.json b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_6.json new file mode 100644 index 00000000..ba9d253d --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_6.json @@ -0,0 +1,663 @@ +{ + "seed": 6, + "grid": { + "width": 200.0, + "height": 200.0, + "nx": 20, + "ny": 20 + }, + "sites": [ + [ + 66.2, + 71.1 + ], + [ + 48.2, + 153.8 + ], + [ + 160.4, + 119.3 + ], + [ + 171.1, + 136.7 + ], + [ + 58.9, + 73.8 + ], + [ + 35.7, + 126.6 + ], + [ + 60.1, + 35.0 + ], + [ + 36.7, + 65.7 + ], + [ + 41.6, + 153.0 + ], + [ + 139.1, + 16.0 + ], + [ + 75.4, + 42.0 + ], + [ + 170.8, + 11.0 + ], + [ + 190.0, + 132.0 + ], + [ + 40.0, + 38.9 + ], + [ + 141.1, + 175.1 + ], + [ + 19.9, + 50.0 + ], + [ + 171.9, + 69.8 + ], + [ + 46.2, + 130.8 + ], + [ + 158.9, + 60.5 + ], + [ + 54.6, + 61.3 + ], + [ + 22.1, + 139.4 + ], + [ + 32.7, + 167.4 + ], + [ + 109.4, + 195.0 + ], + [ + 60.6, + 74.7 + ], + [ + 167.8, + 162.2 + ], + [ + 73.2, + 31.3 + ], + [ + 114.3, + 162.3 + ], + [ + 27.6, + 47.6 + ], + [ + 162.2, + 145.2 + ], + [ + 19.5, + 159.6 + ], + [ + 174.7, + 52.9 + ], + [ + 34.0, + 139.9 + ], + [ + 32.1, + 149.7 + ], + [ + 95.4, + 144.7 + ], + [ + 138.0, + 139.8 + ], + [ + 137.1, + 130.6 + ], + [ + 121.6, + 150.6 + ], + [ + 136.9, + 176.3 + ], + [ + 144.6, + 135.2 + ], + [ + 181.9, + 144.9 + ] + ], + "groups": [ + [ + 6, + 10, + 25 + ], + [ + 16, + 18, + 30 + ], + [ + 34, + 35, + 38 + ], + [ + 0, + 4, + 19, + 23 + ], + [ + 24, + 28 + ], + [ + 2, + 3, + 12, + 39 + ], + [ + 9, + 11 + ], + [ + 1, + 5, + 8, + 17, + 20, + 31, + 32 + ], + [ + 7, + 13, + 15, + 27 + ], + [ + 21, + 29 + ], + [ + 14, + 22, + 37 + ], + [ + 26, + 33, + 36 + ] + ], + "battery": [ + 20.4, + 29.1, + 27.8, + 15.9, + 17.9, + 29.2, + 24.2, + 21.3, + 26.7, + 27.2, + 22.4, + 22.0 + ], + "demand": [ + 0.1, + 0.362, + 0.145, + 0.626, + 0.607, + 0.144, + 0.628, + 0.097, + 0.105, + 0.646, + 0.463, + 0.9, + 0.629, + 0.701, + 0.127, + 0.833, + 0.084, + 0.618, + 0.911, + 0.908, + 0.893, + 0.452, + 0.694, + 0.791, + 0.137, + 0.943, + 0.909, + 0.441, + 0.778, + 0.447, + 0.276, + 0.712, + 0.064, + 0.8, + 0.978, + 0.286, + 0.94, + 0.084, + 0.858, + 0.28, + 0.966, + 0.431, + 0.922, + 0.032, + 0.086, + 0.816, + 0.639, + 0.536, + 0.033, + 0.173, + 0.37, + 0.052, + 0.993, + 0.765, + 0.95, + 0.701, + 0.343, + 0.078, + 0.732, + 0.114, + 0.547, + 0.464, + 0.495, + 0.075, + 0.074, + 0.52, + 0.03, + 0.525, + 0.15, + 0.282, + 0.151, + 0.568, + 0.388, + 0.223, + 0.222, + 0.564, + 0.911, + 0.419, + 0.163, + 0.316, + 0.24, + 0.572, + 0.786, + 0.569, + 0.188, + 0.325, + 0.381, + 0.441, + 0.398, + 0.998, + 0.551, + 0.662, + 0.716, + 0.645, + 0.805, + 0.372, + 0.667, + 0.973, + 0.164, + 0.625, + 0.385, + 0.061, + 0.04, + 0.759, + 0.115, + 0.711, + 0.1, + 0.368, + 0.032, + 0.543, + 0.786, + 0.226, + 0.565, + 0.922, + 0.293, + 0.471, + 0.724, + 0.864, + 0.169, + 0.661, + 0.12, + 0.112, + 0.983, + 0.05, + 0.92, + 0.75, + 0.676, + 0.729, + 0.651, + 0.287, + 0.443, + 0.202, + 0.859, + 0.779, + 0.788, + 0.214, + 0.592, + 0.255, + 0.791, + 0.745, + 0.995, + 0.708, + 0.684, + 0.777, + 0.094, + 0.595, + 0.033, + 0.367, + 0.117, + 0.948, + 0.028, + 0.921, + 0.948, + 0.436, + 0.832, + 0.052, + 0.139, + 0.01, + 0.068, + 0.003, + 0.828, + 0.088, + 0.591, + 0.273, + 0.132, + 0.419, + 0.182, + 0.977, + 0.384, + 0.594, + 0.564, + 0.627, + 0.48, + 0.27, + 0.137, + 0.888, + 0.193, + 0.448, + 0.808, + 0.576, + 0.948, + 0.81, + 0.514, + 0.12, + 0.593, + 0.474, + 0.862, + 0.531, + 0.694, + 0.812, + 0.47, + 0.842, + 0.899, + 0.154, + 0.919, + 0.87, + 0.855, + 0.338, + 0.252, + 0.133, + 0.371, + 0.855, + 0.363, + 0.005, + 0.305, + 0.083, + 0.623, + 0.768, + 0.927, + 0.089, + 0.38, + 0.126, + 0.91, + 0.837, + 0.52, + 0.954, + 0.52, + 0.776, + 0.488, + 0.364, + 0.994, + 0.842, + 0.592, + 0.867, + 0.649, + 0.798, + 0.067, + 0.132, + 0.109, + 0.603, + 0.72, + 0.155, + 0.505, + 0.114, + 0.688, + 0.779, + 0.087, + 0.98, + 0.665, + 0.125, + 0.351, + 0.158, + 0.728, + 0.917, + 0.175, + 0.184, + 0.629, + 0.86, + 0.739, + 0.458, + 0.111, + 0.614, + 0.456, + 0.804, + 0.572, + 0.734, + 0.971, + 0.626, + 0.912, + 0.274, + 0.002, + 0.882, + 0.03, + 0.994, + 0.456, + 0.061, + 0.275, + 0.721, + 0.384, + 0.624, + 0.102, + 0.836, + 0.373, + 0.833, + 0.147, + 0.667, + 0.469, + 0.404, + 0.019, + 0.532, + 0.838, + 0.104, + 0.446, + 0.012, + 0.994, + 0.97, + 0.678, + 0.48, + 0.07, + 0.121, + 0.705, + 0.795, + 0.989, + 0.862, + 0.438, + 0.255, + 0.542, + 0.346, + 0.545, + 0.884, + 0.453, + 0.981, + 0.718, + 0.046, + 0.747, + 0.662, + 0.622, + 0.003, + 0.879, + 0.495, + 0.537, + 0.17, + 0.035, + 0.014, + 0.21, + 0.304, + 0.154, + 0.058, + 0.727, + 0.702, + 0.882, + 0.218, + 0.728, + 0.594, + 0.09, + 0.875, + 0.357, + 0.185, + 0.183, + 0.924, + 0.221, + 0.384, + 0.547, + 0.359, + 0.656, + 0.088, + 0.43, + 0.585, + 0.619, + 0.609, + 0.269, + 0.638, + 0.972, + 0.482, + 0.644, + 0.837, + 0.252, + 0.266, + 0.97, + 0.301, + 0.426, + 0.371, + 0.993, + 0.545, + 0.896, + 0.162, + 0.278, + 0.65, + 0.318, + 0.663, + 0.508, + 0.078, + 0.112, + 0.439, + 0.534, + 0.177, + 0.263, + 0.001, + 0.266, + 0.309, + 0.388, + 0.593, + 0.853, + 0.717, + 0.741, + 0.283, + 0.496, + 0.727, + 0.894, + 0.24, + 0.015, + 0.051, + 0.515, + 0.872, + 0.153, + 0.784, + 0.641, + 0.864, + 0.901, + 0.107, + 0.794, + 0.135, + 0.119, + 0.399, + 0.489, + 0.07, + 0.276, + 0.175, + 0.135, + 0.099 + ], + "pt_dbm": 20.0, + "n_exp": 6.0, + "threshold": -105.0, + "coverage_ratio": 0.8, + "delta_min": 5.0, + "horizon": 96, + "site_cap": 60.0, + "p_silent": 0.05, + "p_work_base": 3.0, + "p_work_coef": 3.0 +} diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_7.json b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_7.json new file mode 100644 index 00000000..80637450 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_7.json @@ -0,0 +1,568 @@ +{ + "seed": 7, + "grid": { + "width": 200.0, + "height": 200.0, + "nx": 20, + "ny": 20 + }, + "sites": [ + [ + 53.9, + 142.0 + ], + [ + 61.7, + 142.5 + ], + [ + 43.0, + 153.1 + ], + [ + 43.4, + 147.8 + ], + [ + 76.7, + 140.8 + ], + [ + 32.2, + 140.6 + ], + [ + 87.9, + 61.4 + ], + [ + 128.2, + 62.1 + ], + [ + 121.9, + 38.8 + ], + [ + 25.1, + 136.0 + ], + [ + 175.8, + 68.1 + ], + [ + 32.8, + 134.1 + ], + [ + 153.8, + 127.9 + ], + [ + 140.2, + 161.6 + ], + [ + 45.3, + 35.8 + ], + [ + 109.7, + 43.6 + ], + [ + 136.8, + 195.0 + ], + [ + 38.0, + 149.1 + ], + [ + 163.4, + 68.9 + ], + [ + 57.8, + 148.1 + ], + [ + 140.0, + 65.7 + ], + [ + 130.6, + 139.4 + ], + [ + 71.3, + 142.3 + ], + [ + 78.9, + 161.6 + ] + ], + "groups": [ + [ + 7, + 8, + 15, + 20 + ], + [ + 10, + 18 + ], + [ + 13, + 16 + ], + [ + 6, + 14 + ], + [ + 2, + 3, + 5, + 9, + 11, + 17 + ], + [ + 12, + 21 + ], + [ + 0, + 1, + 4, + 19, + 22, + 23 + ] + ], + "battery": [ + 15.3, + 21.9, + 17.5, + 16.8, + 15.9, + 26.5, + 16.9 + ], + "demand": [ + 0.248, + 0.391, + 0.871, + 0.081, + 0.449, + 0.549, + 0.883, + 0.819, + 0.864, + 0.278, + 0.415, + 0.359, + 0.884, + 0.958, + 0.151, + 0.176, + 0.232, + 0.233, + 0.485, + 0.589, + 0.263, + 0.004, + 0.419, + 0.369, + 0.566, + 0.953, + 0.69, + 0.515, + 0.618, + 0.676, + 0.054, + 0.9, + 0.78, + 0.875, + 0.798, + 0.392, + 0.399, + 0.104, + 0.634, + 0.062, + 0.067, + 0.209, + 0.162, + 0.34, + 0.053, + 0.0, + 0.151, + 0.101, + 0.364, + 0.026, + 0.874, + 0.614, + 0.149, + 0.252, + 0.347, + 0.364, + 0.123, + 0.849, + 0.993, + 0.466, + 0.484, + 0.086, + 0.102, + 0.343, + 0.265, + 0.829, + 0.161, + 0.023, + 0.951, + 0.528, + 0.147, + 0.543, + 0.027, + 0.528, + 0.979, + 0.863, + 0.696, + 0.261, + 0.367, + 0.167, + 0.772, + 0.533, + 0.779, + 0.33, + 0.223, + 0.812, + 0.985, + 0.853, + 0.806, + 0.818, + 0.74, + 0.227, + 0.518, + 0.356, + 0.029, + 0.028, + 0.279, + 0.259, + 0.693, + 0.957, + 0.447, + 0.937, + 0.988, + 0.955, + 0.365, + 0.22, + 0.227, + 0.197, + 0.204, + 0.624, + 0.9, + 0.84, + 0.479, + 0.653, + 0.8, + 0.085, + 0.661, + 0.91, + 0.782, + 0.75, + 0.478, + 0.179, + 0.789, + 0.333, + 0.801, + 0.972, + 0.396, + 0.401, + 0.947, + 0.725, + 0.17, + 0.127, + 0.151, + 0.905, + 0.807, + 0.146, + 0.827, + 0.98, + 0.657, + 0.35, + 0.549, + 0.131, + 0.014, + 0.971, + 0.65, + 0.527, + 0.934, + 0.434, + 0.872, + 0.826, + 0.211, + 0.252, + 0.293, + 0.241, + 0.586, + 0.259, + 0.419, + 0.131, + 0.91, + 0.354, + 0.458, + 0.583, + 0.904, + 0.421, + 0.918, + 0.502, + 0.532, + 0.524, + 0.019, + 0.44, + 0.183, + 0.004, + 0.799, + 0.172, + 0.473, + 0.725, + 0.556, + 0.326, + 0.518, + 0.555, + 0.784, + 0.106, + 0.56, + 0.248, + 0.277, + 0.772, + 0.508, + 0.562, + 0.76, + 0.912, + 0.443, + 0.613, + 0.506, + 0.512, + 0.693, + 0.452, + 0.533, + 0.478, + 0.942, + 0.699, + 0.877, + 0.942, + 0.26, + 0.56, + 0.943, + 0.84, + 0.137, + 0.122, + 0.442, + 0.073, + 0.241, + 0.073, + 0.669, + 0.784, + 0.897, + 0.154, + 0.716, + 0.66, + 0.143, + 0.883, + 0.968, + 0.22, + 0.953, + 0.398, + 0.487, + 0.99, + 0.832, + 0.161, + 0.432, + 0.516, + 0.339, + 0.196, + 0.319, + 0.722, + 0.019, + 0.554, + 0.44, + 0.018, + 0.331, + 0.624, + 0.512, + 0.064, + 0.985, + 0.788, + 0.972, + 0.105, + 0.266, + 0.04, + 0.779, + 0.27, + 0.13, + 0.422, + 0.911, + 0.819, + 0.259, + 0.149, + 0.919, + 0.571, + 0.7, + 0.089, + 0.058, + 0.688, + 0.425, + 0.072, + 0.938, + 0.634, + 0.802, + 0.084, + 0.856, + 0.067, + 0.863, + 0.454, + 0.339, + 0.553, + 0.927, + 0.268, + 0.129, + 0.527, + 0.238, + 0.109, + 0.161, + 0.05, + 0.202, + 0.312, + 0.305, + 0.759, + 0.29, + 0.5, + 0.178, + 0.347, + 0.018, + 0.25, + 0.015, + 0.733, + 0.551, + 0.189, + 0.475, + 0.935, + 0.106, + 0.819, + 0.432, + 0.495, + 0.835, + 0.393, + 0.507, + 0.688, + 0.982, + 0.343, + 0.832, + 0.707, + 0.636, + 0.405, + 0.348, + 0.054, + 0.13, + 0.071, + 0.741, + 0.256, + 0.163, + 0.084, + 0.841, + 0.871, + 0.671, + 0.282, + 0.242, + 0.293, + 0.459, + 0.158, + 0.446, + 0.263, + 0.962, + 0.973, + 0.547, + 0.244, + 0.966, + 0.31, + 0.357, + 0.001, + 0.382, + 0.475, + 0.503, + 0.201, + 0.505, + 0.005, + 0.264, + 0.09, + 0.4, + 0.042, + 0.022, + 0.304, + 0.233, + 0.586, + 0.529, + 0.751, + 0.658, + 0.716, + 0.879, + 0.39, + 0.326, + 0.985, + 0.149, + 0.724, + 0.643, + 0.044, + 0.835, + 0.892, + 0.627, + 0.734, + 0.812, + 0.139, + 0.524, + 0.504, + 0.835, + 0.805, + 0.826, + 0.584, + 0.893, + 0.683, + 0.693, + 0.23, + 0.031, + 0.133, + 0.361, + 0.105, + 0.836, + 0.559, + 0.628, + 0.626, + 0.681, + 0.489, + 0.003, + 0.798, + 0.748, + 0.503, + 0.535, + 0.659, + 0.066, + 0.737, + 0.252, + 0.074 + ], + "pt_dbm": 20.0, + "n_exp": 6.0, + "threshold": -105.0, + "coverage_ratio": 0.8, + "delta_min": 5.0, + "horizon": 96, + "site_cap": 60.0, + "p_silent": 0.05, + "p_work_base": 3.0, + "p_work_coef": 3.0 +} diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_8.json b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_8.json new file mode 100644 index 00000000..550d197e --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/data/instances/instance_8.json @@ -0,0 +1,617 @@ +{ + "seed": 8, + "grid": { + "width": 200.0, + "height": 200.0, + "nx": 20, + "ny": 20 + }, + "sites": [ + [ + 147.6, + 141.9 + ], + [ + 158.9, + 128.0 + ], + [ + 36.1, + 32.8 + ], + [ + 29.8, + 66.6 + ], + [ + 62.4, + 57.8 + ], + [ + 142.7, + 26.9 + ], + [ + 65.9, + 38.7 + ], + [ + 145.3, + 42.3 + ], + [ + 121.6, + 74.6 + ], + [ + 140.3, + 34.6 + ], + [ + 121.0, + 50.4 + ], + [ + 140.3, + 144.6 + ], + [ + 144.6, + 48.6 + ], + [ + 173.7, + 181.8 + ], + [ + 44.6, + 9.1 + ], + [ + 13.2, + 136.1 + ], + [ + 72.5, + 75.5 + ], + [ + 133.0, + 46.6 + ], + [ + 86.6, + 138.1 + ], + [ + 143.2, + 167.5 + ], + [ + 42.3, + 149.5 + ], + [ + 125.6, + 186.2 + ], + [ + 41.3, + 163.1 + ], + [ + 49.5, + 66.8 + ], + [ + 59.1, + 67.9 + ], + [ + 185.6, + 133.9 + ], + [ + 160.2, + 82.6 + ], + [ + 76.5, + 173.8 + ], + [ + 43.3, + 60.2 + ], + [ + 59.1, + 43.9 + ], + [ + 43.2, + 54.5 + ], + [ + 147.8, + 58.7 + ] + ], + "groups": [ + [ + 5, + 7, + 9, + 10, + 12, + 17 + ], + [ + 13, + 19, + 21 + ], + [ + 3, + 28, + 30 + ], + [ + 4, + 23, + 24 + ], + [ + 16 + ], + [ + 18, + 27 + ], + [ + 0, + 1, + 11, + 25 + ], + [ + 2, + 6, + 14, + 29 + ], + [ + 8, + 26, + 31 + ], + [ + 15, + 20, + 22 + ] + ], + "battery": [ + 25.6, + 29.7, + 15.9, + 15.3, + 17.1, + 20.8, + 23.6, + 26.2, + 20.3, + 29.5 + ], + "demand": [ + 0.003, + 0.836, + 0.21, + 0.875, + 0.033, + 0.965, + 0.759, + 0.99, + 0.58, + 0.178, + 0.97, + 0.442, + 0.083, + 0.605, + 0.497, + 0.473, + 0.155, + 0.122, + 0.581, + 0.453, + 0.126, + 0.439, + 0.726, + 0.715, + 0.471, + 0.394, + 0.361, + 0.226, + 0.198, + 0.446, + 0.108, + 0.725, + 0.708, + 0.612, + 0.023, + 0.364, + 0.432, + 0.093, + 0.968, + 0.756, + 0.503, + 0.705, + 0.201, + 0.229, + 0.526, + 0.744, + 0.647, + 0.007, + 0.613, + 0.909, + 0.743, + 0.928, + 0.982, + 0.224, + 0.296, + 0.547, + 0.977, + 0.369, + 0.941, + 0.85, + 0.516, + 0.625, + 0.957, + 0.783, + 0.296, + 0.846, + 0.196, + 0.37, + 0.959, + 0.847, + 0.312, + 0.29, + 0.613, + 0.843, + 0.684, + 0.936, + 0.728, + 0.821, + 0.605, + 0.362, + 0.428, + 0.49, + 0.529, + 0.838, + 0.269, + 0.069, + 0.733, + 0.271, + 0.42, + 0.117, + 0.829, + 0.274, + 0.292, + 0.804, + 0.504, + 0.277, + 0.888, + 0.412, + 0.361, + 0.965, + 0.79, + 0.747, + 0.25, + 0.813, + 0.597, + 0.83, + 0.368, + 0.781, + 0.793, + 0.388, + 0.655, + 0.78, + 0.993, + 0.92, + 0.786, + 0.137, + 0.61, + 0.685, + 0.087, + 0.463, + 0.869, + 0.905, + 0.925, + 0.093, + 0.607, + 0.865, + 0.653, + 0.32, + 0.039, + 0.92, + 0.859, + 0.654, + 0.366, + 0.828, + 0.815, + 0.725, + 0.6, + 0.319, + 0.297, + 0.357, + 0.327, + 0.52, + 0.114, + 0.721, + 0.348, + 0.16, + 0.255, + 0.389, + 0.431, + 0.782, + 0.1, + 0.223, + 0.293, + 0.177, + 0.749, + 0.109, + 0.051, + 0.556, + 0.736, + 0.825, + 0.559, + 0.327, + 0.882, + 0.736, + 0.479, + 0.563, + 0.936, + 0.997, + 0.931, + 0.133, + 0.327, + 0.408, + 0.169, + 0.762, + 0.061, + 0.887, + 0.586, + 0.614, + 0.415, + 0.937, + 0.578, + 0.897, + 0.923, + 0.692, + 0.983, + 0.49, + 0.367, + 0.844, + 0.528, + 0.309, + 0.68, + 0.476, + 0.128, + 0.838, + 0.5, + 0.972, + 0.534, + 0.937, + 0.289, + 0.217, + 0.527, + 0.767, + 0.825, + 0.647, + 0.124, + 0.394, + 0.035, + 0.371, + 0.945, + 0.347, + 0.699, + 0.621, + 0.143, + 0.688, + 0.08, + 0.185, + 0.58, + 0.127, + 0.964, + 0.426, + 0.63, + 0.965, + 0.108, + 0.999, + 0.04, + 0.601, + 0.656, + 0.986, + 0.358, + 0.947, + 0.65, + 0.477, + 0.17, + 0.236, + 0.365, + 0.346, + 0.372, + 0.808, + 0.375, + 0.629, + 0.039, + 0.141, + 0.71, + 0.891, + 0.313, + 0.032, + 0.233, + 0.36, + 0.366, + 0.561, + 0.302, + 0.647, + 0.786, + 0.783, + 0.741, + 0.584, + 0.261, + 0.841, + 0.29, + 0.86, + 0.447, + 0.629, + 0.499, + 0.654, + 0.341, + 0.578, + 0.737, + 0.361, + 0.6, + 0.456, + 0.47, + 0.384, + 0.547, + 0.709, + 0.174, + 0.146, + 0.823, + 0.088, + 0.793, + 0.759, + 0.218, + 0.636, + 0.253, + 0.882, + 0.197, + 0.116, + 0.037, + 0.719, + 0.212, + 0.314, + 0.977, + 0.736, + 0.738, + 0.576, + 0.641, + 0.011, + 0.813, + 0.329, + 0.855, + 0.415, + 0.911, + 0.0, + 0.98, + 0.096, + 0.511, + 0.482, + 0.986, + 0.717, + 0.982, + 0.906, + 0.425, + 0.148, + 0.146, + 0.709, + 0.951, + 0.665, + 0.346, + 0.597, + 0.657, + 0.253, + 0.694, + 0.199, + 0.028, + 0.064, + 0.488, + 0.696, + 0.226, + 0.397, + 0.693, + 0.263, + 0.522, + 0.603, + 0.406, + 0.208, + 0.693, + 0.701, + 0.189, + 0.807, + 0.655, + 0.866, + 0.78, + 0.494, + 0.716, + 0.314, + 0.654, + 0.082, + 0.346, + 0.506, + 0.886, + 0.684, + 0.909, + 0.869, + 0.971, + 0.618, + 0.617, + 0.469, + 0.221, + 0.797, + 0.61, + 0.162, + 0.24, + 0.221, + 0.723, + 0.597, + 0.201, + 0.688, + 0.563, + 0.267, + 0.445, + 0.993, + 0.141, + 0.237, + 0.594, + 0.84, + 0.922, + 0.399, + 0.769, + 0.119, + 0.386, + 0.857, + 0.665, + 0.447, + 0.599, + 0.487, + 0.548, + 0.884, + 0.976, + 0.2, + 0.755, + 0.945, + 0.607, + 0.5, + 0.373, + 0.484, + 0.581, + 0.611, + 0.114, + 0.999, + 0.276, + 0.548 + ], + "pt_dbm": 20.0, + "n_exp": 6.0, + "threshold": -105.0, + "coverage_ratio": 0.8, + "delta_min": 5.0, + "horizon": 96, + "site_cap": 60.0, + "p_silent": 0.05, + "p_work_base": 3.0, + "p_work_coef": 3.0 +} diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/docker/Dockerfile b/benchmarks/PowerSystems/TelecomBackup/verification/docker/Dockerfile new file mode 100644 index 00000000..7c94d870 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim + +WORKDIR /workspace + +# The evaluator, simulator, reference solver and baseline use only the Python +# standard library. The unified runtime mounts the benchmark sandbox into the +# container, so no benchmark files are baked into the image. The candidate may +# read `verification/simulator.py` (it is part of the scoring objective and is +# allowed by the constraints), but no instance data is baked in. + +ENV PYTHONUNBUFFERED=1 + +CMD ["python"] diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/evaluate.py b/benchmarks/PowerSystems/TelecomBackup/verification/evaluate.py new file mode 100644 index 00000000..640b660b --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/evaluate.py @@ -0,0 +1,472 @@ +"""TelecomBackup evaluation entry point. + +CLI:: + + python verification/evaluate.py [--local] [--generate-seed N] + [--reference] [--time-budget S] [--data-dir DIR] + +For every instance the candidate solver runs in a subprocess (timeout = the +per-instance budget); its stdout is parsed into on-intervals and handed to the +simulator, which returns the backup time. Malformed output / out-of-range +intervals / crash / timeout score 0 for that instance. Score = mean backup time +(minutes) over all scored instances. + +Evaluation modes +---------------- +``local`` (default; development and smoke tests) + Scores the committed fixed instances in ``verification/data/instances``. + Setting ``TELECOM_EVAL_GENERATE_SEED`` additionally scores fresh instances. + +``official`` (``TELECOM_EVAL_MODE=official``; set by ``frontier_eval/eval_command.txt``) + Scores **only** freshly generated instances -- the committed fixed instances + are never part of the official score, so a candidate cannot memorise them. + A generation seed is mandatory: if it is missing the evaluator raises + ``OfficialModeConfigError`` instead of silently falling back to the public + instances (the reviewer-reported hardcoding hole). + +Integrity / anti-cheating (mirrors the CVRP benchmark) + * static candidate checks (EVOLVE-BLOCK markers + fixed-region byte diff, + forbidden imports, absolute paths, per-instance hardcoding) -- validator.py; + * candidate subprocess environment strips ``FRONTIER_*`` / ``TELECOM_EVAL_*`` + (``candidate_env``), closing the host-env side channel; + * runtime-generated instances (written to a temp dir, never to the repo or the + sandbox), so a candidate cannot pre-position solutions for them; + * determinism probe: cross-size instances are each run twice and must agree. + * ``--reference`` deliberately bypasses the candidate checks so the bundled + reference solver can be scored; it exists only to reproduce the documented + reference score and must not be used to score candidates. + +Time budget +----------- +The per-instance budget defaults to 60 s. When the unified framework drives the +evaluation it exports ``FRONTIER_EVAL_EVALUATOR_TIMEOUT_S`` (its whole-evaluation +wall-clock cap); the evaluator then shrinks the per-instance budget if needed so +the run cannot be killed by that cap, and records ``budget_shrunk`` / the +effective budget in its result. The official path should raise the cap (see the +README) so the full per-instance budget is available. + +Environment variables + TELECOM_EVAL_MODE "official" | "local" (default "local") + TELECOM_EVAL_GENERATE_SEED generation seed (mandatory in official mode) + TELECOM_EVAL_GENERATE_COUNT number of generated instances (default 8) + TELECOM_EVAL_REFERENCE non-empty/"1" to bypass candidate checks + FRONTIER_EVAL_EVALUATOR_TIMEOUT_S framework-level wall-clock budget (seconds) + +Interface used by ``frontier_eval/evaluator.py``:: + + evaluate(program_path, *, time_budget=60.0) -> {"combined_score": float, + "valid": float, "per_instance": {...}, ...} +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +# 保证无论从哪个 cwd/以何种方式加载，都能 import 到同目录的模块 +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from simulator import simulate, load_instance # noqa: E402 +from validator import candidate_env, check_candidate, check_determinism # noqa: E402 + +INVALID_SCORE = 0.0 +DATA_DIR = Path(__file__).resolve().parent / "data" / "instances" +DEFAULT_GENERATE_COUNT = 8 +# 运行时生成实例的规模循环（与 generator 默认一致） +GEN_SIZES = (20, 24, 28, 32, 36, 40) +# 框架整体超时未知时的保守默认（仅用于自限；不影响独立运行 CLI） +DEFAULT_FRAMEWORK_TIMEOUT_S = 300.0 +# 预留给进程启动 / 框架开销、不计入每实例预算的时间（秒） +BUDGET_RESERVE_S = 30.0 + +MODES = ("official", "local") + + +class OfficialModeConfigError(RuntimeError): + """official 模式缺少必需配置（如生成种子）时抛出，绝不静默回退到公开实例。""" + + +def _source_benchmark_dir() -> Path | None: + """宿主基准目录（unified 沙箱通过该环境变量暴露），用于读取初始 baseline 做比对。""" + raw = os.environ.get("FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR", "").strip() + if raw: + path = Path(raw) + if path.is_dir(): + return path + return None + + +def _resolve_mode(explicit: str | None = None) -> str: + raw = (explicit if explicit is not None else os.environ.get("TELECOM_EVAL_MODE", "")).strip().lower() + if not raw: + return "local" + if raw not in MODES: + raise OfficialModeConfigError( + f"TELECOM_EVAL_MODE must be one of {MODES}, got {raw!r}" + ) + return raw + + +def _reference_mode(explicit: bool | None = None) -> bool: + if explicit is not None: + return bool(explicit) + return os.environ.get("TELECOM_EVAL_REFERENCE", "").strip() not in ("", "0", "false", "False") + + +def _generate_seed() -> int | None: + raw = os.environ.get("TELECOM_EVAL_GENERATE_SEED", "").strip() + if not raw: + return None + try: + return int(raw) + except ValueError as exc: + raise OfficialModeConfigError( + f"TELECOM_EVAL_GENERATE_SEED must be an integer, got {raw!r}" + ) from exc + + +def _generate_count() -> int: + raw = os.environ.get("TELECOM_EVAL_GENERATE_COUNT", "").strip() + if not raw: + return DEFAULT_GENERATE_COUNT + try: + return max(0, int(raw)) + except ValueError: + return DEFAULT_GENERATE_COUNT + + +def _framework_timeout_s() -> float | None: + """框架整体超时（仅当框架显式设置该变量时返回，独立运行 CLI 时为 None）。""" + raw = os.environ.get("FRONTIER_EVAL_EVALUATOR_TIMEOUT_S", "").strip() + if not raw: + return None + try: + value = float(raw) + except ValueError: + return None + return value if value > 0 else None + + +def _fit_budget(requested: float, n_runs: int) -> tuple[float, bool, float | None]: + """按框架整体超时收缩每实例预算，保证评测总时长不超过观测到的框架上限。 + + 仅在框架显式设置 ``FRONTIER_EVAL_EVALUATOR_TIMEOUT_S`` 时收缩（独立 CLI 不受影响）。 + 返回 (有效预算, 是否收缩, 框架超时)。不设下限：``n_runs * budget`` 恒 ≤ 上限， + 因此合规求解器即使跑满预算也不会被框架硬杀（超过上限会被判 -1e18，比低分更糟）。 + """ + timeout = _framework_timeout_s() + if timeout is None or n_runs <= 0: + return requested, False, timeout + # Monotonic in `timeout` and always < timeout, so `n_runs * budget` never + # exceeds the cap. Above 2*RESERVE keep the reserve; at or below it (a cap + # barely above the fixed overhead) use half the cap. + available = ( + (timeout - BUDGET_RESERVE_S) + if timeout > 2.0 * BUDGET_RESERVE_S + else timeout / 2.0 + ) + fit = available / n_runs + if fit < requested: + return fit, True, timeout + return requested, False, timeout + + +def _parse_on_intervals(raw: str, k: int, horizon: int) -> list[list[list[int]]] | None: + """解析候选 stdout 为 on_intervals；非法返回 None。 + + 格式：{"on": [[[a,b),...], ...]}，on[k] 为电源 k 的开启区间（半开 [a,b)）。 + """ + try: + text = raw.strip() + if not text: + return None + obj = json.loads(text) + if isinstance(obj, dict): + obj = obj.get("on") + if not isinstance(obj, list) or len(obj) != k: + return None + out: list[list[list[int]]] = [] + for ivs in obj: + if not isinstance(ivs, list): + return None + cur: list[list[int]] = [] + for iv in ivs: + if not isinstance(iv, list) or len(iv) != 2: + return None + a, b = iv + if isinstance(a, bool) or isinstance(b, bool): + return None + if not isinstance(a, int) or not isinstance(b, int): + return None + if a < 0 or b > horizon or a > b: + return None + cur.append([a, b]) + out.append(cur) + return out + except Exception: + return None + + +def _run_one(program_path: Path, inst_path: Path, time_budget: float, + python: str) -> tuple[float, dict[str, Any]]: + inst = load_instance(inst_path) + k = len(inst["groups"]) + horizon = int(inst["horizon"]) + started = time.time() + try: + proc = subprocess.run( + [python, str(program_path), str(inst_path)], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=time_budget, + cwd=str(program_path.parent), + env=candidate_env(), + ) + elapsed = time.time() - started + if proc.returncode != 0: + return 0.0, {"status": "crash", "stderr_tail": proc.stderr[-500:]} + on = _parse_on_intervals(proc.stdout, k, horizon) + if on is None: + return 0.0, {"status": "bad_output", "stdout_tail": proc.stdout[-500:]} + minutes = simulate(inst, on) + return minutes, {"status": "ok", "on": on, "minutes": round(minutes, 1), + "elapsed_s": round(elapsed, 2)} + except subprocess.TimeoutExpired: + return 0.0, {"status": "timeout", "budget_s": round(time_budget, 2)} + except Exception as exc: + return 0.0, {"status": "error", "message": str(exc)} + + +def _load_host_module(mod_name: str): + """从宿主 benchmark 目录加载 `verification/.py`（沙箱内不含该模块时）。 + + 沙箱内不复制 generator.py（候选不可见）；评测器需要生成器时经 + FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR 从宿主加载（CVRP 同款模式）。 + """ + import importlib.util + + src = os.environ.get("FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR", "").strip() + if src and Path(src).is_dir(): + path = Path(src) / "verification" / f"{mod_name}.py" + if path.is_file(): + spec = importlib.util.spec_from_file_location(f"_tb_{mod_name}", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + return None + + +def _generate_instances(base_seed: int, count: int, out_dir: Path) -> list[Path]: + """按种子现场生成新实例（防硬编码）。派生种子 base_seed*1000+i，可复现。 + + 生成器从宿主加载；生成实例逐个过 `_stagger_ok`（错峰 ≥ 全程开启 25%）， + 不合格跳过——保证生成实例同样奖励调度（与固定实例的验收一致）。 + """ + gen_mod = _load_host_module("generator") + if gen_mod is None: + try: + import generator as gen_mod # 直跑（非沙箱）时的本地回退 + except ImportError as exc: + raise OfficialModeConfigError( + "runtime instance generation requires verification/generator.py " + "(loaded from the host benchmark dir via " + "FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR, or importable locally); " + "it is intentionally not copied into the sandbox" + ) from exc + + paths: list[Path] = [] + i = 0 + attempts = 0 + while len(paths) < count and attempts < count * 64: + inst = gen_mod.generate(base_seed * 1000 + i, GEN_SIZES[i % len(GEN_SIZES)]) + i += 1 + attempts += 1 + if not gen_mod._stagger_ok(inst): + continue + path = out_dir / f"gen_{base_seed}_{len(paths) + 1}.json" + path.write_text(json.dumps(inst, ensure_ascii=False) + "\n", encoding="utf-8") + paths.append(path) + return paths + + +def _select_probes(instances: list[Path], n: int = 3) -> list[Path]: + """跨规模选确定性探针：按排序取 小/中/大 各一（外加一个生成实例，若有）。 + + 去重：official 模式下实例本身全是生成实例，`gen[0]` 可能已在探针里，不重复添加。 + """ + if len(instances) <= n: + return list(instances) + idxs = sorted({0, len(instances) // 2, len(instances) - 1}) + probes = [instances[i] for i in idxs] + gen = [p for p in instances if p.name.startswith("gen_")] + if gen and gen[0] not in probes: + probes.append(gen[0]) + return probes + + +def evaluate(program_path: str, *, time_budget: float = 60.0, + data_dir: str | Path | None = None, + python: str | None = None, + mode: str | None = None, + reference: bool | None = None) -> dict[str, Any]: + """评测候选求解器：返回 metrics 字典。 + + official 模式缺少生成种子时抛 ``OfficialModeConfigError``（不静默回退）。 + """ + resolved_mode = _resolve_mode(mode) + is_reference = _reference_mode(reference) + + prog = Path(program_path).resolve() + if not prog.exists(): + return {"combined_score": 0.0, "valid": 0.0, "per_instance": {}, + "mode": resolved_mode, "error": f"program not found: {prog}"} + + gen_seed = _generate_seed() + gen_count = _generate_count() + + # official 模式：必须要有生成种子；缺失即显式失败，绝不回退到公开固定实例。 + if resolved_mode == "official" and gen_seed is None: + raise OfficialModeConfigError( + "official mode requires TELECOM_EVAL_GENERATE_SEED to be set " + "(the official score must be computed on freshly generated instances, " + "never on the committed public instances). Set the variable to a " + "non-public seed, or use `--local` for development scoring." + ) + if resolved_mode == "official" and gen_count <= 0: + raise OfficialModeConfigError( + "official mode requires TELECOM_EVAL_GENERATE_COUNT > 0, got " + f"{gen_count!r}" + ) + + # 静态完整性检查（EVOLVE-BLOCK 标记/只读区比对、禁引用、禁绝对路径、禁硬编码）。 + # --reference 有意跳过：仅供复现内置参考求解器的分数。 + violations: list[str] = [] + if not is_reference: + baseline_path = None + src_dir = _source_benchmark_dir() + if src_dir is not None: + candidate_baseline = src_dir / "baseline" / "solver.py" + if candidate_baseline.is_file(): + baseline_path = candidate_baseline + violations = check_candidate(prog, baseline_path=baseline_path) + + # 实例池： + # official -> 仅运行时生成（不含公开固定实例） + # local -> 固定实例（存在时）+ 可选运行时生成 + n_fixed = 0 + instances: list[Path] = [] + generating = gen_seed is not None and gen_count > 0 + tmp_dir: Path | None = ( + Path(tempfile.mkdtemp(prefix="telecom_eval_")) + if (resolved_mode == "official" or generating) + else None + ) + if resolved_mode == "official": + # official 模式必须有种子与正数生成量（上面已校验），此处 tmp_dir 必非空 + instances = _generate_instances(int(gen_seed), gen_count, tmp_dir) + else: + inst_dir = Path(data_dir).resolve() if data_dir else DATA_DIR + if inst_dir.is_dir(): + fixed = sorted(inst_dir.glob("instance_*.json")) + n_fixed = len(fixed) + instances = list(fixed) + if generating and tmp_dir is not None: + instances.extend(_generate_instances(gen_seed, gen_count, tmp_dir)) + + if not instances: + return {"combined_score": 0.0, "valid": 0.0, "per_instance": {}, + "mode": resolved_mode, "generate_seed": gen_seed, + "num_instances": 0, + "error": f"no instances (mode={resolved_mode}, generate_seed={gen_seed!r})"} + + py = python or sys.executable + + # 每实例预算：按框架整体超时自限，保证合规求解器用满预算也不会被框架硬杀。 + probes = [] if is_reference else _select_probes(instances) + n_runs = len(instances) + 2 * len(probes) + eff_budget, shrunk, fw_timeout = _fit_budget(time_budget, n_runs) + + # 确定性探针：跨规模选若干实例（小/中/大 + 一个生成实例，若有）各跑两次， + # 输出必须一致（候选不能只在最小实例上确定）。 + if not violations and not is_reference: + for probe in probes: + det_ok, det_note = check_determinism(py, prog, probe, eff_budget) + if not det_ok: + violations = [f"determinism check failed on {probe.name}: {det_note}"] + break + + per_instance: dict[str, Any] = {} + total = 0.0 + all_valid = True + for inst_path in instances: + if violations: + per_instance[inst_path.name] = {"status": "preflight_failed", + "reasons": violations} + continue + minutes, info = _run_one(prog, inst_path, eff_budget, py) + per_instance[inst_path.name] = info + if minutes <= 0 and info.get("status") != "ok": + all_valid = False + total += minutes + + score = total / len(instances) if instances else 0.0 + return { + "combined_score": round(score, 2), + "valid": 1.0 if all_valid and not violations else 0.0, + "per_instance": per_instance, + "mode": resolved_mode, + "official": resolved_mode == "official", + "reference": is_reference, + "preflight_bypassed": is_reference, + "num_instances": len(instances), + "num_fixed_instances": n_fixed, + "num_generated_instances": len(instances) - n_fixed, + "time_budget_s": time_budget, + "effective_time_budget_s": round(eff_budget, 2) if shrunk else time_budget, + "budget_shrunk": shrunk, + "framework_timeout_s": fw_timeout, + "generate_seed": gen_seed, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="区域备电评测") + parser.add_argument("solver", help="候选求解器脚本路径") + parser.add_argument("--time-budget", type=float, default=60.0, + help="求解时间预算（秒），默认 60") + parser.add_argument("--data-dir", type=str, default=None, + help="实例目录（默认 verification/data/instances）") + parser.add_argument("--generate-seed", type=int, default=None, + help="运行时生成实例的种子（防硬编码；等价于设 TELECOM_EVAL_GENERATE_SEED）") + parser.add_argument("--mode", choices=MODES, default=None, + help="official：仅用运行时生成实例（需 --generate-seed/环境变量）；" + "local（默认）：用固定实例") + parser.add_argument("--local", dest="mode", action="store_const", const="local", + help="等价于 --mode local（开发/冒烟测试）") + parser.add_argument("--reference", action="store_true", + help="跳过候选完整性检查，用于复现内置参考求解器的分数") + args = parser.parse_args(argv) + + if args.generate_seed is not None: + os.environ["TELECOM_EVAL_GENERATE_SEED"] = str(args.generate_seed) + try: + result = evaluate(args.solver, time_budget=args.time_budget, data_dir=args.data_dir, + mode=args.mode, reference=args.reference) + except OfficialModeConfigError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/generator.py b/benchmarks/PowerSystems/TelecomBackup/verification/generator.py new file mode 100644 index 00000000..b4d29fb8 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/generator.py @@ -0,0 +1,154 @@ +"""实例生成器：seed 固定，生成可复现的区域备电实例（JSON）。 + +参数语义： +- 区域 200m x 200m，栅格化为 nx*ny 栅格（默认 20x20 = 400）。 +- 站点围绕 4 个聚类中心高斯散布（城区基站分布），坐标裁剪到区域内。 +- 电源分组：对站点坐标做 K-Means（纯 Python），K = ceil(N/3.5)。 +- 电池电量：每个电源 uniform(15, 30) kWh。 +- 栅格需求：每个栅格 uniform(0, 1)（相对负载单位）。 +- 覆盖参数：pt_dbm=20, n_exp=6（阈值 -105dBm 下站点良好覆盖半径约 120m）。 +""" + +from __future__ import annotations + +import json +import math +import random +from pathlib import Path +from typing import Any + +GRID = {"width": 200.0, "height": 200.0, "nx": 20, "ny": 20} +CLUSTER_CENTERS = [(50.0, 50.0), (150.0, 50.0), (50.0, 150.0), (150.0, 150.0)] +SITE_SIGMA = 20.0 +BATTERY_RANGE = (15.0, 30.0) +P_SILENT = 0.05 # 静默功耗 (kW/站点)：便宜，休息保电收益大 +P_WORK_BASE = 3.0 # 工作功耗基值 (kW/站点)：昂贵，全开惩罚明显 +P_WORK_COEF = 3.0 # 工作功耗负载系数 (kW/负载比) +DEMAND_MAX = 1.0 +SITE_CAP = 60.0 +HORIZON = 96 +DELTA_MIN = 5.0 +PT_DBM = 20.0 +N_EXP = 6.0 +THRESHOLD = -105.0 +COVERAGE_RATIO = 0.8 + + +def _kmeans(points: list[tuple[float, float]], k: int, rng: random.Random, + iters: int = 20) -> list[list[int]]: + n = len(points) + assert n >= k, "not enough sites for k groups" + init = rng.sample(range(n), k) + centers = [points[i] for i in init] + assign: list[int] = [0] * n + for _ in range(iters): + for i, (x, y) in enumerate(points): + assign[i] = min( + range(k), key=lambda c: math.hypot(x - centers[c][0], y - centers[c][1]) + ) + new_centers = [] + for c in range(k): + members = [points[i] for i in range(n) if assign[i] == c] + if members: + new_centers.append( + (sum(p[0] for p in members) / len(members), + sum(p[1] for p in members) / len(members)) + ) + else: + new_centers.append(centers[c]) + centers = new_centers + groups: list[list[int]] = [[] for _ in range(k)] + for i, c in enumerate(assign): + groups[c].append(i) + return [g for g in groups if g] + + +def generate(seed: int, n_sites: int, n_clusters: int = 4) -> dict[str, Any]: + rng = random.Random(seed) + centers = rng.sample(CLUSTER_CENTERS, n_clusters) + sites: list[list[float]] = [] + for _ in range(n_sites): + cx, cy = rng.choice(centers) + x = min(195.0, max(5.0, cx + rng.gauss(0.0, SITE_SIGMA))) + y = min(195.0, max(5.0, cy + rng.gauss(0.0, SITE_SIGMA))) + sites.append([round(x, 1), round(y, 1)]) + + k = max(1, math.ceil(n_sites / 3.5)) + groups = _kmeans([(s[0], s[1]) for s in sites], k, rng=rng) + battery = [round(rng.uniform(*BATTERY_RANGE), 1) for _ in groups] + + nx, ny = GRID["nx"], GRID["ny"] + demand = [round(rng.uniform(0.0, DEMAND_MAX), 3) for _ in range(nx * ny)] + + inst: dict[str, Any] = { + "seed": seed, + "grid": GRID, + "sites": sites, + "groups": groups, + "battery": battery, + "demand": demand, + "pt_dbm": PT_DBM, + "n_exp": N_EXP, + "threshold": THRESHOLD, + "coverage_ratio": COVERAGE_RATIO, + "delta_min": DELTA_MIN, + "horizon": HORIZON, + "site_cap": SITE_CAP, + "p_silent": P_SILENT, + "p_work_base": P_WORK_BASE, + "p_work_coef": P_WORK_COEF, + } + return inst + + + + +def _stagger_ok(inst: dict[str, Any], min_gap: float = 0.25) -> bool: + """验收：错峰调度（同时休息多路）须比全程开启好 >= min_gap。 + + 全程开启电池并行耗尽；让多路电源同时休息（其余保持负载分散）能大幅保电 + 并延长总备电时长。实例几何/电池若不满足，则此实例对"调度"无意义，应重造。 + """ + from simulator import simulate + + horizon = int(inst["horizon"]) + k = len(inst["groups"]) + on_all = [[[0, horizon]] for _ in range(k)] + ao = simulate(inst, on_all) + best = ao + for n_rest in (1, 2, 3): + for r in (4, 8, 12, 16): + on = [[] for _ in range(k)] + slot = seg = 0 + while slot < horizon: + for i in range(k): + if (seg + i) % k >= n_rest: + on[i].append([slot, min(slot + r, horizon)]) + slot += r + seg += 1 + best = max(best, simulate(inst, on)) + return best >= ao * (1.0 + min_gap) + +def main() -> None: + out_dir = Path(__file__).resolve().parent / "data" / "instances" + out_dir.mkdir(parents=True, exist_ok=True) + specs = [ + (1, 20), (2, 24), (3, 28), (4, 32), + (5, 36), (6, 40), (7, 24), (8, 32), + ] + for idx, (seed, n_sites) in enumerate(specs): + inst = None + for trial in range(300): + s = seed + trial * 101 + cand = generate(s, n_sites) + if _stagger_ok(cand): + inst = cand + break + assert inst is not None, f"no acceptable instance for slot {idx}" + path = out_dir / f"instance_{seed}.json" + path.write_text(json.dumps(inst, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") + print(f"instance_{seed}.json N={n_sites} K={len(inst['groups'])} seed={inst['seed']} (stagger ok)") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/ref_solver.py b/benchmarks/PowerSystems/TelecomBackup/verification/ref_solver.py new file mode 100644 index 00000000..88b8e621 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/ref_solver.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""参考求解器：轮流休息一路的错峰调度（"好解"参考，供 agent 对比/超越）。 + +思路：全程开启会让所有电池并行耗尽；"轮流休息一路"（其余保持负载分散） +能让被休息的电源以低静默功耗保电，从而延长总备电时长。对 R（休息窗口） +做小范围搜索取最优。 + +用法：python verification/ref_solver.py +输出：stdout 打印 {"on": [[[a,b),...], ...]} +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +_here = Path(__file__).resolve().parent +if str(_here) not in sys.path: + sys.path.insert(0, str(_here)) +from simulator import simulate # noqa: E402 + + +def solve(inst: dict) -> list[list[list[int]]]: + horizon = int(inst["horizon"]) + k = len(inst["groups"]) + best_on = [[[0, horizon]] for _ in range(k)] + best_s = simulate(inst, best_on) + for n_rest in (1, 2, 3): + for r in (2, 4, 6, 8, 12, 16): + on = [[] for _ in range(k)] + slot = seg = 0 + while slot < horizon: + for i in range(k): + if (seg + i) % k >= n_rest: + on[i].append([slot, min(slot + r, horizon)]) + slot += r + seg += 1 + s = simulate(inst, on) + if s > best_s: + best_s, best_on = s, on + return best_on + + +def main() -> int: + if len(sys.argv) < 2: + print("usage: python ref_solver.py ", file=sys.stderr) + return 2 + inst_path = Path(sys.argv[1]) + inst = json.loads(inst_path.read_text(encoding="utf-8")) + print(json.dumps({"on": solve(inst)})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/requirements.txt b/benchmarks/PowerSystems/TelecomBackup/verification/requirements.txt new file mode 100644 index 00000000..4e66700e --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/requirements.txt @@ -0,0 +1,2 @@ +# The evaluator uses only the Python standard library. +# Runtime requirement: Python >= 3.10 (no third-party dependencies). diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/simulator.py b/benchmarks/PowerSystems/TelecomBackup/verification/simulator.py new file mode 100644 index 00000000..7c929cec --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/simulator.py @@ -0,0 +1,155 @@ +"""计分模拟器：验证候选电源开关策略，返回备电时长（分钟）。 + +规则（自编简化模型，物理语义见 Task.md / README）： +- 区域为 nx*ny 个栅格；N 个站点；K 个电源，每组站点共用一个电池。 +- 覆盖电平 P(g, s) = pt_dbm - 10*n_exp*log10(d(g,s)+1)（dBm，d 为米）。 +- 栅格接入"最强存活站点"；栅格"良好"当且仅当 max P(g,s) > threshold。 +- 站点功耗（由实例提供，默认见下）：工作 P_work = p_work_base + p_work_coef*min(load_s/site_cap, 1.0) kW；静默 p_silent kW。 + 实例自带校准（当前数据：p_silent=0.05, p_work_base=3.0, p_work_coef=3.0），保证错峰调度有明显收益。 +- 负载迁移：电源关闭 -> 站点静默（不提供覆盖，仍耗静默电）-> 栅格即时接入最强存活站点。 +- 电源 k 在时隙 t > close_time[k] 时关闭；电量 E_k 每时隙扣 dt*sum(P)，耗尽 -> 停服（不耗电不覆盖）。 +- 备电时长 = 首次覆盖比例 < coverage_ratio 的时隙的上一时刻（分钟）；撑满 horizon 则 T*delta_min。 +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +# 默认参数（与 generator.py 保持一致） +PT_DBM = 20.0 +N_EXP = 6.0 +THRESHOLD = -105.0 +COVERAGE_RATIO = 0.8 +DELTA_MIN = 5.0 +SITE_CAP = 60.0 + + +def load_instance(path: str | Path) -> dict[str, Any]: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def build_power(inst: dict[str, Any]) -> list[list[float]]: + """返回 (N, G) 的覆盖电平矩阵 P(s, g)（dBm）。""" + grid = inst["grid"] + nx, ny = grid["nx"], grid["ny"] + w, h = grid["width"], grid["height"] + pt = float(inst.get("pt_dbm", PT_DBM)) + n_exp = float(inst.get("n_exp", N_EXP)) + centers = [ + ((ix + 0.5) * w / nx, (iy + 0.5) * h / ny) for iy in range(ny) for ix in range(nx) + ] + rows: list[list[float]] = [] + for sx, sy in inst["sites"]: + row = [ + pt - 10.0 * n_exp * math.log10(math.hypot(sx - cx, sy - cy) + 1.0) + for cx, cy in centers + ] + rows.append(row) + return rows + + +def _assign(power: list[list[float]], served: list[int], demand: list[float], + threshold: float) -> tuple[list[float], int]: + """给每个栅格接入最强存活站点；返回 (每站点负载, 良好栅格数)。""" + n_sites = len(power) + n_cells = len(demand) + best_val = [-1e9] * n_cells + best_site = [-1] * n_cells + for s in served: + prow = power[s] + for g in range(n_cells): + v = prow[g] + if v > best_val[g]: + best_val[g] = v + best_site[g] = s + load = [0.0] * n_sites + good = 0 + for g in range(n_cells): + if best_val[g] > threshold: + good += 1 + s = best_site[g] + if s >= 0: + load[s] += demand[g] + return load, good + + +def _normalize_intervals(intervals: list[list[int]], horizon: int) -> list[list[int]]: + """合并重叠/相邻区间并排序，返回升序不交区间 [a, b)。""" + ivs = sorted((a, b) for a, b in intervals if a < b) + merged: list[list[int]] = [] + for a, b in ivs: + if merged and a <= merged[-1][1]: + merged[-1][1] = max(merged[-1][1], b) + else: + merged.append([a, b]) + return merged + + +def simulate(inst: dict[str, Any], on_intervals: list[list[list[int]]]) -> float: + """返回备电时长（分钟）。 + + on_intervals[k] = 电源 k 的开启时隙区间列表，如 [[0, 48], [60, 96]]， + 表示时隙 0..47 与 60..95 开启（时隙 0-index，半开区间 [a, b)）。 + on_intervals[k] = [[0, horizon]] 表示永不关闭；[] 表示一开始就关。 + """ + power = build_power(inst) + groups: list[list[int]] = inst["groups"] + demand: list[float] = inst["demand"] + # 功耗参数由实例提供（数据自带校准）：静默功耗、工作基值、工作负载系数。 + p_silent = float(inst.get("p_silent", 1.0)) + p_work_base = float(inst.get("p_work_base", 1.2)) + p_work_coef = float(inst.get("p_work_coef", 2.0)) + threshold = float(inst.get("threshold", THRESHOLD)) + coverage_ratio = float(inst.get("coverage_ratio", COVERAGE_RATIO)) + delta_min = float(inst.get("delta_min", DELTA_MIN)) + site_cap = float(inst.get("site_cap", SITE_CAP)) + horizon = int(inst["horizon"]) + n_cells = len(demand) + n_sites = len(inst["sites"]) + required_good = coverage_ratio * n_cells + dt_hour = delta_min / 60.0 + + # 预计算每时隙每电源开关状态（K x horizon 布尔） + on_states: list[list[bool]] = [] + for k in range(len(groups)): + ivs = _normalize_intervals(on_intervals[k], horizon) + row = [False] * horizon + for a, b in ivs: + for s in range(max(0, a), min(b, horizon)): + row[s] = True + on_states.append(row) + + def work_sites(slot: int, battery: list[float]) -> list[int]: + served: list[int] = [] + for k in range(len(groups)): + if on_states[k][slot] and battery[k] > 0: + served.extend(groups[k]) + return served + + battery = [float(e) for e in inst["battery"]] + for slot in range(horizon): + # 本时隙开始时的工作站点（扣电前） + served_now = work_sites(slot, battery) + load, _ = _assign(power, served_now, demand, threshold) + # 电量推进 + for k in range(len(groups)): + if battery[k] <= 0: + continue + e_k = 0.0 + for s in groups[k]: + if on_states[k][slot]: + load_ratio = min(load[s] / site_cap, 1.0) + e_k += p_work_base + p_work_coef * load_ratio # 工作功耗 + else: + e_k += p_silent # 静默功耗 + battery[k] -= dt_hour * e_k + # 扣电后覆盖检查 + served_after = work_sites(slot, battery) + _, good = _assign(power, served_after, demand, threshold) + if good < required_good: + return slot * delta_min + return horizon * delta_min diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/test_evaluator.py b/benchmarks/PowerSystems/TelecomBackup/verification/test_evaluator.py new file mode 100644 index 00000000..fd9bff6a --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/test_evaluator.py @@ -0,0 +1,184 @@ +"""Unit tests for verification/evaluate.py (stdlib unittest, no third-party deps). + +Run from the TelecomBackup task directory: + python verification/test_evaluator.py +""" +from __future__ import annotations + +import json +import os +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import evaluate as ev # noqa: E402 +from generator import generate # noqa: E402 + +TASK_ROOT = Path(__file__).resolve().parents[1] +BASELINE = TASK_ROOT / "baseline" / "solver.py" + +ALWAYS_ON = ( + "# EVOLVE-BLOCK-START\n" + "def solve(inst):\n" + " k = len(inst['groups']); T = inst['horizon']\n" + " return [[[0, T]] for _ in range(k)]\n" + "# EVOLVE-BLOCK-END\n" + "import json, sys\n" + "inst = json.load(open(sys.argv[1], encoding='utf-8'))\n" + "print(json.dumps({'on': solve(inst)}))\n" +) + + +class EvaluatorTestCase(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="telecom_eval_test_")) + self.inst_dir = self.tmp / "instances" + self.inst_dir.mkdir() + # 3 个固定实例（seed 派生），使评测稳定 + for i in range(3): + inst = generate(1000 + i, 20 + 8 * i) + (self.inst_dir / f"instance_{i}.json").write_text( + json.dumps(inst), encoding="utf-8" + ) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + for key in ( + "TELECOM_EVAL_MODE", + "TELECOM_EVAL_GENERATE_SEED", + "TELECOM_EVAL_GENERATE_COUNT", + "TELECOM_EVAL_REFERENCE", + "TELECOM_EVAL_TIME_BUDGET", + "FRONTIER_EVAL_EVALUATOR_TIMEOUT_S", + ): + os.environ.pop(key, None) + + def _write(self, name: str, src: str) -> Path: + p = self.tmp / name + p.write_text(src, encoding="utf-8") + return p + + def _run(self, solver: Path) -> dict: + return ev.evaluate(str(solver), time_budget=30.0, data_dir=self.inst_dir) + + def test_always_on_scores_positive_and_valid(self): + solver = self._write("always_on.py", ALWAYS_ON) + result = self._run(solver) + self.assertEqual(result["valid"], 1.0) + self.assertGreater(result["combined_score"], 0.0) + self.assertEqual(result["num_instances"], 3) + + def test_malformed_output_scores_zero(self): + solver = self._write("bad.py", "print('not json')\n") + result = self._run(solver) + self.assertEqual(result["combined_score"], 0.0) + self.assertEqual(result["valid"], 0.0) + + def test_timeout_scores_zero(self): + solver = self._write("slow.py", "import time; time.sleep(60)\n") + result = ev.evaluate(str(solver), time_budget=0.5, data_dir=self.inst_dir) + self.assertEqual(result["combined_score"], 0.0) + self.assertEqual(result["valid"], 0.0) + + def test_hardcoded_candidate_fails_preflight(self): + src = ALWAYS_ON.replace( + " return [[[0, T]] for _ in range(k)]", + " table = {'instance_0': [[[0, 48]]]}\n" + " return table.get('instance_0', [[[0, T]] for _ in range(k)])", + 1, + ) + solver = self._write("hardcode.py", src) + result = self._run(solver) + self.assertEqual(result["valid"], 0.0) + self.assertTrue( + any("preflight" in v.get("status", "") for v in result["per_instance"].values()) + ) + + def test_missing_program_returns_error(self): + result = self._run(self.tmp / "nope.py") + self.assertEqual(result["valid"], 0.0) + self.assertIn("error", result) + + def test_generate_seed_adds_instances(self): + os.environ["TELECOM_EVAL_GENERATE_SEED"] = "7" + os.environ["TELECOM_EVAL_GENERATE_COUNT"] = "4" + solver = self._write("always_on2.py", ALWAYS_ON) + result = self._run(solver) + self.assertEqual(result["num_instances"], 3 + 4) + self.assertEqual(result["generate_seed"], 7) + self.assertEqual(result["valid"], 1.0) + + def test_generated_instances_are_reproducible(self): + os.environ["TELECOM_EVAL_GENERATE_SEED"] = "7" + os.environ["TELECOM_EVAL_GENERATE_COUNT"] = "2" + solver = self._write("always_on3.py", ALWAYS_ON) + a = self._run(solver) + b = self._run(solver) + self.assertEqual(a["combined_score"], b["combined_score"]) + + # --- official mode: mandatory runtime generation (review blocker 1) --- + + def test_official_mode_requires_seed(self): + """official mode without a seed must fail loudly, not score public instances.""" + solver = self._write("always_on4.py", ALWAYS_ON) + with self.assertRaises(ev.OfficialModeConfigError): + ev.evaluate(str(solver), time_budget=10.0, mode="official", + data_dir=self.inst_dir) + + def test_official_mode_scores_only_generated_instances(self): + os.environ["TELECOM_EVAL_GENERATE_SEED"] = "42" + os.environ["TELECOM_EVAL_GENERATE_COUNT"] = "3" + solver = self._write("always_on5.py", ALWAYS_ON) + result = ev.evaluate(str(solver), time_budget=10.0, mode="official", + data_dir=self.inst_dir) + self.assertTrue(result["official"]) + self.assertEqual(result["num_fixed_instances"], 0) # public instances excluded + self.assertEqual(result["num_generated_instances"], 3) + self.assertEqual(result["num_instances"], 3) + self.assertEqual(result["generate_seed"], 42) + self.assertEqual(result["valid"], 1.0) + + def test_bad_mode_value_rejected(self): + solver = self._write("always_on6.py", ALWAYS_ON) + os.environ["TELECOM_EVAL_MODE"] = "offical" # typo + with self.assertRaises(ev.OfficialModeConfigError): + ev.evaluate(str(solver), time_budget=10.0) + + # --- reference scoring path (review blocker 2) --- + + def test_reference_bypasses_preflight(self): + """--reference scores ref_solver.py, which normal preflight rejects.""" + ref = Path(ev.__file__).resolve().parent / "ref_solver.py" + # generous budget: ref_solver does a real search, so a tight budget can + # flake to 0 under CPU contention + result = ev.evaluate(str(ref), time_budget=30.0, data_dir=self.inst_dir, + mode="local", reference=True) + self.assertTrue(result["preflight_bypassed"]) + self.assertEqual(result["valid"], 1.0) + self.assertGreater(result["combined_score"], 0.0) + + def test_reference_bypass_is_not_the_default(self): + ref = Path(ev.__file__).resolve().parent / "ref_solver.py" + result = ev.evaluate(str(ref), time_budget=10.0, data_dir=self.inst_dir, + mode="local") + self.assertFalse(result["preflight_bypassed"]) + self.assertEqual(result["valid"], 0.0) + + # --- per-instance budget self-limits to the framework timeout (review item 3) --- + + def test_budget_self_limits_to_framework_timeout(self): + os.environ["TELECOM_EVAL_GENERATE_SEED"] = "7" + os.environ["TELECOM_EVAL_GENERATE_COUNT"] = "1" + os.environ["FRONTIER_EVAL_EVALUATOR_TIMEOUT_S"] = "120" + solver = self._write("always_on7.py", ALWAYS_ON) + result = self._run(solver) + self.assertTrue(result["budget_shrunk"]) + self.assertLess(result["effective_time_budget_s"], 60.0) + self.assertEqual(result["framework_timeout_s"], 120.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/test_frontier_eval_evaluator.py b/benchmarks/PowerSystems/TelecomBackup/verification/test_frontier_eval_evaluator.py new file mode 100644 index 00000000..a14acb26 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/test_frontier_eval_evaluator.py @@ -0,0 +1,141 @@ +"""Sandbox-evaluator end-to-end tests (frontier_eval/evaluator.py). + +The unified runtime runs the eval from the sandbox: the sandbox evaluator +loads `verification/evaluate.py` (copied in), and the generation path +host-loads `generator.py` via FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR. +These tests lock that wiring so a refactor cannot silently break the sandbox +path (the review finding that flagged no end-to-end sandbox coverage). +""" +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +TASK_ROOT = Path(__file__).resolve().parents[1] +EVALUATOR = TASK_ROOT / "frontier_eval" / "evaluator.py" +BASELINE = TASK_ROOT / "baseline" / "solver.py" + +_spec = importlib.util.spec_from_file_location("_sandbox_eval", EVALUATOR) +_sb = importlib.util.module_from_spec(_spec) +assert _spec.loader is not None +_spec.loader.exec_module(_sb) + + +class TestSandboxBaseline(unittest.TestCase): + def test_baseline_scores(self): + m = _sb.evaluate(str(BASELINE))["metrics"] + self.assertEqual(m["valid"], 1.0) + self.assertEqual(m["num_instances"], 8) + self.assertGreater(m["combined_score"], 100.0) + + +class TestSandboxHostLoad(unittest.TestCase): + def test_generation_host_loads_generator(self): + # Sandbox view: the host benchmark dir is exposed via the env var. + os.environ["FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR"] = str(TASK_ROOT) + os.environ["TELECOM_EVAL_GENERATE_SEED"] = "7" + os.environ["TELECOM_EVAL_GENERATE_COUNT"] = "4" + try: + m = _sb.evaluate(str(BASELINE))["metrics"] + finally: + os.environ.pop("FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR", None) + os.environ.pop("TELECOM_EVAL_GENERATE_SEED", None) + os.environ.pop("TELECOM_EVAL_GENERATE_COUNT", None) + self.assertEqual(m["valid"], 1.0) + self.assertGreaterEqual(m["num_instances"], 12) # 8 fixed + generated + + +class TestSandboxOfficialMode(unittest.TestCase): + """official mode: mandatory generation, public instances excluded.""" + + def _clear(self): + for key in ( + "TELECOM_EVAL_MODE", + "TELECOM_EVAL_GENERATE_SEED", + "TELECOM_EVAL_GENERATE_COUNT", + "TELECOM_EVAL_TIME_BUDGET", + "FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR", + ): + os.environ.pop(key, None) + + def test_official_mode_requires_seed(self): + self._clear() + os.environ["TELECOM_EVAL_MODE"] = "official" + try: + with self.assertRaises(RuntimeError) as ctx: + _sb.evaluate(str(BASELINE)) + finally: + self._clear() + self.assertIn("TELECOM_EVAL_GENERATE_SEED", str(ctx.exception)) + + def test_official_mode_excludes_public_instances(self): + self._clear() + os.environ["TELECOM_EVAL_MODE"] = "official" + os.environ["TELECOM_EVAL_GENERATE_SEED"] = "5" + os.environ["TELECOM_EVAL_GENERATE_COUNT"] = "4" + os.environ["FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR"] = str(TASK_ROOT) + try: + m = _sb.evaluate(str(BASELINE))["metrics"] + finally: + self._clear() + self.assertEqual(m["num_fixed_instances"], 0) + self.assertEqual(m["num_generated_instances"], 4) + self.assertEqual(m["valid"], 1.0) + + def test_time_budget_env_is_respected(self): + self._clear() + os.environ["TELECOM_EVAL_TIME_BUDGET"] = "10" + try: + m = _sb.evaluate(str(BASELINE))["metrics"] + finally: + self._clear() + self.assertEqual(m["time_budget_s"], 10.0) + + +class TestEvalCommandSelectsOfficialMode(unittest.TestCase): + """The framework reads eval_command.txt; it must select official mode. + + Without this, dropping the prefix would silently revert the framework path + to local (public-instance) scoring -- the exact hole review blocker 1 closed. + """ + + def test_eval_command_file_selects_official_mode(self): + cmd = (TASK_ROOT / "frontier_eval" / "eval_command.txt").read_text(encoding="utf-8") + self.assertIn("TELECOM_EVAL_MODE=official", cmd) + + +class TestSandboxRejectsRefSolver(unittest.TestCase): + def test_ref_solver_cheat_rejected(self): + src = BASELINE.read_text(encoding="utf-8") + start = src.find("# EVOLVE-BLOCK-START") + end = src.find("# EVOLVE-BLOCK-END") + injected = ( + src[: start + len("# EVOLVE-BLOCK-START")] + + "\n from verification.ref_solver import solve\n return solve(inst)\n" + + src[end:] + ) + tmp = Path(tempfile.mkdtemp(prefix="tb_cheat_")) + cand = tmp / "cheat.py" + cand.write_text(injected, encoding="utf-8") + try: + m = _sb.evaluate(str(cand))["metrics"] + finally: + shutil.rmtree(tmp, ignore_errors=True) + self.assertEqual(m["valid"], 0.0) + self.assertEqual(m["combined_score"], 0.0) + # rejected by the static check (not merely by a runtime crash): the + # reported reason must name the forbidden reference token + reasons = [r for info in m["per_instance"].values() + for r in info.get("reasons", [])] + self.assertTrue(any("ref_solver" in r for r in reasons), + f"expected a ref_solver preflight reason, got {reasons!r}") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/test_simulator.py b/benchmarks/PowerSystems/TelecomBackup/verification/test_simulator.py new file mode 100644 index 00000000..abfa8b76 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/test_simulator.py @@ -0,0 +1,110 @@ +"""Unit tests for verification/simulator.py (stdlib unittest, no third-party deps). + +Run from the TelecomBackup task directory: + python verification/test_simulator.py +""" +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import simulator as sim # noqa: E402 + + +def _tiny() -> dict: + """1 站 1 电源 1 栅格：手算用例。""" + return { + "grid": {"width": 200.0, "height": 200.0, "nx": 1, "ny": 1}, + "sites": [[100.0, 100.0]], + "groups": [[0]], + "battery": [10.0], + "demand": [0.5], + "pt_dbm": 20.0, "n_exp": 6.0, "threshold": -105.0, + "coverage_ratio": 0.8, "delta_min": 5.0, "horizon": 8, "site_cap": 60.0, + } + + +class TestTinyManual(unittest.TestCase): + def setUp(self): + self.inst = _tiny() + + def test_always_on(self): + # 工作功耗 1.2+2*min(0.5/60,1)=1.2167kW，8 时隙耗电远小于 10kWh -> 撑满 40min + self.assertEqual(sim.simulate(self.inst, [[[0, 8]]]), 40.0) + + def test_never_on(self): + # 一开始就关 -> t=0 覆盖 0 < 80% -> 0 分钟 + self.assertEqual(sim.simulate(self.inst, [[]]), 0.0) + + def test_half_horizon(self): + # 前 4 时隙开、后关 -> 第 5 时隙跌破 -> 20 分钟 + self.assertEqual(sim.simulate(self.inst, [[[0, 4]]]), 20.0) + + +class TestNormalizeIntervals(unittest.TestCase): + def test_merge_overlap(self): + self.assertEqual(sim._normalize_intervals([[0, 10], [5, 15]], 96), [[0, 15]]) + + def test_merge_adjacent(self): + self.assertEqual(sim._normalize_intervals([[0, 5], [5, 10]], 96), [[0, 10]]) + + def test_empty_interval_list(self): + self.assertEqual(sim._normalize_intervals([], 96), []) + + def test_out_of_order(self): + # 乱序的合法区间：排序后升序返回 + self.assertEqual(sim._normalize_intervals([[3, 8], [0, 2]], 96), [[0, 2], [3, 8]]) + + def test_invalid_interval_filtered(self): + # a >= b 的畸形区间被忽略 + self.assertEqual(sim._normalize_intervals([[8, 3], [0, 2]], 96), [[0, 2]]) + + +class TestBatteryDepletion(unittest.TestCase): + def test_tiny_battery_dies(self): + inst = _tiny() + inst["battery"] = [0.05] # 1 个时隙都撑不过 + # 全开：第 1 时隙扣电后耗尽停服 -> 覆盖掉到 0 -> 该时隙跌破 -> 0 分钟 + self.assertEqual(sim.simulate(inst, [[[0, 8]]]), 0.0) + + def test_multi_power_some_dead(self): + inst = { + "grid": {"width": 200.0, "height": 200.0, "nx": 2, "ny": 2}, + "sites": [[50, 50], [150, 150]], + "groups": [[0], [1]], + "battery": [0.05, 20.0], + "demand": [0.5] * 4, + "pt_dbm": 20.0, "n_exp": 6.0, "threshold": -105.0, + "coverage_ratio": 0.4, "delta_min": 5.0, "horizon": 10, "site_cap": 60.0, + } + # 电源 0 立即耗尽停服，电源 1 继续服务；覆盖要求 40% 很容易满足 -> 撑满 + self.assertEqual(sim.simulate(inst, [[[0, 10]], [[0, 10]]]), 50.0) + + +class TestDeterminism(unittest.TestCase): + def test_same_input_same_output(self): + inst = _tiny() + a = sim.simulate(inst, [[[0, 8]]]) + b = sim.simulate(inst, [[[0, 8]]]) + self.assertEqual(a, b) + + +class TestCoverageConstraint(unittest.TestCase): + def test_coverage_falls_below_ratio_ends_early(self): + inst = _tiny() + inst["coverage_ratio"] = 0.9 # 单站时覆盖不足，需两站才满足 + inst["sites"] = [[50, 50], [150, 150]] + inst["groups"] = [[0], [1]] + inst["battery"] = [20.0, 20.0] + inst["demand"] = [0.5] * 4 # 与 2x2 栅格匹配 + inst["grid"] = {"width": 200.0, "height": 200.0, "nx": 2, "ny": 2} + # 只开电源 0：(150,150) 栅格无良好覆盖 -> 良好 3/4=75% < 90% -> t=0 跌破 -> 0 分钟 + self.assertEqual(sim.simulate(inst, [[[0, 8]], []]), 0.0) + # 两个都开 -> 覆盖 4/4=100% >= 90% -> 撑满 8 时隙（horizon=8）= 40 分钟 + self.assertEqual(sim.simulate(inst, [[[0, 8]], [[0, 8]]]), 40.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/test_validator.py b/benchmarks/PowerSystems/TelecomBackup/verification/test_validator.py new file mode 100644 index 00000000..042cfe97 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/test_validator.py @@ -0,0 +1,170 @@ +"""Unit tests for verification/validator.py (stdlib unittest, no third-party deps). + +Run from the TelecomBackup task directory: + python verification/test_validator.py +""" +from __future__ import annotations + +import json +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import validator as vd # noqa: E402 + +TASK_ROOT = Path(__file__).resolve().parents[1] +BASELINE = TASK_ROOT / "baseline" / "solver.py" +BASELINE_SRC = BASELINE.read_text(encoding="utf-8") + + +class TestStaticChecks(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="telecom_val_")) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _check(self, src: str) -> list[str]: + path = self.tmp / "candidate.py" + path.write_text(src, encoding="utf-8") + return vd.check_candidate(path, baseline_path=BASELINE) + + def test_baseline_is_clean(self): + self.assertEqual(vd.check_candidate(BASELINE, baseline_path=BASELINE), []) + + def test_missing_markers(self): + src = BASELINE_SRC.replace(vd.EVOLVE_START, "").replace(vd.EVOLVE_END, "") + issues = self._check(src) + self.assertTrue(any("EVOLVE-BLOCK" in i for i in issues)) + + def test_outside_block_changed(self): + src = BASELINE_SRC.replace('"""区域备电 baseline', '"""HACKED baseline', 1) + issues = self._check(src) + self.assertTrue(any("outside EVOLVE-BLOCK" in i for i in issues)) + + def test_hardcode_by_instance_name(self): + src = BASELINE_SRC.replace( + 'return [[[0, horizon]] for _ in range(k)]', + 'table = {"instance_1": [[[0, 48]]]}\n return table', + 1, + ) + issues = self._check(src) + self.assertTrue(any("hardcodes" in i for i in issues)) + + def test_import_generator(self): + src = BASELINE_SRC.replace( + 'return [[[0, horizon]] for _ in range(k)]', + "from generator import generate\n return []", + 1, + ) + issues = self._check(src) + self.assertTrue(any("forbidden" in i for i in issues)) + + def test_import_verification_evaluate_dotted(self): + src = BASELINE_SRC.replace( + 'return [[[0, horizon]] for _ in range(k)]', + "import verification.evaluate\n return []", + 1, + ) + issues = self._check(src) + self.assertTrue(any("forbidden" in i for i in issues)) + + def test_import_ref_solver_rejected(self): + # P0：候选不得白嫖参考求解器（ref_solver）。 + src = BASELINE_SRC.replace( + 'return [[[0, horizon]] for _ in range(k)]', + "from verification.ref_solver import solve\n return solve(inst)", + 1, + ) + issues = self._check(src) + self.assertTrue(any("forbidden" in i or "ref_solver" in i for i in issues)) + + def test_absolute_path(self): + src = BASELINE_SRC.replace( + 'return [[[0, horizon]] for _ in range(k)]', + 'p = "C:\\\\Users\\\\secret"\n return []', + 1, + ) + issues = self._check(src) + self.assertTrue(any("absolute" in i for i in issues)) + + def test_whitebox_simulator_import_is_allowed(self): + # 候选被允许用白盒计分器做内部搜索（Task.md 承诺的能力） + src = BASELINE_SRC.replace( + 'return [[[0, horizon]] for _ in range(k)]', + "from simulator import simulate\n return [[[0, horizon]] for _ in range(k)]", + 1, + ) + self.assertEqual(self._check(src), []) + + +class TestCandidateEnv(unittest.TestCase): + def test_strips_frontier_and_telecom_vars(self): + import os + + old = dict(os.environ) + os.environ["FRONTIER_ENGINEERING_ROOT"] = "C:\\repo" + os.environ["FRONTIER_EVAL_UNIFIED_PYTHON"] = "C:\\python" + os.environ["TELECOM_EVAL_GENERATE_SEED"] = "42" + os.environ["KEEP_ME"] = "x" + try: + env = vd.candidate_env() + finally: + os.environ.clear() + os.environ.update(old) + self.assertNotIn("FRONTIER_ENGINEERING_ROOT", env) + self.assertNotIn("FRONTIER_EVAL_UNIFIED_PYTHON", env) + self.assertNotIn("TELECOM_EVAL_GENERATE_SEED", env) + self.assertEqual(env.get("KEEP_ME"), "x") + + +class TestDeterminism(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="telecom_det_")) + self.inst = self.tmp / "inst.json" + self.inst.write_text(json.dumps({ + "grid": {"width": 200.0, "height": 200.0, "nx": 2, "ny": 2}, + "sites": [[50, 50], [150, 150]], + "groups": [[0], [1]], + "battery": [12.0, 8.0], + "demand": [0.5] * 4, + "pt_dbm": 20.0, "n_exp": 6.0, "threshold": -105.0, + "coverage_ratio": 0.4, "delta_min": 5.0, "horizon": 10, "site_cap": 60.0, + }), encoding="utf-8") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write(self, name: str, src: str) -> Path: + p = self.tmp / name + p.write_text(src, encoding="utf-8") + return p + + def test_deterministic_candidate_passes(self): + solver = self._write("det.py", ( + "import json, sys\n" + "inst = json.load(open(sys.argv[1], encoding='utf-8'))\n" + "k = len(inst['groups']); T = inst['horizon']\n" + "print(json.dumps({'on': [[[0, T]] for _ in range(k)]}))\n" + )) + ok, note = vd.check_determinism(sys.executable, solver, self.inst, 30.0) + self.assertTrue(ok, note) + + def test_random_candidate_fails(self): + solver = self._write("rand.py", ( + "import json, random, sys\n" + "inst = json.load(open(sys.argv[1], encoding='utf-8'))\n" + "k = len(inst['groups']); T = inst['horizon']\n" + "iv = [[0, random.randint(0, T)] for _ in range(1)]\n" + "print(json.dumps({'on': [iv for _ in range(k)]}))\n" + )) + ok, note = vd.check_determinism(sys.executable, solver, self.inst, 30.0) + self.assertFalse(ok) + self.assertIn("not deterministic", note) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/PowerSystems/TelecomBackup/verification/validator.py b/benchmarks/PowerSystems/TelecomBackup/verification/validator.py new file mode 100644 index 00000000..093dd384 --- /dev/null +++ b/benchmarks/PowerSystems/TelecomBackup/verification/validator.py @@ -0,0 +1,159 @@ +"""TelecomBackup 候选完整性校验（借鉴 CVRP validator，简化版）。 + +在评分前以"可执行检查"强制任务约束（不只是自然语言）： + 1. EVOLVE-BLOCK 完整性：标记必须存在；标记外的代码必须与初始 baseline 逐字节一致。 + 2. 禁引用：候选不得 import verification 的评测/生成模块（evaluate/generator）， + 不得出现绝对路径，不得按实例名硬编码调度。 + 3. 确定性：同一实例跑两次必须输出一致（调度器需要可复现）。 + 4. candidate_env：候选子进程环境剥离 FRONTIER_*/TELECOM_EVAL_* 变量， + 封死"通过宿主环境变量定位评测基线"的侧信道。 + +候选允许 import `verification/simulator.py`（白盒计分器，任务有意暴露）， +但不允许 import 评测/生成逻辑。纯标准库。 +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import tempfile +from pathlib import Path +from typing import Any + +EVOLVE_START = "EVOLVE-BLOCK-START" +EVOLVE_END = "EVOLVE-BLOCK-END" + +# 禁引用 token：强 token 子串匹配即可（正常求解器几乎不会出现）。 +STRONG_TOKENS = ( + "generate_instance", + "TELECOM_EVAL_", + "from generator", + "import generator", + "ref_solver", +) +# "verification" 只在与评测/生成模块构成 import/路径上下文时禁止 +# （候选被允许 import verification/simulator.py）。 +FORBIDDEN_RE = ( + re.compile(r"verification[\\/](?:evaluate|generator)"), + re.compile(r"verification\s*\.\s*(?:evaluate|generator)\b"), + re.compile(r"\b(?:from|import)\s+(?:evaluate|generator)\b"), +) +# Windows 盘符 / POSIX 家目录绝对路径。 +ABS_PATH_RE = re.compile(r"[A-Za-z]:[\\/]|/home/|/Users/") +# 按实例名硬编码调度，如 "instance_1": [...] 或 "gen_42_1": [...] +HARDCODE_RE = re.compile(r"[\"'](?:instance|gen)_\d+(?:_\d+)?[\"']\s*:") + + +def split_evolve_blocks(src: str) -> tuple[str, str, str] | None: + start = src.find(EVOLVE_START) + end = src.find(EVOLVE_END) + if start == -1 or end == -1 or end <= start: + return None + return ( + src[:start], + src[start + len(EVOLVE_START) : end], + src[end + len(EVOLVE_END) :], + ) + + +def fixed_region(parts: tuple[str, str, str]) -> str: + """EVOLVE-BLOCK 之外的只读部分。 + + 行尾归一化：CRLF/LF 差异与结尾缺换行（保存/复制时的常见无损差异）不计为 + "修改了固定区"——固定区是否被改看内容，不看行尾。 + """ + return (parts[0] + parts[2]).replace("\r\n", "\n").rstrip("\n") + + +def static_check_source(src: str, baseline_src: str | None = None) -> list[str]: + issues: list[str] = [] + parts = split_evolve_blocks(src) + if parts is None: + issues.append("missing EVOLVE-BLOCK-START / EVOLVE-BLOCK-END markers") + elif baseline_src is not None: + init_parts = split_evolve_blocks(baseline_src) + if init_parts is not None and fixed_region(init_parts) != fixed_region(parts): + issues.append("code outside EVOLVE-BLOCK differs from initial baseline") + + for token in STRONG_TOKENS: + if token in src: + issues.append(f"candidate references forbidden token {token!r}") + for pat in FORBIDDEN_RE: + if pat.search(src): + issues.append("candidate references forbidden evaluation/generation module") + + if ABS_PATH_RE.search(src): + issues.append("candidate contains an absolute filesystem path") + if HARDCODE_RE.search(src): + issues.append("candidate hardcodes per-instance schedules by name") + + return issues + + +def check_candidate( + solver_path: Path | str, baseline_path: Path | str | None = None +) -> list[str]: + solver_path = Path(solver_path) + try: + src = solver_path.read_text(encoding="utf-8", errors="replace") + except Exception as exc: + return [f"cannot read candidate source: {exc}"] + + baseline_src = None + if baseline_path is not None: + try: + baseline_src = Path(baseline_path).read_text( + encoding="utf-8", errors="replace" + ) + except Exception: + baseline_src = None + return static_check_source(src, baseline_src) + + +def candidate_env() -> dict[str, str]: + """候选子进程环境：剥离宿主路径与评测相关变量。""" + env = os.environ.copy() + for key in list(env): + upper = key.upper() + if upper.startswith("FRONTIER") or upper.startswith("TELECOM_EVAL"): + del env[key] + return env + + +def check_determinism( + python: str, + solver_path: Path | str, + inst_path: Path | str, + timeout: float, +) -> tuple[bool, str]: + """同一实例跑两次，输出必须字节一致。""" + solver_path = Path(solver_path) + inst_path = Path(inst_path) + outputs: list[Any] = [] + for _ in range(2): + try: + proc = subprocess.run( + [python, str(solver_path), str(inst_path)], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout, + cwd=str(solver_path.parent), + env=candidate_env(), + ) + except subprocess.TimeoutExpired: + return False, "timeout during determinism check" + except Exception as exc: + return False, f"error during determinism check: {exc}" + if proc.returncode != 0: + return False, f"candidate exited with code {proc.returncode}" + try: + outputs.append(json.loads(proc.stdout)) + except Exception as exc: + return False, f"cannot parse determinism output: {exc}" + if outputs[0] != outputs[1]: + return False, "candidate is not deterministic (output differs across two runs)" + return True, ""