diff --git a/edgebench/tasks/vibrating_path_graph_coloring/NOTES.md b/edgebench/tasks/vibrating_path_graph_coloring/NOTES.md index 2f7adf31..e55aeebe 100644 --- a/edgebench/tasks/vibrating_path_graph_coloring/NOTES.md +++ b/edgebench/tasks/vibrating_path_graph_coloring/NOTES.md @@ -1,7 +1,7 @@ # vibrating_path_graph_coloring CodeChef VBR "Vibrating Path", in ALE-Bench style. The agent writes `solution.py`. The judge runs -`python3 solution.py` on 50 fixed cases (8 at a time, 30 s wall clock each), scores each output +`python3 solution.py` on 50 fixed cases (`min(8, SFORGE_CPUS)` at a time, 30 s wall clock each), scores each output with `tools/bin/tester` (the sum of the costs of the hit vertices) and sums the per-case scores. Any case that is not `OK` makes the total `inf`. Raw score = that sum (minimize). The rescale is the v1 one (`piecewise_log_min`). @@ -42,9 +42,22 @@ Deterministic submissions match exactly and equal the historical v1 scores. The searches differ by less than the old judge's own run-to-run spread; historical 2 is very load-sensitive, but old and new run side by side agree within 6%. +## Changes + +### 1.1.0 + +The judge ran 8 cases at once within the official 4 CPUs, so a case got about 15 s of CPU within its +30 s wall-clock limit. The pool is now `min(8, SFORGE_CPUS, cases)`, defaulting to 4. Fewer +submissions time out and become `inf`. Deadline variants of one solver timed out less often, but too +few old grades were valid to establish score stability. Grading takes longer: a valid solver using +28 s of CPU per case took 525-624 s, so `timeout_sec` goes from 600 to 1200 (at 4 workers, nominal +solver and tester budgets total 13 rounds x 90 s = 1170 s, excluding overhead). Score anchors are +unchanged; regrade before comparing with v1 leaderboard entries. +Source, Deviations and Regression summary describe 1.0.0. Issue: https://github.com/ByteDance-Seed/EdgeBench/issues/40 + ## Known issues -Kept as in v1, so scores stay comparable. Fixing any of them changes scores and bumps `version`. +Not fixed. Fixing any of them changes scores and bumps `version`. 1. **The scored cases are reproducible.** They come from the public generator with a fixed seed range; the instruction says to use seeds 0..10000 only, but nothing enforces it, so an agent diff --git a/edgebench/tasks/vibrating_path_graph_coloring/judge/Dockerfile b/edgebench/tasks/vibrating_path_graph_coloring/judge/Dockerfile index b9570ed2..86c5fa3d 100644 --- a/edgebench/tasks/vibrating_path_graph_coloring/judge/Dockerfile +++ b/edgebench/tasks/vibrating_path_graph_coloring/judge/Dockerfile @@ -14,7 +14,7 @@ # 8 heredoc > /solution.py (baseline) -> COPY environment/workspace/solution.py (byte-identical; # v1 grades it when the submission has no solution.py) # 9 rm -rf .git && git init && ... commit -m 'init' -> same, v1's commit dates pinned (the judge never reads it) -# 10 heredoc > /tmp/eval_vibrating_path.py; chmod +x -> COPY judge/eval_vibrating_path.py /judge/ (byte-identical) +# 10 heredoc > /tmp/eval_vibrating_path.py; chmod +x -> COPY judge/eval_vibrating_path.py /judge/ (v1's, except the pool size) # v1 eval_cmd + parser score_sum -> judge/grade.py (task.toml [judge].cmd) # Base = bases/python310 (sforge build passes it as BASE_IMAGE). diff --git a/edgebench/tasks/vibrating_path_graph_coloring/judge/eval_vibrating_path.py b/edgebench/tasks/vibrating_path_graph_coloring/judge/eval_vibrating_path.py index 58b05206..5374def4 100644 --- a/edgebench/tasks/vibrating_path_graph_coloring/judge/eval_vibrating_path.py +++ b/edgebench/tasks/vibrating_path_graph_coloring/judge/eval_vibrating_path.py @@ -42,7 +42,7 @@ def run_case(args): os.chdir(WORK_DIR) case_files = sorted(f for f in glob.glob(os.path.join(CASES_DIR, "*")) if os.path.isfile(f)) results = {} - with ProcessPoolExecutor(max_workers=min(8, len(case_files))) as pool: + with ProcessPoolExecutor(max_workers=min(8, int(os.environ.get("SFORGE_CPUS", 4)), len(case_files))) as pool: futures = {pool.submit(run_case, (i, cf)): i for i, cf in enumerate(case_files)} for fut in as_completed(futures): cid, status, score = fut.result() diff --git a/edgebench/tasks/vibrating_path_graph_coloring/judge/grade.py b/edgebench/tasks/vibrating_path_graph_coloring/judge/grade.py index 0aacda8a..138f9094 100644 --- a/edgebench/tasks/vibrating_path_graph_coloring/judge/grade.py +++ b/edgebench/tasks/vibrating_path_graph_coloring/judge/grade.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 """Grader for vibrating_path_graph_coloring (Verified judge contract, see SCHEMA.md). -Reproduces v1 exactly: +Reproduces v1, except the case pool size (1.1.0): v1 eval_cmd : cd /home/workspace/vibrating_path && python3 /tmp/eval_vibrating_path.py v1 parser : score_sum (sforge/harness/log_parsers/score_sum.py) v1 score : first `TOTAL_SCORE ` in the eval output (sforge/harness/grading.py:extract_score) -/judge/eval_vibrating_path.py is byte-identical to the v1 /tmp/eval_vibrating_path.py. +/judge/eval_vibrating_path.py is the v1 /tmp/eval_vibrating_path.py, except the pool size. This wrapper only runs it and translates its stdout into /opt/sforge/result.json. stdout = logs only. """ diff --git a/edgebench/tasks/vibrating_path_graph_coloring/task.toml b/edgebench/tasks/vibrating_path_graph_coloring/task.toml index f5215ecf..991229a2 100644 --- a/edgebench/tasks/vibrating_path_graph_coloring/task.toml +++ b/edgebench/tasks/vibrating_path_graph_coloring/task.toml @@ -4,7 +4,7 @@ name = "Vibrating Path Graph Coloring" category = "Combinatorial Optimization" language = "Python" description = "Color graph vertices and selectively remove edges to minimize a cost that penalizes both removed edges and monochromatic surviving edges." -version = "1.0.0" +version = "1.1.0" status = "migrated" [environment] @@ -22,11 +22,11 @@ exclude = [] [judge] cmd = "python3 /judge/grade.py" -timeout_sec = 600 +timeout_sec = 1200 cpus = 4 memory = "8g" # Official leaderboard limits (v1 kept them only in the experiment YAMLs). -# eval_vibrating_path.py runs the 50 cases 8 at a time, each `python3 solution.py` under a 30 s +# eval_vibrating_path.py runs the 50 cases min(8, SFORGE_CPUS) at a time, each `python3 solution.py` under a 30 s # wall-clock timeout (tester 60 s). Scores of time-limited (anytime) solutions depend on host CPU # speed and load; see NOTES.md.