From 9ec970bce3ab89a84f497cdf73a2d4e94cdc4c91 Mon Sep 17 00:00:00 2001 From: findalexli Date: Tue, 3 Nov 2026 09:04:36 -0800 Subject: [PATCH 1/7] apple_incremental_game: size the case pool by SFORGE_CPUS --- .../tasks/apple_incremental_game/NOTES.md | 20 +++++++++++++++++-- .../judge/eval_apple_incremental_game.py | 2 +- .../apple_incremental_game/judge/grade.py | 4 ++-- .../tasks/apple_incremental_game/task.toml | 6 +++--- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/edgebench/tasks/apple_incremental_game/NOTES.md b/edgebench/tasks/apple_incremental_game/NOTES.md index b31adfcc..aad8b8d5 100644 --- a/edgebench/tasks/apple_incremental_game/NOTES.md +++ b/edgebench/tasks/apple_incremental_game/NOTES.md @@ -2,7 +2,7 @@ AtCoder Heuristic Contest 058 "Apple Incremental Game", in ALE-Bench style. The agent writes `solution.py` (stdin -> stdout): 500 turns, each either strengthening one of the 4×10 machines or -doing nothing. The judge runs it on 50 fixed cases (8 at a time, 30 s wall clock each), scores each +doing nothing. The judge runs it on 50 fixed cases (`min(8, SFORGE_CPUS)` at a time, 30 s wall clock each), scores each output with `tools/bin/tester` (`round(1e5 · log2(S))`, S = apples after T turns), and sums the per-case scores of the `OK` cases (failed cases count 0). Raw score = that sum (maximize). The rescale is the v1 one (`piecewise_max`, baseline 45000000). @@ -13,7 +13,8 @@ rescale is the v1 one (`piecewise_max`, baseline 45000000). authors' repository (not public), identical by sha256 and mode to the published v1 images. The tools are Python scripts. - `environment/workspace/solution.py` (starter) and `judge/eval_apple_incremental_game.py`: - byte-identical to v1's inline scripts and to the published v1 images. `judge/grade.py` is new and + byte-identical to v1's inline scripts and to the published v1 images (the eval script until 1.1.0). + `judge/grade.py` is new and writes `result.json` with v1's scoring. - `instruction.md`: v1's prompt, verbatim. Base: `bases/python310`. @@ -47,6 +48,21 @@ timeouts depending on host load, on both images. Historical 3 uses unseeded rand extra pairs of runs, both images give overlapping ranges with means 0.05 M apart. The other time-limited searches differ by under 1 point on 0..100. +## Changes + +### 1.1.0 + +The judge ran 8 cases at once on the official 4 CPUs, against SCHEMA's rule that a parallel grader +sizes its pool with `SFORGE_CPUS`, so a case that searches until its own deadline got about half +the CPU that 4 cases at once give it. The pool is now `min(8, SFORGE_CPUS, cases)`; `SFORGE_CPUS` +must be a positive integer, and 4 applies only when it is absent. Deterministic solutions that +produce the same output and finish on both judges keep their scores. The two deadline-based files +measured gained 3.32-5.48 of 100 across eight matched pairs with 4 vCPUs online and one judge at a +time; other resource conditions gave other gains, and there is no mapping from old scores. The +longest measured grade at 4 CPUs is 390.8 s (every solver timing out), so `timeout_sec` stays 600 +for 4 CPUs; a smaller `sforge eval --judge-cpu-limit` needs a larger timeout. Known issue 3 stays: limits are +still wall clock. Regression summary describes 1.0.0. + ## Known issues Kept as in v1, so scores stay comparable. Fixing any of them changes scores and bumps `version`. diff --git a/edgebench/tasks/apple_incremental_game/judge/eval_apple_incremental_game.py b/edgebench/tasks/apple_incremental_game/judge/eval_apple_incremental_game.py index ed36e1b4..0dacb41a 100755 --- a/edgebench/tasks/apple_incremental_game/judge/eval_apple_incremental_game.py +++ b/edgebench/tasks/apple_incremental_game/judge/eval_apple_incremental_game.py @@ -42,7 +42,7 @@ def run_case(args): os.chdir(WORK_DIR) case_files = sorted(f for f in glob.glob(os.path.join(CASES_DIR, "*")) if os.path.isfile(f)) results = {} - with ProcessPoolExecutor(max_workers=min(8, len(case_files))) as pool: + with ProcessPoolExecutor(max_workers=min(8, int(os.environ.get("SFORGE_CPUS", 4)), len(case_files))) as pool: futures = {pool.submit(run_case, (i, cf)): i for i, cf in enumerate(case_files)} for fut in as_completed(futures): cid, status, score = fut.result() diff --git a/edgebench/tasks/apple_incremental_game/judge/grade.py b/edgebench/tasks/apple_incremental_game/judge/grade.py index 5e58e2fb..e79d0c58 100755 --- a/edgebench/tasks/apple_incremental_game/judge/grade.py +++ b/edgebench/tasks/apple_incremental_game/judge/grade.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 """Grader for apple_incremental_game (Verified judge contract, see SCHEMA.md). -Reproduces v1 exactly: +Keeps v1's command, parser and score: v1 eval_cmd : cd /home/workspace/apple_incremental_game && python3 /tmp/eval_apple_incremental_game.py v1 parser : score_sum (sforge/harness/log_parsers/score_sum.py) v1 score : first `TOTAL_SCORE ` in the eval output (sforge/harness/grading.py:extract_score) -/judge/eval_apple_incremental_game.py is byte-identical to the v1 /tmp/eval_apple_incremental_game.py. +/judge/eval_apple_incremental_game.py is the v1 /tmp/eval_apple_incremental_game.py except for its pool size (1.1.0). This wrapper only runs it and translates its stdout into /opt/sforge/result.json. stdout = logs only. """ diff --git a/edgebench/tasks/apple_incremental_game/task.toml b/edgebench/tasks/apple_incremental_game/task.toml index 76d65fe1..89210801 100644 --- a/edgebench/tasks/apple_incremental_game/task.toml +++ b/edgebench/tasks/apple_incremental_game/task.toml @@ -4,7 +4,7 @@ name = "Apple Incremental Game" category = "Combinatorial Optimization" language = "Python" description = "Decide each turn whether to invest in machines or collect output in an incremental production game, balancing short-term gains against long-horizon compounding." -version = "1.0.0" +version = "1.1.0" status = "migrated" [environment] @@ -26,8 +26,8 @@ timeout_sec = 600 cpus = 4 memory = "8g" # Official leaderboard limits (v1 kept them only in the experiment YAMLs). -# eval_apple_incremental_game.py runs the 50 cases 8 at a time (ProcessPoolExecutor, -# max_workers=8), each `python3 solution.py` under a 30 s wall-clock timeout. Scores of +# eval_apple_incremental_game.py runs the 50 cases min(8, SFORGE_CPUS) at a time, +# each `python3 solution.py` under a 30 s wall-clock timeout. Scores of # time-limited (anytime) solutions depend on host CPU speed and load; see NOTES.md. [score] From e2013a27e843c0eda8fa1dbec035533327de8dd5 Mon Sep 17 00:00:00 2001 From: findalexli Date: Tue, 3 Nov 2026 12:25:18 -0800 Subject: [PATCH 2/7] apple_incremental_game: qualify the remaining v1 identity claims --- edgebench/tasks/apple_incremental_game/NOTES.md | 7 ++++--- edgebench/tasks/apple_incremental_game/judge/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/edgebench/tasks/apple_incremental_game/NOTES.md b/edgebench/tasks/apple_incremental_game/NOTES.md index aad8b8d5..ba2abd16 100644 --- a/edgebench/tasks/apple_incremental_game/NOTES.md +++ b/edgebench/tasks/apple_incremental_game/NOTES.md @@ -57,11 +57,12 @@ sizes its pool with `SFORGE_CPUS`, so a case that searches until its own deadlin the CPU that 4 cases at once give it. The pool is now `min(8, SFORGE_CPUS, cases)`; `SFORGE_CPUS` must be a positive integer, and 4 applies only when it is absent. Deterministic solutions that produce the same output and finish on both judges keep their scores. The two deadline-based files -measured gained 3.32-5.48 of 100 across eight matched pairs with 4 vCPUs online and one judge at a -time; other resource conditions gave other gains, and there is no mapping from old scores. The +measured gained 3.32-5.48 of 100 across nine matched pairs with 4 vCPUs online and one judge at a +time; other resource conditions gave other gains, and there is no mapping from old scores, so +regrade before comparing with v1 leaderboard entries. The longest measured grade at 4 CPUs is 390.8 s (every solver timing out), so `timeout_sec` stays 600 for 4 CPUs; a smaller `sforge eval --judge-cpu-limit` needs a larger timeout. Known issue 3 stays: limits are -still wall clock. Regression summary describes 1.0.0. +still wall clock. Source, Deviations and Regression summary describe 1.0.0. ## Known issues diff --git a/edgebench/tasks/apple_incremental_game/judge/Dockerfile b/edgebench/tasks/apple_incremental_game/judge/Dockerfile index 0e2f0968..009e7148 100644 --- a/edgebench/tasks/apple_incremental_game/judge/Dockerfile +++ b/edgebench/tasks/apple_incremental_game/judge/Dockerfile @@ -15,7 +15,7 @@ # 9 rm -rf .git && git init && git config user.* && add && commit -m init # -> kept, author/dates pinned to v1's commit # 10 heredoc > /tmp/eval_apple_incremental_game.py; chmod +x -> COPY judge/eval_apple_incremental_game.py /judge/ -# (byte-identical, moved out of /tmp) +# (v1's, except the pool size, moved out of /tmp) # v1 eval_cmd + parser score_sum -> judge/grade.py (task.toml [judge].cmd) # Base = bases/python310 (sforge build passes it as BASE_IMAGE). From d5eba306a12b127cb2d788a002bd41474b919642 Mon Sep 17 00:00:00 2001 From: findalexli Date: Tue, 3 Nov 2026 14:50:10 -0800 Subject: [PATCH 3/7] apple_incremental_game: drop the comparability claim from Known issues --- edgebench/tasks/apple_incremental_game/NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edgebench/tasks/apple_incremental_game/NOTES.md b/edgebench/tasks/apple_incremental_game/NOTES.md index ba2abd16..1ca2f9b9 100644 --- a/edgebench/tasks/apple_incremental_game/NOTES.md +++ b/edgebench/tasks/apple_incremental_game/NOTES.md @@ -66,7 +66,7 @@ still wall clock. Source, Deviations and Regression summary describe 1.0.0. ## Known issues -Kept as in v1, so scores stay comparable. Fixing any of them changes scores and bumps `version`. +Kept as in v1. Fixing any of them changes scores and bumps `version`. 1. **The submission can forge its score through the tester's error message.** The eval script takes a score from any tester output line that looks like `Score = …` and ignores the tester's From e4f224586ca10c4a4df46e920251ee16cdf4a917 Mon Sep 17 00:00:00 2001 From: findalexli Date: Tue, 3 Nov 2026 18:58:37 -0800 Subject: [PATCH 4/7] apple_incremental_game: NOTES, nine real files --- edgebench/tasks/apple_incremental_game/NOTES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/edgebench/tasks/apple_incremental_game/NOTES.md b/edgebench/tasks/apple_incremental_game/NOTES.md index 1ca2f9b9..d1eaf841 100644 --- a/edgebench/tasks/apple_incremental_game/NOTES.md +++ b/edgebench/tasks/apple_incremental_game/NOTES.md @@ -56,9 +56,9 @@ The judge ran 8 cases at once on the official 4 CPUs, against SCHEMA's rule that sizes its pool with `SFORGE_CPUS`, so a case that searches until its own deadline got about half the CPU that 4 cases at once give it. The pool is now `min(8, SFORGE_CPUS, cases)`; `SFORGE_CPUS` must be a positive integer, and 4 applies only when it is absent. Deterministic solutions that -produce the same output and finish on both judges keep their scores. The two deadline-based files -measured gained 3.32-5.48 of 100 across nine matched pairs with 4 vCPUs online and one judge at a -time; other resource conditions gave other gains, and there is no mapping from old scores, so +produce the same output and finish on both judges keep their scores. The seven deadline-based files +measured gained 3.65-10.86 of 100 across ten pairs with 4 vCPUs online and one judge at a +time, and no case was lost on either judge; other resource conditions gave other gains, and there is no mapping from old scores, so regrade before comparing with v1 leaderboard entries. The longest measured grade at 4 CPUs is 390.8 s (every solver timing out), so `timeout_sec` stays 600 for 4 CPUs; a smaller `sforge eval --judge-cpu-limit` needs a larger timeout. Known issue 3 stays: limits are From 08843a9a1aa294c96273570f8b815ff30f71e99b Mon Sep 17 00:00:00 2001 From: findalexli Date: Tue, 3 Nov 2026 19:33:04 -0800 Subject: [PATCH 5/7] apple_incremental_game: timeout_sec covers one worker --- edgebench/tasks/apple_incremental_game/NOTES.md | 8 ++++---- edgebench/tasks/apple_incremental_game/task.toml | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/edgebench/tasks/apple_incremental_game/NOTES.md b/edgebench/tasks/apple_incremental_game/NOTES.md index d1eaf841..fba1dacd 100644 --- a/edgebench/tasks/apple_incremental_game/NOTES.md +++ b/edgebench/tasks/apple_incremental_game/NOTES.md @@ -59,10 +59,10 @@ must be a positive integer, and 4 applies only when it is absent. Deterministic produce the same output and finish on both judges keep their scores. The seven deadline-based files measured gained 3.65-10.86 of 100 across ten pairs with 4 vCPUs online and one judge at a time, and no case was lost on either judge; other resource conditions gave other gains, and there is no mapping from old scores, so -regrade before comparing with v1 leaderboard entries. The -longest measured grade at 4 CPUs is 390.8 s (every solver timing out), so `timeout_sec` stays 600 -for 4 CPUs; a smaller `sforge eval --judge-cpu-limit` needs a larger timeout. Known issue 3 stays: limits are -still wall clock. Source, Deviations and Regression summary describe 1.0.0. +regrade before comparing with v1 leaderboard entries. `timeout_sec` +goes from 600 to 1800, because a smaller pool takes longer: 50 cases of 30 s run one at a time take +1500 s, and at 4 CPUs the longest measured grade is 390.8 s. Known issue 3 stays: limits are still +wall clock. Source, Deviations and Regression summary describe 1.0.0. ## Known issues diff --git a/edgebench/tasks/apple_incremental_game/task.toml b/edgebench/tasks/apple_incremental_game/task.toml index 89210801..e3298f2c 100644 --- a/edgebench/tasks/apple_incremental_game/task.toml +++ b/edgebench/tasks/apple_incremental_game/task.toml @@ -22,13 +22,14 @@ exclude = [] [judge] cmd = "python3 /judge/grade.py" -timeout_sec = 600 +timeout_sec = 1800 cpus = 4 memory = "8g" # Official leaderboard limits (v1 kept them only in the experiment YAMLs). # eval_apple_incremental_game.py runs the 50 cases min(8, SFORGE_CPUS) at a time, # each `python3 solution.py` under a 30 s wall-clock timeout. Scores of # time-limited (anytime) solutions depend on host CPU speed and load; see NOTES.md. +# timeout_sec covers 50 x 30 s one at a time, for a smaller --judge-cpu-limit. [score] direction = "maximize" From 7dd54d5ed3798f417f5eb6bf92cce77f02a8466c Mon Sep 17 00:00:00 2001 From: findalexli Date: Tue, 3 Nov 2026 21:18:13 -0800 Subject: [PATCH 6/7] apple_incremental_game: pair count in NOTES --- edgebench/tasks/apple_incremental_game/NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edgebench/tasks/apple_incremental_game/NOTES.md b/edgebench/tasks/apple_incremental_game/NOTES.md index fba1dacd..b277e498 100644 --- a/edgebench/tasks/apple_incremental_game/NOTES.md +++ b/edgebench/tasks/apple_incremental_game/NOTES.md @@ -57,7 +57,7 @@ sizes its pool with `SFORGE_CPUS`, so a case that searches until its own deadlin the CPU that 4 cases at once give it. The pool is now `min(8, SFORGE_CPUS, cases)`; `SFORGE_CPUS` must be a positive integer, and 4 applies only when it is absent. Deterministic solutions that produce the same output and finish on both judges keep their scores. The seven deadline-based files -measured gained 3.65-10.86 of 100 across ten pairs with 4 vCPUs online and one judge at a +measured gained 3.65-10.86 of 100 across fourteen pairs with 4 vCPUs online and one judge at a time, and no case was lost on either judge; other resource conditions gave other gains, and there is no mapping from old scores, so regrade before comparing with v1 leaderboard entries. `timeout_sec` goes from 600 to 1800, because a smaller pool takes longer: 50 cases of 30 s run one at a time take From aba7189151b6dc1561bcf71aeb56e3a273ccf3d0 Mon Sep 17 00:00:00 2001 From: findalexli Date: Tue, 3 Nov 2026 23:12:51 -0800 Subject: [PATCH 7/7] apple_incremental_game: shorter NOTES, drop the timeout_sec comment --- edgebench/tasks/apple_incremental_game/NOTES.md | 17 +++++++---------- .../tasks/apple_incremental_game/task.toml | 1 - 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/edgebench/tasks/apple_incremental_game/NOTES.md b/edgebench/tasks/apple_incremental_game/NOTES.md index b277e498..ef6c310c 100644 --- a/edgebench/tasks/apple_incremental_game/NOTES.md +++ b/edgebench/tasks/apple_incremental_game/NOTES.md @@ -53,16 +53,13 @@ time-limited searches differ by under 1 point on 0..100. ### 1.1.0 The judge ran 8 cases at once on the official 4 CPUs, against SCHEMA's rule that a parallel grader -sizes its pool with `SFORGE_CPUS`, so a case that searches until its own deadline got about half -the CPU that 4 cases at once give it. The pool is now `min(8, SFORGE_CPUS, cases)`; `SFORGE_CPUS` -must be a positive integer, and 4 applies only when it is absent. Deterministic solutions that -produce the same output and finish on both judges keep their scores. The seven deadline-based files -measured gained 3.65-10.86 of 100 across fourteen pairs with 4 vCPUs online and one judge at a -time, and no case was lost on either judge; other resource conditions gave other gains, and there is no mapping from old scores, so -regrade before comparing with v1 leaderboard entries. `timeout_sec` -goes from 600 to 1800, because a smaller pool takes longer: 50 cases of 30 s run one at a time take -1500 s, and at 4 CPUs the longest measured grade is 390.8 s. Known issue 3 stays: limits are still -wall clock. Source, Deviations and Regression summary describe 1.0.0. +sizes its pool with `SFORGE_CPUS`, so a deadline search got about half the CPU of 4 at once. The pool +is now `min(8, SFORGE_CPUS, cases)`; `SFORGE_CPUS` must be a positive integer, and 4 applies only when +it is absent. Deterministic solutions that finish on both judges keep their scores. Seven deadline-based +files gained 3.65-10.86 of 100 across fourteen pairs at true 4 CPUs, with no case lost on either judge. +There is no mapping from old scores, so regrade before comparing with v1 entries. `timeout_sec` goes +from 600 to 1800 for a smaller `--judge-cpu-limit`: one file took 1443 s at limit 1, and at 4 CPUs the +longest grade took about 391 s. Known issue 3 stays. Source, Deviations and Regression summary describe 1.0.0. ## Known issues diff --git a/edgebench/tasks/apple_incremental_game/task.toml b/edgebench/tasks/apple_incremental_game/task.toml index e3298f2c..a1070faa 100644 --- a/edgebench/tasks/apple_incremental_game/task.toml +++ b/edgebench/tasks/apple_incremental_game/task.toml @@ -29,7 +29,6 @@ memory = "8g" # eval_apple_incremental_game.py runs the 50 cases min(8, SFORGE_CPUS) at a time, # each `python3 solution.py` under a 30 s wall-clock timeout. Scores of # time-limited (anytime) solutions depend on host CPU speed and load; see NOTES.md. -# timeout_sec covers 50 x 30 s one at a time, for a smaller --judge-cpu-limit. [score] direction = "maximize"