diff --git a/edgebench/tasks/arc_compiler_runtime/NOTES.md b/edgebench/tasks/arc_compiler_runtime/NOTES.md index 86a65028..d8ca4c06 100644 --- a/edgebench/tasks/arc_compiler_runtime/NOTES.md +++ b/edgebench/tasks/arc_compiler_runtime/NOTES.md @@ -20,7 +20,7 @@ and runs each with `tsx` under `timeout 90`. Raw score = `100 × WEIGHTED_PASSED snapshot; all 361 packages that step adds are checked against v1's versions. Base: `bases/python`. - `judge/eval_hidden.py` is v1's inline eval program, verbatim; `judge/run.sh` and `judge/write_result.py` are new and write `result.json` with v1's structured-result logic. -- `instruction.md`: v1's prompt, verbatim. +- `instruction.md`: v1's prompt, verbatim in 1.0.0; 1.0.1 adds the `analyze()` return shape. ## Deviations from v1 @@ -48,6 +48,15 @@ each. The grader is deterministic. Raw score (= 0..100). Every row matches exactly, including raw and weighted counts, and equals the historical v1 scores. +## Changes + +### 1.0.1 + +The hidden type-system suites (1000 of the 2970 weighted points, 33.67 of 100) call `.filter` on the +return value of `analyze()`, but nothing the agent sees gives that shape, and the starter returns an +object. A submission that keeps the object scores 0 on those suites. `instruction.md` now states the +shape. The judge and work images are unchanged (same tags), so grading is unchanged. + ## Known issues Kept as in v1, so scores stay comparable. Fixing any of them changes scores and bumps `version`. diff --git a/edgebench/tasks/arc_compiler_runtime/instruction.md b/edgebench/tasks/arc_compiler_runtime/instruction.md index 33a95db0..5917ca54 100644 --- a/edgebench/tasks/arc_compiler_runtime/instruction.md +++ b/edgebench/tasks/arc_compiler_runtime/instruction.md @@ -1 +1,3 @@ -Implement the missing Arc compiler and runtime modules under compiler/src/. Use CHEATSHEET.md, TASK.md, and the public smoke tests as the language contract. Modify only files under compiler/src/. Do not modify tests, scoring scripts, task metadata, Dockerfiles, package files, node_modules, generated dist artifacts, source maps, or evaluator-owned files. Work offline and do not depend on hidden tests or hard-coded answers. \ No newline at end of file +Implement the missing Arc compiler and runtime modules under compiler/src/. Use CHEATSHEET.md, TASK.md, and the public smoke tests as the language contract. Modify only files under compiler/src/. Do not modify tests, scoring scripts, task metadata, Dockerfiles, package files, node_modules, generated dist artifacts, source maps, or evaluator-owned files. Work offline and do not depend on hidden tests or hard-coded answers. + +The evaluator calls `analyze(parse(lex(source)))`. `analyze` in `compiler/src/semantic.js` must return an array of diagnostics (empty when there are none), each an object with `level` (`"error"` or `"warning"`) and `message` (a string). The starter's `analyze` returns a different shape. \ No newline at end of file diff --git a/edgebench/tasks/arc_compiler_runtime/task.toml b/edgebench/tasks/arc_compiler_runtime/task.toml index af688a74..8adae239 100644 --- a/edgebench/tasks/arc_compiler_runtime/task.toml +++ b/edgebench/tasks/arc_compiler_runtime/task.toml @@ -4,7 +4,7 @@ name = "Arc Compiler Runtime" category = "Systems & Software Engineering" language = "Python" # v1 tasks.jsonl value (base image); the submission itself is JavaScript run by Node 20 description = "Implement a complete TypeScript compiler pipeline (lexer, parser, type checker, code generator) for a novel programming language defined by specification documents." -version = "1.0.0" +version = "1.0.1" status = "migrated" [environment]