diff --git a/controlflow_cli_test.go b/controlflow_cli_test.go new file mode 100644 index 00000000..11fab54c --- /dev/null +++ b/controlflow_cli_test.go @@ -0,0 +1,217 @@ +package main + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/howlcipher/howlframe/internal/bytecode" +) + +const controlflowProgram = `(do + (print (% 7 3) (% -7 3) (% 7 -3) (% -7 -3) (// 7 2) (// -7 2) (// 7 -2) -5 (+ 3 -2) -1.5 (* 3 -2)) + (print (if (= 1 2) "a" (= 2 2) "b" "c") (if false 1 false 2 3) (cond ((= 1 2) "x") (else "y"))) + (print (encode_json (range 4)) (encode_json (range 2 5)) (encode_json (range 3 1))) + (let (ws (list "pear" "fig" "banana" "kiwi")) + (print (max_by w ws (str_len w)) (min_by w ws (str_len w)) (encode_json (sort_by w ws (str_len w))) (encode_json (sort_by w ws (- 0 (str_len w)))))) + (print (str "[" (pad_left 42 5) "|" (pad_left "x" 3 "0") "|" (pad_right "ab" 4) "|" (pad_right "abcdef" 2) "]")) + (print (str "[" (str_repeat "ab" 3) "|" (str_repeat "x" 0) "|" (rtrim " hi \t\n") "|" (ltrim " \t hi ") "]")) + (print (to_base 10 2) (to_base 0 2) (to_base 255 16) (to_base -10 2) (to_base 35 36)) + (let ((a b c) (list 1 2 3) d 4) (print a b c d)) + (let (s 0) + (do + (for x (range 10) (do (when (= x 5) (break)) (set s (+ s x)))) + (print s))) + (let (i 0 found -1) + (do + (while (< i 100) (do (when (= (% i 7) 6) (do (set found i) (break))) (set i (+ i 1)))) + (print i found))) + (let (out (list)) + (do + (for a (range 3) (for b (range 3) (do (when (> b a) (break)) (append out (str a b))))) + (print (str_join out ",")))))` + +const controlflowWant = "1 2 -2 -1 3 -4 -4 -5 1 -1.5 -6\n" + + "b 3 y\n" + + "[0,1,2,3] [2,3,4] []\n" + + "banana fig [\"fig\",\"pear\",\"kiwi\",\"banana\"] [\"banana\",\"pear\",\"kiwi\",\"fig\"]\n" + + "[ 42|00x|ab |abcdef]\n" + + "[ababab|| hi|hi ]\n" + + "1010 0 ff -1010 z\n" + + "1 2 3 4\n" + + "10\n" + + "6 6\n" + + "00,10,11,20,21,22\n" + +func writeTemp(t *testing.T, dir, name, src string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(src), 0600); err != nil { + t.Fatal(err) + } + return p +} + +// The control-flow and core built-ins are pure sugar: no capability under the +// governed profile, and the same output on the interpreter, governed AST +// bytecode and HFIR bytecode. +func TestControlflowBuiltinsArePureOnEveryPath(t *testing.T) { + bin := c1c2Binary(t) + dir := t.TempDir() + src := writeTemp(t, dir, "p.howl", controlflowProgram) + art := filepath.Join(dir, "p.hfbc") + hart := filepath.Join(dir, "h.hfbc") + for _, profile := range []string{"default", "governed"} { + if out, err := exec.Command(bin, "check", "--profile", profile, src).CombinedOutput(); err != nil { + t.Fatalf("check %s: %v %s", profile, err, out) + } + } + if out, err := exec.Command(bin, "build", "--profile", "governed", src, "-o", art).CombinedOutput(); err != nil { + t.Fatalf("build: %v %s", err, out) + } + out, err := exec.Command(bin, "inspect", "--caps", art).CombinedOutput() + if err != nil { + t.Fatalf("inspect: %v %s", err, out) + } + var report bytecode.Report + if err := json.Unmarshal(out, &report); err != nil { + t.Fatalf("%v: %s", err, out) + } + if len(report.Capabilities) != 0 { + t.Fatalf("pure control-flow built-ins required capabilities %v", report.Capabilities) + } + if out, err := exec.Command(bin, "-compile-hfir-bc", src, "-o", hart).CombinedOutput(); err != nil { + t.Fatalf("hfir compile: %v %s", err, out) + } + runs := map[string][]string{ + "interpreter": {"-run", src}, + "bytecode_governed": {"-run-bc", art}, + "hfir_bytecode": {"-run-bc", hart}, + } + for name, args := range runs { + t.Run(name, func(t *testing.T) { + got, err := exec.Command(bin, args...).CombinedOutput() + if err != nil { + t.Fatalf("run: %v %s", err, got) + } + if string(got) != controlflowWant { + t.Fatalf("got %q want %q", got, controlflowWant) + } + }) + } +} + +// Runtime value errors fail closed with the same stderr shape on every path. +func TestControlflowValueErrorsOnEveryPath(t *testing.T) { + bin := c1c2Binary(t) + cases := map[string]string{ + `(do (print (max_by x (list) x)))`: "VALUE_ERROR: max_by of empty list", + `(do (print (min_by x (list) x)))`: "VALUE_ERROR: min_by of empty list", + `(do (print (to_base 5 1)))`: "VALUE_ERROR: to_base base must be 2..36", + `(do (print (to_base 5 37)))`: "VALUE_ERROR: to_base base must be 2..36", + `(do (print (% 5 0)))`: "division by zero", + `(do (print (// 5 0)))`: "division by zero", + // The signed multiply check still catches real overflow after the + // fix that stopped (* 3 -2) from reporting one. + `(do (print (* 9223372036854775807 2)))`: "integer overflow", + `(do (print (* -9223372036854775807 3)))`: "integer overflow", + } + for src, want := range cases { + dir := t.TempDir() + p := writeTemp(t, dir, "e.howl", src) + art := filepath.Join(dir, "e.hfbc") + hart := filepath.Join(dir, "h.hfbc") + if out, err := exec.Command(bin, "build", "--profile", "governed", p, "-o", art).CombinedOutput(); err != nil { + t.Fatalf("%s build: %v %s", src, err, out) + } + if out, err := exec.Command(bin, "-compile-hfir-bc", p, "-o", hart).CombinedOutput(); err != nil { + t.Fatalf("%s hfir: %v %s", src, err, out) + } + for _, args := range [][]string{{"-run", p}, {"-run-bc", art}, {"-run-bc", hart}} { + out, err := exec.Command(bin, args...).CombinedOutput() + if err == nil || !strings.Contains(string(out), want) { + t.Errorf("%s %v: err=%v out=%q want %q", src, args[0], err, out, want) + } + } + } +} + +// A break outside a loop, or in a value position, is a compile error on every +// entry point; it never reaches a backend. +func TestStrayBreakIsRejected(t *testing.T) { + bin := c1c2Binary(t) + for _, src := range []string{ + `(do (break))`, + `(do (for x (list 1) (print (break))))`, + `(do (let (y (break)) (print y)))`, + } { + dir := t.TempDir() + p := writeTemp(t, dir, "b.howl", src) + for _, args := range [][]string{{"-run", p}, {"build", "--profile", "governed", p, "-o", filepath.Join(dir, "b.hfbc")}, {"-compile-hfir-bc", p, "-o", filepath.Join(dir, "h.hfbc")}} { + out, err := exec.Command(bin, args...).CombinedOutput() + if err == nil || !strings.Contains(string(out), "break must be a statement inside for/while") { + t.Errorf("%s %v: err=%v out=%q", src, args[0], err, out) + } + } + } +} + +// Go and JS reject every new form with UNSUPPORTED_ON_TARGET before writing +// anything. Negative literals are plain literals and still build. +func TestControlflowRejectedOnGoJS(t *testing.T) { + bin := c1c2Binary(t) + forms := []string{ + `(print (% 7 3))`, `(print (// 7 2))`, `(print (encode_json (range 3)))`, + `(print (max_by x (list 1) x))`, `(print (min_by x (list 1) x))`, `(print (encode_json (sort_by x (list 1) x)))`, + `(print (pad_left "a" 3))`, `(print (pad_right "a" 3))`, `(print (str_repeat "a" 3))`, + `(print (rtrim "a "))`, `(print (ltrim " a"))`, `(print (to_base 5 2))`, + `(for x (list 1) (break))`, `(print (if false 1 false 2 3))`, `(let ((a b) (list 1 2)) (print a))`, + } + for _, f := range forms { + for _, target := range []string{"go", "js"} { + dir := t.TempDir() + p := writeTemp(t, dir, "g.howl", "(do "+f+")") + o := filepath.Join(dir, "out") + out, err := exec.Command(bin, "build", "--target", target, p, "-o", o).CombinedOutput() + if err == nil || !strings.Contains(string(out), "UNSUPPORTED_ON_TARGET") || strings.Contains(string(out), "Built") { + t.Errorf("%s on %s: err=%v out=%s", f, target, err, out) + } + if _, statErr := os.Stat(o); statErr == nil { + t.Errorf("%s on %s wrote output", f, target) + } + } + } + dir := t.TempDir() + p := writeTemp(t, dir, "n.howl", `(do (print (+ 3 -2) (- 5 -2)))`) + for _, target := range []string{"go", "js"} { + if out, err := exec.Command(bin, "build", "--target", target, p, "-o", filepath.Join(dir, "n."+target)).CombinedOutput(); err != nil { + t.Errorf("negative literal on %s: %v %s", target, err, out) + } + } +} + +// Governed checks fail closed through the new forms: a forbidden construct +// nested in any of them (or in a cond clause) is still rejected, and the +// governed scan no longer mistakes cond's else clause for a construct. +func TestControlflowGovernedFailsClosed(t *testing.T) { + bin := c1c2Binary(t) + for _, src := range []string{ + `(do (print (% (exec "ls") 2)))`, + `(do (print (sort_by x (list 1) (exec "ls"))))`, + `(do (print (cond ((= 1 1) (exec "ls")) (else 1))))`, + `(do (print (cond ((= 1 2) 1) (else (exec "ls")))))`, + `(do (for x (range 2) (do (exec "ls") (break))))`, + `(do (let ((a b) (list (exec "ls") 1)) (print a)))`, + `(do (print (if false 1 false (exec "ls") 3)))`, + } { + dir := t.TempDir() + p := writeTemp(t, dir, "g.howl", src) + out, err := exec.Command(bin, "build", "--profile", "governed", p, "-o", filepath.Join(dir, "g.hfbc")).CombinedOutput() + if err == nil || !strings.Contains(string(out), "PROFILE_FORBIDDEN_CONSTRUCT") { + t.Errorf("%s: err=%v out=%s", src, err, out) + } + } +} diff --git a/docs/reference/construct_coverage.md b/docs/reference/construct_coverage.md index b27a916a..41fc6237 100644 --- a/docs/reference/construct_coverage.md +++ b/docs/reference/construct_coverage.md @@ -5,10 +5,12 @@ | Construct | Standalone Bytecode | Required Capability | Compile-Time Only? | Known Unsupported? | Tracker | Evidence Source | |---|---|---|---|---|---|---| | `!=` | Yes | None | No | No | - | AUTHORITATIVE | +| `%` | No | None | Yes | No | - | AUTHORITATIVE | | `*` | Yes | None | No | No | - | AUTHORITATIVE | | `+` | Yes | None | No | No | - | AUTHORITATIVE | | `-` | Yes | None | No | No | - | AUTHORITATIVE | | `/` | Yes | None | No | No | - | AUTHORITATIVE | +| `//` | No | None | Yes | No | - | AUTHORITATIVE | | `<` | Yes | None | No | No | - | AUTHORITATIVE | | `<=` | Yes | None | No | No | - | AUTHORITATIVE | | `=` | Yes | None | No | No | - | AUTHORITATIVE | @@ -24,6 +26,7 @@ | `assert_semantic` | No | None | No | Yes | - | AUTHORITATIVE | | `attr_escape` | Yes | None | No | No | - | AUTHORITATIVE | | `blank` | No | None | Yes | No | - | AUTHORITATIVE | +| `break` | No | None | Yes | No | - | AUTHORITATIVE | | `bytes_to_string` | Yes | None | No | No | - | AUTHORITATIVE | | `call` | Yes | None | No | No | - | AUTHORITATIVE | | `cli_app` | Yes | None | No | No | - | AUTHORITATIVE | @@ -69,6 +72,7 @@ | `list_len` | Yes | None | No | No | - | AUTHORITATIVE | | `list_slice` | No | None | Yes | No | - | AUTHORITATIVE | | `llm_generate` | Yes | network | No | No | - | AUTHORITATIVE | +| `ltrim` | No | None | Yes | No | - | AUTHORITATIVE | | `map_delete` | Yes | None | No | No | - | AUTHORITATIVE | | `map_get` | Yes | None | No | No | - | AUTHORITATIVE | | `map_has` | No | None | Yes | No | - | AUTHORITATIVE | @@ -77,8 +81,10 @@ | `map_set` | Yes | None | No | No | - | AUTHORITATIVE | | `match` | No | None | No | Yes | - | AUTHORITATIVE | | `max` | No | None | Yes | No | - | AUTHORITATIVE | +| `max_by` | No | None | Yes | No | - | AUTHORITATIVE | | `middleware` | No | None | No | Yes | - | AUTHORITATIVE | | `min` | No | None | Yes | No | - | AUTHORITATIVE | +| `min_by` | No | None | Yes | No | - | AUTHORITATIVE | | `mkdir` | Yes | filesystem | No | No | - | AUTHORITATIVE | | `module` | No | None | Yes | No | - | AUTHORITATIVE | | `neural_circuit` | Yes | network | No | No | - | AUTHORITATIVE | @@ -87,9 +93,12 @@ | `optimize_block` | Yes | None | No | No | - | AUTHORITATIVE | | `optimize_signature` | Yes | None | No | No | - | AUTHORITATIVE | | `or` | Yes | None | No | No | - | AUTHORITATIVE | +| `pad_left` | No | None | Yes | No | - | AUTHORITATIVE | +| `pad_right` | No | None | Yes | No | - | AUTHORITATIVE | | `parse_json` | Yes | None | No | No | - | AUTHORITATIVE | | `patch` | No | None | Yes | No | - | AUTHORITATIVE | | `print` | Yes | None | No | No | - | AUTHORITATIVE | +| `range` | No | None | Yes | No | - | AUTHORITATIVE | | `read_file` | Yes | filesystem | No | No | - | AUTHORITATIVE | | `read_line` | Yes | None | No | No | - | AUTHORITATIVE | | `read_text` | No | None | Yes | No | - | AUTHORITATIVE | @@ -104,6 +113,7 @@ | `res_json` | Yes | network | No | No | - | AUTHORITATIVE | | `return` | Yes | None | No | No | - | AUTHORITATIVE | | `reverse` | No | None | Yes | No | - | AUTHORITATIVE | +| `rtrim` | No | None | Yes | No | - | AUTHORITATIVE | | `schema` | No | None | No | Yes | - | AUTHORITATIVE | | `schema_bridge` | Yes | None | No | No | - | AUTHORITATIVE | | `semantic_match` | No | None | No | Yes | - | AUTHORITATIVE | @@ -112,6 +122,7 @@ | `sinc` | No | None | Yes | No | - | AUTHORITATIVE | | `sleep` | Yes | None | No | No | - | AUTHORITATIVE | | `sort` | No | None | Yes | No | - | AUTHORITATIVE | +| `sort_by` | No | None | Yes | No | - | AUTHORITATIVE | | `spawn` | Yes | process | No | No | - | AUTHORITATIVE | | `spawn_agent` | Yes | process | No | No | - | AUTHORITATIVE | | `sput` | No | None | Yes | No | - | AUTHORITATIVE | @@ -127,6 +138,7 @@ | `str_contains` | No | None | Yes | No | - | AUTHORITATIVE | | `str_join` | Yes | None | No | No | - | AUTHORITATIVE | | `str_len` | No | None | Yes | No | - | AUTHORITATIVE | +| `str_repeat` | No | None | Yes | No | - | AUTHORITATIVE | | `str_replace` | No | None | Yes | No | - | AUTHORITATIVE | | `str_slice` | No | None | Yes | No | - | AUTHORITATIVE | | `str_split` | Yes | None | No | No | - | AUTHORITATIVE | @@ -137,6 +149,7 @@ | `task` | Yes | None | No | No | - | AUTHORITATIVE | | `test` | No | None | No | Yes | improvements.md #96 | AUTHORITATIVE | | `time_now` | Yes | None | No | No | - | AUTHORITATIVE | +| `to_base` | No | None | Yes | No | - | AUTHORITATIVE | | `to_float` | Yes | None | No | No | - | AUTHORITATIVE | | `to_int` | Yes | None | No | No | - | AUTHORITATIVE | | `to_lower` | Yes | None | No | No | - | AUTHORITATIVE | diff --git a/experiments/token-efficiency-heldout-v2/REPORT.md b/experiments/token-efficiency-heldout-v2/REPORT.md index 128ef208..9bce17ad 100644 --- a/experiments/token-efficiency-heldout-v2/REPORT.md +++ b/experiments/token-efficiency-heldout-v2/REPORT.md @@ -18,7 +18,10 @@ unchanged, #90 stays Partial, and no live LLM was used. | HowlFrame vs Go | 0.590 | 0.558 | 0.586 (4084 / 6972) | 37/0/0 | o200k_base: vs Python 1.306 median / 1.321 total (2/0/35); vs JS 1.083 / 1.112 -(10/1/26); vs Go 0.582 / 0.583 (37/0/0). +(10/1/26); vs Go 0.582 / 0.583 (37/0/0). The tokenizer is *not* neutral for +JS: on o200k the HowlFrame-vs-JS median drops from 1.136 to 1.083 and wins rise +from 7 to 10 (the #146 PR body called o200k "nearly identical"; that was wrong +for JS). Python and Go are near-identical across the two encodings. All 148 references (37 tasks x 4 languages) pass the verifier. `gaps.json` is empty: every task is expressible in current HowlFrame, but many only through @@ -92,6 +95,16 @@ Recorded so the next slice can be judged against them; none was implemented. formatting (w31). No right-trim, pad or repeat (w03, w08, w20). - No sort by key: distinct-scores loop for w33. `map_keys` returns sorted string keys, so numeric buckets needed a counted loop (w11). +- Governed profile bans unknown heads such as `count`: any head that is not a + registry construct (including a helper or defun called by bare name) is + `PROFILE_FORBIDDEN_CONSTRUCT` under `--profile governed`, so references + cannot factor small helpers; they inline the logic instead (Motoko, #146). +- `let` binding limits: a `let` takes exactly one body form (multi-statement + bodies need `(do ...)`), and bindings must be flat `(name value ...)` pairs + (no destructuring). Motoko also reported `let` as non-sequential while + golfing; on main a later binding does see an earlier one + (`(let (a 2 b (* a 3)) ...)` gives 6, via nested lets), so the practical + cost is the extra `do` and the lack of destructuring. - Multi-line indentation is counted like every prior set (v1 style). ## Method @@ -108,3 +121,142 @@ byte-identical across reruns. `results_freeze_test.go` pins sha256 of `gaps.json` and every file in `results/`. `verification_test.go` checks every task x language is verified with a current hash. + +## After the control-flow slice (`postcf/`) + +A **separate** results set. The frozen specs (`tasks/`), the frozen baseline +HowlFrame sources (`solutions/howl/`) and baseline results (`results/`, +`gaps.json`) are untouched and keep their pins (`TestHeldoutV2SpecsFrozen`, +`TestHeldoutV2BaselineResultsFrozen`, `TestHeldoutV2VerificationCoverage`). +The Python/JS/Go references are the same baseline files. Only the HowlFrame +sources were rewritten, into `postcf/solutions/howl/`, and verified and counted +with the same verifier (governed build, `-run-bc`, fs read=in/write=out only) +and the same tokenizer rules. Reproduce with `./run_postcf.sh` +(`TOKBENCH_PYTHON` as for `run.sh`); `postcf/results/` is pinned by +`TestHeldoutV2PostCFResultsFrozen` and checked by +`TestHeldoutV2PostCFVerificationCoverage`. 148/148 pass, no gaps. + +**HowlFrame still loses to Python on held-out v2**: 1.154 median per task and +1.167 total on cl100k (5 wins, 32 losses), down from 1.306 / 1.324 (2/35). It +is now roughly level with JS on cl100k (1.028 median, 16/1/20) and ahead of JS +on o200k (0.968 median, 20/0/17). Go stays far behind HowlFrame. + +### What the slice added (pure sugar; no opcode, capability or effect) + +`(% a b)` / `(// a b)` floored like Python; n-ary `if` (elif chains); +`(break)` in for/while bodies; `(range [a] b)`; negative literals (`-1`, +`-2.5`); `max_by` / `min_by` / `sort_by`; `pad_left` / `pad_right` / +`str_repeat` / `rtrim` / `ltrim`; `(to_base n b)`; `let` destructuring +`((a b) lst)`. `map_keys` is **unchanged**: map keys are strings on every +backend (Go/JS included), so sorting numeric-looking keys numerically would +change existing output on every target; `sort_by k (map_keys d) (to_int k)` +covers the w11 case instead. + +### How the sources were rewritten + +- 25 of 37 files changed; the other 12 (w02, w04-w07, w17, w18, w21-w24, w30) + are byte-identical copies of the baseline, because nothing in the slice + applies to them. +- Rewrites use the new forms where the baseline used a workaround, keeping the + baseline layout. Two also collapsed structure with pre-existing features: + w03 (`filter` over `map_list` instead of for/append) and w25 (`filter`/`any` + over `range` instead of nested while). Motoko's #146 golf showed packing and + `filter` alone take w03 from 104 to 83 tokens, so about 21 of w03's 52-token + gain is not attributable to the slice. w25's gain needs `range` and `%`. +- Each changed task also runs the same on the interpreter and on HFIR bytecode, + except the tasks that use `parse_json` (not supported under `-run`, as on + main) and w22 (byte-identical to the baseline; it fails with + CAPABILITY_DENIED under HFIR bytecode on main too, a pre-existing issue). + +### Headline (cl100k_base) + +| Comparison | Median per-task ratio before → after | Total-token ratio before → after | W/T/L before → after | +|---|---|---|---| +| HowlFrame vs python | 1.306 → **1.154** | 1.324 → **1.167** (4084 → 3599 / 3084) | 2/0/35 → **5/0/32** | +| HowlFrame vs js | 1.136 → **1.028** | 1.167 → **1.028** (4084 → 3599 / 3500) | 7/2/28 → **16/1/20** | +| HowlFrame vs go | 0.590 → **0.519** | 0.586 → **0.516** (4084 → 3599 / 6972) | 37/0/0 → **37/0/0** | + +### Per category (cl100k_base, HowlFrame / Python) + +| Category | n | Median per-task before → after | Total-token before → after | +|---|---|---|---| +| text | 4 | 1.254 → 1.097 | 1.314 → 1.117 | +| table | 4 | 1.279 → 1.248 | 1.378 → 1.248 | +| numbers | 4 | 1.433 → 1.090 | 1.470 → 1.153 | +| lists | 4 | 1.423 → 1.186 | 1.391 → 1.210 | +| strings | 4 | 1.344 → 1.138 | 1.321 → 1.220 | +| records | 4 | 1.190 → 1.190 | 1.242 → 1.242 | +| logic | 13 | 1.244 → 1.150 | 1.290 → 1.127 | + +### Headline (o200k_base) + +| Comparison | Median per-task ratio before → after | Total-token ratio before → after | W/T/L before → after | +|---|---|---|---| +| HowlFrame vs python | 1.306 → **1.164** | 1.321 → **1.165** (4095 → 3613 / 3100) | 2/0/35 → **5/0/32** | +| HowlFrame vs js | 1.083 → **0.968** | 1.112 → **0.981** (4095 → 3613 / 3683) | 10/1/26 → **20/0/17** | +| HowlFrame vs go | 0.582 → **0.520** | 0.583 → **0.514** (4095 → 3613 / 7030) | 37/0/0 → **37/0/0** | + +### Per category (o200k_base, HowlFrame / Python) + +| Category | n | Median per-task before → after | Total-token before → after | +|---|---|---|---| +| text | 4 | 1.254 → 1.097 | 1.314 → 1.117 | +| table | 4 | 1.247 → 1.228 | 1.348 → 1.224 | +| numbers | 4 | 1.433 → 1.096 | 1.470 → 1.160 | +| lists | 4 | 1.403 → 1.187 | 1.381 → 1.201 | +| strings | 4 | 1.351 → 1.145 | 1.325 → 1.224 | +| records | 4 | 1.197 → 1.197 | 1.248 → 1.248 | +| logic | 13 | 1.242 → 1.149 | 1.290 → 1.127 | + +### Per task (HowlFrame tokens before → after; cl100k_base and o200k_base) + +| Task | Cat | cl100k howl | py | js | go | howl/py | o200k howl | py | js | go | howl/py | +|---|---|---|---|---|---|---|---|---|---|---|---| +| w01 | text | 111 → 109 | 95 | 106 | 168 | 1.17 → 1.15 | 111 → 109 | 95 | 110 | 170 | 1.17 → 1.15 | +| w02 | text | 79 (=) | 59 | 61 | 133 | 1.34 → 1.34 | 79 (=) | 59 | 65 | 134 | 1.34 → 1.34 | +| w03 | text | 104 → 52 | 57 | 64 | 143 | 1.82 → **0.91** | 104 → 52 | 57 | 68 | 144 | 1.82 → **0.91** | +| w04 | text | 66 (=) | 63 | 80 | 163 | 1.05 → 1.05 | 66 (=) | 63 | 86 | 165 | 1.05 → 1.05 | +| w05 | table | 98 (=) | 76 | 88 | 177 | 1.29 → 1.29 | 98 (=) | 78 | 92 | 178 | 1.26 → 1.26 | +| w06 | table | 97 (=) | 79 | 101 | 179 | 1.23 → 1.23 | 97 (=) | 82 | 105 | 180 | 1.18 → 1.18 | +| w07 | table | 104 (=) | 82 | 91 | 202 | 1.27 → 1.27 | 104 (=) | 84 | 96 | 203 | 1.24 → 1.24 | +| w08 | table | 135 → 94 | 78 | 86 | 181 | 1.73 → 1.21 | 135 → 95 | 78 | 90 | 182 | 1.73 → 1.22 | +| w09 | numbers | 56 → 54 | 50 | 61 | 150 | 1.12 → 1.08 | 56 → 54 | 50 | 65 | 151 | 1.12 → 1.08 | +| w10 | numbers | 117 → 88 | 80 | 108 | 166 | 1.46 → 1.10 | 117 → 89 | 80 | 112 | 167 | 1.46 → 1.11 | +| w11 | numbers | 139 → 106 | 79 | 127 | 220 | 1.76 → 1.34 | 139 → 107 | 79 | 131 | 222 | 1.76 → 1.35 | +| w12 | numbers | 101 → 76 | 72 | 97 | 206 | 1.40 → 1.06 | 101 → 76 | 72 | 101 | 208 | 1.40 → 1.06 | +| w13 | lists | 79 → 70 | 62 | 78 | 171 | 1.27 → 1.13 | 78 → 69 | 61 | 81 | 171 | 1.28 → 1.13 | +| w14 | lists | 123 → 115 | 86 | 105 | 221 | 1.43 → 1.34 | 124 → 116 | 88 | 112 | 223 | 1.41 → 1.32 | +| w15 | lists | 94 → 82 | 66 | 74 | 205 | 1.42 → 1.24 | 94 → 82 | 66 | 80 | 207 | 1.42 → 1.24 | +| w16 | lists | 81 → 61 | 57 | 59 | 164 | 1.42 → 1.07 | 81 → 61 | 58 | 63 | 165 | 1.40 → 1.05 | +| w17 | strings | 92 (=) | 57 | 62 | 156 | 1.61 → 1.61 | 92 (=) | 57 | 66 | 158 | 1.61 → 1.61 | +| w18 | strings | 64 (=) | 63 | 64 | 165 | 1.02 → 1.02 | 64 (=) | 63 | 70 | 165 | 1.02 → 1.02 | +| w19 | strings | 91 → 82 | 69 | 73 | 129 | 1.32 → 1.19 | 92 → 83 | 69 | 81 | 131 | 1.33 → 1.20 | +| w20 | strings | 78 → 62 | 57 | 61 | 125 | 1.37 → 1.09 | 78 → 62 | 57 | 65 | 126 | 1.37 → 1.09 | +| w21 | records | 99 (=) | 65 | 69 | 174 | 1.52 → 1.52 | 99 (=) | 65 | 73 | 175 | 1.52 → 1.52 | +| w22 | records | 99 (=) | 88 | 104 | 226 | 1.12 → 1.12 | 100 (=) | 88 | 110 | 228 | 1.14 → 1.14 | +| w23 | records | 90 (=) | 78 | 83 | 198 | 1.15 → 1.15 | 91 (=) | 78 | 89 | 198 | 1.17 → 1.17 | +| w24 | records | 92 (=) | 75 | 81 | 190 | 1.23 → 1.23 | 92 (=) | 75 | 85 | 190 | 1.23 → 1.23 | +| w25 | logic | 128 → 66 | 77 | 102 | 188 | 1.66 → **0.86** | 128 → 67 | 77 | 106 | 190 | 1.66 → **0.87** | +| w26 | logic | 94 → 87 | 72 | 76 | 157 | 1.31 → 1.21 | 94 → 87 | 72 | 80 | 160 | 1.31 → 1.21 | +| w27 | logic | 91 → 79 | 95 | 98 | 184 | 0.96 → **0.83** | 92 → 80 | 95 | 102 | 185 | 0.97 → **0.84** | +| w28 | logic | 146 → 137 | 133 | 155 | 229 | 1.10 → 1.03 | 147 → 138 | 134 | 162 | 231 | 1.10 → 1.03 | +| w29 | logic | 108 → 105 | 90 | 118 | 214 | 1.20 → 1.17 | 108 → 105 | 90 | 124 | 217 | 1.20 → 1.17 | +| w30 | logic | 163 (=) | 140 | 147 | 254 | 1.16 → 1.16 | 163 (=) | 140 | 151 | 255 | 1.16 → 1.16 | +| w31 | logic | 104 → 41 | 49 | 54 | 151 | 2.12 → **0.84** | 104 → 41 | 49 | 59 | 152 | 2.12 → **0.84** | +| w32 | logic | 142 → 128 | 124 | 142 | 232 | 1.15 → 1.03 | 143 → 129 | 126 | 147 | 237 | 1.13 → 1.02 | +| w33 | logic | 143 → 142 | 95 | 113 | 252 | 1.51 → 1.49 | 145 → 143 | 96 | 120 | 255 | 1.51 → 1.49 | +| w34 | logic | 227 → 208 | 145 | 123 | 250 | 1.57 → 1.43 | 227 → 208 | 145 | 130 | 251 | 1.57 → 1.43 | +| w35 | logic | 140 → 122 | 100 | 123 | 230 | 1.40 → 1.22 | 141 → 123 | 101 | 130 | 234 | 1.40 → 1.22 | +| w36 | logic | 110 → 106 | 111 | 98 | 179 | 0.99 → **0.95** | 111 → 107 | 112 | 102 | 180 | 0.99 → **0.96** | +| w37 | logic | 199 → 184 | 160 | 168 | 240 | 1.24 → 1.15 | 200 → 185 | 161 | 174 | 242 | 1.24 → 1.15 | + +### Where HowlFrame still loses + +- records (w21-w24) and w05-w07, w17, w30 have no slice-relevant workaround; + the cost there is s-expression structure (`(map_set d k (+ (nz (map_get d k) + 0) v))` against `d[k] = d.get(k, 0) + v`) and the explicit + `parse_json Any (read_text ...)` / `write_file` / `encode_json` I/O. +- w33 (sort by two keys) barely moves: two stable `sort_by` passes plus a + dense-rank loop cost as much as the old distinct-score scan. +- w34 (grid neighbours) is still 1.43x Python: indexing a list of lists costs + `(list_get (list_get g y) x)` per access. diff --git a/experiments/token-efficiency-heldout-v2/cmd/heldoutv2verify/main.go b/experiments/token-efficiency-heldout-v2/cmd/heldoutv2verify/main.go index 07f2574b..4058f0c7 100644 --- a/experiments/token-efficiency-heldout-v2/cmd/heldoutv2verify/main.go +++ b/experiments/token-efficiency-heldout-v2/cmd/heldoutv2verify/main.go @@ -47,6 +47,8 @@ var exts = map[string]string{"howl": "howl", "python": "py", "js": "js", "go": " func main() { root := flag.String("root", "experiments/token-efficiency-heldout-v2", "benchmark root") howl := flag.String("howlframe", "/tmp/howlframe", "HowlFrame binary") + howlDir := flag.String("howl-dir", "", "HowlFrame solutions directory (default /solutions/howl)") + resultsDir := flag.String("results", "", "results directory (default /results)") flag.Parse() var manifest struct{ Tasks []Task } must(readJSON(filepath.Join(*root, "tasks/manifest.json"), &manifest)) @@ -69,6 +71,9 @@ func main() { } total++ src, _ := filepath.Abs(filepath.Join(*root, "solutions", lang, t.ID+"."+exts[lang])) + if lang == "howl" && *howlDir != "" { + src, _ = filepath.Abs(filepath.Join(*howlDir, t.ID+"."+exts[lang])) + } data, readErr := os.ReadFile(src) r := Result{Task: t.ID, Lang: lang, SHA256: fmt.Sprintf("%x", sha256.Sum256(data))} if readErr != nil { @@ -88,8 +93,12 @@ func main() { fmt.Printf("%s: %d/%d passed\n", lang, passed, total) } out, _ := json.MarshalIndent(results, "", " ") - must(os.MkdirAll(filepath.Join(*root, "results"), 0o755)) - must(os.WriteFile(filepath.Join(*root, "results/verification.json"), append(out, '\n'), 0o644)) + res := filepath.Join(*root, "results") + if *resultsDir != "" { + res = *resultsDir + } + must(os.MkdirAll(res, 0o755)) + must(os.WriteFile(filepath.Join(res, "verification.json"), append(out, '\n'), 0o644)) if failed { os.Exit(1) } diff --git a/experiments/token-efficiency-heldout-v2/compare_postcf.py b/experiments/token-efficiency-heldout-v2/compare_postcf.py new file mode 100644 index 00000000..2a5108c1 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/compare_postcf.py @@ -0,0 +1,62 @@ +"""Before/after tables: frozen baseline (results/) vs post-control-flow (postcf/results/).""" +import json, statistics +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +LANGS = ("python", "js", "go") + + +def load(d): + rows = json.loads((ROOT / d / "tokens.json").read_text()) + out = {} + for r in rows: + out.setdefault(r["encoding"], {}).setdefault(r["lang"], {})[r["task"]] = r["tokens"] + return out + + +def main(): + before, after = load("results"), load("postcf/results") + cats = {t["id"]: t["category"] for t in json.loads((ROOT / "tasks/manifest.json").read_text())["tasks"]} + ids = sorted(cats) + md = [] + for enc in ("cl100k_base", "o200k_base"): + b, a = before[enc], after[enc] + md.append(f"### Headline ({enc})\n") + md.append("| Comparison | Median per-task ratio before → after | Total-token ratio before → after | W/T/L before → after |") + md.append("|---|---|---|---|") + for l in LANGS: + def stats(h): + rs = [h[t] / b[l][t] for t in ids] + w = sum(h[t] < b[l][t] for t in ids); ti = sum(h[t] == b[l][t] for t in ids) + return statistics.median(rs), sum(h.values()) / sum(b[l].values()), f"{w}/{ti}/{len(ids) - w - ti}" + m0, t0, w0 = stats(b["howl"]); m1, t1, w1 = stats(a["howl"]) + md.append(f"| HowlFrame vs {l} | {m0:.3f} → **{m1:.3f}** | {t0:.3f} → **{t1:.3f}** ({sum(b['howl'].values())} → {sum(a['howl'].values())} / {sum(b[l].values())}) | {w0} → **{w1}** |") + md.append("") + md.append(f"### Per category ({enc}, HowlFrame / Python)\n") + md.append("| Category | n | Median per-task before → after | Total-token before → after |") + md.append("|---|---|---|---|") + for cat in sorted(set(cats.values()), key=lambda c: ["text", "table", "numbers", "lists", "strings", "records", "logic"].index(c)): + cs = [t for t in ids if cats[t] == cat] + f = lambda h: (statistics.median(h[t] / b["python"][t] for t in cs), sum(h[t] for t in cs) / sum(b["python"][t] for t in cs)) + (m0, t0), (m1, t1) = f(b["howl"]), f(a["howl"]) + md.append(f"| {cat} | {len(cs)} | {m0:.3f} → {m1:.3f} | {t0:.3f} → {t1:.3f} |") + md.append("") + md.append("### Per task (HowlFrame tokens before → after; cl100k_base and o200k_base)\n") + md.append("| Task | Cat | cl100k howl | py | js | go | howl/py | o200k howl | py | js | go | howl/py |") + md.append("|---|---|---|---|---|---|---|---|---|---|---|---|") + for t in ids: + cells = [t, cats[t]] + for enc in ("cl100k_base", "o200k_base"): + b, a = before[enc], after[enc] + h0, h1 = b["howl"][t], a["howl"][t] + r0, r1 = h0 / b["python"][t], h1 / b["python"][t] + hv = f"{h0} → {h1}" if h0 != h1 else f"{h0} (=)" + rv = f"{r0:.2f} → **{r1:.2f}**" if r1 < 1 else f"{r0:.2f} → {r1:.2f}" + cells += [hv, str(b["python"][t]), str(b["js"][t]), str(b["go"][t]), rv] + md.append("| " + " | ".join(cells) + " |") + (ROOT / "postcf/results/compare.md").write_text("\n".join(md) + "\n") + print("\n".join(md)) + + +if __name__ == "__main__": + main() diff --git a/experiments/token-efficiency-heldout-v2/count_tokens.py b/experiments/token-efficiency-heldout-v2/count_tokens.py index 27c5e13b..07af90ad 100644 --- a/experiments/token-efficiency-heldout-v2/count_tokens.py +++ b/experiments/token-efficiency-heldout-v2/count_tokens.py @@ -13,6 +13,14 @@ def main(): + # Defaults reproduce the frozen baseline; the post-control-flow set passes + # --howl-dir postcf/solutions/howl --results postcf/results. + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--howl-dir", default="solutions/howl") + ap.add_argument("--results", default="results") + opts = ap.parse_args() + res = ROOT / opts.results if tiktoken.__version__ != "0.8.0": raise RuntimeError("Install requirements.txt: tiktoken==0.8.0 required") tiktoken.load.read_file = v0.offline_read @@ -20,7 +28,7 @@ def main(): tasks = json.loads((ROOT / "tasks/manifest.json").read_text())["tasks"] cats = {t["id"]: t["category"] for t in tasks} gaps = {(g["task"], g["lang"]) for g in json.loads((ROOT / "gaps.json").read_text())} - checks = {(r["task"], r["lang"]): r for r in json.loads((ROOT / "results/verification.json").read_text())} + checks = {(r["task"], r["lang"]): r for r in json.loads((res / "verification.json").read_text())} expected = {(t["id"], l) for t in tasks for l in LANGUAGES} - gaps if set(checks) != expected: raise RuntimeError("verification coverage must be every task x language minus gaps.json") @@ -29,7 +37,8 @@ def main(): for lang, ext in LANGUAGES.items(): if (t["id"], lang) in gaps: continue - raw = (ROOT / f"solutions/{lang}/{t['id']}.{ext}").read_bytes() + src_dir = ROOT / opts.howl_dir if lang == "howl" else ROOT / f"solutions/{lang}" + raw = (src_dir / f"{t['id']}.{ext}").read_bytes() c = checks[t["id"], lang] if hashlib.sha256(raw).hexdigest() != c["sha256"] or not c["pass"]: raise RuntimeError(f"unverified or changed source {t['id']}/{lang}") @@ -40,8 +49,8 @@ def main(): "tokens": len(enc.encode(src, disallowed_special=())), "tokens_nocomments": len(enc.encode(stripped, disallowed_special=())), "bytes": len(raw), "pass": c["pass"]}) - (ROOT / "results/tokens.json").write_text(json.dumps(rows, indent=2) + "\n") - with (ROOT / "results/tokens.csv").open("w", newline="") as f: + (res / "tokens.json").write_text(json.dumps(rows, indent=2) + "\n") + with (res / "tokens.csv").open("w", newline="") as f: w = csv.DictWriter(f, fieldnames=list(rows[0])) w.writeheader() w.writerows(rows) @@ -74,7 +83,7 @@ def main(): "median_ratio_vs_python": statistics.median(by["howl"][t] / by["python"][t] for t in hs) if hs else None, } summary[enc][metric] = s - (ROOT / "results/summary.json").write_text(json.dumps(summary, indent=2) + "\n") + (res / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") print(json.dumps({e: summary[e]["tokens"]["howl_vs"] for e in summary}, indent=1)) diff --git a/experiments/token-efficiency-heldout-v2/postcf/results/compare.md b/experiments/token-efficiency-heldout-v2/postcf/results/compare.md new file mode 100644 index 00000000..e09f0b78 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/results/compare.md @@ -0,0 +1,81 @@ +### Headline (cl100k_base) + +| Comparison | Median per-task ratio before → after | Total-token ratio before → after | W/T/L before → after | +|---|---|---|---| +| HowlFrame vs python | 1.306 → **1.154** | 1.324 → **1.167** (4084 → 3599 / 3084) | 2/0/35 → **5/0/32** | +| HowlFrame vs js | 1.136 → **1.028** | 1.167 → **1.028** (4084 → 3599 / 3500) | 7/2/28 → **16/1/20** | +| HowlFrame vs go | 0.590 → **0.519** | 0.586 → **0.516** (4084 → 3599 / 6972) | 37/0/0 → **37/0/0** | + +### Per category (cl100k_base, HowlFrame / Python) + +| Category | n | Median per-task before → after | Total-token before → after | +|---|---|---|---| +| text | 4 | 1.254 → 1.097 | 1.314 → 1.117 | +| table | 4 | 1.279 → 1.248 | 1.378 → 1.248 | +| numbers | 4 | 1.433 → 1.090 | 1.470 → 1.153 | +| lists | 4 | 1.423 → 1.186 | 1.391 → 1.210 | +| strings | 4 | 1.344 → 1.138 | 1.321 → 1.220 | +| records | 4 | 1.190 → 1.190 | 1.242 → 1.242 | +| logic | 13 | 1.244 → 1.150 | 1.290 → 1.127 | + +### Headline (o200k_base) + +| Comparison | Median per-task ratio before → after | Total-token ratio before → after | W/T/L before → after | +|---|---|---|---| +| HowlFrame vs python | 1.306 → **1.164** | 1.321 → **1.165** (4095 → 3613 / 3100) | 2/0/35 → **5/0/32** | +| HowlFrame vs js | 1.083 → **0.968** | 1.112 → **0.981** (4095 → 3613 / 3683) | 10/1/26 → **20/0/17** | +| HowlFrame vs go | 0.582 → **0.520** | 0.583 → **0.514** (4095 → 3613 / 7030) | 37/0/0 → **37/0/0** | + +### Per category (o200k_base, HowlFrame / Python) + +| Category | n | Median per-task before → after | Total-token before → after | +|---|---|---|---| +| text | 4 | 1.254 → 1.097 | 1.314 → 1.117 | +| table | 4 | 1.247 → 1.228 | 1.348 → 1.224 | +| numbers | 4 | 1.433 → 1.096 | 1.470 → 1.160 | +| lists | 4 | 1.403 → 1.187 | 1.381 → 1.201 | +| strings | 4 | 1.351 → 1.145 | 1.325 → 1.224 | +| records | 4 | 1.197 → 1.197 | 1.248 → 1.248 | +| logic | 13 | 1.242 → 1.149 | 1.290 → 1.127 | + +### Per task (HowlFrame tokens before → after; cl100k_base and o200k_base) + +| Task | Cat | cl100k howl | py | js | go | howl/py | o200k howl | py | js | go | howl/py | +|---|---|---|---|---|---|---|---|---|---|---|---| +| w01 | text | 111 → 109 | 95 | 106 | 168 | 1.17 → 1.15 | 111 → 109 | 95 | 110 | 170 | 1.17 → 1.15 | +| w02 | text | 79 (=) | 59 | 61 | 133 | 1.34 → 1.34 | 79 (=) | 59 | 65 | 134 | 1.34 → 1.34 | +| w03 | text | 104 → 52 | 57 | 64 | 143 | 1.82 → **0.91** | 104 → 52 | 57 | 68 | 144 | 1.82 → **0.91** | +| w04 | text | 66 (=) | 63 | 80 | 163 | 1.05 → 1.05 | 66 (=) | 63 | 86 | 165 | 1.05 → 1.05 | +| w05 | table | 98 (=) | 76 | 88 | 177 | 1.29 → 1.29 | 98 (=) | 78 | 92 | 178 | 1.26 → 1.26 | +| w06 | table | 97 (=) | 79 | 101 | 179 | 1.23 → 1.23 | 97 (=) | 82 | 105 | 180 | 1.18 → 1.18 | +| w07 | table | 104 (=) | 82 | 91 | 202 | 1.27 → 1.27 | 104 (=) | 84 | 96 | 203 | 1.24 → 1.24 | +| w08 | table | 135 → 94 | 78 | 86 | 181 | 1.73 → 1.21 | 135 → 95 | 78 | 90 | 182 | 1.73 → 1.22 | +| w09 | numbers | 56 → 54 | 50 | 61 | 150 | 1.12 → 1.08 | 56 → 54 | 50 | 65 | 151 | 1.12 → 1.08 | +| w10 | numbers | 117 → 88 | 80 | 108 | 166 | 1.46 → 1.10 | 117 → 89 | 80 | 112 | 167 | 1.46 → 1.11 | +| w11 | numbers | 139 → 106 | 79 | 127 | 220 | 1.76 → 1.34 | 139 → 107 | 79 | 131 | 222 | 1.76 → 1.35 | +| w12 | numbers | 101 → 76 | 72 | 97 | 206 | 1.40 → 1.06 | 101 → 76 | 72 | 101 | 208 | 1.40 → 1.06 | +| w13 | lists | 79 → 70 | 62 | 78 | 171 | 1.27 → 1.13 | 78 → 69 | 61 | 81 | 171 | 1.28 → 1.13 | +| w14 | lists | 123 → 115 | 86 | 105 | 221 | 1.43 → 1.34 | 124 → 116 | 88 | 112 | 223 | 1.41 → 1.32 | +| w15 | lists | 94 → 82 | 66 | 74 | 205 | 1.42 → 1.24 | 94 → 82 | 66 | 80 | 207 | 1.42 → 1.24 | +| w16 | lists | 81 → 61 | 57 | 59 | 164 | 1.42 → 1.07 | 81 → 61 | 58 | 63 | 165 | 1.40 → 1.05 | +| w17 | strings | 92 (=) | 57 | 62 | 156 | 1.61 → 1.61 | 92 (=) | 57 | 66 | 158 | 1.61 → 1.61 | +| w18 | strings | 64 (=) | 63 | 64 | 165 | 1.02 → 1.02 | 64 (=) | 63 | 70 | 165 | 1.02 → 1.02 | +| w19 | strings | 91 → 82 | 69 | 73 | 129 | 1.32 → 1.19 | 92 → 83 | 69 | 81 | 131 | 1.33 → 1.20 | +| w20 | strings | 78 → 62 | 57 | 61 | 125 | 1.37 → 1.09 | 78 → 62 | 57 | 65 | 126 | 1.37 → 1.09 | +| w21 | records | 99 (=) | 65 | 69 | 174 | 1.52 → 1.52 | 99 (=) | 65 | 73 | 175 | 1.52 → 1.52 | +| w22 | records | 99 (=) | 88 | 104 | 226 | 1.12 → 1.12 | 100 (=) | 88 | 110 | 228 | 1.14 → 1.14 | +| w23 | records | 90 (=) | 78 | 83 | 198 | 1.15 → 1.15 | 91 (=) | 78 | 89 | 198 | 1.17 → 1.17 | +| w24 | records | 92 (=) | 75 | 81 | 190 | 1.23 → 1.23 | 92 (=) | 75 | 85 | 190 | 1.23 → 1.23 | +| w25 | logic | 128 → 66 | 77 | 102 | 188 | 1.66 → **0.86** | 128 → 67 | 77 | 106 | 190 | 1.66 → **0.87** | +| w26 | logic | 94 → 87 | 72 | 76 | 157 | 1.31 → 1.21 | 94 → 87 | 72 | 80 | 160 | 1.31 → 1.21 | +| w27 | logic | 91 → 79 | 95 | 98 | 184 | 0.96 → **0.83** | 92 → 80 | 95 | 102 | 185 | 0.97 → **0.84** | +| w28 | logic | 146 → 137 | 133 | 155 | 229 | 1.10 → 1.03 | 147 → 138 | 134 | 162 | 231 | 1.10 → 1.03 | +| w29 | logic | 108 → 105 | 90 | 118 | 214 | 1.20 → 1.17 | 108 → 105 | 90 | 124 | 217 | 1.20 → 1.17 | +| w30 | logic | 163 (=) | 140 | 147 | 254 | 1.16 → 1.16 | 163 (=) | 140 | 151 | 255 | 1.16 → 1.16 | +| w31 | logic | 104 → 41 | 49 | 54 | 151 | 2.12 → **0.84** | 104 → 41 | 49 | 59 | 152 | 2.12 → **0.84** | +| w32 | logic | 142 → 128 | 124 | 142 | 232 | 1.15 → 1.03 | 143 → 129 | 126 | 147 | 237 | 1.13 → 1.02 | +| w33 | logic | 143 → 142 | 95 | 113 | 252 | 1.51 → 1.49 | 145 → 143 | 96 | 120 | 255 | 1.51 → 1.49 | +| w34 | logic | 227 → 208 | 145 | 123 | 250 | 1.57 → 1.43 | 227 → 208 | 145 | 130 | 251 | 1.57 → 1.43 | +| w35 | logic | 140 → 122 | 100 | 123 | 230 | 1.40 → 1.22 | 141 → 123 | 101 | 130 | 234 | 1.40 → 1.22 | +| w36 | logic | 110 → 106 | 111 | 98 | 179 | 0.99 → **0.95** | 111 → 107 | 112 | 102 | 180 | 0.99 → **0.96** | +| w37 | logic | 199 → 184 | 160 | 168 | 240 | 1.24 → 1.15 | 200 → 185 | 161 | 174 | 242 | 1.24 → 1.15 | diff --git a/experiments/token-efficiency-heldout-v2/postcf/results/summary.json b/experiments/token-efficiency-heldout-v2/postcf/results/summary.json new file mode 100644 index 00000000..f32918b7 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/results/summary.json @@ -0,0 +1,466 @@ +{ + "cl100k_base": { + "tokens": { + "languages": { + "howl": { + "median_all": 92, + "median_howl_solved": 92, + "mean_howl_solved": 97.27027027027027 + }, + "python": { + "median_all": 78, + "median_howl_solved": 78, + "mean_howl_solved": 83.35135135135135 + }, + "js": { + "median_all": 91, + "median_howl_solved": 91, + "mean_howl_solved": 94.5945945945946 + }, + "go": { + "median_all": 181, + "median_howl_solved": 181, + "mean_howl_solved": 188.43243243243242 + } + }, + "howl_vs": { + "python": { + "median_per_task_ratio": 1.1538461538461537, + "ratio_of_medians": 1.1794871794871795, + "wins": 5, + "ties": 0, + "losses": 32, + "gaps": 0 + }, + "js": { + "median_per_task_ratio": 1.028301886792453, + "ratio_of_medians": 1.010989010989011, + "wins": 16, + "ties": 1, + "losses": 20, + "gaps": 0 + }, + "go": { + "median_per_task_ratio": 0.5193370165745856, + "ratio_of_medians": 0.5082872928176796, + "wins": 37, + "ties": 0, + "losses": 0, + "gaps": 0 + } + }, + "categories": { + "lists": { + "n": 4, + "howl_solved": 4, + "mean_howl": 82, + "mean_python": 67.75, + "mean_js": 79, + "mean_go": 190.25, + "median_ratio_vs_python": 1.1857282502443793 + }, + "logic": { + "n": 13, + "howl_solved": 13, + "mean_howl": 120.61538461538461, + "mean_python": 107, + "mean_js": 116.6923076923077, + "mean_go": 212.30769230769232, + "median_ratio_vs_python": 1.15 + }, + "numbers": { + "n": 4, + "howl_solved": 4, + "mean_howl": 81, + "mean_python": 70.25, + "mean_js": 98.25, + "mean_go": 185.5, + "median_ratio_vs_python": 1.09 + }, + "records": { + "n": 4, + "howl_solved": 4, + "mean_howl": 95, + "mean_python": 76.5, + "mean_js": 84.25, + "mean_go": 197, + "median_ratio_vs_python": 1.1902564102564102 + }, + "strings": { + "n": 4, + "howl_solved": 4, + "mean_howl": 75, + "mean_python": 61.5, + "mean_js": 65, + "mean_go": 143.75, + "median_ratio_vs_python": 1.1380625476735315 + }, + "table": { + "n": 4, + "howl_solved": 4, + "mean_howl": 98.25, + "mean_python": 78.75, + "mean_js": 91.5, + "mean_go": 184.75, + "median_ratio_vs_python": 1.248070392096326 + }, + "text": { + "n": 4, + "howl_solved": 4, + "mean_howl": 76.5, + "mean_python": 68.5, + "mean_js": 77.75, + "mean_go": 151.75, + "median_ratio_vs_python": 1.0974937343358397 + } + } + }, + "tokens_nocomments": { + "languages": { + "howl": { + "median_all": 92, + "median_howl_solved": 92, + "mean_howl_solved": 97.27027027027027 + }, + "python": { + "median_all": 78, + "median_howl_solved": 78, + "mean_howl_solved": 83.35135135135135 + }, + "js": { + "median_all": 91, + "median_howl_solved": 91, + "mean_howl_solved": 94.5945945945946 + }, + "go": { + "median_all": 175, + "median_howl_solved": 175, + "mean_howl_solved": 182.43243243243242 + } + }, + "howl_vs": { + "python": { + "median_per_task_ratio": 1.1538461538461537, + "ratio_of_medians": 1.1794871794871795, + "wins": 5, + "ties": 0, + "losses": 32, + "gaps": 0 + }, + "js": { + "median_per_task_ratio": 1.028301886792453, + "ratio_of_medians": 1.010989010989011, + "wins": 16, + "ties": 1, + "losses": 20, + "gaps": 0 + }, + "go": { + "median_per_task_ratio": 0.5348837209302325, + "ratio_of_medians": 0.5257142857142857, + "wins": 37, + "ties": 0, + "losses": 0, + "gaps": 0 + } + }, + "categories": { + "lists": { + "n": 4, + "howl_solved": 4, + "mean_howl": 82, + "mean_python": 67.75, + "mean_js": 79, + "mean_go": 184.25, + "median_ratio_vs_python": 1.1857282502443793 + }, + "logic": { + "n": 13, + "howl_solved": 13, + "mean_howl": 120.61538461538461, + "mean_python": 107, + "mean_js": 116.6923076923077, + "mean_go": 206.30769230769232, + "median_ratio_vs_python": 1.15 + }, + "numbers": { + "n": 4, + "howl_solved": 4, + "mean_howl": 81, + "mean_python": 70.25, + "mean_js": 98.25, + "mean_go": 179.5, + "median_ratio_vs_python": 1.09 + }, + "records": { + "n": 4, + "howl_solved": 4, + "mean_howl": 95, + "mean_python": 76.5, + "mean_js": 84.25, + "mean_go": 191, + "median_ratio_vs_python": 1.1902564102564102 + }, + "strings": { + "n": 4, + "howl_solved": 4, + "mean_howl": 75, + "mean_python": 61.5, + "mean_js": 65, + "mean_go": 137.75, + "median_ratio_vs_python": 1.1380625476735315 + }, + "table": { + "n": 4, + "howl_solved": 4, + "mean_howl": 98.25, + "mean_python": 78.75, + "mean_js": 91.5, + "mean_go": 178.75, + "median_ratio_vs_python": 1.248070392096326 + }, + "text": { + "n": 4, + "howl_solved": 4, + "mean_howl": 76.5, + "mean_python": 68.5, + "mean_js": 77.75, + "mean_go": 145.75, + "median_ratio_vs_python": 1.0974937343358397 + } + } + } + }, + "o200k_base": { + "tokens": { + "languages": { + "howl": { + "median_all": 92, + "median_howl_solved": 92, + "mean_howl_solved": 97.64864864864865 + }, + "python": { + "median_all": 78, + "median_howl_solved": 78, + "mean_howl_solved": 83.78378378378379 + }, + "js": { + "median_all": 96, + "median_howl_solved": 96, + "mean_howl_solved": 99.54054054054055 + }, + "go": { + "median_all": 182, + "median_howl_solved": 182, + "mean_howl_solved": 190 + } + }, + "howl_vs": { + "python": { + "median_per_task_ratio": 1.1642857142857144, + "ratio_of_medians": 1.1794871794871795, + "wins": 5, + "ties": 0, + "losses": 32, + "gaps": 0 + }, + "js": { + "median_per_task_ratio": 0.9682539682539683, + "ratio_of_medians": 0.9583333333333334, + "wins": 20, + "ties": 0, + "losses": 17, + "gaps": 0 + }, + "go": { + "median_per_task_ratio": 0.5201793721973094, + "ratio_of_medians": 0.5054945054945055, + "wins": 37, + "ties": 0, + "losses": 0, + "gaps": 0 + } + }, + "categories": { + "lists": { + "n": 4, + "howl_solved": 4, + "mean_howl": 82, + "mean_python": 68.25, + "mean_js": 84, + "mean_go": 191.5, + "median_ratio_vs_python": 1.1867858917039245 + }, + "logic": { + "n": 13, + "howl_solved": 13, + "mean_howl": 121.23076923076923, + "mean_python": 107.53846153846153, + "mean_js": 122.07692307692308, + "mean_go": 214.53846153846155, + "median_ratio_vs_python": 1.1490683229813665 + }, + "numbers": { + "n": 4, + "howl_solved": 4, + "mean_howl": 81.5, + "mean_python": 70.25, + "mean_js": 102.25, + "mean_go": 187, + "median_ratio_vs_python": 1.09625 + }, + "records": { + "n": 4, + "howl_solved": 4, + "mean_howl": 95.5, + "mean_python": 76.5, + "mean_js": 89.25, + "mean_go": 197.75, + "median_ratio_vs_python": 1.1966666666666668 + }, + "strings": { + "n": 4, + "howl_solved": 4, + "mean_howl": 75.25, + "mean_python": 61.5, + "mean_js": 70.5, + "mean_go": 145, + "median_ratio_vs_python": 1.1453089244851258 + }, + "table": { + "n": 4, + "howl_solved": 4, + "mean_howl": 98.5, + "mean_python": 80.5, + "mean_js": 95.75, + "mean_go": 185.75, + "median_ratio_vs_python": 1.228021978021978 + }, + "text": { + "n": 4, + "howl_solved": 4, + "mean_howl": 76.5, + "mean_python": 68.5, + "mean_js": 82.25, + "mean_go": 153.25, + "median_ratio_vs_python": 1.0974937343358397 + } + } + }, + "tokens_nocomments": { + "languages": { + "howl": { + "median_all": 92, + "median_howl_solved": 92, + "mean_howl_solved": 97.64864864864865 + }, + "python": { + "median_all": 78, + "median_howl_solved": 78, + "mean_howl_solved": 83.78378378378379 + }, + "js": { + "median_all": 96, + "median_howl_solved": 96, + "mean_howl_solved": 99.54054054054055 + }, + "go": { + "median_all": 176, + "median_howl_solved": 176, + "mean_howl_solved": 184 + } + }, + "howl_vs": { + "python": { + "median_per_task_ratio": 1.1642857142857144, + "ratio_of_medians": 1.1794871794871795, + "wins": 5, + "ties": 0, + "losses": 32, + "gaps": 0 + }, + "js": { + "median_per_task_ratio": 0.9682539682539683, + "ratio_of_medians": 0.9583333333333334, + "wins": 20, + "ties": 0, + "losses": 17, + "gaps": 0 + }, + "go": { + "median_per_task_ratio": 0.5345622119815668, + "ratio_of_medians": 0.5227272727272727, + "wins": 37, + "ties": 0, + "losses": 0, + "gaps": 0 + } + }, + "categories": { + "lists": { + "n": 4, + "howl_solved": 4, + "mean_howl": 82, + "mean_python": 68.25, + "mean_js": 84, + "mean_go": 185.5, + "median_ratio_vs_python": 1.1867858917039245 + }, + "logic": { + "n": 13, + "howl_solved": 13, + "mean_howl": 121.23076923076923, + "mean_python": 107.53846153846153, + "mean_js": 122.07692307692308, + "mean_go": 208.53846153846155, + "median_ratio_vs_python": 1.1490683229813665 + }, + "numbers": { + "n": 4, + "howl_solved": 4, + "mean_howl": 81.5, + "mean_python": 70.25, + "mean_js": 102.25, + "mean_go": 181, + "median_ratio_vs_python": 1.09625 + }, + "records": { + "n": 4, + "howl_solved": 4, + "mean_howl": 95.5, + "mean_python": 76.5, + "mean_js": 89.25, + "mean_go": 191.75, + "median_ratio_vs_python": 1.1966666666666668 + }, + "strings": { + "n": 4, + "howl_solved": 4, + "mean_howl": 75.25, + "mean_python": 61.5, + "mean_js": 70.5, + "mean_go": 139, + "median_ratio_vs_python": 1.1453089244851258 + }, + "table": { + "n": 4, + "howl_solved": 4, + "mean_howl": 98.5, + "mean_python": 80.5, + "mean_js": 95.75, + "mean_go": 179.75, + "median_ratio_vs_python": 1.228021978021978 + }, + "text": { + "n": 4, + "howl_solved": 4, + "mean_howl": 76.5, + "mean_python": 68.5, + "mean_js": 82.25, + "mean_go": 147.25, + "median_ratio_vs_python": 1.0974937343358397 + } + } + } + } +} diff --git a/experiments/token-efficiency-heldout-v2/postcf/results/tokens.csv b/experiments/token-efficiency-heldout-v2/postcf/results/tokens.csv new file mode 100644 index 00000000..7c7f2334 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/results/tokens.csv @@ -0,0 +1,297 @@ +task,category,lang,encoding,tokens,tokens_nocomments,bytes,pass +w01,text,howl,cl100k_base,109,109,312,True +w01,text,howl,o200k_base,109,109,312,True +w01,text,python,cl100k_base,95,95,325,True +w01,text,python,o200k_base,95,95,325,True +w01,text,js,cl100k_base,106,106,350,True +w01,text,js,o200k_base,110,110,350,True +w01,text,go,cl100k_base,168,162,526,True +w01,text,go,o200k_base,170,164,526,True +w02,text,howl,cl100k_base,79,79,227,True +w02,text,howl,o200k_base,79,79,227,True +w02,text,python,cl100k_base,59,59,211,True +w02,text,python,o200k_base,59,59,211,True +w02,text,js,cl100k_base,61,61,210,True +w02,text,js,o200k_base,65,65,210,True +w02,text,go,cl100k_base,133,127,429,True +w02,text,go,o200k_base,134,128,429,True +w03,text,howl,cl100k_base,52,52,162,True +w03,text,howl,o200k_base,52,52,162,True +w03,text,python,cl100k_base,57,57,211,True +w03,text,python,o200k_base,57,57,211,True +w03,text,js,cl100k_base,64,64,234,True +w03,text,js,o200k_base,68,68,234,True +w03,text,go,cl100k_base,143,137,476,True +w03,text,go,o200k_base,144,138,476,True +w04,text,howl,cl100k_base,66,66,199,True +w04,text,howl,o200k_base,66,66,199,True +w04,text,python,cl100k_base,63,63,210,True +w04,text,python,o200k_base,63,63,210,True +w04,text,js,cl100k_base,80,80,273,True +w04,text,js,o200k_base,86,86,273,True +w04,text,go,cl100k_base,163,157,525,True +w04,text,go,o200k_base,165,159,525,True +w05,table,howl,cl100k_base,98,98,271,True +w05,table,howl,o200k_base,98,98,271,True +w05,table,python,cl100k_base,76,76,273,True +w05,table,python,o200k_base,78,78,273,True +w05,table,js,cl100k_base,88,88,305,True +w05,table,js,o200k_base,92,92,305,True +w05,table,go,cl100k_base,177,171,577,True +w05,table,go,o200k_base,178,172,577,True +w06,table,howl,cl100k_base,97,97,265,True +w06,table,howl,o200k_base,97,97,265,True +w06,table,python,cl100k_base,79,79,272,True +w06,table,python,o200k_base,82,82,272,True +w06,table,js,cl100k_base,101,101,332,True +w06,table,js,o200k_base,105,105,332,True +w06,table,go,cl100k_base,179,173,581,True +w06,table,go,o200k_base,180,174,581,True +w07,table,howl,cl100k_base,104,104,292,True +w07,table,howl,o200k_base,104,104,292,True +w07,table,python,cl100k_base,82,82,271,True +w07,table,python,o200k_base,84,84,271,True +w07,table,js,cl100k_base,91,91,321,True +w07,table,js,o200k_base,96,96,321,True +w07,table,go,cl100k_base,202,196,636,True +w07,table,go,o200k_base,203,197,636,True +w08,table,howl,cl100k_base,94,94,272,True +w08,table,howl,o200k_base,95,95,272,True +w08,table,python,cl100k_base,78,78,262,True +w08,table,python,o200k_base,78,78,262,True +w08,table,js,cl100k_base,86,86,311,True +w08,table,js,o200k_base,90,90,311,True +w08,table,go,cl100k_base,181,175,575,True +w08,table,go,o200k_base,182,176,575,True +w09,numbers,howl,cl100k_base,54,54,158,True +w09,numbers,howl,o200k_base,54,54,158,True +w09,numbers,python,cl100k_base,50,50,173,True +w09,numbers,python,o200k_base,50,50,173,True +w09,numbers,js,cl100k_base,61,61,213,True +w09,numbers,js,o200k_base,65,65,213,True +w09,numbers,go,cl100k_base,150,144,497,True +w09,numbers,go,o200k_base,151,145,497,True +w10,numbers,howl,cl100k_base,88,88,226,True +w10,numbers,howl,o200k_base,89,89,226,True +w10,numbers,python,cl100k_base,80,80,229,True +w10,numbers,python,o200k_base,80,80,229,True +w10,numbers,js,cl100k_base,108,108,326,True +w10,numbers,js,o200k_base,112,112,326,True +w10,numbers,go,cl100k_base,166,160,510,True +w10,numbers,go,o200k_base,167,161,510,True +w11,numbers,howl,cl100k_base,106,106,278,True +w11,numbers,howl,o200k_base,107,107,278,True +w11,numbers,python,cl100k_base,79,79,264,True +w11,numbers,python,o200k_base,79,79,264,True +w11,numbers,js,cl100k_base,127,127,383,True +w11,numbers,js,o200k_base,131,131,383,True +w11,numbers,go,cl100k_base,220,214,696,True +w11,numbers,go,o200k_base,222,216,696,True +w12,numbers,howl,cl100k_base,76,76,218,True +w12,numbers,howl,o200k_base,76,76,218,True +w12,numbers,python,cl100k_base,72,72,226,True +w12,numbers,python,o200k_base,72,72,226,True +w12,numbers,js,cl100k_base,97,97,293,True +w12,numbers,js,o200k_base,101,101,293,True +w12,numbers,go,cl100k_base,206,200,628,True +w12,numbers,go,o200k_base,208,202,628,True +w13,lists,howl,cl100k_base,70,70,191,True +w13,lists,howl,o200k_base,69,69,191,True +w13,lists,python,cl100k_base,62,62,203,True +w13,lists,python,o200k_base,61,61,203,True +w13,lists,js,cl100k_base,78,78,256,True +w13,lists,js,o200k_base,81,81,256,True +w13,lists,go,cl100k_base,171,165,536,True +w13,lists,go,o200k_base,171,165,536,True +w14,lists,howl,cl100k_base,115,115,362,True +w14,lists,howl,o200k_base,116,116,362,True +w14,lists,python,cl100k_base,86,86,308,True +w14,lists,python,o200k_base,88,88,308,True +w14,lists,js,cl100k_base,105,105,374,True +w14,lists,js,o200k_base,112,112,374,True +w14,lists,go,cl100k_base,221,215,714,True +w14,lists,go,o200k_base,223,217,714,True +w15,lists,howl,cl100k_base,82,82,247,True +w15,lists,howl,o200k_base,82,82,247,True +w15,lists,python,cl100k_base,66,66,225,True +w15,lists,python,o200k_base,66,66,225,True +w15,lists,js,cl100k_base,74,74,272,True +w15,lists,js,o200k_base,80,80,272,True +w15,lists,go,cl100k_base,205,199,682,True +w15,lists,go,o200k_base,207,201,682,True +w16,lists,howl,cl100k_base,61,61,184,True +w16,lists,howl,o200k_base,61,61,184,True +w16,lists,python,cl100k_base,57,57,187,True +w16,lists,python,o200k_base,58,58,187,True +w16,lists,js,cl100k_base,59,59,196,True +w16,lists,js,o200k_base,63,63,196,True +w16,lists,go,cl100k_base,164,158,501,True +w16,lists,go,o200k_base,165,159,501,True +w17,strings,howl,cl100k_base,92,92,255,True +w17,strings,howl,o200k_base,92,92,255,True +w17,strings,python,cl100k_base,57,57,198,True +w17,strings,python,o200k_base,57,57,198,True +w17,strings,js,cl100k_base,62,62,206,True +w17,strings,js,o200k_base,66,66,206,True +w17,strings,go,cl100k_base,156,150,465,True +w17,strings,go,o200k_base,158,152,465,True +w18,strings,howl,cl100k_base,64,64,194,True +w18,strings,howl,o200k_base,64,64,194,True +w18,strings,python,cl100k_base,63,63,212,True +w18,strings,python,o200k_base,63,63,212,True +w18,strings,js,cl100k_base,64,64,224,True +w18,strings,js,o200k_base,70,70,224,True +w18,strings,go,cl100k_base,165,159,538,True +w18,strings,go,o200k_base,165,159,538,True +w19,strings,howl,cl100k_base,82,82,260,True +w19,strings,howl,o200k_base,83,83,260,True +w19,strings,python,cl100k_base,69,69,194,True +w19,strings,python,o200k_base,69,69,194,True +w19,strings,js,cl100k_base,73,73,236,True +w19,strings,js,o200k_base,81,81,236,True +w19,strings,go,cl100k_base,129,123,397,True +w19,strings,go,o200k_base,131,125,397,True +w20,strings,howl,cl100k_base,62,62,181,True +w20,strings,howl,o200k_base,62,62,181,True +w20,strings,python,cl100k_base,57,57,184,True +w20,strings,python,o200k_base,57,57,184,True +w20,strings,js,cl100k_base,61,61,215,True +w20,strings,js,o200k_base,65,65,215,True +w20,strings,go,cl100k_base,125,119,416,True +w20,strings,go,o200k_base,126,120,416,True +w21,records,howl,cl100k_base,99,99,295,True +w21,records,howl,o200k_base,99,99,295,True +w21,records,python,cl100k_base,65,65,252,True +w21,records,python,o200k_base,65,65,252,True +w21,records,js,cl100k_base,69,69,246,True +w21,records,js,o200k_base,73,73,246,True +w21,records,go,cl100k_base,174,168,591,True +w21,records,go,o200k_base,175,169,591,True +w22,records,howl,cl100k_base,99,99,308,True +w22,records,howl,o200k_base,100,100,308,True +w22,records,python,cl100k_base,88,88,306,True +w22,records,python,o200k_base,88,88,306,True +w22,records,js,cl100k_base,104,104,358,True +w22,records,js,o200k_base,110,110,358,True +w22,records,go,cl100k_base,226,220,748,True +w22,records,go,o200k_base,228,222,748,True +w23,records,howl,cl100k_base,90,90,273,True +w23,records,howl,o200k_base,91,91,273,True +w23,records,python,cl100k_base,78,78,277,True +w23,records,python,o200k_base,78,78,277,True +w23,records,js,cl100k_base,83,83,287,True +w23,records,js,o200k_base,89,89,287,True +w23,records,go,cl100k_base,198,192,638,True +w23,records,go,o200k_base,198,192,638,True +w24,records,howl,cl100k_base,92,92,270,True +w24,records,howl,o200k_base,92,92,270,True +w24,records,python,cl100k_base,75,75,271,True +w24,records,python,o200k_base,75,75,271,True +w24,records,js,cl100k_base,81,81,276,True +w24,records,js,o200k_base,85,85,276,True +w24,records,go,cl100k_base,190,184,603,True +w24,records,go,o200k_base,190,184,603,True +w25,logic,howl,cl100k_base,66,66,175,True +w25,logic,howl,o200k_base,67,67,175,True +w25,logic,python,cl100k_base,77,77,220,True +w25,logic,python,o200k_base,77,77,220,True +w25,logic,js,cl100k_base,102,102,311,True +w25,logic,js,o200k_base,106,106,311,True +w25,logic,go,cl100k_base,188,182,569,True +w25,logic,go,o200k_base,190,184,569,True +w26,logic,howl,cl100k_base,87,87,204,True +w26,logic,howl,o200k_base,87,87,204,True +w26,logic,python,cl100k_base,72,72,195,True +w26,logic,python,o200k_base,72,72,195,True +w26,logic,js,cl100k_base,76,76,215,True +w26,logic,js,o200k_base,80,80,215,True +w26,logic,go,cl100k_base,157,151,458,True +w26,logic,go,o200k_base,160,154,458,True +w27,logic,howl,cl100k_base,79,79,208,True +w27,logic,howl,o200k_base,80,80,208,True +w27,logic,python,cl100k_base,95,95,279,True +w27,logic,python,o200k_base,95,95,279,True +w27,logic,js,cl100k_base,98,98,282,True +w27,logic,js,o200k_base,102,102,282,True +w27,logic,go,cl100k_base,184,178,548,True +w27,logic,go,o200k_base,185,179,548,True +w28,logic,howl,cl100k_base,137,137,409,True +w28,logic,howl,o200k_base,138,138,409,True +w28,logic,python,cl100k_base,133,133,462,True +w28,logic,python,o200k_base,134,134,462,True +w28,logic,js,cl100k_base,155,155,518,True +w28,logic,js,o200k_base,162,162,518,True +w28,logic,go,cl100k_base,229,223,712,True +w28,logic,go,o200k_base,231,225,712,True +w29,logic,howl,cl100k_base,105,105,304,True +w29,logic,howl,o200k_base,105,105,304,True +w29,logic,python,cl100k_base,90,90,319,True +w29,logic,python,o200k_base,90,90,319,True +w29,logic,js,cl100k_base,118,118,403,True +w29,logic,js,o200k_base,124,124,403,True +w29,logic,go,cl100k_base,214,208,696,True +w29,logic,go,o200k_base,217,211,696,True +w30,logic,howl,cl100k_base,163,163,413,True +w30,logic,howl,o200k_base,163,163,413,True +w30,logic,python,cl100k_base,140,140,379,True +w30,logic,python,o200k_base,140,140,379,True +w30,logic,js,cl100k_base,147,147,378,True +w30,logic,js,o200k_base,151,151,378,True +w30,logic,go,cl100k_base,254,248,715,True +w30,logic,go,o200k_base,255,249,715,True +w31,logic,howl,cl100k_base,41,41,121,True +w31,logic,howl,o200k_base,41,41,121,True +w31,logic,python,cl100k_base,49,49,166,True +w31,logic,python,o200k_base,49,49,166,True +w31,logic,js,cl100k_base,54,54,188,True +w31,logic,js,o200k_base,59,59,188,True +w31,logic,go,cl100k_base,151,145,483,True +w31,logic,go,o200k_base,152,146,483,True +w32,logic,howl,cl100k_base,128,128,335,True +w32,logic,howl,o200k_base,129,129,335,True +w32,logic,python,cl100k_base,124,124,375,True +w32,logic,python,o200k_base,126,126,375,True +w32,logic,js,cl100k_base,142,142,404,True +w32,logic,js,o200k_base,147,147,404,True +w32,logic,go,cl100k_base,232,226,674,True +w32,logic,go,o200k_base,237,231,674,True +w33,logic,howl,cl100k_base,142,142,380,True +w33,logic,howl,o200k_base,143,143,380,True +w33,logic,python,cl100k_base,95,95,326,True +w33,logic,python,o200k_base,96,96,326,True +w33,logic,js,cl100k_base,113,113,371,True +w33,logic,js,o200k_base,120,120,371,True +w33,logic,go,cl100k_base,252,246,781,True +w33,logic,go,o200k_base,255,249,781,True +w34,logic,howl,cl100k_base,208,208,506,True +w34,logic,howl,o200k_base,208,208,506,True +w34,logic,python,cl100k_base,145,145,378,True +w34,logic,python,o200k_base,145,145,378,True +w34,logic,js,cl100k_base,123,123,359,True +w34,logic,js,o200k_base,130,130,359,True +w34,logic,go,cl100k_base,250,244,642,True +w34,logic,go,o200k_base,251,245,642,True +w35,logic,howl,cl100k_base,122,122,357,True +w35,logic,howl,o200k_base,123,123,357,True +w35,logic,python,cl100k_base,100,100,336,True +w35,logic,python,o200k_base,101,101,336,True +w35,logic,js,cl100k_base,123,123,420,True +w35,logic,js,o200k_base,130,130,420,True +w35,logic,go,cl100k_base,230,224,742,True +w35,logic,go,o200k_base,234,228,742,True +w36,logic,howl,cl100k_base,106,106,339,True +w36,logic,howl,o200k_base,107,107,339,True +w36,logic,python,cl100k_base,111,111,375,True +w36,logic,python,o200k_base,112,112,375,True +w36,logic,js,cl100k_base,98,98,292,True +w36,logic,js,o200k_base,102,102,292,True +w36,logic,go,cl100k_base,179,173,624,True +w36,logic,go,o200k_base,180,174,624,True +w37,logic,howl,cl100k_base,184,184,428,True +w37,logic,howl,o200k_base,185,185,428,True +w37,logic,python,cl100k_base,160,160,358,True +w37,logic,python,o200k_base,161,161,358,True +w37,logic,js,cl100k_base,168,168,386,True +w37,logic,js,o200k_base,174,174,386,True +w37,logic,go,cl100k_base,240,234,591,True +w37,logic,go,o200k_base,242,236,591,True diff --git a/experiments/token-efficiency-heldout-v2/postcf/results/tokens.json b/experiments/token-efficiency-heldout-v2/postcf/results/tokens.json new file mode 100644 index 00000000..2fc0aa70 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/results/tokens.json @@ -0,0 +1,2962 @@ +[ + { + "task": "w01", + "category": "text", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 109, + "tokens_nocomments": 109, + "bytes": 312, + "pass": true + }, + { + "task": "w01", + "category": "text", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 109, + "tokens_nocomments": 109, + "bytes": 312, + "pass": true + }, + { + "task": "w01", + "category": "text", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 95, + "tokens_nocomments": 95, + "bytes": 325, + "pass": true + }, + { + "task": "w01", + "category": "text", + "lang": "python", + "encoding": "o200k_base", + "tokens": 95, + "tokens_nocomments": 95, + "bytes": 325, + "pass": true + }, + { + "task": "w01", + "category": "text", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 106, + "tokens_nocomments": 106, + "bytes": 350, + "pass": true + }, + { + "task": "w01", + "category": "text", + "lang": "js", + "encoding": "o200k_base", + "tokens": 110, + "tokens_nocomments": 110, + "bytes": 350, + "pass": true + }, + { + "task": "w01", + "category": "text", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 168, + "tokens_nocomments": 162, + "bytes": 526, + "pass": true + }, + { + "task": "w01", + "category": "text", + "lang": "go", + "encoding": "o200k_base", + "tokens": 170, + "tokens_nocomments": 164, + "bytes": 526, + "pass": true + }, + { + "task": "w02", + "category": "text", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 79, + "tokens_nocomments": 79, + "bytes": 227, + "pass": true + }, + { + "task": "w02", + "category": "text", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 79, + "tokens_nocomments": 79, + "bytes": 227, + "pass": true + }, + { + "task": "w02", + "category": "text", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 59, + "tokens_nocomments": 59, + "bytes": 211, + "pass": true + }, + { + "task": "w02", + "category": "text", + "lang": "python", + "encoding": "o200k_base", + "tokens": 59, + "tokens_nocomments": 59, + "bytes": 211, + "pass": true + }, + { + "task": "w02", + "category": "text", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 61, + "tokens_nocomments": 61, + "bytes": 210, + "pass": true + }, + { + "task": "w02", + "category": "text", + "lang": "js", + "encoding": "o200k_base", + "tokens": 65, + "tokens_nocomments": 65, + "bytes": 210, + "pass": true + }, + { + "task": "w02", + "category": "text", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 133, + "tokens_nocomments": 127, + "bytes": 429, + "pass": true + }, + { + "task": "w02", + "category": "text", + "lang": "go", + "encoding": "o200k_base", + "tokens": 134, + "tokens_nocomments": 128, + "bytes": 429, + "pass": true + }, + { + "task": "w03", + "category": "text", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 52, + "tokens_nocomments": 52, + "bytes": 162, + "pass": true + }, + { + "task": "w03", + "category": "text", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 52, + "tokens_nocomments": 52, + "bytes": 162, + "pass": true + }, + { + "task": "w03", + "category": "text", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 57, + "tokens_nocomments": 57, + "bytes": 211, + "pass": true + }, + { + "task": "w03", + "category": "text", + "lang": "python", + "encoding": "o200k_base", + "tokens": 57, + "tokens_nocomments": 57, + "bytes": 211, + "pass": true + }, + { + "task": "w03", + "category": "text", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 64, + "tokens_nocomments": 64, + "bytes": 234, + "pass": true + }, + { + "task": "w03", + "category": "text", + "lang": "js", + "encoding": "o200k_base", + "tokens": 68, + "tokens_nocomments": 68, + "bytes": 234, + "pass": true + }, + { + "task": "w03", + "category": "text", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 143, + "tokens_nocomments": 137, + "bytes": 476, + "pass": true + }, + { + "task": "w03", + "category": "text", + "lang": "go", + "encoding": "o200k_base", + "tokens": 144, + "tokens_nocomments": 138, + "bytes": 476, + "pass": true + }, + { + "task": "w04", + "category": "text", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 66, + "tokens_nocomments": 66, + "bytes": 199, + "pass": true + }, + { + "task": "w04", + "category": "text", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 66, + "tokens_nocomments": 66, + "bytes": 199, + "pass": true + }, + { + "task": "w04", + "category": "text", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 63, + "tokens_nocomments": 63, + "bytes": 210, + "pass": true + }, + { + "task": "w04", + "category": "text", + "lang": "python", + "encoding": "o200k_base", + "tokens": 63, + "tokens_nocomments": 63, + "bytes": 210, + "pass": true + }, + { + "task": "w04", + "category": "text", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 80, + "tokens_nocomments": 80, + "bytes": 273, + "pass": true + }, + { + "task": "w04", + "category": "text", + "lang": "js", + "encoding": "o200k_base", + "tokens": 86, + "tokens_nocomments": 86, + "bytes": 273, + "pass": true + }, + { + "task": "w04", + "category": "text", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 163, + "tokens_nocomments": 157, + "bytes": 525, + "pass": true + }, + { + "task": "w04", + "category": "text", + "lang": "go", + "encoding": "o200k_base", + "tokens": 165, + "tokens_nocomments": 159, + "bytes": 525, + "pass": true + }, + { + "task": "w05", + "category": "table", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 98, + "tokens_nocomments": 98, + "bytes": 271, + "pass": true + }, + { + "task": "w05", + "category": "table", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 98, + "tokens_nocomments": 98, + "bytes": 271, + "pass": true + }, + { + "task": "w05", + "category": "table", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 76, + "tokens_nocomments": 76, + "bytes": 273, + "pass": true + }, + { + "task": "w05", + "category": "table", + "lang": "python", + "encoding": "o200k_base", + "tokens": 78, + "tokens_nocomments": 78, + "bytes": 273, + "pass": true + }, + { + "task": "w05", + "category": "table", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 88, + "tokens_nocomments": 88, + "bytes": 305, + "pass": true + }, + { + "task": "w05", + "category": "table", + "lang": "js", + "encoding": "o200k_base", + "tokens": 92, + "tokens_nocomments": 92, + "bytes": 305, + "pass": true + }, + { + "task": "w05", + "category": "table", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 177, + "tokens_nocomments": 171, + "bytes": 577, + "pass": true + }, + { + "task": "w05", + "category": "table", + "lang": "go", + "encoding": "o200k_base", + "tokens": 178, + "tokens_nocomments": 172, + "bytes": 577, + "pass": true + }, + { + "task": "w06", + "category": "table", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 97, + "tokens_nocomments": 97, + "bytes": 265, + "pass": true + }, + { + "task": "w06", + "category": "table", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 97, + "tokens_nocomments": 97, + "bytes": 265, + "pass": true + }, + { + "task": "w06", + "category": "table", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 79, + "tokens_nocomments": 79, + "bytes": 272, + "pass": true + }, + { + "task": "w06", + "category": "table", + "lang": "python", + "encoding": "o200k_base", + "tokens": 82, + "tokens_nocomments": 82, + "bytes": 272, + "pass": true + }, + { + "task": "w06", + "category": "table", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 101, + "tokens_nocomments": 101, + "bytes": 332, + "pass": true + }, + { + "task": "w06", + "category": "table", + "lang": "js", + "encoding": "o200k_base", + "tokens": 105, + "tokens_nocomments": 105, + "bytes": 332, + "pass": true + }, + { + "task": "w06", + "category": "table", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 179, + "tokens_nocomments": 173, + "bytes": 581, + "pass": true + }, + { + "task": "w06", + "category": "table", + "lang": "go", + "encoding": "o200k_base", + "tokens": 180, + "tokens_nocomments": 174, + "bytes": 581, + "pass": true + }, + { + "task": "w07", + "category": "table", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 104, + "tokens_nocomments": 104, + "bytes": 292, + "pass": true + }, + { + "task": "w07", + "category": "table", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 104, + "tokens_nocomments": 104, + "bytes": 292, + "pass": true + }, + { + "task": "w07", + "category": "table", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 82, + "tokens_nocomments": 82, + "bytes": 271, + "pass": true + }, + { + "task": "w07", + "category": "table", + "lang": "python", + "encoding": "o200k_base", + "tokens": 84, + "tokens_nocomments": 84, + "bytes": 271, + "pass": true + }, + { + "task": "w07", + "category": "table", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 91, + "tokens_nocomments": 91, + "bytes": 321, + "pass": true + }, + { + "task": "w07", + "category": "table", + "lang": "js", + "encoding": "o200k_base", + "tokens": 96, + "tokens_nocomments": 96, + "bytes": 321, + "pass": true + }, + { + "task": "w07", + "category": "table", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 202, + "tokens_nocomments": 196, + "bytes": 636, + "pass": true + }, + { + "task": "w07", + "category": "table", + "lang": "go", + "encoding": "o200k_base", + "tokens": 203, + "tokens_nocomments": 197, + "bytes": 636, + "pass": true + }, + { + "task": "w08", + "category": "table", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 94, + "tokens_nocomments": 94, + "bytes": 272, + "pass": true + }, + { + "task": "w08", + "category": "table", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 95, + "tokens_nocomments": 95, + "bytes": 272, + "pass": true + }, + { + "task": "w08", + "category": "table", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 78, + "tokens_nocomments": 78, + "bytes": 262, + "pass": true + }, + { + "task": "w08", + "category": "table", + "lang": "python", + "encoding": "o200k_base", + "tokens": 78, + "tokens_nocomments": 78, + "bytes": 262, + "pass": true + }, + { + "task": "w08", + "category": "table", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 86, + "tokens_nocomments": 86, + "bytes": 311, + "pass": true + }, + { + "task": "w08", + "category": "table", + "lang": "js", + "encoding": "o200k_base", + "tokens": 90, + "tokens_nocomments": 90, + "bytes": 311, + "pass": true + }, + { + "task": "w08", + "category": "table", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 181, + "tokens_nocomments": 175, + "bytes": 575, + "pass": true + }, + { + "task": "w08", + "category": "table", + "lang": "go", + "encoding": "o200k_base", + "tokens": 182, + "tokens_nocomments": 176, + "bytes": 575, + "pass": true + }, + { + "task": "w09", + "category": "numbers", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 54, + "tokens_nocomments": 54, + "bytes": 158, + "pass": true + }, + { + "task": "w09", + "category": "numbers", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 54, + "tokens_nocomments": 54, + "bytes": 158, + "pass": true + }, + { + "task": "w09", + "category": "numbers", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 50, + "tokens_nocomments": 50, + "bytes": 173, + "pass": true + }, + { + "task": "w09", + "category": "numbers", + "lang": "python", + "encoding": "o200k_base", + "tokens": 50, + "tokens_nocomments": 50, + "bytes": 173, + "pass": true + }, + { + "task": "w09", + "category": "numbers", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 61, + "tokens_nocomments": 61, + "bytes": 213, + "pass": true + }, + { + "task": "w09", + "category": "numbers", + "lang": "js", + "encoding": "o200k_base", + "tokens": 65, + "tokens_nocomments": 65, + "bytes": 213, + "pass": true + }, + { + "task": "w09", + "category": "numbers", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 150, + "tokens_nocomments": 144, + "bytes": 497, + "pass": true + }, + { + "task": "w09", + "category": "numbers", + "lang": "go", + "encoding": "o200k_base", + "tokens": 151, + "tokens_nocomments": 145, + "bytes": 497, + "pass": true + }, + { + "task": "w10", + "category": "numbers", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 88, + "tokens_nocomments": 88, + "bytes": 226, + "pass": true + }, + { + "task": "w10", + "category": "numbers", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 89, + "tokens_nocomments": 89, + "bytes": 226, + "pass": true + }, + { + "task": "w10", + "category": "numbers", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 80, + "tokens_nocomments": 80, + "bytes": 229, + "pass": true + }, + { + "task": "w10", + "category": "numbers", + "lang": "python", + "encoding": "o200k_base", + "tokens": 80, + "tokens_nocomments": 80, + "bytes": 229, + "pass": true + }, + { + "task": "w10", + "category": "numbers", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 108, + "tokens_nocomments": 108, + "bytes": 326, + "pass": true + }, + { + "task": "w10", + "category": "numbers", + "lang": "js", + "encoding": "o200k_base", + "tokens": 112, + "tokens_nocomments": 112, + "bytes": 326, + "pass": true + }, + { + "task": "w10", + "category": "numbers", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 166, + "tokens_nocomments": 160, + "bytes": 510, + "pass": true + }, + { + "task": "w10", + "category": "numbers", + "lang": "go", + "encoding": "o200k_base", + "tokens": 167, + "tokens_nocomments": 161, + "bytes": 510, + "pass": true + }, + { + "task": "w11", + "category": "numbers", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 106, + "tokens_nocomments": 106, + "bytes": 278, + "pass": true + }, + { + "task": "w11", + "category": "numbers", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 107, + "tokens_nocomments": 107, + "bytes": 278, + "pass": true + }, + { + "task": "w11", + "category": "numbers", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 79, + "tokens_nocomments": 79, + "bytes": 264, + "pass": true + }, + { + "task": "w11", + "category": "numbers", + "lang": "python", + "encoding": "o200k_base", + "tokens": 79, + "tokens_nocomments": 79, + "bytes": 264, + "pass": true + }, + { + "task": "w11", + "category": "numbers", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 127, + "tokens_nocomments": 127, + "bytes": 383, + "pass": true + }, + { + "task": "w11", + "category": "numbers", + "lang": "js", + "encoding": "o200k_base", + "tokens": 131, + "tokens_nocomments": 131, + "bytes": 383, + "pass": true + }, + { + "task": "w11", + "category": "numbers", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 220, + "tokens_nocomments": 214, + "bytes": 696, + "pass": true + }, + { + "task": "w11", + "category": "numbers", + "lang": "go", + "encoding": "o200k_base", + "tokens": 222, + "tokens_nocomments": 216, + "bytes": 696, + "pass": true + }, + { + "task": "w12", + "category": "numbers", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 76, + "tokens_nocomments": 76, + "bytes": 218, + "pass": true + }, + { + "task": "w12", + "category": "numbers", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 76, + "tokens_nocomments": 76, + "bytes": 218, + "pass": true + }, + { + "task": "w12", + "category": "numbers", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 72, + "tokens_nocomments": 72, + "bytes": 226, + "pass": true + }, + { + "task": "w12", + "category": "numbers", + "lang": "python", + "encoding": "o200k_base", + "tokens": 72, + "tokens_nocomments": 72, + "bytes": 226, + "pass": true + }, + { + "task": "w12", + "category": "numbers", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 97, + "tokens_nocomments": 97, + "bytes": 293, + "pass": true + }, + { + "task": "w12", + "category": "numbers", + "lang": "js", + "encoding": "o200k_base", + "tokens": 101, + "tokens_nocomments": 101, + "bytes": 293, + "pass": true + }, + { + "task": "w12", + "category": "numbers", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 206, + "tokens_nocomments": 200, + "bytes": 628, + "pass": true + }, + { + "task": "w12", + "category": "numbers", + "lang": "go", + "encoding": "o200k_base", + "tokens": 208, + "tokens_nocomments": 202, + "bytes": 628, + "pass": true + }, + { + "task": "w13", + "category": "lists", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 70, + "tokens_nocomments": 70, + "bytes": 191, + "pass": true + }, + { + "task": "w13", + "category": "lists", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 69, + "tokens_nocomments": 69, + "bytes": 191, + "pass": true + }, + { + "task": "w13", + "category": "lists", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 62, + "tokens_nocomments": 62, + "bytes": 203, + "pass": true + }, + { + "task": "w13", + "category": "lists", + "lang": "python", + "encoding": "o200k_base", + "tokens": 61, + "tokens_nocomments": 61, + "bytes": 203, + "pass": true + }, + { + "task": "w13", + "category": "lists", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 78, + "tokens_nocomments": 78, + "bytes": 256, + "pass": true + }, + { + "task": "w13", + "category": "lists", + "lang": "js", + "encoding": "o200k_base", + "tokens": 81, + "tokens_nocomments": 81, + "bytes": 256, + "pass": true + }, + { + "task": "w13", + "category": "lists", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 171, + "tokens_nocomments": 165, + "bytes": 536, + "pass": true + }, + { + "task": "w13", + "category": "lists", + "lang": "go", + "encoding": "o200k_base", + "tokens": 171, + "tokens_nocomments": 165, + "bytes": 536, + "pass": true + }, + { + "task": "w14", + "category": "lists", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 115, + "tokens_nocomments": 115, + "bytes": 362, + "pass": true + }, + { + "task": "w14", + "category": "lists", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 116, + "tokens_nocomments": 116, + "bytes": 362, + "pass": true + }, + { + "task": "w14", + "category": "lists", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 86, + "tokens_nocomments": 86, + "bytes": 308, + "pass": true + }, + { + "task": "w14", + "category": "lists", + "lang": "python", + "encoding": "o200k_base", + "tokens": 88, + "tokens_nocomments": 88, + "bytes": 308, + "pass": true + }, + { + "task": "w14", + "category": "lists", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 105, + "tokens_nocomments": 105, + "bytes": 374, + "pass": true + }, + { + "task": "w14", + "category": "lists", + "lang": "js", + "encoding": "o200k_base", + "tokens": 112, + "tokens_nocomments": 112, + "bytes": 374, + "pass": true + }, + { + "task": "w14", + "category": "lists", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 221, + "tokens_nocomments": 215, + "bytes": 714, + "pass": true + }, + { + "task": "w14", + "category": "lists", + "lang": "go", + "encoding": "o200k_base", + "tokens": 223, + "tokens_nocomments": 217, + "bytes": 714, + "pass": true + }, + { + "task": "w15", + "category": "lists", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 82, + "tokens_nocomments": 82, + "bytes": 247, + "pass": true + }, + { + "task": "w15", + "category": "lists", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 82, + "tokens_nocomments": 82, + "bytes": 247, + "pass": true + }, + { + "task": "w15", + "category": "lists", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 66, + "tokens_nocomments": 66, + "bytes": 225, + "pass": true + }, + { + "task": "w15", + "category": "lists", + "lang": "python", + "encoding": "o200k_base", + "tokens": 66, + "tokens_nocomments": 66, + "bytes": 225, + "pass": true + }, + { + "task": "w15", + "category": "lists", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 74, + "tokens_nocomments": 74, + "bytes": 272, + "pass": true + }, + { + "task": "w15", + "category": "lists", + "lang": "js", + "encoding": "o200k_base", + "tokens": 80, + "tokens_nocomments": 80, + "bytes": 272, + "pass": true + }, + { + "task": "w15", + "category": "lists", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 205, + "tokens_nocomments": 199, + "bytes": 682, + "pass": true + }, + { + "task": "w15", + "category": "lists", + "lang": "go", + "encoding": "o200k_base", + "tokens": 207, + "tokens_nocomments": 201, + "bytes": 682, + "pass": true + }, + { + "task": "w16", + "category": "lists", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 61, + "tokens_nocomments": 61, + "bytes": 184, + "pass": true + }, + { + "task": "w16", + "category": "lists", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 61, + "tokens_nocomments": 61, + "bytes": 184, + "pass": true + }, + { + "task": "w16", + "category": "lists", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 57, + "tokens_nocomments": 57, + "bytes": 187, + "pass": true + }, + { + "task": "w16", + "category": "lists", + "lang": "python", + "encoding": "o200k_base", + "tokens": 58, + "tokens_nocomments": 58, + "bytes": 187, + "pass": true + }, + { + "task": "w16", + "category": "lists", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 59, + "tokens_nocomments": 59, + "bytes": 196, + "pass": true + }, + { + "task": "w16", + "category": "lists", + "lang": "js", + "encoding": "o200k_base", + "tokens": 63, + "tokens_nocomments": 63, + "bytes": 196, + "pass": true + }, + { + "task": "w16", + "category": "lists", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 164, + "tokens_nocomments": 158, + "bytes": 501, + "pass": true + }, + { + "task": "w16", + "category": "lists", + "lang": "go", + "encoding": "o200k_base", + "tokens": 165, + "tokens_nocomments": 159, + "bytes": 501, + "pass": true + }, + { + "task": "w17", + "category": "strings", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 92, + "tokens_nocomments": 92, + "bytes": 255, + "pass": true + }, + { + "task": "w17", + "category": "strings", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 92, + "tokens_nocomments": 92, + "bytes": 255, + "pass": true + }, + { + "task": "w17", + "category": "strings", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 57, + "tokens_nocomments": 57, + "bytes": 198, + "pass": true + }, + { + "task": "w17", + "category": "strings", + "lang": "python", + "encoding": "o200k_base", + "tokens": 57, + "tokens_nocomments": 57, + "bytes": 198, + "pass": true + }, + { + "task": "w17", + "category": "strings", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 62, + "tokens_nocomments": 62, + "bytes": 206, + "pass": true + }, + { + "task": "w17", + "category": "strings", + "lang": "js", + "encoding": "o200k_base", + "tokens": 66, + "tokens_nocomments": 66, + "bytes": 206, + "pass": true + }, + { + "task": "w17", + "category": "strings", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 156, + "tokens_nocomments": 150, + "bytes": 465, + "pass": true + }, + { + "task": "w17", + "category": "strings", + "lang": "go", + "encoding": "o200k_base", + "tokens": 158, + "tokens_nocomments": 152, + "bytes": 465, + "pass": true + }, + { + "task": "w18", + "category": "strings", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 64, + "tokens_nocomments": 64, + "bytes": 194, + "pass": true + }, + { + "task": "w18", + "category": "strings", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 64, + "tokens_nocomments": 64, + "bytes": 194, + "pass": true + }, + { + "task": "w18", + "category": "strings", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 63, + "tokens_nocomments": 63, + "bytes": 212, + "pass": true + }, + { + "task": "w18", + "category": "strings", + "lang": "python", + "encoding": "o200k_base", + "tokens": 63, + "tokens_nocomments": 63, + "bytes": 212, + "pass": true + }, + { + "task": "w18", + "category": "strings", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 64, + "tokens_nocomments": 64, + "bytes": 224, + "pass": true + }, + { + "task": "w18", + "category": "strings", + "lang": "js", + "encoding": "o200k_base", + "tokens": 70, + "tokens_nocomments": 70, + "bytes": 224, + "pass": true + }, + { + "task": "w18", + "category": "strings", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 165, + "tokens_nocomments": 159, + "bytes": 538, + "pass": true + }, + { + "task": "w18", + "category": "strings", + "lang": "go", + "encoding": "o200k_base", + "tokens": 165, + "tokens_nocomments": 159, + "bytes": 538, + "pass": true + }, + { + "task": "w19", + "category": "strings", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 82, + "tokens_nocomments": 82, + "bytes": 260, + "pass": true + }, + { + "task": "w19", + "category": "strings", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 83, + "tokens_nocomments": 83, + "bytes": 260, + "pass": true + }, + { + "task": "w19", + "category": "strings", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 69, + "tokens_nocomments": 69, + "bytes": 194, + "pass": true + }, + { + "task": "w19", + "category": "strings", + "lang": "python", + "encoding": "o200k_base", + "tokens": 69, + "tokens_nocomments": 69, + "bytes": 194, + "pass": true + }, + { + "task": "w19", + "category": "strings", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 73, + "tokens_nocomments": 73, + "bytes": 236, + "pass": true + }, + { + "task": "w19", + "category": "strings", + "lang": "js", + "encoding": "o200k_base", + "tokens": 81, + "tokens_nocomments": 81, + "bytes": 236, + "pass": true + }, + { + "task": "w19", + "category": "strings", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 129, + "tokens_nocomments": 123, + "bytes": 397, + "pass": true + }, + { + "task": "w19", + "category": "strings", + "lang": "go", + "encoding": "o200k_base", + "tokens": 131, + "tokens_nocomments": 125, + "bytes": 397, + "pass": true + }, + { + "task": "w20", + "category": "strings", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 62, + "tokens_nocomments": 62, + "bytes": 181, + "pass": true + }, + { + "task": "w20", + "category": "strings", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 62, + "tokens_nocomments": 62, + "bytes": 181, + "pass": true + }, + { + "task": "w20", + "category": "strings", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 57, + "tokens_nocomments": 57, + "bytes": 184, + "pass": true + }, + { + "task": "w20", + "category": "strings", + "lang": "python", + "encoding": "o200k_base", + "tokens": 57, + "tokens_nocomments": 57, + "bytes": 184, + "pass": true + }, + { + "task": "w20", + "category": "strings", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 61, + "tokens_nocomments": 61, + "bytes": 215, + "pass": true + }, + { + "task": "w20", + "category": "strings", + "lang": "js", + "encoding": "o200k_base", + "tokens": 65, + "tokens_nocomments": 65, + "bytes": 215, + "pass": true + }, + { + "task": "w20", + "category": "strings", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 125, + "tokens_nocomments": 119, + "bytes": 416, + "pass": true + }, + { + "task": "w20", + "category": "strings", + "lang": "go", + "encoding": "o200k_base", + "tokens": 126, + "tokens_nocomments": 120, + "bytes": 416, + "pass": true + }, + { + "task": "w21", + "category": "records", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 99, + "tokens_nocomments": 99, + "bytes": 295, + "pass": true + }, + { + "task": "w21", + "category": "records", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 99, + "tokens_nocomments": 99, + "bytes": 295, + "pass": true + }, + { + "task": "w21", + "category": "records", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 65, + "tokens_nocomments": 65, + "bytes": 252, + "pass": true + }, + { + "task": "w21", + "category": "records", + "lang": "python", + "encoding": "o200k_base", + "tokens": 65, + "tokens_nocomments": 65, + "bytes": 252, + "pass": true + }, + { + "task": "w21", + "category": "records", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 69, + "tokens_nocomments": 69, + "bytes": 246, + "pass": true + }, + { + "task": "w21", + "category": "records", + "lang": "js", + "encoding": "o200k_base", + "tokens": 73, + "tokens_nocomments": 73, + "bytes": 246, + "pass": true + }, + { + "task": "w21", + "category": "records", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 174, + "tokens_nocomments": 168, + "bytes": 591, + "pass": true + }, + { + "task": "w21", + "category": "records", + "lang": "go", + "encoding": "o200k_base", + "tokens": 175, + "tokens_nocomments": 169, + "bytes": 591, + "pass": true + }, + { + "task": "w22", + "category": "records", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 99, + "tokens_nocomments": 99, + "bytes": 308, + "pass": true + }, + { + "task": "w22", + "category": "records", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 100, + "tokens_nocomments": 100, + "bytes": 308, + "pass": true + }, + { + "task": "w22", + "category": "records", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 88, + "tokens_nocomments": 88, + "bytes": 306, + "pass": true + }, + { + "task": "w22", + "category": "records", + "lang": "python", + "encoding": "o200k_base", + "tokens": 88, + "tokens_nocomments": 88, + "bytes": 306, + "pass": true + }, + { + "task": "w22", + "category": "records", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 104, + "tokens_nocomments": 104, + "bytes": 358, + "pass": true + }, + { + "task": "w22", + "category": "records", + "lang": "js", + "encoding": "o200k_base", + "tokens": 110, + "tokens_nocomments": 110, + "bytes": 358, + "pass": true + }, + { + "task": "w22", + "category": "records", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 226, + "tokens_nocomments": 220, + "bytes": 748, + "pass": true + }, + { + "task": "w22", + "category": "records", + "lang": "go", + "encoding": "o200k_base", + "tokens": 228, + "tokens_nocomments": 222, + "bytes": 748, + "pass": true + }, + { + "task": "w23", + "category": "records", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 90, + "tokens_nocomments": 90, + "bytes": 273, + "pass": true + }, + { + "task": "w23", + "category": "records", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 91, + "tokens_nocomments": 91, + "bytes": 273, + "pass": true + }, + { + "task": "w23", + "category": "records", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 78, + "tokens_nocomments": 78, + "bytes": 277, + "pass": true + }, + { + "task": "w23", + "category": "records", + "lang": "python", + "encoding": "o200k_base", + "tokens": 78, + "tokens_nocomments": 78, + "bytes": 277, + "pass": true + }, + { + "task": "w23", + "category": "records", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 83, + "tokens_nocomments": 83, + "bytes": 287, + "pass": true + }, + { + "task": "w23", + "category": "records", + "lang": "js", + "encoding": "o200k_base", + "tokens": 89, + "tokens_nocomments": 89, + "bytes": 287, + "pass": true + }, + { + "task": "w23", + "category": "records", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 198, + "tokens_nocomments": 192, + "bytes": 638, + "pass": true + }, + { + "task": "w23", + "category": "records", + "lang": "go", + "encoding": "o200k_base", + "tokens": 198, + "tokens_nocomments": 192, + "bytes": 638, + "pass": true + }, + { + "task": "w24", + "category": "records", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 92, + "tokens_nocomments": 92, + "bytes": 270, + "pass": true + }, + { + "task": "w24", + "category": "records", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 92, + "tokens_nocomments": 92, + "bytes": 270, + "pass": true + }, + { + "task": "w24", + "category": "records", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 75, + "tokens_nocomments": 75, + "bytes": 271, + "pass": true + }, + { + "task": "w24", + "category": "records", + "lang": "python", + "encoding": "o200k_base", + "tokens": 75, + "tokens_nocomments": 75, + "bytes": 271, + "pass": true + }, + { + "task": "w24", + "category": "records", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 81, + "tokens_nocomments": 81, + "bytes": 276, + "pass": true + }, + { + "task": "w24", + "category": "records", + "lang": "js", + "encoding": "o200k_base", + "tokens": 85, + "tokens_nocomments": 85, + "bytes": 276, + "pass": true + }, + { + "task": "w24", + "category": "records", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 190, + "tokens_nocomments": 184, + "bytes": 603, + "pass": true + }, + { + "task": "w24", + "category": "records", + "lang": "go", + "encoding": "o200k_base", + "tokens": 190, + "tokens_nocomments": 184, + "bytes": 603, + "pass": true + }, + { + "task": "w25", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 66, + "tokens_nocomments": 66, + "bytes": 175, + "pass": true + }, + { + "task": "w25", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 67, + "tokens_nocomments": 67, + "bytes": 175, + "pass": true + }, + { + "task": "w25", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 77, + "tokens_nocomments": 77, + "bytes": 220, + "pass": true + }, + { + "task": "w25", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 77, + "tokens_nocomments": 77, + "bytes": 220, + "pass": true + }, + { + "task": "w25", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 102, + "tokens_nocomments": 102, + "bytes": 311, + "pass": true + }, + { + "task": "w25", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 106, + "tokens_nocomments": 106, + "bytes": 311, + "pass": true + }, + { + "task": "w25", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 188, + "tokens_nocomments": 182, + "bytes": 569, + "pass": true + }, + { + "task": "w25", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 190, + "tokens_nocomments": 184, + "bytes": 569, + "pass": true + }, + { + "task": "w26", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 87, + "tokens_nocomments": 87, + "bytes": 204, + "pass": true + }, + { + "task": "w26", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 87, + "tokens_nocomments": 87, + "bytes": 204, + "pass": true + }, + { + "task": "w26", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 72, + "tokens_nocomments": 72, + "bytes": 195, + "pass": true + }, + { + "task": "w26", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 72, + "tokens_nocomments": 72, + "bytes": 195, + "pass": true + }, + { + "task": "w26", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 76, + "tokens_nocomments": 76, + "bytes": 215, + "pass": true + }, + { + "task": "w26", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 80, + "tokens_nocomments": 80, + "bytes": 215, + "pass": true + }, + { + "task": "w26", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 157, + "tokens_nocomments": 151, + "bytes": 458, + "pass": true + }, + { + "task": "w26", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 160, + "tokens_nocomments": 154, + "bytes": 458, + "pass": true + }, + { + "task": "w27", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 79, + "tokens_nocomments": 79, + "bytes": 208, + "pass": true + }, + { + "task": "w27", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 80, + "tokens_nocomments": 80, + "bytes": 208, + "pass": true + }, + { + "task": "w27", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 95, + "tokens_nocomments": 95, + "bytes": 279, + "pass": true + }, + { + "task": "w27", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 95, + "tokens_nocomments": 95, + "bytes": 279, + "pass": true + }, + { + "task": "w27", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 98, + "tokens_nocomments": 98, + "bytes": 282, + "pass": true + }, + { + "task": "w27", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 102, + "tokens_nocomments": 102, + "bytes": 282, + "pass": true + }, + { + "task": "w27", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 184, + "tokens_nocomments": 178, + "bytes": 548, + "pass": true + }, + { + "task": "w27", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 185, + "tokens_nocomments": 179, + "bytes": 548, + "pass": true + }, + { + "task": "w28", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 137, + "tokens_nocomments": 137, + "bytes": 409, + "pass": true + }, + { + "task": "w28", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 138, + "tokens_nocomments": 138, + "bytes": 409, + "pass": true + }, + { + "task": "w28", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 133, + "tokens_nocomments": 133, + "bytes": 462, + "pass": true + }, + { + "task": "w28", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 134, + "tokens_nocomments": 134, + "bytes": 462, + "pass": true + }, + { + "task": "w28", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 155, + "tokens_nocomments": 155, + "bytes": 518, + "pass": true + }, + { + "task": "w28", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 162, + "tokens_nocomments": 162, + "bytes": 518, + "pass": true + }, + { + "task": "w28", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 229, + "tokens_nocomments": 223, + "bytes": 712, + "pass": true + }, + { + "task": "w28", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 231, + "tokens_nocomments": 225, + "bytes": 712, + "pass": true + }, + { + "task": "w29", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 105, + "tokens_nocomments": 105, + "bytes": 304, + "pass": true + }, + { + "task": "w29", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 105, + "tokens_nocomments": 105, + "bytes": 304, + "pass": true + }, + { + "task": "w29", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 90, + "tokens_nocomments": 90, + "bytes": 319, + "pass": true + }, + { + "task": "w29", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 90, + "tokens_nocomments": 90, + "bytes": 319, + "pass": true + }, + { + "task": "w29", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 118, + "tokens_nocomments": 118, + "bytes": 403, + "pass": true + }, + { + "task": "w29", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 124, + "tokens_nocomments": 124, + "bytes": 403, + "pass": true + }, + { + "task": "w29", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 214, + "tokens_nocomments": 208, + "bytes": 696, + "pass": true + }, + { + "task": "w29", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 217, + "tokens_nocomments": 211, + "bytes": 696, + "pass": true + }, + { + "task": "w30", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 163, + "tokens_nocomments": 163, + "bytes": 413, + "pass": true + }, + { + "task": "w30", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 163, + "tokens_nocomments": 163, + "bytes": 413, + "pass": true + }, + { + "task": "w30", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 140, + "tokens_nocomments": 140, + "bytes": 379, + "pass": true + }, + { + "task": "w30", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 140, + "tokens_nocomments": 140, + "bytes": 379, + "pass": true + }, + { + "task": "w30", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 147, + "tokens_nocomments": 147, + "bytes": 378, + "pass": true + }, + { + "task": "w30", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 151, + "tokens_nocomments": 151, + "bytes": 378, + "pass": true + }, + { + "task": "w30", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 254, + "tokens_nocomments": 248, + "bytes": 715, + "pass": true + }, + { + "task": "w30", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 255, + "tokens_nocomments": 249, + "bytes": 715, + "pass": true + }, + { + "task": "w31", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 41, + "tokens_nocomments": 41, + "bytes": 121, + "pass": true + }, + { + "task": "w31", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 41, + "tokens_nocomments": 41, + "bytes": 121, + "pass": true + }, + { + "task": "w31", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 49, + "tokens_nocomments": 49, + "bytes": 166, + "pass": true + }, + { + "task": "w31", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 49, + "tokens_nocomments": 49, + "bytes": 166, + "pass": true + }, + { + "task": "w31", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 54, + "tokens_nocomments": 54, + "bytes": 188, + "pass": true + }, + { + "task": "w31", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 59, + "tokens_nocomments": 59, + "bytes": 188, + "pass": true + }, + { + "task": "w31", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 151, + "tokens_nocomments": 145, + "bytes": 483, + "pass": true + }, + { + "task": "w31", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 152, + "tokens_nocomments": 146, + "bytes": 483, + "pass": true + }, + { + "task": "w32", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 128, + "tokens_nocomments": 128, + "bytes": 335, + "pass": true + }, + { + "task": "w32", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 129, + "tokens_nocomments": 129, + "bytes": 335, + "pass": true + }, + { + "task": "w32", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 124, + "tokens_nocomments": 124, + "bytes": 375, + "pass": true + }, + { + "task": "w32", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 126, + "tokens_nocomments": 126, + "bytes": 375, + "pass": true + }, + { + "task": "w32", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 142, + "tokens_nocomments": 142, + "bytes": 404, + "pass": true + }, + { + "task": "w32", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 147, + "tokens_nocomments": 147, + "bytes": 404, + "pass": true + }, + { + "task": "w32", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 232, + "tokens_nocomments": 226, + "bytes": 674, + "pass": true + }, + { + "task": "w32", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 237, + "tokens_nocomments": 231, + "bytes": 674, + "pass": true + }, + { + "task": "w33", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 142, + "tokens_nocomments": 142, + "bytes": 380, + "pass": true + }, + { + "task": "w33", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 143, + "tokens_nocomments": 143, + "bytes": 380, + "pass": true + }, + { + "task": "w33", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 95, + "tokens_nocomments": 95, + "bytes": 326, + "pass": true + }, + { + "task": "w33", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 96, + "tokens_nocomments": 96, + "bytes": 326, + "pass": true + }, + { + "task": "w33", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 113, + "tokens_nocomments": 113, + "bytes": 371, + "pass": true + }, + { + "task": "w33", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 120, + "tokens_nocomments": 120, + "bytes": 371, + "pass": true + }, + { + "task": "w33", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 252, + "tokens_nocomments": 246, + "bytes": 781, + "pass": true + }, + { + "task": "w33", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 255, + "tokens_nocomments": 249, + "bytes": 781, + "pass": true + }, + { + "task": "w34", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 208, + "tokens_nocomments": 208, + "bytes": 506, + "pass": true + }, + { + "task": "w34", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 208, + "tokens_nocomments": 208, + "bytes": 506, + "pass": true + }, + { + "task": "w34", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 145, + "tokens_nocomments": 145, + "bytes": 378, + "pass": true + }, + { + "task": "w34", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 145, + "tokens_nocomments": 145, + "bytes": 378, + "pass": true + }, + { + "task": "w34", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 123, + "tokens_nocomments": 123, + "bytes": 359, + "pass": true + }, + { + "task": "w34", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 130, + "tokens_nocomments": 130, + "bytes": 359, + "pass": true + }, + { + "task": "w34", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 250, + "tokens_nocomments": 244, + "bytes": 642, + "pass": true + }, + { + "task": "w34", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 251, + "tokens_nocomments": 245, + "bytes": 642, + "pass": true + }, + { + "task": "w35", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 122, + "tokens_nocomments": 122, + "bytes": 357, + "pass": true + }, + { + "task": "w35", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 123, + "tokens_nocomments": 123, + "bytes": 357, + "pass": true + }, + { + "task": "w35", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 100, + "tokens_nocomments": 100, + "bytes": 336, + "pass": true + }, + { + "task": "w35", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 101, + "tokens_nocomments": 101, + "bytes": 336, + "pass": true + }, + { + "task": "w35", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 123, + "tokens_nocomments": 123, + "bytes": 420, + "pass": true + }, + { + "task": "w35", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 130, + "tokens_nocomments": 130, + "bytes": 420, + "pass": true + }, + { + "task": "w35", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 230, + "tokens_nocomments": 224, + "bytes": 742, + "pass": true + }, + { + "task": "w35", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 234, + "tokens_nocomments": 228, + "bytes": 742, + "pass": true + }, + { + "task": "w36", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 106, + "tokens_nocomments": 106, + "bytes": 339, + "pass": true + }, + { + "task": "w36", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 107, + "tokens_nocomments": 107, + "bytes": 339, + "pass": true + }, + { + "task": "w36", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 111, + "tokens_nocomments": 111, + "bytes": 375, + "pass": true + }, + { + "task": "w36", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 112, + "tokens_nocomments": 112, + "bytes": 375, + "pass": true + }, + { + "task": "w36", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 98, + "tokens_nocomments": 98, + "bytes": 292, + "pass": true + }, + { + "task": "w36", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 102, + "tokens_nocomments": 102, + "bytes": 292, + "pass": true + }, + { + "task": "w36", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 179, + "tokens_nocomments": 173, + "bytes": 624, + "pass": true + }, + { + "task": "w36", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 180, + "tokens_nocomments": 174, + "bytes": 624, + "pass": true + }, + { + "task": "w37", + "category": "logic", + "lang": "howl", + "encoding": "cl100k_base", + "tokens": 184, + "tokens_nocomments": 184, + "bytes": 428, + "pass": true + }, + { + "task": "w37", + "category": "logic", + "lang": "howl", + "encoding": "o200k_base", + "tokens": 185, + "tokens_nocomments": 185, + "bytes": 428, + "pass": true + }, + { + "task": "w37", + "category": "logic", + "lang": "python", + "encoding": "cl100k_base", + "tokens": 160, + "tokens_nocomments": 160, + "bytes": 358, + "pass": true + }, + { + "task": "w37", + "category": "logic", + "lang": "python", + "encoding": "o200k_base", + "tokens": 161, + "tokens_nocomments": 161, + "bytes": 358, + "pass": true + }, + { + "task": "w37", + "category": "logic", + "lang": "js", + "encoding": "cl100k_base", + "tokens": 168, + "tokens_nocomments": 168, + "bytes": 386, + "pass": true + }, + { + "task": "w37", + "category": "logic", + "lang": "js", + "encoding": "o200k_base", + "tokens": 174, + "tokens_nocomments": 174, + "bytes": 386, + "pass": true + }, + { + "task": "w37", + "category": "logic", + "lang": "go", + "encoding": "cl100k_base", + "tokens": 240, + "tokens_nocomments": 234, + "bytes": 591, + "pass": true + }, + { + "task": "w37", + "category": "logic", + "lang": "go", + "encoding": "o200k_base", + "tokens": 242, + "tokens_nocomments": 236, + "bytes": 591, + "pass": true + } +] diff --git a/experiments/token-efficiency-heldout-v2/postcf/results/verification.json b/experiments/token-efficiency-heldout-v2/postcf/results/verification.json new file mode 100644 index 00000000..4491911a --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/results/verification.json @@ -0,0 +1,890 @@ +[ + { + "task": "w01", + "lang": "howl", + "pass": true, + "sha256": "cb6c8fc01e1022b84f203dac67e012fc8978f13dee8ff6cbecc234f10b798f1e" + }, + { + "task": "w02", + "lang": "howl", + "pass": true, + "sha256": "a48eabcdf78c948bbf9633fce496c42b190fdd830b7b144b1edb6b4778c5b452" + }, + { + "task": "w03", + "lang": "howl", + "pass": true, + "sha256": "59bbdd94391bea3b445d94557501f96ec2a906f1c306c52ac3f15a4375d99757" + }, + { + "task": "w04", + "lang": "howl", + "pass": true, + "sha256": "c39a41f1c33ff2c36ad5f51770bf32cb411f27c05bc998a402fb00db2e34d5d1" + }, + { + "task": "w05", + "lang": "howl", + "pass": true, + "sha256": "f0cc69e3c9b93db019d684a3d661532555e46385c5b91ada0021c1fcc2c26c30" + }, + { + "task": "w06", + "lang": "howl", + "pass": true, + "sha256": "65f4a851178cca27ada225ba7ab5d9b2f1141e70853780214ab23b49481d125e" + }, + { + "task": "w07", + "lang": "howl", + "pass": true, + "sha256": "eaee24b07aa85ba0796496a17f6c63c2c91687ca53aa3b8daf4c26b33c07befa" + }, + { + "task": "w08", + "lang": "howl", + "pass": true, + "sha256": "795a4d9d74eed7cd9a21c0351619d4153fe03943a97243858e5ae9993f38464b" + }, + { + "task": "w09", + "lang": "howl", + "pass": true, + "sha256": "14010cc90c2b75d62d7298a5ef08363f08614e89d9f6959179d525c2692cece0" + }, + { + "task": "w10", + "lang": "howl", + "pass": true, + "sha256": "2bfe3d86ff059b5d6c1841cc4bdf9188c4cc176a9539c4a3e5a3ee40f695bd97" + }, + { + "task": "w11", + "lang": "howl", + "pass": true, + "sha256": "2c1eda4594958d80201316f9d00c74bcca5be5b9b1940c8774b8b1369e7a1852" + }, + { + "task": "w12", + "lang": "howl", + "pass": true, + "sha256": "122024632222ac8671a665ee3e90be402ecdf93475f5fd41ef80916921c4acf6" + }, + { + "task": "w13", + "lang": "howl", + "pass": true, + "sha256": "e555c4676af8ee9e39ec611d20eb7d93bce2c1b43ae83969b3b9c249b3fe1890" + }, + { + "task": "w14", + "lang": "howl", + "pass": true, + "sha256": "987051c414bb453cfbcfb08384c208f3aa5a10acd222d040c70a9fe53e80dfd5" + }, + { + "task": "w15", + "lang": "howl", + "pass": true, + "sha256": "bb350ca8d2ad9821fd7f7d4b046df0a107a76db28b157c75f594c2d65bc720f9" + }, + { + "task": "w16", + "lang": "howl", + "pass": true, + "sha256": "6c5973e158f4d15651c61abd79e19b817daaa3dbce995eee40312a61a354b099" + }, + { + "task": "w17", + "lang": "howl", + "pass": true, + "sha256": "67e12021e5019c40fec17b53990f2f87ea043fdd6ed98d3a7c318a4c63110517" + }, + { + "task": "w18", + "lang": "howl", + "pass": true, + "sha256": "b785f8874ff49d4278b3a8c0ad2a41eaea5652f88305d4529a09b0c0e2c68918" + }, + { + "task": "w19", + "lang": "howl", + "pass": true, + "sha256": "08a057369acd66e0691f16af055a25033a8afb9ff4119912bedee2b762c23b44" + }, + { + "task": "w20", + "lang": "howl", + "pass": true, + "sha256": "ededffbb80a74f0661f596774edc44ffd1b0f02b855c5a50ee92d290f6bfe1a5" + }, + { + "task": "w21", + "lang": "howl", + "pass": true, + "sha256": "64e175ebc99f90964ec9f26d07d6235742394ac34c5714bf89eea40dfa6c3e15" + }, + { + "task": "w22", + "lang": "howl", + "pass": true, + "sha256": "7f5f8c2b87e13c5628869ddcf999ba68af253d23f606208a59c51e090d0828bb" + }, + { + "task": "w23", + "lang": "howl", + "pass": true, + "sha256": "f8ea407bc1785b1c4a2d5a380c867b583c54de4815957b4653155a913065f989" + }, + { + "task": "w24", + "lang": "howl", + "pass": true, + "sha256": "4ba5491355c224b7b7f5214b32c0d9f6f6175ed7bc18c505d17255018f7a0159" + }, + { + "task": "w25", + "lang": "howl", + "pass": true, + "sha256": "9c150008160244591d7f8a3b304e61aeb7b3b2977b9ec2d99b6eec938bd24391" + }, + { + "task": "w26", + "lang": "howl", + "pass": true, + "sha256": "f685d89ee1b1299cf10e41dfeea620df76784ff508f9721973287b3c93b12565" + }, + { + "task": "w27", + "lang": "howl", + "pass": true, + "sha256": "43c1b9e03069a4d484c1eebdbb30bd10376bacdf53e42258fc66c6327be4a296" + }, + { + "task": "w28", + "lang": "howl", + "pass": true, + "sha256": "034399db784d22a8da29643732dfe5c8dfb5a01333cf443c62087f0304896af5" + }, + { + "task": "w29", + "lang": "howl", + "pass": true, + "sha256": "ba652b94657908c081c182ed8e708c177b83856a534d1fac02ee267c05389cfe" + }, + { + "task": "w30", + "lang": "howl", + "pass": true, + "sha256": "4bcc103d7ddf3e61c2351611b9bc3e96938841ea4cf8bbe1c66eb7b18e276670" + }, + { + "task": "w31", + "lang": "howl", + "pass": true, + "sha256": "8d5399fd2a5b48fb834423db7f1044803d2fffd28677f072b08e4938dcfcebeb" + }, + { + "task": "w32", + "lang": "howl", + "pass": true, + "sha256": "35d59d36931a1a8562be20e5d94d93068a103d6f51e8df4ce483fc041d8315cf" + }, + { + "task": "w33", + "lang": "howl", + "pass": true, + "sha256": "806028d0eeb2e1cc7bc88357a56b830c4e02294e5843c5c862bbdbfd2a4768d0" + }, + { + "task": "w34", + "lang": "howl", + "pass": true, + "sha256": "233549878bbd692aa0dc04a94be81e9435e21e879f4f576d21a04d468f60506a" + }, + { + "task": "w35", + "lang": "howl", + "pass": true, + "sha256": "2bc8d2443f74c1dbe2f49a72774b704dc4801909bff59385b7ad3142cefc5445" + }, + { + "task": "w36", + "lang": "howl", + "pass": true, + "sha256": "dd872e213c416002bfb8742a56e25130f291808cbc1d070af78a2278b64023a7" + }, + { + "task": "w37", + "lang": "howl", + "pass": true, + "sha256": "ecdbe67225e7080263707d15cfce12dd90273e47e4b4dde3dc8600955322b526" + }, + { + "task": "w01", + "lang": "python", + "pass": true, + "sha256": "8f3118ecacd3fdc72f4224d7080fb648f6a8326fab9356d4abfb61faa6e2cb12" + }, + { + "task": "w02", + "lang": "python", + "pass": true, + "sha256": "02259e2bc8e0b86000545a529420d4aeb8e1bd69d34ba2fa5b03edad8e4a450d" + }, + { + "task": "w03", + "lang": "python", + "pass": true, + "sha256": "3ebab7139b6bf7a2d2ecfe34c2fb488dffff58fa16203036f86c3e354ac53042" + }, + { + "task": "w04", + "lang": "python", + "pass": true, + "sha256": "d304662da1a6aa1b02f4f34a67a093856dd6d063341991e61ea094b7df6369e8" + }, + { + "task": "w05", + "lang": "python", + "pass": true, + "sha256": "623123a24a3143a438a12617be48c209f78391b273faa1f3d79ee7fedc59ed23" + }, + { + "task": "w06", + "lang": "python", + "pass": true, + "sha256": "16aa2a2f58a3d919bb288d753589854dd387f28831f7f9843c8e733b38e57d8f" + }, + { + "task": "w07", + "lang": "python", + "pass": true, + "sha256": "66819bae1962d4647ad1ed97224e61e7060af6c1c03d7d36c00e30f7ac3e6234" + }, + { + "task": "w08", + "lang": "python", + "pass": true, + "sha256": "e80000e27a0ed99c46f09b6e266be4e96ae25db6ec8cd5af12722623a5f6f5f5" + }, + { + "task": "w09", + "lang": "python", + "pass": true, + "sha256": "0432547b54351daf909eafdf74fb3886ce30f00c40fb2f49867d89535c888eb0" + }, + { + "task": "w10", + "lang": "python", + "pass": true, + "sha256": "d06addd3867c2a4c4170ffc14cdfd5b9bd1134943dd542fdd4a0796b2eb3713d" + }, + { + "task": "w11", + "lang": "python", + "pass": true, + "sha256": "e7342796db038545f62e91487abe9af787b52e632c96d9b3281455b12c5e8344" + }, + { + "task": "w12", + "lang": "python", + "pass": true, + "sha256": "18cc0936cbfbcbc9d95d29314d7292fa46147aeb2975af7fff49acb3edfd04eb" + }, + { + "task": "w13", + "lang": "python", + "pass": true, + "sha256": "4cb516a6b946b6db89f9cca39e36535c8be51ac172bf68204f5aa81e8385353d" + }, + { + "task": "w14", + "lang": "python", + "pass": true, + "sha256": "1f4b3e4fbfa3523cbf467d62e33fc0e466a7e093a46123d1c0b579a160b95ccb" + }, + { + "task": "w15", + "lang": "python", + "pass": true, + "sha256": "c8371730f88f881a22849dd386d0a3be17ed6f15f122e347c03e0c88d12635b3" + }, + { + "task": "w16", + "lang": "python", + "pass": true, + "sha256": "59f91c126ae151e78a9a4491a51e6907b782a056f36250928af95132dbe0edc4" + }, + { + "task": "w17", + "lang": "python", + "pass": true, + "sha256": "059551fcd28f57da72abe17b45a84ef8bb5cdc9a106e0f7b29a18bb454dff36b" + }, + { + "task": "w18", + "lang": "python", + "pass": true, + "sha256": "1f7cc3226d7fd5a75f7d94be22d988806c4cf2ecd10edb2160d98dbf20c5f2db" + }, + { + "task": "w19", + "lang": "python", + "pass": true, + "sha256": "567645c1f53be895af50b1e5e68104347892fa38465e40088ed1b4e8b54324a6" + }, + { + "task": "w20", + "lang": "python", + "pass": true, + "sha256": "a80c124a5229b4ecd88fffc7d0958e411029a999c133314de93af2940d702083" + }, + { + "task": "w21", + "lang": "python", + "pass": true, + "sha256": "493cf8cc59109b80cb7c098b20f59bba8b70191ebf44935a4159007cd51e4976" + }, + { + "task": "w22", + "lang": "python", + "pass": true, + "sha256": "64afb3795b1d858a9027a37081a49ef9b36a55a753c13faa518ccb5d71c667ee" + }, + { + "task": "w23", + "lang": "python", + "pass": true, + "sha256": "d7251212994688840f13c5b67ba0e91f481d0b277c01ece578c9d5b86bd2e3c1" + }, + { + "task": "w24", + "lang": "python", + "pass": true, + "sha256": "edef493f38edef1449f4759b8e29aa9a32bdf2931b82d35eabbe09132fea120a" + }, + { + "task": "w25", + "lang": "python", + "pass": true, + "sha256": "45a08a55f2e5bead6874279c18c9964549f3aeea33f3511de4d7f976f944868f" + }, + { + "task": "w26", + "lang": "python", + "pass": true, + "sha256": "bcaa8eb13bb5dc893672f726e6278f1319b362caedb8664842f71fb4de77c231" + }, + { + "task": "w27", + "lang": "python", + "pass": true, + "sha256": "5761fc1264eb64482bcb002514e8dd51ead52cbf6d1192533b0fabd36ef6b12b" + }, + { + "task": "w28", + "lang": "python", + "pass": true, + "sha256": "2833fbbc0300cabc213dc4e996a0afe533284f9f57efc34ba02d5942d9919e81" + }, + { + "task": "w29", + "lang": "python", + "pass": true, + "sha256": "19176d672273ddbd77b866ecf522970f216039416d860b60f2ed5d38a53267aa" + }, + { + "task": "w30", + "lang": "python", + "pass": true, + "sha256": "19e9dff0c476a39e6f591d978628584c611b3805fe4e5c52c55fbc93a710258a" + }, + { + "task": "w31", + "lang": "python", + "pass": true, + "sha256": "ed559cc8e3ebef4eaf230c8c6b81cfaed3106b2a861df4097df7cd86406003ab" + }, + { + "task": "w32", + "lang": "python", + "pass": true, + "sha256": "ec6220a5dbe978021db89a1649cb949b4dceb62eba03ef74996913a32dcffe43" + }, + { + "task": "w33", + "lang": "python", + "pass": true, + "sha256": "2a47f5194782d5d6241ecc0fe5c6904e35ccefca7932ff9522a9d871e31de068" + }, + { + "task": "w34", + "lang": "python", + "pass": true, + "sha256": "17ee5eafbc9a671dc2d9066bd2a316a79e8d9a7ea215fb44f7a3f4eefde96d2e" + }, + { + "task": "w35", + "lang": "python", + "pass": true, + "sha256": "77a2a5f511d440e272074d180f6f55db74c6ebe40f20b3aa706f9dc19b9067a3" + }, + { + "task": "w36", + "lang": "python", + "pass": true, + "sha256": "f9a81cc29797981a1a5ee17e01b2b0e854cacd5258d1ad152117943ded81e346" + }, + { + "task": "w37", + "lang": "python", + "pass": true, + "sha256": "6198f8e8b6df14d10bb86d30edd92659d12b97ba7ea730ac8e5dd5cb6389a773" + }, + { + "task": "w01", + "lang": "js", + "pass": true, + "sha256": "c5c591a7c44119ccdf87a4d3503e481a2135f5ec437bdb3ef1a81b4d817c0aaf" + }, + { + "task": "w02", + "lang": "js", + "pass": true, + "sha256": "1ad3252a423581c6e32f4452924c17f23dbc8c04f5893c59e356d2b8225a3b47" + }, + { + "task": "w03", + "lang": "js", + "pass": true, + "sha256": "e77fe2c38cb78c8685ecec976812911483a23ea09206090aa90ae9585d3cd3f6" + }, + { + "task": "w04", + "lang": "js", + "pass": true, + "sha256": "b7edb0d60535746e41e7f2c4d5f283ac6003b1164727b29abdbacdf0f134f83b" + }, + { + "task": "w05", + "lang": "js", + "pass": true, + "sha256": "549246ee7ec3c0e57f174cb43c9c2bb9d2ac71fee103a93ff5d21a77fa8abf79" + }, + { + "task": "w06", + "lang": "js", + "pass": true, + "sha256": "746a1e6a08fff40dd688329f8d1bc539800b1ca4630ca2ec9a3d643d98384a02" + }, + { + "task": "w07", + "lang": "js", + "pass": true, + "sha256": "ddf8d9771d37ebd9dfae094a0193483769978f998f47d6d25fc5fe5758908fd6" + }, + { + "task": "w08", + "lang": "js", + "pass": true, + "sha256": "5be9c13a9956a91ffa9793c3314b7b6571fe50cd922fe635ed6c670606ea31bf" + }, + { + "task": "w09", + "lang": "js", + "pass": true, + "sha256": "00a9c1cb7702f1206b31148fd32a6c272f39002d978f487864ac499b3db645a0" + }, + { + "task": "w10", + "lang": "js", + "pass": true, + "sha256": "c2e026ff8b3f64c047bb7bed65f6986b7eb15cfff7b1ac21386931819a3d5d05" + }, + { + "task": "w11", + "lang": "js", + "pass": true, + "sha256": "4ebb05b6730b2adf63cd041f67b2f17c7d5335fe42a1304029dbf18270a9ccd2" + }, + { + "task": "w12", + "lang": "js", + "pass": true, + "sha256": "91b81dd2fd81489dc6d9c4f3da67aed0f637df3d4688017f0ab00628c2fd2ec7" + }, + { + "task": "w13", + "lang": "js", + "pass": true, + "sha256": "9dcd83c6bf7ea6ea8708dc26e5b5efda76dc0d329b0b67611b6d2f1441642cce" + }, + { + "task": "w14", + "lang": "js", + "pass": true, + "sha256": "b5a64fdadb9892daf12a10fef5cd52008076d161aa7c02256a787803edc8d70c" + }, + { + "task": "w15", + "lang": "js", + "pass": true, + "sha256": "bd237511847cc5e65b4a75cc5160fd128bc64d9663fcb82c13bb3840baac783e" + }, + { + "task": "w16", + "lang": "js", + "pass": true, + "sha256": "18122820d494dcd3b38323e17195f4f90b5213574e8728cc3c2e50d3575d4fbb" + }, + { + "task": "w17", + "lang": "js", + "pass": true, + "sha256": "bff46c067ed8799c7d2d9b73e41eb0fd0e839b7a9a7bd1baa0092b4159d405a2" + }, + { + "task": "w18", + "lang": "js", + "pass": true, + "sha256": "b58541be55dd1d539bda7dd93923ff2e81654ebc7669ba3d01c3d2524504e7d3" + }, + { + "task": "w19", + "lang": "js", + "pass": true, + "sha256": "721e5df61cafec6e6ff993356810e8230276e8870e4151cc5d3c886b502fd034" + }, + { + "task": "w20", + "lang": "js", + "pass": true, + "sha256": "d7487549080d89140bb5db72dc7679bfe86f1d67f71989fe197618fa88194432" + }, + { + "task": "w21", + "lang": "js", + "pass": true, + "sha256": "92cb0621d276228ee322c5bc12c28f7242684f0b3cf66ed7c9021ea78b5738ae" + }, + { + "task": "w22", + "lang": "js", + "pass": true, + "sha256": "f48cb0cfd1853763b19afd4c59f0eb3a47ff12dd77d5ce47b8cf07b3e086503c" + }, + { + "task": "w23", + "lang": "js", + "pass": true, + "sha256": "0a7f6d41078841346cfdc85e43b2c51c2a4541d6db5f6b2f416fce66fd8186be" + }, + { + "task": "w24", + "lang": "js", + "pass": true, + "sha256": "3fb08821c62c948b5b29f266428a6d247baf082afd76654edcc9fe4c715ba72c" + }, + { + "task": "w25", + "lang": "js", + "pass": true, + "sha256": "bed28c9def238d97635a6539e096b6a883beb159830b955b42835c12a86da18a" + }, + { + "task": "w26", + "lang": "js", + "pass": true, + "sha256": "f7029b3bced1b3e328be870c0641b95a5afd6daf227cd888304697f9d90cc426" + }, + { + "task": "w27", + "lang": "js", + "pass": true, + "sha256": "554be662d6d96d584286779d08b6d9791ee79a4e979c008cf7f1e8ad2d10f96a" + }, + { + "task": "w28", + "lang": "js", + "pass": true, + "sha256": "659b9420e06ca3db3dab0ded8c7d3eefb129708fec94ed7600593e8a43919afe" + }, + { + "task": "w29", + "lang": "js", + "pass": true, + "sha256": "3581b6a69a80abe5df3033f4864b0525141dc4ef173714b4e0a8e403989c6956" + }, + { + "task": "w30", + "lang": "js", + "pass": true, + "sha256": "f2a7579f2a1e8b2a931c46b8b95d3a31e62fe632c88190c229808c0e6f2e6ae0" + }, + { + "task": "w31", + "lang": "js", + "pass": true, + "sha256": "9c205e26261ff4481a179f8a5b8ce8a77439fbda32ee58b4163ce1f05c0d000c" + }, + { + "task": "w32", + "lang": "js", + "pass": true, + "sha256": "dd96fb4446f03706d44e6913454259f850309b7f0be4bef5ca5bbf86e3be552b" + }, + { + "task": "w33", + "lang": "js", + "pass": true, + "sha256": "9c5ce0654c1cb728867fd7b1c1f0be70cfb49826795d2cef15965807f93c7712" + }, + { + "task": "w34", + "lang": "js", + "pass": true, + "sha256": "b230aa0de6bc6af9a92c1ebd24cb03b7fe493d578f5d42747a721a7e05c20154" + }, + { + "task": "w35", + "lang": "js", + "pass": true, + "sha256": "e46b3e6c8d4596f97b276c0431d1639d2e496937ef134c4ef9dce4c924d1c139" + }, + { + "task": "w36", + "lang": "js", + "pass": true, + "sha256": "d4727175d96f79a05e3515973c5c3d9c5441d623b80e77b6cbdb30378a9a94f0" + }, + { + "task": "w37", + "lang": "js", + "pass": true, + "sha256": "030a045d39e27f7d40aedd6cf4e248e98397087903360d2778d6b7c799c88361" + }, + { + "task": "w01", + "lang": "go", + "pass": true, + "sha256": "1cb610c66e93224c93f999f32efd926186cd6e6cf0cdcb35aebfe5876e872730" + }, + { + "task": "w02", + "lang": "go", + "pass": true, + "sha256": "ce6c25a4c7b3a46a92393570abf855bacc5a0583eba941b0eeb8a75848dbeb8c" + }, + { + "task": "w03", + "lang": "go", + "pass": true, + "sha256": "6f9fc67915d5f8ff281dbd4c3cb0c2da0f766f8815ea019055d4b9604c8c1540" + }, + { + "task": "w04", + "lang": "go", + "pass": true, + "sha256": "7b4203c9fe5d5266bf2601070f5e1eb7cf84bbf37576bdcafb8c9bdd4ed2bd9c" + }, + { + "task": "w05", + "lang": "go", + "pass": true, + "sha256": "27788c55ab109b078df03901e875068011473c2e696b57e50d21556e8f2c79cd" + }, + { + "task": "w06", + "lang": "go", + "pass": true, + "sha256": "f8cab76ebff97739c66e259b0ef372e63d87c87846fe0a7377fe893f86c778b1" + }, + { + "task": "w07", + "lang": "go", + "pass": true, + "sha256": "4691a4acb4bcc607f94f3cdf83d413d31cc7fdf2b731fa6fe7ba7db54e10df15" + }, + { + "task": "w08", + "lang": "go", + "pass": true, + "sha256": "4ead3ca21cc3c84f7b27f831798d0721a23e184467a3d4d5af5b47d1f7e7e805" + }, + { + "task": "w09", + "lang": "go", + "pass": true, + "sha256": "1b01ef1070e578c4e24e32c9391f5f711731a1c9e63b378108d80e0e265e0345" + }, + { + "task": "w10", + "lang": "go", + "pass": true, + "sha256": "9c93eb50f60e5f04cf6e790261ab08f437b20bfea39567b2a5e7aeb44359815e" + }, + { + "task": "w11", + "lang": "go", + "pass": true, + "sha256": "5b1d1cf4beab3fa6fdeaa156b7bcda80c074ee924366cc8ebdd3423667bab01b" + }, + { + "task": "w12", + "lang": "go", + "pass": true, + "sha256": "cdbcbda7889abcec6b5011397ff648210336934faf6095c3b4aa9d8862593c64" + }, + { + "task": "w13", + "lang": "go", + "pass": true, + "sha256": "5fdc1f8426d8402e6967165d67f03a4a5b0c2042ec3eae52adc316b173b3b3a2" + }, + { + "task": "w14", + "lang": "go", + "pass": true, + "sha256": "39ea81b03e7b28187df1f46cf0de5124be5c7ee33ed983e91844552dd925133a" + }, + { + "task": "w15", + "lang": "go", + "pass": true, + "sha256": "6966b093a335e91203216f78908bfcd3a5d4786aaf39224f36667705695a6943" + }, + { + "task": "w16", + "lang": "go", + "pass": true, + "sha256": "a9e7755d4edfcbb341e48ab44e8fe1302eee9314b255f07a500e97971d78d00c" + }, + { + "task": "w17", + "lang": "go", + "pass": true, + "sha256": "36260b5a840d2860ca6a414ebfb8cbe70dc156523869544ea29a81aaad1000d6" + }, + { + "task": "w18", + "lang": "go", + "pass": true, + "sha256": "19863fb20722c8ed3e825ba945cfe96ae744a039645427eb8df2f05a0999d37d" + }, + { + "task": "w19", + "lang": "go", + "pass": true, + "sha256": "43fa83f674ca0f389064f3459690bf56b7c93012d2a0e4509c52d0b7da443179" + }, + { + "task": "w20", + "lang": "go", + "pass": true, + "sha256": "0d33eecd654e1c3bd362984822e89303356c9ca74cdec279c6534697f51d13f6" + }, + { + "task": "w21", + "lang": "go", + "pass": true, + "sha256": "75c8fd4190645f76d0b183df63521c860b9ae8557a6fc42fba46f2f74bfd407a" + }, + { + "task": "w22", + "lang": "go", + "pass": true, + "sha256": "c69dc057e04312eda1006ff2c2e7cc17bfcc19224ba911867248655d60c00795" + }, + { + "task": "w23", + "lang": "go", + "pass": true, + "sha256": "a60e29bb53f740977c3cca0ecbc38db054286077f9fd8ba4b9d0bb9ccb8a99cd" + }, + { + "task": "w24", + "lang": "go", + "pass": true, + "sha256": "746a2303c0e8de6587406c27667c661ce9d6cd57f1706e91743b4cd792256803" + }, + { + "task": "w25", + "lang": "go", + "pass": true, + "sha256": "3210e6658360234681380540ee0688c0a2af03168827a35bdf6042c7cbb55e25" + }, + { + "task": "w26", + "lang": "go", + "pass": true, + "sha256": "cdc2ed6aa7841aacaa6a9a0b30b0b7ce0deabfae212fd38694b5c22295bcdd8f" + }, + { + "task": "w27", + "lang": "go", + "pass": true, + "sha256": "7493a0f8647a10d1d470e3171a0b7d2d129b2281bd2bd4dc873ffa1421bf919b" + }, + { + "task": "w28", + "lang": "go", + "pass": true, + "sha256": "a8af038da85a66be55b02702b8f661d7e06cff191fd2315fbdeed69f5b6e45d3" + }, + { + "task": "w29", + "lang": "go", + "pass": true, + "sha256": "37d65c825351b610ed83c213ddabf18a1e9cf2d3a2420b7cf6f98b2e457d77ba" + }, + { + "task": "w30", + "lang": "go", + "pass": true, + "sha256": "e968bc037313c3744aa2d37d5aeae24ba7a1909e0ee53537abcb76ed33e3cad6" + }, + { + "task": "w31", + "lang": "go", + "pass": true, + "sha256": "9e248610f0ebca5e8f53e52e4949f5b5aee68e276da26dfffdc40240cf15c87c" + }, + { + "task": "w32", + "lang": "go", + "pass": true, + "sha256": "8127f3538eddf69d4d1c3ce9ee62347b3b59c021813134e74339d1b8839b82f8" + }, + { + "task": "w33", + "lang": "go", + "pass": true, + "sha256": "e45a2b82a6aaf636162155e9b3cfed3e0841b392ddb55fc81276c80953fa54ec" + }, + { + "task": "w34", + "lang": "go", + "pass": true, + "sha256": "044b22cd00a02c01988d9df6c6c8eb9283cfdccacc894b59b589f483dd53d177" + }, + { + "task": "w35", + "lang": "go", + "pass": true, + "sha256": "0c6c1b658315c6cc5660163d537d4cd823360c926c70790e14862c6cd904ee9d" + }, + { + "task": "w36", + "lang": "go", + "pass": true, + "sha256": "8886a0d175ffc4f93df5c6493bc21a8b24f1c2fb231f1664c5e1199670578218" + }, + { + "task": "w37", + "lang": "go", + "pass": true, + "sha256": "29249a8e0900d839ac59ec0defbdb79755cc87a440a391b390c26e64c69f60b7" + } +] diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w01.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w01.howl new file mode 100644 index 00000000..97507445 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w01.howl @@ -0,0 +1,9 @@ +(do + (let (out (list) cur "") + (do + (for w (str_split (str_trim (read_text "in/para.txt")) " ") + (if (= cur "") (set cur w) + (<= (str_len (str cur " " w)) 20) (set cur (str cur " " w)) + (do (append out cur) (set cur w)))) + (append out cur) + (write_file "out/result.txt" (str_join out "\n"))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w02.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w02.howl new file mode 100644 index 00000000..ccb142e6 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w02.howl @@ -0,0 +1,6 @@ +(do + (let (out (list)) + (do + (for l (lines (read_file "in/events.txt")) + (when (or (= (list_len out) 0) (!= l (list_get out (- (list_len out) 1)))) (append out l))) + (write_file "out/result.txt" (str_join out "\n"))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w03.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w03.howl new file mode 100644 index 00000000..0bd81553 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w03.howl @@ -0,0 +1 @@ +(do (write_file "out/result.txt" (str_join (filter s (map_list l (lines (read_file "in/settings.txt")) (rtrim (list_get (str_split l "#") 0))) (!= s "")) "\n"))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w04.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w04.howl new file mode 100644 index 00000000..e766c767 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w04.howl @@ -0,0 +1 @@ +(do (write_file "out/result.txt" (str_join (map_list l (lines (read_file "in/headings.txt")) (str_join (map_list w (str_split l " ") (str (to_upper (str_slice w 0 1)) (str_slice w 1))) " ")) "\n"))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w05.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w05.howl new file mode 100644 index 00000000..c6c468b1 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w05.howl @@ -0,0 +1,6 @@ +(do + (let (out (dict)) + (do + (for l (list_slice (lines (read_file "in/ledger.csv")) 1) + (let (r (str_split l ",")) (map_set out (list_get r 1) (+ (nz (map_get out (list_get r 1)) 0) (to_int (list_get r 2)))))) + (write_file "out/result.json" (encode_json out))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w06.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w06.howl new file mode 100644 index 00000000..c9fa56f5 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w06.howl @@ -0,0 +1,6 @@ +(do + (let (out (dict)) + (do + (for l (list_slice (lines (read_file "in/ledger.csv")) 1) + (let (r (str_split l ",") m (str_slice l 0 7)) (map_set out m (+ (nz (map_get out m) 0) (to_int (list_get r 2)))))) + (write_file "out/result.json" (encode_json out))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w07.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w07.howl new file mode 100644 index 00000000..335b4642 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w07.howl @@ -0,0 +1,6 @@ +(do + (let (ls (lines (read_file "in/lines.csv")) out (list (str (list_get ls 0) ",total"))) + (do + (for l (list_slice ls 1) + (let (r (str_split l ",")) (append out (str l "," (* (to_int (list_get r 1)) (to_int (list_get r 2))))))) + (write_file "out/result.csv" (str_join out "\n"))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w08.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w08.howl new file mode 100644 index 00000000..d2a3c4b0 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w08.howl @@ -0,0 +1,3 @@ +(do + (let (rows (map_list l (list_slice (lines (read_file "in/team.csv")) 1) (str_split l ",")) w (max (map_list r rows (str_len (list_get r 0))))) + (write_file "out/result.txt" (str_join (map_list r rows (str (pad_right (list_get r 0) w) " | " (list_get r 1))) "\n")))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w09.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w09.howl new file mode 100644 index 00000000..433f0e80 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w09.howl @@ -0,0 +1 @@ +(do (let (v (sort (map_list x (lines (read_file "in/latency.txt")) (to_int x)))) (write_file "out/result.txt" (to_string (list_get v (// (list_len v) 2)))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w10.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w10.howl new file mode 100644 index 00000000..6bdd53c2 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w10.howl @@ -0,0 +1,7 @@ +(do + (write_file "out/result.txt" + (str_join + (map_list x (lines (read_file "in/seconds.txt")) + (let (s (to_int x)) + (str (// s 3600) ":" (pad_left (% (// s 60) 60) 2 "0") ":" (pad_left (% s 60) 2 "0")))) + "\n"))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w11.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w11.howl new file mode 100644 index 00000000..249d56ac --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w11.howl @@ -0,0 +1,4 @@ +(do + (let (d (count_by x (str_split (str_trim (read_text "in/ages.txt")) ",") (str (// (to_int x) 10)))) + (write_file "out/result.txt" + (str_join (map_list b (sort_by k (map_keys d) (to_int k)) (str (* (to_int b) 10) "-" (+ (* (to_int b) 10) 9) ": " (map_get d b))) "\n")))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w12.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w12.howl new file mode 100644 index 00000000..2522adea --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w12.howl @@ -0,0 +1,3 @@ +(do + (let (a (map_list x (str_split (str_trim (read_text "in/load.txt")) ",") (to_int x))) + (write_file "out/result.json" (encode_json (map_list i (range (- (list_len a) 2)) (// (sum (list_slice a i (+ i 3))) 3)))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w13.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w13.howl new file mode 100644 index 00000000..1fd6184c --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w13.howl @@ -0,0 +1,3 @@ +(do + (let (q (parse_json Any (read_text "in/jobs.json"))) + (write_file "out/result.json" (encode_json (map_list i (range (// (+ (list_len q) 2) 3)) (list_slice q (* i 3) (+ (* i 3) 3))))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w14.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w14.howl new file mode 100644 index 00000000..ed37b471 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w14.howl @@ -0,0 +1,8 @@ +(do + (let (a (parse_json Any (read_text "in/left.json")) b (parse_json Any (read_text "in/right.json")) out (list)) + (do + (for i (range (max (list (list_len a) (list_len b)))) + (do + (when (< i (list_len a)) (append out (list_get a i))) + (when (< i (list_len b)) (append out (list_get b i))))) + (write_file "out/result.json" (encode_json out))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w15.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w15.howl new file mode 100644 index 00000000..9c4482cf --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w15.howl @@ -0,0 +1,5 @@ +(do + (let (r (parse_json Any (read_text "in/ring.json")) k (% (to_int (str_trim (read_text "in/k.txt"))) (list_len r)) out (list_slice r k)) + (do + (for x (list_slice r 0 k) (append out x)) + (write_file "out/result.json" (encode_json out))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w16.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w16.howl new file mode 100644 index 00000000..dbd0c6f9 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w16.howl @@ -0,0 +1,3 @@ +(do + (let (m (parse_json Any (read_text "in/meter.json"))) + (write_file "out/result.json" (encode_json (map_list i (range 1 (list_len m)) (- (list_get m i) (list_get m (- i 1)))))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w17.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w17.howl new file mode 100644 index 00000000..2ffbe324 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w17.howl @@ -0,0 +1,7 @@ +(do + (let (out "" prev "" n 0) + (do + (for c (str_split (str_trim (read_text "in/raw.txt")) "") + (if (= c prev) (inc n) + (do (when (> n 0) (set out (str out prev n))) (set prev c) (set n 1)))) + (write_file "out/result.txt" (str out prev n))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w18.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w18.howl new file mode 100644 index 00000000..099d86d7 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w18.howl @@ -0,0 +1 @@ +(do (write_file "out/result.txt" (str_join (map_list l (lines (read_file "in/idents.txt")) (str_join (map_list c (str_split l "") (if (!= c (to_lower c)) (str "_" (to_lower c)) c)) "")) "\n"))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w19.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w19.howl new file mode 100644 index 00000000..e0f58fe1 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w19.howl @@ -0,0 +1,7 @@ +(do + (let (abc (str_split "abcdefghijklmnopqrstuvwxyz" "")) + (write_file "out/result.txt" + (str_join + (map_list c (str_split (str_trim (read_text "in/msg.txt")) "") + (let (i (index_of abc c)) (if (< i 0) c (list_get abc (% (+ i 3) 26))))) + "")))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w20.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w20.howl new file mode 100644 index 00000000..4268831f --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w20.howl @@ -0,0 +1,6 @@ +(do + (write_file "out/result.txt" + (str_join + (map_list l (lines (read_file "in/cards.txt")) + (let (k (- (str_len l) 4)) (str (str_repeat "*" k) (str_slice l k)))) + "\n"))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w21.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w21.howl new file mode 100644 index 00000000..abc590a7 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w21.howl @@ -0,0 +1,8 @@ +(do + (let (out (dict)) + (do + (for s (parse_json Any (read_text "in/staff.json")) + (let (m (map_get s "manager")) + (when (!= m "") + (let (r (nz (map_get out m) (list))) (do (append r (map_get s "name")) (map_set out m r)))))) + (write_file "out/result.json" (encode_json out))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w22.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w22.howl new file mode 100644 index 00000000..9d2e084a --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w22.howl @@ -0,0 +1,7 @@ +(do + (let (sc (parse_json Any (read_text "in/scores.json"))) + (write_file "out/result.txt" + (str_join + (map_list u (parse_json Any (read_text "in/users.json")) + (str (map_get u "name") " " (sum (map_list s (filter s sc (= (map_get s "user") (map_get u "id"))) (map_get s "points"))))) + "\n")))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w23.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w23.howl new file mode 100644 index 00000000..5b4c85fd --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w23.howl @@ -0,0 +1,6 @@ +(do + (let (st (parse_json Any (read_text "in/stock.json"))) + (do + (for m (parse_json Any (read_text "in/moves.json")) + (map_set st (map_get m "sku") (+ (nz (map_get st (map_get m "sku")) 0) (map_get m "delta")))) + (write_file "out/result.json" (encode_json st))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w24.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w24.howl new file mode 100644 index 00000000..e2413dce --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w24.howl @@ -0,0 +1,7 @@ +(do + (let (out (dict)) + (do + (for c (parse_json Any (read_text "in/carts.json")) + (for i (map_get c "items") + (map_set out (map_get i "sku") (+ (nz (map_get out (map_get i "sku")) 0) (map_get i "qty"))))) + (write_file "out/result.json" (encode_json out))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w25.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w25.howl new file mode 100644 index 00000000..db25a5d7 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w25.howl @@ -0,0 +1,3 @@ +(do + (let (n (to_int (str_trim (read_text "in/n.txt")))) + (write_file "out/result.txt" (str_join (filter p (range 2 (+ n 1)) (not (any d (range 2 p) (= (% p d) 0)))) " ")))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w26.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w26.howl new file mode 100644 index 00000000..9e2840ad --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w26.howl @@ -0,0 +1,6 @@ +(do + (let (n (to_int (str_trim (read_text "in/n.txt"))) k 0) + (do + (while (!= n 1) + (do (set n (if (= (% n 2) 0) (// n 2) (+ (* 3 n) 1))) (inc k))) + (write_file "out/result.txt" (to_string k))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w27.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w27.howl new file mode 100644 index 00000000..5147e09b --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w27.howl @@ -0,0 +1,7 @@ +(do + (write_file "out/result.txt" + (str_join + (map_list l (lines (read_file "in/years.txt")) + (let (y (to_int l)) + (str y (if (= (% y (if (= (% y 100) 0) 400 4)) 0) " leap" " common")))) + "\n"))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w28.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w28.howl new file mode 100644 index 00000000..cea61335 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w28.howl @@ -0,0 +1,10 @@ +(do + (let (t (dict ("closed open" "opened") ("opened close" "closed") ("closed lock" "locked") ("locked unlock" "closed")) st "closed" i 0) + (do + (for c (lines (read_file "in/cmds.txt")) + (let (nx (map_get t (str st " " c))) + (do + (inc i) + (when (is_nil nx) (do (set st (str "error line " i ": cannot " c " when " st)) (break))) + (set st nx)))) + (write_file "out/result.txt" st)))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w29.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w29.howl new file mode 100644 index 00000000..38e4cb15 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w29.howl @@ -0,0 +1,9 @@ +(do + (let (b (to_int (str_trim (read_text "in/budget.txt"))) tot 0 i 0 res "within budget") + (do + (for x (lines (read_file "in/spend.txt")) + (do + (inc i) + (set tot (+ tot (to_int x))) + (when (> tot b) (do (set res (str "over at " i)) (break))))) + (write_file "out/result.txt" res)))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w30.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w30.howl new file mode 100644 index 00000000..14d61cd5 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w30.howl @@ -0,0 +1,12 @@ +(do + (let (v (dict ("I" 1) ("V" 5) ("X" 10) ("L" 50) ("C" 100) ("D" 500) ("M" 1000))) + (write_file "out/result.json" + (encode_json + (map_list l (lines (read_file "in/romans.txt")) + (let (ds (map_list c (str_split l "") (map_get v c)) t 0 i 0) + (do + (for d ds + (do + (inc i) + (set t (if (and (< i (list_len ds)) (< d (list_get ds i))) (- t d) (+ t d))))) + t))))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w31.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w31.howl new file mode 100644 index 00000000..e51dc655 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w31.howl @@ -0,0 +1 @@ +(do (write_file "out/result.txt" (str_join (map_list l (lines (read_file "in/nums.txt")) (to_base (to_int l) 2)) "\n"))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w32.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w32.howl new file mode 100644 index 00000000..974382f6 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w32.howl @@ -0,0 +1,11 @@ +(do + (write_file "out/result.txt" + (str_join + (map_list l (lines (read_file "in/ids.txt")) + (let (t 0 i 0) + (do + (for c (reverse (str_split l "")) + (let (d (* (to_int c) (+ 1 (% i 2)))) + (do (set t (+ t (if (> d 9) (- d 9) d))) (inc i)))) + (str l (if (= (% t 10) 0) " valid" " invalid"))))) + "\n"))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w33.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w33.howl new file mode 100644 index 00000000..c6ceec9e --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w33.howl @@ -0,0 +1,9 @@ +(do + (let (b (sort_by e (sort_by e (parse_json Any (read_text "in/board.json")) (map_get e "name")) (- 0 (map_get e "score"))) r 0 p 0 out (list)) + (do + (for e b + (let (s (map_get e "score")) + (do + (when (or (= r 0) (!= s p)) (do (inc r) (set p s))) + (append out (str r " " (map_get e "name") " " s))))) + (write_file "out/result.txt" (str_join out "\n"))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w34.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w34.howl new file mode 100644 index 00000000..75411f49 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w34.howl @@ -0,0 +1,10 @@ +(do + (let (g (map_list l (lines (read_file "in/maze.txt")) (str_split l "")) r 0 c 0 w 0 i 0) + (do + (for row g + (do (when (list_contains row "S") (do (set r i) (set c (index_of row "S")))) (inc i))) + (for d (list (list -1 0) (list 1 0) (list 0 -1) (list 0 1)) + (let ((dy dx) d y (+ r dy) x (+ c dx)) + (when (and (>= y 0) (< y (list_len g)) (>= x 0) (< x (list_len (list_get g y))) (= (list_get (list_get g y) x) "#")) + (inc w)))) + (write_file "out/result.txt" (str r "," c " " w))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w35.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w35.howl new file mode 100644 index 00000000..476bd7f0 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w35.howl @@ -0,0 +1,9 @@ +(do + (let (p (parse_json Any (read_text "in/prices.json")) t (to_int (str_trim (read_text "in/target.txt"))) res "none") + (do + (for j (range (list_len p)) + (do + (for i (range j) + (when (= (+ (list_get p i) (list_get p j)) t) (do (set res (str i " " j)) (break)))) + (when (!= res "none") (break)))) + (write_file "out/result.txt" res)))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w36.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w36.howl new file mode 100644 index 00000000..2ad60133 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w36.howl @@ -0,0 +1,10 @@ +(do + (write_file "out/result.txt" + (str_join + (map_list l (lines (read_file "in/passwords.txt")) + (let (cs (str_split l "")) + (if (< (str_len l) 8) "too short" + (not (any c cs (str_contains "0123456789" c))) "no digit" + (not (any c cs (str_contains "ABCDEFGHIJKLMNOPQRSTUVWXYZ" c))) "no upper" + "ok"))) + "\n"))) diff --git a/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w37.howl b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w37.howl new file mode 100644 index 00000000..ec244798 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf/solutions/howl/w37.howl @@ -0,0 +1,7 @@ +(do + (let (b (str_split (str_join (lines (read_file "in/board.txt")) "") "") res "") + (do + (for t (list (list 0 1 2) (list 3 4 5) (list 6 7 8) (list 0 3 6) (list 1 4 7) (list 2 5 8) (list 0 4 8) (list 2 4 6)) + (let ((i j k) t a (list_get b i)) + (when (and (!= a ".") (= a (list_get b j)) (= a (list_get b k))) (set res a)))) + (write_file "out/result.txt" (if (!= res "") res (list_contains b ".") "open" "draw"))))) diff --git a/experiments/token-efficiency-heldout-v2/postcf_test.go b/experiments/token-efficiency-heldout-v2/postcf_test.go new file mode 100644 index 00000000..0ff114c7 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/postcf_test.go @@ -0,0 +1,71 @@ +package heldoutv2 + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "testing" +) + +// The post-control-flow HowlFrame set (postcf/) is a separate results set: +// every task x language is verified with a current hash, the Python/JS/Go +// references are the baseline files, and the results are pinned below. +func TestHeldoutV2PostCFVerificationCoverage(t *testing.T) { + var manifest struct { + Tasks []struct { + ID string `json:"id"` + } `json:"tasks"` + } + readJSON(t, "tasks/manifest.json", &manifest) + var results []struct { + Task, Lang, SHA256 string + Pass bool + } + readJSON(t, "postcf/results/verification.json", &results) + seen := map[string]bool{} + for _, r := range results { + key := r.Task + "/" + r.Lang + if seen[key] { + t.Errorf("duplicate %s", key) + } + seen[key] = true + if !r.Pass { + t.Errorf("%s did not pass", key) + } + path := filepath.Join("solutions", r.Lang, r.Task+"."+exts[r.Lang]) + if r.Lang == "howl" { + path = filepath.Join("postcf", "solutions", "howl", r.Task+".howl") + } + src, err := os.ReadFile(path) + if err != nil { + t.Errorf("%s: %v", key, err) + continue + } + if got := fmt.Sprintf("%x", sha256.Sum256(src)); got != r.SHA256 { + t.Errorf("%s: stale hash", key) + } + } + if want := len(manifest.Tasks) * len(exts); len(results) != want { + t.Errorf("%d results, want %d (postcf has no gaps)", len(results), want) + } +} + +func TestHeldoutV2PostCFResultsFrozen(t *testing.T) { + frozen := map[string]string{ + "postcf/results/compare.md": "03020dcb9beb29976460b670a5331e1df009f5115c24699cd4503bf927078cbb", + "postcf/results/summary.json": "4553229c7ce800311b854394017b97b55cb62dbfb79aa39c82be3690930f03e1", + "postcf/results/tokens.csv": "acf707796bc89132c88a31918ec152d8e77843aec4ba818b7ccd8889c9bfcfee", + "postcf/results/tokens.json": "e10b5b43497db767c4fd4e77b6ddf97976854e4ce11c2a57b63899e5f210c08e", + "postcf/results/verification.json": "11b1844f42e57b818c9cb3b31672a8ef8df72feddf0eaa7a723d9f570f5dff16", + } + for path, want := range frozen { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if got := fmt.Sprintf("%x", sha256.Sum256(data)); got != want { + t.Errorf("%s changed after post-control-flow freeze: %s", path, got) + } + } +} diff --git a/experiments/token-efficiency-heldout-v2/run_postcf.sh b/experiments/token-efficiency-heldout-v2/run_postcf.sh new file mode 100755 index 00000000..f9354144 --- /dev/null +++ b/experiments/token-efficiency-heldout-v2/run_postcf.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Post-control-flow rerun: same specs, same Python/JS/Go references and the +# same verifier/tokenizer as run.sh, HowlFrame sources from postcf/. The +# frozen baseline (solutions/howl, results/) is not touched. +set -euo pipefail +cd "$(dirname "$0")/../.." +export GOCACHE="${GOCACHE:-/tmp/hf-token-go-cache}" +go build -o /tmp/howlframe . +go build -o /tmp/heldoutv2verify ./experiments/token-efficiency-heldout-v2/cmd/heldoutv2verify +B=experiments/token-efficiency-heldout-v2 +status=0 +/tmp/heldoutv2verify -howlframe /tmp/howlframe -howl-dir "$B/postcf/solutions/howl" -results "$B/postcf/results" || status=$? +if [ "$status" -eq 0 ]; then + "${TOKBENCH_PYTHON:-python3}" "$B/count_tokens.py" --howl-dir postcf/solutions/howl --results postcf/results + "${TOKBENCH_PYTHON:-python3}" "$B/compare_postcf.py" +fi +exit "$status" diff --git a/howlframe.go b/howlframe.go index 79a3558e..363a1918 100644 --- a/howlframe.go +++ b/howlframe.go @@ -792,6 +792,7 @@ func buildSource() { root = ast.ApplyWithContext(root, nil) root = ast.ApplyWithContext(root, nil) construct.EnforceArity(root) + rejectUnsupportedTargetControlflow(root, strings.ToLower(strings.TrimSpace(*target))) ast.Desugar(root) if diags := hfir.VerifyProfile(root, profile, inputFile, ""); len(diags) > 0 { reportHFIRDiagnostics(diags) @@ -1242,3 +1243,47 @@ func rejectUnsupportedTargetValueForms(root *ast.Node, target string) { } walk(root, false) } + +// rejectUnsupportedTargetControlflow fails closed on --target=go|js for the +// heldout-v2 control-flow and core built-ins (%, //, break, range, max_by, +// min_by, sort_by, pad_left, pad_right, str_repeat, rtrim, ltrim, to_base), +// n-ary if (elif chains) and let destructuring. They are bytecode-target sugar: +// the expansions use let-expressions in value position or loop flags the Go +// and JS backends do not emit. It runs on the source tree before desugaring, +// before any file is written: nonzero exit, no "Built", no output. +func rejectUnsupportedTargetControlflow(root *ast.Node, target string) { + if target != "go" && target != "js" && target != "javascript" { + return + } + heads := map[string]bool{} + for _, h := range ast.ControlflowHeads { + heads[h] = true + } + fail := func(what string, n *ast.Node) { + ast.ReportError(fmt.Sprintf("UNSUPPORTED_ON_TARGET: %s is not supported on --target=%s (control-flow/core built-ins are bytecode-target sugar); build for the bytecode target instead", what, target), n.Line, n.Column) + } + var walk func(n *ast.Node) + walk = func(n *ast.Node) { + if n == nil || n.Type != "List" { + return + } + if len(n.Children) > 0 && n.Children[0].Type == "SYMBOL" { + switch h := n.Children[0].Value; { + case heads[h]: + fail(h, n) + case h == "if" && len(n.Children) > 4: + fail("n-ary if (elif chain)", n) + case h == "let" && len(n.Children) > 1 && n.Children[1] != nil: + for i, b := range n.Children[1].Children { + if i%2 == 0 && b != nil && b.Type == "List" && len(n.Children[1].Children)%2 == 0 { + fail("let destructuring", b) + } + } + } + } + for _, c := range n.Children { + walk(c) + } + } + walk(root) +} diff --git a/internal/ast/desugar.go b/internal/ast/desugar.go index 3af06e27..db254d77 100644 --- a/internal/ast/desugar.go +++ b/internal/ast/desugar.go @@ -46,6 +46,7 @@ func Desugar(node *Node) { collectDefunNames(node, funcs) desugarInContext(node, "root", 0) rewriteAutoCallsInContext(node, funcs, "root", 0) + reportStrayBreaks(node) } // desugarInContext walks the tree with parent context so binding pairs and @@ -68,6 +69,7 @@ func desugarInContext(node *Node, parentHead string, childIndex int) { } // Expand multi-binding let before walking so children see single bindings. if head == "let" { + desugarDestructure(node) desugarMultiLet(node) head = node.Children[0].Value } @@ -225,6 +227,14 @@ func desugarInContext(node *Node, parentHead string, childIndex int) { if isSugarExprContext(parentHead, childIndex) && desugarCorelib(node) { desugarInContext(node, parentHead, childIndex) } + case "%", "//", "range", "max_by", "min_by", "sort_by", "pad_left", "pad_right", "str_repeat", "rtrim", "ltrim", "to_base": + if isSugarExprContext(parentHead, childIndex) && desugarControlflow(node) { + desugarInContext(node, parentHead, childIndex) + } + case "if": + desugarNaryIf(node) + case "for", "while": + desugarLoopBreak(node) case "list_get": // A literal (list …) operand is left alone: backends such as wasm // lower it directly and do not support let. @@ -262,7 +272,9 @@ func isSugarExprContext(parentHead string, childIndex int) bool { "req_header", "req_method", "req_path", "req_query", "llm_generate", "confidence", // Core-library sugar (heldout-v1). - "list_slice", "str_slice", "str_len", "index_of", "map_list", "filter", "reduce", "sum", "min", "max", "reverse", "count_by", "sort": + "list_slice", "str_slice", "str_len", "index_of", "map_list", "filter", "reduce", "sum", "min", "max", "reverse", "count_by", "sort", + // Control-flow and core built-ins (heldout-v2 follow-up). + "%", "//", "range", "max_by", "min_by", "sort_by", "pad_left", "pad_right", "str_repeat", "rtrim", "ltrim", "to_base": return true case "for": return childIndex == 2 || childIndex == 3 @@ -363,6 +375,8 @@ var reservedCallHeads = map[string]bool{ "blank": true, "unix_now": true, "ver_gt": true, "ver_ge": true, "all": true, "any": true, "parse_json": true, "list_slice": true, "str_slice": true, "str_len": true, "index_of": true, "map_list": true, "filter": true, "reduce": true, "sum": true, "min": true, "max": true, "reverse": true, "count_by": true, "sort": true, + "%": true, "//": true, "range": true, "max_by": true, "min_by": true, "sort_by": true, "pad_left": true, + "pad_right": true, "str_repeat": true, "rtrim": true, "ltrim": true, "to_base": true, "break": true, "dict": true, "exit": true, "for": true, "if": true, "let": true, "list": true, "print": true, "return": true, "set": true, "stderr": true, "to_lower": true, "to_upper": true, "str_trim": true, diff --git a/internal/ast/desugar_controlflow.go b/internal/ast/desugar_controlflow.go new file mode 100644 index 00000000..98a5c78a --- /dev/null +++ b/internal/ast/desugar_controlflow.go @@ -0,0 +1,312 @@ +package ast + +import "strings" + +// Control-flow and core built-ins (heldout-v2 follow-up). Like the +// heldout-v1 core library, every form expands into existing, already lowered +// constructs (let, if, while, for, set, append, list_get, list_len, str_split, +// str_join, arithmetic, comparisons), so it adds no opcode, capability or +// effect and runs identically on the interpreter, AST bytecode and HFIR +// bytecode. Go/JS targets reject every form with UNSUPPORTED_ON_TARGET. +// +// (% a b) integer remainder, floored like Python: the result +// has the sign of b. (% 7 3)=1, (% -7 3)=2, (% 7 -3)=-2. +// (// a b) floored integer division: (// -7 2) = -4. +// Both use exact int arithmetic around one float +// division, so |a| must stay below 2^53. b = 0 is the +// existing RUNTIME_ERROR "division by zero". +// (if c1 a c2 b ... [else]) n-ary if: elif chains without cond clauses. +// (break) leaves the innermost for/while. Statement position +// only (directly or via do/if/when/unless/let bodies +// inside the loop); anywhere else is a compile error. +// (range n) / (range a b) list of ints a..b-1 (empty when b <= a). +// (max_by x lst key) / (min_by x lst key) element with the largest / +// smallest key (first on ties). Empty list: +// "VALUE_ERROR: max_by of empty list", exit 1. +// (sort_by x lst key) stable ascending sort by key (numbers or strings). +// Keys are computed once per element. O(n^2). +// (pad_left s n [ch]) / (pad_right s n [ch]) pad (str s) with ch +// (default " ") until it is at least n bytes. +// (str_repeat s n) s repeated n times ("" when n <= 0). +// (rtrim s) / (ltrim s) strip " ", "\t", "\n", "\r" from one end. +// (to_base n b) n in base b (2..36), lowercase digits, "-" prefix +// for negatives. Other bases: "VALUE_ERROR: to_base +// base must be 2..36", exit 1. +// (let ((a b) lst) body) destructuring: a = (list_get lst 0), b = ... . + +var controlflowTemplates = map[string]struct { + arity int + vars []string + temps []string + tpl string + binder int +}{ + "%": {2, []string{"a", "b"}, []string{"A", "B", "R"}, + "(let ($A $a $B $b $R (- $A (* $B (to_int (/ $A $B))))) (if (and (!= $R 0) (!= (< $R 0) (< $B 0))) (+ $R $B) $R))", -1}, + "//": {2, []string{"a", "b"}, []string{"A", "B"}, + "(let ($A $a $B $b) (to_int (/ (- $A (% $A $B)) $B)))", -1}, + "range": {2, []string{"a", "b"}, []string{"A", "B", "R"}, + "(let ($A $a $B $b $R (list)) (do (while (< $A $B) (do (append $R $A) (set $A (+ $A 1)))) $R))", -1}, + "max_by": {3, []string{"x", "lst", "key"}, []string{"L", "KS", "I", "M", "MK", "K"}, + "(let ($L $lst $KS (map_list $x $L $key) $I 0 $M (list_get $L 0) $MK (list_get $KS 0)) (do (when (= (list_len $L) 0) (do (stderr $msg) (exit 1))) (for $K $KS (do (when (> $K $MK) (do (set $M (list_get $L $I)) (set $MK $K))) (set $I (+ $I 1)))) $M))", 0}, + "min_by": {3, []string{"x", "lst", "key"}, []string{"L", "KS", "I", "M", "MK", "K"}, + "(let ($L $lst $KS (map_list $x $L $key) $I 0 $M (list_get $L 0) $MK (list_get $KS 0)) (do (when (= (list_len $L) 0) (do (stderr $msg) (exit 1))) (for $K $KS (do (when (< $K $MK) (do (set $M (list_get $L $I)) (set $MK $K))) (set $I (+ $I 1)))) $M))", 0}, + "sort_by": {3, []string{"x", "lst", "key"}, []string{"L", "KS", "R", "I", "K", "N", "F", "J", "Y"}, + "(let ($L $lst $KS (map_list $x $L $key) $R (list) $I 0) (do (for $K $KS (let ($N (list) $F 0) (do (for $J $R (do (when (and (= $F 0) (< $K (list_get $KS $J))) (do (append $N $I) (set $F 1))) (append $N $J))) (when (= $F 0) (append $N $I)) (set $R $N) (set $I (+ $I 1))))) (map_list $Y $R (list_get $L $Y))))", 0}, + "pad_left": {3, []string{"s", "n", "ch"}, []string{"S", "N", "C", "P"}, + "(let ($S (str $s) $N $n $C $ch $P $E) (do (while (< (+ (str_len $P) (str_len $S)) $N) (set $P (str $P $C))) (str $P $S)))", -1}, + "pad_right": {3, []string{"s", "n", "ch"}, []string{"S", "N", "C"}, + "(let ($S (str $s) $N $n $C $ch) (do (while (< (str_len $S) $N) (set $S (str $S $C))) $S))", -1}, + "str_repeat": {2, []string{"s", "n"}, []string{"S", "N", "R"}, + "(let ($S $s $N $n $R $E) (do (while (> $N 0) (do (set $R (str $R $S)) (set $N (- $N 1)))) $R))", -1}, + "rtrim": {1, []string{"s"}, []string{"C", "N"}, + "(let ($C (str_split $s $E) $N (list_len $C)) (do (while (and (> $N 0) (list_contains $WS (list_get $C (- $N 1)))) (set $N (- $N 1))) (str_join (list_slice $C 0 $N) $E)))", -1}, + "ltrim": {1, []string{"s"}, []string{"C", "N", "I"}, + "(let ($C (str_split $s $E) $N (list_len $C) $I 0) (do (while (and (< $I $N) (list_contains $WS (list_get $C $I))) (set $I (+ $I 1))) (str_join (list_slice $C $I $N) $E)))", -1}, + "to_base": {2, []string{"n", "b"}, []string{"N", "B", "G", "R", "NEG"}, + "(let ($N $n $B $b $G (str_split $D $E) $R $E $NEG (< $N 0)) (do (when (or (< $B 2) (> $B 36)) (do (stderr $msg) (exit 1))) (when $NEG (set $N (- 0 $N))) (while (> $N 0) (do (set $R (str (list_get $G (% $N $B)) $R)) (set $N (// $N $B)))) (when (= $R $E) (set $R $Z)) (when $NEG (set $R (str $MINUS $R))) $R))", -1}, +} + +// ControlflowHeads lists every head added by this file plus break, for the +// Go/JS target rejection and the reserved-call list. +var ControlflowHeads = []string{"%", "//", "range", "max_by", "min_by", "sort_by", "pad_left", "pad_right", "str_repeat", "rtrim", "ltrim", "to_base", "break"} + +// IsControlflowSugar reports whether head is a template form of this file. +func IsControlflowSugar(head string) bool { + _, ok := controlflowTemplates[head] + return ok +} + +func strNode(v string, at *Node) *Node { + return &Node{Type: "STRING", Value: v, Line: at.Line, Column: at.Column, Filename: at.Filename} +} + +func desugarControlflow(node *Node) bool { + head := node.Children[0].Value + spec := controlflowTemplates[head] + switch { + case head == "range" && len(node.Children) == 2: + zero := &Node{Type: "INT", Value: "0", Line: node.Line, Column: node.Column, Filename: node.Filename} + node.Children = []*Node{node.Children[0], zero, node.Children[1]} + case (head == "pad_left" || head == "pad_right") && len(node.Children) == 3: + node.Children = append(node.Children, strNode(" ", node)) + } + if len(node.Children) != spec.arity+1 { + ReportError(head+" expects "+itoa(spec.arity)+" arguments", node.Line, node.Column) + return false + } + if spec.binder >= 0 && node.Children[spec.binder+1].Type != "SYMBOL" { + ReportError(head+" expects a variable name as argument "+itoa(spec.binder+1), node.Line, node.Column) + return false + } + args := map[string]*Node{} + for i, v := range spec.vars { + args[v] = node.Children[i+1] + } + // Literal constants get a fresh node per occurrence (the template + // builder only clones symbols). + consts := map[string]func() *Node{ + "E": func() *Node { return strNode("", node) }, + "Z": func() *Node { return strNode("0", node) }, + "MINUS": func() *Node { return strNode("-", node) }, + "D": func() *Node { return strNode("0123456789abcdefghijklmnopqrstuvwxyz", node) }, + "WS": func() *Node { + return &Node{Type: "List", Line: node.Line, Column: node.Column, Filename: node.Filename, Children: []*Node{ + symNode("list", node), strNode(" ", node), strNode("\t", node), strNode("\n", node), strNode("\r", node)}} + }, + } + tpl := spec.tpl + toks := strings.Fields(strings.NewReplacer("(", " ( ", ")", " ) ").Replace(tpl)) + for i, t := range toks { + if mk, ok := consts[strings.TrimPrefix(t, "$")]; ok && strings.HasPrefix(t, "$") { + key := "const" + itoa(i) + args[key] = mk() + toks[i] = "$" + key + } + } + tpl = strings.Join(toks, " ") + switch head { + case "max_by", "min_by": + args["msg"] = strNode("VALUE_ERROR: "+head+" of empty list\n", node) + case "to_base": + args["msg"] = strNode("VALUE_ERROR: to_base base must be 2..36\n", node) + } + for _, tmp := range spec.temps { + args[tmp] = symNode(compactTemp("cl"+tmp), node) + } + node.Children = buildTemplateFresh(tpl, args, nil, node).Children + return true +} + +// desugarNaryIf turns (if c1 a c2 b ... [else]) with more than three +// arguments into nested binary ifs. Returns true when it rewrote the node. +func desugarNaryIf(node *Node) bool { + if len(node.Children) <= 4 { + return false + } + args := node.Children[1:] + var build func(i int) *Node + build = func(i int) *Node { + rest := len(args) - i + if rest == 1 { + return args[i] + } + n := &Node{Type: "List", Line: node.Line, Column: node.Column, Filename: node.Filename, + Children: []*Node{symNode("if", node), args[i], args[i+1]}} + if rest > 2 { + n.Children = append(n.Children, build(i+2)) + } + return n + } + node.Children = build(0).Children + return true +} + +// desugarDestructure rewrites list-pattern bindings in a let binding list: +// ((a b) v) becomes (T v a (list_get T 0) b (list_get T 1)). +func desugarDestructure(node *Node) { + if len(node.Children) < 3 { + return + } + binding := node.Children[1] + if binding == nil || binding.Type != "List" || len(binding.Children)%2 != 0 { + return + } + var out []*Node + changed := false + for i := 0; i+1 < len(binding.Children); i += 2 { + name, val := binding.Children[i], binding.Children[i+1] + if name == nil || name.Type != "List" || len(name.Children) == 0 || !allSymbols(name.Children) { + out = append(out, name, val) + continue + } + changed = true + tmp := compactTemp("clDS") + out = append(out, symNode(tmp, node), val) + for j, sym := range name.Children { + idx := &Node{Type: "INT", Value: itoa(j), Line: sym.Line, Column: sym.Column, Filename: sym.Filename} + get := &Node{Type: "List", Line: sym.Line, Column: sym.Column, Filename: sym.Filename, + Children: []*Node{symNode("list_get", sym), symNode(tmp, sym), idx}} + out = append(out, sym, get) + } + } + if changed { + binding.Children = out + } +} + +func allSymbols(ns []*Node) bool { + for _, n := range ns { + if n == nil || n.Type != "SYMBOL" { + return false + } + } + return true +} + +func isBreak(n *Node) bool { + return n != nil && n.Type == "List" && len(n.Children) >= 1 && n.Children[0].Type == "SYMBOL" && n.Children[0].Value == "break" +} + +func isLoop(n *Node) bool { + return n != nil && n.Type == "List" && len(n.Children) > 0 && n.Children[0].Type == "SYMBOL" && + (n.Children[0].Value == "for" || n.Children[0].Value == "while") +} + +// rewriteBreaks replaces statement-position (break) forms in a loop body by +// (set F 1) and guards every later statement of an enclosing do with +// (if (= F 0) ...). It does not enter nested loops (their breaks are already +// rewritten bottom-up) and reports whether it rewrote anything. +func rewriteBreaks(n *Node, flag string) bool { + if n == nil || n.Type != "List" || len(n.Children) == 0 || isLoop(n) { + return false + } + if isBreak(n) { + if len(n.Children) != 1 { + ReportError("break takes no arguments", n.Line, n.Column) + } + n.Children = []*Node{symNode("set", n), symNode(flag, n), {Type: "INT", Value: "1", Line: n.Line, Column: n.Column, Filename: n.Filename}} + return true + } + if n.Children[0].Type != "SYMBOL" { + return false + } + switch n.Children[0].Value { + case "do", "cli_app": + for i := 1; i < len(n.Children); i++ { + if !rewriteBreaks(n.Children[i], flag) { + continue + } + if i+1 < len(n.Children) { + rest := &Node{Type: "List", Line: n.Line, Column: n.Column, Filename: n.Filename, + Children: append([]*Node{symNode("do", n)}, n.Children[i+1:]...)} + rewriteBreaks(rest, flag) + guard := &Node{Type: "List", Line: n.Line, Column: n.Column, Filename: n.Filename, + Children: []*Node{symNode("if", n), flagIsZero(flag, n), rest}} + n.Children = append(n.Children[:i+1], guard) + } + return true + } + return false + case "if": + hit := false + for i := 2; i < len(n.Children); i++ { + if rewriteBreaks(n.Children[i], flag) { + hit = true + } + } + return hit + case "let": + if len(n.Children) > 2 { + return rewriteBreaks(n.Children[2], flag) + } + } + return false +} + +func flagIsZero(flag string, at *Node) *Node { + return &Node{Type: "List", Line: at.Line, Column: at.Column, Filename: at.Filename, Children: []*Node{ + symNode("=", at), symNode(flag, at), {Type: "INT", Value: "0", Line: at.Line, Column: at.Column, Filename: at.Filename}}} +} + +// desugarLoopBreak rewrites a for/while whose body breaks into +// (let (F 0) (while (and (= F 0) c) body')) or +// (let (F 0) (for x lst (if (= F 0) body'))). A for keeps iterating its +// (already evaluated) list but does nothing after the break. +func desugarLoopBreak(node *Node) { + if len(node.Children) < 3 { + return + } + bodyIdx := len(node.Children) - 1 + flag := compactTemp("clBRK") + if !rewriteBreaks(node.Children[bodyIdx], flag) { + return + } + loop := &Node{Type: "List", Line: node.Line, Column: node.Column, Filename: node.Filename, Children: node.Children} + if node.Children[0].Value == "while" { + loop.Children[1] = &Node{Type: "List", Line: node.Line, Column: node.Column, Filename: node.Filename, + Children: []*Node{symNode("and", node), flagIsZero(flag, node), loop.Children[1]}} + } else { + loop.Children[bodyIdx] = &Node{Type: "List", Line: node.Line, Column: node.Column, Filename: node.Filename, + Children: []*Node{symNode("if", node), flagIsZero(flag, node), loop.Children[bodyIdx]}} + } + node.Children = []*Node{symNode("let", node), + {Type: "List", Line: node.Line, Column: node.Column, Filename: node.Filename, Children: []*Node{ + symNode(flag, node), {Type: "INT", Value: "0", Line: node.Line, Column: node.Column, Filename: node.Filename}}}, + loop} +} + +// reportStrayBreaks fails closed on any (break) left after desugaring: one +// outside a loop, or in a value position the loop rewrite does not reach. +func reportStrayBreaks(n *Node) { + if n == nil || n.Type != "List" { + return + } + if isBreak(n) { + ReportError("break must be a statement inside for/while (directly or in do/if/when/unless/let bodies)", n.Line, n.Column) + return + } + for _, c := range n.Children { + reportStrayBreaks(c) + } +} diff --git a/internal/bytecode/construct_drift_test.go b/internal/bytecode/construct_drift_test.go index 63e66276..c283bbdd 100644 --- a/internal/bytecode/construct_drift_test.go +++ b/internal/bytecode/construct_drift_test.go @@ -139,6 +139,19 @@ func TestCompileTimeOnlyConstructsReachingCompileNodeHaveCases(t *testing.T) { "reverse": true, // ast.Desugar core library "count_by": true, // ast.Desugar core library "sort": true, // ast.Desugar core library + "%": true, // ast.Desugar control-flow + "//": true, // ast.Desugar control-flow + "break": true, // ast.Desugar control-flow (loop flag) + "range": true, // ast.Desugar control-flow + "max_by": true, // ast.Desugar core library + "min_by": true, // ast.Desugar core library + "sort_by": true, // ast.Desugar core library + "pad_left": true, // ast.Desugar core library + "pad_right": true, // ast.Desugar core library + "str_repeat": true, // ast.Desugar core library + "rtrim": true, // ast.Desugar core library + "ltrim": true, // ast.Desugar core library + "to_base": true, // ast.Desugar core library "ver_ge": true, // ast.Desugar "nz": true, // ast.Desugar "sget": true, // ast.Desugar diff --git a/internal/construct/arity.go b/internal/construct/arity.go index 26c8349e..f51b049d 100644 --- a/internal/construct/arity.go +++ b/internal/construct/arity.go @@ -36,7 +36,7 @@ var arities = map[string]Arity{ "env": {1, 1}, "ephemeral_circuit": {2, Variadic}, "exec": {1, Variadic}, "exit": {1, 1}, "fetch": {2, 3}, "filter": {3, 3}, "for": {3, 3}, "html_escape": {1, 1}, "http_body": {1, 1}, "http_request": {2, 3}, "http_res": {1, 1}, "http_server": {1, Variadic}, - "http_status": {1, 1}, "if": {2, 3}, "inc": {1, 1}, "index_of": {2, 2}, "is_nil": {1, 1}, + "http_status": {1, 1}, "if": {2, Variadic}, "inc": {1, 1}, "index_of": {2, 2}, "is_nil": {1, 1}, "lazy_synthesize": {3, 3}, "let": {2, 2}, "lines": {1, 1}, "list": {0, Variadic}, "list_contains": {2, 2}, "list_get": {2, 2}, "list_len": {1, 1}, "list_slice": {2, 3}, "llm_generate": {1, 2}, "map_delete": {2, 2}, "map_get": {2, 2}, "map_has": {2, 2}, @@ -58,6 +58,10 @@ var arities = map[string]Arity{ "try_let": {2, Variadic}, "unix_now": {0, 0}, "unless": {1, Variadic}, "ver_ge": {2, 2}, "ver_gt": {2, 2}, "wasm_app": {0, Variadic}, "web_app": {0, Variadic}, "when": {1, Variadic}, "while": {2, 2}, "write_file": {2, 2}, + // Control-flow and core built-ins (heldout-v2 follow-up). + "%": {2, 2}, "//": {2, 2}, "break": {0, 0}, "range": {1, 2}, "max_by": {3, 3}, "min_by": {3, 3}, + "sort_by": {3, 3}, "pad_left": {2, 3}, "pad_right": {2, 3}, "str_repeat": {2, 2}, "rtrim": {1, 1}, + "ltrim": {1, 1}, "to_base": {2, 2}, } // arityExempt lists registry constructs with no arity entry, each with the diff --git a/internal/construct/arity_test.go b/internal/construct/arity_test.go index 845a1332..bf1aa783 100644 --- a/internal/construct/arity_test.go +++ b/internal/construct/arity_test.go @@ -57,6 +57,12 @@ func TestArityMessages(t *testing.T) { `(cli_app (fetch "u"))`: "ARITY_ERROR: fetch expects 2 to 3 args, got 1", `(cli_app (defun f (x)))`: "ARITY_ERROR: defun expects at least 3 args, got 2", `(cli_app (write_file "a" "b" "c"))`: "ARITY_ERROR: write_file expects 2 args, got 3", + `(cli_app (print (% 1)))`: "ARITY_ERROR: % expects 2 args, got 1", + `(cli_app (print (range)))`: "ARITY_ERROR: range expects 1 to 2 args, got 0", + `(cli_app (for x (list 1) (break 1)))`: "ARITY_ERROR: break expects 0 args, got 1", + `(cli_app (print (sort_by x (list 1))))`: "ARITY_ERROR: sort_by expects 3 args, got 2", + `(cli_app (print (pad_left "a")))`: "ARITY_ERROR: pad_left expects 2 to 3 args, got 1", + `(cli_app (print (to_base 1 2 3)))`: "ARITY_ERROR: to_base expects 2 args, got 3", `(cli_app (let (x (read_file)) (print x)))`: "ARITY_ERROR: read_file expects 1 arg, got 0", } for src, want := range cases { diff --git a/internal/construct/construct.go b/internal/construct/construct.go index c95991a5..e822a561 100644 --- a/internal/construct/construct.go +++ b/internal/construct/construct.go @@ -80,10 +80,12 @@ type Entry struct { // a copy on every call. construct_test.go enforces the ordering. var table = []Entry{ {Name: "!=", Support: Supported, Note: "compileNode binop case"}, + {Name: "%", Support: CompileTimeOnly, Note: "control-flow sugar; (% a b) floored remainder via to_int(/)/*/- by ast.Desugar (pure)"}, {Name: "*", Support: Supported, Note: "compileNode binop case"}, {Name: "+", Support: Supported, Note: "compileNode binop case"}, {Name: "-", Support: Supported, Note: "compileNode binop case"}, {Name: "/", Support: Supported, Note: "compileNode binop case"}, + {Name: "//", Support: CompileTimeOnly, Note: "control-flow sugar; (// a b) floored integer division via % and / by ast.Desugar (pure)"}, {Name: "<", Support: Supported, Note: "compileNode binop case"}, {Name: "<=", Support: Supported, Note: "compileNode binop case"}, {Name: "=", Support: Supported, Note: "compileNode binop case, normalized to =="}, @@ -99,6 +101,7 @@ var table = []Entry{ {Name: "assert_semantic", Support: Unsupported, Note: "LLM intent assertion; no opcode, Go backend only"}, {Name: "attr_escape", Support: Supported, Note: "ATTR_ESCAPE; same encoding as html_escape, named for attribute context"}, {Name: "blank", Support: CompileTimeOnly, Note: "density sugar; (blank x) → (= x \"\") by ast.Desugar"}, + {Name: "break", Support: CompileTimeOnly, Note: "control-flow sugar; (break) in a for/while body becomes a flag set + guarded rest by ast.Desugar (pure)"}, {Name: "bytes_to_string", Support: Supported, Note: "CONVERT"}, {Name: "call", Support: Supported, Note: "CALL"}, {Name: "cli_app", Support: Supported, Note: "compiles every child in sequence"}, @@ -144,6 +147,7 @@ var table = []Entry{ {Name: "list_len", Support: Supported, Note: "LIST_LEN; list, string, or bytes"}, {Name: "list_slice", Support: CompileTimeOnly, Note: "core-library sugar; (list_slice lst a b) elements a..b-1, negative from end by ast.Desugar (pure)"}, {Name: "llm_generate", Support: Supported, Note: "LLM_GENERATE"}, + {Name: "ltrim", Support: CompileTimeOnly, Note: "core-library sugar; (ltrim s) via str_split/while/list_slice/str_join by ast.Desugar (pure)"}, {Name: "map_delete", Support: Supported, Note: "MAP_DELETE"}, {Name: "map_get", Support: Supported, Note: "MAP_GET"}, {Name: "map_has", Support: CompileTimeOnly, Note: "density sugar; dict key membership via map_keys by ast.Desugar"}, @@ -152,8 +156,10 @@ var table = []Entry{ {Name: "map_set", Support: Supported, Note: "MAP_SET"}, {Name: "match", Support: Unsupported, Note: "no opcode; the interpreter also fails closed on it (internal/vm/vm.go); use switch"}, {Name: "max", Support: CompileTimeOnly, Note: "core-library sugar; (max lst) for/when/> by ast.Desugar (pure)"}, + {Name: "max_by", Support: CompileTimeOnly, Note: "core-library sugar; (max_by x lst key) via map_list/for by ast.Desugar (pure)"}, {Name: "middleware", Support: Unsupported, Note: "HTTP middleware chain; no opcode, Go backend only"}, {Name: "min", Support: CompileTimeOnly, Note: "core-library sugar; (min lst) for/when/< by ast.Desugar (pure)"}, + {Name: "min_by", Support: CompileTimeOnly, Note: "core-library sugar; (min_by x lst key) via map_list/for by ast.Desugar (pure)"}, {Name: "mkdir", Support: Supported, Note: "MKDIR"}, {Name: "module", Support: CompileTimeOnly, Note: "parser.ExpandIncludes/ast.ResolveModules consume (module ...) children of the root before checker.Check or lowering ever run (improvements.md #95); a root that is itself a module (no importer) is a malformed standalone program and fails earlier, in checker.Check"}, {Name: "neural_circuit", Support: Supported, Note: "NEURAL_CIRCUIT"}, @@ -162,9 +168,12 @@ var table = []Entry{ {Name: "optimize_block", Support: Supported, Note: "compiles children[3:]"}, {Name: "optimize_signature", Support: Supported, Note: "compiles the final child; the preceding metric/test/candidate forms are metadata"}, {Name: "or", Support: Supported, Note: "compileNode binop case"}, + {Name: "pad_left", Support: CompileTimeOnly, Note: "core-library sugar; (pad_left s n [ch]) via while/str by ast.Desugar (pure)"}, + {Name: "pad_right", Support: CompileTimeOnly, Note: "core-library sugar; (pad_right s n [ch]) via while/str by ast.Desugar (pure)"}, {Name: "parse_json", Support: Supported, Opaque: true, Note: "PARSE_JSON; reads child .Value directly"}, {Name: "patch", Support: CompileTimeOnly, Note: "applied and removed by ast.ApplyPatches before lowering"}, {Name: "print", Support: Supported, Note: "PRINT"}, + {Name: "range", Support: CompileTimeOnly, Note: "control-flow sugar; (range [a] b) int list via while/append by ast.Desugar (pure)"}, {Name: "read_file", Support: Supported, Note: "READ_FILE"}, {Name: "read_line", Support: Supported, Note: "READ_LINE"}, {Name: "read_text", Support: CompileTimeOnly, Note: "density sugar; expanded to bytes_to_string/read_file by ast.Desugar before lowering"}, @@ -179,6 +188,7 @@ var table = []Entry{ {Name: "res_json", Support: Supported, Note: "RES_JSON"}, {Name: "return", Support: Supported, Note: "RETURN"}, {Name: "reverse", Support: CompileTimeOnly, Note: "core-library sugar; (reverse lst) while/list_get/append by ast.Desugar (pure)"}, + {Name: "rtrim", Support: CompileTimeOnly, Note: "core-library sugar; (rtrim s) via str_split/while/list_slice/str_join by ast.Desugar (pure)"}, {Name: "schema", Support: Unsupported, Note: "declarative SQL migration; no opcode, Go backend only"}, {Name: "schema_bridge", Support: Supported, Note: "compiles the wrapped source expression"}, {Name: "semantic_match", Support: Unsupported, Note: "LLM intent dispatch; no opcode, Go backend only"}, @@ -187,6 +197,7 @@ var table = []Entry{ {Name: "sinc", Support: CompileTimeOnly, Note: "density sugar; store string-counter increment+print by ast.Desugar"}, {Name: "sleep", Support: Supported, Note: "SLEEP"}, {Name: "sort", Support: CompileTimeOnly, Note: "core-library sugar; (sort lst) stable insertion sort via for/when/' || ch == '!' { + if unicode.IsLetter(ch) || ch == '_' || ch == '%' || ch == '/' || ch == '-' || ch == '=' || ch == '.' || ch == '+' || ch == '*' || ch == '<' || ch == '>' || ch == '!' { val := string(ch) for unicode.IsLetter(l.peekChar()) || unicode.IsDigit(l.peekChar()) || l.peekChar() == '_' || l.peekChar() == '/' || l.peekChar() == '-' || l.peekChar() == '=' || l.peekChar() == '.' || l.peekChar() == '+' || l.peekChar() == '*' || l.peekChar() == '<' || l.peekChar() == '>' || l.peekChar() == '!' { val += string(l.nextChar()) diff --git a/internal/lexer/negative_literal_test.go b/internal/lexer/negative_literal_test.go new file mode 100644 index 00000000..ffdf673e --- /dev/null +++ b/internal/lexer/negative_literal_test.go @@ -0,0 +1,32 @@ +package lexer + +import "testing" + +// A '-' directly followed by a digit is a negative number literal; "-" alone, +// "-x" and symbols containing '-' are unchanged, and '%' lexes as a symbol. +func TestNegativeLiteralsAndPercent(t *testing.T) { + cases := []struct { + src string + want []Token + }{ + {"-1", []Token{{Type: TokenInt, Value: "-1"}}}, + {"-2.5", []Token{{Type: TokenFloat, Value: "-2.5"}}}, + {"- 1", []Token{{Type: TokenSymbol, Value: "-"}, {Type: TokenInt, Value: "1"}}}, + {"-x", []Token{{Type: TokenSymbol, Value: "-x"}}}, + {"a-1", []Token{{Type: TokenSymbol, Value: "a-1"}}}, + {"%", []Token{{Type: TokenSymbol, Value: "%"}}}, + {"//", []Token{{Type: TokenSymbol, Value: "//"}}}, + } + for _, c := range cases { + l := NewLexer(c.src) + for i, w := range c.want { + got := l.NextToken() + if got.Type != w.Type || got.Value != w.Value { + t.Errorf("%q token %d: got %s %q want %s %q", c.src, i, got.Type, got.Value, w.Type, w.Value) + } + } + if got := l.NextToken(); got.Type != TokenEOF { + t.Errorf("%q: trailing token %s %q", c.src, got.Type, got.Value) + } + } +} diff --git a/internal/vm/vm.go b/internal/vm/vm.go index 5ebc1855..97c6b66e 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -19,7 +19,6 @@ import ( "github.com/howlcipher/howlframe/internal/parser" "io" "math" - "math/bits" "net/http" "os" "os/exec" @@ -1281,13 +1280,24 @@ func checkedSubInt64(a, b int64, node *ast.Node) int64 { } func checkedMulInt64(a, b int64, node *ast.Node) int64 { - hi, lo := bits.Mul64(uint64(a), uint64(b)) - // sign-extend the low 63 bits; the high word must equal either all 0s or all 1s of that extension - expectedHi := uint64(lo>>63) * ^uint64(0) - if hi != expectedHi { + if mulOverflowsInt64(a, b) { InterpErr("integer overflow", node) } - return int64(lo) + return a * b +} + +// mulOverflowsInt64 reports whether a*b overflows int64. The previous check +// multiplied the operands as uint64, which flagged every product with exactly +// one negative operand ((* 3 -2)) as an overflow. +func mulOverflowsInt64(a, b int64) bool { + if a == 0 || b == 0 { + return false + } + if (a == -1 && b == math.MinInt64) || (b == -1 && a == math.MinInt64) { + return true + } + c := a * b + return c/b != a } // compareNumeric compares two numeric values. It returns the signed comparison @@ -3477,12 +3487,10 @@ func bcCheckedSubInt64(a, b int64) int64 { } func bcCheckedMulInt64(a, b int64) int64 { - hi, lo := bits.Mul64(uint64(a), uint64(b)) - expectedHi := uint64(lo>>63) * ^uint64(0) - if hi != expectedHi { + if mulOverflowsInt64(a, b) { panic(NewRuntimeError("RUNTIME_ERROR", "main", 0, 0, "integer overflow")) } - return int64(lo) + return a * b } func ToBCFloat(v any) float64 {