From d67884b5c69f0b9dd68670f1f13e831210bf8c5d Mon Sep 17 00:00:00 2001 From: howlcipher Date: Mon, 12 Oct 2026 11:48:39 -0400 Subject: [PATCH 1/2] Fix held-out correctness bugs: nested shorthand, short-circuit and/or, map_get miss = nil, list value semantics --- .../release_authority_test.go | 10 +- apps/task_api/task_api.howl | 8 +- .../2026-10-12_heldout_correctness_fixes.md | 125 ++++++++++++++ heldout_correctness_cli_test.go | 163 ++++++++++++++++++ internal/ast/desugar.go | 24 ++- internal/ast/desugar_context_drift_test.go | 41 +++++ internal/backend/gogen/absence_test.go | 6 +- .../backend/gogen/collection_type_test.go | 2 +- internal/backend/gogen/gogen.go | 69 ++++++-- .../backend/gogen/heldout_semantics_test.go | 31 ++++ internal/backend/gogen/map_get_path_test.go | 4 +- internal/backend/javascript/absence_test.go | 12 +- .../javascript/heldout_semantics_test.go | 37 ++++ internal/backend/javascript/javascript.go | 21 ++- .../backend/javascript/map_get_path_test.go | 4 +- internal/bytecode/bytecode.go | 35 +++- internal/hfir/bytecode.go | 11 ++ internal/vm/absence_test.go | 26 ++- internal/vm/map_get_path_test.go | 8 +- internal/vm/vm.go | 58 +++++-- tests/conformance/lowered_hfir_abi_v1.json | 2 +- 21 files changed, 613 insertions(+), 84 deletions(-) create mode 100644 docs/journals/2026-10-12_heldout_correctness_fixes.md create mode 100644 heldout_correctness_cli_test.go create mode 100644 internal/ast/desugar_context_drift_test.go create mode 100644 internal/backend/gogen/heldout_semantics_test.go create mode 100644 internal/backend/javascript/heldout_semantics_test.go diff --git a/apps/release_authority/release_authority_test.go b/apps/release_authority/release_authority_test.go index 0612f2b9..8a544ce6 100644 --- a/apps/release_authority/release_authority_test.go +++ b/apps/release_authority/release_authority_test.go @@ -85,12 +85,18 @@ func TestReleaseAuthority(t *testing.T) { t.Fatalf("extra output after decision: %v", err) } var proposal struct { - Target string `json:"target"` + Target *string `json:"target"` } if err := json.Unmarshal([]byte(proposalJSON), &proposal); err != nil { t.Fatal(err) } - if decision["target"] != proposal.Target { + // A missing target is nil from map_get, so the app's is_nil + // fallback reports "unknown" (2026-10-12 nil semantics). + wantTarget := "unknown" + if proposal.Target != nil { + wantTarget = *proposal.Target + } + if decision["target"] != wantTarget { t.Fatalf("target did not round-trip: %q", decision["target"]) } tokens := json.NewDecoder(strings.NewReader(outStr)) diff --git a/apps/task_api/task_api.howl b/apps/task_api/task_api.howl index 341de2e5..08d33d2c 100644 --- a/apps/task_api/task_api.howl +++ b/apps/task_api/task_api.howl @@ -26,7 +26,7 @@ (res_json 400 (dict ("error" "invalid_json"))) ) (let (title (map_get body "title")) - (if (= title "") + (if (or (is_nil title) (= title "")) (res_json 400 (dict ("error" "title_required"))) (let (id (call next_task_id)) (let (task (dict ("id" id) ("title" title) ("status" "open"))) @@ -84,7 +84,7 @@ (res_json 400 (dict ("error" "invalid_json"))) ) (let (id (map_get body "id")) - (if (= id "") + (if (or (is_nil id) (= id "")) (res_json 400 (dict ("error" "id_required"))) (do (store_open kv "memory://session") @@ -106,7 +106,7 @@ (res_json 400 (dict ("error" "invalid_json"))) ) (let (id (map_get body "id")) - (if (= id "") + (if (or (is_nil id) (= id "")) (res_json 400 (dict ("error" "id_required"))) (do (store_open kv "memory://session") @@ -135,7 +135,7 @@ (res_json 400 (dict ("error" "invalid_json"))) ) (let (id (map_get body "id")) - (if (= id "") + (if (or (is_nil id) (= id "")) (res_json 400 (dict ("error" "id_required"))) (do (store_open kv "memory://session") diff --git a/docs/journals/2026-10-12_heldout_correctness_fixes.md b/docs/journals/2026-10-12_heldout_correctness_fixes.md new file mode 100644 index 00000000..5cb4cd9e --- /dev/null +++ b/docs/journals/2026-10-12_heldout_correctness_fixes.md @@ -0,0 +1,125 @@ +# 2026-10-12: Held-out correctness fixes + +Base: main `c21a029c` (#136 merged). This slice fixes the four correctness +bugs the held-out benchmark (heldout-v0) surfaced. No new built-ins. No token +claims. No production `-compile-bc`/HFIR flip. + +## 1. Nested shorthand under the governed profile + +**Bug:** shorthand (`str_replace`, `read_text`, `str`, ...) nested inside the +argument of a value-taking built-in (`write_file`, `to_lower`, ...) was not +expanded. The governed build then failed with "no lowering for construct". + +**Fix:** `internal/ast/desugar.go` `isSugarExprContext` now lists every +value-taking built-in as a parent. These include write_file, read_file, +to_lower/to_upper, str_trim, the escapes, regex_match, is_nil, map_keys, env, +exec, store_*, the req_*/res_* readers, llm_generate and others. A new drift +test (`desugar_context_drift_test.go`) walks the construct registry. Every +supported, non-opaque construct must expand nested sugar unless it is listed as +structural, with a written reason. + +**Fail-closed is unchanged:** the order is still pre-desugar profile scan, +then desugar, then post-desugar `hfir.VerifyProfile`, then build. A forbidden +construct inside a nested shorthand argument is rejected before any artifact +exists, and nothing runs. `TestHeldoutNestedShorthandGovernedFailsClosed` +covers this: an `exec` hidden in `str_replace` inside `write_file` yields no +artifact and no side file. + +## 2. Short-circuit `and` / `or` + +**Bug:** both operands were always evaluated. + +**Fix:** +- **AST interpreter:** `evalBinop` decides before it evaluates the right side. +- **AST bytecode:** `bytecode.ShortCircuit` emits a JUMP_IF_FALSE guard around + the right operand. The HFIR bytecode lowering uses the same helper. +- **Go/JS backends:** they already used native `&&`/`||`. + +The right operand of `and` runs only when the left is true, and the right of +`or` only when the left is false. Effects (`write_file`) and errors +(`to_int "qq"`) on the skipped side do not happen. + +## 3. `map_get` on a missing key returns nil (behavior change) + +**Before:** a miss returned `""`, so `is_nil` and `nz` defaults never fired on a +miss. + +**Now:** a miss returns nil on every path: +- **VM paths:** interpreter, bytecode and HFIR bytecode. +- **JS:** returns `null`. +- **Go:** returns nil for dynamic maps. + +Text rendering is kept stable. `vm.TextOf` renders nil as `""` in `print`, +`stderr`, `to_string` and `str_join`. The Go/JS backends render nil/null as `""` +in `print` and `to_string`. So `(print (map_get m "zz"))` still prints an empty +field. `is_nil`, `nz` and `=` now tell a miss (`nil`) apart from a present `""`: +`(= (map_get m "zz") "")` is now **false**. + +Other consequences: +- A chained `map_get` through a missing intermediate is still `TYPE_ERROR`, + now "expected dict, got nil". +- A missing `store_get` record now prints as an empty field instead of ``. + `is_nil` is the way to observe absence. + +**Audit and migration:** +- `apps/task_api`: the title/id validation compared with `""`. It now uses + `(or (is_nil x) (= x ""))`, so it still rejects both a missing and an empty + value. Short-circuit makes this safe. +- `apps/release_authority`: it already had `is_nil` fallbacks to `"unknown"`, + which were dead code before. They now fire. The test expects `"unknown"` when + a proposal omits `target`. +- The vm/gogen/js absence and path tests were renamed `...MissIsNil` and their + expectations updated. +- The lowered-HFIR ABI conformance fixture `map_get_absence` now expects + `is_nil` of a miss to be `true`. +- A grep of all `.howl` for `(= (map_get …) "")` found nothing else. + +**Go typed-map caveat:** a Go `map[string]string` cannot hold nil, so a value +read of a miss is still `""`. `is_nil (map_get m k)` on a named map is compiled +as a key-presence check (`howlFrameMapGetIsNil`), so it agrees with the VM. +`nz (map_get m k) d` on a typed string map in the Go target may still see `""`. + +## 4. Lists are values (decision) + +**Decision:** explicit value semantics, applied the same way on every target. +- `append` rebinds only the named variable to a new list. +- A list stored in a map, or bound to another name, is unchanged. +- To update the map, `map_set` the list back. +- Dicts keep their in-place `map_set` mutation, which is unchanged. + +The VM paths already copied on append. The bug was that the backends did not: +- **JS:** used `push`, which aliased. `howlFrameAppend` now returns a copy and + the emitted code is `xs = howlFrameAppend(xs, item)`. +- **Go:** `append` could write into a shared backing array. It now uses a full + slice expression, `append(xs[:len(xs):len(xs)], item)`, in both the direct + form and the helper. + +## Regression tests + +- `heldout_correctness_cli_test.go`: each bug runs on the AST interpreter + (`-run`), governed AST bytecode (`build --profile governed` then `-run-bc`) + and HFIR bytecode (`-compile-hfir-bc` then `-run-bc`). It includes the + effect-skipping short-circuit test with `write_file` on the skipped side. +- `internal/backend/gogen/heldout_semantics_test.go` and + `internal/backend/javascript/heldout_semantics_test.go` cover list aliasing, + nil rendering, `is_nil` of a miss and short-circuit. Go dict literals are + string-valued, so the list-in-map case is covered on the VM and JS paths. +- `desugar_context_drift_test.go` keeps the shorthand parent list complete. + +## Benchmarks (verification only; no claims) + +- **token-efficiency-v0 `run.sh`:** howl 60/60, python 60/60, js 60/60, go 60/60. +- **token-efficiency-heldout-v0 `run.sh`:** exit 0. All 126 references verified + (howl 30, plus 32 each for python, js and go), with the same 2 recorded gaps + (h31, h32). The committed results files are unchanged. +- **h05/h07/h12 natural forms:** rewritten in a scratch run with nested + `str_replace (read_text …)`, `or` with `to_int` on the right, and + `nz (map_get …) 0`. All three pass the verifier. They were **not** committed, + so the frozen solution hashes and the #136 report stay as they are. + +## Known gaps + +- The JS backend's `nz` lowering emits `let` in expression position, which is + a syntax error. This bug predates this slice and is untouched here. +- Go dict literals cannot hold lists or ints (string-valued only). This also + predates this slice. diff --git a/heldout_correctness_cli_test.go b/heldout_correctness_cli_test.go new file mode 100644 index 00000000..bdeb9d28 --- /dev/null +++ b/heldout_correctness_cli_test.go @@ -0,0 +1,163 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Regression tests for the correctness bugs found by the held-out benchmark +// (docs/journals/2026-10-12_heldout_correctness_fixes.md). Each program runs +// on every applicable execution path: the AST interpreter, AST bytecode built +// under the governed profile (-run-bc), and the experimental HFIR bytecode. + +type heldoutPath struct { + name string + run func(t *testing.T, bin, dir, src string) (string, error) +} + +func heldoutPaths() []heldoutPath { + caps := func(dir string) string { + return "filesystem:read=" + dir + ",filesystem:write=" + dir + } + runBC := func(bin, dir, art string) (string, error) { + out, err := exec.Command(bin, "-run-bc", "-allow-caps", caps(dir), art).CombinedOutput() + return string(out), err + } + return []heldoutPath{ + {"interpreter", func(t *testing.T, bin, dir, src string) (string, error) { + out, err := exec.Command(bin, "-run", "-allow-caps", caps(dir), src).CombinedOutput() + return string(out), err + }}, + {"bytecode_governed", func(t *testing.T, bin, dir, src string) (string, error) { + art := filepath.Join(dir, "g.hfbc") + if out, err := exec.Command(bin, "build", "--profile", "governed", src, "-o", art).CombinedOutput(); err != nil { + t.Fatalf("governed build: %v %s", err, out) + } + return runBC(bin, dir, art) + }}, + {"hfir_bytecode", func(t *testing.T, bin, dir, src string) (string, error) { + art := filepath.Join(dir, "h.hfbc") + if out, err := exec.Command(bin, "-compile-hfir-bc", src, "-o", art).CombinedOutput(); err != nil { + t.Fatalf("hfir compile: %v %s", err, out) + } + return runBC(bin, dir, art) + }}, + } +} + +func runHeldoutProgram(t *testing.T, bin, program, want string, after func(t *testing.T, dir string)) { + t.Helper() + for _, p := range heldoutPaths() { + t.Run(p.name, func(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "p.howl") + body := strings.ReplaceAll(program, "$DIR", dir) + if err := os.WriteFile(src, []byte(body), 0600); err != nil { + t.Fatal(err) + } + out, err := p.run(t, bin, dir, src) + if err != nil { + t.Fatalf("run: %v %s", err, out) + } + if out != want { + t.Fatalf("got %q want %q", out, want) + } + if after != nil { + after(t, dir) + } + }) + } +} + +// Bug 1: shorthand nested inside value-taking built-in arguments expands under +// the governed profile. +func TestHeldoutNestedShorthandExpands(t *testing.T) { + bin := c1c2Binary(t) + program := `(do + (let (t "a foo b") + (do + (write_file "$DIR/out.txt" (str_replace t "foo" "bar")) + (print (to_lower (str "AB" "C")) (read_text "$DIR/out.txt")))))` + runHeldoutProgram(t, bin, program, "abc a bar b\n", nil) +} + +// Bug 1, fail-closed half: a forbidden construct hidden in a nested shorthand +// argument is rejected by the governed build; no artifact, no effect. +func TestHeldoutNestedShorthandGovernedFailsClosed(t *testing.T) { + bin := c1c2Binary(t) + dir := t.TempDir() + side := filepath.Join(dir, "side") + src := filepath.Join(dir, "x.howl") + art := filepath.Join(dir, "x.hfbc") + program := `(do (write_file "` + side + `" "ran") (write_file "` + side + `2" (str_replace (exec "touch" "` + side + `.exec") "a" "b")))` + if err := os.WriteFile(src, []byte(program), 0600); err != nil { + t.Fatal(err) + } + if out, err := exec.Command(bin, "build", "--profile", "governed", src, "-o", art).CombinedOutput(); err == nil { + t.Fatalf("governed build accepted nested exec: %s", out) + } + for _, p := range []string{art, side, side + "2", side + ".exec"} { + if _, err := os.Stat(p); err == nil { + t.Fatalf("%s exists after a rejected governed build", p) + } + } +} + +// Bug 2: and/or short-circuit. The right side is not evaluated when the left +// decides, so its effects (write_file) and its errors (to_int "qq") never run. +func TestHeldoutShortCircuitSkipsRightSide(t *testing.T) { + bin := c1c2Binary(t) + program := `(do + (print (and (= 1 2) (do (write_file "$DIR/and.txt" "ran") (= 1 1))) + (or (= 1 1) (do (write_file "$DIR/or.txt" "ran") (= 1 1))) + (and (= 1 2) (= (to_int "qq") 1)) + (or (= 1 1) (= (to_int "zz") 1))) + (print (and (= 1 1) (do (write_file "$DIR/and_rhs.txt" "ran") (= 2 2))) + (or (= 1 2) (= 2 3))))` + runHeldoutProgram(t, bin, program, "false true false true\ntrue false\n", func(t *testing.T, dir string) { + for _, name := range []string{"and.txt", "or.txt"} { + if _, err := os.Stat(filepath.Join(dir, name)); err == nil { + t.Fatalf("short-circuited right side ran: %s exists", name) + } + } + if _, err := os.Stat(filepath.Join(dir, "and_rhs.txt")); err != nil { + t.Fatalf("undecided right side did not run: %v", err) + } + }) +} + +// Bug 3: map_get of a missing key is nil, so is_nil and nz defaults work. nil +// renders as "" in text. +func TestHeldoutMapGetMissIsNil(t *testing.T) { + bin := c1c2Binary(t) + program := `(do + (let (m (dict ("a" "x") ("e" ""))) + (do + (print (is_nil (map_get m "b")) (is_nil (map_get m "e")) (= (map_get m "b") "")) + (print (nz (map_get m "b") "dflt") (nz (map_get m "a") "dflt")) + (print (str "[" (map_get m "b") "]") (map_get m "b")))))` + runHeldoutProgram(t, bin, program, "true false false\ndflt x\n[] \n", nil) +} + +// Bug 4: lists are values. append rebinds only the named variable; a list held +// in a map (or another variable) is unchanged until it is map_set back. +func TestHeldoutListsAreValues(t *testing.T) { + bin := c1c2Binary(t) + program := `(cli_app + (let (m (dict ("xs" (list "a")))) + (let (xs (map_get m "xs")) + (let (ys xs) + (do + (append xs "b") + (print "map" (list_len (map_get m "xs"))) + (print "alias" (list_len ys)) + (print "var" (list_len xs)) + (map_set m "xs" xs) + (print "after" (list_len (map_get m "xs"))) + (print (str_join (map_get m "xs") ",")) + )))))` + runHeldoutProgram(t, bin, program, "map 1\nalias 1\nvar 2\nafter 2\na,b\n", nil) +} diff --git a/internal/ast/desugar.go b/internal/ast/desugar.go index 6a7f12c6..e7cbbc42 100644 --- a/internal/ast/desugar.go +++ b/internal/ast/desugar.go @@ -248,7 +248,15 @@ func isSugarExprContext(parentHead string, childIndex int) bool { "switch", "inc", "dec", "str", "nz", "map_has", "when", "unless", "str_contains", "str_replace", "starts_with", "ends_with", "http_body", "http_status", "http_res", "list_contains", "not", - "blank", "unix_now", "ver_gt", "ver_ge", "all", "any", "parse_json", "list_len": + "blank", "unix_now", "ver_gt", "ver_ge", "all", "any", "parse_json", "list_len", + // Value-taking built-ins: nested sugar in their arguments must expand + // (heldout-v0 h05/h12). Kept complete by the registry drift test. + "write_file", "read_file", "mkdir", "to_lower", "to_upper", "str_trim", + "html_escape", "attr_escape", "regex_match", "is_nil", "map_keys", + "env", "sleep", "exec", "cli_args", "read_line", "time_now", + "store_open", "store_keys", "res_header", + "req_header", "req_method", "req_path", "req_query", + "llm_generate", "confidence": return true case "for": return childIndex == 2 || childIndex == 3 @@ -1435,3 +1443,17 @@ var exprBodyHeads = map[string]bool{ "req_header": true, "req_method": true, "req_path": true, "req_query": true, "http_request": true, "fetch": true, } + +// IsSugarExprParent reports whether sugar nested directly under a form with +// this head is expanded (for heads with structural slots such as for/let, it +// reports whether any child slot is an expression slot). Exported for the +// registry drift test so every construct taking expression arguments expands +// nested sugar. +func IsSugarExprParent(head string) bool { + for i := 1; i <= 3; i++ { + if isSugarExprContext(head, i) { + return true + } + } + return false +} diff --git a/internal/ast/desugar_context_drift_test.go b/internal/ast/desugar_context_drift_test.go new file mode 100644 index 00000000..27569b68 --- /dev/null +++ b/internal/ast/desugar_context_drift_test.go @@ -0,0 +1,41 @@ +package ast_test + +import ( + "sort" + "testing" + + "github.com/howlcipher/howlframe/internal/ast" + "github.com/howlcipher/howlframe/internal/construct" +) + +// structuralSugarParents are constructs whose children are not ordinary +// expression slots (names, parameter lists, type annotations, quoted +// instructions), handled by their own walker context or never containing sugar. +var structuralSugarParents = map[string]string{ + "defun": "name and parameter list are structural; body sugar is handled by desugarDefunExprBody", + "dict": "children are (key value) pairs, walked as bindings", + "neural_circuit": "first child is a structural argument list, second a quoted instruction", + "ephemeral_circuit": "first child is a structural argument list, second a quoted instruction", +} + +// Every supported construct that compiles its children as expressions must +// expand nested density sugar; otherwise a nested (read_text …) reaches +// lowering unexpanded and the build fails (heldout-v0 h05/h12). +func TestEveryExpressionConstructExpandsNestedSugar(t *testing.T) { + var missing []string + for _, e := range construct.Table() { + if e.Support != construct.Supported || e.Opaque { + continue + } + if _, ok := structuralSugarParents[e.Name]; ok { + continue + } + if !ast.IsSugarExprParent(e.Name) { + missing = append(missing, e.Name) + } + } + sort.Strings(missing) + if len(missing) > 0 { + t.Fatalf("constructs that do not expand nested sugar: %v", missing) + } +} diff --git a/internal/backend/gogen/absence_test.go b/internal/backend/gogen/absence_test.go index e7087c47..67c2929e 100644 --- a/internal/backend/gogen/absence_test.go +++ b/internal/backend/gogen/absence_test.go @@ -12,7 +12,7 @@ import ( "github.com/howlcipher/howlframe/internal/parser" ) -func TestGoMapGetMissIsEmptyString(t *testing.T) { +func TestGoMapGetMissIsNil(t *testing.T) { const source = `(cli_app (let (strings (dict ("hit" "yes") ("empty" ""))) (do @@ -26,7 +26,7 @@ func TestGoMapGetMissIsEmptyString(t *testing.T) { "s-empty \n" + "s-miss \n" + "s-empty-nil false\n" + - "s-miss-nil false\n" + "s-miss-nil true\n" root := parser.NewParser(lexer.NewLexer(source), "absence.howl").ParseExpression() checker.Check(root) @@ -67,7 +67,7 @@ func main() { "s-miss string \"\"\n" + "a-hit string \"yes\"\n" + "a-empty string \"\"\n" + - "a-miss string \"\"\n" + + "a-miss nil\n" + "a-n int \"7\"\n" + "a-nested map[string]interface {} \"map[k:v]\"\n" + "a-null nil\n" diff --git a/internal/backend/gogen/collection_type_test.go b/internal/backend/gogen/collection_type_test.go index 6584adbc..bcdefdce 100644 --- a/internal/backend/gogen/collection_type_test.go +++ b/internal/backend/gogen/collection_type_test.go @@ -36,7 +36,7 @@ func TestGoConcreteListAndDictStayDirect(t *testing.T) { (print (map_get d "k")) ))))` code := generateCheckedGo(t, source) - if !strings.Contains(code, `items = append(items, "b")`) { + if !strings.Contains(code, `items = append(items[:len(items):len(items)], "b")`) { t.Fatalf("concrete append changed shape:\n%s", code) } if !strings.Contains(code, `d["k2"] = "v2"`) || !strings.Contains(code, `delete(d, "k")`) { diff --git a/internal/backend/gogen/gogen.go b/internal/backend/gogen/gogen.go index b01e9f0c..b5111bb9 100644 --- a/internal/backend/gogen/gogen.go +++ b/internal/backend/gogen/gogen.go @@ -73,27 +73,35 @@ type goVarFrame struct { var gogenVarStack []goVarFrame // mapGetHelperSource is the Go form of the map_get absence contract. -// map[string]string already yields "" for a missing key. map[string]any -// yields nil, so the helper returns "" for that miss. A present value, -// including "" and a present nil, is returned unchanged. +// map[string]string yields "" for a missing key (it cannot hold nil). +// map[string]any yields nil. A present value is returned unchanged. func mapGetHelperSource() string { return `func howlFrameMapGet[V any](m map[string]V, key string) V { if v, ok := m[key]; ok { return v } + // A missing key is the zero value: nil for dynamic maps. Typed string + // maps (map[string]string) cannot hold nil and return "". var zero V - if any(zero) == nil { - if blank, ok := any("").(V); ok { - return blank - } - } return zero } +// howlFrameMapGetIsNil is is_nil(map_get m key): true for a missing key or a +// present nil, false otherwise, for typed and dynamic maps alike. +func howlFrameMapGetIsNil[V any](m map[string]V, key string) bool { + v, ok := m[key] + if !ok { + return true + } + var a any = v + return a == nil +} + // howlFrameMapGetValue reads a key from a dict expression. A named map_get // stays on howlFrameMapGet so a string map still returns a string. A nested -// map_get receives that result as any. A missing key is still "". A map_get -// of that sentinel, or of any other non-dict, fails closed. +// map_get receives that result as any. A missing key in a dynamic map is nil +// (a typed string map still returns ""). A map_get of nil, or of any other +// non-dict, fails closed. func howlFrameMapGetValue(m any, key string) any { switch d := m.(type) { case map[string]string: @@ -105,7 +113,7 @@ func howlFrameMapGetValue(m any, key string) any { if v, ok := d[key]; ok { return v } - return "" + return nil default: panic(fmt.Sprintf("TYPE_ERROR: map_get expected dict, got %T", m)) } @@ -115,7 +123,7 @@ func howlFrameMapGetValue(m any, key string) any { } // collectionHelperSource is the fail-closed form of the remaining dict and -// list operations. A missing map_get key is still "". A list_get index that +// list operations. A missing map_get key is nil for dynamic maps ("" for typed string maps). A list_get index that // is in range returns the element. An out-of-range index is "". A receiver // of the wrong type is TYPE_ERROR, as is a list_get index that is not a // whole number. @@ -127,9 +135,10 @@ func collectionHelperSource() string { if !ok { panic(fmt.Sprintf("TYPE_ERROR: append expected string item, got %T", item)) } - return append(l, s) + // Full slice expression forces a copy: lists are values. + return append(l[:len(l):len(l)], s) case []any: - return append(l, item) + return append(l[:len(l):len(l)], item) default: panic(fmt.Sprintf("TYPE_ERROR: append expected list, got %T", list)) } @@ -318,6 +327,12 @@ func goValueType(node *ast.Node) string { return "float64" case "is_nil": return "bool" + case "schema_bridge": + // The bridge emits its wrapped source expression. + if len(node.Children) == 3 { + return goValueType(node.Children[2]) + } + return "" default: return "" } @@ -1634,7 +1649,7 @@ func EmitGoIR(ir *ir.IRNode, reqVar string, depth int) string { return fmt.Sprintf("howlFrameToFloat(%s)", valStr) case "to_string": valStr := generateExpression(ir.Kids[0], reqVar, depth+1) - return fmt.Sprintf("fmt.Sprint(%s)", valStr) + return fmt.Sprintf("func() string { var _v any = %s; if _v == nil { return \"\" }; return fmt.Sprint(_v) }()", valStr) case "bytes_to_string": valStr := generateExpression(ir.Kids[0], reqVar, depth+1) return fmt.Sprintf("string(%s)", valStr) @@ -1677,7 +1692,10 @@ func EmitGoIR(ir *ir.IRNode, reqVar string, depth int) string { // when the runtime value is not a list. switch goVarKind(listNode.Value) { case "[]string", "": - return fmt.Sprintf(" %s = append(%s, %s)", listNode.Value, listNode.Value, itemStr) + // Lists are values: the full slice expression forces a fresh + // backing array, so a list stored in a map (or another variable) + // never sees this append. Matches the VM copy-on-append. + return fmt.Sprintf(" %s = append(%s[:len(%s):len(%s)], %s)", listNode.Value, listNode.Value, listNode.Value, listNode.Value, itemStr) case "any": gogenCollection = true return fmt.Sprintf(" %s = howlFrameAppend(%s, %s)", listNode.Value, listNode.Value, itemStr) @@ -1753,6 +1771,17 @@ func EmitGoIR(ir *ir.IRNode, reqVar string, depth int) string { // plane keys. Improvement #109. return fmt.Sprintf("func() []string { switch _m := any(%s).(type) { case map[string]string: _keys := make([]string, 0, len(_m)); for _k := range _m { _keys = append(_keys, _k) }; sort.Strings(_keys); return _keys; case map[string]any: _keys := make([]string, 0, len(_m)); for _k := range _m { _keys = append(_keys, _k) }; sort.Strings(_keys); return _keys; default: panic(fmt.Sprintf(\"TYPE_ERROR: map_keys expected dict, got %%T\", _m)) } }()", dictStr) case "is_nil": + // is_nil of a named map_get checks key presence, so a miss on a typed + // map[string]string (which cannot hold nil) still reports true. + if mg := ir.Kids[0]; mg != nil && mg.Type == "List" && len(mg.Children) == 3 && + mg.Children[0].Type == "SYMBOL" && mg.Children[0].Value == "map_get" && mg.Children[1].Type == "SYMBOL" { + switch goVarKind(mg.Children[1].Value) { + case "map[string]string", "map[string]any": + gogenMapGet = true + keyStr := generateExpression(mg.Children[2], reqVar, depth+1) + return fmt.Sprintf("howlFrameMapGetIsNil(%s, fmt.Sprint(%s))", mg.Children[1].Value, keyStr) + } + } valStr := generateExpression(ir.Kids[0], reqVar, depth+1) return fmt.Sprintf("func() bool { var _v any = %s; return _v == nil || fmt.Sprint(_v) == \"\" }()", valStr) case "list": @@ -1770,7 +1799,13 @@ func EmitGoIR(ir *ir.IRNode, reqVar string, depth int) string { case "print": var args []string for _, kid := range ir.Kids { - args = append(args, generateExpression(kid, reqVar, depth+1)) + // nil (a map_get miss) renders as "", matching vm.TextOf. Only a + // dynamic value can be nil; concrete values print directly. + argStr := generateExpression(kid, reqVar, depth+1) + if typ := goValueType(kid); typ == "" || typ == "any" { + argStr = fmt.Sprintf("func() any { var _v any = %s; if _v == nil { return \"\" }; return _v }()", argStr) + } + args = append(args, argStr) } return fmt.Sprintf(" fmt.Println(%s)", strings.Join(args, ", ")) } diff --git a/internal/backend/gogen/heldout_semantics_test.go b/internal/backend/gogen/heldout_semantics_test.go new file mode 100644 index 00000000..df18192c --- /dev/null +++ b/internal/backend/gogen/heldout_semantics_test.go @@ -0,0 +1,31 @@ +package gogen + +import "testing" + +// Held-out correctness fixes (2026-10-12) on the Go target: lists are values, +// and/or short-circuit, and a map_get miss is nil (rendered as ""). Go dict +// literals are string-valued, so lists-in-maps are covered on the VM paths +// and the JS target; here the value rule is checked on list aliasing. +func TestGoHeldoutSemantics(t *testing.T) { + const source = `(cli_app + (let (a (list "x")) + (do + (append a "y") + (append a "z") + (let (b a) + (do + (append a "p") + (append b "q") + (print (str_join a ",") (str_join b ",")))) + (let (m (dict ("n" "1"))) + (do + (print "miss" (map_get m "zz") (is_nil (map_get m "zz")) (to_string (map_get m "zz"))) + (print (and (= 1 2) (= (list_get a 0) "x")) (or (= 1 1) (= 1 2))))))))` + const want = "x,y,z,p x,y,z,q\n" + + "miss true \n" + + "false true\n" + code := generateCheckedGo(t, source) + if got := goRun(t, "main.go", code); got != want { + t.Fatalf("got %q want %q\n%s", got, want, code) + } +} diff --git a/internal/backend/gogen/map_get_path_test.go b/internal/backend/gogen/map_get_path_test.go index a3827c7a..da233981 100644 --- a/internal/backend/gogen/map_get_path_test.go +++ b/internal/backend/gogen/map_get_path_test.go @@ -48,12 +48,12 @@ func TestGoChainedMapGetMissingIntermediate(t *testing.T) { code := generateCheckedGo(t, source) stdout, crash := goRunCrash(t, code) if stdout != "sentinel \n" { - t.Fatalf("stdout = %q, want the #103 empty-string sentinel and no chained print", stdout) + t.Fatalf("stdout = %q, want the nil miss and no chained print", stdout) } if strings.Contains(stdout, "chained") { t.Fatalf("chained read of the sentinel printed %q", stdout) } - if !strings.Contains(crash, "TYPE_ERROR: map_get expected dict, got string") { + if !strings.Contains(crash, "TYPE_ERROR: map_get expected dict, got ") { t.Fatalf("crash = %q", crash) } } diff --git a/internal/backend/javascript/absence_test.go b/internal/backend/javascript/absence_test.go index 864581c3..c8c540ad 100644 --- a/internal/backend/javascript/absence_test.go +++ b/internal/backend/javascript/absence_test.go @@ -12,7 +12,7 @@ import ( "github.com/howlcipher/howlframe/internal/parser" ) -func TestJSMapGetMissIsEmptyString(t *testing.T) { +func TestJSMapGetMissIsNil(t *testing.T) { const source = `(web_app (let (strings (dict ("hit" "yes") ("empty" ""))) (let (raw "{\"hit\":\"yes\",\"empty\":\"\",\"n\":7,\"nested\":{\"k\":\"v\"}}") @@ -38,22 +38,22 @@ func TestJSMapGetMissIsEmptyString(t *testing.T) { "s-empty \n" + "s-miss \n" + "s-empty-nil false\n" + - "s-miss-nil false\n" + + "s-miss-nil true\n" + "a-hit yes\n" + "a-n 7\n" + "a-empty \n" + "a-miss \n" + "a-empty-nil false\n" + - "a-miss-nil false\n" + + "a-miss-nil true\n" + "nested v\n" + "nested-miss \n" + - "nested-miss-nil false\n" + "nested-miss-nil true\n" root := parser.NewParser(lexer.NewLexer(source), "absence.howl").ParseExpression() checker.Check(root) code, _ := GenerateJSCode(root) - if !strings.Contains(code, `howlFrameMapGet(strings, "missing")`) || !strings.Contains(code, `?? ""`) { - t.Fatalf("generated JavaScript does not coerce a missing map_get to \"\":\n%s", code) + if !strings.Contains(code, `howlFrameMapGet(strings, "missing")`) { + t.Fatalf("generated JavaScript does not route map_get through the helper:\n%s", code) } dir := t.TempDir() diff --git a/internal/backend/javascript/heldout_semantics_test.go b/internal/backend/javascript/heldout_semantics_test.go new file mode 100644 index 00000000..860729bd --- /dev/null +++ b/internal/backend/javascript/heldout_semantics_test.go @@ -0,0 +1,37 @@ +package javascript + +import "testing" + +// Held-out correctness fixes (2026-10-12) on the JS target: lists are values, +// and/or short-circuit, and a map_get miss is null (rendered as ""). +func TestJSHeldoutSemantics(t *testing.T) { + const source = `(web_app + (let (a (list "x")) + (do + (append a "y") + (append a "z") + (let (b a) + (do + (append a "p") + (append b "q") + (print (str_join a ",") (str_join b ",")))) + (let (m (dict ("xs" (list "a")) ("n" "1"))) + (let (xs (map_get m "xs")) + (do + (append xs "b") + (print "map" (list_len (map_get m "xs")) "var" (list_len xs)) + (map_set m "xs" xs) + (print "after" (list_len (map_get m "xs"))) + (print "miss" (map_get m "zz") (is_nil (map_get m "zz")) (to_string (map_get m "zz"))) + (print (and (= 1 2) (= (list_get xs 0) "a")) (or (= 1 1) (= 1 2)))))))))` + const want = "x,y,z,p x,y,z,q\n" + + "map 1 var 2\n" + + "after 2\n" + + "miss true \n" + + "false true\n" + code := generateCheckedJS(t, source) + stdout, stderr, err := runNode(t, code) + if err != nil || stdout != want { + t.Fatalf("err=%v stdout=%q want %q stderr=%q\n%s", err, stdout, want, stderr, code) + } +} diff --git a/internal/backend/javascript/javascript.go b/internal/backend/javascript/javascript.go index 3fe08cc2..e1faad98 100644 --- a/internal/backend/javascript/javascript.go +++ b/internal/backend/javascript/javascript.go @@ -89,7 +89,7 @@ var jsNumeric bool func collectionJSHelper() string { // Dicts are plain objects. Lists are arrays. A missing map_get key is - // still "". list_get of an in-range element returns that element. An + // null (rendered as "" by print/to_string). list_get of an in-range element returns that element. An // out-of-range index is "". A wrong receiver is TYPE_ERROR. A list_get // index that is not a whole number is TYPE_ERROR, matching Atoi. return `function howlFrameValueKind(v) { @@ -104,7 +104,7 @@ function howlFrameMapGet(dict, key) { if (!howlFrameIsDict(dict)) { throw new Error("TYPE_ERROR: map_get expected dict, got " + howlFrameValueKind(dict)); } - return dict[key] ?? ""; + return Object.prototype.hasOwnProperty.call(dict, key) ? dict[key] : null; } function howlFrameMapSet(dict, key, val) { if (!howlFrameIsDict(dict)) { @@ -122,8 +122,10 @@ function howlFrameAppend(list, item) { if (!Array.isArray(list)) { throw new Error("TYPE_ERROR: append expected list, got " + howlFrameValueKind(list)); } - list.push(item); - return list; + // Lists are values: append returns a copy, matching the VM. + var out = list.slice(); + out.push(item); + return out; } function howlFrameListIndex(idx) { if (typeof idx === "number" && Number.isInteger(idx)) return idx; @@ -797,7 +799,7 @@ func EmitJSIR(ir *ir.IRNode, reqVar string, depth int) string { return fmt.Sprintf("howlFrameToFloat(%s)", valStr) case "to_string", "bytes_to_string": valStr := generateJSStatementRaw(ir.Kids[0], reqVar, depth+1) - return fmt.Sprintf("String(%s)", valStr) + return fmt.Sprintf("((_v) => _v == null ? \"\" : String(_v))(%s)", valStr) case "to_lower": valStr := generateJSStatementRaw(ir.Kids[0], reqVar, depth+1) return fmt.Sprintf("String(%s).toLowerCase()", valStr) @@ -830,7 +832,7 @@ func EmitJSIR(ir *ir.IRNode, reqVar string, depth int) string { jsNeedsCollection = true listNode := ir.Kids[0] itemStr := generateJSStatementRaw(ir.Kids[1], reqVar, depth+1) - return fmt.Sprintf("howlFrameAppend(%s, %s)", listNode.Value, itemStr) + return fmt.Sprintf("%s = howlFrameAppend(%s, %s)", listNode.Value, listNode.Value, itemStr) case "map_set": jsNeedsCollection = true dictNode := ir.Kids[0] @@ -908,7 +910,12 @@ func EmitJSIR(ir *ir.IRNode, reqVar string, depth int) string { case "print": var args []string for _, kid := range ir.Kids { - args = append(args, generateJSStatementRaw(kid, reqVar, depth+1)) + // null (a map_get miss) renders as "", matching vm.TextOf. + argStr := generateJSStatementRaw(kid, reqVar, depth+1) + if kid.Type != "STRING" && kid.Type != "INT" && kid.Type != "FLOAT" { + argStr = fmt.Sprintf("((_v) => _v == null ? \"\" : _v)(%s)", argStr) + } + args = append(args, argStr) } return fmt.Sprintf("console.log(%s)", strings.Join(args, ", ")) } diff --git a/internal/backend/javascript/map_get_path_test.go b/internal/backend/javascript/map_get_path_test.go index e9c04dee..a28ec139 100644 --- a/internal/backend/javascript/map_get_path_test.go +++ b/internal/backend/javascript/map_get_path_test.go @@ -52,12 +52,12 @@ func TestJSChainedMapGetMissingIntermediate(t *testing.T) { t.Fatalf("missing intermediate succeeded, stdout=%q stderr=%q", stdout, stderr) } if stdout != "sentinel \n" { - t.Fatalf("stdout = %q, want the #103 empty-string sentinel and no chained print", stdout) + t.Fatalf("stdout = %q, want the nil miss and no chained print", stdout) } if strings.Contains(stdout, "chained") { t.Fatalf("chained read of the sentinel printed %q", stdout) } - if !strings.Contains(stderr, "TYPE_ERROR: map_get expected dict, got string") { + if !strings.Contains(stderr, "TYPE_ERROR: map_get expected dict, got ") { t.Fatalf("stderr = %q", stderr) } } diff --git a/internal/bytecode/bytecode.go b/internal/bytecode/bytecode.go index 4dcae48b..d208fd81 100644 --- a/internal/bytecode/bytecode.go +++ b/internal/bytecode/bytecode.go @@ -719,7 +719,9 @@ func (c *BCCompiler) compileNode(node *ast.Node) []BCInstruction { insts = append(insts, c.compileNode(arg)...) } insts = append(insts, BCInstruction{OpString: "CALL", Op: OpCall, StringOperand: funcName, IntOperand: int64(float64(len(node.Children) - 2))}) - case "+", "-", "*", "/", "<", ">", "<=", ">=", "==", "!=", "=", "and", "or": + case "and", "or": + insts = append(insts, ShortCircuit(head, c.compileNode(node.Children[1]), c.compileNode(node.Children[2]))...) + case "+", "-", "*", "/", "<", ">", "<=", ">=", "==", "!=", "=": insts = append(insts, c.compileNode(node.Children[1])...) insts = append(insts, c.compileNode(node.Children[2])...) op := head @@ -864,3 +866,34 @@ func (c *BCCompiler) compileNode(node *ast.Node) []BCInstruction { func extractParamNames(paramsNode *ast.Node) []string { return ast.ParamNames(paramsNode) } + +// ShortCircuit lowers (and l r) / (or l r) so r is skipped when l decides. +// The left value is tested by JUMP_IF_FALSE (which requires a boolean); a +// skipped branch pushes the decided constant. An evaluated right operand still +// goes through BINOP with the identity constant, so a non-boolean right value +// is the same TYPE_ERROR as before. +// +// and: l; JIF +len(r)+4; r; CONST true; BINOP and; JUMP +2; CONST false +// or: l; JIF +3; CONST true; JUMP +len(r)+3; r; CONST false; BINOP or +func ShortCircuit(op string, left, right []BCInstruction) []BCInstruction { + out := append([]BCInstruction{}, left...) + if op == "and" { + out = append(out, BCInstruction{OpString: "JUMP_IF_FALSE", Op: OpJumpIfFalse, IntOperand: int64(len(right) + 4)}) + out = append(out, right...) + out = append(out, + BCInstruction{OpString: "LOAD_CONST", Op: OpLoadConst, ValueOperand: true}, + BCInstruction{OpString: "BINOP", Op: OpBinop, StringOperand: "and"}, + BCInstruction{OpString: "JUMP", Op: OpJump, IntOperand: 2}, + BCInstruction{OpString: "LOAD_CONST", Op: OpLoadConst, ValueOperand: false}) + return out + } + out = append(out, + BCInstruction{OpString: "JUMP_IF_FALSE", Op: OpJumpIfFalse, IntOperand: 3}, + BCInstruction{OpString: "LOAD_CONST", Op: OpLoadConst, ValueOperand: true}, + BCInstruction{OpString: "JUMP", Op: OpJump, IntOperand: int64(len(right) + 3)}) + out = append(out, right...) + out = append(out, + BCInstruction{OpString: "LOAD_CONST", Op: OpLoadConst, ValueOperand: false}, + BCInstruction{OpString: "BINOP", Op: OpBinop, StringOperand: "or"}) + return out +} diff --git a/internal/hfir/bytecode.go b/internal/hfir/bytecode.go index e295ef02..05e7fb5c 100644 --- a/internal/hfir/bytecode.go +++ b/internal/hfir/bytecode.go @@ -203,6 +203,17 @@ func (c *bytecodeLowerer) compile(node *Node) (instructions []bytecode.BCInstruc diagnostic := c.diagnostic(node, "binary operation requires operator, left, and right") return nil, &diagnostic } + if node.Value == "and" || node.Value == "or" { + left, d := c.compile(children[0]) + if d != nil { + return nil, d + } + right, d := c.compile(children[1]) + if d != nil { + return nil, d + } + return bytecode.ShortCircuit(node.Value, left, right), nil + } insts, childDiagnostic := compileAll() if childDiagnostic != nil { return nil, childDiagnostic diff --git a/internal/vm/absence_test.go b/internal/vm/absence_test.go index 34def96c..9439ee26 100644 --- a/internal/vm/absence_test.go +++ b/internal/vm/absence_test.go @@ -37,16 +37,16 @@ const absenceMapGetWant = "s-hit yes\n" + "s-empty \n" + "s-miss \n" + "s-empty-nil false\n" + - "s-miss-nil false\n" + + "s-miss-nil true\n" + "a-hit yes\n" + "a-n 7\n" + "a-empty \n" + "a-miss \n" + "a-empty-nil false\n" + - "a-miss-nil false\n" + + "a-miss-nil true\n" + "nested v\n" + "nested-miss \n" + - "nested-miss-nil false\n" + "nested-miss-nil true\n" const absenceStoreSource = `(cli_app (let (strings (dict ("empty" ""))) @@ -62,9 +62,9 @@ const absenceStoreSource = `(cli_app const absenceStoreWant = "field-nil false\n" + "record-nil false\n" + "gone-nil true\n" + - "gone \n" + "gone \n" -func TestMapGetMissIsEmptyString(t *testing.T) { +func TestMapGetMissIsNil(t *testing.T) { t.Run("interpreter", func(t *testing.T) { node, _ := parseAndCompile(t, absenceMapGetSource) var out, errOut bytes.Buffer @@ -98,9 +98,7 @@ func TestStoreGetMissIsNil(t *testing.T) { if out.String() != absenceStoreWant { t.Fatalf("stdout = %q, want %q", out.String(), absenceStoreWant) } - if strings.Contains(out.String(), "gone \n") || strings.Contains(out.String(), "gone \"\"") { - t.Fatalf("missing store record printed as an empty string:\n%s", out.String()) - } + // nil renders as "" in print (2026-10-12); absence is observed with is_nil. } func TestAbsenceOpcodeValuesAndCapabilities(t *testing.T) { @@ -153,11 +151,11 @@ func TestAbsenceOpcodeValuesAndCapabilities(t *testing.T) { if got := runMapGet(t, vm, "strings", "empty"); got != "" { t.Fatalf("present empty string = %#v", got) } - if got := runMapGet(t, vm, "strings", "missing"); got != "" { - t.Fatalf("string-map miss = %#v, want \"\"", got) + if got := runMapGet(t, vm, "strings", "missing"); got != nil { + t.Fatalf("string-map miss = %#v, want nil", got) } - if got := runMapGet(t, vm, "records", "missing"); got != "" { - t.Fatalf("map[string]any miss = %#v, want \"\"", got) + if got := runMapGet(t, vm, "records", "missing"); got != nil { + t.Fatalf("map[string]any miss = %#v, want nil", got) } if got := runMapGet(t, vm, "records", "n"); got != int64(7) { t.Fatalf("non-string hit = %#v, want 7", got) @@ -172,8 +170,8 @@ func TestAbsenceOpcodeValuesAndCapabilities(t *testing.T) { if got := runIsNil(t, vm, "strings", "empty"); got { t.Fatal("is_nil(present \"\") = true") } - if got := runIsNil(t, vm, "strings", "missing"); got { - t.Fatal("is_nil(map_get miss) = true, want false") + if got := runIsNil(t, vm, "strings", "missing"); !got { + t.Fatal("is_nil(map_get miss) = false, want true") } if got := runIsNil(t, vm, "records", "null"); !got { t.Fatal("is_nil(present nil) = false") diff --git a/internal/vm/map_get_path_test.go b/internal/vm/map_get_path_test.go index ad9abf38..ee961971 100644 --- a/internal/vm/map_get_path_test.go +++ b/internal/vm/map_get_path_test.go @@ -85,11 +85,11 @@ func TestChainedMapGetMissingIntermediate(t *testing.T) { if evidence.RuntimeFailure == nil || evidence.RuntimeFailure.Code != "TYPE_ERROR" { t.Fatalf("failure = %#v, want TYPE_ERROR", evidence.RuntimeFailure) } - if !strings.Contains(evidence.RuntimeFailure.Message, "map_get expected dict, got string") { + if !strings.Contains(evidence.RuntimeFailure.Message, "map_get expected dict, got ") { t.Fatalf("message = %q", evidence.RuntimeFailure.Message) } if out.String() != "sentinel \n" { - t.Fatalf("stdout = %q, want the #103 empty-string sentinel and no chained print", out.String()) + t.Fatalf("stdout = %q, want the nil miss and no chained print", out.String()) } if strings.Contains(out.String(), "chained") { t.Fatalf("chained read of the sentinel printed %q", out.String()) @@ -145,11 +145,11 @@ func TestChainedMapGetDeclaresNoCapability(t *testing.T) { func assertMapGetPathTypeError(t *testing.T, exitCode int, stdout, stderr string) { t.Helper() - if exitCode == 0 || !strings.Contains(stderr, "TYPE_ERROR: map_get expected dict, got string") { + if exitCode == 0 || !strings.Contains(stderr, "TYPE_ERROR: map_get expected dict, got") || !strings.Contains(stderr, "nil") { t.Fatalf("exit = %d, stdout = %q, stderr = %q", exitCode, stdout, stderr) } if stdout != "sentinel \n" { - t.Fatalf("stdout = %q, want the #103 empty-string sentinel and no chained print", stdout) + t.Fatalf("stdout = %q, want the nil miss and no chained print", stdout) } if strings.Contains(stdout, "chained") { t.Fatalf("chained read of the sentinel printed %q", stdout) diff --git a/internal/vm/vm.go b/internal/vm/vm.go index 30a67cc7..cf470d3d 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -357,13 +357,13 @@ func (interp *Interpreter) evalList(node *ast.Node, env *InterpEnv) any { } var out []string for _, arg := range args { - out = append(out, fmt.Sprint(arg)) + out = append(out, TextOf(arg)) } fmt.Fprintln(interp.Out, strings.Join(out, " ")) return nil case "stderr": val := interp.eval(node.Children[1], env) - fmt.Fprint(interp.ErrOut, val) + fmt.Fprint(interp.ErrOut, TextOf(val)) return nil case "exit": val := interp.eval(node.Children[1], env) @@ -601,7 +601,7 @@ func (interp *Interpreter) evalList(node *ast.Node, env *InterpEnv) any { if len(node.Children) != 2 { InterpErr("to_string expects (to_string val)", node) } - return fmt.Sprint(interp.eval(node.Children[1], env)) + return TextOf(interp.eval(node.Children[1], env)) case "bytes_to_string": if len(node.Children) != 2 { InterpErr("bytes_to_string expects (bytes_to_string val)", node) @@ -709,7 +709,7 @@ func (interp *Interpreter) evalList(node *ast.Node, env *InterpEnv) any { } strs := make([]string, len(items)) for i, it := range items { - strs[i] = fmt.Sprint(it) + strs[i] = TextOf(it) } return strings.Join(strs, sep) case "regex_match": @@ -1099,7 +1099,7 @@ func (interp *Interpreter) evalMapGet(node *ast.Node, env *InterpEnv) any { } d, ok := current.(map[string]any) if !ok { - // A missing key is "". Reading through that sentinel, or any other + // A missing key is nil. Reading through nil, or any other // non-dict, is a type error rather than another absence value. InterpErr(fmt.Sprintf("TYPE_ERROR: map_get expected dict, got %T", current), dictNode) } @@ -1107,8 +1107,8 @@ func (interp *Interpreter) evalMapGet(node *ast.Node, env *InterpEnv) any { if val, ok := d[key]; ok { return val } - // Missing keys are the empty string. A missing store record is nil. - return "" + // A missing key is nil (is_nil true; nz supplies a default). + return nil } func (interp *Interpreter) evalListGet(node *ast.Node, env *InterpEnv) any { @@ -1142,13 +1142,23 @@ func (interp *Interpreter) evalBinop(op string, node *ast.Node, env *InterpEnv) if len(node.Children) != 3 { InterpErr(fmt.Sprintf("%s expects 2 arguments", op), node) } - a := interp.eval(node.Children[1], env) - b := interp.eval(node.Children[2], env) + // and/or short-circuit: the right operand (and any effect in it) is not + // evaluated when the left operand decides the result. switch op { case "and": - return ToBool(a, node.Children[1]) && ToBool(b, node.Children[2]) + if !ToBool(interp.eval(node.Children[1], env), node.Children[1]) { + return false + } + return ToBool(interp.eval(node.Children[2], env), node.Children[2]) case "or": - return ToBool(a, node.Children[1]) || ToBool(b, node.Children[2]) + if ToBool(interp.eval(node.Children[1], env), node.Children[1]) { + return true + } + return ToBool(interp.eval(node.Children[2], env), node.Children[2]) + } + a := interp.eval(node.Children[1], env) + b := interp.eval(node.Children[2], env) + switch op { case "==", "=": return ValuesEqual(a, b) case "!=": @@ -2963,12 +2973,12 @@ func (vm *BCVM) run(insts []bytecode.BCInstruction, env *BcEnv) any { vals = append([]any{vm.pop(inst.Op)}, vals...) } for _, v := range vals { - out = append(out, fmt.Sprint(v)) + out = append(out, TextOf(v)) } fmt.Fprintln(vm.Out, strings.Join(out, " ")) case bytecode.OpStderr: val := vm.pop(inst.Op) - fmt.Fprint(vm.ErrOut, val) + fmt.Fprint(vm.ErrOut, TextOf(val)) case bytecode.OpExit: val := vm.pop(inst.Op) var code int @@ -3034,7 +3044,7 @@ func (vm *BCVM) run(insts []bytecode.BCInstruction, env *BcEnv) any { } var strs []string for _, item := range list { - strs = append(strs, fmt.Sprint(item)) + strs = append(strs, TextOf(item)) } joined := strings.Join(strs, sep) vm.chargeAlloc(len(joined), ip, inst.Op) @@ -3160,8 +3170,8 @@ func (vm *BCVM) run(insts []bytecode.BCInstruction, env *BcEnv) any { } dict, ok := current.(map[string]any) if !ok { - // A missing key is "". map_get of that sentinel is a type - // error, not a second miss. + // A missing key is nil. map_get of nil is a type error, + // not a second miss. panic(NewRuntimeError("TYPE_ERROR", "main", ip, inst.Op, "map_get expected dict, got %T", current)) } if val, ok := dict[key]; ok { @@ -3173,8 +3183,8 @@ func (vm *BCVM) run(insts []bytecode.BCInstruction, env *BcEnv) any { vm.trace.state("map:"+varName, key) nodeID, _ := vm.prog.TrustedMainOriginAt(ip) vm.mapLedger.read(dict, ip, nodeID, key, false, nil) - // Missing keys are the empty string. A missing store record is nil. - vm.push("") + // A missing key is nil (is_nil true; nz supplies a default). + vm.push(nil) } case bytecode.OpListGet: varName := inst.StringOperand @@ -3521,7 +3531,7 @@ func BcConvert(target string, a any) any { panic(NewRuntimeError("CONVERSION_ERROR", "main", 0, bytecode.OpConvert, "cannot convert %T to float", a)) } case "to_string": - return fmt.Sprint(a) + return TextOf(a) case "bytes_to_string": items, ok := a.([]any) if !ok { @@ -3739,3 +3749,13 @@ func filesystemIOPath(path string, grants capability.Grants) string { normalized, _ := filepath.Abs(path) // authorization already checked this error return normalized } + +// TextOf renders a value as text for print, stderr, to_string and str_join. +// nil (a missing map key or store record) renders as the empty string, so +// absence stays distinguishable via is_nil/nz without changing text output. +func TextOf(v any) string { + if v == nil { + return "" + } + return fmt.Sprint(v) +} diff --git a/tests/conformance/lowered_hfir_abi_v1.json b/tests/conformance/lowered_hfir_abi_v1.json index 21e745b6..3a4796e4 100644 --- a/tests/conformance/lowered_hfir_abi_v1.json +++ b/tests/conformance/lowered_hfir_abi_v1.json @@ -26,7 +26,7 @@ "targets": ["hfir_bytecode", "bytecode", "interpreter", "go", "javascript"], "javascript_root": "web_app", "expect": "PASS", - "stdout": "yes\nfalse\nx||y" + "stdout": "yes\ntrue\nx||y" }, { "name": "map_keys_sort_bmp", From 2f5e0aa2350370666844a9b13b019ec2a36d6308 Mon Sep 17 00:00:00 2001 From: howlcipher Date: Mon, 12 Oct 2026 12:59:48 -0400 Subject: [PATCH 2/2] docs: map_get miss is nil (Motoko #137 blocker) --- docs/reference/lowered_hfir_abi_v1.md | 2 +- improvements.md | 10 +++++----- internal/vm/map_get_path_test.go | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/reference/lowered_hfir_abi_v1.md b/docs/reference/lowered_hfir_abi_v1.md index 32adef14..5bdb084c 100644 --- a/docs/reference/lowered_hfir_abi_v1.md +++ b/docs/reference/lowered_hfir_abi_v1.md @@ -53,7 +53,7 @@ These grant nothing. The suite runs them with an empty capability grant. * `let` and `print`. `print` writes one line. Several arguments are separated by a single space. * `dict`, `map_get`, `map_keys`, `list`, `list_len`, `str_join`, `is_nil`, `html_escape`, `attr_escape`, `regex_match`. -`(map_get dict key)` on a missing key is `""`. `is_nil` of that result is false. A present value is returned as stored. This is the #103 absence rule. `map_get` grants nothing. +`(map_get dict key)` on a missing key is nil. `is_nil` of that result is true. Nil renders as `""` in `print` and `to_string`, so a miss still prints as an empty field. A present value is returned as stored. Go caveat: on a typed `map[string]string` a miss still reads as `""` (that map cannot hold nil); `is_nil` of a named `map_get` is compiled as a key-presence check so it still agrees with the VM. This supersedes the #103 empty-string miss rule. `map_get` grants nothing. `(map_keys dict)` returns a list of strings in UTF-8 byte order (Go `sort.Strings`, Go string `<`). An empty dict is an empty list, so its length is `0`. A value that is not a dict fails at runtime with `TYPE_ERROR` and the text `map_keys expected dict`. `map_keys` grants nothing. The order, including keys outside the Basic Multilingual Plane, is the #109 rule. The suite replays `tests/fixtures/map_keys_sort_bmp.howl` and `tests/fixtures/map_keys_sort_nonbmp.howl`. Wasm does not emit `map_keys`; #73 must use this byte order if it ever does. diff --git a/improvements.md b/improvements.md index e67540f7..4b3b6e0c 100644 --- a/improvements.md +++ b/improvements.md @@ -43,7 +43,7 @@ Pending rows are ranked by a diminishing-returns score: | 105 | [html_escape, attribute escape, and handler encode contract tests](#105-html_escape-attribute-escape-and-handler-encode-contract-tests) | Done (2026-09-29) | 1.75 (7×1÷4) | Sonnet 5 | Gemini 3.1 Pro (High) | gpt-5.6-sol | `html_escape` and `attr_escape` encode `& < > " '` on the VM, Go, and JS. No new capability. Handler contract tests keep ids out of `