diff --git a/change_log.md b/change_log.md index ecad0f50..35ad5c75 100644 --- a/change_log.md +++ b/change_log.md @@ -9,6 +9,7 @@ * Dogfood round 2: `apps/notes_cli`, `apps/auth_api`, `apps/grep_cli` built on standalone bytecode; `store_open` accepts a runtime URI expression; AST density sugar `(switch …)` and `(inc x)`. Real-app token harness extended with Python/Go/Node ports and parity. Journal: `docs/journals/2026-10-10_dogfood_apps_round2.md`. ### Fixed +* #159, #155: experimental HFIR bytecode (`-compile-hfir-bc`) now matches the interpreter and governed AST bytecode byte for byte on every frozen benchmark solution (v0, heldout-v0/v1/v2/v3, postcf, postrec: 268 programs; 23 diverged before). Two lowering root causes: (1) `parse_json` evaluated its body symbol onto the operand stack in addition to `PARSE_JSON` reading it by name, so the stray raw string was later popped by `WRITE_FILE` as its path (a false `CAPABILITY_DENIED` inside the grant), by `MAP_GET` as its key (`TYPE_ERROR`), or by `PRINT` (#155 printed the raw text); a content edge that is not the bound symbol now fails closed. (2) `map_get` whose dict is an expression had no HFIR shape and failed to lower; it now emits the AST compiler's exact sequence (`MAP_GET` with an empty variable operand). No opcode, capability or grant change; production `-compile-bc` is unchanged and #90 stays Partial. Every benchmark verifier and `tools/difftest` now run HFIR as a third path, and `TestHFIRBenchmarkSolutionParity` fails CI on any divergence. * #151: float and mixed int/float `%` and `//` follow Python exactly. `%` is `math.fmod` floored to the divisor's sign (computed exactly by an existing-construct doubling/subtraction loop, so `(% MaxInt64 1e33)` is `9.223372036854776e+18`, `(% 1 0.1)` is `0.09999999999999995`, `(% 4.0 -2.0)` is `-0`), and `//` is CPython's float floor division returned as an int (`(// MaxInt64 1e33)` is `0`). Integral floats that print like ints (`189944.0`) paired with a big int no longer run the int core in float arithmetic. Inf/NaN operands fail with `VALUE_ERROR: % and // need finite operands`, and a `//` result outside int64 with `VALUE_ERROR: // result outside int64`. Mixed int/float comparisons no longer treat the float 2^63 as MinInt64 or round the int through float64. A non-numeric operand now reports `% requires numeric operands (int or float)` at the form instead of `< compares incompatible types`, and template VALUE_ERRORs (destructuring included) carry `at line L, column C`. Instruction cost (-run-bc): a 20k-iteration loop with one `%` and one `//` takes 4.30M instructions (3.17M after #149, 1.55M before #149), so about 460 iterations fit the default 100k limit; `to_base` of MinInt64 in base 2 takes about 16.2k (13.4k after #149); the exact float path loops about twice the binary-exponent gap (worst case `1e308 % 5e-324`, about 53k). Still pure sugar: no opcode or capability; Go/JS still reject with `UNSUPPORTED_ON_TARGET`. * #149: integer `%`, `//` and `to_base` use exact int64 arithmetic over the whole range with Python floor semantics (previously one float division, wrong above 2^53; `(// (2^53-1) -3)` was already off by one). `(% MaxInt64 10)` is 7, `(// 123456789012345678 1000)` is 123456789012345, `MinInt64 // -1` is the existing `integer overflow` (not `CONVERSION_ERROR`), and `to_base` handles MinInt64. Operands inside (-2^52, 2^52) and float operands keep the old expansion. `let` destructuring of a list of the wrong length fails with `VALUE_ERROR: destructuring expects N elements`. Still pure sugar: no opcode or capability; Go/JS still reject with `UNSUPPORTED_ON_TARGET`. * HOWL-E2E-UX-001 UX-003: browser JavaScript fetch accepts explicit host grants without reading Node process.env. Missing grants still deny before requests; Node environment authority and exit behavior are preserved. Browser bootstrap errors preserve the original failure. diff --git a/experiments/internal/pathparity/pathparity.go b/experiments/internal/pathparity/pathparity.go new file mode 100644 index 00000000..69cc1c36 --- /dev/null +++ b/experiments/internal/pathparity/pathparity.go @@ -0,0 +1,139 @@ +// Package pathparity replays one HowlFrame program on every execution path the +// benchmark verifiers trust - the AST interpreter (-run), governed AST +// bytecode (build --profile governed, then -run-bc) and the experimental HFIR +// bytecode (-compile-hfir-bc, then -run-bc) - each in a fresh sandbox with the +// same grant, and reports any byte-level divergence (issue #159). +package pathparity + +import ( + "context" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + "time" +) + +// Paths lists the compared execution paths in report order. +var Paths = []string{"interpreter", "bytecode", "hfir"} + +// Outcome is everything a path is allowed to make observable: its combined +// stdout/stderr, its exit status and the full resulting sandbox tree. +type Outcome struct { + Output string + Exit int + Files map[string]string +} + +// Sandbox prepares a fresh, private sandbox under run and returns its path +// together with the -allow-caps grant for it. +type Sandbox func(run string) (dir, caps string, err error) + +// Run executes src on path in a sandbox prepared by prepare under work. +func Run(howl, src, path, work string, prepare Sandbox) (Outcome, error) { + if err := os.MkdirAll(work, 0o700); err != nil { + return Outcome{}, err + } + dir, caps, err := prepare(work) + if err != nil { + return Outcome{}, err + } + src, _ = filepath.Abs(src) + art := filepath.Join(work, "prog.hfbc") + command := func(args ...string) (string, int) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, howl, args...) + cmd.Dir = dir + cmd.Env = []string{"PATH=/usr/bin:/bin", "HOME=" + work, "LANG=C", "TZ=UTC"} + out, err := cmd.CombinedOutput() + code := 0 + if ee, ok := err.(*exec.ExitError); ok { + code = ee.ExitCode() + } else if err != nil { + code = -1 + } + return strings.ReplaceAll(string(out), work, "$WORK"), code + } + var out string + var code int + switch path { + case "interpreter": + out, code = command("-run", "-allow-caps", caps, src) + case "bytecode": + if o, c := command("build", "--profile", "governed", src, "-o", art); c != 0 { + return Outcome{Output: "build failed: " + o, Exit: c}, nil + } + out, code = command("-run-bc", "-allow-caps", caps, art) + case "hfir": + if o, c := command("-compile-hfir-bc", src, "-o", art); c != 0 { + return Outcome{Output: "hfir compile failed: " + o, Exit: c}, nil + } + out, code = command("-run-bc", "-allow-caps", caps, art) + default: + return Outcome{}, fmt.Errorf("unknown path %q", path) + } + return Outcome{Output: out, Exit: code, Files: snapshot(dir)}, nil +} + +// Check runs src on every path and returns "" when all outcomes are +// identical, or a description of the first divergence from the interpreter. +func Check(howl, src, work string, prepare Sandbox) string { + var base Outcome + for i, p := range Paths { + got, err := Run(howl, src, p, filepath.Join(work, p), prepare) + if err != nil { + return p + ": " + err.Error() + } + if i == 0 { + base = got + continue + } + if !reflect.DeepEqual(got, base) { + return fmt.Sprintf("path parity: %s differs from interpreter: %s", p, describe(got, base)) + } + } + return "" +} + +func describe(got, want Outcome) string { + if got.Exit != want.Exit || got.Output != want.Output { + return fmt.Sprintf("exit %d output %q, interpreter exit %d output %q", got.Exit, got.Output, want.Exit, want.Output) + } + var names []string + for k := range got.Files { + names = append(names, k) + } + for k := range want.Files { + if _, ok := got.Files[k]; !ok { + names = append(names, k) + } + } + sort.Strings(names) + for _, k := range names { + g, gok := got.Files[k] + w, wok := want.Files[k] + if gok != wok || g != w { + return fmt.Sprintf("file %s: %q (present=%v), interpreter %q (present=%v)", k, g, gok, w, wok) + } + } + return "outcomes differ" +} + +func snapshot(dir string) map[string]string { + files := map[string]string{} + filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + rel, _ := filepath.Rel(dir, p) + b, _ := os.ReadFile(p) + files[filepath.ToSlash(rel)] = string(b) + return nil + }) + return files +} diff --git a/experiments/token-efficiency-heldout-v0/cmd/heldoutverify/main.go b/experiments/token-efficiency-heldout-v0/cmd/heldoutverify/main.go index ce5535cb..fbe8723c 100644 --- a/experiments/token-efficiency-heldout-v0/cmd/heldoutverify/main.go +++ b/experiments/token-efficiency-heldout-v0/cmd/heldoutverify/main.go @@ -17,6 +17,8 @@ import ( "sort" "strings" "time" + + "github.com/howlcipher/howlframe/experiments/internal/pathparity" ) type Task struct { @@ -75,6 +77,11 @@ func main() { r.Detail = readErr.Error() } else { r.Detail = verify(filepath.Join(work, lang+"-"+t.ID), src, lang, *howl, t) + if r.Detail == "" && lang == "howl" { + // #159: the interpreter, governed bytecode and HFIR bytecode + // must agree byte for byte (output, exit status, file tree). + r.Detail = pathparity.Check(*howl, src, filepath.Join(work, "parity-"+t.ID), heldoutSandbox(t)) + } r.Pass = r.Detail == "" } if r.Pass { @@ -211,3 +218,28 @@ func must(err error) { os.Exit(2) } } + +// heldoutSandbox materializes a task's inputs plus an empty out/ and grants +// exactly the same read and write roots verify uses. +func heldoutSandbox(t Task) pathparity.Sandbox { + return func(run string) (string, string, error) { + dir := filepath.Join(run, "sandbox") + if err := os.MkdirAll(filepath.Join(dir, "out"), 0o755); err != nil { + return "", "", err + } + for p, content := range t.Inputs { + full := filepath.Join(dir, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return "", "", err + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + return "", "", err + } + } + abs, err := filepath.Abs(dir) + if err != nil { + return "", "", err + } + return abs, "filesystem:read=" + filepath.Join(abs, "in") + ",filesystem:write=" + filepath.Join(abs, "out"), nil + } +} diff --git a/experiments/token-efficiency-heldout-v1/cmd/heldoutv1verify/main.go b/experiments/token-efficiency-heldout-v1/cmd/heldoutv1verify/main.go index 328014ca..89d0e126 100644 --- a/experiments/token-efficiency-heldout-v1/cmd/heldoutv1verify/main.go +++ b/experiments/token-efficiency-heldout-v1/cmd/heldoutv1verify/main.go @@ -17,6 +17,8 @@ import ( "sort" "strings" "time" + + "github.com/howlcipher/howlframe/experiments/internal/pathparity" ) type Task struct { @@ -75,6 +77,11 @@ func main() { r.Detail = readErr.Error() } else { r.Detail = verify(filepath.Join(work, lang+"-"+t.ID), src, lang, *howl, t) + if r.Detail == "" && lang == "howl" { + // #159: the interpreter, governed bytecode and HFIR bytecode + // must agree byte for byte (output, exit status, file tree). + r.Detail = pathparity.Check(*howl, src, filepath.Join(work, "parity-"+t.ID), heldoutSandbox(t)) + } r.Pass = r.Detail == "" } if r.Pass { @@ -211,3 +218,28 @@ func must(err error) { os.Exit(2) } } + +// heldoutSandbox materializes a task's inputs plus an empty out/ and grants +// exactly the same read and write roots verify uses. +func heldoutSandbox(t Task) pathparity.Sandbox { + return func(run string) (string, string, error) { + dir := filepath.Join(run, "sandbox") + if err := os.MkdirAll(filepath.Join(dir, "out"), 0o755); err != nil { + return "", "", err + } + for p, content := range t.Inputs { + full := filepath.Join(dir, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return "", "", err + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + return "", "", err + } + } + abs, err := filepath.Abs(dir) + if err != nil { + return "", "", err + } + return abs, "filesystem:read=" + filepath.Join(abs, "in") + ",filesystem:write=" + filepath.Join(abs, "out"), nil + } +} diff --git a/experiments/token-efficiency-heldout-v2/cmd/heldoutv2verify/main.go b/experiments/token-efficiency-heldout-v2/cmd/heldoutv2verify/main.go index 4058f0c7..92f3d7b3 100644 --- a/experiments/token-efficiency-heldout-v2/cmd/heldoutv2verify/main.go +++ b/experiments/token-efficiency-heldout-v2/cmd/heldoutv2verify/main.go @@ -17,6 +17,8 @@ import ( "sort" "strings" "time" + + "github.com/howlcipher/howlframe/experiments/internal/pathparity" ) type Task struct { @@ -80,6 +82,11 @@ func main() { r.Detail = readErr.Error() } else { r.Detail = verify(filepath.Join(work, lang+"-"+t.ID), src, lang, *howl, t) + if r.Detail == "" && lang == "howl" { + // #159: the interpreter, governed bytecode and HFIR bytecode + // must agree byte for byte (output, exit status, file tree). + r.Detail = pathparity.Check(*howl, src, filepath.Join(work, "parity-"+t.ID), heldoutSandbox(t)) + } r.Pass = r.Detail == "" } if r.Pass { @@ -220,3 +227,28 @@ func must(err error) { os.Exit(2) } } + +// heldoutSandbox materializes a task's inputs plus an empty out/ and grants +// exactly the same read and write roots verify uses. +func heldoutSandbox(t Task) pathparity.Sandbox { + return func(run string) (string, string, error) { + dir := filepath.Join(run, "sandbox") + if err := os.MkdirAll(filepath.Join(dir, "out"), 0o755); err != nil { + return "", "", err + } + for p, content := range t.Inputs { + full := filepath.Join(dir, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return "", "", err + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + return "", "", err + } + } + abs, err := filepath.Abs(dir) + if err != nil { + return "", "", err + } + return abs, "filesystem:read=" + filepath.Join(abs, "in") + ",filesystem:write=" + filepath.Join(abs, "out"), nil + } +} diff --git a/experiments/token-efficiency-heldout-v3/cmd/heldoutv3verify/main.go b/experiments/token-efficiency-heldout-v3/cmd/heldoutv3verify/main.go index a2ad05c4..b9ef8c8b 100644 --- a/experiments/token-efficiency-heldout-v3/cmd/heldoutv3verify/main.go +++ b/experiments/token-efficiency-heldout-v3/cmd/heldoutv3verify/main.go @@ -17,6 +17,8 @@ import ( "sort" "strings" "time" + + "github.com/howlcipher/howlframe/experiments/internal/pathparity" ) type Task struct { @@ -81,6 +83,11 @@ func main() { r.Detail = readErr.Error() } else { r.Detail = verify(filepath.Join(work, lang+"-"+t.ID), src, lang, *howl, t) + if r.Detail == "" && lang == "howl" { + // #159: the interpreter, governed bytecode and HFIR bytecode + // must agree byte for byte (output, exit status, file tree). + r.Detail = pathparity.Check(*howl, src, filepath.Join(work, "parity-"+t.ID), heldoutSandbox(t)) + } r.Pass = r.Detail == "" } if r.Pass { @@ -227,3 +234,28 @@ func must(err error) { os.Exit(2) } } + +// heldoutSandbox materializes a task's inputs plus an empty out/ and grants +// exactly the same read and write roots verify uses. +func heldoutSandbox(t Task) pathparity.Sandbox { + return func(run string) (string, string, error) { + dir := filepath.Join(run, "sandbox") + if err := os.MkdirAll(filepath.Join(dir, "out"), 0o755); err != nil { + return "", "", err + } + for p, content := range t.Inputs { + full := filepath.Join(dir, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return "", "", err + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + return "", "", err + } + } + abs, err := filepath.Abs(dir) + if err != nil { + return "", "", err + } + return abs, "filesystem:read=" + filepath.Join(abs, "in") + ",filesystem:write=" + filepath.Join(abs, "out"), nil + } +} diff --git a/experiments/token-efficiency-v0/cmd/tokverify/main.go b/experiments/token-efficiency-v0/cmd/tokverify/main.go index bade59ec..4e581c56 100644 --- a/experiments/token-efficiency-v0/cmd/tokverify/main.go +++ b/experiments/token-efficiency-v0/cmd/tokverify/main.go @@ -10,9 +10,11 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/howlcipher/howlframe/experiments/falsifiable-v0/harness" + "github.com/howlcipher/howlframe/experiments/internal/pathparity" ) type Result struct { @@ -137,6 +139,13 @@ func main() { err = readErr } else { result.Comparison, result.Diagnostics, err = verify(root, filepath.Join(work, lang+"-"+task.ID), source, lang, *hfexpFlag, *howlFlag, task) + if err == nil && lang == "howl" && result.Comparison.Success { + // #159: interpreter, governed bytecode and HFIR bytecode + // must agree byte for byte in identical sandboxes. + if detail := pathparity.Check(*howlFlag, source, filepath.Join(work, "parity-"+task.ID), v0Sandbox(root, task)); detail != "" { + err = fmt.Errorf("%s", detail) + } + } } if err != nil { result.Diagnostics += " " + err.Error() @@ -163,3 +172,16 @@ func main() { os.Exit(1) } } + +// v0Sandbox materializes the same reference sandbox the other languages use +// and grants the task's declared capabilities. +func v0Sandbox(root string, task harness.Task) pathparity.Sandbox { + return func(run string) (string, string, error) { + dir := filepath.Join(run, "ref") + if err := os.Mkdir(dir, 0o700); err != nil { + return "", "", err + } + sandbox, _, err := harness.MaterializeReferenceSandbox(filepath.Join(root, "experiments/falsifiable-v0"), dir, task) + return sandbox, strings.Join(task.AllowedCaps, ","), err + } +} diff --git a/hfir_parity_cli_test.go b/hfir_parity_cli_test.go new file mode 100644 index 00000000..a87603a0 --- /dev/null +++ b/hfir_parity_cli_test.go @@ -0,0 +1,251 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/howlcipher/howlframe/experiments/falsifiable-v0/harness" +) + +// Issue #159: every frozen HowlFrame benchmark solution must behave byte for +// byte the same on the AST interpreter (-run), governed AST bytecode +// (build --profile governed + -run-bc) and the experimental HFIR bytecode +// (-compile-hfir-bc + -run-bc). Each path runs in a fresh sandbox with the +// same grant; the observed outcome is stdout+stderr, the exit status and the +// full resulting file tree. Any divergence fails CI. + +type parityCase struct { + set, id, src string + caps string + setup func(dir string) error +} + +type parityOutcome struct { + Output string + Exit int + Files map[string]string +} + +func parityHeldoutCases(t *testing.T, set, root, solutions string) []parityCase { + var manifest struct { + Tasks []struct { + ID string `json:"id"` + Inputs map[string]string `json:"inputs"` + } `json:"tasks"` + } + data, err := os.ReadFile(filepath.Join(root, "tasks/manifest.json")) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &manifest); err != nil { + t.Fatal(err) + } + var cases []parityCase + for _, task := range manifest.Tasks { + src := filepath.Join(solutions, task.ID+".howl") + if _, err := os.Stat(src); err != nil { + continue // recorded gap + } + inputs := task.Inputs + cases = append(cases, parityCase{set: set, id: task.ID, src: src, caps: "filesystem:read=$DIR/in,filesystem:write=$DIR/out", setup: func(dir string) error { + if err := os.MkdirAll(filepath.Join(dir, "out"), 0o755); err != nil { + return err + } + for p, content := range inputs { + full := filepath.Join(dir, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return err + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + return err + } + } + return nil + }}) + } + return cases +} + +func parityV0Cases(t *testing.T) []parityCase { + corpus, _ := filepath.Abs("experiments/falsifiable-v0") + manifest, err := harness.LoadManifest(corpus) + if err != nil { + t.Fatal(err) + } + var cases []parityCase + for _, task := range manifest.All() { + src := filepath.Join("experiments/token-efficiency-v0/solutions/howl", task.ID+".howl") + if _, err := os.Stat(src); err != nil { + continue + } + // The sandbox is materialized per path in runParityPath. + cases = append(cases, parityCase{set: "v0", id: task.ID, src: src, caps: strings.Join(task.AllowedCaps, ",")}) + } + return cases +} + +func allParityCases(t *testing.T) []parityCase { + var cases []parityCase + for _, s := range []struct{ set, root, sol string }{ + {"heldout-v0", "experiments/token-efficiency-heldout-v0", "experiments/token-efficiency-heldout-v0/solutions/howl"}, + {"heldout-v1", "experiments/token-efficiency-heldout-v1", "experiments/token-efficiency-heldout-v1/solutions/howl"}, + {"heldout-v2", "experiments/token-efficiency-heldout-v2", "experiments/token-efficiency-heldout-v2/solutions/howl"}, + {"heldout-v2-postcf", "experiments/token-efficiency-heldout-v2", "experiments/token-efficiency-heldout-v2/postcf/solutions/howl"}, + {"heldout-v3", "experiments/token-efficiency-heldout-v3", "experiments/token-efficiency-heldout-v3/solutions/howl"}, + {"heldout-v3-postrec", "experiments/token-efficiency-heldout-v3", "experiments/token-efficiency-heldout-v3/postrec/solutions/howl"}, + } { + cases = append(cases, parityHeldoutCases(t, s.set, s.root, s.sol)...) + } + return append(cases, parityV0Cases(t)...) +} + +func snapshotParityTree(dir string) map[string]string { + files := map[string]string{} + filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + rel, _ := filepath.Rel(dir, p) + rel = filepath.ToSlash(rel) + b, _ := os.ReadFile(p) + files[rel] = string(b) + return nil + }) + return files +} + +func runParityPath(t *testing.T, bin string, c parityCase, path string) parityOutcome { + t.Helper() + work := t.TempDir() + dir := filepath.Join(work, "sandbox") + if c.set == "v0" { + corpus, _ := filepath.Abs("experiments/falsifiable-v0") + manifest, err := harness.LoadManifest(corpus) + if err != nil { + t.Fatal(err) + } + var task harness.Task + for _, tk := range manifest.All() { + if tk.ID == c.id { + task = tk + } + } + run := filepath.Join(work, "run") + if err := os.Mkdir(run, 0o700); err != nil { + t.Fatal(err) + } + if dir, _, err = harness.MaterializeReferenceSandbox(corpus, run, task); err != nil { + t.Fatal(err) + } + } else { + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := c.setup(dir); err != nil { + t.Fatal(err) + } + } + caps := strings.ReplaceAll(c.caps, "$DIR", dir) + src, _ := filepath.Abs(c.src) + art := filepath.Join(work, "prog.hfbc") + command := func(args ...string) (string, int) { + cmd := exec.Command(bin, args...) + cmd.Dir = dir + cmd.Env = []string{"PATH=/usr/bin:/bin", "HOME=" + work, "LANG=C", "TZ=UTC"} + out, err := cmd.CombinedOutput() + code := 0 + if ee, ok := err.(*exec.ExitError); ok { + code = ee.ExitCode() + } else if err != nil { + code = -1 + } + return strings.ReplaceAll(string(out), work, "$WORK"), code + } + var out string + var code int + switch path { + case "interpreter": + out, code = command("-run", "-allow-caps", caps, src) + case "bytecode": + if o, e := command("build", "--profile", "governed", src, "-o", art); e != 0 { + return parityOutcome{Output: "build failed: " + o, Exit: e} + } + out, code = command("-run-bc", "-allow-caps", caps, art) + case "hfir": + if o, e := command("-compile-hfir-bc", src, "-o", art); e != 0 { + return parityOutcome{Output: "hfir compile failed: " + o, Exit: e} + } + out, code = command("-run-bc", "-allow-caps", caps, art) + } + return parityOutcome{Output: out, Exit: code, Files: snapshotParityTree(dir)} +} + +func TestHFIRBenchmarkSolutionParity(t *testing.T) { + if testing.Short() { + t.Skip("builds and runs every benchmark solution on three paths") + } + bin := c1c2Binary(t) + cases := allParityCases(t) + if len(cases) < 260 { + t.Fatalf("parity corpus has %d programs, want every benchmark solution", len(cases)) + } + for _, c := range cases { + c := c + t.Run(c.set+"/"+c.id, func(t *testing.T) { + t.Parallel() + interp := runParityPath(t, bin, c, "interpreter") + for _, p := range []string{"bytecode", "hfir"} { + got := runParityPath(t, bin, c, p) + if fmt.Sprint(got) != fmt.Sprint(interp) { + t.Errorf("%s differs from interpreter:\n%s: %+v\ninterpreter: %+v", p, p, got, interp) + } + } + }) + } +} + +// #159 root cause 1 (the x03 false denial): the stray parse_json body left on +// the stack was popped by WRITE_FILE as its path. The fix must not widen the +// grant: inside the write root every path succeeds, and with no write root or +// a different one every path is still denied and writes nothing. +func TestHFIRParseJSONWriteFileStaysFailClosed(t *testing.T) { + bin := c1c2Binary(t) + program := `(do (write_file "out/r.json" (encode_json (map_get (parse_json Any (read_text "in/d.json")) "a"))))` + src := filepath.Join(t.TempDir(), "p.howl") + if err := os.WriteFile(src, []byte(program), 0o644); err != nil { + t.Fatal(err) + } + setup := func(dir string) error { + os.MkdirAll(filepath.Join(dir, "in"), 0o755) + os.MkdirAll(filepath.Join(dir, "out"), 0o755) + os.MkdirAll(filepath.Join(dir, "other"), 0o755) + return os.WriteFile(filepath.Join(dir, "in/d.json"), []byte(`{"a": {"y": [1, 2]}}`), 0o644) + } + grants := map[string]string{ + "granted": "filesystem:read=$DIR/in,filesystem:write=$DIR/out", + "read_only": "filesystem:read=$DIR/in", + "other_write": "filesystem:read=$DIR/in,filesystem:write=$DIR/other", + } + for name, caps := range grants { + c := parityCase{set: "focused", id: name, src: src, caps: caps, setup: setup} + for _, p := range []string{"interpreter", "bytecode", "hfir"} { + got := runParityPath(t, bin, c, p) + written, wrote := got.Files["out/r.json"] + if name == "granted" { + if got.Exit != 0 || written != `{"y":[1,2]}` { + t.Errorf("%s/%s: exit %d output %q file %q", name, p, got.Exit, got.Output, written) + } + continue + } + if got.Exit == 0 || wrote || !strings.Contains(got.Output, "denied") { + t.Errorf("%s/%s must fail closed: exit %d output %q wrote=%v", name, p, got.Exit, got.Output, wrote) + } + } + } +} diff --git a/internal/hfir/bytecode.go b/internal/hfir/bytecode.go index 95840ac1..9757f985 100644 --- a/internal/hfir/bytecode.go +++ b/internal/hfir/bytecode.go @@ -381,11 +381,19 @@ func (c *bytecodeLowerer) compile(node *Node) (instructions []bytecode.BCInstruc diagnostic := c.diagnostic(node, "parse_json requires content") return nil, &diagnostic } - insts, childDiagnostic := compileChild(0) - if childDiagnostic != nil { - return nil, childDiagnostic - } - return append(insts, instruction(bytecode.OpParseJson, "PARSE_JSON", func(inst *bytecode.BCInstruction) { inst.StringOperand = node.Value })), nil + // PARSE_JSON reads its body from the variable named by its operand + // and pushes nothing else, exactly as bytecode.CompileToBytecode + // emits it. Desugar always binds the body to a symbol first, so the + // content edge is that symbol and must not also be evaluated: doing + // so left the raw string on the operand stack, where a later + // WRITE_FILE popped it as its path (a false CAPABILITY_DENIED inside + // the grant) and PRINT/MAP_GET consumed it instead of the parsed + // value (#159, #155). Anything else fails closed. + if children[0].Kind != "symbol" || children[0].Value == "" || children[0].Value != node.Value { + diagnostic := c.diagnostic(node, "parse_json content must be a bound symbol") + return nil, &diagnostic + } + return []bytecode.BCInstruction{instruction(bytecode.OpParseJson, "PARSE_JSON", func(inst *bytecode.BCInstruction) { inst.StringOperand = node.Value })}, nil case "is_nil": if len(children) != 1 || node.DataInputs[0].Name != "value" { diagnostic := c.diagnostic(node, "is_nil requires value") @@ -499,6 +507,16 @@ func (c *bytecodeLowerer) compile(node *Node) (instructions []bytecode.BCInstruc } return append(insts, instruction(op, name, nil)), nil case "map_get", "list_get", "map_delete", "append": + if node.Kind == "map_get" && len(node.DataInputs) == 2 && node.DataInputs[0].Name == "target" && node.DataInputs[1].Name == "key" { + // Expression target: evaluate the dict, then the key, then + // MAP_GET with an empty variable operand pops both - the same + // sequence bytecode.CompileToBytecode emits (#159). + insts, childDiagnostic := compileAll() + if childDiagnostic != nil { + return nil, childDiagnostic + } + return append(insts, instruction(bytecode.OpMapGet, "MAP_GET", nil)), nil + } edgeName := "key" if node.Kind == "list_get" { edgeName = "index" diff --git a/internal/hfir/issue159_test.go b/internal/hfir/issue159_test.go new file mode 100644 index 00000000..d2c278ee --- /dev/null +++ b/internal/hfir/issue159_test.go @@ -0,0 +1,75 @@ +package hfir + +import ( + "testing" + + "github.com/howlcipher/howlframe/internal/ast" + "github.com/howlcipher/howlframe/internal/bytecode" + "github.com/howlcipher/howlframe/internal/checker" + "github.com/howlcipher/howlframe/internal/lexer" + "github.com/howlcipher/howlframe/internal/parser" +) + +func lowerBothIssue159(t *testing.T, source string) (hfirMain, astMain []bytecode.BCInstruction, graph *Graph) { + t.Helper() + root := parser.NewParser(lexer.NewLexer(source), "i159.howl").ParseExpression() + ast.Desugar(root) + checker.Check(root) + graph, err := LowerAST(root, "i159.howl") + if err != nil { + t.Fatalf("LowerAST: %v", err) + } + program, diagnostics := LowerToBytecode(graph) + if len(diagnostics) != 0 { + t.Fatalf("LowerToBytecode diagnostics = %#v", diagnostics) + } + return program.Main, bytecode.CompileToBytecode(root).Main, graph +} + +func sameInstructionStream(t *testing.T, got, want []bytecode.BCInstruction) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("HFIR emitted %d instructions, AST compiler %d:\nhfir %v\nast %v", len(got), len(want), got, want) + } + for i := range got { + named := got[i].Op == bytecode.OpLoadVar || got[i].Op == bytecode.OpStoreVar || got[i].Op == bytecode.OpParseJson || got[i].Op == bytecode.OpMapGet + if got[i].OpString != want[i].OpString || got[i].IntOperand != want[i].IntOperand || (named && got[i].StringOperand != want[i].StringOperand) { + t.Fatalf("instruction %d: HFIR %s %q %d, AST %s %q %d", i, got[i].OpString, got[i].StringOperand, got[i].IntOperand, want[i].OpString, want[i].StringOperand, want[i].IntOperand) + } + } +} + +// Root cause of #155 and of the x02/x03/x07/x08/x15/x27 divergences: +// PARSE_JSON reads its body by variable name, but HFIR also evaluated the +// body symbol, leaving a stray string on the operand stack. +func TestIssue159ParseJSONDoesNotLeakItsBody(t *testing.T) { + got, want, _ := lowerBothIssue159(t, `(cli_app (let (s "{\"z\": 1}") (print (encode_json (parse_json Any s)) (let (v (parse_json Any s)) (map_get v "z")))))`) + sameInstructionStream(t, got, want) + for i, inst := range got { + if inst.Op == bytecode.OpParseJson && i > 0 && got[i-1].Op == bytecode.OpLoadVar && got[i-1].StringOperand == inst.StringOperand { + t.Fatalf("instruction %d: PARSE_JSON preceded by a LOAD_VAR of its own body", i) + } + } +} + +// PARSE_JSON can only name a variable; a content edge that is not that bound +// symbol fails closed instead of compiling to a wrong program. +func TestIssue159ParseJSONNonSymbolContentFailsClosed(t *testing.T) { + _, _, graph := lowerBothIssue159(t, `(cli_app (let (s "[1]") (print (parse_json Any s))))`) + for _, node := range graph.Nodes { + if node.Kind == "parse_json" { + constant := &Node{Kind: "const", LiteralKind: "STRING", Value: "[1]"} + node.DataInputs[0].SourceNode = graph.AddNode(constant) + } + } + if program, diagnostics := LowerToBytecode(graph); program != nil || len(diagnostics) == 0 { + t.Fatalf("non-symbol parse_json content lowered: %v %v", program, diagnostics) + } +} + +// Root cause of the x10/x11 compile failures: map_get whose dict is an +// expression had no HFIR shape. It now matches the AST compiler exactly. +func TestIssue159MapGetExpressionTarget(t *testing.T) { + got, want, _ := lowerBothIssue159(t, `(cli_app (let (d (dict ("m" (dict ("k" 7))))) (print (map_get (map_get d "m") "k"))))`) + sameInstructionStream(t, got, want) +} diff --git a/internal/hfir/lowering.go b/internal/hfir/lowering.go index 658ceb5a..7a38aa55 100644 --- a/internal/hfir/lowering.go +++ b/internal/hfir/lowering.go @@ -331,6 +331,18 @@ func (ctx *LoweringContext) lowerSemanticList(node *Node, astNode *ast.Node, hea if len(astNode.Children) < 2 { return "", false, nil } + if head == "map_get" && len(astNode.Children) == 3 && astNode.Children[1].Type != "SYMBOL" { + // (map_get key): the same shape bytecode.CompileToBytecode + // accepts. The dict expression is evaluated and popped by + // MAP_GET with an empty variable operand, which is what lets + // one map_get be the dict of another (#159). No new opcode. + node.Kind = head + id, err := addNamed([]struct { + name string + child *ast.Node + }{{"target", astNode.Children[1]}, {"key", astNode.Children[2]}}) + return id, true, err + } if (head == "map_get" || head == "map_delete" || head == "list_get" || head == "append" || head == "parse_json") && astNode.Children[1].Type != "SYMBOL" { return "", false, nil } diff --git a/tests/hfir_parity/155_parse_json_operand_stack.howl b/tests/hfir_parity/155_parse_json_operand_stack.howl new file mode 100644 index 00000000..baea69f6 --- /dev/null +++ b/tests/hfir_parity/155_parse_json_operand_stack.howl @@ -0,0 +1,6 @@ +(cli_app + (let (s "{\"z\": 1, \"a\": [2, {\"b\": null}]}") + (do + (print (encode_json (parse_json Any s)) (let (v (parse_json Any s)) (get v "z"))) + (print (str_join (map_list k (map_keys (parse_json Any s)) (str k "!")) ",")) + (print (list_len (map_keys (parse_json Any s))) (encode_json (parse_json Any s)))))) diff --git a/tests/hfir_parity/159_map_get_expression_target.howl b/tests/hfir_parity/159_map_get_expression_target.howl new file mode 100644 index 00000000..26dc6b31 --- /dev/null +++ b/tests/hfir_parity/159_map_get_expression_target.howl @@ -0,0 +1,8 @@ +(cli_app + (let (s "{\"meta\": {\"tags\": [\"ui\", \"urgent\"], \"owner\": {\"name\": \"kim\"}}}") + (let (doc (parse_json Any s)) + (do + (print (map_get (map_get doc "meta") "owner")) + (print (map_get (map_get (map_get doc "meta") "owner") "name")) + (print (list_len (map_get (map_get doc "meta") "tags"))) + (print (map_get (dict ("k" 7)) "k")))))) diff --git a/tools/difftest/hfir_regression_test.go b/tools/difftest/hfir_regression_test.go new file mode 100644 index 00000000..5a4e4116 --- /dev/null +++ b/tools/difftest/hfir_regression_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" +) + +// Focused regressions for #155 and #159: each fixture isolates one HFIR +// lowering root cause and must agree on HFIR bytecode, AST bytecode and the +// interpreter. Go and JavaScript are excluded on purpose: they reject +// parse_json-based programs as UNSUPPORTED_ON_TARGET, which is not what these +// fixtures pin. +func TestHFIRRegressionParity(t *testing.T) { + cases := map[string]string{ + // parse_json evaluated its symbol body onto the stack in addition to + // PARSE_JSON reading it by name; the stray raw string was printed + // instead of the re-encoded JSON. + "155_parse_json_operand_stack.howl": `{"a":[2,{"b":null}],"z":1} 1 +a!,z! +2 {"a":[2,{"b":null}],"z":1}`, + // map_get whose dict is itself an expression failed to lower. + "159_map_get_expression_target.howl": "map[name:kim]\nkim\n2\n7", + } + targets := []Target{TargetHFIRBytecode, TargetBytecode, TargetInterpreter} + for name, want := range cases { + name, want := name, want + t.Run(name, func(t *testing.T) { + report, err := VerifyParity(filepath.Join("..", "..", "tests", "hfir_parity", name), nil, "", targets) + if err != nil { + t.Fatal(err) + } + if report.OverallStatus != StatusPass { + t.Fatalf("parity mismatch:\n%s", strings.Join(report.Discrepancies, "\n")) + } + for target, result := range report.TargetResults { + if NormalizeOutput(result.Stdout) != want { + t.Errorf("%s stdout %q, want %q", target, result.Stdout, want) + } + } + }) + } +} diff --git a/tools/difftest/main.go b/tools/difftest/main.go index 3d3b8211..ad0238c3 100644 --- a/tools/difftest/main.go +++ b/tools/difftest/main.go @@ -117,8 +117,29 @@ func testFixture(file string) (passed bool, skipReason string, failMsg string) { return false, "", fmt.Sprintf("(run-bc): %v\n%s", err, out6) } - if out1 != out4 || out1 != out6 { - msg := fmt.Sprintf("stdout mismatch\n--- interpreter ---\n%s\n--- go backend ---\n%s\n--- bytecode ---\n%s", out1, out4, out6) + // #159: the experimental HFIR bytecode path is a third leg; any + // divergence from the interpreter fails the run. + hfirPath := filepath.Join(outDir, "test.hfir.bc") + // A construct on the locked HFIR promote-blocker fence (see + // TestProdFlipCriteriaLock) is refused at lowering, fail closed; that is + // not a semantic mismatch, so only that refusal skips the leg. + out8, hfirFenced := out1, false + out7, err := runCommand("go", "run", "howlframe.go", "-compile-hfir-bc", file, "-o", hfirPath) + if err != nil { + if !strings.Contains(out7, "is not in the Phase-1 executable subset") { + return false, "", fmt.Sprintf("(compile-hfir-bc): %v\n%s", err, out7) + } + hfirFenced = true + } + if !hfirFenced { + out8, err = runCommand("go", "run", "howlframe.go", "-run-bc", "-allow-caps", "network,filesystem,process,environment,database", hfirPath) + if err != nil { + return false, "", fmt.Sprintf("(run-bc hfir): %v\n%s", err, out8) + } + } + + if out1 != out4 || out1 != out6 || out1 != out8 { + msg := fmt.Sprintf("stdout mismatch\n--- interpreter ---\n%s\n--- go backend ---\n%s\n--- bytecode ---\n%s\n--- hfir bytecode ---\n%s", out1, out4, out6, out8) return false, "", msg }