Bytecode carries the fetch body (fixes #143)

docs/journals/2026-10-12_issue143_fetch_body.md

@@ -0,0 +1,53 @@
+# 2026-10-12 — Issue #143: bytecode carries the fetch body
+
+**Criteria change approved by the Owner (William Elias).** The prod-flip
+criteria lock and the ABI conformance tests used to pin "the bytecode
+fetch body is skipped". They now pin "the body is sent".
+
+## What changed
+- The AST bytecode compiler (`internal/bytecode`) and the HFIR bytecode
+  lowerer (`internal/hfir`) both compile `(fetch url method body)` with the
+  new shared helper `bytecode.FetchWithBody`:
+  `url, method, body, CONVERT to_string, HTTP_REQUEST 1, LOAD_CONST "body", MAP_GET`.
+  - This adds no new opcode, no new capability and no artifact-format
+    change. HTTP_REQUEST already carries a body and already needs `network`.
+  - The result is the response body bytes, the same value FETCH pushes.
+  - Semantics match the interpreter: the body is stringified, so `42` is
+    sent as `"42"`, and the method and URL are unchanged.
+  - A fetch without a body still compiles to the bare FETCH, so existing
+    artifacts and programs are unchanged.
+- Tests updated to the new behavior:
+  - `TestProdFlipCriteriaLock/fetch_body` now requires both compilers to emit
+    HTTP_REQUEST rather than a body-less FETCH, and to keep the body string.
+  - The hfir unit test `TestLowerToBytecodeFetchSkipsBodyOperand` is replaced
+    by `...SendsBodyOperand`, which locks the exact instruction sequence.
+  - `TestHFIRBytecodeFetchOperandOrder`: the PUT body is now received (2
+    bodies, one per compiler), and the operand shapes include the body.
+  - `nested_multi_effect_*` (fixture `21_nested_multi_effect.howl`, difftest
+    and VM tests): the only body fetch sits in a dead `while false` branch.
+    - Its marker is renamed from `phase2d-body-not-sent` to
+      `phase2d-dead-branch-body`.
+    - Both compilers must now load that body (they no longer drop it).
+    - The server still sees no body, because the branch never runs.
+- New regression test `TestIssue143FetchBodyEveryPath` uses a local
+  httptest server only.
+  - It covers the interpreter (`-run`), AST bytecode (`build` + `-run-bc`)
+    and HFIR bytecode (`-compile-hfir-bc` + `-run-bc`).
+  - The cases are a POST with a string body, a PUT with a numeric body, and
+    a GET with no body. All three paths must produce identical stdout and
+    identical requests.
+  - With the network grant denied, the bytecode paths must open no
+    connection and send no body.
+
+## Caveats
+- A transport failure on a fetch with a body reports `http_request failed` /
+  `http_request body exceeds maximum...` instead of `fetch failed`. The
+  code (IO_ERROR or LIMIT_EXCEEDED) is the same; only the opcode and message
+  text differ. The receipt records HTTP_REQUEST for these calls.
+- The body goes through CONVERT to_string. Strings and numbers match the
+  interpreter's `fmt.Sprint`. Other values, such as lists, dicts and bytes,
+  are stringified by the VM's own to_string, which may not format them the
+  way Go's `fmt.Sprint` does.
+- Production `-compile-bc`/HFIR is not flipped. #90 stays Partial. No tip
+  lock, no live LLM, no DOM, no external network. Hange's
+  `experiments/token-efficiency-heldout-v2` is untouched.

internal/bytecode/bytecode.go

@@ -500,6 +500,14 @@ func (c *BCCompiler) compileNode(node *ast.Node) []BCInstruction {
 			// construct.arities table above and before desugaring (#141).
 			insts = append(insts, c.compileNode(node.Children[1])...)
 			insts = append(insts, c.compileNode(node.Children[2])...)
+			if len(node.Children) == 4 {
+				// FETCH has no body operand. A body is sent the way the
+				// interpreter sends it (stringified) by lowering to the
+				// existing HTTP_REQUEST and taking its "body" bytes (#143).
+				// No new opcode and no new capability.
+				insts = append(insts, FetchWithBody(c.compileNode(node.Children[3]))...)
+				break
+			}
 			insts = append(insts, BCInstruction{OpString: "FETCH", Op: OpFetch})
 		case "http_request":
 			// (http_request url method [body]) → status+body dict.
@@ -888,6 +896,19 @@ func extractParamNames(paramsNode *ast.Node) []string {
 	return ast.ParamNames(paramsNode)
 }
 
+// FetchWithBody finishes (fetch url method body) once url and method are on
+// the stack: it stringifies body (as the interpreter's fmt.Sprint does for
+// strings and numbers), sends it with HTTP_REQUEST and keeps the response
+// body bytes, which is what FETCH pushes. Shared with the HFIR lowerer.
+func FetchWithBody(body []BCInstruction) []BCInstruction {
+	out := append([]BCInstruction{}, body...)
+	return append(out,
+		BCInstruction{OpString: "CONVERT", Op: OpConvert, StringOperand: "to_string"},
+		BCInstruction{OpString: "HTTP_REQUEST", Op: OpHttpRequest, IntOperand: 1},
+		BCInstruction{OpString: "LOAD_CONST", Op: OpLoadConst, ValueOperand: "body"},
+		BCInstruction{OpString: "MAP_GET", Op: OpMapGet})
+}
+
 // ShortCircuit lowers (and l r) / (or l r) so r is skipped when l decides.
 // The left value is tested by JUMP_IF_FALSE (which requires a boolean); a
 // skipped branch pushes the decided constant. An evaluated right operand still

internal/hfir/bytecode.go

@@ -365,9 +365,16 @@ func (c *bytecodeLowerer) compile(node *Node) (instructions []bytecode.BCInstruc
 		}
 		insts = append(insts, methodInsts...)
 		// Same operand order as bytecode.CompileToBytecode: URL, then method.
-		// FETCH pops the method and then the URL. The optional body is not
-		// an operand; the AST compiler does not compile that child either.
-		// No new opcode.
+		// FETCH pops the method and then the URL. A body lowers exactly as
+		// the AST compiler lowers it (#143): HTTP_REQUEST with the
+		// stringified body, then its "body" bytes. No new opcode.
+		if len(children) == 3 {
+			bodyInsts, childDiagnostic := compileChild(2)
+			if childDiagnostic != nil {
+				return nil, childDiagnostic
+			}
+			return append(insts, bytecode.FetchWithBody(bodyInsts)...), nil
+		}
 		return append(insts, instruction(bytecode.OpFetch, "FETCH", nil)), nil
 	case "parse_json":
 		if len(children) != 1 || node.DataInputs[0].Name != "content" {

internal/hfir/bytecode_test.go

@@ -209,11 +209,14 @@ func TestLowerToBytecodeFetchEmitsExistingOpcode(t *testing.T) {
 	}
 }
 
-func TestLowerToBytecodeFetchSkipsBodyOperand(t *testing.T) {
+// TestLowerToBytecodeFetchSendsBodyOperand: since #143 (Owner-approved change
+// to the flip criteria) a fetch body is sent, as the interpreter sends it, via
+// the existing HTTP_REQUEST opcode, keeping the response body bytes.
+func TestLowerToBytecodeFetchSendsBodyOperand(t *testing.T) {
 	graph := NewGraph()
 	url := graph.AddNode(&Node{Kind: "const", LiteralKind: "STRING", Value: "http://127.0.0.1:47653/howlframe-abi-v1-phase2d"})
 	method := graph.AddNode(&Node{Kind: "const", LiteralKind: "STRING", Value: "PUT"})
-	body := graph.AddNode(&Node{Kind: "const", LiteralKind: "STRING", Value: "phase2d-body-not-sent"})
+	body := graph.AddNode(&Node{Kind: "const", LiteralKind: "STRING", Value: "phase2d-body-sent"})
 	entry := graph.AddNode(&Node{
 		Kind: "fetch",
 		DataInputs: []DataEdge{
@@ -228,17 +231,24 @@ func TestLowerToBytecodeFetchSkipsBodyOperand(t *testing.T) {
 	if len(diagnostics) != 0 {
 		t.Fatalf("LowerToBytecode() diagnostics = %#v", diagnostics)
 	}
-	if len(program.Main) != 3 || program.Main[2].Op != bytecode.OpFetch {
-		t.Fatalf("instructions = %#v, want URL, method, FETCH", program.Main)
-	}
-	if program.Main[0].ValueOperand != "http://127.0.0.1:47653/howlframe-abi-v1-phase2d" || program.Main[1].ValueOperand != "PUT" {
-		t.Fatalf("operands = %#v, %#v, want URL then method", program.Main[0].ValueOperand, program.Main[1].ValueOperand)
+	want := []bytecode.Opcode{bytecode.OpLoadConst, bytecode.OpLoadConst, bytecode.OpLoadConst, bytecode.OpConvert, bytecode.OpHttpRequest, bytecode.OpLoadConst, bytecode.OpMapGet}
+	if len(program.Main) != len(want) {
+		t.Fatalf("instructions = %#v, want url, method, body, CONVERT, HTTP_REQUEST 1, \"body\", MAP_GET", program.Main)
 	}
-	for _, inst := range program.Main {
-		if inst.ValueOperand == "phase2d-body-not-sent" || inst.StringOperand == "phase2d-body-not-sent" {
-			t.Fatalf("FETCH compiled the body: %#v", program.Main)
+	for i, op := range want {
+		if program.Main[i].Op != op {
+			t.Fatalf("instruction %d = %#v, want op %v", i, program.Main[i], op)
 		}
 	}
+	if program.Main[0].ValueOperand != "http://127.0.0.1:47653/howlframe-abi-v1-phase2d" || program.Main[1].ValueOperand != "PUT" || program.Main[2].ValueOperand != "phase2d-body-sent" {
+		t.Fatalf("operands = %#v", program.Main[:3])
+	}
+	if program.Main[3].StringOperand != "to_string" || program.Main[4].IntOperand != 1 || program.Main[5].ValueOperand != "body" {
+		t.Fatalf("body lowering = %#v", program.Main[3:])
+	}
+	if bytecode.Registry[program.Main[4].Op].Capability != "network" {
+		t.Fatalf("HTTP_REQUEST capability = %q, want network", bytecode.Registry[program.Main[4].Op].Capability)
+	}
 }
 
 func TestLowerToBytecodeFetchRequiresURLAndMethod(t *testing.T) {

internal/hfir/lowering.go

@@ -416,9 +416,8 @@ func (ctx *LoweringContext) lowerSemanticList(node *Node, astNode *ast.Node, hea
 		// (fetch url method [body]). Existing FETCH opcode. The first edge
 		// is the URL and the second is the method. An optional third edge
 		// records a body. Experimental -compile-hfir-bc only. No new opcode
-		// and no new capability. OpFetch has no body operand, and the AST
-		// bytecode compiler does not compile that child, so the bytecode
-		// lowerer does not either.
+		// and no new capability. The bytecode lowerer sends the body via
+		// HTTP_REQUEST, as the AST bytecode compiler does (#143).
 		if len(astNode.Children) != 3 && len(astNode.Children) != 4 {
 			return "", false, nil
 		}

internal/vm/hfir_equivalence_test.go

@@ -1537,14 +1537,15 @@ func TestHFIRBytecodeFetchFixture(t *testing.T) {
 
 // TestHFIRBytecodeFetchOperandOrder locks GET, POST, and a three-argument
 // fetch onto the same FETCH instruction the AST compiler emits: URL, then
-// method. The third argument is a body. OpFetch has no body operand, so
-// neither compiler loads it, and the fixture server sees an empty body.
+// method. The third argument is a body. Since #143 (Owner-approved change
+// to the flip criteria) both compilers send it, as the interpreter does, by
+// lowering to HTTP_REQUEST and keeping the response body.
 func TestHFIRBytecodeFetchOperandOrder(t *testing.T) {
 	hits, bodies := serveABIFetchFixture(t)
 	source := `(cli_app
   (print (bytes_to_string (fetch "` + abiFetchURL + `" "GET")))
   (print (bytes_to_string (fetch "` + abiFetchURL + `" "POST")))
-  (print (bytes_to_string (fetch "` + abiFetchURL + `" "PUT" "phase2d-body-not-sent"))))`
+  (print (bytes_to_string (fetch "` + abiFetchURL + `" "PUT" "phase2d-body-sent"))))`
 	root, graph := checkedHFIRGraph(t, source)
 	var methods []string
 	var bodiesOnGraph int
@@ -1582,7 +1583,7 @@ func TestHFIRBytecodeFetchOperandOrder(t *testing.T) {
 	wantShapes := []string{
 		"main\x00" + abiFetchURL + "\x00GET",
 		"main\x00" + abiFetchURL + "\x00POST",
-		"main\x00" + abiFetchURL + "\x00PUT",
+		"main\x00" + abiFetchURL + "\x00PUT\x00body=phase2d-body-sent",
 	}
 	if got, want := fetchOperandShapes(legacy), fetchOperandShapes(direct); !reflect.DeepEqual(got, want) || !reflect.DeepEqual(got, wantShapes) {
 		t.Fatalf("FETCH operands\nAST %#v\nHFIR %#v\nwant %#v", got, want, wantShapes)
@@ -1595,8 +1596,8 @@ func TestHFIRBytecodeFetchOperandOrder(t *testing.T) {
 	if legacyOutcome.stdout != wantStdout || legacyOutcome.stderr != "" || legacyOutcome.exitCode != 0 || legacyOutcome.vmError != nil {
 		t.Fatalf("granted outcome %#v, want stdout %q", legacyOutcome, wantStdout)
 	}
-	if hits.Load() != 6 || bodies.Load() != 0 {
-		t.Fatalf("hits = %d bodies = %d, want 6 requests and no body", hits.Load(), bodies.Load())
+	if hits.Load() != 6 || bodies.Load() != 2 {
+		t.Fatalf("hits = %d bodies = %d, want 6 requests and 2 bodies (one PUT per compiler)", hits.Load(), bodies.Load())
 	}
 }
 
@@ -1656,8 +1657,9 @@ func TestHFIRBytecodeNestedFetchFixture(t *testing.T) {
 // TestHFIRBytecodeNestedMultiEffectFixture compares production AST bytecode
 // with the experimental HFIR lowerer on one program that nests env,
 // read_file, write_file, mkdir, exec, and fetch inside if, while, for, and
-// defun. One fetch records a body edge. OpFetch still has no body operand,
-// so neither compiler loads that string. The conformance harness runs the
+// defun. One fetch, in a dead while-false branch, records a body edge. Since
+// #143 both compilers load that body the same way, and it is never sent
+// because the branch never runs. The conformance harness runs the
 // same file through -compile-bc and -compile-hfir-bc.
 func TestHFIRBytecodeNestedMultiEffectFixture(t *testing.T) {
 	t.Setenv("HOWLFRAME_ABI_SECRET", "phase1-token")
@@ -1691,12 +1693,12 @@ func TestHFIRBytecodeNestedMultiEffectFixture(t *testing.T) {
 	if got, want := fetchOperandShapes(legacy), fetchOperandShapes(direct); !reflect.DeepEqual(got, want) {
 		t.Fatalf("FETCH operands\nAST %#v\nHFIR %#v", got, want)
 	}
-	const droppedBody = "phase2d-body-not-sent"
-	if strings.Contains(strings.Join(fetchOperandShapes(direct), "\n"), droppedBody) || programHasLiteral(direct, droppedBody) || programHasLiteral(legacy, droppedBody) {
-		t.Fatal("a bytecode compiler loaded the fetch body")
+	const deadBody = "phase2d-dead-branch-body"
+	if !programHasLiteral(direct, deadBody) || !programHasLiteral(legacy, deadBody) {
+		t.Fatal("a bytecode compiler dropped the fetch body")
 	}
 	getShape := "probe\x00" + abiFetchURL + "\x00GET"
-	putShape := "probe\x00" + abiFetchURL + "\x00PUT"
+	putShape := "probe\x00" + abiFetchURL + "\x00PUT\x00body=" + deadBody
 	var gets, puts int
 	for _, shape := range fetchOperandShapes(direct) {
 		switch shape {
@@ -1788,7 +1790,7 @@ func TestHFIRBytecodeNestedMultiEffectFixture(t *testing.T) {
 	if legacyPartial.vmError == nil || legacyPartial.vmError.Code != "CAPABILITY_DENIED" || legacyPartial.vmError.Opcode != "FETCH" || legacyPartial.vmError.Message != "capability denied: network" {
 		t.Fatalf("partial denial = %#v, want CAPABILITY_DENIED FETCH", legacyPartial)
 	}
-	if strings.Contains(legacyPartial.vmError.Message, abiFetchURL) || strings.Contains(legacyPartial.stdout, abiFetchMarker) || strings.Contains(legacyPartial.stdout, droppedBody) {
+	if strings.Contains(legacyPartial.vmError.Message, abiFetchURL) || strings.Contains(legacyPartial.stdout, abiFetchMarker) || strings.Contains(legacyPartial.stdout, deadBody) {
 		t.Fatalf("partial denial leaked the URL, marker, or body: %#v", legacyPartial)
 	}
 	if legacyPartial.stdout != partialStdout {
@@ -1808,7 +1810,7 @@ func TestHFIRBytecodeNestedMultiEffectFixture(t *testing.T) {
 	if legacyGranted.stdout != wantStdout || legacyGranted.stderr != "" || legacyGranted.exitCode != 0 || legacyGranted.vmError != nil || legacyGranted.panicVal != nil {
 		t.Fatalf("granted outcome %#v", legacyGranted)
 	}
-	if strings.Contains(legacyGranted.stdout, "no") || strings.Contains(legacyGranted.stdout, droppedBody) || strings.Contains(legacyGranted.stdout, prefix) {
+	if strings.Contains(legacyGranted.stdout, "no") || strings.Contains(legacyGranted.stdout, deadBody) || strings.Contains(legacyGranted.stdout, prefix) {
 		t.Fatalf("granted stdout leaked an untaken branch, body, or path: %q", legacyGranted.stdout)
 	}
 	if !reflect.DeepEqual(legacyFS, grantedFS) || !reflect.DeepEqual(directFS, grantedFS) {
@@ -2412,7 +2414,7 @@ func assertNestedMultiEffectShape(t *testing.T, graph *hfir.Graph, prefix string
 					t.Fatalf("body fetch edges %#v method %#v", node.DataInputs, method)
 				}
 				body := graph.NodeByID(node.DataInputs[2].SourceNode)
-				if body == nil || body.Kind != "const" || body.Value != "phase2d-body-not-sent" {
+				if body == nil || body.Kind != "const" || body.Value != "phase2d-dead-branch-body" {
 					t.Fatalf("fetch body = %#v", body)
 				}
 				bodyEdges++
@@ -3010,6 +3012,26 @@ func fetchOperandShapes(program *bytecode.BCProgram) []string {
 	var shapes []string
 	collect := func(where string, insts []bytecode.BCInstruction) {
 		for index, inst := range insts {
+			// A fetch with a body lowers to url, method, body, CONVERT,
+			// HTTP_REQUEST 1, LOAD_CONST "body", MAP_GET (#143).
+			if inst.Op == bytecode.OpHttpRequest && inst.IntOperand == 1 && index >= 4 &&
+				insts[index-1].Op == bytecode.OpConvert && insts[index-1].StringOperand == "to_string" &&
+				index+2 < len(insts) && insts[index+1].Op == bytecode.OpLoadConst && insts[index+1].ValueOperand == "body" && insts[index+2].Op == bytecode.OpMapGet {
+				parts := []string{where}
+				for _, prev := range insts[index-4 : index-1] {
+					text, ok := prev.ValueOperand.(string)
+					if prev.Op != bytecode.OpLoadConst || !ok {
+						parts = []string{where, "nonliteral"}
+						break
+					}
+					parts = append(parts, text)
+				}
+				if len(parts) == 4 {
+					parts[3] = "body=" + parts[3]
+				}
+				shapes = append(shapes, strings.Join(parts, "\x00"))
+				continue
+			}
 			if inst.Op != bytecode.OpFetch {
 				continue
 			}

issue143_fetch_body_cli_test.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+)
+
+// TestIssue143FetchBodyEveryPath: a fetch body is sent with the same
+// semantics on the interpreter, AST bytecode and HFIR bytecode (-run-bc).
+// The body is stringified, so a number is sent as its decimal text, and a
+// fetch without a body sends none. A denied network grant sends nothing.
+// Local httptest server only.
+func TestIssue143FetchBodyEveryPath(t *testing.T) {
+	bin := c1c2Binary(t)
+	var mu sync.Mutex
+	var got []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, _ := io.ReadAll(r.Body)
+		mu.Lock()
+		got = append(got, r.Method+" "+string(b))
+		mu.Unlock()
+		_, _ = w.Write([]byte("echo:" + string(b)))
+	}))
+	defer srv.Close()
+	dir := t.TempDir()
+	src := filepath.Join(dir, "fb.howl")
+	program := `(cli_app
+  (print (bytes_to_string (fetch "` + srv.URL + `/a" "POST" "hello")))
+  (let (n 42) (print (bytes_to_string (fetch "` + srv.URL + `/b" "PUT" n))))
+  (print (bytes_to_string (fetch "` + srv.URL + `/c" "GET"))))
+`
+	if err := os.WriteFile(src, []byte(program), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	const wantOut = "echo:hello\necho:42\necho:\n"
+	wantReqs := []string{"POST hello", "PUT 42", "GET "}
+	reset := func() []string {
+		mu.Lock()
+		defer mu.Unlock()
+		r := got
+		got = nil
+		return r
+	}
+	paths := map[string]func() (string, int){
+		"interpreter": func() (string, int) { return issue141Run(bin, "-allow-caps", "network", "-run", src) },
+		"ast_bytecode": func() (string, int) {
+			art := filepath.Join(dir, "ast.hfbc")
+			if out, code := issue141Run(bin, "build", src, "-o", art); code != 0 {
+				return out, code
+			}
+			return issue141Run(bin, "-run-bc", "-allow-caps", "network", art)
+		},
+		"hfir_bytecode": func() (string, int) {
+			art := filepath.Join(dir, "hfir.hfbc")
+			if out, code := issue141Run(bin, "-compile-hfir-bc", src, "-o", art); code != 0 {
+				return out, code
+			}
+			return issue141Run(bin, "-run-bc", "-allow-caps", "network", art)
+		},
+	}
+	for _, name := range []string{"interpreter", "ast_bytecode", "hfir_bytecode"} {
+		reset()
+		out, code := paths[name]()
+		reqs := reset()
+		if code != 0 || out != wantOut {
+			t.Errorf("%s: exit %d stdout %q, want %q", name, code, out, wantOut)
+		}
+		if strings.Join(reqs, "|") != strings.Join(wantReqs, "|") {
+			t.Errorf("%s: requests %q, want %q", name, reqs, wantReqs)
+		}
+	}
+
+	// Denied: the bytecode paths must not open a connection or send the body.
+	for _, compile := range [][]string{{"build", src, "-o"}, {"-compile-hfir-bc", src, "-o"}} {
+		art := filepath.Join(dir, "denied-"+strings.TrimLeft(compile[0], "-")+".hfbc")
+		if out, code := issue141Run(bin, append(compile, art)...); code != 0 {
+			t.Fatalf("%v: %s", compile, out)
+		}
+		reset()
+		out, code := issue141Run(bin, "-run-bc", art)
+		if code == 0 || !strings.Contains(out, "capability denied: network") {
+			t.Errorf("%v denied: exit %d %s", compile, code, out)
+		}
+		if reqs := reset(); len(reqs) != 0 {
+			t.Errorf("%v denied sent %q", compile, reqs)
+		}
+	}
+}