From 1d05abbc51412740a57edacec20a9558cacc17f2 Mon Sep 17 00:00:00 2001 From: Curtis Myzie Date: Fri, 12 Jun 2026 15:14:54 -0400 Subject: [PATCH] Add named bindings, flatMap, sortBy, entries, sort/reverse, and template upgrades Implements the post-v1.1.0 roadmap items from the Mobius Ops integration feedback. Language surface: - Every iterating form (map, filter, flatMap, any, all, find, count, sortBy) now accepts a three-arg shape that names the element binding: filter(orders, o, o.status == "paid"). The named form binds only the chosen name plus index, leaving `it` resolving to an enclosing two-arg form, which closes the nested-forms scoping gap. Two-arg calls are unchanged. - New flatMap form: like map, but list body results splice element-by-element, nil splices as nothing, and any other value appends as a single element. - New sortBy form: stable sort of a copy by a per-element key expression; keys must be all numbers or all strings. - New entries(m) builtin in the default set: sorted key-value pairs of a string-keyed map, making maps iterable through the forms. - New sort(xs) and reverse(xs) builtins in CollectionFuncs(). Templates: - Composite values (maps, slices, arrays, structs) render as compact JSON with HTML escaping disabled instead of Go map syntax. This is a rendering behavior change; see the spec's templates section. - WithTemplateDelimiters("${{", "}}") replaces the default `${`/`}` so shell snippets like ${HOME} pass through literally. - WithTemplateFormatter(fn) is an escape hatch that runs before the default rendering chain. - Template errors now report 1-based line:column with the byte offset as supplementary detail. - New Template.Segments() exposes literal and expression segments with positions and each segment's compiled *Program, for editor hints and per-segment Identifiers(). - Template-only options passed to Compile fail with ErrCompile instead of being silently ignored. Also adds docs/rfcs/0001-pipe-operator.md, a design RFC for repurposing `|` as a pipeline operator. No implementation commitment; the recommendation is opt-in via a WithPipeOperator option if adopted. Spec, guides, examples, llms.txt, README, doc tests, and runnable example companions are updated to match. Template fuzzing now covers custom delimiters. Co-Authored-By: Claude Fable 5 --- README.md | 49 +- builtin_groups.go | 121 ++++- builtins.go | 36 ++ builtins_entries_sort_test.go | 286 ++++++++++++ docs/guides/examples.md | 150 ++++++- docs/guides/higher-order-patterns.md | 295 ++++++++---- docs/guides/templates.md | 231 +++++++--- docs/reference/spec.md | 331 ++++++++++++-- docs/rfcs/0001-pipe-operator.md | 597 +++++++++++++++++++++++++ docs_examples_test.go | 110 +++-- docs_guides_test.go | 219 +++++++++ engine.go | 34 +- examples/higher_order_patterns/main.go | 54 ++- examples/templates_in_anger/main.go | 34 +- higher_order.go | 209 +++++++-- higher_order_binding_test.go | 296 ++++++++++++ identifiers.go | 86 ++-- llms.txt | 145 ++++-- program.go | 27 +- suggest.go | 2 +- template.go | 432 ++++++++++++++---- template_features_test.go | 257 +++++++++++ template_fuzz_test.go | 64 +++ 23 files changed, 3608 insertions(+), 457 deletions(-) create mode 100644 builtins_entries_sort_test.go create mode 100644 docs/rfcs/0001-pipe-operator.md create mode 100644 higher_order_binding_test.go create mode 100644 template_features_test.go diff --git a/README.md b/README.md index 905188a..2019db3 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,9 @@ out, err := tmpl.Render(ctx, env) The text outside `${...}` is just text. The expression inside can use the same selectors, functions, literals, and higher-order forms as any other compiled -expression. +expression. `nil` renders as the empty string; maps, slices, arrays, and +structs render as compact JSON (`${config}` produces `{"retries":3}`, not +`map[retries:3]`). ## Using the Go API @@ -90,9 +92,11 @@ safe to share between goroutines. Compile at startup. Run per request. No functions are registered by default, so the surface area is exactly as wide as you want it. `WithBuiltins()` opts you into a small standard set (`len`, -`contains`, `has`, `keys`, `upper`, `lower`, `int`, `float`, `string`, `bool`, -`sprintf`). `WithFunctions` lets you register any Go function as a callable -identifier: +`contains`, `has`, `keys`, `entries`, `upper`, `lower`, `int`, `float`, +`string`, `bool`, `sprintf`). `entries(m)` returns the sorted key-value pairs +of a string-keyed map as `[{"key":k,"value":v}, ...]`, making maps iterable +through higher-order forms. `WithFunctions` lets you register any Go function +as a callable identifier: ```go p, err := expr.Compile(`greet(upper(name))`, expr.WithFunctions(map[string]any{ @@ -105,8 +109,8 @@ Mix and match, or skip the builtins entirely and expose only the handful that make sense for your sandbox. Opt-in groups — `expr.MathFuncs()` (`min`, `max`, `abs`, `floor`, `ceil`, `round`), `expr.StringFuncs()` (`trim`, `split`, `join`, `replace`, `startsWith`, `endsWith`), and `expr.CollectionFuncs()` -(`first`, `last`, `sum`, `slice`) — add the usual helpers via `WithFunctions` -without widening the default set. +(`first`, `last`, `sum`, `slice`, `sort`, `reverse`) — add the usual helpers +via `WithFunctions` without widening the default set. ## What the environment can be @@ -143,22 +147,33 @@ are left alone, so nothing you already had stops working. ## Higher-order forms -A small set of always-available forms for working with lists: `map`, `filter`, -`any`, `all`, `find`, `count`. Inside the second argument, `it` is the current -element and `index` is its position: +A set of always-available forms for working with lists: `map`, `filter`, +`flatMap`, `any`, `all`, `find`, `count`, `sortBy`. Inside the body, `it` is +the current element and `index` is its position: ```go p, err := expr.Compile(`filter(users, it.age >= 18 && index < 10)`) ``` -The predicate is re-evaluated per element, so they compose naturally: -`any(orders, count(it.items, it.price > 100) > 0)`. Two more special forms -use laziness for control flow instead of iteration: `try(value, default)` -falls back when `value` errors, and `if(cond, then, else)` evaluates only the -branch the condition selects — so `if(n != 0, total/n, 0)` can't divide by -zero. These forms are always registered (no `WithBuiltins` needed), but you -can shadow any of them by registering a function or env value of the same -name. +Every iterating form also accepts a three-argument shape that names the element +explicitly, which makes nested forms readable and lets you reference an outer +element from inside an inner body: + +```go +// Named bindings: r is the review, c is the comment. +map(reviews, r, map(r.comments, c, r.author + ": " + c)) +``` + +`flatMap` works like `map` but splices list body results element-by-element +into the output, which flattens one level of nesting. `sortBy` evaluates a key +expression per element and returns a stable-sorted copy of the list. + +Two more special forms use laziness for control flow instead of iteration: +`try(value, default)` falls back when `value` errors, and `if(cond, then, +else)` evaluates only the branch the condition selects, so `if(n != 0, total/n, +0)` can't divide by zero. These forms are always registered (no `WithBuiltins` +needed), but you can shadow any of them by registering a function or env value +of the same name. ## What it isn't diff --git a/builtin_groups.go b/builtin_groups.go index 44f0e13..6808534 100644 --- a/builtin_groups.go +++ b/builtin_groups.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "reflect" + "sort" "strings" ) @@ -63,12 +64,17 @@ func StringFuncs() map[string]any { // slice(xs, i, j) elements [i, j) of a list, or the rune range // of a string; negative indices count from the // end and out-of-range bounds clamp +// sort(xs) ascending copy; all numbers (numeric order) or +// all strings (lexicographic); mixed types error +// reverse(xs) reversed copy; never mutates input func CollectionFuncs() map[string]any { return map[string]any{ - "first": Func(nativeFirst), - "last": Func(nativeLast), - "sum": Func(nativeSum), - "slice": Func(nativeSlice), + "first": Func(nativeFirst), + "last": Func(nativeLast), + "sum": Func(nativeSum), + "slice": Func(nativeSlice), + "sort": Func(nativeSort), + "reverse": Func(nativeReverse), } } @@ -399,3 +405,110 @@ func resolveIndex(i int64, n int) int { } return int(i) } + +func nativeSort(_ context.Context, args []any) (any, error) { + if err := checkArity("sort", 1, len(args)); err != nil { + return nil, err + } + return builtinSort(args[0]) +} + +// builtinSort returns a sorted copy of a list. All elements must be +// either numbers (any int/float mix, compared with the same rules as +// the < operator) or all strings (lexicographic). The sort is stable +// and reorders the original elements without converting them: ints +// stay ints, floats stay floats. Mixed or non-comparable element +// types produce an ErrEvaluate. An empty or nil input returns an +// empty []any. +func builtinSort(v any) ([]any, error) { + if v == nil { + return []any{}, nil + } + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return nil, fmt.Errorf("%w: sort: expected list, got %T", ErrEvaluate, v) + } + n := rv.Len() + out := make([]any, n) + for i := 0; i < n; i++ { + out[i] = rv.Index(i).Interface() + } + less, err := scalarLessFunc("sort", out) + if err != nil { + return nil, err + } + sort.SliceStable(out, less) + return out, nil +} + +// scalarLessFunc validates that every value in vals is a number (any +// int/float mix) or that every value is a string, and returns the +// matching index-based less function. The mode is chosen by the first +// value; the error names the first value that does not fit. Shared by +// sort and sortBy so the two agree on comparison semantics. +func scalarLessFunc(name string, vals []any) (func(i, j int) bool, error) { + if len(vals) == 0 { + return func(i, j int) bool { return false }, nil + } + if _, ok := toFloat64(vals[0]); ok { + for i, v := range vals { + if _, ok := toFloat64(v); !ok { + return nil, fmt.Errorf("%w: %s: element %d is %T, not a number", ErrEvaluate, name, i, v) + } + } + return func(i, j int) bool { return numericLess(vals[i], vals[j]) }, nil + } + if _, ok := asString(vals[0]); ok { + for i, v := range vals { + if _, ok := asString(v); !ok { + return nil, fmt.Errorf("%w: %s: element %d is %T, not a string", ErrEvaluate, name, i, v) + } + } + return func(i, j int) bool { + a, _ := asString(vals[i]) + b, _ := asString(vals[j]) + return a < b + }, nil + } + return nil, fmt.Errorf("%w: %s: elements must be all numbers or all strings, got %T", + ErrEvaluate, name, vals[0]) +} + +// numericLess compares two numbers with the same rules as the < +// operator: both integral values compare as int64, any other mix +// compares as float64. +func numericLess(a, b any) bool { + if ai, ok := toInt64(a); ok { + if bi, ok := toInt64(b); ok { + return ai < bi + } + } + af, _ := toFloat64(a) + bf, _ := toFloat64(b) + return af < bf +} + +func nativeReverse(_ context.Context, args []any) (any, error) { + if err := checkArity("reverse", 1, len(args)); err != nil { + return nil, err + } + return builtinReverse(args[0]) +} + +// builtinReverse returns a reversed copy of a list. It never mutates +// the input. nil and empty lists return an empty []any. +func builtinReverse(v any) ([]any, error) { + if v == nil { + return []any{}, nil + } + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return nil, fmt.Errorf("%w: reverse: expected list, got %T", ErrEvaluate, v) + } + n := rv.Len() + out := make([]any, n) + for i := 0; i < n; i++ { + out[i] = rv.Index(n - 1 - i).Interface() + } + return out, nil +} diff --git a/builtins.go b/builtins.go index ddc6b64..bab87fc 100644 --- a/builtins.go +++ b/builtins.go @@ -29,6 +29,8 @@ import ( // key presence for string-keyed maps // has(m, k) true if map m has key k; errors if m is not a map // keys(m) sorted string keys of a map +// entries(m) sorted key-value pairs of a string-keyed map; each +// element is map[string]any{"key": k, "value": v} // lower(s), upper(s) case conversion // sprintf(fmt, ...) fmt.Sprintf-style formatting with cycle guards // @@ -46,6 +48,7 @@ func Builtins() map[string]any { "contains": Func(nativeContains), "has": Func(nativeHas), "keys": Func(nativeKeys), + "entries": Func(nativeEntries), "lower": Func(nativeLower), "upper": Func(nativeUpper), "sprintf": Func(nativeSprintf), @@ -116,6 +119,13 @@ func nativeKeys(_ context.Context, args []any) (any, error) { return builtinKeys(args[0]) } +func nativeEntries(_ context.Context, args []any) (any, error) { + if err := checkArity("entries", 1, len(args)); err != nil { + return nil, err + } + return builtinEntries(args[0]) +} + func nativeLower(_ context.Context, args []any) (any, error) { if err := checkArity("lower", 1, len(args)); err != nil { return nil, err @@ -307,3 +317,29 @@ func builtinKeys(m any) ([]any, error) { } return out, nil } + +// builtinEntries returns the key-value pairs of a string-keyed map as a +// []any, sorted by key for determinism. Each element is a +// map[string]any{"key": k, "value": v}, mirroring the sort and key-type +// rules of builtinKeys. +func builtinEntries(m any) ([]any, error) { + if m == nil { + return nil, nil + } + rv := reflect.ValueOf(m) + if rv.Kind() != reflect.Map || rv.Type().Key().Kind() != reflect.String { + return nil, fmt.Errorf("%w: entries: expected map with string keys, got %T", ErrEvaluate, m) + } + mapKeys := rv.MapKeys() + strs := make([]string, len(mapKeys)) + for i, k := range mapKeys { + strs[i] = k.String() + } + sort.Strings(strs) + out := make([]any, len(strs)) + for i, s := range strs { + val := rv.MapIndex(mapStringKey(rv.Type().Key(), s)).Interface() + out[i] = map[string]any{"key": s, "value": val} + } + return out, nil +} diff --git a/builtins_entries_sort_test.go b/builtins_entries_sort_test.go new file mode 100644 index 0000000..b6ad2ea --- /dev/null +++ b/builtins_entries_sort_test.go @@ -0,0 +1,286 @@ +package expr + +import ( + "testing" + + "github.com/deepnoodle-ai/expr/internal/require" +) + +// --------------------------------------------------------------------------- +// entries() +// --------------------------------------------------------------------------- + +func TestEntries_EmptyMap(t *testing.T) { + got, err := evalExpr(t.Context(), `entries(m)`, map[string]any{"m": map[string]any{}}, WithBuiltins()) + require.NoError(t, err) + require.Equal(t, []any{}, got) +} + +func TestEntries_NilMap(t *testing.T) { + got, err := evalExpr(t.Context(), `entries(m)`, map[string]any{"m": nil}, WithBuiltins()) + require.NoError(t, err) + require.Nil(t, got) +} + +func TestEntries_SingleKey(t *testing.T) { + m := map[string]any{"x": int64(42)} + got, err := evalExpr(t.Context(), `entries(m)`, map[string]any{"m": m}, WithBuiltins()) + require.NoError(t, err) + want := []any{ + map[string]any{"key": "x", "value": int64(42)}, + } + require.Equal(t, want, got) +} + +// entries must sort by key for determinism regardless of map iteration order. +func TestEntries_MultiKeySorted(t *testing.T) { + m := map[string]any{"c": int64(3), "a": int64(1), "b": int64(2)} + got, err := evalExpr(t.Context(), `entries(m)`, map[string]any{"m": m}, WithBuiltins()) + require.NoError(t, err) + want := []any{ + map[string]any{"key": "a", "value": int64(1)}, + map[string]any{"key": "b", "value": int64(2)}, + map[string]any{"key": "c", "value": int64(3)}, + } + require.Equal(t, want, got) +} + +// entries must match the sort order produced by keys() for the same map. +func TestEntries_SortMatchesKeys(t *testing.T) { + m := map[string]any{"zebra": "z", "apple": "a", "mango": "m"} + env := map[string]any{"m": m} + + keysGot, err := evalExpr(t.Context(), `keys(m)`, env, WithBuiltins()) + require.NoError(t, err) + + entriesGot, err := evalExpr(t.Context(), `entries(m)`, env, WithBuiltins()) + require.NoError(t, err) + + keySlice := keysGot.([]any) + entrySlice := entriesGot.([]any) + require.Equal(t, len(keySlice), len(entrySlice)) + for i, k := range keySlice { + entry := entrySlice[i].(map[string]any) + require.Equal(t, k, entry["key"], "position %d key mismatch", i) + } +} + +// Non-string-keyed map must produce an ErrEvaluate, same class as keys(). +func TestEntries_NonStringKeyError(t *testing.T) { + _, err := evalExpr(t.Context(), `entries(m)`, map[string]any{"m": map[int]any{1: "x"}}, WithBuiltins()) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "entries") +} + +// entries arity check. +func TestEntries_ArityError(t *testing.T) { + _, err := evalExpr(t.Context(), `entries(m, m)`, map[string]any{"m": map[string]any{}}, WithBuiltins()) + require.ErrorIs(t, err, ErrEvaluate) +} + +// Compose entries with filter: keep only entries whose value > 1. +func TestEntries_ComposeWithFilter(t *testing.T) { + m := map[string]any{"a": int64(1), "b": int64(2), "c": int64(3)} + env := map[string]any{"m": m} + // filter(entries(m), it.value > 1) returns entries with value 2 and 3. + got, err := evalExpr(t.Context(), + `filter(entries(m), it.value > 1)`, + env, + WithBuiltins(), + ) + require.NoError(t, err) + want := []any{ + map[string]any{"key": "b", "value": int64(2)}, + map[string]any{"key": "c", "value": int64(3)}, + } + require.Equal(t, want, got) +} + +// Compose entries with map form to extract keys. +func TestEntries_ComposeWithMapForm(t *testing.T) { + m := map[string]any{"b": int64(2), "a": int64(1)} + env := map[string]any{"m": m} + // map(entries(m), it.key) should equal keys(m) + got, err := evalExpr(t.Context(), `map(entries(m), it.key)`, env, WithBuiltins()) + require.NoError(t, err) + require.Equal(t, []any{"a", "b"}, got) +} + +// --------------------------------------------------------------------------- +// sort() +// --------------------------------------------------------------------------- + +func TestSort_Ints(t *testing.T) { + env := map[string]any{"xs": []any{int64(3), int64(1), int64(2)}} + got, err := evalExpr(t.Context(), `sort(xs)`, env, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{int64(1), int64(2), int64(3)}, got) +} + +func TestSort_Floats(t *testing.T) { + env := map[string]any{"xs": []any{3.5, 1.1, 2.2}} + got, err := evalExpr(t.Context(), `sort(xs)`, env, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + // All floats, all whole-or-fractional: result stays float64. + sl := got.([]any) + require.Equal(t, 3, len(sl)) + require.Equal(t, 1.1, sl[0]) + require.Equal(t, 2.2, sl[1]) + require.Equal(t, 3.5, sl[2]) +} + +func TestSort_MixedIntFloat(t *testing.T) { + // int64 and float64 are both numeric; sort numerically. + env := map[string]any{"xs": []any{int64(3), 1.5, int64(2)}} + got, err := evalExpr(t.Context(), `sort(xs)`, env, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + sl := got.([]any) + require.Equal(t, 3, len(sl)) + // 1.5 first, then 2, then 3 + f0, ok0 := toFloat64(sl[0]) + f1, ok1 := toFloat64(sl[1]) + f2, ok2 := toFloat64(sl[2]) + require.True(t, ok0 && ok1 && ok2) + require.Equal(t, 1.5, f0) + require.Equal(t, float64(2), f1) + require.Equal(t, float64(3), f2) +} + +func TestSort_Strings(t *testing.T) { + env := map[string]any{"xs": []any{"banana", "apple", "cherry"}} + got, err := evalExpr(t.Context(), `sort(xs)`, env, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{"apple", "banana", "cherry"}, got) +} + +func TestSort_Empty(t *testing.T) { + got, err := evalExpr(t.Context(), `sort([])`, nil, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{}, got) +} + +func TestSort_Nil(t *testing.T) { + got, err := evalExpr(t.Context(), `sort(xs)`, map[string]any{"xs": nil}, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{}, got) +} + +func TestSort_SingleElement(t *testing.T) { + got, err := evalExpr(t.Context(), `sort([42])`, nil, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{int64(42)}, got) +} + +func TestSort_MixedTypeError(t *testing.T) { + env := map[string]any{"xs": []any{int64(1), "two", int64(3)}} + _, err := evalExpr(t.Context(), `sort(xs)`, env, WithFunctions(CollectionFuncs())) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "sort") +} + +func TestSort_BoolError(t *testing.T) { + env := map[string]any{"xs": []any{true, false}} + _, err := evalExpr(t.Context(), `sort(xs)`, env, WithFunctions(CollectionFuncs())) + require.ErrorIs(t, err, ErrEvaluate) +} + +func TestSort_NilElementError(t *testing.T) { + env := map[string]any{"xs": []any{int64(1), nil, int64(3)}} + _, err := evalExpr(t.Context(), `sort(xs)`, env, WithFunctions(CollectionFuncs())) + require.ErrorIs(t, err, ErrEvaluate) +} + +func TestSort_NotAListError(t *testing.T) { + _, err := evalExpr(t.Context(), `sort(42)`, nil, WithFunctions(CollectionFuncs())) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "sort") +} + +// sort must not mutate the input slice. +func TestSort_InputNotMutated(t *testing.T) { + original := []any{int64(3), int64(1), int64(2)} + // make a copy to check against + snapshot := []any{int64(3), int64(1), int64(2)} + env := map[string]any{"xs": original} + _, err := evalExpr(t.Context(), `sort(xs)`, env, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, snapshot, original) +} + +// Expressions with inline literals work too. +func TestSort_InlineLiteral(t *testing.T) { + got, err := evalExpr(t.Context(), `sort([3, 1, 2])`, nil, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{int64(1), int64(2), int64(3)}, got) +} + +// --------------------------------------------------------------------------- +// reverse() +// --------------------------------------------------------------------------- + +func TestReverse_List(t *testing.T) { + env := map[string]any{"xs": []any{int64(1), int64(2), int64(3)}} + got, err := evalExpr(t.Context(), `reverse(xs)`, env, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{int64(3), int64(2), int64(1)}, got) +} + +func TestReverse_Strings(t *testing.T) { + env := map[string]any{"xs": []any{"a", "b", "c"}} + got, err := evalExpr(t.Context(), `reverse(xs)`, env, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{"c", "b", "a"}, got) +} + +func TestReverse_Empty(t *testing.T) { + got, err := evalExpr(t.Context(), `reverse([])`, nil, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{}, got) +} + +func TestReverse_Nil(t *testing.T) { + got, err := evalExpr(t.Context(), `reverse(xs)`, map[string]any{"xs": nil}, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{}, got) +} + +func TestReverse_SingleElement(t *testing.T) { + got, err := evalExpr(t.Context(), `reverse([99])`, nil, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{int64(99)}, got) +} + +func TestReverse_NotAListError(t *testing.T) { + _, err := evalExpr(t.Context(), `reverse("hello")`, nil, WithFunctions(CollectionFuncs())) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "reverse") +} + +// reverse must not mutate the input slice. +func TestReverse_InputNotMutated(t *testing.T) { + original := []any{int64(1), int64(2), int64(3)} + snapshot := []any{int64(1), int64(2), int64(3)} + env := map[string]any{"xs": original} + _, err := evalExpr(t.Context(), `reverse(xs)`, env, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, snapshot, original) +} + +// Compose sort + reverse for descending order. +func TestSortReverse_Descending(t *testing.T) { + env := map[string]any{"xs": []any{int64(3), int64(1), int64(2)}} + got, err := evalExpr(t.Context(), `reverse(sort(xs))`, env, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + require.Equal(t, []any{int64(3), int64(2), int64(1)}, got) +} + +// sort and reverse must not appear in the default Builtins set. +func TestSortReverse_NotInDefaultBuiltins(t *testing.T) { + _, err := evalExpr(t.Context(), `sort([1, 2])`, nil, WithBuiltins()) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "unknown function") + + _, err = evalExpr(t.Context(), `reverse([1, 2])`, nil, WithBuiltins()) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "unknown function") +} diff --git a/docs/guides/examples.md b/docs/guides/examples.md index 69104e8..452ec4c 100644 --- a/docs/guides/examples.md +++ b/docs/guides/examples.md @@ -202,11 +202,17 @@ map[string]any{ } ``` -Caveat worth knowing: inside a nested higher-order form, `it` and -`index` always refer to the **innermost** form's current element. -There is no `let` or outer-binding. If you need to reference both -the outer element and inner element in the same predicate, stop -nesting and do the join in Go, or register a helper function. +Named bindings let you reference the outer element by name from inside +an inner body: + +``` +filter(entries(scores), e, e.value >= 80) +``` + +Caveat for the two-arg form: inside a nested two-arg higher-order form, +`it` and `index` always refer to the **innermost** form's current +element. The outer `it` is shadowed. Use named bindings to keep both +visible. --- @@ -238,11 +244,11 @@ map[string]any{ } ``` -The `${...}` result is stringified via `fmt.Sprintf("%v", ...)`, so a -`[]any` of strings prints as a Go slice. For real templating of lists, -either build the final string with `sprintf` in a single expression, or -join the list in the host program and interpolate the joined string back -in through the env. +Maps, slices, arrays, and structs render as compact JSON inside `${...}`: +`${files}` produces `["a.go","b.go"]`, not `[a.go b.go]`. For +variable-length lists rendered as human-readable text, register a `join` +helper and call it from the expression, or join in Go and pass the result +through the env. --- @@ -345,33 +351,49 @@ a redeploy. That's the entire pitch for an embedded expression language. --- -## 8. Extracting + sorting via a registered function +## 8. Extracting + sorting -Higher-order forms don't include `sort`, on purpose — sorting needs -stable comparators and expr stays out of that business. Register a Go -function instead: +`sortBy` is a built-in special form. It evaluates a key expression per +element and returns a stable-sorted copy of the list. Combined with +`filter` and a registered `take`: ```go take( sortBy( filter(users, it.active), - "age", + it.age, + ), + 3, +) +``` + +Or with the named-binding form for clarity: + +```go +take( + sortBy( + filter(users, u, u.active), + u, + u.age, ), 3, ) ``` -Host-side registration: +Host-side `take` registration: ```go expr.WithFunctions(map[string]any{ - "sortBy": func(xs []any, key string) []any { ... }, - "take": func(xs []any, n int) []any { ... }, + "take": func(xs []any, n int) []any { ... }, }) ``` -The philosophy: if expr doesn't have it, register a Go function for it. -Don't fight the language. +`sortBy` keys must be all numbers or all strings. For descending order, +compose with `reverse` from `CollectionFuncs`: + +```go +reverse(sortBy(users, u, u.age)) +``` --- @@ -444,3 +466,91 @@ p, err := expr.Compile(src, With an empty `prices` list, `avg_price` is `0` rather than a division-by-zero error — the untaken branch never runs. + +--- + +## 11. Named bindings and flatMap + +Named element bindings let you refer to the outer element by name from +inside a nested form body. `flatMap` flattens one level of nesting. + +```go +// Extract all order IDs from all users using flatMap with a named binding. +flatMap(users, u, u.orders) +``` + +Env: + +```go +map[string]any{ + "users": []any{ + map[string]any{"orders": []any{int64(1), int64(2)}}, + map[string]any{"orders": []any{int64(3)}}, + }, +} +``` + +Result: `[]any{1, 2, 3}`. + +Nested named forms — the outer `r` stays visible inside the inner body +because the inner named form does not bind `it`: + +```go +map(reviews, r, join(map(r.comments, c, r.author + ": " + c), "; ")) +``` + +Env: + +```go +map[string]any{ + "reviews": []any{ + map[string]any{"author": "ann", "comments": []any{"good", "clear"}}, + map[string]any{"author": "bob", "comments": []any{"ok"}}, + }, +} +``` + +Result (requires `WithFunctions(expr.StringFuncs())`): +`[]any{"ann: good; ann: clear", "bob: ok"}`. + +--- + +## 12. entries, sort, and reverse + +`entries(m)` makes maps iterable through higher-order forms. `sort` and +`reverse` (from `CollectionFuncs`) sort and reverse lists. + +```go +// Format all response headers as "key: value", sorted by key. +map(entries(headers), e, sprintf("%s: %s", e.key, e.value)) +``` + +Env: + +```go +map[string]any{ + "headers": map[string]any{ + "content-type": "application/json", + "x-request-id": "abc123", + }, +} +``` + +Result: `[]any{"content-type: application/json", "x-request-id: abc123"}`. + +```go +// Keep only entries whose value exceeds a threshold. +filter(entries(scores), e, e.value > 80) +``` + +`sort` and `reverse` require `WithFunctions(expr.CollectionFuncs())`: + +```go +// Sort numbers ascending, then reverse for descending. +reverse(sort([3, 1, 2])) // → [3, 2, 1] +sort(["banana", "apple"]) // → ["apple", "banana"] +``` + +`sort` accepts all-numbers or all-strings; mixed types produce +`ErrEvaluate`. It never mutates the input and returns a fresh `[]any`. +`reverse` works on any list type and also returns a fresh copy. diff --git a/docs/guides/higher-order-patterns.md b/docs/guides/higher-order-patterns.md index 43207bf..2cca571 100644 --- a/docs/guides/higher-order-patterns.md +++ b/docs/guides/higher-order-patterns.md @@ -1,40 +1,85 @@ # Higher-order patterns -`map`, `filter`, `any`, `all`, `find`, `count` are the closest thing -expr has to control flow over collections. There's no `for` and no -`let`. These six forms, plus the lazy `if(cond, t, f)` special form -(only the selected branch evaluates) and Go's short-circuit `&&` / -`||`, are how you make decisions and shape data. This guide walks the idioms that come up most often and the -ones you have to work around. +`map`, `filter`, `flatMap`, `any`, `all`, `find`, `count`, and `sortBy` +are the closest thing expr has to control flow over collections. There +is no `for` and no `let`. These eight forms, plus the lazy `if(cond, t, +f)` special form (only the selected branch evaluates) and Go's +short-circuit `&&` / `||`, are how you make decisions and shape data. +This guide walks the idioms that come up most often and the ones you +have to work around. A runnable companion lives in [`../../examples/higher_order_patterns/`](../../examples/higher_order_patterns/). ## The shape of a higher-order form +Every iterating form accepts two call shapes: + ``` -form(list, predicate_or_transform) +form(list, body) // two-arg: binds `it` and `index` +form(list, name, body) // three-arg: binds `name` and `index` ``` -- `list` must be a slice, array, or `nil`. **Maps are not iterated.** - To iterate a map, drive with `keys(m)` and index into `m[k]` inside - the predicate. -- The second argument is an **unevaluated AST** that the form - re-evaluates once per element. Inside that body, `it` is the - current element and `index` is its 0-based position. Both shadow - any outer identifier of the same name. +- `list` must be a slice, array, or `nil`. **Maps are not iterated + directly.** Use `keys(m)` to iterate keys, or `entries(m)` to iterate + key-value pairs. +- In the two-arg form, `it` is the current element and `index` is its + 0-based position. Both shadow any outer identifier of the same name. +- In the three-arg form, the second argument is the element binding name. + Only the chosen name and `index` are bound inside the body; `it` is + **not** bound, so an enclosing scope's `it` remains visible. + +These forms are **always registered**. `WithBuiltins()` is not required. +You can shadow any of them by registering your own function of the same +name, but you lose the per-element re-evaluation. + +## Named element bindings + +The three-arg form solves a problem the two-arg form cannot: nested +forms where you need to refer to the outer element by name from inside +an inner body. + +With the two-arg form only, the inner body shadows the outer `it`: + +``` +// Inside the inner map, `it` is a comment, not a review. +// There is no way to refer to the review from inside this body. +map(reviews, map(it.comments, it)) // it = comment here, review is gone +``` -These forms are **always registered**. `WithBuiltins()` is not -required. You can shadow them by registering your own function of -the same name, but you lose the per-element re-evaluation — see -[registering-functions.md](registering-functions.md). +With named bindings, you choose which name is visible where: + +``` +// Outer two-arg, inner named: outer `it` (the review) stays visible +// because the inner named form does not bind `it`. +map(reviews, map(it.comments, c, it.author + "/" + c)) +// ^^ outer `it` = review ^^ inner `c` = comment + +// Outer named, inner two-arg: `r` (the review) is visible inside +// the inner body alongside inner `it` (the comment). +map(reviews, r, join(map(r.comments, r.author + "/" + it), ",")) +// ^^ binds r ^^ inner `it` = comment +``` + +Named bindings shadow env names and outer bindings of the same name. +An inner form that reuses a name hides the outer one for its body: + +``` +map(users, u, map(u.orders, u, u)) // inner u shadows outer u +``` + +### Reserved binding names + +You cannot use `it`, `index`, `true`, `false`, `nil`, `map`, or `if` +as a binding name. Any of these produces an `ErrEvaluate:
+binding cannot be named ""`. A non-identifier in the name +position produces ` binding must be a plain identifier, got ...`. ## Validation bags -The canonical shape for "run a list of rules over an input and -collect the ones that failed." Every rule is a -`{ok, msg}` literal, you filter for failures, then project to the -message. +The canonical shape for "run a list of rules over an input and collect +the ones that failed." Every rule is a `{ok, msg}` literal, you filter +for failures, then project to the message. ``` { @@ -53,11 +98,10 @@ message. } ``` -Why this beats a flat `&&` chain: you get the failing reasons in -order, every rule is a one-line edit, and adding a check is adding a -row. The top-level `ok` still short-circuits — the engine doesn't -build the error list unless you ask for it in the same composite -literal. +Why this beats a flat `&&` chain: you get the failing reasons in order, +every rule is a one-line edit, and adding a check is adding a row. The +top-level `ok` still short-circuits: the engine doesn't build the error +list unless you ask for it in the same composite literal. This pattern generalizes to anywhere you want "a list of possibly-failing predicates with associated data." For example, validation with @@ -66,8 +110,8 @@ severities: `{"ok": ..., "severity": "warn", "msg": "..."}`. ## Summary objects A single composite literal with many `count(...)` / `any(...)` / -`find(...)` calls is the idiomatic way to build a stats object from -a list: +`find(...)` calls is the idiomatic way to build a stats object from a +list: ``` { @@ -80,11 +124,10 @@ a list: } ``` -Every field is its own pass over the list, but the engine evaluates -them in order inside the enclosing composite literal, so it reads -like a single declarative summary. If you care about doing it in one -pass, register a Go function that takes the list and returns the -summary — don't fight expr. +Every field is its own pass over the list, but the engine evaluates them +in order inside the enclosing composite literal, so it reads like a +single declarative summary. If you care about doing it in one pass, +register a Go function that takes the list and returns the summary. ## Filter, then map @@ -95,23 +138,21 @@ the field I care about" shape: map(filter(orders, it.status == "paid"), it.id) ``` -Reads as "the id of every paid order." Composes cleanly to any depth: +Reads as "the id of every paid order." Named bindings make the elements +explicit when nesting gets deep: ``` map( - filter(users, it.active && len(it.roles) > 0), - upper(it.name), + filter(orders, o, o.status == "paid" && o.total > 100), + o, + {o.id: o.total}, ) ``` -The `it` in the outer `map` refers to *the element that passed the -inner `filter`* — there is no confusion because the inner `filter` -has already been reduced to a value before the outer `map` runs. - ## Nested forms and the `it` rebinding rule -Inside a nested higher-order form, `it` and `index` always refer to -the **innermost** form's current element. The outer binding is gone. +Inside a nested two-arg higher-order form, `it` and `index` always refer +to the **innermost** form's current element. The outer binding is gone. ``` count(orders, it.status == "paid" && count(it.items, it.price >= 100) >= 2) @@ -120,20 +161,105 @@ count(orders, it.status == "paid" && count(it.items, it.price >= 100) >= 2) The outer `it.status` runs before the inner `count(it.items, ...)` starts, so the two references don't collide. But inside the inner -predicate, `it` is an item, not an order. There is no way to spell -"outer it" from inside the inner body. +predicate, `it` is an item, not an order. + +Named bindings solve this when you need the outer element inside an +inner body: + +``` +// Before: no way to reference the review from inside the inner body. +map(reviews, map(it.comments, it)) + +// After: r is the review, it is the comment. +map(reviews, r, map(r.comments, c, sprintf("%s: %s", r.author, c))) +``` + +## flatMap: flatten and collect + +`flatMap(xs, body)` is like `map`, but body results that are lists get +spliced into the output element-by-element. Use it to flatten a +collection of collections, or to expand each element into zero or more +output elements. + +``` +// Flatten orders-per-user into a single order list. +flatMap(users, u, u.orders) + +// Two-arg form: same thing using `it`. +flatMap(users, it.orders) +``` + +Splicing is one level deep only: + +``` +flatMap([1, [2, 3], 4], it) // → [1, 2, 3, 4] +flatMap([[1, [2]], [3]], it) // → [1, [2], 3] (inner list kept whole) +``` + +`nil` body results splice as nothing (the nil-is-an-empty-list rule): + +``` +flatMap([1, 2, 3], if(it > 1, [it, it], nil)) // → [2, 2, 3, 3] +``` + +Strings are never split into runes; they append as a single element: + +``` +flatMap(["ab", "c"], it) // → ["ab", "c"] +``` + +## sortBy: stable sort by key + +`sortBy(xs, key)` evaluates the key expression once per element and +returns a **new** list sorted in ascending order. The sort is stable: +elements with equal keys preserve their input order. The input list is +never mutated. + +``` +// Sort orders by total, ascending. +sortBy(orders, it.total) + +// Named binding form. +sortBy(orders, o, o.total) +``` + +Keys must be all numbers (any int/float mix) or all strings. Mixed or +non-comparable key types produce an `ErrEvaluate` naming the offending +element. Combined with `reverse` (from `CollectionFuncs`): + +``` +// Descending sort. +reverse(sortBy(orders, o, o.total)) +``` + +## Iterating maps with `entries` + +`entries(m)` (from `WithBuiltins`) returns the key-value pairs of a +string-keyed map as a sorted `[]any`, each element being +`map[string]any{"key": k, "value": v}`. It makes maps iterable through +all the higher-order forms: + +``` +// Format all headers as "key: value". +map(entries(headers), e, sprintf("%s: %s", e.key, e.value)) + +// Keep only entries whose score is above 80. +filter(entries(scores), e, e.value > 80) + +// Sort a map's entries by value. +sortBy(entries(scores), e, e.value) +``` -## Why there's no `let`, and what to do instead +## Why there is no `let`, and what to do instead -People routinely ask for `let` or `with` to bind an intermediate -value. expr doesn't have it, by design — every `let` would add a -scope, and scopes are where expression languages start growing -teeth. There are three workarounds depending on what you need. +Named bindings scoped to a single form body cover the main pain point +without general scoping machinery: the `o` in `filter(orders, o, ...)` is +a one-form binding, not a declaration that leaks into siblings. If you +need a value bound across a whole expression or across siblings of a +composite literal, the answer is still to move it outside the expression. -**1. Duplicate the sub-expression.** If the value is cheap, just -write it twice. The AST walker has per-element caching for many -common subexpressions, but even without that, two reads of -`user.profile.name` is usually fine. +**1. Duplicate the sub-expression.** If the value is cheap, write it +twice: ``` { @@ -142,10 +268,9 @@ common subexpressions, but even without that, two reads of } ``` -**2. Register a helper.** If you need the value bound across a whole -subtree (a join on the outer element from inside a nested form, for -example), register a Go function that takes both and returns the -computed shape. +**2. Register a helper.** If you need to combine the outer and inner +elements in a way named bindings can't help with (e.g., you need both +elements as arguments to a Go function), register the helper: ```go expr.WithFunctions(map[string]any{ @@ -158,8 +283,7 @@ ordersFor(orders, user.id) ``` **3. Pre-compute in Go.** Move the binding out of the expression -entirely. If your template needs `user.profile.name` three times, -add it as an env key: +entirely: ```go env := map[string]any{ @@ -169,10 +293,8 @@ env := map[string]any{ } ``` -Expressions read `name` and `isAdmin` directly. You lose nothing -except the ability to change the derivation from inside an -expression — and if that mattered, you probably didn't want to -precompute. +Expressions read `name` and `isAdmin` directly. You lose nothing except +the ability to change the derivation from inside an expression. ## `any` / `all` short-circuit, `count` does not @@ -181,35 +303,34 @@ precompute. - `filter` and `map` run the predicate on every element. - `count` runs the predicate on every element (no early exit). - `find` returns as soon as a match is found. +- `flatMap` and `sortBy` evaluate the body/key on every element. -If you need "at least two matches," `count(list, pred) >= 2` is -correct but iterates the whole list. For large lists where the -cutoff matters, write the check as a function and register it. +If you need "at least two matches," `count(list, pred) >= 2` is correct +but iterates the whole list. ## Empty-list behavior -| Form | On empty list | -| --------------------- | ------------------------------------ | -| `map(xs, it)` | `[]any{}` (empty slice) | -| `filter(xs, pred)` | `[]any{}` | -| `any(xs, pred)` | `false` | -| `all(xs, pred)` | `true` (vacuously) | -| `find(xs, pred)` | `nil` | -| `count(xs, pred)` | `0` | - -`all([]) == true` is the usual mathematical convention and usually -what you want for "every X must be valid" over a possibly-empty list. -If you need "non-empty and all valid," spell it out: +| Form | On empty or nil list | +| ---------------------- | ---------------------------- | +| `map(xs, it)` | `[]any{}` (empty slice) | +| `filter(xs, pred)` | `[]any{}` | +| `flatMap(xs, it)` | `[]any{}` | +| `any(xs, pred)` | `false` | +| `all(xs, pred)` | `true` (vacuously) | +| `find(xs, pred)` | `nil` | +| `count(xs, pred)` | `0` | +| `sortBy(xs, key)` | `[]any{}` | + +`all([]) == true` is the usual mathematical convention and usually what +you want for "every X must be valid" over a possibly-empty list. If you +need "non-empty and all valid," spell it out: `len(xs) > 0 && all(xs, pred)`. ## When to stop reaching for higher-order -expr's higher-order set covers single-pass list operations. It does -**not** cover grouping, sorting, or zipping. If you find yourself -writing a `sort`-shaped expression, register a `sortBy` in Go and -call it — see example 8 in [examples.md](examples.md). Same for -group-by, reduce with accumulator, and anything that needs to bind -values across iterations. +expr's higher-order set covers single-pass list operations and sorting. +It does **not** cover grouping, zipping, or reduce with an accumulator. +If you find yourself needing those, register a Go function and call it. -The rule: if it fits in one pass with a per-element predicate, use -the forms. Otherwise, register a Go function. +The rule: if it fits in one pass with a per-element predicate or key, +use the forms. Otherwise, register a Go function. diff --git a/docs/guides/templates.md b/docs/guides/templates.md index d903377..81fd452 100644 --- a/docs/guides/templates.md +++ b/docs/guides/templates.md @@ -3,10 +3,10 @@ `NewTemplate` is the smallest string interpolator that still pulls its weight. You write `${...}` around an expression, the template compiles every expression **once** at construction time, and `Render` walks the -pre-compiled segments against an env. This guide is about using it -well — the patterns, the stringification rules, and the places where -it pays to preprocess in Go instead of cramming everything inside -`${...}`. +pre-compiled segments against an env. This guide is about using it well: +the patterns, the stringification rules, custom delimiters, formatters, +and the places where it pays to preprocess in Go instead of cramming +everything inside `${...}`. A runnable companion lives in [`../../examples/templates_in_anger/`](../../examples/templates_in_anger/). @@ -27,77 +27,176 @@ for _, user := range users { ``` Every `${...}` body is parsed and compiled during `NewTemplate`. Each -`Render` is pure AST walking against the fresh env — no parsing, no +`Render` is pure AST walking against the fresh env: no parsing, no reflection on the template shape itself. If your template is -request-scoped, cache it. If it's static, compile at package init: +request-scoped, cache it. If it is static, compile at package init: ```go var orderConfirm = mustTemplate(`Order ${id}: ${len(items)} item(s) for $${total}`) ``` (`$$` escapes to a literal `$`, so `$${total}` emits the literal `$` -followed by the interpolated `total`. That's how you print a dollar +followed by the interpolated `total`. That is how you print a dollar sign without confusing the template parser.) ## How `${...}` stringifies -Each `${...}` result is converted to a string with these rules: +Each `${...}` result is converted to a string with these rules (a +custom formatter from `WithTemplateFormatter` runs first and can +override any of them): -| Result type | Output | -| ----------- | ----------------------------------------------- | -| `nil` | empty string | -| `string` | passthrough | -| anything else | `fmt.Sprintf("%v", v)` | +| Result type | Output | +| ---------------------------- | ------------------------------------------------------- | +| `nil` | empty string | +| `string` | passthrough unchanged | +| map, slice, array, struct | compact JSON (see below) | +| everything else | `fmt.Sprintf("%v", v)` | -The `nil` → empty-string rule is deliberate: optional fields that +**JSON rendering for composite values.** Maps, slices, arrays, and +structs (following pointers) now render as compact JSON with HTML +escaping disabled, so `&`, `<`, and `>` survive intact in webhook +payloads and prompts: + +``` +${config} // was: map[retries:3 timeout:30s] + // now: {"retries":3,"timeout":"30s"} + +${files} // was: [a.go b.go] + // now: ["a.go","b.go"] +``` + +A composite that marshals to a JSON string (e.g. `time.Time`, or a +type with a custom `MarshalJSON`) renders as the unquoted string rather +than as a JSON string literal with surrounding quotes: + +``` +${createdAt} // time.Time value renders as: 2026-06-12T09:30:00Z +``` + +Values JSON cannot encode (cycles, channels, functions) fall back to +`fmt.Sprintf("%v", v)` rather than erroring. + +**This is a behavior change from earlier releases.** If you relied on +the previous `fmt.Sprintf("%v", v)` rendering for composite values, use +`WithTemplateFormatter` to restore the old behavior for the types you +need: + +```go +expr.WithTemplateFormatter(func(v any) (string, bool) { + if m, ok := v.(map[string]any); ok { + return fmt.Sprintf("%v", m), true + } + return "", false +}) +``` + +The `nil` to empty-string rule is deliberate: optional fields that resolve to `nil` produce no output rather than the literal `""`. -Matches Jinja / Liquid / Handlebars convention. Downside: a template -can't distinguish "value was nil" from "value was the empty string" -in its output. If you need to, emit a sentinel in the expression: +This matches Jinja/Liquid/Handlebars convention. A template cannot +distinguish "value was nil" from "value was the empty string" in its +output. If you need that distinction, emit a sentinel: ``` Nickname: ${if(user.nickname == nil, "(none)", user.nickname)} ``` -`if(cond, t, f)` is the canonical ternary in expr. It is lazy — -only the branch the condition selects evaluates — so guards like -`${if(n != 0, total/n, 0)}` are safe. `try(...)` and -operand-returning `||` remain useful when the *condition itself* -might error. +`if(cond, t, f)` is the canonical ternary in expr. It is lazy: only +the branch the condition selects evaluates, so guards like +`${if(n != 0, total/n, 0)}` are safe. -## The list-stringification footgun +## Custom delimiters: `WithTemplateDelimiters` -`fmt.Sprintf("%v", []any{...})` produces Go's slice syntax: -`[a b c]`. That's almost never what you want in a user-facing -template. Two ways to handle it: +When the default `${` / `}` collides with your surrounding text (shell +scripts, JavaScript template literals, Kubernetes YAML with `${...}` +references), switch delimiters per template: + +```go +tmpl, err := expr.NewTemplate(src, expr.WithTemplateDelimiters("${{", "}}")) +``` -**Option 1 — build the final string inside the expression** with -`sprintf` and friends. Good for small, fixed-shape lists: +With `${{ }}`, text like `echo ${HOME}` passes through as a literal. +Only `${{ ... }}` is treated as an expression: ``` -${sprintf("%s, %s, and %s", items[0], items[1], items[2])} +Deploy ${{ service.name }} via: echo ${HOME} +// renders: Deploy api via: echo ${HOME} ``` -But expr has no `join` builtin and no loop, so arbitrary-length lists -don't work this way. +Delimiter rules: + +- The opener must end with one or more `{`; the closer must be the + matching `}` run. `${{` requires `}}`. +- The opener may not contain `$$` (collides with the escape). +- The `$$` escape applies only when the opener starts with `$`. + With `${{`, `$${{foo}}` emits the literal text `${{foo}}`. +- With a non-`$` opener like `{{`, no `$$` escape applies and `$$` + passes through as two literal dollar signs. + +Passing `WithTemplateDelimiters` to `Compile` fails with `ErrCompile`. -**Option 2 — preprocess in Go** and pass the joined string back in -through the env. This is the idiomatic answer for variable-length -lists: +## Custom value formatter: `WithTemplateFormatter` + +Install a hook that runs first for every interpolated result, including +`nil` and strings. Return `false` to fall through to the default chain: ```go -env := map[string]any{ - "user": user, - "taskList": strings.Join(titles(user.OpenTasks), ", "), -} +tmpl, err := expr.NewTemplate(src, + expr.WithBuiltins(), + expr.WithTemplateFormatter(func(v any) (string, bool) { + switch x := v.(type) { + case time.Time: + return x.Format("Jan 2 2006"), true + case nil: + return "N/A", true + } + return "", false + }), +) +``` + +Passing `WithTemplateFormatter` to `Compile` fails with `ErrCompile`. + +## Error messages + +Runtime errors from `Render` include a 1-based `line:column` (byte +columns) with the offset as supplementary detail: + ``` +template: evaluating ${boom} at 3:3 (offset 21): expr: evaluate error: ... +``` + +Parse-time errors (empty expression, invalid expression, unclosed opener) +use the same format: ``` -Hi ${user.name}! Open tasks: ${taskList}. +template: empty expression `${}` at 2:1 (offset 9) +template: invalid expression `${x +}` at 2:3 (offset 11): ... +template: unclosed `${` at 3:1 (offset 18, missing `}`) ``` -A template's job is to interpolate values. A Go function's job is -data shaping. When in doubt, reach for Go. +## Inspecting segments: `Template.Segments()` + +`Segments()` returns the parsed segments in source order as +`[]TemplateSegment`. Each segment carries: + +- `Literal` and `Source`: one is non-empty (literal text vs. expression + body); never both. +- `Offset`, `Line`, `Column`: 1-based position in the raw template. +- `Program`: the compiled `*Program` for expression segments, `nil` for + literals. Call `Program.Identifiers()` to get the env references for + that expression alone: useful for editor hints, live validation, and + variable dependency tracking. + +```go +tmpl, _ := expr.NewTemplate("Hi ${user.name}, you have ${count} items") +for _, seg := range tmpl.Segments() { + if seg.Program != nil { + fmt.Printf("expr at %d:%d: %s (needs: %v)\n", + seg.Line, seg.Column, seg.Source, + seg.Program.Identifiers()) + } +} +``` ## Multi-line expressions inside `${...}` @@ -115,18 +214,27 @@ Total: ${ The parser keeps reading until the closing `}`, using `go/scanner` to track brace depth, so expression bodies with JSON literals or nested -calls work without extra escaping. +calls work without extra escaping. Multi-line bodies work with custom +delimiters too: + +```go +tmpl, _ := expr.NewTemplate( + "header\n${{\n join(names, \", \")\n}}\nfooter", + expr.WithTemplateDelimiters("${{", "}}"), + expr.WithFunctions(expr.StringFuncs()), +) +``` ## When to register a helper instead -Three signs that you're pushing the template too hard: +Three signs that you are pushing the template too hard: -1. You're writing the same non-trivial expression in multiple - templates. Register a Go function and call it from both. -2. You need a `join` or `format-date` or `pluralize`. Not present by +1. You are writing the same non-trivial expression in multiple templates. + Register a Go function and call it from both. +2. You need a `join`, `format-date`, or `pluralize`. Not present by default; register what you need. 3. The `${...}` body is longer than the literal text around it. The - template is now a wrapper around an expression — just use + template is now a wrapper around an expression: just use `Program.Run` directly and format in Go. A useful helper set for text templates, registered once at startup: @@ -150,32 +258,27 @@ var textOpts = []expr.Option{ } ``` -Now your templates can say -`${pluralize(count, "task", "tasks")}` and -`${join(names, ", ")}` without resorting to in-Go preprocessing. Trade -gain: every template that uses these options pays for the full set. -Keep helpers cheap and pure. +Now your templates can say `${pluralize(count, "task", "tasks")}` and +`${join(names, ", ")}` without resorting to in-Go preprocessing. ## Errors from `Render` -A failed expression inside a `${...}` body bubbles up through -`Render` as an `ErrEvaluate`-wrapped error that names the original -source of the expression. Other segments before the failure are still -evaluated — but the final output is discarded on error, so `Render` -is either "here's the full rendered string" or "here's an error," not -"here's a partially rendered string." +A failed expression inside a `${...}` body bubbles up through `Render` +as an `ErrEvaluate`-wrapped error that names the original source of the +expression. Other segments before the failure are still evaluated, but +the final output is discarded on error, so `Render` is either "here is +the full rendered string" or "here is an error," not "here is a +partially rendered string." ## A few patterns worth knowing -- **Boolean flags:** `${admin && " (admin)"}` emits - `" (admin)"` when truthy and `false` otherwise; `false` renders as - `false` via `%v`, which is usually what you want for debugging but - wrong for user-facing text. Use `if` to give the false branch an +- **Boolean flags:** `${admin && " (admin)"}` emits `" (admin)"` when + truthy and `false` otherwise; use `if` to give the false branch an explicit value: `${if(admin, " (admin)", "")}`. - **Counts with the right singular/plural:** register a `pluralize` helper, or compute the label in Go and pass it in. -- **Currency:** format in Go. Templates are not the right place to - deal with locale-aware money formatting. +- **Currency:** format in Go. Templates are not the right place for + locale-aware money formatting. - **Escaping HTML:** expr templates do no escaping. If your output is HTML, run the result through `html/template` or escape inside a registered helper. diff --git a/docs/reference/spec.md b/docs/reference/spec.md index 26e0e62..69bd489 100644 --- a/docs/reference/spec.md +++ b/docs/reference/spec.md @@ -332,6 +332,7 @@ The standard set is: | `contains(h,n)` | `(any, any) -> bool, error` | Substring for string haystacks, element membership for slices/arrays (using [loose equality](#equality)), key presence for string-keyed maps. | | `has(m,k)` | `(any, string) -> bool, error` | True if map `m` has key `k`. Maps only. Nil → `false`. | | `keys(m)` | `(any) -> []any, error` | Sorted string keys. Other key types → error. | +| `entries(m)` | `(any) -> []any, error` | Sorted key-value pairs of a string-keyed map. Each element is `map[string]any{"key": k, "value": v}`. Nil → `nil`. Other key types → error. Useful for iterating maps through the higher-order forms: `map(entries(m), e, e.key + "=" + e.value)`. | | `lower(s)` | `(string) -> string` | `strings.ToLower`. | | `upper(s)` | `(string) -> string` | `strings.ToUpper`. | | `sprintf(f,...)`| `(string, ...any) -> string` | `fmt.Sprintf`. | @@ -394,42 +395,135 @@ p, err := expr.Compile(src, | `last(xs)` | `(list) -> any, error` | Last element; `nil` for nil or empty lists. | | `sum(xs)` | `(list) -> num, error` | Numeric sum. `int64` (overflow-checked) when every element is integral, `float64` otherwise. Nil/empty → `0`. | | `slice(xs, i, j)` | `(list\|string, int, int) -> list\|string, error` | Half-open range `[i, j)`. Lists yield `[]any`; strings slice by rune. Negative indices count from the end; out-of-range bounds clamp; `i > j` → empty. Covers the rejected `xs[i:j]` syntax. | +| `sort(xs)` | `(list) -> []any, error` | Ascending stable sort. All elements must be numbers (any int/float mix, compared numerically) or all strings (lexicographic). Elements are not converted: ints stay ints. Nil/empty → `[]any{}`. Mixed or non-comparable types → error. Never mutates the input. | +| `reverse(xs)` | `(list) -> []any, error` | Reversed copy. Never mutates the input. Nil/empty → `[]any{}`. | ## Higher-order special forms -expr also provides a fixed set of **special forms**. Unlike the -standard builtins, special forms are always registered and do not -require `WithBuiltins`. They look like ordinary function calls in -source, but their arguments are not all evaluated eagerly. Six of -them iterate lists (`map`, `filter`, `any`, `all`, `find`, `count`); -`try` and `if` instead use laziness for error recovery and -branching. For the iterating forms, the second argument (the -predicate) is re-evaluated once per element with two extra -identifiers in scope: - -- `it` — the current element -- `index` — the 0-based position as an `int64` - -Inside the predicate, `it` and `index` shadow any identifier of the -same name from the outer env. Nested forms nest naturally: -`map(matrix, map(it, it * 10))` binds the inner `it` to each inner -element and the outer `it` is no longer reachable until the inner -`map` returns. - -| Name | Returns | Description | -| -------------------- | -------------------------- | ----------- | -| `map(list, expr)` | `[]any` | New list with `expr` evaluated per element. | -| `filter(list, pred)` | `[]any` | Elements where `pred` is truthy, in original order. | -| `any(list, pred)` | `bool` | `true` if `pred` is truthy for any element; short-circuits. | -| `all(list, pred)` | `bool` | `true` if `pred` is truthy for every element; short-circuits. Empty list → `true`. | -| `find(list, pred)` | element or `nil` | First element for which `pred` is truthy, or `nil`. | -| `count(list, pred)` | `int64` | Number of elements for which `pred` is truthy. | -| `try(value, default)`| value or `default` | Evaluates `value`; returns `default` if `value` raised an `ErrEvaluate` (missing key, type error, out-of-range index, etc.). The `default` expression is only evaluated when the primary fails. | -| `if(cond, then, else)`| `then` or `else` value | Lazy three-argument selector: evaluates `cond`, then **only** the branch selected by `cond`'s [truthiness](#truthiness). Binds no `it`/`index`. | +expr provides a fixed set of **special forms**. Unlike the standard +builtins, special forms are always registered and do not require +`WithBuiltins`. They look like ordinary function calls in source, but +their arguments are not all evaluated eagerly. Eight of them iterate +lists (`map`, `filter`, `flatMap`, `any`, `all`, `find`, `count`, +`sortBy`); `try` and `if` instead use laziness for error recovery and +branching. + +### Two-arg vs. three-arg iterating forms + +Every iterating form accepts two call shapes: + +``` +form(collection, body) // two-arg: binds `it` and `index` +form(collection, name, body) // three-arg: binds `name` and `index` +``` + +**Two-arg form.** The body is re-evaluated once per element with `it` +(current element) and `index` (0-based position as `int64`) in scope. +Both shadow any outer identifier of the same name. + +**Three-arg form.** The second argument is the element binding name. +It must be a plain identifier. Only the chosen name and `index` are +bound inside the body; `it` is **not** bound, so an enclosing +two-arg form's `it` remains reachable from inside the body. This +closes the gap where nested `it`-based forms offered no way to refer +to the outer element. + +Example: outer named form, inner two-arg — `r` is the outer element, +`it` is the inner element from the two-arg `map`: + +``` +map(reviews, r, join(map(r.comments, r.author + "/" + it), ",")) +``` + +Example: outer two-arg, inner named — outer `it` stays visible from +inside the named form body because the named form does not bind `it`: + +``` +map(reviews, map(it.comments, c, it.author + "/" + c)) +``` + +Bindings shadow env names lexically. An inner named form whose name +matches an outer named form's name shadows the outer one for the +duration of its own body: + +``` +map(users, u, map(u.orders, u, u)) // inner u shadows outer u +``` + +### Reserved binding names + +The following identifiers may not be used as a binding name in the +three-arg form. Any of them produces `ErrEvaluate: binding +cannot be named ""`: + +- `it`, `index` (already have special meaning in the two-arg form) +- `true`, `false`, `nil` (reserved literals) +- `map`, `if` (Go keyword rewrites used internally by expr) + +A non-identifier in the name position (selector, call, parenthesized +expression) produces `ErrEvaluate: binding must be a plain +identifier, got `. Wrong arity produces `ErrEvaluate: +expects 2 arguments (collection, predicate) or 3 (collection, name, +predicate), got N`. + +### Form table + +| Name | Returns | Description | +| -------------------------- | ---------------- | ----------- | +| `map(list, expr)` | `[]any` | New list with `expr` evaluated per element. | +| `map(list, name, expr)` | `[]any` | Same, binding element as `name`. | +| `filter(list, pred)` | `[]any` | Elements where `pred` is truthy, in original order. | +| `filter(list, name, pred)` | `[]any` | Same, binding element as `name`. | +| `flatMap(list, expr)` | `[]any` | Like `map`, but a list body result is spliced element-by-element; nil splices as nothing; non-list appends as one element. Splicing is one level deep only. Strings are never split to runes. | +| `flatMap(list, name, expr)`| `[]any` | Same, binding element as `name`. | +| `any(list, pred)` | `bool` | `true` if `pred` is truthy for any element; short-circuits. | +| `any(list, name, pred)` | `bool` | Same, binding element as `name`. | +| `all(list, pred)` | `bool` | `true` if `pred` is truthy for every element; short-circuits. Empty list → `true`. | +| `all(list, name, pred)` | `bool` | Same, binding element as `name`. | +| `find(list, pred)` | element or `nil` | First element for which `pred` is truthy, or `nil`. | +| `find(list, name, pred)` | element or `nil` | Same, binding element as `name`. | +| `count(list, pred)` | `int64` | Number of elements for which `pred` is truthy. | +| `count(list, name, pred)` | `int64` | Same, binding element as `name`. | +| `sortBy(list, key)` | `[]any` | Stable sort of a copy of list by the key expression. Keys must be all numbers or all strings. | +| `sortBy(list, name, key)` | `[]any` | Same, binding element as `name`. | +| `try(value, default)` | value or default | Evaluates `value`; returns `default` if `value` raised an `ErrEvaluate`. The `default` is only evaluated when the primary fails. | +| `if(cond, then, else)` | `then` or `else` | Lazy ternary: evaluates `cond`, then only the branch selected by `cond`'s [truthiness](#truthiness). | The `list` argument must be a slice or array (or `nil`, which is treated as empty). Maps are not iterated by these forms; use -`keys(m)` (from `WithBuiltins`) to drive a map iteration manually. +`keys(m)` or `entries(m)` (both in `WithBuiltins`) to drive map +iteration manually. + +### flatMap splicing rules + +`flatMap(xs, body)` / `flatMap(xs, name, body)`: + +- Body result is `[]any` or a typed slice/array: each element is + appended individually to the output (splice). +- Body result is `nil`: nothing is appended (nil is treated as an + empty list, matching `iterItems`). +- Body result is any other value, including a string: appended as a + single element. Strings are never split into runes. +- Splicing is one level deep only: `flatMap([[1, [2]], [3]], it)` + yields `[1, [2], 3]`, not `[1, 2, 3]`. + +### sortBy key comparison rules + +`sortBy` evaluates the key expression once per element, then sorts a +copy of the list using a stable sort. Key comparison follows the same +rules as the `<` operator and `sort`: + +- All numbers (any int/float mix): both-integral values compare as + `int64`; any float in either operand promotes both to `float64`. +- All strings: lexicographic order. +- Mixed or non-comparable key types: `ErrEvaluate` naming the + offending element and its type. The sort always returns a fresh copy; + the input is never mutated. + +Key-expression errors are reported like other predicate errors: +`sortBy predicate '' failed on element N: ...`. + +### Predicate error wrapping When a predicate raises an `ErrEvaluate`, the iterating forms wrap the error with the form name, the predicate's source text, and the @@ -447,10 +541,12 @@ typed it. The wrapping preserves the underlying error chain (`errors.Is(err, ErrEvaluate)` still matches); context cancellation passes through unchanged. -`try(value, default)` is the odd one out: it does not iterate a list -and binds no implicit `it`/`index`. Both arguments are arbitrary -expressions. The `default` is **only** evaluated when `value` failed, -so users can supply expensive or side-effecting fallbacks safely. +### `try` and `if` + +`try(value, default)` does not iterate a list and binds no implicit +`it`/`index`. Both arguments are arbitrary expressions. The `default` +is **only** evaluated when `value` failed, so users can supply +expensive or side-effecting fallbacks safely. `try` traps anything wrapping `ErrEvaluate`: missing fields/keys, nil selectors, out-of-range indices, type-coercion failures from `int`, @@ -478,18 +574,21 @@ if(user != nil, user.name, "?") // no nil-selector error All three arguments see the enclosing scope; `if` binds no implicit `it`/`index`. (`if` is a Go statement keyword, so like `map` it is -rewritten to an internal token before parsing — invisible except that +rewritten to an internal token before parsing, invisible except that it is why `if` can be called at all.) Special-form names can be shadowed: if `WithFunctions` registers a function with the same name, or the caller's env contains an entry with that name, the user binding wins. This lets consumers replace -the built-in behavior when they need to — note that a shadowed `if` -or `try` goes through the ordinary call path, where all arguments -evaluate eagerly. The `map` keyword is special because Go's parser -reserves it: expr rewrites `map` to an internal token before parsing -so the form can still be called as `map(xs, it * 2)`, and translates -it back for error messages and method lookups. +the built-in behavior when they need to. A shadowed `if` or `try` +goes through the ordinary call path, where all arguments evaluate +eagerly. Three-arg calls to shadowed forms are equally shadowed: a +user function named `flatMap` receiving three arguments is called as +an ordinary function with three evaluated arguments. The `map` +keyword is special because Go's parser reserves it: expr rewrites +`map` to an internal token before parsing so the form can still be +called as `map(xs, it * 2)`, and translates it back for error +messages and method lookups. ## Optional access (`?.` and `?[`) @@ -702,6 +801,156 @@ composite literals (`[]any{1, 2}`, `map[string]any{...}`, `[]int{}`, slice/index expressions like `xs[0]`, array types like `[3]int`) untouched, so expressions that never use bare literals are unaffected. +## Templates + +`NewTemplate` pre-compiles a `${...}` string interpolator. Every +`${...}` body is compiled once at construction time and re-evaluated +on each `Render` call. The same options accepted by `Compile` are +accepted by `NewTemplate`. + +```go +t, err := expr.NewTemplate("Hello ${user.name}!", expr.WithBuiltins()) +out, err := t.Render(ctx, env) +``` + +### Value rendering + +Each `${...}` result is converted to a string with these rules +(a custom formatter installed with `WithTemplateFormatter` runs first +and can override any of them): + +1. `nil` → empty string. Optional fields that resolve to `nil` silently + produce no output, matching Jinja/Liquid/Handlebars convention. +2. `string` → passthrough unchanged. +3. Maps, slices, arrays, and structs (following pointers) → compact JSON + with HTML escaping disabled, so `&`, `<`, and `>` survive intact. + A composite that marshals to a JSON string (e.g. `time.Time` with its + default `MarshalJSON`, or any custom marshaler) renders as the + unquoted string rather than as a JSON string literal. + Values JSON cannot encode (cycles, channels, functions) fall back to + `fmt.Sprintf("%v", v)`. +4. Everything else → `fmt.Sprintf("%v", v)`. + +**Behavior change from earlier versions.** In prior releases, composite +values rendered via `fmt.Sprintf("%v", v)`, producing Go syntax like +`map[retries:3]`. They now render as compact JSON: `{"retries":3}`. +Callers that relied on the old rendering must either install a +`WithTemplateFormatter` that replicates the old behavior, or update +their expected output. + +### Escaping + +`$$` is rewritten to a literal `$`. `$${name}` therefore emits the +literal text `${name}`. A bare `$` not followed by `$` or `{` is +emitted verbatim, so `$5` and `$foo` pass through unchanged. The `$$` +escape applies only when the configured opener starts with `$`. + +### Custom delimiters: `WithTemplateDelimiters(open, close)` + +Replaces `${` / `}` for a single `NewTemplate` call: + +```go +t, err := expr.NewTemplate(src, expr.WithTemplateDelimiters("${{", "}}")) +``` + +Rules: + +- The opener must end with one or more `{`; the closer must be the + matching `}` run. `${{` requires `}}`, `${{{` requires `}}}`. +- The opener may not contain `$$` (collides with the escape). +- The `$$` escape applies only when the opener starts with `$`. + With `WithTemplateDelimiters("${{", "}}")`, `$${{expr}}` emits + the literal `${{expr}}`. +- With `WithTemplateDelimiters("{{", "}}")`, no `$$` escape applies + and `$$` passes through as two literal dollar signs. + +GitHub-Actions-style `${{ expr }}` avoids collisions with shell +parameter expansion (`${HOME}`) and JavaScript template literals. + +Passing `WithTemplateDelimiters` to `Compile` fails with `ErrCompile`. + +### Custom formatter: `WithTemplateFormatter(fn)` + +Installs a custom value renderer that runs first for every +interpolated result, including `nil` and strings. Returning `false` +falls through to the default rendering chain described above: + +```go +expr.WithTemplateFormatter(func(v any) (string, bool) { + if t, ok := v.(time.Time); ok { + return t.Format(time.Stamp), true + } + return "", false +}) +``` + +Passing `WithTemplateFormatter` to `Compile` fails with `ErrCompile`. + +### Error messages + +Runtime errors from `Render` include a 1-based `line:column` +(byte-based columns) with the offset as supplementary detail: + +``` +template: evaluating ${boom} at 3:3 (offset 21): expr: evaluate error: ... +``` + +Parse-time errors (empty expression, invalid expression, unclosed +opener) carry the same `line:column (offset N)` format. + +### `Template.Segments()` + +Returns the parsed segments of the template in source order, as a +`[]TemplateSegment`. Each segment carries: + +```go +type TemplateSegment struct { + Literal string // non-empty for literal runs; empty for expressions + Source string // expression body text; empty for literals + Offset int // byte offset of the segment start in the raw template + Line int // 1-based line number + Column int // 1-based byte column + Program *Program // compiled expression; nil for literals +} +``` + +Literal segments have `Source == ""` and `Program == nil`. Expression +segments carry the compiled `*Program`, so hosts can call +`Program.Identifiers()` per segment for editor hints, live validation, +or variable extraction. For literal segments containing `$$` escapes, +`Literal` holds the decoded text. + +## `Program.Identifiers()` + +Returns the sorted, deduplicated set of top-level identifier names the +expression references through the environment. Hosts use it to validate +an expression against a known env shape at load time, to track +dependencies for cache invalidation, or to decide which values are worth +computing before a `Run`. + +Excluded from the result: + +- The literals `true`, `false`, and `nil`. +- `it` and `index` where they are bound by an enclosing iterating form + (two-arg form binds `it`; all forms bind `index`). +- The named element binding of a three-arg form: both the binding + identifier itself (the `o` in `filter(orders, o, o.paid)`) and all + references to that name inside the body are excluded. +- `it` inside a three-arg form body is **not** excluded: the three-arg + form does not bind `it`, so `it` inside such a body is a genuine env + reference (it would be bound by an enclosing two-arg form at run time, + or fail as undefined). +- Names registered via `WithFunctions` / `WithBuiltins`, which resolve + without the env. +- Special-form names (`map`, `filter`, `flatMap`, `try`, `if`, `sortBy`, + etc.) in call position, which are always available. + +The analysis is static and best-effort in one corner: env entries can +shadow registered functions and special forms at run time, so an +excluded name may still be read from the env when a host deliberately +shadows it. Every name that can only resolve through the env is always +included. + ## Error model All runtime failures wrap `ErrEvaluate`; all parse failures wrap diff --git a/docs/rfcs/0001-pipe-operator.md b/docs/rfcs/0001-pipe-operator.md new file mode 100644 index 0000000..0c8b357 --- /dev/null +++ b/docs/rfcs/0001-pipe-operator.md @@ -0,0 +1,597 @@ +# RFC 0001: Pipe Operator (`|`) + +**Status:** Draft +**Date:** 2026-06-12 +**No implementation commitment has been made. This document exists to think the design through.** + +--- + +## Summary + +Repurpose the bitwise-OR token `|` as a pipeline operator. `a | f(x, y)` +desugars at compile time to `f(a, x, y)`, threading the left operand as +the first argument of the right-side call. The desugar is purely an AST +rewrite; the evaluator and all existing behavior remain untouched. + +--- + +## 1. Motivation + +Expressions that chain several higher-order forms on a single collection +are the most common "hard to read" complaint from expr users. The nesting +grows from the outside in, the innermost operation is buried in the middle +of the source, and each closing parenthesis has to be mentally matched to +its opener: + +``` +join(map(filter(checks, !it.ok), sprintf("- %s: %s", it.name, it.msg)), "\n") +``` + +With a pipe operator the same logic reads left-to-right, matching the +order operations actually run: + +``` +checks | filter(!it.ok) | map(sprintf("- %s: %s", it.name, it.msg)) | join("\n") +``` + +A second common case is a chain of string or collection operations where +the data flows through several transforms: + +``` +split(replace(lower(trim(input)), " ", "_"), ",") +``` + +versus + +``` +input | trim() | lower() | replace(" ", "_") | split(",") +``` + +(The right side of each pipe must be a call expression, so +single-argument functions are written with empty parentheses: +`input | trim()` desugars to `trim(input)`. See section 5 for why a +bare `input | trim` is rejected.) + +Neither example is contrived. Both appear in real expr usage in the +codebase that motivated this RFC. The motivating case is a health-check +report renderer where the before version requires four levels of nesting +that obscure what is being computed. + +--- + +## 2. Semantics + +### 2.1 Basic desugar + +The pipe operator `|` desugars at compile time, during the AST +validation pass, before the evaluator ever sees the tree. The rule is: + +> `a | f(x, y, ...)` rewrites to `f(a, x, y, ...)` + +The left operand `a` is injected as the **first** argument of the +right-side call. The remaining arguments shift right. The result is an +ordinary `*ast.CallExpr` that the evaluator handles through the normal +function-call path. + +### 2.2 Chaining is left-associative + +Go's parser makes `|` left-associative at the same precedence level as +`+` and `-` (precedence 4). A chain like: + +``` +a | f() | g() +``` + +parses as: + +``` +(a | f()) | g() +``` + +The first rewrite produces `f(a)`, then the outer `|` desugars that +result as `g(f(a))`. Left-to-right order is preserved: `a` flows into +`f`, and `f`'s result flows into `g`. This matches the intuition that +reading left-to-right follows execution order. + +For the motivating example: + +``` +checks | filter(!it.ok) | map(sprintf("- %s: %s", it.name, it.msg)) | join("\n") +``` + +The chain desugars step by step as: + +``` +join(map(filter(checks, !it.ok), sprintf("- %s: %s", it.name, it.msg)), "\n") +``` + +which is exactly the nested form. + +### 2.3 Special forms receive the rewritten CallExpr + +Because the desugar happens at compile time before dispatch, special +forms (`filter`, `map`, `flatMap`, `any`, `all`, `find`, `count`, +`sortBy`, `try`, `if`) receive the rewritten `*ast.CallExpr` like any +other call. Their arity checks run against the post-rewrite argument +count. So: + +``` +checks | filter(!it.ok) +``` + +desugars to `filter(checks, !it.ok)`, which has two arguments and passes +the forms' argument validation (`splitFormArgs`) normally. The named +three-arg binding form composes the same way: `checks | filter(c, !c.ok)` +desugars to `filter(checks, c, !c.ok)`. The predicate argument remains in its +original AST position (now index 1 rather than 0); no special handling is +needed for `it`/`index` binding. + +The `try` and `if` forms work the same way. `value | try(fallback)` +desugars to `try(value, fallback)`, which is the standard two-argument +form. `cond | if(then, else)` is syntactically valid but semantically +odd; no special case is needed since `if(cond, then, else)` already does +what a user would expect. + +### 2.4 Non-call right-hand side + +`a | b` where `b` is not a call expression is rejected at compile time +with: + +``` +compile error: pipe operator | requires a function call on the right-hand side +``` + +See section 6 for the full error taxonomy. + +--- + +## 3. Precedence + +### 3.1 The table + +Go's precedence levels, and where `|` sits, are fixed by the parser: + +| Precedence | Operators | Associativity | +| ---------- | ---------------------------- | ------------- | +| 5 (high) | `* / %` | left | +| 4 | `+ - \| ^` | left | +| 3 | `== != < <= > >=` | left | +| 2 | `&&` | left | +| 1 (low) | `\|\|` | left | + +`|` is at precedence level 4, the same as `+` and `-`. Unary `!`, `-`, +`+` bind tighter than any binary operator. + +The key consequences: + +- `|` binds **tighter** than comparisons (`==`, `!=`, `<`, etc.). +- `|` binds **tighter** than logical operators (`&&`, `||`). +- `|` binds at the **same level** as `+` and `-`, left-associative. + +### 3.2 Worked examples + +**Example 1: pipe with comparison** + +``` +checks | filter(!it.ok) == []any{} +``` + +Parses as: + +``` +(checks | filter(!it.ok)) == []any{} +``` + +which desugars to: + +``` +filter(checks, !it.ok) == []any{} +``` + +This is probably what the user meant: compare the filtered result to an +empty list. The precedence works in the user's favor here. + +**Example 2: pipe on the right of a comparison** + +``` +len(errors) == 0 | f() +``` + +Parses as: + +``` +len(errors) == (0 | f()) +``` + +which desugars to: + +``` +len(errors) == f(0) +``` + +This is almost certainly not what the user meant. They likely wanted +`(len(errors) == 0) | f()`, i.e., pipe the boolean result into `f`. The +silent misparsing is a trap. + +**Example 3: pipe and `&&`** + +``` +active | filter(it.enabled) && len(errors) == 0 +``` + +Parses as: + +``` +(active | filter(it.enabled)) && (len(errors) == 0) +``` + +which desugars to: + +``` +filter(active, it.enabled) && (len(errors) == 0) +``` + +This is the natural reading: "the filtered list, and the error count is +zero." The precedence works correctly because `&&` binds looser than `|`. + +**Example 4: pipe with unary `!`** + +``` +checks | filter(!it.ok) | any(it.severity == "critical") +``` + +Parses as: + +``` +((checks | filter(!it.ok)) | any(it.severity == "critical")) +``` + +which desugars to: + +``` +any(filter(checks, !it.ok), it.severity == "critical") +``` + +The unary `!` inside the filter predicate binds to `it.ok` before any +binary operator is considered, so the pipe sees `filter(!it.ok)` as a +complete call. This is correct and unsurprising. + +### 3.3 Recommendation: compile error for suspicious mixes + +Example 2 demonstrates a real trap: when `|` appears as an operand of a +comparison without parentheses, the user almost certainly did not intend +the pipe to consume the comparison's right operand. The same applies to +`|` appearing as the right operand of `+` or `-`. + +This RFC recommends adding a compile-time diagnostic for one pattern, +where "pipe node" means a `*ast.BinaryExpr` with `Op == token.OR` that +will be rewritten as a pipe: a pipe node appearing as the `Y` (right +operand) of a comparison operator (`==`, `!=`, `<`, `<=`, `>`, `>=`). +This catches `a == b | f()` parsing as `a == f(b)`. + +Mixing `|` with `+` and `-` needs no diagnostic: they share precedence +level 4 and group left-to-right, so `a + b | f()` is `(a + b) | f()`, +which desugars to the intended `f(a + b)`. + +The one genuinely unsafe case is `|` appearing as the **right** operand +of a **comparison**, because the comparison then consumes the pipe's left +operand rather than the other way around. The exact compile error is: + +``` +compile error: ambiguous expression: | on the right of == may parse differently + than expected; use parentheses to clarify: write (a | f()) == b or a == f(b) +``` + +More precisely, the diagnostic fires when the parent node of a `|` +BinaryExpr is a comparison BinaryExpr and the pipe is the right-hand +child (`parent.Y`). This is a conservative set: it catches the known trap +without requiring heuristics about intent. + +Adopting this diagnostic is strongly recommended. Silent precedence +surprises in an embedded expression language are especially damaging +because the author of an expression is often not the author of the +embedding code, and there may be no test coverage for the specific +combination. + +--- + +## 4. Interaction with optional access (`?.` and `?[`) + +### 4.1 Nil propagation + +The `?.` and `?[` operators already handle the "receiver may be nil" +case: they short-circuit to `nil` when the receiver is nil or the lookup +produces nothing, rather than raising `ErrEvaluate`. The pipe operator +does not change this behavior. + +Consider: + +``` +user?.orders | filter(it.paid) +``` + +This desugars to: + +``` +filter(user?.orders, it.paid) +``` + +`user?.orders` evaluates via the existing `__try_select__` sentinel. If +`user` is nil or has no `orders` field, the result is `nil`. `filter` +receives `nil` as its collection, which the existing `iterItems` function +already handles: `nil` is treated as an empty list. So the full +expression returns `[]any{}` when `user` is nil, with no error. + +The behavior is therefore: + +- nil receiver through `?.` or `?[`: pipe receives `nil`, passes it to + the called function, which sees an empty or nil collection. +- For `filter`, `map`, `any`, `all`, `find`, `count`: nil input returns + the appropriate empty result. +- For other functions that do not handle `nil`: they receive `nil` and + behave according to their own contract. This is no different from + calling them directly with a nil argument. + +### 4.2 Chained optional access before a pipe + +``` +events?[0]?.tags | filter(!it.internal) +``` + +desugars to: + +``` +filter(events?[0]?.tags, !it.internal) +``` + +The `events?[0]?.tags` subexpression is evaluated first (as it always +would be). If the index is out of range or the `tags` field is absent, +the result is `nil`, and `filter` returns `[]any{}`. + +### 4.3 Calling the result of a pipe + +`(a | f())?.field` is allowed: the pipe desugars to `f(a)`, a call +expression, and `?.field` on a call result is already supported. +`(a | f())()` is rejected because calling the result of a call is +already rejected for all calls (`call target must be a function name or +selector`). + +### 4.4 Consistency verdict + +Optional access and pipe compose cleanly. The nil-propagation semantics +are consistent with the existing `?.`/`?[` design: missing data flows +forward as `nil` rather than erroring, and functions that already handle +`nil` (the higher-order forms) absorb it silently. No new nil-handling +rules are needed. + +--- + +## 5. Non-call right-hand side + +`a | b` where `b` is not a call expression cannot be given a useful +meaning without introducing semantics that are not present elsewhere in +expr. Possible interpretations (bitwise OR, value threading to a +non-function) are either already rejected or do not fit the model. + +This RFC recommends rejecting with a compile error at the same point +where the pipe desugar would otherwise fire: + +``` +compile error: pipe operator | requires a function call on the right-hand side; + "b" is not a call (did you mean to write b(...)?) +``` + +If `b` is an identifier that resolves to a known higher-order form, the +error can reference the expected signature, following the same "did you +mean" style the rest of expr uses: + +``` +compile error: pipe operator | requires a function call on the right-hand side; + "filter" is a special form, did you mean to write filter(predicate)? +``` + +--- + +## 6. Error message quality + +A summary of all new compile errors introduced by this feature, with +exact text: + +**Non-call right-hand side:** + +``` +compile error: pipe operator | requires a function call on the right-hand side; + "" is not a call +``` + +When the RHS is a recognized special form name (an identifier matching a +known higher-order form): + +``` +compile error: pipe operator | requires a function call on the right-hand side; + "" is a special form, did you mean to write ()? +``` + +**Ambiguous precedence (pipe as right operand of comparison):** + +``` +compile error: ambiguous expression: | on the right of may parse differently + than expected; use parentheses to clarify: write ( | ()) + or () +``` + +Where `` is the comparison token (`==`, `!=`, `<`, `<=`, `>`, `>=`). + +All errors wrap `ErrCompile` so `errors.Is(err, ErrCompile)` continues +to work. + +--- + +## 7. The identity question + +### 7.1 The case against + +expr's identity claim is that it "accepts a strict subset of Go's +expression syntax." Users who know Go can read expr expressions without +learning anything new; Go tooling (syntax highlighting, formatters) works +on expr source with no modification. This identity is a meaningful part +of the value proposition: the language is small because it has edges, and +those edges are Go's edges. + +Repurposing `|` breaks this. In Go, `a | b` is bitwise OR. An expr +expression containing `a | f()` is not valid Go and does not mean the +same thing as the closest valid Go. A Go developer reading an expr +expression with `|` cannot apply their existing mental model. Syntax +highlighting will not help, because the token looks like bitwise OR. + +The existing `?.` and `?[` operators are a precedent for divergence, but +they are unusual in that Go simply does not have those tokens, so there is +no conflict with existing Go meaning. `|` is different: it has a Go +meaning that expr users will know, and repurposing it silently replaces +that meaning. + +There is also a maintenance argument. Every new operator that diverges +from Go adds surface area to the "not quite Go" subset that users and +tools must track. The current divergences (`?.`, `?[`, `map` rewrite, `if` +rewrite) each had a clear necessity. Pipelines are a convenience, not a +necessity: the nested form works today. + +### 7.2 The case for + +The "strict Go subset" claim already has caveats. Bitwise operators are +rejected, so `|` is not available to users at all today. A user writing +`a | b` already gets a compile error; the change from "bitwise OR is +rejected" to "this is a pipe operator" does not take away any working +functionality. In that sense, the token is genuinely free. + +The ergonomic case is real. The motivating example is not contrived, and +deeply nested higher-order calls are the most common readability complaint +about expr. Pipelines are an established idiom (shells, Elixir, Haskell, +Rust iterators, JavaScript `.then()` chains) that many developers +recognize immediately, even if the specific token `|` evokes bitwise OR +in C-family languages. + +An opt-in option (`WithPipeOperator()`) would let the host application +decide. Users of that application would encounter a clear capability +boundary, and the spec could document it explicitly. + +### 7.3 Recommendation + +Adopt the feature behind an opt-in `WithPipeOperator()` compile option, +clearly documented as a deviation from the strict Go subset. Keep the +default behavior unchanged: without `WithPipeOperator()`, `a | b` +continues to produce "bitwise operator | is not supported." + +The opt-in framing resolves the identity tension: the core language +remains a Go subset, and applications that want pipeline ergonomics +declare that choice explicitly. Documentation should be frank about the +trade-off rather than framing the opt-in as a mere "safety valve." + +--- + +## 8. Alternatives considered + +### 8.1 Method-chain style (`xs.filter(...).map(...)`) + +A chaining syntax like `xs.filter(!it.ok).map(it.name)` would look more +Go-idiomatic and sidestep the token-reuse issue. However, expr's selector +calls (`x.f(...)`) resolve `f` on the runtime value of `x`, not on the +type. There is no mechanism to attach `filter` as a method of a `[]any`. +Implementing this would require either method injection at compile time +(significant complexity) or a distinct parse pass that recognizes +`.filter(...)` differently from ordinary selector calls (surprising +behavior for users). Either path is substantially more invasive than the +pipe desugar and produces a syntax that looks like Go but behaves +differently in a less obvious way. + +### 8.2 A `pipe()` builtin function + +``` +pipe(checks, filter(!it.ok), map(sprintf("- %s: %s", it.name, it.msg)), join("\n")) +``` + +This is honest about being different from Go, avoids any token-reuse +issue, and could be implemented as a variadic higher-order form. The +problems: the syntax is verbose and does not improve readability much +over the nested form; the arguments are positionally significant in a +non-obvious way (the forms are not called, their ASTs are passed and +re-dispatched, which is surprising); and the "pipe" concept buried in a +function call loses the visual left-to-right flow that makes pipelines +useful in the first place. + +### 8.3 Nested calls as status quo + +The current nested form is unambiguous, compiles fine, and is what all +existing users already know. The readability cost is real but bounded: +expr expressions are usually short, and the host application can provide +a multi-line editor with parenthesis matching. This is the lowest-risk +option and the correct choice if the ergonomic gains of the pipe do not +justify the language complexity cost. + +--- + +## 9. Open questions and suggested decision process + +### 9.1 Open questions + +**Q1. Desugar timing.** The proposal places the desugar in the compile- +time validation pass. Should it instead be a pre-parse source rewrite +(like `map`/`if`), a post-parse AST rewrite (like `?.`/`?[`), or a +phase of its own? The answer affects how error positions and the +displayed predicate text in higher-order error messages behave. + +**Q2. Error position reporting.** After desugaring, the injected first +argument's source position may not exist in the original source. How +should compiler error positions be reported for the synthesized +arguments? + +**Q3. Interaction with `identifiers.go`.** `Program.Identifiers()` walks +the AST to collect env-referenced names. If the desugar rewrites the AST +in place, `Identifiers()` sees the desugared tree and reports correctly. +If the desugar is a source-level rewrite, the stored source (`p.source`) +diverges from the tree. Which canonical form is preferred? + +**Q4. Display in error messages.** `formatPredicate` and `exprDisplayString` +reverse internal rewrites back to user-visible form. Should the pipe form +be reversed in error messages (showing `a | f()` rather than `f(a)`) or +shown in desugared form? + +**Q5. Right-to-left composition.** Is there any use case for `f(x) | g` +where `g` receives `f(x)` as a first argument? This is the normal pipe +direction and is handled. But the converse (inserting as the *last* +argument) would require a different token or explicit syntax. + +**Q6. Interaction with `WithEvalBudget`.** The desugar is purely +structural; it does not change how many AST nodes are evaluated. No +budget impact expected, but worth verifying in the prototype. + +### 9.2 Suggested decision process + +1. **Settle the identity question first.** If the team decides that + strict Go-subset identity is non-negotiable, the feature is closed. + If the team is open to a documented deviation, proceed. + +2. **Prototype behind `WithPipeOperator()`.** Implement the desugar in a + branch, adding the option flag, the non-call RHS error, and the + ambiguous-precedence diagnostic. Do not change the default behavior. + No documentation changes outside the option itself. + +3. **Validate with real expressions.** Run the prototype against the + motivating health-check expressions and any other real-world pipelines + that motivated the RFC. Confirm that the desugar and error messages + behave as expected. + +4. **Resolve open questions Q1-Q4** based on prototype experience. + Particularly, confirm that `identifiers.go` and `formatPredicate` + are correct across the rewrite. + +5. **Decide on default-on vs. opt-in.** After the prototype is proven, + the team can reconsider whether `WithPipeOperator()` should become + the default (which makes it part of the baseline language) or remain + opt-in (which keeps the strict Go-subset default). This decision + carries the most long-term weight and should not be rushed. + +6. **Update spec and docs in the same change.** If the feature ships, + `docs/reference/spec.md`, `docs/guides/higher-order-patterns.md`, + `docs/guides/examples.md`, and `llms.txt` all need updating in the + same commit, per the project conventions. diff --git a/docs_examples_test.go b/docs_examples_test.go index 0118fc7..23ecad7 100644 --- a/docs_examples_test.go +++ b/docs_examples_test.go @@ -280,31 +280,16 @@ func TestDocsExample7_RBAC(t *testing.T) { assertDeepEqual(t, got, want) } -// Example 8: Extracting + sorting via a registered function. -func TestDocsExample8_RegisteredSort(t *testing.T) { +// Example 8: Extracting + sorting with the built-in sortBy form. +func TestDocsExample8_SortBy(t *testing.T) { + // Two-arg form: sortBy key expression is `it.age`. src := `take( sortBy( filter(users, it.active), - "age", + it.age, ), 3, )` - sortBy := func(xs []any, key string) []any { - out := append([]any{}, xs...) - // stable insertion sort by numeric key - for i := 1; i < len(out); i++ { - for j := i; j > 0; j-- { - a := out[j-1].(map[string]any)[key] - b := out[j].(map[string]any)[key] - if toFloat(a) > toFloat(b) { - out[j-1], out[j] = out[j], out[j-1] - } else { - break - } - } - } - return out - } take := func(xs []any, n int) []any { if n > len(xs) { n = len(xs) @@ -313,23 +298,92 @@ func TestDocsExample8_RegisteredSort(t *testing.T) { } env := map[string]any{ "users": []any{ - map[string]any{"name": "Ada", "age": 36, "active": true}, - map[string]any{"name": "Alan", "age": 41, "active": false}, - map[string]any{"name": "Grace", "age": 29, "active": true}, - map[string]any{"name": "Linus", "age": 54, "active": true}, - map[string]any{"name": "Mira", "age": 22, "active": true}, + map[string]any{"name": "Ada", "age": int64(36), "active": true}, + map[string]any{"name": "Alan", "age": int64(41), "active": false}, + map[string]any{"name": "Grace", "age": int64(29), "active": true}, + map[string]any{"name": "Linus", "age": int64(54), "active": true}, + map[string]any{"name": "Mira", "age": int64(22), "active": true}, }, } got := runDocExample(t, src, env, WithBuiltins(), - WithFunctions(map[string]any{"sortBy": sortBy, "take": take}), + WithFunctions(map[string]any{"take": take}), ) want := []any{ - map[string]any{"name": "Mira", "age": 22, "active": true}, - map[string]any{"name": "Grace", "age": 29, "active": true}, - map[string]any{"name": "Ada", "age": 36, "active": true}, + map[string]any{"name": "Mira", "age": int64(22), "active": true}, + map[string]any{"name": "Grace", "age": int64(29), "active": true}, + map[string]any{"name": "Ada", "age": int64(36), "active": true}, } assertDeepEqual(t, got, want) + + // Named-binding form: `sortBy(filter(...), u, u.age)`. + srcNamed := `take( + sortBy( + filter(users, u, u.active), + u, + u.age, + ), + 3, + )` + got = runDocExample(t, srcNamed, env, + WithBuiltins(), + WithFunctions(map[string]any{"take": take}), + ) + assertDeepEqual(t, got, want) +} + +// Example 11: Named bindings and flatMap. +func TestDocsExample11_NamedBindingsFlatMap(t *testing.T) { + env := map[string]any{ + "users": []any{ + map[string]any{"orders": []any{int64(1), int64(2)}}, + map[string]any{"orders": []any{int64(3)}}, + }, + "reviews": []any{ + map[string]any{"author": "ann", "comments": []any{"a1", "a2"}}, + map[string]any{"author": "bob", "comments": []any{"b1"}}, + }, + } + + // flatMap with named binding flattens orders per user. + got := runDocExample(t, `flatMap(users, u, u.orders)`, env) + assertDeepEqual(t, got, []any{int64(1), int64(2), int64(3)}) + + // Outer named, inner named: r.author stays visible inside inner body. + opts := []Option{WithBuiltins(), WithFunctions(StringFuncs())} + got = runDocExample(t, `map(reviews, r, join(map(r.comments, c, r.author + ": " + c), "; "))`, env, opts...) + assertDeepEqual(t, got, []any{"ann: a1; ann: a2", "bob: b1"}) +} + +// Example 12: entries, sort, and reverse. +func TestDocsExample12_EntriesSortReverse(t *testing.T) { + env := map[string]any{ + "headers": map[string]any{ + "content-type": "application/json", + "x-request-id": "abc123", + }, + "scores": map[string]any{ + "alice": int64(90), + "bob": int64(70), + "carol": int64(85), + }, + } + + // Format all response headers as "key: value", sorted by key. + got := runDocExample(t, `map(entries(headers), e, e.key + ": " + e.value)`, env) + assertDeepEqual(t, got, []any{"content-type: application/json", "x-request-id: abc123"}) + + // Keep only entries whose value exceeds a threshold. + got = runDocExample(t, `map(filter(entries(scores), e, e.value > 80), e, e.key)`, env) + assertDeepEqual(t, got, []any{"alice", "carol"}) + + // sort and reverse require CollectionFuncs. + collOpts := []Option{WithFunctions(CollectionFuncs())} + got = runDocExample(t, `reverse(sort([3, 1, 2]))`, nil, collOpts...) + assertDeepEqual(t, got, []any{int64(3), int64(2), int64(1)}) + + got = runDocExample(t, `sort(["banana", "apple"])`, nil, collOpts...) + assertDeepEqual(t, got, []any{"apple", "banana"}) } func toFloat(v any) float64 { diff --git a/docs_guides_test.go b/docs_guides_test.go index 4433844..74ed709 100644 --- a/docs_guides_test.go +++ b/docs_guides_test.go @@ -482,3 +482,222 @@ func TestGuide_HigherOrder_LazyIf(t *testing.T) { got := runGuide(t, `if(len(xs) > 5, xs[5], "small")`, env) assertDeepEqual(t, got, "small") } + +// --- higher-order-patterns.md: named bindings --------------------------------- + +func TestGuide_HigherOrder_NamedBinding_AllForms(t *testing.T) { + // higher-order-patterns.md: every iterating form accepts the three-arg shape. + orders := []any{ + map[string]any{"status": "paid", "n": int64(2)}, + map[string]any{"status": "open", "n": int64(1)}, + map[string]any{"status": "paid", "n": int64(3)}, + } + env := map[string]any{"orders": orders} + + got := runGuide(t, `map(orders, o, o.n)`, env) + assertDeepEqual(t, got, []any{int64(2), int64(1), int64(3)}) + + got = runGuide(t, `filter(orders, o, o.status == "paid")`, env) + if len(got.([]any)) != 2 { + t.Fatalf("filter: expected 2, got %v", got) + } + + got = runGuide(t, `any(orders, o, o.n > 2)`, env) + assertDeepEqual(t, got, true) + + got = runGuide(t, `all(orders, o, o.n > 0)`, env) + assertDeepEqual(t, got, true) + + got = runGuide(t, `count(orders, o, o.status == "paid")`, env) + assertDeepEqual(t, got, int64(2)) +} + +func TestGuide_HigherOrder_NamedBinding_NestedScoping(t *testing.T) { + // higher-order-patterns.md: outer named form + inner two-arg: `r` (review) + // stays visible inside the inner body because the inner named form doesn't + // bind `it`. + reviews := []any{ + map[string]any{"author": "ann", "comments": []any{"a1", "a2"}}, + map[string]any{"author": "bob", "comments": []any{"b1"}}, + } + env := map[string]any{"reviews": reviews} + + // Outer named (r), inner two-arg: `it` is the comment. + got := runGuide(t, `map(reviews, r, map(r.comments, r.author + "/" + it))`, env) + assertDeepEqual(t, got, []any{ + []any{"ann/a1", "ann/a2"}, + []any{"bob/b1"}, + }) +} + +// --- higher-order-patterns.md: flatMap ---------------------------------------- + +func TestGuide_HigherOrder_FlatMap(t *testing.T) { + // higher-order-patterns.md: flatMap(users, u, u.orders) flattens orders. + env := map[string]any{ + "users": []any{ + map[string]any{"orders": []any{int64(1), int64(2)}}, + map[string]any{"orders": []any{int64(3)}}, + }, + } + + got := runGuide(t, `flatMap(users, u, u.orders)`, env) + assertDeepEqual(t, got, []any{int64(1), int64(2), int64(3)}) + + // One-level-deep splice. + got = runGuide(t, `flatMap([1, [2, 3], 4], it)`, nil, WithBuiltins()) + assertDeepEqual(t, got, []any{int64(1), int64(2), int64(3), int64(4)}) + + // nil splices as nothing. + got = runGuide(t, `flatMap([1, 2, 3], if(it > 1, [it, it], nil))`, nil, WithBuiltins()) + assertDeepEqual(t, got, []any{int64(2), int64(2), int64(3), int64(3)}) + + // Strings are not split. + got = runGuide(t, `flatMap(["ab", "c"], it)`, nil, WithBuiltins()) + assertDeepEqual(t, got, []any{"ab", "c"}) +} + +// --- higher-order-patterns.md: sortBy ----------------------------------------- + +func TestGuide_HigherOrder_SortBy(t *testing.T) { + // higher-order-patterns.md: sortBy evaluates a key expr and returns a stable copy. + env := map[string]any{ + "orders": []any{ + map[string]any{"status": "paid", "n": int64(2)}, + map[string]any{"status": "open", "n": int64(1)}, + map[string]any{"status": "paid", "n": int64(3)}, + }, + } + + got := runGuide(t, `map(sortBy(orders, o, o.n), o, o.n)`, env) + assertDeepEqual(t, got, []any{int64(1), int64(2), int64(3)}) + + // String keys sort lexicographically. + got = runGuide(t, `map(sortBy(orders, o, o.status), o, o.status)`, env) + assertDeepEqual(t, got, []any{"open", "paid", "paid"}) + + // Input list is not mutated. + first := env["orders"].([]any)[0].(map[string]any) + if first["n"] != int64(2) { + t.Fatalf("sortBy mutated input: first element changed to %v", first) + } +} + +// --- higher-order-patterns.md: entries ---------------------------------------- + +func TestGuide_HigherOrder_Entries(t *testing.T) { + // higher-order-patterns.md: entries makes maps iterable through the forms. + env := map[string]any{ + "headers": map[string]any{ + "content-type": "application/json", + "x-request-id": "abc123", + }, + } + + got := runGuide(t, `map(entries(headers), e, e.key + ": " + e.value)`, env) + assertDeepEqual(t, got, []any{"content-type: application/json", "x-request-id: abc123"}) + + // filter through entries. + scores := map[string]any{"alice": int64(90), "bob": int64(70), "carol": int64(85)} + got = runGuide(t, `map(filter(entries(scores), e, e.value > 80), e, e.key)`, + map[string]any{"scores": scores}) + // keys are sorted by entries, so alice and carol qualify in sorted order. + assertDeepEqual(t, got, []any{"alice", "carol"}) +} + +// --- templates.md: JSON composite rendering ----------------------------------- + +func TestGuide_Templates_JSONCompositeRendering(t *testing.T) { + // templates.md: maps, slices, arrays, and structs render as compact JSON. + env := map[string]any{ + "config": map[string]any{"retries": int64(3)}, + "files": []any{"a.go", "b.go"}, + } + + tmpl, err := NewTemplate("${config} | ${files}", WithBuiltins()) + if err != nil { + t.Fatalf("NewTemplate: %v", err) + } + out, err := tmpl.Render(t.Context(), env) + if err != nil { + t.Fatalf("Render: %v", err) + } + if out != `{"retries":3} | ["a.go","b.go"]` { + t.Fatalf("got %q", out) + } +} + +func TestGuide_Templates_CustomDelimiters(t *testing.T) { + // templates.md: WithTemplateDelimiters swaps the opener/closer. + env := map[string]any{"service": map[string]any{"name": "api"}} + + tmpl, err := NewTemplate( + "Deploy ${{ service.name }} via: echo ${HOME}", + WithTemplateDelimiters("${{", "}}"), + ) + if err != nil { + t.Fatalf("NewTemplate: %v", err) + } + out, err := tmpl.Render(t.Context(), env) + if err != nil { + t.Fatalf("Render: %v", err) + } + if out != "Deploy api via: echo ${HOME}" { + t.Fatalf("got %q", out) + } +} + +func TestGuide_Templates_CustomFormatter(t *testing.T) { + // templates.md: WithTemplateFormatter runs first for every interpolated value. + env := map[string]any{"price": 12.5, "label": "total", "empty": nil} + + tmpl, err := NewTemplate("${label}: ${price} (${empty})", + WithTemplateFormatter(func(v any) (string, bool) { + switch x := v.(type) { + case float64: + return fmt.Sprintf("%.2f", x), true + case nil: + return "N/A", true + } + return "", false + }), + ) + if err != nil { + t.Fatalf("NewTemplate: %v", err) + } + out, err := tmpl.Render(t.Context(), env) + if err != nil { + t.Fatalf("Render: %v", err) + } + if out != "total: 12.50 (N/A)" { + t.Fatalf("got %q", out) + } +} + +func TestGuide_Templates_CompileRejectsTemplateOnlyOptions(t *testing.T) { + // templates.md: template-only options passed to Compile fail at load time. + _, err := Compile("1+1", WithTemplateDelimiters("${{", "}}")) + if !errors.Is(err, ErrCompile) { + t.Fatalf("expected ErrCompile, got %v", err) + } + _, err = Compile("1+1", WithTemplateFormatter(func(any) (string, bool) { return "", false })) + if !errors.Is(err, ErrCompile) { + t.Fatalf("expected ErrCompile, got %v", err) + } +} + +func TestGuide_Templates_ErrorsReportLineColumn(t *testing.T) { + // templates.md: runtime errors include line:column in the message. + src := "line one\nline two\n ${boom} end" + tmpl, err := NewTemplate(src) + if err != nil { + t.Fatalf("NewTemplate: %v", err) + } + _, rerr := tmpl.Render(t.Context(), map[string]any{}) + if rerr == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(rerr.Error(), "at 3:3") { + t.Fatalf("expected line:col in error, got %v", rerr) + } +} diff --git a/engine.go b/engine.go index c760889..76c07db 100644 --- a/engine.go +++ b/engine.go @@ -66,15 +66,23 @@ const MaxEvalDepth = 256 // WithBuiltins for any shared name. type Option func(*compileConfig) -// compileConfig is the resolved set of options for a single Compile. -// It is consumed during parsing to build the function dispatch tables -// baked into the resulting Program. +// compileConfig is the resolved set of options for a single Compile +// or NewTemplate. It is consumed during parsing to build the function +// dispatch tables baked into the resulting Program. The tmpl* fields +// are consumed only by NewTemplate; Compile rejects options that set +// them (tracked in templateOnly) so a misplaced template option fails +// at load time instead of being silently ignored. type compileConfig struct { funcs map[string]any prepared map[string]*preparedFunc fieldTags *structTagConfig evalBudget int errs []error + + tmplOpen string + tmplClose string + tmplFormatter func(v any) (string, bool) + templateOnly []string } func newCompileConfig() *compileConfig { @@ -174,14 +182,26 @@ func WithFieldTags(names ...string) Option { // Program. JSON-style array and object literals ([1, 2, 3], {"k": v}) are // always accepted; see docs/reference/spec.md for the exact rules. func Compile(code string, opts ...Option) (*Program, error) { - if len(code) > MaxSourceLength { - return nil, fmt.Errorf("%w: source length %d exceeds maximum %d", - ErrCompile, len(code), MaxSourceLength) - } cfg := newCompileConfig() for _, opt := range opts { opt(cfg) } + if len(cfg.templateOnly) > 0 { + return nil, fmt.Errorf("%w: %s applies only to NewTemplate", + ErrCompile, strings.Join(cfg.templateOnly, ", ")) + } + return compileWithConfig(code, cfg) +} + +// compileWithConfig compiles code against an already-resolved option +// set. NewTemplate uses it to compile every `${...}` segment without +// re-applying the option closures per segment (and without tripping +// the template-only option check that guards Compile). +func compileWithConfig(code string, cfg *compileConfig) (*Program, error) { + if len(code) > MaxSourceLength { + return nil, fmt.Errorf("%w: source length %d exceeds maximum %d", + ErrCompile, len(code), MaxSourceLength) + } if len(cfg.errs) > 0 { return nil, fmt.Errorf("%w: %w", ErrCompile, errors.Join(cfg.errs...)) } diff --git a/examples/higher_order_patterns/main.go b/examples/higher_order_patterns/main.go index 95c480b..12a2d7d 100644 --- a/examples/higher_order_patterns/main.go +++ b/examples/higher_order_patterns/main.go @@ -1,5 +1,5 @@ -// Higher-order patterns: validation bag, summary object, and -// filter+map projection. All three are single-expression shapes. +// Higher-order patterns: validation bag, summary object, filter+map projection, +// named bindings, flatMap, sortBy, and entries. All are single-expression shapes. package main import ( @@ -13,8 +13,9 @@ func main() { ctx := context.Background() opts := []expr.Option{expr.WithBuiltins()} - run := func(label, src string, env any) { - p, err := expr.Compile(src, opts...) + run := func(label, src string, env any, extra ...expr.Option) { + allOpts := append(opts, extra...) + p, err := expr.Compile(src, allOpts...) if err != nil { panic(err) } @@ -74,4 +75,49 @@ func main() { }, }, ) + + // 4. Named binding: named form + two-arg inner form. + reviews := []any{ + map[string]any{"author": "ann", "comments": []any{"a1", "a2"}}, + map[string]any{"author": "bob", "comments": []any{"b1"}}, + } + // Outer named (r), inner two-arg (it = comment): r.author stays visible. + run("nested named binding", + `map(reviews, r, map(r.comments, r.author + "/" + it))`, + map[string]any{"reviews": reviews}, + ) + + // 5. flatMap: flatten orders across users. + run("flatMap orders", + `flatMap(users, u, u.orders)`, + map[string]any{ + "users": []any{ + map[string]any{"orders": []any{int64(1), int64(2)}}, + map[string]any{"orders": []any{int64(3)}}, + }, + }, + ) + + // 6. sortBy: sort orders by a field. + run("sortBy total", + `map(sortBy(orders, o, o.n), o, o.n)`, + map[string]any{ + "orders": []any{ + map[string]any{"n": int64(3)}, + map[string]any{"n": int64(1)}, + map[string]any{"n": int64(2)}, + }, + }, + ) + + // 7. entries: iterate a map's key-value pairs. + run("entries", + `map(entries(headers), e, e.key + ": " + e.value)`, + map[string]any{ + "headers": map[string]any{ + "content-type": "application/json", + "x-request-id": "abc123", + }, + }, + ) } diff --git a/examples/templates_in_anger/main.go b/examples/templates_in_anger/main.go index adc74b7..89560fb 100644 --- a/examples/templates_in_anger/main.go +++ b/examples/templates_in_anger/main.go @@ -1,6 +1,6 @@ // Templates in anger: compile once, render many, with a registered // `join` helper so variable-length lists render as human-readable -// text instead of Go slice syntax. +// text instead of JSON, and a formatter that overrides nil rendering. package main import ( @@ -57,4 +57,36 @@ func main() { panic(err) } fmt.Println(out) + + // Composite values render as compact JSON by default. + configTmpl, err := expr.NewTemplate(`Config: ${config}`, expr.WithBuiltins()) + if err != nil { + panic(err) + } + out, err = configTmpl.Render(ctx, map[string]any{ + "config": map[string]any{"retries": int64(3), "timeout": "30s"}, + }) + if err != nil { + panic(err) + } + fmt.Println(out) // Config: {"retries":3,"timeout":"30s"} + + // Custom formatter overrides rendering for specific types. + nilTmpl, err := expr.NewTemplate( + `Nickname: ${nickname}`, + expr.WithTemplateFormatter(func(v any) (string, bool) { + if v == nil { + return "(none)", true + } + return "", false + }), + ) + if err != nil { + panic(err) + } + out, err = nilTmpl.Render(ctx, map[string]any{"nickname": nil}) + if err != nil { + panic(err) + } + fmt.Println(out) // Nickname: (none) } diff --git a/higher_order.go b/higher_order.go index d77d19a..9746c17 100644 --- a/higher_order.go +++ b/higher_order.go @@ -9,20 +9,39 @@ import ( "go/printer" "go/token" "reflect" + "sort" "strconv" "strings" ) // itEnv is a scope chain used by the higher-order special forms. It -// binds `it` (current element) and `index` (0-based position) while -// delegating every other identifier to the parent env. Scopes nest -// naturally: an itEnv whose parent is itself an itEnv resolves inner -// `it`/`index` to the innermost loop, matching lexical expectations -// for `map(users, map(it.friends, it.name))`. +// binds the current element and `index` (0-based position) while +// delegating every other identifier to the parent env. The two-arg +// form of an iterating form binds the element as `it`; the three-arg +// form binds it under the user-chosen name instead, recorded here in +// name. Scopes nest naturally: an itEnv whose parent is itself an +// itEnv resolves inner bindings to the innermost loop, matching +// lexical expectations for `map(users, map(it.friends, it.name))`. +// Because a named scope does not bind `it` at all, a nested two-arg +// form's `it` stays reachable from inside a named form and vice +// versa: `map(reviews, r, map(r.comments, sprintf("%s: %s", r.author, it)))`. type itEnv struct { parent any it any index int64 + // name is the element binding's identifier for the three-arg + // form. Empty means the two-arg form, which binds `it`. + name string +} + +// elementName returns the identifier under which this scope binds the +// current element: `it` for the two-arg form, the user-chosen name +// for the three-arg form. +func (s *itEnv) elementName() string { + if s.name != "" { + return s.name + } + return "it" } // higherOrderForm is the uniform signature for every built-in @@ -46,8 +65,9 @@ type userForm struct { // callHint is the signature shown in the "is a special form" // suggester message, e.g. `map(xs, predicate)`. callHint string - // bindsIt marks the iterating forms, which bind `it`/`index` - // inside their second argument. Drives the identifier collector. + // bindsIt marks the iterating forms, which bind an element and + // `index` inside their body: `it` in the two-arg form, a named + // binding in the three-arg form. Drives the identifier collector. bindsIt bool fn higherOrderForm } @@ -79,10 +99,12 @@ func init() { userForms = []userForm{ {name: "map", internal: mapFormName, callHint: "map(xs, predicate)", bindsIt: true, fn: formMap}, {name: "filter", internal: "filter", callHint: "filter(xs, predicate)", bindsIt: true, fn: formFilter}, + {name: "flatMap", internal: "flatMap", callHint: "flatMap(xs, predicate)", bindsIt: true, fn: formFlatMap}, {name: "any", internal: "any", callHint: "any(xs, predicate)", bindsIt: true, fn: formAny}, {name: "all", internal: "all", callHint: "all(xs, predicate)", bindsIt: true, fn: formAll}, {name: "find", internal: "find", callHint: "find(xs, predicate)", bindsIt: true, fn: formFind}, {name: "count", internal: "count", callHint: "count(xs, predicate)", bindsIt: true, fn: formCount}, + {name: "sortBy", internal: "sortBy", callHint: "sortBy(xs, key)", bindsIt: true, fn: formSortBy}, {name: "try", internal: "try", callHint: "try(value, default)", fn: formTry}, {name: "if", internal: ifFuncName, callHint: "if(cond, then, else)", fn: formIf}, } @@ -142,13 +164,41 @@ func (p *Program) iterItems(ctx context.Context, name string, collExpr ast.Expr, ErrEvaluate, name, coll) } -// checkFormArity reports a consistent error across every form. -func checkFormArity(name string, got int) error { - if got == 2 { - return nil +// splitFormArgs splits an iterating form's arguments into the +// collection expression, the element binding name, and the body +// expression. The two-arg form binds the element as `it` (bind is +// ""); the three-arg form names the binding explicitly: argument 2 +// must be a plain identifier, and the body then sees the element +// under that name instead of `it`, so nested forms can still +// reference an outer `it`. The arity and binding checks happen at +// eval time, like every other form check, because forms can be +// shadowed by env entries that only exist at Run. +func splitFormArgs(name string, n *ast.CallExpr) (coll ast.Expr, bind string, body ast.Expr, err error) { + switch len(n.Args) { + case 2: + return n.Args[0], "", n.Args[1], nil + case 3: + ident, ok := n.Args[1].(*ast.Ident) + if !ok { + if src := exprDisplayString(n.Args[1]); src != "" { + return nil, "", nil, fmt.Errorf("%w: %s binding must be a plain identifier, got `%s`", + ErrEvaluate, name, src) + } + return nil, "", nil, fmt.Errorf("%w: %s binding must be a plain identifier", + ErrEvaluate, name) + } + bind = displayIdent(ident.Name) + switch { + case bind == "it", bind == "index", + bind == "true", bind == "false", bind == "nil", + bind != ident.Name: // keyword sentinel: user wrote `map` or `if` + return nil, "", nil, fmt.Errorf("%w: %s binding cannot be named %q", + ErrEvaluate, name, bind) + } + return n.Args[0], bind, n.Args[2], nil } - return fmt.Errorf("%w: %s expects 2 arguments (collection, predicate), got %d", - ErrEvaluate, name, got) + return nil, "", nil, fmt.Errorf("%w: %s expects 2 arguments (collection, predicate) or 3 (collection, name, predicate), got %d", + ErrEvaluate, name, len(n.Args)) } // forEach is the shared loop used by every higher-order form. The @@ -165,12 +215,13 @@ func (p *Program) forEach( ctx context.Context, name string, items itemSeq, + bind string, predicate ast.Expr, env any, depth int, body func(item any, result any) (stop bool, err error), ) error { - scope := &itEnv{parent: env} + scope := &itEnv{parent: env, name: bind} for i := 0; i < items.n; i++ { item := items.at(i) scope.it = item @@ -367,15 +418,16 @@ func displayJSONLit(n *ast.CompositeLit) (ast.Expr, bool) { } func formMap(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { - if err := checkFormArity("map", len(n.Args)); err != nil { + coll, bind, body, err := splitFormArgs("map", n) + if err != nil { return nil, err } - items, err := p.iterItems(ctx, "map", n.Args[0], env, depth) + items, err := p.iterItems(ctx, "map", coll, env, depth) if err != nil { return nil, err } out := make([]any, 0, items.n) - err = p.forEach(ctx, "map", items, n.Args[1], env, depth, func(_ any, v any) (bool, error) { + err = p.forEach(ctx, "map", items, bind, body, env, depth, func(_ any, v any) (bool, error) { out = append(out, v) return false, nil }) @@ -386,15 +438,16 @@ func formMap(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth in } func formFilter(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { - if err := checkFormArity("filter", len(n.Args)); err != nil { + coll, bind, body, err := splitFormArgs("filter", n) + if err != nil { return nil, err } - items, err := p.iterItems(ctx, "filter", n.Args[0], env, depth) + items, err := p.iterItems(ctx, "filter", coll, env, depth) if err != nil { return nil, err } out := make([]any, 0, items.n) - err = p.forEach(ctx, "filter", items, n.Args[1], env, depth, func(item any, v any) (bool, error) { + err = p.forEach(ctx, "filter", items, bind, body, env, depth, func(item any, v any) (bool, error) { if isTruthy(v) { out = append(out, item) } @@ -406,16 +459,105 @@ func formFilter(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth return out, nil } +// formFlatMap implements `flatMap(xs, body)` / `flatMap(xs, x, body)`. +// Like map, except a body result that is a list is spliced into the +// output element-by-element, and a nil body result is spliced as +// nothing (mirroring iterItems, which treats nil as an empty list). +// Any other body result is appended as a single element, so flatMap +// over mixed data never errors on a non-list value. +func formFlatMap(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { + coll, bind, body, err := splitFormArgs("flatMap", n) + if err != nil { + return nil, err + } + items, err := p.iterItems(ctx, "flatMap", coll, env, depth) + if err != nil { + return nil, err + } + out := make([]any, 0, items.n) + err = p.forEach(ctx, "flatMap", items, bind, body, env, depth, func(_ any, v any) (bool, error) { + switch s := v.(type) { + case nil: + return false, nil + case []any: + out = append(out, s...) + return false, nil + } + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.Slice || rv.Kind() == reflect.Array { + for i := 0; i < rv.Len(); i++ { + out = append(out, rv.Index(i).Interface()) + } + return false, nil + } + out = append(out, v) + return false, nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +// formSortBy implements `sortBy(xs, key)` / `sortBy(xs, x, key)`. The +// key expression is evaluated once per element; elements are then +// reordered by their keys with a stable sort, so equal keys preserve +// input order. Keys follow the same comparison rules as sort: all +// numbers (any int/float mix) or all strings, anything else is an +// ErrEvaluate naming the offending element. +func formSortBy(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { + coll, bind, body, err := splitFormArgs("sortBy", n) + if err != nil { + return nil, err + } + items, err := p.iterItems(ctx, "sortBy", coll, env, depth) + if err != nil { + return nil, err + } + keys := make([]any, 0, items.n) + elems := make([]any, 0, items.n) + err = p.forEach(ctx, "sortBy", items, bind, body, env, depth, func(item any, v any) (bool, error) { + keys = append(keys, v) + elems = append(elems, item) + return false, nil + }) + if err != nil { + return nil, err + } + less, err := scalarLessFunc("sortBy", keys) + if err != nil { + return nil, err + } + sort.Stable(&keyedSorter{keys: keys, elems: elems, less: less}) + return elems, nil +} + +// keyedSorter reorders elems and keys in tandem so sortBy can sort +// elements by their computed keys. +type keyedSorter struct { + keys []any + elems []any + less func(i, j int) bool +} + +func (s *keyedSorter) Len() int { return len(s.keys) } +func (s *keyedSorter) Swap(i, j int) { + s.keys[i], s.keys[j] = s.keys[j], s.keys[i] + s.elems[i], s.elems[j] = s.elems[j], s.elems[i] +} +func (s *keyedSorter) Less(i, j int) bool { return s.less(i, j) } + func formAny(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { - if err := checkFormArity("any", len(n.Args)); err != nil { + coll, bind, body, err := splitFormArgs("any", n) + if err != nil { return nil, err } - items, err := p.iterItems(ctx, "any", n.Args[0], env, depth) + items, err := p.iterItems(ctx, "any", coll, env, depth) if err != nil { return nil, err } found := false - err = p.forEach(ctx, "any", items, n.Args[1], env, depth, func(_ any, v any) (bool, error) { + err = p.forEach(ctx, "any", items, bind, body, env, depth, func(_ any, v any) (bool, error) { if isTruthy(v) { found = true return true, nil @@ -429,15 +571,16 @@ func formAny(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth in } func formAll(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { - if err := checkFormArity("all", len(n.Args)); err != nil { + coll, bind, body, err := splitFormArgs("all", n) + if err != nil { return nil, err } - items, err := p.iterItems(ctx, "all", n.Args[0], env, depth) + items, err := p.iterItems(ctx, "all", coll, env, depth) if err != nil { return nil, err } ok := true - err = p.forEach(ctx, "all", items, n.Args[1], env, depth, func(_ any, v any) (bool, error) { + err = p.forEach(ctx, "all", items, bind, body, env, depth, func(_ any, v any) (bool, error) { if !isTruthy(v) { ok = false return true, nil @@ -451,16 +594,17 @@ func formAll(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth in } func formFind(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { - if err := checkFormArity("find", len(n.Args)); err != nil { + coll, bind, body, err := splitFormArgs("find", n) + if err != nil { return nil, err } - items, err := p.iterItems(ctx, "find", n.Args[0], env, depth) + items, err := p.iterItems(ctx, "find", coll, env, depth) if err != nil { return nil, err } var match any matched := false - err = p.forEach(ctx, "find", items, n.Args[1], env, depth, func(item any, v any) (bool, error) { + err = p.forEach(ctx, "find", items, bind, body, env, depth, func(item any, v any) (bool, error) { if isTruthy(v) { match = item matched = true @@ -478,15 +622,16 @@ func formFind(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth i } func formCount(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { - if err := checkFormArity("count", len(n.Args)); err != nil { + coll, bind, body, err := splitFormArgs("count", n) + if err != nil { return nil, err } - items, err := p.iterItems(ctx, "count", n.Args[0], env, depth) + items, err := p.iterItems(ctx, "count", coll, env, depth) if err != nil { return nil, err } var total int64 - err = p.forEach(ctx, "count", items, n.Args[1], env, depth, func(_ any, v any) (bool, error) { + err = p.forEach(ctx, "count", items, bind, body, env, depth, func(_ any, v any) (bool, error) { if isTruthy(v) { total++ } diff --git a/higher_order_binding_test.go b/higher_order_binding_test.go new file mode 100644 index 0000000..a97b535 --- /dev/null +++ b/higher_order_binding_test.go @@ -0,0 +1,296 @@ +package expr + +import ( + "testing" + + "github.com/deepnoodle-ai/expr/internal/require" +) + +func bindingEnv() map[string]any { + return map[string]any{ + "orders": []any{ + map[string]any{"status": "paid", "n": int64(2)}, + map[string]any{"status": "open", "n": int64(1)}, + map[string]any{"status": "paid", "n": int64(3)}, + }, + "reviews": []any{ + map[string]any{"author": "ann", "comments": []any{"a1", "a2"}}, + map[string]any{"author": "bob", "comments": []any{"b1"}}, + }, + "users": []any{ + map[string]any{"orders": []any{int64(1), int64(2)}}, + map[string]any{"orders": []any{int64(3)}}, + }, + } +} + +func TestNamedBinding_AllIteratingForms(t *testing.T) { + env := bindingEnv() + + v, err := evalExpr(t.Context(), `map(orders, o, o.n)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(2), int64(1), int64(3)}, v) + + v, err = evalExpr(t.Context(), `filter(orders, o, o.status == "paid")`, env) + require.NoError(t, err) + require.Len(t, v.([]any), 2) + + v, err = evalExpr(t.Context(), `any(orders, o, o.n > 2)`, env) + require.NoError(t, err) + require.Equal(t, true, v) + + v, err = evalExpr(t.Context(), `all(orders, o, o.n > 0)`, env) + require.NoError(t, err) + require.Equal(t, true, v) + + v, err = evalExpr(t.Context(), `find(orders, o, o.status == "open")`, env) + require.NoError(t, err) + require.Equal(t, map[string]any{"status": "open", "n": int64(1)}, v) + + v, err = evalExpr(t.Context(), `count(orders, o, o.status == "paid")`, env) + require.NoError(t, err) + require.Equal(t, int64(2), v) +} + +// The three-arg form binds only the chosen name plus index. `it` is +// not bound, so it resolves to an enclosing two-arg form (or fails as +// undefined at top level). This is the motivating nested case: the +// guide used to say there was no way to spell outer `it` from inside +// an inner body. +func TestNamedBinding_NestedScoping(t *testing.T) { + env := bindingEnv() + + // Outer named, inner two-arg: `it` is the inner element, `r` the outer. + v, err := evalExpr(t.Context(), `map(reviews, r, join(map(r.comments, r.author + "/" + it), ","))`, env, + WithBuiltins(), WithFunctions(StringFuncs())) + require.NoError(t, err) + require.Equal(t, []any{"ann/a1,ann/a2", "bob/b1"}, v) + + // Outer two-arg, inner named: outer `it` reachable from inner body. + v, err = evalExpr(t.Context(), `map(reviews, map(it.comments, c, it.author + "/" + c))`, env) + require.NoError(t, err) + require.Equal(t, []any{[]any{"ann/a1", "ann/a2"}, []any{"bob/b1"}}, v) + + // index inside a named form refers to the innermost form. + v, err = evalExpr(t.Context(), `map(orders, o, index)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(0), int64(1), int64(2)}, v) + + // Inner named binding shadows an equally-named outer binding. + v, err = evalExpr(t.Context(), `map(users, u, map(u.orders, u, u))[0]`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(1), int64(2)}, v) +} + +func TestNamedBinding_ItUndefinedInsideNamedForm(t *testing.T) { + env := bindingEnv() + _, err := evalExpr(t.Context(), `map(orders, o, it)`, env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), `undefined identifier "it"`) +} + +func TestNamedBinding_ShadowsEnvName(t *testing.T) { + env := map[string]any{ + "x": "outer", + "xs": []any{"inner"}, + } + v, err := evalExpr(t.Context(), `map(xs, x, x)`, env) + require.NoError(t, err) + require.Equal(t, []any{"inner"}, v) +} + +func TestNamedBinding_InvalidBindings(t *testing.T) { + env := bindingEnv() + cases := []struct { + src string + want string + }{ + {`map(orders, it, 1)`, `map binding cannot be named "it"`}, + {`map(orders, index, 1)`, `map binding cannot be named "index"`}, + {`map(orders, nil, 1)`, `map binding cannot be named "nil"`}, + {`map(orders, true, 1)`, `map binding cannot be named "true"`}, + {`map(orders, map, 1)`, `map binding cannot be named "map"`}, + {`map(orders, if, 1)`, `map binding cannot be named "if"`}, + {`map(orders, o.x, 1)`, "map binding must be a plain identifier, got `o.x`"}, + {`map(orders, (o), 1)`, "map binding must be a plain identifier, got `(o)`"}, + {`map(orders, o, 1, 2)`, "map expects 2 arguments (collection, predicate) or 3 (collection, name, predicate), got 4"}, + } + for _, tc := range cases { + _, err := evalExpr(t.Context(), tc.src, env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), tc.want) + } +} + +// The streamed filter(xs, p)[n] fast path must honor the three-arg +// form, including its binding validation errors. +func TestNamedBinding_FilterIndexFastPath(t *testing.T) { + env := bindingEnv() + + v, err := evalExpr(t.Context(), `filter(orders, o, o.status == "paid")[1]`, env) + require.NoError(t, err) + require.Equal(t, map[string]any{"status": "paid", "n": int64(3)}, v) + + _, err = evalExpr(t.Context(), `filter(orders, it, 1)[0]`, env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), `filter binding cannot be named "it"`) +} + +func TestFlatMap(t *testing.T) { + env := bindingEnv() + + v, err := evalExpr(t.Context(), `flatMap(users, u, u.orders)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(1), int64(2), int64(3)}, v) + + v, err = evalExpr(t.Context(), `flatMap(users, it.orders)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(1), int64(2), int64(3)}, v) + + // Non-list body results append as single elements; lists splice. + v, err = evalExpr(t.Context(), `flatMap([1, [2, 3], 4], it)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(1), int64(2), int64(3), int64(4)}, v) + + // Splicing is one level deep only. + v, err = evalExpr(t.Context(), `flatMap([[1, [2]], [3]], it)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(1), []any{int64(2)}, int64(3)}, v) + + // nil body results are spliced as nothing, mirroring the + // nil-is-an-empty-list rule used by the forms' first argument. + v, err = evalExpr(t.Context(), `flatMap([1, 2, 3], if(it > 1, [it, it], nil))`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(2), int64(2), int64(3), int64(3)}, v) + + // Strings are not lists: they append whole, never splice to runes. + v, err = evalExpr(t.Context(), `flatMap(["ab", "c"], it)`, env) + require.NoError(t, err) + require.Equal(t, []any{"ab", "c"}, v) + + // Typed slices from the env splice like []any does. + v, err = evalExpr(t.Context(), `flatMap(pairs, it)`, map[string]any{ + "pairs": []any{[]int{1, 2}, []int{3}}, + }) + require.NoError(t, err) + require.Equal(t, []any{1, 2, 3}, v) + + // Empty and nil collections behave like the other forms. + v, err = evalExpr(t.Context(), `flatMap(nil, it)`, env) + require.NoError(t, err) + require.Equal(t, []any{}, v) + + _, err = evalExpr(t.Context(), `flatMap(orders)`, env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "flatMap expects 2 arguments") +} + +func TestSortBy(t *testing.T) { + env := bindingEnv() + + v, err := evalExpr(t.Context(), `map(sortBy(orders, o, o.n), o, o.n)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(1), int64(2), int64(3)}, v) + + v, err = evalExpr(t.Context(), `map(sortBy(orders, it.n), it.n)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(1), int64(2), int64(3)}, v) + + // String keys sort lexically. + v, err = evalExpr(t.Context(), `map(sortBy(orders, o, o.status), o, o.status)`, env) + require.NoError(t, err) + require.Equal(t, []any{"open", "paid", "paid"}, v) + + // Mixed int/float keys compare numerically. + v, err = evalExpr(t.Context(), `sortBy([2.5, 1, 3], it)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(1), 2.5, int64(3)}, v) + + // Stable: equal keys preserve input order. + v, err = evalExpr(t.Context(), `map(sortBy(orders, o, o.status), o, o.n)`, env) + require.NoError(t, err) + require.Equal(t, []any{int64(1), int64(2), int64(3)}, v) + + // The input list is reordered as a copy, never in place. + orders := env["orders"].([]any) + require.Equal(t, map[string]any{"status": "paid", "n": int64(2)}, orders[0]) + + // Mixed or non-comparable key types are errors. + _, err = evalExpr(t.Context(), `sortBy([1, "a"], it)`, env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "sortBy: element 1 is string, not a number") + + _, err = evalExpr(t.Context(), `sortBy(orders, o, o)`, env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "sortBy: elements must be all numbers or all strings") + + // Key expressions that error are reported like any predicate error. + _, err = evalExpr(t.Context(), `sortBy(orders, o, o.missing)`, env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "sortBy predicate `o.missing` failed on element 0") + + // Empty and nil collections sort to empty. + v, err = evalExpr(t.Context(), `sortBy(nil, it)`, env) + require.NoError(t, err) + require.Equal(t, []any{}, v) +} + +func TestNamedBinding_Identifiers(t *testing.T) { + p, err := Compile(`map(orders, o, o.n + tax)`) + require.NoError(t, err) + require.Equal(t, []string{"orders", "tax"}, p.Identifiers()) + + // The binding identifier itself is not an env reference. + p, err = Compile(`flatMap(users, u, u.orders)`) + require.NoError(t, err) + require.Equal(t, []string{"users"}, p.Identifiers()) + + // `it` inside a three-arg form is NOT bound by it: only an + // enclosing two-arg form can bind it, otherwise it is an env name. + p, err = Compile(`map(orders, o, it)`) + require.NoError(t, err) + require.Equal(t, []string{"it", "orders"}, p.Identifiers()) + + // Nested: outer two-arg binds it, inner named binds c; index is + // bound by both. + p, err = Compile(`map(reviews, map(it.comments, c, c + it.author + index))`) + require.NoError(t, err) + require.Equal(t, []string{"reviews"}, p.Identifiers()) + + // Outside the body, the binding name is a plain env reference. + p, err = Compile(`map(orders, o, o.n) + len(o)`, WithBuiltins()) + require.NoError(t, err) + require.Equal(t, []string{"o", "orders"}, p.Identifiers()) + + // sortBy participates like the other binding forms. + p, err = Compile(`sortBy(files, f, f.additions)`) + require.NoError(t, err) + require.Equal(t, []string{"files"}, p.Identifiers()) +} + +// A binding name should appear in did-you-mean candidates for typos +// inside the body. +func TestNamedBinding_Suggestion(t *testing.T) { + env := bindingEnv() + _, err := evalExpr(t.Context(), `map(orders, order, ordr.n)`, env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), `did you mean "order"?`) +} + +// Env entries and registered functions still shadow the forms, +// three-arg calls included: a user function named flatMap receives +// three evaluated arguments instead of form treatment. +func TestNamedBinding_FormShadowing(t *testing.T) { + env := bindingEnv() + called := false + fn := func(args ...any) any { + called = true + return "shadowed" + } + v, err := evalExpr(t.Context(), `flatMap(users, 1, 2)`, env, WithFunctions(map[string]any{ + "flatMap": fn, + })) + require.NoError(t, err) + require.Equal(t, "shadowed", v) + require.True(t, called) +} diff --git a/identifiers.go b/identifiers.go index 686d86a..2ced9a7 100644 --- a/identifiers.go +++ b/identifiers.go @@ -16,7 +16,11 @@ import ( // // - the literals true, false, and nil // - `it` and `index` where they are bound by an iterating -// higher-order form (map, filter, any, all, find, count) +// higher-order form (map, filter, flatMap, any, all, find, +// count, sortBy) +// - the named element binding of a three-arg form (the `o` in +// `filter(orders, o, o.paid)`), both the binding identifier +// itself and references to it inside the body // - names registered via WithFunctions / WithBuiltins, which // resolve without the env // - special-form names (map, filter, try, if, ...) in call @@ -37,7 +41,7 @@ func (p *Program) Identifiers() []string { // env-resolved identifier set per the Identifiers contract. func collectIdentifiers(root ast.Expr, funcs map[string]any) []string { seen := map[string]struct{}{} - walkIdentifiers(root, false, funcs, seen) + walkIdentifiers(root, nil, funcs, seen) out := make([]string, 0, len(seen)) for name := range seen { out = append(out, name) @@ -46,11 +50,32 @@ func collectIdentifiers(root ast.Expr, funcs map[string]any) []string { return out } +// boundIdents is an immutable stack of element bindings introduced by +// enclosing iterating forms. Each frame binds one element name (`it` +// for the two-arg form, the user-chosen name for the three-arg form); +// `index` is bound by every frame. +type boundIdents struct { + element string + parent *boundIdents +} + +func (b *boundIdents) has(name string) bool { + if b != nil && name == "index" { + return true + } + for s := b; s != nil; s = s.parent { + if name == s.element { + return true + } + } + return false +} + // walkIdentifiers visits node and records env-resolved identifier -// names into seen. itBound reports whether the walk is inside the -// predicate of an iterating form, where `it` and `index` are bound by -// the form rather than the env. -func walkIdentifiers(node ast.Expr, itBound bool, funcs map[string]any, seen map[string]struct{}) { +// names into seen. bound carries the element bindings of the +// enclosing iterating forms, where names resolve to the form rather +// than the env. +func walkIdentifiers(node ast.Expr, bound *boundIdents, funcs map[string]any, seen map[string]struct{}) { switch n := node.(type) { case *ast.Ident: name := displayIdent(n.Name) @@ -58,7 +83,7 @@ func walkIdentifiers(node ast.Expr, itBound bool, funcs map[string]any, seen map case "true", "false", "nil": return } - if itBound && (name == "it" || name == "index") { + if bound.has(name) { return } if _, registered := funcs[name]; registered { @@ -66,50 +91,59 @@ func walkIdentifiers(node ast.Expr, itBound bool, funcs map[string]any, seen map } seen[name] = struct{}{} case *ast.ParenExpr: - walkIdentifiers(n.X, itBound, funcs, seen) + walkIdentifiers(n.X, bound, funcs, seen) case *ast.UnaryExpr: - walkIdentifiers(n.X, itBound, funcs, seen) + walkIdentifiers(n.X, bound, funcs, seen) case *ast.BinaryExpr: - walkIdentifiers(n.X, itBound, funcs, seen) - walkIdentifiers(n.Y, itBound, funcs, seen) + walkIdentifiers(n.X, bound, funcs, seen) + walkIdentifiers(n.Y, bound, funcs, seen) case *ast.SelectorExpr: // n.Sel is a field/key/method name on the receiver, not an // env identifier. - walkIdentifiers(n.X, itBound, funcs, seen) + walkIdentifiers(n.X, bound, funcs, seen) case *ast.IndexExpr: - walkIdentifiers(n.X, itBound, funcs, seen) - walkIdentifiers(n.Index, itBound, funcs, seen) + walkIdentifiers(n.X, bound, funcs, seen) + walkIdentifiers(n.Index, bound, funcs, seen) case *ast.CallExpr: if ident, ok := n.Fun.(*ast.Ident); ok { if _, isForm := higherOrderForms[ident.Name]; isForm { if _, shadowed := funcs[displayIdent(ident.Name)]; !shadowed { // Special-form call: the form name resolves without - // the env, and iterating forms bind it/index inside - // their predicate argument. - if itBindingForms[ident.Name] && len(n.Args) == 2 { - walkIdentifiers(n.Args[0], itBound, funcs, seen) - walkIdentifiers(n.Args[1], true, funcs, seen) - return + // the env, and iterating forms bind an element and + // index inside their body argument. + if itBindingForms[ident.Name] { + if coll, bind, body, err := splitFormArgs(ident.Name, n); err == nil { + walkIdentifiers(coll, bound, funcs, seen) + element := bind + if element == "" { + element = "it" + } + // The binding identifier of a three-arg + // form is declared by the form, not read + // from the env, so it is never walked. + walkIdentifiers(body, &boundIdents{element: element, parent: bound}, funcs, seen) + return + } } for _, a := range n.Args { - walkIdentifiers(a, itBound, funcs, seen) + walkIdentifiers(a, bound, funcs, seen) } return } } } - walkIdentifiers(n.Fun, itBound, funcs, seen) + walkIdentifiers(n.Fun, bound, funcs, seen) for _, a := range n.Args { - walkIdentifiers(a, itBound, funcs, seen) + walkIdentifiers(a, bound, funcs, seen) } case *ast.CompositeLit: for _, e := range n.Elts { if kv, ok := e.(*ast.KeyValueExpr); ok { - walkIdentifiers(kv.Key, itBound, funcs, seen) - walkIdentifiers(kv.Value, itBound, funcs, seen) + walkIdentifiers(kv.Key, bound, funcs, seen) + walkIdentifiers(kv.Value, bound, funcs, seen) continue } - walkIdentifiers(e, itBound, funcs, seen) + walkIdentifiers(e, bound, funcs, seen) } } } diff --git a/llms.txt b/llms.txt index bb3bc5a..5bfe6a6 100644 --- a/llms.txt +++ b/llms.txt @@ -110,6 +110,7 @@ registered yourself. This keeps the sandbox surface as narrow as you want. | `contains(h, n)` | `(any, any) -> bool` | Substring, element, or map-key presence | | `has(m, k)` | `(any, string) -> bool` | Key presence on a map (nil → false) | | `keys(m)` | `(any) -> []any` | Sorted string keys | +| `entries(m)` | `(any) -> []any` | Sorted key-value pairs: `[{"key":k,"value":v}, ...]`; string-keyed maps only; nil → nil | | `upper(s)` | `(string) -> string` | `strings.ToUpper` | | `lower(s)` | `(string) -> string` | `strings.ToLower` | | `sprintf(f, ...)` | `(string, ...any) -> string` | `fmt.Sprintf` | @@ -124,26 +125,48 @@ Opt-in helper groups (register with `WithFunctions`, kept out of `replace(s, old, new)`, `startsWith(s, p)`, `endsWith(s, p)` - `expr.CollectionFuncs()` — `first(xs)`, `last(xs)` (nil for empty), `sum(xs)` (int64 unless a float appears), `slice(xs, i, j)` - (half-open, negative indices from end, clamps; lists and strings) + (half-open, negative indices from end, clamps; lists and strings), + `sort(xs)` (ascending stable copy; all-numbers or all-strings; never + mutates input; nil/empty → `[]any{}`), + `reverse(xs)` (reversed copy; never mutates; nil/empty → `[]any{}`) ## Higher-order forms (always registered) -The second argument is a predicate AST that is re-evaluated per element -with `it` (current element) and `index` (0-based position) in scope. -These shadow any outer identifier of the same name. `list` must be a -slice, array, or nil (maps are not iterated — drive with `keys(m)` if -you need to). - -| Form | Returns | Description | -| --------------------- | ---------------- | ----------------------------------------------------- | -| `map(list, expr)` | `[]any` | Transform every element | -| `filter(list, pred)` | `[]any` | Keep elements matching predicate | -| `any(list, pred)` | `bool` | True if any match (short-circuits) | -| `all(list, pred)` | `bool` | True if all match (empty list → true, short-circuits) | -| `find(list, pred)` | element or `nil` | First match or nil | -| `count(list, pred)` | `int64` | Number of matches | -| `try(value, default)` | value or default | `value` if it evaluates cleanly, else `default` | -| `if(cond, then, else)`| branch value | Lazy ternary: only the selected branch evaluates | +Every iterating form accepts two call shapes: + +``` +form(list, body) // two-arg: binds `it` (element) and `index` (int64, 0-based) +form(list, name, body) // three-arg: binds `name` and `index`; `it` is NOT bound +``` + +Three-arg: the second argument is the element binding name (a plain +identifier). Only `name` and `index` are bound; `it` from an enclosing +two-arg form stays reachable. Reserved names (`it`, `index`, `true`, +`false`, `nil`, `map`, `if`) produce ErrEvaluate. + +`list` must be a slice, array, or nil. Use `keys(m)` or `entries(m)` to +iterate maps. + +| Form | Returns | Description | +| ---------------------------- | ---------------- | ----------------------------------------------------- | +| `map(list, expr)` | `[]any` | Transform every element | +| `map(list, name, expr)` | `[]any` | Same, element bound as `name` | +| `filter(list, pred)` | `[]any` | Keep elements matching predicate | +| `filter(list, name, pred)` | `[]any` | Same, element bound as `name` | +| `flatMap(list, expr)` | `[]any` | Like map; list results splice, nil splices as nothing, other values append whole; one level deep only; strings not split | +| `flatMap(list, name, expr)` | `[]any` | Same, element bound as `name` | +| `any(list, pred)` | `bool` | True if any match (short-circuits) | +| `any(list, name, pred)` | `bool` | Same, element bound as `name` | +| `all(list, pred)` | `bool` | True if all match (empty → true, short-circuits) | +| `all(list, name, pred)` | `bool` | Same, element bound as `name` | +| `find(list, pred)` | element or `nil` | First match or nil | +| `find(list, name, pred)` | element or `nil` | Same, element bound as `name` | +| `count(list, pred)` | `int64` | Number of matches | +| `count(list, name, pred)` | `int64` | Same, element bound as `name` | +| `sortBy(list, key)` | `[]any` | Stable sorted copy by key expr; all-numbers or all-strings; mixed → ErrEvaluate; never mutates input | +| `sortBy(list, name, key)` | `[]any` | Same, element bound as `name` | +| `try(value, default)` | value or default | `value` if it evaluates cleanly, else `default` | +| `if(cond, then, else)` | branch value | Lazy ternary: only the selected branch evaluates | When a predicate inside an iterating form errors, the form wraps the error with its name, the predicate's source text, and the failing @@ -153,25 +176,27 @@ add their own layer. The wrapping preserves the underlying chain so `errors.Is(err, ErrEvaluate)` still matches; cancellation passes through unchanged. -`try` and `if` are the odd ones out: they do not iterate a list and -bind no `it` or `index`. `try` traps anything wrapping `ErrEvaluate` -(missing keys, nil selectors, out-of-range indices, `int`/`float` -parse failures); its `default` runs only when the primary fails. -`if(cond, then, else)` evaluates only the branch selected by `cond`'s -truthiness, so `if(n != 0, total/n, 0)` cannot divide by zero. -`try` does **not** trap raw `context.Canceled`, +`try` and `if` do not iterate a list and bind no `it` or `index`. +`try` traps anything wrapping `ErrEvaluate` (missing keys, nil selectors, +out-of-range indices, `int`/`float` parse failures); its `default` runs +only when the primary fails. `if(cond, then, else)` evaluates only the +branch selected by `cond`'s truthiness, so `if(n != 0, total/n, 0)` +cannot divide by zero. `try` does **not** trap `context.Canceled`, `context.DeadlineExceeded`, or anything wrapping `ErrCompile`. ``` try(int(s), 0) // parse with fallback try(find(events, it.kind == "purchase")?.user, "—") // optional chain try(user.nickname, nil) || "(none)" // present nil +flatMap(users, u, u.orders) // flatten orders per user +sortBy(orders, o, o.total) // sort by field +map(entries(headers), e, e.key + ": " + e.value) // iterate a map ``` Names can be shadowed by `WithFunctions` or an env entry of the same -name. The literal token `map` is rewritten to an internal sentinel -before parsing (Go treats `map` as a keyword), then translated back for -error messages. +name, three-arg calls included. The literal token `map` is rewritten to +an internal sentinel before parsing (Go treats `map` as a keyword), then +translated back for error messages. ## Optional access (`?.` and `?[`) @@ -271,6 +296,55 @@ out, _ := t.Render(ctx, env) Each `${...}` is compiled once at construction time and re-evaluated on every `Render`. Outside of `${...}` the template text is literal. +**Value rendering:** nil → empty string; string → passthrough; maps, +slices, arrays, structs → compact JSON with HTML escaping disabled +(`&`, `<`, `>` survive); a composite marshaling to a JSON string (e.g. +`time.Time`) renders as the unquoted string; JSON-incompatible values +(cycles, funcs) fall back to `fmt.Sprintf("%v", v)`. This is a +**behavior change**: composite values previously rendered via +`fmt.Sprintf("%v", v)` (e.g. `map[k:v]`), now render as JSON +(`{"k":"v"}`). + +**`$$` escape:** `$$` → literal `$`. `$${name}` emits `${name}`. +Applies only when the opener starts with `$`. + +**Custom delimiters (NewTemplate only):** +```go +expr.WithTemplateDelimiters("${{", "}}") // GitHub Actions style +expr.WithTemplateDelimiters("{{", "}}") // Mustache style +``` +Opener must end with one or more `{`; closer is the matching `}` run. +Passing to `Compile` fails with `ErrCompile`. + +**Custom formatter (NewTemplate only):** +```go +expr.WithTemplateFormatter(func(v any) (string, bool) { + if t, ok := v.(time.Time); ok { return t.Format(time.Stamp), true } + return "", false +}) +``` +Runs first for every interpolated value; return `false` to fall through. +Passing to `Compile` fails with `ErrCompile`. + +**Error messages** include 1-based `line:column (offset N)`: +``` +template: evaluating ${boom} at 3:3 (offset 21): ... +``` + +**`Template.Segments() []TemplateSegment`** — parsed segments in source +order. Each `TemplateSegment` carries `Literal`, `Source`, `Offset`, +`Line`, `Column`, and `Program` (compiled expression for expression +segments, nil for literals). Call `Program.Identifiers()` per segment +for editor hints or live validation. + +`Program.Identifiers()` returns the sorted, deduplicated env-resolved +names the expression references. Excludes: `true`/`false`/`nil`; +`it`/`index` bound by an enclosing iterating form; the named binding of +a three-arg form (both the name itself and its body references); names +registered via `WithFunctions`/`WithBuiltins`; special-form names in +call position. `it` inside a three-arg form body IS included (the form +does not bind it). + ## JSON-style literals Bare `[...]` and `{"k": v}` are rewritten before parsing into @@ -329,15 +403,18 @@ All parse failures wrap `ErrCompile`. All runtime failures wrap are wrapped so `errors.Is` and `errors.As` still find the original cause. Unknown identifiers, fields, and keys get a Levenshtein-based "did you mean...?" hint drawn from what's actually in scope. -Higher-order forms (`map`, `filter`, `any`, `all`, `find`, `count`, -`try`, `if`) referenced as bare identifiers get a tailored hint -showing their call signature, e.g. `"count" is a special form, did -you mean to call count(xs, predicate)?`. +Higher-order forms (`map`, `filter`, `flatMap`, `any`, `all`, `find`, +`count`, `sortBy`, `try`, `if`) referenced as bare identifiers get a +tailored hint showing their call signature, e.g. `"count" is a special +form, did you mean to call count(xs, predicate)?`. `Program.Identifiers()` returns the sorted set of env-resolved names -the expression references (excluding literals, bound `it`/`index`, -registered functions, and special forms) — useful for validating an -expression against a known env shape at load time. +the expression references. Excludes: literals, bound `it`/`index`, +the named binding name and its body references in three-arg forms, +registered functions, and special forms in call position. `it` inside +a three-arg form body is included (the form does not bind it). +Useful for validating an expression against a known env shape at load +time or tracking dependency sets for cache invalidation. ```go if errors.Is(err, expr.ErrCompile) { /* ... */ } diff --git a/program.go b/program.go index f750be2..18ddb85 100644 --- a/program.go +++ b/program.go @@ -453,15 +453,16 @@ func evalIdent(n *ast.Ident, env any, funcs map[string]any, fieldTags *structTag // struct, or an *itEnv wrapping one of the above. For structs, configured // field tags participate in field lookup, and fields are preferred over // methods when both match. Methods are returned as bound function values -// so they can be invoked by a CallExpr node. An itEnv binds `it` and -// `index` ahead of anything in its parent. +// so they can be invoked by a CallExpr node. An itEnv binds its element +// name (`it`, or the named binding of a three-arg form) and `index` +// ahead of anything in its parent. func lookupEnv(env any, name string, fieldTags *structTagConfig) (any, bool, error) { if env == nil { return nil, false, nil } if it, ok := env.(*itEnv); ok { switch name { - case "it": + case it.elementName(): return it.it, true, nil case "index": return it.index, true, nil @@ -950,7 +951,7 @@ func lookupEnvRV(env any, name string, fieldTags *structTagConfig) (reflect.Valu } if it, ok := env.(*itEnv); ok { switch name { - case "it": + case it.elementName(): return reflect.ValueOf(it.it), true, nil case "index": return reflect.ValueOf(it.index), true, nil @@ -1285,7 +1286,7 @@ func (p *Program) tryFilterIndex(ctx context.Context, n *ast.IndexExpr, env any, return nil, false, nil } ident, ok := call.Fun.(*ast.Ident) - if !ok || ident.Name != "filter" || len(call.Args) != 2 { + if !ok || ident.Name != "filter" || len(call.Args) < 2 || len(call.Args) > 3 { return nil, false, nil } // Respect identifier shadowing: a user-registered or env-bound @@ -1307,11 +1308,19 @@ func (p *Program) tryFilterIndex(ctx context.Context, n *ast.IndexExpr, env any, return nil, false, nil } + // Both the two-arg (`it`) and three-arg (named binding) forms are + // streamed; splitFormArgs validates the binding identifier with + // the same error the general form path would produce. + collExpr, bindName, predicate, err := splitFormArgs("filter", call) + if err != nil { + return nil, true, err + } + // Stream the collection via reflect rather than materializing it // through iterItems — for filter(xs, p)[0] over a 1000-element // slice, the unstreamed path allocates 1000 any-boxes only to // throw 999 of them away. - coll, err := p.eval(ctx, call.Args[0], env, depth) + coll, err := p.eval(ctx, collExpr, env, depth) if err != nil { return nil, true, err } @@ -1323,15 +1332,15 @@ func (p *Program) tryFilterIndex(ctx context.Context, n *ast.IndexExpr, env any, return nil, true, fmt.Errorf("%w: filter expects a list as its first argument, got %T", ErrEvaluate, coll) } - scope := &itEnv{parent: env} + scope := &itEnv{parent: env, name: bindName} var seen int64 for i := 0; i < rv.Len(); i++ { item := rv.Index(i).Interface() scope.it = item scope.index = int64(i) - v, err := p.eval(ctx, call.Args[1], scope, depth) + v, err := p.eval(ctx, predicate, scope, depth) if err != nil { - return nil, true, wrapPredicateErr("filter", call.Args[1], i, err) + return nil, true, wrapPredicateErr("filter", predicate, i, err) } if !isTruthy(v) { continue diff --git a/suggest.go b/suggest.go index f3044a4..152b99f 100644 --- a/suggest.go +++ b/suggest.go @@ -193,7 +193,7 @@ func availableFields(recv any, fieldTags *structTagConfig) []string { return out } if it, ok := recv.(*itEnv); ok { - out := []string{"it", "index"} + out := []string{it.elementName(), "index"} return append(out, availableFields(it.parent, fieldTags)...) } rv := reflect.ValueOf(recv) diff --git a/template.go b/template.go index 162b7a3..e1794ed 100644 --- a/template.go +++ b/template.go @@ -1,13 +1,24 @@ package expr import ( + "bytes" "context" + "encoding/json" + "errors" "fmt" "go/scanner" "go/token" + "reflect" "strings" ) +// defaultTemplateOpen and defaultTemplateClose are the delimiters +// used when WithTemplateDelimiters is not supplied. +const ( + defaultTemplateOpen = "${" + defaultTemplateClose = "}" +) + // runner is the minimal interface templateSegment needs to evaluate a // compiled expression. *Program is the only production implementation; // the interface exists so template tests can drive parseTemplate with @@ -29,21 +40,120 @@ type runner interface { // therefore emits the literal text `${name}`. A bare `$` that is not // followed by `$` or `{` is emitted verbatim, so `$5` or `$foo` pass // through unchanged. +// +// The `${` / `}` delimiters can be replaced per template with +// [WithTemplateDelimiters]; the `$$` escape applies only when the +// configured opener starts with `$`. type Template struct { - raw string - segments []templateSegment + raw string + segments []templateSegment + open string + close string + formatter func(v any) (string, bool) } // templateSegment is either a literal chunk of the source (script == // nil) or a compiled expression to evaluate at runtime. For script // segments, source and offset describe the original `${...}` body and -// its starting byte offset in the raw template, so runtime errors can -// point at the exact expression that failed. +// the starting byte offset of its opening delimiter in the raw +// template; line and column are the 1-based position of that offset, +// so runtime errors can point at the exact expression that failed. type templateSegment struct { literal string script runner source string offset int + line int + column int +} + +// TemplateSegment is the public projection of one parsed template +// segment, exposed by [Template.Segments]. A segment is either a +// literal run of text (Literal non-empty, Source empty) or an +// interpolated expression (Source holds the expression body). +// +// Offset is the byte offset of the segment's start in the raw +// template: the first byte of the literal text, or the first byte of +// the opening delimiter for expression segments. Line and Column are +// the 1-based line and byte-based column of that offset. For literal +// segments containing `$$` escapes, Literal holds the decoded text, +// which may be shorter than the raw source it spans. +type TemplateSegment struct { + Literal string + Source string + Offset int + Line int + Column int + // Program is the compiled expression for expression segments, + // nil for literal segments. Hosts can call Identifiers() on it + // for per-segment variable extraction, editor hints, or live + // validation. + Program *Program +} + +// WithTemplateDelimiters replaces the default `${` / `}` expression +// delimiters for a [NewTemplate] call. The opener must end with at +// least one `{` and the closer must be the matching run of `}`: +// +// tmpl, err := expr.NewTemplate(src, expr.WithTemplateDelimiters("${{", "}}")) +// +// A GitHub-Actions-style `${{ expr }}` opener avoids collisions with +// shell parameter expansion and JavaScript template literals, so text +// like `echo ${HOME}` passes through as a literal. The `$$` escape +// applies only when the opener starts with `$`. +// +// The option applies only to NewTemplate; passing it to [Compile] +// fails with ErrCompile. +func WithTemplateDelimiters(open, close string) Option { + return func(c *compileConfig) { + c.templateOnly = append(c.templateOnly, "WithTemplateDelimiters") + braces := trailingBraces(open) + switch { + case open == "": + c.errs = append(c.errs, errors.New("WithTemplateDelimiters: opener must not be empty")) + case braces == 0: + c.errs = append(c.errs, fmt.Errorf("WithTemplateDelimiters: opener %q must end with `{`", open)) + case strings.Contains(open, "$$"): + c.errs = append(c.errs, fmt.Errorf("WithTemplateDelimiters: opener %q collides with the `$$` escape", open)) + case close != strings.Repeat("}", braces): + c.errs = append(c.errs, fmt.Errorf("WithTemplateDelimiters: closer %q must be %q to match opener %q", + close, strings.Repeat("}", braces), open)) + default: + c.tmplOpen = open + c.tmplClose = close + } + } +} + +// WithTemplateFormatter installs a custom value renderer for a +// [NewTemplate] call. It runs first for every interpolated result, +// including nil and strings; returning false falls through to the +// default rendering chain (nil to empty, strings pass through, +// composites to JSON, everything else fmt-style). +// +// expr.WithTemplateFormatter(func(v any) (string, bool) { +// if t, ok := v.(time.Time); ok { +// return t.Format(time.Stamp), true +// } +// return "", false +// }) +// +// The option applies only to NewTemplate; passing it to [Compile] +// fails with ErrCompile. +func WithTemplateFormatter(fn func(v any) (string, bool)) Option { + return func(c *compileConfig) { + c.templateOnly = append(c.templateOnly, "WithTemplateFormatter") + c.tmplFormatter = fn + } +} + +// trailingBraces counts the `{` run at the end of s. +func trailingBraces(s string) int { + n := 0 + for i := len(s) - 1; i >= 0 && s[i] == '{'; i-- { + n++ + } + return n } // NewTemplate parses raw and pre-compiles every `${...}` expression @@ -51,21 +161,63 @@ type templateSegment struct { // accepted and become constant templates that return raw unchanged // from [Template.Render]. func NewTemplate(raw string, opts ...Option) (*Template, error) { - return parseTemplate(raw, func(code string) (runner, error) { - return Compile(code, opts...) + cfg := newCompileConfig() + for _, opt := range opts { + opt(cfg) + } + if len(cfg.errs) > 0 { + return nil, fmt.Errorf("%w: %w", ErrCompile, errors.Join(cfg.errs...)) + } + open, close := cfg.tmplOpen, cfg.tmplClose + if open == "" { + open, close = defaultTemplateOpen, defaultTemplateClose + } + t, err := parseTemplateWith(raw, open, close, func(code string) (runner, error) { + return compileWithConfig(code, cfg) }) + if err != nil { + return nil, err + } + t.formatter = cfg.tmplFormatter + return t, nil } // Source returns the unparsed template source. func (t *Template) Source() string { return t.raw } +// Segments returns the parsed segments of the template in source +// order: literal runs and compiled expressions, each carrying its +// offset and line:column position in the raw source. Hosts use it for +// syntax highlighting, per-expression variable extraction (via +// TemplateSegment.Program's Identifiers method), and live validation, +// without re-implementing the template scanner. +func (t *Template) Segments() []TemplateSegment { + out := make([]TemplateSegment, 0, len(t.segments)) + for _, seg := range t.segments { + s := TemplateSegment{ + Literal: seg.literal, + Source: seg.source, + Offset: seg.offset, + Line: seg.line, + Column: seg.column, + } + if p, ok := seg.script.(*Program); ok { + s.Program = p + } + out = append(out, s) + } + return out +} + // Render evaluates each `${...}` expression against env and // concatenates the results with the surrounding literal text. env // follows the same rules as [Program.Run]: it may be a map[string]any, // a struct, or a pointer to a struct. Templates with no expressions // return the raw source unchanged without invoking any script. // -// Value rendering rules for each `${...}` result: +// Value rendering rules for each `${...}` result (a formatter +// installed with [WithTemplateFormatter] runs first and can override +// any of them): // // - nil renders as the empty string. Optional fields that resolve // to nil silently produce no output rather than the literal @@ -74,6 +226,13 @@ func (t *Template) Source() string { return t.raw } // callers do not need a null-coalescing operator for the common // case of optional values. // - string values pass through unchanged. +// - maps, slices, arrays, and structs render as compact JSON with +// HTML escaping disabled, so `${config}` interpolates as +// `{"retries":3}` rather than Go's map syntax. A composite that +// marshals to a JSON string (time.Time, custom marshalers) +// renders as the string itself, unquoted. Values JSON cannot +// represent (cycles, channels, funcs) fall back to the fmt-style +// formatting used for scalars. // - Everything else is formatted with fmt.Sprintf("%v", v). // // The nil-to-empty rule means a template cannot distinguish "value @@ -95,133 +254,232 @@ func (t *Template) Render(ctx context.Context, env any) (string, error) { } v, err := seg.script.Run(ctx, env) if err != nil { - return "", fmt.Errorf("template: evaluating ${%s} at offset %d: %w", seg.source, seg.offset, err) + return "", fmt.Errorf("template: evaluating %s%s%s at %d:%d (offset %d): %w", + t.open, seg.source, t.close, seg.line, seg.column, seg.offset, err) } - b.WriteString(formatTemplateValue(v)) + b.WriteString(t.formatValue(v)) } return b.String(), nil } -// formatTemplateValue renders a Run result for interpolation. nil -// becomes the empty string; strings pass through unchanged; everything -// else uses Go's default formatting. +// formatValue renders a Run result for interpolation per the rules +// documented on Render. +func (t *Template) formatValue(v any) string { + if t.formatter != nil { + if s, ok := t.formatter(v); ok { + return s + } + } + return formatTemplateValue(v) +} + +// formatTemplateValue is the default rendering chain: nil becomes the +// empty string, strings pass through unchanged, composite values +// marshal to compact JSON, and everything else (or a value JSON +// cannot encode) uses Go's default formatting. func formatTemplateValue(v any) string { switch x := v.(type) { case nil: return "" case string: return x - default: - return safeFormatValue(v) } + if isCompositeKind(v) { + if s, ok := marshalTemplateJSON(v); ok { + return s + } + } + return safeFormatValue(v) +} + +// isCompositeKind reports whether v is a map, slice, array, or struct +// (following pointers), the shapes that render as JSON in templates. +func isCompositeKind(v any) bool { + rv := reflect.ValueOf(v) + for rv.Kind() == reflect.Pointer { + if rv.IsNil() { + return false + } + rv = rv.Elem() + } + switch rv.Kind() { + case reflect.Map, reflect.Slice, reflect.Array, reflect.Struct: + return true + } + return false +} + +// marshalTemplateJSON renders v as compact JSON with HTML escaping +// disabled, so `&`, `<`, and `>` survive intact in webhook payloads +// and prompts. A result that is itself a JSON string (a composite +// with a custom marshaler, like time.Time) is unquoted so it renders +// like any other string. ok is false when v cannot be marshaled +// (cycles, channels, funcs, NaN) and the caller should fall back. +func marshalTemplateJSON(v any) (string, bool) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return "", false + } + out := strings.TrimSuffix(buf.String(), "\n") + if len(out) >= 2 && out[0] == '"' { + var s string + if err := json.Unmarshal([]byte(out), &s); err == nil { + return s, true + } + } + return out, true } -// parseTemplate walks raw once, emitting literal segments for plain -// text and compiling each `${...}` body via the supplied compile -// function. It preserves a single literal segment for constant -// templates so Render can take the fast path. The compile parameter is -// a function so tests can drive the parser with mock compile functions -// that never touch the real engine. +// parseTemplate walks raw once with the default `${` / `}` delimiters, +// emitting literal segments for plain text and compiling each `${...}` +// body via the supplied compile function. It preserves a single literal +// segment for constant templates so Render can take the fast path. The +// compile parameter is a function so tests can drive the parser with +// mock compile functions that never touch the real engine. func parseTemplate(raw string, compile func(string) (runner, error)) (*Template, error) { - segs, err := parseTemplateSegments(raw, compile) + return parseTemplateWith(raw, defaultTemplateOpen, defaultTemplateClose, compile) +} + +// parseTemplateWith is parseTemplate generalized over the expression +// delimiters. open must end with one or more `{` and close must be +// the matching `}` run; WithTemplateDelimiters validates this before +// any caller reaches here. +func parseTemplateWith(raw, open, close string, compile func(string) (runner, error)) (*Template, error) { + segs, err := parseTemplateSegments(raw, open, close, compile) if err != nil { return nil, err } - return &Template{raw: raw, segments: segs}, nil + return &Template{raw: raw, segments: segs, open: open, close: close}, nil } -func parseTemplateSegments(raw string, compile func(string) (runner, error)) ([]templateSegment, error) { +func parseTemplateSegments(raw, open, close string, compile func(string) (runner, error)) ([]templateSegment, error) { if raw == "" { - return []templateSegment{{literal: ""}}, nil + return []templateSegment{{literal: "", line: 1, column: 1}}, nil + } + + braces := trailingBraces(open) + // The `$$` escape only exists for `$`-prefixed openers; hosts that + // switch to delimiters like `{{` chose them to avoid `$` entirely. + escape := "" + if open[0] == '$' { + escape = "$$" } var segs []templateSegment var lit strings.Builder + litStart := 0 + appendLit := func(s string, at int) { + if lit.Len() == 0 { + litStart = at + } + lit.WriteString(s) + } flushLit := func() { if lit.Len() > 0 { - segs = append(segs, templateSegment{literal: lit.String()}) + line, col := templateLineCol(raw, litStart) + segs = append(segs, templateSegment{literal: lit.String(), offset: litStart, line: line, column: col}) lit.Reset() } } i := 0 for i < len(raw) { - // Look for the next '$' that might begin something interesting. - // Anything up to that point is literal. - dollar := strings.IndexByte(raw[i:], '$') - if dollar < 0 { - lit.WriteString(raw[i:]) + // Find whichever comes first: the escape sequence or the + // opening delimiter. Everything before it is literal text, as + // is the entire tail when neither occurs again. The escape + // wins when both match at the same region (`$${` is an + // escaped `$` followed by plain text), which is what makes + // `$${name}` emit the literal `${name}`. + openIdx := strings.Index(raw[i:], open) + escIdx := -1 + if escape != "" { + escIdx = strings.Index(raw[i:], escape) + } + if openIdx < 0 && escIdx < 0 { + appendLit(raw[i:], i) break } - lit.WriteString(raw[i : i+dollar]) - i += dollar - - // '$' at end of string: literal. - if i+1 >= len(raw) { - lit.WriteByte('$') - i++ + useEscape := escIdx >= 0 && (openIdx < 0 || escIdx <= openIdx) + cut := openIdx + if useEscape { + cut = escIdx + } + appendLit(raw[i:i+cut], i) + i += cut + if useEscape { + appendLit("$", i) + i += 2 continue } - switch raw[i+1] { - case '$': - // '$$' always collapses to a single literal '$'. Advancing - // past both characters means a following '{' is plain text, - // which is how `$${name}` emits the literal `${name}`. - lit.WriteByte('$') - i += 2 - case '{': - // Expression opener. Flush any buffered literal, locate - // the matching '}', compile the body, emit a script - // segment. - flushLit() - openOffset := i - exprStart := i + 2 - exprEnd, hadContent, err := scanTemplateExprEnd(raw, exprStart, openOffset) - if err != nil { - return nil, err - } - body := strings.TrimSpace(raw[exprStart:exprEnd]) - if body == "" || !hadContent { - return nil, fmt.Errorf("template: empty expression `${}` at offset %d", openOffset) - } - script, err := compile(body) - if err != nil { - return nil, fmt.Errorf("template: invalid expression `${%s}` at offset %d: %w", body, openOffset, err) - } - segs = append(segs, templateSegment{script: script, source: body, offset: openOffset}) - i = exprEnd + 1 - default: - // Bare '$' followed by something else: literal. - lit.WriteByte('$') - i++ + // Expression opener. Flush any buffered literal, locate the + // matching close, compile the body, emit a script segment. + flushLit() + openOffset := i + exprStart := i + len(open) + bodyEnd, hadContent, err := scanTemplateExprEnd(raw, exprStart, openOffset, open, close, braces) + if err != nil { + return nil, err + } + line, col := templateLineCol(raw, openOffset) + body := strings.TrimSpace(raw[exprStart:bodyEnd]) + if body == "" || !hadContent { + return nil, fmt.Errorf("template: empty expression `%s%s` at %d:%d (offset %d)", + open, close, line, col, openOffset) + } + script, err := compile(body) + if err != nil { + return nil, fmt.Errorf("template: invalid expression `%s%s%s` at %d:%d (offset %d): %w", + open, body, close, line, col, openOffset, err) } + segs = append(segs, templateSegment{script: script, source: body, offset: openOffset, line: line, column: col}) + i = bodyEnd + len(close) } flushLit() // A template with no expressions still needs one segment so Eval // can return the raw string directly. if len(segs) == 0 { - segs = []templateSegment{{literal: raw}} + segs = []templateSegment{{literal: raw, line: 1, column: 1}} } return segs, nil } -// scanTemplateExprEnd returns the byte offset within src of the `}` -// that closes the `${` whose body begins at start. It drives go/scanner -// over src[start:] and counts brace depth, so string literals, rune -// literals, comments, and nested composite literals are all handled -// correctly by virtue of using the same tokenizer that go/parser uses -// when it later compiles the expression body. +// templateLineCol converts a byte offset in raw to a 1-based line and +// byte-based column, the form editors and humans expect from +// multiline templates. +func templateLineCol(raw string, offset int) (line, col int) { + if offset > len(raw) { + offset = len(raw) + } + line = 1 + strings.Count(raw[:offset], "\n") + col = offset - strings.LastIndexByte(raw[:offset], '\n') + return line, col +} + +// scanTemplateExprEnd returns the byte offset within src of the first +// `}` of the run that closes the opener whose body begins at start. +// It drives go/scanner over src[start:] and counts brace depth, so +// string literals, rune literals, comments, and nested composite +// literals are all handled correctly by virtue of using the same +// tokenizer that go/parser uses when it later compiles the expression +// body. +// +// For multi-brace openers like `${{`, depth starts at the opener's +// brace count and the closing `}` run must be contiguous: the scanner +// reaching depth zero on a `}` that is not the end of a `}}` run is +// reported as an error rather than silently splitting the closer. // // hadContent reports whether the scanner saw at least one token inside // the body that wasn't a brace or auto-inserted semicolon, so that // comment-only bodies like `${/* hi */}` are rejected up front with // the same "empty expression" error as `${}`. // -// openOffset is the byte offset of the opening `${` in the original -// raw template source; it's only used for error messages. -func scanTemplateExprEnd(src string, start, openOffset int) (end int, hadContent bool, err error) { +// openOffset is the byte offset of the opening delimiter in the +// original raw template source; it's only used for error messages. +func scanTemplateExprEnd(src string, start, openOffset int, open, close string, braces int) (end int, hadContent bool, err error) { fset := token.NewFileSet() file := fset.AddFile("", fset.Base(), len(src)-start) @@ -231,18 +489,28 @@ func scanTemplateExprEnd(src string, start, openOffset int) (end int, hadContent // body is later fed to go/parser. s.Init(file, []byte(src[start:]), func(token.Position, string) {}, 0) - depth := 1 // we are already inside the '{' of '${' + line, col := templateLineCol(src, openOffset) + depth := braces // we are already inside the opener's brace run for { pos, tok, _ := s.Scan() switch tok { case token.EOF: - return 0, false, fmt.Errorf("template: unclosed `${` at offset %d (missing `}`)", openOffset) + return 0, false, fmt.Errorf("template: unclosed `%s` at %d:%d (offset %d, missing `%s`)", + open, line, col, openOffset, close) case token.LBRACE: depth++ case token.RBRACE: depth-- if depth == 0 { - return start + file.Offset(pos), hadContent, nil + last := start + file.Offset(pos) + first := last - (braces - 1) + for k := first; k <= last; k++ { + if k < 0 || src[k] != '}' { + return 0, false, fmt.Errorf("template: expression `%s` at %d:%d (offset %d) must be closed by `%s`", + open, line, col, openOffset, close) + } + } + return first, hadContent, nil } case token.SEMICOLON: // Automatic semicolon insertion produces these even when diff --git a/template_features_test.go b/template_features_test.go new file mode 100644 index 0000000..ba4da51 --- /dev/null +++ b/template_features_test.go @@ -0,0 +1,257 @@ +package expr + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/deepnoodle-ai/expr/internal/require" +) + +func renderTemplate(t *testing.T, src string, env any, opts ...Option) string { + t.Helper() + tmpl, err := NewTemplate(src, opts...) + require.NoError(t, err) + out, err := tmpl.Render(context.Background(), env) + require.NoError(t, err) + return out +} + +// Composite values render as compact JSON, not Go map/slice syntax. +func TestTemplate_CompositeValuesRenderAsJSON(t *testing.T) { + env := map[string]any{ + "config": map[string]any{"retries": int64(3), "timeout": "30s"}, + "files": []any{"a.go", "b.go"}, + "name": "alice", + "n": int64(42), + } + + require.Equal(t, `{"retries":3,"timeout":"30s"}`, renderTemplate(t, "${config}", env)) + require.Equal(t, `["a.go","b.go"]`, renderTemplate(t, "${files}", env)) + // Strings and scalars are untouched; nil still renders empty. + require.Equal(t, "alice", renderTemplate(t, "${name}", env)) + require.Equal(t, "42", renderTemplate(t, "${n}", env)) + require.Equal(t, "", renderTemplate(t, "${missing_thing}", map[string]any{"missing_thing": nil})) +} + +// HTML-significant characters survive: the default json.Marshal would +// rewrite & < > as &-style escapes, mangling webhook payloads +// and prompts. +func TestTemplate_JSONDoesNotEscapeHTML(t *testing.T) { + env := map[string]any{ + "q": map[string]any{"filter": "a&b "}, + } + require.Equal(t, `{"filter":"a&b "}`, renderTemplate(t, "${q}", env)) +} + +func TestTemplate_StructsRenderAsJSON(t *testing.T) { + type point struct { + X int `json:"x"` + Y string `json:"y"` + } + env := map[string]any{"p": point{X: 1, Y: "up"}} + require.Equal(t, `{"x":1,"y":"up"}`, renderTemplate(t, "${p}", env)) +} + +// A composite that marshals to a JSON string (time.Time, custom +// marshalers) renders unquoted, like any other string. +func TestTemplate_TimeRendersUnquoted(t *testing.T) { + ts := time.Date(2026, 6, 12, 9, 30, 0, 0, time.UTC) + env := map[string]any{"at": ts} + require.Equal(t, "2026-06-12T09:30:00Z", renderTemplate(t, "${at}", env)) +} + +// Values JSON cannot encode fall back to the previous fmt-style +// formatting rather than erroring. +func TestTemplate_JSONFallbackOnMarshalFailure(t *testing.T) { + env := map[string]any{ + "bad": map[string]any{"fn": func() {}}, + } + out := renderTemplate(t, "${bad}", env) + require.Contains(t, out, "map[") +} + +func TestTemplate_WithTemplateFormatter(t *testing.T) { + env := map[string]any{ + "price": 12.5, + "label": "total", + "empty": nil, + } + formatter := func(v any) (string, bool) { + switch x := v.(type) { + case float64: + return fmt.Sprintf("%.2f", x), true + case nil: + return "N/A", true + } + return "", false + } + out := renderTemplate(t, "${label}: ${price} (${empty})", env, WithTemplateFormatter(formatter)) + // Floats and nil hit the formatter; the string falls through. + require.Equal(t, "total: 12.50 (N/A)", out) +} + +func TestTemplate_CustomDelimiters(t *testing.T) { + env := map[string]any{"service": map[string]any{"name": "api"}} + + // ${HOME} is now literal text: shell snippets pass through. + out := renderTemplate(t, "Deploy ${{ service.name }} via: echo ${HOME}", env, + WithTemplateDelimiters("${{", "}}")) + require.Equal(t, "Deploy api via: echo ${HOME}", out) + + // Nested braces inside the body still scan correctly. + out = renderTemplate(t, `v=${{ {"k": 1}["k"] }}`, nil, + WithTemplateDelimiters("${{", "}}")) + require.Equal(t, "v=1", out) + + // $$ escaping continues to work with a $-prefixed opener. + out = renderTemplate(t, "$${{literal}}", nil, WithTemplateDelimiters("${{", "}}")) + require.Equal(t, "${{literal}}", out) + + // Openers without $ work too; no escape sequence applies. + out = renderTemplate(t, "Hello {{ service.name }}, $5 and $$ are plain", env, + WithTemplateDelimiters("{{", "}}")) + require.Equal(t, "Hello api, $5 and $$ are plain", out) +} + +func TestTemplate_CustomDelimiterErrors(t *testing.T) { + _, err := NewTemplate("x", WithTemplateDelimiters("", "}")) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), "opener must not be empty") + + _, err = NewTemplate("x", WithTemplateDelimiters("<<", ">>")) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), "must end with `{`") + + _, err = NewTemplate("x", WithTemplateDelimiters("${{", "}")) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), `closer "}" must be "}}"`) + + _, err = NewTemplate("x", WithTemplateDelimiters("$${", "}")) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), "collides with the `$$` escape") + + // Unclosed custom opener names the configured delimiters. + _, err = NewTemplate("x ${{ y }", WithTemplateDelimiters("${{", "}}")) + require.Error(t, err) + require.Contains(t, err.Error(), "unclosed `${{`") + require.Contains(t, err.Error(), "missing `}}`") + + // The closing brace run must be contiguous. + _, err = NewTemplate(`${{ {"a": 1} } }`, WithTemplateDelimiters("${{", "}}")) + require.Error(t, err) + require.Contains(t, err.Error(), "must be closed by `}}`") +} + +// Template-only options passed to Compile fail at load time instead +// of being silently ignored. +func TestCompile_RejectsTemplateOnlyOptions(t *testing.T) { + _, err := Compile("1 + 1", WithTemplateDelimiters("${{", "}}")) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), "WithTemplateDelimiters applies only to NewTemplate") + + _, err = Compile("1 + 1", WithTemplateFormatter(func(any) (string, bool) { return "", false })) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), "WithTemplateFormatter applies only to NewTemplate") +} + +func TestTemplate_ErrorsReportLineColumn(t *testing.T) { + src := "line one\nline two\n ${boom} end" + tmpl, err := NewTemplate(src) + require.NoError(t, err) + _, rerr := tmpl.Render(context.Background(), map[string]any{}) + require.Error(t, rerr) + // ${boom} starts at line 3, byte column 3. + require.Contains(t, rerr.Error(), "at 3:3") + require.Contains(t, rerr.Error(), "evaluating ${boom}") + + // Parse-time errors carry line:column too. + _, err = NewTemplate("ok\n${}") + require.Error(t, err) + require.Contains(t, err.Error(), "empty expression `${}` at 2:1") + + _, err = NewTemplate("a\nb ${x +}") + require.Error(t, err) + require.Contains(t, err.Error(), "invalid expression `${x +}` at 2:3") + + _, err = NewTemplate("\n\n${open") + require.Error(t, err) + require.Contains(t, err.Error(), "unclosed `${` at 3:1") +} + +func TestTemplate_Segments(t *testing.T) { + src := "Hi ${user.name},\nyou have ${count} items" + tmpl, err := NewTemplate(src) + require.NoError(t, err) + + segs := tmpl.Segments() + require.Len(t, segs, 5) + + require.Equal(t, "Hi ", segs[0].Literal) + require.Equal(t, 0, segs[0].Offset) + require.Equal(t, 1, segs[0].Line) + require.Equal(t, 1, segs[0].Column) + require.Nil(t, segs[0].Program) + + require.Equal(t, "user.name", segs[1].Source) + require.Equal(t, 3, segs[1].Offset) + require.Equal(t, 1, segs[1].Line) + require.Equal(t, 4, segs[1].Column) + require.NotNil(t, segs[1].Program) + require.Equal(t, []string{"user"}, segs[1].Program.Identifiers()) + + require.Equal(t, ",\nyou have ", segs[2].Literal) + + require.Equal(t, "count", segs[3].Source) + require.Equal(t, 2, segs[3].Line) + require.Equal(t, 10, segs[3].Column) + require.Equal(t, []string{"count"}, segs[3].Program.Identifiers()) + + require.Equal(t, " items", segs[4].Literal) +} + +// Literal segments record where they started even when `$$` escapes +// make the decoded text shorter than the raw span. +func TestTemplate_SegmentsWithEscapes(t *testing.T) { + tmpl, err := NewTemplate("a $$ b ${x}") + require.NoError(t, err) + segs := tmpl.Segments() + require.Len(t, segs, 2) + require.Equal(t, "a $ b ", segs[0].Literal) + require.Equal(t, 0, segs[0].Offset) + require.Equal(t, "x", segs[1].Source) + require.Equal(t, 7, segs[1].Offset) +} + +func TestTemplate_SegmentsConstantTemplate(t *testing.T) { + tmpl, err := NewTemplate("no expressions here") + require.NoError(t, err) + segs := tmpl.Segments() + require.Len(t, segs, 1) + require.Equal(t, "no expressions here", segs[0].Literal) + require.Equal(t, 1, segs[0].Line) + require.Equal(t, 1, segs[0].Column) +} + +// Multiline bodies inside custom delimiters keep scanning across +// lines, and the segment position points at the opener. +func TestTemplate_CustomDelimiterMultiline(t *testing.T) { + src := "header\n${{\n join(names, \", \")\n}}\nfooter" + tmpl, err := NewTemplate(src, + WithTemplateDelimiters("${{", "}}"), + WithFunctions(StringFuncs())) + require.NoError(t, err) + out, err := tmpl.Render(context.Background(), map[string]any{ + "names": []any{"a", "b"}, + }) + require.NoError(t, err) + require.Equal(t, "header\na, b\nfooter", out) + + segs := tmpl.Segments() + require.Len(t, segs, 3) + require.Equal(t, 2, segs[1].Line) + require.Equal(t, 1, segs[1].Column) + require.True(t, strings.HasPrefix(segs[1].Source, "join")) +} diff --git a/template_fuzz_test.go b/template_fuzz_test.go index 2f441ac..398f9de 100644 --- a/template_fuzz_test.go +++ b/template_fuzz_test.go @@ -125,3 +125,67 @@ func FuzzTemplateEval(f *testing.F) { _ = out }) } + +// customDelimSeeds adapts the corpus to the delimiter styles +// WithTemplateDelimiters enables, plus cases unique to multi-brace +// openers (split closers, shell collisions, nested composites). +var customDelimSeeds = []string{ + "", + "plain ${HOME} text", + "${{x}}", + "${{ x }}", + "${{x}}${{y}}", + "pre ${{x}} post", + "$${{x}}", + "${{ map[string]any{\"k\": 1} }}", + "${{ {\"k\": 1} }}", + "${{ \"}}\" }}", + "${{ `}` }}", + "${{ a /* }} */ + b }}", + "${{a}}}} tail", + "${{a", + "${{", + "${{}}", + "${{ }}", + "${{ x }", + "${{ x } }", + "${ x }", + "{{x}}", + "\ufeff${{x}}", +} + +// FuzzTemplateParseCustomDelims confirms the generalized scanner +// holds the same invariants as FuzzTemplateParse when the opener +// spans multiple braces ("${{"/"}}") and when it has no "$" prefix +// ("{{"/"}}"). +func FuzzTemplateParseCustomDelims(f *testing.F) { + for _, s := range customDelimSeeds { + f.Add(s) + } + for _, s := range fuzzSeeds { + f.Add(s) + } + f.Fuzz(func(t *testing.T, src string) { + if !utf8.ValidString(src) { + t.Skip() + } + for _, delims := range [][2]string{{"${{", "}}"}, {"{{", "}}"}} { + tmpl, err := parseTemplateWith(src, delims[0], delims[1], acceptAllCompile) + if err != nil { + if !strings.HasPrefix(err.Error(), "template:") { + t.Fatalf("error missing template: prefix: %v", err) + } + continue + } + if tmpl == nil { + t.Fatalf("nil template with nil error for %q", src) + } + if got := tmpl.Source(); got != src { + t.Fatalf("Source() drift: got %q want %q", got, src) + } + if _, err := tmpl.Render(context.Background(), nil); err != nil { + t.Fatalf("Render error on accept-all compiler: %v", err) + } + } + }) +}