From 5fd871422f8aa480e7ff02cab093f902ed743232 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Jun 2026 12:14:17 +0000 Subject: [PATCH] Add lazy if, eval budget, compile-time registration checks, Identifiers, and helper groups A batch of reliability and ergonomics improvements: - `if(cond, then, else)` is now a lazy special form instead of an eager builtin: only the branch the condition selects evaluates, so guard idioms like `if(n != 0, total/n, 0)` work. It is always available (no WithBuiltins needed) and shadowable like every other form. - `WithEvalBudget(n)` bounds the AST nodes a single Run may evaluate, giving hostile nesting (`map(xs, map(xs, map(xs, it)))`) a deterministic, cheap failure instead of burning a core until the context deadline. - `WithFunctions` registrations are validated at Compile time: nil entries, non-function values, and unsupported signatures fail with ErrCompile instead of hiding until the first call. - `Program.Identifiers()` exposes the sorted set of env-resolved names the expression references, for load-time env validation and dependency tracking. - Runtime-error panic recovery now covers every reflect dispatch path (registered functions and env callables, not just bound methods). - New opt-in builtin groups kept out of the default sandbox surface: MathFuncs (min, max, abs, floor, ceil, round), StringFuncs (trim, split, join, replace, startsWith, endsWith), and CollectionFuncs (first, last, sum, slice). - CI gains a short fuzz smoke step over all six fuzz targets. Docs (spec, guides, llms.txt, README) updated in lockstep, with new docs-honesty tests pinning the changed claims. https://claude.ai/code/session_01Nw3onLb6YHSRU1kjnKdaph --- .github/workflows/test.yml | 11 + CLAUDE.md | 6 +- README.md | 16 +- boundaries1_test.go | 53 ++-- budget_test.go | 101 +++++++ builtin_groups.go | 401 +++++++++++++++++++++++++++ builtin_groups_test.go | 223 +++++++++++++++ builtins.go | 22 +- docs/guides/examples.md | 46 +++ docs/guides/higher-order-patterns.md | 6 +- docs/guides/registering-functions.md | 43 ++- docs/guides/sandboxing.md | 51 +++- docs/guides/templates.md | 8 +- docs/reference/spec.md | 110 +++++++- docs_examples_test.go | 46 +++ docs_guides_test.go | 75 ++++- engine.go | 58 +++- engine_test.go | 10 +- examples/sandboxing/main.go | 5 +- higher_order.go | 50 +++- identifiers.go | 115 ++++++++ identifiers_test.go | 87 ++++++ llms.txt | 48 +++- prepared.go | 15 +- program.go | 63 +++-- reflect.go | 20 +- registration_test.go | 108 ++++++++ suggest.go | 5 +- 28 files changed, 1660 insertions(+), 142 deletions(-) create mode 100644 budget_test.go create mode 100644 builtin_groups.go create mode 100644 builtin_groups_test.go create mode 100644 identifiers.go create mode 100644 identifiers_test.go create mode 100644 registration_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 37b6908..d70baf6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,3 +32,14 @@ jobs: - name: Test run: go test -race -count=1 ./... + + # Short fuzz runs catch pre-parse rewriter regressions that the + # seed corpora alone miss (e.g. `?` combined with literals). + - name: Fuzz smoke + run: | + go test -run='^$' -fuzz='^FuzzCompile$' -fuzztime=15s . + go test -run='^$' -fuzz='^FuzzEval$' -fuzztime=15s . + go test -run='^$' -fuzz='^FuzzTemplateParse$' -fuzztime=15s . + go test -run='^$' -fuzz='^FuzzTemplateEval$' -fuzztime=15s . + go test -run='^$' -fuzz='^FuzzRewrite$' -fuzztime=15s ./internal/jsonlit + go test -run='^$' -fuzz='^FuzzRewrite$' -fuzztime=15s ./internal/optaccess diff --git a/CLAUDE.md b/CLAUDE.md index de8d1eb..446ced8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,11 +18,13 @@ parameter interpolation. | File | Purpose | | ------------------- | ------------------------------------------------------------------ | -| `engine.go` | `Compile`, `Option`, `map` keyword preprocessing | +| `engine.go` | `Compile`, `Option`, `map`/`if` keyword preprocessing | | `program.go` | AST walker — the evaluator | | `reflect.go` + `prepared.go` | Env lookup, function dispatch, cached signatures | | `builtins.go` | Default function set exposed by `Builtins()` / `WithBuiltins()` | -| `higher_order.go` | `map`, `filter`, `any`, `all`, `find`, `count` special forms | +| `builtin_groups.go` | Opt-in `MathFuncs()` / `StringFuncs()` / `CollectionFuncs()` sets | +| `higher_order.go` | `map`, `filter`, `any`, `all`, `find`, `count`, `try`, `if` forms | +| `identifiers.go` | `Program.Identifiers()` — env-referenced name collection | | `template.go` | `${...}` interpolation via `NewTemplate` / `Render` | | `truthy.go` | `IsTruthy` rules used by `!`, `&&`, `\|\|`, `bool(v)` | | `suggest.go` | "Did you mean…" for unknown identifiers | diff --git a/README.md b/README.md index 2998f6b..905188a 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,11 @@ p, err := expr.Compile(`greet(upper(name))`, expr.WithFunctions(map[string]any{ ``` Mix and match, or skip the builtins entirely and expose only the handful that -make sense for your sandbox. +make sense for your sandbox. Opt-in groups — `expr.MathFuncs()` (`min`, `max`, +`abs`, `floor`, `ceil`, `round`), `expr.StringFuncs()` (`trim`, `split`, +`join`, `replace`, `startsWith`, `endsWith`), and `expr.CollectionFuncs()` +(`first`, `last`, `sum`, `slice`) — add the usual helpers via `WithFunctions` +without widening the default set. ## What the environment can be @@ -148,9 +152,13 @@ p, err := expr.Compile(`filter(users, it.age >= 18 && index < 10)`) ``` The predicate is re-evaluated per element, so they compose naturally: -`any(orders, count(it.items, it.price > 100) > 0)`. These forms are always -registered (no `WithBuiltins` needed), but you can shadow any of them by -registering a function or env value of the same name. +`any(orders, count(it.items, it.price > 100) > 0)`. Two more special forms +use laziness for control flow instead of iteration: `try(value, default)` +falls back when `value` errors, and `if(cond, then, else)` evaluates only the +branch the condition selects — so `if(n != 0, total/n, 0)` can't divide by +zero. These forms are always registered (no `WithBuiltins` needed), but you +can shadow any of them by registering a function or env value of the same +name. ## What it isn't diff --git a/boundaries1_test.go b/boundaries1_test.go index 80c65d8..791ece8 100644 --- a/boundaries1_test.go +++ b/boundaries1_test.go @@ -437,21 +437,22 @@ func TestCall_UnknownFunction(t *testing.T) { require.Contains(t, err.Error(), "unknown function") } +// Registering a function with an unsupported signature fails at +// Compile time — the registration could never be called successfully, +// so surfacing it at load time beats hiding it until the first call. func TestCall_TooManyReturns(t *testing.T) { - opts := []Option{WithFunctions(map[string]any{ + _, err := Compile("three()", WithFunctions(map[string]any{ "three": func() (int, int, int) { return 1, 2, 3 }, - })} - _, err := evalExpr(t.Context(), "three()", nil, opts...) - require.ErrorIs(t, err, ErrEvaluate) + })) + require.ErrorIs(t, err, ErrCompile) require.Contains(t, err.Error(), "returns") } func TestCall_SecondReturnNotError(t *testing.T) { - opts := []Option{WithFunctions(map[string]any{ + _, err := Compile("bad()", WithFunctions(map[string]any{ "bad": func() (int, string) { return 1, "x" }, - })} - _, err := evalExpr(t.Context(), "bad()", nil, opts...) - require.ErrorIs(t, err, ErrEvaluate) + })) + require.ErrorIs(t, err, ErrCompile) require.Contains(t, err.Error(), "second return must be error") } @@ -747,23 +748,33 @@ func TestBuiltin_If_Arity(t *testing.T) { } } -// Without WithBuiltins, `if(...)` must report a clean "unknown -// function" error using the user-visible name, not the internal -// rewrite sentinel. +// `if` is a special form like map/filter/try: always available, no +// WithBuiltins required. func TestBuiltin_If_NotRegistered(t *testing.T) { - _, err := evalExpr(t.Context(), `if(true, 1, 2)`, nil) - require.ErrorIs(t, err, ErrEvaluate) - require.Contains(t, err.Error(), `unknown function "if"`) + got, err := evalExpr(t.Context(), `if(true, 1, 2)`, nil) + require.NoError(t, err) + require.Equal(t, int64(1), got) } -// `if` is eager: the unselected branch still evaluates, so a -// runtime error there propagates. Users who need laziness reach for -// try, &&, or ||. -func TestBuiltin_If_EagerEvaluation(t *testing.T) { +// `if` is lazy: only the branch selected by the condition evaluates, +// so the guard idiom protects against errors in the untaken branch. +func TestBuiltin_If_LazyEvaluation(t *testing.T) { opts := []Option{WithBuiltins()} - env := map[string]any{"xs": []any{1, 2, 3}} - _, err := evalExpr(t.Context(), `if(true, xs[0], xs[99])`, env, opts...) - require.Error(t, err) + env := map[string]any{"xs": []any{int64(1), int64(2), int64(3)}} + + got, err := evalExpr(t.Context(), `if(true, xs[0], xs[99])`, env, opts...) + require.NoError(t, err) + require.Equal(t, int64(1), got) + + // The canonical division guard. + got, err = evalExpr(t.Context(), `if(n != 0, 10/n, 0)`, map[string]any{"n": int64(0)}, opts...) + require.NoError(t, err) + require.Equal(t, int64(0), got) + + // An error in the *taken* branch still propagates. + _, err = evalExpr(t.Context(), `if(false, xs[0], xs[99])`, env, opts...) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "out of range") } // A user-registered `if` must shadow the builtin, matching the diff --git a/budget_test.go b/budget_test.go new file mode 100644 index 0000000..2b7b2e8 --- /dev/null +++ b/budget_test.go @@ -0,0 +1,101 @@ +package expr + +import ( + "strings" + "sync" + "testing" + + "github.com/deepnoodle-ai/expr/internal/require" +) + +func TestBudget_UnderLimitSucceeds(t *testing.T) { + got, err := evalExpr(t.Context(), "1 + 2 * 3", nil, WithEvalBudget(100)) + require.NoError(t, err) + require.Equal(t, int64(7), got) +} + +func TestBudget_ExceededReturnsErrEvaluate(t *testing.T) { + env := map[string]any{"xs": make([]any, 1000)} + _, err := evalExpr(t.Context(), "map(xs, map(xs, map(xs, it)))", env, WithEvalBudget(10_000)) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "evaluation budget exceeded") +} + +// The budget is deterministic: the same program, env, and limit either +// always succeeds or always fails, independent of wall-clock speed. +func TestBudget_Deterministic(t *testing.T) { + env := map[string]any{"xs": []any{int64(1), int64(2), int64(3)}} + p, err := Compile("count(xs, it > 1)", WithEvalBudget(5)) + require.NoError(t, err) + for i := 0; i < 10; i++ { + _, err := p.Run(t.Context(), env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "evaluation budget exceeded") + } +} + +// Each Run gets the full budget; a Run that consumed most of the +// budget must not starve the next one. +func TestBudget_PerRun(t *testing.T) { + env := map[string]any{"xs": []any{int64(1), int64(2), int64(3)}} + p, err := Compile("count(xs, it > 1)", WithEvalBudget(100)) + require.NoError(t, err) + for i := 0; i < 5; i++ { + got, err := p.Run(t.Context(), env) + require.NoError(t, err) + require.Equal(t, int64(2), got) + } +} + +func TestBudget_ConcurrentRunsIndependent(t *testing.T) { + env := map[string]any{"xs": []any{int64(1), int64(2), int64(3)}} + p, err := Compile("count(xs, it > 1)", WithEvalBudget(100)) + require.NoError(t, err) + var wg sync.WaitGroup + errs := make([]error, 16) + for i := range errs { + wg.Add(1) + go func(i int) { + defer wg.Done() + _, errs[i] = p.Run(t.Context(), env) + }(i) + } + wg.Wait() + for _, err := range errs { + require.NoError(t, err) + } +} + +// try cannot be used to escape the budget: once exhausted, every +// subsequent node evaluation (including try's fallback) fails too. +func TestBudget_TryDoesNotEscape(t *testing.T) { + env := map[string]any{"xs": make([]any, 1000)} + _, err := evalExpr(t.Context(), "try(map(xs, map(xs, it)), 42)", env, WithEvalBudget(100)) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "evaluation budget exceeded") +} + +func TestBudget_ZeroAndNegativeMeanUnlimited(t *testing.T) { + env := map[string]any{"xs": make([]any, 100)} + for _, n := range []int{0, -1} { + got, err := evalExpr(t.Context(), "len(map(xs, it))", env, WithBuiltins(), WithEvalBudget(n)) + require.NoError(t, err) + require.Equal(t, 100, got) + } +} + +// A budget on a template bounds each placeholder expression. +func TestBudget_Template(t *testing.T) { + tpl, err := NewTemplate("total: ${count(xs, it > 0)}", WithEvalBudget(10)) + require.NoError(t, err) + xs := make([]any, 1000) + for i := range xs { + xs[i] = int64(i) + } + env := map[string]any{"xs": xs} + _, err = tpl.Render(t.Context(), env) + require.Error(t, err) + if !strings.Contains(err.Error(), "evaluation budget exceeded") { + t.Fatalf("expected budget error, got: %v", err) + } +} diff --git a/builtin_groups.go b/builtin_groups.go new file mode 100644 index 0000000..44f0e13 --- /dev/null +++ b/builtin_groups.go @@ -0,0 +1,401 @@ +package expr + +import ( + "context" + "fmt" + "math" + "reflect" + "strings" +) + +// MathFuncs returns the opt-in numeric helper set. Like [Builtins], +// the returned map is a fresh copy owned by the caller. Register it +// with [WithFunctions]: +// +// expr.Compile(src, expr.WithBuiltins(), expr.WithFunctions(expr.MathFuncs())) +// +// The groups are separate from Builtins so a minimal sandbox stays +// minimal: hosts opt in to exactly the surface they want. +// +// min(a, ...), max(a, ...) smallest/largest argument; int64 when every +// argument is integral, float64 otherwise +// abs(n) absolute value; int64 in, int64 out +// floor(v), ceil(v), round(v) float64 results; integers pass through +func MathFuncs() map[string]any { + return map[string]any{ + "min": Func(nativeMin), + "max": Func(nativeMax), + "abs": Func(nativeAbs), + "floor": Func(nativeFloor), + "ceil": Func(nativeCeil), + "round": Func(nativeRound), + } +} + +// StringFuncs returns the opt-in string helper set. Like [Builtins], +// the returned map is a fresh copy owned by the caller. Register it +// with [WithFunctions]. +// +// trim(s) strings.TrimSpace +// split(s, sep) list of substrings +// join(xs, sep) concatenate a list of strings +// replace(s, old, new) strings.ReplaceAll +// startsWith(s, prefix) strings.HasPrefix +// endsWith(s, suffix) strings.HasSuffix +func StringFuncs() map[string]any { + return map[string]any{ + "trim": Func(nativeTrim), + "split": Func(nativeSplit), + "join": Func(nativeJoin), + "replace": Func(nativeReplace), + "startsWith": Func(nativeStartsWith), + "endsWith": Func(nativeEndsWith), + } +} + +// CollectionFuncs returns the opt-in list helper set. Like +// [Builtins], the returned map is a fresh copy owned by the caller. +// Register it with [WithFunctions]. +// +// first(xs), last(xs) first/last element; nil for empty or nil lists +// sum(xs) numeric sum; int64 when every element is +// integral, float64 otherwise; empty → 0 +// slice(xs, i, j) elements [i, j) of a list, or the rune range +// of a string; negative indices count from the +// end and out-of-range bounds clamp +func CollectionFuncs() map[string]any { + return map[string]any{ + "first": Func(nativeFirst), + "last": Func(nativeLast), + "sum": Func(nativeSum), + "slice": Func(nativeSlice), + } +} + +func nativeMin(_ context.Context, args []any) (any, error) { + return builtinMinMax("min", args, false) +} + +func nativeMax(_ context.Context, args []any) (any, error) { + return builtinMinMax("max", args, true) +} + +// builtinMinMax compares all arguments, staying in int64 when every +// argument is integral and promoting the comparison to float64 +// otherwise. NaN propagates (matching math.Min / math.Max). +func builtinMinMax(name string, args []any, wantMax bool) (any, error) { + if len(args) == 0 { + return nil, fmt.Errorf("%w: %s expects at least 1 arg, got 0", ErrEvaluate, name) + } + allInt := true + for _, a := range args { + if _, ok := toInt64(a); ok { + continue + } + if _, ok := toFloat64(a); !ok { + return nil, fmt.Errorf("%w: %s: expected number, got %T", ErrEvaluate, name, a) + } + allInt = false + } + if allInt { + best, _ := toInt64(args[0]) + for _, a := range args[1:] { + v, _ := toInt64(a) + if (wantMax && v > best) || (!wantMax && v < best) { + best = v + } + } + return best, nil + } + best, _ := toFloat64(args[0]) + for _, a := range args[1:] { + v, _ := toFloat64(a) + if wantMax { + best = math.Max(best, v) + } else { + best = math.Min(best, v) + } + } + return best, nil +} + +func nativeAbs(_ context.Context, args []any) (any, error) { + if err := checkArity("abs", 1, len(args)); err != nil { + return nil, err + } + if i, ok := toInt64(args[0]); ok { + if i == math.MinInt64 { + return nil, fmt.Errorf("%w: integer overflow", ErrEvaluate) + } + if i < 0 { + return -i, nil + } + return i, nil + } + if f, ok := toFloat64(args[0]); ok { + return math.Abs(f), nil + } + return nil, fmt.Errorf("%w: abs: expected number, got %T", ErrEvaluate, args[0]) +} + +func nativeFloor(_ context.Context, args []any) (any, error) { + return builtinRounding("floor", args, math.Floor) +} + +func nativeCeil(_ context.Context, args []any) (any, error) { + return builtinRounding("ceil", args, math.Ceil) +} + +func nativeRound(_ context.Context, args []any) (any, error) { + return builtinRounding("round", args, math.Round) +} + +// builtinRounding applies fn to float arguments and passes integral +// arguments through unchanged (already exact, and keeping int64 avoids +// a surprising type change for `ceil(n)` over an int env value). +func builtinRounding(name string, args []any, fn func(float64) float64) (any, error) { + if err := checkArity(name, 1, len(args)); err != nil { + return nil, err + } + if i, ok := toInt64(args[0]); ok { + return i, nil + } + if f, ok := toFloat64(args[0]); ok { + return fn(f), nil + } + return nil, fmt.Errorf("%w: %s: expected number, got %T", ErrEvaluate, name, args[0]) +} + +func nativeTrim(_ context.Context, args []any) (any, error) { + if err := checkArity("trim", 1, len(args)); err != nil { + return nil, err + } + s, ok := asString(args[0]) + if !ok { + return nil, fmt.Errorf("%w: trim: expected string, got %T", ErrEvaluate, args[0]) + } + return strings.TrimSpace(s), nil +} + +func nativeSplit(_ context.Context, args []any) (any, error) { + if err := checkArity("split", 2, len(args)); err != nil { + return nil, err + } + s, ok := asString(args[0]) + if !ok { + return nil, fmt.Errorf("%w: split: expected string, got %T", ErrEvaluate, args[0]) + } + sep, ok := asString(args[1]) + if !ok { + return nil, fmt.Errorf("%w: split: separator must be string, got %T", ErrEvaluate, args[1]) + } + parts := strings.Split(s, sep) + out := make([]any, len(parts)) + for i, p := range parts { + out[i] = p + } + return out, nil +} + +func nativeJoin(_ context.Context, args []any) (any, error) { + if err := checkArity("join", 2, len(args)); err != nil { + return nil, err + } + sep, ok := asString(args[1]) + if !ok { + return nil, fmt.Errorf("%w: join: separator must be string, got %T", ErrEvaluate, args[1]) + } + if args[0] == nil { + return "", nil + } + rv := reflect.ValueOf(args[0]) + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return nil, fmt.Errorf("%w: join: expected list, got %T", ErrEvaluate, args[0]) + } + parts := make([]string, rv.Len()) + for i := 0; i < rv.Len(); i++ { + elem := rv.Index(i).Interface() + s, ok := asString(elem) + if !ok { + return nil, fmt.Errorf("%w: join: element %d is %T, not string", ErrEvaluate, i, elem) + } + parts[i] = s + } + return strings.Join(parts, sep), nil +} + +func nativeReplace(_ context.Context, args []any) (any, error) { + if err := checkArity("replace", 3, len(args)); err != nil { + return nil, err + } + s, ok := asString(args[0]) + if !ok { + return nil, fmt.Errorf("%w: replace: expected string, got %T", ErrEvaluate, args[0]) + } + old, ok := asString(args[1]) + if !ok { + return nil, fmt.Errorf("%w: replace: old must be string, got %T", ErrEvaluate, args[1]) + } + new_, ok := asString(args[2]) + if !ok { + return nil, fmt.Errorf("%w: replace: new must be string, got %T", ErrEvaluate, args[2]) + } + return strings.ReplaceAll(s, old, new_), nil +} + +func nativeStartsWith(_ context.Context, args []any) (any, error) { + return builtinAffix("startsWith", args, strings.HasPrefix) +} + +func nativeEndsWith(_ context.Context, args []any) (any, error) { + return builtinAffix("endsWith", args, strings.HasSuffix) +} + +func builtinAffix(name string, args []any, fn func(s, affix string) bool) (any, error) { + if err := checkArity(name, 2, len(args)); err != nil { + return nil, err + } + s, ok := asString(args[0]) + if !ok { + return nil, fmt.Errorf("%w: %s: expected string, got %T", ErrEvaluate, name, args[0]) + } + affix, ok := asString(args[1]) + if !ok { + return nil, fmt.Errorf("%w: %s: expected string, got %T", ErrEvaluate, name, args[1]) + } + return fn(s, affix), nil +} + +func nativeFirst(_ context.Context, args []any) (any, error) { + return builtinEnd("first", args, 0) +} + +func nativeLast(_ context.Context, args []any) (any, error) { + return builtinEnd("last", args, -1) +} + +// builtinEnd returns the element at the front (at == 0) or back +// (at == -1) of a list, or nil when the list is nil or empty — +// mirroring how the higher-order forms treat nil as an empty list. +func builtinEnd(name string, args []any, at int) (any, error) { + if err := checkArity(name, 1, len(args)); err != nil { + return nil, err + } + if args[0] == nil { + return nil, nil + } + rv := reflect.ValueOf(args[0]) + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return nil, fmt.Errorf("%w: %s: expected list, got %T", ErrEvaluate, name, args[0]) + } + if rv.Len() == 0 { + return nil, nil + } + if at < 0 { + return rv.Index(rv.Len() - 1).Interface(), nil + } + return rv.Index(0).Interface(), nil +} + +func nativeSum(_ context.Context, args []any) (any, error) { + if err := checkArity("sum", 1, len(args)); err != nil { + return nil, err + } + if args[0] == nil { + return int64(0), nil + } + rv := reflect.ValueOf(args[0]) + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return nil, fmt.Errorf("%w: sum: expected list, got %T", ErrEvaluate, args[0]) + } + var intSum int64 + var floatSum float64 + floating := false + for i := 0; i < rv.Len(); i++ { + elem := rv.Index(i).Interface() + if !floating { + if v, ok := toInt64(elem); ok { + next, ok := checkedAddInt64(intSum, v) + if !ok { + return nil, fmt.Errorf("%w: integer overflow", ErrEvaluate) + } + intSum = next + continue + } + // First non-integral element: switch to float accumulation. + floating = true + floatSum = float64(intSum) + } + v, ok := toFloat64(elem) + if !ok { + return nil, fmt.Errorf("%w: sum: element %d is %T, not a number", ErrEvaluate, i, elem) + } + floatSum += v + } + if floating { + return floatSum, nil + } + return intSum, nil +} + +// nativeSlice implements slice(xs, i, j): the half-open range [i, j) +// of a list (returned as []any) or string (rune-based, returned as +// string). Negative indices count from the end, out-of-range bounds +// clamp, and i > j yields an empty result — so slice never fails on +// range, only on type. This stands in for `xs[i:j]` syntax, which the +// language rejects by design. +func nativeSlice(_ context.Context, args []any) (any, error) { + if err := checkArity("slice", 3, len(args)); err != nil { + return nil, err + } + i, err := toIndexInt(args[1]) + if err != nil { + return nil, err + } + j, err := toIndexInt(args[2]) + if err != nil { + return nil, err + } + if args[0] == nil { + return []any{}, nil + } + if s, ok := asString(args[0]); ok { + runes := []rune(s) + lo, hi := clampRange(i, j, len(runes)) + return string(runes[lo:hi]), nil + } + rv := reflect.ValueOf(args[0]) + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return nil, fmt.Errorf("%w: slice: expected list or string, got %T", ErrEvaluate, args[0]) + } + lo, hi := clampRange(i, j, rv.Len()) + out := make([]any, 0, hi-lo) + for k := lo; k < hi; k++ { + out = append(out, rv.Index(k).Interface()) + } + return out, nil +} + +// clampRange resolves negative indices against n and clamps both +// bounds into [0, n], collapsing inverted ranges to empty. +func clampRange(i, j int64, n int) (int, int) { + lo := resolveIndex(i, n) + hi := resolveIndex(j, n) + if hi < lo { + hi = lo + } + return lo, hi +} + +func resolveIndex(i int64, n int) int { + if i < 0 { + i += int64(n) + } + if i < 0 { + return 0 + } + if i > int64(n) { + return n + } + return int(i) +} diff --git a/builtin_groups_test.go b/builtin_groups_test.go new file mode 100644 index 0000000..ef14f1a --- /dev/null +++ b/builtin_groups_test.go @@ -0,0 +1,223 @@ +package expr + +import ( + "testing" + + "github.com/deepnoodle-ai/expr/internal/require" +) + +func mathOpts() []Option { return []Option{WithFunctions(MathFuncs())} } +func stringOpts() []Option { return []Option{WithFunctions(StringFuncs())} } +func collectionOpts() []Option { return []Option{WithFunctions(CollectionFuncs())} } + +func TestMathFuncs_MinMax(t *testing.T) { + cases := []struct { + expr string + want any + }{ + {`min(3, 1, 2)`, int64(1)}, + {`max(3, 1, 2)`, int64(3)}, + {`min(5)`, int64(5)}, + {`min(2, 1.5)`, 1.5}, + {`max(2, 1.5)`, float64(2)}, + {`min(-1, 1)`, int64(-1)}, + } + for _, tc := range cases { + t.Run(tc.expr, func(t *testing.T) { + got, err := evalExpr(t.Context(), tc.expr, nil, mathOpts()...) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + + _, err := evalExpr(t.Context(), `min()`, nil, mathOpts()...) + require.ErrorIs(t, err, ErrEvaluate) + + _, err = evalExpr(t.Context(), `min(1, "x")`, nil, mathOpts()...) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "expected number") +} + +func TestMathFuncs_Abs(t *testing.T) { + got, err := evalExpr(t.Context(), `abs(-3)`, nil, mathOpts()...) + require.NoError(t, err) + require.Equal(t, int64(3), got) + + got, err = evalExpr(t.Context(), `abs(2.5) + abs(-2.5)`, nil, mathOpts()...) + require.NoError(t, err) + require.Equal(t, float64(5), got) + + // MinInt64 has no positive counterpart; checked like unary minus. + _, err = evalExpr(t.Context(), `abs(n)`, map[string]any{"n": int64(-9223372036854775808)}, mathOpts()...) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "integer overflow") +} + +func TestMathFuncs_Rounding(t *testing.T) { + cases := []struct { + expr string + want any + }{ + {`floor(2.7)`, float64(2)}, + {`ceil(2.1)`, float64(3)}, + {`round(2.5)`, float64(3)}, + {`round(-2.5)`, float64(-3)}, + // Integers pass through without a type change. + {`floor(4)`, int64(4)}, + {`ceil(4)`, int64(4)}, + {`round(4)`, int64(4)}, + } + for _, tc := range cases { + t.Run(tc.expr, func(t *testing.T) { + got, err := evalExpr(t.Context(), tc.expr, nil, mathOpts()...) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +func TestStringFuncs(t *testing.T) { + cases := []struct { + expr string + want any + }{ + {`trim(" hi ")`, "hi"}, + {`split("a,b,c", ",")`, []any{"a", "b", "c"}}, + {`join(["a", "b"], "-")`, "a-b"}, + {`join([], "-")`, ""}, + {`replace("a.b.c", ".", "/")`, "a/b/c"}, + {`startsWith("hello", "he")`, true}, + {`startsWith("hello", "lo")`, false}, + {`endsWith("hello", "lo")`, true}, + {`endsWith("hello", "he")`, false}, + } + for _, tc := range cases { + t.Run(tc.expr, func(t *testing.T) { + got, err := evalExpr(t.Context(), tc.expr, nil, stringOpts()...) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +func TestStringFuncs_SplitJoinRoundTrip(t *testing.T) { + got, err := evalExpr(t.Context(), `join(split("a b c", " "), "_")`, nil, stringOpts()...) + require.NoError(t, err) + require.Equal(t, "a_b_c", got) +} + +func TestStringFuncs_Errors(t *testing.T) { + for _, src := range []string{ + `trim(1)`, + `split(1, ",")`, + `split("a", 1)`, + `join("not-a-list", ",")`, + `join([1], ",")`, + `replace(1, "a", "b")`, + `startsWith(1, "a")`, + `endsWith("a", 1)`, + } { + _, err := evalExpr(t.Context(), src, nil, stringOpts()...) + require.ErrorIs(t, err, ErrEvaluate, src) + } +} + +func TestCollectionFuncs_FirstLast(t *testing.T) { + env := map[string]any{"xs": []any{int64(1), int64(2), int64(3)}} + + got, err := evalExpr(t.Context(), `first(xs)`, env, collectionOpts()...) + require.NoError(t, err) + require.Equal(t, int64(1), got) + + got, err = evalExpr(t.Context(), `last(xs)`, env, collectionOpts()...) + require.NoError(t, err) + require.Equal(t, int64(3), got) + + // Empty and nil lists yield nil, mirroring find's no-match result. + for _, src := range []string{`first([])`, `last([])`, `first(nil)`, `last(nil)`} { + got, err = evalExpr(t.Context(), src, nil, collectionOpts()...) + require.NoError(t, err, src) + require.Nil(t, got, src) + } + + _, err = evalExpr(t.Context(), `first("str")`, nil, collectionOpts()...) + require.ErrorIs(t, err, ErrEvaluate) +} + +func TestCollectionFuncs_Sum(t *testing.T) { + cases := []struct { + expr string + want any + }{ + {`sum([1, 2, 3])`, int64(6)}, + {`sum([1, 2.5])`, 3.5}, + {`sum([])`, int64(0)}, + {`sum(nil)`, int64(0)}, + } + for _, tc := range cases { + t.Run(tc.expr, func(t *testing.T) { + got, err := evalExpr(t.Context(), tc.expr, nil, collectionOpts()...) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + + _, err := evalExpr(t.Context(), `sum([1, "x"])`, nil, collectionOpts()...) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "not a number") + + env := map[string]any{"big": []any{int64(9223372036854775807), int64(1)}} + _, err = evalExpr(t.Context(), `sum(big)`, env, collectionOpts()...) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "integer overflow") +} + +func TestCollectionFuncs_Slice(t *testing.T) { + env := map[string]any{"xs": []any{int64(0), int64(1), int64(2), int64(3)}} + cases := []struct { + expr string + want any + }{ + {`slice(xs, 1, 3)`, []any{int64(1), int64(2)}}, + {`slice(xs, 0, 99)`, []any{int64(0), int64(1), int64(2), int64(3)}}, + {`slice(xs, -2, 99)`, []any{int64(2), int64(3)}}, + {`slice(xs, 3, 1)`, []any{}}, + {`slice(nil, 0, 2)`, []any{}}, + {`slice("héllo", 1, 3)`, "él"}, + {`slice("hello", -3, 99)`, "llo"}, + } + for _, tc := range cases { + t.Run(tc.expr, func(t *testing.T) { + got, err := evalExpr(t.Context(), tc.expr, env, collectionOpts()...) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + + _, err := evalExpr(t.Context(), `slice(42, 0, 1)`, nil, collectionOpts()...) + require.ErrorIs(t, err, ErrEvaluate) + + _, err = evalExpr(t.Context(), `slice(xs, "a", 1)`, env, collectionOpts()...) + require.ErrorIs(t, err, ErrEvaluate) +} + +// The groups must not leak into the default builtin set. +func TestGroups_NotInDefaultBuiltins(t *testing.T) { + _, err := evalExpr(t.Context(), `min(1, 2)`, nil, WithBuiltins()) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), "unknown function") +} + +// Groups compose with Builtins and with each other through the usual +// last-wins option ordering. +func TestGroups_ComposeWithBuiltins(t *testing.T) { + got, err := evalExpr(t.Context(), + `upper(join(slice(split("a,b,c,d", ","), 0, max(2, 1)), "-"))`, nil, + WithBuiltins(), + WithFunctions(MathFuncs()), + WithFunctions(StringFuncs()), + WithFunctions(CollectionFuncs()), + ) + require.NoError(t, err) + require.Equal(t, "A-B", got) +} diff --git a/builtins.go b/builtins.go index 1aaf30c..ddc6b64 100644 --- a/builtins.go +++ b/builtins.go @@ -24,8 +24,6 @@ import ( // as base-10 integers // float(v) numeric conversion to float64; strings parse strictly // bool(v) truthiness check (matches IsTruthy) -// if(cond, t, f) pick t when cond is truthy, else f; both branches -// are evaluated eagerly // contains(h, n) substring for strings, element membership for // slices/arrays (using loose numeric equality), or // key presence for string-keyed maps @@ -33,6 +31,11 @@ import ( // keys(m) sorted string keys of a map // lower(s), upper(s) case conversion // sprintf(fmt, ...) fmt.Sprintf-style formatting with cycle guards +// +// if(cond, then, else) is not in this map: it is a special form +// (always available, lazily evaluated) — see higher_order.go. Opt-in +// extension sets live in [MathFuncs], [StringFuncs], and +// [CollectionFuncs]. func Builtins() map[string]any { return map[string]any{ "len": Func(nativeLen), @@ -40,7 +43,6 @@ func Builtins() map[string]any { "int": Func(nativeInt), "float": Func(nativeFloat), "bool": Func(nativeBool), - "if": Func(nativeIf), "contains": Func(nativeContains), "has": Func(nativeHas), "keys": Func(nativeKeys), @@ -93,20 +95,6 @@ func nativeBool(_ context.Context, args []any) (any, error) { return IsTruthy(args[0]), nil } -// nativeIf is the eager three-argument selector. Both branches are -// evaluated before the call, so use try / && / || when laziness -// matters. The condition is interpreted via IsTruthy, so any value -// (number, string, slice, ...) can drive the choice. -func nativeIf(_ context.Context, args []any) (any, error) { - if err := checkArity("if", 3, len(args)); err != nil { - return nil, err - } - if IsTruthy(args[0]) { - return args[1], nil - } - return args[2], nil -} - func nativeContains(_ context.Context, args []any) (any, error) { if err := checkArity("contains", 2, len(args)); err != nil { return nil, err diff --git a/docs/guides/examples.md b/docs/guides/examples.md index 0fae98a..69104e8 100644 --- a/docs/guides/examples.md +++ b/docs/guides/examples.md @@ -398,3 +398,49 @@ Env is a typical webhook payload: a `request` map with `method`, `headers`, and `body` keys. This kind of expression is exactly what `expr` is built for — authorization and routing rules that you want your operators to edit without redeploying the host program. + +--- + +## 10. Guarded math and the opt-in helper groups + +`if(cond, then, else)` is lazy — only the selected branch evaluates — +so it doubles as a guard: dividing by a count that might be zero, +selecting into a list that might be empty. Combined with the opt-in +helper groups (`expr.MathFuncs()`, `expr.StringFuncs()`, +`expr.CollectionFuncs()`), per-order stats stay in the expression +instead of leaking into Go: + +```go +{ + "avg_price": if(len(prices) > 0, sum(prices) / len(prices), 0), + "spread": if(len(prices) > 0, max(0, last(prices) - first(prices)), 0), + "top_three": slice(prices, 0, 3), + "sku_list": join(map(items, upper(it.sku)), ", "), +} +``` + +Env: + +```go +map[string]any{ + "prices": []any{int64(10), int64(20), int64(60)}, + "items": []any{ + map[string]any{"sku": "a-1"}, + map[string]any{"sku": "b-2"}, + }, +} +``` + +Compile with the groups registered alongside the standard builtins: + +```go +p, err := expr.Compile(src, + expr.WithBuiltins(), + expr.WithFunctions(expr.MathFuncs()), + expr.WithFunctions(expr.StringFuncs()), + expr.WithFunctions(expr.CollectionFuncs()), +) +``` + +With an empty `prices` list, `avg_price` is `0` rather than a +division-by-zero error — the untaken branch never runs. diff --git a/docs/guides/higher-order-patterns.md b/docs/guides/higher-order-patterns.md index 84daa4b..43207bf 100644 --- a/docs/guides/higher-order-patterns.md +++ b/docs/guides/higher-order-patterns.md @@ -2,9 +2,9 @@ `map`, `filter`, `any`, `all`, `find`, `count` are the closest thing expr has to control flow over collections. There's no `for` and no -`let`. These six forms, plus the eager `if(cond, t, f)` builtin and -Go's short-circuit `&&` / `||`, are how you make decisions and shape -data. This guide walks the idioms that come up most often and the +`let`. These six forms, plus the lazy `if(cond, t, f)` special form +(only the selected branch evaluates) and Go's short-circuit `&&` / +`||`, are how you make decisions and shape data. This guide walks the idioms that come up most often and the ones you have to work around. A runnable companion lives in diff --git a/docs/guides/registering-functions.md b/docs/guides/registering-functions.md index 2c97010..fd593c0 100644 --- a/docs/guides/registering-functions.md +++ b/docs/guides/registering-functions.md @@ -34,6 +34,28 @@ expr.Compile(src, ) ``` +## Opt-in helper sets + +Before writing your own string/math/list helpers, check the bundled +groups. Each returns a fresh `map[string]any` ready for +`WithFunctions`, and they stay out of `WithBuiltins()` so a minimal +sandbox stays minimal: + +```go +expr.Compile(src, + expr.WithBuiltins(), + expr.WithFunctions(expr.MathFuncs()), // min, max, abs, floor, ceil, round + expr.WithFunctions(expr.StringFuncs()), // trim, split, join, replace, startsWith, endsWith + expr.WithFunctions(expr.CollectionFuncs()), // first, last, sum, slice +) +``` + +All entries are pure and deterministic; see the +[spec](../reference/spec.md#optional-builtin-groups) for exact +signatures and typing rules. Hosts used to re-implement these +slightly differently from each other — prefer the shared versions +unless your semantics genuinely differ. + ## Supported return signatures Only three shapes are legal: @@ -45,9 +67,13 @@ Only three shapes are legal: | `()` | `func(event Event)` (returns `nil` to expr) | Anything else — multiple non-error returns, a second return that -isn't `error` — errors at call time, not compile time. expr can't tell -what's in your map without actually looking at the function's reflect -type at dispatch time. +isn't `error` — fails `Compile` with `ErrCompile`. So does a nil +entry or a value that isn't a Go function at all. The registration +is validated when `Compile` applies its options, which is the point +of the Compile/Run split: a bad entry surfaces when you load the +expression, not on the first request that happens to call it. If you +want to expose a constant, put it in the env instead of the function +map. A function that returns `(T, error)` and returns a non-nil error propagates that error up through `Program.Run`. The error chain is @@ -189,6 +215,11 @@ at the top level. `Compile` — expr doesn't expect that and may read from it again. - **`context.Context` detection is exact.** The parameter type must be literally `context.Context` (the interface), not a type alias. -- **A registered function that panics panics the caller.** expr does - not `recover` around user code. If you want panic safety, wrap in the - function, not the expression. +- **Runtime panics are contained; deliberate panics are not.** A + `runtime.Error` panic inside a registered function (nil deref, + index out of range) comes back from `Run` as an `ErrEvaluate` + instead of crashing the host. An explicit `panic("...")` with any + other value still propagates — that's a deliberate signal expr + won't swallow. Either way, a function that panics on input an + expression can supply is a bug; fix it rather than relying on the + recovery. diff --git a/docs/guides/sandboxing.md b/docs/guides/sandboxing.md index baac091..95ec570 100644 --- a/docs/guides/sandboxing.md +++ b/docs/guides/sandboxing.md @@ -79,6 +79,41 @@ checking `ctx` cannot be interrupted — Go has no way to kill a goroutine. If you register I/O, take a `context.Context` as the first parameter (expr injects it automatically) and honor it. +## Deterministic work bound: `WithEvalBudget` + +Source-length and depth limits bound memory and stack, and context +deadlines bound wall-clock time — but none of them bound *work*. +Higher-order forms multiply: each `map` layer re-evaluates its +predicate per element, so + +``` +map(xs, map(xs, map(xs, it))) +``` + +over a 10k-element env list is 10¹² predicate evaluations from a +~30-byte expression. With only a deadline, that expression burns a +core for the full timeout, every time it is evaluated. + +`WithEvalBudget(n)` makes the bound deterministic: every AST node +evaluated — including each per-element predicate re-evaluation — +consumes one unit, and exhausting the budget fails the `Run` with an +`ErrEvaluate` immediately, not at the deadline: + +```go +p, err := expr.Compile(src, expr.WithBuiltins(), expr.WithEvalBudget(100_000)) +... +_, err = p.Run(ctx, env) // "evaluation budget exceeded (limit 100000)" +``` + +Each `Run` gets the full budget (concurrent Runs do not share a +counter), and `try(...)` cannot catch its way past exhaustion. Size +the budget generously — honest policy expressions rarely evaluate +more than a few thousand nodes, so 10⁵–10⁶ leaves orders of magnitude +of headroom while still rejecting the hostile cases in microseconds. +The budget counts evaluator steps only; time spent *inside* a +registered function is invisible to it, so the context deadline +remains the backstop for slow callees. + ## What to register, and what not to Every function you register is a capability the expression can @@ -103,13 +138,22 @@ Do not register: proxy for SSRF). - Functions that execute code or shell commands (`exec.Command`, `template.Execute`, `reflect.Call`). -- Functions that panic on bad input. +- Functions that panic on bad input. (As defense in depth, expr + converts `runtime.Error` panics — nil derefs, out-of-range indexing + — in any registered function, env callable, or bound method into an + `ErrEvaluate` instead of crashing the host. Deliberate `panic(...)` + calls with other values still propagate. Don't lean on this: a + function that panics on attacker-supplied input is still a bug.) The default set from `WithBuiltins()` is deliberately small and all deterministic / side-effect free: `len`, `string`, `int`, `float`, `bool`, `contains`, `has`, `keys`, `upper`, `lower`, `sprintf`. Nothing there can reach outside the process. If you want a minimal -sandbox, start there. +sandbox, start there. The opt-in groups `expr.MathFuncs()`, +`expr.StringFuncs()`, and `expr.CollectionFuncs()` keep the same +properties (pure, deterministic, allocation bounded by input size) — +register the ones your expressions need via `WithFunctions` without +widening the default surface. ## Auditing a function surface @@ -170,6 +214,9 @@ authors but can reveal the shape of the env to attackers. - [ ] `MaxSourceLength` tuned to your inputs (default 64 KiB). - [ ] `MaxEvalDepth` left at 256 unless you have a reason to raise it. +- [ ] `WithEvalBudget` set if expression authors are untrusted, so + hostile nesting fails deterministically instead of spinning + until the deadline. - [ ] Every `Run` call uses a context with a deadline. - [ ] No registered function does I/O without honoring context. - [ ] No registered function mutates shared state. diff --git a/docs/guides/templates.md b/docs/guides/templates.md index 38577b8..d903377 100644 --- a/docs/guides/templates.md +++ b/docs/guides/templates.md @@ -59,9 +59,11 @@ in its output. If you need to, emit a sentinel in the expression: Nickname: ${if(user.nickname == nil, "(none)", user.nickname)} ``` -`if(cond, t, f)` is the canonical ternary in expr. Both branches -evaluate eagerly, so reach for `try(...)` or operand-returning -`||` when you need to dodge a runtime error in one branch. +`if(cond, t, f)` is the canonical ternary in expr. It is lazy — +only the branch the condition selects evaluates — so guards like +`${if(n != 0, total/n, 0)}` are safe. `try(...)` and +operand-returning `||` remain useful when the *condition itself* +might error. ## The list-stringification footgun diff --git a/docs/reference/spec.md b/docs/reference/spec.md index f0531ec..44778e0 100644 --- a/docs/reference/spec.md +++ b/docs/reference/spec.md @@ -329,7 +329,6 @@ The standard set is: | `int(v)` | `(any) -> int64, error` | Numeric values convert (float truncates toward zero). Strings are parsed strictly with `strconv.ParseInt` base-10 (trimmed whitespace, no `0x`, no trailing garbage). | | `float(v)` | `(any) -> float64, error` | Like `int`, but `strconv.ParseFloat` 64-bit. | | `bool(v)` | `(any) -> bool` | Same semantics as [truthiness](#truthiness). | -| `if(c,t,f)` | `(any, any, any) -> any` | Eager three-argument selector: returns `t` when `c` is truthy, else `f`. Both branches always evaluate; reach for `try`, `&&`, or `\|\|` when one branch must be skipped. | | `contains(h,n)` | `(any, any) -> bool, error` | Substring for string haystacks, element membership for slices/arrays (using [loose equality](#equality)), key presence for string-keyed maps. | | `has(m,k)` | `(any, string) -> bool, error` | True if map `m` has key `k`. Maps only. Nil → `false`. | | `keys(m)` | `(any) -> []any, error` | Sorted string keys. Other key types → error. | @@ -337,14 +336,76 @@ The standard set is: | `upper(s)` | `(string) -> string` | `strings.ToUpper`. | | `sprintf(f,...)`| `(string, ...any) -> string` | `fmt.Sprintf`. | +`if(cond, then, else)` is not a builtin: it is a lazily evaluated +[special form](#higher-order-special-forms), always available without +`WithBuiltins`. + +### Registration validation + +`WithFunctions` entries are validated when `Compile` applies its +options. A nil entry, a value that is not a Go function, or a function +with an unsupported signature (more than two return values, or a +second return that is not `error`) fails `Compile` with `ErrCompile` — +the mistake surfaces at load time rather than on the expression's +first call. Constants belong in the env, not in `WithFunctions`. + +### Optional builtin groups + +Three opt-in helper sets extend the default builtins without widening +a minimal sandbox. Each returns a fresh `map[string]any` for +`WithFunctions`; all entries are deterministic and side-effect free: + +```go +p, err := expr.Compile(src, + expr.WithBuiltins(), + expr.WithFunctions(expr.MathFuncs()), + expr.WithFunctions(expr.StringFuncs()), + expr.WithFunctions(expr.CollectionFuncs()), +) +``` + +`expr.MathFuncs()`: + +| Name | Signature | Notes | +| ----------------- | ------------------------------- | ----- | +| `min(a, ...)` | `(num, ...num) -> num, error` | Smallest argument. `int64` when every argument is integral, `float64` otherwise. At least one argument. | +| `max(a, ...)` | `(num, ...num) -> num, error` | Largest argument; same typing rule as `min`. | +| `abs(n)` | `(num) -> num, error` | Absolute value. `int64` in → `int64` out; `abs(MinInt64)` errors like unary minus. | +| `floor(v)` | `(num) -> num, error` | `math.Floor` for floats; integers pass through unchanged. | +| `ceil(v)` | `(num) -> num, error` | `math.Ceil`; integers pass through. | +| `round(v)` | `(num) -> num, error` | `math.Round` (half away from zero); integers pass through. | + +`expr.StringFuncs()`: + +| Name | Signature | Notes | +| ----------------------- | ------------------------------------ | ----- | +| `trim(s)` | `(string) -> string, error` | `strings.TrimSpace`. | +| `split(s, sep)` | `(string, string) -> []any, error` | `strings.Split`; elements are strings. | +| `join(xs, sep)` | `(list, string) -> string, error` | Elements must be strings. Nil list → `""`. | +| `replace(s, old, new)` | `(string, string, string) -> string, error` | `strings.ReplaceAll`. | +| `startsWith(s, prefix)` | `(string, string) -> bool, error` | `strings.HasPrefix`. | +| `endsWith(s, suffix)` | `(string, string) -> bool, error` | `strings.HasSuffix`. | + +`expr.CollectionFuncs()`: + +| Name | Signature | Notes | +| ----------------- | ------------------------------- | ----- | +| `first(xs)` | `(list) -> any, error` | First element; `nil` for nil or empty lists. | +| `last(xs)` | `(list) -> any, error` | Last element; `nil` for nil or empty lists. | +| `sum(xs)` | `(list) -> num, error` | Numeric sum. `int64` (overflow-checked) when every element is integral, `float64` otherwise. Nil/empty → `0`. | +| `slice(xs, i, j)` | `(list\|string, int, int) -> list\|string, error` | Half-open range `[i, j)`. Lists yield `[]any`; strings slice by rune. Negative indices count from the end; out-of-range bounds clamp; `i > j` → empty. Covers the rejected `xs[i:j]` syntax. | + ## Higher-order special forms -expr also provides a fixed set of **special forms** for iterating -lists. Unlike the standard builtins, the higher-order forms are always -registered and do not require `WithBuiltins`. They look like ordinary -function calls in source, but the second argument (the predicate) is -not evaluated eagerly. Instead, the form re-evaluates the predicate -AST once per element with two extra identifiers in scope: +expr also provides a fixed set of **special forms**. Unlike the +standard builtins, special forms are always registered and do not +require `WithBuiltins`. They look like ordinary function calls in +source, but their arguments are not all evaluated eagerly. Six of +them iterate lists (`map`, `filter`, `any`, `all`, `find`, `count`); +`try` and `if` instead use laziness for error recovery and +branching. For the iterating forms, the second argument (the +predicate) is re-evaluated once per element with two extra +identifiers in scope: - `it` — the current element - `index` — the 0-based position as an `int64` @@ -364,6 +425,7 @@ element and the outer `it` is no longer reachable until the inner | `find(list, pred)` | element or `nil` | First element for which `pred` is truthy, or `nil`. | | `count(list, pred)` | `int64` | Number of elements for which `pred` is truthy. | | `try(value, default)`| value or `default` | Evaluates `value`; returns `default` if `value` raised an `ErrEvaluate` (missing key, type error, out-of-range index, etc.). The `default` expression is only evaluated when the primary fails. | +| `if(cond, then, else)`| `then` or `else` value | Lazy three-argument selector: evaluates `cond`, then **only** the branch selected by `cond`'s [truthiness](#truthiness). Binds no `it`/`index`. | The `list` argument must be a slice or array (or `nil`, which is treated as empty). Maps are not iterated by these forms; use @@ -404,14 +466,30 @@ try(int(input), 0) > 0 try(user.nickname, nil) || "(none)" ``` +`if(cond, then, else)` is the other non-iterating form. It is the +language's ternary: the condition decides via truthiness, and only +the selected branch evaluates, so an error in the untaken branch is +never raised. That makes the guard idiom safe: + +``` +if(n != 0, total / n, 0) // no division-by-zero from the guard +if(user != nil, user.name, "?") // no nil-selector error +``` + +All three arguments see the enclosing scope; `if` binds no implicit +`it`/`index`. (`if` is a Go statement keyword, so like `map` it is +rewritten to an internal token before parsing — invisible except that +it is why `if` can be called at all.) + Special-form names can be shadowed: if `WithFunctions` registers a function with the same name, or the caller's env contains an entry with that name, the user binding wins. This lets consumers replace -the built-in behavior when they need to. The `map` keyword is -special because Go's parser reserves it: expr rewrites `map` to an -internal token before parsing so the form can still be called as -`map(xs, it * 2)`, and translates it back for error messages and -method lookups. +the built-in behavior when they need to — note that a shadowed `if` +or `try` goes through the ordinary call path, where all arguments +evaluate eagerly. The `map` keyword is special because Go's parser +reserves it: expr rewrites `map` to an internal token before parsing +so the form can still be called as `map(xs, it * 2)`, and translates +it back for error messages and method lookups. ## Optional access (`?.` and `?[`) @@ -496,6 +574,14 @@ The following are hard limits: trees deeper than this return `ErrEvaluate: expression nested too deeply`. This caps selector chains (`a.b.c...`), nested binary expressions, and nested calls. +- **Evaluation budget** (opt-in): `WithEvalBudget(n)` bounds the total + number of AST nodes a single `Run` may evaluate, counting every + per-element re-evaluation of a higher-order predicate. Exhausting + the budget returns `ErrEvaluate: evaluation budget exceeded`. This + is the only deterministic CPU bound — source length and depth limits + do not stop nested higher-order forms from multiplying work + (`map(xs, map(xs, map(xs, it)))` is `len(xs)³` predicate + evaluations from a ~30-byte expression). The default is unlimited. Under adversarial input, expr must never: diff --git a/docs_examples_test.go b/docs_examples_test.go index 88e1d32..0118fc7 100644 --- a/docs_examples_test.go +++ b/docs_examples_test.go @@ -379,3 +379,49 @@ func TestDocsExample9_WebhookPredicate(t *testing.T) { } assertDeepEqual(t, got, want) } + +// Example 10: Guarded math and the opt-in helper groups. +func TestDocsExample10_GuardedMathAndGroups(t *testing.T) { + src := `{ + "avg_price": if(len(prices) > 0, sum(prices) / len(prices), 0), + "spread": if(len(prices) > 0, max(0, last(prices) - first(prices)), 0), + "top_three": slice(prices, 0, 3), + "sku_list": join(map(items, upper(it.sku)), ", "), + }` + opts := []Option{ + WithBuiltins(), + WithFunctions(MathFuncs()), + WithFunctions(StringFuncs()), + WithFunctions(CollectionFuncs()), + } + env := map[string]any{ + "prices": []any{int64(10), int64(20), int64(60)}, + "items": []any{ + map[string]any{"sku": "a-1"}, + map[string]any{"sku": "b-2"}, + }, + } + got := runDocExample(t, src, env, opts...) + want := map[string]any{ + "avg_price": int64(30), + "spread": int64(50), + "top_three": []any{int64(10), int64(20), int64(60)}, + "sku_list": "A-1, B-2", + } + assertDeepEqual(t, got, want) + + // The doc claims the empty-list case returns 0 instead of a + // division-by-zero error, because if() is lazy. + emptyEnv := map[string]any{ + "prices": []any{}, + "items": []any{}, + } + got = runDocExample(t, src, emptyEnv, opts...) + want = map[string]any{ + "avg_price": int64(0), + "spread": int64(0), + "top_three": []any{}, + "sku_list": "", + } + assertDeepEqual(t, got, want) +} diff --git a/docs_guides_test.go b/docs_guides_test.go index 7264c4f..4433844 100644 --- a/docs_guides_test.go +++ b/docs_guides_test.go @@ -128,9 +128,9 @@ func (v guideOrderView) Subtotal() float64 { return v.subtotal } func TestGuide_DesigningEnv_PointerStruct(t *testing.T) { o := &guideOrder{ - ID: "A-1", + ID: "A-1", Items: []guideLineItem{{SKU: "a", Price: 60}, {SKU: "b", Price: 80}}, - Meta: map[string]any{"source": "web"}, + Meta: map[string]any{"source": "web"}, } src := `Subtotal() > 100 && len(Items) >= 2 && !has(Meta, "refunded")` got := runGuide(t, src, o) @@ -411,3 +411,74 @@ func TestGuide_HigherOrder_EmptyListSemantics(t *testing.T) { } } } + +func TestGuide_Sandboxing_EvalBudget(t *testing.T) { + // The guide claims WithEvalBudget fails hostile nesting + // deterministically and immediately, not at the deadline. + p, err := Compile(`map(xs, map(xs, map(xs, it)))`, WithBuiltins(), WithEvalBudget(100_000)) + if err != nil { + t.Fatalf("compile: %v", err) + } + xs := make([]any, 10_000) + for i := range xs { + xs[i] = int64(i) + } + _, err = p.Run(context.Background(), map[string]any{"xs": xs}) + if !errors.Is(err, ErrEvaluate) { + t.Fatalf("expected ErrEvaluate, got %v", err) + } + if !strings.Contains(err.Error(), "evaluation budget exceeded") { + t.Fatalf("expected budget error, got %v", err) + } +} + +func TestGuide_Templates_LazyIfGuard(t *testing.T) { + // templates.md claims `${if(n != 0, total/n, 0)}` is safe when n + // is zero because only the selected branch evaluates. + tpl, err := NewTemplate(`${if(n != 0, total/n, 0)}`, WithBuiltins()) + if err != nil { + t.Fatalf("compile: %v", err) + } + out, err := tpl.Render(context.Background(), map[string]any{"n": int64(0), "total": int64(10)}) + if err != nil { + t.Fatalf("render: %v", err) + } + assertDeepEqual(t, out, "0") +} + +func TestGuide_RegisteringFunctions_CompileTimeValidation(t *testing.T) { + // registering-functions.md claims invalid registrations fail + // Compile with ErrCompile rather than erroring at call time. + _, err := Compile(`bad()`, WithFunctions(map[string]any{ + "bad": func() (int, string) { return 1, "x" }, + })) + if !errors.Is(err, ErrCompile) { + t.Fatalf("expected ErrCompile, got %v", err) + } +} + +func TestGuide_RegisteringFunctions_HelperGroups(t *testing.T) { + // The opt-in groups snippet from registering-functions.md. + p, err := Compile(`join(map(split(trim(" a,b,c "), ","), upper(it)), "-")`, + WithBuiltins(), + WithFunctions(MathFuncs()), + WithFunctions(StringFuncs()), + WithFunctions(CollectionFuncs()), + ) + if err != nil { + t.Fatalf("compile: %v", err) + } + got, err := p.Run(context.Background(), map[string]any{}) + if err != nil { + t.Fatalf("run: %v", err) + } + assertDeepEqual(t, got, "A-B-C") +} + +func TestGuide_HigherOrder_LazyIf(t *testing.T) { + // higher-order-patterns.md claims only the selected branch of + // if(cond, t, f) evaluates. + env := map[string]any{"xs": []any{int64(1)}} + got := runGuide(t, `if(len(xs) > 5, xs[5], "small")`, env) + assertDeepEqual(t, got, "small") +} diff --git a/engine.go b/engine.go index 52f196d..c760889 100644 --- a/engine.go +++ b/engine.go @@ -70,9 +70,11 @@ type Option func(*compileConfig) // It is consumed during parsing to build the function dispatch tables // baked into the resulting Program. type compileConfig struct { - funcs map[string]any - prepared map[string]*preparedFunc - fieldTags *structTagConfig + funcs map[string]any + prepared map[string]*preparedFunc + fieldTags *structTagConfig + evalBudget int + errs []error } func newCompileConfig() *compileConfig { @@ -98,15 +100,22 @@ func WithBuiltins() Option { // values to the declared parameter types at call time. Return signatures of // `T`, `(T, error)`, and `()` are supported. Variadic functions are also // supported. +// +// Invalid registrations — a nil entry, a value that is not a Go +// function, or a function with an unsupported signature (more than two +// return values, or a second return that is not error) — cause Compile +// to fail with ErrCompile. Surfacing the mistake at load time is the +// point of the Compile/Run split; before this check the error would +// hide until the expression first called the bad entry. func WithFunctions(funcs map[string]any) Option { return func(c *compileConfig) { for name, fn := range funcs { c.funcs[name] = fn pf, err := prepareFunc(name, fn) if err != nil { - // Defer error surfacing to call-time; store a - // nil-native preparedFunc entry so lookups still - // find the name (useful for better error hints). + c.errs = append(c.errs, err) + // Keep a name-only entry so later options can still + // override it and lookups find the name. c.prepared[name] = &preparedFunc{name: name} continue } @@ -115,6 +124,27 @@ func WithFunctions(funcs map[string]any) Option { } } +// WithEvalBudget bounds the total work a single Run may perform. Each +// AST node evaluated — including every per-element re-evaluation of a +// higher-order form's predicate — consumes one unit; when the budget +// is exhausted, Run fails with an ErrEvaluate-wrapped error. +// +// MaxSourceLength and MaxEvalDepth bound memory and stack, but not +// CPU: nested higher-order forms multiply, so a ~60-byte expression +// like map(xs, map(xs, map(xs, it))) over a 10k-element list is 10^12 +// predicate evaluations. Context deadlines cap wall-clock time but +// still let one expression burn a core for the full timeout; a budget +// makes hostile-input behavior deterministic and cheap to reject. +// +// n <= 0 means unlimited (the default). The budget counts evaluator +// steps, not time spent inside registered functions — bound those +// separately (see docs/guides/sandboxing.md). +func WithEvalBudget(n int) Option { + return func(c *compileConfig) { + c.evalBudget = n + } +} + // WithStructTags enables struct field lookup by the named struct tags. // Tags are checked in the order provided before falling back to the Go // exported field name. Tag options after a comma are ignored, so @@ -152,6 +182,9 @@ func Compile(code string, opts ...Option) (*Program, error) { for _, opt := range opts { opt(cfg) } + if len(cfg.errs) > 0 { + return nil, fmt.Errorf("%w: %w", ErrCompile, errors.Join(cfg.errs...)) + } // Pipeline order: optaccess turns `?.`/`?[` into sentinel calls // while the source still uses raw operator syntax; jsonlit then // rewrites bare composite literals; preprocessSource handles the @@ -167,11 +200,12 @@ func Compile(code string, opts ...Option) (*Program, error) { return nil, err } p := &Program{ - source: code, - root: node, - funcs: cfg.funcs, - prepared: cfg.prepared, - fieldTags: cfg.fieldTags, + source: code, + root: node, + funcs: cfg.funcs, + prepared: cfg.prepared, + fieldTags: cfg.fieldTags, + evalBudget: cfg.evalBudget, } p.compile() return p, nil @@ -305,7 +339,7 @@ func matchRewrite(tok token.Token) (struct { tok token.Token src string internal string -}{}, false + }{}, false } // displayIdent converts an internal rewritten identifier back to the diff --git a/engine_test.go b/engine_test.go index 69f44c3..fa4a4e6 100644 --- a/engine_test.go +++ b/engine_test.go @@ -414,10 +414,10 @@ func TestCompile_SyntaxError(t *testing.T) { func TestEval_UnsupportedSyntax(t *testing.T) { cases := []string{ - "state.items[1:3]", // slice expression - "x.(int)", // type assertion - "func() int { 1 }()", // function literal - "[]int{1, 2, 3}", // composite literal + "state.items[1:3]", // slice expression + "x.(int)", // type assertion + "func() int { 1 }()", // function literal + "[]int{1, 2, 3}", // composite literal } for _, expr := range cases { t.Run(expr, func(t *testing.T) { @@ -455,7 +455,7 @@ func TestEval_EnvCallable(t *testing.T) { env := map[string]any{ "name": "ada", "upper": strings.ToUpper, - "addN": func(n, m int) int { return n + m }, + "addN": func(n, m int) int { return n + m }, "greet": func(who string) (string, error) { if who == "" { return "", errors.New("empty name") diff --git a/examples/sandboxing/main.go b/examples/sandboxing/main.go index 723f8c7..e98b2af 100644 --- a/examples/sandboxing/main.go +++ b/examples/sandboxing/main.go @@ -24,7 +24,10 @@ func main() { } // Compile once. Only builtins are registered — no I/O, no mutation. - p, err := expr.Compile(src, expr.WithBuiltins()) + // The eval budget caps total work per Run, so hostile nesting like + // map(xs, map(xs, map(xs, it))) fails deterministically instead of + // burning a core until the deadline. + p, err := expr.Compile(src, expr.WithBuiltins(), expr.WithEvalBudget(100_000)) if err != nil { panic(err) } diff --git a/higher_order.go b/higher_order.go index 120d762..085abfb 100644 --- a/higher_order.go +++ b/higher_order.go @@ -46,7 +46,10 @@ type userForm struct { // callHint is the signature shown in the "is a special form" // suggester message, e.g. `map(xs, predicate)`. callHint string - fn higherOrderForm + // bindsIt marks the iterating forms, which bind `it`/`index` + // inside their second argument. Drives the identifier collector. + bindsIt bool + fn higherOrderForm } // userForms enumerates every user-visible special form. Order is @@ -68,19 +71,28 @@ var userForms []userForm // expr. See the dispatch in evalCall. var higherOrderForms map[string]higherOrderForm +// itBindingForms holds the dispatch keys of the forms that bind +// `it`/`index` in their second argument, derived from userForms. +var itBindingForms map[string]bool + func init() { userForms = []userForm{ - {name: "map", internal: mapFormName, callHint: "map(xs, predicate)", fn: formMap}, - {name: "filter", internal: "filter", callHint: "filter(xs, predicate)", fn: formFilter}, - {name: "any", internal: "any", callHint: "any(xs, predicate)", fn: formAny}, - {name: "all", internal: "all", callHint: "all(xs, predicate)", fn: formAll}, - {name: "find", internal: "find", callHint: "find(xs, predicate)", fn: formFind}, - {name: "count", internal: "count", callHint: "count(xs, predicate)", fn: formCount}, + {name: "map", internal: mapFormName, callHint: "map(xs, predicate)", bindsIt: true, fn: formMap}, + {name: "filter", internal: "filter", callHint: "filter(xs, predicate)", bindsIt: true, fn: formFilter}, + {name: "any", internal: "any", callHint: "any(xs, predicate)", bindsIt: true, fn: formAny}, + {name: "all", internal: "all", callHint: "all(xs, predicate)", bindsIt: true, fn: formAll}, + {name: "find", internal: "find", callHint: "find(xs, predicate)", bindsIt: true, fn: formFind}, + {name: "count", internal: "count", callHint: "count(xs, predicate)", bindsIt: true, fn: formCount}, {name: "try", internal: "try", callHint: "try(value, default)", fn: formTry}, + {name: "if", internal: ifFuncName, callHint: "if(cond, then, else)", fn: formIf}, } higherOrderForms = make(map[string]higherOrderForm, len(userForms)+2) + itBindingForms = make(map[string]bool, len(userForms)) for _, f := range userForms { higherOrderForms[f.internal] = f.fn + if f.bindsIt { + itBindingForms[f.internal] = true + } } // Sentinel forms emitted by the optaccess pre-parse rewrite. // Users do not type these names; they appear only as the @@ -637,6 +649,30 @@ func tryIndexValue(recv, idx any) (any, error) { return nil, fmt.Errorf("%w: cannot index %T", ErrEvaluate, recv) } +// formIf implements `if(cond, then, else)` as a lazy special form: +// only the branch selected by the condition's truthiness is +// evaluated, so the guard idiom `if(n != 0, total/n, 0)` works +// without tripping over the untaken branch. `if` binds no implicit +// `it`/`index`; all three arguments see the enclosing scope. +// +// Like every special form it can be shadowed: a function registered +// under "if" or an env entry of that name wins, restoring eager +// argument evaluation through the normal call path. +func formIf(p *Program, ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { + if len(n.Args) != 3 { + return nil, fmt.Errorf("%w: if expects 3 arguments (cond, then, else), got %d", + ErrEvaluate, len(n.Args)) + } + cond, err := p.eval(ctx, n.Args[0], env, depth) + if err != nil { + return nil, err + } + if isTruthy(cond) { + return p.eval(ctx, n.Args[1], env, depth) + } + return p.eval(ctx, n.Args[2], env, depth) +} + // formTry implements `try(value, default)`. It evaluates the first // argument; if evaluation returns an ErrEvaluate, it evaluates and // returns the second argument instead. The default expression is diff --git a/identifiers.go b/identifiers.go new file mode 100644 index 0000000..686d86a --- /dev/null +++ b/identifiers.go @@ -0,0 +1,115 @@ +package expr + +import ( + "go/ast" + "sort" +) + +// Identifiers returns the sorted, de-duplicated set of top-level +// identifier names the expression references — the names Run will +// try to resolve through the environment. Hosts can use it to +// validate an expression against a known env shape at load time, to +// track dependencies for cache invalidation, or to decide which +// values are worth computing before a Run. +// +// Excluded from the result: +// +// - the literals true, false, and nil +// - `it` and `index` where they are bound by an iterating +// higher-order form (map, filter, any, all, find, count) +// - names registered via WithFunctions / WithBuiltins, which +// resolve without the env +// - special-form names (map, filter, try, if, ...) in call +// position, which are always available +// +// The analysis is static and therefore best-effort in one corner: +// env entries can shadow registered functions and special forms at +// Run time, so an excluded name may still be read from the env when +// a host deliberately shadows it. Every name that can only resolve +// through the env is always included. +func (p *Program) Identifiers() []string { + out := make([]string, len(p.identifiers)) + copy(out, p.identifiers) + return out +} + +// collectIdentifiers walks the compiled AST once and gathers the +// env-resolved identifier set per the Identifiers contract. +func collectIdentifiers(root ast.Expr, funcs map[string]any) []string { + seen := map[string]struct{}{} + walkIdentifiers(root, false, funcs, seen) + out := make([]string, 0, len(seen)) + for name := range seen { + out = append(out, name) + } + sort.Strings(out) + return out +} + +// walkIdentifiers visits node and records env-resolved identifier +// names into seen. itBound reports whether the walk is inside the +// predicate of an iterating form, where `it` and `index` are bound by +// the form rather than the env. +func walkIdentifiers(node ast.Expr, itBound bool, funcs map[string]any, seen map[string]struct{}) { + switch n := node.(type) { + case *ast.Ident: + name := displayIdent(n.Name) + switch name { + case "true", "false", "nil": + return + } + if itBound && (name == "it" || name == "index") { + return + } + if _, registered := funcs[name]; registered { + return + } + seen[name] = struct{}{} + case *ast.ParenExpr: + walkIdentifiers(n.X, itBound, funcs, seen) + case *ast.UnaryExpr: + walkIdentifiers(n.X, itBound, funcs, seen) + case *ast.BinaryExpr: + walkIdentifiers(n.X, itBound, funcs, seen) + walkIdentifiers(n.Y, itBound, funcs, seen) + case *ast.SelectorExpr: + // n.Sel is a field/key/method name on the receiver, not an + // env identifier. + walkIdentifiers(n.X, itBound, funcs, seen) + case *ast.IndexExpr: + walkIdentifiers(n.X, itBound, funcs, seen) + walkIdentifiers(n.Index, itBound, funcs, seen) + case *ast.CallExpr: + if ident, ok := n.Fun.(*ast.Ident); ok { + if _, isForm := higherOrderForms[ident.Name]; isForm { + if _, shadowed := funcs[displayIdent(ident.Name)]; !shadowed { + // Special-form call: the form name resolves without + // the env, and iterating forms bind it/index inside + // their predicate argument. + if itBindingForms[ident.Name] && len(n.Args) == 2 { + walkIdentifiers(n.Args[0], itBound, funcs, seen) + walkIdentifiers(n.Args[1], true, funcs, seen) + return + } + for _, a := range n.Args { + walkIdentifiers(a, itBound, funcs, seen) + } + return + } + } + } + walkIdentifiers(n.Fun, itBound, funcs, seen) + for _, a := range n.Args { + walkIdentifiers(a, itBound, funcs, seen) + } + case *ast.CompositeLit: + for _, e := range n.Elts { + if kv, ok := e.(*ast.KeyValueExpr); ok { + walkIdentifiers(kv.Key, itBound, funcs, seen) + walkIdentifiers(kv.Value, itBound, funcs, seen) + continue + } + walkIdentifiers(e, itBound, funcs, seen) + } + } +} diff --git a/identifiers_test.go b/identifiers_test.go new file mode 100644 index 0000000..2cc8335 --- /dev/null +++ b/identifiers_test.go @@ -0,0 +1,87 @@ +package expr + +import ( + "context" + "testing" + + "github.com/deepnoodle-ai/expr/internal/require" +) + +func compileIdents(t *testing.T, src string, opts ...Option) []string { + t.Helper() + p, err := Compile(src, opts...) + require.NoError(t, err) + return p.Identifiers() +} + +func TestIdentifiers_Basic(t *testing.T) { + require.Equal(t, []string{"a", "b", "d"}, compileIdents(t, "a + b.c[d]")) +} + +func TestIdentifiers_SortedAndDeduped(t *testing.T) { + require.Equal(t, []string{"a", "z"}, compileIdents(t, "z + a + z + a")) +} + +func TestIdentifiers_LiteralsExcluded(t *testing.T) { + require.Equal(t, []string{"x"}, compileIdents(t, "x == nil || true || false")) +} + +func TestIdentifiers_ItIndexBoundInsideForms(t *testing.T) { + // it/index inside an iterating form are bound by the form, not + // read from the env. + require.Equal(t, []string{"xs"}, compileIdents(t, "filter(xs, it.age > 18 && index < 10)")) + // The map keyword goes through the keyword rewrite; same rule. + require.Equal(t, []string{"xs"}, compileIdents(t, "map(xs, it * 2)")) + // Nested forms keep the binding. + require.Equal(t, []string{"users"}, compileIdents(t, "map(users, map(it.friends, it.name))")) +} + +func TestIdentifiers_ItAtTopLevelIncluded(t *testing.T) { + // Outside a form, `it` is an ordinary env identifier. + require.Equal(t, []string{"it"}, compileIdents(t, "it + 1")) +} + +func TestIdentifiers_PredicateOuterNamesIncluded(t *testing.T) { + // Names other than it/index inside a predicate still resolve + // through the env. + require.Equal(t, []string{"threshold", "xs"}, compileIdents(t, "any(xs, it > threshold)")) +} + +func TestIdentifiers_FormNamesExcluded(t *testing.T) { + require.Equal(t, []string{"a", "b", "v"}, compileIdents(t, "try(v.x, nil) || if(a, b, 0)")) +} + +func TestIdentifiers_RegisteredFunctionsExcluded(t *testing.T) { + require.Equal(t, []string{"name"}, compileIdents(t, "upper(name)", WithBuiltins())) + // Without registration the same call target needs the env. + require.Equal(t, []string{"name", "upper"}, compileIdents(t, "upper(name)")) +} + +func TestIdentifiers_ShadowedFormIsOrdinaryCall(t *testing.T) { + // A registered "filter" turns the call into an ordinary function + // call: no it/index binding, and the name resolves via funcs. + idents := compileIdents(t, "filter(xs, it)", WithFunctions(map[string]any{ + "filter": Func(func(_ context.Context, _ []any) (any, error) { return nil, nil }), + })) + require.Equal(t, []string{"it", "xs"}, idents) +} + +func TestIdentifiers_OptionalAccess(t *testing.T) { + require.Equal(t, []string{"i", "user"}, compileIdents(t, "user?.profile?.name == user?[i]")) +} + +func TestIdentifiers_JSONLiterals(t *testing.T) { + require.Equal(t, []string{"v", "x", "y"}, compileIdents(t, `{"k": v, "xs": [x, y]}`)) +} + +func TestIdentifiers_ReturnsCopy(t *testing.T) { + p, err := Compile("a + b") + require.NoError(t, err) + first := p.Identifiers() + first[0] = "mutated" + require.Equal(t, []string{"a", "b"}, p.Identifiers()) +} + +func TestIdentifiers_EmptyForPureLiteral(t *testing.T) { + require.Len(t, compileIdents(t, `1 + 2`), 0) +} diff --git a/llms.txt b/llms.txt index c0b9511..bb3bc5a 100644 --- a/llms.txt +++ b/llms.txt @@ -24,9 +24,11 @@ language reference is [`docs/reference/spec.md`](docs/reference/spec.md). no bytecode, no VM, no optimizer. This is deliberate — the whole library is ~3.9k non-test Go LOC. - **Safe by default.** `MaxSourceLength` (64 KiB) caps the parser; - `MaxEvalDepth` (256) caps the evaluator. Bitwise ops, pointer ops, - channel ops, spread args, type assertions, and function literals - are all rejected at eval time. + `MaxEvalDepth` (256) caps the evaluator; opt-in `WithEvalBudget(n)` + caps total nodes evaluated per Run so hostile nesting fails + deterministically. Bitwise ops, pointer ops, channel ops, spread + args, type assertions, and function literals are all rejected at + eval time. - **Loosely typed at runtime.** Integers are `int64`, floats are `float64`, numeric comparisons mix freely, and truthiness covers nil/false/zero/empty-string/empty-collection. @@ -86,6 +88,11 @@ any shared name. | `expr.WithBuiltins()` | Registers the standard builtin function set (see below) | | `expr.WithFunctions(map[string]any{})`| Registers arbitrary Go functions as callable identifiers | | `expr.WithStructTags("json")` | Opts struct field lookup into tag names before Go names | +| `expr.WithEvalBudget(n)` | Caps AST nodes evaluated per Run (deterministic CPU bound) | + +`WithFunctions` entries are validated at Compile time: a nil entry, a +non-function value, or an unsupported signature fails Compile with +`ErrCompile`. Constants go in the env, not the function map. Builtins are opt-in: if you don't pass `WithBuiltins`, the only callable names are the higher-order forms (always registered) and anything you @@ -100,7 +107,6 @@ registered yourself. This keeps the sandbox surface as narrow as you want. | `int(v)` | `(any) -> int64` | Truncates floats; strict base-10 parse for strings | | `float(v)` | `(any) -> float64` | `strconv.ParseFloat` 64-bit for strings | | `bool(v)` | `(any) -> bool` | Same rules as truthiness | -| `if(c, t, f)` | `(any, any, any) -> any` | Eager ternary; `t` and `f` both evaluate before the call | | `contains(h, n)` | `(any, any) -> bool` | Substring, element, or map-key presence | | `has(m, k)` | `(any, string) -> bool` | Key presence on a map (nil → false) | | `keys(m)` | `(any) -> []any` | Sorted string keys | @@ -108,6 +114,18 @@ registered yourself. This keeps the sandbox surface as narrow as you want. | `lower(s)` | `(string) -> string` | `strings.ToLower` | | `sprintf(f, ...)` | `(string, ...any) -> string` | `fmt.Sprintf` | +Opt-in helper groups (register with `WithFunctions`, kept out of +`WithBuiltins` so minimal sandboxes stay minimal): + +- `expr.MathFuncs()` — `min`, `max` (variadic; int64 when all args + integral, else float64), `abs`, `floor`, `ceil`, `round` (integers + pass through unchanged) +- `expr.StringFuncs()` — `trim(s)`, `split(s, sep)`, `join(xs, sep)`, + `replace(s, old, new)`, `startsWith(s, p)`, `endsWith(s, p)` +- `expr.CollectionFuncs()` — `first(xs)`, `last(xs)` (nil for empty), + `sum(xs)` (int64 unless a float appears), `slice(xs, i, j)` + (half-open, negative indices from end, clamps; lists and strings) + ## Higher-order forms (always registered) The second argument is a predicate AST that is re-evaluated per element @@ -125,6 +143,7 @@ you need to). | `find(list, pred)` | element or `nil` | First match or nil | | `count(list, pred)` | `int64` | Number of matches | | `try(value, default)` | value or default | `value` if it evaluates cleanly, else `default` | +| `if(cond, then, else)`| branch value | Lazy ternary: only the selected branch evaluates | When a predicate inside an iterating form errors, the form wraps the error with its name, the predicate's source text, and the failing @@ -134,10 +153,12 @@ add their own layer. The wrapping preserves the underlying chain so `errors.Is(err, ErrEvaluate)` still matches; cancellation passes through unchanged. -`try` is the odd one out: it does not iterate a list, binds no `it` -or `index`, and traps anything wrapping `ErrEvaluate` (missing keys, -nil selectors, out-of-range indices, `int`/`float` parse failures). -The `default` expression is lazy: it runs only when the primary fails. +`try` and `if` are the odd ones out: they do not iterate a list and +bind no `it` or `index`. `try` traps anything wrapping `ErrEvaluate` +(missing keys, nil selectors, out-of-range indices, `int`/`float` +parse failures); its `default` runs only when the primary fails. +`if(cond, then, else)` evaluates only the branch selected by `cond`'s +truthiness, so `if(n != 0, total/n, 0)` cannot divide by zero. `try` does **not** trap raw `context.Canceled`, `context.DeadlineExceeded`, or anything wrapping `ErrCompile`. @@ -309,9 +330,14 @@ are wrapped so `errors.Is` and `errors.As` still find the original cause. Unknown identifiers, fields, and keys get a Levenshtein-based "did you mean...?" hint drawn from what's actually in scope. Higher-order forms (`map`, `filter`, `any`, `all`, `find`, `count`, -`try`) referenced as bare identifiers get a tailored hint showing -their call signature, e.g. `"count" is a special form, did you -mean to call count(xs, predicate)?`. +`try`, `if`) referenced as bare identifiers get a tailored hint +showing their call signature, e.g. `"count" is a special form, did +you mean to call count(xs, predicate)?`. + +`Program.Identifiers()` returns the sorted set of env-resolved names +the expression references (excluding literals, bound `it`/`index`, +registered functions, and special forms) — useful for validating an +expression against a known env shape at load time. ```go if errors.Is(err, expr.ErrCompile) { /* ... */ } diff --git a/prepared.go b/prepared.go index a4949fb..1148f3d 100644 --- a/prepared.go +++ b/prepared.go @@ -45,17 +45,17 @@ type preparedFunc struct { // function's reflect metadata is cached for the fallback path. func prepareFunc(name string, fn any) (*preparedFunc, error) { if fn == nil { - return nil, fmt.Errorf("expr: function %q is nil", name) + return nil, fmt.Errorf("function %q is nil", name) } if nf, ok := fn.(Func); ok { return &preparedFunc{name: name, native: nf}, nil } fv := reflect.ValueOf(fn) if fv.Kind() != reflect.Func { - return nil, fmt.Errorf("expr: function %q is not a function (got %T)", name, fn) + return nil, fmt.Errorf("function %q is not a function (got %T)", name, fn) } if fv.IsNil() { - return nil, fmt.Errorf("expr: function %q is a nil function value", name) + return nil, fmt.Errorf("function %q is a nil function value", name) } ft := fv.Type() p := &preparedFunc{ @@ -82,11 +82,11 @@ func prepareFunc(name string, fn any) (*preparedFunc, error) { if p.numOut == 2 { errType := reflect.TypeOf((*error)(nil)).Elem() if !ft.Out(1).Implements(errType) { - return nil, fmt.Errorf("expr: function %q: second return must be error, got %v", name, ft.Out(1)) + return nil, fmt.Errorf("function %q: second return must be error, got %v", name, ft.Out(1)) } p.hasErrRet = true } else if p.numOut > 2 { - return nil, fmt.Errorf("expr: function %q returns %d values (expected 0, 1, or (T, error))", name, p.numOut) + return nil, fmt.Errorf("function %q returns %d values (expected 0, 1, or (T, error))", name, p.numOut) } return p, nil } @@ -139,7 +139,10 @@ func callPreparedReflect(ctx context.Context, pf *preparedFunc, args []any) (any } } - out := pf.fv.Call(in) + out, err := callReflectRecoverRuntime(pf.name, pf.fv, in) + if err != nil { + return nil, err + } switch pf.numOut { case 0: return nil, nil diff --git a/program.go b/program.go index 0b8e39b..f3c8229 100644 --- a/program.go +++ b/program.go @@ -30,6 +30,18 @@ type Program struct { // evalLiteral is a single map lookup instead of repeating // strconv work on every Run. litCache map[*ast.BasicLit]any + + // identifiers is the sorted set of environment-resolved names the + // expression references, computed once during compile(). See + // Identifiers for the exact rules. + identifiers []string + + // evalBudget is the per-Run node-evaluation limit configured via + // WithEvalBudget; 0 means unlimited. budgetLeft is nil on the + // shared Program — Run works on a shallow copy holding a fresh + // counter so concurrent Runs never share budget state. + evalBudget int + budgetLeft *int64 } // compile walks the root AST once to populate the lookup caches used @@ -40,6 +52,7 @@ func (p *Program) compile() { p.callCache = map[*ast.CallExpr]*preparedFunc{} p.litCache = map[*ast.BasicLit]any{} p.root = p.prewalk(p.root) + p.identifiers = collectIdentifiers(p.root, p.funcs) } // constValue reports the pre-computed value of a folded/literal node, @@ -260,6 +273,16 @@ func (p *Program) Run(ctx context.Context, env any) (any, error) { if ctx == nil { ctx = context.Background() } + if p.evalBudget > 0 { + // Evaluate through a shallow copy carrying a fresh counter so + // the shared Program stays immutable and concurrent Runs each + // get the full budget. The copy shares the (read-only) AST + // and caches. + run := *p + left := int64(p.evalBudget) + run.budgetLeft = &left + return run.eval(ctx, run.root, env, 0) + } return p.eval(ctx, p.root, env, 0) } @@ -270,6 +293,12 @@ func (p *Program) eval(ctx context.Context, node ast.Expr, env any, depth int) ( if depth >= MaxEvalDepth { return nil, fmt.Errorf("%w: expression nested too deeply (limit %d)", ErrEvaluate, MaxEvalDepth) } + if p.budgetLeft != nil { + if *p.budgetLeft <= 0 { + return nil, fmt.Errorf("%w: evaluation budget exceeded (limit %d)", ErrEvaluate, p.evalBudget) + } + *p.budgetLeft-- + } depth++ switch n := node.(type) { case *ast.BasicLit: @@ -1092,33 +1121,25 @@ func (p *Program) evalCall(ctx context.Context, n *ast.CallExpr, env any, depth // unevaluated so it can be re-run per element with `it`/`index` // bound in an itEnv scope. Identifier shadowing follows the same // env→funcs order used by resolveCallable, so users can always - // register their own `map` or `filter` if they prefer. + // register their own `map`, `filter`, or `if` if they prefer. // - // `map` is special: the preprocessing pass rewrote its token to - // mapFormName so Go's parser would accept it, but shadowing - // checks still have to use the user-visible name "map" — a user - // who calls WithFunctions({"map": ...}) expects their function - // to win, and env entries named "map" should be honored too. + // Keyword-backed forms (`map`, `if`) dispatch on their internal + // sentinel name, but shadowing checks use the user-visible name + // via displayIdent — a user who calls WithFunctions({"map": ...}) + // expects their function to win, and env entries named "map" + // should be honored too. if ident, isIdent := n.Fun.(*ast.Ident); isIdent { - if ident.Name == mapFormName { - if v, ok, err := lookupEnv(env, "map", p.fieldTags); err != nil { + if form, isForm := higherOrderForms[ident.Name]; isForm { + name := displayIdent(ident.Name) + if v, ok, err := lookupEnv(env, name, p.fieldTags); err != nil { return nil, err } else if ok { - return p.callValue(ctx, "map", v, n.Args, env, depth) + return p.callValue(ctx, name, v, n.Args, env, depth) } - if v, ok := p.funcs["map"]; ok { - return p.callValue(ctx, "map", v, n.Args, env, depth) - } - return formMap(p, ctx, n, env, depth) - } - if form, isForm := higherOrderForms[ident.Name]; isForm { - if _, inEnv, err := lookupEnv(env, ident.Name, p.fieldTags); err != nil { - return nil, err - } else if !inEnv { - if _, inFuncs := p.funcs[ident.Name]; !inFuncs { - return form(p, ctx, n, env, depth) - } + if v, ok := p.funcs[name]; ok { + return p.callValue(ctx, name, v, n.Args, env, depth) } + return form(p, ctx, n, env, depth) } } diff --git a/reflect.go b/reflect.go index 6e15cb9..9d9a5f6 100644 --- a/reflect.go +++ b/reflect.go @@ -35,19 +35,27 @@ func callFunction(ctx context.Context, name string, fn any, args []any) (any, er return nil, err } - return finishCall(name, ft, callReflect(fv, in)) -} - -func callReflect(fv reflect.Value, in []reflect.Value) []reflect.Value { - return fv.Call(in) + out, err := callReflectRecoverRuntime(name, fv, in) + if err != nil { + return nil, err + } + return finishCall(name, ft, out) } +// callReflectRecoverRuntime invokes fv and converts runtime.Error +// panics (nil deref, index out of range, ...) in the callee into an +// ErrEvaluate so one buggy function cannot take down the host through +// Run. Every reflect-based dispatch path — bound methods, env-stored +// functions, and WithFunctions-registered functions — goes through +// this wrapper. Deliberate panics with non-runtime values are +// re-raised: those signal programmer intent that expr should not +// swallow. func callReflectRecoverRuntime(name string, fv reflect.Value, in []reflect.Value) (out []reflect.Value, err error) { defer func() { if r := recover(); r != nil { if _, ok := r.(runtime.Error); ok { out = nil - err = fmt.Errorf("%w: %q panicked during method dispatch: %v", ErrEvaluate, name, r) + err = fmt.Errorf("%w: %q panicked during call: %v", ErrEvaluate, name, r) return } panic(r) diff --git a/registration_test.go b/registration_test.go new file mode 100644 index 0000000..b1effd6 --- /dev/null +++ b/registration_test.go @@ -0,0 +1,108 @@ +package expr + +import ( + "context" + "testing" + + "github.com/deepnoodle-ai/expr/internal/require" +) + +// --- Compile-time registration validation --------------------------------- + +func TestRegistration_NonFunctionFailsCompile(t *testing.T) { + _, err := Compile("f()", WithFunctions(map[string]any{"f": 42})) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), `function "f" is not a function`) +} + +func TestRegistration_NilEntryFailsCompile(t *testing.T) { + _, err := Compile("f()", WithFunctions(map[string]any{"f": nil})) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), `function "f" is nil`) +} + +func TestRegistration_TypedNilFuncFailsCompile(t *testing.T) { + var fn func() int + _, err := Compile("f()", WithFunctions(map[string]any{"f": fn})) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), "nil function value") +} + +// The check fires even when the expression never calls the bad entry: +// the registration itself is the host bug being surfaced. +func TestRegistration_UnreferencedBadEntryStillFails(t *testing.T) { + _, err := Compile("1 + 1", WithFunctions(map[string]any{"unused": "nope"})) + require.ErrorIs(t, err, ErrCompile) +} + +func TestRegistration_MultipleErrorsAllReported(t *testing.T) { + _, err := Compile("1", WithFunctions(map[string]any{ + "a": 1, + "b": func() (int, int, int) { return 0, 0, 0 }, + })) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), `"a"`) + require.Contains(t, err.Error(), `"b"`) +} + +// A later option can override a bad earlier entry... but the bad +// registration still fails Compile: options are validated as written, +// not after merging, so the mistake never hides behind an override. +func TestRegistration_BadEntryNotMaskedByOverride(t *testing.T) { + _, err := Compile("f()", + WithFunctions(map[string]any{"f": 42}), + WithFunctions(map[string]any{"f": func() int { return 1 }}), + ) + require.ErrorIs(t, err, ErrCompile) +} + +// Non-function values belong in the env, which continues to work. +func TestRegistration_ConstantsGoInEnv(t *testing.T) { + got, err := evalExpr(t.Context(), "pi * 2", map[string]any{"pi": 3.14}) + require.NoError(t, err) + require.Equal(t, 6.28, got) +} + +// --- Panic recovery across dispatch paths ---------------------------------- + +// A registered function that hits a runtime panic (nil deref, index +// out of range) is converted to ErrEvaluate instead of crashing the +// host through Run. Prepared fast path. +func TestPanicRecovery_RegisteredFunction(t *testing.T) { + opts := []Option{WithFunctions(map[string]any{ + "boom": func(i int) int { + var xs []int + return xs[i] // index out of range + }, + })} + _, err := evalExpr(t.Context(), "boom(3)", nil, opts...) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), `"boom" panicked`) +} + +// Same protection for callables stored in the env, which dispatch +// through the non-prepared reflect path. +func TestPanicRecovery_EnvFunction(t *testing.T) { + env := map[string]any{ + "boom": func() int { + var p *int + return *p // nil dereference + }, + } + _, err := evalExpr(t.Context(), "boom()", env) + require.ErrorIs(t, err, ErrEvaluate) + require.Contains(t, err.Error(), `"boom" panicked`) +} + +// Deliberate panics with non-runtime values are not swallowed: they +// signal programmer intent and propagate to the caller. +func TestPanicRecovery_ExplicitPanicPropagates(t *testing.T) { + env := map[string]any{ + "boom": func() int { panic("deliberate") }, + } + p, err := Compile("boom()") + require.NoError(t, err) + require.Panics(t, func() { + _, _ = p.Run(context.Background(), env) + }) +} diff --git a/suggest.go b/suggest.go index 5ee54d4..f3044a4 100644 --- a/suggest.go +++ b/suggest.go @@ -55,7 +55,10 @@ func formatHint(name string, candidates []string) string { if closest, ok := closestName(name, candidates); ok { return fmt.Sprintf(" (did you mean %q?)", closest) } - const maxList = 8 + // The special forms alone contribute 8 names to the candidate + // set, so the cap must leave room for a few env entries on top or + // the list hint would never fire for small envs. + const maxList = 12 if len(candidates) > maxList { return "" }