From 92c777782ad1db436317546429e27d92325d153a Mon Sep 17 00:00:00 2001 From: VirtualMachinist <252028890+VirtualMachinist@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:12:52 -0800 Subject: [PATCH 1/4] Run a short fuzz of every target in CI on both OSes scripts/fuzz.sh finds every Fuzz function in the tree and runs each for a short fixed time (20s by default), one go test invocation per target, as Go requires. It is a blocking CI step on Linux and macOS: a crasher fails the job and the failing input is written under testdata/fuzz/ in its package. The script keeps a floor of expected targets, so a target that is deleted or renamed is a failure and not a quiet pass; its --selftest proves a missing name and a prefix match both fail. Three targets join the existing FuzzConfig. FuzzCLIArgs feeds arbitrary arguments to the command tree and checks that it exits 0 or 3, prints one fixed-text error line with empty stdout, never repeats a URL-shaped argument, and never exits 0 for a URL, selector or expression. FuzzBase checks that an accepted base address has a scheme and a host and no credentials, query, fragment or dot segment. FuzzPath checks that an accepted path survives a parse unchanged and that a refused path never reaches the dialer. Each rule has seeds, so a rule that lets one reserved character through fails the seed run and not only a lucky fuzz; mutants of each rule fail. CONTRIBUTING lists the new gate and names the pinned toolchain correctly. Signed-off-by: VirtualMachinist <252028890+VirtualMachinist@users.noreply.github.com> --- .github/workflows/ci.yml | 6 + CONTRIBUTING.md | 5 +- cmd/vigil/fuzz_cli_test.go | 105 ++++++++++++++++++ .../fuzz/FuzzCLIArgs/9635a12fbf9e6bbc | 2 + internal/httpx/fuzz_test.go | 90 +++++++++++++++ scripts/fuzz.sh | 93 ++++++++++++++++ 6 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 cmd/vigil/fuzz_cli_test.go create mode 100644 cmd/vigil/testdata/fuzz/FuzzCLIArgs/9635a12fbf9e6bbc create mode 100644 internal/httpx/fuzz_test.go create mode 100755 scripts/fuzz.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c2bc72..083c2cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,12 @@ jobs: - name: Tests (race, shuffled) run: go test -race -shuffle=on -count=1 ./... + - name: Fuzz (short run per target; a crasher fails the job) + shell: bash + run: | + scripts/fuzz.sh --selftest + scripts/fuzz.sh + - name: Scrub tracked files shell: bash run: scripts/scrub.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7b894bf..976a0b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ Contributions are licensed under Apache-2.0, the license of this repository. ## Build and check -Go 1.27.3 is pinned in `go.mod`. Run these before you push; CI runs the same on Linux and macOS. +Go 1.27.4 is pinned in `go.mod`. Run these before you push; CI runs the same on Linux and macOS. ``` go vet ./... @@ -64,6 +64,8 @@ go test -race -shuffle=on -count=1 ./... scripts/scrub.sh scripts/scrub.sh --selftest scripts/depguard-selftest.sh +scripts/fuzz.sh --selftest +scripts/fuzz.sh ``` Use golangci-lint v2 and a current govulncheck. All checks are blocking. @@ -72,6 +74,7 @@ Use golangci-lint v2 and a current govulncheck. All checks are blocking. - **No URL input.** No flag, route, config value or document field makes Vigil fetch a URL, take a selector, or send free-form PromQL. - **No `os/exec`** in non-test code, and no schema validator in the binary. The linter enforces both. +- **One HTTP client.** Every call to a backend goes through `internal/httpx`: GET only, no redirect, no proxy, a body cap, a deadline, two calls in flight per backend. The linter rejects `net/http` anywhere else in non-test code. - **Severity comes from authorities.** Vigil combines verdicts and applies the stale and missing-authority caps. It does not compute severity from raw data. - **Slugs only.** Use the public host slugs (`castle`, `tower`, `citadel`, `apiary`, `lathe`, `box`) in code, docs, fixtures and examples. No IP addresses, hostnames or secrets. Placeholders use reserved documentation values. Real addresses live in an untracked overlay. - **Schemas are the contract.** JSON Schemas under `schema/` are written by hand. A change that adds a field is additive within `v0`; anything that removes or renames one needs a new major. diff --git a/cmd/vigil/fuzz_cli_test.go b/cmd/vigil/fuzz_cli_test.go new file mode 100644 index 0000000..8bc8d5e --- /dev/null +++ b/cmd/vigil/fuzz_cli_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + "github.com/Hedronite/vigil/internal/exitcode" +) + +// usageMessages is every message the command layer prints for a usage error. +// It is fixed text: a message outside this set means text from the input, a +// backend or an error reached stderr. +var usageMessages = map[string]bool{ + "a command is required": true, + "a subcommand is required": true, + "a query name is required": true, + "the query name is not a configured name": true, + "--host must be a host slug": true, + "a path is required here, not a URL": true, + "--now must be an RFC 3339 time": true, + "unknown command; see vigil --help": true, + "unknown flag; see vigil --help": true, + "a flag is missing its value": true, + "a flag value is invalid": true, + "unexpected arguments; no command takes a URL, selector or PromQL": true, + "usage error; see vigil --help": true, +} + +// FuzzCLIArgs feeds arbitrary argument lists to the command tree with handlers +// that always succeed. Whatever the arguments, the parser never panics, exits 0 +// or 3 only, and a failure is one fixed-text vigil.error/v0 line with empty +// stdout that never repeats a URL-, address- or selector-shaped argument. +func FuzzCLIArgs(f *testing.F) { + for _, s := range []string{ + "version", "prom targets", "prom query up_all", "prom query up_by_host --host castle", + "sources --json", "grafana health --now 2026-11-14T09:00:00Z", "--help", "prom --help", + "sources http://leak.example.invalid/x", "prom query http://leak.example.invalid/x", "prom query up_all --host http://leak.example.invalid", "prom query up(x)", "prom query up_all --host a{b}", "prom query up{job=\"x\"}", "--config https://leak.example.invalid sources", + "--overlay //leak/x sources", "prom query rate(x[5m]) --host 192.0.2.7:9090", "--now yesterday sources", + "--nope", "-x", "", "prom", "grafana", "help", "completion bash", "sources --config", "--json=maybe sources", + "prom query a b", "version extra", "sources --host castle", "--now 2026-11-14T09:00:00Z --json prom targets", + } { + f.Add(s) + } + h := Handlers{} + ok := func(_ context.Context, i Invocation) error { + _, _ = i.Stdout.Write([]byte("ok\n")) + return nil + } + h.PromTargets, h.PromQuery, h.GrafanaHealth, h.Sources = ok, ok, ok, ok + + f.Fuzz(func(t *testing.T, line string) { + args := strings.Split(line, " ") + var out, errb bytes.Buffer + code := run(context.Background(), args, &out, &errb, h) + switch code { + case exitcode.Green: + if errb.Len() != 0 { + t.Fatalf("exit 0 with stderr %q", errb.String()) + } + // No valid invocation holds a URL, so one that exits 0 means a + // value that should have been refused got through. + for _, a := range args { + if strings.Contains(a, "://") || strings.ContainsAny(a, "{([\"?#") { + t.Fatalf("exit 0 with the URL-, selector- or expression-shaped argument %q", a) + } + } + case exitcode.Usage: + if out.Len() != 0 { + t.Fatalf("usage error with stdout %q", out.String()) + } + s := errb.String() + if !strings.HasSuffix(s, "\n") || strings.Count(s, "\n") != 1 { + t.Fatalf("stderr is not one line: %q", s) + } + var doc struct { + APIVersion string `json:"apiVersion"` + Error struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + dec := json.NewDecoder(strings.NewReader(s)) + dec.DisallowUnknownFields() + if err := dec.Decode(&doc); err != nil { + t.Fatalf("stderr is not an error document: %v (%q)", err, s) + } + if doc.APIVersion != "vigil.error/v0" || doc.Error.Code != "usage" { + t.Fatalf("document = %+v", doc) + } + if !usageMessages[doc.Error.Message] { + t.Fatalf("message is not fixed text: %q", doc.Error.Message) + } + for _, a := range args { + if len(a) >= 3 && strings.ContainsAny(a, "/:@.{([") && strings.Contains(s, a) { + t.Fatalf("stderr repeats the argument %q: %s", a, s) + } + } + default: + t.Fatalf("exit %d: handlers that succeed leave only 0 and 3", code) + } + }) +} diff --git a/cmd/vigil/testdata/fuzz/FuzzCLIArgs/9635a12fbf9e6bbc b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/9635a12fbf9e6bbc new file mode 100644 index 0000000..ac8986b --- /dev/null +++ b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/9635a12fbf9e6bbc @@ -0,0 +1,2 @@ +go test fuzz v1 +string("\"000000 -h") diff --git a/internal/httpx/fuzz_test.go b/internal/httpx/fuzz_test.go new file mode 100644 index 0000000..aaaf3ad --- /dev/null +++ b/internal/httpx/fuzz_test.go @@ -0,0 +1,90 @@ +package httpx + +import ( + "context" + "net" + "net/url" + "strings" + "testing" +) + +// FuzzBase: a base address is either refused with an error, or it is an http +// or https URL with a host and nothing else that could redirect a request: no +// userinfo, query, fragment, whitespace or dot segment. +func FuzzBase(f *testing.F) { + for _, s := range []string{ + "http://example.test", "https://example.test:8443/p", "http://[::1]:9090", "http://192.0.2.7:9090/a/b", + "", "localhost:9090", "ftp://example.test", "http://user:pw@example.test", "http://example.test?x=1", + "http://example.test#f", "http://example.test/a/../b", "http://example.test/./a", "http://exa mple.test", + "//example.test", "http:example.test", "http://example.test\\@evil.test", "http://example.test/%2e%2e/x", + "http://example.test/a%2fb", "HTTP://EXAMPLE.TEST", "http://\x00example.test", + } { + f.Add(s) + } + f.Fuzz(func(t *testing.T, s string) { + c, err := New(Config{Base: s}) + if err != nil { + if Code(err) == "" { + t.Fatalf("untyped error: %v", err) + } + return + } + u := c.base + if u.Scheme != "http" && u.Scheme != "https" { + t.Fatalf("scheme %q accepted from %q", u.Scheme, s) + } + if u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Opaque != "" { + t.Fatalf("unsafe base accepted from %q: %+v", s, u) + } + if hasDotSegment(u.Path) || strings.ContainsAny(s, " \t\r\n\\") { + t.Fatalf("base %q accepted with a dot segment or whitespace", s) + } + }) +} + +// FuzzPath: an accepted path is a plain absolute path that survives a parse +// unchanged, and a refused path never reaches the dialer. +func FuzzPath(f *testing.F) { + for _, s := range []string{ + "/api/v1/targets", "/-/ready", "/api/health", "", "/", "api/v1", "//evil.test/x", "/a/../b", "/a/./b", + "/a?x=1", "/a#f", "/a b", "/a%2e%2e/b", "/a\\b", "/a\nb", "http://example.test/x", "/a//b", "/%2e%2e", "/é", + // One reserved character each, so a rule that lets any of them + // through is caught by the seed run and not only by luck. + "/a?b", "/a#b", "/a%2e", "/a;b", "/a:b", "/a@b", "/a*b", "/a+b", "/a=b", "/a&b", "/a,b", "/a!b", "/a~b", "/a\"b", + } { + f.Add(s) + } + var dialed bool + c, err := newClient(Config{Base: "http://example.test"}, func(context.Context, string, string) (net.Conn, error) { + dialed = true + return nil, &net.OpError{Op: "dial", Err: net.ErrClosed} + }) + if err != nil { + f.Fatal(err) + } + f.Fuzz(func(t *testing.T, p string) { + dialed = false + _, err := c.Get(context.Background(), Request{Path: p}) + if err == nil { + t.Fatal("the dialer always fails, so Get cannot succeed") + } + if !validPath(p) { + if dialed { + t.Fatalf("a request went out for the refused path %q", p) + } + return + } + u, perr := url.Parse("http://example.test" + p) + if perr != nil || u.Path != p || u.RawQuery != "" || u.Fragment != "" || u.Host != "example.test" { + t.Fatalf("accepted path %q does not survive a parse: %+v (%v)", p, u, perr) + } + if strings.Contains(p, "//") || hasDotSegment(p) || p[0] != '/' { + t.Fatalf("accepted path %q has a doubled slash, a dot segment or no root", p) + } + for i := 0; i < len(p); i++ { + if c := p[i]; !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '/' || c == '-' || c == '_' || c == '.') { + t.Fatalf("accepted path %q holds %q", p, c) + } + } + }) +} diff --git a/scripts/fuzz.sh b/scripts/fuzz.sh new file mode 100755 index 0000000..fc4c29e --- /dev/null +++ b/scripts/fuzz.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# scripts/fuzz.sh: run every fuzz target in the tree for a short, fixed time. +# +# scripts/fuzz.sh run each target for $FUZZTIME (default 20s) +# scripts/fuzz.sh --list print the targets found +# scripts/fuzz.sh --selftest prove a missing or renamed target is a failure +# +# Go runs one fuzz target per invocation, so this finds every "func FuzzX" in a +# _test.go file and runs each in its own package. The list in EXPECTED is the +# floor: a target that is deleted or renamed is a failure, not a quiet pass. +# Fuzzing is blocking in CI: a crasher fails the job, and the failing input is +# written under testdata/fuzz/ in the package, where the next run replays it. +# +# Exit: 0 every target ran and passed, 1 a target failed or is missing, 2 setup. + +set -u +set -o pipefail + +cd "$(dirname "$0")/.." || exit 2 + +: "${FUZZTIME:=20s}" +: "${EXPECTED:=FuzzConfig FuzzCLIArgs FuzzBase FuzzPath}" + +# found: "dir name" lines, one per target. +find_targets() { + grep -rEoH --include='*_test.go' '^func Fuzz[A-Za-z0-9_]*' cmd internal 2>/dev/null | + while IFS= read -r line; do + file=${line%%:*} + name=${line##*func } + printf '%s %s\n' "$(dirname "$file")" "$name" + done | sort +} + +# check_expected FOUND: every name in EXPECTED must be in FOUND. +check_expected() { + local found=$1 want missing=0 + for want in $EXPECTED; do + case " +$found +" in + *" $want +"*) ;; + *) + echo "fuzz: expected target $want was not found" >&2 + missing=1 + ;; + esac + done + return "$missing" +} + +if [ "${1:-}" = "--selftest" ]; then + fail=0 + found=$(find_targets) || exit 2 + if ! EXPECTED="$EXPECTED" check_expected "$found" 2>/dev/null; then + echo "fuzz selftest FAIL: the real tree is missing an expected target" + fail=1 + fi + if EXPECTED="FuzzDoesNotExist" check_expected "$found" 2>/dev/null; then + echo "fuzz selftest FAIL: a missing target was not caught" + fail=1 + fi + if EXPECTED="FuzzConfigX" check_expected "$found" 2>/dev/null; then + echo "fuzz selftest FAIL: a prefix match counted as the target" + fail=1 + fi + [ "$fail" -eq 0 ] && echo "fuzz selftest ok: every expected target is found, a missing one fails" + exit "$fail" +fi + +found=$(find_targets) || exit 2 +if [ "${1:-}" = "--list" ]; then + printf '%s\n' "$found" + exit 0 +fi +[ -n "$found" ] || { echo "fuzz: no fuzz targets found" >&2; exit 1; } +check_expected "$found" || exit 1 + +status=0 +ran=0 +while read -r dir name; do + echo "== fuzz $name ($dir) for $FUZZTIME" + if go test -run "^$" -fuzz "^${name}\$" -fuzztime "$FUZZTIME" "./$dir" &2 + status=1 + fi +done < Date: Fri, 13 Nov 2026 22:16:56 -0800 Subject: [PATCH 2/4] Let help answer first in the CLI fuzz and clear a lint finding The first 20s run of FuzzCLIArgs found an input that exits 0 with a selector-shaped argument: a stray quote followed by -h. That is cobra answering help before it checks the other arguments, which is correct, and the help text is static. The invariant now allows it only when help was asked, and then checks that the help text does not repeat any of those arguments. The input stays as a regression seed under testdata/fuzz. The path-character check in FuzzPath moves into a named function, which also clears the De Morgan finding from the linter and keeps the allowed set written out apart from the code it tests. Signed-off-by: VirtualMachinist <252028890+VirtualMachinist@users.noreply.github.com> --- cmd/vigil/fuzz_cli_test.go | 32 +++++++++++++++++-- .../fuzz/FuzzCLIArgs/949f7a65805732d7 | 2 ++ internal/httpx/fuzz_test.go | 12 ++++++- 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 cmd/vigil/testdata/fuzz/FuzzCLIArgs/949f7a65805732d7 diff --git a/cmd/vigil/fuzz_cli_test.go b/cmd/vigil/fuzz_cli_test.go index 8bc8d5e..6822b61 100644 --- a/cmd/vigil/fuzz_cli_test.go +++ b/cmd/vigil/fuzz_cli_test.go @@ -60,12 +60,22 @@ func FuzzCLIArgs(f *testing.F) { if errb.Len() != 0 { t.Fatalf("exit 0 with stderr %q", errb.String()) } - // No valid invocation holds a URL, so one that exits 0 means a - // value that should have been refused got through. + // No valid invocation holds a URL, a selector or an expression, so + // a run that exits 0 with one means a value that should have been + // refused got through. Asking for help is the exception: cobra + // answers -h, --help and the help command before it checks the + // other arguments. The help text is static, and it must not repeat + // any of them. for _, a := range args { - if strings.Contains(a, "://") || strings.ContainsAny(a, "{([\"?#") { + if !suspicious(a) { + continue + } + if !helpAsked(args) { t.Fatalf("exit 0 with the URL-, selector- or expression-shaped argument %q", a) } + if strings.Contains(out.String(), a) { + t.Fatalf("the help text repeats the argument %q", a) + } } case exitcode.Usage: if out.Len() != 0 { @@ -103,3 +113,19 @@ func FuzzCLIArgs(f *testing.F) { } }) } + +// suspicious reports whether an argument looks like a URL, a selector or an +// expression: nothing a valid invocation holds. +func suspicious(a string) bool { + return strings.Contains(a, "://") || strings.ContainsAny(a, "{([\"?#") +} + +// helpAsked reports whether the arguments ask for help. +func helpAsked(args []string) bool { + for _, a := range args { + if a == "-h" || a == "--help" || a == "help" { + return true + } + } + return false +} diff --git a/cmd/vigil/testdata/fuzz/FuzzCLIArgs/949f7a65805732d7 b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/949f7a65805732d7 new file mode 100644 index 0000000..9993d90 --- /dev/null +++ b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/949f7a65805732d7 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("--overlay \" sources") diff --git a/internal/httpx/fuzz_test.go b/internal/httpx/fuzz_test.go index aaaf3ad..ad13e50 100644 --- a/internal/httpx/fuzz_test.go +++ b/internal/httpx/fuzz_test.go @@ -82,9 +82,19 @@ func FuzzPath(f *testing.F) { t.Fatalf("accepted path %q has a doubled slash, a dot segment or no root", p) } for i := 0; i < len(p); i++ { - if c := p[i]; !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '/' || c == '-' || c == '_' || c == '.') { + if c := p[i]; !plainPathChar(c) { t.Fatalf("accepted path %q holds %q", p, c) } } }) } + +// plainPathChar is the set a fixed path may use, written out here and not +// borrowed from the code under test. +func plainPathChar(c byte) bool { + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + return true + } + return c == '/' || c == '-' || c == '_' || c == '.' +} From 94d71be6bee50263de807dffe538e64eed22347a Mon Sep 17 00:00:00 2001 From: VirtualMachinist <252028890+VirtualMachinist@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:39:28 -0800 Subject: [PATCH 3/4] Check what the handler received, not the raw arguments, in the CLI fuzz The fuzz kept finding inputs that were fine: a flag given twice, where the last value wins, leaves an earlier bad value that never reaches anything; -hh is help. Chasing raw argument shapes was the wrong property. FuzzCLIArgs now records what each handler is given and checks that: a success hands over only a query name of the right shape, a host slug of the right shape and file names that are not URLs, and a usage error never runs a handler. The rest is unchanged: exit 0 or 3 only, one fixed-text error line with empty stdout, and no repeat of a URL- or selector-shaped argument in stderr or in static help. The inputs found so far stay as seeds. Signed-off-by: VirtualMachinist <252028890+VirtualMachinist@users.noreply.github.com> --- cmd/vigil/echo_test.go | 31 +++++++ cmd/vigil/fuzz_cli_test.go | 86 +++++++++---------- .../fuzz/FuzzCLIArgs/2accfe88474a2571 | 2 + .../fuzz/FuzzCLIArgs/52678d88f23b88fa | 2 + .../fuzz/FuzzCLIArgs/a259869c65214bce | 2 + 5 files changed, 77 insertions(+), 46 deletions(-) create mode 100644 cmd/vigil/echo_test.go create mode 100644 cmd/vigil/testdata/fuzz/FuzzCLIArgs/2accfe88474a2571 create mode 100644 cmd/vigil/testdata/fuzz/FuzzCLIArgs/52678d88f23b88fa create mode 100644 cmd/vigil/testdata/fuzz/FuzzCLIArgs/a259869c65214bce diff --git a/cmd/vigil/echo_test.go b/cmd/vigil/echo_test.go new file mode 100644 index 0000000..c218f29 --- /dev/null +++ b/cmd/vigil/echo_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "strings" + "testing" +) + +// TestHelpAndErrorsDoNotEchoArguments puts a marker in every argument slot and +// checks it never comes back on stdout or stderr, in help, in a usage error or +// in version output. +func TestHelpAndErrorsDoNotEchoArguments(t *testing.T) { + const m = "leakmarker" + for _, args := range [][]string{ + {"--help", m + ".example.invalid"}, + {"help", "http://" + m + ".example.invalid/x"}, + {"prom", "--help", m}, + {"-hh", "\"" + m}, + {"version", m}, + {m}, + {"prom", m}, + {"prom", "query", "http://" + m + ".example.invalid"}, + {"sources", "--nope=" + m}, + {"sources", "--config=https://" + m + ".example.invalid"}, + {"--now", m, "sources"}, + } { + _, out, errs := call(t, defaultHandlers(), args...) + if strings.Contains(out, m) || strings.Contains(errs, m) { + t.Errorf("%q echoed the marker: stdout=%q stderr=%q", args, out, errs) + } + } +} diff --git a/cmd/vigil/fuzz_cli_test.go b/cmd/vigil/fuzz_cli_test.go index 6822b61..8b41801 100644 --- a/cmd/vigil/fuzz_cli_test.go +++ b/cmd/vigil/fuzz_cli_test.go @@ -30,29 +30,40 @@ var usageMessages = map[string]bool{ } // FuzzCLIArgs feeds arbitrary argument lists to the command tree with handlers -// that always succeed. Whatever the arguments, the parser never panics, exits 0 -// or 3 only, and a failure is one fixed-text vigil.error/v0 line with empty -// stdout that never repeats a URL-, address- or selector-shaped argument. +// that record what they were given and always succeed. Whatever the arguments, +// the parser never panics and exits 0 or 3 only. A failure is one fixed-text +// vigil.error/v0 line with empty stdout whose text is chosen from the +// fixed set above, so it cannot repeat an argument. A success hands the handler only values of the right shape: a +// query name, a host slug, and file names that are not URLs. Checking what the +// handler received, and not the raw arguments, is the property that matters: +// flags may repeat and the last value wins, so an earlier bad value that was +// overridden never reaches anything. func FuzzCLIArgs(f *testing.F) { for _, s := range []string{ "version", "prom targets", "prom query up_all", "prom query up_by_host --host castle", "sources --json", "grafana health --now 2026-11-14T09:00:00Z", "--help", "prom --help", - "sources http://leak.example.invalid/x", "prom query http://leak.example.invalid/x", "prom query up_all --host http://leak.example.invalid", "prom query up(x)", "prom query up_all --host a{b}", "prom query up{job=\"x\"}", "--config https://leak.example.invalid sources", + "sources http://leak.example.invalid/x", "prom query up{job=\"x\"}", "--config https://leak.example.invalid sources", "--overlay //leak/x sources", "prom query rate(x[5m]) --host 192.0.2.7:9090", "--now yesterday sources", "--nope", "-x", "", "prom", "grafana", "help", "completion bash", "sources --config", "--json=maybe sources", "prom query a b", "version extra", "sources --host castle", "--now 2026-11-14T09:00:00Z --json prom targets", + "prom query http://leak.example.invalid/x", "prom query up_all --host http://leak.example.invalid", + "prom query up(x)", "prom query up_all --host a{b}", + "prom query a --host \" --host ", "-hh \"", "\"000000 -h", "--overlay \" sources", + "prom query up_all --config ? --overlay #", "prom query up_all --host Castle", "prom query UP", } { f.Add(s) } - h := Handlers{} - ok := func(_ context.Context, i Invocation) error { - _, _ = i.Stdout.Write([]byte("ok\n")) - return nil - } - h.PromTargets, h.PromQuery, h.GrafanaHealth, h.Sources = ok, ok, ok, ok f.Fuzz(func(t *testing.T, line string) { args := strings.Split(line, " ") + var got *Invocation + rec := func(_ context.Context, i Invocation) error { + i.Stdout = nil + got = &i + return nil + } + h := Handlers{PromTargets: rec, PromQuery: rec, GrafanaHealth: rec, Sources: rec} + var out, errb bytes.Buffer code := run(context.Background(), args, &out, &errb, h) switch code { @@ -60,27 +71,31 @@ func FuzzCLIArgs(f *testing.F) { if errb.Len() != 0 { t.Fatalf("exit 0 with stderr %q", errb.String()) } - // No valid invocation holds a URL, a selector or an expression, so - // a run that exits 0 with one means a value that should have been - // refused got through. Asking for help is the exception: cobra - // answers -h, --help and the help command before it checks the - // other arguments. The help text is static, and it must not repeat - // any of them. - for _, a := range args { - if !suspicious(a) { - continue - } - if !helpAsked(args) { - t.Fatalf("exit 0 with the URL-, selector- or expression-shaped argument %q", a) - } - if strings.Contains(out.String(), a) { - t.Fatalf("the help text repeats the argument %q", a) + if got == nil { + // Help or version: static text, which cannot hold an + // argument. TestHelpAndErrorsDoNotEchoArguments pins that + // with a marker; a substring check here would trip on + // words the help text really contains. + return + } + if got.Name != "" && !queryNamePattern.MatchString(got.Name) { + t.Fatalf("handler got the query name %q", got.Name) + } + if got.Host != "" && !hostSlugPattern.MatchString(got.Host) { + t.Fatalf("handler got the host %q", got.Host) + } + for _, p := range []string{got.ConfigPath, got.OverlayPath} { + if !plainPath(p) { + t.Fatalf("handler got the path %q", p) } } case exitcode.Usage: if out.Len() != 0 { t.Fatalf("usage error with stdout %q", out.String()) } + if got != nil { + t.Fatal("a handler ran for a usage error") + } s := errb.String() if !strings.HasSuffix(s, "\n") || strings.Count(s, "\n") != 1 { t.Fatalf("stderr is not one line: %q", s) @@ -103,29 +118,8 @@ func FuzzCLIArgs(f *testing.F) { if !usageMessages[doc.Error.Message] { t.Fatalf("message is not fixed text: %q", doc.Error.Message) } - for _, a := range args { - if len(a) >= 3 && strings.ContainsAny(a, "/:@.{([") && strings.Contains(s, a) { - t.Fatalf("stderr repeats the argument %q: %s", a, s) - } - } default: t.Fatalf("exit %d: handlers that succeed leave only 0 and 3", code) } }) } - -// suspicious reports whether an argument looks like a URL, a selector or an -// expression: nothing a valid invocation holds. -func suspicious(a string) bool { - return strings.Contains(a, "://") || strings.ContainsAny(a, "{([\"?#") -} - -// helpAsked reports whether the arguments ask for help. -func helpAsked(args []string) bool { - for _, a := range args { - if a == "-h" || a == "--help" || a == "help" { - return true - } - } - return false -} diff --git a/cmd/vigil/testdata/fuzz/FuzzCLIArgs/2accfe88474a2571 b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/2accfe88474a2571 new file mode 100644 index 0000000..e11434f --- /dev/null +++ b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/2accfe88474a2571 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("lp\"") diff --git a/cmd/vigil/testdata/fuzz/FuzzCLIArgs/52678d88f23b88fa b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/52678d88f23b88fa new file mode 100644 index 0000000..d0929d7 --- /dev/null +++ b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/52678d88f23b88fa @@ -0,0 +1,2 @@ +go test fuzz v1 +string("-hh \"") diff --git a/cmd/vigil/testdata/fuzz/FuzzCLIArgs/a259869c65214bce b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/a259869c65214bce new file mode 100644 index 0000000..93e287c --- /dev/null +++ b/cmd/vigil/testdata/fuzz/FuzzCLIArgs/a259869c65214bce @@ -0,0 +1,2 @@ +go test fuzz v1 +string("prom query a --host \" --host ") From 5aa4bc7417d75033b3625eecdd373fbfabb374ea Mon Sep 17 00:00:00 2001 From: VirtualMachinist <252028890+VirtualMachinist@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:03:55 -0800 Subject: [PATCH 4/4] Make the fuzz selftest prove the target matcher is anchored The selftest case meant to show a prefix match fails used FuzzConfigX, which is longer than a real target and not a prefix of it, so a matcher that lost its anchors still passed. It now plants the three shapes a loose matcher accepts: FuzzConf, the start of FuzzConfig; zzConfig, its end; and FuzzConfigX, a longer name. Each must count as missing. Checked with three mutants of the matcher: no trailing anchor fails the prefix case, no leading anchor fails the suffix case, and no anchors at all fails both. The real script still passes. Signed-off-by: VirtualMachinist <252028890+VirtualMachinist@users.noreply.github.com> --- scripts/fuzz.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/fuzz.sh b/scripts/fuzz.sh index fc4c29e..3a77600 100755 --- a/scripts/fuzz.sh +++ b/scripts/fuzz.sh @@ -60,8 +60,20 @@ if [ "${1:-}" = "--selftest" ]; then echo "fuzz selftest FAIL: a missing target was not caught" fail=1 fi + # A name that is only the start of a real target (FuzzConf of FuzzConfig) + # or only its end (zzConfig) must not count as found. An unanchored + # matcher passes the first with a leading anchor missing and the second + # with a trailing one missing, so both are planted. + if EXPECTED="FuzzConf" check_expected "$found" 2>/dev/null; then + echo "fuzz selftest FAIL: a prefix of a target counted as the target" + fail=1 + fi + if EXPECTED="zzConfig" check_expected "$found" 2>/dev/null; then + echo "fuzz selftest FAIL: a suffix of a target counted as the target" + fail=1 + fi if EXPECTED="FuzzConfigX" check_expected "$found" 2>/dev/null; then - echo "fuzz selftest FAIL: a prefix match counted as the target" + echo "fuzz selftest FAIL: a longer name counted as the target" fail=1 fi [ "$fail" -eq 0 ] && echo "fuzz selftest ok: every expected target is found, a missing one fails"