diff --git a/README.md b/README.md index 08657cf..b6b123e 100644 --- a/README.md +++ b/README.md @@ -67,21 +67,29 @@ per-target evidence — `serial.log` (the guest kernel boot), `qemu.log`, and The bundled validator answers "does this `.bpf.o` load/attach?" Sometimes you want to answer "does **my project's actual loader** come up on this kernel?" — which also exercises your userspace path and needs no manifest kept in sync with -that loader. Command mode does exactly that: it runs a command (optionally a -binary you ship in) **inside each matrix kernel VM**, and the per-kernel verdict -is its exit code. +that loader. Command mode does exactly that: it runs your loader command +(optionally a binary you ship in) **inside each matrix kernel VM**, and the +per-kernel verdict is its **exit code**. The bundled validator is *not* used, so +this tests the real userspace loader path. ```bash -# Run your statically-built loader across the matrix; pass == exit 0 per kernel. -bpfcompat test --command '$BPFCOMPAT_BIN --self-test' \ - --command-binary ./build/myloader --matrix matrices/mvp.yaml --out report.json +# Dedicated verb: ship your loader, run it on every kernel, exit code = verdict. +bpfcompat test-command --cmd '$BPFCOMPAT_BIN --self-test' \ + --bin ./build/myloader --matrix matrices/mvp.yaml --out report.json -# Or drive an already-installed tool against a shipped .bpf.o. +# Equivalent flag form on `test`, e.g. driving a loader against a shipped .bpf.o: bpfcompat test --command '$BPFCOMPAT_BIN --obj $BPFCOMPAT_ARTIFACT' \ --command-binary ./build/loader --artifact ./build/probe.bpf.o \ --matrix matrices/mvp.yaml --out report.json ``` +A real run, shipping a libbpf loader and pointing it at the +[known-tricky kernel library](docs/kernel-quirk-library.md) — the loader's exit +code catches the ring-buffer incompatibility on 5.4 and passes 5.15, with no +validator load in between: + +![bpfcompat test-command running a loader across kernels; ubuntu-20.04-5.4 fails (loader exit 2, ring buffer needs >= 5.8) and ubuntu-22.04-5.15 passes (loader exit 0), with libbpf load/attach skipped](docs/images/test-command-demo.png) + The command runs as root in the disposable guest with `$BPFCOMPAT_BIN` (your shipped binary), `$BPFCOMPAT_ARTIFACT` (a staged `.bpf.o`, if given), and `$BPFCOMPAT_REMOTE_ROOT` exported. See @@ -252,10 +260,10 @@ not a production runtime loader and it is not a production multi-tenant SaaS. Implemented: - VM-backed `.bpf.o` validation through QEMU/KVM cloud images. -- **Command/binary validation** (`bpfcompat test --command`) — run *your own* - loader binary/command inside each kernel VM and take its **exit code** as the - per-kernel verdict. The bundled validator is **not** used in this mode; this - tests the real userspace loader path. See +- **Command/binary validation** (`bpfcompat test-command` / `test --command`) — + run *your own* loader binary/command inside each kernel VM and take its **exit + code** as the per-kernel verdict. The bundled validator is **not** used in this + mode; this tests the real userspace loader path. See [docs/command-validation.md](docs/command-validation.md). - **Library of known-tricky vendor kernels** (`matrices/quirk-library.yaml`) — the kernels where "version ≠ feature support" bites; run a `.bpf.o` or your diff --git a/cmd/bpfcompat/main.go b/cmd/bpfcompat/main.go index 6db7f40..fdd896f 100644 --- a/cmd/bpfcompat/main.go +++ b/cmd/bpfcompat/main.go @@ -37,6 +37,8 @@ func run(args []string) int { switch args[0] { case "test": return runTest(args[1:]) + case "test-command": + return runTestCommand(args[1:]) case "suite": return runSuite(args[1:]) case "profile": @@ -210,6 +212,70 @@ func runTest(args []string) int { cfg.KeepVMOnFailure = *keepVMOnFailure cfg.UnsafeAllowHostRunner = unsafeAllowHostRunner + return executeTestConfig(cfg) +} + +// runTestCommand is command/binary validation as an explicit verb: provide your +// own loader command (and optionally a binary to ship in), and bpfcompat runs it +// inside each matrix kernel VM, taking the exit code as the per-kernel verdict. +// It is a thin, discoverable front for `bpfcompat test --command` — the bundled +// validator is not used, so this tests the real userspace loader path. +func runTestCommand(args []string) int { + fs := flag.NewFlagSet("test-command", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + + var cfg runner.Config + fs.StringVar(&cfg.Command, "cmd", "", "Loader command run as root inside each kernel VM (verdict = exit code). Exposes $BPFCOMPAT_BIN/$BPFCOMPAT_ARTIFACT/$BPFCOMPAT_REMOTE_ROOT") + fs.StringVar(&cfg.CommandBinary, "bin", "", "Local loader executable shipped into each guest and exposed to --cmd as $BPFCOMPAT_BIN") + fs.IntVar(&cfg.CommandExpectExit, "expect-exit", 0, "Exit code that counts as a pass (default 0)") + fs.StringVar(&cfg.ArtifactPath, "artifact", "", "Optional .bpf.o staged into each guest and exposed as $BPFCOMPAT_ARTIFACT") + fs.StringVar(&cfg.ArtifactName, "artifact-name", "", "Logical artifact family name for version history (optional)") + fs.StringVar(&cfg.ArtifactVersion, "artifact-version", "", "Artifact version label for version history (optional)") + fs.StringVar(&cfg.ArtifactVariant, "artifact-variant", "", "Artifact variant label (optional)") + fs.StringVar(&cfg.MatrixPath, "matrix", "", "Path to matrix YAML") + fs.BoolVar(&cfg.Quick, "quick", false, "Use the built-in quick-check kernel set instead of --matrix") + fs.StringVar(&cfg.OutPath, "out", "", "Path to JSON report output") + fs.StringVar(&cfg.MarkdownPath, "markdown", "", "Path to Markdown report output (optional)") + fs.StringVar(&cfg.WorkDir, "workdir", ".bpfcompat", "Working directory root") + fs.IntVar(&cfg.Concurrency, "concurrency", 2, "Maximum concurrent VM jobs") + timeoutText := fs.String("timeout", "180s", "Per-target timeout duration") + keepVMOnFailure := fs.Bool("keep-vm-on-failure", false, "Keep VM overlays/logs on failure") + + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "Usage:\n bpfcompat test-command --cmd --matrix --out [--bin ] [--artifact ] [flags]\n\n") + fs.PrintDefaults() + } + + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return runner.ExitToolError + } + if fs.NArg() != 0 { + fmt.Fprintf(os.Stderr, "unexpected positional arguments: %v\n", fs.Args()) + return runner.ExitToolError + } + if strings.TrimSpace(cfg.Command) == "" { + fmt.Fprintln(os.Stderr, "--cmd is required") + return runner.ExitToolError + } + + timeout, err := time.ParseDuration(*timeoutText) + if err != nil { + fmt.Fprintf(os.Stderr, "invalid --timeout value %q: %v\n", *timeoutText, err) + return runner.ExitToolError + } + cfg.Timeout = timeout + cfg.KeepVMOnFailure = *keepVMOnFailure + cfg.Runner = runner.RunnerVM + + return executeTestConfig(cfg) +} + +// executeTestConfig validates a built test Config and runs it, printing the +// summary. Shared by `test` and `test-command`. +func executeTestConfig(cfg runner.Config) int { if err := cfg.Validate(); err != nil { fmt.Fprintf(os.Stderr, "invalid arguments: %v\n", err) return runner.ExitToolError @@ -1357,6 +1423,7 @@ func printRootUsage() { fmt.Println("Usage:") fmt.Println(" bpfcompat test --artifact --matrix --out [flags]") fmt.Println(" bpfcompat test --command --matrix --out [--command-binary ] [flags]") + fmt.Println(" bpfcompat test-command --cmd --matrix --out [--bin ] [--artifact ] [flags]") fmt.Println(" bpfcompat suite --suite --out [flags]") fmt.Println(" bpfcompat profile list --matrix ") fmt.Println(" bpfcompat history list [flags]") diff --git a/cmd/bpfcompat/main_test.go b/cmd/bpfcompat/main_test.go index 3fcacd3..75f06b4 100644 --- a/cmd/bpfcompat/main_test.go +++ b/cmd/bpfcompat/main_test.go @@ -40,3 +40,30 @@ func TestSplitCSVUpper(t *testing.T) { t.Fatalf("unexpected splitCSVUpper result: got=%v want=%v", got, want) } } + +func TestRunTestCommandValidation(t *testing.T) { + cases := []struct { + name string + args []string + want int + }{ + {"missing cmd", []string{"--matrix", "m.yaml", "--out", "o.json"}, 1}, + {"help", []string{"-h"}, 0}, + {"unexpected positional", []string{"--cmd", "x", "--matrix", "m.yaml", "--out", "o.json", "stray"}, 1}, + {"bad timeout", []string{"--cmd", "x", "--matrix", "m.yaml", "--out", "o.json", "--timeout", "nope"}, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := runTestCommand(tc.args); got != tc.want { + t.Fatalf("runTestCommand(%v) = %d, want %d", tc.args, got, tc.want) + } + }) + } +} + +// test-command must route through the same command-mode config as `test --command`. +func TestRunTestCommandDispatch(t *testing.T) { + if got := run([]string{"test-command", "-h"}); got != 0 { + t.Fatalf("run test-command -h = %d, want 0", got) + } +} diff --git a/docs/command-validation.md b/docs/command-validation.md index b81469d..1f4723c 100644 --- a/docs/command-validation.md +++ b/docs/command-validation.md @@ -25,9 +25,20 @@ that the `.bpf.o` flow uses. ## Usage +There are two equivalent front-ends: the dedicated `test-command` verb (shorter +flags: `--cmd`, `--bin`, `--expect-exit`) and `test --command` (the same flags +prefixed with `--command`). Use whichever reads better. + ```bash -# Ship a statically-linked loader and run it on every matrix kernel. -# Pass == exit 0 (override with --command-expect-exit N). +# Dedicated verb — ship a statically-linked loader, run it on every matrix kernel. +# Pass == exit 0 (override with --expect-exit N). +bpfcompat test-command \ + --cmd '$BPFCOMPAT_BIN --self-test' \ + --bin ./build/myloader \ + --matrix matrices/mvp.yaml \ + --out report.json + +# Equivalent flag form on `test`: bpfcompat test \ --command '$BPFCOMPAT_BIN --self-test' \ --command-binary ./build/myloader \ diff --git a/docs/images/test-command-demo.png b/docs/images/test-command-demo.png new file mode 100644 index 0000000..532712b Binary files /dev/null and b/docs/images/test-command-demo.png differ