From 120617e3b52c38fb66c1dce61eff298a3fd318bc Mon Sep 17 00:00:00 2001 From: Geno Roupsky <185580+groupsky@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:55:03 +0300 Subject: [PATCH] test(e2e): end-to-end test with sshd in a container and a fake Telegram The real broker, nodkey run, nodkey ssh-agent with real ssh and git, and nodkey ssh-grant, with throwaway keys; a canary search over every output; a CI job. Closes #15 --- .github/dependabot.yml | 13 + .github/workflows/ci.yml | 30 ++ AGENTS.md | 16 +- DESIGN.md | 103 +++- test/e2e/audit_test.go | 140 +++++ test/e2e/canary_test.go | 171 +++++++ test/e2e/doc.go | 9 + test/e2e/docker_test.go | 348 +++++++++++++ test/e2e/e2e_test.go | 731 +++++++++++++++++++++++++++ test/e2e/env_test.go | 615 ++++++++++++++++++++++ test/e2e/main_test.go | 166 ++++++ test/e2e/norace_test.go | 6 + test/e2e/process_test.go | 323 ++++++++++++ test/e2e/race_test.go | 6 + test/e2e/run_test.go | 389 ++++++++++++++ test/e2e/telegram_test.go | 650 ++++++++++++++++++++++++ test/e2e/testdata/sshd/Dockerfile | 25 + test/e2e/testdata/sshd/entrypoint.sh | 29 ++ test/e2e/testdata/sshd/sshd_config | 17 + 19 files changed, 3768 insertions(+), 19 deletions(-) create mode 100644 test/e2e/audit_test.go create mode 100644 test/e2e/canary_test.go create mode 100644 test/e2e/doc.go create mode 100644 test/e2e/docker_test.go create mode 100644 test/e2e/e2e_test.go create mode 100644 test/e2e/env_test.go create mode 100644 test/e2e/main_test.go create mode 100644 test/e2e/norace_test.go create mode 100644 test/e2e/process_test.go create mode 100644 test/e2e/race_test.go create mode 100644 test/e2e/run_test.go create mode 100644 test/e2e/telegram_test.go create mode 100644 test/e2e/testdata/sshd/Dockerfile create mode 100644 test/e2e/testdata/sshd/entrypoint.sh create mode 100644 test/e2e/testdata/sshd/sshd_config diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 49c8daa..203b1ab 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,6 +14,19 @@ updates: prefix: chore include: scope + # The base image of the SSH targets of the end-to-end test, pinned by + # its digest. Dependabot updates the digest. Pull request titles look + # like "test(deps): bump debian ...". + - package-ecosystem: docker + directory: /test/e2e/testdata/sshd + schedule: + interval: weekly + cooldown: + default-days: 7 + commit-message: + prefix: test + include: scope + # Pull request titles look like "ci(deps): bump actions/checkout ...". # Dependabot also updates the pinned commit SHA and its version comment. - package-ecosystem: github-actions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11beb7a..056be93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,36 @@ jobs: with: # Dependabot does not update this version. Update it by hand. version: v2.14.0 + # With the tag e2e, the files of the end-to-end test are checked + # too. No file needs the tag to be missing, so nothing is left out. + args: --build-tags e2e + + e2e: + name: end-to-end test + runs-on: ubuntu-latest + # The test itself takes about 10 seconds; without caches, building + # the image and the binaries takes a few minutes more. + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Show the programs that the test uses + # The runner has Docker, the OpenSSH client and git. The test + # needs OpenSSH 8.9 or newer. + run: | + docker version + ssh -V + git --version + - run: go vet -tags e2e ./test/e2e/... + - name: End-to-end test + # See DESIGN.md section 17. The test builds nodkey and the image + # of test/e2e/testdata/sshd, and removes every container, network + # and image that it makes. It fails if one is left. + run: go test -race -tags e2e ./test/e2e/... vuln: runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index b82522a..2c5c81c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,23 +28,25 @@ Planned. Each v1 issue adds its part. (sections 6 and 15) and `nodkey ssh-grant` (sections 13 and 15). - `contrib/systemd/` - the example systemd unit of the broker, with encrypted credentials and a sandbox (DESIGN.md section 16). -- `test/e2e/` - end-to-end tests with `sshd` in a container. Build tag - `e2e`. +- `test/e2e/` - the end-to-end test with `sshd` in a container (DESIGN.md + section 17). Build tag `e2e`. The image of the container is + `test/e2e/testdata/sshd`. - `.claude/agents/security-reviewer.md` - the security review every change gets. ## Commands -These work once the Go module exists (issue #2); the e2e test comes with -issue #15. - ```sh go test -race ./... go vet ./... -golangci-lint run -go test -race -tags e2e ./test/e2e/... # needs Docker +golangci-lint run --build-tags e2e # the tag adds the files of test/e2e +go test -race -tags e2e ./test/e2e/... # needs Docker, ssh and git ``` +The last one is the end-to-end test (DESIGN.md section 17). It builds +nodkey and a Docker image, and removes every container, network and image +that it makes. + ## Rules - **This repository is public.** Never add a real host name, domain name, diff --git a/DESIGN.md b/DESIGN.md index a95f725..f446571 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -2490,18 +2490,97 @@ bws 2.1.0 and bw 2026.9.1 on 2026-10-14 `bw` when it runs under that name. It records its arguments and its environment, so the tests can check that no token, password or session key is ever an argument. -- **End-to-end test**: a throwaway key, `sshd` in a container, the broker and - the client on the test machine, and the fake approver. It checks that - `ssh` and `git` over SSH work, that a deny blocks the login, that an - unknown host key is refused, that `ssh-grant` covers exactly the listed - hosts, and that `nodkey run` passes a secret into a command without - printing it. It needs Docker, so it has its own build tag and its own CI - job. -- **Canary test**: the end-to-end test uses a planted secret value and then - searches every log, the audit log, every message sent to the fake - Telegram server, and the client's output for it. A second canary sits in - a parent process's command line, to test the masking. Any hit fails the - test. +- **End-to-end test** (`test/e2e`): the real programs, with only Telegram + and the store faked. It builds the `nodkey` binary from the repository, + with the race detector when the test runs with it. + - The SSH targets run in one Docker container, built from + `test/e2e/testdata/sshd`: OpenSSH's `sshd` and git from Debian, on a + base image pinned by its digest. Each target is an `sshd` of its own, + with its own throwaway host key: `web-1` (Ed25519), `web-2` (ECDSA), + `db-1` (RSA) and `web-3` (Ed25519). The broker's `known_hosts` has the + first three, not `web-3`. The remote users `deploy` and `ops` have the + public half of the broker's throwaway key `deploy` in their + `authorized_keys`. Only the broker holds its private half. `deploy` + has a bare git repository. The container has no capability but those + that `sshd` needs, no new privileges, limits on its processes and + memory, and `DisableForwarding yes`. It publishes its ports on + `127.0.0.1` only, from a network of its own. + - `nodkey broker`, `nodkey run`, `nodkey ssh-agent`, `nodkey ssh-grant`, + `ssh` and `git` run on the test machine. The files of the broker and + the clients are in a private directory that passes every check of + section 14, with TLS keys from `nodkey keygen`. The store is a fake + `bws`, a shell script. The broker's Telegram code is the real one: it + reaches a fake Telegram server through a fake HTTPS proxy and a test + CA, as in the tests of `nodkey broker`. The test taps "Approve once", + "Approve 15 min" or "Deny" there. Two clients, `laptop` and `ci`, have + their own keys and home directories. + - The cases: `nodkey run` passes the secret into a command, which proves + it with the SHA-256 of the value; after a Deny, the command does not + run; an SSH login after "Approve once"; an unknown host key is refused + before any message; after `ssh-grant` for `web-1` and `web-2`, these + hosts need no tap, and `db-1` and the remote user `ops` still ask; + `git clone` and `git push` over SSH, where one "Approve 15 min" covers + the logins after it; a denied login, after which the grants of the key + have ended. Each case also checks what the message shows, what the + clients print, the entries in the audit log, and that no other + message goes out. The config has no "never ask" rule, and nothing in + the test makes an alert, so every message must have buttons. At the + end, `nodkey audit verify` checks the whole audit log with every + anchor of the broker's journal, and a copy of the log with one changed + entry fails. + - The root-only test of `internal/client` runs as root in a container + without a network, with only the capabilities `SETUID` and `SETGID`: + a process of another user connects to the socket of + `nodkey ssh-agent`, and the agent refuses it (`SO_PEERCRED`). + - Run it with `go test -race -tags e2e ./test/e2e/...`. It needs + Docker, OpenSSH's `ssh` 8.9 or newer, and `git`. Once the image and + the binaries are built, it takes about 10 seconds. The CI job `e2e` + runs it. + - Each run of the test has a run ID: its start time and a random part. + The names and labels of its containers, its network and its image + hold it, and so does the name of its private directory, in + `$XDG_RUNTIME_DIR`, or else in the cache directory (`$XDG_CACHE_HOME`, + or else `~/.cache`). While it runs, the test holds a lock on the file + `nodkey-e2e-RUNID.lock` in the temporary directory (`$TMPDIR`, or else + `/tmp`). At the end, it removes everything that it made, and fails if + something is left. A run that is killed, such as by the `-timeout` of + `go test` or by Ctrl-C, cannot do that: the kernel stops its programs, + its containers end on their own after 15 minutes, and the rest stays. + So each run first removes + what runs that have ended left: those whose lock file nobody holds, + and those older than an hour whose lock file it cannot check: it is + not there, or not a regular file. It never touches a run that holds + its lock, or anything without a run ID. By hand, when no end-to-end + test runs: + + ```sh + docker ps --all --quiet --filter label=nodkey-e2e | xargs -r docker rm --force + docker network ls --quiet --filter label=nodkey-e2e | xargs -r docker network rm + docker image ls nodkey-e2e-sshd --format '{{.Repository}}:{{.Tag}}' | xargs -r docker image rm + rm -rf "${XDG_CACHE_HOME:-$HOME/.cache}"/nodkey-e2e-* "${TMPDIR:-/tmp}"/nodkey-e2e-*.lock + if [ -n "$XDG_RUNTIME_DIR" ]; then rm -rf "$XDG_RUNTIME_DIR"/nodkey-e2e-*; fi + ``` +- **Canary test**: part of the end-to-end test. The first canary is the + value of the secret in the store. The second canary is a random word in + the command line of a wrapper process, which runs `ssh` and `git`: as + the value of `--api-token=`, and as a word of its own. The test searches + every log (the broker's journal, `nodkey ssh-agent` and the `sshd`s), + the audit log, every call to the fake Telegram server, and every output + of the clients. It looks for the first canary as it is, in lower and + upper case, in hex, and in base64 and base64url at each offset, and for + each part of 12 characters of its random part in the same forms, so + that a value that was cut is found too. It looks for the made-up bot + token and the token of the store in the same forms, and for the second + canary in any case. Any hit fails the test. The command line of the + wrapper is claimed text, so section 7 shows it only in the message and + in the audit log, masked. The test checks that every message and every + audit entry of an SSH login shows `***` in place of both forms. It also + checks that no other call to Telegram, and no output of `nodkey`, holds + claimed text: a reason, a word of a command line, the working + directory, or the variable name. Guards make sure that there was + something to find: the broker handed out the secret (a `secret_sent` + entry), and SSH logins went through the wrapper. So a `-run` filter + that leaves out the cases makes the search fail, not pass. - **One manual test with a real Telegram bot**, written down in `docs/manual-telegram-test.md`. The token comes from a local file and is never committed. diff --git a/test/e2e/audit_test.go b/test/e2e/audit_test.go new file mode 100644 index 0000000..f67f0d1 --- /dev/null +++ b/test/e2e/audit_test.go @@ -0,0 +1,140 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "encoding/json" + "errors" + "io/fs" + "os" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +// auditLog reads the audit log of the broker, one case after the other: +// each case checks the entries that it caused. +type auditLog struct { + path string + seen int // the entries that the cases have checked +} + +// entry is one entry of the audit log. +type entry map[string]any + +// field returns the field k as text: a string as it is, a list of strings +// joined with ",", a number in decimal, and "" when the entry does not +// have it. +func (en entry) field(k string) string { + switch v := en[k].(type) { + case string: + return v + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + case []any: + parts := make([]string, len(v)) + for i, p := range v { + parts[i], _ = p.(string) + } + return strings.Join(parts, ",") + case nil: + return "" + } + return "(not a string, a number or a list)" +} + +// name returns the event of the entry, with its answer, cause, via or +// code, such as "decision:approve_once" or "denied:not_allowed". +func (en entry) name() string { + name := en.field("event") + for _, k := range []string{"answer", "cause", "via", "code"} { + if v := en.field(k); v != "" { + name += ":" + v + } + } + return name +} + +// eventNames returns the names of entries. +func eventNames(entries []entry) []string { + out := make([]string, len(entries)) + for i, en := range entries { + out[i] = en.name() + } + return out +} + +// entries returns the complete lines of the audit log, as entries. The +// broker writes each line with one write call, but a read can come in the +// middle of it, so a last line without its newline is left for later. +func (a *auditLog) entries(t *testing.T) []entry { + t.Helper() + data, err := os.ReadFile(a.path) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil // the broker makes it at its start + case err != nil: + t.Fatal(err) + } + data = data[:bytes.LastIndexByte(data, '\n')+1] + var out []entry + for line := range strings.Lines(string(data)) { + var en entry + if err := json.Unmarshal([]byte(line), &en); err != nil { + t.Fatalf("audit log line %d is not JSON: %v", len(out)+1, err) + } + out = append(out, en) + } + return out +} + +// next waits until the audit log has len(want) entries after those that +// the cases have checked, at most settleLimit, and returns them. The +// broker writes the entries of a request before it answers the client. It fails unless their names +// are want, in this order, and unless each of them has the fields of +// subject, where "" means that the entry does not have the field. The +// entries of one request also share its request ID. +func (a *auditLog) next(t *testing.T, subject map[string]string, want ...string) []entry { + t.Helper() + deadline := time.Now().Add(settleLimit) + var got []entry + for { + all := a.entries(t) + got = all[min(a.seen, len(all)):] + if len(got) >= len(want) || time.Now().After(deadline) { + break + } + time.Sleep(20 * time.Millisecond) + } + got = got[:min(len(got), len(want))] + a.seen += len(got) + if !slices.Equal(eventNames(got), want) { + t.Fatalf("audit entries %q, want %q", eventNames(got), want) + } + for i, en := range got { + for k, v := range subject { + if g := en.field(k); g != v { + t.Errorf("audit entry %d (%s) has %s %q, want %q", a.seen-len(got)+i+1, en.name(), k, redact(g), v) + } + } + if id := got[0].field("request_id"); en.field("request_id") != id { + t.Errorf("audit entry %d (%s) has another request ID than the entries before it", a.seen-len(got)+i+1, en.name()) + } + } + return got +} + +// unchecked reports the entries that no case has checked yet, and marks +// them as checked, so that one failed case does not make every later case +// fail. +func (a *auditLog) unchecked(t *testing.T) { + t.Helper() + all := a.entries(t) + if len(all) > a.seen { + t.Errorf("audit entries that no case expected: %q", eventNames(all[a.seen:])) + a.seen = len(all) + } +} diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go new file mode 100644 index 0000000..a6afaef --- /dev/null +++ b/test/e2e/canary_test.go @@ -0,0 +1,171 @@ +//go:build e2e + +package e2e + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "strings" + "sync" + "testing" +) + +// The canary search of DESIGN.md section 17. The first canary is the +// value of the secret in the store: nodkey must never show it, log it or +// send it anywhere but into the environment of the command of nodkey run. +// The second canary is a word in the command line of a parent process of +// ssh: nodkey ssh-agent sends it to the broker as claimed text, and the +// broker masks it before it shows or logs it (DESIGN.md section 7). + +// canaryForm is one form of a canary, as the search looks for it. +type canaryForm struct { + name string + value string +} + +// formsOf returns the forms of value to look for: as it is, in lower and +// upper case, in hex, and in base64 and base64url. A base64 text holds a +// value in one of 3 ways, by its offset modulo 3 in the bytes that were +// encoded, so there are 3 forms of each: the characters that depend only on +// the value (base64Core). Each name starts with label. +func formsOf(label, value string) []canaryForm { + h := hex.EncodeToString([]byte(value)) + forms := []canaryForm{ + {label + " as it is", value}, + {label + " in lower case", strings.ToLower(value)}, + {label + " in upper case", strings.ToUpper(value)}, + {label + " in hex", h}, + {label + " in upper-case hex", strings.ToUpper(h)}, + } + for shift := range 3 { + forms = append(forms, + canaryForm{fmt.Sprintf("%s in base64 at offset %d", label, shift), base64Core(base64.RawStdEncoding, value, shift)}, + canaryForm{fmt.Sprintf("%s in base64url at offset %d", label, shift), base64Core(base64.RawURLEncoding, value, shift)}) + } + return forms +} + +// canary1Prefix starts the first canary; the rest of it is random. +const canary1Prefix = "e2e-canary-" + +// windowLen is the length of the parts of the first canary that the +// search looks for, so that it finds a value that was cut, too. +const windowLen = 12 + +// secretForms returns the forms of the first canary to look for (formsOf): +// the whole value, and each part of windowLen characters of its random +// part. +func secretForms(canary string) []canaryForm { + forms := formsOf("the first canary", canary) + random := strings.TrimPrefix(canary, canary1Prefix) + for i := 0; i+windowLen <= len(random); i++ { + forms = append(forms, formsOf(fmt.Sprintf("a part of the first canary (at %d)", i), random[i:i+windowLen])...) + } + return forms +} + +// base64Core returns the characters of the base64 of value after shift +// other bytes that depend only on value. Every base64 text that encodes +// value at an offset of shift modulo 3 holds them, whatever comes before +// and after the value. +func base64Core(enc *base64.Encoding, value string, shift int) string { + s := enc.EncodeToString(append(make([]byte, shift), value...)) + // Character i holds bits 6i to 6i+5; the value has bits 8*shift to + // 8*(shift+len(value))-1. + start := (8*shift + 5) / 6 + end := 8 * (shift + len(value)) / 6 + return s[start:end] +} + +// found returns the names of the forms that text holds. +func found(text string, forms []canaryForm) []string { + var names []string + for _, f := range forms { + if strings.Contains(text, f.value) { + names = append(names, f.name) + } + } + return names +} + +// TestCanaryForms: the search finds the first canary in each form that it +// looks for, also inside other data, whatever its offset in a base64 +// text, and also when only a part of it is there. So a search that finds +// nothing has looked. +func TestCanaryForms(t *testing.T) { + value := newCanary1(t) + forms := secretForms(value) + random := strings.TrimPrefix(value, canary1Prefix) + for _, part := range []string{value, random[:windowLen], random[len(random)-windowLen-1:], random[3 : 3+windowLen+2]} { + for _, before := range []string{"", "a", "ab", "abc"} { + for _, after := range []string{"", "z", "yz"} { + data := []byte(before + part + after) + h := hex.EncodeToString(data) + for name, text := range map[string]string{ + "plain": string(data), + "lower case": strings.ToLower(string(data)), + "upper case": strings.ToUpper(string(data)), + "hex": h, + "upper hex": strings.ToUpper(h), + "base64": base64.StdEncoding.EncodeToString(data), + "raw base64": base64.RawStdEncoding.EncodeToString(data), + "base64url": base64.URLEncoding.EncodeToString(data), + "raw base64url": base64.RawURLEncoding.EncodeToString(data), + } { + if len(found("{"+text+"}", forms)) == 0 { + t.Errorf("the search does not find %d characters of the canary %s after %q and before %q", + len(part), name, before, after) + } + } + } + } + } + if got := found(strings.Repeat(canary1Prefix, 10)+random[:windowLen-1], forms); len(got) != 0 { + t.Errorf("the search finds %q in a text without the canary", got) + } +} + +// hidden holds the canaries while the test runs, so that a failure +// message never shows them (redact). +var hidden struct { + mu sync.Mutex + values []string +} + +// hide makes redact replace each of values. +func hide(values ...string) { + hidden.mu.Lock() + defer hidden.mu.Unlock() + hidden.values = append(hidden.values, values...) +} + +// redact returns s with each canary replaced by "[canary]", in any case. +// The test prints text only through it, so that a test failure never +// prints a value (AGENTS.md). +func redact(s string) string { + hidden.mu.Lock() + defer hidden.mu.Unlock() + for _, v := range hidden.values { + for { + i := strings.Index(asciiLower(s), asciiLower(v)) + if i < 0 { + break + } + s = s[:i] + "[canary]" + s[i+len(v):] + } + } + return s +} + +// asciiLower returns s with the ASCII letters A to Z in lower case. It +// keeps the length of s, unlike strings.ToLower. +func asciiLower(s string) string { + b := []byte(s) + for i, c := range b { + if 'A' <= c && c <= 'Z' { + b[i] = c + 'a' - 'A' + } + } + return string(b) +} diff --git a/test/e2e/doc.go b/test/e2e/doc.go new file mode 100644 index 0000000..a974e62 --- /dev/null +++ b/test/e2e/doc.go @@ -0,0 +1,9 @@ +// Package e2e is the end-to-end test of nodkey (DESIGN.md section 17, +// issue #15). It builds the nodkey binary, starts sshd in a Docker +// container, runs the real nodkey broker with a fake Telegram server, and +// logs in with the real ssh and git through nodkey ssh-agent. It needs +// Docker, so its test files have the build tag e2e, and a plain +// "go test ./..." does not run them: +// +// go test -race -tags e2e ./test/e2e/... +package e2e diff --git a/test/e2e/docker_test.go b/test/e2e/docker_test.go new file mode 100644 index 0000000..5142f1f --- /dev/null +++ b/test/e2e/docker_test.go @@ -0,0 +1,348 @@ +//go:build e2e + +package e2e + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// The containers of the test. Every container, network and image that the +// test makes has a name with runID, and every container and network has +// the label runLabel, so that TestMain can check at the end that none is +// left. The container of the SSH targets ends on its own after +// containerLifetime, in case the test process ends without its cleanups, +// such as when go test kills it after its -timeout. A later run removes +// what is left (removeStale). + +// runLabel is the label of every container and network of this run. +var runLabel = "nodkey-e2e=" + runID + +// imageTag is the tag of the image of testdata/sshd that this run builds. +// TestMain removes it after the tests. +var imageTag = "nodkey-e2e-sshd:" + runID + +// containerLifetime is how long a container may run: it ends on its own +// after this time. +const containerLifetime = 15 * time.Minute + +// Time limits of the docker commands. The build installs packages with +// apt-get, unless Docker's build cache has them. +const ( + dockerBuildLimit = 10 * time.Minute + dockerLimit = 2 * time.Minute +) + +// runDocker runs docker with args and the time limit limit, and returns +// what it wrote to stdout and to stderr. It does not use the test's +// context, so that it also works in the cleanups. +func runDocker(limit time.Duration, args ...string) (string, string, error) { + return runDockerWith(limit, nil, args...) +} + +// runDockerWith is runDocker with stdin. +func runDockerWith(limit time.Duration, stdin io.Reader, args ...string) (string, string, error) { + ctx, cancel := context.WithTimeout(context.Background(), limit) + defer cancel() + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, "docker", args...) //nolint:gosec // G204: the test runs docker with its own arguments. + cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, &stdout, &stderr + err := cmd.Run() + return stdout.String(), stderr.String(), err +} + +// copyIn copies the files of the directory dir into the directory /e2e of +// the container id, as files of root, with their modes. docker cp would +// keep the owner of each file, this user, and root without capabilities +// cannot read a file of another user with mode 0600, such as a host key. +// So it sends docker cp a tar archive whose files belong to root. +func copyIn(t *testing.T, id, dir string) { + t.Helper() + var archive bytes.Buffer + tw := tar.NewWriter(&archive) + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, en := range entries { + info, err := en.Info() + if err != nil || !info.Mode().IsRegular() { + t.Fatalf("%s is not a regular file: %v", en.Name(), err) + } + data := readFile(t, filepath.Join(dir, en.Name())) + hdr := &tar.Header{Typeflag: tar.TypeReg, Name: en.Name(), Mode: int64(info.Mode().Perm()), Size: int64(len(data)), + ModTime: info.ModTime()} + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(data); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if stdout, stderr, err := runDockerWith(dockerLimit, &archive, "cp", "-", id+":/e2e"); err != nil { + t.Fatalf("docker cp: %v\n%s%s", err, stdout, stderr) + } +} + +// mustDocker runs docker with args, and fails the test on an error. It +// returns stdout without the space around it. +func mustDocker(t *testing.T, limit time.Duration, args ...string) string { + t.Helper() + stdout, stderr, err := runDocker(limit, args...) + if err != nil { + t.Fatalf("docker %s: %v\n%s%s", strings.Join(args, " "), err, stdout, stderr) + } + return strings.TrimSpace(stdout) +} + +// requireDocker fails the test when Docker cannot be used. The test is run +// only on purpose, with the build tag e2e, so it never skips. +func requireDocker(t *testing.T) { + t.Helper() + if _, stderr, err := runDocker(dockerLimit, "version", "--format", "{{.Server.Version}}"); err != nil { + t.Fatalf("the end-to-end test needs Docker: docker version: %v\n%s", err, stderr) + } +} + +// image is the image of testdata/sshd, built once per test process. +var image struct { + once sync.Once + built bool // the build ran and made imageTag + err string // why the build failed +} + +// sshdImage builds the image of testdata/sshd once, and returns its tag. +// The Dockerfile pins its base image by digest. Before the build, it +// removes what earlier runs left (removeStale). +func sshdImage(t *testing.T) string { + t.Helper() + requireDocker(t) + image.once.Do(func() { + removeStale(t) + stdout, stderr, err := runDocker(dockerBuildLimit, "build", "--quiet", "--tag", imageTag, "testdata/sshd") + image.built = true // remove the tag at the end, also after a failed build + if err != nil { + image.err = fmt.Sprintf("docker build: %v\n%s%s", err, stdout, stderr) + } + }) + if image.err != "" { + t.Fatal(image.err) + } + return imageTag +} + +// removeImage removes the image of this test process, if it was built. It +// reports what it could not remove. +func removeImage() error { + if !image.built { + return nil + } + if _, stderr, err := runDocker(dockerLimit, "image", "rm", imageTag); err != nil && + !strings.Contains(stderr, "No such image") { + return fmt.Errorf("docker image rm %s: %w: %s", imageTag, err, strings.TrimSpace(stderr)) + } + return nil +} + +// leftovers returns the containers, networks and images of this test +// process that are still there. +func leftovers() ([]string, error) { + var left []string + for _, args := range [][]string{ + {"ps", "--all", "--quiet", "--filter", "label=" + runLabel}, + {"network", "ls", "--quiet", "--filter", "label=" + runLabel}, + {"image", "ls", "--quiet", imageTag}, + } { + stdout, stderr, err := runDocker(dockerLimit, args...) + if err != nil { + return nil, fmt.Errorf("docker %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr)) + } + for id := range strings.FieldsSeq(stdout) { + left = append(left, args[0]+" "+id) + } + } + return left, nil +} + +// removeContainer removes the container id, and waits until it is gone. +// The root container runs with --rm, so Docker may already be removing it +// when it has ended. +func removeContainer(id string) error { + _, stderr, err := runDocker(dockerLimit, "rm", "--force", id) + switch { + case err == nil, strings.Contains(stderr, "No such container"): + return nil + case !strings.Contains(stderr, "already in progress"): + return fmt.Errorf("docker rm %s: %w: %s", id, err, strings.TrimSpace(stderr)) + } + for deadline := time.Now().Add(dockerLimit); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + if _, stderr, err := runDocker(dockerLimit, "container", "inspect", id); err != nil && + strings.Contains(stderr, "No such container") { + return nil + } + } + return fmt.Errorf("docker did not remove the container %s within %s", id, dockerLimit) +} + +// hardening are the options of every container of the test: no +// capability but those that a container adds, no new privileges through +// a program file, and limits on processes and memory. +var hardening = []string{ + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--pids-limit", "256", "--memory", "512m", +} + +// targets is the container of the SSH targets: one sshd for each target, +// each with its own port and host key. +type targets struct { + id string + ports map[string]int // the port of each target on 127.0.0.1 of this machine +} + +// startTargets starts the container of the SSH targets on a network of its +// own, until the test ends. files is a directory with the files of +// entrypoint.sh; names are the targets, in the order of the lines of its +// file "targets", whose ports in the container start at 2201. Each port +// is published on 127.0.0.1 only. startTargets waits until each sshd +// answers. +func startTargets(t *testing.T, img, files string, names []string) *targets { + t.Helper() + network := "nodkey-e2e-" + runID + mustDocker(t, dockerLimit, "network", "create", "--label", runLabel, network) + t.Cleanup(func() { + if _, stderr, err := runDocker(dockerLimit, "network", "rm", network); err != nil { + t.Errorf("docker network rm %s: %v: %s", network, err, stderr) + } + }) + // No --rm: when sshd stops at once, its log stays for the failure. + args := append([]string{"create", "--label", runLabel, "--name", "nodkey-e2e-" + runID, "--network", network, + // sshd runs the part before the login as its own user in a chroot, + // and then changes to the remote user. Nothing else in the + // container needs a capability: the test passes without + // AUDIT_WRITE and KILL, which Docker gives by default. + "--cap-add", "SETUID", "--cap-add", "SETGID", "--cap-add", "SYS_CHROOT"}, hardening...) + for i := range names { + args = append(args, "--publish", "127.0.0.1::"+strconv.Itoa(2201+i)) + } + args = append(args, img, strconv.Itoa(int(containerLifetime/time.Second))) + tg := &targets{id: mustDocker(t, dockerLimit, args...), ports: make(map[string]int)} + t.Cleanup(func() { + if t.Failed() { + t.Logf("the log of the SSH targets:\n%s", tg.logs(t)) + } + if err := removeContainer(tg.id); err != nil { + t.Error(err) + } + }) + copyIn(t, tg.id, files) + mustDocker(t, dockerLimit, "start", tg.id) + + // Lines such as "2201/tcp -> 127.0.0.1:32768". + published := mustDocker(t, dockerLimit, "port", tg.id) + for i, name := range names { + prefix := strconv.Itoa(2201+i) + "/tcp -> 127.0.0.1:" + for line := range strings.Lines(published) { + if rest, ok := strings.CutPrefix(strings.TrimSpace(line), prefix); ok { + port, err := strconv.Atoi(rest) + if err != nil { + t.Fatalf("docker port: %q", line) + } + tg.ports[name] = port + } + } + if tg.ports[name] == 0 { + t.Fatalf("docker port has no port for %s:\n%s\nthe log of the targets:\n%s", name, published, tg.logs(t)) + } + } + for _, name := range names { + tg.waitSSHD(t, name) + } + return tg +} + +// waitSSHD waits until the sshd of the target name sends its banner. A +// connection to a published port can open before sshd listens, so only +// the banner counts. +func (tg *targets) waitSSHD(t *testing.T, name string) { + t.Helper() + addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(tg.ports[name])) + deadline := time.Now().Add(waitLimit) + for { + if bannerOf(t, addr) { + return + } + if time.Now().After(deadline) { + t.Fatalf("the sshd of %s does not answer on %s; the log of the targets:\n%s", name, addr, tg.logs(t)) + } + time.Sleep(100 * time.Millisecond) + } +} + +// bannerOf reports whether an SSH server answers on addr with its banner. +func bannerOf(t *testing.T, addr string) bool { + t.Helper() + c, err := (&net.Dialer{Timeout: 2 * time.Second}).DialContext(t.Context(), "tcp", addr) + if err != nil { + return false + } + defer func() { _ = c.Close() }() + _ = c.SetReadDeadline(time.Now().Add(2 * time.Second)) + line, err := bufio.NewReader(c).ReadString('\n') + return err == nil && strings.HasPrefix(line, "SSH-2.0-") +} + +// logs returns the log of the container: the lines of each sshd. +func (tg *targets) logs(t *testing.T) string { + t.Helper() + stdout, stderr, err := runDocker(dockerLimit, "logs", tg.id) + if err != nil { + return fmt.Sprintf("(docker logs: %v: %s)", err, stderr) + } + return stdout + stderr +} + +// runAsRoot runs the program bin in a new container of img, as root, +// without a network, with args. It returns what the program wrote and its +// exit code. +func runAsRoot(t *testing.T, img, bin string, args ...string) (string, int) { + t.Helper() + name := "nodkey-e2e-root-" + runID + create := append([]string{"create", "--rm", "--label", runLabel, "--name", name, "--network", "none", + "--user", "0:0", "--entrypoint", "/e2e/" + filepath.Base(bin), + // The test starts a child as another user (65534). + "--cap-add", "SETUID", "--cap-add", "SETGID"}, hardening...) + create = append(append(create, img), args...) + id := mustDocker(t, dockerLimit, create...) + t.Cleanup(func() { + if err := removeContainer(id); err != nil { + t.Error(err) + } + }) + copyIn(t, id, filepath.Dir(bin)) + stdout, stderr, err := runDocker(dockerLimit, "start", "--attach", id) + var exit *exec.ExitError + switch { + case err == nil: + return stdout + stderr, 0 + case errors.As(err, &exit): + return stdout + stderr, exit.ExitCode() + } + t.Fatalf("docker start: %v\n%s%s", err, stdout, stderr) + return "", -1 +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 0000000..1eb10e2 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,731 @@ +//go:build e2e + +package e2e + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "syscall" + "testing" + + "golang.org/x/crypto/ssh" +) + +// TestEndToEnd is the end-to-end test of version 1 (issue #15, DESIGN.md +// section 17): the real nodkey broker, nodkey run, nodkey ssh-agent and +// nodkey ssh-grant, the real ssh and git, sshd in a container, and the +// person's taps at a fake Telegram server. Each case also checks the +// entries that it adds to the audit log. At the end, nodkey audit verify +// checks the whole log with every anchor of the broker's journal, and +// the canary search looks at every log and every output. +// +// The cases run in this order on purpose. A Deny pauses the key for 60 +// seconds and ends its grants (DESIGN.md section 8), so the denied login +// comes last, and the grants of ssh-grant and of "Approve 15 min" cover +// hosts that the later cases do not expect to be covered. +func TestEndToEnd(t *testing.T) { + t.Parallel() + e := newEnv(t) + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"nodkey run passes the secret into the command", e.runApproved}, + {"nodkey run denied", e.runDenied}, + {"ssh login approved", e.loginApproved}, + {"unknown host key refused", e.unknownHostKey}, + {"ssh-grant covers exactly the listed hosts", e.sshGrant}, + {"git over SSH", e.gitOverSSH}, + {"ssh login denied, and the Deny ends the grants", e.loginDenied}, + } { + t.Run(c.name, func(t *testing.T) { + e.audit.unchecked(t) + c.run(t) + }) + } + t.Run("the clients and the broker stop", e.stop) + t.Run("nodkey audit verify passes with every anchor", e.verifyAudit) + t.Run("only the messages of requests and the audit log hold claimed text", e.searchClaimed) + t.Run("the canaries and the tokens are nowhere", e.searchCanaries) +} + +// TestAgentRefusesAnotherRealUserInDocker runs the root-only test of +// internal/client (issue #13) as root in a container, where it can switch +// a child process to another user: that process connects to the socket of +// nodkey ssh-agent, and the agent refuses it (SO_PEERCRED). The test must +// pass there, not skip. +func TestAgentRefusesAnotherRealUserInDocker(t *testing.T) { + t.Parallel() + img := sshdImage(t) + bin := buildClientTest(t, subdirectory(t, "root-")) + out, code := runAsRoot(t, img, bin, + "-test.run=^TestAgentRefusesAnotherRealUser$", "-test.v", "-test.count=1", "-test.timeout=2m") + switch { + case code != 0: + t.Fatalf("the test of internal/client failed in the container (exit %d):\n%s", code, out) + case !strings.Contains(out, "--- PASS: TestAgentRefusesAnotherRealUser "): + t.Fatalf("the test of internal/client did not pass in the container:\n%s", out) + case strings.Contains(out, "--- SKIP"): + t.Fatalf("the test of internal/client skipped in the container:\n%s", out) + } + t.Logf("in the container, as root:\n%s", out) +} + +// Patterns of the lines of a message (DESIGN.md section 7). +const ( + codePattern = `[2-9a-hjkmnp-z]{4}` // a short code + clockPattern = `\d\d:\d\d:\d\d` + expiryPattern = `^expires ` + clockPattern + ` \(\d+ s\)$` + // wrapperShown is how the command line of the wrapper shows in a + // message: both forms of the second canary masked. + wrapperShown = wrapperArg + " --api-token=*** *** -- " +) + +// lit returns a pattern that matches exactly s. +func lit(s string) string { return "^" + regexp.QuoteMeta(s) + "$" } + +// checkLines fails unless text has one line for each pattern, and each +// line matches its pattern. +func checkLines(t *testing.T, what, text string, patterns ...string) { + t.Helper() + lines := strings.Split(text, "\n") + if len(lines) != len(patterns) { + t.Errorf("%s has %d lines, want %d:\n%s", what, len(lines), len(patterns), redact(text)) + return + } + for i, p := range patterns { + if !regexp.MustCompile(p).MatchString(lines[i]) { + t.Errorf("%s: line %d is %q, want it to match %q", what, i+1, redact(lines[i]), p) + } + } +} + +// shortCode returns the short code in the first line of a message. +func shortCode(t *testing.T, m tgMessage) string { + t.Helper() + first, _, _ := strings.Cut(m.text, "\n") + code := first[strings.LastIndexByte(first, ' ')+1:] + if !regexp.MustCompile("^" + codePattern + "$").MatchString(code) { + t.Fatalf("the first line of the message has no short code: %q", redact(first)) + } + return code +} + +// checkMessageEntry checks the message of the request entry en: the exact +// text that the person saw, and the buttons as its last line (DESIGN.md +// section 9). +func checkMessageEntry(t *testing.T, en entry, m tgMessage, buttons string) { + t.Helper() + if got, want := en.field("message"), m.text+"\n"+buttons; got != want { + t.Errorf("the request entry has the message\n%s\nwant the message that the person saw\n%s", + redact(got), redact(want)) + } +} + +// checkResult waits until the broker shows the end of the request on its +// message, and checks that line. +func (e *env) checkResult(t *testing.T, m tgMessage, pattern string) { + t.Helper() + if got := e.tg.waitResult(t, m); !regexp.MustCompile(pattern).MatchString(got) { + t.Errorf("the message shows the result %q, want it to match %q", redact(got), pattern) + } +} + +// tapAndCheck taps the button prefix on m, and checks the text that the +// phone shows. +func (e *env) tapAndCheck(t *testing.T, m tgMessage, prefix, want string) { + t.Helper() + if got := e.tg.tap(t, m, prefix); got != want { + t.Errorf("the phone shows %q after the tap, want %q", redact(got), want) + } +} + +// checkMessages fails unless want messages went out since there were n: +// with buttons or without. An alert or a notice counts too. +func (e *env) checkMessages(t *testing.T, n, want int) { + t.Helper() + if got := e.tg.sent() - n; got != want { + t.Errorf("%d messages went out, want %d", got, want) + } +} + +// nodkeyRun starts nodkey run as the client ci, with the secret of the +// broker in secretVar, and the command argv. +func (e *env) nodkeyRun(t *testing.T, reason string, argv ...string) *proc { + t.Helper() + args := append([]string{e.bin, "run", "--secret", secretVar + "=" + secretName, "--reason", reason, "--"}, argv...) + return e.procs.start(t, "nodkey run", filepath.Join(e.ci.home, "work"), e.ci.env(), nil, args...) +} + +// lookIn returns the path of the program name in the directories of path, +// as nodkey run finds it. +func lookIn(t *testing.T, path, name string) string { + t.Helper() + for dir := range strings.SplitSeq(path, ":") { + p := filepath.Join(dir, name) + if fi, err := os.Stat(p); err == nil && fi.Mode().IsRegular() && fi.Mode()&0o111 != 0 { + return p + } + } + t.Fatalf("no %s in %s", name, path) + return "" +} + +// secretsSubject returns the fields of every audit entry of a request of +// nodkey run by ci. +func secretsSubject(command, reason string) map[string]string { + return map[string]string{ + "client": "ci", "kind": "secrets", "secrets": secretName, "key": "", "hosts": "", "host_key": "", + "remote_user": "", "command": command, "reason": reason, + } +} + +// runApproved: nodkey run asks, the person taps "Approve once", and the +// command gets the value of the secret in its environment. It proves this +// by the SHA-256 of the value, and prints nothing else; nodkey prints only +// the short code (DESIGN.md section 5). +func (e *env) runApproved(t *testing.T) { + reason := e.reason("e2e: give the command the canary secret") + script := `printf %s "$` + secretVar + `" | sha256sum` + n := e.tg.sent() + p := e.nodkeyRun(t, reason, "sh", "-c", script) + m := e.tg.waitMessage(t, n+1, p) + code := shortCode(t, m) + command := lookIn(t, clientPath, "sh") + " -c '" + script + "'" + checkLines(t, "the message", m.text, + lit("๐Ÿ”‘ Secrets ยท request "+code), + lit("โœ… machine: ci"), + lit("โœ… secrets: "+secretName), + lit("โ€” claimed by the agent โ€”"), + lit("as: "+secretVar), + lit("command: "+command), + `^in: \S+/ci/work$`, + lit("reason: "+reason), + expiryPattern) + if want := []string{tapOnce + m.requestID, tapDeny + m.requestID}; !slices.Equal(m.buttons, want) { + t.Errorf("the buttons %q, want %q: the rule is always", m.buttons, want) + } + e.tapAndCheck(t, m, tapOnce, "Approved.") + p.mustExit(t, 0) + + sum := sha256.Sum256([]byte(e.canary1)) + if got, want := p.stdout.all(), []string{hex.EncodeToString(sum[:]) + " -"}; !slices.Equal(got, want) { + t.Errorf("the command printed %q, want the SHA-256 of the secret: %q", redact(strings.Join(got, "\n")), want) + } + if got, want := p.stderr.all(), []string{"nodkey: waiting for approval (request " + code + ")"}; !slices.Equal(got, want) { + t.Errorf("nodkey run wrote %q to stderr, want %q", redact(strings.Join(got, "\n")), want) + } + entries := e.audit.next(t, secretsSubject(command, reason), "request", "decision:approve_once", "secret_sent") + checkMessageEntry(t, entries[0], m, "[ Approve once ] [ Deny ]") + if id := entries[1].field("user_id"); id != strconv.FormatInt(approverID, 10) { + t.Errorf("the decision entry has the user ID %q, want the approver's", id) + } + e.checkResult(t, m, `^๐Ÿ‘ approved once by you at `+clockPattern+`$`) + e.checkMessages(t, n, 1) +} + +// runDenied: the person taps Deny. nodkey run exits with 77, and the +// command does not run. +func (e *env) runDenied(t *testing.T) { + reason := e.reason("e2e: a command that must not run") + const file = "the-command-ran" // in the working directory of nodkey run + marker := filepath.Join(e.ci.home, "work", file) + n := e.tg.sent() + p := e.nodkeyRun(t, reason, "touch", file) + m := e.tg.waitMessage(t, n+1, p) + e.tapAndCheck(t, m, tapDeny, "Denied.") + p.mustExit(t, 77) + if last := p.stderr.last(); last != "nodkey: denied (exit 77)" { + t.Errorf("the last line of nodkey run is %q, want %q", redact(last), "nodkey: denied (exit 77)") + } + if fileExists(marker) { + t.Error("the command ran after a Deny") + } + command := lookIn(t, clientPath, "touch") + " " + file + e.audit.next(t, secretsSubject(command, reason), "request", "decision:deny", "denied:approver") + e.checkResult(t, m, `^โ›” denied by you at `+clockPattern+`$`) + e.checkMessages(t, n, 1) +} + +// loginSubject returns the fields of every audit entry of an SSH login by +// laptop as user to host, which the broker checked. +func (e *env) loginSubject(user, host, reason string) map[string]string { + return map[string]string{ + "client": "laptop", "kind": "ssh_login", "key": keyName, "secrets": "", "hosts": host, + "host_key": ssh.FingerprintSHA256(e.hostKeys[host]), "remote_user": user, "reason": reason, + } +} + +// checkLoginMessage checks the message of an SSH login as user to host +// that ssh made through the wrapper (DESIGN.md section 7): the verified +// lines, and the claimed command lines of ssh and its parents, with the +// second canary masked in the command line of the wrapper. +func (e *env) checkLoginMessage(t *testing.T, m tgMessage, user, host, reason string) { + t.Helper() + fp := ssh.FingerprintSHA256(e.hostKeys[host]) + checkLines(t, "the message", m.text, + `^๐Ÿ” SSH login ยท request `+codePattern+`$`, + lit("โœ… machine: laptop"), + lit("โœ… key: "+keyName), + lit("โœ… host: "+host+" (host key "+fp[:len("SHA256:")+4]+"โ€ฆ)"), + lit("โœ… remote user: "+user), + lit("โ€” claimed by the agent โ€”"), + "^command: "+regexp.QuoteMeta(e.sshPath+" -F ")+`\S+`+regexp.QuoteMeta(" "+user+"@"+host+" echo e2e-ok โ† ")+ + `\S+`+regexp.QuoteMeta(" "+wrapperShown+e.sshPath+" -F "), + lit("reason: "+reason), + expiryPattern) + if want := []string{tapOnce + m.requestID, tapWindow + m.requestID, tapDeny + m.requestID}; !slices.Equal(m.buttons, want) { + t.Errorf("the buttons %q, want %q: the rule is window", m.buttons, want) + } +} + +// login logs in with ssh as user to host, and checks what the message +// shows. It taps the button prefix, and checks that ssh logs in. It +// returns the audit entries of the login. +func (e *env) login(t *testing.T, user, host, prefix string) []entry { + t.Helper() + reason := e.reason("e2e: log in to " + host + " as " + user) + n, agentLines := e.tg.sent(), e.agent.stderr.count() + p := e.ssh(t, user, host, reason) + m := e.tg.waitMessage(t, n+1, p) + code := shortCode(t, m) + e.checkLoginMessage(t, m, user, host, reason) + e.tapAndCheck(t, m, prefix, "Approved.") + p.mustExit(t, 0) + if got := p.stdout.all(); !slices.Equal(got, []string{"e2e-ok"}) { + t.Errorf("ssh printed %q, want the output of the command", redact(strings.Join(got, "\n"))) + } + want := []string{"nodkey: waiting for approval (request " + code + ")"} + if got := e.agent.stderr.next(agentLines, len(want)); !slices.Equal(got, want) { + t.Errorf("nodkey ssh-agent wrote %q, want %q", redact(strings.Join(got, "\n")), want) + } + answer := map[string]string{tapOnce: "decision:approve_once", tapWindow: "decision:approve_window"}[prefix] + entries := e.audit.next(t, e.loginSubject(user, host, reason), "request", answer, "signed") + checkMessageEntry(t, entries[0], m, "[ Approve once ] [ Approve 15 min ] [ Deny ]") + if command := entries[0].field("command"); !strings.Contains(command, wrapperShown) { + t.Errorf("the request entry has the command %q, want the masked command line of the wrapper", redact(command)) + } + result := `^๐Ÿ‘ approved once by you at ` + clockPattern + `$` + if prefix == tapWindow { + result = `^๐Ÿ‘ approved for 15 min by you at ` + clockPattern + `$` + if w := entries[1].field("window_minutes"); w != "15" { + t.Errorf("the decision entry has window_minutes %q, want 15", w) + } + } + e.checkResult(t, m, result) + e.checkMessages(t, n, 1) + return entries +} + +// loginByGrant logs in with ssh as user to host, and checks that a grant +// let it in, without a message. +func (e *env) loginByGrant(t *testing.T, user, host string) { + t.Helper() + reason := e.reason("e2e: log in to " + host + " as " + user) + n := e.tg.sent() + p := e.ssh(t, user, host, reason) + p.mustExit(t, 0) + if e.tg.sent() != n { + t.Errorf("a message went out for the login to %s; a grant should cover it, without a message", host) + } + subject := e.loginSubject(user, host, reason) + subject["message"] = "" // no message + e.audit.next(t, subject, "request", "grant_used:grant", "signed") +} + +// loginApproved: ssh logs in to web-1 after the person taps "Approve +// once". The message names the host from the broker's own known_hosts, +// and shows the claimed command lines, masked. +func (e *env) loginApproved(t *testing.T) { + e.login(t, "deploy", "web-1", tapOnce) +} + +// unknownHostKey: web-3's host key is not in the broker's known_hosts. The +// broker refuses the login before any message, and its log says why. +func (e *env) unknownHostKey(t *testing.T) { + reason := e.reason("e2e: log in to web-3") + n, agentLines := e.tg.sent(), e.agent.stderr.count() + p := e.ssh(t, "deploy", unknownTarget, reason) + p.mustExit(t, 255) + if !p.stderr.has("Permission denied (publickey)") { + t.Errorf("ssh does not say that the login was refused:\n%s", p.output()) + } + if e.tg.sent() != n { + t.Error("a message went out about a host that is not in the broker's known_hosts") + } + want := []string{"nodkey: no signature: denied"} + if got := e.agent.stderr.next(agentLines, len(want)); !slices.Equal(got, want) { + t.Errorf("nodkey ssh-agent wrote %q, want %q", redact(strings.Join(got, "\n")), want) + } + e.broker.stderr.settled(t, "nodkey broker", "the host key is not in the broker's known_hosts") + // The entries name the key, and no host: the check of the host failed. + e.audit.next(t, map[string]string{ + "client": "laptop", "kind": "ssh_login", "key": keyName, "hosts": "", "host_key": "", "remote_user": "", + "reason": reason, "message": "", + }, "request", "denied:not_allowed") +} + +// sshGrant: one tap on the message of nodkey ssh-grant lets ssh log in to +// the listed hosts, web-1 and web-2, as deploy, without another tap. A +// host that is not listed, db-1, still asks, and so does another remote +// user, ops, on a listed host (DESIGN.md section 13). +func (e *env) sshGrant(t *testing.T) { + reason := e.reason("e2e: apply the nginx fix") + n := e.tg.sent() + p := e.procs.start(t, "nodkey ssh-grant", filepath.Join(e.laptop.home, "work"), e.laptop.env(), nil, + e.bin, "ssh-grant", "--key", keyName, "--user", "deploy", "--host", "web-1", "--host", "web-2", + "--for", "15m", "--reason", reason) + m := e.tg.waitMessage(t, n+1, p) + code := shortCode(t, m) + checkLines(t, "the message", m.text, + lit("๐Ÿ” SSH grant ยท request "+code), + lit("โœ… machine: laptop"), + lit("โœ… key: "+keyName), + lit("โœ… hosts: web-1, web-2"), + lit("โœ… remote user: deploy"), + `^โœ… grants end: `+clockPattern+` \(15 min\)$`, + lit("โ€” claimed by the agent โ€”"), + lit("reason: "+reason), + expiryPattern) + if want := []string{tapWindow + m.requestID, tapDeny + m.requestID}; !slices.Equal(m.buttons, want) { + t.Errorf("the buttons %q, want %q: Approve and Deny", m.buttons, want) + } + e.tapAndCheck(t, m, tapWindow, "Approved.") + p.mustExit(t, 0) + if lines := p.stderr.all(); len(lines) != 2 || lines[0] != "nodkey: waiting for approval (request "+code+")" || + !regexp.MustCompile(`^nodkey: SSH logins as deploy to 2 hosts allowed until \d\d:\d\d \(exit 0\)$`).MatchString(lines[1]) { + t.Errorf("nodkey ssh-grant wrote %q", redact(strings.Join(lines, "\n"))) + } + if out := p.stdout.text(); out != "" { + t.Errorf("nodkey ssh-grant printed %q on stdout", redact(out)) + } + entries := e.audit.next(t, map[string]string{ + "client": "laptop", "kind": "ssh_grant", "key": keyName, "hosts": "web-1,web-2", "host_key": "", + "remote_user": "deploy", "reason": reason, "command": "", + }, "request", "decision:approve_window") + checkMessageEntry(t, entries[0], m, "[ Approve ] [ Deny ]") + if w := entries[1].field("window_minutes"); w != "15" { + t.Errorf("the decision entry has window_minutes %q, want 15", w) + } + e.checkResult(t, m, `^๐Ÿ‘ approved by you at `+clockPattern+`, until `+clockPattern+`$`) + e.checkMessages(t, n, 1) + + // The listed hosts: no tap. + e.loginByGrant(t, "deploy", "web-1") + e.loginByGrant(t, "deploy", "web-2") + // A host that is not listed still asks, and so does another remote + // user on a listed host. + e.login(t, "deploy", "db-1", tapOnce) + e.login(t, "ops", "web-1", tapOnce) +} + +// gitOverSSH: git clones the bare repository on db-1, pushes a commit to +// it, and clones it again, with ssh through nodkey ssh-agent. The first +// login asks, and the person taps "Approve 15 min"; that grant covers the +// two logins after it (DESIGN.md section 8). +func (e *env) gitOverSSH(t *testing.T) { + work := filepath.Join(e.laptop.home, "git") + mkdirs(t, work) + reason := e.reason("e2e: git over SSH") + n := e.tg.sent() + clone := e.git(t, work, reason, "clone", "deploy@db-1:repo.git", "one") + m := e.tg.waitMessage(t, n+1, clone) + code := shortCode(t, m) + if !strings.Contains(m.text, "โœ… host: db-1 (host key ") || !strings.Contains(m.text, "git-upload-pack") { + t.Errorf("the message of git clone:\n%s", redact(m.text)) + } + e.tapAndCheck(t, m, tapWindow, "Approved.") + clone.mustExit(t, 0) + e.agent.stderr.settled(t, "nodkey ssh-agent", "nodkey: waiting for approval (request "+code+")") + entries := e.audit.next(t, e.loginSubject("deploy", "db-1", reason), "request", "decision:approve_window", "signed") + checkMessageEntry(t, entries[0], m, "[ Approve once ] [ Approve 15 min ] [ Deny ]") + e.checkResult(t, m, `^๐Ÿ‘ approved for 15 min by you at `+clockPattern+`$`) + + // A commit, and a push: the grant covers the login of git push. + one := filepath.Join(work, "one") + writeFile(t, filepath.Join(one, "README"), []byte("nodkey end-to-end test\n")) + e.git(t, one, reason, "add", "README").mustExit(t, 0) + e.git(t, one, reason, "commit", "--quiet", "--message", "e2e: a commit over SSH").mustExit(t, 0) + e.git(t, one, reason, "push", "--quiet", "origin", "HEAD:refs/heads/main").mustExit(t, 0) + subject := e.loginSubject("deploy", "db-1", reason) + subject["message"] = "" + e.audit.next(t, subject, "request", "grant_used:grant", "signed") + + // A new clone has the commit. + e.git(t, work, reason, "clone", "--quiet", "deploy@db-1:repo.git", "two").mustExit(t, 0) + e.audit.next(t, subject, "request", "grant_used:grant", "signed") + if e.tg.sent() != n+1 { + t.Errorf("%d messages went out for git, want 1: the grant covers the push and the second clone", e.tg.sent()-n) + } + head := func(dir string) string { + p := e.git(t, dir, reason, "rev-parse", "HEAD") + p.mustExit(t, 0) + return p.stdout.text() + } + if h1, h2 := head(one), head(filepath.Join(work, "two")); h1 != h2 || len(h1) < 40 { + t.Errorf("the second clone has the commit %q, want %q", h2, h1) + } + if got := string(readFile(t, filepath.Join(work, "two", "README"))); got != "nodkey end-to-end test\n" { + t.Errorf("the second clone has README %q", got) + } +} + +// loginDenied: the person taps Deny, and ssh does not log in. The Deny +// also ends every grant of the key at once, and new requests for the key +// are denied without a message for 60 seconds (DESIGN.md section 8): a +// grant is checked before that pause, so the login to web-1, which the +// grant of ssh-grant allowed before, is denied now. +func (e *env) loginDenied(t *testing.T) { + reason := e.reason("e2e: log in to db-1 as ops") + n, agentLines := e.tg.sent(), e.agent.stderr.count() + p := e.ssh(t, "ops", "db-1", reason) + m := e.tg.waitMessage(t, n+1, p) + code := shortCode(t, m) + e.checkLoginMessage(t, m, "ops", "db-1", reason) + e.tapAndCheck(t, m, tapDeny, "Denied.") + p.mustExit(t, 255) + if !p.stderr.has("Permission denied (publickey)") { + t.Errorf("ssh does not say that the login was refused:\n%s", p.output()) + } + want := []string{"nodkey: waiting for approval (request " + code + ")", "nodkey: no signature (request " + code + "): denied"} + if got := e.agent.stderr.next(agentLines, len(want)); !slices.Equal(got, want) { + t.Errorf("nodkey ssh-agent wrote %q, want %q", redact(strings.Join(got, "\n")), want) + } + entries := e.audit.next(t, e.loginSubject("ops", "db-1", reason), "request", "decision:deny", "denied:approver") + checkMessageEntry(t, entries[0], m, "[ Approve once ] [ Approve 15 min ] [ Deny ]") + e.checkResult(t, m, `^โ›” denied by you at `+clockPattern+`$`) + + // The grant of ssh-grant for web-1 has ended. + reason = e.reason("e2e: log in to web-1 as deploy") + agentLines = e.agent.stderr.count() + p = e.ssh(t, "deploy", "web-1", reason) + p.mustExit(t, 255) + if e.tg.sent() != n+1 { + t.Error("a message went out during the pause after a Deny") + } + want = []string{"nodkey: no signature: denied by a limit of the broker"} + if got := e.agent.stderr.next(agentLines, len(want)); !slices.Equal(got, want) { + t.Errorf("nodkey ssh-agent wrote %q, want %q", redact(strings.Join(got, "\n")), want) + } + subject := e.loginSubject("deploy", "web-1", reason) + subject["message"] = "" + e.audit.next(t, subject, "request", "denied:limit") +} + +// stop stops nodkey ssh-agent and nodkey broker with SIGTERM, as systemd +// does, and checks that each ends as DESIGN.md section 15 says. +func (e *env) stop(t *testing.T) { + e.agent.signal(t, syscall.SIGTERM) + if code := e.agent.wait(t); code != 128+int(syscall.SIGTERM) { + t.Errorf("nodkey ssh-agent ended with %d, want to die of SIGTERM", code) + } + if last := e.agent.stderr.last(); last != "nodkey: stopped by SIGTERM (exit 143)" { + t.Errorf("the last line of nodkey ssh-agent is %q", redact(last)) + } + if fileExists(e.socket) { + t.Error("nodkey ssh-agent left its socket") + } + if !fileExists(e.socket + ".lock") { + t.Error("the lock file of nodkey ssh-agent is gone; it stays after the agent ends") + } + e.broker.signal(t, syscall.SIGTERM) + e.broker.mustExit(t, 0) + e.broker.stderr.inOrder(t, "nodkey broker", "nodkey broker: stopping, as a signal asked", + "nodkey broker: the audit log is closed", lineStopped+`" exit_code=0`) + e.audit.next(t, map[string]string{}, "broker_stopped") + e.audit.unchecked(t) + if entries := e.audit.entries(t); len(entries) == 0 || entries[0].name() != "broker_started" { + t.Errorf("the audit log does not start with broker_started: %q", eventNames(entries)) + } + for _, p := range e.procs.all() { + if p.stderr.has("DATA RACE") { + t.Errorf("the race detector found a data race in %s", p.name) + } + } + for _, c := range e.tg.proxied() { + if c != "CONNECT "+telegramHost+":443" { + t.Errorf("the fake proxy was asked for %q", c) + } + } +} + +// verifyAudit: nodkey audit verify checks the whole audit log with every +// anchor that the broker wrote to its journal (stderr), as DESIGN.md +// section 9 says: journalctl -u nodkey -o cat | nodkey audit verify FILE +// --anchors -. Every entry has its anchor. A copy of the log with one +// changed entry fails. +func (e *env) verifyAudit(t *testing.T) { + data := readFile(t, e.auditPath) + lines := strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") + n := len(lines) + journal := e.broker.stderr.text() + if got := strings.Count("\n"+journal, "\naudit anchor "); got != n { + t.Errorf("the journal has %d anchors, want one for each of the %d entries", got, n) + } + verify := func(name, path string) *proc { + return e.procs.start(t, name, "", append([]string{"PATH=" + clientPath}, quickExit()...), + strings.NewReader(journal), e.bin, "audit", "verify", path, "--anchors", "-") + } + p := verify("nodkey audit verify", e.auditPath) + p.mustExit(t, 0) + last := sha256.Sum256([]byte(lines[n-1])) + checkLines(t, "the output of nodkey audit verify", strings.TrimSuffix(p.stdout.text(), "\n"), + lit("ok: "+strconv.Itoa(n)+" entries (torn lines: 0)"), + lit("anchors: "+strconv.Itoa(n)+" checked, all match; the newest is entry "+strconv.Itoa(n)), + lit("last entry: "+strconv.Itoa(n)+":"+hex.EncodeToString(last[:]))) + + // A copy with one changed field: the client of the first request. + i := slices.IndexFunc(lines, func(l string) bool { return strings.Contains(l, `"client":"`) }) + if i < 0 { + t.Fatal("no entry of the audit log names a client") + } + lines[i] = strings.Replace(lines[i], `"client":"`, `"client":"x`, 1) + changed := filepath.Join(e.dir, "state", "changed.jsonl") + writeFile(t, changed, []byte(strings.Join(lines, "\n")+"\n")) + p = verify("nodkey audit verify (a changed entry)", changed) + p.mustExit(t, 65) + // The changed line fails its anchor, or the line after it fails its + // link to it, whichever the check finds first. + if !p.stderr.has(": line "+strconv.Itoa(i+1)+": ") && !p.stderr.has(": line "+strconv.Itoa(i+2)+": ") { + t.Errorf("nodkey audit verify does not name the changed line %d:\n%s", i+1, p.output()) + } +} + +// claimedTexts returns parts of the claimed text that the cases sent: +// each reason, and words of the command lines, of the working directory +// and of the variable name. +func (e *env) claimedTexts() []string { + return append(slices.Clone(e.reasons), wrapperArg, "--api-token", "echo e2e-ok", "sha256sum", "the-command-ran", + "git-upload-pack", "git-receive-pack", secretVar, "/ci/work") +} + +// firstOf returns the first of parts that text holds, or "". +func firstOf(text string, parts []string) string { + for _, p := range parts { + if strings.Contains(text, p) { + return p + } + } + return "" +} + +// searchClaimed checks where the claimed text of the cases went. DESIGN.md +// section 7 shows claimed text only in the message of a request, and +// section 9 keeps it in the audit log. So no other call to Telegram holds +// it, such as an alert, a notice or the answer to a tap, and no output of +// nodkey: not the broker's journal, not nodkey ssh-agent, and not the +// lines of nodkey run, nodkey ssh-grant, nodkey keygen and nodkey audit +// verify. The config of the test has no "never ask" rule, so no notice +// goes out, and nothing in the test makes an alert: every message must +// have buttons. It runs after the broker has stopped, when its alerts in +// the background have gone out too. +func (e *env) searchClaimed(t *testing.T) { + for _, text := range e.tg.unasked(t) { + t.Errorf("a message without buttons went out, an alert or a notice:\n%s", redact(text)) + } + if len(e.reasons) == 0 { + t.Fatal("no case sent claimed text; the search does not test") + } + claimed := e.claimedTexts() + for i, call := range e.tg.otherCalls() { + if c := firstOf(call, claimed); c != "" { + t.Errorf("call %d to Telegram, which is not the message of a request, holds the claimed text %q: %s", + i, c, redact(call)) + } + } + for _, p := range e.procs.all() { + if !strings.HasPrefix(p.name, "nodkey ") { + continue // ssh and git print what the person ran + } + outputs := map[string]string{"stderr": p.stderr.text()} + if p.name != "nodkey run" { // its stdout is the command's + outputs["stdout"] = p.stdout.text() + } + for which, text := range outputs { + if c := firstOf(text, claimed); c != "" { + t.Errorf("the %s of %s holds the claimed text %q", which, p.name, c) + } + } + } +} + +// searchCanaries is the canary search of DESIGN.md section 17. It looks +// at every log (the broker's journal, nodkey ssh-agent, the sshd of the +// targets), the audit log, every call to the fake Telegram server, and +// every output of a client: nodkey run, nodkey ssh-grant, ssh, git, +// nodkey keygen and nodkey audit verify. +// +// The first canary, the secret value, must be nowhere, in no form, not +// even a part of it. Neither may the made-up bot token or the token of +// the store. The second canary sat in the command line of the wrapper, a +// parent of ssh. That is claimed text: DESIGN.md section 7 shows it only +// in the message and in the audit log (the command and message fields), +// always masked. So it must be nowhere either, and its masked form must +// be in each message and audit entry of a login through the wrapper. +// searchClaimed checks that the logs hold no claimed text at all. +// +// The search finds nothing only if there was something to find: a guard +// checks that the broker handed out the secret, and that SSH logins went +// through the wrapper, so that a -run filter cannot make it pass empty. +func (e *env) searchCanaries(t *testing.T) { + entries := e.audit.entries(t) + if !slices.ContainsFunc(entries, func(en entry) bool { + return en.field("event") == "secret_sent" && en.field("secrets") == secretName + }) { + t.Error("the broker handed out no secret (no secret_sent entry); the search of the first canary does not test") + } + texts := map[string]string{ + "the audit log": string(readFile(t, e.auditPath)), + "the log of the SSH targets": e.targets.logs(t), + } + for i, p := range e.procs.all() { + texts["stdout of "+p.name+" #"+strconv.Itoa(i)] = p.stdout.text() + texts["stderr of "+p.name+" #"+strconv.Itoa(i)] = p.stderr.text() + } + for i, b := range e.tg.bodies() { + texts["call "+strconv.Itoa(i)+" to Telegram"] = b + } + for i, s := range e.tg.shownTexts(t) { + texts["message text "+strconv.Itoa(i)] = s + } + _, botSecret, _ := strings.Cut(e.tg.token, ":") + forms := secretForms(e.canary1) + forms = append(forms, formsOf("the bot token", e.tg.token)...) + forms = append(forms, formsOf("the secret part of the bot token", botSecret)...) + forms = append(forms, formsOf("the token of the store", e.bwsToken)...) + for where, text := range texts { + if names := found(text, forms); len(names) > 0 { + t.Errorf("%s holds %q", where, names) + } + if strings.Contains(asciiLower(text), asciiLower(e.canary2)) { + t.Errorf("%s holds the second canary (from the command line of a parent of ssh)", where) + } + } + + // Where the masked second canary is. Every SSH login of the test went + // through the wrapper. + logins := 0 + for _, s := range e.tg.shownTexts(t) { + if strings.HasPrefix(s, "๐Ÿ” SSH login") { + logins++ + if !strings.Contains(s, wrapperShown) { + t.Errorf("a message of an SSH login does not show the masked command line of the wrapper:\n%s", redact(s)) + } + } + } + if logins == 0 { + t.Error("no message of an SSH login; the search of the second canary does not test") + } + for _, en := range entries { + if en.field("kind") == "ssh_login" && !strings.Contains(en.field("command"), wrapperShown) { + t.Errorf("the %s entry of the request %s does not have the masked command line of the wrapper", + en.name(), en.field("request_id")) + } + } +} diff --git a/test/e2e/env_test.go b/test/e2e/env_test.go new file mode 100644 index 0000000..892b07b --- /dev/null +++ b/test/e2e/env_test.go @@ -0,0 +1,615 @@ +//go:build e2e + +package e2e + +import ( + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "encoding/hex" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + "golang.org/x/crypto/ssh" +) + +// The setup of the end-to-end test (DESIGN.md section 17): +// +// - the nodkey binary, built from this repository; +// - a private directory for the files of the broker and the clients, +// which pass every file check of nodkey; +// - the SSH targets: one container with an sshd for each of web-1, +// web-2, db-1 and web-3, each with its own throwaway host key. The +// broker's known_hosts has web-1, web-2 and db-1, not web-3. The users +// deploy and ops have the public half of the broker's key deploy in +// their authorized_keys; only the broker holds its private half; +// - the fake Telegram server, behind its fake HTTPS proxy; +// - nodkey broker, with a fake store program (a bws) that holds the +// first canary as the value of the secret canary-token; +// - two clients on this machine: laptop, which logs in with ssh and git +// through its nodkey ssh-agent, and ci, which runs nodkey run. Each +// has its own TLS key, made with nodkey keygen, and its own home +// directory with its client config. +// +// The broker limits each client to 6 new messages in any 60 seconds +// (DESIGN.md section 8). The cases of laptop send exactly 6 messages, and +// those of ci 2: a new case that sends a message needs a client of its own, +// or it waits in the broker's queue. + +// The names of the test, from the example names of AGENTS.md. +const ( + keyName = "deploy" // the broker's SSH key + secretName = "canary-token" // the broker's secret, whose value is the first canary + secretVar = "E2E_TOKEN" // the variable of nodkey run for it +) + +// knownTargets are the targets in the broker's known_hosts, and +// unknownTarget is a target whose host key is not there. ssh on this +// machine knows all four (allTargets). +var ( + knownTargets = []string{"web-1", "web-2", "db-1"} + unknownTarget = "web-3" +) + +// allTargets returns every target: the lines of the file "targets" of the +// container, in this order. +func allTargets() []string { + return append(append([]string(nil), knownTargets...), unknownTarget) +} + +// hostKeyTypes are the types of the host keys of the targets, so that the +// broker checks a session-bind of each type that it takes. +var hostKeyTypes = map[string]string{"web-1": "ed25519", "web-2": "ecdsa", "db-1": "rsa", "web-3": "ed25519"} + +// clientPath is the PATH of the clients that look up programs: nodkey run +// finds sh there. +const clientPath = "/usr/local/bin:/usr/bin:/bin" + +// env is the setup of the end-to-end test. Its methods take the test or +// subtest that uses them. +type env struct { + procs procs + + bin string // the nodkey binary + self string // the test binary, for the wrapper of ssh and git + sshPath string + gitPath string + dir string // the private directory + + canary1 string // the value of the secret in the store + canary2 string // a word in the command line of a parent of ssh + bwsToken string // the made-up token of the store + reasons []string // every claimed reason of the cases (reason) + + tg *fakeTelegram + targets *targets + hostKeys map[string]ssh.PublicKey // the host key of each target + + brokerAddr string + brokerPin string + secretID string // the ID of the secret in the store + auditPath string + broker *proc + audit *auditLog + + laptop, ci *client + agent *proc // nodkey ssh-agent of laptop + socket string // its socket + sshConfig string // the ssh config of laptop +} + +// client is a client machine of the broker: its name, and its home +// directory with ~/.config/nodkey/client.toml and its TLS key. +type client struct { + name string + home string + pin string // the pin of its TLS key, from nodkey keygen +} + +// newEnv makes the setup and starts it: the targets, the broker and the +// SSH agent of laptop. Everything ends when the test ends. +func newEnv(t *testing.T) *env { + t.Helper() + img := sshdImage(t) + e := &env{canary1: newCanary1(t), canary2: newCanary2(t), hostKeys: make(map[string]ssh.PublicKey)} + hide(e.canary1, e.canary2) + e.sshPath, e.gitPath = requireOpenSSH(t), requireProgram(t, "git") + var err error + if e.self, err = os.Executable(); err != nil { + t.Fatal(err) + } + _, e.bin = runDirectory(t) + e.dir = subdirectory(t, "e2e-") + for _, sub := range []string{"creds", "etc", "bin", "state", "ssh", "container", "ca"} { + mkdirs(t, filepath.Join(e.dir, sub)) + } + e.auditPath = filepath.Join(e.dir, "state", "audit.jsonl") + e.audit = &auditLog{path: e.auditPath} + + // The keys: the broker's SSH key, and a host key for each target. + deployKey, deployPub := newSSHKey(t, "ed25519") + writeFile(t, filepath.Join(e.dir, "creds", "ssh-"+keyName), deployKey) + files := filepath.Join(e.dir, "container") // the files of entrypoint.sh + writeFile(t, filepath.Join(files, "authorized_keys"), ssh.MarshalAuthorizedKey(deployPub)) + var lines strings.Builder + for i, name := range allTargets() { + priv, pub := newSSHKey(t, hostKeyTypes[name]) + e.hostKeys[name] = pub + writeFile(t, filepath.Join(files, name+".key"), priv) + fmt.Fprintf(&lines, "%s %d\n", name, 2201+i) + } + writeFile(t, filepath.Join(files, "targets"), []byte(lines.String())) + e.targets = startTargets(t, img, files, allTargets()) + + e.tg = newFakeTelegram(t, filepath.Join(e.dir, "ca")) + e.bwsToken = "made-up-bws-token-" + rand.Text() + _, botSecret, _ := strings.Cut(e.tg.token, ":") + hide(e.tg.token, botSecret, e.bwsToken) + e.writeBrokerFiles(t) + e.laptop = e.newClient(t, "laptop") + e.ci = e.newClient(t, "ci") + e.startBroker(t) + e.audit.next(t, map[string]string{"version": "dev"}, "broker_started") + e.startAgent(t) + e.writeSSHConfig(t) + return e +} + +// newCanary1 returns the first canary: the secret value in the store. It +// is random, made at run time, so the source holds no value. +func newCanary1(t *testing.T) string { + t.Helper() + return canary1Prefix + rand.Text() +} + +// newCanary2 returns the second canary: 32 random upper-case letters and +// digits, one after the other, so that it looks random as a whole (DESIGN.md +// section 7, "long random-looking words"). +func newCanary2(t *testing.T) string { + t.Helper() + const letters, digits = "ABCDEFGHJKLMNPQRSTUVWXYZ", "23456789" + var b strings.Builder + for i := range 32 { + set := letters + if i%2 == 1 { + set = digits + } + n, err := rand.Int(rand.Reader, big.NewInt(int64(len(set)))) + if err != nil { + t.Fatal(err) + } + b.WriteByte(set[n.Int64()]) + } + return b.String() +} + +// requireOpenSSH returns the path of the OpenSSH client, and fails the +// test when it is missing or older than 8.9, which sends no session-bind. +func requireOpenSSH(t *testing.T) string { + t.Helper() + path := requireProgram(t, "ssh") + out, err := exec.CommandContext(t.Context(), path, "-V").CombinedOutput() //nolint:gosec // G204: ssh from PATH. + if err != nil { + t.Fatalf("ssh -V: %v", err) + } + m := regexp.MustCompile(`OpenSSH_(\d+)\.(\d+)`).FindSubmatch(out) + if m == nil { + t.Fatalf("ssh is not OpenSSH: %q", out) + } + major, _ := strconv.Atoi(string(m[1])) + minor, _ := strconv.Atoi(string(m[2])) + if major < 8 || major == 8 && minor < 9 { + t.Fatalf("OpenSSH %d.%d sends no session-bind; the test needs 8.9 or newer", major, minor) + } + return path +} + +// requireProgram returns the path of name in PATH, and fails the test when +// it is missing. +func requireProgram(t *testing.T, name string) string { + t.Helper() + path, err := exec.LookPath(name) + if err != nil { + t.Fatalf("the end-to-end test needs %s: %v", name, err) + } + return path +} + +// mkdirs makes dir and its missing parents with mode 0700. +func mkdirs(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } +} + +// writeFile writes data to path with mode 0600. +func writeFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} + +// writeProgram writes a program to path with mode 0700. +func writeProgram(t *testing.T, path, text string) { + t.Helper() + writeFile(t, path, []byte(text)) + if err := os.Chmod(path, 0o700); err != nil { //nolint:gosec // G302: a program of the test, private to its owner. + t.Fatal(err) + } +} + +// newSSHKey returns a new SSH private key of kind (ed25519, ecdsa or +// rsa), in the OpenSSH format that sshd and the broker read, and its +// public key. +func newSSHKey(t *testing.T, kind string) ([]byte, ssh.PublicKey) { + t.Helper() + var priv any + var err error + switch kind { + case "ed25519": + _, priv, err = ed25519.GenerateKey(rand.Reader) + case "ecdsa": + priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + case "rsa": + priv, err = rsa.GenerateKey(rand.Reader, 3072) + default: + t.Fatalf("no key type %q", kind) + } + if err != nil { + t.Fatal(err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatal(err) + } + block, err := ssh.MarshalPrivateKey(priv, "") + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(block), signer.PublicKey() +} + +// nodkey runs nodkey with args, and returns its stdout. It fails the test +// unless nodkey exits with 0. +func (e *env) nodkey(t *testing.T, args ...string) string { + t.Helper() + p := e.procs.start(t, "nodkey "+args[0], "", append([]string{"PATH=" + clientPath}, quickExit()...), nil, + append([]string{e.bin}, args...)...) + p.mustExit(t, 0) + return strings.TrimSpace(p.stdout.text()) +} + +// writeBrokerFiles writes the files of the broker, except its config, +// which needs the listen address (writeBrokerConfig). +func (e *env) writeBrokerFiles(t *testing.T) { + t.Helper() + creds := filepath.Join(e.dir, "creds") + e.brokerPin = e.nodkey(t, "keygen", "--out", filepath.Join(creds, "broker-tls-key")) + writeFile(t, filepath.Join(creds, "telegram-token"), []byte(e.tg.token+"\n")) + writeFile(t, filepath.Join(creds, "bws-token"), []byte(e.bwsToken+"\n")) + + // A fake bws: it prints the secret with the ID $3, as + // "bws secret get ID --output json --color no" does, and fails for + // any other ID. + id := newUUID(t) + writeProgram(t, filepath.Join(e.dir, "bin", "bws"), fmt.Sprintf("#!/bin/sh\n"+ + "# A fake bws of the end-to-end test of nodkey.\n"+ + "case \"$1 $2 $3\" in\n"+ + "'secret get %s') printf '%%s' '{\"id\":\"%s\",\"value\":\"%s\"}' ;;\n"+ + "*) exit 1 ;;\n"+ + "esac\n", id, id, e.canary1)) + e.secretID = id + + var known strings.Builder + for _, name := range knownTargets { + known.WriteString(name + " " + string(ssh.MarshalAuthorizedKey(e.hostKeys[name]))) + } + writeFile(t, filepath.Join(e.dir, "etc", "known_hosts"), []byte(known.String())) +} + +// newUUID returns a random UUID (version 4). +func newUUID(t *testing.T) string { + t.Helper() + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + t.Fatal(err) + } + b[6] = b[6]&0x0f | 0x40 + b[8] = b[8]&0x3f | 0x80 + h := hex.EncodeToString(b) + return h[:8] + "-" + h[8:12] + "-" + h[12:16] + "-" + h[16:20] + "-" + h[20:] +} + +// writeBrokerConfig writes the config of the broker, with its listen +// address. +func (e *env) writeBrokerConfig(t *testing.T) { + t.Helper() + cfg := fmt.Sprintf(`# The broker of the end-to-end test of nodkey. +listen = %q +tls_key = "${CREDENTIALS_DIRECTORY}/broker-tls-key" +audit_log = %q +known_hosts = %q + +[telegram] +token_file = "${CREDENTIALS_DIRECTORY}/telegram-token" +user_id = %d +timeout = "60s" +ssh_timeout = "60s" + +[client.laptop] +pin = %q + +[client.ci] +pin = %q + +[store.main] +type = "bws" +program = %q +token_file = "${CREDENTIALS_DIRECTORY}/bws-token" + +[secret.%s] +store = "main" +id = %q +clients = ["ci"] +ask = "always" + +[key.%s] +file = "${CREDENTIALS_DIRECTORY}/ssh-%s" +clients = ["laptop"] +ask = "window" +window = "15m" +`, e.brokerAddr, e.auditPath, filepath.Join(e.dir, "etc", "known_hosts"), approverID, e.laptop.pin, e.ci.pin, + filepath.Join(e.dir, "bin", "bws"), secretName, e.secretID, keyName, keyName) + writeFile(t, filepath.Join(e.dir, "etc", "broker.toml"), []byte(cfg)) +} + +// newClient makes the home directory of a client, with its TLS key from +// nodkey keygen. Its config needs the broker's address +// (writeClientConfig). +func (e *env) newClient(t *testing.T, name string) *client { + t.Helper() + c := &client{name: name, home: filepath.Join(e.dir, name)} + for _, sub := range []string{".config/nodkey", "run", "work"} { + mkdirs(t, filepath.Join(c.home, sub)) + } + c.pin = e.nodkey(t, "keygen", "--out", filepath.Join(c.home, ".config", "nodkey", "client.key")) + return c +} + +// writeClientConfig writes ~/.config/nodkey/client.toml of c. +func (e *env) writeClientConfig(t *testing.T, c *client) { + t.Helper() + cfg := fmt.Sprintf("broker = %q\nbroker_pin = %q\ntls_key = \"~/.config/nodkey/client.key\"\n", e.brokerAddr, e.brokerPin) + writeFile(t, filepath.Join(c.home, ".config", "nodkey", "client.toml"), []byte(cfg)) +} + +// env returns the environment of a nodkey client of c: its home, and +// clientPath. Nothing comes from the environment of the test, such as +// SSH_AUTH_SOCK. +func (c *client) env(more ...string) []string { + return append(append([]string{"HOME=" + c.home, "PATH=" + clientPath}, quickExit()...), more...) +} + +// quickExit returns the environment that makes a program built with the +// race detector exit at once: by default it waits a second at its exit +// (GORACE option atexit_sleep_ms), and the test starts many such +// programs, nodkey and the wrapper. A GORACE of the test's environment +// comes first, so its other options stay. +func quickExit() []string { + if !raceEnabled { + return goraceEnv() + } + return []string{"GORACE=" + strings.TrimSpace(os.Getenv("GORACE")+" atexit_sleep_ms=0")} +} + +// goraceEnv returns GORACE of the test's environment, if it is set, for +// nodkey broker and nodkey ssh-agent. +func goraceEnv() []string { + if v, ok := os.LookupEnv("GORACE"); ok { + return []string{"GORACE=" + v} + } + return nil +} + +// The lines of nodkey broker that the test looks for. +const ( + lineListening = "nodkey broker: listening; the broker is ready" + lineProxy = "telegram: calls to Telegram go through a proxy from the environment" + lineStopped = "nodkey broker: stopped" +) + +// startBroker starts nodkey broker, and waits until it is ready. The port +// of its listen address was free a moment before, but another program can +// take it in between; then the broker stops, and the test starts it again +// with another port. +func (e *env) startBroker(t *testing.T) { + t.Helper() + brokerEnv := append([]string{ + "CREDENTIALS_DIRECTORY=" + filepath.Join(e.dir, "creds"), + "HTTPS_PROXY=http://" + e.tg.proxyAddr, + "SSL_CERT_FILE=" + e.tg.caFile, + "SSL_CERT_DIR=" + e.tg.caDir, + }, goraceEnv()...) + for try := 1; ; try++ { + e.brokerAddr = freeAddress(t) + e.writeBrokerConfig(t) + e.writeClientConfig(t, e.laptop) + e.writeClientConfig(t, e.ci) + p := e.procs.start(t, "nodkey broker", "", brokerEnv, nil, + e.bin, "broker", "--config", filepath.Join(e.dir, "etc", "broker.toml")) + if p.stderr.waitOrEnd(t, "nodkey broker", lineListening) { + e.broker = p + break + } + p.wait(t) + if try == 3 || !p.stderr.has("address already in use") { + t.Fatalf("nodkey broker did not start; its log:\n%s", redact(p.stderr.text())) + } + } + if !e.broker.stderr.has(lineProxy) || !e.broker.stderr.has("proxy_host="+e.tg.proxyAddr) { + t.Errorf("the broker does not log that it calls Telegram through the fake proxy; its log:\n%s", + redact(e.broker.stderr.text())) + } +} + +// freeAddress returns an address on the loopback interface with a port +// that is free now, outside the range of ports that the kernel and Docker +// give out by themselves (net.ipv4.ip_local_port_range), as in the tests +// of nodkey broker. +func freeAddress(t *testing.T) string { + t.Helper() + low, high := int64(10000), int64(32768) + if data, err := os.ReadFile("/proc/sys/net/ipv4/ip_local_port_range"); err == nil { + if f := strings.Fields(string(data)); len(f) == 2 { + lo, err1 := strconv.ParseInt(f[0], 10, 64) + hi, err2 := strconv.ParseInt(f[1], 10, 64) + switch { + case err1 != nil || err2 != nil: + case lo-10000 >= 1000: + low, high = 10000, lo + case 65535-hi >= 1000: + low, high = hi+1, 65536 + default: + low, high = 0, 0 + } + } + } + for range 200 { + addr := "127.0.0.1:0" + if high > low { + n, err := rand.Int(rand.Reader, big.NewInt(high-low)) + if err != nil { + t.Fatal(err) + } + addr = "127.0.0.1:" + strconv.FormatInt(low+n.Int64(), 10) + } + ln, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", addr) + if err != nil { + continue // in use + } + addr = ln.Addr().String() + if err := ln.Close(); err != nil { + t.Fatal(err) + } + return addr + } + t.Fatal("found no free port for the broker") + return "" +} + +// startAgent starts nodkey ssh-agent of laptop on its default socket, +// $XDG_RUNTIME_DIR/nodkey/agent.sock, and waits until it listens. +func (e *env) startAgent(t *testing.T) { + t.Helper() + run := filepath.Join(e.laptop.home, "run") + e.socket = filepath.Join(run, "nodkey", "agent.sock") + agentEnv := append([]string{"HOME=" + e.laptop.home, "XDG_RUNTIME_DIR=" + run}, goraceEnv()...) + e.agent = e.procs.start(t, "nodkey ssh-agent", "", agentEnv, nil, e.bin, "ssh-agent") + e.agent.stderr.wait(t, "nodkey ssh-agent", "nodkey: the SSH agent listens on "+e.socket) +} + +// writeSSHConfig writes the ssh config of laptop, and its known_hosts: ssh +// knows each target by its name (HostKeyAlias), also web-3, which the +// broker does not know. ssh uses no key file, only the agent. The config +// keeps ssh away from the files of the user who runs the test: -F skips +// ~/.ssh/config, and the known_hosts files and the key files are named. +func (e *env) writeSSHConfig(t *testing.T) { + t.Helper() + dir := filepath.Join(e.dir, "ssh") + var known, cfg strings.Builder + for _, name := range allTargets() { + known.WriteString(name + " " + string(ssh.MarshalAuthorizedKey(e.hostKeys[name]))) + fmt.Fprintf(&cfg, "Host %s\n\tHostName 127.0.0.1\n\tPort %d\n\tHostKeyAlias %s\n", name, e.targets.ports[name], name) + } + writeFile(t, filepath.Join(dir, "known_hosts"), []byte(known.String())) + fmt.Fprintf(&cfg, "Host *\n"+ + "\tIdentityFile %s\n"+ // no such file: no key file, only the agent + "\tUserKnownHostsFile %s\n"+ + "\tGlobalKnownHostsFile /dev/null\n"+ + "\tStrictHostKeyChecking yes\n"+ + "\tCheckHostIP no\n"+ + "\tUpdateHostKeys no\n"+ + "\tBatchMode yes\n"+ + "\tPreferredAuthentications publickey\n"+ + "\tControlMaster no\n"+ + "\tControlPath none\n"+ + "\tForwardAgent no\n"+ + "\tConnectTimeout 20\n"+ + "\tLogLevel ERROR\n", + filepath.Join(dir, "no-such-key"), filepath.Join(dir, "known_hosts")) + e.sshConfig = filepath.Join(dir, "config") + writeFile(t, e.sshConfig, []byte(cfg.String())) +} + +// reason returns s, a claimed reason of a case, and keeps it: claimed text +// may be shown only in a message and the audit log (searchClaimed). +func (e *env) reason(s string) string { + e.reasons = append(e.reasons, s) + return s +} + +// sshEnv returns the environment of ssh and git on laptop: SSH_AUTH_SOCK +// is the socket of its nodkey ssh-agent, and NODKEY_REASON the reason. +func (e *env) sshEnv(reason string, more ...string) []string { + return e.laptop.env(append([]string{"SSH_AUTH_SOCK=" + e.socket, "NODKEY_REASON=" + reason}, more...)...) +} + +// wrapped returns argv run by the wrapper, whose own arguments hold the +// second canary twice: as the value of an option with a secret name, and +// as a word of its own. nodkey ssh-agent sends the command line of the +// wrapper to the broker as claimed text, when the wrapper is one of the +// first 3 parents of ssh. +func (e *env) wrapped(argv ...string) []string { + return append([]string{e.self, wrapperArg, "--api-token=" + e.canary2, e.canary2, "--"}, argv...) +} + +// ssh starts ssh as user to host, through the wrapper, with the command +// "echo e2e-ok". +func (e *env) ssh(t *testing.T, user, host, reason string) *proc { + t.Helper() + return e.procs.start(t, "ssh "+user+"@"+host, filepath.Join(e.laptop.home, "work"), e.sshEnv(reason), nil, + e.wrapped(e.sshPath, "-F", e.sshConfig, user+"@"+host, "echo", "e2e-ok")...) +} + +// git starts git with args in dir, through the wrapper. git runs ssh with +// the config of laptop. +func (e *env) git(t *testing.T, dir, reason string, args ...string) *proc { + t.Helper() + gitEnv := e.sshEnv(reason, + "GIT_SSH_COMMAND="+shellQuote(e.sshPath)+" -F "+shellQuote(e.sshConfig), + "GIT_CONFIG_NOSYSTEM=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_AUTHOR_NAME=dev", "GIT_AUTHOR_EMAIL=dev@example.com", + "GIT_COMMITTER_NAME=dev", "GIT_COMMITTER_EMAIL=dev@example.com", + "LC_ALL=C", + ) + return e.procs.start(t, "git "+args[0], dir, gitEnv, nil, e.wrapped(append([]string{e.gitPath}, args...)...)...) +} + +// shellQuote quotes s for sh. +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +// readFile reads the file at path. +func readFile(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) //nolint:gosec // G304: a file of the test. + if err != nil { + t.Fatal(err) + } + return data +} diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go new file mode 100644 index 0000000..b644011 --- /dev/null +++ b/test/e2e/main_test.go @@ -0,0 +1,166 @@ +//go:build e2e + +package e2e + +import ( + "context" + "debug/buildinfo" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "syscall" + "testing" + "time" +) + +// wrapperArg is the first argument of the test binary when it runs as the +// wrapper of a client program (wrapper). +const wrapperArg = "nodkey-e2e-wrapper" + +func TestMain(m *testing.M) { + if len(os.Args) > 1 && os.Args[1] == wrapperArg { + os.Exit(wrapper(os.Args[2:])) + } + stderr := os.Stderr //nolint:forbidigo // TestMain reports on stderr, as the testing package does. + // The lock tells later runs that this run has not ended (removeStale). + // Without it, they wait an hour before they remove what it left. + lock, err := takeRunLock() + if err != nil { + _, _ = fmt.Fprintln(stderr, "e2e: cannot make the lock file of this run:", err) + } + code := m.Run() + // The cleanups of the tests removed their containers and networks. + // Remove the image and the private directory, and check that nothing + // of this run is left. + if err := removeImage(); err != nil { + _, _ = fmt.Fprintln(stderr, "e2e:", err) + code = 1 + } + if image.built { + left, err := leftovers() + switch { + case err != nil: + _, _ = fmt.Fprintln(stderr, "e2e: cannot list what the test left in Docker:", err) + code = 1 + case len(left) > 0: + _, _ = fmt.Fprintf(stderr, "e2e: the test left these in Docker: %q\n", left) + code = 1 + } + } + if run.dir != "" { + if err := os.RemoveAll(run.dir); err != nil { + _, _ = fmt.Fprintln(stderr, "e2e: cannot remove the private directory of the test:", err) + code = 1 + } + } + if lock != nil { + releaseRunLock(lock) + } + os.Exit(code) +} + +// wrapper runs the command after "--" as its child, with the same stdin, +// stdout and stderr, and returns its exit code. The arguments before "--" +// do nothing: they are there to be seen in /proc/PID/cmdline of a parent +// of ssh, where nodkey ssh-agent reads them as claimed text. The test puts +// the second canary there. +func wrapper(args []string) int { + i := slices.Index(args, "--") + if i < 0 || i == len(args)-1 { + return 2 + } + cmd := exec.CommandContext(context.Background(), args[i+1], args[i+2:]...) //nolint:gosec // G204: the wrapper runs the command of the test. + //nolint:forbidigo // the wrapper passes its own stdin, stdout and stderr to the command. + cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr + // When the wrapper dies, so does the command: ssh or git. + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + err := cmd.Run() + var exit *exec.ExitError + switch { + case err == nil: + return 0 + case errors.As(err, &exit) && exit.ExitCode() >= 0: + return exit.ExitCode() + } + return 127 +} + +// goTool returns the go command of the Go version that built this test, +// so that nodkey is built with the same version: such as when the test +// runs with another Go than the one first in PATH. +func goTool() string { + //nolint:staticcheck // SA1019: the root of the toolchain that built this test binary is what is wanted here. + if root := runtime.GOROOT(); root != "" { + if p := filepath.Join(root, "bin", "go"); fileExists(p) { + return p + } + } + return "go" +} + +// fileExists reports whether path exists. +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// goBuildLimit bounds a build of the go command, also without a cache. +const goBuildLimit = 10 * time.Minute + +// goBuild runs the go command with args in the module root, with env added +// to the environment of the test. +func goBuild(env []string, args ...string) error { + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + return err + } + if _, err := os.Stat(filepath.Join(root, "go.mod")); err != nil { + return fmt.Errorf("no go.mod two levels up from the test: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), goBuildLimit) + defer cancel() + cmd := exec.CommandContext(ctx, goTool(), args...) //nolint:gosec // G204: the test builds nodkey with its own arguments. + cmd.Dir = root + cmd.Env = append(os.Environ(), env...) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("go %q: %w\n%s", args, err, out) + } + return nil +} + +// buildNodkey builds the nodkey binary from this repository as bin. When +// the test runs with the race detector, so does nodkey: the broker and the +// clients then report data races too. +func buildNodkey(bin string) error { + args := []string{"build", "-o", bin} + if raceEnabled { + args = append(args, "-race") + } + if err := goBuild(nil, append(args, "./cmd/nodkey")...); err != nil { + return err + } + info, err := buildinfo.ReadFile(bin) + if err != nil { + return err + } + if info.GoVersion != runtime.Version() { + return fmt.Errorf("nodkey was built with %s, the test with %s", info.GoVersion, runtime.Version()) + } + return nil +} + +// buildClientTest builds the test binary of internal/client into dir, for +// the root-only test in a container, and returns its path. It is static +// (no cgo), so it runs in the container whatever C library the host has. +func buildClientTest(t *testing.T, dir string) string { + t.Helper() + bin := filepath.Join(dir, "client.test") + if err := goBuild([]string{"CGO_ENABLED=0"}, "test", "-c", "-o", bin, "./internal/client"); err != nil { + t.Fatal(err) + } + return bin +} diff --git a/test/e2e/norace_test.go b/test/e2e/norace_test.go new file mode 100644 index 0000000..9c1ed14 --- /dev/null +++ b/test/e2e/norace_test.go @@ -0,0 +1,6 @@ +//go:build e2e && !race + +package e2e + +// raceEnabled reports whether the test runs with the race detector. +const raceEnabled = false diff --git a/test/e2e/process_test.go b/test/e2e/process_test.go new file mode 100644 index 0000000..ca79e98 --- /dev/null +++ b/test/e2e/process_test.go @@ -0,0 +1,323 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "slices" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// waitLimit bounds every wait of the test for one step: a line of a log, a +// message at the fake Telegram, the end of a command. +const waitLimit = 60 * time.Second + +// settleLimit bounds a wait for what a program wrote before it answered +// its client: a line of its log, or an audit entry. When the client has +// its answer, it is there, or on its way through a pipe. +const settleLimit = 10 * time.Second + +// proc is a program that the test started, such as nodkey broker, nodkey +// ssh-agent, ssh or git, with its output. +type proc struct { + name string + cmd *exec.Cmd + stdout *lineLog + stderr *lineLog + done chan struct{} // closed when the program ended and its output is read + err error // the error of Wait; set before done is closed +} + +// procs are the programs of one test, for the canary search: every log and +// every output of a client. +type procs struct { + mu sync.Mutex + list []*proc +} + +// start starts argv in the directory dir ("" for the directory of the +// test), with exactly the environment env, and its stdin from stdin (nil +// for none). The program gets a process group of its own: when the test +// ends and it still runs, the test kills it with every process that it +// started, such as ssh under the wrapper. When the test process dies +// without its cleanups, such as after the -timeout of go test or Ctrl-C, +// the kernel kills the program (Pdeathsig). +func (ps *procs) start(t *testing.T, name, dir string, env []string, stdin io.Reader, argv ...string) *proc { + t.Helper() + // Not exec.CommandContext with the test's context: that context ends + // before the cleanups run, and the test stops its long-running + // programs itself (nodkey broker and nodkey ssh-agent with SIGTERM). + cmd := exec.CommandContext(context.Background(), argv[0], argv[1:]...) //nolint:gosec // G204: the test runs its own programs. + p := &proc{name: name, cmd: cmd, stdout: newLineLog(), stderr: newLineLog(), done: make(chan struct{})} + cmd.Dir, cmd.Env, cmd.Stdin = dir, env, stdin + cmd.Stdout, cmd.Stderr = p.stdout, p.stderr + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL} + // A process that the program leaves behind may keep its output open. + cmd.WaitDelay = 5 * time.Second + if err := cmd.Start(); err != nil { + t.Fatalf("%s: %v", name, err) + } + go func() { + p.err = cmd.Wait() + p.stdout.close() + p.stderr.close() + close(p.done) + }() + ps.mu.Lock() + ps.list = append(ps.list, p) + ps.mu.Unlock() + t.Cleanup(func() { + select { + case <-p.done: + default: + p.kill() + <-p.done + } + }) + return p +} + +// kill kills the process group of the program. +func (p *proc) kill() { + _ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL) +} + +// all returns every program so far. +func (ps *procs) all() []*proc { + ps.mu.Lock() + defer ps.mu.Unlock() + return slices.Clone(ps.list) +} + +// wait waits until the program ends, and returns its exit code, or 128 +// and the number of the signal that ended it. +func (p *proc) wait(t *testing.T) int { + t.Helper() + select { + case <-p.done: + case <-time.After(waitLimit): + p.kill() + <-p.done + t.Fatalf("%s did not end within %s; its output:\n%s", p.name, waitLimit, p.output()) + } + var exit *exec.ExitError + switch { + case p.err == nil: + return 0 + case errors.As(p.err, &exit): + if ws, ok := exit.Sys().(syscall.WaitStatus); ok && ws.Signaled() { + return 128 + int(ws.Signal()) + } + return exit.ExitCode() + } + t.Fatalf("%s: %v", p.name, p.err) + return -1 +} + +// mustExit waits until the program ends, and fails unless it exited with +// code. +func (p *proc) mustExit(t *testing.T, code int) { + t.Helper() + if got := p.wait(t); got != code { + t.Fatalf("%s exited with %d, want %d; its output:\n%s", p.name, got, code, p.output()) + } +} + +// signal sends sig to the program. +func (p *proc) signal(t *testing.T, sig os.Signal) { + t.Helper() + if err := p.cmd.Process.Signal(sig); err != nil { + t.Fatalf("%s: %v", p.name, err) + } +} + +// output returns what the program wrote so far, for a failure message. +func (p *proc) output() string { + return redact(fmt.Sprintf("--- stdout of %s:\n%s--- stderr of %s:\n%s", p.name, p.stdout.text(), p.name, + p.stderr.text())) +} + +// lineLog holds the lines of an output as they come. It is the stdout +// or the stderr of a program. +type lineLog struct { + mu sync.Mutex + lines []string + partial []byte // the start of the next line + done bool // the output is closed + changed chan struct{} // closed and made new on each change +} + +func newLineLog() *lineLog { return &lineLog{changed: make(chan struct{})} } + +// Write adds the lines of p. The end of p without a newline waits for the +// rest of its line. +func (l *lineLog) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + l.partial = append(l.partial, p...) + for { + i := bytes.IndexByte(l.partial, '\n') + if i < 0 { + break + } + l.lines = append(l.lines, string(l.partial[:i])) + l.partial = l.partial[i+1:] + } + l.notifyLocked() + return len(p), nil +} + +// close ends the output. A last line without a newline counts as a line. +func (l *lineLog) close() { + l.mu.Lock() + defer l.mu.Unlock() + if len(l.partial) > 0 { + l.lines = append(l.lines, string(l.partial)) + l.partial = nil + } + l.done = true + l.notifyLocked() +} + +func (l *lineLog) notifyLocked() { + close(l.changed) + l.changed = make(chan struct{}) +} + +// all returns the lines so far. +func (l *lineLog) all() []string { + l.mu.Lock() + defer l.mu.Unlock() + return slices.Clone(l.lines) +} + +// text returns the lines so far, each with a newline. +func (l *lineLog) text() string { + var b strings.Builder + for _, line := range l.all() { + b.WriteString(line + "\n") + } + return b.String() +} + +// has reports whether a line holds s. +func (l *lineLog) has(s string) bool { + return slices.ContainsFunc(l.all(), func(line string) bool { return strings.Contains(line, s) }) +} + +// last returns the last line, or "". +func (l *lineLog) last() string { + lines := l.all() + if len(lines) == 0 { + return "" + } + return lines[len(lines)-1] +} + +// next waits until there are count lines after the first n, and returns +// the lines after the first n. A program writes a line before it answers +// its client, but the line reaches the test through a pipe, so the test +// waits for it. After settleLimit, it returns the lines that came. +func (l *lineLog) next(n, count int) []string { + deadline := time.NewTimer(settleLimit) + defer deadline.Stop() + for { + l.mu.Lock() + lines := slices.Clone(l.lines[min(n, len(l.lines)):]) + done, changed := l.done, l.changed + l.mu.Unlock() + if len(lines) >= count || done { + return lines + } + select { + case <-changed: + case <-deadline.C: + return lines + } + } +} + +// count returns the number of lines so far. +func (l *lineLog) count() int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.lines) +} + +// waitOrEnd waits until a line holds s, and reports true, or until the +// output ends without such a line, and reports false. It waits at most +// waitLimit. +func (l *lineLog) waitOrEnd(t *testing.T, what, s string) bool { + t.Helper() + return l.waitOrEndWithin(t, what, s, waitLimit) +} + +// waitOrEndWithin is waitOrEnd, with the limit limit. +func (l *lineLog) waitOrEndWithin(t *testing.T, what, s string, limit time.Duration) bool { + t.Helper() + deadline := time.NewTimer(limit) + defer deadline.Stop() + for { + l.mu.Lock() + found := slices.ContainsFunc(l.lines, func(line string) bool { return strings.Contains(line, s) }) + done, changed := l.done, l.changed + l.mu.Unlock() + switch { + case found: + return true + case done: + return false + } + select { + case <-changed: + case <-deadline.C: + t.Fatalf("no line of %s holds %q after %s; it wrote:\n%s", what, s, limit, redact(l.text())) + } + } +} + +// wait waits until a line holds s. It fails when the output ends without +// such a line. +func (l *lineLog) wait(t *testing.T, what, s string) { + t.Helper() + if !l.waitOrEnd(t, what, s) { + t.Fatalf("%s ended, and no line holds %q; it wrote:\n%s", what, s, redact(l.text())) + } +} + +// settled waits until a line holds s, a line that the program wrote before +// it answered its client. It fails when none comes within settleLimit. +func (l *lineLog) settled(t *testing.T, what, s string) { + t.Helper() + if !l.waitOrEndWithin(t, what, s, settleLimit) { + t.Fatalf("%s ended, and no line holds %q; it wrote:\n%s", what, s, redact(l.text())) + } +} + +// inOrder fails unless lines hold each of parts, in this order. +func (l *lineLog) inOrder(t *testing.T, what string, parts ...string) { + t.Helper() + lines := l.all() + i := 0 + for _, p := range parts { + for i < len(lines) && !strings.Contains(lines[i], p) { + i++ + } + if i == len(lines) { + t.Errorf("no line of %s holds %q after the lines of %q; it wrote:\n%s", what, p, parts, + redact(strings.Join(lines, "\n"))) + return + } + i++ + } +} diff --git a/test/e2e/race_test.go b/test/e2e/race_test.go new file mode 100644 index 0000000..0546058 --- /dev/null +++ b/test/e2e/race_test.go @@ -0,0 +1,6 @@ +//go:build e2e && race + +package e2e + +// raceEnabled reports whether the test runs with the race detector. +const raceEnabled = true diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go new file mode 100644 index 0000000..0f1594a --- /dev/null +++ b/test/e2e/run_test.go @@ -0,0 +1,389 @@ +//go:build e2e + +package e2e + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// One run of the test is one test process. Its Docker objects and its +// files carry its run ID, and while it runs, it holds a lock on its lock +// file. A run that is killed, such as by the -timeout of go test or by +// Ctrl-C, cannot clean up. So each run first removes what earlier runs left +// (removeStale): only objects and files with a run ID, and only of runs +// that have ended. + +// runID names the Docker objects and the files of this run: its start time +// in Unix seconds and 4 random bytes, such as "1760511234-a1b2c3d4". The +// time tells a later run how old they are. +var runID = func() string { + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + panic(err) + } + return strconv.FormatInt(time.Now().Unix(), 10) + "-" + hex.EncodeToString(b) +}() + +// runIDPattern matches a run ID, and takes its start time. +var runIDPattern = regexp.MustCompile(`^([0-9]{10})-[0-9a-f]{8}$`) + +// The prefix of the names of the Docker objects, the private directories +// and the lock files of the runs, before the run ID. +const runPrefix = "nodkey-e2e-" + +// How long until the objects of an ended run count as left behind. +const ( + // staleDead is for a run whose lock file is there and not locked. A + // run locks its file right after it makes it, at its start. + staleDead = 10 * time.Second + // staleUnknown is for a run whose lock file this run cannot see: one + // with another TMPDIR, another host that uses the same Docker, or one + // from before a restart. A run takes seconds, and go test ends it + // after 10 minutes by default. + staleUnknown = time.Hour +) + +// lockPath returns the lock file of the run id. It is in the system's +// temporary directory, so that the runs of every user see it. +func lockPath(id string) string { + return filepath.Join(os.TempDir(), runPrefix+id+".lock") +} + +// takeRunLock makes the lock file of this run and locks it. The lock holds +// until the process ends, also when it is killed. The file is empty, and +// others may read it, so that their runs can test the lock too. +func takeRunLock() (*os.File, error) { + path := lockPath(runID) + //nolint:gosec // G302, G304: an empty lock file, readable by others on purpose. + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o644) + if err != nil { + return nil, err + } + // The umask may have taken the read bit of others: set it again. + if err := f.Chmod(0o644); err != nil { + _ = f.Close() + _ = os.Remove(path) + return nil, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + _ = os.Remove(path) + return nil, err + } + return f, nil +} + +// releaseRunLock removes the lock file of this run, and unlocks it. +func releaseRunLock(f *os.File) { + _ = os.Remove(f.Name()) + _ = f.Close() +} + +// runState is what a run can learn about another run from its lock file. +type runState int + +const ( + runLive runState = iota // its lock is held: it runs + runDead // its lock file is there, and nobody holds the lock + runUnknown // its lock file is not there, not a regular file, or cannot be read +) + +// stateOf returns the state of the run id. +func stateOf(id string) runState { + return stateAt(lockPath(id)) +} + +// stateAt returns the state of the run whose lock file is path. Any local +// user can make a file in the shared temporary directory, with the name +// of a lock file. So stateAt never waits in open (a FIFO would wait for a +// writer, for ever), follows no link, and takes only a regular file: +// anything else is runUnknown. +func stateAt(path string) runState { + f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0) //nolint:gosec // G304: a lock file of a run of this test, which it only locks. + if err != nil { + return runUnknown + } + defer func() { _ = f.Close() }() + if fi, err := f.Stat(); err != nil || !fi.Mode().IsRegular() { + return runUnknown + } + err = syscall.Flock(int(f.Fd()), syscall.LOCK_SH|syscall.LOCK_NB) + switch { + case errors.Is(err, syscall.EWOULDBLOCK): + return runLive + case err != nil: + return runUnknown + } + return runDead // the lock ends when f closes +} + +// isStale reports whether id is the run ID of another run that has ended: +// its lock file is not locked, or, when this run cannot see its lock file, +// it started long ago. Anything else is not a run ID of this test, and is +// never stale. +func isStale(id string) bool { + m := runIDPattern.FindStringSubmatch(id) + if m == nil || id == runID { + return false + } + secs, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + return false + } + age := time.Since(time.Unix(secs, 0)) + switch stateOf(id) { + case runDead: + return age > staleDead + case runUnknown: + return age > staleUnknown + } + return false +} + +// bases are the directories that may hold the private directory of a run +// (runDirectory): $XDG_RUNTIME_DIR, and the user's cache directory. +func bases() []string { + var out []string + if d := os.Getenv("XDG_RUNTIME_DIR"); d != "" { + out = append(out, d) + } + if d, err := os.UserCacheDir(); err == nil { + out = append(out, d) + } + return out +} + +// removeStale removes what earlier runs of this test left, when they have +// ended (isStale): their containers, networks and images, their private +// directories, and their lock files. It never touches an object or a file +// without a run ID in its name or its label. It logs what it removes. +func removeStale(t *testing.T) { + t.Helper() + for _, kind := range []struct { + what string + list []string // a docker command that prints "ID RUNID" lines + rm []string // the docker command that removes one of them + }{ + {"container", []string{"ps", "--all", "--filter", "label=nodkey-e2e", "--format", `{{.ID}} {{.Label "nodkey-e2e"}}`}, nil}, + {"network", []string{"network", "ls", "--filter", "label=nodkey-e2e", "--format", `{{.ID}} {{.Label "nodkey-e2e"}}`}, + []string{"network", "rm"}}, + {"image", []string{"image", "ls", "nodkey-e2e-sshd", "--format", `nodkey-e2e-sshd:{{.Tag}} {{.Tag}}`}, + []string{"image", "rm"}}, + } { + stdout, stderr, err := runDocker(dockerLimit, kind.list...) + if err != nil { + t.Logf("cannot list the %ss of earlier runs: %v: %s", kind.what, err, strings.TrimSpace(stderr)) + continue + } + for line := range strings.Lines(stdout) { + name, id, _ := strings.Cut(strings.TrimSpace(line), " ") + if !isStale(id) { + continue + } + var rmErr error + if kind.rm == nil { + rmErr = removeContainer(name) + } else if _, stderr, err := runDocker(dockerLimit, append(kind.rm, name)...); err != nil { + rmErr = fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr)) + } + if rmErr != nil { + t.Logf("cannot remove the %s %s of the ended run %s: %v", kind.what, name, id, rmErr) + continue + } + t.Logf("removed the %s %s of the ended run %s", kind.what, name, id) + } + } + for _, base := range bases() { + entries, err := os.ReadDir(base) + if err != nil { + continue + } + for _, en := range entries { + if id, ok := strings.CutPrefix(en.Name(), runPrefix); ok && en.IsDir() && isStale(id) { + dir := filepath.Join(base, en.Name()) + if err := os.RemoveAll(dir); err != nil { + t.Logf("cannot remove the private directory %s of the ended run %s: %v", dir, id, err) + continue + } + t.Logf("removed the private directory %s of the ended run %s", dir, id) + } + } + } + // The lock files last: the checks above read them. + if entries, err := os.ReadDir(os.TempDir()); err == nil { + for _, en := range entries { + name := en.Name() + if id, ok := strings.CutPrefix(strings.TrimSuffix(name, ".lock"), runPrefix); ok && + strings.HasSuffix(name, ".lock") && isStale(id) { + // The lock file of another user stays: the directory has its sticky bit. + _ = os.Remove(filepath.Join(os.TempDir(), name)) + } + } + } +} + +// run holds the private directory of this run, made once. +var run struct { + once sync.Once + dir string // the private directory, base/nodkey-e2e- + nodkey string // the nodkey binary in it + err string +} + +// runDirectory returns the private directory of this run, and the nodkey +// binary in it. It holds every file of the test, also the throwaway keys, +// and TestMain removes it at the end. +// +// Every directory on the way to it passes the file checks of nodkey +// (DESIGN.md section 14): the system temporary directory fails them, so it +// is in one of bases, as in the tests of nodkey broker. nodkey keygen +// checks the directories on the way and its new file as the broker checks +// a key file, so a key that it makes there is the probe. A program there +// must run too: the fake store. +func runDirectory(t *testing.T) (dir, nodkey string) { + t.Helper() + run.once.Do(func() { + var tried []string + for _, base := range bases() { + d := filepath.Join(base, runPrefix+runID) + if err := os.Mkdir(d, 0o700); err != nil { + tried = append(tried, fmt.Sprintf("%s: %v", base, err)) + continue + } + bin := filepath.Join(d, "nodkey") + if err := buildNodkey(bin); err != nil { + _ = os.RemoveAll(d) + run.err = err.Error() + return + } + if err := probeDir(d, bin); err != nil { + _ = os.RemoveAll(d) + tried = append(tried, fmt.Sprintf("%s: %v", base, err)) + continue + } + run.dir, run.nodkey = d, bin + return + } + run.err = fmt.Sprintf("no directory passes the file checks of nodkey: tried %q", tried) + }) + if run.err != "" { + t.Fatal(run.err) + } + return run.dir, run.nodkey +} + +// probeDir reports whether dir passes the checks of a key file, and +// whether a program there can run. +func probeDir(dir, bin string) error { + if err := os.Chmod(dir, 0o700); err != nil { //nolint:gosec // G302: a directory, private to its owner. + return err + } + probe := filepath.Join(dir, "probe.key") + keygen := exec.CommandContext(context.Background(), bin, "keygen", "--out", probe) //nolint:gosec // G204: the nodkey binary of the test. + keygen.Env = append([]string{"PATH=" + clientPath}, quickExit()...) + out, err := keygen.CombinedOutput() + _ = os.Remove(probe) + if err != nil { + return fmt.Errorf("nodkey keygen: %w: %s", err, strings.TrimSpace(string(out))) + } + script := filepath.Join(dir, "probe.sh") + if err := os.WriteFile(script, []byte("#!/bin/sh\nexit 0\n"), 0o700); err != nil { //nolint:gosec // G306: a program of the test. + return err + } + defer func() { _ = os.Remove(script) }() + if err := exec.CommandContext(context.Background(), script).Run(); err != nil { //nolint:gosec // G204: a program that the test wrote. + return fmt.Errorf("a program there cannot run: %w", err) + } + return nil +} + +// subdirectory makes a new directory with the prefix name in the private +// directory of this run, for one test, and removes it when the test ends. +func subdirectory(t *testing.T, name string) string { + t.Helper() + parent, _ := runDirectory(t) + dir, err := os.MkdirTemp(parent, name) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return dir +} + +// TestLockState: a run learns the state of another run from its lock file +// at once, and takes only a regular file. A FIFO, a link or a directory +// with the name of a lock file, which any local user can make in the +// shared temporary directory, is runUnknown: removeStale never waits on it. +func TestLockState(t *testing.T) { + dir := t.TempDir() + free := filepath.Join(dir, "free.lock") + writeFile(t, free, nil) + held := filepath.Join(dir, "held.lock") + writeFile(t, held, nil) + lock, err := os.Open(held) //nolint:gosec // G304: a file of the test. + if err != nil { + t.Fatal(err) + } + defer func() { _ = lock.Close() }() + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + t.Fatal(err) + } + fifo := filepath.Join(dir, "fifo.lock") + if err := syscall.Mkfifo(fifo, 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "link.lock") + if err := os.Symlink(free, link); err != nil { + t.Fatal(err) + } + // The case that the review found: a FIFO with the name of the lock file + // of an old run, in the shared temporary directory. That run counts as + // one whose lock file cannot be checked, so it is stale after an hour. + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + t.Fatal(err) + } + old := "1700000000-" + hex.EncodeToString(b) + if err := syscall.Mkfifo(lockPath(old), 0o600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Remove(lockPath(old)) }) + + for _, tt := range []struct { + name string + got func() bool + }{ + {"a regular file that nobody locks", func() bool { return stateAt(free) == runDead }}, + {"a regular file that a run locks", func() bool { return stateAt(held) == runLive }}, + {"a FIFO", func() bool { return stateAt(fifo) == runUnknown }}, + {"a link to a regular file", func() bool { return stateAt(link) == runUnknown }}, + {"a directory", func() bool { return stateAt(dir) == runUnknown }}, + {"no file", func() bool { return stateAt(filepath.Join(dir, "missing.lock")) == runUnknown }}, + {"a FIFO as the lock file of an old run", func() bool { return isStale(old) }}, + } { + done := make(chan bool, 1) + go func() { done <- tt.got() }() + select { + case ok := <-done: + if !ok { + t.Errorf("%s: the wrong state", tt.name) + } + case <-time.After(5 * time.Second): + t.Fatalf("%s: the check waits", tt.name) + } + } +} diff --git a/test/e2e/telegram_test.go b/test/e2e/telegram_test.go new file mode 100644 index 0000000..1d48ecc --- /dev/null +++ b/test/e2e/telegram_test.go @@ -0,0 +1,650 @@ +//go:build e2e + +package e2e + +import ( + "bufio" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "html" + "io" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// The fake Telegram server of the end-to-end test, and the fake HTTPS +// proxy in front of it. This is the approach of the tests of nodkey +// broker in cmd/nodkey (brokerfake_linux_test.go): the broker reaches the +// fake through HTTPS_PROXY. The proxy takes only a CONNECT to +// api.telegram.org, and answers the TLS handshake itself, with a +// certificate of a test CA that the broker trusts through SSL_CERT_FILE +// (and an empty SSL_CERT_DIR). So the broker runs its real Telegram code, +// and nothing goes to the real Bot API: no other CA is trusted, so even a +// call that missed the proxy could not finish its TLS handshake, and the +// bot token is only sent after it. + +// telegramHost is the host of the Bot API. +const telegramHost = "api.telegram.org" + +// approverID is the made-up Telegram user ID of the approver. The private +// chat with the approver has the same ID. +const approverID int64 = 7000001 + +// The button data of DESIGN.md section 7: the prefix and the request ID. +const ( + tapOnce = "o:" // Approve once + tapWindow = "w:" // Approve N min, and Approve on an SSH grant + tapDeny = "d:" // Deny +) + +// fakeTelegram is a fake Bot API server behind a fake HTTPS proxy. It +// answers like Telegram, and keeps every call, for the checks and for the +// canary search. +type fakeTelegram struct { + token string // the made-up bot token + proxyAddr string // the fake proxy + caFile string // SSL_CERT_FILE of the broker: the test CA + caDir string // SSL_CERT_DIR of the broker: an empty directory + + mu sync.Mutex + calls []tgCall + updates []tgUpdate + nextUpdate int64 + nextMsg int64 + markups map[int64]json.RawMessage // the buttons of each message, by message ID + connects []string // what the proxy was asked to connect to + changed chan struct{} // closed and made new on each change +} + +// tgCall is one call that the fake got. +type tgCall struct { + method string + body []byte // the request body + msgID int64 // sendMessage: the message ID of its answer; editMessageText: the message it edits + buttons []string // sendMessage: the data of its buttons +} + +// tgUpdate is a queued update. +type tgUpdate struct { + id int64 + raw []byte +} + +// tgMessage is a message with buttons: the message of a request. +type tgMessage struct { + id int64 // the message ID + requestID string // the request ID in the button data + buttons []string // the button data + text string // the text that the person sees: HTML tags removed, entities decoded +} + +// newFakeTelegram starts a fake Telegram server and its proxy, until the +// test ends. The certificate of its test CA goes into the directory dir. +func newFakeTelegram(t *testing.T, dir string) *fakeTelegram { + t.Helper() + f := &fakeTelegram{ + token: newBotToken(t), + nextUpdate: 1000, + nextMsg: 500, + markups: make(map[int64]json.RawMessage), + changed: make(chan struct{}), + } + caPEM, leaf := newTestCA(t) + f.caFile = filepath.Join(dir, "ca.pem") + if err := os.WriteFile(f.caFile, caPEM, 0o600); err != nil { + t.Fatal(err) + } + f.caDir = filepath.Join(dir, "none") + if err := os.Mkdir(f.caDir, 0o700); err != nil { + t.Fatal(err) + } + + ln, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f.proxyAddr = ln.Addr().String() + tunnels := &chanListener{conns: make(chan net.Conn), done: make(chan struct{}), addr: ln.Addr()} + srv := &http.Server{Handler: f, ReadHeaderTimeout: waitLimit} + tlsConfig := &tls.Config{Certificates: []tls.Certificate{leaf}, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12} + var wg sync.WaitGroup + wg.Go(func() { _ = srv.Serve(tunnels) }) + wg.Go(func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + wg.Go(func() { f.connect(c, tlsConfig, tunnels) }) + } + }) + t.Cleanup(func() { + _ = ln.Close() + _ = srv.Close() + wg.Wait() + }) + return f +} + +// newBotToken returns a made-up bot token: digits, ":", and 35 random +// characters. +func newBotToken(t *testing.T) string { + t.Helper() + n, err := rand.Int(rand.Reader, big.NewInt(1_000_000_000)) + if err != nil { + t.Fatal(err) + } + secret := strings.Map(func(r rune) rune { + if r >= '2' && r <= '7' { + return r + } + return r | 0x20 // some lower-case letters + }, rand.Text()+rand.Text()) + return strconv.FormatInt(1_000_000_000+n.Int64(), 10) + ":" + secret[:35] +} + +// connect handles one connection to the fake proxy: a CONNECT to +// api.telegram.org, then TLS with the test CA's certificate, and the +// fake's HTTP server. Anything else is refused. +func (f *fakeTelegram) connect(c net.Conn, cfg *tls.Config, tunnels *chanListener) { + _ = c.SetReadDeadline(time.Now().Add(waitLimit)) + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil { + _ = c.Close() + return + } + f.mu.Lock() + f.connects = append(f.connects, req.Method+" "+req.Host) + f.mu.Unlock() + if req.Method != http.MethodConnect || req.Host != telegramHost+":443" { + _, _ = io.WriteString(c, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n") + _ = c.Close() + return + } + _ = c.SetReadDeadline(time.Time{}) + if _, err := io.WriteString(c, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + _ = c.Close() + return + } + select { + case tunnels.conns <- tls.Server(&bufferedConn{Conn: c, r: br}, cfg): + case <-tunnels.done: + _ = c.Close() + } +} + +// bufferedConn is a connection whose first bytes may wait in r. +type bufferedConn struct { + net.Conn + r *bufio.Reader +} + +func (c *bufferedConn) Read(p []byte) (int, error) { return c.r.Read(p) } + +// chanListener is a net.Listener of the connections in conns. +type chanListener struct { + conns chan net.Conn + done chan struct{} + once sync.Once + addr net.Addr +} + +func (l *chanListener) Accept() (net.Conn, error) { + select { + case c := <-l.conns: + return c, nil + case <-l.done: + return nil, net.ErrClosed + } +} + +func (l *chanListener) Close() error { + l.once.Do(func() { close(l.done) }) + return nil +} + +func (l *chanListener) Addr() net.Addr { return l.addr } + +// newTestCA returns a new CA certificate in PEM, and a certificate for +// api.telegram.org that it signed, with its key. +func newTestCA(t *testing.T) ([]byte, tls.Certificate) { + t.Helper() + now := time.Now() + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "nodkey e2e test CA"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + ca, err := x509.ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + leafTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: telegramHost}, + DNSNames: []string{telegramHost}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + leafDER, err := x509.CreateCertificate(rand.Reader, leafTemplate, ca, &leafKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), + tls.Certificate{Certificate: [][]byte{leafDER}, PrivateKey: leafKey} +} + +func (f *fakeTelegram) notifyLocked() { + close(f.changed) + f.changed = make(chan struct{}) +} + +// writeTelegram writes an answer of the Bot API. +func writeTelegram(w http.ResponseWriter, status int, body string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, body) +} + +func (f *fakeTelegram) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rest, ok := strings.CutPrefix(r.URL.Path, "/bot") + token, method, ok2 := strings.Cut(rest, "/") + if !ok || !ok2 || token != f.token { + writeTelegram(w, http.StatusUnauthorized, `{"ok":false,"error_code":401,"description":"Unauthorized"}`) + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + return + } + var p struct { + ChatID int64 `json:"chat_id"` + MessageID int64 `json:"message_id"` + Offset int64 `json:"offset"` + Timeout int `json:"timeout"` + ReplyMarkup json.RawMessage `json:"reply_markup"` + } + _ = json.Unmarshal(body, &p) // the fields come from the broker; a test checks what it needs + call := tgCall{method: method, body: body, msgID: p.MessageID} + f.mu.Lock() + if method == "sendMessage" { + f.nextMsg++ + call.msgID = f.nextMsg + call.buttons = buttonData(p.ReplyMarkup) + f.markups[call.msgID] = p.ReplyMarkup + } + f.calls = append(f.calls, call) + f.notifyLocked() + f.mu.Unlock() + + message := func(id int64) string { + return fmt.Sprintf(`{"ok":true,"result":{"message_id":%d,"date":1,"chat":{"id":%d,"type":"private"},`+ + `"text":"the text"}}`, id, p.ChatID) + } + switch method { + case "getUpdates": + f.serveUpdates(w, r, p.Offset, time.Duration(p.Timeout)*time.Second) + case "sendMessage": + writeTelegram(w, http.StatusOK, message(call.msgID)) + case "editMessageText", "editMessageReplyMarkup": + writeTelegram(w, http.StatusOK, message(p.MessageID)) + case "answerCallbackQuery": + writeTelegram(w, http.StatusOK, `{"ok":true,"result":true}`) + default: + writeTelegram(w, http.StatusNotFound, `{"ok":false,"error_code":404,"description":"Not Found"}`) + } +} + +// serveUpdates answers getUpdates: the queued updates from offset on, or +// none after wait, as Telegram's long polling does. +func (f *fakeTelegram) serveUpdates(w http.ResponseWriter, r *http.Request, offset int64, wait time.Duration) { + timer := time.NewTimer(min(max(wait, time.Second), 50*time.Second)) + defer timer.Stop() + for { + f.mu.Lock() + f.updates = slices.DeleteFunc(f.updates, func(u tgUpdate) bool { return u.id < offset }) + var parts []string + for _, u := range f.updates { + parts = append(parts, string(u.raw)) + } + changed := f.changed + f.mu.Unlock() + if len(parts) > 0 { + writeTelegram(w, http.StatusOK, `{"ok":true,"result":[`+strings.Join(parts, ",")+`]}`) + return + } + select { + case <-changed: + case <-timer.C: + writeTelegram(w, http.StatusOK, `{"ok":true,"result":[]}`) + return + case <-r.Context().Done(): + return + } + } +} + +// buttonData returns the callback data of each button in reply_markup. +func buttonData(markup json.RawMessage) []string { + var kb struct { + InlineKeyboard [][]struct { + CallbackData string `json:"callback_data"` + } `json:"inline_keyboard"` + } + if len(markup) == 0 || json.Unmarshal(markup, &kb) != nil { + return nil + } + var data []string + for _, row := range kb.InlineKeyboard { + for _, b := range row { + data = append(data, b.CallbackData) + } + } + return data +} + +// waitFor waits until cond, which runs with f.mu held, is true. +func (f *fakeTelegram) waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.NewTimer(waitLimit) + defer deadline.Stop() + for { + f.mu.Lock() + ok, changed := cond(), f.changed + f.mu.Unlock() + if ok { + return + } + select { + case <-changed: + case <-deadline.C: + t.Fatalf("waited %s in vain for %s", waitLimit, what) + } + } +} + +// messagesLocked returns every message that the broker sent: those with +// buttons, of requests, and those without, alerts and notices. +func (f *fakeTelegram) messagesLocked() []tgCall { + var out []tgCall + for _, c := range f.calls { + if c.method == "sendMessage" { + out = append(out, c) + } + } + return out +} + +// sent returns how many messages the fake got, with buttons or without. +func (f *fakeTelegram) sent() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.messagesLocked()) +} + +// waitMessage waits for the n-th message, and returns it. The program p +// asks for it, and waits for the tap. waitMessage fails when that message +// has no buttons, an alert or a notice, which this test never expects, +// and when p ends first: then no message comes. +func (f *fakeTelegram) waitMessage(t *testing.T, n int, p *proc) tgMessage { + t.Helper() + deadline := time.NewTimer(waitLimit) + defer deadline.Stop() + for ended := false; ; { + f.mu.Lock() + got, changed := len(f.messagesLocked()) >= n, f.changed + f.mu.Unlock() + switch { + case got: + case ended: + t.Fatalf("%s ended before message %d came; its output:\n%s", p.name, n, p.output()) + default: + select { + case <-changed: + case <-p.done: + ended = true // look once more: the message may have come just before + case <-deadline.C: + t.Fatalf("waited %s in vain for message %d", waitLimit, n) + } + continue + } + break + } + f.mu.Lock() + c := f.messagesLocked()[n-1] + f.mu.Unlock() + text := shownText(t, c.body) + if len(c.buttons) == 0 { + t.Fatalf("message %d has no buttons: an alert or a notice went out:\n%s", n, redact(text)) + } + _, id, _ := strings.Cut(c.buttons[len(c.buttons)-1], ":") + return tgMessage{id: c.msgID, requestID: id, buttons: c.buttons, text: text} +} + +// unasked returns the text of each message without buttons: an alert or a +// notice. +func (f *fakeTelegram) unasked(t *testing.T) []string { + t.Helper() + f.mu.Lock() + calls := f.messagesLocked() + f.mu.Unlock() + var out []string + for _, c := range calls { + if len(c.buttons) == 0 { + out = append(out, shownText(t, c.body)) + } + } + return out +} + +// otherCalls returns every call to the fake, as the method and the body, +// except the messages with buttons and the edits of them: the calls that +// must not hold claimed text. +func (f *fakeTelegram) otherCalls() []string { + f.mu.Lock() + defer f.mu.Unlock() + asked := make(map[int64]bool) + for _, c := range f.calls { + if c.method == "sendMessage" && len(c.buttons) > 0 { + asked[c.msgID] = true + } + } + var out []string + for _, c := range f.calls { + switch { + case c.method == "sendMessage" && len(c.buttons) > 0: + case c.method == "editMessageText" && asked[c.msgID]: + default: + out = append(out, c.method+" "+string(c.body)) + } + } + return out +} + +// shownText returns the text of a sendMessage or editMessageText body as +// the person sees it: the HTML of the message without its tags, with its +// entities decoded. The broker writes only tags. +func shownText(t *testing.T, body []byte) string { + t.Helper() + var p struct { + Text string `json:"text"` + ParseMode string `json:"parse_mode"` + } + if err := json.Unmarshal(body, &p); err != nil { + t.Fatalf("a message body is not JSON: %v", err) + } + if p.ParseMode != "HTML" { + t.Errorf("a message has parse_mode %q, want HTML", p.ParseMode) + } + plain := strings.NewReplacer("", "", "", "").Replace(p.Text) + if strings.ContainsAny(plain, "<>") { + t.Errorf("a message has HTML tags other than : %q", redact(p.Text)) + } + return html.UnescapeString(plain) +} + +// tap presses the button of m whose data starts with prefix, as the +// approver, in the private chat, on the message with the buttons that the +// broker sent there. It returns the text that the phone shows. A tap can +// come before the broker knows that its message was sent; then the phone +// shows "Not ready yet. Tap again in a moment.", and the person taps +// again. +func (f *fakeTelegram) tap(t *testing.T, m tgMessage, prefix string) string { + t.Helper() + if !slices.Contains(m.buttons, prefix+m.requestID) { + t.Fatalf("message %d has no button %s: its buttons are %q", m.id, prefix, m.buttons) + } + for try := 1; ; try++ { + answer := f.waitTapAnswer(t, f.queueTap(t, m.id, prefix+m.requestID)) + if !strings.Contains(answer, "Not ready yet") { + return answer + } + if try == 100 { + t.Fatalf("every tap on message %d came too early", m.id) + } + time.Sleep(10 * time.Millisecond) + } +} + +// queueTap queues a press of the approver on the message msgID with the +// button data data, and returns the ID of the press (the callback query). +func (f *fakeTelegram) queueTap(t *testing.T, msgID int64, data string) string { + t.Helper() + f.mu.Lock() + defer f.mu.Unlock() + f.nextUpdate++ + id := "cb" + strconv.FormatInt(f.nextUpdate, 10) + msg := map[string]any{ + "message_id": msgID, "date": 1, "text": "the text", + "chat": map[string]any{"id": approverID, "type": "private"}, + } + if m := f.markups[msgID]; len(m) > 0 { + msg["reply_markup"] = m + } + raw, err := json.Marshal(map[string]any{ + "update_id": f.nextUpdate, + "callback_query": map[string]any{ + "id": id, + "from": map[string]any{"id": approverID, "is_bot": false, "first_name": "Test"}, + "chat_instance": "1", + "data": data, + "message": msg, + }, + }) + if err != nil { + t.Fatal(err) + } + f.updates = append(f.updates, tgUpdate{id: f.nextUpdate, raw: raw}) + f.notifyLocked() + return id +} + +// waitTapAnswer waits for the answer to the press id +// (answerCallbackQuery), and returns the text that the phone shows. +func (f *fakeTelegram) waitTapAnswer(t *testing.T, id string) string { + t.Helper() + var text string + f.waitFor(t, "the answer to the press "+id, func() bool { + for _, c := range f.calls { + var p struct { + ID string `json:"callback_query_id"` + Text string `json:"text"` + } + if c.method == "answerCallbackQuery" && json.Unmarshal(c.body, &p) == nil && p.ID == id { + text = p.Text + return true + } + } + return false + }) + return text +} + +// waitResult waits until the broker has edited the message m to show the +// end of its request, and returns the last line of the new text, such as +// "๐Ÿ‘ approved once by you at 14:02:41". +func (f *fakeTelegram) waitResult(t *testing.T, m tgMessage) string { + t.Helper() + var body []byte + f.waitFor(t, fmt.Sprintf("the edit of message %d", m.id), func() bool { + for _, c := range f.calls { + if c.method == "editMessageText" && c.msgID == m.id { + body = c.body + return true + } + } + return false + }) + text := shownText(t, body) + return text[strings.LastIndex(text, "\n")+1:] +} + +// bodies returns every request body that the fake got, each after the name +// of its method. +func (f *fakeTelegram) bodies() []string { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for _, c := range f.calls { + out = append(out, c.method+" "+string(c.body)) + } + return out +} + +// shownTexts returns the text of every message and every edit of a +// message, as the person sees it. +func (f *fakeTelegram) shownTexts(t *testing.T) []string { + t.Helper() + f.mu.Lock() + calls := slices.Clone(f.calls) + f.mu.Unlock() + var out []string + for _, c := range calls { + if c.method == "sendMessage" || c.method == "editMessageText" { + out = append(out, shownText(t, c.body)) + } + } + return out +} + +// proxied returns what the proxy was asked to connect to. +func (f *fakeTelegram) proxied() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.connects) +} diff --git a/test/e2e/testdata/sshd/Dockerfile b/test/e2e/testdata/sshd/Dockerfile new file mode 100644 index 0000000..b4992aa --- /dev/null +++ b/test/e2e/testdata/sshd/Dockerfile @@ -0,0 +1,25 @@ +# The SSH targets of the end-to-end test of nodkey (test/e2e): OpenSSH's +# sshd and git, from Debian. The image holds no key: the host keys that +# the openssh-server package makes are removed. Before the container +# starts, the test copies its own throwaway keys to /e2e (see +# entrypoint.sh). The same image runs the root-only test of +# internal/client. +# +# The base image is pinned by the digest of its multi-platform index. +FROM debian:bookworm-slim@sha256:7c7b2c966bc9ee8cedfeef67e0e279108992c77681fa595db4a9d65c06ccc587 + +# deploy and ops are the remote users. Their password is "*": no password +# works, and the account is not locked, so sshd lets the key in (sshd +# refuses a locked account when UsePAM is off). The bare repository is +# the target of git over SSH. +RUN apt-get update \ + && apt-get install --yes --no-install-recommends openssh-server git \ + && rm -rf /var/lib/apt/lists/* /etc/ssh/ssh_host_* \ + && useradd --create-home --shell /bin/sh --password '*' deploy \ + && useradd --create-home --shell /bin/sh --password '*' ops \ + && su deploy -c 'git init --quiet --bare --initial-branch=main /home/deploy/repo.git' \ + && mkdir -p /run/sshd /e2e /etc/ssh/authorized_keys + +COPY sshd_config /etc/ssh/sshd_config.e2e +COPY entrypoint.sh /usr/local/bin/e2e-entrypoint.sh +ENTRYPOINT ["/bin/sh", "/usr/local/bin/e2e-entrypoint.sh"] diff --git a/test/e2e/testdata/sshd/entrypoint.sh b/test/e2e/testdata/sshd/entrypoint.sh new file mode 100644 index 0000000..e66754f --- /dev/null +++ b/test/e2e/testdata/sshd/entrypoint.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# The entrypoint of the SSH targets of the end-to-end test of nodkey. +# +# Before the container starts, the test copies these files to /e2e: +# authorized_keys the public half of the broker's SSH key +# targets one line for each target: its name and its port +# NAME.key the host key of the target NAME, such as web-1.key +# +# Each target is an sshd of its own, with its own port and host key. Its +# log lines go to the container's log, each with the name of the target. +# The container ends on its own after $1 seconds (default 900), in case +# the test could not remove it. +# +# The container has only the capabilities that sshd needs, so this script +# writes only files of root: docker cp makes the files in /e2e files of +# root too. +set -eu + +for user in deploy ops; do + install -m 644 /e2e/authorized_keys "/etc/ssh/authorized_keys/$user" +done + +while read -r name port; do + install -m 600 "/e2e/$name.key" "/etc/ssh/e2e_$name.key" + (/usr/sbin/sshd -D -e -f /etc/ssh/sshd_config.e2e -p "$port" -h "/etc/ssh/e2e_$name.key" 2>&1 | + sed -u "s/^/$name: /") & +done