diff --git a/DESIGN.md b/DESIGN.md index f446571..9c00773 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -406,7 +406,9 @@ NODKEY_REASON="apply the nginx fix" ansible-playbook site.yml the broker cancels the request (section 7). It sees this also when `ssh` sent more bytes before: it asks the kernel whether the other side hung up (`poll` with `POLLRDHUP`), and does not read those - bytes. + bytes. After a hang-up, the client gives `ssh` no signature, also + when an answer of the broker comes at that moment, and its line says + that `ssh` closed its connection. 6. The broker checks, and denies on any failure. It makes these checks before any message or place in a queue (section 8). A request that fails them gets no message, and the audit log gets it with the key name, if diff --git a/cmd/nodkey/brokerenv_linux_test.go b/cmd/nodkey/brokerenv_linux_test.go index 51dd8b4..d9aeab4 100644 --- a/cmd/nodkey/brokerenv_linux_test.go +++ b/cmd/nodkey/brokerenv_linux_test.go @@ -208,8 +208,7 @@ func usableTestDir(t *testing.T, dir string) bool { return false } probe := filepath.Join(dir, "probe") - //nolint:gosec // G703: a file in the test's own new directory. - if os.WriteFile(probe, []byte("#!/bin/sh\nexit 0\n"), 0o600) != nil || config.CheckPrivateFile(probe) != nil { + if writeUninherited(probe, []byte("#!/bin/sh\nexit 0\n")) != nil || config.CheckPrivateFile(probe) != nil { return false } if os.Chmod(probe, 0o700) != nil { //nolint:gosec // G302: a program of the test. @@ -223,7 +222,7 @@ func usableTestDir(t *testing.T, dir string) bool { // writeTestFile writes data to path with exactly mode. func writeTestFile(t *testing.T, path string, data []byte, mode os.FileMode) { t.Helper() - if err := os.WriteFile(path, data, 0o600); err != nil { //nolint:gosec // G703: a file in the test's own directory. + if err := writeUninherited(path, data); err != nil { t.Fatal(err) } if err := os.Chmod(path, mode); err != nil { @@ -231,6 +230,22 @@ func writeTestFile(t *testing.T, path string, data []byte, mode os.FileMode) { } } +// writeUninherited writes data to path with mode 0600, so that no child +// process gets a copy of the file descriptor that writes it. The tests run +// in parallel, and many start child processes. A child gets a copy of each +// open descriptor of the test process and keeps it until it runs its own +// program. While a child keeps a copy of the descriptor that writes a +// program, the start of that program fails with ETXTBSY, "text file busy" +// (go.dev/issue/22315): then the probe of privateTestDir failed, and the +// test failed or was skipped. Each start of a child holds syscall.ForkLock +// for writing until the child runs its program, so no child starts while +// this holds it for reading. +func writeUninherited(path string, data []byte) error { + syscall.ForkLock.RLock() + defer syscall.ForkLock.RUnlock() + return os.WriteFile(path, data, 0o600) //nolint:gosec // G703: a file in the test's own directory. +} + // madeUp returns a made-up value with what at its start, made at run time, // so the source holds nothing that looks like a token. func madeUp(what string) string { @@ -458,11 +473,13 @@ var ( // freeAddress returns an address on the loopback interface with a port // that is free now, for the broker of one test. The broker listens on it -// a moment later. So the port is outside the range of ports that the -// kernel gives out by itself, to sockets that ask for any port or connect -// (net.ipv4.ip_local_port_range): no other socket of the tests takes it in -// between. No two tests of this process get the same port. When the -// kernel's range leaves no room, the kernel chooses the port, and +// a moment later, and again when the test starts it again. So the port is +// outside the range of ports that the kernel gives out by itself, to +// sockets that ask for any port or connect (net.ipv4.ip_local_port_range): +// no other socket of the tests takes it in between. No two tests of this +// process get the same port, and no two tests of two processes that run +// at the same time (reservePort), such as the tests of two checkouts. When +// the kernel's range leaves no room, the kernel chooses the port, and // startListening tries again if another socket takes it first. func freeAddress(t *testing.T) string { t.Helper() @@ -489,7 +506,7 @@ func freeAddress(t *testing.T) string { taken := portsTaken[port] portsTaken[port] = true portsMu.Unlock() - if taken { + if taken || !reservePort(t, port) { continue } addr := "127.0.0.1:" + strconv.FormatInt(port, 10) @@ -515,7 +532,7 @@ func freeAddress(t *testing.T) string { taken := portsTaken[int64(addr.Port)] portsTaken[int64(addr.Port)] = true portsMu.Unlock() - if !taken { + if !taken && reservePort(t, int64(addr.Port)) { return addr.String() } } @@ -523,6 +540,27 @@ func freeAddress(t *testing.T) string { return "" } +// reservePort reserves port for the test t against other processes of +// these tests, such as a second go test run at the same time. Until t +// ends, it holds the abstract Unix socket "@nodkey-test-port-PORT", and +// only one socket can hold such a name. It reports false when another +// process holds the name. Without it, such a process could take the port +// while the broker of t was stopped, and the next start of that broker +// failed; or it could listen there, and TestBrokerNoListenBeforeTheAuditLog +// saw a broker listen. The kernel frees the name when the socket closes, +// also when the process ends without cleanup. A port that turns out to be +// in use stays reserved until t ends; that does no harm. +func reservePort(t *testing.T, port int64) bool { + t.Helper() + c, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "unixgram", + "@nodkey-test-port-"+strconv.FormatInt(port, 10)) + if err != nil { + return false + } + t.Cleanup(func() { _ = c.Close() }) + return true +} + // portsOutside returns the ports [low, high) for the brokers of the tests, // outside the kernel's own range of ports from lo to hi: the larger of the // ports from 10000 to below lo, and the ports above hi. high is not more diff --git a/internal/client/client.go b/internal/client/client.go index 9cf5351..99155e8 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -212,7 +212,9 @@ func (tr *trace) isConnected() bool { // answer: no other request goes on that connection, so there is no // pipelining. The request is never sent again. When ctx ends, the // connection closes, so the broker cancels the request; Post then returns -// context.Cause(ctx). +// context.Cause(ctx). An answer that comes while the connection closes can +// still be returned instead, as Go's HTTP client does: a caller that must +// deny after ctx ends checks ctx after Post. // // Every other failure is an *Error: a final answer other than 200, also a // redirect; a 200 whose body api.ReadAnswer refuses; more than a few diff --git a/internal/client/sshagent.go b/internal/client/sshagent.go index 6756b10..114eecc 100644 --- a/internal/client/sshagent.go +++ b/internal/client/sshagent.go @@ -588,6 +588,8 @@ func (c *agentConn) bind(contents []byte) []byte { // claimed context of the process that connected. It checks the answer // with checkSignAnswer before it gives the signature to ssh. When ssh // closes the connection while the request waits, it ends the request. +// After a hang-up, the deadline or a stop of the agent, ssh gets no +// signature, whatever the broker answers then. func (c *agentConn) sign(ctx context.Context, key, data []byte, flags uint32) []byte { a := c.a switch { @@ -618,6 +620,15 @@ func (c *agentConn) sign(ctx context.Context, key, data []byte, flags uint32) [] a.log.waitingFor(s) }) stop() + if cause := context.Cause(waitCtx); cause != nil { + // The request was cancelled: ssh hung up, the deadline passed, or the + // agent stops. Go's HTTP client can still return an answer that came + // while it closed the connection to the broker: the 403 that the + // broker sends when it sees the request cancelled, or a signature + // after a tap at that moment. A cancel means deny: the cause is the + // result, and ssh gets no signature after it. + err = cause + } if err != nil { a.log.note("no signature%s: %s", code.label(), failureText(err)) return failureMessage diff --git a/internal/client/sshagent_helpers_test.go b/internal/client/sshagent_helpers_test.go index 446e306..be24a71 100644 --- a/internal/client/sshagent_helpers_test.go +++ b/internal/client/sshagent_helpers_test.go @@ -392,9 +392,11 @@ type sshServer struct { addr string hostKey ssh.PublicKey - mu sync.Mutex - kex [][]byte // the session-bind of each key exchange, in order - logins int // the logins that the server accepted + mu sync.Mutex + kex [][]byte // the session-bind of each key exchange, in order + ended int // the connections whose handshake ended, with a login or without + logins int // the logins that the server accepted + changed chan struct{} // closed and made new when a handshake ends } // recordingSigner signs with a host key and records each signature. @@ -429,7 +431,7 @@ func marshalBind(hostKey ssh.PublicKey, sessionID []byte, sig *ssh.Signature, fo // address. Only userKey may log in. func startSSHServer(t *testing.T, host ssh.AlgorithmSigner, userKey ssh.PublicKey) *sshServer { t.Helper() - s := &sshServer{hostKey: host.PublicKey()} + s := &sshServer{hostKey: host.PublicKey(), changed: make(chan struct{})} cfg := &ssh.ServerConfig{ PublicKeyCallback: func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { if !bytes.Equal(key.Marshal(), userKey.Marshal()) { @@ -471,13 +473,21 @@ func (s *sshServer) serve(nc net.Conn, cfg *ssh.ServerConfig) { defer func() { _ = nc.Close() }() _ = nc.SetDeadline(time.Now().Add(2 * waitLimit)) conn, chans, reqs, err := ssh.NewServerConn(nc, cfg) + // NewServerConn sends the client its success before it returns, so the + // client can see its login work before the server counts it here. A + // test waits for the count with loginsAfter. + s.mu.Lock() + s.ended++ + if err == nil { + s.logins++ + } + close(s.changed) + s.changed = make(chan struct{}) + s.mu.Unlock() if err != nil { return } defer func() { _ = conn.Close() }() - s.mu.Lock() - s.logins++ - s.mu.Unlock() go ssh.DiscardRequests(reqs) for nch := range chans { ch, chReqs, err := nch.Accept() @@ -504,11 +514,28 @@ func (s *sshServer) binds() [][]byte { return append([][]byte(nil), s.kex...) } -// loginCount returns how many logins the server accepted. -func (s *sshServer) loginCount() int { - s.mu.Lock() - defer s.mu.Unlock() - return s.logins +// loginsAfter waits until the handshake of n connections has ended, with +// a login or without, and returns how many logins the server accepted. +// Only then is the count final: the server answers a login before it +// counts it, and a refused login ends after the client has given up. +func (s *sshServer) loginsAfter(t testing.TB, n int) int { + t.Helper() + deadline := time.NewTimer(waitLimit) + defer deadline.Stop() + for { + s.mu.Lock() + ended, logins, changed := s.ended, s.logins, s.changed + s.mu.Unlock() + if ended >= n { + return logins + } + select { + case <-changed: + case <-deadline.C: + t.Fatalf("the SSH server ended %d handshakes within %s, want %d", ended, waitLimit, n) + return 0 + } + } } // fakeSSHBroker is a test handler in place of the broker: it lists keys, diff --git a/internal/client/sshagent_openssh_linux_test.go b/internal/client/sshagent_openssh_linux_test.go index 1a2d494..3eb317c 100644 --- a/internal/client/sshagent_openssh_linux_test.go +++ b/internal/client/sshagent_openssh_linux_test.go @@ -155,10 +155,10 @@ func TestOpenSSHLogin(t *testing.T) { } res := <-done if tt.answer == audit.AnswerApproveOnce { - if res.err != nil || srv.loginCount() != 1 { + if res.err != nil || srv.loginsAfter(t, 1) != 1 { t.Fatalf("ssh: %v; agent lines %q\n%s", res.err, au.lines(), res.out) } - } else if res.err == nil || srv.loginCount() != 0 { + } else if res.err == nil || srv.loginsAfter(t, 1) != 0 { t.Fatalf("ssh logged in: %v\n%s", res.err, res.out) } p := asked.Prompt @@ -216,8 +216,11 @@ func TestOpenSSHSignRequest(t *testing.T) { cmd := exec.CommandContext(ctx, sh, append([]string{"-c", script, sshPath}, args...)...) //nolint:gosec // G204: the test runs ssh through sh. cmd.Env = sshEnv(au.socket, dir, "NODKEY_REASON=from the shell", "SECRET_OF_THE_SHELL="+canary) out, err := cmd.CombinedOutput() - if err != nil || srv.loginCount() != 0 { - t.Fatalf("sh: %v; %d logins\n%s", err, srv.loginCount(), out) + if err != nil { + t.Fatalf("sh: %v\n%s", err, out) + } + if n := srv.loginsAfter(t, 1); n != 0 { + t.Fatalf("%d logins\n%s", n, out) } var body []byte select { @@ -296,12 +299,12 @@ func TestOpenSSHAgentChecksTheSignature(t *testing.T) { dir := shortTempDir(t) out, err := runSSH(t, sshPath, sshArgs(t, srv.addr, "deploy", host.PublicKey(), dir), sshEnv(au.socket, dir)) if tt.want == "" { - if err != nil || srv.loginCount() != 1 { + if err != nil || srv.loginsAfter(t, 1) != 1 { t.Fatalf("ssh: %v; agent lines %q\n%s", err, au.lines(), out) } return } - if err == nil || srv.loginCount() != 0 { + if err == nil || srv.loginsAfter(t, 1) != 0 { t.Fatalf("ssh logged in\n%s", out) } if got := au.lines(); len(got) != 1 || got[0] != "nodkey: no signature: "+tt.want { @@ -326,7 +329,7 @@ func TestOpenSSHAgentChecksTheSignature(t *testing.T) { au := startAgent(t, b, agentOptions{}) dir := shortTempDir(t) out, err := runSSH(t, sshPath, sshArgs(t, srv.addr, "deploy", host.PublicKey(), dir), sshEnv(au.socket, dir)) - if err == nil || srv.loginCount() != 0 { + if err == nil || srv.loginsAfter(t, 1) != 0 { t.Fatalf("ssh logged in\n%s", out) } // Another test may take the free port at once: then the TLS diff --git a/internal/client/sshagent_test.go b/internal/client/sshagent_test.go index 79c88fc..aaf1b45 100644 --- a/internal/client/sshagent_test.go +++ b/internal/client/sshagent_test.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "crypto/rsa" "encoding/binary" + "encoding/json" "errors" "fmt" "io" @@ -63,7 +64,8 @@ func (s *agentSigner) SignWithAlgorithm(_ io.Reader, data []byte, algorithm stri // login logs in to srv as user, with the first key that the agent ag // lists, as OpenSSH does with an agent. Give it a new agent connection. -// It may run in a goroutine of its own: it only returns errors. +// It may run in a goroutine of its own: it only returns errors. It can +// return before srv counts the login: see sshServer.loginsAfter. func login(ctx context.Context, ag agent.ExtendedAgent, srv *sshServer, user string) error { keys, err := ag.List() if err != nil { @@ -128,11 +130,11 @@ func TestAgentLogin(t *testing.T) { waiting := "nodkey: waiting for approval (request " + code + ")" want := []string{waiting} if tt.answer == audit.AnswerApproveOnce { - if err != nil || srv.loginCount() != 1 { + if err != nil || srv.loginsAfter(t, 1) != 1 { t.Fatalf("the login failed: %v; agent lines %q", err, au.lines()) } } else { - if err == nil || srv.loginCount() != 0 { + if err == nil || srv.loginsAfter(t, 1) != 0 { t.Fatalf("the login worked, %v", err) } want = append(want, "nodkey: no signature (request "+code+"): denied") @@ -1198,8 +1200,8 @@ func TestAgentCancelsWhenSSHHangsUp(t *testing.T) { go func() { done <- login(t.Context(), ag, srv, "deploy") }() code := sb.waitAsked(t, 1)[0].Prompt.ShortCode _ = c.Close() - if err := <-done; err == nil { - t.Fatal("the login worked") + if err := <-done; err == nil || srv.loginsAfter(t, 1) != 0 { + t.Fatalf("the login worked, %v", err) } if got := eventNames(sb.audit.waitFor(t, 2)); got != "request denied:cancelled" { t.Errorf("audit entries %q", got) @@ -1325,6 +1327,107 @@ func TestAgentCancelsWhenSSHHangsUpAfterMoreBytes(t *testing.T) { } } +// lateAnswer is an HTTP transport in place of the broker's connection. +// For each request, it sends the body to got, waits until the context of +// the request ends, and then returns the answer of answer all the same. +// Go's HTTP client does that with an answer that comes while it closes the +// connection after a cancel: the broker answers a cancelled request with +// 403 at once, and a tap can come at that moment. +type lateAnswer struct { + got chan<- []byte + answer func(body []byte) (int, []byte) +} + +func (l lateAnswer) RoundTrip(r *http.Request) (*http.Response, error) { + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + l.got <- body + select { + case <-r.Context().Done(): + case <-time.After(waitLimit): + return nil, errors.New("the request was not cancelled") + } + status, answer := l.answer(body) + return &http.Response{StatusCode: status, Status: http.StatusText(status), Proto: "HTTP/1.1", ProtoMajor: 1, + ProtoMinor: 1, Header: http.Header{"Content-Type": {api.ContentType}}, Body: io.NopCloser(bytes.NewReader(answer)), + ContentLength: int64(len(answer)), Request: r}, nil +} + +// TestAgentCancelWinsOverALateAnswer: when ssh hangs up while its request +// waits, or the deadline of the request passes, the agent cancels the +// request, and ssh gets no signature, also when an answer of the broker +// comes at that moment. The line names the cancel. (Issue #63: the agent +// wrote "denied", the broker's answer to the cancel, and it would have +// passed on a signature.) For the hang-up, ssh closes only its half for +// writing: the agent sees a hang-up, and the test can still read the +// agent's answer. +func TestAgentCancelWinsOverALateAnswer(t *testing.T) { + t.Parallel() + deploy, _ := newSSHKey(t) + signature := func(body []byte) (int, []byte) { + var req api.SignRequest + if err := req.UnmarshalJSON(body); err != nil { + return http.StatusBadRequest, nil + } + sig, err := deploy.Sign(rand.Reader, req.Data) + if err != nil { + return http.StatusInternalServerError, nil + } + answer, err := json.Marshal(api.SignAnswer{Format: sig.Format, Blob: sig.Blob}) + if err != nil { + return http.StatusInternalServerError, nil + } + return http.StatusOK, answer + } + forbidden := func([]byte) (int, []byte) { return http.StatusForbidden, []byte(`{"error":"Forbidden"}`) } + const hungUp = "nodkey: no signature: ssh closed its connection while the request waited" + for _, tt := range []struct { + name string + answer func(body []byte) (int, []byte) + deadline time.Duration // the deadline of the request; 0: ssh hangs up + line string + }{ + {"a valid signature after a hang-up", signature, 0, hungUp}, + {"403 after a hang-up", forbidden, 0, hungUp}, + {"a valid signature after the deadline", signature, 100 * time.Millisecond, + "nodkey: no signature: no answer from the broker within 100ms"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := make(chan []byte, 1) + b := &Broker{addr: "192.0.2.10:7743", timeout: RequestTimeout, + client: &http.Client{Transport: lateAnswer{got: got, answer: tt.answer}}} + au := startAgent(t, b, agentOptions{signTimeout: tt.deadline}) + c := au.conn(t) + if !bindAgent(t, c, []byte("bind")) { + t.Fatal("the session-bind was refused") + } + if err := writeAgentMessage(c, signRequestMessage(deploy.PublicKey().Marshal(), []byte("data"), 0)); err != nil { + t.Fatal(err) + } + <-got + if tt.deadline == 0 { + uc, ok := c.(*net.UnixConn) + if !ok { + t.Fatal("not a Unix connection") + } + if err := uc.CloseWrite(); err != nil { + t.Fatal(err) + } + } + _ = c.SetReadDeadline(time.Now().Add(waitLimit)) + if answer, err := readAgentMessage(c); err != nil || !bytes.Equal(answer, failureMessage) { + t.Fatalf("after the cancel, the agent answered % x, %v; want SSH_AGENT_FAILURE", answer, err) + } + if got := au.lines(); !slices.Equal(got, []string{tt.line}) { + t.Errorf("the agent wrote %q, want %q", got, tt.line) + } + }) + } +} + // captureSigner records the data that the ssh client of golang.org/x/crypto // asks it to sign, and fails the login. type captureSigner struct { diff --git a/internal/client/sshgrant.go b/internal/client/sshgrant.go index 53de10f..c84017d 100644 --- a/internal/client/sshgrant.go +++ b/internal/client/sshgrant.go @@ -139,6 +139,13 @@ func (g granter) run(ctx context.Context, args []string, w io.Writer) int { ans, err := Post[api.GrantAnswer](ctx, b, api.PathSSHGrant, body, func(code string) { _, _ = fmt.Fprintf(stderr, "nodkey: waiting for approval (request %s)\n", code) }) + if err == nil && ctx.Err() != nil { + // A signal came while the request waited, and Post still returned + // the answer that came at that moment (see Post). As DESIGN.md + // sections 13 and 15 say, the client says cancelled then, and dies + // of the signal, so that a shell loop stops. + err = context.Cause(ctx) + } if err != nil { return requestFailed(ctx, stderr, err) } diff --git a/internal/client/sshgrant_test.go b/internal/client/sshgrant_test.go index af60d97..c76a6cf 100644 --- a/internal/client/sshgrant_test.go +++ b/internal/client/sshgrant_test.go @@ -3,6 +3,7 @@ package client import ( "context" "errors" + "fmt" "io" "net" "net/http" @@ -736,6 +737,38 @@ func TestSSHGrantCancel(t *testing.T) { gb.mustAsk(t, laptop, gb.signRequest(t, "deploy", "web-1", 0, "deploy"), 2) } +// TestSSHGrantCancelWinsOverALateAnswer: a signal comes while nodkey +// ssh-grant waits, and the broker's 200 comes at that moment, which Go's +// HTTP client can still return (lateAnswer). As DESIGN.md sections 13 and +// 15 say, the client says cancelled, with the status of the signal, so +// that main ends nodkey with it and a shell loop stops. (From the review +// of issue #63: it said "allowed until" and exited with 0.) +func TestSSHGrantCancelWinsOverALateAnswer(t *testing.T) { + t.Parallel() + for _, sig := range []syscall.Signal{syscall.SIGINT, syscall.SIGTERM} { + in := interrupted{sig: sig} + t.Run(in.name(), func(t *testing.T) { + t.Parallel() + got := make(chan []byte, 1) + b := &Broker{addr: "192.0.2.10:7743", timeout: RequestTimeout, + client: &http.Client{Transport: lateAnswer{got: got, answer: func([]byte) (int, []byte) { + return http.StatusOK, []byte(`{"minutes":15}`) + }}}} + ctx, cancel := context.WithCancelCause(t.Context()) + defer cancel(nil) + go func() { + <-got + cancel(in) + }() + code, stderr := grantIn(ctx, testGranter(b, nil), grantArgs()...) + want := fmt.Sprintf("nodkey: cancelled by %s (exit %d)", in.name(), in.status()) + if code != in.status() || lastLine(stderr) != want || strings.Contains(stderr, "allowed") { + t.Errorf("exit code %d, stderr %q; want the last line %q", code, stderr, want) + } + }) + } +} + // TestSSHGrantUnreachable: nodkey ssh-grant exits with 69 when it cannot // read its config, cannot reach the broker, or when the other side is not // the broker of broker_pin. It never asks again.