Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,9 @@ NODKEY_REASON="apply the nginx fix" ansible-playbook site.yml
the broker cancels the request (section 7). It sees this also when
`ssh` sent more bytes before: it asks the kernel whether the other
side hung up (`poll` with `POLLRDHUP`), and does not read those
bytes.
bytes. After a hang-up, the client gives `ssh` no signature, also
when an answer of the broker comes at that moment, and its line says
that `ssh` closed its connection.
6. The broker checks, and denies on any failure. It makes these checks
before any message or place in a queue (section 8). A request that fails
them gets no message, and the audit log gets it with the key name, if
Expand Down
58 changes: 48 additions & 10 deletions cmd/nodkey/brokerenv_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,7 @@ func usableTestDir(t *testing.T, dir string) bool {
return false
}
probe := filepath.Join(dir, "probe")
//nolint:gosec // G703: a file in the test's own new directory.
if os.WriteFile(probe, []byte("#!/bin/sh\nexit 0\n"), 0o600) != nil || config.CheckPrivateFile(probe) != nil {
if writeUninherited(probe, []byte("#!/bin/sh\nexit 0\n")) != nil || config.CheckPrivateFile(probe) != nil {
return false
}
if os.Chmod(probe, 0o700) != nil { //nolint:gosec // G302: a program of the test.
Expand All @@ -223,14 +222,30 @@ func usableTestDir(t *testing.T, dir string) bool {
// writeTestFile writes data to path with exactly mode.
func writeTestFile(t *testing.T, path string, data []byte, mode os.FileMode) {
t.Helper()
if err := os.WriteFile(path, data, 0o600); err != nil { //nolint:gosec // G703: a file in the test's own directory.
if err := writeUninherited(path, data); err != nil {
t.Fatal(err)
}
if err := os.Chmod(path, mode); err != nil {
t.Fatal(err)
}
}

// writeUninherited writes data to path with mode 0600, so that no child
// process gets a copy of the file descriptor that writes it. The tests run
// in parallel, and many start child processes. A child gets a copy of each
// open descriptor of the test process and keeps it until it runs its own
// program. While a child keeps a copy of the descriptor that writes a
// program, the start of that program fails with ETXTBSY, "text file busy"
// (go.dev/issue/22315): then the probe of privateTestDir failed, and the
// test failed or was skipped. Each start of a child holds syscall.ForkLock
// for writing until the child runs its program, so no child starts while
// this holds it for reading.
func writeUninherited(path string, data []byte) error {
syscall.ForkLock.RLock()
defer syscall.ForkLock.RUnlock()
return os.WriteFile(path, data, 0o600) //nolint:gosec // G703: a file in the test's own directory.
}

// madeUp returns a made-up value with what at its start, made at run time,
// so the source holds nothing that looks like a token.
func madeUp(what string) string {
Expand Down Expand Up @@ -458,11 +473,13 @@ var (

// freeAddress returns an address on the loopback interface with a port
// that is free now, for the broker of one test. The broker listens on it
// a moment later. So the port is outside the range of ports that the
// kernel gives out by itself, to sockets that ask for any port or connect
// (net.ipv4.ip_local_port_range): no other socket of the tests takes it in
// between. No two tests of this process get the same port. When the
// kernel's range leaves no room, the kernel chooses the port, and
// a moment later, and again when the test starts it again. So the port is
// outside the range of ports that the kernel gives out by itself, to
// sockets that ask for any port or connect (net.ipv4.ip_local_port_range):
// no other socket of the tests takes it in between. No two tests of this
// process get the same port, and no two tests of two processes that run
// at the same time (reservePort), such as the tests of two checkouts. When
// the kernel's range leaves no room, the kernel chooses the port, and
// startListening tries again if another socket takes it first.
func freeAddress(t *testing.T) string {
t.Helper()
Expand All @@ -489,7 +506,7 @@ func freeAddress(t *testing.T) string {
taken := portsTaken[port]
portsTaken[port] = true
portsMu.Unlock()
if taken {
if taken || !reservePort(t, port) {
continue
}
addr := "127.0.0.1:" + strconv.FormatInt(port, 10)
Expand All @@ -515,14 +532,35 @@ func freeAddress(t *testing.T) string {
taken := portsTaken[int64(addr.Port)]
portsTaken[int64(addr.Port)] = true
portsMu.Unlock()
if !taken {
if !taken && reservePort(t, int64(addr.Port)) {
return addr.String()
}
}
t.Fatal("found no free port for the broker")
return ""
}

// reservePort reserves port for the test t against other processes of
// these tests, such as a second go test run at the same time. Until t
// ends, it holds the abstract Unix socket "@nodkey-test-port-PORT", and
// only one socket can hold such a name. It reports false when another
// process holds the name. Without it, such a process could take the port
// while the broker of t was stopped, and the next start of that broker
// failed; or it could listen there, and TestBrokerNoListenBeforeTheAuditLog
// saw a broker listen. The kernel frees the name when the socket closes,
// also when the process ends without cleanup. A port that turns out to be
// in use stays reserved until t ends; that does no harm.
func reservePort(t *testing.T, port int64) bool {
t.Helper()
c, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "unixgram",
"@nodkey-test-port-"+strconv.FormatInt(port, 10))
if err != nil {
return false
}
t.Cleanup(func() { _ = c.Close() })
return true
}

// portsOutside returns the ports [low, high) for the brokers of the tests,
// outside the kernel's own range of ports from lo to hi: the larger of the
// ports from 10000 to below lo, and the ports above hi. high is not more
Expand Down
4 changes: 3 additions & 1 deletion internal/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,9 @@ func (tr *trace) isConnected() bool {
// answer: no other request goes on that connection, so there is no
// pipelining. The request is never sent again. When ctx ends, the
// connection closes, so the broker cancels the request; Post then returns
// context.Cause(ctx).
// context.Cause(ctx). An answer that comes while the connection closes can
// still be returned instead, as Go's HTTP client does: a caller that must
// deny after ctx ends checks ctx after Post.
//
// Every other failure is an *Error: a final answer other than 200, also a
// redirect; a 200 whose body api.ReadAnswer refuses; more than a few
Expand Down
11 changes: 11 additions & 0 deletions internal/client/sshagent.go
Original file line number Diff line number Diff line change
Expand Up @@ -588,6 +588,8 @@ func (c *agentConn) bind(contents []byte) []byte {
// claimed context of the process that connected. It checks the answer
// with checkSignAnswer before it gives the signature to ssh. When ssh
// closes the connection while the request waits, it ends the request.
// After a hang-up, the deadline or a stop of the agent, ssh gets no
// signature, whatever the broker answers then.
func (c *agentConn) sign(ctx context.Context, key, data []byte, flags uint32) []byte {
a := c.a
switch {
Expand Down Expand Up @@ -618,6 +620,15 @@ func (c *agentConn) sign(ctx context.Context, key, data []byte, flags uint32) []
a.log.waitingFor(s)
})
stop()
if cause := context.Cause(waitCtx); cause != nil {
// The request was cancelled: ssh hung up, the deadline passed, or the
// agent stops. Go's HTTP client can still return an answer that came
// while it closed the connection to the broker: the 403 that the
// broker sends when it sees the request cancelled, or a signature
// after a tap at that moment. A cancel means deny: the cause is the
// result, and ssh gets no signature after it.
err = cause
}
if err != nil {
a.log.note("no signature%s: %s", code.label(), failureText(err))
return failureMessage
Expand Down
51 changes: 39 additions & 12 deletions internal/client/sshagent_helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -392,9 +392,11 @@ type sshServer struct {
addr string
hostKey ssh.PublicKey

mu sync.Mutex
kex [][]byte // the session-bind of each key exchange, in order
logins int // the logins that the server accepted
mu sync.Mutex
kex [][]byte // the session-bind of each key exchange, in order
ended int // the connections whose handshake ended, with a login or without
logins int // the logins that the server accepted
changed chan struct{} // closed and made new when a handshake ends
}

// recordingSigner signs with a host key and records each signature.
Expand Down Expand Up @@ -429,7 +431,7 @@ func marshalBind(hostKey ssh.PublicKey, sessionID []byte, sig *ssh.Signature, fo
// address. Only userKey may log in.
func startSSHServer(t *testing.T, host ssh.AlgorithmSigner, userKey ssh.PublicKey) *sshServer {
t.Helper()
s := &sshServer{hostKey: host.PublicKey()}
s := &sshServer{hostKey: host.PublicKey(), changed: make(chan struct{})}
cfg := &ssh.ServerConfig{
PublicKeyCallback: func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
if !bytes.Equal(key.Marshal(), userKey.Marshal()) {
Expand Down Expand Up @@ -471,13 +473,21 @@ func (s *sshServer) serve(nc net.Conn, cfg *ssh.ServerConfig) {
defer func() { _ = nc.Close() }()
_ = nc.SetDeadline(time.Now().Add(2 * waitLimit))
conn, chans, reqs, err := ssh.NewServerConn(nc, cfg)
// NewServerConn sends the client its success before it returns, so the
// client can see its login work before the server counts it here. A
// test waits for the count with loginsAfter.
s.mu.Lock()
s.ended++
if err == nil {
s.logins++
}
close(s.changed)
s.changed = make(chan struct{})
s.mu.Unlock()
if err != nil {
return
}
defer func() { _ = conn.Close() }()
s.mu.Lock()
s.logins++
s.mu.Unlock()
go ssh.DiscardRequests(reqs)
for nch := range chans {
ch, chReqs, err := nch.Accept()
Expand All @@ -504,11 +514,28 @@ func (s *sshServer) binds() [][]byte {
return append([][]byte(nil), s.kex...)
}

// loginCount returns how many logins the server accepted.
func (s *sshServer) loginCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.logins
// loginsAfter waits until the handshake of n connections has ended, with
// a login or without, and returns how many logins the server accepted.
// Only then is the count final: the server answers a login before it
// counts it, and a refused login ends after the client has given up.
func (s *sshServer) loginsAfter(t testing.TB, n int) int {
t.Helper()
deadline := time.NewTimer(waitLimit)
defer deadline.Stop()
for {
s.mu.Lock()
ended, logins, changed := s.ended, s.logins, s.changed
s.mu.Unlock()
if ended >= n {
return logins
}
select {
case <-changed:
case <-deadline.C:
t.Fatalf("the SSH server ended %d handshakes within %s, want %d", ended, waitLimit, n)
return 0
}
}
}

// fakeSSHBroker is a test handler in place of the broker: it lists keys,
Expand Down
17 changes: 10 additions & 7 deletions internal/client/sshagent_openssh_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,10 @@ func TestOpenSSHLogin(t *testing.T) {
}
res := <-done
if tt.answer == audit.AnswerApproveOnce {
if res.err != nil || srv.loginCount() != 1 {
if res.err != nil || srv.loginsAfter(t, 1) != 1 {
t.Fatalf("ssh: %v; agent lines %q\n%s", res.err, au.lines(), res.out)
}
} else if res.err == nil || srv.loginCount() != 0 {
} else if res.err == nil || srv.loginsAfter(t, 1) != 0 {
t.Fatalf("ssh logged in: %v\n%s", res.err, res.out)
}
p := asked.Prompt
Expand Down Expand Up @@ -216,8 +216,11 @@ func TestOpenSSHSignRequest(t *testing.T) {
cmd := exec.CommandContext(ctx, sh, append([]string{"-c", script, sshPath}, args...)...) //nolint:gosec // G204: the test runs ssh through sh.
cmd.Env = sshEnv(au.socket, dir, "NODKEY_REASON=from the shell", "SECRET_OF_THE_SHELL="+canary)
out, err := cmd.CombinedOutput()
if err != nil || srv.loginCount() != 0 {
t.Fatalf("sh: %v; %d logins\n%s", err, srv.loginCount(), out)
if err != nil {
t.Fatalf("sh: %v\n%s", err, out)
}
if n := srv.loginsAfter(t, 1); n != 0 {
t.Fatalf("%d logins\n%s", n, out)
}
var body []byte
select {
Expand Down Expand Up @@ -296,12 +299,12 @@ func TestOpenSSHAgentChecksTheSignature(t *testing.T) {
dir := shortTempDir(t)
out, err := runSSH(t, sshPath, sshArgs(t, srv.addr, "deploy", host.PublicKey(), dir), sshEnv(au.socket, dir))
if tt.want == "" {
if err != nil || srv.loginCount() != 1 {
if err != nil || srv.loginsAfter(t, 1) != 1 {
t.Fatalf("ssh: %v; agent lines %q\n%s", err, au.lines(), out)
}
return
}
if err == nil || srv.loginCount() != 0 {
if err == nil || srv.loginsAfter(t, 1) != 0 {
t.Fatalf("ssh logged in\n%s", out)
}
if got := au.lines(); len(got) != 1 || got[0] != "nodkey: no signature: "+tt.want {
Expand All @@ -326,7 +329,7 @@ func TestOpenSSHAgentChecksTheSignature(t *testing.T) {
au := startAgent(t, b, agentOptions{})
dir := shortTempDir(t)
out, err := runSSH(t, sshPath, sshArgs(t, srv.addr, "deploy", host.PublicKey(), dir), sshEnv(au.socket, dir))
if err == nil || srv.loginCount() != 0 {
if err == nil || srv.loginsAfter(t, 1) != 0 {
t.Fatalf("ssh logged in\n%s", out)
}
// Another test may take the free port at once: then the TLS
Expand Down
Loading
Loading