Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-proxy-tunnel-destination-policy.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The agent proxy now applies its plain-HTTP destination policy to uninspected CONNECT and transparent-TLS tunnels as well, so all three egress legs dial only globally routable addresses; `HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` lifts it for every leg at once
9 changes: 5 additions & 4 deletions src/crux/proxy/github_proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,10 @@ type GitHubProxy struct {

// allowPrivateForward snapshots HIVE_PROXY_ALLOW_PRIVATE_FORWARD at
// construction (a boot-time deployment choice like proxyAdvisoryOK). The
// zero value keeps the plain-HTTP forward guard ON: non-routable
// destinations (loopback, link-local/metadata, RFC 1918, CGNAT, multicast)
// are refused on the plain path. See plain_forward_guard.go.
// zero value keeps the destination guard ON: non-routable destinations
// (loopback, link-local/metadata, RFC 1918, CGNAT, multicast) are refused
// on the plain-HTTP forward path and on uninspected CONNECT/transparent
// tunnels. See plain_forward_guard.go.
allowPrivateForward bool

// injectGHAuth snapshots config.ProxyInjectGHAuth() at construction (#1861,
Expand Down Expand Up @@ -365,7 +366,7 @@ func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logg
// dial the private estate on an agent's behalf.
allowPrivateForward := plainForwardAllowPrivate()
if allowPrivateForward {
logger.Warn("proxy: plain-HTTP forwards to non-routable destinations are ALLOWED — agents can reach loopback, link-local (cloud metadata) and private-network services through this proxy",
logger.Warn("proxy: plain-HTTP forwards and uninspected tunnels to non-routable destinations are ALLOWED — agents can reach loopback, link-local (cloud metadata) and private-network services through this proxy",
"env", plainForwardAllowPrivateEnv)
}

Expand Down
43 changes: 39 additions & 4 deletions src/crux/proxy/github_proxy_connect.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,12 @@ func (p *GitHubProxy) handleTransparentTLS(conn net.Conn, peeked []byte) {
// so the forced-egress redirect exempts this proxy-originated dial.
upstream, err := p.dialTunnelUpstream(host)
if err != nil {
var denied *plainForwardDenied
if errors.As(err, &denied) && denied.reason != plainForwardReasonLoopback {
p.logger.Warn("proxy: transparent tunnel refused — destination is not globally routable",
"agent", agentName, "host", host, "target", denied.target, "class", denied.reason)
p.countViolation(agentName)
}
return
}
defer func() { _ = upstream.Close() }()
Expand Down Expand Up @@ -268,12 +274,15 @@ func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string {

// dialTunnelUpstream opens the raw TCP leg of a non-inspected transparent
// tunnel. The socket is SO_MARKed so the forced-egress redirect exempts this
// proxy-originated dial. tunnelDial, when set, overrides it (tests).
// proxy-originated dial, and the destination is guarded like every other
// uninspected egress leg (see plain_forward_guard.go): the host here comes
// from the client's SNI, which names whatever the client chose. tunnelDial,
// when set, overrides it (tests).
func (p *GitHubProxy) dialTunnelUpstream(host string) (net.Conn, error) {
if p.tunnelDial != nil {
return p.tunnelDial(host)
}
return markDialer(transparentProxyTimeout).Dial("tcp", host+":443")
return p.tunnelDialer(transparentProxyTimeout).Dial("tcp", host+":443")
}

const tlsClientHelloMaxSize = 4096
Expand Down Expand Up @@ -598,9 +607,17 @@ func (p *GitHubProxy) logClientRead(msg string, err error, attrs ...any) {
func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) {
const tunnelDialTimeout = 10 * time.Second
// SO_MARK the socket so the forced-egress redirect exempts this
// proxy-originated tunnel dial (CONNECT targets include :443).
upstream, err := markDialer(tunnelDialTimeout).Dial("tcp", r.Host)
// proxy-originated tunnel dial (CONNECT targets include :443). The dialer
// is destination-guarded like the plain forward path: a CONNECT names an
// arbitrary host:port, and without the guard it was a raw TCP tunnel from
// the proxy's uid to the cloud metadata service or the cluster estate.
upstream, err := p.tunnelDialer(tunnelDialTimeout).Dial("tcp", r.Host)
if err != nil {
var denied *plainForwardDenied
if errors.As(err, &denied) {
p.refuseTunnel(conn, r, p.identifyAgentFromConn(conn, r), denied)
return
}
p.logger.Warn("proxy: CONNECT dial failed", "host", r.Host, "error", err)
_, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nconnection failed\n")
return
Expand All @@ -612,6 +629,24 @@ func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) {
relayTunnel(conn, upstream)
}

// refuseTunnel answers a CONNECT whose destination the guard barred, with the
// same loopback-vs-estate split as refusePlainForward: loopback is a client
// that ignored NO_PROXY and is logged at Debug without a violation; every
// other class is counted against the caller.
func (p *GitHubProxy) refuseTunnel(conn net.Conn, r *http.Request, agentName string, denied *plainForwardDenied) {
if denied.reason == plainForwardReasonLoopback {
p.logger.Debug("proxy: CONNECT to loopback refused (clients must bypass the proxy for loopback; NO_PROXY)",
"agent", agentName, "host", r.Host, "target", denied.target)
} else {
p.logger.Warn("proxy: CONNECT refused — destination is not globally routable",
"agent", agentName, "host", r.Host, "target", denied.target, "class", denied.reason)
p.countViolation(agentName)
}
_, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+
"the hive proxy does not tunnel to %s addresses (%s); set %s=true to allow it\n",
denied.reason, denied.target, plainForwardAllowPrivateEnv)
}

// forwardPlainDirect handles non-CONNECT (plain HTTP) requests on a raw
// connection.
//
Expand Down
62 changes: 46 additions & 16 deletions src/crux/proxy/plain_forward_guard.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,22 @@ import (
"os"
"strings"
"syscall"
"time"
)

// The plain-HTTP (non-CONNECT) forward path is the one egress path through
// The plain-HTTP (non-CONNECT) forward path and the uninspected tunnel paths
// (a CONNECT to a host the proxy neither inspects nor intercepts, and the
// transparent-TLS passthrough for such a host) are the egress paths through
// this proxy that carried NO destination policy at all: handleConn hands every
// non-CONNECT request to forwardPlainDirect, which RoundTripped it unchanged.
// An agent that sets a proxy URL by hand — or simply asks its CLI for
// `http://<anything>` — therefore had the proxy open a connection on its
// behalf, from the PROXY's uid, to any address the agent named. The forced
// egress gate does not cover it: the iptables rule only redirects :443, so
// plain HTTP is reachable only through this path, and the proxy's own uid is
// exempted from the redirect precisely so its upstream dials work.
// non-CONNECT request to forwardPlainDirect, which RoundTripped it unchanged,
// and tunnelDirect dialled whatever host:port the CONNECT line named. An agent
// that sets a proxy URL by hand — or simply asks its CLI for
// `http://<anything>`, or sends `CONNECT 169.254.169.254:80` — therefore had
// the proxy open a connection on its behalf, from the PROXY's uid, to any
// address the agent named. The forced egress gate does not cover it: the
// iptables rule only redirects :443, so plain HTTP is reachable only through
// this path, and the proxy's own uid is exempted from the redirect precisely
// so its upstream dials work.
//
// The interesting targets are not on the public internet, where the agent's
// own egress would have reached anyway. They are the addresses only the pod
Expand All @@ -29,7 +34,8 @@ import (
// (node credentials on most clouds), and the cluster-internal RFC1918 estate
// (the Kubernetes API, other spokes, internal registries).
//
// So the plain forward path dials only globally routable addresses. The check
// So the plain forward and uninspected tunnel paths dial only globally
// routable addresses. The check
// runs in the dialer's Control hook rather than on the request's host string:
// Control is called AFTER name resolution with the concrete remote address, so
// a hostname that resolves to 127.0.0.1 or 169.254.169.254 — a public DNS
Expand Down Expand Up @@ -132,6 +138,36 @@ var (
unguardedPlainForwardTransport = newPlainForwardTransport(false)
)

// guardedMarkDialer is markDialer with the destination check layered into the
// Control hook, so the address refused is the resolved one the kernel is about
// to connect to. It is the single dialer both the plain forward transport and
// the uninspected tunnel paths build on; keep the policy here so the two
// paths cannot drift apart.
func guardedMarkDialer(timeout time.Duration) *net.Dialer {
dialer := markDialer(timeout)
mark := dialer.Control
dialer.Control = func(network, address string, c syscall.RawConn) error {
if reason := plainForwardDenyReason(address); reason != "" {
return &plainForwardDenied{target: address, reason: reason}
}
return mark(network, address, c)
}
return dialer
}

// tunnelDialer returns the dialer an uninspected tunnel (CONNECT passthrough
// or transparent-TLS passthrough) opens its upstream leg with: guarded unless
// the deployment lifted the guard with HIVE_PROXY_ALLOW_PRIVATE_FORWARD, the
// same boot-time choice the plain forward path honours. The zero value of
// allowPrivateForward is guarded, so a struct-literal GitHubProxy is
// protected without opting in.
func (p *GitHubProxy) tunnelDialer(timeout time.Duration) *net.Dialer {
if p.allowPrivateForward {
return markDialer(timeout)
}
return guardedMarkDialer(timeout)
}

// newPlainForwardTransport clones the standard transport (keeping its
// connection pooling and timeouts), drops its ProxyFromEnvironment so an
// HTTP_PROXY/HTTPS_PROXY on the proxy process is deliberately ignored — with
Expand All @@ -146,13 +182,7 @@ func newPlainForwardTransport(guard bool) *http.Transport {
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
dialer := markDialer(upstreamDialTimeout)
if guard {
mark := dialer.Control
dialer.Control = func(network, address string, c syscall.RawConn) error {
if reason := plainForwardDenyReason(address); reason != "" {
return &plainForwardDenied{target: address, reason: reason}
}
return mark(network, address, c)
}
dialer = guardedMarkDialer(upstreamDialTimeout)
}
return dialer.DialContext(ctx, network, address)
}
Expand Down
2 changes: 2 additions & 0 deletions src/crux/proxy/proxy_deep2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,7 @@ func TestProxyHTTPGraphQLMutationAllowed(t *testing.T) {

func TestHandleConnectDirectAnthropicNoRoute(t *testing.T) {
p := newTestProxy()
p.allowPrivateForward = true // the tunnel target under test is a loopback listener
p.inference = newInferenceRouter()
// No route set for any agent

Expand Down Expand Up @@ -641,6 +642,7 @@ func TestProxyHTTPUpstreamWriteError(t *testing.T) {

func TestTunnelDirectSuccess(t *testing.T) {
p := newTestProxy()
p.allowPrivateForward = true // the tunnel target under test is a loopback listener

// Create a local TCP server
ln, err := net.Listen("tcp", "127.0.0.1:0")
Expand Down
2 changes: 2 additions & 0 deletions src/crux/proxy/proxy_deep_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@ func TestUpdateMaxContextLenStaleGuard(t *testing.T) {

func TestHandleConnCONNECTNonGitHub(t *testing.T) {
p := newTestProxy()
p.allowPrivateForward = true // the tunnel target under test is a loopback listener

// Start a mock TCP server to tunnel to
ln, err := net.Listen("tcp", "127.0.0.1:0")
Expand Down Expand Up @@ -957,6 +958,7 @@ func TestExtractTextFromContentInvalid(t *testing.T) {

func TestHandleConnectDirectNonGitHub(t *testing.T) {
p := newTestProxy()
p.allowPrivateForward = true // the tunnel target under test is a loopback listener
p.inference = newInferenceRouter()

// Create a local TCP server to tunnel to
Expand Down
149 changes: 149 additions & 0 deletions src/crux/proxy/tunnel_destination_guard_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
package proxy

import (
"bufio"
"errors"
"io"
"net"
"net/http"
"strings"
"testing"
"time"
)

// connectRoundTrip drives handleConnectDirect over a pipe for a CONNECT to
// target and returns the response the agent-side client would see.
func connectRoundTrip(t *testing.T, p *GitHubProxy, target string) *http.Response {
t.Helper()
req, err := http.NewRequest(http.MethodConnect, "http://"+target, nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Host = target
req.Header.Set("Proxy-Authorization", "hive scanner")

clientConn, proxyClient := net.Pipe()
t.Cleanup(func() { _ = clientConn.Close() })

go func() {
p.handleConnectDirect(proxyClient, req)
_ = proxyClient.Close()
}()

_ = clientConn.SetReadDeadline(time.Now().Add(10 * time.Second))
resp, err := http.ReadResponse(bufio.NewReader(clientConn), req)
if err != nil {
t.Fatalf("read response: %v", err)
}
t.Cleanup(func() { _ = resp.Body.Close() })
return resp
}

// TestTunnelDirectRefusesNonRoutableDestination pins that the uninspected
// CONNECT path applies the same destination policy as the plain-HTTP forward:
// a CONNECT to the cloud metadata address or the RFC 1918 estate is answered
// 403 and counted against the caller, not tunnelled from the proxy's uid.
func TestTunnelDirectRefusesNonRoutableDestination(t *testing.T) {
p := newTestProxy()
p.proxyAdvisoryOK = true // no UID map in tests; accept the header identity

for _, target := range []string{"169.254.169.254:80", "10.255.255.1:6443", "100.64.0.1:443"} {
resp := connectRoundTrip(t, p, target)
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("CONNECT %s: status = %d, want 403", target, resp.StatusCode)
}
body, _ := io.ReadAll(resp.Body)
if !strings.Contains(string(body), plainForwardAllowPrivateEnv) {
t.Errorf("CONNECT %s: body %q should name the override env var", target, body)
}
}
if got := p.violations["scanner"]; got != 3 {
t.Errorf("violations[scanner] = %d, want 3 — each estate-bound CONNECT must be attributed and counted", got)
}
}

// TestTunnelDirectLoopbackRefusalIsNotAViolation mirrors the plain path: a
// loopback CONNECT is a client that ignored NO_PROXY, refused but not held
// against the caller.
func TestTunnelDirectLoopbackRefusalIsNotAViolation(t *testing.T) {
p := newTestProxy()
p.proxyAdvisoryOK = true

if resp := connectRoundTrip(t, p, "127.0.0.1:9"); resp.StatusCode != http.StatusForbidden {
t.Fatalf("status = %d, want 403 for a loopback CONNECT", resp.StatusCode)
}
if got := p.violations["scanner"]; got != 0 {
t.Errorf("violations[scanner] = %d, want 0 — loopback is a misconfiguration, not a violation", got)
}
}

// TestTunnelDirectAllowPrivateForwardLiftsGuard pins that the same boot-time
// override that lifts the plain-forward guard lifts the tunnel guard, so a
// deployment that opted in keeps working as before.
func TestTunnelDirectAllowPrivateForwardLiftsGuard(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer func() { _ = ln.Close() }()
go func() {
conn, err := ln.Accept()
if err != nil {
return
}
_ = conn.Close()
}()

p := newTestProxy()
p.allowPrivateForward = true

if resp := connectRoundTrip(t, p, ln.Addr().String()); resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want 200 with the guard lifted", resp.StatusCode)
}
}

// TestDialTunnelUpstreamRefusesNonRoutableSNI covers the transparent-TLS
// passthrough leg: the host there comes from the client's SNI, so an SNI that
// names a private address must be refused on the same evidence.
func TestDialTunnelUpstreamRefusesNonRoutableSNI(t *testing.T) {
p := newTestProxy()

_, err := p.dialTunnelUpstream("10.255.255.1")
var denied *plainForwardDenied
if !errors.As(err, &denied) {
t.Fatalf("err = %v, want *plainForwardDenied", err)
}
if denied.reason != plainForwardReasonPrivate {
t.Errorf("reason = %q, want %q", denied.reason, plainForwardReasonPrivate)
}
}

// TestHandleTransparentTLSRefusalCountsEstateNotLoopback pins the handler-level
// split in handleTransparentTLS: a refused estate-bound SNI is counted against
// the caller, a refused loopback SNI is not. Over net.Pipe attribution yields
// "", so the counter lands under that key.
func TestHandleTransparentTLSRefusalCountsEstateNotLoopback(t *testing.T) {
for _, tc := range []struct {
sni string
want int
}{{"10.255.255.1", 1}, {"127.0.0.1", 0}} {
p := newTestProxy()
clientHello := buildClientHello(tc.sni)
clientConn, proxyConn := net.Pipe()
done := make(chan struct{})
go func() {
defer close(done)
p.handleTransparentTLS(proxyConn, clientHello[:1])
_ = proxyConn.Close()
}()
_, _ = clientConn.Write(clientHello[1:])
<-done
_ = clientConn.Close()
p.mu.Lock()
got := p.violations[""]
p.mu.Unlock()
if got != tc.want {
t.Errorf("SNI %s: violations[\"\"] = %d, want %d", tc.sni, got, tc.want)
}
}
}
Loading