diff --git a/examples/sds-test-server/Dockerfile b/examples/sds-test-server/Dockerfile new file mode 100644 index 0000000000..6f1cadb9fb --- /dev/null +++ b/examples/sds-test-server/Dockerfile @@ -0,0 +1,20 @@ +FROM golang:1.26.5@sha256:63f132d58c1f589f0dcda584933a9bb44bfda1150f1506377f5a902f34d86033 AS builder + +ARG GO_LDFLAGS="" + +WORKDIR /workspace +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY . ./ +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 \ + GOOS=${TARGETOS} \ + GOARCH=${TARGETARCH} \ + go build -o /bin/sds-test-server -ldflags "${GO_LDFLAGS}" . + +FROM gcr.io/distroless/static-debian11 +COPY --from=builder /bin/sds-test-server / + +ENTRYPOINT ["/sds-test-server"] diff --git a/examples/sds-test-server/Makefile b/examples/sds-test-server/Makefile new file mode 100644 index 0000000000..2d1887ae6e --- /dev/null +++ b/examples/sds-test-server/Makefile @@ -0,0 +1,9 @@ + +IMAGE_PREFIX ?= envoyproxy/gateway- +APP_NAME ?= sds-test-server +TAG ?= latest +GO_LDFLAGS ?= + +.PHONY: docker-buildx +docker-buildx: + docker buildx build . -t $(IMAGE_PREFIX)$(APP_NAME):$(TAG) --build-arg GO_LDFLAGS="$(GO_LDFLAGS)" --load diff --git a/examples/sds-test-server/go.mod b/examples/sds-test-server/go.mod new file mode 100644 index 0000000000..7567808d3c --- /dev/null +++ b/examples/sds-test-server/go.mod @@ -0,0 +1,28 @@ +module github.com/envoyproxy/gateway-sds-test-server + +go 1.26.5 + +require ( + github.com/envoyproxy/go-control-plane v0.14.1-0.20260409050421-3f47accd6e14 + github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260627225610-70ff85c381ff + github.com/stretchr/testify v1.11.1 + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af +) + +require ( + cel.dev/expr v0.25.1 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/examples/sds-test-server/go.sum b/examples/sds-test-server/go.sum new file mode 100644 index 0000000000..de4bf30098 --- /dev/null +++ b/examples/sds-test-server/go.sum @@ -0,0 +1,74 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.14.1-0.20260409050421-3f47accd6e14 h1:7g8SJv4OrVcLT4yfkzIbsTcwLBwyLu8gKb/yCf3Loxk= +github.com/envoyproxy/go-control-plane v0.14.1-0.20260409050421-3f47accd6e14/go.mod h1:18SVzvkoF8AL2O7baVikhojMZ+7rFPh3o8tOOsBVyok= +github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260627225610-70ff85c381ff h1:stwP9x94QfAFs+RF+YFkSrSuTxBuVrj6Sv+PJXJkXzo= +github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260627225610-70ff85c381ff/go.mod h1:RgJXVdNtBhId0AeGnDEqPRSejRMoz//JumYvSTcJTvM= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/sds-test-server/main.go b/examples/sds-test-server/main.go new file mode 100644 index 0000000000..c92a7e6c34 --- /dev/null +++ b/examples/sds-test-server/main.go @@ -0,0 +1,143 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log" + "net" + "os" + "os/signal" + "syscall" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + secretservicev3 "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + cachetypes "github.com/envoyproxy/go-control-plane/pkg/cache/types" + cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" + logv3 "github.com/envoyproxy/go-control-plane/pkg/log" + resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + serverv3 "github.com/envoyproxy/go-control-plane/pkg/server/v3" + "google.golang.org/grpc" +) + +const snapshotKey = "sds-test-server" + +type config struct { + socketPath string + secretName string + certificatePath string + privateKeyPath string +} + +type staticNodeHash struct{} + +func (staticNodeHash) ID(*corev3.Node) string { + return snapshotKey +} + +func main() { + var cfg config + flag.StringVar(&cfg.socketPath, "socket-path", "", "path to the SDS Unix domain socket") + flag.StringVar(&cfg.secretName, "secret-name", "", "name of the SDS TLS secret") + flag.StringVar(&cfg.certificatePath, "cert-path", "", "path to the PEM certificate chain") + flag.StringVar(&cfg.privateKeyPath, "key-path", "", "path to the PEM private key") + flag.Parse() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + if err := run(ctx, cfg); err != nil { + log.Fatalf("SDS server failed: %v", err) + } +} + +func run(ctx context.Context, cfg config) error { + grpcServer, listener, err := newSDSServer(ctx, cfg) + if err != nil { + return err + } + defer func() { + _ = listener.Close() + _ = os.Remove(cfg.socketPath) + }() + + serveErr := make(chan error, 1) + go func() { + serveErr <- grpcServer.Serve(listener) + }() + + select { + case <-ctx.Done(): + grpcServer.Stop() + return nil + case err := <-serveErr: + return fmt.Errorf("serve SDS requests: %w", err) + } +} + +func newSDSServer(ctx context.Context, cfg config) (*grpc.Server, net.Listener, error) { + certificate, err := os.ReadFile(cfg.certificatePath) + if err != nil { + return nil, nil, fmt.Errorf("read certificate: %w", err) + } + privateKey, err := os.ReadFile(cfg.privateKeyPath) + if err != nil { + return nil, nil, fmt.Errorf("read private key: %w", err) + } + + secret := &tlsv3.Secret{ + Name: cfg.secretName, + Type: &tlsv3.Secret_TlsCertificate{ + TlsCertificate: &tlsv3.TlsCertificate{ + CertificateChain: inlineBytes(certificate), + PrivateKey: inlineBytes(privateKey), + }, + }, + } + snapshot, err := cachev3.NewSnapshot("1", map[resourcev3.Type][]cachetypes.Resource{ + resourcev3.SecretType: {secret}, + }) + if err != nil { + return nil, nil, fmt.Errorf("create SDS snapshot: %w", err) + } + if err := snapshot.Consistent(); err != nil { + return nil, nil, fmt.Errorf("validate SDS snapshot: %w", err) + } + + snapshotCache := cachev3.NewSnapshotCache(false, staticNodeHash{}, logv3.NewDefaultLogger()) + if err := snapshotCache.SetSnapshot(ctx, snapshotKey, snapshot); err != nil { + return nil, nil, fmt.Errorf("store SDS snapshot: %w", err) + } + + if err := os.Remove(cfg.socketPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, nil, fmt.Errorf("remove stale socket: %w", err) + } + listener, err := net.Listen("unix", cfg.socketPath) + if err != nil { + return nil, nil, fmt.Errorf("listen on SDS socket: %w", err) + } + if err := os.Chmod(cfg.socketPath, 0o666); err != nil { + _ = listener.Close() + return nil, nil, fmt.Errorf("set SDS socket permissions: %w", err) + } + + grpcServer := grpc.NewServer() + secretservicev3.RegisterSecretDiscoveryServiceServer( + grpcServer, + serverv3.NewServer(ctx, snapshotCache, nil), + ) + return grpcServer, listener, nil +} + +func inlineBytes(data []byte) *corev3.DataSource { + return &corev3.DataSource{ + Specifier: &corev3.DataSource_InlineBytes{InlineBytes: data}, + } +} diff --git a/examples/sds-test-server/main_test.go b/examples/sds-test-server/main_test.go new file mode 100644 index 0000000000..f0fcc2d9b8 --- /dev/null +++ b/examples/sds-test-server/main_test.go @@ -0,0 +1,81 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package main + +import ( + "context" + "net" + "os" + "path/filepath" + "testing" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + secretservicev3 "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/proto" +) + +func TestSDSServerServesTLSSecretOverUnixSocket(t *testing.T) { + // Given a server configured with certificate material on disk. + tempDir := t.TempDir() + socketFile, err := os.CreateTemp("", "sds-*.sock") + require.NoError(t, err) + socketPath := socketFile.Name() + require.NoError(t, socketFile.Close()) + require.NoError(t, os.Remove(socketPath)) + t.Cleanup(func() { _ = os.Remove(socketPath) }) + certificate := []byte("test certificate") + privateKey := []byte("test private key") + certificatePath := filepath.Join(tempDir, "tls.crt") + privateKeyPath := filepath.Join(tempDir, "tls.key") + require.NoError(t, os.WriteFile(certificatePath, certificate, 0o600)) + require.NoError(t, os.WriteFile(privateKeyPath, privateKey, 0o600)) + cfg := config{ + socketPath: socketPath, + secretName: "listener-certificate", + certificatePath: certificatePath, + privateKeyPath: privateKeyPath, + } + grpcServer, listener, err := newSDSServer(t.Context(), cfg) + require.NoError(t, err) + t.Cleanup(grpcServer.Stop) + t.Cleanup(func() { require.NoError(t, listener.Close()) }) + go func() { _ = grpcServer.Serve(listener) }() + + connection, err := grpc.NewClient( + "passthrough:///sds-test-server", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", cfg.socketPath) + }), + ) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, connection.Close()) }) + stream, err := secretservicev3.NewSecretDiscoveryServiceClient(connection).StreamSecrets(t.Context()) + require.NoError(t, err) + + // When the listener certificate is requested through the SDS stream. + require.NoError(t, stream.Send(&discoveryv3.DiscoveryRequest{ + Node: &corev3.Node{Id: "envoy-test-node"}, + ResourceNames: []string{cfg.secretName}, + TypeUrl: resourcev3.SecretType, + })) + response, err := stream.Recv() + require.NoError(t, err) + + // Then the named secret contains the configured certificate and key. + require.Len(t, response.Resources, 1) + secret := &tlsv3.Secret{} + require.NoError(t, response.Resources[0].UnmarshalTo(secret)) + require.Equal(t, cfg.secretName, secret.Name) + require.True(t, proto.Equal(inlineBytes(certificate), secret.GetTlsCertificate().CertificateChain)) + require.True(t, proto.Equal(inlineBytes(privateKey), secret.GetTlsCertificate().PrivateKey)) +} diff --git a/internal/gatewayapi/helpers.go b/internal/gatewayapi/helpers.go index 006062e8c0..a32636b32f 100644 --- a/internal/gatewayapi/helpers.go +++ b/internal/gatewayapi/helpers.go @@ -590,8 +590,7 @@ func irTLSConfigs(config *ListenerTLSConfig) *ir.TLSConfig { Certificates: make([]ir.TLSCertificate, len(config.secrets)), } for i, tlsSecret := range config.secrets { - cert := getTLSCertificateFromSecret(tlsSecret) - tlsListenerConfigs.Certificates[i] = cert + tlsListenerConfigs.Certificates[i] = getTLSCertificateFromSecret(tlsSecret) } if config.frontendTLSValidation != nil && config.frontendTLSValidation.ValidateError == nil { @@ -629,6 +628,11 @@ func isValidClientCertificateRef(tlsSecret *corev1.Secret) bool { } func getTLSCertificateFromSecret(tlsSecret *corev1.Secret) ir.TLSCertificate { + if tlsSecret.Type == egv1a1.SDSSecretType { + sdsConfig, _ := ir.NewSDSConfig(tlsSecret) + return ir.TLSCertificate{Name: irTLSListenerConfigName(tlsSecret), SDS: sdsConfig} + } + cert := ir.TLSCertificate{ Name: irTLSListenerConfigName(tlsSecret), Certificate: tlsSecret.Data[corev1.TLSCertKey], diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go index b7f6ca661c..5ef92ea7f0 100644 --- a/internal/gatewayapi/listener.go +++ b/internal/gatewayapi/listener.go @@ -34,6 +34,8 @@ import ( var _ ListenersTranslator = (*Translator)(nil) +const sdsCertificateOpaqueConditionMessage = "HTTP/2 is disabled by default because one or more HTTPS listeners on this port use an SDS-backed certificate whose DNS names cannot be inspected. Configure ALPN explicitly with ClientTrafficPolicy to override this default." + type ListenersTranslator interface { ProcessListeners(gateways []*GatewayContext, xdsIR resource.XdsIRMap, infraIR resource.InfraIRMap, resources *resource.Resources) } @@ -338,6 +340,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR resource containerPort := t.servicePortToContainerPort(listener.Port, gateway.envoyProxy) switch listener.Protocol { case gwapiv1.HTTPProtocolType, gwapiv1.HTTPSProtocolType: + tlsConfig := irTLSConfigs(&listener.tls) irListener := &ir.HTTPListener{ CoreListenerDetails: ir.CoreListenerDetails{ Name: irListenerName(listener), @@ -347,7 +350,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR resource Metadata: buildListenerMetadata(listener, gateway), IPFamily: ipFamily, }, - TLS: irTLSConfigs(&listener.tls), + TLS: tlsConfig, Path: ir.PathSettings{ MergeSlashes: true, EscapedSlashesAction: ir.UnescapeAndRedirect, @@ -368,6 +371,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR resource // Store the HTTPListener IR in the listener context for use in the overlapping TLS config check. listener.httpIR = irListener case gwapiv1.TCPProtocolType, gwapiv1.TLSProtocolType: + tlsConfig := irTLSConfigsForTCPListener(&listener.tls) irListener := &ir.TCPListener{ CoreListenerDetails: ir.CoreListenerDetails{ Name: irListenerName(listener), @@ -382,7 +386,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR resource // TLS field should be added to TCPListener as ClientTrafficPolicy will affect // Listener TLS. Then TCPRoute whose TLS should be configured as Terminate just // refers to the Listener TLS. - TLS: irTLSConfigsForTCPListener(&listener.tls), + TLS: tlsConfig, } xdsIR[irKey].TCP = append(xdsIR[irKey].TCP, irListener) case gwapiv1.UDPProtocolType: @@ -536,6 +540,61 @@ func checkOverlappingHostnames(httpsListeners []*ListenerContext) { // checkOverlappingCertificates checks for overlapping certificates SANs between HTTPSlisteners and sets // the `OverlappingTLSConfig` condition if there are overlapping certificates. func checkOverlappingCertificates(httpsListeners []*ListenerContext) { + // Envoy Gateway cannot inspect certificates served over SDS. When multiple + // valid listeners share a port, disable HTTP/2 on SDS-backed listeners and on + // peers whose known certificate SANs may overlap the SDS listener hostname. A + // nil SDS listener hostname matches every peer because it accepts all hostnames. + validListenerCountByPort := make(map[gwapiv1.PortNumber]int) + sdsListenersByPort := make(map[gwapiv1.PortNumber][]*ListenerContext) + for _, listener := range httpsListeners { + if hasInvalidCondition(listener) { + continue + } + validListenerCountByPort[listener.Port]++ + + for _, secret := range listener.tls.secrets { + if secret.Type == egv1a1.SDSSecretType { + sdsListenersByPort[listener.Port] = append(sdsListenersByPort[listener.Port], listener) + break + } + } + } + + for _, listener := range httpsListeners { + if hasInvalidCondition(listener) || validListenerCountByPort[listener.Port] < 2 { + continue + } + + disableHTTP2 := false + for _, sdsListener := range sdsListenersByPort[listener.Port] { + if listener == sdsListener || sdsListener.Hostname == nil { + disableHTTP2 = true + break + } + for _, dnsName := range listener.tls.certDNSNames { + if areOverlappingHostnames(sdsListener.Hostname, new(gwapiv1.Hostname(dnsName))) { + disableHTTP2 = true + break + } + } + if disableHTTP2 { + break + } + } + if !disableHTTP2 { + continue + } + if listener.httpIR != nil { + listener.httpIR.TLSOverlaps = true + } + listener.SetCondition( + gwapiv1.ListenerConditionOverlappingTLSConfig, + metav1.ConditionTrue, + status.ListenerReasonSDSCertificateOpaque, + sdsCertificateOpaqueConditionMessage, + ) + } + type overlappingListener struct { gateway1 *GatewayContext gateway2 *GatewayContext diff --git a/internal/gatewayapi/listener_test.go b/internal/gatewayapi/listener_test.go index 13005bcaf7..8d2edcec8b 100644 --- a/internal/gatewayapi/listener_test.go +++ b/internal/gatewayapi/listener_test.go @@ -477,9 +477,11 @@ func TestCheckOverlappingHostnames(t *testing.T) { func TestCheckOverlappingCertificates(t *testing.T) { tests := []struct { - name string - listeners []*ListenerContext - expectedStatus []expectedListenerStatus + name string + listeners []*ListenerContext + invalidListeners []string + expectedStatus []expectedListenerStatus + expectedTLSOverlaps []string }{ { name: "No overlapping certificates", @@ -551,6 +553,7 @@ func TestCheckOverlappingCertificates(t *testing.T) { message: "The certificate SAN foo.example.com overlaps with the certificate SAN foo.example.com in listener listener-1. ALPN will default to HTTP/1.1 to prevent HTTP/2 connection coalescing, unless explicitly configured via ClientTrafficPolicy", }, }, + expectedTLSOverlaps: []string{"listener-1", "listener-2"}, }, { name: "Overlapping certificates with different ports", @@ -622,6 +625,7 @@ func TestCheckOverlappingCertificates(t *testing.T) { message: "The certificate SAN foo.example.com overlaps with the certificate SAN *.example.com in listener listener-1. ALPN will default to HTTP/1.1 to prevent HTTP/2 connection coalescing, unless explicitly configured via ClientTrafficPolicy", }, }, + expectedTLSOverlaps: []string{"listener-1", "listener-2"}, }, { name: "Overlapping certificates with multiple dns names", @@ -665,6 +669,143 @@ func TestCheckOverlappingCertificates(t *testing.T) { message: "The certificate SAN *.example.org overlaps with the certificate SAN bar.example.org in listener listener-1. ALPN will default to HTTP/1.1 to prevent HTTP/2 connection coalescing, unless explicitly configured via ClientTrafficPolicy", }, }, + expectedTLSOverlaps: []string{"listener-1", "listener-2"}, + }, + { + name: "SDS certificate leaves distinct-hostname peers on HTTP/2", + listeners: []*ListenerContext{ + { + Listener: &gwapiv1.Listener{Name: "listener-1", Protocol: gwapiv1.HTTPSProtocolType, Port: 443, Hostname: new(gwapiv1.Hostname("foo.example.com"))}, + tls: ListenerTLSConfig{ + certDNSNames: []string{"foo.example.com"}, + }, + }, + { + Listener: &gwapiv1.Listener{Name: "listener-2", Protocol: gwapiv1.HTTPSProtocolType, Port: 443, Hostname: new(gwapiv1.Hostname("sds.example.com"))}, + tls: ListenerTLSConfig{ + secrets: []*corev1.Secret{{Type: egv1a1.SDSSecretType}}, + }, + }, + { + Listener: &gwapiv1.Listener{Name: "listener-3", Protocol: gwapiv1.HTTPSProtocolType, Port: 443, Hostname: new(gwapiv1.Hostname("bar.example.com"))}, + tls: ListenerTLSConfig{ + certDNSNames: []string{"bar.example.com"}, + }, + }, + }, + expectedStatus: []expectedListenerStatus{ + { + listenerName: "listener-2", + condition: gwapiv1.ListenerConditionOverlappingTLSConfig, + status: metav1.ConditionTrue, + reason: status.ListenerReasonSDSCertificateOpaque, + message: "HTTP/2 is disabled by default because one or more HTTPS listeners on this port use an SDS-backed certificate whose DNS names cannot be inspected. Configure ALPN explicitly with ClientTrafficPolicy to override this default.", + }, + }, + expectedTLSOverlaps: []string{"listener-2"}, + }, + { + name: "SDS listener hostname overlaps a peer certificate SAN", + listeners: []*ListenerContext{ + { + Listener: &gwapiv1.Listener{Name: "listener-1", Protocol: gwapiv1.HTTPSProtocolType, Port: 443, Hostname: new(gwapiv1.Hostname("*.example.com"))}, + tls: ListenerTLSConfig{ + secrets: []*corev1.Secret{{Type: egv1a1.SDSSecretType}}, + }, + }, + { + Listener: &gwapiv1.Listener{Name: "listener-2", Protocol: gwapiv1.HTTPSProtocolType, Port: 443, Hostname: new(gwapiv1.Hostname("foo.example.com"))}, + tls: ListenerTLSConfig{ + certDNSNames: []string{"foo.example.com"}, + }, + }, + }, + expectedStatus: []expectedListenerStatus{ + { + listenerName: "listener-1", + condition: gwapiv1.ListenerConditionOverlappingTLSConfig, + status: metav1.ConditionTrue, + reason: status.ListenerReasonSDSCertificateOpaque, + message: "HTTP/2 is disabled by default because one or more HTTPS listeners on this port use an SDS-backed certificate whose DNS names cannot be inspected. Configure ALPN explicitly with ClientTrafficPolicy to override this default.", + }, + { + listenerName: "listener-2", + condition: gwapiv1.ListenerConditionOverlappingTLSConfig, + status: metav1.ConditionTrue, + reason: status.ListenerReasonSDSCertificateOpaque, + message: "HTTP/2 is disabled by default because one or more HTTPS listeners on this port use an SDS-backed certificate whose DNS names cannot be inspected. Configure ALPN explicitly with ClientTrafficPolicy to override this default.", + }, + }, + expectedTLSOverlaps: []string{"listener-1", "listener-2"}, + }, + { + name: "SDS certificate does not affect listeners on a different port", + listeners: []*ListenerContext{ + { + Listener: &gwapiv1.Listener{Name: "listener-1", Protocol: gwapiv1.HTTPSProtocolType, Port: 443}, + tls: ListenerTLSConfig{ + secrets: []*corev1.Secret{{Type: egv1a1.SDSSecretType}}, + }, + }, + { + Listener: &gwapiv1.Listener{Name: "listener-2", Protocol: gwapiv1.HTTPSProtocolType, Port: 8443}, + tls: ListenerTLSConfig{ + certDNSNames: []string{"foo.example.com"}, + }, + }, + }, + }, + { + name: "SDS certificate ignores an invalid same-port peer", + listeners: []*ListenerContext{ + { + Listener: &gwapiv1.Listener{Name: "listener-1", Protocol: gwapiv1.HTTPSProtocolType, Port: 443}, + tls: ListenerTLSConfig{ + secrets: []*corev1.Secret{{Type: egv1a1.SDSSecretType}}, + }, + }, + { + Listener: &gwapiv1.Listener{Name: "listener-2", Protocol: gwapiv1.HTTPSProtocolType, Port: 443}, + tls: ListenerTLSConfig{ + certDNSNames: []string{"foo.example.com"}, + }, + }, + { + Listener: &gwapiv1.Listener{Name: "listener-3", Protocol: gwapiv1.HTTPSProtocolType, Port: 443}, + tls: ListenerTLSConfig{ + certDNSNames: []string{"bar.example.com"}, + }, + }, + }, + invalidListeners: []string{"listener-3"}, + expectedStatus: []expectedListenerStatus{ + { + listenerName: "listener-1", + condition: gwapiv1.ListenerConditionOverlappingTLSConfig, + status: metav1.ConditionTrue, + reason: gwapiv1.ListenerConditionReason("SDSCertificateOpaque"), + message: "HTTP/2 is disabled by default because one or more HTTPS listeners on this port use an SDS-backed certificate whose DNS names cannot be inspected. Configure ALPN explicitly with ClientTrafficPolicy to override this default.", + }, + { + listenerName: "listener-2", + condition: gwapiv1.ListenerConditionOverlappingTLSConfig, + status: metav1.ConditionTrue, + reason: gwapiv1.ListenerConditionReason("SDSCertificateOpaque"), + message: "HTTP/2 is disabled by default because one or more HTTPS listeners on this port use an SDS-backed certificate whose DNS names cannot be inspected. Configure ALPN explicitly with ClientTrafficPolicy to override this default.", + }, + }, + expectedTLSOverlaps: []string{"listener-1", "listener-2"}, + }, + { + name: "Lone SDS certificate does not require an ALPN downgrade", + listeners: []*ListenerContext{ + { + Listener: &gwapiv1.Listener{Name: "listener-1", Protocol: gwapiv1.HTTPSProtocolType, Port: 443}, + tls: ListenerTLSConfig{ + secrets: []*corev1.Secret{{Type: egv1a1.SDSSecretType}}, + }, + }, + }, }, } @@ -690,6 +831,18 @@ func TestCheckOverlappingCertificates(t *testing.T) { gateway.listeners[i].gateway = gateway gateway.listeners[i].httpIR = &ir.HTTPListener{} } + for _, invalidListener := range tt.invalidListeners { + for _, listener := range gateway.listeners { + if string(listener.Name) == invalidListener { + listener.SetCondition( + gwapiv1.ListenerConditionAccepted, + metav1.ConditionFalse, + gwapiv1.ListenerReasonInvalid, + "Listener is invalid.", + ) + } + } + } // Process overlapping certificates checkOverlappingCertificates(tt.listeners) @@ -729,6 +882,7 @@ func TestCheckOverlappingCertificates(t *testing.T) { found := false for _, expected := range tt.expectedStatus { if string(listener.Name) == expected.listenerName && + condition.Type == string(expected.condition) && condition.Status == expected.status && condition.Reason == string(expected.reason) && condition.Message == expected.message { @@ -743,9 +897,9 @@ func TestCheckOverlappingCertificates(t *testing.T) { } } - expectedTLSOverlaps := map[string]bool{} - for _, expected := range tt.expectedStatus { - expectedTLSOverlaps[expected.listenerName] = true + expectedTLSOverlaps := make(map[string]bool, len(tt.expectedTLSOverlaps)) + for _, listenerName := range tt.expectedTLSOverlaps { + expectedTLSOverlaps[listenerName] = true } for _, listener := range gateway.listeners { require.NotNil(t, listener.httpIR) diff --git a/internal/gatewayapi/status/error.go b/internal/gatewayapi/status/error.go index 57b712fb8e..832d812cd8 100644 --- a/internal/gatewayapi/status/error.go +++ b/internal/gatewayapi/status/error.go @@ -38,6 +38,7 @@ const ( // Listener condition reasons for various error scenarios const ( ListenerReasonPartiallyInvalidCertificateRef gwapiv1.ListenerConditionReason = "PartiallyInvalidCertificateRef" + ListenerReasonSDSCertificateOpaque gwapiv1.ListenerConditionReason = "SDSCertificateOpaque" ) // ListenerError is an error interface that represents errors that need to be reflected diff --git a/internal/gatewayapi/testdata/sds-listener-disabled.in.yaml b/internal/gatewayapi/testdata/sds-listener-disabled.in.yaml new file mode 100644 index 0000000000..a5fc075f88 --- /dev/null +++ b/internal/gatewayapi/testdata/sds-listener-disabled.in.yaml @@ -0,0 +1,72 @@ +gatewayClass: + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: envoy-gateway-class + spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: + - apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: sds-only + protocol: HTTPS + port: 443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-cert-default + - name: sds-mixed + protocol: HTTPS + port: 8443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: envoy-valid + namespace: envoy-gateway-system + - name: sds-cert-default +referenceGrants: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: ReferenceGrant + metadata: + namespace: envoy-gateway-system + name: allow-base-certificate + spec: + from: + - group: gateway.networking.k8s.io + kind: Gateway + namespace: envoy-gateway + to: + - group: "" + kind: Secret + name: envoy-valid +secrets: + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: sds-cert-default + type: gateway.envoyproxy.io/sds + data: + url: L3Zhci9ydW4vc2VjcmV0cy93b3JrbG9hZC1zcGlmZmUtdWRzL3NvY2tldA== + secretName: ZGVmYXVsdA== + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway-system + name: envoy-valid + type: kubernetes.io/tls + data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnVENDQVNlZ0F3SUJBZ0lVRm1sOExCRzBvL1FLNFErWjdrODI0c0MyaUZ3d0NnWUlLb1pJemowRUF3SXcKRmpFVU1CSUdBMVVFQXd3TFptOXZMbUpoY2k1amIyMHdIaGNOTWpRd01qSTVNRGt6TURFd1doY05NelF3TWpJMgpNRGt6TURFd1dqQVdNUlF3RWdZRFZRUUREQXRtYjI4dVltRnlMbU52YlRCWk1CTUdCeXFHU000OUFnRUdDQ3FHClNNNDlBd0VIQTBJQUJMYVl2cUt1VlZveERvNTJlV3p2WUI1anc3RU1GODZybXlvaTVadWF5emRNdnBnNHpCcjgKUktCak5zK1QxakI4T0t1Y1MvN1JVRHgwcHorOTc2ek0zaU9qVXpCUk1CMEdBMVVkRGdRV0JCVE82K2NnMFIwZAp3dHJ6SlFQRzZnNzZoQkJVelRBZkJnTlZIU01FR0RBV2dCVE82K2NnMFIwZHd0cnpKUVBHNmc3NmhCQlV6VEFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSVFDMlhwUFFnUXpXYWUzYjVwWnQKR2N1TWZESjBjME9QS2NuZWdrWFoyQzRCM2dJZ1Uvc1Jrd0lwTFFOUlYrRWFZdzRQNVQ1Z1BFNlkrVnBtQzk4aApvVmpaL3pRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + tls.key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUF5ZktRdlBCdWRYUmgwTExtdFZSSlBZbDZlK0dnenZnY3RGZXhLaDlhMUdvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFdHBpK29xNVZXakVPam5aNWJPOWdIbVBEc1F3WHpxdWJLaUxsbTVyTE4weSttRGpNR3Z4RQpvR00yejVQV01IdzRxNXhML3RGUVBIU25QNzN2ck16ZUl3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/internal/gatewayapi/testdata/sds-listener-disabled.out.yaml b/internal/gatewayapi/testdata/sds-listener-disabled.out.yaml new file mode 100644 index 0000000000..464ebeaa6d --- /dev/null +++ b/internal/gatewayapi/testdata/sds-listener-disabled.out.yaml @@ -0,0 +1,161 @@ +gatewayClass: + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: envoy-gateway-class + spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: sds-only + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - name: sds-cert-default + mode: Terminate + - allowedRoutes: + namespaces: + from: All + name: sds-mixed + port: 8443 + protocol: HTTPS + tls: + certificateRefs: + - name: envoy-valid + namespace: envoy-gateway-system + - name: sds-cert-default + mode: Terminate + status: + listeners: + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: 'No valid secrets exist: certificate refs 0: SDS Secret envoy-gateway/sds-cert-default + cannot be used because SDS Secret references are not enabled in EnvoyGateway + configuration.' + reason: InvalidCertificateRef + status: "False" + type: ResolvedRefs + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener is invalid, see other Conditions for details. + reason: Invalid + status: "False" + type: Programmed + name: sds-only + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: 'Some secrets are invalid: certificate refs 1: SDS Secret envoy-gateway/sds-cert-default + cannot be used because SDS Secret references are not enabled in EnvoyGateway + configuration.' + reason: PartiallyInvalidCertificateRef + status: "False" + type: ResolvedRefs + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + name: sds-mixed + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - name: envoy-gateway/gateway-1/sds-mixed + ports: + - containerPort: 8443 + name: https-8443 + protocol: HTTPS + servicePort: 8443 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + ownerReference: + kind: GatewayClass + name: envoy-gateway-class + name: envoy-gateway/gateway-1 + namespace: envoy-gateway-system +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + globalResources: + proxyServiceCluster: + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + settings: + - addressType: IP + endpoints: + - host: 7.6.5.4 + port: 8080 + zone: zone1 + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + protocol: TCP + http: + - address: 0.0.0.0 + externalPort: 8443 + hostnames: + - '*' + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: sds-mixed + name: envoy-gateway/gateway-1/sds-mixed + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 8443 + tls: + alpnProtocols: null + certificates: + - certificate: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnVENDQVNlZ0F3SUJBZ0lVRm1sOExCRzBvL1FLNFErWjdrODI0c0MyaUZ3d0NnWUlLb1pJemowRUF3SXcKRmpFVU1CSUdBMVVFQXd3TFptOXZMbUpoY2k1amIyMHdIaGNOTWpRd01qSTVNRGt6TURFd1doY05NelF3TWpJMgpNRGt6TURFd1dqQVdNUlF3RWdZRFZRUUREQXRtYjI4dVltRnlMbU52YlRCWk1CTUdCeXFHU000OUFnRUdDQ3FHClNNNDlBd0VIQTBJQUJMYVl2cUt1VlZveERvNTJlV3p2WUI1anc3RU1GODZybXlvaTVadWF5emRNdnBnNHpCcjgKUktCak5zK1QxakI4T0t1Y1MvN1JVRHgwcHorOTc2ek0zaU9qVXpCUk1CMEdBMVVkRGdRV0JCVE82K2NnMFIwZAp3dHJ6SlFQRzZnNzZoQkJVelRBZkJnTlZIU01FR0RBV2dCVE82K2NnMFIwZHd0cnpKUVBHNmc3NmhCQlV6VEFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSVFDMlhwUFFnUXpXYWUzYjVwWnQKR2N1TWZESjBjME9QS2NuZWdrWFoyQzRCM2dJZ1Uvc1Jrd0lwTFFOUlYrRWFZdzRQNVQ1Z1BFNlkrVnBtQzk4aApvVmpaL3pRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + name: envoy-gateway-system/envoy-valid + privateKey: '[redacted]' + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/gatewayapi/testdata/sds-listener-invalid.in.yaml b/internal/gatewayapi/testdata/sds-listener-invalid.in.yaml new file mode 100644 index 0000000000..1b16ea710f --- /dev/null +++ b/internal/gatewayapi/testdata/sds-listener-invalid.in.yaml @@ -0,0 +1,157 @@ +gatewayClass: + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: envoy-gateway-class + spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: + - apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: missing-url + protocol: HTTPS + port: 443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-missing-url + - name: missing-secret-name + protocol: HTTPS + port: 8443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-missing-secret-name + - name: mixed-invalid-sds + protocol: HTTPS + port: 9443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: envoy-valid + namespace: envoy-gateway-system + - name: sds-missing-url + - name: cross-namespace-without-grant + protocol: HTTPS + port: 10444 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-cross-without-grant + namespace: sds-secrets + - name: valid-sds-malformed-tls + protocol: HTTPS + port: 11443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-valid + - name: malformed-tls + - name: malformed-sds-url + protocol: HTTPS + port: 12443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-malformed-url +referenceGrants: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: ReferenceGrant + metadata: + namespace: envoy-gateway-system + name: allow-base-certificate + spec: + from: + - group: gateway.networking.k8s.io + kind: Gateway + namespace: envoy-gateway + to: + - group: "" + kind: Secret + name: envoy-valid +secrets: + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: sds-missing-url + type: gateway.envoyproxy.io/sds + data: + secretName: ZGVmYXVsdA== + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: sds-missing-secret-name + type: gateway.envoyproxy.io/sds + data: + url: dW5peDovLy92YXIvcnVuL3NlY3JldHMvd29ya2xvYWQtc3BpZmZlLXVkcy9zb2NrZXQ= + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: sds-valid + type: gateway.envoyproxy.io/sds + data: + url: dW5peDovLy92YXIvcnVuL3NlY3JldHMvd29ya2xvYWQtc3BpZmZlLXVkcy9zb2NrZXQ= + secretName: dmFsaWQ= + - apiVersion: v1 + kind: Secret + metadata: + namespace: sds-secrets + name: sds-cross-without-grant + type: gateway.envoyproxy.io/sds + data: + url: dW5peDovLy92YXIvcnVuL3NlY3JldHMvd29ya2xvYWQtc3BpZmZlLXVkcy9zb2NrZXQ= + secretName: Y3Jvc3M= + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: malformed-tls + type: kubernetes.io/tls + data: + tls.crt: bm90IGEgcGVtIGNlcnRpZmljYXRl + tls.key: bm90IGEgcGVtIHByaXZhdGUga2V5 + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: sds-malformed-url + type: gateway.envoyproxy.io/sds + data: + url: L3Zhci9ydW4vc2VjcmV0cy93b3JrbG9hZC1zcGlmZmUtdWRzL3NvY2tldA== + secretName: dGVzdA== + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway-system + name: envoy-valid + type: kubernetes.io/tls + data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnVENDQVNlZ0F3SUJBZ0lVRm1sOExCRzBvL1FLNFErWjdrODI0c0MyaUZ3d0NnWUlLb1pJemowRUF3SXcKRmpFVU1CSUdBMVVFQXd3TFptOXZMbUpoY2k1amIyMHdIaGNOTWpRd01qSTVNRGt6TURFd1doY05NelF3TWpJMgpNRGt6TURFd1dqQVdNUlF3RWdZRFZRUUREQXRtYjI4dVltRnlMbU52YlRCWk1CTUdCeXFHU000OUFnRUdDQ3FHClNNNDlBd0VIQTBJQUJMYVl2cUt1VlZveERvNTJlV3p2WUI1anc3RU1GODZybXlvaTVadWF5emRNdnBnNHpCcjgKUktCak5zK1QxakI4T0t1Y1MvN1JVRHgwcHorOTc2ek0zaU9qVXpCUk1CMEdBMVVkRGdRV0JCVE82K2NnMFIwZAp3dHJ6SlFQRzZnNzZoQkJVelRBZkJnTlZIU01FR0RBV2dCVE82K2NnMFIwZHd0cnpKUVBHNmc3NmhCQlV6VEFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSVFDMlhwUFFnUXpXYWUzYjVwWnQKR2N1TWZESjBjME9QS2NuZWdrWFoyQzRCM2dJZ1Uvc1Jrd0lwTFFOUlYrRWFZdzRQNVQ1Z1BFNlkrVnBtQzk4aApvVmpaL3pRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + tls.key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUF5ZktRdlBCdWRYUmgwTExtdFZSSlBZbDZlK0dnenZnY3RGZXhLaDlhMUdvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFdHBpK29xNVZXakVPam5aNWJPOWdIbVBEc1F3WHpxdWJLaUxsbTVyTE4weSttRGpNR3Z4RQpvR00yejVQV01IdzRxNXhML3RGUVBIU25QNzN2ck16ZUl3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/internal/gatewayapi/testdata/sds-listener-invalid.out.yaml b/internal/gatewayapi/testdata/sds-listener-invalid.out.yaml new file mode 100644 index 0000000000..a1ed6861ba --- /dev/null +++ b/internal/gatewayapi/testdata/sds-listener-invalid.out.yaml @@ -0,0 +1,325 @@ +gatewayClass: + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: envoy-gateway-class + spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: missing-url + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - name: sds-missing-url + mode: Terminate + - allowedRoutes: + namespaces: + from: All + name: missing-secret-name + port: 8443 + protocol: HTTPS + tls: + certificateRefs: + - name: sds-missing-secret-name + mode: Terminate + - allowedRoutes: + namespaces: + from: All + name: mixed-invalid-sds + port: 9443 + protocol: HTTPS + tls: + certificateRefs: + - name: envoy-valid + namespace: envoy-gateway-system + - name: sds-missing-url + mode: Terminate + - allowedRoutes: + namespaces: + from: All + name: cross-namespace-without-grant + port: 10444 + protocol: HTTPS + tls: + certificateRefs: + - name: sds-cross-without-grant + namespace: sds-secrets + mode: Terminate + - allowedRoutes: + namespaces: + from: All + name: valid-sds-malformed-tls + port: 11443 + protocol: HTTPS + tls: + certificateRefs: + - name: sds-valid + - name: malformed-tls + mode: Terminate + - allowedRoutes: + namespaces: + from: All + name: malformed-sds-url + port: 12443 + protocol: HTTPS + tls: + certificateRefs: + - name: sds-malformed-url + mode: Terminate + status: + listeners: + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: 'No valid secrets exist: certificate refs 0: Secret envoy-gateway/sds-missing-url + is not a valid SDS reference secret: no url found in SDS reference secret + envoy-gateway/sds-missing-url.' + reason: InvalidCertificateRef + status: "False" + type: ResolvedRefs + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener is invalid, see other Conditions for details. + reason: Invalid + status: "False" + type: Programmed + name: missing-url + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: 'No valid secrets exist: certificate refs 0: Secret envoy-gateway/sds-missing-secret-name + is not a valid SDS reference secret: no secretName found in SDS reference + secret envoy-gateway/sds-missing-secret-name.' + reason: InvalidCertificateRef + status: "False" + type: ResolvedRefs + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener is invalid, see other Conditions for details. + reason: Invalid + status: "False" + type: Programmed + name: missing-secret-name + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: 'Some secrets are invalid: certificate refs 1: Secret envoy-gateway/sds-missing-url + is not a valid SDS reference secret: no url found in SDS reference secret + envoy-gateway/sds-missing-url.' + reason: PartiallyInvalidCertificateRef + status: "False" + type: ResolvedRefs + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + name: mixed-invalid-sds + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: 'No valid secrets exist: certificate refs 0: Certificate ref to secret + sds-secrets/sds-cross-without-grant not permitted by any ReferenceGrant.' + reason: RefNotPermitted + status: "False" + type: ResolvedRefs + - lastTransitionTime: null + message: Listener is invalid, see other Conditions for details. + reason: Invalid + status: "False" + type: Programmed + name: cross-namespace-without-grant + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: 'Some secrets are invalid: envoy-gateway/malformed-tls must contain + valid tls.crt and tls.key, unable to validate certificate in tls.crt: unable + to decode pem data for certificate' + reason: PartiallyInvalidCertificateRef + status: "False" + type: ResolvedRefs + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + name: valid-sds-malformed-tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: 'No valid secrets exist: certificate refs 0: Secret envoy-gateway/sds-malformed-url + is not a valid SDS reference secret: invalid URL in SDS reference secret + envoy-gateway/sds-malformed-url: unsupported URL scheme: .' + reason: InvalidCertificateRef + status: "False" + type: ResolvedRefs + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener is invalid, see other Conditions for details. + reason: Invalid + status: "False" + type: Programmed + name: malformed-sds-url + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - name: envoy-gateway/gateway-1/mixed-invalid-sds + ports: + - containerPort: 9443 + name: https-9443 + protocol: HTTPS + servicePort: 9443 + - name: envoy-gateway/gateway-1/valid-sds-malformed-tls + ports: + - containerPort: 11443 + name: https-11443 + protocol: HTTPS + servicePort: 11443 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + ownerReference: + kind: GatewayClass + name: envoy-gateway-class + name: envoy-gateway/gateway-1 + namespace: envoy-gateway-system +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + globalResources: + proxyServiceCluster: + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + settings: + - addressType: IP + endpoints: + - host: 7.6.5.4 + port: 8080 + zone: zone1 + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + protocol: TCP + http: + - address: 0.0.0.0 + externalPort: 9443 + hostnames: + - '*' + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: mixed-invalid-sds + name: envoy-gateway/gateway-1/mixed-invalid-sds + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 9443 + tls: + alpnProtocols: null + certificates: + - certificate: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnVENDQVNlZ0F3SUJBZ0lVRm1sOExCRzBvL1FLNFErWjdrODI0c0MyaUZ3d0NnWUlLb1pJemowRUF3SXcKRmpFVU1CSUdBMVVFQXd3TFptOXZMbUpoY2k1amIyMHdIaGNOTWpRd01qSTVNRGt6TURFd1doY05NelF3TWpJMgpNRGt6TURFd1dqQVdNUlF3RWdZRFZRUUREQXRtYjI4dVltRnlMbU52YlRCWk1CTUdCeXFHU000OUFnRUdDQ3FHClNNNDlBd0VIQTBJQUJMYVl2cUt1VlZveERvNTJlV3p2WUI1anc3RU1GODZybXlvaTVadWF5emRNdnBnNHpCcjgKUktCak5zK1QxakI4T0t1Y1MvN1JVRHgwcHorOTc2ek0zaU9qVXpCUk1CMEdBMVVkRGdRV0JCVE82K2NnMFIwZAp3dHJ6SlFQRzZnNzZoQkJVelRBZkJnTlZIU01FR0RBV2dCVE82K2NnMFIwZHd0cnpKUVBHNmc3NmhCQlV6VEFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSVFDMlhwUFFnUXpXYWUzYjVwWnQKR2N1TWZESjBjME9QS2NuZWdrWFoyQzRCM2dJZ1Uvc1Jrd0lwTFFOUlYrRWFZdzRQNVQ1Z1BFNlkrVnBtQzk4aApvVmpaL3pRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + name: envoy-gateway-system/envoy-valid + privateKey: '[redacted]' + - address: 0.0.0.0 + externalPort: 11443 + hostnames: + - '*' + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: valid-sds-malformed-tls + name: envoy-gateway/gateway-1/valid-sds-malformed-tls + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 11443 + tls: + alpnProtocols: null + certificates: + - name: envoy-gateway/sds-valid + sds: + address: /var/run/secrets/workload-spiffe-uds/socket + scheme: unix + secretName: valid + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/gatewayapi/testdata/sds-listener.in.yaml b/internal/gatewayapi/testdata/sds-listener.in.yaml new file mode 100644 index 0000000000..b2fbec0d00 --- /dev/null +++ b/internal/gatewayapi/testdata/sds-listener.in.yaml @@ -0,0 +1,154 @@ +gatewayClass: + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: envoy-gateway-class + spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: + - apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: sds-multiple + hostname: sds.example.com + protocol: HTTPS + port: 443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-cert-default + - name: sds-cert-other + - name: inline-same-port + hostname: inline.example.com + protocol: HTTPS + port: 443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: envoy-valid + namespace: envoy-gateway-system + - name: sds-mixed + protocol: HTTPS + port: 8443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-cert-default + - name: envoy-valid + namespace: envoy-gateway-system + - name: sds-tls-terminate + protocol: TLS + port: 9443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-cert-default + - name: sds-cross-namespace + protocol: HTTPS + port: 11443 + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - name: sds-cert-cross + namespace: sds-secrets +tcpRoutes: + - apiVersion: gateway.networking.k8s.io/v1 + kind: TCPRoute + metadata: + namespace: default + name: sds-tcproute + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: sds-tls-terminate + rules: + - backendRefs: + - name: service-1 + port: 8080 +referenceGrants: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: ReferenceGrant + metadata: + namespace: envoy-gateway-system + name: allow-base-certificate + spec: + from: + - group: gateway.networking.k8s.io + kind: Gateway + namespace: envoy-gateway + to: + - group: "" + kind: Secret + name: envoy-valid + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: ReferenceGrant + metadata: + namespace: sds-secrets + name: allow-sds-certificate + spec: + from: + - group: gateway.networking.k8s.io + kind: Gateway + namespace: envoy-gateway + to: + - group: "" + kind: Secret + name: sds-cert-cross +secrets: + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: sds-cert-default + type: gateway.envoyproxy.io/sds + data: + url: dW5peDovLy92YXIvcnVuL3NlY3JldHMvd29ya2xvYWQtc3BpZmZlLXVkcy9zb2NrZXQ= + secretName: ZGVmYXVsdA== + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: sds-cert-other + type: gateway.envoyproxy.io/sds + data: + url: dW5peDovLy92YXIvcnVuL3NlY3JldHMvd29ya2xvYWQtc3BpZmZlLXVkcy9zb2NrZXQ= + secretName: b3RoZXI= + - apiVersion: v1 + kind: Secret + metadata: + namespace: sds-secrets + name: sds-cert-cross + type: gateway.envoyproxy.io/sds + data: + url: dW5peDovLy92YXIvcnVuL3NlY3JldHMvd29ya2xvYWQtc3BpZmZlLXVkcy9zb2NrZXQ= + secretName: Y3Jvc3M= + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway-system + name: envoy-valid + type: kubernetes.io/tls + data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnVENDQVNlZ0F3SUJBZ0lVRm1sOExCRzBvL1FLNFErWjdrODI0c0MyaUZ3d0NnWUlLb1pJemowRUF3SXcKRmpFVU1CSUdBMVVFQXd3TFptOXZMbUpoY2k1amIyMHdIaGNOTWpRd01qSTVNRGt6TURFd1doY05NelF3TWpJMgpNRGt6TURFd1dqQVdNUlF3RWdZRFZRUUREQXRtYjI4dVltRnlMbU52YlRCWk1CTUdCeXFHU000OUFnRUdDQ3FHClNNNDlBd0VIQTBJQUJMYVl2cUt1VlZveERvNTJlV3p2WUI1anc3RU1GODZybXlvaTVadWF5emRNdnBnNHpCcjgKUktCak5zK1QxakI4T0t1Y1MvN1JVRHgwcHorOTc2ek0zaU9qVXpCUk1CMEdBMVVkRGdRV0JCVE82K2NnMFIwZAp3dHJ6SlFQRzZnNzZoQkJVelRBZkJnTlZIU01FR0RBV2dCVE82K2NnMFIwZHd0cnpKUVBHNmc3NmhCQlV6VEFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSVFDMlhwUFFnUXpXYWUzYjVwWnQKR2N1TWZESjBjME9QS2NuZWdrWFoyQzRCM2dJZ1Uvc1Jrd0lwTFFOUlYrRWFZdzRQNVQ1Z1BFNlkrVnBtQzk4aApvVmpaL3pRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + tls.key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUF5ZktRdlBCdWRYUmgwTExtdFZSSlBZbDZlK0dnenZnY3RGZXhLaDlhMUdvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFdHBpK29xNVZXakVPam5aNWJPOWdIbVBEc1F3WHpxdWJLaUxsbTVyTE4weSttRGpNR3Z4RQpvR00yejVQV01IdzRxNXhML3RGUVBIU25QNzN2ck16ZUl3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/internal/gatewayapi/testdata/sds-listener.out.yaml b/internal/gatewayapi/testdata/sds-listener.out.yaml new file mode 100644 index 0000000000..d4f3cf0a3b --- /dev/null +++ b/internal/gatewayapi/testdata/sds-listener.out.yaml @@ -0,0 +1,446 @@ +gatewayClass: + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: envoy-gateway-class + spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + hostname: sds.example.com + name: sds-multiple + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - name: sds-cert-default + - name: sds-cert-other + mode: Terminate + - allowedRoutes: + namespaces: + from: All + hostname: inline.example.com + name: inline-same-port + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - name: envoy-valid + namespace: envoy-gateway-system + mode: Terminate + - allowedRoutes: + namespaces: + from: All + name: sds-mixed + port: 8443 + protocol: HTTPS + tls: + certificateRefs: + - name: sds-cert-default + - name: envoy-valid + namespace: envoy-gateway-system + mode: Terminate + - allowedRoutes: + namespaces: + from: All + name: sds-tls-terminate + port: 9443 + protocol: TLS + tls: + certificateRefs: + - name: sds-cert-default + mode: Terminate + - allowedRoutes: + namespaces: + from: All + name: sds-cross-namespace + port: 11443 + protocol: HTTPS + tls: + certificateRefs: + - name: sds-cert-cross + namespace: sds-secrets + mode: Terminate + status: + listeners: + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + - lastTransitionTime: null + message: HTTP/2 is disabled by default because one or more HTTPS listeners + on this port use an SDS-backed certificate whose DNS names cannot be inspected. + Configure ALPN explicitly with ClientTrafficPolicy to override this default. + reason: SDSCertificateOpaque + status: "True" + type: OverlappingTLSConfig + name: sds-multiple + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: inline-same-port + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: sds-mixed + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: sds-tls-terminate + supportedKinds: + - group: gateway.networking.k8s.io + kind: TCPRoute + - group: gateway.networking.k8s.io + kind: TLSRoute + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: sds-cross-namespace + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - name: envoy-gateway/gateway-1/sds-multiple + ports: + - containerPort: 10443 + name: https-443 + protocol: HTTPS + servicePort: 443 + - name: envoy-gateway/gateway-1/sds-mixed + ports: + - containerPort: 8443 + name: https-8443 + protocol: HTTPS + servicePort: 8443 + - name: envoy-gateway/gateway-1/sds-tls-terminate + ports: + - containerPort: 9443 + name: tls-9443 + protocol: TLS + servicePort: 9443 + - name: envoy-gateway/gateway-1/sds-cross-namespace + ports: + - containerPort: 11443 + name: https-11443 + protocol: HTTPS + servicePort: 11443 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + ownerReference: + kind: GatewayClass + name: envoy-gateway-class + name: envoy-gateway/gateway-1 + namespace: envoy-gateway-system +tcpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: TCPRoute + metadata: + name: sds-tcproute + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: sds-tls-terminate + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: sds-tls-terminate +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + globalResources: + proxyServiceCluster: + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + settings: + - addressType: IP + endpoints: + - host: 7.6.5.4 + port: 8080 + zone: zone1 + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + protocol: TCP + http: + - address: 0.0.0.0 + externalPort: 443 + hostnames: + - sds.example.com + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: sds-multiple + name: envoy-gateway/gateway-1/sds-multiple + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10443 + tls: + alpnProtocols: null + certificates: + - name: envoy-gateway/sds-cert-default + sds: + address: /var/run/secrets/workload-spiffe-uds/socket + scheme: unix + secretName: default + - name: envoy-gateway/sds-cert-other + sds: + address: /var/run/secrets/workload-spiffe-uds/socket + scheme: unix + secretName: other + tlsOverlaps: true + - address: 0.0.0.0 + externalPort: 443 + hostnames: + - inline.example.com + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: inline-same-port + name: envoy-gateway/gateway-1/inline-same-port + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10443 + tls: + alpnProtocols: null + certificates: + - certificate: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnVENDQVNlZ0F3SUJBZ0lVRm1sOExCRzBvL1FLNFErWjdrODI0c0MyaUZ3d0NnWUlLb1pJemowRUF3SXcKRmpFVU1CSUdBMVVFQXd3TFptOXZMbUpoY2k1amIyMHdIaGNOTWpRd01qSTVNRGt6TURFd1doY05NelF3TWpJMgpNRGt6TURFd1dqQVdNUlF3RWdZRFZRUUREQXRtYjI4dVltRnlMbU52YlRCWk1CTUdCeXFHU000OUFnRUdDQ3FHClNNNDlBd0VIQTBJQUJMYVl2cUt1VlZveERvNTJlV3p2WUI1anc3RU1GODZybXlvaTVadWF5emRNdnBnNHpCcjgKUktCak5zK1QxakI4T0t1Y1MvN1JVRHgwcHorOTc2ek0zaU9qVXpCUk1CMEdBMVVkRGdRV0JCVE82K2NnMFIwZAp3dHJ6SlFQRzZnNzZoQkJVelRBZkJnTlZIU01FR0RBV2dCVE82K2NnMFIwZHd0cnpKUVBHNmc3NmhCQlV6VEFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSVFDMlhwUFFnUXpXYWUzYjVwWnQKR2N1TWZESjBjME9QS2NuZWdrWFoyQzRCM2dJZ1Uvc1Jrd0lwTFFOUlYrRWFZdzRQNVQ1Z1BFNlkrVnBtQzk4aApvVmpaL3pRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + name: envoy-gateway-system/envoy-valid + privateKey: '[redacted]' + - address: 0.0.0.0 + externalPort: 8443 + hostnames: + - '*' + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: sds-mixed + name: envoy-gateway/gateway-1/sds-mixed + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 8443 + tls: + alpnProtocols: null + certificates: + - name: envoy-gateway/sds-cert-default + sds: + address: /var/run/secrets/workload-spiffe-uds/socket + scheme: unix + secretName: default + - certificate: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnVENDQVNlZ0F3SUJBZ0lVRm1sOExCRzBvL1FLNFErWjdrODI0c0MyaUZ3d0NnWUlLb1pJemowRUF3SXcKRmpFVU1CSUdBMVVFQXd3TFptOXZMbUpoY2k1amIyMHdIaGNOTWpRd01qSTVNRGt6TURFd1doY05NelF3TWpJMgpNRGt6TURFd1dqQVdNUlF3RWdZRFZRUUREQXRtYjI4dVltRnlMbU52YlRCWk1CTUdCeXFHU000OUFnRUdDQ3FHClNNNDlBd0VIQTBJQUJMYVl2cUt1VlZveERvNTJlV3p2WUI1anc3RU1GODZybXlvaTVadWF5emRNdnBnNHpCcjgKUktCak5zK1QxakI4T0t1Y1MvN1JVRHgwcHorOTc2ek0zaU9qVXpCUk1CMEdBMVVkRGdRV0JCVE82K2NnMFIwZAp3dHJ6SlFQRzZnNzZoQkJVelRBZkJnTlZIU01FR0RBV2dCVE82K2NnMFIwZHd0cnpKUVBHNmc3NmhCQlV6VEFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSVFDMlhwUFFnUXpXYWUzYjVwWnQKR2N1TWZESjBjME9QS2NuZWdrWFoyQzRCM2dJZ1Uvc1Jrd0lwTFFOUlYrRWFZdzRQNVQ1Z1BFNlkrVnBtQzk4aApvVmpaL3pRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + name: envoy-gateway-system/envoy-valid + privateKey: '[redacted]' + - address: 0.0.0.0 + externalPort: 11443 + hostnames: + - '*' + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: sds-cross-namespace + name: envoy-gateway/gateway-1/sds-cross-namespace + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 11443 + tls: + alpnProtocols: null + certificates: + - name: sds-secrets/sds-cert-cross + sds: + address: /var/run/secrets/workload-spiffe-uds/socket + scheme: unix + secretName: cross + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 + tcp: + - address: 0.0.0.0 + externalPort: 9443 + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: sds-tls-terminate + name: envoy-gateway/gateway-1/sds-tls-terminate + port: 9443 + routes: + - destination: + metadata: + kind: TCPRoute + name: sds-tcproute + namespace: default + name: tcproute/default/sds-tcproute/rule/-1 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: "8080" + name: tcproute/default/sds-tcproute/rule/-1/backend/0 + protocol: TCP + weight: 1 + metadata: + kind: TCPRoute + name: sds-tcproute + namespace: default + name: tcproute/default/sds-tcproute + tls: + terminate: + alpnProtocols: [] + certificates: + - name: envoy-gateway/sds-cert-default + sds: + address: /var/run/secrets/workload-spiffe-uds/socket + scheme: unix + secretName: default + tls: + alpnProtocols: [] + certificates: + - name: envoy-gateway/sds-cert-default + sds: + address: /var/run/secrets/workload-spiffe-uds/socket + scheme: unix + secretName: default diff --git a/internal/gatewayapi/tls_test.go b/internal/gatewayapi/tls_test.go index f9c887cacb..35d9cb361f 100644 --- a/internal/gatewayapi/tls_test.go +++ b/internal/gatewayapi/tls_test.go @@ -746,3 +746,51 @@ func TestParseCertsExpiredLeafChainRejected(t *testing.T) { require.Equal(t, gwapiv1.ListenerReasonInvalidCertificateRef, listenerErr.Reason()) require.Contains(t, listenerErr.Error(), "has expired") } + +func TestValidateTerminateModeDeduplicatesSDSCertificateRefs(t *testing.T) { + sdsSecretOne := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "sds-one", Namespace: secretNamespace}, + Type: egv1a1.SDSSecretType, + Data: map[string][]byte{ + "secretName": []byte("listener-one"), + "url": []byte("unix:///var/run/sds/one.sock"), + }, + } + sdsSecretTwo := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "sds-two", Namespace: secretNamespace}, + Type: egv1a1.SDSSecretType, + Data: map[string][]byte{ + "secretName": []byte("listener-two"), + "url": []byte("unix:///var/run/sds/two.sock"), + }, + } + + gateway := &GatewayContext{Gateway: &gwapiv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: "gateway", Namespace: secretNamespace}, + Spec: gwapiv1.GatewaySpec{Listeners: []gwapiv1.Listener{{ + Name: "https", + TLS: &gwapiv1.ListenerTLSConfig{CertificateRefs: []gwapiv1.SecretObjectReference{ + {Name: "sds-one"}, + {Name: "sds-one"}, + {Name: "sds-two"}, + {Name: "sds-one"}, + }}, + }}}, + }} + gateway.ResetListeners() + + translator := &Translator{ + TranslatorContext: &TranslatorContext{}, + SDSSecretRefEnabled: true, + } + translator.SetSecrets([]*corev1.Secret{sdsSecretOne, sdsSecretTwo}) + + resolvedSecrets, certs, ok := translator.validateTerminateModeAndGetTLSSecrets( + gateway.listeners[0], + &resource.Resources{}, + ) + + require.True(t, ok) + require.Empty(t, certs) + require.Equal(t, []*corev1.Secret{sdsSecretOne, sdsSecretTwo}, resolvedSecrets) +} diff --git a/internal/gatewayapi/translator_test.go b/internal/gatewayapi/translator_test.go index f17ee67748..45106d970d 100644 --- a/internal/gatewayapi/translator_test.go +++ b/internal/gatewayapi/translator_test.go @@ -124,6 +124,16 @@ func TestTranslate(t *testing.T) { BackendEnabled: true, PerResourceSystemCASecret: true, }, + { + name: "sds-listener", + BackendEnabled: true, + SDSEnabled: true, + }, + { + name: "sds-listener-invalid", + BackendEnabled: true, + SDSEnabled: true, + }, } inputFiles, err := filepath.Glob(filepath.Join("testdata", "*.in.yaml")) diff --git a/internal/gatewayapi/validate.go b/internal/gatewayapi/validate.go index 5549d76b55..35c4e4cb71 100644 --- a/internal/gatewayapi/validate.go +++ b/internal/gatewayapi/validate.go @@ -24,6 +24,7 @@ import ( egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/gatewayapi/resource" "github.com/envoyproxy/gateway/internal/gatewayapi/status" + "github.com/envoyproxy/gateway/internal/ir" ) func (t *Translator) validateBackendRef(backendRefContext BackendRefContext, route RouteContext, @@ -454,7 +455,10 @@ func (t *Translator) validateTerminateModeAndGetTLSSecrets( } var errs []status.ListenerError - secrets := make([]*corev1.Secret, 0, len(listener.TLS.CertificateRefs)) + tlsSecrets := make([]*corev1.Secret, 0, len(listener.TLS.CertificateRefs)) + sdsSecrets := make([]*corev1.Secret, 0, len(listener.TLS.CertificateRefs)) + orderedSecrets := make([]*corev1.Secret, 0, len(listener.TLS.CertificateRefs)) + resolvedSDSSecretNames := make(map[types.NamespacedName]struct{}) for idx, certificateRef := range listener.TLS.CertificateRefs { if certificateRef.Group != nil && string(*certificateRef.Group) != "" { errs = append(errs, status.NewListenerStatusError( @@ -518,6 +522,34 @@ func (t *Translator) validateTerminateModeAndGetTLSSecrets( continue } + if secret.Type == egv1a1.SDSSecretType { + if !t.SDSSecretRefEnabled { + errs = append(errs, status.NewListenerStatusError( + fmt.Errorf("certificate refs %d: SDS Secret %s/%s cannot be used because SDS Secret references are not enabled in EnvoyGateway configuration.", idx, secretNamespace, certificateRef.Name), + gwapiv1.ListenerReasonInvalidCertificateRef, + )) + continue + } + + _, err := ir.NewSDSConfig(secret) + if err != nil { + errs = append(errs, status.NewListenerStatusError( + fmt.Errorf("certificate refs %d: Secret %s/%s is not a valid SDS reference secret: %w.", idx, secretNamespace, certificateRef.Name, err), + gwapiv1.ListenerReasonInvalidCertificateRef, + )) + continue + } + name := types.NamespacedName{Namespace: secret.Namespace, Name: secret.Name} + // Repeated SDS refs are ignored after their first occurrence to avoid duplicate xDS secret configs. + if _, ok := resolvedSDSSecretNames[name]; ok { + continue + } + resolvedSDSSecretNames[name] = struct{}{} + sdsSecrets = append(sdsSecrets, secret) + orderedSecrets = append(orderedSecrets, secret) + continue + } + if secret.Type != corev1.SecretTypeTLS { errs = append(errs, status.NewListenerStatusError( fmt.Errorf("certificate refs %d: Secret %s/%s must be of type %s.", idx, secretNamespace, certificateRef.Name, corev1.SecretTypeTLS), @@ -534,10 +566,11 @@ func (t *Translator) validateTerminateModeAndGetTLSSecrets( continue } - secrets = append(secrets, secret) + tlsSecrets = append(tlsSecrets, secret) + orderedSecrets = append(orderedSecrets, secret) } - if len(secrets) == 0 { + if len(tlsSecrets)+len(sdsSecrets) == 0 { // Use RefNotPermitted only if ALL errors are RefNotPermitted // Otherwise use InvalidCertificateRef as the general catch-all reason := gwapiv1.ListenerReasonRefNotPermitted @@ -563,16 +596,20 @@ func (t *Translator) validateTerminateModeAndGetTLSSecrets( return nil, nil, false } - validSecrets, certs, err := parseCertsFromTLSSecretsData(secrets) + validSecrets, certs, err := parseCertsFromTLSSecretsData(tlsSecrets) if err != nil { if err.Reason() != status.ListenerReasonPartiallyInvalidCertificateRef { - listener.SetCondition( - gwapiv1.ListenerConditionResolvedRefs, - metav1.ConditionFalse, - err.Reason(), - fmt.Sprintf("No valid secrets exist: %v.", err.Error()), - ) - return nil, nil, false + if len(validSecrets) == 0 && len(sdsSecrets) > 0 { + errs = append(errs, err) + } else { + listener.SetCondition( + gwapiv1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + err.Reason(), + fmt.Sprintf("No valid secrets exist: %v.", err.Error()), + ) + return nil, nil, false + } } else { errs = append(errs, err) } @@ -591,7 +628,29 @@ func (t *Translator) validateTerminateModeAndGetTLSSecrets( fmt.Sprintf("Some secrets are invalid: %v", errors.Join(errList...)), ) } - return validSecrets, certs, true + validTLSSecretsByName := make(map[types.NamespacedName][]*corev1.Secret, len(validSecrets)) + for _, secret := range validSecrets { + name := types.NamespacedName{Namespace: secret.Namespace, Name: secret.Name} + validTLSSecretsByName[name] = append(validTLSSecretsByName[name], secret) + } + + resolvedSecrets := make([]*corev1.Secret, 0, len(validSecrets)+len(sdsSecrets)) + for _, secret := range orderedSecrets { + if secret.Type == egv1a1.SDSSecretType { + resolvedSecrets = append(resolvedSecrets, secret) + continue + } + + name := types.NamespacedName{Namespace: secret.Namespace, Name: secret.Name} + matchingSecrets := validTLSSecretsByName[name] + if len(matchingSecrets) == 0 { + continue + } + resolvedSecrets = append(resolvedSecrets, matchingSecrets[0]) + validTLSSecretsByName[name] = matchingSecrets[1:] + } + + return resolvedSecrets, certs, true } // validateTLSConfiguration validates TLS configuration per protocol. diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 0cf1778775..72fd2002d5 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -47,6 +47,10 @@ var ( ErrTCPRouteSNIsEmpty = errors.New("field SNIs must be specified with at least a single server name entry") ErrTLSCertEmpty = errors.New("field certificate must be specified") ErrTLSPrivateKey = errors.New("field PrivateKey must be specified") + ErrTLSSDSSecretNameEmpty = errors.New("field SDS SecretName must be specified") + ErrTLSSDSSchemeEmpty = errors.New("field SDS Scheme must be specified") + ErrTLSSDSAddressEmpty = errors.New("field SDS Address must be specified") + ErrTLSCertificateMultipleSources = errors.New("only one of SDS or inline certificate fields may be specified") ErrRouteNameEmpty = errors.New("field Name must be specified") ErrHTTPRouteHostnameEmpty = errors.New("field Hostname must be specified") ErrDestinationNameEmpty = errors.New("field Name must be specified") @@ -356,7 +360,8 @@ type HTTPListener struct { Hostnames []string `json:"hostnames" yaml:"hostnames"` // Tls configuration. If omitted, the gateway will expose a plain text HTTP server. TLS *TLSConfig `json:"tls,omitempty" yaml:"tls,omitempty"` - // TLSOverlaps indicates if the listener's certificate SANs overlap with another listener's certificate SANs. + // TLSOverlaps indicates that another listener on the same port either has overlapping certificate SANs or uses an + // SDS-backed certificate whose SANs cannot be inspected. // HTTP/2 should be disabled if this is true to avoid the HTTP/2 Connection Coalescing issue (see https://gateway-api.sigs.k8s.io/geps/gep-3567/) // We use a standalone field to avoid messing with the ClientTrafficPolicy ALPN config. TLSOverlaps bool `json:"tlsOverlaps,omitempty" yaml:"tlsOverlaps,omitempty"` @@ -627,6 +632,21 @@ type SubjectAltName struct { func (t *TLSCertificate) Validate() error { var errs error + if t.SDS != nil { + if len(t.Certificate) > 0 || len(t.PrivateKey) > 0 || len(t.OCSPStaple) > 0 { + errs = errors.Join(errs, ErrTLSCertificateMultipleSources) + } + if t.SDS.SecretName == "" { + errs = errors.Join(errs, ErrTLSSDSSecretNameEmpty) + } + if t.SDS.Scheme == "" { + errs = errors.Join(errs, ErrTLSSDSSchemeEmpty) + } + if t.SDS.Address == "" { + errs = errors.Join(errs, ErrTLSSDSAddressEmpty) + } + return errs + } if len(t.Certificate) == 0 { errs = errors.Join(errs, ErrTLSCertEmpty) } diff --git a/internal/ir/xds_test.go b/internal/ir/xds_test.go index ae7c33c3ac..bb0591d7b8 100644 --- a/internal/ir/xds_test.go +++ b/internal/ir/xds_test.go @@ -634,6 +634,33 @@ func TestValidateXds(t *testing.T) { } } +func TestTLSCertificateValidateDistinguishesMissingSDSFields(t *testing.T) { + tests := []struct { + name string + sds *SDSConfig + want error + }{ + { + name: "scheme", + sds: &SDSConfig{SecretName: "listener", Address: "/var/run/sds.sock"}, + want: ErrTLSSDSSchemeEmpty, + }, + { + name: "address", + sds: &SDSConfig{SecretName: "listener", Scheme: "unix"}, + want: ErrTLSSDSAddressEmpty, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := (&TLSCertificate{SDS: test.sds}).Validate() + + require.ErrorIs(t, err, test.want) + }) + } +} + func TestValidateHTTPListener(t *testing.T) { tests := []struct { name string @@ -741,9 +768,10 @@ func TestValidateTCPListener(t *testing.T) { func TestValidateTLSListenerConfig(t *testing.T) { tests := []struct { - name string - input TLSConfig - want error + name string + input TLSConfig + want error + wantErr bool }{ { name: "happy", @@ -755,6 +783,75 @@ func TestValidateTLSListenerConfig(t *testing.T) { }, want: nil, }, + { + name: "SDS happy", + input: TLSConfig{ + Certificates: []TLSCertificate{{ + SDS: &SDSConfig{ + SecretName: "default", + Scheme: "unix", + Address: "/var/run/secrets/workload-spiffe-uds/socket", + }, + }}, + }, + want: nil, + }, + { + name: "SDS with inline certificate and private key", + input: TLSConfig{ + Certificates: []TLSCertificate{{ + SDS: &SDSConfig{ + SecretName: "default", + Scheme: "unix", + Address: "/var/run/secrets/workload-spiffe-uds/socket", + }, + Certificate: []byte("server-cert"), + PrivateKey: []byte("priv-key"), + }}, + }, + want: ErrTLSCertificateMultipleSources, + }, + { + name: "SDS with inline OCSP staple", + input: TLSConfig{ + Certificates: []TLSCertificate{{ + SDS: &SDSConfig{ + SecretName: "default", + Scheme: "unix", + Address: "/var/run/secrets/workload-spiffe-uds/socket", + }, + OCSPStaple: []byte("ocsp-staple"), + }}, + }, + want: ErrTLSCertificateMultipleSources, + }, + { + name: "SDS empty", + input: TLSConfig{ + Certificates: []TLSCertificate{{ + SDS: &SDSConfig{}, + }}, + }, + wantErr: true, + }, + { + name: "SDS missing URL", + input: TLSConfig{ + Certificates: []TLSCertificate{{ + SDS: &SDSConfig{SecretName: "default"}, + }}, + }, + wantErr: true, + }, + { + name: "SDS missing secret name", + input: TLSConfig{ + Certificates: []TLSCertificate{{ + SDS: &SDSConfig{Scheme: "unix", Address: "/x"}, + }}, + }, + wantErr: true, + }, { name: "invalid server cert", input: TLSConfig{ @@ -776,6 +873,10 @@ func TestValidateTLSListenerConfig(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + if test.wantErr { + require.Error(t, (&test.input).Validate()) + return + } if test.want == nil { require.NoError(t, (&test.input).Validate()) } else { diff --git a/internal/xds/translator/sds.go b/internal/xds/translator/sds.go index f47f358939..37a6e73c44 100644 --- a/internal/xds/translator/sds.go +++ b/internal/xds/translator/sds.go @@ -9,14 +9,17 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "slices" "strings" "time" + "unicode/utf8" cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "github.com/envoyproxy/gateway/internal/ir" @@ -25,6 +28,23 @@ import ( const defaultConnectionTimeout = 10 * time.Second +func sdsClusterNameFromURL(url string) string { + address := strings.TrimPrefix(url, "unix://") + hash := sha256.Sum256([]byte(address)) + const maxReadablePrefixLength = 48 + + hashSuffix := hex.EncodeToString(hash[:16]) + readablePrefix := strings.Trim(strings.ReplaceAll(address, "/", "_"), "_") + for len(readablePrefix) > maxReadablePrefixLength { + _, size := utf8.DecodeLastRuneInString(readablePrefix) + readablePrefix = readablePrefix[:len(readablePrefix)-size] + } + if readablePrefix != "" { + return fmt.Sprintf("sds_%s_%s", readablePrefix, hashSuffix) + } + return fmt.Sprintf("sds_%s", hashSuffix) +} + func sdsSecretConfig(secretName, clusterName string) *tlsv3.SdsSecretConfig { return &tlsv3.SdsSecretConfig{ Name: secretName, @@ -47,50 +67,11 @@ func sdsSecretConfig(secretName, clusterName string) *tlsv3.SdsSecretConfig { } } -// sdsClusterNameFromURL generates a unique cluster name from an SDS URL -func sdsClusterNameFromURL(url string) string { - // Sanitize the URL to create a valid cluster name - // For Unix domain sockets like "unix:///var/run/sds" or "/var/run/sds", create a meaningful name - if strings.HasPrefix(url, "unix://") { - // Unix domain socket with scheme - extract the path - path := strings.TrimPrefix(url, "unix://") - sanitized := strings.ReplaceAll(path, "/", "_") - sanitized = strings.Trim(sanitized, "_") - return fmt.Sprintf("sds_%s", sanitized) - } - if strings.HasPrefix(url, "/") { - // Unix domain socket path without scheme - sanitized := strings.ReplaceAll(url, "/", "_") - sanitized = strings.Trim(sanitized, "_") - return fmt.Sprintf("sds_%s", sanitized) - } - // For other URLs, use a hash - hash := sha256.Sum256([]byte(url)) - return fmt.Sprintf("sds_%s", hex.EncodeToString(hash[:8])) -} - -// createSDSCluster creates an SDS cluster for the given URL -func createSDSCluster(tCtx *types.ResourceVersionTable, sdsURL string) error { +func buildSDSCluster(sdsURL string) *cluster.Cluster { clusterName := sdsClusterNameFromURL(sdsURL) + pipePath := strings.TrimPrefix(sdsURL, "unix://") - // Check if cluster already exists - if tCtx.XdsResources[resourcev3.ClusterType] != nil { - for _, resource := range tCtx.XdsResources[resourcev3.ClusterType] { - if c, ok := resource.(*cluster.Cluster); ok && c.Name == clusterName { - // Cluster already exists - return nil - } - } - } - - // Create the cluster based on the URL type - pipePath := sdsURL - // Extract path for Unix domain sockets - if strings.HasPrefix(sdsURL, "unix://") { - pipePath = strings.TrimPrefix(sdsURL, "unix://") - } - - c := &cluster.Cluster{ + return &cluster.Cluster{ Name: clusterName, ClusterDiscoveryType: &cluster.Cluster_Type{ Type: cluster.Cluster_STATIC, @@ -120,6 +101,18 @@ func createSDSCluster(tCtx *types.ResourceVersionTable, sdsURL string) error { ConnectTimeout: durationpb.New(defaultConnectionTimeout), Http2ProtocolOptions: &corev3.Http2ProtocolOptions{}, } +} + +// createSDSCluster creates an SDS cluster for the given URL +func createSDSCluster(tCtx *types.ResourceVersionTable, sdsURL string) error { + c := buildSDSCluster(sdsURL) + + if existing := findXdsCluster(tCtx, c.Name); existing != nil { + if !proto.Equal(existing, c) { + return fmt.Errorf("SDS cluster %q conflicts with an existing cluster", c.Name) + } + return nil + } if err := tCtx.AddXdsResource(resourcev3.ClusterType, c); err != nil { return err @@ -129,7 +122,7 @@ func createSDSCluster(tCtx *types.ResourceVersionTable, sdsURL string) error { // processSDSClusters scans the IR for SDS URLs and creates clusters for them func processSDSClusters(tCtx *types.ResourceVersionTable, xdsIR *ir.Xds) error { - sdsURLs := make(map[string]bool) + sdsURLs := make(map[string]struct{}) collectSDSURLs := func(dest []*ir.DestinationSetting) { for _, d := range dest { @@ -137,11 +130,11 @@ func processSDSClusters(tCtx *types.ResourceVersionTable, xdsIR *ir.Xds) error { continue } if caCert := d.TLS.CACertificate; caCert != nil && caCert.SDS != nil && caCert.SDS.GetURL() != "" { - sdsURLs[caCert.SDS.GetURL()] = true + sdsURLs[caCert.SDS.GetURL()] = struct{}{} } for _, cert := range d.TLS.ClientCertificates { if cert.SDS != nil && cert.SDS.GetURL() != "" { - sdsURLs[cert.SDS.GetURL()] = true + sdsURLs[cert.SDS.GetURL()] = struct{}{} } } } @@ -152,21 +145,44 @@ func processSDSClusters(tCtx *types.ResourceVersionTable, xdsIR *ir.Xds) error { } for _, httpListener := range xdsIR.HTTP { + if httpListener.TLS != nil { + for _, cert := range httpListener.TLS.Certificates { + if cert.SDS != nil && cert.SDS.GetURL() != "" { + sdsURLs[cert.SDS.GetURL()] = struct{}{} + } + } + } + for _, route := range httpListener.Routes { if route.Destination != nil { collectSDSURLs(route.Destination.Settings) } } } + for _, tcpListener := range xdsIR.TCP { for _, route := range tcpListener.Routes { + if route.TLS != nil && route.TLS.Terminate != nil { + for _, cert := range route.TLS.Terminate.Certificates { + if cert.SDS != nil && cert.SDS.GetURL() != "" { + sdsURLs[cert.SDS.GetURL()] = struct{}{} + } + } + } + if route.Destination != nil { collectSDSURLs(route.Destination.Settings) } } } + urls := make([]string, 0, len(sdsURLs)) for url := range sdsURLs { + urls = append(urls, url) + } + slices.Sort(urls) + + for _, url := range urls { if err := createSDSCluster(tCtx, url); err != nil { return err } diff --git a/internal/xds/translator/sds_test.go b/internal/xds/translator/sds_test.go new file mode 100644 index 0000000000..a86e4b31b0 --- /dev/null +++ b/internal/xds/translator/sds_test.go @@ -0,0 +1,195 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + "strings" + "testing" + "time" + "unicode/utf8" + + cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" + + "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/xds/types" +) + +func TestSDSClusterNameFromURLIncludesReadableUnixPathAndHash(t *testing.T) { + require.Equal(t, "sds_var_run_secrets_workload-spiffe-uds_socket_2465711674b623c4f815575b37404800", sdsClusterNameFromURL("unix:///var/run/secrets/workload-spiffe-uds/socket")) +} + +func TestSDSClusterNameFromURLDistinguishesUnixSocketPaths(t *testing.T) { + first := sdsClusterNameFromURL("/run/a/b/socket") + second := sdsClusterNameFromURL("/run/a_b/socket") + + require.Equal(t, "sds_run_a_b_socket_73917e80488448b0df63fc687c91913f", first) + require.NotEqual(t, first, second) + require.Contains(t, first, "run_a_b_socket") + require.Contains(t, second, "run_a_b_socket") +} + +func TestSDSClusterNameFromURLPreservesValidUTF8(t *testing.T) { + url := "unix:///" + strings.Repeat("a", 47) + "é/socket" + + name := sdsClusterNameFromURL(url) + + require.True(t, utf8.ValidString(name)) +} + +func TestSDSClusterNameFromURLUsesStrongHashWithoutReadablePrefix(t *testing.T) { + name := sdsClusterNameFromURL("unix:///") + + require.Len(t, strings.TrimPrefix(name, "sds_"), 32) +} + +func TestProcessSDSClustersCreatesDistinctClustersForAmbiguousUnixPaths(t *testing.T) { + tCtx := &types.ResourceVersionTable{} + xdsIR := xdsWithHTTPSDSURLs("unix:///run/a/b/socket", "unix:///run/a_b/socket") + + err := processSDSClusters(tCtx, xdsIR) + + require.NoError(t, err) + require.Len(t, tCtx.XdsResources[resourcev3.ClusterType], 2) + require.NotEqual(t, sdsClusterNameFromURL("unix:///run/a/b/socket"), sdsClusterNameFromURL("unix:///run/a_b/socket")) +} + +func TestProcessSDSClustersDeduplicatesSameURLAcrossHTTPAndTCP(t *testing.T) { + sdsURL := "unix:///var/run/secrets/workload-spiffe-uds/socket" + tCtx := &types.ResourceVersionTable{} + xdsIR := xdsWithHTTPSDSURLs(sdsURL) + xdsIR.TCP = []*ir.TCPListener{{ + Routes: []*ir.TCPRoute{{ + TLS: &ir.TLS{Terminate: tlsConfigWithSDSURLs(sdsURL)}, + }}, + }} + + err := processSDSClusters(tCtx, xdsIR) + + require.NoError(t, err) + require.Len(t, tCtx.XdsResources[resourcev3.ClusterType], 1) +} + +func TestProcessSDSClustersOrdersClustersByURL(t *testing.T) { + tCtx := &types.ResourceVersionTable{} + xdsIR := xdsWithHTTPSDSURLs( + "unix:///var/run/sds/z.sock", + "unix:///var/run/sds/a.sock", + "unix:///var/run/sds/m.sock", + ) + + err := processSDSClusters(tCtx, xdsIR) + + require.NoError(t, err) + clusters := tCtx.XdsResources[resourcev3.ClusterType] + clusterA := clusters[0].(*cluster.Cluster) + clusterM := clusters[1].(*cluster.Cluster) + clusterZ := clusters[2].(*cluster.Cluster) + require.Equal(t, []string{ + sdsClusterNameFromURL("unix:///var/run/sds/a.sock"), + sdsClusterNameFromURL("unix:///var/run/sds/m.sock"), + sdsClusterNameFromURL("unix:///var/run/sds/z.sock"), + }, []string{ + clusterA.GetName(), + clusterM.GetName(), + clusterZ.GetName(), + }) +} + +func TestProcessSDSClustersReturnsCollisionForPreexistingNonSDSCluster(t *testing.T) { + sdsURL := "unix:///var/run/sds/socket" + clusterName := sdsClusterNameFromURL(sdsURL) + tCtx := &types.ResourceVersionTable{} + tCtx.XdsResources = types.XdsResources{ + resourcev3.ClusterType: {&cluster.Cluster{Name: clusterName}}, + } + + err := processSDSClusters(tCtx, xdsWithHTTPSDSURLs(sdsURL)) + + require.EqualError(t, err, `SDS cluster "`+clusterName+`" conflicts with an existing cluster`) + require.Len(t, tCtx.XdsResources[resourcev3.ClusterType], 1) +} + +func TestProcessSDSClustersAcceptsPreexistingCanonicalSDSCluster(t *testing.T) { + sdsURL := "unix:///var/run/sds/socket" + tCtx := &types.ResourceVersionTable{} + require.NoError(t, createSDSCluster(tCtx, sdsURL)) + + err := processSDSClusters(tCtx, xdsWithHTTPSDSURLs(sdsURL)) + + require.NoError(t, err) + require.Len(t, tCtx.XdsResources[resourcev3.ClusterType], 1) +} + +func TestProcessSDSClustersReturnsCollisionWhenPreexistingClusterMissesHTTP2Options(t *testing.T) { + sdsURL := "unix:///var/run/sds/socket" + tCtx := &types.ResourceVersionTable{} + require.NoError(t, createSDSCluster(tCtx, sdsURL)) + existing := findXdsCluster(tCtx, sdsClusterNameFromURL(sdsURL)) + require.NotNil(t, existing) + http2Options := existing.ProtoReflect().Descriptor().Fields().ByName("http2_protocol_options") + existing.ProtoReflect().Clear(http2Options) + + err := processSDSClusters(tCtx, xdsWithHTTPSDSURLs(sdsURL)) + + require.EqualError(t, err, sdsCollisionError(sdsURL)) + require.Len(t, tCtx.XdsResources[resourcev3.ClusterType], 1) +} + +func TestProcessSDSClustersReturnsCollisionWhenPreexistingClusterHasWrongConnectTimeout(t *testing.T) { + sdsURL := "unix:///var/run/sds/socket" + tCtx := &types.ResourceVersionTable{} + require.NoError(t, createSDSCluster(tCtx, sdsURL)) + findXdsCluster(tCtx, sdsClusterNameFromURL(sdsURL)).ConnectTimeout = durationpb.New(5 * time.Second) + + err := processSDSClusters(tCtx, xdsWithHTTPSDSURLs(sdsURL)) + + require.EqualError(t, err, sdsCollisionError(sdsURL)) + require.Len(t, tCtx.XdsResources[resourcev3.ClusterType], 1) +} + +func TestProcessSDSClustersReturnsCollisionWhenPreexistingClusterHasEmptyLoadAssignment(t *testing.T) { + sdsURL := "unix:///var/run/sds/socket" + tCtx := &types.ResourceVersionTable{} + require.NoError(t, createSDSCluster(tCtx, sdsURL)) + clusterName := sdsClusterNameFromURL(sdsURL) + findXdsCluster(tCtx, clusterName).LoadAssignment = &endpoint.ClusterLoadAssignment{ClusterName: clusterName} + + err := processSDSClusters(tCtx, xdsWithHTTPSDSURLs(sdsURL)) + + require.EqualError(t, err, sdsCollisionError(sdsURL)) + require.Len(t, tCtx.XdsResources[resourcev3.ClusterType], 1) +} + +func xdsWithHTTPSDSURLs(urls ...string) *ir.Xds { + return &ir.Xds{ + HTTP: []*ir.HTTPListener{{ + TLS: tlsConfigWithSDSURLs(urls...), + }}, + } +} + +func tlsConfigWithSDSURLs(urls ...string) *ir.TLSConfig { + certificates := make([]ir.TLSCertificate, 0, len(urls)) + for _, url := range urls { + certificates = append(certificates, ir.TLSCertificate{ + Name: strings.Trim(url, "/"), + SDS: &ir.SDSConfig{ + SecretName: "default", + Scheme: "unix", + Address: strings.TrimPrefix(url, "unix://"), + }, + }) + } + return &ir.TLSConfig{Certificates: certificates} +} + +func sdsCollisionError(sdsURL string) string { + return `SDS cluster "` + sdsClusterNameFromURL(sdsURL) + `" conflicts with an existing cluster` +} diff --git a/internal/xds/translator/testdata/in/xds-ir/sds-listener.yaml b/internal/xds/translator/testdata/in/xds-ir/sds-listener.yaml new file mode 100644 index 0000000000..7154fb5b59 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/sds-listener.yaml @@ -0,0 +1,86 @@ +http: +- name: test1 + address: 0.0.0.0 + port: 10080 + hostnames: + - example.com + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + tls: + certificates: + - name: sds-default + sds: + secretName: default + scheme: unix + address: /var/run/secrets/workload-spiffe-uds/socket + - name: sds-other + sds: + secretName: other + scheme: unix + address: /var/run/sds/two/socket + - name: sds-underscore + sds: + secretName: underscore + scheme: unix + address: /var/run/sds/underscore/socket + - name: inline + certificate: Y2VydC1kYXRh + privateKey: a2V5LWRhdGE= + tlsOverlaps: true + routes: + - name: route1 + hostname: '*' + destination: + name: test1-dest + settings: + - endpoints: + - host: 1.2.3.4 + port: 50000 + name: test1-dest/backend/0 +- name: test2 + address: 0.0.0.0 + port: 10081 + hostnames: + - '*' + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + routes: + - name: route2 + hostname: '*' + destination: + name: test2-dest + settings: + - endpoints: + - host: 2.3.4.5 + port: 50001 + name: test2-dest/backend/0 +tcp: +- name: test3 + address: 0.0.0.0 + port: 10082 + tls: + certificates: + - name: sds-default + sds: + secretName: default + scheme: unix + address: /var/run/secrets/workload-spiffe-uds/socket + routes: + - name: route3 + tls: + terminate: + certificates: + - name: sds-default + sds: + secretName: default + scheme: unix + address: /var/run/secrets/workload-spiffe-uds/socket + destination: + name: test3-dest + settings: + - endpoints: + - host: 3.4.5.6 + port: 50002 + name: test3-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/sds-listener.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/sds-listener.clusters.yaml new file mode 100644 index 0000000000..cfddbf7ccb --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/sds-listener.clusters.yaml @@ -0,0 +1,105 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: test1-dest + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: test1-dest + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: test2-dest + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: test2-dest + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: test3-dest + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: test3-dest + perConnectionBufferLimitBytes: 32768 + type: EDS +- connectTimeout: 10s + http2ProtocolOptions: {} + loadAssignment: + clusterName: sds_var_run_sds_two_socket_bf395108472af10505ea7b43dde5a5e9 + endpoints: + - lbEndpoints: + - endpoint: + address: + pipe: + path: /var/run/sds/two/socket + name: sds_var_run_sds_two_socket_bf395108472af10505ea7b43dde5a5e9 + type: STATIC +- connectTimeout: 10s + http2ProtocolOptions: {} + loadAssignment: + clusterName: sds_var_run_sds_underscore_socket_e84255b605ecebe05308d449b6fe20d6 + endpoints: + - lbEndpoints: + - endpoint: + address: + pipe: + path: /var/run/sds/underscore/socket + name: sds_var_run_sds_underscore_socket_e84255b605ecebe05308d449b6fe20d6 + type: STATIC +- connectTimeout: 10s + http2ProtocolOptions: {} + loadAssignment: + clusterName: sds_var_run_secrets_workload-spiffe-uds_socket_2465711674b623c4f815575b37404800 + endpoints: + - lbEndpoints: + - endpoint: + address: + pipe: + path: /var/run/secrets/workload-spiffe-uds/socket + name: sds_var_run_secrets_workload-spiffe-uds_socket_2465711674b623c4f815575b37404800 + type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/sds-listener.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/sds-listener.endpoints.yaml new file mode 100644 index 0000000000..a013cab95d --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/sds-listener.endpoints.yaml @@ -0,0 +1,36 @@ +- clusterName: test1-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: test1-dest/backend/0 +- clusterName: test2-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 2.3.4.5 + portValue: 50001 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: test2-dest/backend/0 +- clusterName: test3-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 3.4.5.6 + portValue: 50002 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: test3-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/sds-listener.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/sds-listener.listeners.yaml new file mode 100644 index 0000000000..abe5c767a4 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/sds-listener.listeners.yaml @@ -0,0 +1,148 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + filterChains: + - filterChainMatch: + serverNames: + - example.com + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + initialFetchTimeout: 0s + resourceApiVersion: V3 + routeConfigName: test1 + serverHeaderTransformation: PASS_THROUGH + statPrefix: https-10080 + useRemoteAddress: true + name: test1 + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + commonTlsContext: + alpnProtocols: + - http/1.1 + tlsCertificateSdsSecretConfigs: + - name: default + sdsConfig: + apiConfigSource: + apiType: GRPC + grpcServices: + - envoyGrpc: + clusterName: sds_var_run_secrets_workload-spiffe-uds_socket_2465711674b623c4f815575b37404800 + - name: other + sdsConfig: + apiConfigSource: + apiType: GRPC + grpcServices: + - envoyGrpc: + clusterName: sds_var_run_sds_two_socket_bf395108472af10505ea7b43dde5a5e9 + - name: underscore + sdsConfig: + apiConfigSource: + apiType: GRPC + grpcServices: + - envoyGrpc: + clusterName: sds_var_run_sds_underscore_socket_e84255b605ecebe05308d449b6fe20d6 + - name: inline + sdsConfig: + ads: {} + initialFetchTimeout: 0s + resourceApiVersion: V3 + disableStatefulSessionResumption: true + disableStatelessSessionResumption: true + listenerFilters: + - name: envoy.filters.listener.tls_inspector + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + maxConnectionsToAcceptPerSocketEvent: 1 + name: test1 + perConnectionBufferLimitBytes: 32768 +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10081 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + initialFetchTimeout: 0s + resourceApiVersion: V3 + routeConfigName: test2 + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10081 + useRemoteAddress: true + name: test2 + maxConnectionsToAcceptPerSocketEvent: 1 + name: test2 + perConnectionBufferLimitBytes: 32768 +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10082 + filterChains: + - filters: + - name: envoy.filters.network.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: test3-dest + statPrefix: tls-terminate-10082 + name: route3 + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + commonTlsContext: + alpnProtocols: + - h2 + - http/1.1 + tlsCertificateSdsSecretConfigs: + - name: default + sdsConfig: + apiConfigSource: + apiType: GRPC + grpcServices: + - envoyGrpc: + clusterName: sds_var_run_secrets_workload-spiffe-uds_socket_2465711674b623c4f815575b37404800 + disableStatefulSessionResumption: true + disableStatelessSessionResumption: true + maxConnectionsToAcceptPerSocketEvent: 1 + name: test3 + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/sds-listener.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/sds-listener.routes.yaml new file mode 100644 index 0000000000..1dd09e2cea --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/sds-listener.routes.yaml @@ -0,0 +1,28 @@ +- ignorePortInHostMatching: true + name: test1 + virtualHosts: + - domains: + - '*' + name: test1/* + routes: + - match: + prefix: / + name: route1 + route: + cluster: test1-dest + upgradeConfigs: + - upgradeType: websocket +- ignorePortInHostMatching: true + name: test2 + virtualHosts: + - domains: + - '*' + name: test2/* + routes: + - match: + prefix: / + name: route2 + route: + cluster: test2-dest + upgradeConfigs: + - upgradeType: websocket diff --git a/internal/xds/translator/testdata/out/xds-ir/sds-listener.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/sds-listener.secrets.yaml new file mode 100644 index 0000000000..33574a4256 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/sds-listener.secrets.yaml @@ -0,0 +1,6 @@ +- name: inline + tlsCertificate: + certificateChain: + inlineBytes: Y2VydC1kYXRh + privateKey: + inlineBytes: a2V5LWRhdGE= diff --git a/internal/xds/translator/testdata/out/xds-ir/sds.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/sds.clusters.yaml index 97e9bec4c6..74008c34dd 100644 --- a/internal/xds/translator/testdata/out/xds-ir/sds.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/sds.clusters.yaml @@ -56,7 +56,7 @@ apiType: GRPC grpcServices: - envoyGrpc: - clusterName: sds_var_run_secrets_workload-spiffe-uds_socket + clusterName: sds_var_run_secrets_workload-spiffe-uds_socket_2465711674b623c4f815575b37404800 tlsCertificateSdsSecretConfigs: - name: default sdsConfig: @@ -64,7 +64,7 @@ apiType: GRPC grpcServices: - envoyGrpc: - clusterName: sds_var_run_secrets_workload-spiffe-uds_socket + clusterName: sds_var_run_secrets_workload-spiffe-uds_socket_2465711674b623c4f815575b37404800 sni: example.com type: EDS typedExtensionProtocolOptions: @@ -78,12 +78,12 @@ - connectTimeout: 10s http2ProtocolOptions: {} loadAssignment: - clusterName: sds_var_run_secrets_workload-spiffe-uds_socket + clusterName: sds_var_run_secrets_workload-spiffe-uds_socket_2465711674b623c4f815575b37404800 endpoints: - lbEndpoints: - endpoint: address: pipe: path: /var/run/secrets/workload-spiffe-uds/socket - name: sds_var_run_secrets_workload-spiffe-uds_socket + name: sds_var_run_secrets_workload-spiffe-uds_socket_2465711674b623c4f815575b37404800 type: STATIC diff --git a/release-notes/current/breaking_changes/9525-hash-sds-unix-socket-cluster-names.md b/release-notes/current/breaking_changes/9525-hash-sds-unix-socket-cluster-names.md new file mode 100644 index 0000000000..d89ce61741 --- /dev/null +++ b/release-notes/current/breaking_changes/9525-hash-sds-unix-socket-cluster-names.md @@ -0,0 +1 @@ +SDS clusters generated for Unix socket URLs now include a hash suffix in their xDS names to prevent collisions between distinct paths. EnvoyPatchPolicies or extension servers that match the previous path-derived cluster names must update those references. diff --git a/release-notes/current/new_features/9525-sds-listener-certificate-refs.md b/release-notes/current/new_features/9525-sds-listener-certificate-refs.md new file mode 100644 index 0000000000..106e7a4184 --- /dev/null +++ b/release-notes/current/new_features/9525-sds-listener-certificate-refs.md @@ -0,0 +1 @@ +Added support for referencing a Secret of type `gateway.envoyproxy.io/sds` in a Gateway listener's `tls.certificateRefs`, letting Envoy fetch the listener certificate from an external SDS server instead of an inline `kubernetes.io/tls` Secret, gated by the existing `enableSDSSecretRef` EnvoyGateway extension API flag. When multiple valid HTTPS listeners share a port, SDS-backed listeners default to HTTP/1.1 because their certificate DNS names are opaque; same-port listeners are also downgraded when their known certificate DNS names overlap the SDS listener hostname, or when that hostname is unspecified. Affected listeners report `gateway.envoyproxy.io/TLSCertificateNamesUnknown=True` with reason `SDSCertificateOpaque`. diff --git a/site/content/en/latest/tasks/security/secure-gateways.md b/site/content/en/latest/tasks/security/secure-gateways.md index 182873f424..badfb856e2 100644 --- a/site/content/en/latest/tasks/security/secure-gateways.md +++ b/site/content/en/latest/tasks/security/secure-gateways.md @@ -16,6 +16,7 @@ This task uses a self-signed CA, so it should be used for testing and demonstrat ## TLS Certificates Generate the certificates and keys used by the Gateway to terminate client TLS connections. +For certificates delivered by an external SDS server, see [SDS Certificate References][]. Create a root certificate and private key to sign certificates: @@ -779,3 +780,4 @@ Checkout the [Developer Guide](/community/develop) to get involved in the projec [ReferenceGrant]: https://gateway-api.sigs.k8s.io/reference/api-types/referencegrant/ [ClientTrafficPolicy]: ../../api/extension_types#clienttrafficpolicy +[SDS Certificate References]: ../tls-termination/#sds-certificate-references diff --git a/site/content/en/latest/tasks/security/tls-termination.md b/site/content/en/latest/tasks/security/tls-termination.md index 9f79b8b8af..3b16d03383 100644 --- a/site/content/en/latest/tasks/security/tls-termination.md +++ b/site/content/en/latest/tasks/security/tls-termination.md @@ -251,6 +251,119 @@ curl -v -HHost:backend-2.example.com --resolve "backend-2.example.com:443:${GATE The echo response includes the serving pod's name (`POD_NAME`), so each request lands on a different backend Service even though they share one listener. +## SDS Certificate References + +Besides inline `kubernetes.io/tls` Secrets, a listener's `tls.certificateRefs` can reference a Secret of type `gateway.envoyproxy.io/sds`. Rather than embedding a certificate and key directly, this Secret tells Envoy to fetch the certificate at runtime from an external Secret Discovery Service (SDS) server, identified by two keys: `url` (the scheme-qualified SDS server Unix domain socket URL) and `secretName` (the resource name Envoy requests from that server): + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: sds-cert +type: gateway.envoyproxy.io/sds +stringData: + url: unix:///var/run/secrets/workload-spiffe-uds/socket + secretName: default +``` + +The server in `examples/sds-test-server` is an E2E fixture, not a production SDS implementation. + +A listener references it the same way it references any other certificate Secret: + +```yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: eg +spec: + gatewayClassName: eg + listeners: + - name: tls + protocol: TLS + port: 443 + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: sds-cert +``` + +This feature is disabled by default. Add `enableSDSSecretRef` under `extensionApis` in the `EnvoyGateway` configuration stored in the `envoy-gateway-config` ConfigMap: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: envoy-gateway-config + namespace: envoy-gateway-system +data: + envoy-gateway.yaml: | + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyGateway + gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + provider: + type: Kubernetes + extensionApis: + enableSDSSecretRef: true +``` + +Preserve any other settings already present in `envoy-gateway.yaml`, then apply the ConfigMap and restart Envoy Gateway: + +```shell +kubectl apply -f envoy-gateway-config.yaml +kubectl rollout restart deployment/envoy-gateway -n envoy-gateway-system +kubectl rollout status deployment/envoy-gateway -n envoy-gateway-system +``` + +The SDS Unix domain socket must also be mounted into the Envoy Proxy pod at the path specified by `url`. For example, the following `EnvoyProxy` mounts a socket directory provided on every Kubernetes node: + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: sds-socket + namespace: envoy-gateway-system +spec: + provider: + type: Kubernetes + kubernetes: + envoyDeployment: + patch: + type: StrategicMerge + value: + spec: + template: + spec: + volumes: + - name: sds-socket + hostPath: + path: /var/run/secrets/workload-spiffe-uds + type: Directory + containers: + - name: envoy + volumeMounts: + - name: sds-socket + mountPath: /var/run/secrets/workload-spiffe-uds +``` + +Attach the `EnvoyProxy` to the `GatewayClass` used by the Gateway: + +```shell +kubectl apply -f envoyproxy-sds.yaml +kubectl patch gatewayclass eg --type=merge --patch '{"spec":{"parametersRef":{"group":"gateway.envoyproxy.io","kind":"EnvoyProxy","name":"sds-socket","namespace":"envoy-gateway-system"}}}' +``` + +Replace the `hostPath` with the directory used by the SDS provider. The directory must exist on every node that can run an Envoy Proxy pod. The Envoy process must have permission to connect to the socket. If the `GatewayClass` already references an `EnvoyProxy`, add the volume and mount to that resource instead of replacing `parametersRef`. Enabling SDS Secret references does not add the socket or change the proxy deployment automatically. + +Kubernetes authorization and ReferenceGrant control access to the Secret containing the SDS connection details. The SDS server separately authorizes the `secretName` requested by Envoy using the proxy's SDS identity. Only enable SDS Secret references when users who can create or reference these Secrets are trusted to request the SDS resources available to that identity. Use separate Envoy Proxy deployments or SDS identities for mutually untrusted tenants. + +Envoy waits for SDS-backed listener certificates before activating the listener. Envoy Gateway combines HTTPS listeners on the same address and port into one Envoy listener, so an unavailable SDS server or resource can delay activation or reset connections for every listener sharing that address and port. + +To prevent HTTP/2 connection coalescing across HTTPS listeners that share a port, Envoy Gateway normally compares certificate DNS/SAN names and defaults listeners with overlapping certificates to HTTP/1.1 unless ALPN is explicitly configured. Because SDS-backed certificates are opaque to Envoy Gateway, an SDS-backed listener defaults to HTTP/1.1 when multiple valid HTTPS listeners share its port. A same-port listener using an inline TLS Secret also defaults to HTTP/1.1 when one of its known certificate DNS/SAN names overlaps the SDS listener's configured hostname. If the SDS listener omits `hostname` and therefore matches all hostnames, every valid same-port HTTPS listener defaults to HTTP/1.1. A standalone SDS-backed listener retains HTTP/2. + +Affected listeners report the Envoy Gateway condition `gateway.envoyproxy.io/TLSCertificateNamesUnknown=True` with reason `SDSCertificateOpaque`. Configure ALPN explicitly with ClientTrafficPolicy to override the default. + [TCPRoute]: https://gateway-api.sigs.k8s.io/reference/api-spec/main/spec/#tcproute [TLSRoute]: https://gateway-api.sigs.k8s.io/reference/api-spec/main/spec/#tlsroute [tls-passthrough]: ../tls-passthrough/ diff --git a/test/config/envoy-gateaway-config/default.yaml b/test/config/envoy-gateaway-config/default.yaml index 7b37e9818b..1e3f4d7427 100644 --- a/test/config/envoy-gateaway-config/default.yaml +++ b/test/config/envoy-gateaway-config/default.yaml @@ -15,6 +15,7 @@ data: enableEnvoyPatchPolicy: true enableBackend: true enableLua: true + enableSDSSecretRef: true gatewayAPI: enabled: [XListenerSet] rateLimit: diff --git a/test/config/envoy-gateaway-config/gateway-namespace-mode.yaml b/test/config/envoy-gateaway-config/gateway-namespace-mode.yaml index 93183361f3..166a222e0c 100644 --- a/test/config/envoy-gateaway-config/gateway-namespace-mode.yaml +++ b/test/config/envoy-gateaway-config/gateway-namespace-mode.yaml @@ -18,6 +18,7 @@ data: enableEnvoyPatchPolicy: true enableBackend: true enableLua: true + enableSDSSecretRef: true gatewayAPI: enabled: [XListenerSet] rateLimit: diff --git a/test/config/envoy-gateaway-config/watch-namespaces.yaml b/test/config/envoy-gateaway-config/watch-namespaces.yaml index 62675a6138..fdddb880c4 100644 --- a/test/config/envoy-gateaway-config/watch-namespaces.yaml +++ b/test/config/envoy-gateaway-config/watch-namespaces.yaml @@ -32,6 +32,7 @@ data: enableEnvoyPatchPolicy: true enableBackend: true enableLua: true + enableSDSSecretRef: true gatewayAPI: enabled: [XListenerSet] rateLimit: diff --git a/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml b/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml index ec28c305f5..6e2ac490e4 100644 --- a/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml +++ b/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml @@ -15,6 +15,7 @@ data: enableEnvoyPatchPolicy: true enableBackend: true enableLua: true + enableSDSSecretRef: true gatewayAPI: enabled: [XListenerSet] rateLimit: diff --git a/test/e2e/testdata/sds-listener-certificate.yaml b/test/e2e/testdata/sds-listener-certificate.yaml new file mode 100644 index 0000000000..f27105795d --- /dev/null +++ b/test/e2e/testdata/sds-listener-certificate.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +kind: Secret +metadata: + name: sds-listener-certificate-source + namespace: gateway-conformance-infra +type: kubernetes.io/tls +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUIzVENDQVlPZ0F3SUJBZ0lVSjl1Z1JKaDJNL3VZU2ZnZk13UkFMK1RSa3Njd0NnWUlLb1pJemowRUF3SXcKTmpFYU1CZ0dBMVVFQ2d3UlJXNTJiM2tnUjJGMFpYZGhlU0JGTWtVeEdEQVdCZ05WQkFNTUQzTmtjeTVsZUdGdApjR3hsTG1OdmJUQWVGdzB5TmpBM01UY3hORFExTWpKYUZ3MHpOakEzTVRReE5EUTFNakphTURZeEdqQVlCZ05WCkJBb01FVVZ1ZG05NUlFZGhkR1YzWVhrZ1JUSkZNUmd3RmdZRFZRUUREQTl6WkhNdVpYaGhiWEJzWlM1amIyMHcKV1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVRwVC9WNGxKZ0puS2hBcE5vRDZZemE2VGFOV1JDcQpDc1ZjVWlxTnhWNjNzd21RQnlUSjU5Z3hwTTY4SU1UbXhHMyticVljdW9PNnhKRFozcTFsVHhHc28yOHdiVEFkCkJnTlZIUTRFRmdRVTh2dHo0c01WRk85dyswR0h2LzFBZENhU0ZBQXdId1lEVlIwakJCZ3dGb0FVOHZ0ejRzTVYKRk85dyswR0h2LzFBZENhU0ZBQXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWFCZ05WSFJFRUV6QVJnZzl6WkhNdQpaWGhoYlhCc1pTNWpiMjB3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnY05CN284OFR4M2tDUUFkczRENXlPcldBCjhWa2Z1MHhuRlZsU1JIU0szSE1DSVFDU0lSVXVOMnJacmd4eG9sN2R4L0l3OXhQWFdVVFdxM24vNlhwQU45V2gKZ2c9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUVWWVVGTU4wUEpsdlQyZ0VEeGhWenNQTUtJcFAxYlE3UmhKNkY4aFE5dWNvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFNlUvMWVKU1lDWnlvUUtUYUErbU0ydWsyalZrUXFnckZYRklxamNWZXQ3TUprQWNreWVmWQpNYVRPdkNERTVzUnQvbTZtSExxRHVzU1EyZDZ0WlU4UnJBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +--- +# The data-plane pod can run in either namespace depending on the test suite's namespace mode, so both namespaces need the same source key material. +apiVersion: v1 +kind: Secret +metadata: + name: sds-listener-certificate-source + namespace: envoy-gateway-system +type: kubernetes.io/tls +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUIzVENDQVlPZ0F3SUJBZ0lVSjl1Z1JKaDJNL3VZU2ZnZk13UkFMK1RSa3Njd0NnWUlLb1pJemowRUF3SXcKTmpFYU1CZ0dBMVVFQ2d3UlJXNTJiM2tnUjJGMFpYZGhlU0JGTWtVeEdEQVdCZ05WQkFNTUQzTmtjeTVsZUdGdApjR3hsTG1OdmJUQWVGdzB5TmpBM01UY3hORFExTWpKYUZ3MHpOakEzTVRReE5EUTFNakphTURZeEdqQVlCZ05WCkJBb01FVVZ1ZG05NUlFZGhkR1YzWVhrZ1JUSkZNUmd3RmdZRFZRUUREQTl6WkhNdVpYaGhiWEJzWlM1amIyMHcKV1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVRwVC9WNGxKZ0puS2hBcE5vRDZZemE2VGFOV1JDcQpDc1ZjVWlxTnhWNjNzd21RQnlUSjU5Z3hwTTY4SU1UbXhHMyticVljdW9PNnhKRFozcTFsVHhHc28yOHdiVEFkCkJnTlZIUTRFRmdRVTh2dHo0c01WRk85dyswR0h2LzFBZENhU0ZBQXdId1lEVlIwakJCZ3dGb0FVOHZ0ejRzTVYKRk85dyswR0h2LzFBZENhU0ZBQXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWFCZ05WSFJFRUV6QVJnZzl6WkhNdQpaWGhoYlhCc1pTNWpiMjB3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnY05CN284OFR4M2tDUUFkczRENXlPcldBCjhWa2Z1MHhuRlZsU1JIU0szSE1DSVFDU0lSVXVOMnJacmd4eG9sN2R4L0l3OXhQWFdVVFdxM24vNlhwQU45V2gKZ2c9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUVWWVVGTU4wUEpsdlQyZ0VEeGhWenNQTUtJcFAxYlE3UmhKNkY4aFE5dWNvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFNlUvMWVKU1lDWnlvUUtUYUErbU0ydWsyalZrUXFnckZYRklxamNWZXQ3TUprQWNreWVmWQpNYVRPdkNERTVzUnQvbTZtSExxRHVzU1EyZDZ0WlU4UnJBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +--- +apiVersion: v1 +kind: Secret +metadata: + name: sds-listener-certificate + namespace: gateway-conformance-infra +type: gateway.envoyproxy.io/sds +stringData: + url: unix:///var/run/sds/server.sock + secretName: sds-listener-certificate +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: sds-listener-certificate + namespace: gateway-conformance-infra +spec: + ipFamily: IPv4 + provider: + type: Kubernetes + kubernetes: + envoyDeployment: + patch: + type: StrategicMerge + value: + spec: + template: + spec: + volumes: + - name: sds-socket + emptyDir: {} + - name: sds-listener-certificate-source + secret: + secretName: sds-listener-certificate-source + containers: + - name: envoy + volumeMounts: + - name: sds-socket + mountPath: /var/run/sds + - name: sds-test-server + image: envoyproxy/gateway-sds-test-server:latest + imagePullPolicy: IfNotPresent + args: + - --socket-path=/var/run/sds/server.sock + - --secret-name=sds-listener-certificate + - --cert-path=/var/run/certificate/tls.crt + - --key-path=/var/run/certificate/tls.key + volumeMounts: + - name: sds-socket + mountPath: /var/run/sds + - name: sds-listener-certificate-source + mountPath: /var/run/certificate + readOnly: true +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: sds-listener-certificate + namespace: gateway-conformance-infra +spec: + gatewayClassName: "{GATEWAY_CLASS_NAME}" + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: sds-listener-certificate + listeners: + - name: https + protocol: HTTPS + port: 443 + hostname: sds.example.com + tls: + mode: Terminate + certificateRefs: + - group: "" + kind: Secret + name: sds-listener-certificate +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: sds-listener-certificate + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: sds-listener-certificate + sectionName: https + hostnames: + - sds.example.com + rules: + - matches: + - path: + type: PathPrefix + value: /sds-listener + backendRefs: + - name: infra-backend-v1 + port: 8080 diff --git a/test/e2e/tests/sds_listener_certificate.go b/test/e2e/tests/sds_listener_certificate.go new file mode 100644 index 0000000000..0ae15627c6 --- /dev/null +++ b/test/e2e/tests/sds_listener_certificate.go @@ -0,0 +1,123 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build e2e + +package tests + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api/conformance/utils/suite" + tlsutils "sigs.k8s.io/gateway-api/conformance/utils/tls" +) + +func init() { + ConformanceTests = append(ConformanceTests, SDSListenerCertificateTest) +} + +var SDSListenerCertificateTest = suite.ConformanceTest{ + ShortName: "SDSListenerCertificate", + Description: "Use an SDS-backed Secret for HTTPS listener certificate delivery", + Manifests: []string{"testdata/sds-listener-certificate.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + const ( + namespace = "gateway-conformance-infra" + serverName = "sds.example.com" + ) + gatewayName := types.NamespacedName{Name: "sds-listener-certificate", Namespace: namespace} + routeName := types.NamespacedName{Name: "sds-listener-certificate", Namespace: namespace} + + // Given an accepted HTTPS Gateway whose certificate reference is backed by SDS. + gatewayHost := kubernetes.GatewayAndRoutesMustBeAccepted( + t, + suite.Client, + suite.TimeoutConfig, + suite.ControllerName, + kubernetes.NewGatewayRef(gatewayName), + &gwapiv1.HTTPRoute{}, + false, + routeName, + ) + certificatePEM, _, _, err := GetTLSSecret( + suite.Client, + types.NamespacedName{Name: "sds-listener-certificate-source", Namespace: namespace}, + ) + require.NoError(t, err) + gwPodNamespace := GetGatewayResourceNamespace() + WaitForPods( + t, + suite.Client, + gwPodNamespace, + map[string]string{ + "gateway.envoyproxy.io/owning-gateway-name": gatewayName.Name, + "gateway.envoyproxy.io/owning-gateway-namespace": gatewayName.Namespace, + }, + corev1.PodRunning, + &PodReady, + ) + + // When traffic is sent through the SDS-backed HTTPS listener. + expected := http.ExpectedResponse{ + Request: http.Request{Host: serverName, Path: "/sds-listener"}, + Response: http.Response{ + StatusCodes: []int{200}, + }, + Namespace: namespace, + } + tlsutils.MakeTLSRequestAndExpectEventuallyConsistentResponse( + t, + suite.RoundTripper, + suite.TimeoutConfig, + gatewayHost, + certificatePEM, + nil, + nil, + serverName, + expected, + ) + + // Then Envoy presents the exact certificate delivered by the SDS server. + expectedCertificate := parseCertificate(t, certificatePEM) + rootCAs := x509.NewCertPool() + require.True(t, rootCAs.AppendCertsFromPEM(certificatePEM)) + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: rootCAs, + ServerName: serverName, + } + connection, err := tls.DialWithDialer( + &net.Dialer{Timeout: 10 * time.Second}, + "tcp", + net.JoinHostPort(gatewayHost, "443"), + tlsConfig, + ) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, connection.Close()) }) + require.NotEmpty(t, connection.ConnectionState().PeerCertificates) + require.True(t, bytes.Equal(expectedCertificate.Raw, connection.ConnectionState().PeerCertificates[0].Raw)) + }, +} + +func parseCertificate(t *testing.T, certificatePEM []byte) *x509.Certificate { + t.Helper() + block, _ := pem.Decode(certificatePEM) + require.NotNil(t, block) + certificate, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + return certificate +} diff --git a/tools/make/examples.mk b/tools/make/examples.mk index 7d904e0cfd..f7b8708a94 100644 --- a/tools/make/examples.mk +++ b/tools/make/examples.mk @@ -1,4 +1,4 @@ -EXAMPLE_APPS := simple-extension-server extension-server envoy-ext-auth grpc-ext-proc preserve-case-backend static-file-server dynamic-module-test backend-utilization remote-infra +EXAMPLE_APPS := simple-extension-server extension-server envoy-ext-auth grpc-ext-proc preserve-case-backend static-file-server dynamic-module-test backend-utilization remote-infra sds-test-server EXAMPLE_IMAGE_PREFIX ?= envoyproxy/gateway- EXAMPLE_TAG ?= latest