Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions internal/provider/kubernetes/controller_offline.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ func newOfflineGatewayAPIClient(extensionPolicies []schema.GroupVersionKind, ena
WithIndex(&gwapiv1.Gateway{}, classGatewayIndex, gatewayIndexFunc).
WithIndex(&gwapiv1.Gateway{}, secretGatewayIndex, secretGatewayIndexFunc).
WithIndex(&gwapiv1.ListenerSet{}, gatewayListenerSetIndex, gatewayListenerSetIndexFunc).
WithIndex(&gwapiv1.ListenerSet{}, secretListenerSetIndex, secretListenerSetIndexFunc).
WithIndex(&gwapiv1.HTTPRoute{}, gatewayHTTPRouteIndex, gatewayHTTPRouteIndexFunc).
WithIndex(&gwapiv1.HTTPRoute{}, backendHTTPRouteIndex, backendHTTPRouteIndexFunc).
WithIndex(&gwapiv1.HTTPRoute{}, listenerSetHTTPRouteIndex, listenerSetHTTPRouteIndexFunc).
Expand Down
2 changes: 2 additions & 0 deletions internal/provider/kubernetes/controller_offline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ func TestNewOfflineGatewayAPIControllerIndexRegistration(t *testing.T) {
t.Run("ListenerSet index", func(t *testing.T) {
err := cli.List(context.Background(), &gwapiv1.ListenerSetList{}, client.MatchingFields{gatewayListenerSetIndex: "any"})
require.NoError(t, err)
err = cli.List(context.Background(), &gwapiv1.ListenerSetList{}, client.MatchingFields{secretListenerSetIndex: "any"})
require.NoError(t, err)
})

t.Run("HTTPRoute indices", func(t *testing.T) {
Expand Down
27 changes: 27 additions & 0 deletions internal/provider/kubernetes/indexers.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ const (
gatewayTCPRouteIndex = "gatewayTCPRouteIndex"
gatewayUDPRouteIndex = "gatewayUDPRouteIndex"
secretGatewayIndex = "secretGatewayIndex"
secretListenerSetIndex = "secretListenerSetIndex"
targetRefGrantRouteIndex = "targetRefGrantRouteIndex"
backendHTTPRouteIndex = "backendHTTPRouteIndex"
backendGRPCRouteIndex = "backendGRPCRouteIndex"
Expand Down Expand Up @@ -139,6 +140,9 @@ func addListenerSetIndexers(ctx context.Context, mgr manager.Manager) error {
if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1.ListenerSet{}, gatewayListenerSetIndex, gatewayListenerSetIndexFunc); err != nil {
return err
}
if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1.ListenerSet{}, secretListenerSetIndex, secretListenerSetIndexFunc); err != nil {
return err
}
return nil
}

Expand Down Expand Up @@ -746,6 +750,29 @@ func secretGatewayIndexFunc(rawObj client.Object) []string {
return secretReferences
}

// secretListenerSetIndexFunc indexes ListenerSet objects by the Secrets they
// reference in listeners[].tls.certificateRefs, mirroring secretGatewayIndexFunc.
func secretListenerSetIndexFunc(rawObj client.Object) []string {
listenerSet := rawObj.(*gwapiv1.ListenerSet)
var secretReferences []string
for _, listener := range listenerSet.Spec.Listeners {
if listener.TLS == nil || *listener.TLS.Mode != gwapiv1.TLSModeTerminate {
continue
}
for _, cert := range listener.TLS.CertificateRefs {
if *cert.Kind == resource.KindSecret {
secretReferences = append(secretReferences,
types.NamespacedName{
Namespace: gatewayapi.NamespaceDerefOr(cert.Namespace, listenerSet.Namespace),
Name: string(cert.Name),
}.String(),
)
}
}
}
return secretReferences
}

func gatewayIndexFunc(rawObj client.Object) []string {
gateway := rawObj.(*gwapiv1.Gateway)
return []string{string(gateway.Spec.GatewayClassName)}
Expand Down
45 changes: 42 additions & 3 deletions internal/provider/kubernetes/predicates.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,12 @@ func (r *gatewayAPIReconciler) validateSecretForReconcile(secret *corev1.Secret)
return true
}

if r.listenerSetCRDExists {
if r.isListenerSetReferencingSecret(&nsName) {
return true
}
}

if r.spCRDExists {
if r.isSecurityPolicyReferencingSecret(&nsName) {
return true
Expand Down Expand Up @@ -383,11 +389,44 @@ func (r *gatewayAPIReconciler) isGatewayReferencingSecret(nsName *types.Namespac

for i := range gwList.Items {
gw := &gwList.Items[i]
if !r.validateGatewayForReconcile(gw) {
return false
if r.validateGatewayForReconcile(gw) {
return true
}
}
return true
return false
}

func (r *gatewayAPIReconciler) isListenerSetReferencingSecret(nsName *types.NamespacedName) bool {
lsList := &gwapiv1.ListenerSetList{}
if err := r.client.List(context.Background(), lsList, &client.ListOptions{
FieldSelector: fields.OneTermEqualSelector(secretListenerSetIndex, nsName.String()),
}); err != nil {
r.log.Error(err, "unable to find associated ListenerSets")
return false
}

if len(lsList.Items) == 0 {
return false
}

for i := range lsList.Items {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we return false for 1 bad parentRef instead of finding any valid secrets that need to be reconciled
this bug exists in another place

  diff --git a/internal/provider/kubernetes/predicates.go b/internal/provider/kubernetes/predicates.go
  --- a/internal/provider/kubernetes/predicates.go
  +++ b/internal/provider/kubernetes/predicates.go
  @@ -388,11 +388,11 @@ func (r *gatewayAPIReconciler) isGatewayReferencingSecret(nsName *types.Namespa

   	for i := range gwList.Items {
   		gw := &gwList.Items[i]
  -		if !r.validateGatewayForReconcile(gw) {
  -			return false
  +		if r.validateGatewayForReconcile(gw) {
  +			return true
   		}
   	}
  -	return true
  +	return false
   }

   func (r *gatewayAPIReconciler) isListenerSetReferencingSecret(nsName *types.NamespacedName) bool {
  @@ -419,12 +419,12 @@ func (r *gatewayAPIReconciler) isListenerSetReferencingSecret(nsName *types.Nam
   		if err := r.client.Get(context.Background(), key, gw); err != nil {
   			r.log.Error(err, "failed to get parent Gateway for ListenerSet",
   				"namespace", ls.Namespace, "name", ls.Name)
  -			return false
  +			continue
   		}
  -		if !r.validateGatewayForReconcile(gw) {
  -			return false
  +		if r.validateGatewayForReconcile(gw) {
  +			return true
   		}
   	}
  -	return true
  +	return false
   }

can this be addressed ?

ls := &lsList.Items[i]
parent := ls.Spec.ParentRef
gw := &gwapiv1.Gateway{}
key := types.NamespacedName{
Namespace: gatewayapi.NamespaceDerefOr(parent.Namespace, ls.Namespace),
Name: string(parent.Name),
}
if err := r.client.Get(context.Background(), key, gw); err != nil {
r.log.Error(err, "failed to get parent Gateway for ListenerSet",
"namespace", ls.Namespace, "name", ls.Name)
continue
}
if r.validateGatewayForReconcile(gw) {
return true
}
}
return false
}

func (r *gatewayAPIReconciler) isSecurityPolicyReferencingSecret(nsName *types.NamespacedName) bool {
Expand Down
182 changes: 175 additions & 7 deletions internal/provider/kubernetes/predicates_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -910,19 +910,186 @@ func TestValidateSecretForReconcile(t *testing.T) {
secret: test.GetSecret(types.NamespacedName{Namespace: "default", Name: "unrelated-secret"}),
expect: false,
},
{
name: "references ListenerSet TLS certificate",
configs: []client.Object{
test.GetGatewayClass("test-gc", egv1a1.GatewayControllerName, nil),
test.GetGateway(types.NamespacedName{Namespace: "default", Name: "parent-gw"}, "test-gc", 8080),
func() *gwapiv1.ListenerSet {
ls := test.GetListenerSet(
types.NamespacedName{Namespace: "default", Name: "tls-ls"},
types.NamespacedName{Namespace: "default", Name: "parent-gw"},
443,
)
secretKind := gwapiv1.Kind(resource.KindSecret)
mode := gwapiv1.TLSModeTerminate
ls.Spec.Listeners[0].Protocol = gwapiv1.HTTPSProtocolType
ls.Spec.Listeners[0].TLS = &gwapiv1.ListenerTLSConfig{
Mode: &mode,
CertificateRefs: []gwapiv1.SecretObjectReference{{
Kind: &secretKind,
Name: "ls-tls-secret",
}},
}
return ls
}(),
},
secret: test.GetSecret(types.NamespacedName{Namespace: "default", Name: "ls-tls-secret"}),
expect: true,
},
{
name: "ListenerSet exists but secret is unrelated",
configs: []client.Object{
test.GetGatewayClass("test-gc", egv1a1.GatewayControllerName, nil),
test.GetGateway(types.NamespacedName{Namespace: "default", Name: "parent-gw"}, "test-gc", 8080),
func() *gwapiv1.ListenerSet {
ls := test.GetListenerSet(
types.NamespacedName{Namespace: "default", Name: "tls-ls"},
types.NamespacedName{Namespace: "default", Name: "parent-gw"},
443,
)
secretKind := gwapiv1.Kind(resource.KindSecret)
mode := gwapiv1.TLSModeTerminate
ls.Spec.Listeners[0].Protocol = gwapiv1.HTTPSProtocolType
ls.Spec.Listeners[0].TLS = &gwapiv1.ListenerTLSConfig{
Mode: &mode,
CertificateRefs: []gwapiv1.SecretObjectReference{{
Kind: &secretKind,
Name: "ls-tls-secret",
}},
}
return ls
}(),
},
secret: test.GetSecret(types.NamespacedName{Namespace: "default", Name: "unrelated-secret"}),
expect: false,
},
{
name: "ListenerSet references secret but parent gateway has invalid controller",
configs: []client.Object{
test.GetGatewayClass("test-gc", "not.configured/controller", nil),
test.GetGateway(types.NamespacedName{Namespace: "default", Name: "parent-gw"}, "test-gc", 8080),
func() *gwapiv1.ListenerSet {
ls := test.GetListenerSet(
types.NamespacedName{Namespace: "default", Name: "tls-ls"},
types.NamespacedName{Namespace: "default", Name: "parent-gw"},
443,
)
secretKind := gwapiv1.Kind(resource.KindSecret)
mode := gwapiv1.TLSModeTerminate
ls.Spec.Listeners[0].Protocol = gwapiv1.HTTPSProtocolType
ls.Spec.Listeners[0].TLS = &gwapiv1.ListenerTLSConfig{
Mode: &mode,
CertificateRefs: []gwapiv1.SecretObjectReference{{
Kind: &secretKind,
Name: "ls-tls-secret",
}},
}
return ls
}(),
},
secret: test.GetSecret(types.NamespacedName{Namespace: "default", Name: "ls-tls-secret"}),
expect: false,
},
{
// One unmanaged parent must not hide another ListenerSet whose parent is managed.
name: "mixed ListenerSet parents: one invalid controller, one managed",
configs: []client.Object{
test.GetGatewayClass("test-gc", egv1a1.GatewayControllerName, nil),
test.GetGatewayClass("other-gc", "not.configured/controller", nil),
test.GetGateway(types.NamespacedName{Namespace: "default", Name: "good-gw"}, "test-gc", 8080),
test.GetGateway(types.NamespacedName{Namespace: "default", Name: "bad-gw"}, "other-gc", 8080),
func() *gwapiv1.ListenerSet {
ls := test.GetListenerSet(
types.NamespacedName{Namespace: "default", Name: "bad-ls"},
types.NamespacedName{Namespace: "default", Name: "bad-gw"},
443,
)
secretKind := gwapiv1.Kind(resource.KindSecret)
mode := gwapiv1.TLSModeTerminate
ls.Spec.Listeners[0].Protocol = gwapiv1.HTTPSProtocolType
ls.Spec.Listeners[0].TLS = &gwapiv1.ListenerTLSConfig{
Mode: &mode,
CertificateRefs: []gwapiv1.SecretObjectReference{{
Kind: &secretKind,
Name: "shared-ls-tls",
}},
}
return ls
}(),
func() *gwapiv1.ListenerSet {
ls := test.GetListenerSet(
types.NamespacedName{Namespace: "default", Name: "good-ls"},
types.NamespacedName{Namespace: "default", Name: "good-gw"},
443,
)
secretKind := gwapiv1.Kind(resource.KindSecret)
mode := gwapiv1.TLSModeTerminate
ls.Spec.Listeners[0].Protocol = gwapiv1.HTTPSProtocolType
ls.Spec.Listeners[0].TLS = &gwapiv1.ListenerTLSConfig{
Mode: &mode,
CertificateRefs: []gwapiv1.SecretObjectReference{{
Kind: &secretKind,
Name: "shared-ls-tls",
}},
}
return ls
}(),
},
secret: test.GetSecret(types.NamespacedName{Namespace: "default", Name: "shared-ls-tls"}),
expect: true,
},
{
name: "mixed Gateways referencing secret: one invalid controller, one managed",
configs: []client.Object{
test.GetGatewayClass("test-gc", egv1a1.GatewayControllerName, nil),
test.GetGatewayClass("other-gc", "not.configured/controller", nil),
func() *gwapiv1.Gateway {
gw := test.GetGateway(types.NamespacedName{Namespace: "default", Name: "bad-gw"}, "other-gc", 443)
secretKind := gwapiv1.Kind(resource.KindSecret)
mode := gwapiv1.TLSModeTerminate
gw.Spec.Listeners[0].Protocol = gwapiv1.HTTPSProtocolType
gw.Spec.Listeners[0].TLS = &gwapiv1.ListenerTLSConfig{
Mode: &mode,
CertificateRefs: []gwapiv1.SecretObjectReference{{
Kind: &secretKind,
Name: "shared-gw-tls",
}},
}
return gw
}(),
func() *gwapiv1.Gateway {
gw := test.GetGateway(types.NamespacedName{Namespace: "default", Name: "good-gw"}, "test-gc", 443)
secretKind := gwapiv1.Kind(resource.KindSecret)
mode := gwapiv1.TLSModeTerminate
gw.Spec.Listeners[0].Protocol = gwapiv1.HTTPSProtocolType
gw.Spec.Listeners[0].TLS = &gwapiv1.ListenerTLSConfig{
Mode: &mode,
CertificateRefs: []gwapiv1.SecretObjectReference{{
Kind: &secretKind,
Name: "shared-gw-tls",
}},
}
return gw
}(),
},
secret: test.GetSecret(types.NamespacedName{Namespace: "default", Name: "shared-gw-tls"}),
expect: true,
},
}

// Create the reconciler.
logger := logging.DefaultLogger(os.Stdout, egv1a1.LogLevelInfo)

r := gatewayAPIReconciler{
classController: egv1a1.GatewayControllerName,
log: logger,
backendCRDExists: true,
spCRDExists: true,
epCRDExists: true,
eepCRDExists: true,
hrfCRDExists: true,
classController: egv1a1.GatewayControllerName,
log: logger,
backendCRDExists: true,
spCRDExists: true,
epCRDExists: true,
eepCRDExists: true,
hrfCRDExists: true,
listenerSetCRDExists: true,
envoyGateway: &egv1a1.EnvoyGateway{
EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
ExtensionAPIs: &egv1a1.ExtensionAPISettings{
Expand All @@ -937,6 +1104,7 @@ func TestValidateSecretForReconcile(t *testing.T) {
WithScheme(envoygateway.GetScheme()).
WithObjects(tc.configs...).
WithIndex(&gwapiv1.Gateway{}, secretGatewayIndex, secretGatewayIndexFunc).
WithIndex(&gwapiv1.ListenerSet{}, secretListenerSetIndex, secretListenerSetIndexFunc).
WithIndex(&egv1a1.SecurityPolicy{}, secretSecurityPolicyIndex, secretSecurityPolicyIndexFunc).
WithIndex(&egv1a1.EnvoyProxy{}, secretEnvoyProxyIndex, secretEnvoyProxyIndexFunc).
WithIndex(&egv1a1.EnvoyExtensionPolicy{}, secretEnvoyExtensionPolicyIndex, secretEnvoyExtensionPolicyIndexFunc).
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fixed ListenerSet not being reconciled when a referenced TLS Secret is
created or updated after the ListenerSet. Secret watches previously only
indexed Gateway certificateRefs, so cert-manager style late Secret creation
left the ListenerSet stuck with Programmed=False until an unrelated reconcile.
Loading