diff --git a/README.md b/README.md index 407fe99..d64ebe5 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ In dry-run, relative and recurring schedules are logged without saving state or By default, the controller discovers resources across all namespaces and watches those that support `get`, `list`, `watch`, `patch`, and `delete`. It skips subresources and APIs that lack these verbs. An explicit watch pattern that selects an unsupported API causes a discovery error unless an ignore rule excludes that API. -Resource filters select which API types to watch. Namespace filters limit which objects the controller acts on, but the cache still reads across all namespaces. Namespace filters do not provide tenant isolation. +Resource filters select which API types to watch. The controller reads, deletes, and restarts resources only in the filtered namespaces. Namespace patterns limit deletions and restarts but require cluster-wide reads. - `--watch-resource` (repeatable) - glob pattern for resources to watch. - Format - `.` for grouped APIs (e.g., `deployments.apps`) or `` for core APIs (e.g., `pods`). @@ -175,7 +175,7 @@ Resource filters select which API types to watch. Namespace filters limit which - Broad patterns can select APIs that lack the required verbs. Prefer exact resource names. - `--ignore-resource` (repeatable) - glob pattern for resources to strictly ignore. Takes precedence over watch rules. - `--watch-namespace` (repeatable) - glob pattern for namespaces to watch (e.g. `default`, `dev-*`). - - If provided, the controller acts only on objects in matching namespaces. It excludes other cluster-scoped objects, except watched `Namespace` objects whose names match the pattern. + - If provided, the controller deletes or restarts only objects in matching namespaces. Namespace objects also require `--watch-resource=namespaces`. Other cluster-scoped resources are excluded. - `--ignore-namespace` (repeatable) - glob pattern for namespaces to strictly ignore. Takes precedence over watch rules. ### RBAC diff --git a/cmd/main.go b/cmd/main.go index 779964e..9f75cf8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -166,7 +166,20 @@ func main() { metricsServerOptions.KeyName = metricsCertKey } + scopeConfig := controller.ScopeConfig{ + WatchResources: watchResources, + IgnoreResources: ignoreResources, + WatchNamespaces: watchNamespaces, + IgnoreNamespaces: ignoreNamespaces, + } + cacheOptions, err := scopeConfig.CacheOptions() + if err != nil { + setupLog.Error(err, "invalid namespace watch configuration") + os.Exit(1) + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Cache: cacheOptions, Scheme: scheme, Metrics: metricsServerOptions, WebhookServer: webhookServer, @@ -190,13 +203,6 @@ func main() { os.Exit(1) } - scopeConfig := controller.ScopeConfig{ - WatchResources: watchResources, - IgnoreResources: ignoreResources, - WatchNamespaces: watchNamespaces, - IgnoreNamespaces: ignoreNamespaces, - } - setupLog.Info("Starting with scope configuration", "watchResources", watchResources, "ignoreResources", ignoreResources, diff --git a/internal/controller/cache_scope.go b/internal/controller/cache_scope.go new file mode 100644 index 0000000..9df4869 --- /dev/null +++ b/internal/controller/cache_scope.go @@ -0,0 +1,36 @@ +package controller + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/cache" +) + +// CacheOptions restricts API reads when the watch list contains exact namespace names. +func (c ScopeConfig) CacheOptions() (cache.Options, error) { + if len(c.WatchNamespaces) == 0 { + return cache.Options{}, nil + } + for _, namespace := range c.WatchNamespaces { + if strings.ContainsAny(namespace, "*?[]\\") { + return cache.Options{}, nil + } + } + for _, namespace := range c.WatchNamespaces { + if problems := validation.IsDNS1123Label(namespace); len(problems) > 0 { + return cache.Options{}, fmt.Errorf("invalid namespace name %q", namespace) + } + } + options := cache.Options{DefaultNamespaces: make(map[string]cache.Config)} + for _, namespace := range c.WatchNamespaces { + if c.IsNamespaceAllowed(namespace) { + options.DefaultNamespaces[namespace] = cache.Config{} + } + } + if len(options.DefaultNamespaces) == 0 { + return cache.Options{}, fmt.Errorf("watch list requires at least one namespace after ignore rules") + } + return options, nil +} diff --git a/internal/controller/discovery.go b/internal/controller/discovery.go index 842ea6e..142e810 100644 --- a/internal/controller/discovery.go +++ b/internal/controller/discovery.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "net/http" - "sort" - "strings" "sync" "time" @@ -14,7 +12,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/discovery" "k8s.io/client-go/util/workqueue" @@ -28,9 +25,6 @@ import ( const discoveryRetryInterval = 30 * time.Second -// Every watched resource may carry either deletion or restart annotations. -var requiredWatchVerbs = sets.New("get", "list", "watch", "patch", "delete") - type preferredResourceDiscovery interface { ServerPreferredResources() ([]*metav1.APIResourceList, error) } @@ -231,133 +225,6 @@ func (*coverageSyncRunnable) NeedLeaderElection() bool { return true } -func planResources(lists []*metav1.APIResourceList, config ScopeConfig, logger logr.Logger) (*resourcePlan, error) { - plan := &resourcePlan{counts: make(map[[2]string]int)} - resolvedPatterns := make(map[string]bool, len(config.WatchResources)) - var unwatchable []string - - for _, list := range lists { - gv, err := schema.ParseGroupVersion(list.GroupVersion) - if err != nil { - plan.count(discoveryResultFailed, discoveryReasonInvalidGroup) - publishPlanMetrics(plan) - logger.Error(err, "Invalid group version returned by discovery", "groupVersion", list.GroupVersion) - return nil, fmt.Errorf("invalid discovered group version %q: %w", list.GroupVersion, err) - } - - for _, resource := range list.APIResources { - key := resourceKey(resource.Name, gv.Group) - requested := matchingPatterns(config.WatchResources, key) - ignored := matches(config.IgnoreResources, key) - - if ignored { - // Ignore precedence removes this match from the required effective watch set. - for _, pattern := range requested { - resolvedPatterns[pattern] = true - } - plan.skip(logger, key, discoveryReasonConfigured) - continue - } - if len(config.WatchResources) > 0 && len(requested) == 0 { - plan.skip(logger, key, discoveryReasonConfigured) - continue - } - for _, pattern := range requested { - resolvedPatterns[pattern] = true - } - if strings.Contains(resource.Name, "/") { - plan.skip(logger, key, discoveryReasonSubresource) - if len(requested) > 0 { - unwatchable = append(unwatchable, key) - } - continue - } - - verbs := sets.New(resource.Verbs...) - if !verbs.HasAll(requiredWatchVerbs.UnsortedList()...) { - plan.skip(logger, key, discoveryReasonUnsupportedVerbs) - if len(requested) > 0 { - unwatchable = append(unwatchable, key) - } - continue - } - - plan.resources = append(plan.resources, discoveredResource{ - key: key, - gvk: schema.GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: resource.Kind}, - }) - plan.count(discoveryResultDiscovered, discoveryReasonWatchable) - } - } - - var missing []string - ignoredPatterns := sets.New(config.IgnoreResources...) - for _, pattern := range config.WatchResources { - if !resolvedPatterns[pattern] && !ignoredPatterns.Has(pattern) { - missing = append(missing, pattern) - } - } - sort.Strings(missing) - sort.Strings(unwatchable) - if len(missing) > 0 || len(unwatchable) > 0 { - if len(missing) > 0 { - plan.countBy(discoveryResultFailed, discoveryReasonRequestMissing, len(missing)) - for _, pattern := range missing { - logger.Error(errors.New("requested resource pattern was not discovered"), "Required resource is missing", "pattern", pattern) - } - } - if len(unwatchable) > 0 { - plan.countBy(discoveryResultFailed, discoveryReasonRequestUnusable, len(unwatchable)) - for _, key := range unwatchable { - logger.Error(errors.New("required watch verbs are unavailable"), "Required resource is unwatchable", "resource", key) - } - } - publishPlanMetrics(plan) - return nil, fmt.Errorf("explicit resource coverage incomplete: missing patterns=%v, unwatchable resources=%v", missing, unwatchable) - } - - sort.Slice(plan.resources, func(i, j int) bool { return plan.resources[i].key < plan.resources[j].key }) - publishPlanMetrics(plan) - return plan, nil -} - -func resourceKey(resource, group string) string { - if group == "" { - return resource - } - return resource + "." + group -} - -func matchingPatterns(patterns []string, key string) []string { - var matched []string - for _, pattern := range patterns { - if matches([]string{pattern}, key) { - matched = append(matched, pattern) - } - } - return matched -} - -func (p *resourcePlan) count(result, reason string) { - p.counts[[2]string{result, reason}]++ -} - -func (p *resourcePlan) countBy(result, reason string, count int) { - p.counts[[2]string{result, reason}] += count -} - -func (p *resourcePlan) skip(logger logr.Logger, key, reason string) { - p.count(discoveryResultSkipped, reason) - logger.V(1).Info("Skipping discovered resource", "resource", key, "reason", reason) -} - -func publishPlanMetrics(plan *resourcePlan) { - resetDiscoveryResourceMetrics() - for labels, count := range plan.counts { - setDiscoveryResourceMetric(labels[0], labels[1], count) - } -} - type trackedSource struct { name string source source.TypedSyncingSource[resourceRequest] diff --git a/internal/controller/discovery_plan.go b/internal/controller/discovery_plan.go new file mode 100644 index 0000000..a5fe574 --- /dev/null +++ b/internal/controller/discovery_plan.go @@ -0,0 +1,154 @@ +package controller + +import ( + "errors" + "fmt" + "sort" + "strings" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" +) + +// Every watched resource may carry deletion or restart annotations. +var requiredWatchVerbs = sets.New("get", "list", "watch", "patch", "delete") + +func planResources(lists []*metav1.APIResourceList, config ScopeConfig, logger logr.Logger) (*resourcePlan, error) { + plan := &resourcePlan{counts: make(map[[2]string]int)} + resolvedPatterns := make(map[string]bool, len(config.WatchResources)) + var unwatchable []string + + for _, list := range lists { + gv, err := schema.ParseGroupVersion(list.GroupVersion) + if err != nil { + plan.count(discoveryResultFailed, discoveryReasonInvalidGroup) + publishPlanMetrics(plan) + logger.Error(err, "Invalid group version returned by discovery", "groupVersion", list.GroupVersion) + return nil, fmt.Errorf("invalid discovered group version %q: %w", list.GroupVersion, err) + } + + for _, resource := range list.APIResources { + key := resourceKey(resource.Name, gv.Group) + requested := matchingPatterns(config.WatchResources, key) + ignored := matches(config.IgnoreResources, key) + + if ignored { + // Ignore precedence removes this match from the required effective watch set. + for _, pattern := range requested { + resolvedPatterns[pattern] = true + } + plan.skip(logger, key, discoveryReasonConfigured) + continue + } + if len(config.WatchResources) > 0 && len(requested) == 0 { + plan.skip(logger, key, discoveryReasonConfigured) + continue + } + for _, pattern := range requested { + resolvedPatterns[pattern] = true + } + explicitNamespace := gv.Group == "" && resource.Name == "namespaces" && config.watchesNamespaceObjects() + if len(config.WatchNamespaces) > 0 && !resource.Namespaced && !explicitNamespace { + if len(requested) > 0 { + plan.count(discoveryResultFailed, discoveryReasonRequestUnusable) + publishPlanMetrics(plan) + return nil, fmt.Errorf("cluster-scoped resource %s requires an empty namespace watch list; "+ + "only an exact namespaces resource selection can be combined with namespace filters", key) + } + plan.skip(logger, key, discoveryReasonConfigured) + continue + } + if strings.Contains(resource.Name, "/") { + plan.skip(logger, key, discoveryReasonSubresource) + if len(requested) > 0 { + unwatchable = append(unwatchable, key) + } + continue + } + + verbs := sets.New(resource.Verbs...) + if !verbs.HasAll(requiredWatchVerbs.UnsortedList()...) { + plan.skip(logger, key, discoveryReasonUnsupportedVerbs) + if len(requested) > 0 { + unwatchable = append(unwatchable, key) + } + continue + } + + plan.resources = append(plan.resources, discoveredResource{ + key: key, + gvk: schema.GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: resource.Kind}, + }) + plan.count(discoveryResultDiscovered, discoveryReasonWatchable) + } + } + + var missing []string + ignoredPatterns := sets.New(config.IgnoreResources...) + for _, pattern := range config.WatchResources { + if !resolvedPatterns[pattern] && !ignoredPatterns.Has(pattern) { + missing = append(missing, pattern) + } + } + sort.Strings(missing) + sort.Strings(unwatchable) + if len(missing) > 0 || len(unwatchable) > 0 { + if len(missing) > 0 { + plan.countBy(discoveryResultFailed, discoveryReasonRequestMissing, len(missing)) + for _, pattern := range missing { + logger.Error(errors.New("requested resource pattern was not discovered"), "Required resource is missing", "pattern", pattern) + } + } + if len(unwatchable) > 0 { + plan.countBy(discoveryResultFailed, discoveryReasonRequestUnusable, len(unwatchable)) + for _, key := range unwatchable { + logger.Error(errors.New("required watch verbs are unavailable"), "Required resource is unwatchable", "resource", key) + } + } + publishPlanMetrics(plan) + return nil, fmt.Errorf("explicit resource coverage incomplete: missing patterns=%v, unwatchable resources=%v", missing, unwatchable) + } + + sort.Slice(plan.resources, func(i, j int) bool { return plan.resources[i].key < plan.resources[j].key }) + publishPlanMetrics(plan) + return plan, nil +} + +func resourceKey(resource, group string) string { + if group == "" { + return resource + } + return resource + "." + group +} + +func matchingPatterns(patterns []string, key string) []string { + var matched []string + for _, pattern := range patterns { + if matches([]string{pattern}, key) { + matched = append(matched, pattern) + } + } + return matched +} + +func (p *resourcePlan) count(result, reason string) { + p.counts[[2]string{result, reason}]++ +} + +func (p *resourcePlan) countBy(result, reason string, count int) { + p.counts[[2]string{result, reason}] += count +} + +func (p *resourcePlan) skip(logger logr.Logger, key, reason string) { + p.count(discoveryResultSkipped, reason) + logger.V(1).Info("Skipping discovered resource", "resource", key, "reason", reason) +} + +func publishPlanMetrics(plan *resourcePlan) { + resetDiscoveryResourceMetrics() + for labels, count := range plan.counts { + setDiscoveryResourceMetric(labels[0], labels[1], count) + } +} diff --git a/internal/controller/discovery_test.go b/internal/controller/discovery_test.go index f6ea9a8..3e84294 100644 --- a/internal/controller/discovery_test.go +++ b/internal/controller/discovery_test.go @@ -109,6 +109,10 @@ func TestExplicitDiscoverySetupRejectsDiscoveryErrors(t *testing.T) { } func TestExplicitDiscoverySetupRejectsMissingAndUnwatchableResources(t *testing.T) { + node := apiResource("nodes", "Node", watchVerbs) + node.Namespaced = false + namespace := apiResource("namespaces", "Namespace", watchVerbs) + namespace.Namespaced = false tests := map[string]struct { config ScopeConfig response discoveryResponse @@ -128,6 +132,21 @@ func TestExplicitDiscoverySetupRejectsMissingAndUnwatchableResources(t *testing. }}, want: "unwatchable resources=[pods]", }, + "cluster resource with namespace filters": { + config: ScopeConfig{WatchResources: []string{"nodes"}, WatchNamespaces: []string{"team-a"}}, + response: discoveryResponse{lists: []*metav1.APIResourceList{coreResources(node)}}, + want: "requires an empty namespace watch list", + }, + "Namespace resource glob with exact namespace filters": { + config: ScopeConfig{WatchResources: []string{"namespace*"}, WatchNamespaces: []string{"team-a"}}, + response: discoveryResponse{lists: []*metav1.APIResourceList{coreResources(namespace)}}, + want: "only an exact namespaces resource selection", + }, + "Namespace resource glob with namespace patterns": { + config: ScopeConfig{WatchResources: []string{"namespace*"}, WatchNamespaces: []string{"team-*"}}, + response: discoveryResponse{lists: []*metav1.APIResourceList{coreResources(namespace)}}, + want: "only an exact namespaces resource selection", + }, } for name, test := range tests { @@ -154,68 +173,90 @@ func TestExplicitDiscoverySetupRejectsMissingAndUnwatchableResources(t *testing. } func TestBroadDiscoveryRecoversBeforeBecomingReady(t *testing.T) { - partialErr := &discovery.ErrGroupDiscoveryFailed{Groups: map[schema.GroupVersion]error{ - {Group: "apps", Version: "v1"}: errors.New("temporarily unavailable"), - }} - client := &scriptedDiscovery{responses: []discoveryResponse{ - { - lists: []*metav1.APIResourceList{coreResources(apiResource("pods", "Pod", watchVerbs))}, - err: partialErr, - }, - {err: errors.New("API server unavailable")}, - {lists: []*metav1.APIResourceList{coreResources(apiResource("pods", "Pod", watchVerbs))}}, - }} - syncStarted := make(chan struct{}) - releaseSync := make(chan struct{}) - coordinator := newTestCoordinator(client, ScopeConfig{}, func(resources []discoveredResource) (func(context.Context) error, error) { - if len(resources) != 1 || resources[0].key != "pods" { - return nil, fmt.Errorf("unexpected recovered resources: %#v", resources) - } - return func(ctx context.Context) error { - close(syncStarted) - select { - case <-releaseSync: - return nil - case <-ctx.Done(): - return ctx.Err() + for _, test := range []struct { + name string + config ScopeConfig + want []string + }{ + {name: "cluster defaults", want: []string{"namespaces", "nodes", "pods"}}, + {name: "exact namespace list", config: ScopeConfig{WatchNamespaces: []string{"team-a"}}, want: []string{"pods"}}, + {name: "namespace patterns", config: ScopeConfig{WatchNamespaces: []string{"team-*"}}, want: []string{"pods"}}, + } { + t.Run(test.name, func(t *testing.T) { + namespace := apiResource("namespaces", "Namespace", watchVerbs) + namespace.Namespaced = false + node := apiResource("nodes", "Node", watchVerbs) + node.Namespaced = false + resourceList := coreResources(namespace, node, apiResource("pods", "Pod", watchVerbs)) + partialErr := &discovery.ErrGroupDiscoveryFailed{Groups: map[schema.GroupVersion]error{ + {Group: "apps", Version: "v1"}: errors.New("temporarily unavailable"), + }} + client := &scriptedDiscovery{responses: []discoveryResponse{ + { + lists: []*metav1.APIResourceList{resourceList}, + err: partialErr, + }, + {err: errors.New("API server unavailable")}, + {lists: []*metav1.APIResourceList{resourceList}}, + }} + syncStarted := make(chan struct{}) + releaseSync := make(chan struct{}) + coordinator := newTestCoordinator(client, test.config, func(resources []discoveredResource) (func(context.Context) error, error) { + keys := make([]string, 0, len(resources)) + for _, resource := range resources { + keys = append(keys, resource.key) + } + if strings.Join(keys, ",") != strings.Join(test.want, ",") { + return nil, fmt.Errorf("unexpected recovered resources: %#v", resources) + } + return func(ctx context.Context) error { + close(syncStarted) + select { + case <-releaseSync: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }, nil + }) + + _, retry, err := coordinator.prepareInitial() + if err != nil || !retry { + t.Fatalf("broad partial discovery should defer and retry: retry=%v err=%v", retry, err) + } + if err := coordinator.state.check(); err == nil { + t.Fatal("partial discovery must be unready") } - }, nil - }) - _, retry, err := coordinator.prepareInitial() - if err != nil || !retry { - t.Fatalf("broad partial discovery should defer and retry: retry=%v err=%v", retry, err) - } - if err := coordinator.state.check(); err == nil { - t.Fatal("partial discovery must be unready") - } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- coordinator.Start(ctx) }() - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- coordinator.Start(ctx) }() + select { + case <-syncStarted: + case <-time.After(time.Second): + t.Fatal("discovery did not recover") + } + if err := coordinator.state.check(); err == nil { + t.Fatal("complete discovery without cache sync must remain unready") + } + close(releaseSync) + eventually(t, time.Second, func() bool { return coordinator.state.check() == nil }) + if client.callCount() < 3 { + t.Fatalf("expected retries through total error to recovery, got %d calls", client.callCount()) + } - select { - case <-syncStarted: - case <-time.After(time.Second): - t.Fatal("discovery did not recover") - } - if err := coordinator.state.check(); err == nil { - t.Fatal("complete discovery without cache sync must remain unready") - } - close(releaseSync) - eventually(t, time.Second, func() bool { return coordinator.state.check() == nil }) - if client.callCount() < 3 { - t.Fatalf("expected retries through total error to recovery, got %d calls", client.callCount()) - } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("coordinator shutdown failed: %v", err) + } + case <-time.After(time.Second): + t.Fatal("coordinator did not stop") + } - cancel() - select { - case err := <-done: - if err != nil { - t.Fatalf("coordinator shutdown failed: %v", err) - } - case <-time.After(time.Second): - t.Fatal("coordinator did not stop") + }) } } @@ -263,14 +304,31 @@ func TestReadinessCheckTreatsOnlyPassiveLeaderReplicaAsReady(t *testing.T) { } func TestExplicitIgnorePrecedenceCanProduceEmptyWatchSet(t *testing.T) { - for name, lists := range map[string][]*metav1.APIResourceList{ - "resource present": {coreResources(apiResource("configmaps", "ConfigMap", watchVerbs))}, - "resource absent": {coreResources(apiResource("pods", "Pod", watchVerbs))}, + namespace := apiResource("namespaces", "Namespace", watchVerbs) + namespace.Namespaced = false + for name, test := range map[string]struct { + lists []*metav1.APIResourceList + config ScopeConfig + }{ + "resource present": { + lists: []*metav1.APIResourceList{coreResources(apiResource("configmaps", "ConfigMap", watchVerbs))}, + config: ScopeConfig{WatchResources: []string{"configmaps"}, IgnoreResources: []string{"configmaps"}}, + }, + "resource absent": { + lists: []*metav1.APIResourceList{coreResources(apiResource("pods", "Pod", watchVerbs))}, + config: ScopeConfig{WatchResources: []string{"configmaps"}, IgnoreResources: []string{"configmaps"}}, + }, + "cluster resource ignored with namespace filters": { + lists: []*metav1.APIResourceList{coreResources(namespace)}, + config: ScopeConfig{ + WatchResources: []string{"namespaces"}, IgnoreResources: []string{"namespaces"}, WatchNamespaces: []string{"team-a"}, + }, + }, } { t.Run(name, func(t *testing.T) { plan, err := planResources( - lists, - ScopeConfig{WatchResources: []string{"configmaps"}, IgnoreResources: []string{"configmaps"}}, + test.lists, + test.config, logr.Discard(), ) if err != nil { diff --git a/internal/controller/scope.go b/internal/controller/scope.go index f553946..7bae1ab 100644 --- a/internal/controller/scope.go +++ b/internal/controller/scope.go @@ -3,7 +3,9 @@ package controller import ( "fmt" "path/filepath" + "slices" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -17,6 +19,11 @@ type ScopeConfig struct { IgnoreNamespaces []string } +// watchesNamespaceObjects requires an exact selection when namespace filters are set. +func (c ScopeConfig) watchesNamespaceObjects() bool { + return slices.Contains(c.WatchResources, "namespaces") && c.IsResourceAllowed("namespaces", "") +} + // matches checks if value matches any of the glob patterns. func matches(patterns []string, value string) bool { for _, p := range patterns { @@ -95,12 +102,12 @@ func allow(config ScopeConfig, obj client.Object) bool { // Case 2: Resource is Cluster-Scoped (ns == "") - // Special Handling: The "Namespace" resource itself. - // If we are watching namespace "foo", we want to allow actions on the Namespace object named "foo". - // We check the Kind. Note: Unstructured objects (used by this controller) always have GVK populated. gvk := obj.GetObjectKind().GroupVersionKind() - if gvk.Kind == "Namespace" { - // For a Namespace object, its Name IS the namespace identifier. + _, nativeNamespace := obj.(*corev1.Namespace) + if nativeNamespace || (gvk.Group == "" && gvk.Version == "v1" && gvk.Kind == "Namespace") { + if len(config.WatchNamespaces) > 0 && !config.watchesNamespaceObjects() { + return false + } return config.IsNamespaceAllowed(obj.GetName()) } diff --git a/internal/controller/scope_integration_test.go b/internal/controller/scope_integration_test.go index b54b5d6..f88b4a3 100644 --- a/internal/controller/scope_integration_test.go +++ b/internal/controller/scope_integration_test.go @@ -8,128 +8,221 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + eventsv1 "k8s.io/api/events/v1" + rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + runtimeconfig "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/envtest" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) var _ = Describe("Scoped Watch Integration", Serial, func() { - - const ( - AllowedNamespace = "integration-allowed" - IgnoredNamespace = "integration-ignored" - ) - BeforeEach(func() { - ns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: AllowedNamespace}} - ns2 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: IgnoredNamespace}} - _ = k8sClient.Create(context.Background(), ns1) - _ = k8sClient.Create(context.Background(), ns2) - }) - - AfterEach(func() { - // Reset configuration to allow all - Reconciler.Config = ScopeConfig{} + // Only the restricted manager can process objects in these scenarios. + previous := Reconciler.Config + Reconciler.Config = ScopeConfig{WatchNamespaces: []string{"inactive-scope-controller"}} + DeferCleanup(func() { Reconciler.Config = previous }) }) Context("When filtering by namespace", func() { It("should ignore events in ignored namespaces", func() { - // Configure the controller to ignore the specific namespace - Reconciler.Config = ScopeConfig{ - IgnoreNamespaces: []string{IgnoredNamespace}, - } - - ctx := context.Background() - - By("Creating a Deployment in the Ignored Namespace with immediate delete") - ignoredDepName := "ignored-dep" - ignoredDep := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: ignoredDepName, - Namespace: IgnoredNamespace, - Annotations: map[string]string{ - DeleteAfterAnnotation: "1s", - }, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "test"}}, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "nginx", Image: "nginx"}}}, + const userName = "namespace-scoped-controller" + allowed := []string{"integration-allowed", "integration-allowed-b"} + const ignored = "integration-ignored" + namespaces := append(append([]string{}, allowed...), ignored) + limited, userConfig := scopeUser(userName) + for _, namespace := range namespaces { + createScopeObjects(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}) + if namespace == ignored { + continue + } + role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "scope-controller", Namespace: namespace}, + Rules: []rbacv1.PolicyRule{ + {APIGroups: []string{"apps"}, Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch", "patch", "update", "delete"}}, + {APIGroups: []string{"", "events.k8s.io"}, Resources: []string{"events"}, Verbs: []string{"create", "patch"}}, }, - }, + } + binding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: role.Name, Namespace: namespace}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: role.Name}, + Subjects: scopeSubjects(userName), + } + createScopeObjects(role, binding) } - Expect(k8sClient.Create(ctx, ignoredDep)).To(Succeed()) - - By("Creating a Deployment in the Allowed Namespace with immediate delete") - allowedDepName := "allowed-dep" - allowedDep := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: allowedDepName, - Namespace: AllowedNamespace, - Annotations: map[string]string{ - DeleteAfterAnnotation: "1s", + for _, namespace := range allowed { + Eventually(func() error { + return limited.List(ctx, &appsv1.DeploymentList{}, client.InNamespace(namespace)) + }, 10*time.Second).Should(Succeed()) + } + By("checking denied API reads before controller startup") + for _, namespace := range []string{"", ignored} { + Expect(apierrors.IsForbidden(limited.List(ctx, &appsv1.DeploymentList{}, client.InNamespace(namespace)))).To(BeTrue()) + } + Expect(apierrors.IsForbidden(limited.List(ctx, &corev1.SecretList{}, client.InNamespace(allowed[0])))).To(BeTrue()) + Expect(apierrors.IsForbidden(limited.List(ctx, &corev1.NamespaceList{}))).To(BeTrue()) + manager := startScopeManager(userConfig, ScopeConfig{ + WatchResources: []string{"deployments.apps"}, + WatchNamespaces: namespaces, IgnoreNamespaces: []string{ignored}, + }) + + By("creating annotated Deployments in both allowed namespaces and the ignored namespace") + for _, namespace := range namespaces { + createScopeObjects(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "scheduled-delete", Namespace: namespace, + Annotations: map[string]string{DeleteAfterAnnotation: "1s"}, }, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "test"}}, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "nginx", Image: "nginx"}}}, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "test"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "nginx", Image: "nginx"}}}, + }, }, - }, + }) } - Expect(k8sClient.Create(ctx, allowedDep)).To(Succeed()) - - By("Verifying the Allowed deployment is processed and deleted") - Eventually(func(g Gomega) { - err := k8sClient.Get(ctx, types.NamespacedName{Name: allowedDepName, Namespace: AllowedNamespace}, &appsv1.Deployment{}) - g.Expect(err).To(HaveOccurred()) - }, 10*time.Second, 500*time.Millisecond).Should(Succeed()) - - By("Verifying the Ignored deployment is NOT processed and still exists") + for _, namespace := range allowed { + Eventually(func() bool { + return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKey{ + Name: "scheduled-delete", Namespace: namespace, + }, &appsv1.Deployment{})) + }, 10*time.Second, 500*time.Millisecond).Should(BeTrue()) + } + By("verifying ignored objects remain unchanged for the existing observation window") + key := client.ObjectKey{Name: "scheduled-delete", Namespace: ignored} Consistently(func(g Gomega) { - dep := &appsv1.Deployment{} - err := k8sClient.Get(ctx, types.NamespacedName{Name: ignoredDepName, Namespace: IgnoredNamespace}, dep) - g.Expect(err).NotTo(HaveOccurred()) - // Also verify it wasn't mutated (annotation should still be delete-after, not converted to delete-at) - g.Expect(dep.Annotations).To(HaveKey(DeleteAfterAnnotation)) - g.Expect(dep.Annotations).NotTo(HaveKey(DeleteAtAnnotation)) + object := &appsv1.Deployment{} + g.Expect(k8sClient.Get(ctx, key, object)).To(Succeed()) + g.Expect(object.Annotations).To(HaveKey(DeleteAfterAnnotation)) + g.Expect(object.Annotations).NotTo(HaveKey(DeleteAtAnnotation)) }, 5*time.Second, 500*time.Millisecond).Should(Succeed()) - - _ = k8sClient.Delete(ctx, ignoredDep) + outside := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}} + Expect(apierrors.IsForbidden(limited.Delete(ctx, outside))).To(BeTrue()) + Expect(manager.GetClient().Get(ctx, key, &appsv1.Deployment{})).NotTo(Succeed()) }) It("should allow watching the Namespace object itself when its name matches the watch list", func() { - // This verifies the user's request: "the actual namespace(s, if they provide a pattern) provided" - nsToWatch := "scope-test-ns-obj" - Reconciler.Config = ScopeConfig{ - WatchNamespaces: []string{nsToWatch}, + const ( + allowed = "scope-test-ns-obj" + denied = "scope-test-ns-denied" + userName = "namespace-object-controller" + ) + limited, userConfig := scopeUser(userName) + for _, name := range []string{allowed, denied} { + createScopeObjects(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}}) } - ctx := context.Background() - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: nsToWatch, - Annotations: map[string]string{ - DeleteAfterAnnotation: "1s", - }, + // The API server provides default; this test does not own it. + err := k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}}) + Expect(err == nil || apierrors.IsAlreadyExists(err)).To(BeTrue()) + role := &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "scope-namespace-controller"}, + Rules: []rbacv1.PolicyRule{ + {APIGroups: []string{""}, Resources: []string{"namespaces"}, Verbs: []string{"get", "list", "watch"}}, + {APIGroups: []string{""}, Resources: []string{"namespaces"}, ResourceNames: []string{allowed}, + Verbs: []string{"patch", "update", "delete"}}, }, } - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - - By("Verifying the Namespace object itself is deleted (or terminating)") - Eventually(func(g Gomega) { - fetchedNS := &corev1.Namespace{} - err := k8sClient.Get(ctx, types.NamespacedName{Name: nsToWatch}, fetchedNS) - if err != nil { - // If error occurred, it MUST be NotFound - g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) - } else { - // If found, it MUST be marked for deletion - g.Expect(fetchedNS.DeletionTimestamp).NotTo(BeNil(), "Namespace should be in terminating state") + binding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: role.Name}, Subjects: scopeSubjects(userName), + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: role.Name}, + } + eventRole := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "scope-namespace-events", Namespace: "default"}, + Rules: []rbacv1.PolicyRule{{APIGroups: []string{"", "events.k8s.io"}, Resources: []string{"events"}, + Verbs: []string{"create", "patch"}}}, + } + eventBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: eventRole.Name, Namespace: "default"}, Subjects: scopeSubjects(userName), + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: eventRole.Name}, + } + createScopeObjects(role, binding, eventRole, eventBinding) + Eventually(func() error { return limited.List(ctx, &corev1.NamespaceList{}) }, 10*time.Second).Should(Succeed()) + startScopeManager(userConfig, ScopeConfig{WatchNamespaces: []string{allowed}, WatchResources: []string{"namespaces"}}) + + for _, name := range []string{allowed, denied} { + object := &corev1.Namespace{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: name}, object)).To(Succeed()) + before := object.DeepCopy() + object.Annotations = map[string]string{DeleteAfterAnnotation: "1s"} + if name == denied { + Expect(apierrors.IsForbidden(limited.Patch(ctx, object, client.MergeFrom(before)))).To(BeTrue()) + Expect(apierrors.IsForbidden(limited.Delete(ctx, object))).To(BeTrue()) + } + Expect(k8sClient.Patch(ctx, object, client.MergeFrom(before))).To(Succeed()) + } + Eventually(func() bool { + object := &corev1.Namespace{} + err := k8sClient.Get(ctx, client.ObjectKey{Name: allowed}, object) + return apierrors.IsNotFound(err) || (err == nil && object.DeletionTimestamp != nil) + }, 10*time.Second, 500*time.Millisecond).Should(BeTrue()) + outside := &corev1.Namespace{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: denied}, outside)).To(Succeed()) + Expect(outside.DeletionTimestamp).To(BeNil()) + Expect(outside.Annotations).NotTo(HaveKey(DeleteAtAnnotation)) + Eventually(func() bool { + events := &eventsv1.EventList{} + if err := k8sClient.List(ctx, events, client.InNamespace("default")); err != nil { + return false + } + for _, event := range events.Items { + if event.Regarding.Name == allowed && event.Reason == "Deleted" { + return true + } } - }, 10*time.Second, 500*time.Millisecond).Should(Succeed()) + return false + }, 10*time.Second).Should(BeTrue()) }) }) }) + +func scopeUser(name string) (client.Client, *rest.Config) { + GinkgoHelper() + user, err := testEnv.AddUser(envtest.User{Name: name}, cfg) + Expect(err).NotTo(HaveOccurred()) + limited, err := client.New(user.Config(), client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + return limited, user.Config() +} + +func scopeSubjects(name string) []rbacv1.Subject { + return []rbacv1.Subject{{APIGroup: rbacv1.GroupName, Kind: "User", Name: name}} +} + +func createScopeObjects(objects ...client.Object) { + GinkgoHelper() + for _, object := range objects { + Expect(k8sClient.Create(ctx, object)).To(Succeed()) + DeferCleanup(func() { + err := k8sClient.Delete(ctx, object) + Expect(err == nil || apierrors.IsNotFound(err)).To(BeTrue()) + }) + } +} + +func startScopeManager(config *rest.Config, scope ScopeConfig) ctrl.Manager { + GinkgoHelper() + options, err := scope.CacheOptions() + Expect(err).NotTo(HaveOccurred()) + manager, err := ctrl.NewManager(config, ctrl.Options{ + Scheme: scheme.Scheme, Cache: options, Metrics: metricsserver.Options{BindAddress: "0"}, + // The suite manager already uses the lifecycle controller name. + Controller: runtimeconfig.Controller{SkipNameValidation: ptr.To(true)}, + }) + Expect(err).NotTo(HaveOccurred()) + reconciler := &LifecycleReconciler{Client: manager.GetClient(), Scheme: manager.GetScheme(), Config: scope} + Expect(reconciler.SetupWithManager(manager)).To(Succeed()) + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- manager.Start(runCtx) }() + DeferCleanup(func() { + stop() + Eventually(done, 10*time.Second).Should(Receive(BeNil())) + }) + Eventually(func() error { return reconciler.coverage.check() }, 20*time.Second).Should(Succeed()) + return manager +} diff --git a/internal/controller/scope_test.go b/internal/controller/scope_test.go index 374bfba..35e8ca6 100644 --- a/internal/controller/scope_test.go +++ b/internal/controller/scope_test.go @@ -1,10 +1,16 @@ package controller import ( + "reflect" + "sort" + "strings" "testing" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" ) func TestIsResourceAllowed(t *testing.T) { @@ -87,6 +93,8 @@ func TestIsNamespaceAllowed(t *testing.T) { config ScopeConfig ns string expected bool + cached []string + cacheErr string }{ { name: "Default Allow", @@ -118,12 +126,70 @@ func TestIsNamespaceAllowed(t *testing.T) { ns: "default", expected: false, }, + { + name: "Exact lists remove exclusions and duplicates", + config: ScopeConfig{WatchNamespaces: []string{"team-a", "team-b", "team-a"}, IgnoreNamespaces: []string{"team-b"}}, + ns: "team-a", expected: true, cached: []string{"team-a"}, + }, + { + name: "Two exact namespaces", + config: ScopeConfig{WatchNamespaces: []string{"team-b", "team-a"}}, + ns: "team-a", expected: true, cached: []string{"team-a", "team-b"}, + }, + { + name: "Mixed exact names and patterns require cluster reads", + config: ScopeConfig{WatchNamespaces: []string{"team-a", "dev-?"}}, + ns: "team-a", expected: true, + }, + { + name: "Ignore glob removes one exact namespace", + config: ScopeConfig{WatchNamespaces: []string{"team-a", "dev-b"}, IgnoreNamespaces: []string{"dev-*"}}, + ns: "dev-b", cached: []string{"team-a"}, + }, + { + name: "All exact namespaces ignored", + config: ScopeConfig{WatchNamespaces: []string{"team-a"}, IgnoreNamespaces: []string{"team-a"}}, + ns: "team-a", cacheErr: "at least one namespace", + }, + { + name: "Ignore glob excludes the entire exact list", + config: ScopeConfig{WatchNamespaces: []string{"team-a"}, IgnoreNamespaces: []string{"team-*"}}, + ns: "team-a", cacheErr: "at least one namespace", + }, + { + name: "Invalid namespace name", + config: ScopeConfig{WatchNamespaces: []string{"INVALID"}}, + ns: "team-a", cacheErr: "invalid namespace name", + }, + { + name: "Empty namespace name", + config: ScopeConfig{WatchNamespaces: []string{""}}, + expected: true, cacheErr: "invalid namespace name", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { allowed := tt.config.IsNamespaceAllowed(tt.ns) g.Expect(allowed).To(Equal(tt.expected)) + options, err := tt.config.CacheOptions() + if tt.cacheErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.cacheErr) { + t.Fatalf("want cache error containing %q, got %v", tt.cacheErr, err) + } + return + } + if err != nil { + t.Fatal(err) + } + var namespaces []string + for namespace := range options.DefaultNamespaces { + namespaces = append(namespaces, namespace) + } + sort.Strings(namespaces) + if !reflect.DeepEqual(namespaces, tt.cached) { + t.Fatalf("cache namespaces: got %v, want %v", namespaces, tt.cached) + } }) } } @@ -135,6 +201,7 @@ func TestAllowLogic(t *testing.T) { mkObj := func(kind, name, namespace string) *unstructured.Unstructured { u := &unstructured.Unstructured{} u.SetKind(kind) + u.SetAPIVersion("v1") u.SetName(name) u.SetNamespace(namespace) return u @@ -143,7 +210,7 @@ func TestAllowLogic(t *testing.T) { tests := []struct { name string config ScopeConfig - obj *unstructured.Unstructured + obj client.Object expected bool }{ { @@ -160,16 +227,31 @@ func TestAllowLogic(t *testing.T) { }, { name: "Namespace Object Allowed (Matches Watch)", - config: ScopeConfig{WatchNamespaces: []string{"foo"}}, + config: ScopeConfig{WatchNamespaces: []string{"foo"}, WatchResources: []string{"namespaces"}}, obj: mkObj("Namespace", "foo", ""), expected: true, }, { name: "Namespace Object Denied (Mismatch Watch)", - config: ScopeConfig{WatchNamespaces: []string{"foo"}}, + config: ScopeConfig{WatchNamespaces: []string{"foo"}, WatchResources: []string{"namespaces"}}, obj: mkObj("Namespace", "bar", ""), expected: false, }, + { + name: "Namespace filter alone does not permit Namespace actions", + config: ScopeConfig{WatchNamespaces: []string{"foo"}}, + obj: mkObj("Namespace", "foo", ""), + }, + { + name: "Resource wildcard does not permit Namespace actions with namespace filters", + config: ScopeConfig{WatchNamespaces: []string{"foo"}, WatchResources: []string{"*"}}, + obj: mkObj("Namespace", "foo", ""), + }, + { + name: "Ignore rule excludes explicitly selected Namespace objects", + config: ScopeConfig{WatchNamespaces: []string{"foo"}, WatchResources: []string{"namespaces"}, IgnoreResources: []string{"namespace*"}}, + obj: mkObj("Namespace", "foo", ""), + }, { name: "Cluster Resource Denied (Strict Mode)", config: ScopeConfig{WatchNamespaces: []string{"foo"}}, @@ -194,6 +276,18 @@ func TestAllowLogic(t *testing.T) { obj: &unstructured.Unstructured{Object: map[string]interface{}{"metadata": map[string]interface{}{"namespace": "foo"}}}, expected: true, }, + { + name: "Native Namespace without TypeMeta", + config: ScopeConfig{WatchNamespaces: []string{"foo"}, WatchResources: []string{"namespaces"}}, + obj: &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, expected: true, + }, + { + name: "Custom Namespace kind is not a native Namespace", + config: ScopeConfig{WatchNamespaces: []string{"foo"}, WatchResources: []string{"namespaces"}}, + obj: &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "example.com/v1", "kind": "Namespace", "metadata": map[string]interface{}{"name": "foo"}, + }}, + }, } for _, tt := range tests {