From 055c39068515082711323609f443f9673e672ffb Mon Sep 17 00:00:00 2001 From: Cezary Stanislawski Date: Fri, 2 Oct 2026 18:27:07 +0200 Subject: [PATCH] perf(test): overlap E2E namespace cleanup and reduce logs --- Makefile | 4 +- test/e2e/e2e_test.go | 395 ++---------------- test/e2e/lifecycle_actions_test.go | 339 +++++++++++++++ test/e2e/namespace_cleanup_regression_test.go | 79 ++++ test/e2e/namespace_cleanup_test.go | 90 ++++ 5 files changed, 537 insertions(+), 370 deletions(-) create mode 100644 test/e2e/lifecycle_actions_test.go create mode 100644 test/e2e/namespace_cleanup_regression_test.go create mode 100644 test/e2e/namespace_cleanup_test.go diff --git a/Makefile b/Makefile index a2fc7dd..a51e63f 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests .PHONY: test-e2e test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. - KIND=$(KIND) KIND_CLUSTER=$(E2E_KIND_CLUSTER) IMG=$(IMG) NAMESPACE=$(NAMESPACE) go test -tags=e2e ./test/e2e/ -v -ginkgo.v + KIND=$(KIND) KIND_CLUSTER=$(E2E_KIND_CLUSTER) IMG=$(IMG) NAMESPACE=$(NAMESPACE) go test -tags=e2e ./test/e2e/ -v $(MAKE) cleanup-test-e2e .PHONY: lint @@ -300,5 +300,5 @@ undeploy-helm: ## Undeploy controller using Helm. .PHONY: test-e2e-helm test-e2e-helm: setup-test-e2e manifests generate fmt vet ## Run the Helm e2e tests. - KIND=$(KIND) KIND_CLUSTER=$(E2E_KIND_CLUSTER) IMG=$(IMG) go test -tags=e2e_helm ./test/e2e-helm/ -v -ginkgo.v + KIND=$(KIND) KIND_CLUSTER=$(E2E_KIND_CLUSTER) IMG=$(IMG) go test -tags=e2e_helm ./test/e2e-helm/ -v $(MAKE) cleanup-test-e2e diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c6f86d1..51d4e3a 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -29,6 +29,7 @@ const pollInterval = 250 * time.Millisecond var _ = Describe("Lifecycle Controller E2E", Ordered, func() { var controllerPodName string + namespaces := &namespaceCleanup{} BeforeAll(func() { By("creating manager namespace") @@ -43,6 +44,9 @@ var _ = Describe("Lifecycle Controller E2E", Ordered, func() { }) AfterAll(func() { + By("waiting for test namespace cleanup") + cleanupErr := namespaces.wait(namespaceCleanupTimeout) + By("undeploying the controller-manager") cmd := exec.Command("make", "undeploy") _, _ = utils.Run(cmd) @@ -50,6 +54,7 @@ var _ = Describe("Lifecycle Controller E2E", Ordered, func() { By("removing manager namespace") cmd = exec.Command("kubectl", "delete", "ns", managerNamespace, "--ignore-not-found") _, _ = utils.Run(cmd) + Expect(cleanupErr).To(Succeed()) }) Context("Controller Health", func() { @@ -76,350 +81,13 @@ var _ = Describe("Lifecycle Controller E2E", Ordered, func() { }) }) - Context("Lifecycle Actions", func() { - const testNamespace = "lifecycle-e2e-tests" - - BeforeEach(func() { - By("creating test namespace") - cmd := exec.Command("kubectl", "create", "ns", testNamespace) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - By("deleting test namespace") - cmd := exec.Command("kubectl", "delete", "ns", testNamespace, "--ignore-not-found") - _, _ = utils.Run(cmd) - }) - - It("should delete a Deployment after the 'delete-after' duration", func() { - By("creating a Deployment with a short delete-after annotation") - deploymentName := "e2e-delete-after" - deploymentYAML := fmt.Sprintf(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: %s - namespace: %s - annotations: - lifecycle.cezary.dev/delete-after: "2s" -spec: - replicas: 1 - selector: - matchLabels: - app: test-delete - template: - metadata: - labels: - app: test-delete - spec: - containers: - - name: nginx - image: nginx:latest -`, deploymentName, testNamespace) - - utils.ApplyYAML(deploymentYAML) - - By("verifying the deployment is eventually deleted") - Eventually(func(g Gomega) { - cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", testNamespace) - _, err := utils.Run(cmd) - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("not found")) - }).WithTimeout(30 * time.Second).WithPolling(pollInterval).Should(Succeed()) - }) - - It("should reset the 'delete-after' timer on re-apply", func() { - By("creating a Deployment with a 'delete-after' annotation") - deploymentName := "e2e-delete-after-reset" - deploymentYAML := fmt.Sprintf(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: %s - namespace: %s - annotations: - lifecycle.cezary.dev/delete-after: "4s" -spec: - replicas: 1 - selector: - matchLabels: - app: test-delete-reset - template: - metadata: - labels: - app: test-delete-reset - spec: - containers: - - name: nginx - image: nginx:latest -`, deploymentName, testNamespace) - - utils.ApplyYAML(deploymentYAML) - - var firstDeleteAt time.Time - By("verifying the initial 'delete-at' annotation is set") - Eventually(func(g Gomega) { - dep := utils.GetDeployment(deploymentName, testNamespace, g) - deleteAtStr, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] - g.Expect(found).To(BeTrue(), "'delete-at' annotation should be present") - - var err error - firstDeleteAt, err = time.Parse(time.RFC3339, deleteAtStr) - g.Expect(err).NotTo(HaveOccurred()) - }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) - - By("waiting a moment before re-applying") - time.Sleep(1500 * time.Millisecond) - - By("re-applying the same manifest to reset the timer") - utils.ApplyYAML(deploymentYAML) - - By("verifying the 'delete-at' annotation is updated to a later time") - Eventually(func(g Gomega) { - dep := utils.GetDeployment(deploymentName, testNamespace, g) - newDeleteAtStr, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] - g.Expect(found).To(BeTrue()) - - newDeleteAt, err := time.Parse(time.RFC3339, newDeleteAtStr) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(newDeleteAt.After(firstDeleteAt)).To(BeTrue(), "The new deletion time should be later than the original") - }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) - }) - - It("should not change a fixed 'delete-at' time on re-apply", func() { - By("creating a Deployment with a fixed 'delete-at' annotation") - deploymentName := "e2e-delete-at-fixed" - deleteAtTime := time.Now().Add(6 * time.Second).UTC().Format(time.RFC3339) - - deploymentYAML := fmt.Sprintf(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: %s - namespace: %s - annotations: - lifecycle.cezary.dev/delete-at: "%s" -spec: - replicas: 1 - selector: - matchLabels: - app: test-delete-fixed - template: - metadata: - labels: - app: test-delete-fixed - spec: - containers: - - name: nginx - image: nginx:latest -`, deploymentName, testNamespace, deleteAtTime) - - utils.ApplyYAML(deploymentYAML) - - By("re-applying the same manifest after a delay") - time.Sleep(1 * time.Second) - utils.ApplyYAML(deploymentYAML) - - By("verifying the 'delete-at' annotation has not changed") - // We use Consistently here to ensure it doesn't flap - Consistently(func(g Gomega) { - dep := utils.GetDeployment(deploymentName, testNamespace, g) - currentDeleteAt, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] - g.Expect(found).To(BeTrue()) - g.Expect(currentDeleteAt).To(Equal(deleteAtTime)) - }).WithTimeout(2 * time.Second).WithPolling(pollInterval).Should(Succeed()) - - By("verifying the deployment is eventually deleted at the fixed time") - Eventually(func(g Gomega) { - cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", testNamespace) - _, err := utils.Run(cmd) - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("not found")) - }).WithTimeout(15 * time.Second).WithPolling(pollInterval).Should(Succeed()) - }) - - It("should not update the 'delete-at' time on unrelated resource updates", func() { - By("creating a Deployment with a 'delete-after' annotation") - deploymentName := "e2e-unrelated-update" - deploymentYAML := fmt.Sprintf(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: %s - namespace: %s - annotations: - lifecycle.cezary.dev/delete-after: "8s" -spec: - replicas: 1 - selector: - matchLabels: - app: test-unrelated - template: - metadata: - labels: - app: test-unrelated - spec: - containers: - - name: nginx - image: nginx:latest -`, deploymentName, testNamespace) - - utils.ApplyYAML(deploymentYAML) - - var initialDeleteAt string - By("verifying the initial 'delete-at' annotation is set") - Eventually(func(g Gomega) { - dep := utils.GetDeployment(deploymentName, testNamespace, g) - deleteAtStr, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] - g.Expect(found).To(BeTrue(), "'delete-at' annotation should be present") - g.Expect(dep.Annotations).NotTo(HaveKey("lifecycle.cezary.dev/delete-after"), "'delete-after' should be removed") - initialDeleteAt = deleteAtStr - }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) - - By("making an unrelated update to the deployment (adding an annotation)") - time.Sleep(1 * time.Second) - cmd := exec.Command("kubectl", "annotate", "deployment", deploymentName, "-n", testNamespace, "unrelated.io/test=true") - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred()) - - By("verifying the 'delete-at' annotation has not changed") - Consistently(func(g Gomega) { - dep := utils.GetDeployment(deploymentName, testNamespace, g) - currentDeleteAt, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] - g.Expect(found).To(BeTrue()) - g.Expect(currentDeleteAt).To(Equal(initialDeleteAt)) - }).WithTimeout(2 * time.Second).WithPolling(pollInterval).Should(Succeed()) - - By("verifying the deployment is eventually deleted at the original fixed time") - Eventually(func(g Gomega) { - cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", testNamespace) - _, err := utils.Run(cmd) - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("not found")) - }).WithTimeout(15 * time.Second).WithPolling(pollInterval).Should(Succeed()) - }) - - It("should restart a Deployment when 'restart-at' is in the past", func() { - By("creating a Deployment with a past restart-at annotation") - deploymentName := "e2e-restart-at" - pastTime := time.Now().Add(-5 * time.Minute).Format(time.RFC3339) - - deploymentYAML := fmt.Sprintf(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: %s - namespace: %s - annotations: - lifecycle.cezary.dev/restart-at: "%s" -spec: - replicas: 1 - selector: - matchLabels: - app: test-restart - template: - metadata: - labels: - app: test-restart - spec: - containers: - - name: nginx - image: nginx:latest -`, deploymentName, testNamespace, pastTime) - - utils.ApplyYAML(deploymentYAML) - - By("verifying the deployment is restarted and annotations are updated") - Eventually(func(g Gomega) { - dep := utils.GetDeployment(deploymentName, testNamespace, g) - - // Check that the main annotation is gone - _, found := dep.Annotations["lifecycle.cezary.dev/restart-at"] - g.Expect(found).To(BeFalse(), "restart-at annotation should be removed") - - // Check that the template annotation was added - _, found = dep.Spec.Template.Annotations["lifecycle.cezary.dev/restartedAt"] - g.Expect(found).To(BeTrue(), "restartedAt annotation should be added to the template") - - }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) - }) - - It("should reset the 'restart-after' timer on re-apply", func() { - By("creating a Deployment with a 'restart-after' annotation") - deploymentName := "e2e-restart-after-reset" - deploymentYAML := fmt.Sprintf(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: %s - namespace: %s - annotations: - lifecycle.cezary.dev/restart-after: "4s" -spec: - replicas: 1 - selector: - matchLabels: - app: test-restart-reset - template: - metadata: - labels: - app: test-restart-reset - spec: - containers: - - name: nginx - image: nginx:latest -`, deploymentName, testNamespace) - - utils.ApplyYAML(deploymentYAML) - - var firstRestartAt time.Time - By("verifying the initial 'restart-at' annotation is set") - Eventually(func(g Gomega) { - dep := utils.GetDeployment(deploymentName, testNamespace, g) - restartAtStr, found := dep.Annotations["lifecycle.cezary.dev/restart-at"] - g.Expect(found).To(BeTrue(), "'restart-at' annotation should be present") - - var err error - firstRestartAt, err = time.Parse(time.RFC3339, restartAtStr) - g.Expect(err).NotTo(HaveOccurred()) - }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) - - By("waiting a moment before re-applying") - time.Sleep(1500 * time.Millisecond) - - By("re-applying the same manifest to reset the timer") - utils.ApplyYAML(deploymentYAML) - - By("verifying the 'restart-at' annotation is updated to a later time") - Eventually(func(g Gomega) { - dep := utils.GetDeployment(deploymentName, testNamespace, g) - newRestartAtStr, found := dep.Annotations["lifecycle.cezary.dev/restart-at"] - g.Expect(found).To(BeTrue()) - - newRestartAt, err := time.Parse(time.RFC3339, newRestartAtStr) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(newRestartAt.After(firstRestartAt)).To(BeTrue(), "The new restart time should be later than the original") - }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) - }) - }) + lifecycleActionTests(namespaces) Context("Recurring and Advanced Actions", func() { - const testNamespace = "lifecycle-e2e-advanced" + var testNamespace string BeforeEach(func() { - By("creating advanced test namespace") - cmd := exec.Command("kubectl", "create", "ns", testNamespace) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - By("deleting advanced test namespace") - cmd := exec.Command("kubectl", "delete", "ns", testNamespace, "--ignore-not-found") - _, _ = utils.Run(cmd) + testNamespace = namespaces.create("lifecycle-e2e-advanced") }) It("should coalesce missed 'restart-every' occurrences into one rollout", func() { @@ -557,7 +225,8 @@ spec: Context("Cross-Resource Type and Error Handling", func() { It("should delete a Namespace and its contents", func() { - namespaceName := "e2e-ns-to-delete" + namespaceName := namespaces.reserve("e2e-ns-to-delete") + DeferCleanup(func() { Expect(namespaces.delete(namespaceName)).To(Succeed()) }) deleteAtTime := time.Now().Add(3 * time.Second).UTC().Format(time.RFC3339) namespaceYAML := fmt.Sprintf(` @@ -591,13 +260,14 @@ data: }) It("should ignore conflicting delete and restart annotations and post an event", func() { + testNamespace := namespaces.create("lifecycle-e2e-conflict") deploymentName := "e2e-conflict" deploymentYAML := fmt.Sprintf(` apiVersion: apps/v1 kind: Deployment metadata: name: %s - namespace: default + namespace: %s annotations: lifecycle.cezary.dev/delete-after: "1s" lifecycle.cezary.dev/restart-after: "1s" @@ -607,24 +277,24 @@ spec: template: metadata: { labels: { app: test-conflict }} spec: { containers: [ { name: nginx, image: nginx:latest } ] } -`, deploymentName) +`, deploymentName, testNamespace) utils.ApplyYAML(deploymentYAML) By("verifying the deployment is NOT deleted or restarted") Consistently(func(g Gomega) { // Check it still exists - cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", "default") + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", testNamespace) _, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred()) // Check it was not restarted - dep := utils.GetDeployment(deploymentName, "default", g) + dep := utils.GetDeployment(deploymentName, testNamespace, g) _, found := dep.Spec.Template.Annotations["lifecycle.cezary.dev/restartedAt"] g.Expect(found).To(BeFalse()) }).WithTimeout(5 * time.Second).WithPolling(pollInterval).Should(Succeed()) By("verifying a warning event was posted") Eventually(func(g Gomega) { - events := utils.GetEvents(g, "default", deploymentName, "Deployment") + events := utils.GetEvents(g, testNamespace, deploymentName, "Deployment") g.Expect(events).NotTo(BeEmpty()) foundEvent := false for _, event := range events { @@ -635,19 +305,17 @@ spec: } g.Expect(foundEvent).To(BeTrue(), "expected to find a ConflictingAnnotations warning event") }).WithTimeout(30 * time.Second).WithPolling(pollInterval).Should(Succeed()) - - cmd := exec.Command("kubectl", "delete", "deployment", deploymentName, "-n", "default") - _, _ = utils.Run(cmd) }) It("should not perform actions when 'dry-run' is true", func() { + testNamespace := namespaces.create("lifecycle-e2e-dry-run") deploymentName := "e2e-dry-run" deploymentYAML := fmt.Sprintf(` apiVersion: apps/v1 kind: Deployment metadata: name: %s - namespace: default + namespace: %s annotations: lifecycle.cezary.dev/delete-after: "2s" lifecycle.cezary.dev/dry-run: "true" @@ -657,19 +325,19 @@ spec: template: metadata: { labels: { app: test-dry-run }} spec: { containers: [ { name: nginx, image: nginx:latest } ] } -`, deploymentName) +`, deploymentName, testNamespace) utils.ApplyYAML(deploymentYAML) By("verifying the deployment is NOT deleted") Consistently(func(g Gomega) { - cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", "default") + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", testNamespace) _, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred()) }).WithTimeout(5 * time.Second).WithPolling(pollInterval).Should(Succeed()) By("verifying a dry-run event was posted") Eventually(func(g Gomega) { - events := utils.GetEvents(g, "default", deploymentName, "Deployment") + events := utils.GetEvents(g, testNamespace, deploymentName, "Deployment") g.Expect(events).NotTo(BeEmpty(), "expected events for dry-run") foundEvent := false for _, event := range events { @@ -680,15 +348,11 @@ spec: } g.Expect(foundEvent).To(BeTrue(), "expected to find DryRunDelete event") }).WithTimeout(20 * time.Second).WithPolling(pollInterval).Should(Succeed()) - - // Cleanup - cmd := exec.Command("kubectl", "delete", "deployment", deploymentName, "-n", "default") - _, _ = utils.Run(cmd) }) }) Context("Dynamic CRD Handling", func() { - const testNamespace = "lifecycle-e2e-crd" + var testNamespace string const crdName = "lifecyclee2etests.testing.lifecycle.cezary.dev" const crdYAML = ` apiVersion: apiextensions.k8s.io/v1 @@ -742,22 +406,17 @@ spec: }) AfterAll(func() { + // Keep the CRD available until namespace finalization completes. + By("waiting for namespace cleanup before removing the test CRD") + cleanupErr := namespaces.wait(namespaceCleanupTimeout) By("deleting the test CRD") cmd := exec.Command("kubectl", "delete", "crd", crdName, "--ignore-not-found") _, _ = utils.Run(cmd) + Expect(cleanupErr).To(Succeed()) }) BeforeEach(func() { - By("creating CRD test namespace") - cmd := exec.Command("kubectl", "create", "ns", testNamespace) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - By("deleting CRD test namespace") - cmd := exec.Command("kubectl", "delete", "ns", testNamespace, "--ignore-not-found") - _, _ = utils.Run(cmd) + testNamespace = namespaces.create("lifecycle-e2e-crd") }) It("should delete a custom resource instance after the 'delete-after' duration", func() { diff --git a/test/e2e/lifecycle_actions_test.go b/test/e2e/lifecycle_actions_test.go new file mode 100644 index 0000000..47266af --- /dev/null +++ b/test/e2e/lifecycle_actions_test.go @@ -0,0 +1,339 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "fmt" + "os/exec" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/cstanislawski/lifecycle-controller/test/utils" +) + +func lifecycleActionTests(namespaces *namespaceCleanup) { + Context("Lifecycle Actions", func() { + var testNamespace string + + BeforeEach(func() { + testNamespace = namespaces.create("lifecycle-e2e-tests") + }) + + It("should delete a Deployment after the 'delete-after' duration", func() { + By("creating a Deployment with a short delete-after annotation") + deploymentName := "e2e-delete-after" + deploymentYAML := fmt.Sprintf(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %s + namespace: %s + annotations: + lifecycle.cezary.dev/delete-after: "2s" +spec: + replicas: 1 + selector: + matchLabels: + app: test-delete + template: + metadata: + labels: + app: test-delete + spec: + containers: + - name: nginx + image: nginx:latest +`, deploymentName, testNamespace) + + utils.ApplyYAML(deploymentYAML) + + By("verifying the deployment is eventually deleted") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", testNamespace) + _, err := utils.Run(cmd) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("not found")) + }).WithTimeout(30 * time.Second).WithPolling(pollInterval).Should(Succeed()) + }) + + It("should reset the 'delete-after' timer on re-apply", func() { + By("creating a Deployment with a 'delete-after' annotation") + deploymentName := "e2e-delete-after-reset" + deploymentYAML := fmt.Sprintf(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %s + namespace: %s + annotations: + lifecycle.cezary.dev/delete-after: "4s" +spec: + replicas: 1 + selector: + matchLabels: + app: test-delete-reset + template: + metadata: + labels: + app: test-delete-reset + spec: + containers: + - name: nginx + image: nginx:latest +`, deploymentName, testNamespace) + + utils.ApplyYAML(deploymentYAML) + + var firstDeleteAt time.Time + By("verifying the initial 'delete-at' annotation is set") + Eventually(func(g Gomega) { + dep := utils.GetDeployment(deploymentName, testNamespace, g) + deleteAtStr, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] + g.Expect(found).To(BeTrue(), "'delete-at' annotation should be present") + + var err error + firstDeleteAt, err = time.Parse(time.RFC3339, deleteAtStr) + g.Expect(err).NotTo(HaveOccurred()) + }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) + + By("waiting a moment before re-applying") + time.Sleep(1500 * time.Millisecond) + + By("re-applying the same manifest to reset the timer") + utils.ApplyYAML(deploymentYAML) + + By("verifying the 'delete-at' annotation is updated to a later time") + Eventually(func(g Gomega) { + dep := utils.GetDeployment(deploymentName, testNamespace, g) + newDeleteAtStr, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] + g.Expect(found).To(BeTrue()) + + newDeleteAt, err := time.Parse(time.RFC3339, newDeleteAtStr) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(newDeleteAt.After(firstDeleteAt)).To(BeTrue(), "The new deletion time should be later than the original") + }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) + }) + + It("should not change a fixed 'delete-at' time on re-apply", func() { + By("creating a Deployment with a fixed 'delete-at' annotation") + deploymentName := "e2e-delete-at-fixed" + deleteAtTime := time.Now().Add(6 * time.Second).UTC().Format(time.RFC3339) + + deploymentYAML := fmt.Sprintf(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %s + namespace: %s + annotations: + lifecycle.cezary.dev/delete-at: "%s" +spec: + replicas: 1 + selector: + matchLabels: + app: test-delete-fixed + template: + metadata: + labels: + app: test-delete-fixed + spec: + containers: + - name: nginx + image: nginx:latest +`, deploymentName, testNamespace, deleteAtTime) + + utils.ApplyYAML(deploymentYAML) + + By("re-applying the same manifest after a delay") + time.Sleep(1 * time.Second) + utils.ApplyYAML(deploymentYAML) + + By("verifying the 'delete-at' annotation has not changed") + // We use Consistently here to ensure it doesn't flap + Consistently(func(g Gomega) { + dep := utils.GetDeployment(deploymentName, testNamespace, g) + currentDeleteAt, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] + g.Expect(found).To(BeTrue()) + g.Expect(currentDeleteAt).To(Equal(deleteAtTime)) + }).WithTimeout(2 * time.Second).WithPolling(pollInterval).Should(Succeed()) + + By("verifying the deployment is eventually deleted at the fixed time") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", testNamespace) + _, err := utils.Run(cmd) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("not found")) + }).WithTimeout(15 * time.Second).WithPolling(pollInterval).Should(Succeed()) + }) + + It("should not update the 'delete-at' time on unrelated resource updates", func() { + By("creating a Deployment with a 'delete-after' annotation") + deploymentName := "e2e-unrelated-update" + deploymentYAML := fmt.Sprintf(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %s + namespace: %s + annotations: + lifecycle.cezary.dev/delete-after: "8s" +spec: + replicas: 1 + selector: + matchLabels: + app: test-unrelated + template: + metadata: + labels: + app: test-unrelated + spec: + containers: + - name: nginx + image: nginx:latest +`, deploymentName, testNamespace) + + utils.ApplyYAML(deploymentYAML) + + var initialDeleteAt string + By("verifying the initial 'delete-at' annotation is set") + Eventually(func(g Gomega) { + dep := utils.GetDeployment(deploymentName, testNamespace, g) + deleteAtStr, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] + g.Expect(found).To(BeTrue(), "'delete-at' annotation should be present") + g.Expect(dep.Annotations).NotTo(HaveKey("lifecycle.cezary.dev/delete-after"), "'delete-after' should be removed") + initialDeleteAt = deleteAtStr + }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) + + By("making an unrelated update to the deployment (adding an annotation)") + time.Sleep(1 * time.Second) + cmd := exec.Command("kubectl", "annotate", "deployment", deploymentName, "-n", testNamespace, "unrelated.io/test=true") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying the 'delete-at' annotation has not changed") + Consistently(func(g Gomega) { + dep := utils.GetDeployment(deploymentName, testNamespace, g) + currentDeleteAt, found := dep.Annotations["lifecycle.cezary.dev/delete-at"] + g.Expect(found).To(BeTrue()) + g.Expect(currentDeleteAt).To(Equal(initialDeleteAt)) + }).WithTimeout(2 * time.Second).WithPolling(pollInterval).Should(Succeed()) + + By("verifying the deployment is eventually deleted at the original fixed time") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", testNamespace) + _, err := utils.Run(cmd) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("not found")) + }).WithTimeout(15 * time.Second).WithPolling(pollInterval).Should(Succeed()) + }) + + It("should restart a Deployment when 'restart-at' is in the past", func() { + By("creating a Deployment with a past restart-at annotation") + deploymentName := "e2e-restart-at" + pastTime := time.Now().Add(-5 * time.Minute).Format(time.RFC3339) + + deploymentYAML := fmt.Sprintf(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %s + namespace: %s + annotations: + lifecycle.cezary.dev/restart-at: "%s" +spec: + replicas: 1 + selector: + matchLabels: + app: test-restart + template: + metadata: + labels: + app: test-restart + spec: + containers: + - name: nginx + image: nginx:latest +`, deploymentName, testNamespace, pastTime) + + utils.ApplyYAML(deploymentYAML) + + By("verifying the deployment is restarted and annotations are updated") + Eventually(func(g Gomega) { + dep := utils.GetDeployment(deploymentName, testNamespace, g) + + // Check that the main annotation is gone + _, found := dep.Annotations["lifecycle.cezary.dev/restart-at"] + g.Expect(found).To(BeFalse(), "restart-at annotation should be removed") + + // Check that the template annotation was added + _, found = dep.Spec.Template.Annotations["lifecycle.cezary.dev/restartedAt"] + g.Expect(found).To(BeTrue(), "restartedAt annotation should be added to the template") + + }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) + }) + + It("should reset the 'restart-after' timer on re-apply", func() { + By("creating a Deployment with a 'restart-after' annotation") + deploymentName := "e2e-restart-after-reset" + deploymentYAML := fmt.Sprintf(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %s + namespace: %s + annotations: + lifecycle.cezary.dev/restart-after: "4s" +spec: + replicas: 1 + selector: + matchLabels: + app: test-restart-reset + template: + metadata: + labels: + app: test-restart-reset + spec: + containers: + - name: nginx + image: nginx:latest +`, deploymentName, testNamespace) + + utils.ApplyYAML(deploymentYAML) + + var firstRestartAt time.Time + By("verifying the initial 'restart-at' annotation is set") + Eventually(func(g Gomega) { + dep := utils.GetDeployment(deploymentName, testNamespace, g) + restartAtStr, found := dep.Annotations["lifecycle.cezary.dev/restart-at"] + g.Expect(found).To(BeTrue(), "'restart-at' annotation should be present") + + var err error + firstRestartAt, err = time.Parse(time.RFC3339, restartAtStr) + g.Expect(err).NotTo(HaveOccurred()) + }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) + + By("waiting a moment before re-applying") + time.Sleep(1500 * time.Millisecond) + + By("re-applying the same manifest to reset the timer") + utils.ApplyYAML(deploymentYAML) + + By("verifying the 'restart-at' annotation is updated to a later time") + Eventually(func(g Gomega) { + dep := utils.GetDeployment(deploymentName, testNamespace, g) + newRestartAtStr, found := dep.Annotations["lifecycle.cezary.dev/restart-at"] + g.Expect(found).To(BeTrue()) + + newRestartAt, err := time.Parse(time.RFC3339, newRestartAtStr) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(newRestartAt.After(firstRestartAt)).To(BeTrue(), "The new restart time should be later than the original") + }).WithTimeout(time.Minute).WithPolling(pollInterval).Should(Succeed()) + }) + }) + +} diff --git a/test/e2e/namespace_cleanup_regression_test.go b/test/e2e/namespace_cleanup_regression_test.go new file mode 100644 index 0000000..cd90594 --- /dev/null +++ b/test/e2e/namespace_cleanup_regression_test.go @@ -0,0 +1,79 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "os/exec" + "strings" + "testing" + "time" + + "github.com/cstanislawski/lifecycle-controller/test/utils" +) + +func TestNamespaceCleanup(t *testing.T) { + cleanup := &namespaceCleanup{} + first := cleanup.reserve("lifecycle-e2e-cleanup") + second := cleanup.reserve("lifecycle-e2e-cleanup") + if first == second { + t.Fatal("test namespaces must be unique") + } + run := func(args ...string) string { + t.Helper() + output, err := utils.Run(exec.Command("kubectl", args...)) + if err != nil { + t.Fatal(err) + } + return output + } + t.Cleanup(func() { + // Release the probe finalizer even if an assertion fails. + _, _ = utils.Run(exec.Command("kubectl", "patch", "configmap", "cleanup-probe", "-n", first, + "--type=merge", "-p", `{"metadata":{"finalizers":[]}}`, "--request-timeout=10s")) + if err := cleanup.wait(30 * time.Second); err != nil { + t.Error(err) + } + }) + run("create", "namespace", first, "--request-timeout=10s") + run("create", "configmap", "cleanup-probe", "-n", first, "--from-literal=key=value") + run("patch", "configmap", "cleanup-probe", "-n", first, "--type=merge", "-p", + `{"metadata":{"finalizers":["testing.lifecycle.cezary.dev/cleanup-probe"]}}`) + if err := cleanup.delete(first); err != nil { + t.Fatal(err) + } + if run("get", "namespace", first, "-o", "jsonpath={.metadata.deletionTimestamp}") == "" { + t.Fatal("cleanup must request deletion before it returns") + } + // The first namespace cannot finish deletion while its resource has a finalizer. + run("create", "namespace", second, "--request-timeout=10s") + run("create", "configmap", "cleanup-probe", "-n", second, "--from-literal=key=second") + if run("get", "configmap", "cleanup-probe", "-n", second, "-o", "jsonpath={.data.key}") != "second" { + t.Fatal("subsequent cases must use isolated resources while cleanup is pending") + } + run("get", "configmap", "cleanup-probe", "-n", first) + err := cleanup.wait(time.Second) + if err == nil || !strings.Contains(err.Error(), first) || !strings.Contains(err.Error(), "Namespaces:") { + t.Fatalf("blocked cleanup must fail with namespace diagnostics, got %v", err) + } + if len(cleanup.names) != 2 { + t.Fatal("failed cleanup must retain namespaces for retry") + } + run("patch", "configmap", "cleanup-probe", "-n", first, "--type=merge", "-p", + `{"metadata":{"finalizers":[]}}`) + if err := cleanup.wait(30 * time.Second); err != nil { + t.Fatal(err) + } + if len(cleanup.names) != 0 { + t.Fatal("successful cleanup must drain all tracked namespaces") + } + output := run("get", "namespace", first, second, "--ignore-not-found", "-o", "name") + if strings.TrimSpace(output) != "" { + t.Fatalf("cleanup left namespaces in the cluster: %s", output) + } + // An API create failure can leave a tracked name that never existed. + cleanup.reserve("lifecycle-e2e-missing") + if err := cleanup.wait(time.Second); err != nil { + t.Fatalf("cleanup must accept a namespace that is already absent: %v", err) + } +} diff --git a/test/e2e/namespace_cleanup_test.go b/test/e2e/namespace_cleanup_test.go new file mode 100644 index 0000000..775919a --- /dev/null +++ b/test/e2e/namespace_cleanup_test.go @@ -0,0 +1,90 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "context" + "fmt" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/uuid" + + "github.com/cstanislawski/lifecycle-controller/test/utils" +) + +const namespaceCleanupTimeout = 2 * time.Minute + +type namespaceCleanup struct { + names []string +} + +func (c *namespaceCleanup) reserve(prefix string) string { + name := prefix + "-" + string(uuid.NewUUID()) + c.names = append(c.names, name) + return name +} + +func (c *namespaceCleanup) create(prefix string) string { + name := c.reserve(prefix) + // Register cleanup before creation. An API response can be lost after creation. + DeferCleanup(func() { + By("requesting namespace cleanup: " + name) + Expect(c.delete(name)).To(Succeed()) + }) + By("creating test namespace: " + name) + _, err := utils.Run(exec.Command("kubectl", "create", "namespace", name, "--request-timeout=30s")) + Expect(err).NotTo(HaveOccurred()) + return name +} + +func (c *namespaceCleanup) delete(names ...string) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + args := append([]string{"delete", "namespace"}, names...) + args = append(args, "--ignore-not-found", "--wait=false", "--request-timeout=30s") + _, err := utils.Run(exec.CommandContext(ctx, "kubectl", args...)) + return err +} + +func (c *namespaceCleanup) wait(timeout time.Duration) error { + if len(c.names) == 0 { + return nil + } + // Retry requests that failed during case cleanup before waiting for all namespaces. + if err := c.delete(c.names...); err != nil { + return c.failure(err) + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + args := append([]string{"wait", "--for=delete", "namespace"}, c.names...) + args = append(args, "--timeout="+timeout.String()) + _, err := utils.Run(exec.CommandContext(ctx, "kubectl", args...)) + if err != nil { + return c.failure(err) + } + c.names = nil + return nil +} + +func (c *namespaceCleanup) failure(cause error) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + var diagnostics strings.Builder + args := append([]string{"get", "namespace"}, c.names...) + args = append(args, "--ignore-not-found", "-o", "yaml") + output, err := utils.Run(exec.CommandContext(ctx, "kubectl", args...)) + _, _ = fmt.Fprintf(&diagnostics, "Namespaces:\n%s\nError: %v\n", output, err) + for _, name := range c.names { + if ctx.Err() != nil { + break + } + output, err = utils.Run(exec.CommandContext(ctx, "kubectl", "get", "pods", "-n", name, "-o", "yaml")) + _, _ = fmt.Fprintf(&diagnostics, "Pods in %s:\n%s\nError: %v\n", name, output, err) + } + return fmt.Errorf("namespace cleanup failed for %v: %w\n%s", c.names, cause, diagnostics.String()) +}