From e8c02e7b11ccaeaa804e5cda5da63c9e01ef0483 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Fri, 26 Jun 2026 13:47:55 +0600 Subject: [PATCH 1/3] Re-enable controller test suite and harden scheme registration The ported upstream StatefulSet controller tests in pkg/controller/tests/ were entirely commented out (~6500 lines) and lived in a package that could not access the unexported identifiers they test, so none of them ran. Move the five test files into pkg/controller/petset/ as white-box tests and adapt them to the fork's API drift: - Update the fakeObjectManager to the current StatefulPodControlObjectManager interface (PetSet params, ListPods, GetPlacementPolicy). - Add thin test adapters for newPetSetPod and NewStatefulPodControl, and wire the PlacementPolicy informer plus a fake OCM ManifestWork client/informer into newFakePetSetController and NewPetSetController. - Point feature-gate tests at the fork's features.DefaultFeatureGate and the new SetFeatureGateDuringTest signature. - Restore lost helpers (ascendingOrdinal, overlappingPetSets). Fix bugs surfaced while re-enabling the tests: - setupController listed StatefulSets from the kube client instead of PetSets from the api client, so the set never entered the indexer. - TestPetSetControl_getSetRevisions waited on a kube StatefulSets informer that was instantiated after Start() and never synced, hanging the suite. - newFakePetSetController seeded CRD objects into the core kube fake client, which panics; route PetSet/PlacementPolicy to the versioned fake. Harden patchCodec: register the PetSet/PlacementPolicy types with the client-go scheme in an init() next to patchCodec instead of relying on a PersistentPreRunE hook in cmd wiring. Otherwise constructing the controller through any other path panics with "no kind is registered for the type v1.PetSet" the first time a ControllerRevision is created. Add placement_test.go covering the AppsCode placement logic: the upsert helpers, spread/anti-affinity and node-affinity generation, domain selection in getAppropriateDomainIndex, and the CEL evaluator with its program cache. Signed-off-by: Tamal Saha --- pkg/cmds/root.go | 9 +- pkg/controller/petset/pet_pod_control_test.go | 885 +++++ pkg/controller/petset/pet_set_control_test.go | 3416 +++++++++++++++++ .../petset/pet_set_status_updater_test.go | 142 + pkg/controller/petset/pet_set_test.go | 1109 ++++++ pkg/controller/petset/pet_set_utils.go | 11 + pkg/controller/petset/pet_set_utils_test.go | 1009 +++++ pkg/controller/placement_test.go | 497 +++ pkg/controller/tests/pet_pod_control_test.go | 865 ----- pkg/controller/tests/pet_set_control_test.go | 3391 ---------------- .../tests/pet_set_status_updater_test.go | 143 - pkg/controller/tests/pet_set_test.go | 1092 ------ pkg/controller/tests/pet_set_utils_test.go | 983 ----- .../featuregate/testing/feature_gate.go | 200 + .../klog/v2/internal/verbosity/verbosity.go | 303 ++ vendor/k8s.io/klog/v2/ktesting/options.go | 132 + vendor/k8s.io/klog/v2/ktesting/setup.go | 38 + .../k8s.io/klog/v2/ktesting/testinglogger.go | 406 ++ vendor/modules.txt | 6 + .../versioned/fake/clientset_generated.go | 86 + .../work/clientset/versioned/fake/doc.go | 5 + .../work/clientset/versioned/fake/register.go | 43 + .../versioned/typed/work/v1/fake/doc.go | 5 + .../work/v1/fake/fake_appliedmanifestwork.go | 37 + .../typed/work/v1/fake/fake_manifestwork.go | 35 + .../typed/work/v1/fake/fake_work_client.go | 29 + .../versioned/typed/work/v1alpha1/fake/doc.go | 5 + .../fake/fake_manifestworkreplicaset.go | 37 + .../work/v1alpha1/fake/fake_work_client.go | 25 + 29 files changed, 8464 insertions(+), 6480 deletions(-) create mode 100644 pkg/controller/petset/pet_pod_control_test.go create mode 100644 pkg/controller/petset/pet_set_control_test.go create mode 100644 pkg/controller/petset/pet_set_status_updater_test.go create mode 100644 pkg/controller/petset/pet_set_test.go create mode 100644 pkg/controller/petset/pet_set_utils_test.go create mode 100644 pkg/controller/placement_test.go delete mode 100644 pkg/controller/tests/pet_pod_control_test.go delete mode 100644 pkg/controller/tests/pet_set_control_test.go delete mode 100644 pkg/controller/tests/pet_set_status_updater_test.go delete mode 100644 pkg/controller/tests/pet_set_test.go delete mode 100644 pkg/controller/tests/pet_set_utils_test.go create mode 100644 vendor/k8s.io/component-base/featuregate/testing/feature_gate.go create mode 100644 vendor/k8s.io/klog/v2/internal/verbosity/verbosity.go create mode 100644 vendor/k8s.io/klog/v2/ktesting/options.go create mode 100644 vendor/k8s.io/klog/v2/ktesting/setup.go create mode 100644 vendor/k8s.io/klog/v2/ktesting/testinglogger.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/clientset_generated.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/doc.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/register.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/doc.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_appliedmanifestwork.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_manifestwork.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_work_client.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/doc.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/fake_manifestworkreplicaset.go create mode 100644 vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/fake_work_client.go diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index 15456f72..a9ea1199 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -17,21 +17,18 @@ limitations under the License. package cmds import ( - api "kubeops.dev/petset/apis/apps/v1" - "github.com/spf13/cobra" v "gomodules.xyz/x/version" genericapiserver "k8s.io/apiserver/pkg/server" - clientscheme "k8s.io/client-go/kubernetes/scheme" ) func NewRootCmd() *cobra.Command { + // The PetSet/PlacementPolicy API types are registered with the client-go + // scheme via the petset controller package's init(), so there is no need to + // do it here in a PersistentPreRunE hook. rootCmd := &cobra.Command{ Use: "petset", DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return api.AddToScheme(clientscheme.Scheme) - }, } rootCmd.AddCommand(v.NewCmdVersion()) diff --git a/pkg/controller/petset/pet_pod_control_test.go b/pkg/controller/petset/pet_pod_control_test.go new file mode 100644 index 00000000..7b2de609 --- /dev/null +++ b/pkg/controller/petset/pet_pod_control_test.go @@ -0,0 +1,885 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package petset + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + api "kubeops.dev/petset/apis/apps/v1" + // _ "kubeops.dev/petset/pkg/apis/apps/install" + // _ "kubeops.dev/petset/pkg/apis/core/install" + "kubeops.dev/petset/pkg/features" + + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + corelisters "k8s.io/client-go/listers/core/v1" + core "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/klog/v2/ktesting" +) + +// newTestStatefulPodControl adapts the upstream 4-argument NewStatefulPodControl +// call signature (client, podLister, claimLister, recorder) to the forked +// constructor which also accepts a ManifestWork client, ManifestWork lister and +// PlacementPolicy lister for distributed PetSets (all nil in these tests). +func newTestStatefulPodControl(client clientset.Interface, podLister corelisters.PodLister, claimLister corelisters.PersistentVolumeClaimLister, recorder record.EventRecorder) *StatefulPodControl { + return NewStatefulPodControl(client, nil, podLister, nil, nil, claimLister, recorder) +} + +// overlappingPetSets sorts a list of PetSets by creation timestamp, breaking +// ties by name. Mirrors the upstream overlappingStatefulSets test helper. +type overlappingPetSets []*api.PetSet + +func (o overlappingPetSets) Len() int { return len(o) } +func (o overlappingPetSets) Swap(i, j int) { o[i], o[j] = o[j], o[i] } +func (o overlappingPetSets) Less(i, j int) bool { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { + return o[i].Name < o[j].Name + } + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) +} + +func TestStatefulPodControlCreatesPods(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := newTestStatefulPodControl(fakeClient, nil, claimLister, recorder) + fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), action.GetResource().Resource) + }) + fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + create := action.(core.CreateAction) + claimIndexer.Add(create.GetObject()) + return true, create.GetObject(), nil + }) + fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { + create := action.(core.CreateAction) + return true, create.GetObject(), nil + }) + if err := control.CreateStatefulPod(context.TODO(), set, pod); err != nil { + t.Errorf("StatefulPodControl failed to create Pod error: %s", err) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 2 { + t.Errorf("Expected 2 events for successful create found %d", eventCount) + } + for i := range events { + if !strings.Contains(events[i], v1.EventTypeNormal) { + t.Errorf("Found unexpected non-normal event %s", events[i]) + } + } +} + +func TestStatefulPodControlCreatePodExists(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + pvcs := getPersistentVolumeClaims(set, pod) + pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for k := range pvcs { + pvc := pvcs[k] + pvcIndexer.Add(&pvc) + } + pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) + control := newTestStatefulPodControl(fakeClient, nil, pvcLister, recorder) + fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + create := action.(core.CreateAction) + return true, create.GetObject(), nil + }) + fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { + return true, pod, apierrors.NewAlreadyExists(action.GetResource().GroupResource(), pod.Name) + }) + if err := control.CreateStatefulPod(context.TODO(), set, pod); !apierrors.IsAlreadyExists(err) { + t.Errorf("Failed to create Pod error: %s", err) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 0 { + t.Errorf("Pod and PVC exist: got %d events, but want 0", eventCount) + for i := range events { + t.Log(events[i]) + } + } +} + +func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) + control := newTestStatefulPodControl(fakeClient, nil, pvcLister, recorder) + fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("API server down")) + }) + fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { + create := action.(core.CreateAction) + return true, create.GetObject(), nil + }) + if err := control.CreateStatefulPod(context.TODO(), set, pod); err == nil { + t.Error("Failed to produce error on PVC creation failure") + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 2 { + t.Errorf("PVC create failure: got %d events, but want 2", eventCount) + } + for i := range events { + if !strings.Contains(events[i], v1.EventTypeWarning) { + t.Errorf("Found unexpected non-warning event %s", events[i]) + } + } +} + +func TestStatefulPodControlCreatePodPVCDeleting(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + pvcs := getPersistentVolumeClaims(set, pod) + pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + deleteTime := time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC) + for k := range pvcs { + pvc := pvcs[k] + pvc.DeletionTimestamp = &metav1.Time{Time: deleteTime} + pvcIndexer.Add(&pvc) + } + pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) + control := newTestStatefulPodControl(fakeClient, nil, pvcLister, recorder) + fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + create := action.(core.CreateAction) + return true, create.GetObject(), nil + }) + fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { + create := action.(core.CreateAction) + return true, create.GetObject(), nil + }) + if err := control.CreateStatefulPod(context.TODO(), set, pod); err == nil { + t.Error("Failed to produce error on deleting PVC") + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("Deleting PVC: got %d events, but want 1", eventCount) + } + for i := range events { + if !strings.Contains(events[i], v1.EventTypeWarning) { + t.Errorf("Found unexpected non-warning event %s", events[i]) + } + } +} + +type fakeIndexer struct { + cache.Indexer + getError error +} + +func (f *fakeIndexer) GetByKey(key string) (interface{}, bool, error) { + return nil, false, f.getError +} + +func TestStatefulPodControlCreatePodPvcGetFailure(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + pvcIndexer := &fakeIndexer{getError: errors.New("API server down")} + pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) + control := newTestStatefulPodControl(fakeClient, nil, pvcLister, recorder) + fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("API server down")) + }) + fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { + create := action.(core.CreateAction) + return true, create.GetObject(), nil + }) + if err := control.CreateStatefulPod(context.TODO(), set, pod); err == nil { + t.Error("Failed to produce error on PVC creation failure") + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 2 { + t.Errorf("PVC create failure: got %d events, but want 2", eventCount) + } + for i := range events { + if !strings.Contains(events[i], v1.EventTypeWarning) { + t.Errorf("Found unexpected non-warning event: %s", events[i]) + } + } +} + +func TestStatefulPodControlCreatePodFailed(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) + control := newTestStatefulPodControl(fakeClient, nil, pvcLister, recorder) + fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + create := action.(core.CreateAction) + return true, create.GetObject(), nil + }) + fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("API server down")) + }) + if err := control.CreateStatefulPod(context.TODO(), set, pod); err == nil { + t.Error("Failed to produce error on Pod creation failure") + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 2 { + t.Errorf("Pod create failed: got %d events, but want 2", eventCount) + } else if !strings.Contains(events[0], v1.EventTypeNormal) { + t.Errorf("Found unexpected non-normal event %s", events[0]) + } else if !strings.Contains(events[1], v1.EventTypeWarning) { + t.Errorf("Found unexpected non-warning event %s", events[1]) + } +} + +func TestStatefulPodControlNoOpUpdate(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + claims := getPersistentVolumeClaims(set, pod) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for k := range claims { + claim := claims[k] + indexer.Add(&claim) + } + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + control := newTestStatefulPodControl(fakeClient, nil, claimLister, recorder) + fakeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) { + t.Error("no-op update should not make any client invocation") + return true, nil, apierrors.NewInternalError(errors.New("if we are here we have a problem")) + }) + if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { + t.Errorf("Error returned on no-op update error: %s", err) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 0 { + t.Errorf("no-op update: got %d events, but want 0", eventCount) + } +} + +func TestStatefulPodControlUpdatesIdentity(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := fake.NewSimpleClientset(pod) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + control := newTestStatefulPodControl(fakeClient, nil, claimLister, recorder) + var updated *v1.Pod + fakeClient.PrependReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + updated = update.GetObject().(*v1.Pod) + return true, update.GetObject(), nil + }) + pod.Name = "goo-0" + if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { + t.Errorf("Successful update returned an error: %s", err) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("Pod update successful:got %d events,but want 1", eventCount) + } else if !strings.Contains(events[0], v1.EventTypeNormal) { + t.Errorf("Found unexpected non-normal event %s", events[0]) + } + if !identityMatches(set, updated) { + t.Error("Name update failed identity does not match") + } +} + +func TestStatefulPodControlUpdateIdentityFailure(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + gooPod := newTestPetSetPod(set, 0) + gooPod.Name = "goo-0" + podIndexer.Add(gooPod) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := newTestStatefulPodControl(fakeClient, podLister, claimLister, recorder) + fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { + pod.Name = "goo-0" + return true, nil, apierrors.NewInternalError(errors.New("API server down")) + }) + pod.Name = "goo-0" + if err := control.UpdateStatefulPod(ctx, set, pod); err == nil { + t.Error("Failed update does not generate an error") + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("Pod update failed: got %d events, but want 1", eventCount) + } else if !strings.Contains(events[0], v1.EventTypeWarning) { + t.Errorf("Found unexpected non-warning event %s", events[0]) + } + if identityMatches(set, pod) { + t.Error("Failed update mutated Pod identity") + } +} + +func TestStatefulPodControlUpdatesPodStorage(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) + control := newTestStatefulPodControl(fakeClient, nil, pvcLister, recorder) + pvcs := getPersistentVolumeClaims(set, pod) + volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) + for i := range pod.Spec.Volumes { + if _, contains := pvcs[pod.Spec.Volumes[i].Name]; !contains { + volumes = append(volumes, pod.Spec.Volumes[i]) + } + } + pod.Spec.Volumes = volumes + fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + return true, update.GetObject(), nil + }) + fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + return true, update.GetObject(), nil + }) + var updated *v1.Pod + fakeClient.PrependReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + updated = update.GetObject().(*v1.Pod) + return true, update.GetObject(), nil + }) + if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { + t.Errorf("Successful update returned an error: %s", err) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 2 { + t.Errorf("Pod storage update successful: got %d events, but want 2", eventCount) + } + for i := range events { + if !strings.Contains(events[i], v1.EventTypeNormal) { + t.Errorf("Found unexpected non-normal event %s", events[i]) + } + } + if !storageMatches(set, updated) { + t.Error("Name update failed identity does not match") + } +} + +func TestStatefulPodControlUpdatePodStorageFailure(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) + control := newTestStatefulPodControl(fakeClient, nil, pvcLister, recorder) + pvcs := getPersistentVolumeClaims(set, pod) + volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) + for i := range pod.Spec.Volumes { + if _, contains := pvcs[pod.Spec.Volumes[i].Name]; !contains { + volumes = append(volumes, pod.Spec.Volumes[i]) + } + } + pod.Spec.Volumes = volumes + fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + return true, update.GetObject(), nil + }) + fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("API server down")) + }) + if err := control.UpdateStatefulPod(ctx, set, pod); err == nil { + t.Error("Failed Pod storage update did not return an error") + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 2 { + t.Errorf("Pod storage update failed: got %d events, but want 2", eventCount) + } + for i := range events { + if !strings.Contains(events[i], v1.EventTypeWarning) { + t.Errorf("Found unexpected non-normal event %s", events[i]) + } + } +} + +func TestStatefulPodControlUpdatePodConflictSuccess(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(podIndexer) + gooPod := newTestPetSetPod(set, 0) + gooPod.Labels[apps.StatefulSetPodNameLabel] = "goo-starts" + podIndexer.Add(gooPod) + claims := getPersistentVolumeClaims(set, gooPod) + for k := range claims { + claim := claims[k] + claimIndexer.Add(&claim) + } + control := newTestStatefulPodControl(fakeClient, podLister, claimLister, recorder) + conflict := false + fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + if !conflict { + conflict = true + return true, update.GetObject(), apierrors.NewConflict(action.GetResource().GroupResource(), pod.Name, errors.New("conflict")) + } + return true, update.GetObject(), nil + }) + pod.Labels[apps.StatefulSetPodNameLabel] = "goo-0" + if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { + t.Errorf("Successful update returned an error: %s", err) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("Pod update successful: got %d, but want 1", eventCount) + } else if !strings.Contains(events[0], v1.EventTypeNormal) { + t.Errorf("Found unexpected non-normal event %s", events[0]) + } + if !identityMatches(set, pod) { + t.Error("Name update failed identity does not match") + } +} + +func TestStatefulPodControlDeletesStatefulPod(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + control := newTestStatefulPodControl(fakeClient, nil, nil, recorder) + fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + if err := control.DeleteStatefulPod(set, pod); err != nil { + t.Errorf("Error returned on successful delete: %s", err) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("delete successful: got %d events, but want 1", eventCount) + } else if !strings.Contains(events[0], v1.EventTypeNormal) { + t.Errorf("Found unexpected non-normal event %s", events[0]) + } +} + +func TestStatefulPodControlDeleteFailure(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + control := newTestStatefulPodControl(fakeClient, nil, nil, recorder) + fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("API server down")) + }) + if err := control.DeleteStatefulPod(set, pod); err == nil { + t.Error("Failed to return error on failed delete") + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("delete failed: got %d events, but want 1", eventCount) + } else if !strings.Contains(events[0], v1.EventTypeWarning) { + t.Errorf("Found unexpected non-warning event %s", events[0]) + } +} + +func TestStatefulPodControlClaimsMatchDeletionPolcy(t *testing.T) { + // The claimOwnerMatchesSetAndPod is tested exhaustively in stateful_set_utils_test; this + // test is for the wiring to the method tested there. + _, ctx := ktesting.NewTestContext(t) + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + claims := getPersistentVolumeClaims(set, pod) + for k := range claims { + claim := claims[k] + indexer.Add(&claim) + } + control := newTestStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{}) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if matches, err := control.ClaimsMatchRetentionPolicy(ctx, set, pod); err != nil { + t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (retain): %v", err) + } else if !matches { + t.Error("Unexpected non-match for ClaimsMatchRetentionPolicy (retain)") + } + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if matches, err := control.ClaimsMatchRetentionPolicy(ctx, set, pod); err != nil { + t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (set deletion): %v", err) + } else if matches { + t.Error("Unexpected match for ClaimsMatchRetentionPolicy (set deletion)") + } +} + +func TestStatefulPodControlUpdatePodClaimForRetentionPolicy(t *testing.T) { + // All the update conditions are tested exhaustively in stateful_set_utils_test. This + // tests the wiring from the pod control to that method. + testFn := func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, true) + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + indexer.Update(update.GetObject()) + return true, update.GetObject(), nil + }) + set := newPetSet(3) + set.GetObjectMeta().SetUID("set-123") + pod := newTestPetSetPod(set, 0) + claims := getPersistentVolumeClaims(set, pod) + for k := range claims { + claim := claims[k] + indexer.Add(&claim) + } + control := newTestStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{}) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if err := control.UpdatePodClaimForRetentionPolicy(ctx, set, pod); err != nil { + t.Errorf("Unexpected error for UpdatePodClaimForRetentionPolicy (retain): %v", err) + } + expectRef := features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) + for k := range claims { + claim, err := claimLister.PersistentVolumeClaims(claims[k].Namespace).Get(claims[k].Name) + if err != nil { + t.Errorf("Unexpected error getting Claim %s/%s: %v", claim.Namespace, claim.Name, err) + } + if hasOwnerRef(claim, set) != expectRef { + t.Errorf("Claim %s/%s bad set owner ref", claim.Namespace, claim.Name) + } + } + } + t.Run("PetSetAutoDeletePVCEnabled", func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, true) + testFn(t) + }) + t.Run("PetSetAutoDeletePVCDisabled", func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, false) + testFn(t) + }) +} + +func TestPodClaimIsStale(t *testing.T) { + const missing = "missing" + const exists = "exists" + const stale = "stale" + const withRef = "with-ref" + testCases := []struct { + name string + claimStates []string + expected bool + skipPodUID bool + }{ + { + name: "all missing", + claimStates: []string{missing, missing}, + expected: false, + }, + { + name: "no claims", + claimStates: []string{}, + expected: false, + }, + { + name: "exists", + claimStates: []string{missing, exists}, + expected: false, + }, + { + name: "all refs", + claimStates: []string{withRef, withRef}, + expected: false, + }, + { + name: "stale & exists", + claimStates: []string{stale, exists}, + expected: true, + }, + { + name: "stale & missing", + claimStates: []string{stale, missing}, + expected: true, + }, + { + name: "withRef & stale", + claimStates: []string{withRef, stale}, + expected: true, + }, + { + name: "withRef, no UID", + claimStates: []string{withRef}, + skipPodUID: true, + expected: true, + }, + } + for _, tc := range testCases { + set := api.PetSet{} + set.Name = "set" + set.Namespace = "default" + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + } + set.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"key": "value"}} + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i, claimState := range tc.claimStates { + claim := v1.PersistentVolumeClaim{} + claim.Name = fmt.Sprintf("claim-%d", i) + set.Spec.VolumeClaimTemplates = append(set.Spec.VolumeClaimTemplates, claim) + claim.Name = fmt.Sprintf("%s-set-3", claim.Name) + claim.Namespace = set.Namespace + switch claimState { + case missing: + // Do nothing, the claim shouldn't exist. + case exists: + claimIndexer.Add(&claim) + case stale: + claim.SetOwnerReferences([]metav1.OwnerReference{ + {Name: "set-3", UID: types.UID("stale")}, + }) + claimIndexer.Add(&claim) + case withRef: + claim.SetOwnerReferences([]metav1.OwnerReference{ + {Name: "set-3", UID: types.UID("123")}, + }) + claimIndexer.Add(&claim) + } + } + pod := v1.Pod{} + pod.Name = "set-3" + if !tc.skipPodUID { + pod.SetUID("123") + } + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := newTestStatefulPodControl(&fake.Clientset{}, nil, claimLister, &noopRecorder{}) + expected := tc.expected + // Note that the error isn't / can't be tested. + if stale, _ := control.PodClaimIsStale(&set, &pod); stale != expected { + t.Errorf("unexpected stale for %s", tc.name) + } + } +} + +func TestStatefulPodControlRetainDeletionPolicyUpdate(t *testing.T) { + testFn := func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + recorder := record.NewFakeRecorder(10) + set := newPetSet(1) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + podIndexer.Add(pod) + claims := getPersistentVolumeClaims(set, pod) + if len(claims) < 1 { + t.Errorf("Unexpected missing PVCs") + } + for k := range claims { + claim := claims[k] + setOwnerRef(&claim, set, &set.TypeMeta) // This ownerRef should be removed in the update. + claimIndexer.Add(&claim) + } + control := newTestStatefulPodControl(fakeClient, podLister, claimLister, recorder) + if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { + t.Errorf("Successful update returned an error: %s", err) + } + for k := range claims { + claim := claims[k] + if hasOwnerRef(&claim, set) { + t.Errorf("ownerRef not removed: %s/%s", claim.Namespace, claim.Name) + } + } + events := collectEvents(recorder.Events) + if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) { + if eventCount := len(events); eventCount != 1 { + t.Errorf("delete failed: got %d events, but want 1", eventCount) + } + } else { + if len(events) != 0 { + t.Errorf("delete failed: expected no events, but got %v", events) + } + } + } + t.Run("PetSetAutoDeletePVCEnabled", func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, true) + testFn(t) + }) + t.Run("PetSetAutoDeletePVCDisabled", func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, false) + testFn(t) + }) +} + +func TestStatefulPodControlRetentionPolicyUpdate(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + // Only applicable when the feature gate is on; the off case is tested in TestStatefulPodControlRetainRetentionPolicyUpdate. + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, true) + + recorder := record.NewFakeRecorder(10) + set := newPetSet(1) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podIndexer.Add(pod) + claims := getPersistentVolumeClaims(set, pod) + if len(claims) != 1 { + t.Errorf("Unexpected or missing PVCs") + } + var claim v1.PersistentVolumeClaim + for k := range claims { + claim = claims[k] + claimIndexer.Add(&claim) + } + fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + claimIndexer.Update(update.GetObject()) + return true, update.GetObject(), nil + }) + podLister := corelisters.NewPodLister(podIndexer) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := newTestStatefulPodControl(fakeClient, podLister, claimLister, recorder) + if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { + t.Errorf("Successful update returned an error: %s", err) + } + updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) + if err != nil { + t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err) + } + if !hasOwnerRef(updatedClaim, set) { + t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("update failed: got %d events, but want 1", eventCount) + } +} + +func TestStatefulPodControlRetentionPolicyUpdateMissingClaims(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + // Only applicable when the feature gate is on. + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, true) + + recorder := record.NewFakeRecorder(10) + set := newPetSet(1) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + pod := newTestPetSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + podIndexer.Add(pod) + fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + claimIndexer.Update(update.GetObject()) + return true, update.GetObject(), nil + }) + control := newTestStatefulPodControl(fakeClient, podLister, claimLister, recorder) + if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { + t.Error("Unexpected error on pod update when PVCs are missing") + } + claims := getPersistentVolumeClaims(set, pod) + if len(claims) != 1 { + t.Errorf("Unexpected or missing PVCs") + } + var claim v1.PersistentVolumeClaim + for k := range claims { + claim = claims[k] + claimIndexer.Add(&claim) + } + + if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { + t.Errorf("Expected update to succeed, saw error %v", err) + } + updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) + if err != nil { + t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err) + } + if !hasOwnerRef(updatedClaim, set) { + t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("update failed: got %d events, but want 2", eventCount) + } + if !strings.Contains(events[0], "SuccessfulUpdate") { + t.Errorf("expected first event to be a successful update: %s", events[1]) + } +} + +func collectEvents(source <-chan string) []string { + done := false + events := make([]string, 0) + for !done { + select { + case event := <-source: + events = append(events, event) + default: + done = true + } + } + return events +} diff --git a/pkg/controller/petset/pet_set_control_test.go b/pkg/controller/petset/pet_set_control_test.go new file mode 100644 index 00000000..fe3caf0f --- /dev/null +++ b/pkg/controller/petset/pet_set_control_test.go @@ -0,0 +1,3416 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package petset + +import ( + "context" + "errors" + "fmt" + "math/rand" + "reflect" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + api "kubeops.dev/petset/apis/apps/v1" + "kubeops.dev/petset/client/clientset/versioned" + apifake "kubeops.dev/petset/client/clientset/versioned/fake" + apiinformers "kubeops.dev/petset/client/informers/externalversions" + stsinformers "kubeops.dev/petset/client/informers/externalversions/apps/v1" + apilisters "kubeops.dev/petset/client/listers/apps/v1" + podutil "kubeops.dev/petset/pkg/api/v1/pod" + "kubeops.dev/petset/pkg/controller" + "kubeops.dev/petset/pkg/controller/history" + "kubeops.dev/petset/pkg/features" + + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + featuregatetesting "k8s.io/component-base/featuregate/testing" +) + +type invariantFunc func(set *api.PetSet, om *fakeObjectManager) error + +func setupController(client clientset.Interface, apiclient versioned.Interface) (*fakeObjectManager, *fakeStatefulSetStatusUpdater, PetSetControlInterface) { + informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) + apiinformerFactory := apiinformers.NewSharedInformerFactory(apiclient, controller.NoResyncPeriodFunc()) + om := newFakeObjectManager(informerFactory, apiinformerFactory) + spc := NewStatefulPodControlFromManager(om, &noopRecorder{}) + ssu := newFakeStatefulSetStatusUpdater(apiinformerFactory.Apps().V1().PetSets()) + recorder := &noopRecorder{} + ssc := NewDefaultPetSetControl(spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder) + + // The informer is not started. The tests here manipulate the local cache (indexers) directly, and there is no waiting + // for client state to sync. In fact, because the client is not updated during tests, informer updates will break tests + // by unexpectedly deleting objects. + // + // TODO: It might be better to rewrite all these tests manipulate the client an explicitly sync to ensure consistent + // state, or to create a fake client that does not use a local cache. + + // The client is passed initial sets, so we have to put them in the local setsIndexer cache. + if sets, err := apiclient.AppsV1().PetSets("").List(context.TODO(), metav1.ListOptions{}); err != nil { + panic(err) + } else { + for i := range sets.Items { + if err := om.setsIndexer.Update(&sets.Items[i]); err != nil { + panic(err) + } + } + } + + return om, ssu, ssc +} + +func burst(set *api.PetSet) *api.PetSet { + set.Spec.PodManagementPolicy = apps.ParallelPodManagement + return set +} + +func setMinReadySeconds(set *api.PetSet, minReadySeconds int32) *api.PetSet { + set.Spec.MinReadySeconds = minReadySeconds + return set +} + +func runTestOverPVCRetentionPolicies(t *testing.T, testName string, testFn func(*testing.T, *apps.StatefulSetPersistentVolumeClaimRetentionPolicy)) { + subtestName := "PetSetAutoDeletePVCDisabled" + if testName != "" { + subtestName = fmt.Sprintf("%s/%s", testName, subtestName) + } + t.Run(subtestName, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, false) + testFn(t, &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }) + }) + + for _, policy := range []*apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + { + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + // tests the case when no policy is set. + nil, + } { + subtestName := pvcDeletePolicyString(policy) + "/PetSetAutoDeletePVCEnabled" + if testName != "" { + subtestName = fmt.Sprintf("%s/%s", testName, subtestName) + } + t.Run(subtestName, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, true) + testFn(t, policy) + }) + } +} + +func pvcDeletePolicyString(policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) string { + if policy == nil { + return "nullPolicy" + } + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + case policy.WhenScaled == retain && policy.WhenDeleted == retain: + return "Retain" + case policy.WhenScaled == retain && policy.WhenDeleted == delete: + return "SetDeleteOnly" + case policy.WhenScaled == delete && policy.WhenDeleted == retain: + return "ScaleDownOnly" + case policy.WhenScaled == delete && policy.WhenDeleted == delete: + return "Delete" + } + return "invalid" +} + +func TestPetSetControl(t *testing.T) { + simpleSetFn := func() *api.PetSet { return newPetSet(3) } + largeSetFn := func() *api.PetSet { return newPetSet(5) } + + testCases := []struct { + fn func(*testing.T, *api.PetSet, invariantFunc) + obj func() *api.PetSet + }{ + {CreatesPods, simpleSetFn}, + {ScalesUp, simpleSetFn}, + {ScalesDown, simpleSetFn}, + {ReplacesPods, largeSetFn}, + {RecreatesFailedPod, simpleSetFn}, + {RecreatesSucceededPod, simpleSetFn}, + {CreatePodFailure, simpleSetFn}, + {UpdatePodFailure, simpleSetFn}, + {UpdateSetStatusFailure, simpleSetFn}, + {PodRecreateDeleteFailure, simpleSetFn}, + {NewRevisionDeletePodFailure, simpleSetFn}, + {RecreatesPVCForPendingPod, simpleSetFn}, + } + + for _, testCase := range testCases { + fnName := runtime.FuncForPC(reflect.ValueOf(testCase.fn).Pointer()).Name() + if i := strings.LastIndex(fnName, "."); i != -1 { + fnName = fnName[i+1:] + } + testObj := testCase.obj + testFn := testCase.fn + runTestOverPVCRetentionPolicies( + t, + fmt.Sprintf("%s/Monotonic", fnName), + func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := testObj() + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + testFn(t, set, assertMonotonicInvariants) + }, + ) + runTestOverPVCRetentionPolicies( + t, + fmt.Sprintf("%s/Burst", fnName), + func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := burst(testObj()) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + testFn(t, set, assertBurstInvariants) + }, + ) + } +} + +func CreatesPods(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 3 { + t.Error("Failed to scale petset to 3 replicas") + } + if set.Status.ReadyReplicas != 3 { + t.Error("Failed to set ReadyReplicas correctly") + } + if set.Status.UpdatedReplicas != 3 { + t.Error("Failed to set UpdatedReplicas correctly") + } + // Check all pods have correct pod index label. + if features.DefaultFeatureGate.Enabled(features.PodIndexLabel) { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if len(pods) != 3 { + t.Errorf("Expected 3 pods, got %d", len(pods)) + } + for _, pod := range pods { + podIndexFromLabel, exists := pod.Labels[apps.PodIndexLabel] + if !exists { + t.Errorf("Missing pod index label: %s", apps.PodIndexLabel) + continue + } + podIndexFromName := strconv.Itoa(getOrdinal(pod)) + if podIndexFromLabel != podIndexFromName { + t.Errorf("Pod index label value (%s) does not match pod index in pod name (%s)", podIndexFromLabel, podIndexFromName) + } + } + } +} + +func ScalesUp(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + *set.Spec.Replicas = 4 + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to scale PetSet : %s", err) + } + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 4 { + t.Error("Failed to scale petset to 4 replicas") + } + if set.Status.ReadyReplicas != 4 { + t.Error("Failed to set readyReplicas correctly") + } + if set.Status.UpdatedReplicas != 4 { + t.Error("Failed to set updatedReplicas correctly") + } +} + +func ScalesDown(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + *set.Spec.Replicas = 0 + if err := scaleDownPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to scale PetSet : %s", err) + } + + // Check updated set. + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 0 { + t.Error("Failed to scale petset to 0 replicas") + } + if set.Status.ReadyReplicas != 0 { + t.Error("Failed to set readyReplicas correctly") + } + if set.Status.UpdatedReplicas != 0 { + t.Error("Failed to set updatedReplicas correctly") + } +} + +func ReplacesPods(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 5 { + t.Error("Failed to scale petset to 5 replicas") + } + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + for _, pod := range pods { + podClaims := getPersistentVolumeClaims(set, pod) + for _, claim := range claims { + if _, found := podClaims[claim.Name]; found { + if hasOwnerRef(claim, pod) { + t.Errorf("Unexpected ownerRef on %s", claim.Name) + } + } + } + } + sort.Sort(ascendingOrdinal(pods)) + om.podsIndexer.Delete(pods[0]) + om.podsIndexer.Delete(pods[2]) + om.podsIndexer.Delete(pods[4]) + for i := 0; i < 5; i += 2 { + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Errorf("Failed to update PetSet : %s", err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if pods, err = om.setPodRunning(set, i); err != nil { + t.Error(err) + } + if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Errorf("Failed to update PetSet : %s", err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if _, err = om.setPodReady(set, i); err != nil { + t.Error(err) + } + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Errorf("Failed to update PetSet : %s", err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if e, a := int32(5), set.Status.Replicas; e != a { + t.Errorf("Expected to scale to %d, got %d", e, a) + } +} + +func recreatesPod(t *testing.T, set *api.PetSet, invariants invariantFunc, phase v1.PodPhase) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset() + om, _, ssc := setupController(client, apiclient) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Errorf("Error updating PetSet %s", err) + } + if err := invariants(set, om); err != nil { + t.Error(err) + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + pods[0].Status.Phase = phase + om.podsIndexer.Update(pods[0]) + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Errorf("Error updating PetSet %s", err) + } + if err := invariants(set, om); err != nil { + t.Error(err) + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if isCreated(pods[0]) { + t.Error("PetSet did not recreate failed Pod") + } +} + +func RecreatesFailedPod(t *testing.T, set *api.PetSet, invariants invariantFunc) { + recreatesPod(t, set, invariants, v1.PodFailed) +} + +func RecreatesSucceededPod(t *testing.T, set *api.PetSet, invariants invariantFunc) { + recreatesPod(t, set, invariants, v1.PodSucceeded) +} + +func CreatePodFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + om.SetCreateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) + + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { + t.Errorf("PetSetControl did not return InternalError found %s", err) + } + // Update so set.Status is set for the next scaleUpPetSetControl call. + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 3 { + t.Error("Failed to scale PetSet to 3 replicas") + } + if set.Status.ReadyReplicas != 3 { + t.Error("Failed to set readyReplicas correctly") + } + if set.Status.UpdatedReplicas != 3 { + t.Error("Failed to updatedReplicas correctly") + } +} + +func UpdatePodFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + om.SetUpdateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) + + // have to have 1 successful loop first + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 3 { + t.Error("Failed to scale PetSet to 3 replicas") + } + if set.Status.ReadyReplicas != 3 { + t.Error("Failed to set readyReplicas correctly") + } + if set.Status.UpdatedReplicas != 3 { + t.Error("Failed to set updatedReplicas correctly") + } + + // now mutate a pod's identity + pods, err := om.podsLister.List(labels.Everything()) + if err != nil { + t.Fatalf("Error listing pods: %v", err) + } + if len(pods) != 3 { + t.Fatalf("Expected 3 pods, got %d", len(pods)) + } + sort.Sort(ascendingOrdinal(pods)) + pods[0].Name = "goo-0" + om.podsIndexer.Update(pods[0]) + + // now it should fail + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil && isOrHasInternalError(err) { + t.Errorf("PetSetControl did not return InternalError found %s", err) + } +} + +func UpdateSetStatusFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, ssu, ssc := setupController(client, apiclient) + ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) + + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { + t.Errorf("PetSetControl did not return InternalError found %s", err) + } + // Update so set.Status is set for the next scaleUpPetSetControl call. + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 3 { + t.Error("Failed to scale PetSet to 3 replicas") + } + if set.Status.ReadyReplicas != 3 { + t.Error("Failed to set readyReplicas to 3") + } + if set.Status.UpdatedReplicas != 3 { + t.Error("Failed to set updatedReplicas to 3") + } +} + +func PodRecreateDeleteFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Errorf("Error updating PetSet %s", err) + } + if err := invariants(set, om); err != nil { + t.Error(err) + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + pods[0].Status.Phase = v1.PodFailed + om.podsIndexer.Update(pods[0]) + om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil && isOrHasInternalError(err) { + t.Errorf("PetSet failed to %s", err) + } + if err := invariants(set, om); err != nil { + t.Error(err) + } + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Errorf("Error updating PetSet %s", err) + } + if err := invariants(set, om); err != nil { + t.Error(err) + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if isCreated(pods[0]) { + t.Error("PetSet did not recreate failed Pod") + } +} + +func NewRevisionDeletePodFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 3 { + t.Error("Failed to scale PetSet to 3 replicas") + } + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + + // trigger a new revision + updateSet := set.DeepCopy() + updateSet.Spec.Template.Spec.Containers[0].Image = "nginx-new" + if err := om.setsIndexer.Update(updateSet); err != nil { + t.Error("Failed to update PetSet") + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + + // delete fails + om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) + _, err = ssc.UpdatePetSet(context.TODO(), set, pods) + if err == nil { + t.Error("Expected err in update PetSet when deleting a pod") + } + + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if err := invariants(set, om); err != nil { + t.Error(err) + } + if set.Status.CurrentReplicas != 3 { + t.Fatalf("Failed pod deletion should not update CurrentReplicas: want 3, got %d", set.Status.CurrentReplicas) + } + if set.Status.CurrentRevision == set.Status.UpdateRevision { + t.Error("Failed to create new revision") + } + + // delete works + om.SetDeleteStatefulPodError(nil, 0) + status, err := ssc.UpdatePetSet(context.TODO(), set, pods) + if err != nil { + t.Fatalf("Unexpected err in update PetSet: %v", err) + } + if status.CurrentReplicas != 2 { + t.Fatalf("Pod deletion should update CurrentReplicas: want 2, got %d", status.CurrentReplicas) + } + if err := invariants(set, om); err != nil { + t.Error(err) + } +} + +func emptyInvariants(set *api.PetSet, om *fakeObjectManager) error { + return nil +} + +func TestPetSetControlWithStartOrdinal(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetStartOrdinal, true) + + simpleSetFn := func() *api.PetSet { + statefulSet := newPetSet(3) + statefulSet.Spec.Ordinals = &apps.StatefulSetOrdinals{Start: int32(2)} + return statefulSet + } + + testCases := []struct { + fn func(*testing.T, *api.PetSet, invariantFunc) + obj func() *api.PetSet + }{ + {CreatesPodsWithStartOrdinal, simpleSetFn}, + } + + for _, testCase := range testCases { + testObj := testCase.obj + testFn := testCase.fn + + set := testObj() + testFn(t, set, emptyInvariants) + } +} + +func CreatesPodsWithStartOrdinal(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 3 { + t.Error("Failed to scale petset to 3 replicas") + } + if set.Status.ReadyReplicas != 3 { + t.Error("Failed to set ReadyReplicas correctly") + } + if set.Status.UpdatedReplicas != 3 { + t.Error("Failed to set UpdatedReplicas correctly") + } + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + sort.Sort(ascendingOrdinal(pods)) + for i, pod := range pods { + expectedOrdinal := 2 + i + actualPodOrdinal := getOrdinal(pod) + if actualPodOrdinal != expectedOrdinal { + t.Errorf("Expected pod ordinal %d. Got %d", expectedOrdinal, actualPodOrdinal) + } + } +} + +func RecreatesPVCForPendingPod(t *testing.T, set *api.PetSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset() + om, _, ssc := setupController(client, apiclient) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Errorf("Error updating PetSet %s", err) + } + if err := invariants(set, om); err != nil { + t.Error(err) + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + for _, claim := range getPersistentVolumeClaims(set, pods[0]) { + om.claimsIndexer.Delete(&claim) + } + pods[0].Status.Phase = v1.PodPending + om.podsIndexer.Update(pods[0]) + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Errorf("Error updating PetSet %s", err) + } + // invariants check if there any missing PVCs for the Pods + if err := invariants(set, om); err != nil { + t.Error(err) + } + _, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } +} + +func TestPetSetControlScaleDownDeleteError(t *testing.T) { + runTestOverPVCRetentionPolicies( + t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := newPetSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + invariants := assertMonotonicInvariants + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + *set.Spec.Replicas = 0 + om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) + if err := scaleDownPetSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { + t.Errorf("PetSetControl failed to throw error on delete %s", err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if err := scaleDownPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn down PetSet %s", err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != 0 { + t.Error("Failed to scale petset to 0 replicas") + } + if set.Status.ReadyReplicas != 0 { + t.Error("Failed to set readyReplicas to 0") + } + if set.Status.UpdatedReplicas != 0 { + t.Error("Failed to set updatedReplicas to 0") + } + }) +} + +func TestPetSetControl_getSetRevisions(t *testing.T) { + type testcase struct { + name string + existing []*apps.ControllerRevision + set *api.PetSet + expectedCount int + expectedCurrent *apps.ControllerRevision + expectedUpdate *apps.ControllerRevision + err bool + } + + testFn := func(test *testcase, t *testing.T) { + client := fake.NewSimpleClientset() + informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) + apiclient := apifake.NewSimpleClientset() + apiinformerFactory := apiinformers.NewSharedInformerFactory(apiclient, controller.NoResyncPeriodFunc()) + spc := NewStatefulPodControlFromManager(newFakeObjectManager(informerFactory, apiinformerFactory), &noopRecorder{}) + ssu := newFakeStatefulSetStatusUpdater(apiinformerFactory.Apps().V1().PetSets()) + recorder := &noopRecorder{} + ssc := defaultPetSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder} + + stop := make(chan struct{}) + defer close(stop) + informerFactory.Start(stop) + cache.WaitForCacheSync( + stop, + informerFactory.Core().V1().Pods().Informer().HasSynced, + informerFactory.Apps().V1().ControllerRevisions().Informer().HasSynced, + ) + test.set.Status.CollisionCount = new(int32) + for i := range test.existing { + ssc.controllerHistory.CreateControllerRevision(test.set, test.existing[i], test.set.Status.CollisionCount) + } + revisions, err := ssc.ListRevisions(test.set) + if err != nil { + t.Fatal(err) + } + current, update, _, err := ssc.getPetSetRevisions(test.set, revisions) + if err != nil { + t.Fatalf("error getting petset revisions:%v", err) + } + revisions, err = ssc.ListRevisions(test.set) + if err != nil { + t.Fatal(err) + } + if len(revisions) != test.expectedCount { + t.Errorf("%s: want %d revisions got %d", test.name, test.expectedCount, len(revisions)) + } + if test.err && err == nil { + t.Errorf("%s: expected error", test.name) + } + if !test.err && !history.EqualRevision(current, test.expectedCurrent) { + t.Errorf("%s: for current want %v got %v", test.name, test.expectedCurrent, current) + } + if !test.err && !history.EqualRevision(update, test.expectedUpdate) { + t.Errorf("%s: for update want %v got %v", test.name, test.expectedUpdate, update) + } + if !test.err && test.expectedCurrent != nil && current != nil && test.expectedCurrent.Revision != current.Revision { + t.Errorf("%s: for current revision want %d got %d", test.name, test.expectedCurrent.Revision, current.Revision) + } + if !test.err && test.expectedUpdate != nil && update != nil && test.expectedUpdate.Revision != update.Revision { + t.Errorf("%s: for update revision want %d got %d", test.name, test.expectedUpdate.Revision, update.Revision) + } + } + + updateRevision := func(cr *apps.ControllerRevision, revision int64) *apps.ControllerRevision { + clone := cr.DeepCopy() + clone.Revision = revision + return clone + } + + runTestOverPVCRetentionPolicies( + t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := newPetSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + set.Status.CollisionCount = new(int32) + rev0 := newRevisionOrDie(set, 1) + set1 := set.DeepCopy() + set1.Spec.Template.Spec.Containers[0].Image = "foo" + set1.Status.CurrentRevision = rev0.Name + set1.Status.CollisionCount = new(int32) + rev1 := newRevisionOrDie(set1, 2) + set2 := set1.DeepCopy() + set2.Spec.Template.Labels["new"] = "label" + set2.Status.CurrentRevision = rev0.Name + set2.Status.CollisionCount = new(int32) + rev2 := newRevisionOrDie(set2, 3) + tests := []testcase{ + { + name: "creates initial revision", + existing: nil, + set: set, + expectedCount: 1, + expectedCurrent: rev0, + expectedUpdate: rev0, + err: false, + }, + { + name: "creates revision on update", + existing: []*apps.ControllerRevision{rev0}, + set: set1, + expectedCount: 2, + expectedCurrent: rev0, + expectedUpdate: rev1, + err: false, + }, + { + name: "must not recreate a new revision of same set", + existing: []*apps.ControllerRevision{rev0, rev1}, + set: set1, + expectedCount: 2, + expectedCurrent: rev0, + expectedUpdate: rev1, + err: false, + }, + { + name: "must rollback to a previous revision", + existing: []*apps.ControllerRevision{rev0, rev1, rev2}, + set: set1, + expectedCount: 3, + expectedCurrent: rev0, + expectedUpdate: updateRevision(rev1, 4), + err: false, + }, + } + for i := range tests { + testFn(&tests[i], t) + } + }) +} + +func setupPodManagementPolicy(podManagementPolicy apps.PodManagementPolicyType, set *api.PetSet) *api.PetSet { + set.Spec.PodManagementPolicy = podManagementPolicy + return set +} + +func TestPetSetControlRollingUpdateWithMaxUnavailable(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.MaxUnavailablePetSet, true) + + simpleParallelVerificationFn := func( + set *api.PetSet, + spc *fakeObjectManager, + ssc PetSetControlInterface, + pods []*v1.Pod, + totalPods int, + selector labels.Selector, + ) []*v1.Pod { + // in burst mode, 2 pods got deleted, so 2 new pods will be created at the same time + if len(pods) != totalPods { + t.Fatalf("Expected create pods 4/5, got pods %v", len(pods)) + } + + // if pod 4 ready, start to update pod 3, even though 5 is not ready + spc.setPodRunning(set, 4) + spc.setPodRunning(set, 5) + originalPods, _ := spc.setPodReady(set, 4) + sort.Sort(ascendingOrdinal(originalPods)) + if _, err := ssc.UpdatePetSet(context.TODO(), set, originalPods); err != nil { + t.Fatal(err) + } + pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + sort.Sort(ascendingOrdinal(pods)) + // pods 0, 1,2, 4,5 should be present(note 3 is missing) + if !reflect.DeepEqual(pods, append(originalPods[:3], originalPods[4:]...)) { + t.Fatalf("Expected pods %v, got pods %v", append(originalPods[:3], originalPods[4:]...), pods) + } + + // create new pod 3 + if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Fatal(err) + } + pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + if len(pods) != totalPods { + t.Fatalf("Expected create pods 2/3, got pods %v", pods) + } + + return pods + } + simpleOrderedVerificationFn := func( + set *api.PetSet, + spc *fakeObjectManager, + ssc PetSetControlInterface, + pods []*v1.Pod, + totalPods int, + selector labels.Selector, + ) []*v1.Pod { + // only one pod gets created at a time due to OrderedReady + if len(pods) != 5 { + t.Fatalf("Expected create pods 5, got pods %v", len(pods)) + } + spc.setPodRunning(set, 4) + pods, _ = spc.setPodReady(set, 4) + + // create new pods 4(only one pod gets created at a time due to OrderedReady) + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Fatal(err) + } + pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + + if len(pods) != totalPods { + t.Fatalf("Expected create pods 4, got pods %v", len(pods)) + } + // if pod 4 ready, start to update pod 3 + spc.setPodRunning(set, 5) + originalPods, _ := spc.setPodReady(set, 5) + sort.Sort(ascendingOrdinal(originalPods)) + if _, err = ssc.UpdatePetSet(context.TODO(), set, originalPods); err != nil { + t.Fatal(err) + } + pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + sort.Sort(ascendingOrdinal(pods)) + + // verify the remaining pods are 0,1,2,4,5 (3 got deleted) + if !reflect.DeepEqual(pods, append(originalPods[:3], originalPods[4:]...)) { + t.Fatalf("Expected pods %v, got pods %v", append(originalPods[:3], originalPods[4:]...), pods) + } + + // create new pod 3 + if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Fatal(err) + } + pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + if len(pods) != totalPods { + t.Fatalf("Expected create pods 2/3, got pods %v", pods) + } + + return pods + } + testCases := []struct { + policyType apps.PodManagementPolicyType + verifyFn func( + set *api.PetSet, + spc *fakeObjectManager, + ssc PetSetControlInterface, + pods []*v1.Pod, + totalPods int, + selector labels.Selector, + ) []*v1.Pod + }{ + {apps.OrderedReadyPodManagement, simpleOrderedVerificationFn}, + {apps.ParallelPodManagement, simpleParallelVerificationFn}, + } + for _, tc := range testCases { + // Setup the statefulSet controller + var totalPods int32 = 6 + var partition int32 = 3 + maxUnavailable := intstr.FromInt32(2) + set := setupPodManagementPolicy(tc.policyType, newPetSet(totalPods)) + set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{ + Type: apps.RollingUpdateStatefulSetStrategyType, + RollingUpdate: func() *apps.RollingUpdateStatefulSetStrategy { + return &apps.RollingUpdateStatefulSetStrategy{ + Partition: &partition, + MaxUnavailable: &maxUnavailable, + } + }(), + } + + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + spc, _, ssc := setupController(client, apiclient) + if err := scaleUpPetSetControl(set, ssc, spc, assertBurstInvariants); err != nil { + t.Fatal(err) + } + set, err := spc.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatal(err) + } + + // Change the image to trigger an update + set.Spec.Template.Spec.Containers[0].Image = "foo" + + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatal(err) + } + originalPods, err := spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + sort.Sort(ascendingOrdinal(originalPods)) + + // since maxUnavailable is 2, update pods 4 and 5, this will delete the pod 4 and 5, + if _, err = ssc.UpdatePetSet(context.TODO(), set, originalPods); err != nil { + t.Fatal(err) + } + pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + + sort.Sort(ascendingOrdinal(pods)) + + // expected number of pod is 0,1,2,3 + if !reflect.DeepEqual(pods, originalPods[:4]) { + t.Fatalf("Expected pods %v, got pods %v", originalPods[:4], pods) + } + + // create new pods + if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + t.Fatal(err) + } + pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + + tc.verifyFn(set, spc, ssc, pods, int(totalPods), selector) + + // pods 3/4/5 ready, should not update other pods + spc.setPodRunning(set, 3) + spc.setPodRunning(set, 5) + spc.setPodReady(set, 5) + originalPods, _ = spc.setPodReady(set, 3) + sort.Sort(ascendingOrdinal(originalPods)) + if _, err = ssc.UpdatePetSet(context.TODO(), set, originalPods); err != nil { + t.Fatal(err) + } + pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + sort.Sort(ascendingOrdinal(pods)) + if !reflect.DeepEqual(pods, originalPods) { + t.Fatalf("Expected pods %v, got pods %v", originalPods, pods) + } + } +} + +func setupForInvariant(t *testing.T) (*api.PetSet, *fakeObjectManager, PetSetControlInterface, intstr.IntOrString, int32) { + var totalPods int32 = 6 + set := newPetSet(totalPods) + // update all pods >=3(3,4,5) + var partition int32 = 3 + maxUnavailable := intstr.FromInt32(2) + set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{ + Type: apps.RollingUpdateStatefulSetStrategyType, + RollingUpdate: func() *apps.RollingUpdateStatefulSetStrategy { + return &apps.RollingUpdateStatefulSetStrategy{ + Partition: &partition, + MaxUnavailable: &maxUnavailable, + } + }(), + } + + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + spc, _, ssc := setupController(client, apiclient) + if err := scaleUpPetSetControl(set, ssc, spc, assertBurstInvariants); err != nil { + t.Fatal(err) + } + set, err := spc.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatal(err) + } + + return set, spc, ssc, maxUnavailable, totalPods +} + +func TestPetSetControlRollingUpdateWithMaxUnavailableInOrderedModeVerifyInvariant(t *testing.T) { + // Make all pods in petset unavailable one by one + // and verify that RollingUpdate doesnt proceed with maxUnavailable set + // this could have been a simple loop, keeping it like this to be able + // to add more params here. + testCases := []struct { + ordinalOfPodToTerminate []int + }{ + {[]int{}}, + {[]int{5}}, + {[]int{3}}, + {[]int{4}}, + {[]int{5, 4}}, + {[]int{5, 3}}, + {[]int{4, 3}}, + {[]int{5, 4, 3}}, + {[]int{2}}, // note this is an ordinal greater than partition(3) + {[]int{1}}, // note this is an ordinal greater than partition(3) + } + for _, tc := range testCases { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.MaxUnavailablePetSet, true) + set, spc, ssc, maxUnavailable, totalPods := setupForInvariant(t) + t.Run(fmt.Sprintf("terminating pod at ordinal %d", tc.ordinalOfPodToTerminate), func(t *testing.T) { + status := apps.StatefulSetStatus{Replicas: int32(totalPods)} + updateRevision := &apps.ControllerRevision{} + + for i := 0; i < len(tc.ordinalOfPodToTerminate); i++ { + // Ensure at least one pod is unavailable before trying to update + _, err := spc.addTerminatingPod(set, tc.ordinalOfPodToTerminate[i]) + if err != nil { + t.Fatal(err) + } + } + + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatal(err) + } + + originalPods, err := spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + + sort.Sort(ascendingOrdinal(originalPods)) + + // start to update + set.Spec.Template.Spec.Containers[0].Image = "foo" + + // try to update the petset + // this function is only called in main code when feature gate is enabled + if _, err = updatePetSetAfterInvariantEstablished(context.TODO(), ssc.(*defaultPetSetControl), set, originalPods, updateRevision, status); err != nil { + t.Fatal(err) + } + pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + + sort.Sort(ascendingOrdinal(pods)) + + expecteddPodsToBeDeleted := maxUnavailable.IntValue() - len(tc.ordinalOfPodToTerminate) + if expecteddPodsToBeDeleted < 0 { + expecteddPodsToBeDeleted = 0 + } + + expectedPodsAfterUpdate := int(totalPods) - expecteddPodsToBeDeleted + + if len(pods) != expectedPodsAfterUpdate { + t.Errorf("Expected pods %v, got pods %v", expectedPodsAfterUpdate, len(pods)) + } + }) + } +} + +func TestPetSetControlRollingUpdate(t *testing.T) { + type testcase struct { + name string + invariants func(set *api.PetSet, om *fakeObjectManager) error + initial func() *api.PetSet + update func(set *api.PetSet) *api.PetSet + validate func(set *api.PetSet, pods []*v1.Pod) error + } + + testFn := func(test *testcase, t *testing.T) { + set := test.initial() + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set = test.update(set) + if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if err := test.validate(set, pods); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + } + + tests := []testcase{ + { + name: "monotonic image update", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(3) + }, + update: func(set *api.PetSet) *api.PetSet { + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "monotonic image update and scale up", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(3) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 5 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "monotonic image update and scale down", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(5) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 3 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(3)) + }, + update: func(set *api.PetSet) *api.PetSet { + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update and scale up", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(3)) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 5 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update and scale down", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(5)) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 3 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + } + for i := range tests { + testFn(&tests[i], t) + } +} + +func TestPetSetControlOnDeleteUpdate(t *testing.T) { + type testcase struct { + name string + invariants func(set *api.PetSet, om *fakeObjectManager) error + initial func() *api.PetSet + update func(set *api.PetSet) *api.PetSet + validateUpdate func(set *api.PetSet, pods []*v1.Pod) error + validateRestart func(set *api.PetSet, pods []*v1.Pod) error + } + + originalImage := newPetSet(3).Spec.Template.Spec.Containers[0].Image + + testFn := func(t *testing.T, test *testcase, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := test.initial() + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{Type: apps.OnDeleteStatefulSetStrategyType} + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set = test.update(set) + if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + + // Pods may have been deleted in the update. Delete any claims with a pod ownerRef. + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + for _, claim := range claims { + for _, ref := range claim.GetOwnerReferences() { + if strings.HasPrefix(ref.Name, "foo-") { + om.claimsIndexer.Delete(claim) + break + } + } + } + + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if err := test.validateUpdate(set, pods); err != nil { + for i := range pods { + t.Log(pods[i].Name) + } + t.Fatalf("%s: %s", test.name, err) + + } + claims, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + for _, claim := range claims { + for _, ref := range claim.GetOwnerReferences() { + if strings.HasPrefix(ref.Name, "foo-") { + t.Fatalf("Unexpected pod reference on %s: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + + replicas := *set.Spec.Replicas + *set.Spec.Replicas = 0 + if err := scaleDownPetSetControl(set, ssc, om, test.invariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + *set.Spec.Replicas = replicas + + claims, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + for _, claim := range claims { + for _, ref := range claim.GetOwnerReferences() { + if strings.HasPrefix(ref.Name, "foo-") { + t.Fatalf("Unexpected pod reference on %s: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + + if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if err := test.validateRestart(set, pods); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + } + + tests := []testcase{ + { + name: "monotonic image update", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(3) + }, + update: func(set *api.PetSet) *api.PetSet { + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "monotonic image update and scale up", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(3) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 5 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if i < 3 && pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + if i >= 3 && pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "monotonic image update and scale down", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(5) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 3 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(3)) + }, + update: func(set *api.PetSet) *api.PetSet { + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update and scale up", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(3)) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 5 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if i < 3 && pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + if i >= 3 && pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update and scale down", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(5)) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 3 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + } + runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + for i := range tests { + testFn(t, &tests[i], policy) + } + }) +} + +func TestPetSetControlRollingUpdateWithPartition(t *testing.T) { + type testcase struct { + name string + partition int32 + invariants func(set *api.PetSet, om *fakeObjectManager) error + initial func() *api.PetSet + update func(set *api.PetSet) *api.PetSet + validate func(set *api.PetSet, pods []*v1.Pod) error + } + + testFn := func(t *testing.T, test *testcase, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := test.initial() + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{ + Type: apps.RollingUpdateStatefulSetStrategyType, + RollingUpdate: func() *apps.RollingUpdateStatefulSetStrategy { + return &apps.RollingUpdateStatefulSetStrategy{Partition: &test.partition} + }(), + } + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set = test.update(set) + if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if err := test.validate(set, pods); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + } + + originalImage := newPetSet(3).Spec.Template.Spec.Containers[0].Image + + tests := []testcase{ + { + name: "monotonic image update", + invariants: assertMonotonicInvariants, + partition: 2, + initial: func() *api.PetSet { + return newPetSet(3) + }, + update: func(set *api.PetSet) *api.PetSet { + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if i < 2 && pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "monotonic image update and scale up", + partition: 2, + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(3) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 5 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if i < 2 && pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update", + partition: 2, + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(3)) + }, + update: func(set *api.PetSet) *api.PetSet { + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if i < 2 && pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update and scale up", + invariants: assertBurstInvariants, + partition: 2, + initial: func() *api.PetSet { + return burst(newPetSet(3)) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 5 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if i < 2 && pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + } + runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + for i := range tests { + testFn(t, &tests[i], policy) + } + }) +} + +func TestPetSetHonorRevisionHistoryLimit(t *testing.T) { + runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + invariants := assertMonotonicInvariants + set := newPetSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, ssu, ssc := setupController(client, apiclient) + + if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + var err error + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + + for i := 0; i < int(*set.Spec.RevisionHistoryLimit)+5; i++ { + set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) + ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) + updatePetSetControl(set, ssc, om, assertUpdateInvariants) + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + revisions, err := ssc.ListRevisions(set) + if err != nil { + t.Fatalf("Error listing revisions: %v", err) + } + // the extra 2 revisions are `currentRevision` and `updateRevision` + // They're considered as `live`, and truncateHistory only cleans up non-live revisions + if len(revisions) > int(*set.Spec.RevisionHistoryLimit)+2 { + t.Fatalf("%s: %d greater than limit %d", "", len(revisions), *set.Spec.RevisionHistoryLimit) + } + } + }) +} + +func TestPetSetControlLimitsHistory(t *testing.T) { + type testcase struct { + name string + invariants func(set *api.PetSet, om *fakeObjectManager) error + initial func() *api.PetSet + } + + testFn := func(t *testing.T, test *testcase) { + set := test.initial() + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + for i := 0; i < 10; i++ { + set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) + if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + _, err = ssc.UpdatePetSet(context.TODO(), set, pods) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + revisions, err := ssc.ListRevisions(set) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if len(revisions) > int(*set.Spec.RevisionHistoryLimit)+2 { + t.Fatalf("%s: %d greater than limit %d", test.name, len(revisions), *set.Spec.RevisionHistoryLimit) + } + } + } + + tests := []testcase{ + { + name: "monotonic update", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(3) + }, + }, + { + name: "burst update", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(3)) + }, + }, + } + for i := range tests { + testFn(t, &tests[i]) + } +} + +func TestPetSetControlRollback(t *testing.T) { + type testcase struct { + name string + invariants func(set *api.PetSet, om *fakeObjectManager) error + initial func() *api.PetSet + update func(set *api.PetSet) *api.PetSet + validateUpdate func(set *api.PetSet, pods []*v1.Pod) error + validateRollback func(set *api.PetSet, pods []*v1.Pod) error + } + + originalImage := newPetSet(3).Spec.Template.Spec.Containers[0].Image + + testFn := func(t *testing.T, test *testcase) { + set := test.initial() + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set = test.update(set) + if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if err := test.validateUpdate(set, pods); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + revisions, err := ssc.ListRevisions(set) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + history.SortControllerRevisions(revisions) + set, err = ApplyRevision(set, revisions[0]) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if err := test.validateRollback(set, pods); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + } + + tests := []testcase{ + { + name: "monotonic image update", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(3) + }, + update: func(set *api.PetSet) *api.PetSet { + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "monotonic image update and scale up", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(3) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 5 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "monotonic image update and scale down", + invariants: assertMonotonicInvariants, + initial: func() *api.PetSet { + return newPetSet(5) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 3 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(3)) + }, + update: func(set *api.PetSet) *api.PetSet { + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update and scale up", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(3)) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 5 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + { + name: "burst image update and scale down", + invariants: assertBurstInvariants, + initial: func() *api.PetSet { + return burst(newPetSet(5)) + }, + update: func(set *api.PetSet) *api.PetSet { + *set.Spec.Replicas = 3 + set.Spec.Template.Spec.Containers[0].Image = "foo" + return set + }, + validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != "foo" { + return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { + sort.Sort(ascendingOrdinal(pods)) + for i := range pods { + if pods[i].Spec.Containers[0].Image != originalImage { + return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) + } + } + return nil + }, + }, + } + for i := range tests { + testFn(t, &tests[i]) + } +} + +func TestPetSetAvailability(t *testing.T) { + tests := []struct { + name string + inputSTS *api.PetSet + expectedActiveReplicas int32 + readyDuration time.Duration + }{ + { + name: "replicas running for required time, when minReadySeconds is enabled", + inputSTS: setMinReadySeconds(newPetSet(1), int32(3600)), + readyDuration: -120 * time.Minute, + expectedActiveReplicas: int32(1), + }, + { + name: "replicas not running for required time, when minReadySeconds is enabled", + inputSTS: setMinReadySeconds(newPetSet(1), int32(3600)), + readyDuration: -30 * time.Minute, + expectedActiveReplicas: int32(0), + }, + } + for _, test := range tests { + set := test.inputSTS + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + spc, _, ssc := setupController(client, apiclient) + if err := scaleUpPetSetControl(set, ssc, spc, assertBurstInvariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + _, err = spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = spc.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err := spc.setPodAvailable(set, 0, time.Now().Add(test.readyDuration)) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + status, err := ssc.UpdatePetSet(context.TODO(), set, pods) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + if status.AvailableReplicas != test.expectedActiveReplicas { + t.Fatalf("expected %d active replicas got %d", test.expectedActiveReplicas, status.AvailableReplicas) + } + } +} + +func TestStatefulSetStatusUpdate(t *testing.T) { + var ( + syncErr = fmt.Errorf("sync error") + statusErr = fmt.Errorf("status error") + ) + + testCases := []struct { + desc string + + hasSyncErr bool + hasStatusErr bool + + expectedErr error + }{ + { + desc: "no error", + hasSyncErr: false, + hasStatusErr: false, + expectedErr: nil, + }, + { + desc: "sync error", + hasSyncErr: true, + hasStatusErr: false, + expectedErr: syncErr, + }, + { + desc: "status error", + hasSyncErr: false, + hasStatusErr: true, + expectedErr: statusErr, + }, + { + desc: "sync and status error", + hasSyncErr: true, + hasStatusErr: true, + expectedErr: syncErr, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + set := newPetSet(3) + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, ssu, ssc := setupController(client, apiclient) + + if tc.hasSyncErr { + om.SetCreateStatefulPodError(syncErr, 0) + } + if tc.hasStatusErr { + ssu.SetUpdateStatefulSetStatusError(statusErr, 0) + } + + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + _, err = ssc.UpdatePetSet(context.TODO(), set, pods) + if ssu.updateStatusTracker.requests != 1 { + t.Errorf("Did not update status") + } + if !errors.Is(err, tc.expectedErr) { + t.Errorf("Expected error: %v, got: %v", tc.expectedErr, err) + } + }) + } +} + +type requestTracker struct { + sync.Mutex + requests int + err error + after int + + parallelLock sync.Mutex + parallel int + maxParallel int + + delay time.Duration +} + +func (rt *requestTracker) errorReady() bool { + rt.Lock() + defer rt.Unlock() + return rt.err != nil && rt.requests >= rt.after +} + +func (rt *requestTracker) inc() { + rt.parallelLock.Lock() + rt.parallel++ + if rt.maxParallel < rt.parallel { + rt.maxParallel = rt.parallel + } + rt.parallelLock.Unlock() + + rt.Lock() + defer rt.Unlock() + rt.requests++ + if rt.delay != 0 { + time.Sleep(rt.delay) + } +} + +func (rt *requestTracker) reset() { + rt.parallelLock.Lock() + rt.parallel = 0 + rt.parallelLock.Unlock() + + rt.Lock() + defer rt.Unlock() + rt.err = nil + rt.after = 0 + rt.delay = 0 +} + +func (rt *requestTracker) getErr() error { + rt.Lock() + defer rt.Unlock() + return rt.err +} + +func newRequestTracker(requests int, err error, after int) requestTracker { + return requestTracker{ + requests: requests, + err: err, + after: after, + } +} + +type fakeObjectManager struct { + podsLister corelisters.PodLister + claimsLister corelisters.PersistentVolumeClaimLister + setsLister apilisters.PetSetLister + podsIndexer cache.Indexer + claimsIndexer cache.Indexer + setsIndexer cache.Indexer + revisionsIndexer cache.Indexer + createPodTracker requestTracker + updatePodTracker requestTracker + deletePodTracker requestTracker +} + +func newFakeObjectManager(informerFactory informers.SharedInformerFactory, apiinformerFactory apiinformers.SharedInformerFactory) *fakeObjectManager { + podInformer := informerFactory.Core().V1().Pods() + claimInformer := informerFactory.Core().V1().PersistentVolumeClaims() + setInformer := apiinformerFactory.Apps().V1().PetSets() + revisionInformer := informerFactory.Apps().V1().ControllerRevisions() + + return &fakeObjectManager{ + podInformer.Lister(), + claimInformer.Lister(), + setInformer.Lister(), + podInformer.Informer().GetIndexer(), + claimInformer.Informer().GetIndexer(), + setInformer.Informer().GetIndexer(), + revisionInformer.Informer().GetIndexer(), + newRequestTracker(0, nil, 0), + newRequestTracker(0, nil, 0), + newRequestTracker(0, nil, 0), + } +} + +func (om *fakeObjectManager) CreatePod(ctx context.Context, pod *v1.Pod, set *api.PetSet) error { + defer om.createPodTracker.inc() + if om.createPodTracker.errorReady() { + defer om.createPodTracker.reset() + return om.createPodTracker.getErr() + } + // Mutate and store an independent copy rather than the caller's pod. In burst + // mode the controller runs processReplica in parallel via slowStartBatch, and + // the fork's newVersionedPetSetPod lists pods (to compute placement) while + // other goroutines are creating pods. The caller's pod object may be aliased + // with an entry in the shared pod cache, so writing to it here would race with + // those concurrent ListPods reads. The real clientset likewise persists a copy + // rather than the object handed to Create. + cp := pod.DeepCopy() + cp.SetUID(types.UID(cp.Name + "-uid")) + return om.podsIndexer.Update(cp) +} + +func (om *fakeObjectManager) GetPod(namespace, podName string, set *api.PetSet) (*v1.Pod, error) { + pod, err := om.podsLister.Pods(namespace).Get(podName) + if err != nil { + return nil, err + } + return pod.DeepCopy(), nil +} + +func (om *fakeObjectManager) UpdatePod(pod *v1.Pod, set *api.PetSet) error { + return om.podsIndexer.Update(pod.DeepCopy()) +} + +func (om *fakeObjectManager) ListPods(ns, labelSelector string, set *api.PetSet) (*v1.PodList, error) { + pods, err := om.podsLister.Pods(ns).List(labels.Everything()) + if err != nil { + return nil, err + } + podList := &v1.PodList{} + for _, pod := range pods { + podList.Items = append(podList.Items, *pod.DeepCopy()) + } + return podList, nil +} + +func (om *fakeObjectManager) GetPlacementPolicy(name string) (*api.PlacementPolicy, error) { + return nil, nil +} + +func (om *fakeObjectManager) DeletePod(pod *v1.Pod, set *api.PetSet) error { + defer om.deletePodTracker.inc() + if om.deletePodTracker.errorReady() { + defer om.deletePodTracker.reset() + return om.deletePodTracker.getErr() + } + if key, err := controller.KeyFunc(pod); err != nil { + return err + } else if obj, found, err := om.podsIndexer.GetByKey(key); err != nil { + return err + } else if found { + return om.podsIndexer.Delete(obj) + } + return nil // Not found, no error in deleting. +} + +func (om *fakeObjectManager) CreateClaim(claim *v1.PersistentVolumeClaim, set *api.PetSet) error { + om.claimsIndexer.Update(claim) + return nil +} + +func (om *fakeObjectManager) GetClaim(namespace, claimName string, set *api.PetSet) (*v1.PersistentVolumeClaim, error) { + return om.claimsLister.PersistentVolumeClaims(namespace).Get(claimName) +} + +func (om *fakeObjectManager) UpdateClaim(claim *v1.PersistentVolumeClaim, set *api.PetSet) error { + // Validate ownerRefs. + refs := claim.GetOwnerReferences() + for _, ref := range refs { + if ref.APIVersion == "" || ref.Kind == "" || ref.Name == "" { + return fmt.Errorf("invalid ownerRefs: %s %v", claim.Name, refs) + } + } + om.claimsIndexer.Update(claim) + return nil +} + +func (om *fakeObjectManager) SetCreateStatefulPodError(err error, after int) { + om.createPodTracker.err = err + om.createPodTracker.after = after +} + +func (om *fakeObjectManager) SetUpdateStatefulPodError(err error, after int) { + om.updatePodTracker.err = err + om.updatePodTracker.after = after +} + +func (om *fakeObjectManager) SetDeleteStatefulPodError(err error, after int) { + om.deletePodTracker.err = err + om.deletePodTracker.after = after +} + +func findPodByOrdinal(pods []*v1.Pod, ordinal int) *v1.Pod { + for _, pod := range pods { + if getOrdinal(pod) == ordinal { + return pod.DeepCopy() + } + } + + return nil +} + +func (om *fakeObjectManager) setPodPending(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return nil, err + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return nil, err + } + pod := findPodByOrdinal(pods, ordinal) + if pod == nil { + return nil, fmt.Errorf("setPodPending: pod ordinal %d not found", ordinal) + } + pod.Status.Phase = v1.PodPending + fakeResourceVersion(pod) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) +} + +func (om *fakeObjectManager) setPodRunning(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return nil, err + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return nil, err + } + pod := findPodByOrdinal(pods, ordinal) + if pod == nil { + return nil, fmt.Errorf("setPodRunning: pod ordinal %d not found", ordinal) + } + pod.Status.Phase = v1.PodRunning + fakeResourceVersion(pod) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) +} + +func (om *fakeObjectManager) setPodReady(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return nil, err + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return nil, err + } + pod := findPodByOrdinal(pods, ordinal) + if pod == nil { + return nil, fmt.Errorf("setPodReady: pod ordinal %d not found", ordinal) + } + condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} + podutil.UpdatePodCondition(&pod.Status, &condition) + fakeResourceVersion(pod) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) +} + +func (om *fakeObjectManager) setPodAvailable(set *api.PetSet, ordinal int, lastTransitionTime time.Time) ([]*v1.Pod, error) { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return nil, err + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return nil, err + } + pod := findPodByOrdinal(pods, ordinal) + if pod == nil { + return nil, fmt.Errorf("setPodAvailable: pod ordinal %d not found", ordinal) + } + condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: metav1.Time{Time: lastTransitionTime}} + _, existingCondition := podutil.GetPodCondition(&pod.Status, condition.Type) + if existingCondition != nil { + existingCondition.Status = v1.ConditionTrue + existingCondition.LastTransitionTime = metav1.Time{Time: lastTransitionTime} + } else { + existingCondition = &v1.PodCondition{ + Type: v1.PodReady, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: lastTransitionTime}, + } + pod.Status.Conditions = append(pod.Status.Conditions, *existingCondition) + } + podutil.UpdatePodCondition(&pod.Status, &condition) + fakeResourceVersion(pod) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) +} + +func (om *fakeObjectManager) addTerminatingPod(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { + pod := newTestPetSetPod(set, ordinal) + pod.SetUID(types.UID(pod.Name + "-uid")) // To match fakeObjectManager.CreatePod + pod.Status.Phase = v1.PodRunning + deleted := metav1.NewTime(time.Now()) + pod.DeletionTimestamp = &deleted + condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} + fakeResourceVersion(pod) + podutil.UpdatePodCondition(&pod.Status, &condition) + om.podsIndexer.Update(pod) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return nil, err + } + return om.podsLister.Pods(set.Namespace).List(selector) +} + +func (om *fakeObjectManager) setPodTerminated(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { + pod := newTestPetSetPod(set, ordinal) + deleted := metav1.NewTime(time.Now()) + pod.DeletionTimestamp = &deleted + fakeResourceVersion(pod) + om.podsIndexer.Update(pod) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return nil, err + } + return om.podsLister.Pods(set.Namespace).List(selector) +} + +var _ StatefulPodControlObjectManager = &fakeObjectManager{} + +type fakeStatefulSetStatusUpdater struct { + setsLister apilisters.PetSetLister + setsIndexer cache.Indexer + updateStatusTracker requestTracker +} + +func newFakeStatefulSetStatusUpdater(setInformer stsinformers.PetSetInformer) *fakeStatefulSetStatusUpdater { + return &fakeStatefulSetStatusUpdater{ + setInformer.Lister(), + setInformer.Informer().GetIndexer(), + newRequestTracker(0, nil, 0), + } +} + +func (ssu *fakeStatefulSetStatusUpdater) UpdateStatefulSetStatus(ctx context.Context, set *api.PetSet, status *apps.StatefulSetStatus) error { + defer ssu.updateStatusTracker.inc() + if ssu.updateStatusTracker.errorReady() { + defer ssu.updateStatusTracker.reset() + return ssu.updateStatusTracker.err + } + set.Status = *status + ssu.setsIndexer.Update(set) + return nil +} + +func (ssu *fakeStatefulSetStatusUpdater) SetUpdateStatefulSetStatusError(err error, after int) { + ssu.updateStatusTracker.err = err + ssu.updateStatusTracker.after = after +} + +var _ StatefulSetStatusUpdaterInterface = &fakeStatefulSetStatusUpdater{} + +func assertMonotonicInvariants(set *api.PetSet, om *fakeObjectManager) error { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + sort.Sort(ascendingOrdinal(pods)) + for idx := 0; idx < len(pods); idx++ { + if idx > 0 && isRunningAndReady(pods[idx]) && !isRunningAndReady(pods[idx-1]) { + return fmt.Errorf("successor %s is Running and Ready while %s is not", pods[idx].Name, pods[idx-1].Name) + } + + if ord := idx + getStartOrdinal(set); getOrdinal(pods[idx]) != ord { + return fmt.Errorf("pods %s deployed in the wrong order %d", pods[idx].Name, ord) + } + + if !storageMatches(set, pods[idx]) { + return fmt.Errorf("pods %s does not match the storage specification of PetSet %s ", pods[idx].Name, set.Name) + } + + for _, claim := range getPersistentVolumeClaims(set, pods[idx]) { + claim, _ := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) + if err := checkClaimInvarients(set, pods[idx], claim); err != nil { + return err + } + } + + if !identityMatches(set, pods[idx]) { + return fmt.Errorf("pods %s does not match the identity specification of PetSet %s ", pods[idx].Name, set.Name) + } + } + return nil +} + +func assertBurstInvariants(set *api.PetSet, om *fakeObjectManager) error { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + sort.Sort(ascendingOrdinal(pods)) + for _, pod := range pods { + if !storageMatches(set, pod) { + return fmt.Errorf("pods %s does not match the storage specification of PetSet %s ", pod.Name, set.Name) + } + + for _, claim := range getPersistentVolumeClaims(set, pod) { + claim, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) + if err != nil { + return err + } + if err := checkClaimInvarients(set, pod, claim); err != nil { + return err + } + } + + if !identityMatches(set, pod) { + return fmt.Errorf("pods %s does not match the identity specification of PetSet %s ", + pod.Name, + set.Name) + } + } + return nil +} + +func assertUpdateInvariants(set *api.PetSet, om *fakeObjectManager) error { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + sort.Sort(ascendingOrdinal(pods)) + for _, pod := range pods { + + if !storageMatches(set, pod) { + return fmt.Errorf("pod %s does not match the storage specification of PetSet %s ", pod.Name, set.Name) + } + + for _, claim := range getPersistentVolumeClaims(set, pod) { + claim, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) + if err != nil { + return err + } + if err := checkClaimInvarients(set, pod, claim); err != nil { + return err + } + } + + if !identityMatches(set, pod) { + return fmt.Errorf("pod %s does not match the identity specification of PetSet %s ", pod.Name, set.Name) + } + } + if set.Spec.UpdateStrategy.Type == apps.OnDeleteStatefulSetStrategyType { + return nil + } + if set.Spec.UpdateStrategy.Type == apps.RollingUpdateStatefulSetStrategyType { + for i := 0; i < int(set.Status.CurrentReplicas) && i < len(pods); i++ { + if want, got := set.Status.CurrentRevision, getPodRevision(pods[i]); want != got { + return fmt.Errorf("pod %s want current revision %s got %s", pods[i].Name, want, got) + } + } + for i, j := len(pods)-1, 0; j < int(set.Status.UpdatedReplicas); i, j = i-1, j+1 { + if want, got := set.Status.UpdateRevision, getPodRevision(pods[i]); want != got { + return fmt.Errorf("pod %s want update revision %s got %s", pods[i].Name, want, got) + } + } + } + return nil +} + +func checkClaimInvarients(set *api.PetSet, pod *v1.Pod, claim *v1.PersistentVolumeClaim) error { + policy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if set.Spec.PersistentVolumeClaimRetentionPolicy != nil && features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) { + policy = *set.Spec.PersistentVolumeClaimRetentionPolicy + } + claimShouldBeRetained := policy.WhenScaled == apps.RetainPersistentVolumeClaimRetentionPolicyType + if claim == nil { + if claimShouldBeRetained { + return fmt.Errorf("claim for Pod %s was not created", pod.Name) + } + return nil // A non-retained claim has no invariants to satisfy. + } + + if pod.Status.Phase != v1.PodRunning || !podutil.IsPodReady(pod) { + // The pod has spun up yet, we do not expect the owner refs on the claim to have been set. + return nil + } + + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + case policy.WhenScaled == retain && policy.WhenDeleted == retain: + if hasOwnerRef(claim, set) { + return fmt.Errorf("claim %s has unexpected owner ref on %s for PetSet retain", claim.Name, set.Name) + } + if hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s has unexpected owner ref on pod %s for PetSet retain", claim.Name, pod.Name) + } + case policy.WhenScaled == retain && policy.WhenDeleted == delete: + if !hasOwnerRef(claim, set) { + return fmt.Errorf("claim %s does not have owner ref on %s for PetSet deletion", claim.Name, set.Name) + } + if hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s has unexpected owner ref on pod %s for PetSet deletion", claim.Name, pod.Name) + } + case policy.WhenScaled == delete && policy.WhenDeleted == retain: + if hasOwnerRef(claim, set) { + return fmt.Errorf("claim %s has unexpected owner ref on %s for scaledown only", claim.Name, set.Name) + } + if !podInOrdinalRange(pod, set) && !hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s does not have owner ref on condemned pod %s for scaledown delete", claim.Name, pod.Name) + } + if podInOrdinalRange(pod, set) && hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s has unexpected owner ref on condemned pod %s for scaledown delete", claim.Name, pod.Name) + } + case policy.WhenScaled == delete && policy.WhenDeleted == delete: + if !podInOrdinalRange(pod, set) { + if !hasOwnerRef(claim, pod) || hasOwnerRef(claim, set) { + return fmt.Errorf("condemned claim %s has bad owner refs: %v", claim.Name, claim.GetOwnerReferences()) + } + } else { + if hasOwnerRef(claim, pod) || !hasOwnerRef(claim, set) { + return fmt.Errorf("live claim %s has bad owner refs: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + return nil +} + +func fakeResourceVersion(object interface{}) { + obj, isObj := object.(metav1.Object) + if !isObj { + return + } + if version := obj.GetResourceVersion(); version == "" { + obj.SetResourceVersion("1") + } else if intValue, err := strconv.ParseInt(version, 10, 32); err == nil { + obj.SetResourceVersion(strconv.FormatInt(intValue+1, 10)) + } +} + +func TestParallelScale(t *testing.T) { + for _, tc := range []struct { + desc string + replicas int32 + desiredReplicas int32 + }{ + { + desc: "scale up from 3 to 30", + replicas: 3, + desiredReplicas: 30, + }, + { + desc: "scale down from 10 to 1", + replicas: 10, + desiredReplicas: 1, + }, + + { + desc: "scale down to 0", + replicas: 501, + desiredReplicas: 0, + }, + { + desc: "scale up from 0", + replicas: 0, + desiredReplicas: 1000, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + set := burst(newPetSet(0)) + set.Spec.VolumeClaimTemplates[0].ObjectMeta.Labels = map[string]string{"test": "test"} + parallelScale(t, set, tc.replicas, tc.desiredReplicas, assertBurstInvariants) + }) + } +} + +func parallelScale(t *testing.T, set *api.PetSet, replicas, desiredReplicas int32, invariants invariantFunc) { + var err error + diff := desiredReplicas - replicas + client := fake.NewSimpleClientset() + apiclient := apifake.NewSimpleClientset(set) + om, _, ssc := setupController(client, apiclient) + om.createPodTracker.delay = time.Millisecond + + *set.Spec.Replicas = replicas + if err := parallelScaleUpPetSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + if set.Status.Replicas != replicas { + t.Errorf("want %v, got %v replicas", replicas, set.Status.Replicas) + } + + fn := parallelScaleUpPetSetControl + if diff < 0 { + fn = parallelScaleDownPetSetControl + } + *set.Spec.Replicas = desiredReplicas + if err := fn(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to scale PetSet : %s", err) + } + + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated PetSet: %v", err) + } + + if set.Status.Replicas != desiredReplicas { + t.Errorf("Failed to scale petset to %v replicas, got %v replicas", desiredReplicas, set.Status.Replicas) + } + + if (diff < -1 || diff > 1) && om.createPodTracker.maxParallel <= 1 { + t.Errorf("want max parallel requests > 1, got %v", om.createPodTracker.maxParallel) + } +} + +func parallelScaleUpPetSetControl(set *api.PetSet, + ssc PetSetControlInterface, + om *fakeObjectManager, + invariants invariantFunc, +) error { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + + // Give up after 2 loops. + // 2 * 500 pods per loop = 1000 max pods <- this should be enough for all test cases. + // Anything slower than that (requiring more iterations) indicates a problem and should fail the test. + maxLoops := 2 + loops := maxLoops + for set.Status.Replicas < *set.Spec.Replicas { + if loops < 1 { + return fmt.Errorf("after %v loops: want %v, got replicas %v", maxLoops, *set.Spec.Replicas, set.Status.Replicas) + } + loops-- + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + sort.Sort(ascendingOrdinal(pods)) + + ordinals := []int{} + for _, pod := range pods { + if pod.Status.Phase == "" { + ordinals = append(ordinals, getOrdinal(pod)) + } + } + // ensure all pods are valid (have a phase) + for _, ord := range ordinals { + if pods, err = om.setPodPending(set, ord); err != nil { + return err + } + } + + // run the controller once and check invariants + _, err = ssc.UpdatePetSet(context.TODO(), set, pods) + if err != nil { + return err + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + return err + } + if err := invariants(set, om); err != nil { + return err + } + } + return invariants(set, om) +} + +func parallelScaleDownPetSetControl(set *api.PetSet, ssc PetSetControlInterface, om *fakeObjectManager, invariants invariantFunc) error { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + + // Give up after 2 loops. + // 2 * 500 pods per loop = 1000 max pods <- this should be enough for all test cases. + // Anything slower than that (requiring more iterations) indicates a problem and should fail the test. + maxLoops := 2 + loops := maxLoops + for set.Status.Replicas > *set.Spec.Replicas { + if loops < 1 { + return fmt.Errorf("after %v loops: want %v replicas, got %v", maxLoops, *set.Spec.Replicas, set.Status.Replicas) + } + loops-- + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + sort.Sort(ascendingOrdinal(pods)) + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + return err + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + return err + } + if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + return err + } + } + + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + return err + } + if err := invariants(set, om); err != nil { + return err + } + + return nil +} + +func scaleUpPetSetControl(set *api.PetSet, + ssc PetSetControlInterface, + om *fakeObjectManager, + invariants invariantFunc, +) error { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + for set.Status.ReadyReplicas < *set.Spec.Replicas { + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + sort.Sort(ascendingOrdinal(pods)) + + // ensure all pods are valid (have a phase) + for _, pod := range pods { + if pod.Status.Phase == "" { + if pods, err = om.setPodPending(set, getOrdinal(pod)); err != nil { + return err + } + break + } + } + + // select one of the pods and move it forward in status + if len(pods) > 0 { + idx := int(rand.Int63n(int64(len(pods)))) + pod := pods[idx] + switch pod.Status.Phase { + case v1.PodPending: + if pods, err = om.setPodRunning(set, getOrdinal(pod)); err != nil { + return err + } + case v1.PodRunning: + if pods, err = om.setPodReady(set, getOrdinal(pod)); err != nil { + return err + } + default: + continue + } + } + // run the controller once and check invariants + _, err = ssc.UpdatePetSet(context.TODO(), set, pods) + if err != nil { + return err + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + return err + } + if err := invariants(set, om); err != nil { + return err + } + } + return invariants(set, om) +} + +func scaleDownPetSetControl(set *api.PetSet, ssc PetSetControlInterface, om *fakeObjectManager, invariants invariantFunc) error { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + + for set.Status.Replicas > *set.Spec.Replicas { + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + sort.Sort(ascendingOrdinal(pods)) + if idx := len(pods) - 1; idx >= 0 { + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + return err + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + return err + } + if pods, err = om.addTerminatingPod(set, getOrdinal(pods[idx])); err != nil { + return err + } + if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + return err + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + return err + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + sort.Sort(ascendingOrdinal(pods)) + + if len(pods) > 0 { + om.podsIndexer.Delete(pods[len(pods)-1]) + } + } + if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + return err + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + return err + } + + if err := invariants(set, om); err != nil { + return err + } + } + // If there are claims with ownerRefs on pods that have been deleted, delete them. + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + currentPods := map[string]bool{} + for _, pod := range pods { + currentPods[pod.Name] = true + } + claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { + return err + } + for _, claim := range claims { + claimPodName := getClaimPodName(set, claim) + if claimPodName == "" { + continue // Skip claims not related to a stateful set pod. + } + if _, found := currentPods[claimPodName]; found { + continue // Skip claims which still have a current pod. + } + for _, refs := range claim.GetOwnerReferences() { + if refs.Name == claimPodName { + om.claimsIndexer.Delete(claim) + break + } + } + } + + return invariants(set, om) +} + +func updateComplete(set *api.PetSet, pods []*v1.Pod) bool { + sort.Sort(ascendingOrdinal(pods)) + if len(pods) != int(*set.Spec.Replicas) { + return false + } + if set.Status.ReadyReplicas != *set.Spec.Replicas { + return false + } + + switch set.Spec.UpdateStrategy.Type { + case apps.OnDeleteStatefulSetStrategyType: + return true + case apps.RollingUpdateStatefulSetStrategyType: + if set.Spec.UpdateStrategy.RollingUpdate == nil || *set.Spec.UpdateStrategy.RollingUpdate.Partition <= 0 { + if set.Status.CurrentReplicas < *set.Spec.Replicas { + return false + } + for i := range pods { + if getPodRevision(pods[i]) != set.Status.CurrentRevision { + return false + } + } + } else { + partition := int(*set.Spec.UpdateStrategy.RollingUpdate.Partition) + if len(pods) < partition { + return false + } + for i := partition; i < len(pods); i++ { + if getPodRevision(pods[i]) != set.Status.UpdateRevision { + return false + } + } + } + } + return true +} + +func updatePetSetControl(set *api.PetSet, + ssc PetSetControlInterface, + om *fakeObjectManager, + invariants invariantFunc, +) error { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + return err + } + + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + return err + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + for !updateComplete(set, pods) { + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + sort.Sort(ascendingOrdinal(pods)) + initialized := false + for _, pod := range pods { + if pod.Status.Phase == "" { + if pods, err = om.setPodPending(set, getOrdinal(pod)); err != nil { + return err + } + break + } + } + if initialized { + continue + } + + if len(pods) > 0 { + idx := int(rand.Int63n(int64(len(pods)))) + pod := pods[idx] + switch pod.Status.Phase { + case v1.PodPending: + if pods, err = om.setPodRunning(set, getOrdinal(pod)); err != nil { + return err + } + case v1.PodRunning: + if pods, err = om.setPodReady(set, getOrdinal(pod)); err != nil { + return err + } + default: + continue + } + } + + if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { + return err + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + return err + } + if err := invariants(set, om); err != nil { + return err + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + } + return invariants(set, om) +} + +func newRevisionOrDie(set *api.PetSet, revision int64) *apps.ControllerRevision { + rev, err := newRevision(set, revision, set.Status.CollisionCount) + if err != nil { + panic(err) + } + return rev +} + +func isOrHasInternalError(err error) bool { + agg, ok := err.(utilerrors.Aggregate) + return !ok && !apierrors.IsInternalError(err) || ok && len(agg.Errors()) > 0 && !apierrors.IsInternalError(agg.Errors()[0]) +} diff --git a/pkg/controller/petset/pet_set_status_updater_test.go b/pkg/controller/petset/pet_set_status_updater_test.go new file mode 100644 index 00000000..7091ae52 --- /dev/null +++ b/pkg/controller/petset/pet_set_status_updater_test.go @@ -0,0 +1,142 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package petset + +import ( + "context" + "errors" + "testing" + + api "kubeops.dev/petset/apis/apps/v1" + "kubeops.dev/petset/client/clientset/versioned/fake" + appslisters "kubeops.dev/petset/client/listers/apps/v1" + + apps "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + core "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" +) + +func TestPetSetUpdaterUpdatesSetStatus(t *testing.T) { + set := newPetSet(3) + status := apps.StatefulSetStatus{ObservedGeneration: 1, Replicas: 2} + fakeClient := &fake.Clientset{} + updater := NewRealStatefulSetStatusUpdater(fakeClient, nil) + fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + return true, update.GetObject(), nil + }) + if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err != nil { + t.Errorf("Error returned on successful status update: %s", err) + } + if set.Status.Replicas != 2 { + t.Errorf("UpdateStatefulSetStatus mutated the sets replicas %d", set.Status.Replicas) + } +} + +func TestStatefulSetStatusUpdaterUpdatesObservedGeneration(t *testing.T) { + set := newPetSet(3) + status := apps.StatefulSetStatus{ObservedGeneration: 3, Replicas: 2} + fakeClient := &fake.Clientset{} + updater := NewRealStatefulSetStatusUpdater(fakeClient, nil) + fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + sts := update.GetObject().(*api.PetSet) + if sts.Status.ObservedGeneration != 3 { + t.Errorf("expected observedGeneration to be synced with generation for petset %q", sts.Name) + } + return true, sts, nil + }) + if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err != nil { + t.Errorf("Error returned on successful status update: %s", err) + } +} + +func TestStatefulSetStatusUpdaterUpdateReplicasFailure(t *testing.T) { + set := newPetSet(3) + status := apps.StatefulSetStatus{ObservedGeneration: 3, Replicas: 2} + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + indexer.Add(set) + setLister := appslisters.NewPetSetLister(indexer) + updater := NewRealStatefulSetStatusUpdater(fakeClient, setLister) + fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("API server down")) + }) + if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err == nil { + t.Error("Failed update did not return error") + } +} + +func TestStatefulSetStatusUpdaterUpdateReplicasConflict(t *testing.T) { + set := newPetSet(3) + status := apps.StatefulSetStatus{ObservedGeneration: 3, Replicas: 2} + conflict := false + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + indexer.Add(set) + setLister := appslisters.NewPetSetLister(indexer) + updater := NewRealStatefulSetStatusUpdater(fakeClient, setLister) + fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + if !conflict { + conflict = true + return true, update.GetObject(), apierrors.NewConflict(action.GetResource().GroupResource(), set.Name, errors.New("object already exists")) + } + return true, update.GetObject(), nil + }) + if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err != nil { + t.Errorf("UpdateStatefulSetStatus returned an error: %s", err) + } + if set.Status.Replicas != 2 { + t.Errorf("UpdateStatefulSetStatus mutated the sets replicas %d", set.Status.Replicas) + } +} + +func TestStatefulSetStatusUpdaterUpdateReplicasConflictFailure(t *testing.T) { + set := newPetSet(3) + status := apps.StatefulSetStatus{ObservedGeneration: 3, Replicas: 2} + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + indexer.Add(set) + setLister := appslisters.NewPetSetLister(indexer) + updater := NewRealStatefulSetStatusUpdater(fakeClient, setLister) + fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + return true, update.GetObject(), apierrors.NewConflict(action.GetResource().GroupResource(), set.Name, errors.New("object already exists")) + }) + if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err == nil { + t.Error("UpdateStatefulSetStatus failed to return an error on get failure") + } +} + +func TestStatefulSetStatusUpdaterGetAvailableReplicas(t *testing.T) { + set := newPetSet(3) + status := apps.StatefulSetStatus{ObservedGeneration: 1, Replicas: 2, AvailableReplicas: 3} + fakeClient := &fake.Clientset{} + updater := NewRealStatefulSetStatusUpdater(fakeClient, nil) + fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + return true, update.GetObject(), nil + }) + if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err != nil { + t.Errorf("Error returned on successful status update: %s", err) + } + if set.Status.AvailableReplicas != 3 { + t.Errorf("UpdateStatefulSetStatus mutated the sets replicas %d", set.Status.AvailableReplicas) + } +} diff --git a/pkg/controller/petset/pet_set_test.go b/pkg/controller/petset/pet_set_test.go new file mode 100644 index 00000000..7c43b063 --- /dev/null +++ b/pkg/controller/petset/pet_set_test.go @@ -0,0 +1,1109 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package petset + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sort" + "testing" + + api "kubeops.dev/petset/apis/apps/v1" + apifake "kubeops.dev/petset/client/clientset/versioned/fake" + apiinformers "kubeops.dev/petset/client/informers/externalversions" + "kubeops.dev/petset/pkg/controller" + "kubeops.dev/petset/pkg/controller/history" + "kubeops.dev/petset/pkg/features" + + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" + ocmfake "open-cluster-management.io/api/client/work/clientset/versioned/fake" + manifestinformers "open-cluster-management.io/api/client/work/informers/externalversions" +) + +var parentKind = api.SchemeGroupVersion.WithKind("PetSet") + +func alwaysReady() bool { return true } + +func TestPetSetControllerCreates(t *testing.T) { + set := newPetSet(3) + logger, ctx := ktesting.NewTestContext(t) + ssc, spc, om, _ := newFakePetSetController(ctx, set) + if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + if obj, _, err := om.setsIndexer.Get(set); err != nil { + t.Error(err) + } else { + set = obj.(*api.PetSet) + } + if set.Status.Replicas != 3 { + t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) + } +} + +func TestPetSetControllerDeletes(t *testing.T) { + set := newPetSet(3) + logger, ctx := ktesting.NewTestContext(t) + ssc, spc, om, _ := newFakePetSetController(ctx, set) + if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + if obj, _, err := om.setsIndexer.Get(set); err != nil { + t.Error(err) + } else { + set = obj.(*api.PetSet) + } + if set.Status.Replicas != 3 { + t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) + } + *set.Spec.Replicas = 0 + if err := scaleDownPetSetController(logger, set, ssc, spc, om); err != nil { + t.Errorf("Failed to turn down PetSet : %s", err) + } + if obj, _, err := om.setsIndexer.Get(set); err != nil { + t.Error(err) + } else { + set = obj.(*api.PetSet) + } + if set.Status.Replicas != 0 { + t.Errorf("set.Status.Replicas = %v; want 0", set.Status.Replicas) + } +} + +func TestPetSetControllerRespectsTermination(t *testing.T) { + set := newPetSet(3) + logger, ctx := ktesting.NewTestContext(t) + ssc, spc, om, _ := newFakePetSetController(ctx, set) + if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + if obj, _, err := om.setsIndexer.Get(set); err != nil { + t.Error(err) + } else { + set = obj.(*api.PetSet) + } + if set.Status.Replicas != 3 { + t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) + } + _, err := om.addTerminatingPod(set, 3) + if err != nil { + t.Error(err) + } + pods, err := om.addTerminatingPod(set, 4) + if err != nil { + t.Error(err) + } + ssc.syncPetSet(ctx, set, pods) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if len(pods) != 5 { + t.Error("PetSet does not respect termination") + } + sort.Sort(ascendingOrdinal(pods)) + spc.DeleteStatefulPod(set, pods[3]) + spc.DeleteStatefulPod(set, pods[4]) + *set.Spec.Replicas = 0 + if err := scaleDownPetSetController(logger, set, ssc, spc, om); err != nil { + t.Errorf("Failed to turn down PetSet : %s", err) + } + if obj, _, err := om.setsIndexer.Get(set); err != nil { + t.Error(err) + } else { + set = obj.(*api.PetSet) + } + if set.Status.Replicas != 0 { + t.Errorf("set.Status.Replicas = %v; want 0", set.Status.Replicas) + } +} + +func TestPetSetControllerBlocksScaling(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + set := newPetSet(3) + ssc, spc, om, _ := newFakePetSetController(ctx, set) + if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { + t.Errorf("Failed to turn up PetSet : %s", err) + } + if obj, _, err := om.setsIndexer.Get(set); err != nil { + t.Error(err) + } else { + set = obj.(*api.PetSet) + } + if set.Status.Replicas != 3 { + t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) + } + *set.Spec.Replicas = 5 + fakeResourceVersion(set) + om.setsIndexer.Update(set) + _, err := om.setPodTerminated(set, 0) + if err != nil { + t.Error("Failed to set pod terminated at ordinal 0") + } + ssc.enqueuePetSet(set) + fakeWorker(ssc) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if len(pods) != 3 { + t.Error("PetSet does not block scaling") + } + sort.Sort(ascendingOrdinal(pods)) + spc.DeleteStatefulPod(set, pods[0]) + ssc.enqueuePetSet(set) + fakeWorker(ssc) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if len(pods) != 3 { + t.Error("PetSet does not resume when terminated Pod is removed") + } +} + +func TestPetSetControllerDeletionTimestamp(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + set := newPetSet(3) + set.DeletionTimestamp = new(metav1.Time) + ssc, _, om, _ := newFakePetSetController(ctx, set) + + om.setsIndexer.Add(set) + + // Force a sync. It should not try to create any Pods. + ssc.enqueuePetSet(set) + fakeWorker(ssc) + + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatal(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + if got, want := len(pods), 0; got != want { + t.Errorf("len(pods) = %v, want %v", got, want) + } +} + +func TestPetSetControllerDeletionTimestampRace(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + set := newPetSet(3) + // The bare client says it IS deleted. + set.DeletionTimestamp = new(metav1.Time) + ssc, _, om, ssh := newFakePetSetController(ctx, set) + + // The lister (cache) says it's NOT deleted. + set2 := *set + set2.DeletionTimestamp = nil + om.setsIndexer.Add(&set2) + + // The recheck occurs in the presence of a matching orphan. + pod := newTestPetSetPod(set, 1) + pod.OwnerReferences = nil + om.podsIndexer.Add(pod) + set.Status.CollisionCount = new(int32) + revision, err := newRevision(set, 1, set.Status.CollisionCount) + if err != nil { + t.Fatal(err) + } + revision.OwnerReferences = nil + _, err = ssh.CreateControllerRevision(set, revision, set.Status.CollisionCount) + if err != nil { + t.Fatal(err) + } + + // Force a sync. It should not try to create any Pods. + ssc.enqueuePetSet(set) + fakeWorker(ssc) + + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatal(err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Fatal(err) + } + if got, want := len(pods), 1; got != want { + t.Errorf("len(pods) = %v, want %v", got, want) + } + + // It should not adopt pods. + for _, pod := range pods { + if len(pod.OwnerReferences) > 0 { + t.Errorf("unexpected pod owner references: %v", pod.OwnerReferences) + } + } + + // It should not adopt revisions. + revisions, err := ssh.ListControllerRevisions(set, selector) + if err != nil { + t.Fatal(err) + } + if got, want := len(revisions), 1; got != want { + t.Errorf("len(revisions) = %v, want %v", got, want) + } + for _, revision := range revisions { + if len(revision.OwnerReferences) > 0 { + t.Errorf("unexpected revision owner references: %v", revision.OwnerReferences) + } + } +} + +func TestPetSetControllerAddPod(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set1 := newPetSet(3) + set2 := newPetSet(3) + pod1 := newTestPetSetPod(set1, 0) + pod2 := newTestPetSetPod(set2, 0) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + + ssc.addPod(logger, pod1) + key, done := ssc.queue.Get() + if key == nil || done { + t.Error("failed to enqueue PetSet") + } else if key, ok := key.(string); !ok { + t.Error("key is not a string") + } else if expectedKey, _ := controller.KeyFunc(set1); expectedKey != key { + t.Errorf("expected PetSet key %s found %s", expectedKey, key) + } + ssc.queue.Done(key) + + ssc.addPod(logger, pod2) + key, done = ssc.queue.Get() + if key == nil || done { + t.Error("failed to enqueue PetSet") + } else if key, ok := key.(string); !ok { + t.Error("key is not a string") + } else if expectedKey, _ := controller.KeyFunc(set2); expectedKey != key { + t.Errorf("expected PetSet key %s found %s", expectedKey, key) + } + ssc.queue.Done(key) +} + +func TestPetSetControllerAddPodOrphan(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set1 := newPetSet(3) + set2 := newPetSet(3) + set2.Name = "foo2" + set3 := newPetSet(3) + set3.Name = "foo3" + set3.Spec.Selector.MatchLabels = map[string]string{"foo3": "bar"} + pod := newTestPetSetPod(set1, 0) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + om.setsIndexer.Add(set3) + + // Make pod an orphan. Expect matching sets to be queued. + pod.OwnerReferences = nil + ssc.addPod(logger, pod) + if got, want := ssc.queue.Len(), 2; got != want { + t.Errorf("queue.Len() = %v, want %v", got, want) + } +} + +func TestPetSetControllerAddPodNoSet(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, _, _ := newFakePetSetController(ctx) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + ssc.addPod(logger, pod) + ssc.queue.ShutDown() + key, _ := ssc.queue.Get() + if key != nil { + t.Errorf("PetSet enqueued key for Pod with no Set %s", key) + } +} + +func TestPetSetControllerUpdatePod(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set1 := newPetSet(3) + set2 := newPetSet(3) + set2.Name = "foo2" + pod1 := newTestPetSetPod(set1, 0) + pod2 := newTestPetSetPod(set2, 0) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + + prev := *pod1 + fakeResourceVersion(pod1) + ssc.updatePod(logger, &prev, pod1) + key, done := ssc.queue.Get() + if key == nil || done { + t.Error("failed to enqueue PetSet") + } else if key, ok := key.(string); !ok { + t.Error("key is not a string") + } else if expectedKey, _ := controller.KeyFunc(set1); expectedKey != key { + t.Errorf("expected PetSet key %s found %s", expectedKey, key) + } + + prev = *pod2 + fakeResourceVersion(pod2) + ssc.updatePod(logger, &prev, pod2) + key, done = ssc.queue.Get() + if key == nil || done { + t.Error("failed to enqueue PetSet") + } else if key, ok := key.(string); !ok { + t.Error("key is not a string") + } else if expectedKey, _ := controller.KeyFunc(set2); expectedKey != key { + t.Errorf("expected PetSet key %s found %s", expectedKey, key) + } +} + +func TestPetSetControllerUpdatePodWithNoSet(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, _, _ := newFakePetSetController(ctx) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + prev := *pod + fakeResourceVersion(pod) + ssc.updatePod(logger, &prev, pod) + ssc.queue.ShutDown() + key, _ := ssc.queue.Get() + if key != nil { + t.Errorf("PetSet enqueued key for Pod with no Set %s", key) + } +} + +func TestPetSetControllerUpdatePodWithSameVersion(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + om.setsIndexer.Add(set) + ssc.updatePod(logger, pod, pod) + ssc.queue.ShutDown() + key, _ := ssc.queue.Get() + if key != nil { + t.Errorf("PetSet enqueued key for Pod with no Set %s", key) + } +} + +func TestPetSetControllerUpdatePodOrphanWithNewLabels(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + pod.OwnerReferences = nil + set2 := newPetSet(3) + set2.Name = "foo2" + om.setsIndexer.Add(set) + om.setsIndexer.Add(set2) + clone := *pod + clone.Labels = map[string]string{"foo2": "bar2"} + fakeResourceVersion(&clone) + ssc.updatePod(logger, &clone, pod) + if got, want := ssc.queue.Len(), 2; got != want { + t.Errorf("queue.Len() = %v, want %v", got, want) + } +} + +func TestPetSetControllerUpdatePodChangeControllerRef(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set := newPetSet(3) + set2 := newPetSet(3) + set2.Name = "foo2" + pod := newTestPetSetPod(set, 0) + pod2 := newTestPetSetPod(set2, 0) + om.setsIndexer.Add(set) + om.setsIndexer.Add(set2) + clone := *pod + clone.OwnerReferences = pod2.OwnerReferences + fakeResourceVersion(&clone) + ssc.updatePod(logger, &clone, pod) + if got, want := ssc.queue.Len(), 2; got != want { + t.Errorf("queue.Len() = %v, want %v", got, want) + } +} + +func TestPetSetControllerUpdatePodRelease(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set := newPetSet(3) + set2 := newPetSet(3) + set2.Name = "foo2" + pod := newTestPetSetPod(set, 0) + om.setsIndexer.Add(set) + om.setsIndexer.Add(set2) + clone := *pod + clone.OwnerReferences = nil + fakeResourceVersion(&clone) + ssc.updatePod(logger, pod, &clone) + if got, want := ssc.queue.Len(), 2; got != want { + t.Errorf("queue.Len() = %v, want %v", got, want) + } +} + +func TestPetSetControllerDeletePod(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set1 := newPetSet(3) + set2 := newPetSet(3) + set2.Name = "foo2" + pod1 := newTestPetSetPod(set1, 0) + pod2 := newTestPetSetPod(set2, 0) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + + ssc.deletePod(logger, pod1) + key, done := ssc.queue.Get() + if key == nil || done { + t.Error("failed to enqueue PetSet") + } else if key, ok := key.(string); !ok { + t.Error("key is not a string") + } else if expectedKey, _ := controller.KeyFunc(set1); expectedKey != key { + t.Errorf("expected PetSet key %s found %s", expectedKey, key) + } + + ssc.deletePod(logger, pod2) + key, done = ssc.queue.Get() + if key == nil || done { + t.Error("failed to enqueue PetSet") + } else if key, ok := key.(string); !ok { + t.Error("key is not a string") + } else if expectedKey, _ := controller.KeyFunc(set2); expectedKey != key { + t.Errorf("expected PetSet key %s found %s", expectedKey, key) + } +} + +func TestPetSetControllerDeletePodOrphan(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set1 := newPetSet(3) + set2 := newPetSet(3) + set2.Name = "foo2" + pod1 := newTestPetSetPod(set1, 0) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + + pod1.OwnerReferences = nil + ssc.deletePod(logger, pod1) + if got, want := ssc.queue.Len(), 0; got != want { + t.Errorf("queue.Len() = %v, want %v", got, want) + } +} + +func TestPetSetControllerDeletePodTombstone(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set := newPetSet(3) + pod := newTestPetSetPod(set, 0) + om.setsIndexer.Add(set) + tombstoneKey, _ := controller.KeyFunc(pod) + tombstone := cache.DeletedFinalStateUnknown{Key: tombstoneKey, Obj: pod} + ssc.deletePod(logger, tombstone) + key, done := ssc.queue.Get() + if key == nil || done { + t.Error("failed to enqueue PetSet") + } else if key, ok := key.(string); !ok { + t.Error("key is not a string") + } else if expectedKey, _ := controller.KeyFunc(set); expectedKey != key { + t.Errorf("expected PetSet key %s found %s", expectedKey, key) + } +} + +func TestPetSetControllerGetPetSetsForPod(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx) + set1 := newPetSet(3) + set2 := newPetSet(3) + set2.Name = "foo2" + pod := newTestPetSetPod(set1, 0) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + om.podsIndexer.Add(pod) + sets := ssc.getPetSetsForPod(pod) + if got, want := len(sets), 2; got != want { + t.Errorf("len(sets) = %v, want %v", got, want) + } +} + +func TestGetPodsForPetSetAdopt(t *testing.T) { + set := newPetSet(5) + pod1 := newTestPetSetPod(set, 1) + // pod2 is an orphan with matching labels and name. + pod2 := newTestPetSetPod(set, 2) + pod2.OwnerReferences = nil + // pod3 has wrong labels. + pod3 := newTestPetSetPod(set, 3) + pod3.OwnerReferences = nil + pod3.Labels = nil + // pod4 has wrong name. + pod4 := newTestPetSetPod(set, 4) + pod4.OwnerReferences = nil + pod4.Name = "x" + pod4.Name + + _, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx, set, pod1, pod2, pod3, pod4) + + om.podsIndexer.Add(pod1) + om.podsIndexer.Add(pod2) + om.podsIndexer.Add(pod3) + om.podsIndexer.Add(pod4) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatal(err) + } + pods, err := ssc.getPodsForPetSet(context.TODO(), set, selector) + if err != nil { + t.Fatalf("getPodsForPetSet() error: %v", err) + } + got := sets.New[string]() + for _, pod := range pods { + got.Insert(pod.Name) + } + // pod2 should be claimed, pod3 and pod4 ignored + want := sets.New[string](pod1.Name, pod2.Name) + if !got.Equal(want) { + t.Errorf("getPodsForPetSet() = %v, want %v", got, want) + } +} + +func TestAdoptOrphanRevisions(t *testing.T) { + ss1 := newPetSetWithLabels(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) + ss1Rev1, err := history.NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) + if err != nil { + t.Fatal(err) + } + ss1Rev1.Namespace = ss1.Namespace + ss1.Spec.Template.Annotations = make(map[string]string) + ss1.Spec.Template.Annotations["ss1"] = "ss1" + ss1Rev2, err := history.NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) + if err != nil { + t.Fatal(err) + } + ss1Rev2.Namespace = ss1.Namespace + ss1Rev2.OwnerReferences = []metav1.OwnerReference{} + + _, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx, ss1, ss1Rev1, ss1Rev2) + + om.revisionsIndexer.Add(ss1Rev1) + om.revisionsIndexer.Add(ss1Rev2) + + err = ssc.adoptOrphanRevisions(context.TODO(), ss1) + if err != nil { + t.Errorf("adoptOrphanRevisions() error: %v", err) + } + + if revisions, err := ssc.control.ListRevisions(ss1); err != nil { + t.Errorf("ListRevisions() error: %v", err) + } else { + var adopted bool + for i := range revisions { + if revisions[i].Name == ss1Rev2.Name && metav1.GetControllerOf(revisions[i]) != nil { + adopted = true + } + } + if !adopted { + t.Error("adoptOrphanRevisions() not adopt orphan revisions") + } + } +} + +func TestGetPodsForPetSetRelease(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + set := newPetSet(3) + ssc, _, om, _ := newFakePetSetController(ctx, set) + pod1 := newTestPetSetPod(set, 1) + // pod2 is owned but has wrong name. + pod2 := newTestPetSetPod(set, 2) + pod2.Name = "x" + pod2.Name + // pod3 is owned but has wrong labels. + pod3 := newTestPetSetPod(set, 3) + pod3.Labels = nil + // pod4 is an orphan that doesn't match. + pod4 := newTestPetSetPod(set, 4) + pod4.OwnerReferences = nil + pod4.Labels = nil + + om.podsIndexer.Add(pod1) + om.podsIndexer.Add(pod2) + om.podsIndexer.Add(pod3) + om.podsIndexer.Add(pod4) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Fatal(err) + } + pods, err := ssc.getPodsForPetSet(context.TODO(), set, selector) + if err != nil { + t.Fatalf("getPodsForPetSet() error: %v", err) + } + got := sets.New[string]() + for _, pod := range pods { + got.Insert(pod.Name) + } + + // Expect only pod1 (pod2 and pod3 should be released, pod4 ignored). + want := sets.New[string](pod1.Name) + if !got.Equal(want) { + t.Errorf("getPodsForPetSet() = %v, want %v", got, want) + } +} + +func TestOrphanedPodsWithPVCDeletePolicy(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, true) + + testFn := func(t *testing.T, scaledownPolicy, deletionPolicy apps.PersistentVolumeClaimRetentionPolicyType) { + set := newPetSet(4) + *set.Spec.Replicas = 2 + set.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = scaledownPolicy + set.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = deletionPolicy + _, ctx := ktesting.NewTestContext(t) + ssc, _, om, _ := newFakePetSetController(ctx, set) + om.setsIndexer.Add(set) + + pods := []*v1.Pod{} + pods = append(pods, newTestPetSetPod(set, 0)) + // pod1 is orphaned + pods = append(pods, newTestPetSetPod(set, 1)) + pods[1].OwnerReferences = nil + // pod2 is owned but has wrong name. + pods = append(pods, newTestPetSetPod(set, 2)) + pods[2].Name = "x" + pods[2].Name + + ssc.apiClient.(*apifake.Clientset).PrependReactor("patch", "pods", func(action core.Action) (bool, runtime.Object, error) { + patch := action.(core.PatchAction).GetPatch() + target := action.(core.PatchAction).GetName() + var pod *v1.Pod + for _, p := range pods { + if p.Name == target { + pod = p + break + } + } + if pod == nil { + t.Fatalf("Can't find patch target %s", target) + } + original, err := json.Marshal(pod) + if err != nil { + t.Fatalf("failed to marshal original pod %s: %v", pod.Name, err) + } + updated, err := strategicpatch.StrategicMergePatch(original, patch, v1.Pod{}) + if err != nil { + t.Fatalf("failed to apply strategic merge patch %q on node %s: %v", patch, pod.Name, err) + } + if err := json.Unmarshal(updated, pod); err != nil { + t.Fatalf("failed to unmarshal updated pod %s: %v", pod.Name, err) + } + + return true, pod, nil + }) + + for _, pod := range pods { + om.podsIndexer.Add(pod) + claims := getPersistentVolumeClaims(set, pod) + for _, claim := range claims { + om.CreateClaim(&claim, set) + } + } + + for i := range pods { + if _, err := om.setPodReady(set, i); err != nil { + t.Errorf("%d: %v", i, err) + } + if _, err := om.setPodRunning(set, i); err != nil { + t.Errorf("%d: %v", i, err) + } + } + + // First sync to manage orphaned pod, then set replicas. + ssc.enqueuePetSet(set) + fakeWorker(ssc) + *set.Spec.Replicas = 0 // Put an ownerRef for all scale-down deleted PVCs. + ssc.enqueuePetSet(set) + fakeWorker(ssc) + + hasNamedOwnerRef := func(claim *v1.PersistentVolumeClaim, name string) bool { + for _, ownerRef := range claim.GetOwnerReferences() { + if ownerRef.Name == name { + return true + } + } + return false + } + verifyOwnerRefs := func(claim *v1.PersistentVolumeClaim, condemned bool) { + podName := getClaimPodName(set, claim) + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + case scaledownPolicy == retain && deletionPolicy == retain: + if hasNamedOwnerRef(claim, podName) || hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + case scaledownPolicy == retain && deletionPolicy == delete: + if hasNamedOwnerRef(claim, podName) || !hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + case scaledownPolicy == delete && deletionPolicy == retain: + if hasNamedOwnerRef(claim, podName) != condemned || hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + case scaledownPolicy == delete && deletionPolicy == delete: + if hasNamedOwnerRef(claim, podName) != condemned || !hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + + claims, _ := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(labels.Everything()) + if len(claims) != len(pods) { + t.Errorf("Unexpected number of claims: %d", len(claims)) + } + for _, claim := range claims { + // Only the first pod and the reclaimed orphan pod should have owner refs. + switch claim.Name { + case "datadir-foo-0", "datadir-foo-1": + verifyOwnerRefs(claim, false) + case "datadir-foo-2": + if hasNamedOwnerRef(claim, getClaimPodName(set, claim)) || hasNamedOwnerRef(claim, set.Name) { + t.Errorf("unexpected ownerRefs for %s: %v", claim.Name, claim.GetOwnerReferences()) + } + default: + t.Errorf("Unexpected claim %s", claim.Name) + } + } + } + policies := []apps.PersistentVolumeClaimRetentionPolicyType{ + apps.RetainPersistentVolumeClaimRetentionPolicyType, + apps.DeletePersistentVolumeClaimRetentionPolicyType, + } + for _, scaledownPolicy := range policies { + for _, deletionPolicy := range policies { + testName := fmt.Sprintf("ScaleDown:%s/SetDeletion:%s", scaledownPolicy, deletionPolicy) + t.Run(testName, func(t *testing.T) { testFn(t, scaledownPolicy, deletionPolicy) }) + } + } +} + +func TestStaleOwnerRefOnScaleup(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PetSetAutoDeletePVC, true) + + for _, policy := range []*apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + } { + onPolicy := func(msg string, args ...interface{}) string { + return fmt.Sprintf(fmt.Sprintf("(%s) %s", policy, msg), args...) + } + set := newPetSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + logger, ctx := ktesting.NewTestContext(t) + ssc, spc, om, _ := newFakePetSetController(ctx, set) + if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { + t.Error(onPolicy("Failed to turn up PetSet : %s", err)) + } + var err error + if set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name); err != nil { + t.Error(onPolicy("Could not get scaled up set: %v", err)) + } + if set.Status.Replicas != 3 { + t.Error(onPolicy("set.Status.Replicas = %v; want 3", set.Status.Replicas)) + } + *set.Spec.Replicas = 2 + if err := scaleDownPetSetController(logger, set, ssc, spc, om); err != nil { + t.Error(onPolicy("Failed to scale down PetSet : msg, %s", err)) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Error(onPolicy("Could not get scaled down PetSet: %v", err)) + } + if set.Status.Replicas != 2 { + t.Error(onPolicy("Failed to scale petset to 2 replicas")) + } + + var claim *v1.PersistentVolumeClaim + claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2") + if err != nil { + t.Error(onPolicy("Could not find expected pvc datadir-foo-2")) + } + refs := claim.GetOwnerReferences() + if len(refs) != 1 { + t.Error(onPolicy("Expected only one refs: %v", refs)) + } + // Make the pod ref stale. + for i := range refs { + if refs[i].Name == "foo-2" { + refs[i].UID = "stale" + break + } + } + claim.SetOwnerReferences(refs) + if err = om.claimsIndexer.Update(claim); err != nil { + t.Error(onPolicy("Could not update claim with new owner ref: %v", err)) + } + + *set.Spec.Replicas = 3 + // Until the stale PVC goes away, the scale up should never finish. Run 10 iterations, then delete the PVC. + if err := scaleUpPetSetControllerBounded(logger, set, ssc, spc, om, 10); err != nil { + t.Error(onPolicy("Failed attempt to scale PetSet back up: %v", err)) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Error(onPolicy("Could not get scaled down PetSet: %v", err)) + } + if set.Status.Replicas != 2 { + t.Error(onPolicy("Expected set to stay at two replicas")) + } + + claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2") + if err != nil { + t.Error(onPolicy("Could not find expected pvc datadir-foo-2")) + } + refs = claim.GetOwnerReferences() + if len(refs) != 1 { + t.Error(onPolicy("Unexpected change to condemned pvc ownerRefs: %v", refs)) + } + foundPodRef := false + for i := range refs { + if refs[i].UID == "stale" { + foundPodRef = true + break + } + } + if !foundPodRef { + t.Error(onPolicy("Claim ref unexpectedly changed: %v", refs)) + } + if err = om.claimsIndexer.Delete(claim); err != nil { + t.Error(onPolicy("Could not delete stale pvc: %v", err)) + } + + if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { + t.Error(onPolicy("Failed to scale PetSet back up: %v", err)) + } + set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) + if err != nil { + t.Error(onPolicy("Could not get scaled down PetSet: %v", err)) + } + if set.Status.Replicas != 3 { + t.Error(onPolicy("Failed to scale set back up once PVC was deleted")) + } + } +} + +func newFakePetSetController(ctx context.Context, initialObjects ...runtime.Object) (*PetSetController, *StatefulPodControl, *fakeObjectManager, history.Interface) { + // PetSet/PlacementPolicy are CRD types known only to the versioned (api) fake + // client; the core kube fake client's scheme would panic on them. Route each + // object to the client that understands it. + var kubeObjects, apiObjects []runtime.Object + for _, obj := range initialObjects { + switch obj.(type) { + case *api.PetSet, *api.PlacementPolicy: + apiObjects = append(apiObjects, obj) + default: + kubeObjects = append(kubeObjects, obj) + } + } + client := fake.NewSimpleClientset(kubeObjects...) + informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) + apiclient := apifake.NewSimpleClientset(apiObjects...) + apiinformerFactory := apiinformers.NewSharedInformerFactory(apiclient, controller.NoResyncPeriodFunc()) + om := newFakeObjectManager(informerFactory, apiinformerFactory) + spc := NewStatefulPodControlFromManager(om, &noopRecorder{}) + ssu := newFakeStatefulSetStatusUpdater(apiinformerFactory.Apps().V1().PetSets()) + ocmClient := ocmfake.NewSimpleClientset() + manifestInformerFactory := manifestinformers.NewSharedInformerFactory(ocmClient, controller.NoResyncPeriodFunc()) + ssc := NewPetSetController( + ctx, + informerFactory.Core().V1().Pods(), + apiinformerFactory.Apps().V1().PetSets(), + apiinformerFactory.Apps().V1().PlacementPolicies(), + informerFactory.Core().V1().PersistentVolumeClaims(), + informerFactory.Apps().V1().ControllerRevisions(), + manifestInformerFactory.Work().V1().ManifestWorks(), + client, + apiclient, + ocmClient, + ) + ssh := history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()) + ssc.podListerSynced = alwaysReady + ssc.setListerSynced = alwaysReady + recorder := record.NewFakeRecorder(10) + ssc.control = NewDefaultPetSetControl(spc, ssu, ssh, recorder) + + return ssc, spc, om, ssh +} + +func fakeWorker(ssc *PetSetController) { + if obj, done := ssc.queue.Get(); !done { + ssc.sync(context.TODO(), obj.(string)) + ssc.queue.Done(obj) + } +} + +func getPodAtOrdinal(pods []*v1.Pod, ordinal int) *v1.Pod { + if 0 > ordinal || ordinal >= len(pods) { + return nil + } + sort.Sort(ascendingOrdinal(pods)) + return pods[ordinal] +} + +func scaleUpPetSetController(logger klog.Logger, set *api.PetSet, ssc *PetSetController, spc *StatefulPodControl, om *fakeObjectManager) error { + return scaleUpPetSetControllerBounded(logger, set, ssc, spc, om, -1) +} + +func scaleUpPetSetControllerBounded(logger klog.Logger, set *api.PetSet, ssc *PetSetController, spc *StatefulPodControl, om *fakeObjectManager, maxIterations int) error { + om.setsIndexer.Add(set) + ssc.enqueuePetSet(set) + fakeWorker(ssc) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + iterations := 0 + for (maxIterations < 0 || iterations < maxIterations) && set.Status.ReadyReplicas < *set.Spec.Replicas { + iterations++ + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + ord := len(pods) - 1 + if pods, err = om.setPodPending(set, ord); err != nil { + return err + } + pod := getPodAtOrdinal(pods, ord) + ssc.addPod(logger, pod) + fakeWorker(ssc) + pod = getPodAtOrdinal(pods, ord) + prev := *pod + if pods, err = om.setPodRunning(set, ord); err != nil { + return err + } + pod = getPodAtOrdinal(pods, ord) + ssc.updatePod(logger, &prev, pod) + fakeWorker(ssc) + pod = getPodAtOrdinal(pods, ord) + prev = *pod + if pods, err = om.setPodReady(set, ord); err != nil { + return err + } + pod = getPodAtOrdinal(pods, ord) + ssc.updatePod(logger, &prev, pod) + fakeWorker(ssc) + if err := assertMonotonicInvariants(set, om); err != nil { + return err + } + obj, _, err := om.setsIndexer.Get(set) + if err != nil { + return err + } + set = obj.(*api.PetSet) + + } + return assertMonotonicInvariants(set, om) +} + +func scaleDownPetSetController(logger klog.Logger, set *api.PetSet, ssc *PetSetController, spc *StatefulPodControl, om *fakeObjectManager) error { + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + return err + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + ord := len(pods) - 1 + pod := getPodAtOrdinal(pods, ord) + prev := *pod + fakeResourceVersion(set) + om.setsIndexer.Add(set) + ssc.enqueuePetSet(set) + fakeWorker(ssc) + pods, err = om.addTerminatingPod(set, ord) + if err != nil { + return err + } + pod = getPodAtOrdinal(pods, ord) + ssc.updatePod(logger, &prev, pod) + fakeWorker(ssc) + spc.DeleteStatefulPod(set, pod) + ssc.deletePod(logger, pod) + fakeWorker(ssc) + for set.Status.Replicas > *set.Spec.Replicas { + pods, err = om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + + ord := len(pods) + pods, err = om.addTerminatingPod(set, ord) + if err != nil { + return err + } + pod = getPodAtOrdinal(pods, ord) + ssc.updatePod(logger, &prev, pod) + fakeWorker(ssc) + spc.DeleteStatefulPod(set, pod) + ssc.deletePod(logger, pod) + fakeWorker(ssc) + obj, _, err := om.setsIndexer.Get(set) + if err != nil { + return err + } + set = obj.(*api.PetSet) + + } + return assertMonotonicInvariants(set, om) +} + +func rawTemplate(template *api.PodTemplateSpec) runtime.RawExtension { + buf := new(bytes.Buffer) + enc := json.NewEncoder(buf) + if err := enc.Encode(template); err != nil { + panic(err) + } + return runtime.RawExtension{Raw: buf.Bytes()} +} diff --git a/pkg/controller/petset/pet_set_utils.go b/pkg/controller/petset/pet_set_utils.go index a4770964..23b8a20b 100644 --- a/pkg/controller/petset/pet_set_utils.go +++ b/pkg/controller/petset/pet_set_utils.go @@ -34,11 +34,22 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog/v2" ) +func init() { + // patchCodec serializes PetSets into ControllerRevisions using the client-go + // global scheme, so the PetSet/PlacementPolicy types must be registered there. + // Register them here rather than relying on a caller (e.g. cmd wiring) to do + // it; otherwise constructing the controller through any other path panics with + // "no kind is registered for the type v1.PetSet" the first time a revision is + // created. + utilruntime.Must(api.AddToScheme(scheme.Scheme)) +} + var patchCodec = scheme.Codecs.LegacyCodec(api.SchemeGroupVersion) // statefulPodRegex is a regular expression that extracts the parent PetSet and ordinal from the Name of a Pod diff --git a/pkg/controller/petset/pet_set_utils_test.go b/pkg/controller/petset/pet_set_utils_test.go new file mode 100644 index 00000000..c9d000c4 --- /dev/null +++ b/pkg/controller/petset/pet_set_utils_test.go @@ -0,0 +1,1009 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package petset + +import ( + "fmt" + "math/rand" + "reflect" + "regexp" + "sort" + "strconv" + "testing" + "time" + + api "kubeops.dev/petset/apis/apps/v1" + podutil "kubeops.dev/petset/pkg/api/v1/pod" + "kubeops.dev/petset/pkg/controller/history" + + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" + "k8s.io/utils/ptr" +) + +// noopRecorder is an EventRecorder that does nothing. record.FakeRecorder has a fixed +// buffer size, which causes tests to hang if that buffer's exceeded. +type noopRecorder struct{} + +func (r *noopRecorder) Event(object runtime.Object, eventtype, reason, message string) {} +func (r *noopRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +} + +func (r *noopRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +} + +// getClaimPodName gets the name of the Pod associated with the Claim, or an empty string if this doesn't look matching. +func getClaimPodName(set *api.PetSet, claim *v1.PersistentVolumeClaim) string { + podName := "" + + statefulClaimRegex := regexp.MustCompile(fmt.Sprintf(".*-(%s-[0-9]+)$", set.Name)) + matches := statefulClaimRegex.FindStringSubmatch(claim.Name) + if len(matches) != 2 { + return podName + } + return matches[1] +} + +func TestGetParentNameAndOrdinal(t *testing.T) { + set := newPetSet(3) + pod := newTestPetSetPod(set, 1) + if parent, ordinal := getParentNameAndOrdinal(pod); parent != set.Name { + t.Errorf("Extracted the wrong parent name expected %s found %s", set.Name, parent) + } else if ordinal != 1 { + t.Errorf("Extracted the wrong ordinal expected %d found %d", 1, ordinal) + } + pod.Name = "1-bar" + if parent, ordinal := getParentNameAndOrdinal(pod); parent != "" { + t.Error("Expected empty string for non-member Pod parent") + } else if ordinal != -1 { + t.Error("Expected -1 for non member Pod ordinal") + } +} + +func TestGetClaimPodName(t *testing.T) { + set := api.PetSet{} + set.Name = "my-set" + claim := v1.PersistentVolumeClaim{} + claim.Name = "volume-my-set-2" + if pod := getClaimPodName(&set, &claim); pod != "my-set-2" { + t.Errorf("Expected my-set-2 found %s", pod) + } + claim.Name = "long-volume-my-set-20" + if pod := getClaimPodName(&set, &claim); pod != "my-set-20" { + t.Errorf("Expected my-set-20 found %s", pod) + } + claim.Name = "volume-2-my-set" + if pod := getClaimPodName(&set, &claim); pod != "" { + t.Errorf("Expected empty string found %s", pod) + } + claim.Name = "volume-pod-2" + if pod := getClaimPodName(&set, &claim); pod != "" { + t.Errorf("Expected empty string found %s", pod) + } +} + +func TestIsMemberOf(t *testing.T) { + set := newPetSet(3) + set2 := newPetSet(3) + set2.Name = "foo2" + pod := newTestPetSetPod(set, 1) + if !isMemberOf(set, pod) { + t.Error("isMemberOf returned false negative") + } + if isMemberOf(set2, pod) { + t.Error("isMemberOf returned false positive") + } +} + +func TestIdentityMatches(t *testing.T) { + set := newPetSet(3) + pod := newTestPetSetPod(set, 1) + if !identityMatches(set, pod) { + t.Error("Newly created Pod has a bad identity") + } + pod.Name = "foo" + if identityMatches(set, pod) { + t.Error("identity matches for a Pod with the wrong name") + } + pod = newTestPetSetPod(set, 1) + pod.Namespace = "" + if identityMatches(set, pod) { + t.Error("identity matches for a Pod with the wrong namespace") + } + pod = newTestPetSetPod(set, 1) + delete(pod.Labels, apps.StatefulSetPodNameLabel) + if identityMatches(set, pod) { + t.Error("identity matches for a Pod with the wrong statefulSetPodNameLabel") + } +} + +func TestStorageMatches(t *testing.T) { + set := newPetSet(3) + pod := newTestPetSetPod(set, 1) + if !storageMatches(set, pod) { + t.Error("Newly created Pod has a invalid storage") + } + pod.Spec.Volumes = nil + if storageMatches(set, pod) { + t.Error("Pod with invalid Volumes has valid storage") + } + pod = newTestPetSetPod(set, 1) + for i := range pod.Spec.Volumes { + pod.Spec.Volumes[i].PersistentVolumeClaim = nil + } + if storageMatches(set, pod) { + t.Error("Pod with invalid Volumes claim valid storage") + } + pod = newTestPetSetPod(set, 1) + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].PersistentVolumeClaim != nil { + pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName = "foo" + } + } + if storageMatches(set, pod) { + t.Error("Pod with invalid Volumes claim valid storage") + } + pod = newTestPetSetPod(set, 1) + pod.Name = "bar" + if storageMatches(set, pod) { + t.Error("Pod with invalid ordinal has valid storage") + } +} + +func TestUpdateIdentity(t *testing.T) { + set := newPetSet(3) + pod := newTestPetSetPod(set, 1) + if !identityMatches(set, pod) { + t.Error("Newly created Pod has a bad identity") + } + pod.Namespace = "" + if identityMatches(set, pod) { + t.Error("identity matches for a Pod with the wrong namespace") + } + updateIdentity(set, pod) + if !identityMatches(set, pod) { + t.Error("updateIdentity failed to update the Pods namespace") + } + delete(pod.Labels, apps.StatefulSetPodNameLabel) + updateIdentity(set, pod) + if !identityMatches(set, pod) { + t.Error("updateIdentity failed to restore the statefulSetPodName label") + } +} + +func TestUpdateStorage(t *testing.T) { + set := newPetSet(3) + pod := newTestPetSetPod(set, 1) + if !storageMatches(set, pod) { + t.Error("Newly created Pod has a invalid storage") + } + pod.Spec.Volumes = nil + if storageMatches(set, pod) { + t.Error("Pod with invalid Volumes has valid storage") + } + updateStorage(set, pod) + if !storageMatches(set, pod) { + t.Error("updateStorage failed to recreate volumes") + } + pod = newTestPetSetPod(set, 1) + for i := range pod.Spec.Volumes { + pod.Spec.Volumes[i].PersistentVolumeClaim = nil + } + if storageMatches(set, pod) { + t.Error("Pod with invalid Volumes claim valid storage") + } + updateStorage(set, pod) + if !storageMatches(set, pod) { + t.Error("updateStorage failed to recreate volume claims") + } + pod = newTestPetSetPod(set, 1) + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].PersistentVolumeClaim != nil { + pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName = "foo" + } + } + if storageMatches(set, pod) { + t.Error("Pod with invalid Volumes claim valid storage") + } + updateStorage(set, pod) + if !storageMatches(set, pod) { + t.Error("updateStorage failed to recreate volume claim names") + } +} + +func TestGetPersistentVolumeClaimRetentionPolicy(t *testing.T) { + retainPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + scaledownPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + + set := api.PetSet{} + set.Spec.PersistentVolumeClaimRetentionPolicy = &retainPolicy + got := getPersistentVolumeClaimRetentionPolicy(&set) + if got.WhenScaled != apps.RetainPersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType { + t.Errorf("Expected retain policy") + } + set.Spec.PersistentVolumeClaimRetentionPolicy = &scaledownPolicy + got = getPersistentVolumeClaimRetentionPolicy(&set) + if got.WhenScaled != apps.DeletePersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType { + t.Errorf("Expected scaledown policy") + } +} + +func TestClaimOwnerMatchesSetAndPod(t *testing.T) { + testCases := []struct { + name string + scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType + setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType + needsPodRef bool + needsSetRef bool + replicas int32 + ordinal int + }{ + { + name: "retain", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: false, + }, + { + name: "on SS delete", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: true, + }, + { + name: "on scaledown only, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + needsPodRef: true, + needsSetRef: false, + replicas: 2, + ordinal: 2, + }, + { + name: "on scaledown only, remains", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: false, + replicas: 2, + ordinal: 1, + }, + { + name: "on both, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + needsPodRef: true, + needsSetRef: false, + replicas: 2, + ordinal: 2, + }, + { + name: "on both, remains", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: true, + replicas: 2, + ordinal: 1, + }, + } + + for _, tc := range testCases { + for _, useOtherRefs := range []bool{false, true} { + for _, setPodRef := range []bool{false, true} { + for _, setSetRef := range []bool{false, true} { + _, ctx := ktesting.NewTestContext(t) + logger := klog.FromContext(ctx) + claim := v1.PersistentVolumeClaim{} + claim.Name = "target-claim" + pod := v1.Pod{} + pod.Name = fmt.Sprintf("pod-%d", tc.ordinal) + pod.GetObjectMeta().SetUID("pod-123") + set := api.PetSet{} + set.Name = "stateful-set" + set.GetObjectMeta().SetUID("ss-456") + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: tc.scaleDownPolicy, + WhenDeleted: tc.setDeletePolicy, + } + set.Spec.Replicas = &tc.replicas + if setPodRef { + setOwnerRef(&claim, &pod, &pod.TypeMeta) + } + if setSetRef { + setOwnerRef(&claim, &set, &set.TypeMeta) + } + if useOtherRefs { + randomObject1 := v1.Pod{} + randomObject1.Name = "rand1" + randomObject1.GetObjectMeta().SetUID("rand1-abc") + randomObject2 := v1.Pod{} + randomObject2.Name = "rand2" + randomObject2.GetObjectMeta().SetUID("rand2-def") + setOwnerRef(&claim, &randomObject1, &randomObject1.TypeMeta) + setOwnerRef(&claim, &randomObject2, &randomObject2.TypeMeta) + } + shouldMatch := setPodRef == tc.needsPodRef && setSetRef == tc.needsSetRef + if claimOwnerMatchesSetAndPod(logger, &claim, &set, &pod) != shouldMatch { + t.Errorf("Bad match for %s with pod=%v,set=%v,others=%v", tc.name, setPodRef, setSetRef, useOtherRefs) + } + } + } + } + } +} + +func TestUpdateClaimOwnerRefForSetAndPod(t *testing.T) { + testCases := []struct { + name string + scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType + setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType + condemned bool + needsPodRef bool + needsSetRef bool + }{ + { + name: "retain", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: false, + }, + { + name: "delete with set", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: true, + }, + { + name: "delete with scaledown, not condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: false, + }, + { + name: "delete on scaledown, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + condemned: true, + needsPodRef: true, + needsSetRef: false, + }, + { + name: "delete on both, not condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: true, + }, + { + name: "delete on both, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + condemned: true, + needsPodRef: true, + needsSetRef: false, + }, + } + for _, tc := range testCases { + for _, hasPodRef := range []bool{true, false} { + for _, hasSetRef := range []bool{true, false} { + _, ctx := ktesting.NewTestContext(t) + logger := klog.FromContext(ctx) + set := api.PetSet{} + set.Name = "ss" + numReplicas := int32(5) + set.Spec.Replicas = &numReplicas + set.SetUID("ss-123") + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: tc.scaleDownPolicy, + WhenDeleted: tc.setDeletePolicy, + } + pod := v1.Pod{} + if tc.condemned { + pod.Name = "pod-8" + } else { + pod.Name = "pod-1" + } + pod.SetUID("pod-456") + claim := v1.PersistentVolumeClaim{} + if hasPodRef { + setOwnerRef(&claim, &pod, &pod.TypeMeta) + } + if hasSetRef { + setOwnerRef(&claim, &set, &set.TypeMeta) + } + needsUpdate := hasPodRef != tc.needsPodRef || hasSetRef != tc.needsSetRef + shouldUpdate := updateClaimOwnerRefForSetAndPod(logger, &claim, &set, &pod) + if shouldUpdate != needsUpdate { + t.Errorf("Bad update for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) + } + if hasOwnerRef(&claim, &pod) != tc.needsPodRef { + t.Errorf("Bad pod ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) + } + if hasOwnerRef(&claim, &set) != tc.needsSetRef { + t.Errorf("Bad set ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) + } + } + } + } +} + +func TestHasOwnerRef(t *testing.T) { + target := v1.Pod{} + target.SetOwnerReferences([]metav1.OwnerReference{ + {UID: "123"}, {UID: "456"}, + }) + ownerA := v1.Pod{} + ownerA.GetObjectMeta().SetUID("123") + ownerB := v1.Pod{} + ownerB.GetObjectMeta().SetUID("789") + if !hasOwnerRef(&target, &ownerA) { + t.Error("Missing owner") + } + if hasOwnerRef(&target, &ownerB) { + t.Error("Unexpected owner") + } +} + +func TestHasStaleOwnerRef(t *testing.T) { + target := v1.Pod{} + target.SetOwnerReferences([]metav1.OwnerReference{ + {Name: "bob", UID: "123"}, {Name: "shirley", UID: "456"}, + }) + ownerA := v1.Pod{} + ownerA.SetUID("123") + ownerA.Name = "bob" + ownerB := v1.Pod{} + ownerB.Name = "shirley" + ownerB.SetUID("789") + ownerC := v1.Pod{} + ownerC.Name = "yvonne" + ownerC.SetUID("345") + if hasStaleOwnerRef(&target, &ownerA) { + t.Error("ownerA should not be stale") + } + if !hasStaleOwnerRef(&target, &ownerB) { + t.Error("ownerB should be stale") + } + if hasStaleOwnerRef(&target, &ownerC) { + t.Error("ownerC should not be stale") + } +} + +func TestSetOwnerRef(t *testing.T) { + target := v1.Pod{} + ownerA := v1.Pod{} + ownerA.Name = "A" + ownerA.GetObjectMeta().SetUID("ABC") + if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != true { + t.Errorf("Unexpected lack of update") + } + ownerRefs := target.GetObjectMeta().GetOwnerReferences() + if len(ownerRefs) != 1 { + t.Errorf("Unexpected owner ref count: %d", len(ownerRefs)) + } + if ownerRefs[0].UID != "ABC" { + t.Errorf("Unexpected owner UID %v", ownerRefs[0].UID) + } + if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != false { + t.Errorf("Unexpected update") + } + if len(target.GetObjectMeta().GetOwnerReferences()) != 1 { + t.Error("Unexpected duplicate reference") + } + ownerB := v1.Pod{} + ownerB.Name = "B" + ownerB.GetObjectMeta().SetUID("BCD") + if setOwnerRef(&target, &ownerB, &ownerB.TypeMeta) != true { + t.Error("Unexpected lack of second update") + } + ownerRefs = target.GetObjectMeta().GetOwnerReferences() + if len(ownerRefs) != 2 { + t.Errorf("Unexpected owner ref count: %d", len(ownerRefs)) + } + if ownerRefs[0].UID != "ABC" || ownerRefs[1].UID != "BCD" { + t.Errorf("Bad second ownerRefs: %v", ownerRefs) + } +} + +func TestRemoveOwnerRef(t *testing.T) { + target := v1.Pod{} + ownerA := v1.Pod{} + ownerA.Name = "A" + ownerA.GetObjectMeta().SetUID("ABC") + if removeOwnerRef(&target, &ownerA) != false { + t.Error("Unexpected update on empty remove") + } + setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) + if removeOwnerRef(&target, &ownerA) != true { + t.Error("Unexpected lack of update") + } + if len(target.GetObjectMeta().GetOwnerReferences()) != 0 { + t.Error("Unexpected owner reference remains") + } + + ownerB := v1.Pod{} + ownerB.Name = "B" + ownerB.GetObjectMeta().SetUID("BCD") + + setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) + if removeOwnerRef(&target, &ownerB) != false { + t.Error("Unexpected update for mismatched owner") + } + if len(target.GetObjectMeta().GetOwnerReferences()) != 1 { + t.Error("Missing ref after no-op remove") + } + setOwnerRef(&target, &ownerB, &ownerB.TypeMeta) + if removeOwnerRef(&target, &ownerA) != true { + t.Error("Missing update for second remove") + } + ownerRefs := target.GetObjectMeta().GetOwnerReferences() + if len(ownerRefs) != 1 { + t.Error("Extra ref after second remove") + } + if ownerRefs[0].UID != "BCD" { + t.Error("Bad UID after second remove") + } +} + +func TestIsRunningAndReady(t *testing.T) { + set := newPetSet(3) + pod := newTestPetSetPod(set, 1) + if isRunningAndReady(pod) { + t.Error("isRunningAndReady does not respect Pod phase") + } + pod.Status.Phase = v1.PodRunning + if isRunningAndReady(pod) { + t.Error("isRunningAndReady does not respect Pod condition") + } + condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} + podutil.UpdatePodCondition(&pod.Status, &condition) + if !isRunningAndReady(pod) { + t.Error("Pod should be running and ready") + } +} + +func TestAscendingOrdinal(t *testing.T) { + set := newPetSet(10) + pods := make([]*v1.Pod, 10) + perm := rand.Perm(10) + for i, v := range perm { + pods[i] = newTestPetSetPod(set, v) + } + sort.Sort(ascendingOrdinal(pods)) + if !sort.IsSorted(ascendingOrdinal(pods)) { + t.Error("ascendingOrdinal fails to sort Pods") + } +} + +func TestOverlappingPetSets(t *testing.T) { + sets := make([]*api.PetSet, 10) + perm := rand.Perm(10) + for i, v := range perm { + sets[i] = newPetSet(10) + sets[i].CreationTimestamp = metav1.NewTime(sets[i].CreationTimestamp.Add(time.Duration(v) * time.Second)) + } + sort.Sort(overlappingPetSets(sets)) + if !sort.IsSorted(overlappingPetSets(sets)) { + t.Error("ascendingOrdinal fails to sort Pods") + } + for i, v := range perm { + sets[i] = newPetSet(10) + sets[i].Name = strconv.FormatInt(int64(v), 10) + } + sort.Sort(overlappingPetSets(sets)) + if !sort.IsSorted(overlappingPetSets(sets)) { + t.Error("ascendingOrdinal fails to sort Pods") + } +} + +func TestNewPodControllerRef(t *testing.T) { + set := newPetSet(1) + pod := newTestPetSetPod(set, 0) + controllerRef := metav1.GetControllerOf(pod) + if controllerRef == nil { + t.Fatalf("No ControllerRef found on new pod") + } + if got, want := controllerRef.APIVersion, api.SchemeGroupVersion.String(); got != want { + t.Errorf("controllerRef.APIVersion = %q, want %q", got, want) + } + if got, want := controllerRef.Kind, "PetSet"; got != want { + t.Errorf("controllerRef.Kind = %q, want %q", got, want) + } + if got, want := controllerRef.Name, set.Name; got != want { + t.Errorf("controllerRef.Name = %q, want %q", got, want) + } + if got, want := controllerRef.UID, set.UID; got != want { + t.Errorf("controllerRef.UID = %q, want %q", got, want) + } + if got, want := *controllerRef.Controller, true; got != want { + t.Errorf("controllerRef.Controller = %v, want %v", got, want) + } +} + +func TestCreateApplyRevision(t *testing.T) { + set := newPetSet(1) + set.Status.CollisionCount = new(int32) + revision, err := newRevision(set, 1, set.Status.CollisionCount) + if err != nil { + t.Fatal(err) + } + set.Spec.Template.Spec.Containers[0].Name = "foo" + if set.Annotations == nil { + set.Annotations = make(map[string]string) + } + key := "foo" + expectedValue := "bar" + set.Annotations[key] = expectedValue + restoredSet, err := ApplyRevision(set, revision) + if err != nil { + t.Fatal(err) + } + restoredRevision, err := newRevision(restoredSet, 2, restoredSet.Status.CollisionCount) + if err != nil { + t.Fatal(err) + } + if !history.EqualRevision(revision, restoredRevision) { + t.Errorf("wanted %v got %v", string(revision.Data.Raw), string(restoredRevision.Data.Raw)) + } + value, ok := restoredRevision.Annotations[key] + if !ok { + t.Errorf("missing annotation %s", key) + } + if value != expectedValue { + t.Errorf("for annotation %s wanted %s got %s", key, expectedValue, value) + } +} + +func TestRollingUpdateApplyRevision(t *testing.T) { + set := newPetSet(1) + set.Status.CollisionCount = new(int32) + currentSet := set.DeepCopy() + currentRevision, err := newRevision(set, 1, set.Status.CollisionCount) + if err != nil { + t.Fatal(err) + } + + set.Spec.Template.Spec.Containers[0].Env = []v1.EnvVar{{Name: "foo", Value: "bar"}} + updateSet := set.DeepCopy() + updateRevision, err := newRevision(set, 2, set.Status.CollisionCount) + if err != nil { + t.Fatal(err) + } + + restoredCurrentSet, err := ApplyRevision(set, currentRevision) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(currentSet.Spec.Template, restoredCurrentSet.Spec.Template) { + t.Errorf("want %v got %v", currentSet.Spec.Template, restoredCurrentSet.Spec.Template) + } + + restoredUpdateSet, err := ApplyRevision(set, updateRevision) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(updateSet.Spec.Template, restoredUpdateSet.Spec.Template) { + t.Errorf("want %v got %v", updateSet.Spec.Template, restoredUpdateSet.Spec.Template) + } +} + +func TestGetPersistentVolumeClaims(t *testing.T) { + // nil inherits petset labels + pod := newPod() + statefulSet := newPetSet(1) + statefulSet.Spec.Selector.MatchLabels = nil + claims := getPersistentVolumeClaims(statefulSet, pod) + pvc := newPVC("datadir-foo-0") + resultClaims := map[string]v1.PersistentVolumeClaim{"datadir": pvc} + + if !reflect.DeepEqual(claims, resultClaims) { + t.Fatalf("Unexpected pvc:\n %+v\n, desired pvc:\n %+v", claims, resultClaims) + } + + // nil inherits petset labels + statefulSet.Spec.Selector.MatchLabels = map[string]string{"test": "test"} + claims = getPersistentVolumeClaims(statefulSet, pod) + pvc.SetLabels(map[string]string{"test": "test"}) + resultClaims = map[string]v1.PersistentVolumeClaim{"datadir": pvc} + if !reflect.DeepEqual(claims, resultClaims) { + t.Fatalf("Unexpected pvc:\n %+v\n, desired pvc:\n %+v", claims, resultClaims) + } + + // non-nil with non-overlapping labels merge pvc and petset labels + statefulSet.Spec.Selector.MatchLabels = map[string]string{"name": "foo"} + statefulSet.Spec.VolumeClaimTemplates[0].ObjectMeta.Labels = map[string]string{"test": "test"} + claims = getPersistentVolumeClaims(statefulSet, pod) + pvc.SetLabels(map[string]string{"test": "test", "name": "foo"}) + resultClaims = map[string]v1.PersistentVolumeClaim{"datadir": pvc} + if !reflect.DeepEqual(claims, resultClaims) { + t.Fatalf("Unexpected pvc:\n %+v\n, desired pvc:\n %+v", claims, resultClaims) + } + + // non-nil with overlapping labels merge pvc and petset labels and prefer petset labels + statefulSet.Spec.Selector.MatchLabels = map[string]string{"test": "foo"} + statefulSet.Spec.VolumeClaimTemplates[0].ObjectMeta.Labels = map[string]string{"test": "test"} + claims = getPersistentVolumeClaims(statefulSet, pod) + pvc.SetLabels(map[string]string{"test": "foo"}) + resultClaims = map[string]v1.PersistentVolumeClaim{"datadir": pvc} + if !reflect.DeepEqual(claims, resultClaims) { + t.Fatalf("Unexpected pvc:\n %+v\n, desired pvc:\n %+v", claims, resultClaims) + } +} + +func newPod() *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-0", + Namespace: v1.NamespaceDefault, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + } +} + +func newPVC(name string) v1.PersistentVolumeClaim { + return v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: name, + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: *resource.NewQuantity(1, resource.BinarySI), + }, + }, + }, + } +} + +func newPetSetWithVolumes(replicas int32, name string, petMounts []v1.VolumeMount, podMounts []v1.VolumeMount) *api.PetSet { + mounts := append(petMounts, podMounts...) + claims := []v1.PersistentVolumeClaim{} + for _, m := range petMounts { + claims = append(claims, newPVC(m.Name)) + } + + vols := []v1.Volume{} + for _, m := range podMounts { + vols = append(vols, v1.Volume{ + Name: m.Name, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: fmt.Sprintf("/tmp/%v", m.Name), + }, + }, + }) + } + + template := api.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx", + VolumeMounts: mounts, + }, + }, + Volumes: vols, + }, + } + + template.Labels = map[string]string{"foo": "bar"} + + return &api.PetSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "PetSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: v1.NamespaceDefault, + UID: types.UID("test"), + }, + Spec: api.PetSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"foo": "bar"}, + }, + Replicas: ptr.To(replicas), + Template: template, + VolumeClaimTemplates: claims, + ServiceName: "governingsvc", + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + RevisionHistoryLimit: func() *int32 { + limit := int32(2) + return &limit + }(), + }, + } +} + +// newTestPetSetPod is a test helper that builds a Pod for the given PetSet and +// ordinal without any PlacementPolicy. It mirrors the upstream +// newStatefulSetPod(set, ordinal) helper, adapting to the forked newPetSetPod +// signature which also takes a PlacementPolicy, a PodList and returns an error. +func newTestPetSetPod(set *api.PetSet, ordinal int) *v1.Pod { + pod, err := newPetSetPod(set, nil, ordinal, nil) + if err != nil { + panic(err) + } + return pod +} + +// ascendingOrdinal sorts a list of Pods by their ordinal index, ascending. +type ascendingOrdinal []*v1.Pod + +func (ao ascendingOrdinal) Len() int { + return len(ao) +} + +func (ao ascendingOrdinal) Swap(i, j int) { + ao[i], ao[j] = ao[j], ao[i] +} + +func (ao ascendingOrdinal) Less(i, j int) bool { + return getOrdinal(ao[i]) < getOrdinal(ao[j]) +} + +func newPetSet(replicas int32) *api.PetSet { + petMounts := []v1.VolumeMount{ + {Name: "datadir", MountPath: "/tmp/zookeeper"}, + } + podMounts := []v1.VolumeMount{ + {Name: "home", MountPath: "/home"}, + } + return newPetSetWithVolumes(replicas, "foo", petMounts, podMounts) +} + +func newPetSetWithLabels(replicas int32, name string, uid types.UID, labels map[string]string) *api.PetSet { + // Converting all the map-only selectors to set-based selectors. + var testMatchExpressions []metav1.LabelSelectorRequirement + for key, value := range labels { + sel := metav1.LabelSelectorRequirement{ + Key: key, + Operator: metav1.LabelSelectorOpIn, + Values: []string{value}, + } + testMatchExpressions = append(testMatchExpressions, sel) + } + return &api.PetSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "PetSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: v1.NamespaceDefault, + UID: uid, + }, + Spec: api.PetSetSpec{ + Selector: &metav1.LabelSelector{ + // Purposely leaving MatchLabels nil, so to ensure it will break if any link + // in the chain ignores the set-based MatchExpressions. + MatchLabels: nil, + MatchExpressions: testMatchExpressions, + }, + Replicas: ptr.To(replicas), + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + Template: api.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx", + VolumeMounts: []v1.VolumeMount{ + {Name: "datadir", MountPath: "/tmp/"}, + {Name: "home", MountPath: "/home"}, + }, + }, + }, + Volumes: []v1.Volume{{ + Name: "home", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: fmt.Sprintf("/tmp/%v", "home"), + }, + }, + }}, + }, + }, + VolumeClaimTemplates: []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "datadir"}, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: *resource.NewQuantity(1, resource.BinarySI), + }, + }, + }, + }, + }, + ServiceName: "governingsvc", + }, + } +} + +func TestGetPetSetMaxUnavailable(t *testing.T) { + testCases := []struct { + maxUnavailable *intstr.IntOrString + replicaCount int + expectedMaxUnavailable int + }{ + // it wouldn't hurt to also test 0 and 0%, even if they should have been forbidden by API validation. + {maxUnavailable: nil, replicaCount: 10, expectedMaxUnavailable: 1}, + {maxUnavailable: ptr.To(intstr.FromInt32(3)), replicaCount: 10, expectedMaxUnavailable: 3}, + {maxUnavailable: ptr.To(intstr.FromInt32(3)), replicaCount: 0, expectedMaxUnavailable: 3}, + {maxUnavailable: ptr.To(intstr.FromInt32(0)), replicaCount: 0, expectedMaxUnavailable: 1}, + {maxUnavailable: ptr.To(intstr.FromString("10%")), replicaCount: 25, expectedMaxUnavailable: 2}, + {maxUnavailable: ptr.To(intstr.FromString("100%")), replicaCount: 5, expectedMaxUnavailable: 5}, + {maxUnavailable: ptr.To(intstr.FromString("50%")), replicaCount: 5, expectedMaxUnavailable: 2}, + {maxUnavailable: ptr.To(intstr.FromString("10%")), replicaCount: 5, expectedMaxUnavailable: 1}, + {maxUnavailable: ptr.To(intstr.FromString("1%")), replicaCount: 0, expectedMaxUnavailable: 1}, + {maxUnavailable: ptr.To(intstr.FromString("0%")), replicaCount: 0, expectedMaxUnavailable: 1}, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { + gotMaxUnavailable, err := getPetSetMaxUnavailable(tc.maxUnavailable, tc.replicaCount) + if err != nil { + t.Fatal(err) + } + if gotMaxUnavailable != tc.expectedMaxUnavailable { + t.Errorf("Expected maxUnavailable %v, got pods %v", tc.expectedMaxUnavailable, gotMaxUnavailable) + } + }) + } +} diff --git a/pkg/controller/placement_test.go b/pkg/controller/placement_test.go new file mode 100644 index 00000000..3f6b1c3e --- /dev/null +++ b/pkg/controller/placement_test.go @@ -0,0 +1,497 @@ +/* +Copyright AppsCode Inc. and Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + api "kubeops.dev/petset/apis/apps/v1" + + "github.com/google/go-cmp/cmp" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func placementTestPetSet(replicas int32) *api.PetSet { + return &api.PetSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, + Spec: api.PetSetSpec{ + Replicas: &replicas, + }, + } +} + +func placementTestTemplate(labels map[string]string) *api.PodTemplateSpec { + return &api.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "main", Image: "busybox"}}, + }, + } +} + +// --------------------------------------------------------------------------- +// Upsert helpers +// --------------------------------------------------------------------------- + +func TestUpsertTopologySpreadConstraint(t *testing.T) { + a := v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone, MaxSkew: 1} + b := v1.TopologySpreadConstraint{TopologyKey: v1.LabelHostname, MaxSkew: 2} + + // append into empty list + lst := UpsertTopologySpreadConstraint(nil, a) + if len(lst) != 1 || lst[0].TopologyKey != v1.LabelTopologyZone { + t.Fatalf("expected zone constraint appended, got %+v", lst) + } + + // append a different topology key + lst = UpsertTopologySpreadConstraint(lst, b) + if len(lst) != 2 { + t.Fatalf("expected 2 constraints, got %d: %+v", len(lst), lst) + } + + // upsert the existing zone key replaces it in place (no growth) + updated := v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone, MaxSkew: 5} + lst = UpsertTopologySpreadConstraint(lst, updated) + if len(lst) != 2 { + t.Fatalf("expected list to stay at 2 after replace, got %d", len(lst)) + } + for _, c := range lst { + if c.TopologyKey == v1.LabelTopologyZone && c.MaxSkew != 5 { + t.Errorf("expected zone MaxSkew updated to 5, got %d", c.MaxSkew) + } + } +} + +func TestUpsertWeightedPodAffinityTerm(t *testing.T) { + zone := v1.WeightedPodAffinityTerm{Weight: 10, PodAffinityTerm: v1.PodAffinityTerm{TopologyKey: v1.LabelTopologyZone}} + host := v1.WeightedPodAffinityTerm{Weight: 20, PodAffinityTerm: v1.PodAffinityTerm{TopologyKey: v1.LabelHostname}} + + lst := UpsertWeightedPodAffinityTerm(nil, zone) + lst = UpsertWeightedPodAffinityTerm(lst, host) + if len(lst) != 2 { + t.Fatalf("expected 2 weighted terms, got %d", len(lst)) + } + + // replace zone by topology key + lst = UpsertWeightedPodAffinityTerm(lst, v1.WeightedPodAffinityTerm{Weight: 99, PodAffinityTerm: v1.PodAffinityTerm{TopologyKey: v1.LabelTopologyZone}}) + if len(lst) != 2 { + t.Fatalf("expected list to stay at 2 after replace, got %d", len(lst)) + } + for _, term := range lst { + if term.PodAffinityTerm.TopologyKey == v1.LabelTopologyZone && term.Weight != 99 { + t.Errorf("expected zone weight updated to 99, got %d", term.Weight) + } + } +} + +func TestUpsertPodAffinityTerm(t *testing.T) { + zone := v1.PodAffinityTerm{TopologyKey: v1.LabelTopologyZone} + host := v1.PodAffinityTerm{TopologyKey: v1.LabelHostname} + + lst := UpsertPodAffinityTerm(nil, zone) + lst = UpsertPodAffinityTerm(lst, host) + if len(lst) != 2 { + t.Fatalf("expected 2 terms, got %d", len(lst)) + } + // replacing the same topology key does not grow the slice + lst = UpsertPodAffinityTerm(lst, zone) + if len(lst) != 2 { + t.Fatalf("expected list to stay at 2 after replace, got %d", len(lst)) + } +} + +func TestUpsertNodeSelectorRequirements(t *testing.T) { + a := v1.NodeSelectorRequirement{Key: "zone", Operator: v1.NodeSelectorOpIn, Values: []string{"a"}} + b := v1.NodeSelectorRequirement{Key: "rack", Operator: v1.NodeSelectorOpIn, Values: []string{"1"}} + + lst := UpsertNodeSelectorRequirements(nil, a) + lst = UpsertNodeSelectorRequirements(lst, b) + if len(lst) != 2 { + t.Fatalf("expected 2 requirements, got %d", len(lst)) + } + // replace existing key "zone" with new values + lst = UpsertNodeSelectorRequirements(lst, v1.NodeSelectorRequirement{Key: "zone", Operator: v1.NodeSelectorOpIn, Values: []string{"b", "c"}}) + if len(lst) != 2 { + t.Fatalf("expected list to stay at 2 after replace, got %d", len(lst)) + } + for _, req := range lst { + if req.Key == "zone" && !cmp.Equal(req.Values, []string{"b", "c"}) { + t.Errorf("expected zone values replaced, got %v", req.Values) + } + } +} + +// --------------------------------------------------------------------------- +// CalculateForPodPlacement: nil policy +// --------------------------------------------------------------------------- + +func TestCalculateForPodPlacement_NilPolicy(t *testing.T) { + tmpl := placementTestTemplate(map[string]string{"app": "test"}) + pInfo := NewPodInfo(placementTestPetSet(3), tmpl, nil, 0, &v1.PodList{}) + + got, err := CalculateForPodPlacement(&pInfo) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !cmp.Equal(got, tmpl.Spec) { + t.Errorf("expected pod spec unchanged for nil policy, diff:\n%s", cmp.Diff(tmpl.Spec, got)) + } + if got.Affinity != nil { + t.Errorf("expected no affinity injected for nil policy, got %+v", got.Affinity) + } +} + +// --------------------------------------------------------------------------- +// Spread constraints + pod anti-affinity +// --------------------------------------------------------------------------- + +func TestCalculateForPodPlacement_ZoneSpreadDoNotSchedule(t *testing.T) { + labels := map[string]string{"app": "test"} + pp := &api.PlacementPolicy{ + Spec: api.PlacementPolicySpec{ + ZoneSpreadConstraint: &api.ZoneSpreadConstraint{ + MaxSkew: 1, + WhenUnsatisfiable: v1.DoNotSchedule, + }, + }, + } + pInfo := NewPodInfo(placementTestPetSet(3), placementTestTemplate(labels), pp, 0, &v1.PodList{}) + + got, err := CalculateForPodPlacement(&pInfo) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(got.TopologySpreadConstraints) != 1 { + t.Fatalf("expected 1 topology spread constraint, got %d", len(got.TopologySpreadConstraints)) + } + tsc := got.TopologySpreadConstraints[0] + if tsc.TopologyKey != v1.LabelTopologyZone || tsc.MaxSkew != 1 || tsc.WhenUnsatisfiable != v1.DoNotSchedule { + t.Errorf("unexpected topology spread constraint: %+v", tsc) + } + if !cmp.Equal(tsc.LabelSelector.MatchLabels, labels) { + t.Errorf("expected label selector %v, got %v", labels, tsc.LabelSelector.MatchLabels) + } + + if got.Affinity == nil || got.Affinity.PodAntiAffinity == nil { + t.Fatalf("expected pod anti-affinity to be set") + } + req := got.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution + if len(req) != 1 || req[0].TopologyKey != v1.LabelTopologyZone { + t.Errorf("expected required zone anti-affinity, got %+v", req) + } + if len(got.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0 { + t.Errorf("did not expect preferred anti-affinity for DoNotSchedule") + } +} + +func TestCalculateForPodPlacement_NodeSpreadScheduleAnyway(t *testing.T) { + labels := map[string]string{"app": "test"} + pp := &api.PlacementPolicy{ + Spec: api.PlacementPolicySpec{ + NodeSpreadConstraint: &api.NodeSpreadConstraint{ + MaxSkew: 2, + WhenUnsatisfiable: v1.ScheduleAnyway, + }, + }, + } + pInfo := NewPodInfo(placementTestPetSet(3), placementTestTemplate(labels), pp, 0, &v1.PodList{}) + + got, err := CalculateForPodPlacement(&pInfo) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(got.TopologySpreadConstraints) != 1 || got.TopologySpreadConstraints[0].TopologyKey != v1.LabelHostname { + t.Fatalf("expected 1 hostname topology spread constraint, got %+v", got.TopologySpreadConstraints) + } + + if got.Affinity == nil || got.Affinity.PodAntiAffinity == nil { + t.Fatalf("expected pod anti-affinity to be set") + } + preferred := got.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution + if len(preferred) != 1 { + t.Fatalf("expected 1 preferred anti-affinity term, got %d", len(preferred)) + } + if preferred[0].Weight != 100 || preferred[0].PodAffinityTerm.TopologyKey != v1.LabelHostname { + t.Errorf("unexpected preferred term: %+v", preferred[0]) + } + if len(got.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 { + t.Errorf("did not expect required anti-affinity for ScheduleAnyway") + } +} + +// --------------------------------------------------------------------------- +// Node affinity from placement +// --------------------------------------------------------------------------- + +func TestCalculateForPodPlacement_NodeAffinityDoNotSchedule(t *testing.T) { + labels := map[string]string{"app": "test"} + pp := &api.PlacementPolicy{ + Spec: api.PlacementPolicySpec{ + Affinity: &api.Affinity{ + NodeAffinity: []api.NodeAffinityRule{ + { + TopologyKey: "topology.kubernetes.io/zone", + WhenUnsatisfiable: v1.DoNotSchedule, + Domains: []api.TopologyDomain{ + {Values: []string{"zone-a"}, Replicas: ""}, // unlimited + }, + }, + }, + }, + }, + } + pInfo := NewPodInfo(placementTestPetSet(3), placementTestTemplate(labels), pp, 0, &v1.PodList{}) + + got, err := CalculateForPodPlacement(&pInfo) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Affinity == nil || got.Affinity.NodeAffinity == nil || got.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + t.Fatalf("expected required node affinity to be set") + } + terms := got.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + if len(terms) != 1 || len(terms[0].MatchExpressions) != 1 { + t.Fatalf("expected single required node selector term, got %+v", terms) + } + req := terms[0].MatchExpressions[0] + if req.Key != "topology.kubernetes.io/zone" || req.Operator != v1.NodeSelectorOpIn || !cmp.Equal(req.Values, []string{"zone-a"}) { + t.Errorf("unexpected node selector requirement: %+v", req) + } +} + +func TestCalculateForPodPlacement_NodeAffinityScheduleAnyway(t *testing.T) { + labels := map[string]string{"app": "test"} + pp := &api.PlacementPolicy{ + Spec: api.PlacementPolicySpec{ + Affinity: &api.Affinity{ + NodeAffinity: []api.NodeAffinityRule{ + { + TopologyKey: "topology.kubernetes.io/zone", + WhenUnsatisfiable: v1.ScheduleAnyway, + Weight: 50, + Domains: []api.TopologyDomain{ + {Values: []string{"zone-a"}, Replicas: ""}, + }, + }, + }, + }, + }, + } + pInfo := NewPodInfo(placementTestPetSet(3), placementTestTemplate(labels), pp, 0, &v1.PodList{}) + + got, err := CalculateForPodPlacement(&pInfo) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Affinity == nil || got.Affinity.NodeAffinity == nil { + t.Fatalf("expected node affinity to be set") + } + if got.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + t.Errorf("did not expect required node affinity for ScheduleAnyway") + } + preferred := got.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution + if len(preferred) != 1 || preferred[0].Weight != 50 { + t.Fatalf("expected one preferred scheduling term with weight 50, got %+v", preferred) + } +} + +// --------------------------------------------------------------------------- +// getAppropriateDomainIndex +// --------------------------------------------------------------------------- + +func podWithRequiredZone(zone string) v1.Pod { + return v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: "topology.kubernetes.io/zone", Operator: v1.NodeSelectorOpIn, Values: []string{zone}}, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestGetAppropriateDomainIndex(t *testing.T) { + rule := api.NodeAffinityRule{ + TopologyKey: "topology.kubernetes.io/zone", + WhenUnsatisfiable: v1.DoNotSchedule, + Domains: []api.TopologyDomain{ + {Values: []string{"zone-a"}, Replicas: "2"}, + {Values: []string{"zone-b"}, Replicas: "2"}, + }, + } + + tests := []struct { + name string + pods []v1.Pod + wantIndex int + wantErr bool + }{ + { + name: "empty pod list picks first domain", + pods: nil, + wantIndex: 0, + }, + { + name: "first domain partially filled still picks first", + pods: []v1.Pod{podWithRequiredZone("zone-a")}, + wantIndex: 0, + }, + { + name: "first domain full rolls over to second", + pods: []v1.Pod{podWithRequiredZone("zone-a"), podWithRequiredZone("zone-a")}, + wantIndex: 1, + }, + { + name: "all domains full returns error", + pods: []v1.Pod{ + podWithRequiredZone("zone-a"), podWithRequiredZone("zone-a"), + podWithRequiredZone("zone-b"), podWithRequiredZone("zone-b"), + }, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pInfo := NewPodInfo(placementTestPetSet(4), placementTestTemplate(map[string]string{"app": "test"}), + &api.PlacementPolicy{ObjectMeta: metav1.ObjectMeta{Name: "pp"}}, 0, + &v1.PodList{Items: tc.pods}) + if err := preCalc(&pInfo); err != nil { + t.Fatalf("preCalc failed: %v", err) + } + idx, err := getAppropriateDomainIndex(rule, pInfo) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got index %d", idx) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if idx != tc.wantIndex { + t.Errorf("got domain index %d, want %d", idx, tc.wantIndex) + } + }) + } +} + +func TestGetAppropriateDomainIndex_UnlimitedDomain(t *testing.T) { + rule := api.NodeAffinityRule{ + TopologyKey: "topology.kubernetes.io/zone", + WhenUnsatisfiable: v1.DoNotSchedule, + Domains: []api.TopologyDomain{ + {Values: []string{"zone-a"}, Replicas: "1"}, + {Values: []string{"zone-b"}, Replicas: ""}, // unlimited + }, + } + // zone-a is full, but zone-b is unlimited so it should always be selectable. + pInfo := NewPodInfo(placementTestPetSet(10), placementTestTemplate(map[string]string{"app": "test"}), + &api.PlacementPolicy{ObjectMeta: metav1.ObjectMeta{Name: "pp"}}, 0, + &v1.PodList{Items: []v1.Pod{ + podWithRequiredZone("zone-a"), + podWithRequiredZone("zone-b"), podWithRequiredZone("zone-b"), + }}) + if err := preCalc(&pInfo); err != nil { + t.Fatalf("preCalc failed: %v", err) + } + idx, err := getAppropriateDomainIndex(rule, pInfo) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if idx != 1 { + t.Errorf("expected unlimited zone-b (index 1), got %d", idx) + } +} + +// --------------------------------------------------------------------------- +// evaluateCEL +// --------------------------------------------------------------------------- + +func TestEvaluateCEL(t *testing.T) { + pInfo := NewPodInfo(placementTestPetSet(7), placementTestTemplate(map[string]string{"app": "test"}), nil, 0, &v1.PodList{}) + if err := preCalc(&pInfo); err != nil { + t.Fatalf("preCalc failed: %v", err) + } + + tests := []struct { + name string + rule string + want int64 + wantErr bool + }{ + {name: "empty string means unlimited", rule: "", want: -1}, + {name: "plain integer", rule: "5", want: 5}, + {name: "negative integer", rule: "-3", want: -3}, + {name: "cel reads replicas", rule: "obj.spec.replicas", want: 7}, + {name: "cel arithmetic", rule: "obj.spec.replicas / 2", want: 3}, + {name: "cel returning string errors", rule: "obj.metadata.name", wantErr: true}, + {name: "invalid cel compile error", rule: "this is ((( not valid", wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := evaluateCEL(&pInfo, tc.rule) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for rule %q, got %d", tc.rule, got) + } + return + } + if err != nil { + t.Fatalf("unexpected error for rule %q: %v", tc.rule, err) + } + if got != tc.want { + t.Errorf("evaluateCEL(%q) = %d, want %d", tc.rule, got, tc.want) + } + }) + } +} + +func TestEvaluateCEL_ProgramCache(t *testing.T) { + pInfo := NewPodInfo(placementTestPetSet(4), placementTestTemplate(map[string]string{"app": "test"}), nil, 0, &v1.PodList{}) + if err := preCalc(&pInfo); err != nil { + t.Fatalf("preCalc failed: %v", err) + } + + const rule = "obj.spec.replicas" + for range 3 { + if _, err := evaluateCEL(&pInfo, rule); err != nil { + t.Fatalf("evaluateCEL failed: %v", err) + } + } + // Plain integers and the empty string short-circuit before compilation, + // so only the single CEL expression should be cached. + if _, err := evaluateCEL(&pInfo, "5"); err != nil { + t.Fatalf("evaluateCEL failed: %v", err) + } + if got := len(pInfo.programCache); got != 1 { + t.Errorf("expected 1 cached CEL program, got %d", got) + } +} diff --git a/pkg/controller/tests/pet_pod_control_test.go b/pkg/controller/tests/pet_pod_control_test.go deleted file mode 100644 index dc53b129..00000000 --- a/pkg/controller/tests/pet_pod_control_test.go +++ /dev/null @@ -1,865 +0,0 @@ -// /* -// Copyright 2016 The Kubernetes Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// */ -package petset - -// -//import ( -// "context" -// "errors" -// "fmt" -// "strings" -// "testing" -// "time" -// -// api "kubeops.dev/petset/apis/apps/v1" -// // _ "kubeops.dev/petset/pkg/apis/apps/install" -// // _ "kubeops.dev/petset/pkg/apis/core/install" -// "kubeops.dev/petset/pkg/features" -// -// apps "k8s.io/api/apps/v1" -// v1 "k8s.io/api/core/v1" -// apierrors "k8s.io/apimachinery/pkg/api/errors" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// "k8s.io/apimachinery/pkg/runtime" -// "k8s.io/apimachinery/pkg/types" -// utilfeature "k8s.io/apiserver/pkg/util/feature" -// "k8s.io/client-go/kubernetes/fake" -// corelisters "k8s.io/client-go/listers/core/v1" -// core "k8s.io/client-go/testing" -// "k8s.io/client-go/tools/cache" -// "k8s.io/client-go/tools/record" -// featuregatetesting "k8s.io/component-base/featuregate/testing" -// "k8s.io/klog/v2/ktesting" -//) -// -//func TestStatefulPodControlCreatesPods(t *testing.T) { -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) -// control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder) -// fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), action.GetResource().Resource) -// }) -// fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// create := action.(core.CreateAction) -// claimIndexer.Add(create.GetObject()) -// return true, create.GetObject(), nil -// }) -// fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { -// create := action.(core.CreateAction) -// return true, create.GetObject(), nil -// }) -// if err := control.CreateStatefulPod(context.TODO(), set, pod); err != nil { -// t.Errorf("StatefulPodControl failed to create Pod error: %s", err) -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 2 { -// t.Errorf("Expected 2 events for successful create found %d", eventCount) -// } -// for i := range events { -// if !strings.Contains(events[i], v1.EventTypeNormal) { -// t.Errorf("Found unexpected non-normal event %s", events[i]) -// } -// } -//} -// -//func TestStatefulPodControlCreatePodExists(t *testing.T) { -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// pvcs := getPersistentVolumeClaims(set, pod) -// pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// for k := range pvcs { -// pvc := pvcs[k] -// pvcIndexer.Add(&pvc) -// } -// pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) -// control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) -// fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// create := action.(core.CreateAction) -// return true, create.GetObject(), nil -// }) -// fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { -// return true, pod, apierrors.NewAlreadyExists(action.GetResource().GroupResource(), pod.Name) -// }) -// if err := control.CreateStatefulPod(context.TODO(), set, pod); !apierrors.IsAlreadyExists(err) { -// t.Errorf("Failed to create Pod error: %s", err) -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 0 { -// t.Errorf("Pod and PVC exist: got %d events, but want 0", eventCount) -// for i := range events { -// t.Log(events[i]) -// } -// } -//} -// -//func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) { -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) -// control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) -// fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// return true, nil, apierrors.NewInternalError(errors.New("API server down")) -// }) -// fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { -// create := action.(core.CreateAction) -// return true, create.GetObject(), nil -// }) -// if err := control.CreateStatefulPod(context.TODO(), set, pod); err == nil { -// t.Error("Failed to produce error on PVC creation failure") -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 2 { -// t.Errorf("PVC create failure: got %d events, but want 2", eventCount) -// } -// for i := range events { -// if !strings.Contains(events[i], v1.EventTypeWarning) { -// t.Errorf("Found unexpected non-warning event %s", events[i]) -// } -// } -//} -// -//func TestStatefulPodControlCreatePodPVCDeleting(t *testing.T) { -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// pvcs := getPersistentVolumeClaims(set, pod) -// pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// deleteTime := time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC) -// for k := range pvcs { -// pvc := pvcs[k] -// pvc.DeletionTimestamp = &metav1.Time{Time: deleteTime} -// pvcIndexer.Add(&pvc) -// } -// pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) -// control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) -// fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// create := action.(core.CreateAction) -// return true, create.GetObject(), nil -// }) -// fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { -// create := action.(core.CreateAction) -// return true, create.GetObject(), nil -// }) -// if err := control.CreateStatefulPod(context.TODO(), set, pod); err == nil { -// t.Error("Failed to produce error on deleting PVC") -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 1 { -// t.Errorf("Deleting PVC: got %d events, but want 1", eventCount) -// } -// for i := range events { -// if !strings.Contains(events[i], v1.EventTypeWarning) { -// t.Errorf("Found unexpected non-warning event %s", events[i]) -// } -// } -//} -// -//type fakeIndexer struct { -// cache.Indexer -// getError error -//} -// -//func (f *fakeIndexer) GetByKey(key string) (interface{}, bool, error) { -// return nil, false, f.getError -//} -// -//func TestStatefulPodControlCreatePodPvcGetFailure(t *testing.T) { -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// pvcIndexer := &fakeIndexer{getError: errors.New("API server down")} -// pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) -// control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) -// fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// return true, nil, apierrors.NewInternalError(errors.New("API server down")) -// }) -// fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { -// create := action.(core.CreateAction) -// return true, create.GetObject(), nil -// }) -// if err := control.CreateStatefulPod(context.TODO(), set, pod); err == nil { -// t.Error("Failed to produce error on PVC creation failure") -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 2 { -// t.Errorf("PVC create failure: got %d events, but want 2", eventCount) -// } -// for i := range events { -// if !strings.Contains(events[i], v1.EventTypeWarning) { -// t.Errorf("Found unexpected non-warning event: %s", events[i]) -// } -// } -//} -// -//func TestStatefulPodControlCreatePodFailed(t *testing.T) { -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) -// control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) -// fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// create := action.(core.CreateAction) -// return true, create.GetObject(), nil -// }) -// fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { -// return true, nil, apierrors.NewInternalError(errors.New("API server down")) -// }) -// if err := control.CreateStatefulPod(context.TODO(), set, pod); err == nil { -// t.Error("Failed to produce error on Pod creation failure") -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 2 { -// t.Errorf("Pod create failed: got %d events, but want 2", eventCount) -// } else if !strings.Contains(events[0], v1.EventTypeNormal) { -// t.Errorf("Found unexpected non-normal event %s", events[0]) -// } else if !strings.Contains(events[1], v1.EventTypeWarning) { -// t.Errorf("Found unexpected non-warning event %s", events[1]) -// } -//} -// -//func TestStatefulPodControlNoOpUpdate(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// claims := getPersistentVolumeClaims(set, pod) -// indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// for k := range claims { -// claim := claims[k] -// indexer.Add(&claim) -// } -// claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) -// control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder) -// fakeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) { -// t.Error("no-op update should not make any client invocation") -// return true, nil, apierrors.NewInternalError(errors.New("if we are here we have a problem")) -// }) -// if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { -// t.Errorf("Error returned on no-op update error: %s", err) -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 0 { -// t.Errorf("no-op update: got %d events, but want 0", eventCount) -// } -//} -// -//func TestStatefulPodControlUpdatesIdentity(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := fake.NewSimpleClientset(set, pod) -// indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) -// control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder) -// var updated *v1.Pod -// fakeClient.PrependReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// updated = update.GetObject().(*v1.Pod) -// return true, update.GetObject(), nil -// }) -// pod.Name = "goo-0" -// if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { -// t.Errorf("Successful update returned an error: %s", err) -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 1 { -// t.Errorf("Pod update successful:got %d events,but want 1", eventCount) -// } else if !strings.Contains(events[0], v1.EventTypeNormal) { -// t.Errorf("Found unexpected non-normal event %s", events[0]) -// } -// if !identityMatches(set, updated) { -// t.Error("Name update failed identity does not match") -// } -//} -// -//func TestStatefulPodControlUpdateIdentityFailure(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// gooPod := newPetSetPod(set, 0) -// gooPod.Name = "goo-0" -// podIndexer.Add(gooPod) -// podLister := corelisters.NewPodLister(podIndexer) -// claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) -// control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) -// fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { -// pod.Name = "goo-0" -// return true, nil, apierrors.NewInternalError(errors.New("API server down")) -// }) -// pod.Name = "goo-0" -// if err := control.UpdateStatefulPod(ctx, set, pod); err == nil { -// t.Error("Failed update does not generate an error") -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 1 { -// t.Errorf("Pod update failed: got %d events, but want 1", eventCount) -// } else if !strings.Contains(events[0], v1.EventTypeWarning) { -// t.Errorf("Found unexpected non-warning event %s", events[0]) -// } -// if identityMatches(set, pod) { -// t.Error("Failed update mutated Pod identity") -// } -//} -// -//func TestStatefulPodControlUpdatesPodStorage(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) -// control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) -// pvcs := getPersistentVolumeClaims(set, pod) -// volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) -// for i := range pod.Spec.Volumes { -// if _, contains := pvcs[pod.Spec.Volumes[i].Name]; !contains { -// volumes = append(volumes, pod.Spec.Volumes[i]) -// } -// } -// pod.Spec.Volumes = volumes -// fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// return true, update.GetObject(), nil -// }) -// fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// return true, update.GetObject(), nil -// }) -// var updated *v1.Pod -// fakeClient.PrependReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// updated = update.GetObject().(*v1.Pod) -// return true, update.GetObject(), nil -// }) -// if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { -// t.Errorf("Successful update returned an error: %s", err) -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 2 { -// t.Errorf("Pod storage update successful: got %d events, but want 2", eventCount) -// } -// for i := range events { -// if !strings.Contains(events[i], v1.EventTypeNormal) { -// t.Errorf("Found unexpected non-normal event %s", events[i]) -// } -// } -// if !storageMatches(set, updated) { -// t.Error("Name update failed identity does not match") -// } -//} -// -//func TestStatefulPodControlUpdatePodStorageFailure(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) -// control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) -// pvcs := getPersistentVolumeClaims(set, pod) -// volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) -// for i := range pod.Spec.Volumes { -// if _, contains := pvcs[pod.Spec.Volumes[i].Name]; !contains { -// volumes = append(volumes, pod.Spec.Volumes[i]) -// } -// } -// pod.Spec.Volumes = volumes -// fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// return true, update.GetObject(), nil -// }) -// fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// return true, nil, apierrors.NewInternalError(errors.New("API server down")) -// }) -// if err := control.UpdateStatefulPod(ctx, set, pod); err == nil { -// t.Error("Failed Pod storage update did not return an error") -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 2 { -// t.Errorf("Pod storage update failed: got %d events, but want 2", eventCount) -// } -// for i := range events { -// if !strings.Contains(events[i], v1.EventTypeWarning) { -// t.Errorf("Found unexpected non-normal event %s", events[i]) -// } -// } -//} -// -//func TestStatefulPodControlUpdatePodConflictSuccess(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// podLister := corelisters.NewPodLister(podIndexer) -// claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// claimLister := corelisters.NewPersistentVolumeClaimLister(podIndexer) -// gooPod := newPetSetPod(set, 0) -// gooPod.Labels[apps.StatefulSetPodNameLabel] = "goo-starts" -// podIndexer.Add(gooPod) -// claims := getPersistentVolumeClaims(set, gooPod) -// for k := range claims { -// claim := claims[k] -// claimIndexer.Add(&claim) -// } -// control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) -// conflict := false -// fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// if !conflict { -// conflict = true -// return true, update.GetObject(), apierrors.NewConflict(action.GetResource().GroupResource(), pod.Name, errors.New("conflict")) -// } -// return true, update.GetObject(), nil -// }) -// pod.Labels[apps.StatefulSetPodNameLabel] = "goo-0" -// if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { -// t.Errorf("Successful update returned an error: %s", err) -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 1 { -// t.Errorf("Pod update successful: got %d, but want 1", eventCount) -// } else if !strings.Contains(events[0], v1.EventTypeNormal) { -// t.Errorf("Found unexpected non-normal event %s", events[0]) -// } -// if !identityMatches(set, pod) { -// t.Error("Name update failed identity does not match") -// } -//} -// -//func TestStatefulPodControlDeletesStatefulPod(t *testing.T) { -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// control := NewStatefulPodControl(fakeClient, nil, nil, recorder) -// fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { -// return true, nil, nil -// }) -// if err := control.DeleteStatefulPod(set, pod); err != nil { -// t.Errorf("Error returned on successful delete: %s", err) -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 1 { -// t.Errorf("delete successful: got %d events, but want 1", eventCount) -// } else if !strings.Contains(events[0], v1.EventTypeNormal) { -// t.Errorf("Found unexpected non-normal event %s", events[0]) -// } -//} -// -//func TestStatefulPodControlDeleteFailure(t *testing.T) { -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// control := NewStatefulPodControl(fakeClient, nil, nil, recorder) -// fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { -// return true, nil, apierrors.NewInternalError(errors.New("API server down")) -// }) -// if err := control.DeleteStatefulPod(set, pod); err == nil { -// t.Error("Failed to return error on failed delete") -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 1 { -// t.Errorf("delete failed: got %d events, but want 1", eventCount) -// } else if !strings.Contains(events[0], v1.EventTypeWarning) { -// t.Errorf("Found unexpected non-warning event %s", events[0]) -// } -//} -// -//func TestStatefulPodControlClaimsMatchDeletionPolcy(t *testing.T) { -// // The claimOwnerMatchesSetAndPod is tested exhaustively in stateful_set_utils_test; this -// // test is for the wiring to the method tested there. -// _, ctx := ktesting.NewTestContext(t) -// fakeClient := &fake.Clientset{} -// indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// claims := getPersistentVolumeClaims(set, pod) -// for k := range claims { -// claim := claims[k] -// indexer.Add(&claim) -// } -// control := NewStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{}) -// set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// } -// if matches, err := control.ClaimsMatchRetentionPolicy(ctx, set, pod); err != nil { -// t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (retain): %v", err) -// } else if !matches { -// t.Error("Unexpected non-match for ClaimsMatchRetentionPolicy (retain)") -// } -// set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// } -// if matches, err := control.ClaimsMatchRetentionPolicy(ctx, set, pod); err != nil { -// t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (set deletion): %v", err) -// } else if matches { -// t.Error("Unexpected match for ClaimsMatchRetentionPolicy (set deletion)") -// } -//} -// -//func TestStatefulPodControlUpdatePodClaimForRetentionPolicy(t *testing.T) { -// // All the update conditions are tested exhaustively in stateful_set_utils_test. This -// // tests the wiring from the pod control to that method. -// testFn := func(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, true)() -// fakeClient := &fake.Clientset{} -// indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) -// fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// indexer.Update(update.GetObject()) -// return true, update.GetObject(), nil -// }) -// set := newPetSet(3) -// set.GetObjectMeta().SetUID("set-123") -// pod := newPetSetPod(set, 0) -// claims := getPersistentVolumeClaims(set, pod) -// for k := range claims { -// claim := claims[k] -// indexer.Add(&claim) -// } -// control := NewStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{}) -// set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// } -// if err := control.UpdatePodClaimForRetentionPolicy(ctx, set, pod); err != nil { -// t.Errorf("Unexpected error for UpdatePodClaimForRetentionPolicy (retain): %v", err) -// } -// expectRef := features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) -// for k := range claims { -// claim, err := claimLister.PersistentVolumeClaims(claims[k].Namespace).Get(claims[k].Name) -// if err != nil { -// t.Errorf("Unexpected error getting Claim %s/%s: %v", claim.Namespace, claim.Name, err) -// } -// if hasOwnerRef(claim, set) != expectRef { -// t.Errorf("Claim %s/%s bad set owner ref", claim.Namespace, claim.Name) -// } -// } -// } -// t.Run("PetSetAutoDeletePVCEnabled", func(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, true)() -// testFn(t) -// }) -// t.Run("PetSetAutoDeletePVCDisabled", func(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, false)() -// testFn(t) -// }) -//} -// -//func TestPodClaimIsStale(t *testing.T) { -// const missing = "missing" -// const exists = "exists" -// const stale = "stale" -// const withRef = "with-ref" -// testCases := []struct { -// name string -// claimStates []string -// expected bool -// skipPodUID bool -// }{ -// { -// name: "all missing", -// claimStates: []string{missing, missing}, -// expected: false, -// }, -// { -// name: "no claims", -// claimStates: []string{}, -// expected: false, -// }, -// { -// name: "exists", -// claimStates: []string{missing, exists}, -// expected: false, -// }, -// { -// name: "all refs", -// claimStates: []string{withRef, withRef}, -// expected: false, -// }, -// { -// name: "stale & exists", -// claimStates: []string{stale, exists}, -// expected: true, -// }, -// { -// name: "stale & missing", -// claimStates: []string{stale, missing}, -// expected: true, -// }, -// { -// name: "withRef & stale", -// claimStates: []string{withRef, stale}, -// expected: true, -// }, -// { -// name: "withRef, no UID", -// claimStates: []string{withRef}, -// skipPodUID: true, -// expected: true, -// }, -// } -// for _, tc := range testCases { -// set := api.PetSet{} -// set.Name = "set" -// set.Namespace = "default" -// set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// } -// set.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"key": "value"}} -// claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// for i, claimState := range tc.claimStates { -// claim := v1.PersistentVolumeClaim{} -// claim.Name = fmt.Sprintf("claim-%d", i) -// set.Spec.VolumeClaimTemplates = append(set.Spec.VolumeClaimTemplates, claim) -// claim.Name = fmt.Sprintf("%s-set-3", claim.Name) -// claim.Namespace = set.Namespace -// switch claimState { -// case missing: -// // Do nothing, the claim shouldn't exist. -// case exists: -// claimIndexer.Add(&claim) -// case stale: -// claim.SetOwnerReferences([]metav1.OwnerReference{ -// {Name: "set-3", UID: types.UID("stale")}, -// }) -// claimIndexer.Add(&claim) -// case withRef: -// claim.SetOwnerReferences([]metav1.OwnerReference{ -// {Name: "set-3", UID: types.UID("123")}, -// }) -// claimIndexer.Add(&claim) -// } -// } -// pod := v1.Pod{} -// pod.Name = "set-3" -// if !tc.skipPodUID { -// pod.SetUID("123") -// } -// claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) -// control := NewStatefulPodControl(&fake.Clientset{}, nil, claimLister, &noopRecorder{}) -// expected := tc.expected -// // Note that the error isn't / can't be tested. -// if stale, _ := control.PodClaimIsStale(&set, &pod); stale != expected { -// t.Errorf("unexpected stale for %s", tc.name) -// } -// } -//} -// -//func TestStatefulPodControlRetainDeletionPolicyUpdate(t *testing.T) { -// testFn := func(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(1) -// set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// } -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// podLister := corelisters.NewPodLister(podIndexer) -// claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) -// podIndexer.Add(pod) -// claims := getPersistentVolumeClaims(set, pod) -// if len(claims) < 1 { -// t.Errorf("Unexpected missing PVCs") -// } -// for k := range claims { -// claim := claims[k] -// setOwnerRef(&claim, set, &set.TypeMeta) // This ownerRef should be removed in the update. -// claimIndexer.Add(&claim) -// } -// control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) -// if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { -// t.Errorf("Successful update returned an error: %s", err) -// } -// for k := range claims { -// claim := claims[k] -// if hasOwnerRef(&claim, set) { -// t.Errorf("ownerRef not removed: %s/%s", claim.Namespace, claim.Name) -// } -// } -// events := collectEvents(recorder.Events) -// if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) { -// if eventCount := len(events); eventCount != 1 { -// t.Errorf("delete failed: got %d events, but want 1", eventCount) -// } -// } else { -// if len(events) != 0 { -// t.Errorf("delete failed: expected no events, but got %v", events) -// } -// } -// } -// t.Run("PetSetAutoDeletePVCEnabled", func(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, true)() -// testFn(t) -// }) -// t.Run("PetSetAutoDeletePVCDisabled", func(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, false)() -// testFn(t) -// }) -//} -// -//func TestStatefulPodControlRetentionPolicyUpdate(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// // Only applicable when the feature gate is on; the off case is tested in TestStatefulPodControlRetainRetentionPolicyUpdate. -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, true)() -// -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(1) -// set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// } -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// podIndexer.Add(pod) -// claims := getPersistentVolumeClaims(set, pod) -// if len(claims) != 1 { -// t.Errorf("Unexpected or missing PVCs") -// } -// var claim v1.PersistentVolumeClaim -// for k := range claims { -// claim = claims[k] -// claimIndexer.Add(&claim) -// } -// fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// claimIndexer.Update(update.GetObject()) -// return true, update.GetObject(), nil -// }) -// podLister := corelisters.NewPodLister(podIndexer) -// claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) -// control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) -// if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { -// t.Errorf("Successful update returned an error: %s", err) -// } -// updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) -// if err != nil { -// t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err) -// } -// if !hasOwnerRef(updatedClaim, set) { -// t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name) -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 1 { -// t.Errorf("update failed: got %d events, but want 1", eventCount) -// } -//} -// -//func TestStatefulPodControlRetentionPolicyUpdateMissingClaims(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// // Only applicable when the feature gate is on. -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, true)() -// -// recorder := record.NewFakeRecorder(10) -// set := newPetSet(1) -// set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// } -// pod := newPetSetPod(set, 0) -// fakeClient := &fake.Clientset{} -// podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// podLister := corelisters.NewPodLister(podIndexer) -// claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) -// podIndexer.Add(pod) -// fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// claimIndexer.Update(update.GetObject()) -// return true, update.GetObject(), nil -// }) -// control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) -// if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { -// t.Error("Unexpected error on pod update when PVCs are missing") -// } -// claims := getPersistentVolumeClaims(set, pod) -// if len(claims) != 1 { -// t.Errorf("Unexpected or missing PVCs") -// } -// var claim v1.PersistentVolumeClaim -// for k := range claims { -// claim = claims[k] -// claimIndexer.Add(&claim) -// } -// -// if err := control.UpdateStatefulPod(ctx, set, pod); err != nil { -// t.Errorf("Expected update to succeed, saw error %v", err) -// } -// updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) -// if err != nil { -// t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err) -// } -// if !hasOwnerRef(updatedClaim, set) { -// t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name) -// } -// events := collectEvents(recorder.Events) -// if eventCount := len(events); eventCount != 1 { -// t.Errorf("update failed: got %d events, but want 2", eventCount) -// } -// if !strings.Contains(events[0], "SuccessfulUpdate") { -// t.Errorf("expected first event to be a successful update: %s", events[1]) -// } -//} -// -//func collectEvents(source <-chan string) []string { -// done := false -// events := make([]string, 0) -// for !done { -// select { -// case event := <-source: -// events = append(events, event) -// default: -// done = true -// } -// } -// return events -//} diff --git a/pkg/controller/tests/pet_set_control_test.go b/pkg/controller/tests/pet_set_control_test.go deleted file mode 100644 index c8e88947..00000000 --- a/pkg/controller/tests/pet_set_control_test.go +++ /dev/null @@ -1,3391 +0,0 @@ -// /* -// Copyright 2016 The Kubernetes Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// */ -package petset - -// -//import ( -// "context" -// "errors" -// "fmt" -// "math/rand" -// "reflect" -// "runtime" -// "sort" -// "strconv" -// "strings" -// "sync" -// "testing" -// "time" -// -// api "kubeops.dev/petset/apis/apps/v1" -// "kubeops.dev/petset/client/clientset/versioned" -// apifake "kubeops.dev/petset/client/clientset/versioned/fake" -// apiinformers "kubeops.dev/petset/client/informers/externalversions" -// stsinformers "kubeops.dev/petset/client/informers/externalversions/apps/v1" -// apilisters "kubeops.dev/petset/client/listers/apps/v1" -// podutil "kubeops.dev/petset/pkg/api/v1/pod" -// "kubeops.dev/petset/pkg/controller" -// "kubeops.dev/petset/pkg/controller/history" -// "kubeops.dev/petset/pkg/features" -// -// apps "k8s.io/api/apps/v1" -// v1 "k8s.io/api/core/v1" -// apierrors "k8s.io/apimachinery/pkg/api/errors" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// "k8s.io/apimachinery/pkg/labels" -// "k8s.io/apimachinery/pkg/types" -// utilerrors "k8s.io/apimachinery/pkg/util/errors" -// "k8s.io/apimachinery/pkg/util/intstr" -// utilfeature "k8s.io/apiserver/pkg/util/feature" -// "k8s.io/client-go/informers" -// clientset "k8s.io/client-go/kubernetes" -// "k8s.io/client-go/kubernetes/fake" -// corelisters "k8s.io/client-go/listers/core/v1" -// "k8s.io/client-go/tools/cache" -// featuregatetesting "k8s.io/component-base/featuregate/testing" -//) -// -//type invariantFunc func(set *api.PetSet, om *fakeObjectManager) error -// -//func setupController(client clientset.Interface, apiclient versioned.Interface) (*fakeObjectManager, *fakeStatefulSetStatusUpdater, PetSetControlInterface) { -// informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) -// apiinformerFactory := apiinformers.NewSharedInformerFactory(apiclient, controller.NoResyncPeriodFunc()) -// om := newFakeObjectManager(informerFactory, apiinformerFactory) -// spc := NewStatefulPodControlFromManager(om, &noopRecorder{}) -// ssu := newFakeStatefulSetStatusUpdater(apiinformerFactory.Apps().V1().PetSets()) -// recorder := &noopRecorder{} -// ssc := NewDefaultPetSetControl(spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder) -// -// // The informer is not started. The tests here manipulate the local cache (indexers) directly, and there is no waiting -// // for client state to sync. In fact, because the client is not updated during tests, informer updates will break tests -// // by unexpectedly deleting objects. -// // -// // TODO: It might be better to rewrite all these tests manipulate the client an explicitly sync to ensure consistent -// // state, or to create a fake client that does not use a local cache. -// -// // The client is passed initial sets, so we have to put them in the local setsIndexer cache. -// if sets, err := client.AppsV1().StatefulSets("").List(context.TODO(), metav1.ListOptions{}); err != nil { -// panic(err) -// } else { -// for _, set := range sets.Items { -// if err := om.setsIndexer.Update(&set); err != nil { -// panic(err) -// } -// } -// } -// -// return om, ssu, ssc -//} -// -//func burst(set *api.PetSet) *api.PetSet { -// set.Spec.PodManagementPolicy = apps.ParallelPodManagement -// return set -//} -// -//func setMinReadySeconds(set *api.PetSet, minReadySeconds int32) *api.PetSet { -// set.Spec.MinReadySeconds = minReadySeconds -// return set -//} -// -//func runTestOverPVCRetentionPolicies(t *testing.T, testName string, testFn func(*testing.T, *apps.StatefulSetPersistentVolumeClaimRetentionPolicy)) { -// subtestName := "PetSetAutoDeletePVCDisabled" -// if testName != "" { -// subtestName = fmt.Sprintf("%s/%s", testName, subtestName) -// } -// t.Run(subtestName, func(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, false)() -// testFn(t, &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// }) -// }) -// -// for _, policy := range []*apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// { -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// }, -// { -// WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// }, -// { -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// }, -// { -// WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// }, -// // tests the case when no policy is set. -// nil, -// } { -// subtestName := pvcDeletePolicyString(policy) + "/PetSetAutoDeletePVCEnabled" -// if testName != "" { -// subtestName = fmt.Sprintf("%s/%s", testName, subtestName) -// } -// t.Run(subtestName, func(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, true)() -// testFn(t, policy) -// }) -// } -//} -// -//func pvcDeletePolicyString(policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) string { -// if policy == nil { -// return "nullPolicy" -// } -// const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType -// const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType -// switch { -// case policy.WhenScaled == retain && policy.WhenDeleted == retain: -// return "Retain" -// case policy.WhenScaled == retain && policy.WhenDeleted == delete: -// return "SetDeleteOnly" -// case policy.WhenScaled == delete && policy.WhenDeleted == retain: -// return "ScaleDownOnly" -// case policy.WhenScaled == delete && policy.WhenDeleted == delete: -// return "Delete" -// } -// return "invalid" -//} -// -//func TestPetSetControl(t *testing.T) { -// simpleSetFn := func() *api.PetSet { return newPetSet(3) } -// largeSetFn := func() *api.PetSet { return newPetSet(5) } -// -// testCases := []struct { -// fn func(*testing.T, *api.PetSet, invariantFunc) -// obj func() *api.PetSet -// }{ -// {CreatesPods, simpleSetFn}, -// {ScalesUp, simpleSetFn}, -// {ScalesDown, simpleSetFn}, -// {ReplacesPods, largeSetFn}, -// {RecreatesFailedPod, simpleSetFn}, -// {RecreatesSucceededPod, simpleSetFn}, -// {CreatePodFailure, simpleSetFn}, -// {UpdatePodFailure, simpleSetFn}, -// {UpdateSetStatusFailure, simpleSetFn}, -// {PodRecreateDeleteFailure, simpleSetFn}, -// {NewRevisionDeletePodFailure, simpleSetFn}, -// {RecreatesPVCForPendingPod, simpleSetFn}, -// } -// -// for _, testCase := range testCases { -// fnName := runtime.FuncForPC(reflect.ValueOf(testCase.fn).Pointer()).Name() -// if i := strings.LastIndex(fnName, "."); i != -1 { -// fnName = fnName[i+1:] -// } -// testObj := testCase.obj -// testFn := testCase.fn -// runTestOverPVCRetentionPolicies( -// t, -// fmt.Sprintf("%s/Monotonic", fnName), -// func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { -// set := testObj() -// set.Spec.PersistentVolumeClaimRetentionPolicy = policy -// testFn(t, set, assertMonotonicInvariants) -// }, -// ) -// runTestOverPVCRetentionPolicies( -// t, -// fmt.Sprintf("%s/Burst", fnName), -// func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { -// set := burst(testObj()) -// set.Spec.PersistentVolumeClaimRetentionPolicy = policy -// testFn(t, set, assertBurstInvariants) -// }, -// ) -// } -//} -// -//func CreatesPods(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 3 { -// t.Error("Failed to scale petset to 3 replicas") -// } -// if set.Status.ReadyReplicas != 3 { -// t.Error("Failed to set ReadyReplicas correctly") -// } -// if set.Status.UpdatedReplicas != 3 { -// t.Error("Failed to set UpdatedReplicas correctly") -// } -// // Check all pods have correct pod index label. -// if features.DefaultFeatureGate.Enabled(features.PodIndexLabel) { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if len(pods) != 3 { -// t.Errorf("Expected 3 pods, got %d", len(pods)) -// } -// for _, pod := range pods { -// podIndexFromLabel, exists := pod.Labels[apps.PodIndexLabel] -// if !exists { -// t.Errorf("Missing pod index label: %s", apps.PodIndexLabel) -// continue -// } -// podIndexFromName := strconv.Itoa(getOrdinal(pod)) -// if podIndexFromLabel != podIndexFromName { -// t.Errorf("Pod index label value (%s) does not match pod index in pod name (%s)", podIndexFromLabel, podIndexFromName) -// } -// } -// } -//} -// -//func ScalesUp(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// *set.Spec.Replicas = 4 -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to scale PetSet : %s", err) -// } -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 4 { -// t.Error("Failed to scale petset to 4 replicas") -// } -// if set.Status.ReadyReplicas != 4 { -// t.Error("Failed to set readyReplicas correctly") -// } -// if set.Status.UpdatedReplicas != 4 { -// t.Error("Failed to set updatedReplicas correctly") -// } -//} -// -//func ScalesDown(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// *set.Spec.Replicas = 0 -// if err := scaleDownPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to scale PetSet : %s", err) -// } -// -// // Check updated set. -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 0 { -// t.Error("Failed to scale petset to 0 replicas") -// } -// if set.Status.ReadyReplicas != 0 { -// t.Error("Failed to set readyReplicas correctly") -// } -// if set.Status.UpdatedReplicas != 0 { -// t.Error("Failed to set updatedReplicas correctly") -// } -//} -// -//func ReplacesPods(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 5 { -// t.Error("Failed to scale petset to 5 replicas") -// } -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// for _, pod := range pods { -// podClaims := getPersistentVolumeClaims(set, pod) -// for _, claim := range claims { -// if _, found := podClaims[claim.Name]; found { -// if hasOwnerRef(claim, pod) { -// t.Errorf("Unexpected ownerRef on %s", claim.Name) -// } -// } -// } -// } -// sort.Sort(ascendingOrdinal(pods)) -// om.podsIndexer.Delete(pods[0]) -// om.podsIndexer.Delete(pods[2]) -// om.podsIndexer.Delete(pods[4]) -// for i := 0; i < 5; i += 2 { -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Errorf("Failed to update PetSet : %s", err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if pods, err = om.setPodRunning(set, i); err != nil { -// t.Error(err) -// } -// if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Errorf("Failed to update PetSet : %s", err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if _, err = om.setPodReady(set, i); err != nil { -// t.Error(err) -// } -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Errorf("Failed to update PetSet : %s", err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if e, a := int32(5), set.Status.Replicas; e != a { -// t.Errorf("Expected to scale to %d, got %d", e, a) -// } -//} -// -//func recreatesPod(t *testing.T, set *api.PetSet, invariants invariantFunc, phase v1.PodPhase) { -// client := fake.NewSimpleClientset() -// apiclient := apifake.NewSimpleClientset() -// om, _, ssc := setupController(client, apiclient) -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Errorf("Error updating PetSet %s", err) -// } -// if err := invariants(set, om); err != nil { -// t.Error(err) -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// pods[0].Status.Phase = phase -// om.podsIndexer.Update(pods[0]) -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Errorf("Error updating PetSet %s", err) -// } -// if err := invariants(set, om); err != nil { -// t.Error(err) -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if isCreated(pods[0]) { -// t.Error("PetSet did not recreate failed Pod") -// } -//} -// -//func RecreatesFailedPod(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// recreatesPod(t, set, invariants, v1.PodFailed) -//} -// -//func RecreatesSucceededPod(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// recreatesPod(t, set, invariants, v1.PodSucceeded) -//} -// -//func CreatePodFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// om.SetCreateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) -// -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { -// t.Errorf("PetSetControl did not return InternalError found %s", err) -// } -// // Update so set.Status is set for the next scaleUpPetSetControl call. -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 3 { -// t.Error("Failed to scale PetSet to 3 replicas") -// } -// if set.Status.ReadyReplicas != 3 { -// t.Error("Failed to set readyReplicas correctly") -// } -// if set.Status.UpdatedReplicas != 3 { -// t.Error("Failed to updatedReplicas correctly") -// } -//} -// -//func UpdatePodFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// om.SetUpdateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) -// -// // have to have 1 successful loop first -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Fatalf("Unexpected error: %v", err) -// } -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 3 { -// t.Error("Failed to scale PetSet to 3 replicas") -// } -// if set.Status.ReadyReplicas != 3 { -// t.Error("Failed to set readyReplicas correctly") -// } -// if set.Status.UpdatedReplicas != 3 { -// t.Error("Failed to set updatedReplicas correctly") -// } -// -// // now mutate a pod's identity -// pods, err := om.podsLister.List(labels.Everything()) -// if err != nil { -// t.Fatalf("Error listing pods: %v", err) -// } -// if len(pods) != 3 { -// t.Fatalf("Expected 3 pods, got %d", len(pods)) -// } -// sort.Sort(ascendingOrdinal(pods)) -// pods[0].Name = "goo-0" -// om.podsIndexer.Update(pods[0]) -// -// // now it should fail -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil && isOrHasInternalError(err) { -// t.Errorf("PetSetControl did not return InternalError found %s", err) -// } -//} -// -//func UpdateSetStatusFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, ssu, ssc := setupController(client, apiclient) -// ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) -// -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { -// t.Errorf("PetSetControl did not return InternalError found %s", err) -// } -// // Update so set.Status is set for the next scaleUpPetSetControl call. -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 3 { -// t.Error("Failed to scale PetSet to 3 replicas") -// } -// if set.Status.ReadyReplicas != 3 { -// t.Error("Failed to set readyReplicas to 3") -// } -// if set.Status.UpdatedReplicas != 3 { -// t.Error("Failed to set updatedReplicas to 3") -// } -//} -// -//func PodRecreateDeleteFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Errorf("Error updating PetSet %s", err) -// } -// if err := invariants(set, om); err != nil { -// t.Error(err) -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// pods[0].Status.Phase = v1.PodFailed -// om.podsIndexer.Update(pods[0]) -// om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil && isOrHasInternalError(err) { -// t.Errorf("PetSet failed to %s", err) -// } -// if err := invariants(set, om); err != nil { -// t.Error(err) -// } -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Errorf("Error updating PetSet %s", err) -// } -// if err := invariants(set, om); err != nil { -// t.Error(err) -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if isCreated(pods[0]) { -// t.Error("PetSet did not recreate failed Pod") -// } -//} -// -//func NewRevisionDeletePodFailure(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 3 { -// t.Error("Failed to scale PetSet to 3 replicas") -// } -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// -// // trigger a new revision -// updateSet := set.DeepCopy() -// updateSet.Spec.Template.Spec.Containers[0].Image = "nginx-new" -// if err := om.setsIndexer.Update(updateSet); err != nil { -// t.Error("Failed to update PetSet") -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// -// // delete fails -// om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) -// _, err = ssc.UpdatePetSet(context.TODO(), set, pods) -// if err == nil { -// t.Error("Expected err in update PetSet when deleting a pod") -// } -// -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if err := invariants(set, om); err != nil { -// t.Error(err) -// } -// if set.Status.CurrentReplicas != 3 { -// t.Fatalf("Failed pod deletion should not update CurrentReplicas: want 3, got %d", set.Status.CurrentReplicas) -// } -// if set.Status.CurrentRevision == set.Status.UpdateRevision { -// t.Error("Failed to create new revision") -// } -// -// // delete works -// om.SetDeleteStatefulPodError(nil, 0) -// status, err := ssc.UpdatePetSet(context.TODO(), set, pods) -// if err != nil { -// t.Fatalf("Unexpected err in update PetSet: %v", err) -// } -// if status.CurrentReplicas != 2 { -// t.Fatalf("Pod deletion should update CurrentReplicas: want 2, got %d", status.CurrentReplicas) -// } -// if err := invariants(set, om); err != nil { -// t.Error(err) -// } -//} -// -//func emptyInvariants(set *api.PetSet, om *fakeObjectManager) error { -// return nil -//} -// -//func TestPetSetControlWithStartOrdinal(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetStartOrdinal, true)() -// -// simpleSetFn := func() *api.PetSet { -// statefulSet := newPetSet(3) -// statefulSet.Spec.Ordinals = &apps.StatefulSetOrdinals{Start: int32(2)} -// return statefulSet -// } -// -// testCases := []struct { -// fn func(*testing.T, *api.PetSet, invariantFunc) -// obj func() *api.PetSet -// }{ -// {CreatesPodsWithStartOrdinal, simpleSetFn}, -// } -// -// for _, testCase := range testCases { -// testObj := testCase.obj -// testFn := testCase.fn -// -// set := testObj() -// testFn(t, set, emptyInvariants) -// } -//} -// -//func CreatesPodsWithStartOrdinal(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 3 { -// t.Error("Failed to scale petset to 3 replicas") -// } -// if set.Status.ReadyReplicas != 3 { -// t.Error("Failed to set ReadyReplicas correctly") -// } -// if set.Status.UpdatedReplicas != 3 { -// t.Error("Failed to set UpdatedReplicas correctly") -// } -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// sort.Sort(ascendingOrdinal(pods)) -// for i, pod := range pods { -// expectedOrdinal := 2 + i -// actualPodOrdinal := getOrdinal(pod) -// if actualPodOrdinal != expectedOrdinal { -// t.Errorf("Expected pod ordinal %d. Got %d", expectedOrdinal, actualPodOrdinal) -// } -// } -//} -// -//func RecreatesPVCForPendingPod(t *testing.T, set *api.PetSet, invariants invariantFunc) { -// client := fake.NewSimpleClientset() -// apiclient := apifake.NewSimpleClientset() -// om, _, ssc := setupController(client, apiclient) -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Errorf("Error updating PetSet %s", err) -// } -// if err := invariants(set, om); err != nil { -// t.Error(err) -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// for _, claim := range getPersistentVolumeClaims(set, pods[0]) { -// om.claimsIndexer.Delete(&claim) -// } -// pods[0].Status.Phase = v1.PodPending -// om.podsIndexer.Update(pods[0]) -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Errorf("Error updating PetSet %s", err) -// } -// // invariants check if there any missing PVCs for the Pods -// if err := invariants(set, om); err != nil { -// t.Error(err) -// } -// _, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -//} -// -//func TestPetSetControlScaleDownDeleteError(t *testing.T) { -// runTestOverPVCRetentionPolicies( -// t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { -// set := newPetSet(3) -// set.Spec.PersistentVolumeClaimRetentionPolicy = policy -// invariants := assertMonotonicInvariants -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// *set.Spec.Replicas = 0 -// om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) -// if err := scaleDownPetSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { -// t.Errorf("PetSetControl failed to throw error on delete %s", err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if err := scaleDownPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn down PetSet %s", err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != 0 { -// t.Error("Failed to scale petset to 0 replicas") -// } -// if set.Status.ReadyReplicas != 0 { -// t.Error("Failed to set readyReplicas to 0") -// } -// if set.Status.UpdatedReplicas != 0 { -// t.Error("Failed to set updatedReplicas to 0") -// } -// }) -//} -// -//func TestPetSetControl_getSetRevisions(t *testing.T) { -// type testcase struct { -// name string -// existing []*apps.ControllerRevision -// set *api.PetSet -// expectedCount int -// expectedCurrent *apps.ControllerRevision -// expectedUpdate *apps.ControllerRevision -// err bool -// } -// -// testFn := func(test *testcase, t *testing.T) { -// client := fake.NewSimpleClientset() -// informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) -// apiclient := apifake.NewSimpleClientset() -// apiinformerFactory := apiinformers.NewSharedInformerFactory(apiclient, controller.NoResyncPeriodFunc()) -// spc := NewStatefulPodControlFromManager(newFakeObjectManager(informerFactory, apiinformerFactory), &noopRecorder{}) -// ssu := newFakeStatefulSetStatusUpdater(apiinformerFactory.Apps().V1().PetSets()) -// recorder := &noopRecorder{} -// ssc := defaultPetSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder} -// -// stop := make(chan struct{}) -// defer close(stop) -// informerFactory.Start(stop) -// cache.WaitForCacheSync( -// stop, -// informerFactory.Apps().V1().StatefulSets().Informer().HasSynced, -// informerFactory.Core().V1().Pods().Informer().HasSynced, -// informerFactory.Apps().V1().ControllerRevisions().Informer().HasSynced, -// ) -// test.set.Status.CollisionCount = new(int32) -// for i := range test.existing { -// ssc.controllerHistory.CreateControllerRevision(test.set, test.existing[i], test.set.Status.CollisionCount) -// } -// revisions, err := ssc.ListRevisions(test.set) -// if err != nil { -// t.Fatal(err) -// } -// current, update, _, err := ssc.getPetSetRevisions(test.set, revisions) -// if err != nil { -// t.Fatalf("error getting petset revisions:%v", err) -// } -// revisions, err = ssc.ListRevisions(test.set) -// if err != nil { -// t.Fatal(err) -// } -// if len(revisions) != test.expectedCount { -// t.Errorf("%s: want %d revisions got %d", test.name, test.expectedCount, len(revisions)) -// } -// if test.err && err == nil { -// t.Errorf("%s: expected error", test.name) -// } -// if !test.err && !history.EqualRevision(current, test.expectedCurrent) { -// t.Errorf("%s: for current want %v got %v", test.name, test.expectedCurrent, current) -// } -// if !test.err && !history.EqualRevision(update, test.expectedUpdate) { -// t.Errorf("%s: for update want %v got %v", test.name, test.expectedUpdate, update) -// } -// if !test.err && test.expectedCurrent != nil && current != nil && test.expectedCurrent.Revision != current.Revision { -// t.Errorf("%s: for current revision want %d got %d", test.name, test.expectedCurrent.Revision, current.Revision) -// } -// if !test.err && test.expectedUpdate != nil && update != nil && test.expectedUpdate.Revision != update.Revision { -// t.Errorf("%s: for update revision want %d got %d", test.name, test.expectedUpdate.Revision, update.Revision) -// } -// } -// -// updateRevision := func(cr *apps.ControllerRevision, revision int64) *apps.ControllerRevision { -// clone := cr.DeepCopy() -// clone.Revision = revision -// return clone -// } -// -// runTestOverPVCRetentionPolicies( -// t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { -// set := newPetSet(3) -// set.Spec.PersistentVolumeClaimRetentionPolicy = policy -// set.Status.CollisionCount = new(int32) -// rev0 := newRevisionOrDie(set, 1) -// set1 := set.DeepCopy() -// set1.Spec.Template.Spec.Containers[0].Image = "foo" -// set1.Status.CurrentRevision = rev0.Name -// set1.Status.CollisionCount = new(int32) -// rev1 := newRevisionOrDie(set1, 2) -// set2 := set1.DeepCopy() -// set2.Spec.Template.Labels["new"] = "label" -// set2.Status.CurrentRevision = rev0.Name -// set2.Status.CollisionCount = new(int32) -// rev2 := newRevisionOrDie(set2, 3) -// tests := []testcase{ -// { -// name: "creates initial revision", -// existing: nil, -// set: set, -// expectedCount: 1, -// expectedCurrent: rev0, -// expectedUpdate: rev0, -// err: false, -// }, -// { -// name: "creates revision on update", -// existing: []*apps.ControllerRevision{rev0}, -// set: set1, -// expectedCount: 2, -// expectedCurrent: rev0, -// expectedUpdate: rev1, -// err: false, -// }, -// { -// name: "must not recreate a new revision of same set", -// existing: []*apps.ControllerRevision{rev0, rev1}, -// set: set1, -// expectedCount: 2, -// expectedCurrent: rev0, -// expectedUpdate: rev1, -// err: false, -// }, -// { -// name: "must rollback to a previous revision", -// existing: []*apps.ControllerRevision{rev0, rev1, rev2}, -// set: set1, -// expectedCount: 3, -// expectedCurrent: rev0, -// expectedUpdate: updateRevision(rev1, 4), -// err: false, -// }, -// } -// for i := range tests { -// testFn(&tests[i], t) -// } -// }) -//} -// -//func setupPodManagementPolicy(podManagementPolicy apps.PodManagementPolicyType, set *api.PetSet) *api.PetSet { -// set.Spec.PodManagementPolicy = podManagementPolicy -// return set -//} -// -//func TestPetSetControlRollingUpdateWithMaxUnavailable(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MaxUnavailablePetSet, true)() -// -// simpleParallelVerificationFn := func( -// set *api.PetSet, -// spc *fakeObjectManager, -// ssc PetSetControlInterface, -// pods []*v1.Pod, -// totalPods int, -// selector labels.Selector, -// ) []*v1.Pod { -// // in burst mode, 2 pods got deleted, so 2 new pods will be created at the same time -// if len(pods) != totalPods { -// t.Fatalf("Expected create pods 4/5, got pods %v", len(pods)) -// } -// -// // if pod 4 ready, start to update pod 3, even though 5 is not ready -// spc.setPodRunning(set, 4) -// spc.setPodRunning(set, 5) -// originalPods, _ := spc.setPodReady(set, 4) -// sort.Sort(ascendingOrdinal(originalPods)) -// if _, err := ssc.UpdatePetSet(context.TODO(), set, originalPods); err != nil { -// t.Fatal(err) -// } -// pods, err := spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// sort.Sort(ascendingOrdinal(pods)) -// // pods 0, 1,2, 4,5 should be present(note 3 is missing) -// if !reflect.DeepEqual(pods, append(originalPods[:3], originalPods[4:]...)) { -// t.Fatalf("Expected pods %v, got pods %v", append(originalPods[:3], originalPods[4:]...), pods) -// } -// -// // create new pod 3 -// if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Fatal(err) -// } -// pods, err = spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// if len(pods) != totalPods { -// t.Fatalf("Expected create pods 2/3, got pods %v", pods) -// } -// -// return pods -// } -// simpleOrderedVerificationFn := func( -// set *api.PetSet, -// spc *fakeObjectManager, -// ssc PetSetControlInterface, -// pods []*v1.Pod, -// totalPods int, -// selector labels.Selector, -// ) []*v1.Pod { -// // only one pod gets created at a time due to OrderedReady -// if len(pods) != 5 { -// t.Fatalf("Expected create pods 5, got pods %v", len(pods)) -// } -// spc.setPodRunning(set, 4) -// pods, _ = spc.setPodReady(set, 4) -// -// // create new pods 4(only one pod gets created at a time due to OrderedReady) -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Fatal(err) -// } -// pods, err := spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// -// if len(pods) != totalPods { -// t.Fatalf("Expected create pods 4, got pods %v", len(pods)) -// } -// // if pod 4 ready, start to update pod 3 -// spc.setPodRunning(set, 5) -// originalPods, _ := spc.setPodReady(set, 5) -// sort.Sort(ascendingOrdinal(originalPods)) -// if _, err = ssc.UpdatePetSet(context.TODO(), set, originalPods); err != nil { -// t.Fatal(err) -// } -// pods, err = spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// sort.Sort(ascendingOrdinal(pods)) -// -// // verify the remaining pods are 0,1,2,4,5 (3 got deleted) -// if !reflect.DeepEqual(pods, append(originalPods[:3], originalPods[4:]...)) { -// t.Fatalf("Expected pods %v, got pods %v", append(originalPods[:3], originalPods[4:]...), pods) -// } -// -// // create new pod 3 -// if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Fatal(err) -// } -// pods, err = spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// if len(pods) != totalPods { -// t.Fatalf("Expected create pods 2/3, got pods %v", pods) -// } -// -// return pods -// } -// testCases := []struct { -// policyType apps.PodManagementPolicyType -// verifyFn func( -// set *api.PetSet, -// spc *fakeObjectManager, -// ssc PetSetControlInterface, -// pods []*v1.Pod, -// totalPods int, -// selector labels.Selector, -// ) []*v1.Pod -// }{ -// {apps.OrderedReadyPodManagement, simpleOrderedVerificationFn}, -// {apps.ParallelPodManagement, simpleParallelVerificationFn}, -// } -// for _, tc := range testCases { -// // Setup the statefulSet controller -// var totalPods int32 = 6 -// var partition int32 = 3 -// maxUnavailable := intstr.FromInt32(2) -// set := setupPodManagementPolicy(tc.policyType, newPetSet(totalPods)) -// set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{ -// Type: apps.RollingUpdateStatefulSetStrategyType, -// RollingUpdate: func() *apps.RollingUpdateStatefulSetStrategy { -// return &apps.RollingUpdateStatefulSetStrategy{ -// Partition: &partition, -// MaxUnavailable: &maxUnavailable, -// } -// }(), -// } -// -// client := fake.NewSimpleClientset() -// apiclient := apifake.NewSimpleClientset(set) -// spc, _, ssc := setupController(client, apiclient) -// if err := scaleUpPetSetControl(set, ssc, spc, assertBurstInvariants); err != nil { -// t.Fatal(err) -// } -// set, err := spc.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatal(err) -// } -// -// // Change the image to trigger an update -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatal(err) -// } -// originalPods, err := spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// sort.Sort(ascendingOrdinal(originalPods)) -// -// // since maxUnavailable is 2, update pods 4 and 5, this will delete the pod 4 and 5, -// if _, err = ssc.UpdatePetSet(context.TODO(), set, originalPods); err != nil { -// t.Fatal(err) -// } -// pods, err := spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// -// sort.Sort(ascendingOrdinal(pods)) -// -// // expected number of pod is 0,1,2,3 -// if !reflect.DeepEqual(pods, originalPods[:4]) { -// t.Fatalf("Expected pods %v, got pods %v", originalPods[:4], pods) -// } -// -// // create new pods -// if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// t.Fatal(err) -// } -// pods, err = spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// -// tc.verifyFn(set, spc, ssc, pods, int(totalPods), selector) -// -// // pods 3/4/5 ready, should not update other pods -// spc.setPodRunning(set, 3) -// spc.setPodRunning(set, 5) -// spc.setPodReady(set, 5) -// originalPods, _ = spc.setPodReady(set, 3) -// sort.Sort(ascendingOrdinal(originalPods)) -// if _, err = ssc.UpdatePetSet(context.TODO(), set, originalPods); err != nil { -// t.Fatal(err) -// } -// pods, err = spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// sort.Sort(ascendingOrdinal(pods)) -// if !reflect.DeepEqual(pods, originalPods) { -// t.Fatalf("Expected pods %v, got pods %v", originalPods, pods) -// } -// } -//} -// -//func setupForInvariant(t *testing.T) (*api.PetSet, *fakeObjectManager, PetSetControlInterface, intstr.IntOrString, int32) { -// var totalPods int32 = 6 -// set := newPetSet(totalPods) -// // update all pods >=3(3,4,5) -// var partition int32 = 3 -// maxUnavailable := intstr.FromInt32(2) -// set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{ -// Type: apps.RollingUpdateStatefulSetStrategyType, -// RollingUpdate: func() *apps.RollingUpdateStatefulSetStrategy { -// return &apps.RollingUpdateStatefulSetStrategy{ -// Partition: &partition, -// MaxUnavailable: &maxUnavailable, -// } -// }(), -// } -// -// client := fake.NewSimpleClientset() -// apiclient := apifake.NewSimpleClientset(set) -// spc, _, ssc := setupController(client, apiclient) -// if err := scaleUpPetSetControl(set, ssc, spc, assertBurstInvariants); err != nil { -// t.Fatal(err) -// } -// set, err := spc.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatal(err) -// } -// -// return set, spc, ssc, maxUnavailable, totalPods -//} -// -//func TestPetSetControlRollingUpdateWithMaxUnavailableInOrderedModeVerifyInvariant(t *testing.T) { -// // Make all pods in petset unavailable one by one -// // and verify that RollingUpdate doesnt proceed with maxUnavailable set -// // this could have been a simple loop, keeping it like this to be able -// // to add more params here. -// testCases := []struct { -// ordinalOfPodToTerminate []int -// }{ -// {[]int{}}, -// {[]int{5}}, -// {[]int{3}}, -// {[]int{4}}, -// {[]int{5, 4}}, -// {[]int{5, 3}}, -// {[]int{4, 3}}, -// {[]int{5, 4, 3}}, -// {[]int{2}}, // note this is an ordinal greater than partition(3) -// {[]int{1}}, // note this is an ordinal greater than partition(3) -// } -// for _, tc := range testCases { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MaxUnavailablePetSet, true)() -// set, spc, ssc, maxUnavailable, totalPods := setupForInvariant(t) -// t.Run(fmt.Sprintf("terminating pod at ordinal %d", tc.ordinalOfPodToTerminate), func(t *testing.T) { -// status := apps.StatefulSetStatus{Replicas: int32(totalPods)} -// updateRevision := &apps.ControllerRevision{} -// -// for i := 0; i < len(tc.ordinalOfPodToTerminate); i++ { -// // Ensure at least one pod is unavailable before trying to update -// _, err := spc.addTerminatingPod(set, tc.ordinalOfPodToTerminate[i]) -// if err != nil { -// t.Fatal(err) -// } -// } -// -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatal(err) -// } -// -// originalPods, err := spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// -// sort.Sort(ascendingOrdinal(originalPods)) -// -// // start to update -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// -// // try to update the petset -// // this function is only called in main code when feature gate is enabled -// if _, err = updatePetSetAfterInvariantEstablished(context.TODO(), ssc.(*defaultPetSetControl), set, originalPods, updateRevision, status); err != nil { -// t.Fatal(err) -// } -// pods, err := spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// -// sort.Sort(ascendingOrdinal(pods)) -// -// expecteddPodsToBeDeleted := maxUnavailable.IntValue() - len(tc.ordinalOfPodToTerminate) -// if expecteddPodsToBeDeleted < 0 { -// expecteddPodsToBeDeleted = 0 -// } -// -// expectedPodsAfterUpdate := int(totalPods) - expecteddPodsToBeDeleted -// -// if len(pods) != expectedPodsAfterUpdate { -// t.Errorf("Expected pods %v, got pods %v", expectedPodsAfterUpdate, len(pods)) -// } -// }) -// } -//} -// -//func TestPetSetControlRollingUpdate(t *testing.T) { -// type testcase struct { -// name string -// invariants func(set *api.PetSet, om *fakeObjectManager) error -// initial func() *api.PetSet -// update func(set *api.PetSet) *api.PetSet -// validate func(set *api.PetSet, pods []*v1.Pod) error -// } -// -// testFn := func(test *testcase, t *testing.T) { -// set := test.initial() -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set = test.update(set) -// if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if err := test.validate(set, pods); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// } -// -// tests := []testcase{ -// { -// name: "monotonic image update", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(3) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "monotonic image update and scale up", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(3) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 5 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "monotonic image update and scale down", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(5) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 3 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(3)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update and scale up", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(3)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 5 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update and scale down", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(5)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 3 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// } -// for i := range tests { -// testFn(&tests[i], t) -// } -//} -// -//func TestPetSetControlOnDeleteUpdate(t *testing.T) { -// type testcase struct { -// name string -// invariants func(set *api.PetSet, om *fakeObjectManager) error -// initial func() *api.PetSet -// update func(set *api.PetSet) *api.PetSet -// validateUpdate func(set *api.PetSet, pods []*v1.Pod) error -// validateRestart func(set *api.PetSet, pods []*v1.Pod) error -// } -// -// originalImage := newPetSet(3).Spec.Template.Spec.Containers[0].Image -// -// testFn := func(t *testing.T, test *testcase, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { -// set := test.initial() -// set.Spec.PersistentVolumeClaimRetentionPolicy = policy -// set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{Type: apps.OnDeleteStatefulSetStrategyType} -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set = test.update(set) -// if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// -// // Pods may have been deleted in the update. Delete any claims with a pod ownerRef. -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// for _, claim := range claims { -// for _, ref := range claim.GetOwnerReferences() { -// if strings.HasPrefix(ref.Name, "foo-") { -// om.claimsIndexer.Delete(claim) -// break -// } -// } -// } -// -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if err := test.validateUpdate(set, pods); err != nil { -// for i := range pods { -// t.Log(pods[i].Name) -// } -// t.Fatalf("%s: %s", test.name, err) -// -// } -// claims, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// for _, claim := range claims { -// for _, ref := range claim.GetOwnerReferences() { -// if strings.HasPrefix(ref.Name, "foo-") { -// t.Fatalf("Unexpected pod reference on %s: %v", claim.Name, claim.GetOwnerReferences()) -// } -// } -// } -// -// replicas := *set.Spec.Replicas -// *set.Spec.Replicas = 0 -// if err := scaleDownPetSetControl(set, ssc, om, test.invariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// *set.Spec.Replicas = replicas -// -// claims, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// for _, claim := range claims { -// for _, ref := range claim.GetOwnerReferences() { -// if strings.HasPrefix(ref.Name, "foo-") { -// t.Fatalf("Unexpected pod reference on %s: %v", claim.Name, claim.GetOwnerReferences()) -// } -// } -// } -// -// if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if err := test.validateRestart(set, pods); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// } -// -// tests := []testcase{ -// { -// name: "monotonic image update", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(3) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "monotonic image update and scale up", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(3) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 5 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if i < 3 && pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// if i >= 3 && pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "monotonic image update and scale down", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(5) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 3 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(3)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update and scale up", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(3)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 5 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if i < 3 && pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// if i >= 3 && pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update and scale down", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(5)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 3 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRestart: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// } -// runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { -// for i := range tests { -// testFn(t, &tests[i], policy) -// } -// }) -//} -// -//func TestPetSetControlRollingUpdateWithPartition(t *testing.T) { -// type testcase struct { -// name string -// partition int32 -// invariants func(set *api.PetSet, om *fakeObjectManager) error -// initial func() *api.PetSet -// update func(set *api.PetSet) *api.PetSet -// validate func(set *api.PetSet, pods []*v1.Pod) error -// } -// -// testFn := func(t *testing.T, test *testcase, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { -// set := test.initial() -// set.Spec.PersistentVolumeClaimRetentionPolicy = policy -// set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{ -// Type: apps.RollingUpdateStatefulSetStrategyType, -// RollingUpdate: func() *apps.RollingUpdateStatefulSetStrategy { -// return &apps.RollingUpdateStatefulSetStrategy{Partition: &test.partition} -// }(), -// } -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set = test.update(set) -// if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if err := test.validate(set, pods); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// } -// -// originalImage := newPetSet(3).Spec.Template.Spec.Containers[0].Image -// -// tests := []testcase{ -// { -// name: "monotonic image update", -// invariants: assertMonotonicInvariants, -// partition: 2, -// initial: func() *api.PetSet { -// return newPetSet(3) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if i < 2 && pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "monotonic image update and scale up", -// partition: 2, -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(3) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 5 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if i < 2 && pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update", -// partition: 2, -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(3)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if i < 2 && pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update and scale up", -// invariants: assertBurstInvariants, -// partition: 2, -// initial: func() *api.PetSet { -// return burst(newPetSet(3)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 5 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if i < 2 && pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// } -// runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { -// for i := range tests { -// testFn(t, &tests[i], policy) -// } -// }) -//} -// -//func TestPetSetHonorRevisionHistoryLimit(t *testing.T) { -// runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { -// invariants := assertMonotonicInvariants -// set := newPetSet(3) -// set.Spec.PersistentVolumeClaimRetentionPolicy = policy -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, ssu, ssc := setupController(client, apiclient) -// -// if err := scaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// var err error -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// -// for i := 0; i < int(*set.Spec.RevisionHistoryLimit)+5; i++ { -// set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) -// ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) -// updatePetSetControl(set, ssc, om, assertUpdateInvariants) -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// revisions, err := ssc.ListRevisions(set) -// if err != nil { -// t.Fatalf("Error listing revisions: %v", err) -// } -// // the extra 2 revisions are `currentRevision` and `updateRevision` -// // They're considered as `live`, and truncateHistory only cleans up non-live revisions -// if len(revisions) > int(*set.Spec.RevisionHistoryLimit)+2 { -// t.Fatalf("%s: %d greater than limit %d", "", len(revisions), *set.Spec.RevisionHistoryLimit) -// } -// } -// }) -//} -// -//func TestPetSetControlLimitsHistory(t *testing.T) { -// type testcase struct { -// name string -// invariants func(set *api.PetSet, om *fakeObjectManager) error -// initial func() *api.PetSet -// } -// -// testFn := func(t *testing.T, test *testcase) { -// set := test.initial() -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// for i := 0; i < 10; i++ { -// set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) -// if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// _, err = ssc.UpdatePetSet(context.TODO(), set, pods) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// revisions, err := ssc.ListRevisions(set) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if len(revisions) > int(*set.Spec.RevisionHistoryLimit)+2 { -// t.Fatalf("%s: %d greater than limit %d", test.name, len(revisions), *set.Spec.RevisionHistoryLimit) -// } -// } -// } -// -// tests := []testcase{ -// { -// name: "monotonic update", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(3) -// }, -// }, -// { -// name: "burst update", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(3)) -// }, -// }, -// } -// for i := range tests { -// testFn(t, &tests[i]) -// } -//} -// -//func TestPetSetControlRollback(t *testing.T) { -// type testcase struct { -// name string -// invariants func(set *api.PetSet, om *fakeObjectManager) error -// initial func() *api.PetSet -// update func(set *api.PetSet) *api.PetSet -// validateUpdate func(set *api.PetSet, pods []*v1.Pod) error -// validateRollback func(set *api.PetSet, pods []*v1.Pod) error -// } -// -// originalImage := newPetSet(3).Spec.Template.Spec.Containers[0].Image -// -// testFn := func(t *testing.T, test *testcase) { -// set := test.initial() -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// if err := scaleUpPetSetControl(set, ssc, om, test.invariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err := om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set = test.update(set) -// if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if err := test.validateUpdate(set, pods); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// revisions, err := ssc.ListRevisions(set) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// history.SortControllerRevisions(revisions) -// set, err = ApplyRevision(set, revisions[0]) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if err := updatePetSetControl(set, ssc, om, assertUpdateInvariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if err := test.validateRollback(set, pods); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// } -// -// tests := []testcase{ -// { -// name: "monotonic image update", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(3) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "monotonic image update and scale up", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(3) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 5 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "monotonic image update and scale down", -// invariants: assertMonotonicInvariants, -// initial: func() *api.PetSet { -// return newPetSet(5) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 3 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(3)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update and scale up", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(3)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 5 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// { -// name: "burst image update and scale down", -// invariants: assertBurstInvariants, -// initial: func() *api.PetSet { -// return burst(newPetSet(5)) -// }, -// update: func(set *api.PetSet) *api.PetSet { -// *set.Spec.Replicas = 3 -// set.Spec.Template.Spec.Containers[0].Image = "foo" -// return set -// }, -// validateUpdate: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != "foo" { -// return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// validateRollback: func(set *api.PetSet, pods []*v1.Pod) error { -// sort.Sort(ascendingOrdinal(pods)) -// for i := range pods { -// if pods[i].Spec.Containers[0].Image != originalImage { -// return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image) -// } -// } -// return nil -// }, -// }, -// } -// for i := range tests { -// testFn(t, &tests[i]) -// } -//} -// -//func TestPetSetAvailability(t *testing.T) { -// tests := []struct { -// name string -// inputSTS *api.PetSet -// expectedActiveReplicas int32 -// readyDuration time.Duration -// }{ -// { -// name: "replicas running for required time, when minReadySeconds is enabled", -// inputSTS: setMinReadySeconds(newPetSet(1), int32(3600)), -// readyDuration: -120 * time.Minute, -// expectedActiveReplicas: int32(1), -// }, -// { -// name: "replicas not running for required time, when minReadySeconds is enabled", -// inputSTS: setMinReadySeconds(newPetSet(1), int32(3600)), -// readyDuration: -30 * time.Minute, -// expectedActiveReplicas: int32(0), -// }, -// } -// for _, test := range tests { -// set := test.inputSTS -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// spc, _, ssc := setupController(client, apiclient) -// if err := scaleUpPetSetControl(set, ssc, spc, assertBurstInvariants); err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// _, err = spc.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// set, err = spc.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// pods, err := spc.setPodAvailable(set, 0, time.Now().Add(test.readyDuration)) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// status, err := ssc.UpdatePetSet(context.TODO(), set, pods) -// if err != nil { -// t.Fatalf("%s: %s", test.name, err) -// } -// if status.AvailableReplicas != test.expectedActiveReplicas { -// t.Fatalf("expected %d active replicas got %d", test.expectedActiveReplicas, status.AvailableReplicas) -// } -// } -//} -// -//func TestStatefulSetStatusUpdate(t *testing.T) { -// var ( -// syncErr = fmt.Errorf("sync error") -// statusErr = fmt.Errorf("status error") -// ) -// -// testCases := []struct { -// desc string -// -// hasSyncErr bool -// hasStatusErr bool -// -// expectedErr error -// }{ -// { -// desc: "no error", -// hasSyncErr: false, -// hasStatusErr: false, -// expectedErr: nil, -// }, -// { -// desc: "sync error", -// hasSyncErr: true, -// hasStatusErr: false, -// expectedErr: syncErr, -// }, -// { -// desc: "status error", -// hasSyncErr: false, -// hasStatusErr: true, -// expectedErr: statusErr, -// }, -// { -// desc: "sync and status error", -// hasSyncErr: true, -// hasStatusErr: true, -// expectedErr: syncErr, -// }, -// } -// -// for _, tc := range testCases { -// t.Run(tc.desc, func(t *testing.T) { -// set := newPetSet(3) -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, ssu, ssc := setupController(client, apiclient) -// -// if tc.hasSyncErr { -// om.SetCreateStatefulPodError(syncErr, 0) -// } -// if tc.hasStatusErr { -// ssu.SetUpdateStatefulSetStatusError(statusErr, 0) -// } -// -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// _, err = ssc.UpdatePetSet(context.TODO(), set, pods) -// if ssu.updateStatusTracker.requests != 1 { -// t.Errorf("Did not update status") -// } -// if !errors.Is(err, tc.expectedErr) { -// t.Errorf("Expected error: %v, got: %v", tc.expectedErr, err) -// } -// }) -// } -//} -// -//type requestTracker struct { -// sync.Mutex -// requests int -// err error -// after int -// -// parallelLock sync.Mutex -// parallel int -// maxParallel int -// -// delay time.Duration -//} -// -//func (rt *requestTracker) errorReady() bool { -// rt.Lock() -// defer rt.Unlock() -// return rt.err != nil && rt.requests >= rt.after -//} -// -//func (rt *requestTracker) inc() { -// rt.parallelLock.Lock() -// rt.parallel++ -// if rt.maxParallel < rt.parallel { -// rt.maxParallel = rt.parallel -// } -// rt.parallelLock.Unlock() -// -// rt.Lock() -// defer rt.Unlock() -// rt.requests++ -// if rt.delay != 0 { -// time.Sleep(rt.delay) -// } -//} -// -//func (rt *requestTracker) reset() { -// rt.parallelLock.Lock() -// rt.parallel = 0 -// rt.parallelLock.Unlock() -// -// rt.Lock() -// defer rt.Unlock() -// rt.err = nil -// rt.after = 0 -// rt.delay = 0 -//} -// -//func (rt *requestTracker) getErr() error { -// rt.Lock() -// defer rt.Unlock() -// return rt.err -//} -// -//func newRequestTracker(requests int, err error, after int) requestTracker { -// return requestTracker{ -// requests: requests, -// err: err, -// after: after, -// } -//} -// -//type fakeObjectManager struct { -// podsLister corelisters.PodLister -// claimsLister corelisters.PersistentVolumeClaimLister -// setsLister apilisters.PetSetLister -// podsIndexer cache.Indexer -// claimsIndexer cache.Indexer -// setsIndexer cache.Indexer -// revisionsIndexer cache.Indexer -// createPodTracker requestTracker -// updatePodTracker requestTracker -// deletePodTracker requestTracker -//} -// -//func newFakeObjectManager(informerFactory informers.SharedInformerFactory, apiinformerFactory apiinformers.SharedInformerFactory) *fakeObjectManager { -// podInformer := informerFactory.Core().V1().Pods() -// claimInformer := informerFactory.Core().V1().PersistentVolumeClaims() -// setInformer := apiinformerFactory.Apps().V1().PetSets() -// revisionInformer := informerFactory.Apps().V1().ControllerRevisions() -// -// return &fakeObjectManager{ -// podInformer.Lister(), -// claimInformer.Lister(), -// setInformer.Lister(), -// podInformer.Informer().GetIndexer(), -// claimInformer.Informer().GetIndexer(), -// setInformer.Informer().GetIndexer(), -// revisionInformer.Informer().GetIndexer(), -// newRequestTracker(0, nil, 0), -// newRequestTracker(0, nil, 0), -// newRequestTracker(0, nil, 0), -// } -//} -// -//func (om *fakeObjectManager) CreatePod(ctx context.Context, pod *v1.Pod) error { -// defer om.createPodTracker.inc() -// if om.createPodTracker.errorReady() { -// defer om.createPodTracker.reset() -// return om.createPodTracker.getErr() -// } -// pod.SetUID(types.UID(pod.Name + "-uid")) -// return om.podsIndexer.Update(pod) -//} -// -//func (om *fakeObjectManager) GetPod(namespace, podName string) (*v1.Pod, error) { -// return om.podsLister.Pods(namespace).Get(podName) -//} -// -//func (om *fakeObjectManager) UpdatePod(pod *v1.Pod) error { -// return om.podsIndexer.Update(pod) -//} -// -//func (om *fakeObjectManager) DeletePod(pod *v1.Pod) error { -// defer om.deletePodTracker.inc() -// if om.deletePodTracker.errorReady() { -// defer om.deletePodTracker.reset() -// return om.deletePodTracker.getErr() -// } -// if key, err := controller.KeyFunc(pod); err != nil { -// return err -// } else if obj, found, err := om.podsIndexer.GetByKey(key); err != nil { -// return err -// } else if found { -// return om.podsIndexer.Delete(obj) -// } -// return nil // Not found, no error in deleting. -//} -// -//func (om *fakeObjectManager) CreateClaim(claim *v1.PersistentVolumeClaim) error { -// om.claimsIndexer.Update(claim) -// return nil -//} -// -//func (om *fakeObjectManager) GetClaim(namespace, claimName string) (*v1.PersistentVolumeClaim, error) { -// return om.claimsLister.PersistentVolumeClaims(namespace).Get(claimName) -//} -// -//func (om *fakeObjectManager) UpdateClaim(claim *v1.PersistentVolumeClaim) error { -// // Validate ownerRefs. -// refs := claim.GetOwnerReferences() -// for _, ref := range refs { -// if ref.APIVersion == "" || ref.Kind == "" || ref.Name == "" { -// return fmt.Errorf("invalid ownerRefs: %s %v", claim.Name, refs) -// } -// } -// om.claimsIndexer.Update(claim) -// return nil -//} -// -//func (om *fakeObjectManager) SetCreateStatefulPodError(err error, after int) { -// om.createPodTracker.err = err -// om.createPodTracker.after = after -//} -// -//func (om *fakeObjectManager) SetUpdateStatefulPodError(err error, after int) { -// om.updatePodTracker.err = err -// om.updatePodTracker.after = after -//} -// -//func (om *fakeObjectManager) SetDeleteStatefulPodError(err error, after int) { -// om.deletePodTracker.err = err -// om.deletePodTracker.after = after -//} -// -//func findPodByOrdinal(pods []*v1.Pod, ordinal int) *v1.Pod { -// for _, pod := range pods { -// if getOrdinal(pod) == ordinal { -// return pod.DeepCopy() -// } -// } -// -// return nil -//} -// -//func (om *fakeObjectManager) setPodPending(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return nil, err -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return nil, err -// } -// pod := findPodByOrdinal(pods, ordinal) -// if pod == nil { -// return nil, fmt.Errorf("setPodPending: pod ordinal %d not found", ordinal) -// } -// pod.Status.Phase = v1.PodPending -// fakeResourceVersion(pod) -// om.podsIndexer.Update(pod) -// return om.podsLister.Pods(set.Namespace).List(selector) -//} -// -//func (om *fakeObjectManager) setPodRunning(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return nil, err -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return nil, err -// } -// pod := findPodByOrdinal(pods, ordinal) -// if pod == nil { -// return nil, fmt.Errorf("setPodRunning: pod ordinal %d not found", ordinal) -// } -// pod.Status.Phase = v1.PodRunning -// fakeResourceVersion(pod) -// om.podsIndexer.Update(pod) -// return om.podsLister.Pods(set.Namespace).List(selector) -//} -// -//func (om *fakeObjectManager) setPodReady(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return nil, err -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return nil, err -// } -// pod := findPodByOrdinal(pods, ordinal) -// if pod == nil { -// return nil, fmt.Errorf("setPodReady: pod ordinal %d not found", ordinal) -// } -// condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} -// podutil.UpdatePodCondition(&pod.Status, &condition) -// fakeResourceVersion(pod) -// om.podsIndexer.Update(pod) -// return om.podsLister.Pods(set.Namespace).List(selector) -//} -// -//func (om *fakeObjectManager) setPodAvailable(set *api.PetSet, ordinal int, lastTransitionTime time.Time) ([]*v1.Pod, error) { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return nil, err -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return nil, err -// } -// pod := findPodByOrdinal(pods, ordinal) -// if pod == nil { -// return nil, fmt.Errorf("setPodAvailable: pod ordinal %d not found", ordinal) -// } -// condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: metav1.Time{Time: lastTransitionTime}} -// _, existingCondition := podutil.GetPodCondition(&pod.Status, condition.Type) -// if existingCondition != nil { -// existingCondition.Status = v1.ConditionTrue -// existingCondition.LastTransitionTime = metav1.Time{Time: lastTransitionTime} -// } else { -// existingCondition = &v1.PodCondition{ -// Type: v1.PodReady, -// Status: v1.ConditionTrue, -// LastTransitionTime: metav1.Time{Time: lastTransitionTime}, -// } -// pod.Status.Conditions = append(pod.Status.Conditions, *existingCondition) -// } -// podutil.UpdatePodCondition(&pod.Status, &condition) -// fakeResourceVersion(pod) -// om.podsIndexer.Update(pod) -// return om.podsLister.Pods(set.Namespace).List(selector) -//} -// -//func (om *fakeObjectManager) addTerminatingPod(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { -// pod := newPetSetPod(set, ordinal) -// pod.SetUID(types.UID(pod.Name + "-uid")) // To match fakeObjectManager.CreatePod -// pod.Status.Phase = v1.PodRunning -// deleted := metav1.NewTime(time.Now()) -// pod.DeletionTimestamp = &deleted -// condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} -// fakeResourceVersion(pod) -// podutil.UpdatePodCondition(&pod.Status, &condition) -// om.podsIndexer.Update(pod) -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return nil, err -// } -// return om.podsLister.Pods(set.Namespace).List(selector) -//} -// -//func (om *fakeObjectManager) setPodTerminated(set *api.PetSet, ordinal int) ([]*v1.Pod, error) { -// pod := newPetSetPod(set, ordinal) -// deleted := metav1.NewTime(time.Now()) -// pod.DeletionTimestamp = &deleted -// fakeResourceVersion(pod) -// om.podsIndexer.Update(pod) -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return nil, err -// } -// return om.podsLister.Pods(set.Namespace).List(selector) -//} -// -//var _ StatefulPodControlObjectManager = &fakeObjectManager{} -// -//type fakeStatefulSetStatusUpdater struct { -// setsLister apilisters.PetSetLister -// setsIndexer cache.Indexer -// updateStatusTracker requestTracker -//} -// -//func newFakeStatefulSetStatusUpdater(setInformer stsinformers.PetSetInformer) *fakeStatefulSetStatusUpdater { -// return &fakeStatefulSetStatusUpdater{ -// setInformer.Lister(), -// setInformer.Informer().GetIndexer(), -// newRequestTracker(0, nil, 0), -// } -//} -// -//func (ssu *fakeStatefulSetStatusUpdater) UpdateStatefulSetStatus(ctx context.Context, set *api.PetSet, status *apps.StatefulSetStatus) error { -// defer ssu.updateStatusTracker.inc() -// if ssu.updateStatusTracker.errorReady() { -// defer ssu.updateStatusTracker.reset() -// return ssu.updateStatusTracker.err -// } -// set.Status = *status -// ssu.setsIndexer.Update(set) -// return nil -//} -// -//func (ssu *fakeStatefulSetStatusUpdater) SetUpdateStatefulSetStatusError(err error, after int) { -// ssu.updateStatusTracker.err = err -// ssu.updateStatusTracker.after = after -//} -// -//var _ StatefulSetStatusUpdaterInterface = &fakeStatefulSetStatusUpdater{} -// -//func assertMonotonicInvariants(set *api.PetSet, om *fakeObjectManager) error { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// sort.Sort(ascendingOrdinal(pods)) -// for idx := 0; idx < len(pods); idx++ { -// if idx > 0 && isRunningAndReady(pods[idx]) && !isRunningAndReady(pods[idx-1]) { -// return fmt.Errorf("successor %s is Running and Ready while %s is not", pods[idx].Name, pods[idx-1].Name) -// } -// -// if ord := idx + getStartOrdinal(set); getOrdinal(pods[idx]) != ord { -// return fmt.Errorf("pods %s deployed in the wrong order %d", pods[idx].Name, ord) -// } -// -// if !storageMatches(set, pods[idx]) { -// return fmt.Errorf("pods %s does not match the storage specification of PetSet %s ", pods[idx].Name, set.Name) -// } -// -// for _, claim := range getPersistentVolumeClaims(set, pods[idx]) { -// claim, _ := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) -// if err := checkClaimInvarients(set, pods[idx], claim); err != nil { -// return err -// } -// } -// -// if !identityMatches(set, pods[idx]) { -// return fmt.Errorf("pods %s does not match the identity specification of PetSet %s ", pods[idx].Name, set.Name) -// } -// } -// return nil -//} -// -//func assertBurstInvariants(set *api.PetSet, om *fakeObjectManager) error { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// sort.Sort(ascendingOrdinal(pods)) -// for _, pod := range pods { -// if !storageMatches(set, pod) { -// return fmt.Errorf("pods %s does not match the storage specification of PetSet %s ", pod.Name, set.Name) -// } -// -// for _, claim := range getPersistentVolumeClaims(set, pod) { -// claim, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) -// if err != nil { -// return err -// } -// if err := checkClaimInvarients(set, pod, claim); err != nil { -// return err -// } -// } -// -// if !identityMatches(set, pod) { -// return fmt.Errorf("pods %s does not match the identity specification of PetSet %s ", -// pod.Name, -// set.Name) -// } -// } -// return nil -//} -// -//func assertUpdateInvariants(set *api.PetSet, om *fakeObjectManager) error { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// sort.Sort(ascendingOrdinal(pods)) -// for _, pod := range pods { -// -// if !storageMatches(set, pod) { -// return fmt.Errorf("pod %s does not match the storage specification of PetSet %s ", pod.Name, set.Name) -// } -// -// for _, claim := range getPersistentVolumeClaims(set, pod) { -// claim, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) -// if err != nil { -// return err -// } -// if err := checkClaimInvarients(set, pod, claim); err != nil { -// return err -// } -// } -// -// if !identityMatches(set, pod) { -// return fmt.Errorf("pod %s does not match the identity specification of PetSet %s ", pod.Name, set.Name) -// } -// } -// if set.Spec.UpdateStrategy.Type == apps.OnDeleteStatefulSetStrategyType { -// return nil -// } -// if set.Spec.UpdateStrategy.Type == apps.RollingUpdateStatefulSetStrategyType { -// for i := 0; i < int(set.Status.CurrentReplicas) && i < len(pods); i++ { -// if want, got := set.Status.CurrentRevision, getPodRevision(pods[i]); want != got { -// return fmt.Errorf("pod %s want current revision %s got %s", pods[i].Name, want, got) -// } -// } -// for i, j := len(pods)-1, 0; j < int(set.Status.UpdatedReplicas); i, j = i-1, j+1 { -// if want, got := set.Status.UpdateRevision, getPodRevision(pods[i]); want != got { -// return fmt.Errorf("pod %s want update revision %s got %s", pods[i].Name, want, got) -// } -// } -// } -// return nil -//} -// -//func checkClaimInvarients(set *api.PetSet, pod *v1.Pod, claim *v1.PersistentVolumeClaim) error { -// policy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// } -// if set.Spec.PersistentVolumeClaimRetentionPolicy != nil && features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) { -// policy = *set.Spec.PersistentVolumeClaimRetentionPolicy -// } -// claimShouldBeRetained := policy.WhenScaled == apps.RetainPersistentVolumeClaimRetentionPolicyType -// if claim == nil { -// if claimShouldBeRetained { -// return fmt.Errorf("claim for Pod %s was not created", pod.Name) -// } -// return nil // A non-retained claim has no invariants to satisfy. -// } -// -// if pod.Status.Phase != v1.PodRunning || !podutil.IsPodReady(pod) { -// // The pod has spun up yet, we do not expect the owner refs on the claim to have been set. -// return nil -// } -// -// const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType -// const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType -// switch { -// case policy.WhenScaled == retain && policy.WhenDeleted == retain: -// if hasOwnerRef(claim, set) { -// return fmt.Errorf("claim %s has unexpected owner ref on %s for PetSet retain", claim.Name, set.Name) -// } -// if hasOwnerRef(claim, pod) { -// return fmt.Errorf("claim %s has unexpected owner ref on pod %s for PetSet retain", claim.Name, pod.Name) -// } -// case policy.WhenScaled == retain && policy.WhenDeleted == delete: -// if !hasOwnerRef(claim, set) { -// return fmt.Errorf("claim %s does not have owner ref on %s for PetSet deletion", claim.Name, set.Name) -// } -// if hasOwnerRef(claim, pod) { -// return fmt.Errorf("claim %s has unexpected owner ref on pod %s for PetSet deletion", claim.Name, pod.Name) -// } -// case policy.WhenScaled == delete && policy.WhenDeleted == retain: -// if hasOwnerRef(claim, set) { -// return fmt.Errorf("claim %s has unexpected owner ref on %s for scaledown only", claim.Name, set.Name) -// } -// if !podInOrdinalRange(pod, set) && !hasOwnerRef(claim, pod) { -// return fmt.Errorf("claim %s does not have owner ref on condemned pod %s for scaledown delete", claim.Name, pod.Name) -// } -// if podInOrdinalRange(pod, set) && hasOwnerRef(claim, pod) { -// return fmt.Errorf("claim %s has unexpected owner ref on condemned pod %s for scaledown delete", claim.Name, pod.Name) -// } -// case policy.WhenScaled == delete && policy.WhenDeleted == delete: -// if !podInOrdinalRange(pod, set) { -// if !hasOwnerRef(claim, pod) || hasOwnerRef(claim, set) { -// return fmt.Errorf("condemned claim %s has bad owner refs: %v", claim.Name, claim.GetOwnerReferences()) -// } -// } else { -// if hasOwnerRef(claim, pod) || !hasOwnerRef(claim, set) { -// return fmt.Errorf("live claim %s has bad owner refs: %v", claim.Name, claim.GetOwnerReferences()) -// } -// } -// } -// return nil -//} -// -//func fakeResourceVersion(object interface{}) { -// obj, isObj := object.(metav1.Object) -// if !isObj { -// return -// } -// if version := obj.GetResourceVersion(); version == "" { -// obj.SetResourceVersion("1") -// } else if intValue, err := strconv.ParseInt(version, 10, 32); err == nil { -// obj.SetResourceVersion(strconv.FormatInt(intValue+1, 10)) -// } -//} -// -//func TestParallelScale(t *testing.T) { -// for _, tc := range []struct { -// desc string -// replicas int32 -// desiredReplicas int32 -// }{ -// { -// desc: "scale up from 3 to 30", -// replicas: 3, -// desiredReplicas: 30, -// }, -// { -// desc: "scale down from 10 to 1", -// replicas: 10, -// desiredReplicas: 1, -// }, -// -// { -// desc: "scale down to 0", -// replicas: 501, -// desiredReplicas: 0, -// }, -// { -// desc: "scale up from 0", -// replicas: 0, -// desiredReplicas: 1000, -// }, -// } { -// t.Run(tc.desc, func(t *testing.T) { -// set := burst(newPetSet(0)) -// set.Spec.VolumeClaimTemplates[0].ObjectMeta.Labels = map[string]string{"test": "test"} -// parallelScale(t, set, tc.replicas, tc.desiredReplicas, assertBurstInvariants) -// }) -// } -//} -// -//func parallelScale(t *testing.T, set *api.PetSet, replicas, desiredReplicas int32, invariants invariantFunc) { -// var err error -// diff := desiredReplicas - replicas -// client := fake.NewSimpleClientset(set) -// apiclient := apifake.NewSimpleClientset(set) -// om, _, ssc := setupController(client, apiclient) -// om.createPodTracker.delay = time.Millisecond -// -// *set.Spec.Replicas = replicas -// if err := parallelScaleUpPetSetControl(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// if set.Status.Replicas != replicas { -// t.Errorf("want %v, got %v replicas", replicas, set.Status.Replicas) -// } -// -// fn := parallelScaleUpPetSetControl -// if diff < 0 { -// fn = parallelScaleDownPetSetControl -// } -// *set.Spec.Replicas = desiredReplicas -// if err := fn(set, ssc, om, invariants); err != nil { -// t.Errorf("Failed to scale PetSet : %s", err) -// } -// -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Fatalf("Error getting updated PetSet: %v", err) -// } -// -// if set.Status.Replicas != desiredReplicas { -// t.Errorf("Failed to scale petset to %v replicas, got %v replicas", desiredReplicas, set.Status.Replicas) -// } -// -// if (diff < -1 || diff > 1) && om.createPodTracker.maxParallel <= 1 { -// t.Errorf("want max parallel requests > 1, got %v", om.createPodTracker.maxParallel) -// } -//} -// -//func parallelScaleUpPetSetControl(set *api.PetSet, -// ssc PetSetControlInterface, -// om *fakeObjectManager, -// invariants invariantFunc, -//) error { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// -// // Give up after 2 loops. -// // 2 * 500 pods per loop = 1000 max pods <- this should be enough for all test cases. -// // Anything slower than that (requiring more iterations) indicates a problem and should fail the test. -// maxLoops := 2 -// loops := maxLoops -// for set.Status.Replicas < *set.Spec.Replicas { -// if loops < 1 { -// return fmt.Errorf("after %v loops: want %v, got replicas %v", maxLoops, *set.Spec.Replicas, set.Status.Replicas) -// } -// loops-- -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// sort.Sort(ascendingOrdinal(pods)) -// -// ordinals := []int{} -// for _, pod := range pods { -// if pod.Status.Phase == "" { -// ordinals = append(ordinals, getOrdinal(pod)) -// } -// } -// // ensure all pods are valid (have a phase) -// for _, ord := range ordinals { -// if pods, err = om.setPodPending(set, ord); err != nil { -// return err -// } -// } -// -// // run the controller once and check invariants -// _, err = ssc.UpdatePetSet(context.TODO(), set, pods) -// if err != nil { -// return err -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// return err -// } -// if err := invariants(set, om); err != nil { -// return err -// } -// } -// return invariants(set, om) -//} -// -//func parallelScaleDownPetSetControl(set *api.PetSet, ssc PetSetControlInterface, om *fakeObjectManager, invariants invariantFunc) error { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// -// // Give up after 2 loops. -// // 2 * 500 pods per loop = 1000 max pods <- this should be enough for all test cases. -// // Anything slower than that (requiring more iterations) indicates a problem and should fail the test. -// maxLoops := 2 -// loops := maxLoops -// for set.Status.Replicas > *set.Spec.Replicas { -// if loops < 1 { -// return fmt.Errorf("after %v loops: want %v replicas, got %v", maxLoops, *set.Spec.Replicas, set.Status.Replicas) -// } -// loops-- -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// sort.Sort(ascendingOrdinal(pods)) -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// return err -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// return err -// } -// if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// return err -// } -// } -// -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// return err -// } -// if err := invariants(set, om); err != nil { -// return err -// } -// -// return nil -//} -// -//func scaleUpPetSetControl(set *api.PetSet, -// ssc PetSetControlInterface, -// om *fakeObjectManager, -// invariants invariantFunc, -//) error { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// for set.Status.ReadyReplicas < *set.Spec.Replicas { -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// sort.Sort(ascendingOrdinal(pods)) -// -// // ensure all pods are valid (have a phase) -// for _, pod := range pods { -// if pod.Status.Phase == "" { -// if pods, err = om.setPodPending(set, getOrdinal(pod)); err != nil { -// return err -// } -// break -// } -// } -// -// // select one of the pods and move it forward in status -// if len(pods) > 0 { -// idx := int(rand.Int63n(int64(len(pods)))) -// pod := pods[idx] -// switch pod.Status.Phase { -// case v1.PodPending: -// if pods, err = om.setPodRunning(set, getOrdinal(pod)); err != nil { -// return err -// } -// case v1.PodRunning: -// if pods, err = om.setPodReady(set, getOrdinal(pod)); err != nil { -// return err -// } -// default: -// continue -// } -// } -// // run the controller once and check invariants -// _, err = ssc.UpdatePetSet(context.TODO(), set, pods) -// if err != nil { -// return err -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// return err -// } -// if err := invariants(set, om); err != nil { -// return err -// } -// } -// return invariants(set, om) -//} -// -//func scaleDownPetSetControl(set *api.PetSet, ssc PetSetControlInterface, om *fakeObjectManager, invariants invariantFunc) error { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// -// for set.Status.Replicas > *set.Spec.Replicas { -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// sort.Sort(ascendingOrdinal(pods)) -// if idx := len(pods) - 1; idx >= 0 { -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// return err -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// return err -// } -// if pods, err = om.addTerminatingPod(set, getOrdinal(pods[idx])); err != nil { -// return err -// } -// if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// return err -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// return err -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// sort.Sort(ascendingOrdinal(pods)) -// -// if len(pods) > 0 { -// om.podsIndexer.Delete(pods[len(pods)-1]) -// } -// } -// if _, err := ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// return err -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// return err -// } -// -// if err := invariants(set, om); err != nil { -// return err -// } -// } -// // If there are claims with ownerRefs on pods that have been deleted, delete them. -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// currentPods := map[string]bool{} -// for _, pod := range pods { -// currentPods[pod.Name] = true -// } -// claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// for _, claim := range claims { -// claimPodName := getClaimPodName(set, claim) -// if claimPodName == "" { -// continue // Skip claims not related to a stateful set pod. -// } -// if _, found := currentPods[claimPodName]; found { -// continue // Skip claims which still have a current pod. -// } -// for _, refs := range claim.GetOwnerReferences() { -// if refs.Name == claimPodName { -// om.claimsIndexer.Delete(claim) -// break -// } -// } -// } -// -// return invariants(set, om) -//} -// -//func updateComplete(set *api.PetSet, pods []*v1.Pod) bool { -// sort.Sort(ascendingOrdinal(pods)) -// if len(pods) != int(*set.Spec.Replicas) { -// return false -// } -// if set.Status.ReadyReplicas != *set.Spec.Replicas { -// return false -// } -// -// switch set.Spec.UpdateStrategy.Type { -// case apps.OnDeleteStatefulSetStrategyType: -// return true -// case apps.RollingUpdateStatefulSetStrategyType: -// if set.Spec.UpdateStrategy.RollingUpdate == nil || *set.Spec.UpdateStrategy.RollingUpdate.Partition <= 0 { -// if set.Status.CurrentReplicas < *set.Spec.Replicas { -// return false -// } -// for i := range pods { -// if getPodRevision(pods[i]) != set.Status.CurrentRevision { -// return false -// } -// } -// } else { -// partition := int(*set.Spec.UpdateStrategy.RollingUpdate.Partition) -// if len(pods) < partition { -// return false -// } -// for i := partition; i < len(pods); i++ { -// if getPodRevision(pods[i]) != set.Status.UpdateRevision { -// return false -// } -// } -// } -// } -// return true -//} -// -//func updatePetSetControl(set *api.PetSet, -// ssc PetSetControlInterface, -// om *fakeObjectManager, -// invariants invariantFunc, -//) error { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// return err -// } -// -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// return err -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// for !updateComplete(set, pods) { -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// sort.Sort(ascendingOrdinal(pods)) -// initialized := false -// for _, pod := range pods { -// if pod.Status.Phase == "" { -// if pods, err = om.setPodPending(set, getOrdinal(pod)); err != nil { -// return err -// } -// break -// } -// } -// if initialized { -// continue -// } -// -// if len(pods) > 0 { -// idx := int(rand.Int63n(int64(len(pods)))) -// pod := pods[idx] -// switch pod.Status.Phase { -// case v1.PodPending: -// if pods, err = om.setPodRunning(set, getOrdinal(pod)); err != nil { -// return err -// } -// case v1.PodRunning: -// if pods, err = om.setPodReady(set, getOrdinal(pod)); err != nil { -// return err -// } -// default: -// continue -// } -// } -// -// if _, err = ssc.UpdatePetSet(context.TODO(), set, pods); err != nil { -// return err -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// return err -// } -// if err := invariants(set, om); err != nil { -// return err -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// } -// return invariants(set, om) -//} -// -//func newRevisionOrDie(set *api.PetSet, revision int64) *apps.ControllerRevision { -// rev, err := newRevision(set, revision, set.Status.CollisionCount) -// if err != nil { -// panic(err) -// } -// return rev -//} -// -//func isOrHasInternalError(err error) bool { -// agg, ok := err.(utilerrors.Aggregate) -// return !ok && !apierrors.IsInternalError(err) || ok && len(agg.Errors()) > 0 && !apierrors.IsInternalError(agg.Errors()[0]) -//} diff --git a/pkg/controller/tests/pet_set_status_updater_test.go b/pkg/controller/tests/pet_set_status_updater_test.go deleted file mode 100644 index dfdd9bea..00000000 --- a/pkg/controller/tests/pet_set_status_updater_test.go +++ /dev/null @@ -1,143 +0,0 @@ -// /* -// Copyright 2017 The Kubernetes Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// */ -package petset - -// -//import ( -// "context" -// "errors" -// "testing" -// -// api "kubeops.dev/petset/apis/apps/v1" -// "kubeops.dev/petset/client/clientset/versioned/fake" -// appslisters "kubeops.dev/petset/client/listers/apps/v1" -// -// apps "k8s.io/api/apps/v1" -// apierrors "k8s.io/apimachinery/pkg/api/errors" -// "k8s.io/apimachinery/pkg/runtime" -// core "k8s.io/client-go/testing" -// "k8s.io/client-go/tools/cache" -//) -// -//func TestPetSetUpdaterUpdatesSetStatus(t *testing.T) { -// set := newPetSet(3) -// status := apps.StatefulSetStatus{ObservedGeneration: 1, Replicas: 2} -// fakeClient := &fake.Clientset{} -// updater := NewRealStatefulSetStatusUpdater(fakeClient, nil) -// fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// return true, update.GetObject(), nil -// }) -// if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err != nil { -// t.Errorf("Error returned on successful status update: %s", err) -// } -// if set.Status.Replicas != 2 { -// t.Errorf("UpdateStatefulSetStatus mutated the sets replicas %d", set.Status.Replicas) -// } -//} -// -//func TestStatefulSetStatusUpdaterUpdatesObservedGeneration(t *testing.T) { -// set := newPetSet(3) -// status := apps.StatefulSetStatus{ObservedGeneration: 3, Replicas: 2} -// fakeClient := &fake.Clientset{} -// updater := NewRealStatefulSetStatusUpdater(fakeClient, nil) -// fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// sts := update.GetObject().(*api.PetSet) -// if sts.Status.ObservedGeneration != 3 { -// t.Errorf("expected observedGeneration to be synced with generation for petset %q", sts.Name) -// } -// return true, sts, nil -// }) -// if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err != nil { -// t.Errorf("Error returned on successful status update: %s", err) -// } -//} -// -//func TestStatefulSetStatusUpdaterUpdateReplicasFailure(t *testing.T) { -// set := newPetSet(3) -// status := apps.StatefulSetStatus{ObservedGeneration: 3, Replicas: 2} -// fakeClient := &fake.Clientset{} -// indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// indexer.Add(set) -// setLister := appslisters.NewPetSetLister(indexer) -// updater := NewRealStatefulSetStatusUpdater(fakeClient, setLister) -// fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { -// return true, nil, apierrors.NewInternalError(errors.New("API server down")) -// }) -// if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err == nil { -// t.Error("Failed update did not return error") -// } -//} -// -//func TestStatefulSetStatusUpdaterUpdateReplicasConflict(t *testing.T) { -// set := newPetSet(3) -// status := apps.StatefulSetStatus{ObservedGeneration: 3, Replicas: 2} -// conflict := false -// fakeClient := &fake.Clientset{} -// indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// indexer.Add(set) -// setLister := appslisters.NewPetSetLister(indexer) -// updater := NewRealStatefulSetStatusUpdater(fakeClient, setLister) -// fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// if !conflict { -// conflict = true -// return true, update.GetObject(), apierrors.NewConflict(action.GetResource().GroupResource(), set.Name, errors.New("object already exists")) -// } -// return true, update.GetObject(), nil -// }) -// if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err != nil { -// t.Errorf("UpdateStatefulSetStatus returned an error: %s", err) -// } -// if set.Status.Replicas != 2 { -// t.Errorf("UpdateStatefulSetStatus mutated the sets replicas %d", set.Status.Replicas) -// } -//} -// -//func TestStatefulSetStatusUpdaterUpdateReplicasConflictFailure(t *testing.T) { -// set := newPetSet(3) -// status := apps.StatefulSetStatus{ObservedGeneration: 3, Replicas: 2} -// fakeClient := &fake.Clientset{} -// indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) -// indexer.Add(set) -// setLister := appslisters.NewPetSetLister(indexer) -// updater := NewRealStatefulSetStatusUpdater(fakeClient, setLister) -// fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// return true, update.GetObject(), apierrors.NewConflict(action.GetResource().GroupResource(), set.Name, errors.New("object already exists")) -// }) -// if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err == nil { -// t.Error("UpdateStatefulSetStatus failed to return an error on get failure") -// } -//} -// -//func TestStatefulSetStatusUpdaterGetAvailableReplicas(t *testing.T) { -// set := newPetSet(3) -// status := apps.StatefulSetStatus{ObservedGeneration: 1, Replicas: 2, AvailableReplicas: 3} -// fakeClient := &fake.Clientset{} -// updater := NewRealStatefulSetStatusUpdater(fakeClient, nil) -// fakeClient.AddReactor("update", "petsets", func(action core.Action) (bool, runtime.Object, error) { -// update := action.(core.UpdateAction) -// return true, update.GetObject(), nil -// }) -// if err := updater.UpdateStatefulSetStatus(context.TODO(), set, &status); err != nil { -// t.Errorf("Error returned on successful status update: %s", err) -// } -// if set.Status.AvailableReplicas != 3 { -// t.Errorf("UpdateStatefulSetStatus mutated the sets replicas %d", set.Status.AvailableReplicas) -// } -//} diff --git a/pkg/controller/tests/pet_set_test.go b/pkg/controller/tests/pet_set_test.go deleted file mode 100644 index 838644c3..00000000 --- a/pkg/controller/tests/pet_set_test.go +++ /dev/null @@ -1,1092 +0,0 @@ -// /* -// Copyright 2016 The Kubernetes Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// */ -package petset - -// -//import ( -// "bytes" -// "context" -// "encoding/json" -// "fmt" -// "sort" -// "testing" -// -// api "kubeops.dev/petset/apis/apps/v1" -// apifake "kubeops.dev/petset/client/clientset/versioned/fake" -// apiinformers "kubeops.dev/petset/client/informers/externalversions" -// "kubeops.dev/petset/pkg/controller" -// "kubeops.dev/petset/pkg/controller/history" -// "kubeops.dev/petset/pkg/features" -// -// apps "k8s.io/api/apps/v1" -// v1 "k8s.io/api/core/v1" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// "k8s.io/apimachinery/pkg/labels" -// "k8s.io/apimachinery/pkg/runtime" -// "k8s.io/apimachinery/pkg/types" -// "k8s.io/apimachinery/pkg/util/sets" -// "k8s.io/apimachinery/pkg/util/strategicpatch" -// utilfeature "k8s.io/apiserver/pkg/util/feature" -// "k8s.io/client-go/informers" -// "k8s.io/client-go/kubernetes/fake" -// core "k8s.io/client-go/testing" -// "k8s.io/client-go/tools/cache" -// "k8s.io/client-go/tools/record" -// featuregatetesting "k8s.io/component-base/featuregate/testing" -// "k8s.io/klog/v2" -// "k8s.io/klog/v2/ktesting" -//) -// -//var parentKind = api.SchemeGroupVersion.WithKind("PetSet") -// -//func alwaysReady() bool { return true } -// -//func TestPetSetControllerCreates(t *testing.T) { -// set := newPetSet(3) -// logger, ctx := ktesting.NewTestContext(t) -// ssc, spc, om, _ := newFakePetSetController(ctx, set) -// if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// if obj, _, err := om.setsIndexer.Get(set); err != nil { -// t.Error(err) -// } else { -// set = obj.(*api.PetSet) -// } -// if set.Status.Replicas != 3 { -// t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) -// } -//} -// -//func TestPetSetControllerDeletes(t *testing.T) { -// set := newPetSet(3) -// logger, ctx := ktesting.NewTestContext(t) -// ssc, spc, om, _ := newFakePetSetController(ctx, set) -// if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// if obj, _, err := om.setsIndexer.Get(set); err != nil { -// t.Error(err) -// } else { -// set = obj.(*api.PetSet) -// } -// if set.Status.Replicas != 3 { -// t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) -// } -// *set.Spec.Replicas = 0 -// if err := scaleDownPetSetController(logger, set, ssc, spc, om); err != nil { -// t.Errorf("Failed to turn down PetSet : %s", err) -// } -// if obj, _, err := om.setsIndexer.Get(set); err != nil { -// t.Error(err) -// } else { -// set = obj.(*api.PetSet) -// } -// if set.Status.Replicas != 0 { -// t.Errorf("set.Status.Replicas = %v; want 0", set.Status.Replicas) -// } -//} -// -//func TestPetSetControllerRespectsTermination(t *testing.T) { -// set := newPetSet(3) -// logger, ctx := ktesting.NewTestContext(t) -// ssc, spc, om, _ := newFakePetSetController(ctx, set) -// if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// if obj, _, err := om.setsIndexer.Get(set); err != nil { -// t.Error(err) -// } else { -// set = obj.(*api.PetSet) -// } -// if set.Status.Replicas != 3 { -// t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) -// } -// _, err := om.addTerminatingPod(set, 3) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.addTerminatingPod(set, 4) -// if err != nil { -// t.Error(err) -// } -// ssc.syncPetSet(ctx, set, pods) -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if len(pods) != 5 { -// t.Error("PetSet does not respect termination") -// } -// sort.Sort(ascendingOrdinal(pods)) -// spc.DeleteStatefulPod(set, pods[3]) -// spc.DeleteStatefulPod(set, pods[4]) -// *set.Spec.Replicas = 0 -// if err := scaleDownPetSetController(logger, set, ssc, spc, om); err != nil { -// t.Errorf("Failed to turn down PetSet : %s", err) -// } -// if obj, _, err := om.setsIndexer.Get(set); err != nil { -// t.Error(err) -// } else { -// set = obj.(*api.PetSet) -// } -// if set.Status.Replicas != 0 { -// t.Errorf("set.Status.Replicas = %v; want 0", set.Status.Replicas) -// } -//} -// -//func TestPetSetControllerBlocksScaling(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// set := newPetSet(3) -// ssc, spc, om, _ := newFakePetSetController(ctx, set) -// if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { -// t.Errorf("Failed to turn up PetSet : %s", err) -// } -// if obj, _, err := om.setsIndexer.Get(set); err != nil { -// t.Error(err) -// } else { -// set = obj.(*api.PetSet) -// } -// if set.Status.Replicas != 3 { -// t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) -// } -// *set.Spec.Replicas = 5 -// fakeResourceVersion(set) -// om.setsIndexer.Update(set) -// _, err := om.setPodTerminated(set, 0) -// if err != nil { -// t.Error("Failed to set pod terminated at ordinal 0") -// } -// ssc.enqueuePetSet(set) -// fakeWorker(ssc) -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Error(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if len(pods) != 3 { -// t.Error("PetSet does not block scaling") -// } -// sort.Sort(ascendingOrdinal(pods)) -// spc.DeleteStatefulPod(set, pods[0]) -// ssc.enqueuePetSet(set) -// fakeWorker(ssc) -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Error(err) -// } -// if len(pods) != 3 { -// t.Error("PetSet does not resume when terminated Pod is removed") -// } -//} -// -//func TestPetSetControllerDeletionTimestamp(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// set := newPetSet(3) -// set.DeletionTimestamp = new(metav1.Time) -// ssc, _, om, _ := newFakePetSetController(ctx, set) -// -// om.setsIndexer.Add(set) -// -// // Force a sync. It should not try to create any Pods. -// ssc.enqueuePetSet(set) -// fakeWorker(ssc) -// -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatal(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// if got, want := len(pods), 0; got != want { -// t.Errorf("len(pods) = %v, want %v", got, want) -// } -//} -// -//func TestPetSetControllerDeletionTimestampRace(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// set := newPetSet(3) -// // The bare client says it IS deleted. -// set.DeletionTimestamp = new(metav1.Time) -// ssc, _, om, ssh := newFakePetSetController(ctx, set) -// -// // The lister (cache) says it's NOT deleted. -// set2 := *set -// set2.DeletionTimestamp = nil -// om.setsIndexer.Add(&set2) -// -// // The recheck occurs in the presence of a matching orphan. -// pod := newPetSetPod(set, 1) -// pod.OwnerReferences = nil -// om.podsIndexer.Add(pod) -// set.Status.CollisionCount = new(int32) -// revision, err := newRevision(set, 1, set.Status.CollisionCount) -// if err != nil { -// t.Fatal(err) -// } -// revision.OwnerReferences = nil -// _, err = ssh.CreateControllerRevision(set, revision, set.Status.CollisionCount) -// if err != nil { -// t.Fatal(err) -// } -// -// // Force a sync. It should not try to create any Pods. -// ssc.enqueuePetSet(set) -// fakeWorker(ssc) -// -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatal(err) -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// t.Fatal(err) -// } -// if got, want := len(pods), 1; got != want { -// t.Errorf("len(pods) = %v, want %v", got, want) -// } -// -// // It should not adopt pods. -// for _, pod := range pods { -// if len(pod.OwnerReferences) > 0 { -// t.Errorf("unexpected pod owner references: %v", pod.OwnerReferences) -// } -// } -// -// // It should not adopt revisions. -// revisions, err := ssh.ListControllerRevisions(set, selector) -// if err != nil { -// t.Fatal(err) -// } -// if got, want := len(revisions), 1; got != want { -// t.Errorf("len(revisions) = %v, want %v", got, want) -// } -// for _, revision := range revisions { -// if len(revision.OwnerReferences) > 0 { -// t.Errorf("unexpected revision owner references: %v", revision.OwnerReferences) -// } -// } -//} -// -//func TestPetSetControllerAddPod(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set1 := newPetSet(3) -// set2 := newPetSet(3) -// pod1 := newPetSetPod(set1, 0) -// pod2 := newPetSetPod(set2, 0) -// om.setsIndexer.Add(set1) -// om.setsIndexer.Add(set2) -// -// ssc.addPod(logger, pod1) -// key, done := ssc.queue.Get() -// if key == nil || done { -// t.Error("failed to enqueue PetSet") -// } else if key, ok := key.(string); !ok { -// t.Error("key is not a string") -// } else if expectedKey, _ := controller.KeyFunc(set1); expectedKey != key { -// t.Errorf("expected PetSet key %s found %s", expectedKey, key) -// } -// ssc.queue.Done(key) -// -// ssc.addPod(logger, pod2) -// key, done = ssc.queue.Get() -// if key == nil || done { -// t.Error("failed to enqueue PetSet") -// } else if key, ok := key.(string); !ok { -// t.Error("key is not a string") -// } else if expectedKey, _ := controller.KeyFunc(set2); expectedKey != key { -// t.Errorf("expected PetSet key %s found %s", expectedKey, key) -// } -// ssc.queue.Done(key) -//} -// -//func TestPetSetControllerAddPodOrphan(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set1 := newPetSet(3) -// set2 := newPetSet(3) -// set2.Name = "foo2" -// set3 := newPetSet(3) -// set3.Name = "foo3" -// set3.Spec.Selector.MatchLabels = map[string]string{"foo3": "bar"} -// pod := newPetSetPod(set1, 0) -// om.setsIndexer.Add(set1) -// om.setsIndexer.Add(set2) -// om.setsIndexer.Add(set3) -// -// // Make pod an orphan. Expect matching sets to be queued. -// pod.OwnerReferences = nil -// ssc.addPod(logger, pod) -// if got, want := ssc.queue.Len(), 2; got != want { -// t.Errorf("queue.Len() = %v, want %v", got, want) -// } -//} -// -//func TestPetSetControllerAddPodNoSet(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, _, _ := newFakePetSetController(ctx) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// ssc.addPod(logger, pod) -// ssc.queue.ShutDown() -// key, _ := ssc.queue.Get() -// if key != nil { -// t.Errorf("PetSet enqueued key for Pod with no Set %s", key) -// } -//} -// -//func TestPetSetControllerUpdatePod(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set1 := newPetSet(3) -// set2 := newPetSet(3) -// set2.Name = "foo2" -// pod1 := newPetSetPod(set1, 0) -// pod2 := newPetSetPod(set2, 0) -// om.setsIndexer.Add(set1) -// om.setsIndexer.Add(set2) -// -// prev := *pod1 -// fakeResourceVersion(pod1) -// ssc.updatePod(logger, &prev, pod1) -// key, done := ssc.queue.Get() -// if key == nil || done { -// t.Error("failed to enqueue PetSet") -// } else if key, ok := key.(string); !ok { -// t.Error("key is not a string") -// } else if expectedKey, _ := controller.KeyFunc(set1); expectedKey != key { -// t.Errorf("expected PetSet key %s found %s", expectedKey, key) -// } -// -// prev = *pod2 -// fakeResourceVersion(pod2) -// ssc.updatePod(logger, &prev, pod2) -// key, done = ssc.queue.Get() -// if key == nil || done { -// t.Error("failed to enqueue PetSet") -// } else if key, ok := key.(string); !ok { -// t.Error("key is not a string") -// } else if expectedKey, _ := controller.KeyFunc(set2); expectedKey != key { -// t.Errorf("expected PetSet key %s found %s", expectedKey, key) -// } -//} -// -//func TestPetSetControllerUpdatePodWithNoSet(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, _, _ := newFakePetSetController(ctx) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// prev := *pod -// fakeResourceVersion(pod) -// ssc.updatePod(logger, &prev, pod) -// ssc.queue.ShutDown() -// key, _ := ssc.queue.Get() -// if key != nil { -// t.Errorf("PetSet enqueued key for Pod with no Set %s", key) -// } -//} -// -//func TestPetSetControllerUpdatePodWithSameVersion(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// om.setsIndexer.Add(set) -// ssc.updatePod(logger, pod, pod) -// ssc.queue.ShutDown() -// key, _ := ssc.queue.Get() -// if key != nil { -// t.Errorf("PetSet enqueued key for Pod with no Set %s", key) -// } -//} -// -//func TestPetSetControllerUpdatePodOrphanWithNewLabels(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// pod.OwnerReferences = nil -// set2 := newPetSet(3) -// set2.Name = "foo2" -// om.setsIndexer.Add(set) -// om.setsIndexer.Add(set2) -// clone := *pod -// clone.Labels = map[string]string{"foo2": "bar2"} -// fakeResourceVersion(&clone) -// ssc.updatePod(logger, &clone, pod) -// if got, want := ssc.queue.Len(), 2; got != want { -// t.Errorf("queue.Len() = %v, want %v", got, want) -// } -//} -// -//func TestPetSetControllerUpdatePodChangeControllerRef(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set := newPetSet(3) -// set2 := newPetSet(3) -// set2.Name = "foo2" -// pod := newPetSetPod(set, 0) -// pod2 := newPetSetPod(set2, 0) -// om.setsIndexer.Add(set) -// om.setsIndexer.Add(set2) -// clone := *pod -// clone.OwnerReferences = pod2.OwnerReferences -// fakeResourceVersion(&clone) -// ssc.updatePod(logger, &clone, pod) -// if got, want := ssc.queue.Len(), 2; got != want { -// t.Errorf("queue.Len() = %v, want %v", got, want) -// } -//} -// -//func TestPetSetControllerUpdatePodRelease(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set := newPetSet(3) -// set2 := newPetSet(3) -// set2.Name = "foo2" -// pod := newPetSetPod(set, 0) -// om.setsIndexer.Add(set) -// om.setsIndexer.Add(set2) -// clone := *pod -// clone.OwnerReferences = nil -// fakeResourceVersion(&clone) -// ssc.updatePod(logger, pod, &clone) -// if got, want := ssc.queue.Len(), 2; got != want { -// t.Errorf("queue.Len() = %v, want %v", got, want) -// } -//} -// -//func TestPetSetControllerDeletePod(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set1 := newPetSet(3) -// set2 := newPetSet(3) -// set2.Name = "foo2" -// pod1 := newPetSetPod(set1, 0) -// pod2 := newPetSetPod(set2, 0) -// om.setsIndexer.Add(set1) -// om.setsIndexer.Add(set2) -// -// ssc.deletePod(logger, pod1) -// key, done := ssc.queue.Get() -// if key == nil || done { -// t.Error("failed to enqueue PetSet") -// } else if key, ok := key.(string); !ok { -// t.Error("key is not a string") -// } else if expectedKey, _ := controller.KeyFunc(set1); expectedKey != key { -// t.Errorf("expected PetSet key %s found %s", expectedKey, key) -// } -// -// ssc.deletePod(logger, pod2) -// key, done = ssc.queue.Get() -// if key == nil || done { -// t.Error("failed to enqueue PetSet") -// } else if key, ok := key.(string); !ok { -// t.Error("key is not a string") -// } else if expectedKey, _ := controller.KeyFunc(set2); expectedKey != key { -// t.Errorf("expected PetSet key %s found %s", expectedKey, key) -// } -//} -// -//func TestPetSetControllerDeletePodOrphan(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set1 := newPetSet(3) -// set2 := newPetSet(3) -// set2.Name = "foo2" -// pod1 := newPetSetPod(set1, 0) -// om.setsIndexer.Add(set1) -// om.setsIndexer.Add(set2) -// -// pod1.OwnerReferences = nil -// ssc.deletePod(logger, pod1) -// if got, want := ssc.queue.Len(), 0; got != want { -// t.Errorf("queue.Len() = %v, want %v", got, want) -// } -//} -// -//func TestPetSetControllerDeletePodTombstone(t *testing.T) { -// logger, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set := newPetSet(3) -// pod := newPetSetPod(set, 0) -// om.setsIndexer.Add(set) -// tombstoneKey, _ := controller.KeyFunc(pod) -// tombstone := cache.DeletedFinalStateUnknown{Key: tombstoneKey, Obj: pod} -// ssc.deletePod(logger, tombstone) -// key, done := ssc.queue.Get() -// if key == nil || done { -// t.Error("failed to enqueue PetSet") -// } else if key, ok := key.(string); !ok { -// t.Error("key is not a string") -// } else if expectedKey, _ := controller.KeyFunc(set); expectedKey != key { -// t.Errorf("expected PetSet key %s found %s", expectedKey, key) -// } -//} -// -//func TestPetSetControllerGetPetSetsForPod(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx) -// set1 := newPetSet(3) -// set2 := newPetSet(3) -// set2.Name = "foo2" -// pod := newPetSetPod(set1, 0) -// om.setsIndexer.Add(set1) -// om.setsIndexer.Add(set2) -// om.podsIndexer.Add(pod) -// sets := ssc.getPetSetsForPod(pod) -// if got, want := len(sets), 2; got != want { -// t.Errorf("len(sets) = %v, want %v", got, want) -// } -//} -// -//func TestGetPodsForPetSetAdopt(t *testing.T) { -// set := newPetSet(5) -// pod1 := newPetSetPod(set, 1) -// // pod2 is an orphan with matching labels and name. -// pod2 := newPetSetPod(set, 2) -// pod2.OwnerReferences = nil -// // pod3 has wrong labels. -// pod3 := newPetSetPod(set, 3) -// pod3.OwnerReferences = nil -// pod3.Labels = nil -// // pod4 has wrong name. -// pod4 := newPetSetPod(set, 4) -// pod4.OwnerReferences = nil -// pod4.Name = "x" + pod4.Name -// -// _, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx, set, pod1, pod2, pod3, pod4) -// -// om.podsIndexer.Add(pod1) -// om.podsIndexer.Add(pod2) -// om.podsIndexer.Add(pod3) -// om.podsIndexer.Add(pod4) -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatal(err) -// } -// pods, err := ssc.getPodsForPetSet(context.TODO(), set, selector) -// if err != nil { -// t.Fatalf("getPodsForPetSet() error: %v", err) -// } -// got := sets.New[string]() -// for _, pod := range pods { -// got.Insert(pod.Name) -// } -// // pod2 should be claimed, pod3 and pod4 ignored -// want := sets.New[string](pod1.Name, pod2.Name) -// if !got.Equal(want) { -// t.Errorf("getPodsForPetSet() = %v, want %v", got, want) -// } -//} -// -//func TestAdoptOrphanRevisions(t *testing.T) { -// ss1 := newPetSetWithLabels(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) -// ss1.Status.CollisionCount = new(int32) -// ss1Rev1, err := history.NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) -// if err != nil { -// t.Fatal(err) -// } -// ss1Rev1.Namespace = ss1.Namespace -// ss1.Spec.Template.Annotations = make(map[string]string) -// ss1.Spec.Template.Annotations["ss1"] = "ss1" -// ss1Rev2, err := history.NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) -// if err != nil { -// t.Fatal(err) -// } -// ss1Rev2.Namespace = ss1.Namespace -// ss1Rev2.OwnerReferences = []metav1.OwnerReference{} -// -// _, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx, ss1, ss1Rev1, ss1Rev2) -// -// om.revisionsIndexer.Add(ss1Rev1) -// om.revisionsIndexer.Add(ss1Rev2) -// -// err = ssc.adoptOrphanRevisions(context.TODO(), ss1) -// if err != nil { -// t.Errorf("adoptOrphanRevisions() error: %v", err) -// } -// -// if revisions, err := ssc.control.ListRevisions(ss1); err != nil { -// t.Errorf("ListRevisions() error: %v", err) -// } else { -// var adopted bool -// for i := range revisions { -// if revisions[i].Name == ss1Rev2.Name && metav1.GetControllerOf(revisions[i]) != nil { -// adopted = true -// } -// } -// if !adopted { -// t.Error("adoptOrphanRevisions() not adopt orphan revisions") -// } -// } -//} -// -//func TestGetPodsForPetSetRelease(t *testing.T) { -// _, ctx := ktesting.NewTestContext(t) -// set := newPetSet(3) -// ssc, _, om, _ := newFakePetSetController(ctx, set) -// pod1 := newPetSetPod(set, 1) -// // pod2 is owned but has wrong name. -// pod2 := newPetSetPod(set, 2) -// pod2.Name = "x" + pod2.Name -// // pod3 is owned but has wrong labels. -// pod3 := newPetSetPod(set, 3) -// pod3.Labels = nil -// // pod4 is an orphan that doesn't match. -// pod4 := newPetSetPod(set, 4) -// pod4.OwnerReferences = nil -// pod4.Labels = nil -// -// om.podsIndexer.Add(pod1) -// om.podsIndexer.Add(pod2) -// om.podsIndexer.Add(pod3) -// om.podsIndexer.Add(pod4) -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// t.Fatal(err) -// } -// pods, err := ssc.getPodsForPetSet(context.TODO(), set, selector) -// if err != nil { -// t.Fatalf("getPodsForPetSet() error: %v", err) -// } -// got := sets.New[string]() -// for _, pod := range pods { -// got.Insert(pod.Name) -// } -// -// // Expect only pod1 (pod2 and pod3 should be released, pod4 ignored). -// want := sets.New[string](pod1.Name) -// if !got.Equal(want) { -// t.Errorf("getPodsForPetSet() = %v, want %v", got, want) -// } -//} -// -//func TestOrphanedPodsWithPVCDeletePolicy(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, true)() -// -// testFn := func(t *testing.T, scaledownPolicy, deletionPolicy apps.PersistentVolumeClaimRetentionPolicyType) { -// set := newPetSet(4) -// *set.Spec.Replicas = 2 -// set.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = scaledownPolicy -// set.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = deletionPolicy -// _, ctx := ktesting.NewTestContext(t) -// ssc, _, om, _ := newFakePetSetController(ctx, set) -// om.setsIndexer.Add(set) -// -// pods := []*v1.Pod{} -// pods = append(pods, newPetSetPod(set, 0)) -// // pod1 is orphaned -// pods = append(pods, newPetSetPod(set, 1)) -// pods[1].OwnerReferences = nil -// // pod2 is owned but has wrong name. -// pods = append(pods, newPetSetPod(set, 2)) -// pods[2].Name = "x" + pods[2].Name -// -// ssc.apiClient.(*apifake.Clientset).PrependReactor("patch", "pods", func(action core.Action) (bool, runtime.Object, error) { -// patch := action.(core.PatchAction).GetPatch() -// target := action.(core.PatchAction).GetName() -// var pod *v1.Pod -// for _, p := range pods { -// if p.Name == target { -// pod = p -// break -// } -// } -// if pod == nil { -// t.Fatalf("Can't find patch target %s", target) -// } -// original, err := json.Marshal(pod) -// if err != nil { -// t.Fatalf("failed to marshal original pod %s: %v", pod.Name, err) -// } -// updated, err := strategicpatch.StrategicMergePatch(original, patch, v1.Pod{}) -// if err != nil { -// t.Fatalf("failed to apply strategic merge patch %q on node %s: %v", patch, pod.Name, err) -// } -// if err := json.Unmarshal(updated, pod); err != nil { -// t.Fatalf("failed to unmarshal updated pod %s: %v", pod.Name, err) -// } -// -// return true, pod, nil -// }) -// -// for _, pod := range pods { -// om.podsIndexer.Add(pod) -// claims := getPersistentVolumeClaims(set, pod) -// for _, claim := range claims { -// om.CreateClaim(&claim) -// } -// } -// -// for i := range pods { -// if _, err := om.setPodReady(set, i); err != nil { -// t.Errorf("%d: %v", i, err) -// } -// if _, err := om.setPodRunning(set, i); err != nil { -// t.Errorf("%d: %v", i, err) -// } -// } -// -// // First sync to manage orphaned pod, then set replicas. -// ssc.enqueuePetSet(set) -// fakeWorker(ssc) -// *set.Spec.Replicas = 0 // Put an ownerRef for all scale-down deleted PVCs. -// ssc.enqueuePetSet(set) -// fakeWorker(ssc) -// -// hasNamedOwnerRef := func(claim *v1.PersistentVolumeClaim, name string) bool { -// for _, ownerRef := range claim.GetOwnerReferences() { -// if ownerRef.Name == name { -// return true -// } -// } -// return false -// } -// verifyOwnerRefs := func(claim *v1.PersistentVolumeClaim, condemned bool) { -// podName := getClaimPodName(set, claim) -// const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType -// const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType -// switch { -// case scaledownPolicy == retain && deletionPolicy == retain: -// if hasNamedOwnerRef(claim, podName) || hasNamedOwnerRef(claim, set.Name) { -// t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) -// } -// case scaledownPolicy == retain && deletionPolicy == delete: -// if hasNamedOwnerRef(claim, podName) || !hasNamedOwnerRef(claim, set.Name) { -// t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) -// } -// case scaledownPolicy == delete && deletionPolicy == retain: -// if hasNamedOwnerRef(claim, podName) != condemned || hasNamedOwnerRef(claim, set.Name) { -// t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) -// } -// case scaledownPolicy == delete && deletionPolicy == delete: -// if hasNamedOwnerRef(claim, podName) != condemned || !hasNamedOwnerRef(claim, set.Name) { -// t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) -// } -// } -// } -// -// claims, _ := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(labels.Everything()) -// if len(claims) != len(pods) { -// t.Errorf("Unexpected number of claims: %d", len(claims)) -// } -// for _, claim := range claims { -// // Only the first pod and the reclaimed orphan pod should have owner refs. -// switch claim.Name { -// case "datadir-foo-0", "datadir-foo-1": -// verifyOwnerRefs(claim, false) -// case "datadir-foo-2": -// if hasNamedOwnerRef(claim, getClaimPodName(set, claim)) || hasNamedOwnerRef(claim, set.Name) { -// t.Errorf("unexpected ownerRefs for %s: %v", claim.Name, claim.GetOwnerReferences()) -// } -// default: -// t.Errorf("Unexpected claim %s", claim.Name) -// } -// } -// } -// policies := []apps.PersistentVolumeClaimRetentionPolicyType{ -// apps.RetainPersistentVolumeClaimRetentionPolicyType, -// apps.DeletePersistentVolumeClaimRetentionPolicyType, -// } -// for _, scaledownPolicy := range policies { -// for _, deletionPolicy := range policies { -// testName := fmt.Sprintf("ScaleDown:%s/SetDeletion:%s", scaledownPolicy, deletionPolicy) -// t.Run(testName, func(t *testing.T) { testFn(t, scaledownPolicy, deletionPolicy) }) -// } -// } -//} -// -//func TestStaleOwnerRefOnScaleup(t *testing.T) { -// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PetSetAutoDeletePVC, true)() -// -// for _, policy := range []*apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// { -// WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// }, -// { -// WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// }, -// } { -// onPolicy := func(msg string, args ...interface{}) string { -// return fmt.Sprintf(fmt.Sprintf("(%s) %s", policy, msg), args...) -// } -// set := newPetSet(3) -// set.Spec.PersistentVolumeClaimRetentionPolicy = policy -// logger, ctx := ktesting.NewTestContext(t) -// ssc, spc, om, _ := newFakePetSetController(ctx, set) -// if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { -// t.Errorf(onPolicy("Failed to turn up PetSet : %s", err)) -// } -// var err error -// if set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name); err != nil { -// t.Errorf(onPolicy("Could not get scaled up set: %v", err)) -// } -// if set.Status.Replicas != 3 { -// t.Errorf(onPolicy("set.Status.Replicas = %v; want 3", set.Status.Replicas)) -// } -// *set.Spec.Replicas = 2 -// if err := scaleDownPetSetController(logger, set, ssc, spc, om); err != nil { -// t.Errorf(onPolicy("Failed to scale down PetSet : msg, %s", err)) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Errorf(onPolicy("Could not get scaled down PetSet: %v", err)) -// } -// if set.Status.Replicas != 2 { -// t.Errorf(onPolicy("Failed to scale petset to 2 replicas")) -// } -// -// var claim *v1.PersistentVolumeClaim -// claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2") -// if err != nil { -// t.Errorf(onPolicy("Could not find expected pvc datadir-foo-2")) -// } -// refs := claim.GetOwnerReferences() -// if len(refs) != 1 { -// t.Errorf(onPolicy("Expected only one refs: %v", refs)) -// } -// // Make the pod ref stale. -// for i := range refs { -// if refs[i].Name == "foo-2" { -// refs[i].UID = "stale" -// break -// } -// } -// claim.SetOwnerReferences(refs) -// if err = om.claimsIndexer.Update(claim); err != nil { -// t.Errorf(onPolicy("Could not update claim with new owner ref: %v", err)) -// } -// -// *set.Spec.Replicas = 3 -// // Until the stale PVC goes away, the scale up should never finish. Run 10 iterations, then delete the PVC. -// if err := scaleUpPetSetControllerBounded(logger, set, ssc, spc, om, 10); err != nil { -// t.Errorf(onPolicy("Failed attempt to scale PetSet back up: %v", err)) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Errorf(onPolicy("Could not get scaled down PetSet: %v", err)) -// } -// if set.Status.Replicas != 2 { -// t.Errorf(onPolicy("Expected set to stay at two replicas")) -// } -// -// claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2") -// if err != nil { -// t.Errorf(onPolicy("Could not find expected pvc datadir-foo-2")) -// } -// refs = claim.GetOwnerReferences() -// if len(refs) != 1 { -// t.Errorf(onPolicy("Unexpected change to condemned pvc ownerRefs: %v", refs)) -// } -// foundPodRef := false -// for i := range refs { -// if refs[i].UID == "stale" { -// foundPodRef = true -// break -// } -// } -// if !foundPodRef { -// t.Errorf(onPolicy("Claim ref unexpectedly changed: %v", refs)) -// } -// if err = om.claimsIndexer.Delete(claim); err != nil { -// t.Errorf(onPolicy("Could not delete stale pvc: %v", err)) -// } -// -// if err := scaleUpPetSetController(logger, set, ssc, spc, om); err != nil { -// t.Errorf(onPolicy("Failed to scale PetSet back up: %v", err)) -// } -// set, err = om.setsLister.PetSets(set.Namespace).Get(set.Name) -// if err != nil { -// t.Errorf(onPolicy("Could not get scaled down PetSet: %v", err)) -// } -// if set.Status.Replicas != 3 { -// t.Errorf(onPolicy("Failed to scale set back up once PVC was deleted")) -// } -// } -//} -// -//func newFakePetSetController(ctx context.Context, initialObjects ...runtime.Object) (*PetSetController, *StatefulPodControl, *fakeObjectManager, history.Interface) { -// client := fake.NewSimpleClientset(initialObjects...) -// informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) -// apiclient := apifake.NewSimpleClientset(initialObjects...) -// apiinformerFactory := apiinformers.NewSharedInformerFactory(apiclient, controller.NoResyncPeriodFunc()) -// om := newFakeObjectManager(informerFactory, apiinformerFactory) -// spc := NewStatefulPodControlFromManager(om, &noopRecorder{}) -// ssu := newFakeStatefulSetStatusUpdater(apiinformerFactory.Apps().V1().PetSets()) -// ssc := NewPetSetController( -// ctx, -// informerFactory.Core().V1().Pods(), -// apiinformerFactory.Apps().V1().PetSets(), -// informerFactory.Core().V1().PersistentVolumeClaims(), -// informerFactory.Apps().V1().ControllerRevisions(), -// client, -// apiclient, -// ) -// ssh := history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()) -// ssc.podListerSynced = alwaysReady -// ssc.setListerSynced = alwaysReady -// recorder := record.NewFakeRecorder(10) -// ssc.control = NewDefaultPetSetControl(spc, ssu, ssh, recorder) -// -// return ssc, spc, om, ssh -//} -// -//func fakeWorker(ssc *PetSetController) { -// if obj, done := ssc.queue.Get(); !done { -// ssc.sync(context.TODO(), obj.(string)) -// ssc.queue.Done(obj) -// } -//} -// -//func getPodAtOrdinal(pods []*v1.Pod, ordinal int) *v1.Pod { -// if 0 > ordinal || ordinal >= len(pods) { -// return nil -// } -// sort.Sort(ascendingOrdinal(pods)) -// return pods[ordinal] -//} -// -//func scaleUpPetSetController(logger klog.Logger, set *api.PetSet, ssc *PetSetController, spc *StatefulPodControl, om *fakeObjectManager) error { -// return scaleUpPetSetControllerBounded(logger, set, ssc, spc, om, -1) -//} -// -//func scaleUpPetSetControllerBounded(logger klog.Logger, set *api.PetSet, ssc *PetSetController, spc *StatefulPodControl, om *fakeObjectManager, maxIterations int) error { -// om.setsIndexer.Add(set) -// ssc.enqueuePetSet(set) -// fakeWorker(ssc) -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// iterations := 0 -// for (maxIterations < 0 || iterations < maxIterations) && set.Status.ReadyReplicas < *set.Spec.Replicas { -// iterations++ -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// ord := len(pods) - 1 -// if pods, err = om.setPodPending(set, ord); err != nil { -// return err -// } -// pod := getPodAtOrdinal(pods, ord) -// ssc.addPod(logger, pod) -// fakeWorker(ssc) -// pod = getPodAtOrdinal(pods, ord) -// prev := *pod -// if pods, err = om.setPodRunning(set, ord); err != nil { -// return err -// } -// pod = getPodAtOrdinal(pods, ord) -// ssc.updatePod(logger, &prev, pod) -// fakeWorker(ssc) -// pod = getPodAtOrdinal(pods, ord) -// prev = *pod -// if pods, err = om.setPodReady(set, ord); err != nil { -// return err -// } -// pod = getPodAtOrdinal(pods, ord) -// ssc.updatePod(logger, &prev, pod) -// fakeWorker(ssc) -// if err := assertMonotonicInvariants(set, om); err != nil { -// return err -// } -// obj, _, err := om.setsIndexer.Get(set) -// if err != nil { -// return err -// } -// set = obj.(*api.PetSet) -// -// } -// return assertMonotonicInvariants(set, om) -//} -// -//func scaleDownPetSetController(logger klog.Logger, set *api.PetSet, ssc *PetSetController, spc *StatefulPodControl, om *fakeObjectManager) error { -// selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) -// if err != nil { -// return err -// } -// pods, err := om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// ord := len(pods) - 1 -// pod := getPodAtOrdinal(pods, ord) -// prev := *pod -// fakeResourceVersion(set) -// om.setsIndexer.Add(set) -// ssc.enqueuePetSet(set) -// fakeWorker(ssc) -// pods, err = om.addTerminatingPod(set, ord) -// if err != nil { -// return err -// } -// pod = getPodAtOrdinal(pods, ord) -// ssc.updatePod(logger, &prev, pod) -// fakeWorker(ssc) -// spc.DeleteStatefulPod(set, pod) -// ssc.deletePod(logger, pod) -// fakeWorker(ssc) -// for set.Status.Replicas > *set.Spec.Replicas { -// pods, err = om.podsLister.Pods(set.Namespace).List(selector) -// if err != nil { -// return err -// } -// -// ord := len(pods) -// pods, err = om.addTerminatingPod(set, ord) -// if err != nil { -// return err -// } -// pod = getPodAtOrdinal(pods, ord) -// ssc.updatePod(logger, &prev, pod) -// fakeWorker(ssc) -// spc.DeleteStatefulPod(set, pod) -// ssc.deletePod(logger, pod) -// fakeWorker(ssc) -// obj, _, err := om.setsIndexer.Get(set) -// if err != nil { -// return err -// } -// set = obj.(*api.PetSet) -// -// } -// return assertMonotonicInvariants(set, om) -//} -// -//func rawTemplate(template *api.PodTemplateSpec) runtime.RawExtension { -// buf := new(bytes.Buffer) -// enc := json.NewEncoder(buf) -// if err := enc.Encode(template); err != nil { -// panic(err) -// } -// return runtime.RawExtension{Raw: buf.Bytes()} -//} diff --git a/pkg/controller/tests/pet_set_utils_test.go b/pkg/controller/tests/pet_set_utils_test.go deleted file mode 100644 index efaacccb..00000000 --- a/pkg/controller/tests/pet_set_utils_test.go +++ /dev/null @@ -1,983 +0,0 @@ -// /* -// Copyright 2016 The Kubernetes Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// */ -package petset - -// -//import ( -// "fmt" -// "math/rand" -// "reflect" -// "regexp" -// "sort" -// "strconv" -// "testing" -// "time" -// -// api "kubeops.dev/petset/apis/apps/v1" -// podutil "kubeops.dev/petset/pkg/api/v1/pod" -// "kubeops.dev/petset/pkg/controller/history" -// -// apps "k8s.io/api/apps/v1" -// v1 "k8s.io/api/core/v1" -// "k8s.io/apimachinery/pkg/api/resource" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// "k8s.io/apimachinery/pkg/runtime" -// "k8s.io/apimachinery/pkg/types" -// "k8s.io/apimachinery/pkg/util/intstr" -// "k8s.io/klog/v2" -// "k8s.io/klog/v2/ktesting" -// "k8s.io/utils/ptr" -//) -// -//// noopRecorder is an EventRecorder that does nothing. record.FakeRecorder has a fixed -//// buffer size, which causes tests to hang if that buffer's exceeded. -//type noopRecorder struct{} -// -//func (r *noopRecorder) Event(object runtime.Object, eventtype, reason, message string) {} -//func (r *noopRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { -//} -// -//func (r *noopRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { -//} -// -//// getClaimPodName gets the name of the Pod associated with the Claim, or an empty string if this doesn't look matching. -//func getClaimPodName(set *api.PetSet, claim *v1.PersistentVolumeClaim) string { -// podName := "" -// -// statefulClaimRegex := regexp.MustCompile(fmt.Sprintf(".*-(%s-[0-9]+)$", set.Name)) -// matches := statefulClaimRegex.FindStringSubmatch(claim.Name) -// if len(matches) != 2 { -// return podName -// } -// return matches[1] -//} -// -//func TestGetParentNameAndOrdinal(t *testing.T) { -// set := newPetSet(3) -// pod := newPetSetPod(set, 1) -// if parent, ordinal := getParentNameAndOrdinal(pod); parent != set.Name { -// t.Errorf("Extracted the wrong parent name expected %s found %s", set.Name, parent) -// } else if ordinal != 1 { -// t.Errorf("Extracted the wrong ordinal expected %d found %d", 1, ordinal) -// } -// pod.Name = "1-bar" -// if parent, ordinal := getParentNameAndOrdinal(pod); parent != "" { -// t.Error("Expected empty string for non-member Pod parent") -// } else if ordinal != -1 { -// t.Error("Expected -1 for non member Pod ordinal") -// } -//} -// -//func TestGetClaimPodName(t *testing.T) { -// set := api.PetSet{} -// set.Name = "my-set" -// claim := v1.PersistentVolumeClaim{} -// claim.Name = "volume-my-set-2" -// if pod := getClaimPodName(&set, &claim); pod != "my-set-2" { -// t.Errorf("Expected my-set-2 found %s", pod) -// } -// claim.Name = "long-volume-my-set-20" -// if pod := getClaimPodName(&set, &claim); pod != "my-set-20" { -// t.Errorf("Expected my-set-20 found %s", pod) -// } -// claim.Name = "volume-2-my-set" -// if pod := getClaimPodName(&set, &claim); pod != "" { -// t.Errorf("Expected empty string found %s", pod) -// } -// claim.Name = "volume-pod-2" -// if pod := getClaimPodName(&set, &claim); pod != "" { -// t.Errorf("Expected empty string found %s", pod) -// } -//} -// -//func TestIsMemberOf(t *testing.T) { -// set := newPetSet(3) -// set2 := newPetSet(3) -// set2.Name = "foo2" -// pod := newPetSetPod(set, 1) -// if !isMemberOf(set, pod) { -// t.Error("isMemberOf returned false negative") -// } -// if isMemberOf(set2, pod) { -// t.Error("isMemberOf returned false positive") -// } -//} -// -//func TestIdentityMatches(t *testing.T) { -// set := newPetSet(3) -// pod := newPetSetPod(set, 1) -// if !identityMatches(set, pod) { -// t.Error("Newly created Pod has a bad identity") -// } -// pod.Name = "foo" -// if identityMatches(set, pod) { -// t.Error("identity matches for a Pod with the wrong name") -// } -// pod = newPetSetPod(set, 1) -// pod.Namespace = "" -// if identityMatches(set, pod) { -// t.Error("identity matches for a Pod with the wrong namespace") -// } -// pod = newPetSetPod(set, 1) -// delete(pod.Labels, apps.StatefulSetPodNameLabel) -// if identityMatches(set, pod) { -// t.Error("identity matches for a Pod with the wrong statefulSetPodNameLabel") -// } -//} -// -//func TestStorageMatches(t *testing.T) { -// set := newPetSet(3) -// pod := newPetSetPod(set, 1) -// if !storageMatches(set, pod) { -// t.Error("Newly created Pod has a invalid storage") -// } -// pod.Spec.Volumes = nil -// if storageMatches(set, pod) { -// t.Error("Pod with invalid Volumes has valid storage") -// } -// pod = newPetSetPod(set, 1) -// for i := range pod.Spec.Volumes { -// pod.Spec.Volumes[i].PersistentVolumeClaim = nil -// } -// if storageMatches(set, pod) { -// t.Error("Pod with invalid Volumes claim valid storage") -// } -// pod = newPetSetPod(set, 1) -// for i := range pod.Spec.Volumes { -// if pod.Spec.Volumes[i].PersistentVolumeClaim != nil { -// pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName = "foo" -// } -// } -// if storageMatches(set, pod) { -// t.Error("Pod with invalid Volumes claim valid storage") -// } -// pod = newPetSetPod(set, 1) -// pod.Name = "bar" -// if storageMatches(set, pod) { -// t.Error("Pod with invalid ordinal has valid storage") -// } -//} -// -//func TestUpdateIdentity(t *testing.T) { -// set := newPetSet(3) -// pod := newPetSetPod(set, 1) -// if !identityMatches(set, pod) { -// t.Error("Newly created Pod has a bad identity") -// } -// pod.Namespace = "" -// if identityMatches(set, pod) { -// t.Error("identity matches for a Pod with the wrong namespace") -// } -// updateIdentity(set, pod) -// if !identityMatches(set, pod) { -// t.Error("updateIdentity failed to update the Pods namespace") -// } -// delete(pod.Labels, apps.StatefulSetPodNameLabel) -// updateIdentity(set, pod) -// if !identityMatches(set, pod) { -// t.Error("updateIdentity failed to restore the statefulSetPodName label") -// } -//} -// -//func TestUpdateStorage(t *testing.T) { -// set := newPetSet(3) -// pod := newPetSetPod(set, 1) -// if !storageMatches(set, pod) { -// t.Error("Newly created Pod has a invalid storage") -// } -// pod.Spec.Volumes = nil -// if storageMatches(set, pod) { -// t.Error("Pod with invalid Volumes has valid storage") -// } -// updateStorage(set, pod) -// if !storageMatches(set, pod) { -// t.Error("updateStorage failed to recreate volumes") -// } -// pod = newPetSetPod(set, 1) -// for i := range pod.Spec.Volumes { -// pod.Spec.Volumes[i].PersistentVolumeClaim = nil -// } -// if storageMatches(set, pod) { -// t.Error("Pod with invalid Volumes claim valid storage") -// } -// updateStorage(set, pod) -// if !storageMatches(set, pod) { -// t.Error("updateStorage failed to recreate volume claims") -// } -// pod = newPetSetPod(set, 1) -// for i := range pod.Spec.Volumes { -// if pod.Spec.Volumes[i].PersistentVolumeClaim != nil { -// pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName = "foo" -// } -// } -// if storageMatches(set, pod) { -// t.Error("Pod with invalid Volumes claim valid storage") -// } -// updateStorage(set, pod) -// if !storageMatches(set, pod) { -// t.Error("updateStorage failed to recreate volume claim names") -// } -//} -// -//func TestGetPersistentVolumeClaimRetentionPolicy(t *testing.T) { -// retainPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// } -// scaledownPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// } -// -// set := api.PetSet{} -// set.Spec.PersistentVolumeClaimRetentionPolicy = &retainPolicy -// got := getPersistentVolumeClaimRetentionPolicy(&set) -// if got.WhenScaled != apps.RetainPersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType { -// t.Errorf("Expected retain policy") -// } -// set.Spec.PersistentVolumeClaimRetentionPolicy = &scaledownPolicy -// got = getPersistentVolumeClaimRetentionPolicy(&set) -// if got.WhenScaled != apps.DeletePersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType { -// t.Errorf("Expected scaledown policy") -// } -//} -// -//func TestClaimOwnerMatchesSetAndPod(t *testing.T) { -// testCases := []struct { -// name string -// scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType -// setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType -// needsPodRef bool -// needsSetRef bool -// replicas int32 -// ordinal int -// }{ -// { -// name: "retain", -// scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// needsPodRef: false, -// needsSetRef: false, -// }, -// { -// name: "on SS delete", -// scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// needsPodRef: false, -// needsSetRef: true, -// }, -// { -// name: "on scaledown only, condemned", -// scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// needsPodRef: true, -// needsSetRef: false, -// replicas: 2, -// ordinal: 2, -// }, -// { -// name: "on scaledown only, remains", -// scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// needsPodRef: false, -// needsSetRef: false, -// replicas: 2, -// ordinal: 1, -// }, -// { -// name: "on both, condemned", -// scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// needsPodRef: true, -// needsSetRef: false, -// replicas: 2, -// ordinal: 2, -// }, -// { -// name: "on both, remains", -// scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// needsPodRef: false, -// needsSetRef: true, -// replicas: 2, -// ordinal: 1, -// }, -// } -// -// for _, tc := range testCases { -// for _, useOtherRefs := range []bool{false, true} { -// for _, setPodRef := range []bool{false, true} { -// for _, setSetRef := range []bool{false, true} { -// _, ctx := ktesting.NewTestContext(t) -// logger := klog.FromContext(ctx) -// claim := v1.PersistentVolumeClaim{} -// claim.Name = "target-claim" -// pod := v1.Pod{} -// pod.Name = fmt.Sprintf("pod-%d", tc.ordinal) -// pod.GetObjectMeta().SetUID("pod-123") -// set := api.PetSet{} -// set.Name = "stateful-set" -// set.GetObjectMeta().SetUID("ss-456") -// set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenScaled: tc.scaleDownPolicy, -// WhenDeleted: tc.setDeletePolicy, -// } -// set.Spec.Replicas = &tc.replicas -// if setPodRef { -// setOwnerRef(&claim, &pod, &pod.TypeMeta) -// } -// if setSetRef { -// setOwnerRef(&claim, &set, &set.TypeMeta) -// } -// if useOtherRefs { -// randomObject1 := v1.Pod{} -// randomObject1.Name = "rand1" -// randomObject1.GetObjectMeta().SetUID("rand1-abc") -// randomObject2 := v1.Pod{} -// randomObject2.Name = "rand2" -// randomObject2.GetObjectMeta().SetUID("rand2-def") -// setOwnerRef(&claim, &randomObject1, &randomObject1.TypeMeta) -// setOwnerRef(&claim, &randomObject2, &randomObject2.TypeMeta) -// } -// shouldMatch := setPodRef == tc.needsPodRef && setSetRef == tc.needsSetRef -// if claimOwnerMatchesSetAndPod(logger, &claim, &set, &pod) != shouldMatch { -// t.Errorf("Bad match for %s with pod=%v,set=%v,others=%v", tc.name, setPodRef, setSetRef, useOtherRefs) -// } -// } -// } -// } -// } -//} -// -//func TestUpdateClaimOwnerRefForSetAndPod(t *testing.T) { -// testCases := []struct { -// name string -// scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType -// setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType -// condemned bool -// needsPodRef bool -// needsSetRef bool -// }{ -// { -// name: "retain", -// scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// condemned: false, -// needsPodRef: false, -// needsSetRef: false, -// }, -// { -// name: "delete with set", -// scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// condemned: false, -// needsPodRef: false, -// needsSetRef: true, -// }, -// { -// name: "delete with scaledown, not condemned", -// scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// condemned: false, -// needsPodRef: false, -// needsSetRef: false, -// }, -// { -// name: "delete on scaledown, condemned", -// scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// condemned: true, -// needsPodRef: true, -// needsSetRef: false, -// }, -// { -// name: "delete on both, not condemned", -// scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// condemned: false, -// needsPodRef: false, -// needsSetRef: true, -// }, -// { -// name: "delete on both, condemned", -// scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, -// condemned: true, -// needsPodRef: true, -// needsSetRef: false, -// }, -// } -// for _, tc := range testCases { -// for _, hasPodRef := range []bool{true, false} { -// for _, hasSetRef := range []bool{true, false} { -// _, ctx := ktesting.NewTestContext(t) -// logger := klog.FromContext(ctx) -// set := api.PetSet{} -// set.Name = "ss" -// numReplicas := int32(5) -// set.Spec.Replicas = &numReplicas -// set.SetUID("ss-123") -// set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenScaled: tc.scaleDownPolicy, -// WhenDeleted: tc.setDeletePolicy, -// } -// pod := v1.Pod{} -// if tc.condemned { -// pod.Name = "pod-8" -// } else { -// pod.Name = "pod-1" -// } -// pod.SetUID("pod-456") -// claim := v1.PersistentVolumeClaim{} -// if hasPodRef { -// setOwnerRef(&claim, &pod, &pod.TypeMeta) -// } -// if hasSetRef { -// setOwnerRef(&claim, &set, &set.TypeMeta) -// } -// needsUpdate := hasPodRef != tc.needsPodRef || hasSetRef != tc.needsSetRef -// shouldUpdate := updateClaimOwnerRefForSetAndPod(logger, &claim, &set, &pod) -// if shouldUpdate != needsUpdate { -// t.Errorf("Bad update for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) -// } -// if hasOwnerRef(&claim, &pod) != tc.needsPodRef { -// t.Errorf("Bad pod ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) -// } -// if hasOwnerRef(&claim, &set) != tc.needsSetRef { -// t.Errorf("Bad set ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) -// } -// } -// } -// } -//} -// -//func TestHasOwnerRef(t *testing.T) { -// target := v1.Pod{} -// target.SetOwnerReferences([]metav1.OwnerReference{ -// {UID: "123"}, {UID: "456"}, -// }) -// ownerA := v1.Pod{} -// ownerA.GetObjectMeta().SetUID("123") -// ownerB := v1.Pod{} -// ownerB.GetObjectMeta().SetUID("789") -// if !hasOwnerRef(&target, &ownerA) { -// t.Error("Missing owner") -// } -// if hasOwnerRef(&target, &ownerB) { -// t.Error("Unexpected owner") -// } -//} -// -//func TestHasStaleOwnerRef(t *testing.T) { -// target := v1.Pod{} -// target.SetOwnerReferences([]metav1.OwnerReference{ -// {Name: "bob", UID: "123"}, {Name: "shirley", UID: "456"}, -// }) -// ownerA := v1.Pod{} -// ownerA.SetUID("123") -// ownerA.Name = "bob" -// ownerB := v1.Pod{} -// ownerB.Name = "shirley" -// ownerB.SetUID("789") -// ownerC := v1.Pod{} -// ownerC.Name = "yvonne" -// ownerC.SetUID("345") -// if hasStaleOwnerRef(&target, &ownerA) { -// t.Error("ownerA should not be stale") -// } -// if !hasStaleOwnerRef(&target, &ownerB) { -// t.Error("ownerB should be stale") -// } -// if hasStaleOwnerRef(&target, &ownerC) { -// t.Error("ownerC should not be stale") -// } -//} -// -//func TestSetOwnerRef(t *testing.T) { -// target := v1.Pod{} -// ownerA := v1.Pod{} -// ownerA.Name = "A" -// ownerA.GetObjectMeta().SetUID("ABC") -// if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != true { -// t.Errorf("Unexpected lack of update") -// } -// ownerRefs := target.GetObjectMeta().GetOwnerReferences() -// if len(ownerRefs) != 1 { -// t.Errorf("Unexpected owner ref count: %d", len(ownerRefs)) -// } -// if ownerRefs[0].UID != "ABC" { -// t.Errorf("Unexpected owner UID %v", ownerRefs[0].UID) -// } -// if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != false { -// t.Errorf("Unexpected update") -// } -// if len(target.GetObjectMeta().GetOwnerReferences()) != 1 { -// t.Error("Unexpected duplicate reference") -// } -// ownerB := v1.Pod{} -// ownerB.Name = "B" -// ownerB.GetObjectMeta().SetUID("BCD") -// if setOwnerRef(&target, &ownerB, &ownerB.TypeMeta) != true { -// t.Error("Unexpected lack of second update") -// } -// ownerRefs = target.GetObjectMeta().GetOwnerReferences() -// if len(ownerRefs) != 2 { -// t.Errorf("Unexpected owner ref count: %d", len(ownerRefs)) -// } -// if ownerRefs[0].UID != "ABC" || ownerRefs[1].UID != "BCD" { -// t.Errorf("Bad second ownerRefs: %v", ownerRefs) -// } -//} -// -//func TestRemoveOwnerRef(t *testing.T) { -// target := v1.Pod{} -// ownerA := v1.Pod{} -// ownerA.Name = "A" -// ownerA.GetObjectMeta().SetUID("ABC") -// if removeOwnerRef(&target, &ownerA) != false { -// t.Error("Unexpected update on empty remove") -// } -// setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) -// if removeOwnerRef(&target, &ownerA) != true { -// t.Error("Unexpected lack of update") -// } -// if len(target.GetObjectMeta().GetOwnerReferences()) != 0 { -// t.Error("Unexpected owner reference remains") -// } -// -// ownerB := v1.Pod{} -// ownerB.Name = "B" -// ownerB.GetObjectMeta().SetUID("BCD") -// -// setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) -// if removeOwnerRef(&target, &ownerB) != false { -// t.Error("Unexpected update for mismatched owner") -// } -// if len(target.GetObjectMeta().GetOwnerReferences()) != 1 { -// t.Error("Missing ref after no-op remove") -// } -// setOwnerRef(&target, &ownerB, &ownerB.TypeMeta) -// if removeOwnerRef(&target, &ownerA) != true { -// t.Error("Missing update for second remove") -// } -// ownerRefs := target.GetObjectMeta().GetOwnerReferences() -// if len(ownerRefs) != 1 { -// t.Error("Extra ref after second remove") -// } -// if ownerRefs[0].UID != "BCD" { -// t.Error("Bad UID after second remove") -// } -//} -// -//func TestIsRunningAndReady(t *testing.T) { -// set := newPetSet(3) -// pod := newPetSetPod(set, 1) -// if isRunningAndReady(pod) { -// t.Error("isRunningAndReady does not respect Pod phase") -// } -// pod.Status.Phase = v1.PodRunning -// if isRunningAndReady(pod) { -// t.Error("isRunningAndReady does not respect Pod condition") -// } -// condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} -// podutil.UpdatePodCondition(&pod.Status, &condition) -// if !isRunningAndReady(pod) { -// t.Error("Pod should be running and ready") -// } -//} -// -//func TestAscendingOrdinal(t *testing.T) { -// set := newPetSet(10) -// pods := make([]*v1.Pod, 10) -// perm := rand.Perm(10) -// for i, v := range perm { -// pods[i] = newPetSetPod(set, v) -// } -// sort.Sort(ascendingOrdinal(pods)) -// if !sort.IsSorted(ascendingOrdinal(pods)) { -// t.Error("ascendingOrdinal fails to sort Pods") -// } -//} -// -//func TestOverlappingPetSets(t *testing.T) { -// sets := make([]*api.PetSet, 10) -// perm := rand.Perm(10) -// for i, v := range perm { -// sets[i] = newPetSet(10) -// sets[i].CreationTimestamp = metav1.NewTime(sets[i].CreationTimestamp.Add(time.Duration(v) * time.Second)) -// } -// sort.Sort(overlappingPetSets(sets)) -// if !sort.IsSorted(overlappingPetSets(sets)) { -// t.Error("ascendingOrdinal fails to sort Pods") -// } -// for i, v := range perm { -// sets[i] = newPetSet(10) -// sets[i].Name = strconv.FormatInt(int64(v), 10) -// } -// sort.Sort(overlappingPetSets(sets)) -// if !sort.IsSorted(overlappingPetSets(sets)) { -// t.Error("ascendingOrdinal fails to sort Pods") -// } -//} -// -//func TestNewPodControllerRef(t *testing.T) { -// set := newPetSet(1) -// pod := newPetSetPod(set, 0) -// controllerRef := metav1.GetControllerOf(pod) -// if controllerRef == nil { -// t.Fatalf("No ControllerRef found on new pod") -// } -// if got, want := controllerRef.APIVersion, api.SchemeGroupVersion.String(); got != want { -// t.Errorf("controllerRef.APIVersion = %q, want %q", got, want) -// } -// if got, want := controllerRef.Kind, "PetSet"; got != want { -// t.Errorf("controllerRef.Kind = %q, want %q", got, want) -// } -// if got, want := controllerRef.Name, set.Name; got != want { -// t.Errorf("controllerRef.Name = %q, want %q", got, want) -// } -// if got, want := controllerRef.UID, set.UID; got != want { -// t.Errorf("controllerRef.UID = %q, want %q", got, want) -// } -// if got, want := *controllerRef.Controller, true; got != want { -// t.Errorf("controllerRef.Controller = %v, want %v", got, want) -// } -//} -// -//func TestCreateApplyRevision(t *testing.T) { -// set := newPetSet(1) -// set.Status.CollisionCount = new(int32) -// revision, err := newRevision(set, 1, set.Status.CollisionCount) -// if err != nil { -// t.Fatal(err) -// } -// set.Spec.Template.Spec.Containers[0].Name = "foo" -// if set.Annotations == nil { -// set.Annotations = make(map[string]string) -// } -// key := "foo" -// expectedValue := "bar" -// set.Annotations[key] = expectedValue -// restoredSet, err := ApplyRevision(set, revision) -// if err != nil { -// t.Fatal(err) -// } -// restoredRevision, err := newRevision(restoredSet, 2, restoredSet.Status.CollisionCount) -// if err != nil { -// t.Fatal(err) -// } -// if !history.EqualRevision(revision, restoredRevision) { -// t.Errorf("wanted %v got %v", string(revision.Data.Raw), string(restoredRevision.Data.Raw)) -// } -// value, ok := restoredRevision.Annotations[key] -// if !ok { -// t.Errorf("missing annotation %s", key) -// } -// if value != expectedValue { -// t.Errorf("for annotation %s wanted %s got %s", key, expectedValue, value) -// } -//} -// -//func TestRollingUpdateApplyRevision(t *testing.T) { -// set := newPetSet(1) -// set.Status.CollisionCount = new(int32) -// currentSet := set.DeepCopy() -// currentRevision, err := newRevision(set, 1, set.Status.CollisionCount) -// if err != nil { -// t.Fatal(err) -// } -// -// set.Spec.Template.Spec.Containers[0].Env = []v1.EnvVar{{Name: "foo", Value: "bar"}} -// updateSet := set.DeepCopy() -// updateRevision, err := newRevision(set, 2, set.Status.CollisionCount) -// if err != nil { -// t.Fatal(err) -// } -// -// restoredCurrentSet, err := ApplyRevision(set, currentRevision) -// if err != nil { -// t.Fatal(err) -// } -// if !reflect.DeepEqual(currentSet.Spec.Template, restoredCurrentSet.Spec.Template) { -// t.Errorf("want %v got %v", currentSet.Spec.Template, restoredCurrentSet.Spec.Template) -// } -// -// restoredUpdateSet, err := ApplyRevision(set, updateRevision) -// if err != nil { -// t.Fatal(err) -// } -// if !reflect.DeepEqual(updateSet.Spec.Template, restoredUpdateSet.Spec.Template) { -// t.Errorf("want %v got %v", updateSet.Spec.Template, restoredUpdateSet.Spec.Template) -// } -//} -// -//func TestGetPersistentVolumeClaims(t *testing.T) { -// // nil inherits petset labels -// pod := newPod() -// statefulSet := newPetSet(1) -// statefulSet.Spec.Selector.MatchLabels = nil -// claims := getPersistentVolumeClaims(statefulSet, pod) -// pvc := newPVC("datadir-foo-0") -// resultClaims := map[string]v1.PersistentVolumeClaim{"datadir": pvc} -// -// if !reflect.DeepEqual(claims, resultClaims) { -// t.Fatalf("Unexpected pvc:\n %+v\n, desired pvc:\n %+v", claims, resultClaims) -// } -// -// // nil inherits petset labels -// statefulSet.Spec.Selector.MatchLabels = map[string]string{"test": "test"} -// claims = getPersistentVolumeClaims(statefulSet, pod) -// pvc.SetLabels(map[string]string{"test": "test"}) -// resultClaims = map[string]v1.PersistentVolumeClaim{"datadir": pvc} -// if !reflect.DeepEqual(claims, resultClaims) { -// t.Fatalf("Unexpected pvc:\n %+v\n, desired pvc:\n %+v", claims, resultClaims) -// } -// -// // non-nil with non-overlapping labels merge pvc and petset labels -// statefulSet.Spec.Selector.MatchLabels = map[string]string{"name": "foo"} -// statefulSet.Spec.VolumeClaimTemplates[0].ObjectMeta.Labels = map[string]string{"test": "test"} -// claims = getPersistentVolumeClaims(statefulSet, pod) -// pvc.SetLabels(map[string]string{"test": "test", "name": "foo"}) -// resultClaims = map[string]v1.PersistentVolumeClaim{"datadir": pvc} -// if !reflect.DeepEqual(claims, resultClaims) { -// t.Fatalf("Unexpected pvc:\n %+v\n, desired pvc:\n %+v", claims, resultClaims) -// } -// -// // non-nil with overlapping labels merge pvc and petset labels and prefer petset labels -// statefulSet.Spec.Selector.MatchLabels = map[string]string{"test": "foo"} -// statefulSet.Spec.VolumeClaimTemplates[0].ObjectMeta.Labels = map[string]string{"test": "test"} -// claims = getPersistentVolumeClaims(statefulSet, pod) -// pvc.SetLabels(map[string]string{"test": "foo"}) -// resultClaims = map[string]v1.PersistentVolumeClaim{"datadir": pvc} -// if !reflect.DeepEqual(claims, resultClaims) { -// t.Fatalf("Unexpected pvc:\n %+v\n, desired pvc:\n %+v", claims, resultClaims) -// } -//} -// -//func newPod() *v1.Pod { -// return &v1.Pod{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: "foo-0", -// Namespace: v1.NamespaceDefault, -// }, -// Spec: v1.PodSpec{ -// Containers: []v1.Container{ -// { -// Name: "nginx", -// Image: "nginx", -// }, -// }, -// }, -// } -//} -// -//func newPVC(name string) v1.PersistentVolumeClaim { -// return v1.PersistentVolumeClaim{ -// ObjectMeta: metav1.ObjectMeta{ -// Namespace: v1.NamespaceDefault, -// Name: name, -// }, -// Spec: v1.PersistentVolumeClaimSpec{ -// Resources: v1.VolumeResourceRequirements{ -// Requests: v1.ResourceList{ -// v1.ResourceStorage: *resource.NewQuantity(1, resource.BinarySI), -// }, -// }, -// }, -// } -//} -// -//func newPetSetWithVolumes(replicas int32, name string, petMounts []v1.VolumeMount, podMounts []v1.VolumeMount) *api.PetSet { -// mounts := append(petMounts, podMounts...) -// claims := []v1.PersistentVolumeClaim{} -// for _, m := range petMounts { -// claims = append(claims, newPVC(m.Name)) -// } -// -// vols := []v1.Volume{} -// for _, m := range podMounts { -// vols = append(vols, v1.Volume{ -// Name: m.Name, -// VolumeSource: v1.VolumeSource{ -// HostPath: &v1.HostPathVolumeSource{ -// Path: fmt.Sprintf("/tmp/%v", m.Name), -// }, -// }, -// }) -// } -// -// template := api.PodTemplateSpec{ -// Spec: v1.PodSpec{ -// Containers: []v1.Container{ -// { -// Name: "nginx", -// Image: "nginx", -// VolumeMounts: mounts, -// }, -// }, -// Volumes: vols, -// }, -// } -// -// template.Labels = map[string]string{"foo": "bar"} -// -// return &api.PetSet{ -// TypeMeta: metav1.TypeMeta{ -// Kind: "PetSet", -// APIVersion: "apps/v1", -// }, -// ObjectMeta: metav1.ObjectMeta{ -// Name: name, -// Namespace: v1.NamespaceDefault, -// UID: types.UID("test"), -// }, -// Spec: api.PetSetSpec{ -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{"foo": "bar"}, -// }, -// Replicas: ptr.To(replicas), -// Template: template, -// VolumeClaimTemplates: claims, -// ServiceName: "governingsvc", -// UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, -// PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// }, -// RevisionHistoryLimit: func() *int32 { -// limit := int32(2) -// return &limit -// }(), -// }, -// } -//} -// -//func newPetSet(replicas int32) *api.PetSet { -// petMounts := []v1.VolumeMount{ -// {Name: "datadir", MountPath: "/tmp/zookeeper"}, -// } -// podMounts := []v1.VolumeMount{ -// {Name: "home", MountPath: "/home"}, -// } -// return newPetSetWithVolumes(replicas, "foo", petMounts, podMounts) -//} -// -//func newPetSetWithLabels(replicas int32, name string, uid types.UID, labels map[string]string) *api.PetSet { -// // Converting all the map-only selectors to set-based selectors. -// var testMatchExpressions []metav1.LabelSelectorRequirement -// for key, value := range labels { -// sel := metav1.LabelSelectorRequirement{ -// Key: key, -// Operator: metav1.LabelSelectorOpIn, -// Values: []string{value}, -// } -// testMatchExpressions = append(testMatchExpressions, sel) -// } -// return &api.PetSet{ -// TypeMeta: metav1.TypeMeta{ -// Kind: "PetSet", -// APIVersion: "apps/v1", -// }, -// ObjectMeta: metav1.ObjectMeta{ -// Name: name, -// Namespace: v1.NamespaceDefault, -// UID: uid, -// }, -// Spec: api.PetSetSpec{ -// Selector: &metav1.LabelSelector{ -// // Purposely leaving MatchLabels nil, so to ensure it will break if any link -// // in the chain ignores the set-based MatchExpressions. -// MatchLabels: nil, -// MatchExpressions: testMatchExpressions, -// }, -// Replicas: ptr.To(replicas), -// PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ -// WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, -// }, -// Template: api.PodTemplateSpec{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: labels, -// }, -// Spec: v1.PodSpec{ -// Containers: []v1.Container{ -// { -// Name: "nginx", -// Image: "nginx", -// VolumeMounts: []v1.VolumeMount{ -// {Name: "datadir", MountPath: "/tmp/"}, -// {Name: "home", MountPath: "/home"}, -// }, -// }, -// }, -// Volumes: []v1.Volume{{ -// Name: "home", -// VolumeSource: v1.VolumeSource{ -// HostPath: &v1.HostPathVolumeSource{ -// Path: fmt.Sprintf("/tmp/%v", "home"), -// }, -// }, -// }}, -// }, -// }, -// VolumeClaimTemplates: []v1.PersistentVolumeClaim{ -// { -// ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "datadir"}, -// Spec: v1.PersistentVolumeClaimSpec{ -// Resources: v1.VolumeResourceRequirements{ -// Requests: v1.ResourceList{ -// v1.ResourceStorage: *resource.NewQuantity(1, resource.BinarySI), -// }, -// }, -// }, -// }, -// }, -// ServiceName: "governingsvc", -// }, -// } -//} -// -//func TestGetPetSetMaxUnavailable(t *testing.T) { -// testCases := []struct { -// maxUnavailable *intstr.IntOrString -// replicaCount int -// expectedMaxUnavailable int -// }{ -// // it wouldn't hurt to also test 0 and 0%, even if they should have been forbidden by API validation. -// {maxUnavailable: nil, replicaCount: 10, expectedMaxUnavailable: 1}, -// {maxUnavailable: ptr.To(intstr.FromInt32(3)), replicaCount: 10, expectedMaxUnavailable: 3}, -// {maxUnavailable: ptr.To(intstr.FromInt32(3)), replicaCount: 0, expectedMaxUnavailable: 3}, -// {maxUnavailable: ptr.To(intstr.FromInt32(0)), replicaCount: 0, expectedMaxUnavailable: 1}, -// {maxUnavailable: ptr.To(intstr.FromString("10%")), replicaCount: 25, expectedMaxUnavailable: 2}, -// {maxUnavailable: ptr.To(intstr.FromString("100%")), replicaCount: 5, expectedMaxUnavailable: 5}, -// {maxUnavailable: ptr.To(intstr.FromString("50%")), replicaCount: 5, expectedMaxUnavailable: 2}, -// {maxUnavailable: ptr.To(intstr.FromString("10%")), replicaCount: 5, expectedMaxUnavailable: 1}, -// {maxUnavailable: ptr.To(intstr.FromString("1%")), replicaCount: 0, expectedMaxUnavailable: 1}, -// {maxUnavailable: ptr.To(intstr.FromString("0%")), replicaCount: 0, expectedMaxUnavailable: 1}, -// } -// -// for i, tc := range testCases { -// t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { -// gotMaxUnavailable, err := getPetSetMaxUnavailable(tc.maxUnavailable, tc.replicaCount) -// if err != nil { -// t.Fatal(err) -// } -// if gotMaxUnavailable != tc.expectedMaxUnavailable { -// t.Errorf("Expected maxUnavailable %v, got pods %v", tc.expectedMaxUnavailable, gotMaxUnavailable) -// } -// }) -// } -//} diff --git a/vendor/k8s.io/component-base/featuregate/testing/feature_gate.go b/vendor/k8s.io/component-base/featuregate/testing/feature_gate.go new file mode 100644 index 00000000..1d7fc467 --- /dev/null +++ b/vendor/k8s.io/component-base/featuregate/testing/feature_gate.go @@ -0,0 +1,200 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "strings" + "sync" + + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" +) + +var ( + overrideLock sync.Mutex + featureFlagOverride map[featuregate.Feature]string + emulationVersionOverride string + emulationVersionOverrideValue *version.Version +) + +func init() { + featureFlagOverride = map[featuregate.Feature]string{} +} + +// SetFeatureGateDuringTest sets the specified gate to the specified value for duration of the test. +// Fails when it detects second call to the same flag or is unable to set or restore feature flag. +// +// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set the same feature flag will cause fatal. +// +// Example use: +// +// featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., true) +func SetFeatureGateDuringTest(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) { + tb.Helper() + detectParallelOverrideCleanup := detectParallelOverride(tb, f) + originalValue := gate.Enabled(f) + originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + originalExplicitlySet := gate.(featuregate.MutableVersionedFeatureGate).ExplicitlySet(f) + + // Specially handle AllAlpha and AllBeta + if f == "AllAlpha" || f == "AllBeta" { + // Iterate over individual gates so their individual values get restored + for k, v := range gate.(featuregate.MutableFeatureGate).GetAll() { + if k == "AllAlpha" || k == "AllBeta" { + continue + } + if (f == "AllAlpha" && v.PreRelease == featuregate.Alpha) || (f == "AllBeta" && v.PreRelease == featuregate.Beta) { + SetFeatureGateDuringTest(tb, gate, k, value) + } + } + } + + if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, value)); err != nil { + if s := suggestChangeEmulationVersion(tb, gate, f, value); s != "" { + tb.Errorf("error setting %s=%v: %v. %s", f, value, err, s) + } else { + tb.Errorf("error setting %s=%v: %v", f, value, err) + } + } + + tb.Cleanup(func() { + tb.Helper() + detectParallelOverrideCleanup() + emuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + if !emuVer.EqualTo(originalEmuVer) { + tb.Fatalf("change of feature gate emulation version from %s to %s in the chain of SetFeatureGateDuringTest is not allowed\nuse SetFeatureGateEmulationVersionDuringTest to change emulation version in tests", + originalEmuVer.String(), emuVer.String()) + } + if originalExplicitlySet { + if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil { + tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + } + } else { + if err := gate.(featuregate.MutableVersionedFeatureGate).ResetFeatureValueToDefault(f); err != nil { + tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + } + } + }) +} + +func suggestChangeEmulationVersion(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) string { + mutableVersionedFeatureGate, ok := gate.(featuregate.MutableVersionedFeatureGate) + if !ok { + return "" + } + + emuVer := mutableVersionedFeatureGate.EmulationVersion() + versionedSpecs, ok := mutableVersionedFeatureGate.GetAllVersioned()[f] + if !ok { + return "" + } + if len(versionedSpecs) > 1 { + // check if the feature is locked + lastLifecycle := versionedSpecs[len(versionedSpecs)-1] + if lastLifecycle.LockToDefault && !lastLifecycle.Version.GreaterThan(emuVer) && lastLifecycle.Default != value { + // if the feature is locked, set the emulation version to the previous version when the feature is not locked. + return fmt.Sprintf("Feature %s is locked at version %s. Try adding SetFeatureGateEmulationVersionDuringTest(t, gate, version.MustParse(\"1.%d\")) at the beginning of your test.", f, emuVer.String(), lastLifecycle.Version.SubtractMinor(1).Minor()) + } + } + return "" +} + +// SetFeatureGateEmulationVersionDuringTest sets the specified gate to the specified emulation version for duration of the test. +// Fails when it detects second call to set a different emulation version or is unable to set or restore emulation version. +// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version will cause fatal. +// Example use: + +// featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) +func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGate, ver *version.Version) { + tb.Helper() + detectParallelOverrideCleanup := detectParallelOverrideEmulationVersion(tb, ver) + originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(ver); err != nil { + tb.Fatalf("failed to set emulation version to %s during test: %v", ver.String(), err) + } + tb.Cleanup(func() { + tb.Helper() + detectParallelOverrideCleanup() + if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(originalEmuVer); err != nil { + tb.Fatalf("failed to restore emulation version to %s during test", originalEmuVer.String()) + } + }) +} + +func detectParallelOverride(tb TB, f featuregate.Feature) func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + beforeOverrideTestName := featureFlagOverride[f] + if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) { + tb.Fatalf("Detected parallel setting of a feature gate by both %q and %q", beforeOverrideTestName, tb.Name()) + } + featureFlagOverride[f] = tb.Name() + + return func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + if afterOverrideTestName := featureFlagOverride[f]; afterOverrideTestName != tb.Name() { + tb.Fatalf("Detected parallel setting of a feature gate between both %q and %q", afterOverrideTestName, tb.Name()) + } + featureFlagOverride[f] = beforeOverrideTestName + } +} + +func detectParallelOverrideEmulationVersion(tb TB, ver *version.Version) func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + beforeOverrideTestName := emulationVersionOverride + beforeOverrideValue := emulationVersionOverrideValue + if ver.EqualTo(beforeOverrideValue) { + return func() {} + } + if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) { + tb.Fatalf("Detected parallel setting of a feature gate emulation version by both %q and %q", beforeOverrideTestName, tb.Name()) + } + emulationVersionOverride = tb.Name() + emulationVersionOverrideValue = ver + + return func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + if afterOverrideTestName := emulationVersionOverride; afterOverrideTestName != tb.Name() { + tb.Fatalf("Detected parallel setting of a feature gate emulation version between both %q and %q", afterOverrideTestName, tb.Name()) + } + emulationVersionOverride = beforeOverrideTestName + emulationVersionOverrideValue = beforeOverrideValue + } +} + +func sameTestOrSubtest(tb TB, testName string) bool { + // Assumes that "/" is not used in test names. + return tb.Name() == testName || strings.HasPrefix(tb.Name(), testName+"/") +} + +type TB interface { + Cleanup(func()) + Error(args ...any) + Errorf(format string, args ...any) + Fatal(args ...any) + Fatalf(format string, args ...any) + Helper() + Name() string +} diff --git a/vendor/k8s.io/klog/v2/internal/verbosity/verbosity.go b/vendor/k8s.io/klog/v2/internal/verbosity/verbosity.go new file mode 100644 index 00000000..40ec27d8 --- /dev/null +++ b/vendor/k8s.io/klog/v2/internal/verbosity/verbosity.go @@ -0,0 +1,303 @@ +/* +Copyright 2013 Google Inc. All Rights Reserved. +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package verbosity + +import ( + "bytes" + "errors" + "flag" + "fmt" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" +) + +// New returns a struct that implements -v and -vmodule support. Changing and +// checking these settings is thread-safe, with all concurrency issues handled +// internally. +func New() *VState { + vs := new(VState) + + // The two fields must have a pointer to the overal struct for their + // implementation of Set. + vs.vmodule.vs = vs + vs.verbosity.vs = vs + + return vs +} + +// Value is an extension that makes it possible to use the values in pflag. +type Value interface { + flag.Value + Type() string +} + +func (vs *VState) V() Value { + return &vs.verbosity +} + +func (vs *VState) VModule() Value { + return &vs.vmodule +} + +// VState contains settings and state. Some of its fields can be accessed +// through atomic read/writes, in other cases a mutex must be held. +type VState struct { + mu sync.Mutex + + // These flags are modified only under lock, although verbosity may be fetched + // safely using atomic.LoadInt32. + vmodule moduleSpec // The state of the -vmodule flag. + verbosity levelSpec // V logging level, the value of the -v flag/ + + // pcs is used in V to avoid an allocation when computing the caller's PC. + pcs [1]uintptr + // vmap is a cache of the V Level for each V() call site, identified by PC. + // It is wiped whenever the vmodule flag changes state. + vmap map[uintptr]Level + // filterLength stores the length of the vmodule filter chain. If greater + // than zero, it means vmodule is enabled. It may be read safely + // using sync.LoadInt32, but is only modified under mu. + filterLength int32 +} + +// Level must be an int32 to support atomic read/writes. +type Level int32 + +type levelSpec struct { + vs *VState + l Level +} + +// get returns the value of the level. +func (l *levelSpec) get() Level { + return Level(atomic.LoadInt32((*int32)(&l.l))) +} + +// set sets the value of the level. +func (l *levelSpec) set(val Level) { + atomic.StoreInt32((*int32)(&l.l), int32(val)) +} + +// String is part of the flag.Value interface. +func (l *levelSpec) String() string { + return strconv.FormatInt(int64(l.l), 10) +} + +// Get is part of the flag.Getter interface. It returns the +// verbosity level as int32. +func (l *levelSpec) Get() interface{} { + return int32(l.l) +} + +// Type is part of pflag.Value. +func (l *levelSpec) Type() string { + return "Level" +} + +// Set is part of the flag.Value interface. +func (l *levelSpec) Set(value string) error { + v, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return err + } + l.vs.mu.Lock() + defer l.vs.mu.Unlock() + l.vs.set(Level(v), l.vs.vmodule.filter, false) + return nil +} + +// moduleSpec represents the setting of the -vmodule flag. +type moduleSpec struct { + vs *VState + filter []modulePat +} + +// modulePat contains a filter for the -vmodule flag. +// It holds a verbosity level and a file pattern to match. +type modulePat struct { + pattern string + literal bool // The pattern is a literal string + level Level +} + +// match reports whether the file matches the pattern. It uses a string +// comparison if the pattern contains no metacharacters. +func (m *modulePat) match(file string) bool { + if m.literal { + return file == m.pattern + } + match, _ := filepath.Match(m.pattern, file) + return match +} + +func (m *moduleSpec) String() string { + // Lock because the type is not atomic. TODO: clean this up. + // Empty instances don't have and don't need a lock (can + // happen when flag uses introspection). + if m.vs != nil { + m.vs.mu.Lock() + defer m.vs.mu.Unlock() + } + var b bytes.Buffer + for i, f := range m.filter { + if i > 0 { + b.WriteRune(',') + } + fmt.Fprintf(&b, "%s=%d", f.pattern, f.level) + } + return b.String() +} + +// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the +// struct is not exported. +func (m *moduleSpec) Get() interface{} { + return nil +} + +// Type is part of pflag.Value +func (m *moduleSpec) Type() string { + return "pattern=N,..." +} + +var errVmoduleSyntax = errors.New("syntax error: expect comma-separated list of filename=N") + +// Set will sets module value +// Syntax: -vmodule=recordio=2,file=1,gfs*=3 +func (m *moduleSpec) Set(value string) error { + var filter []modulePat + for _, pat := range strings.Split(value, ",") { + if len(pat) == 0 { + // Empty strings such as from a trailing comma can be ignored. + continue + } + patLev := strings.Split(pat, "=") + if len(patLev) != 2 || len(patLev[0]) == 0 || len(patLev[1]) == 0 { + return errVmoduleSyntax + } + pattern := patLev[0] + v, err := strconv.ParseInt(patLev[1], 10, 32) + if err != nil { + return errors.New("syntax error: expect comma-separated list of filename=N") + } + if v < 0 { + return errors.New("negative value for vmodule level") + } + if v == 0 { + continue // Ignore. It's harmless but no point in paying the overhead. + } + // TODO: check syntax of filter? + filter = append(filter, modulePat{pattern, isLiteral(pattern), Level(v)}) + } + m.vs.mu.Lock() + defer m.vs.mu.Unlock() + m.vs.set(m.vs.verbosity.l, filter, true) + return nil +} + +// isLiteral reports whether the pattern is a literal string, that is, has no metacharacters +// that require filepath.Match to be called to match the pattern. +func isLiteral(pattern string) bool { + return !strings.ContainsAny(pattern, `\*?[]`) +} + +// set sets a consistent state for V logging. +// The mutex must be held. +func (vs *VState) set(l Level, filter []modulePat, setFilter bool) { + // Turn verbosity off so V will not fire while we are in transition. + vs.verbosity.set(0) + // Ditto for filter length. + atomic.StoreInt32(&vs.filterLength, 0) + + // Set the new filters and wipe the pc->Level map if the filter has changed. + if setFilter { + vs.vmodule.filter = filter + vs.vmap = make(map[uintptr]Level) + } + + // Things are consistent now, so enable filtering and verbosity. + // They are enabled in order opposite to that in V. + atomic.StoreInt32(&vs.filterLength, int32(len(filter))) + vs.verbosity.set(l) +} + +// Enabled checks whether logging is enabled at the given level. This must be +// called with depth=0 when the caller of enabled will do the logging and +// higher values when more stack levels need to be skipped. +// +// The mutex will be locked only if needed. +func (vs *VState) Enabled(level Level, depth int) bool { + // This function tries hard to be cheap unless there's work to do. + // The fast path is two atomic loads and compares. + + // Here is a cheap but safe test to see if V logging is enabled globally. + if vs.verbosity.get() >= level { + return true + } + + // It's off globally but vmodule may still be set. + // Here is another cheap but safe test to see if vmodule is enabled. + if atomic.LoadInt32(&vs.filterLength) > 0 { + // Now we need a proper lock to use the logging structure. The pcs field + // is shared so we must lock before accessing it. This is fairly expensive, + // but if V logging is enabled we're slow anyway. + vs.mu.Lock() + defer vs.mu.Unlock() + if runtime.Callers(depth+2, vs.pcs[:]) == 0 { + return false + } + // runtime.Callers returns "return PCs", but we want + // to look up the symbolic information for the call, + // so subtract 1 from the PC. runtime.CallersFrames + // would be cleaner, but allocates. + pc := vs.pcs[0] - 1 + v, ok := vs.vmap[pc] + if !ok { + v = vs.setV(pc) + } + return v >= level + } + return false +} + +// setV computes and remembers the V level for a given PC +// when vmodule is enabled. +// File pattern matching takes the basename of the file, stripped +// of its .go suffix, and uses filepath.Match, which is a little more +// general than the *? matching used in C++. +// Mutex is held. +func (vs *VState) setV(pc uintptr) Level { + fn := runtime.FuncForPC(pc) + file, _ := fn.FileLine(pc) + // The file is something like /a/b/c/d.go. We want just the d. + file = strings.TrimSuffix(file, ".go") + if slash := strings.LastIndex(file, "/"); slash >= 0 { + file = file[slash+1:] + } + for _, filter := range vs.vmodule.filter { + if filter.match(file) { + vs.vmap[pc] = filter.level + return filter.level + } + } + vs.vmap[pc] = 0 + return 0 +} diff --git a/vendor/k8s.io/klog/v2/ktesting/options.go b/vendor/k8s.io/klog/v2/ktesting/options.go new file mode 100644 index 00000000..2119a222 --- /dev/null +++ b/vendor/k8s.io/klog/v2/ktesting/options.go @@ -0,0 +1,132 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "flag" + "strconv" + + "k8s.io/klog/v2/internal/serialize" + "k8s.io/klog/v2/internal/verbosity" +) + +// Config influences logging in a test logger. To make this configurable via +// command line flags, instantiate this once per program and use AddFlags to +// bind command line flags to the instance before passing it to NewTestContext. +// +// Must be constructed with NewConfig. +type Config struct { + vstate *verbosity.VState + co configOptions +} + +// Verbosity returns a value instance that can be used to query (via String) or +// modify (via Set) the verbosity threshold. This is thread-safe and can be +// done at runtime. +func (c *Config) Verbosity() flag.Value { + return c.vstate.V() +} + +// VModule returns a value instance that can be used to query (via String) or +// modify (via Set) the vmodule settings. This is thread-safe and can be done +// at runtime. +func (c *Config) VModule() flag.Value { + return c.vstate.VModule() +} + +// ConfigOption implements functional parameters for NewConfig. +type ConfigOption func(co *configOptions) + +type configOptions struct { + anyToString serialize.AnyToStringFunc + verbosityFlagName string + vmoduleFlagName string + verbosityDefault int + bufferLogs bool +} + +// AnyToString overrides the default formatter for values that are not +// supported directly by klog. The default is `fmt.Sprintf("%+v")`. +// The formatter must not panic. +func AnyToString(anyToString func(value interface{}) string) ConfigOption { + return func(co *configOptions) { + co.anyToString = anyToString + } +} + +// VerbosityFlagName overrides the default -testing.v for the verbosity level. +func VerbosityFlagName(name string) ConfigOption { + return func(co *configOptions) { + co.verbosityFlagName = name + } +} + +// VModulFlagName overrides the default -testing.vmodule for the per-module +// verbosity levels. +func VModuleFlagName(name string) ConfigOption { + return func(co *configOptions) { + co.vmoduleFlagName = name + } +} + +// Verbosity overrides the default verbosity level of 5. That default is higher +// than in klog itself because it enables logging entries for "the steps +// leading up to errors and warnings" and "troubleshooting" (see +// https://github.com/kubernetes/community/blob/9406b4352fe2d5810cb21cc3cb059ce5886de157/contributors/devel/sig-instrumentation/logging.md#logging-conventions), +// which is useful when debugging a failed test. `go test` only shows the log +// output for failed tests. To see all output, use `go test -v`. +func Verbosity(level int) ConfigOption { + return func(co *configOptions) { + co.verbosityDefault = level + } +} + +// BufferLogs controls whether log entries are captured in memory in addition +// to being printed. Off by default. Unit tests that want to verify that +// log entries are emitted as expected can turn this on and then retrieve +// the captured log through the Underlier LogSink interface. +func BufferLogs(enabled bool) ConfigOption { + return func(co *configOptions) { + co.bufferLogs = enabled + } +} + +// NewConfig returns a configuration with recommended defaults and optional +// modifications. Command line flags are not bound to any FlagSet yet. +func NewConfig(opts ...ConfigOption) *Config { + c := &Config{ + co: configOptions{ + verbosityFlagName: "testing.v", + vmoduleFlagName: "testing.vmodule", + verbosityDefault: 5, + }, + } + for _, opt := range opts { + opt(&c.co) + } + + c.vstate = verbosity.New() + // Cannot fail for this input. + _ = c.vstate.V().Set(strconv.FormatInt(int64(c.co.verbosityDefault), 10)) + return c +} + +// AddFlags registers the command line flags that control the configuration. +func (c *Config) AddFlags(fs *flag.FlagSet) { + fs.Var(c.vstate.V(), c.co.verbosityFlagName, "number for the log level verbosity of the testing logger") + fs.Var(c.vstate.VModule(), c.co.vmoduleFlagName, "comma-separated list of pattern=N log level settings for files matching the patterns") +} diff --git a/vendor/k8s.io/klog/v2/ktesting/setup.go b/vendor/k8s.io/klog/v2/ktesting/setup.go new file mode 100644 index 00000000..bf1d0344 --- /dev/null +++ b/vendor/k8s.io/klog/v2/ktesting/setup.go @@ -0,0 +1,38 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "context" + + "github.com/go-logr/logr" +) + +// DefaultConfig is the global default logging configuration for a unit +// test. It is used by NewTestContext and k8s.io/klogr/testing/init. +var DefaultConfig = NewConfig() + +// NewTestContext returns a logger and context for use in a unit test case or +// benchmark. The tl parameter can be a testing.T or testing.B pointer that +// will receive all log output. Importing k8s.io/klogr/testing/init will add +// command line flags that modify the configuration of that log output. +func NewTestContext(tl TL) (logr.Logger, context.Context) { + logger := NewLogger(tl, DefaultConfig) + ctx := logr.NewContext(context.Background(), logger) + return logger, ctx + +} diff --git a/vendor/k8s.io/klog/v2/ktesting/testinglogger.go b/vendor/k8s.io/klog/v2/ktesting/testinglogger.go new file mode 100644 index 00000000..b6c7bb2e --- /dev/null +++ b/vendor/k8s.io/klog/v2/ktesting/testinglogger.go @@ -0,0 +1,406 @@ +/* +Copyright 2019 The Kubernetes Authors. +Copyright 2020 Intel Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package testinglogger contains an implementation of the logr interface +// which is logging through a function like testing.TB.Log function. +// Therefore it can be used in standard Go tests and Gingko test suites +// to ensure that output is associated with the currently running test. +// +// In addition, the log data is captured in a buffer and can be used by the +// test to verify that the code under test is logging as expected. To get +// access to that data, cast the LogSink into the Underlier type and retrieve +// it: +// +// logger := ktesting.NewLogger(...) +// if testingLogger, ok := logger.GetSink().(ktesting.Underlier); ok { +// t := testingLogger.GetUnderlying() +// buffer := testingLogger.GetBuffer() +// text := buffer.String() +// log := buffer.Data() +// +// Serialization of the structured log parameters is done in the same way +// as for klog.InfoS. +package ktesting + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/go-logr/logr" + + "k8s.io/klog/v2" + "k8s.io/klog/v2/internal/buffer" + "k8s.io/klog/v2/internal/dbg" + "k8s.io/klog/v2/internal/serialize" + "k8s.io/klog/v2/internal/severity" + "k8s.io/klog/v2/internal/verbosity" +) + +// TL is the relevant subset of testing.TB. +type TL interface { + Helper() + Log(args ...interface{}) +} + +// NopTL implements TL with empty stubs. It can be used when only capturing +// output in memory is relevant. +type NopTL struct{} + +func (n NopTL) Helper() {} +func (n NopTL) Log(...interface{}) {} + +var _ TL = NopTL{} + +// BufferTL implements TL with an in-memory buffer. +type BufferTL struct { + strings.Builder +} + +func (n *BufferTL) Helper() {} +func (n *BufferTL) Log(args ...interface{}) { + n.Builder.WriteString(fmt.Sprintln(args...)) +} + +var _ TL = &BufferTL{} + +// NewLogger constructs a new logger for the given test interface. +// +// Beware that testing.T does not support logging after the test that +// it was created for has completed. If a test leaks goroutines +// and those goroutines log something after test completion, +// that output will be printed via the global klog logger with +// ` leaked goroutine` as prefix. +// +// Verbosity can be modified at any time through the Config.V and +// Config.VModule API. +func NewLogger(t TL, c *Config) logr.Logger { + l := tlogger{ + shared: &tloggerShared{ + t: t, + config: c, + }, + } + if c.co.anyToString != nil { + l.shared.formatter.AnyToStringHook = c.co.anyToString + } + + type testCleanup interface { + Cleanup(func()) + Name() string + } + + // Stopping the logging is optional and only done (and required) + // for testing.T/B/F. + if tb, ok := t.(testCleanup); ok { + tb.Cleanup(l.shared.stop) + l.shared.testName = tb.Name() + } + return logr.New(l) +} + +// Buffer stores log entries as formatted text and structured data. +// It is safe to use this concurrently. +type Buffer interface { + // String returns the log entries in a format that is similar to the + // klog text output. + String() string + + // Data returns the log entries as structs. + Data() Log +} + +// Log contains log entries in the order in which they were generated. +type Log []LogEntry + +// DeepCopy returns a copy of the log. The error instance and key/value +// pairs remain shared. +func (l Log) DeepCopy() Log { + log := make(Log, 0, len(l)) + log = append(log, l...) + return log +} + +// LogEntry represents all information captured for a log entry. +type LogEntry struct { + // Timestamp stores the time when the log entry was created. + Timestamp time.Time + + // Type is either LogInfo or LogError. + Type LogType + + // Prefix contains the WithName strings concatenated with a slash. + Prefix string + + // Message is the fixed log message string. + Message string + + // Verbosity is always 0 for LogError. + Verbosity int + + // Err is always nil for LogInfo. It may or may not be + // nil for LogError. + Err error + + // WithKVList are the concatenated key/value pairs from WithValues + // calls. It's guaranteed to have an even number of entries because + // the logger ensures that when WithValues is called. + WithKVList []interface{} + + // ParameterKVList are the key/value pairs passed into the call, + // without any validation. + ParameterKVList []interface{} +} + +// LogType determines whether a log entry was created with an Error or Info +// call. +type LogType string + +const ( + // LogError is the special value used for Error log entries. + LogError = LogType("ERROR") + + // LogInfo is the special value used for Info log entries. + LogInfo = LogType("INFO") +) + +// Underlier is implemented by the LogSink of this logger. It provides access +// to additional APIs that are normally hidden behind the Logger API. +type Underlier interface { + // GetUnderlying returns the testing instance that logging goes to. + // It returns nil when the test has completed already. + GetUnderlying() TL + + // GetBuffer grants access to the in-memory copy of the log entries. + GetBuffer() Buffer +} + +type logBuffer struct { + mutex sync.Mutex + text strings.Builder + log Log +} + +func (b *logBuffer) String() string { + b.mutex.Lock() + defer b.mutex.Unlock() + return b.text.String() +} + +func (b *logBuffer) Data() Log { + b.mutex.Lock() + defer b.mutex.Unlock() + return b.log.DeepCopy() +} + +// tloggerShared holds values that are the same for all LogSink instances. It +// gets referenced by pointer in the tlogger struct. +type tloggerShared struct { + // mutex protects access to t. + mutex sync.Mutex + + // t gets cleared when the test is completed. + t TL + + // The time when the test completed. + stopTime time.Time + + // We warn once when a leaked goroutine logs after test completion. + // + // Not terminating immediately is fairly normal: many controllers + // log "terminating" messages while they shut down, which often is + // right after test completion, if that is when the test cancels the + // context and then doesn't wait for goroutines (which is often + // not possible). + // + // Therefore there is the [stopGracePeriod] during which messages get + // logged to the global logger without the warning. + goroutineWarningDone bool + + formatter serialize.Formatter + testName string + config *Config + buffer logBuffer + callDepth int +} + +// Log output of a leaked goroutine during this period after test completion +// does not trigger the warning. +const stopGracePeriod = 10 * time.Second + +func (ls *tloggerShared) stop() { + ls.mutex.Lock() + defer ls.mutex.Unlock() + ls.t = nil + ls.stopTime = time.Now() +} + +// tlogger is the actual LogSink implementation. +type tlogger struct { + shared *tloggerShared + prefix string + values []interface{} +} + +// fallbackLogger is called while l.shared.mutex is locked and after it has +// been determined that the original testing.TB is no longer usable. +func (l tlogger) fallbackLogger() logr.Logger { + logger := klog.Background().WithValues(l.values...).WithName(l.shared.testName + " leaked goroutine") + if l.prefix != "" { + logger = logger.WithName(l.prefix) + } + // Skip direct caller (= Error or Info) plus the logr wrapper. + logger = logger.WithCallDepth(l.shared.callDepth + 1) + + if !l.shared.goroutineWarningDone { + duration := time.Since(l.shared.stopTime) + if duration > stopGracePeriod { + + logger.WithCallDepth(1).Info("WARNING: test kept at least one goroutine running after test completion", "timeSinceCompletion", duration.Round(time.Second), "callstack", string(dbg.Stacks(false))) + l.shared.goroutineWarningDone = true + } + } + return logger +} + +func (l tlogger) Init(info logr.RuntimeInfo) { + l.shared.callDepth = info.CallDepth +} + +func (l tlogger) GetCallStackHelper() func() { + l.shared.mutex.Lock() + defer l.shared.mutex.Unlock() + if l.shared.t == nil { + return func() {} + } + + return l.shared.t.Helper +} + +func (l tlogger) Info(level int, msg string, kvList ...interface{}) { + l.shared.mutex.Lock() + defer l.shared.mutex.Unlock() + if l.shared.t == nil { + l.fallbackLogger().V(level).Info(msg, kvList...) + return + } + + l.shared.t.Helper() + buf := buffer.GetBuffer() + l.shared.formatter.MergeAndFormatKVs(&buf.Buffer, l.values, kvList) + l.log(LogInfo, msg, level, buf, nil, kvList) +} + +func (l tlogger) Enabled(level int) bool { + return l.shared.config.vstate.Enabled(verbosity.Level(level), 1) +} + +func (l tlogger) Error(err error, msg string, kvList ...interface{}) { + l.shared.mutex.Lock() + defer l.shared.mutex.Unlock() + if l.shared.t == nil { + l.fallbackLogger().Error(err, msg, kvList...) + return + } + + l.shared.t.Helper() + buf := buffer.GetBuffer() + if err != nil { + l.shared.formatter.KVFormat(&buf.Buffer, "err", err) + } + l.shared.formatter.MergeAndFormatKVs(&buf.Buffer, l.values, kvList) + l.log(LogError, msg, 0, buf, err, kvList) +} + +func (l tlogger) log(what LogType, msg string, level int, buf *buffer.Buffer, err error, kvList []interface{}) { + l.shared.t.Helper() + s := severity.InfoLog + if what == LogError { + s = severity.ErrorLog + } + args := []interface{}{buf.SprintHeader(s, time.Now())} + if l.prefix != "" { + args = append(args, l.prefix+":") + } + args = append(args, msg) + if buf.Len() > 0 { + // Skip leading space inserted by serialize.KVListFormat. + args = append(args, string(buf.Bytes()[1:])) + } + l.shared.t.Log(args...) + + if !l.shared.config.co.bufferLogs { + return + } + + l.shared.buffer.mutex.Lock() + defer l.shared.buffer.mutex.Unlock() + + // Store as text. + l.shared.buffer.text.WriteString(string(what)) + for i := 1; i < len(args); i++ { + l.shared.buffer.text.WriteByte(' ') + l.shared.buffer.text.WriteString(args[i].(string)) + } + lastArg := args[len(args)-1].(string) + if lastArg[len(lastArg)-1] != '\n' { + l.shared.buffer.text.WriteByte('\n') + } + + // Store as raw data. + l.shared.buffer.log = append(l.shared.buffer.log, + LogEntry{ + Timestamp: time.Now(), + Type: what, + Prefix: l.prefix, + Message: msg, + Verbosity: level, + Err: err, + WithKVList: l.values, + ParameterKVList: kvList, + }, + ) +} + +// WithName returns a new logr.Logger with the specified name appended. klogr +// uses '/' characters to separate name elements. Callers should not pass '/' +// in the provided name string, but this library does not actually enforce that. +func (l tlogger) WithName(name string) logr.LogSink { + if len(l.prefix) > 0 { + l.prefix = l.prefix + "/" + } + l.prefix += name + return l +} + +func (l tlogger) WithValues(kvList ...interface{}) logr.LogSink { + l.values = serialize.WithValues(l.values, kvList) + return l +} + +func (l tlogger) GetUnderlying() TL { + return l.shared.t +} + +func (l tlogger) GetBuffer() Buffer { + return &l.shared.buffer +} + +var _ logr.LogSink = &tlogger{} +var _ logr.CallStackHelperLogSink = &tlogger{} +var _ Underlier = &tlogger{} diff --git a/vendor/modules.txt b/vendor/modules.txt index a65e65ce..7236e087 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1517,6 +1517,7 @@ k8s.io/component-base/compatibility k8s.io/component-base/config k8s.io/component-base/config/v1alpha1 k8s.io/component-base/featuregate +k8s.io/component-base/featuregate/testing k8s.io/component-base/metrics k8s.io/component-base/metrics/features k8s.io/component-base/metrics/legacyregistry @@ -1546,6 +1547,8 @@ k8s.io/klog/v2/internal/dbg k8s.io/klog/v2/internal/serialize k8s.io/klog/v2/internal/severity k8s.io/klog/v2/internal/sloghandler +k8s.io/klog/v2/internal/verbosity +k8s.io/klog/v2/ktesting # k8s.io/kube-controller-manager v0.32.8 ## explicit; go 1.23.0 k8s.io/kube-controller-manager/config/v1alpha1 @@ -1615,9 +1618,12 @@ kmodules.xyz/monitoring-agent-api/api/v1 # open-cluster-management.io/api v1.1.0 ## explicit; go 1.24.0 open-cluster-management.io/api/client/work/clientset/versioned +open-cluster-management.io/api/client/work/clientset/versioned/fake open-cluster-management.io/api/client/work/clientset/versioned/scheme open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1 +open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1 +open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake open-cluster-management.io/api/client/work/informers/externalversions open-cluster-management.io/api/client/work/informers/externalversions/internalinterfaces open-cluster-management.io/api/client/work/informers/externalversions/work diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/clientset_generated.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 00000000..1206f303 --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,86 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" + clientset "open-cluster-management.io/api/client/work/clientset/versioned" + workv1 "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1" + fakeworkv1 "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake" + workv1alpha1 "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1" + fakeworkv1alpha1 "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +// +// DEPRECATED: NewClientset replaces this with support for field management, which significantly improves +// server side apply testing. NewClientset is only available when apply configurations are generated (e.g. +// via --with-applyconfig). +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + var opts metav1.ListOptions + if watchActcion, ok := action.(testing.WatchActionImpl); ok { + opts = watchActcion.ListOptions + } + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns, opts) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// WorkV1 retrieves the WorkV1Client +func (c *Clientset) WorkV1() workv1.WorkV1Interface { + return &fakeworkv1.FakeWorkV1{Fake: &c.Fake} +} + +// WorkV1alpha1 retrieves the WorkV1alpha1Client +func (c *Clientset) WorkV1alpha1() workv1alpha1.WorkV1alpha1Interface { + return &fakeworkv1alpha1.FakeWorkV1alpha1{Fake: &c.Fake} +} diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/doc.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/doc.go new file mode 100644 index 00000000..e97b6f23 --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/doc.go @@ -0,0 +1,5 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/register.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/register.go new file mode 100644 index 00000000..7777a236 --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/fake/register.go @@ -0,0 +1,43 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + workv1 "open-cluster-management.io/api/work/v1" + workv1alpha1 "open-cluster-management.io/api/work/v1alpha1" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + workv1.AddToScheme, + workv1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/doc.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/doc.go new file mode 100644 index 00000000..97b0a07e --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/doc.go @@ -0,0 +1,5 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_appliedmanifestwork.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_appliedmanifestwork.go new file mode 100644 index 00000000..d4dee657 --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_appliedmanifestwork.go @@ -0,0 +1,37 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + gentype "k8s.io/client-go/gentype" + workv1 "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1" + v1 "open-cluster-management.io/api/work/v1" +) + +// fakeAppliedManifestWorks implements AppliedManifestWorkInterface +type fakeAppliedManifestWorks struct { + *gentype.FakeClientWithList[*v1.AppliedManifestWork, *v1.AppliedManifestWorkList] + Fake *FakeWorkV1 +} + +func newFakeAppliedManifestWorks(fake *FakeWorkV1) workv1.AppliedManifestWorkInterface { + return &fakeAppliedManifestWorks{ + gentype.NewFakeClientWithList[*v1.AppliedManifestWork, *v1.AppliedManifestWorkList]( + fake.Fake, + "", + v1.SchemeGroupVersion.WithResource("appliedmanifestworks"), + v1.SchemeGroupVersion.WithKind("AppliedManifestWork"), + func() *v1.AppliedManifestWork { return &v1.AppliedManifestWork{} }, + func() *v1.AppliedManifestWorkList { return &v1.AppliedManifestWorkList{} }, + func(dst, src *v1.AppliedManifestWorkList) { dst.ListMeta = src.ListMeta }, + func(list *v1.AppliedManifestWorkList) []*v1.AppliedManifestWork { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1.AppliedManifestWorkList, items []*v1.AppliedManifestWork) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_manifestwork.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_manifestwork.go new file mode 100644 index 00000000..f0ee6a01 --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_manifestwork.go @@ -0,0 +1,35 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + gentype "k8s.io/client-go/gentype" + workv1 "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1" + v1 "open-cluster-management.io/api/work/v1" +) + +// fakeManifestWorks implements ManifestWorkInterface +type fakeManifestWorks struct { + *gentype.FakeClientWithList[*v1.ManifestWork, *v1.ManifestWorkList] + Fake *FakeWorkV1 +} + +func newFakeManifestWorks(fake *FakeWorkV1, namespace string) workv1.ManifestWorkInterface { + return &fakeManifestWorks{ + gentype.NewFakeClientWithList[*v1.ManifestWork, *v1.ManifestWorkList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("manifestworks"), + v1.SchemeGroupVersion.WithKind("ManifestWork"), + func() *v1.ManifestWork { return &v1.ManifestWork{} }, + func() *v1.ManifestWorkList { return &v1.ManifestWorkList{} }, + func(dst, src *v1.ManifestWorkList) { dst.ListMeta = src.ListMeta }, + func(list *v1.ManifestWorkList) []*v1.ManifestWork { return gentype.ToPointerSlice(list.Items) }, + func(list *v1.ManifestWorkList, items []*v1.ManifestWork) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_work_client.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_work_client.go new file mode 100644 index 00000000..ed467c83 --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fake/fake_work_client.go @@ -0,0 +1,29 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" + v1 "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1" +) + +type FakeWorkV1 struct { + *testing.Fake +} + +func (c *FakeWorkV1) AppliedManifestWorks() v1.AppliedManifestWorkInterface { + return newFakeAppliedManifestWorks(c) +} + +func (c *FakeWorkV1) ManifestWorks(namespace string) v1.ManifestWorkInterface { + return newFakeManifestWorks(c, namespace) +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeWorkV1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/doc.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/doc.go new file mode 100644 index 00000000..97b0a07e --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/doc.go @@ -0,0 +1,5 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/fake_manifestworkreplicaset.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/fake_manifestworkreplicaset.go new file mode 100644 index 00000000..c188c131 --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/fake_manifestworkreplicaset.go @@ -0,0 +1,37 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + gentype "k8s.io/client-go/gentype" + workv1alpha1 "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1" + v1alpha1 "open-cluster-management.io/api/work/v1alpha1" +) + +// fakeManifestWorkReplicaSets implements ManifestWorkReplicaSetInterface +type fakeManifestWorkReplicaSets struct { + *gentype.FakeClientWithList[*v1alpha1.ManifestWorkReplicaSet, *v1alpha1.ManifestWorkReplicaSetList] + Fake *FakeWorkV1alpha1 +} + +func newFakeManifestWorkReplicaSets(fake *FakeWorkV1alpha1, namespace string) workv1alpha1.ManifestWorkReplicaSetInterface { + return &fakeManifestWorkReplicaSets{ + gentype.NewFakeClientWithList[*v1alpha1.ManifestWorkReplicaSet, *v1alpha1.ManifestWorkReplicaSetList]( + fake.Fake, + namespace, + v1alpha1.SchemeGroupVersion.WithResource("manifestworkreplicasets"), + v1alpha1.SchemeGroupVersion.WithKind("ManifestWorkReplicaSet"), + func() *v1alpha1.ManifestWorkReplicaSet { return &v1alpha1.ManifestWorkReplicaSet{} }, + func() *v1alpha1.ManifestWorkReplicaSetList { return &v1alpha1.ManifestWorkReplicaSetList{} }, + func(dst, src *v1alpha1.ManifestWorkReplicaSetList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha1.ManifestWorkReplicaSetList) []*v1alpha1.ManifestWorkReplicaSet { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha1.ManifestWorkReplicaSetList, items []*v1alpha1.ManifestWorkReplicaSet) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/fake_work_client.go b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/fake_work_client.go new file mode 100644 index 00000000..e7b023ef --- /dev/null +++ b/vendor/open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1/fake/fake_work_client.go @@ -0,0 +1,25 @@ +// Copyright Contributors to the Open Cluster Management project +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" + v1alpha1 "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1alpha1" +) + +type FakeWorkV1alpha1 struct { + *testing.Fake +} + +func (c *FakeWorkV1alpha1) ManifestWorkReplicaSets(namespace string) v1alpha1.ManifestWorkReplicaSetInterface { + return newFakeManifestWorkReplicaSets(c, namespace) +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeWorkV1alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} From 9ecabb340acffe28b844ac15f78f8d8f9d5b7bb5 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Fri, 26 Jun 2026 13:58:25 +0600 Subject: [PATCH 2/3] Revive controller_utils_test.go Restore the upstream pkg/controller/controller_utils_test.go that was dropped in c547220 ("Use golangci-lint 2.x"). All the helpers it exercises still exist in the fork, so the coverage is worth keeping. Adapt it to the fork and the vendored dependency set: - Swap k8s.io/kubernetes/test/utils/ktesting for k8s.io/klog/v2/ktesting (identical NewTestContext API, and the former is not vendored). - Point feature-gate tests at features.DefaultFeatureGate and the new SetFeatureGateDuringTest signature. - ComputeHash now takes *api.PodTemplateSpec; update TestComputeHash. - gofmt interface{} -> any to satisfy the repo formatter config. Drop the tests that would require pulling additional heavy test-only deps into vendor for little benefit: - TestRemoveTaintOffNode / TestAddOrUpdateTaintOnNode need k8s.io/kubernetes/pkg/controller/testutil (FakeNodeHandler); the taint helpers they cover are unused in this fork. - TestCreatePodsWithGenerateName needs k8s.io/client-go/util/testing (FakeHandler) plus a rewrite to the PodInfo-based CreatePods signature. Vendor k8s.io/utils/clock/testing for the fake clock used by the ControllerExpectations tests. Relax errcheck/unparam on _test.go files in .golangci.yml: the revived controller tests mirror the upstream Kubernetes StatefulSet controller tests, which freely ignore errors from in-memory fake stores and use helper signatures that trip unparam; excluding test files keeps the suite rebasable against upstream while production code stays strictly linted. Signed-off-by: Tamal Saha --- .golangci.yml | 16 + pkg/controller/controller_utils_test.go | 713 ++++++++++++++++++ pkg/controller/petset/pet_pod_control_test.go | 2 +- pkg/controller/petset/pet_set_control_test.go | 2 +- pkg/controller/petset/pet_set_test.go | 2 +- pkg/controller/petset/pet_set_utils_test.go | 4 +- .../k8s.io/utils/clock/testing/fake_clock.go | 374 +++++++++ .../clock/testing/simple_interval_clock.go | 44 ++ vendor/modules.txt | 1 + 9 files changed, 1153 insertions(+), 5 deletions(-) create mode 100644 pkg/controller/controller_utils_test.go create mode 100644 vendor/k8s.io/utils/clock/testing/fake_clock.go create mode 100644 vendor/k8s.io/utils/clock/testing/simple_interval_clock.go diff --git a/.golangci.yml b/.golangci.yml index cdd2defc..d1666b37 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,6 +3,22 @@ linters: default: standard enable: - unparam + exclusions: + rules: + # The controller test files are kept close to the upstream Kubernetes + # StatefulSet controller tests, which freely ignore errors from in-memory + # fake stores and use helper signatures that trip unparam. Relax these + # checks for test files so the suite stays rebasable against upstream. + - path: _test\.go + linters: + - errcheck + - unparam + # QF1008 (remove embedded field from selector) is a stylistic quickfix; + # the upstream tests reference .ObjectMeta explicitly. + - path: _test\.go + linters: + - staticcheck + text: "QF1008:" formatters: enable: diff --git a/pkg/controller/controller_utils_test.go b/pkg/controller/controller_utils_test.go new file mode 100644 index 00000000..9964ab36 --- /dev/null +++ b/pkg/controller/controller_utils_test.go @@ -0,0 +1,713 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "math" + "math/rand" + "sort" + "sync" + "testing" + "time" + + api "kubeops.dev/petset/apis/apps/v1" + "kubeops.dev/petset/pkg/features" + "kubeops.dev/petset/pkg/securitycontext" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/klog/v2/ktesting" + "k8s.io/kubernetes/pkg/apis/core" + testingclock "k8s.io/utils/clock/testing" + "k8s.io/utils/ptr" +) + +// NewFakeControllerExpectationsLookup creates a fake store for PodExpectations. +func NewFakeControllerExpectationsLookup(ttl time.Duration) (*ControllerExpectations, *testingclock.FakeClock) { + fakeTime := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) + fakeClock := testingclock.NewFakeClock(fakeTime) + ttlPolicy := &cache.TTLPolicy{TTL: ttl, Clock: fakeClock} + ttlStore := cache.NewFakeExpirationStore( + ExpKeyFunc, nil, ttlPolicy, fakeClock) + return &ControllerExpectations{ttlStore}, fakeClock +} + +func newReplicationController(replicas int) *v1.ReplicationController { + rc := &v1.ReplicationController{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + UID: uuid.NewUUID(), + Name: "foobar", + Namespace: metav1.NamespaceDefault, + ResourceVersion: "18", + }, + Spec: v1.ReplicationControllerSpec{ + Replicas: ptr.To(int32(replicas)), + Selector: map[string]string{"foo": "bar"}, + Template: &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": "foo", + "type": "production", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: "foo/bar", + TerminationMessagePath: v1.TerminationMessagePathDefault, + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults(), + }, + }, + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSDefault, + NodeSelector: map[string]string{ + "baz": "blah", + }, + }, + }, + }, + } + return rc +} + +// create count pods with the given phase for the given rc (same selectors and namespace), and add them to the store. +func newPodList(store cache.Store, count int, status v1.PodPhase, rc *v1.ReplicationController) *v1.PodList { + pods := []v1.Pod{} + for i := 0; i < count; i++ { + newPod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("pod%d", i), + Labels: rc.Spec.Selector, + Namespace: rc.Namespace, + }, + Status: v1.PodStatus{Phase: status}, + } + if store != nil { + store.Add(&newPod) + } + pods = append(pods, newPod) + } + return &v1.PodList{ + Items: pods, + } +} + +func newReplicaSet(name string, replicas int, rsUuid types.UID) *apps.ReplicaSet { + return &apps.ReplicaSet{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + UID: rsUuid, + Name: name, + Namespace: metav1.NamespaceDefault, + ResourceVersion: "18", + }, + Spec: apps.ReplicaSetSpec{ + Replicas: ptr.To(int32(replicas)), + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": "foo", + "type": "production", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: "foo/bar", + TerminationMessagePath: v1.TerminationMessagePathDefault, + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults(), + }, + }, + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSDefault, + NodeSelector: map[string]string{ + "baz": "blah", + }, + }, + }, + }, + } +} + +func TestControllerExpectations(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) + ttl := 30 * time.Second + e, fakeClock := NewFakeControllerExpectationsLookup(ttl) + // In practice we can't really have add and delete expectations since we only either create or + // delete replicas in one rc pass, and the rc goes to sleep soon after until the expectations are + // either fulfilled or timeout. + adds, dels := 10, 30 + rc := newReplicationController(1) + + // RC fires off adds and deletes at apiserver, then sets expectations + rcKey, err := KeyFunc(rc) + assert.NoError(t, err, "Couldn't get key for object %#v: %v", rc, err) + + e.SetExpectations(logger, rcKey, adds, dels) + var wg sync.WaitGroup + for i := 0; i < adds+1; i++ { + wg.Add(1) + go func() { + // In prod this can happen either because of a failed create by the rc + // or after having observed a create via informer + e.CreationObserved(logger, rcKey) + wg.Done() + }() + } + wg.Wait() + + // There are still delete expectations + assert.False(t, e.SatisfiedExpectations(logger, rcKey), "Rc will sync before expectations are met") + + for i := 0; i < dels+1; i++ { + wg.Add(1) + go func() { + e.DeletionObserved(logger, rcKey) + wg.Done() + }() + } + wg.Wait() + + tests := []struct { + name string + expectationsToSet []int + expireExpectations bool + wantPodExpectations []int64 + wantExpectationsSatisfied bool + }{ + { + name: "Expectations have been surpassed", + expireExpectations: false, + wantPodExpectations: []int64{int64(-1), int64(-1)}, + wantExpectationsSatisfied: true, + }, + { + name: "Old expectations are cleared because of ttl", + expectationsToSet: []int{1, 2}, + expireExpectations: true, + wantPodExpectations: []int64{int64(1), int64(2)}, + wantExpectationsSatisfied: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if len(test.expectationsToSet) > 0 { + e.SetExpectations(logger, rcKey, test.expectationsToSet[0], test.expectationsToSet[1]) + } + podExp, exists, err := e.GetExpectations(rcKey) + assert.NoError(t, err, "Could not get expectations for rc, exists %v and err %v", exists, err) + assert.True(t, exists, "Could not get expectations for rc, exists %v and err %v", exists, err) + + add, del := podExp.GetExpectations() + assert.Equal(t, test.wantPodExpectations[0], add, "Unexpected pod expectations %#v", podExp) + assert.Equal(t, test.wantPodExpectations[1], del, "Unexpected pod expectations %#v", podExp) + assert.Equal(t, test.wantExpectationsSatisfied, e.SatisfiedExpectations(logger, rcKey), "Expectations are met but the rc will not sync") + + if test.expireExpectations { + fakeClock.Step(ttl + 1) + assert.True(t, e.SatisfiedExpectations(logger, rcKey), "Expectations should have expired but didn't") + } + }) + } +} + +func TestUIDExpectations(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) + uidExp := NewUIDTrackingControllerExpectations(NewControllerExpectations()) + type test struct { + name string + numReplicas int + } + + shuffleTests := func(tests []test) { + for i := range tests { + j := rand.Intn(i + 1) + tests[i], tests[j] = tests[j], tests[i] + } + } + + getRcDataFrom := func(test test) (string, []string) { + rc := newReplicationController(test.numReplicas) + + rcName := fmt.Sprintf("rc-%v", test.numReplicas) + rc.Name = rcName + rc.Spec.Selector[rcName] = rcName + + podList := newPodList(nil, 5, v1.PodRunning, rc) + rcKey, err := KeyFunc(rc) + if err != nil { + t.Fatalf("Couldn't get key for object %#v: %v", rc, err) + } + + rcPodNames := []string{} + for i := range podList.Items { + p := &podList.Items[i] + p.Name = fmt.Sprintf("%v-%v", p.Name, rc.Name) + rcPodNames = append(rcPodNames, PodKey(p)) + } + uidExp.ExpectDeletions(logger, rcKey, rcPodNames) + + return rcKey, rcPodNames + } + + tests := []test{ + {name: "Replication controller with 2 replicas", numReplicas: 2}, + {name: "Replication controller with 1 replica", numReplicas: 1}, + {name: "Replication controller with no replicas", numReplicas: 0}, + {name: "Replication controller with 5 replicas", numReplicas: 5}, + } + + shuffleTests(tests) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + rcKey, rcPodNames := getRcDataFrom(test) + assert.False(t, uidExp.SatisfiedExpectations(logger, rcKey), + "Controller %v satisfied expectations before deletion", rcKey) + + for _, p := range rcPodNames { + uidExp.DeletionObserved(logger, rcKey, p) + } + + assert.True(t, uidExp.SatisfiedExpectations(logger, rcKey), + "Controller %v didn't satisfy expectations after deletion", rcKey) + + uidExp.DeleteExpectations(logger, rcKey) + + assert.Nil(t, uidExp.GetUIDs(rcKey), + "Failed to delete uid expectations for %v", rcKey) + }) + } +} + +func TestDeletePodsAllowsMissing(t *testing.T) { + fakeClient := fake.NewSimpleClientset() + podControl := RealPodControl{ + KubeClient: fakeClient, + Recorder: &record.FakeRecorder{}, + } + + controllerSpec := newReplicationController(1) + + err := podControl.DeletePod(context.TODO(), "namespace-name", "podName", controllerSpec) + assert.True(t, apierrors.IsNotFound(err)) +} + +func TestCountTerminatingPods(t *testing.T) { + now := metav1.Now() + + // This rc is not needed by the test, only the newPodList to give the pods labels/a namespace. + rc := newReplicationController(0) + podList := newPodList(nil, 7, v1.PodRunning, rc) + podList.Items[0].Status.Phase = v1.PodSucceeded + podList.Items[1].Status.Phase = v1.PodFailed + podList.Items[2].Status.Phase = v1.PodPending + podList.Items[2].SetDeletionTimestamp(&now) + podList.Items[3].Status.Phase = v1.PodRunning + podList.Items[3].SetDeletionTimestamp(&now) + var podPointers []*v1.Pod + for i := range podList.Items { + podPointers = append(podPointers, &podList.Items[i]) + } + + terminatingPods := CountTerminatingPods(podPointers) + + assert.Equal(t, terminatingPods, int32(2)) + + terminatingList := FilterTerminatingPods(podPointers) + assert.Equal(t, len(terminatingList), int(2)) +} + +func TestActivePodFiltering(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) + type podData struct { + podName string + podPhase v1.PodPhase + } + + type test struct { + name string + pods []podData + wantPodNames []string + } + + tests := []test{ + { + name: "Filters active pods", + pods: []podData{ + {podName: "pod-1", podPhase: v1.PodSucceeded}, + {podName: "pod-2", podPhase: v1.PodFailed}, + {podName: "pod-3"}, + {podName: "pod-4"}, + {podName: "pod-5"}, + }, + wantPodNames: []string{"pod-3", "pod-4", "pod-5"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // This rc is not needed by the test, only the newPodList to give the pods labels/a namespace. + rc := newReplicationController(0) + podList := newPodList(nil, 5, v1.PodRunning, rc) + for idx, testPod := range test.pods { + podList.Items[idx].Name = testPod.podName + podList.Items[idx].Status.Phase = testPod.podPhase + } + + var podPointers []*v1.Pod + for i := range podList.Items { + podPointers = append(podPointers, &podList.Items[i]) + } + got := FilterActivePods(logger, podPointers) + gotNames := sets.New[string]() + for _, pod := range got { + gotNames.Insert(pod.Name) + } + + if diff := cmp.Diff(test.wantPodNames, sets.List(gotNames)); diff != "" { + t.Errorf("Active pod names (-want,+got):\n%s", diff) + } + }) + } +} + +func TestSortingActivePods(t *testing.T) { + now := metav1.Now() + then := metav1.Time{Time: now.AddDate(0, -1, 0)} + + tests := []struct { + name string + pods []v1.Pod + wantOrder []string + }{ + { + name: "Sorts by active pod", + pods: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "unscheduled"}, + Spec: v1.PodSpec{NodeName: ""}, + Status: v1.PodStatus{Phase: v1.PodPending}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "scheduledButPending"}, + Spec: v1.PodSpec{NodeName: "bar"}, + Status: v1.PodStatus{Phase: v1.PodPending}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "unknownPhase"}, + Spec: v1.PodSpec{NodeName: "foo"}, + Status: v1.PodStatus{Phase: v1.PodUnknown}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "runningButNotReady"}, + Spec: v1.PodSpec{NodeName: "foo"}, + Status: v1.PodStatus{Phase: v1.PodRunning}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "runningNoLastTransitionTime"}, + Spec: v1.PodSpec{NodeName: "foo"}, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue}}, + ContainerStatuses: []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "runningWithLastTransitionTime"}, + Spec: v1.PodSpec{NodeName: "foo"}, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: now}}, + ContainerStatuses: []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "runningLongerTime"}, + Spec: v1.PodSpec{NodeName: "foo"}, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}}, + ContainerStatuses: []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "lowerContainerRestartCount", CreationTimestamp: now}, + Spec: v1.PodSpec{NodeName: "foo"}, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}}, + ContainerStatuses: []v1.ContainerStatus{{RestartCount: 2}, {RestartCount: 1}}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "oldest", CreationTimestamp: then}, + Spec: v1.PodSpec{NodeName: "foo"}, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}}, + ContainerStatuses: []v1.ContainerStatus{{RestartCount: 2}, {RestartCount: 1}}, + }, + }, + }, + wantOrder: []string{ + "unscheduled", + "scheduledButPending", + "unknownPhase", + "runningButNotReady", + "runningNoLastTransitionTime", + "runningWithLastTransitionTime", + "runningLongerTime", + "lowerContainerRestartCount", + "oldest", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + numPods := len(test.pods) + + for i := 0; i < 20; i++ { + idx := rand.Perm(numPods) + randomizedPods := make([]*v1.Pod, numPods) + for j := 0; j < numPods; j++ { + randomizedPods[j] = &test.pods[idx[j]] + } + + sort.Sort(ActivePods(randomizedPods)) + gotOrder := make([]string, len(randomizedPods)) + for i := range randomizedPods { + gotOrder[i] = randomizedPods[i].Name + } + + if diff := cmp.Diff(test.wantOrder, gotOrder); diff != "" { + t.Errorf("Sorted active pod names (-want,+got):\n%s", diff) + } + } + }) + } +} + +func TestSortingActivePodsWithRanks(t *testing.T) { + now := metav1.Now() + then1Month := metav1.Time{Time: now.AddDate(0, -1, 0)} + then2Hours := metav1.Time{Time: now.Add(-2 * time.Hour)} + then5Hours := metav1.Time{Time: now.Add(-5 * time.Hour)} + then8Hours := metav1.Time{Time: now.Add(-8 * time.Hour)} + zeroTime := metav1.Time{} + pod := func(podName, nodeName string, phase v1.PodPhase, ready bool, restarts int32, readySince metav1.Time, created metav1.Time, annotations map[string]string) *v1.Pod { + var conditions []v1.PodCondition + var containerStatuses []v1.ContainerStatus + if ready { + conditions = []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: readySince}} + containerStatuses = []v1.ContainerStatus{{RestartCount: restarts}} + } + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + CreationTimestamp: created, + Name: podName, + Annotations: annotations, + }, + Spec: v1.PodSpec{NodeName: nodeName}, + Status: v1.PodStatus{ + Conditions: conditions, + ContainerStatuses: containerStatuses, + Phase: phase, + }, + } + } + var ( + unscheduledPod = pod("unscheduled", "", v1.PodPending, false, 0, zeroTime, zeroTime, nil) + scheduledPendingPod = pod("pending", "node", v1.PodPending, false, 0, zeroTime, zeroTime, nil) + unknownPhasePod = pod("unknown-phase", "node", v1.PodUnknown, false, 0, zeroTime, zeroTime, nil) + runningNotReadyPod = pod("not-ready", "node", v1.PodRunning, false, 0, zeroTime, zeroTime, nil) + runningReadyNoLastTransitionTimePod = pod("ready-no-last-transition-time", "node", v1.PodRunning, true, 0, zeroTime, zeroTime, nil) + runningReadyNow = pod("ready-now", "node", v1.PodRunning, true, 0, now, now, nil) + runningReadyThen = pod("ready-then", "node", v1.PodRunning, true, 0, then1Month, then1Month, nil) + runningReadyNowHighRestarts = pod("ready-high-restarts", "node", v1.PodRunning, true, 9001, now, now, nil) + runningReadyNowCreatedThen = pod("ready-now-created-then", "node", v1.PodRunning, true, 0, now, then1Month, nil) + lowPodDeletionCost = pod("low-deletion-cost", "node", v1.PodRunning, true, 0, now, then1Month, map[string]string{core.PodDeletionCost: "10"}) + highPodDeletionCost = pod("high-deletion-cost", "node", v1.PodRunning, true, 0, now, then1Month, map[string]string{core.PodDeletionCost: "100"}) + unscheduled5Hours = pod("unscheduled-5-hours", "", v1.PodPending, false, 0, then5Hours, then5Hours, nil) + unscheduled8Hours = pod("unscheduled-10-hours", "", v1.PodPending, false, 0, then8Hours, then8Hours, nil) + ready2Hours = pod("ready-2-hours", "", v1.PodRunning, true, 0, then2Hours, then1Month, nil) + ready5Hours = pod("ready-5-hours", "", v1.PodRunning, true, 0, then5Hours, then1Month, nil) + ready10Hours = pod("ready-10-hours", "", v1.PodRunning, true, 0, then8Hours, then1Month, nil) + ) + equalityTests := []struct { + p1 *v1.Pod + p2 *v1.Pod + disableLogarithmicScaleDown bool + }{ + {p1: unscheduledPod}, + {p1: scheduledPendingPod}, + {p1: unknownPhasePod}, + {p1: runningNotReadyPod}, + {p1: runningReadyNowCreatedThen}, + {p1: runningReadyNow}, + {p1: runningReadyThen}, + {p1: runningReadyNowHighRestarts}, + {p1: runningReadyNowCreatedThen}, + {p1: unscheduled5Hours, p2: unscheduled8Hours}, + {p1: ready5Hours, p2: ready10Hours}, + } + for i, test := range equalityTests { + t.Run(fmt.Sprintf("Equality tests %d", i), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.LogarithmicScaleDown, !test.disableLogarithmicScaleDown) + if test.p2 == nil { + test.p2 = test.p1 + } + podsWithRanks := ActivePodsWithRanks{ + Pods: []*v1.Pod{test.p1, test.p2}, + Rank: []int{1, 1}, + Now: now, + } + if podsWithRanks.Less(0, 1) || podsWithRanks.Less(1, 0) { + t.Errorf("expected pod %q to be equivalent to %q", test.p1.Name, test.p2.Name) + } + }) + } + + type podWithRank struct { + pod *v1.Pod + rank int + } + inequalityTests := []struct { + lesser, greater podWithRank + disablePodDeletioncost bool + disableLogarithmicScaleDown bool + }{ + {lesser: podWithRank{unscheduledPod, 1}, greater: podWithRank{scheduledPendingPod, 2}}, + {lesser: podWithRank{unscheduledPod, 2}, greater: podWithRank{scheduledPendingPod, 1}}, + {lesser: podWithRank{scheduledPendingPod, 1}, greater: podWithRank{unknownPhasePod, 2}}, + {lesser: podWithRank{unknownPhasePod, 1}, greater: podWithRank{runningNotReadyPod, 2}}, + {lesser: podWithRank{runningNotReadyPod, 1}, greater: podWithRank{runningReadyNoLastTransitionTimePod, 1}}, + {lesser: podWithRank{runningReadyNoLastTransitionTimePod, 1}, greater: podWithRank{runningReadyNow, 1}}, + {lesser: podWithRank{runningReadyNow, 2}, greater: podWithRank{runningReadyNoLastTransitionTimePod, 1}}, + {lesser: podWithRank{runningReadyNow, 1}, greater: podWithRank{runningReadyThen, 1}}, + {lesser: podWithRank{runningReadyNow, 2}, greater: podWithRank{runningReadyThen, 1}}, + {lesser: podWithRank{runningReadyNowHighRestarts, 1}, greater: podWithRank{runningReadyNow, 1}}, + {lesser: podWithRank{runningReadyNow, 2}, greater: podWithRank{runningReadyNowHighRestarts, 1}}, + {lesser: podWithRank{runningReadyNow, 1}, greater: podWithRank{runningReadyNowCreatedThen, 1}}, + {lesser: podWithRank{runningReadyNowCreatedThen, 2}, greater: podWithRank{runningReadyNow, 1}}, + {lesser: podWithRank{lowPodDeletionCost, 2}, greater: podWithRank{highPodDeletionCost, 1}}, + {lesser: podWithRank{highPodDeletionCost, 2}, greater: podWithRank{lowPodDeletionCost, 1}, disablePodDeletioncost: true}, + {lesser: podWithRank{ready2Hours, 1}, greater: podWithRank{ready5Hours, 1}}, + } + + for i, test := range inequalityTests { + t.Run(fmt.Sprintf("Inequality tests %d", i), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.PodDeletionCost, !test.disablePodDeletioncost) + featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.LogarithmicScaleDown, !test.disableLogarithmicScaleDown) + + podsWithRanks := ActivePodsWithRanks{ + Pods: []*v1.Pod{test.lesser.pod, test.greater.pod}, + Rank: []int{test.lesser.rank, test.greater.rank}, + Now: now, + } + if !podsWithRanks.Less(0, 1) { + t.Errorf("expected pod %q with rank %v to be less than %q with rank %v", podsWithRanks.Pods[0].Name, podsWithRanks.Rank[0], podsWithRanks.Pods[1].Name, podsWithRanks.Rank[1]) + } + if podsWithRanks.Less(1, 0) { + t.Errorf("expected pod %q with rank %v not to be less than %v with rank %v", podsWithRanks.Pods[1].Name, podsWithRanks.Rank[1], podsWithRanks.Pods[0].Name, podsWithRanks.Rank[0]) + } + }) + } +} + +func TestActiveReplicaSetsFiltering(t *testing.T) { + rsUuid := uuid.NewUUID() + tests := []struct { + name string + replicaSets []*apps.ReplicaSet + wantReplicaSets []*apps.ReplicaSet + }{ + { + name: "Filters active replica sets", + replicaSets: []*apps.ReplicaSet{ + newReplicaSet("zero", 0, rsUuid), + nil, + newReplicaSet("foo", 1, rsUuid), + newReplicaSet("bar", 2, rsUuid), + }, + wantReplicaSets: []*apps.ReplicaSet{ + newReplicaSet("foo", 1, rsUuid), + newReplicaSet("bar", 2, rsUuid), + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotReplicaSets := FilterActiveReplicaSets(test.replicaSets) + + if diff := cmp.Diff(test.wantReplicaSets, gotReplicaSets); diff != "" { + t.Errorf("Active replica set names (-want,+got):\n%s", diff) + } + }) + } +} + +func TestComputeHash(t *testing.T) { + collisionCount := int32(1) + otherCollisionCount := int32(2) + maxCollisionCount := int32(math.MaxInt32) + tests := []struct { + name string + template *api.PodTemplateSpec + collisionCount *int32 + otherCollisionCount *int32 + }{ + { + name: "simple", + template: &api.PodTemplateSpec{}, + collisionCount: &collisionCount, + otherCollisionCount: &otherCollisionCount, + }, + { + name: "using math.MaxInt64", + template: &api.PodTemplateSpec{}, + collisionCount: nil, + otherCollisionCount: &maxCollisionCount, + }, + } + + for _, test := range tests { + hash := ComputeHash(test.template, test.collisionCount) + otherHash := ComputeHash(test.template, test.otherCollisionCount) + + assert.NotEqual(t, hash, otherHash, "expected different hashes but got the same: %d", hash) + } +} diff --git a/pkg/controller/petset/pet_pod_control_test.go b/pkg/controller/petset/pet_pod_control_test.go index 7b2de609..ea4a2188 100644 --- a/pkg/controller/petset/pet_pod_control_test.go +++ b/pkg/controller/petset/pet_pod_control_test.go @@ -202,7 +202,7 @@ type fakeIndexer struct { getError error } -func (f *fakeIndexer) GetByKey(key string) (interface{}, bool, error) { +func (f *fakeIndexer) GetByKey(key string) (any, bool, error) { return nil, false, f.getError } diff --git a/pkg/controller/petset/pet_set_control_test.go b/pkg/controller/petset/pet_set_control_test.go index fe3caf0f..53d93ddd 100644 --- a/pkg/controller/petset/pet_set_control_test.go +++ b/pkg/controller/petset/pet_set_control_test.go @@ -2953,7 +2953,7 @@ func checkClaimInvarients(set *api.PetSet, pod *v1.Pod, claim *v1.PersistentVolu return nil } -func fakeResourceVersion(object interface{}) { +func fakeResourceVersion(object any) { obj, isObj := object.(metav1.Object) if !isObj { return diff --git a/pkg/controller/petset/pet_set_test.go b/pkg/controller/petset/pet_set_test.go index 7c43b063..b8ec04f9 100644 --- a/pkg/controller/petset/pet_set_test.go +++ b/pkg/controller/petset/pet_set_test.go @@ -835,7 +835,7 @@ func TestStaleOwnerRefOnScaleup(t *testing.T) { WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, }, } { - onPolicy := func(msg string, args ...interface{}) string { + onPolicy := func(msg string, args ...any) string { return fmt.Sprintf(fmt.Sprintf("(%s) %s", policy, msg), args...) } set := newPetSet(3) diff --git a/pkg/controller/petset/pet_set_utils_test.go b/pkg/controller/petset/pet_set_utils_test.go index c9d000c4..20034dc0 100644 --- a/pkg/controller/petset/pet_set_utils_test.go +++ b/pkg/controller/petset/pet_set_utils_test.go @@ -46,10 +46,10 @@ import ( type noopRecorder struct{} func (r *noopRecorder) Event(object runtime.Object, eventtype, reason, message string) {} -func (r *noopRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +func (r *noopRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...any) { } -func (r *noopRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +func (r *noopRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...any) { } // getClaimPodName gets the name of the Pod associated with the Claim, or an empty string if this doesn't look matching. diff --git a/vendor/k8s.io/utils/clock/testing/fake_clock.go b/vendor/k8s.io/utils/clock/testing/fake_clock.go new file mode 100644 index 00000000..9503690b --- /dev/null +++ b/vendor/k8s.io/utils/clock/testing/fake_clock.go @@ -0,0 +1,374 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "sync" + "time" + + "k8s.io/utils/clock" +) + +var ( + _ = clock.PassiveClock(&FakePassiveClock{}) + _ = clock.WithTicker(&FakeClock{}) + _ = clock.Clock(&IntervalClock{}) +) + +// FakePassiveClock implements PassiveClock, but returns an arbitrary time. +type FakePassiveClock struct { + lock sync.RWMutex + time time.Time +} + +// FakeClock implements clock.Clock, but returns an arbitrary time. +type FakeClock struct { + FakePassiveClock + + // waiters are waiting for the fake time to pass their specified time + waiters []*fakeClockWaiter +} + +type fakeClockWaiter struct { + targetTime time.Time + stepInterval time.Duration + skipIfBlocked bool + destChan chan time.Time + afterFunc func() +} + +// NewFakePassiveClock returns a new FakePassiveClock. +func NewFakePassiveClock(t time.Time) *FakePassiveClock { + return &FakePassiveClock{ + time: t, + } +} + +// NewFakeClock constructs a fake clock set to the provided time. +func NewFakeClock(t time.Time) *FakeClock { + return &FakeClock{ + FakePassiveClock: *NewFakePassiveClock(t), + } +} + +// Now returns f's time. +func (f *FakePassiveClock) Now() time.Time { + f.lock.RLock() + defer f.lock.RUnlock() + return f.time +} + +// Since returns time since the time in f. +func (f *FakePassiveClock) Since(ts time.Time) time.Duration { + f.lock.RLock() + defer f.lock.RUnlock() + return f.time.Sub(ts) +} + +// SetTime sets the time on the FakePassiveClock. +func (f *FakePassiveClock) SetTime(t time.Time) { + f.lock.Lock() + defer f.lock.Unlock() + f.time = t +} + +// After is the fake version of time.After(d). +func (f *FakeClock) After(d time.Duration) <-chan time.Time { + f.lock.Lock() + defer f.lock.Unlock() + stopTime := f.time.Add(d) + ch := make(chan time.Time, 1) // Don't block! + f.waiters = append(f.waiters, &fakeClockWaiter{ + targetTime: stopTime, + destChan: ch, + }) + return ch +} + +// NewTimer constructs a fake timer, akin to time.NewTimer(d). +func (f *FakeClock) NewTimer(d time.Duration) clock.Timer { + f.lock.Lock() + defer f.lock.Unlock() + stopTime := f.time.Add(d) + ch := make(chan time.Time, 1) // Don't block! + timer := &fakeTimer{ + fakeClock: f, + waiter: fakeClockWaiter{ + targetTime: stopTime, + destChan: ch, + }, + } + f.waiters = append(f.waiters, &timer.waiter) + return timer +} + +// AfterFunc is the Fake version of time.AfterFunc(d, cb). +func (f *FakeClock) AfterFunc(d time.Duration, cb func()) clock.Timer { + f.lock.Lock() + defer f.lock.Unlock() + stopTime := f.time.Add(d) + ch := make(chan time.Time, 1) // Don't block! + + timer := &fakeTimer{ + fakeClock: f, + waiter: fakeClockWaiter{ + targetTime: stopTime, + destChan: ch, + afterFunc: cb, + }, + } + f.waiters = append(f.waiters, &timer.waiter) + return timer +} + +// Tick constructs a fake ticker, akin to time.Tick +func (f *FakeClock) Tick(d time.Duration) <-chan time.Time { + if d <= 0 { + return nil + } + f.lock.Lock() + defer f.lock.Unlock() + tickTime := f.time.Add(d) + ch := make(chan time.Time, 1) // hold one tick + f.waiters = append(f.waiters, &fakeClockWaiter{ + targetTime: tickTime, + stepInterval: d, + skipIfBlocked: true, + destChan: ch, + }) + + return ch +} + +// NewTicker returns a new Ticker. +func (f *FakeClock) NewTicker(d time.Duration) clock.Ticker { + f.lock.Lock() + defer f.lock.Unlock() + tickTime := f.time.Add(d) + ch := make(chan time.Time, 1) // hold one tick + f.waiters = append(f.waiters, &fakeClockWaiter{ + targetTime: tickTime, + stepInterval: d, + skipIfBlocked: true, + destChan: ch, + }) + + return &fakeTicker{ + c: ch, + } +} + +// Step moves the clock by Duration and notifies anyone that's called After, +// Tick, or NewTimer. +func (f *FakeClock) Step(d time.Duration) { + f.lock.Lock() + defer f.lock.Unlock() + f.setTimeLocked(f.time.Add(d)) +} + +// SetTime sets the time. +func (f *FakeClock) SetTime(t time.Time) { + f.lock.Lock() + defer f.lock.Unlock() + f.setTimeLocked(t) +} + +// Actually changes the time and checks any waiters. f must be write-locked. +func (f *FakeClock) setTimeLocked(t time.Time) { + f.time = t + newWaiters := make([]*fakeClockWaiter, 0, len(f.waiters)) + for i := range f.waiters { + w := f.waiters[i] + if !w.targetTime.After(t) { + if w.skipIfBlocked { + select { + case w.destChan <- t: + default: + } + } else { + w.destChan <- t + } + + if w.afterFunc != nil { + w.afterFunc() + } + + if w.stepInterval > 0 { + for !w.targetTime.After(t) { + w.targetTime = w.targetTime.Add(w.stepInterval) + } + newWaiters = append(newWaiters, w) + } + + } else { + newWaiters = append(newWaiters, f.waiters[i]) + } + } + f.waiters = newWaiters +} + +// HasWaiters returns true if Waiters() returns non-0 (so you can write race-free tests). +func (f *FakeClock) HasWaiters() bool { + f.lock.RLock() + defer f.lock.RUnlock() + return len(f.waiters) > 0 +} + +// Waiters returns the number of "waiters" on the clock (so you can write race-free +// tests). A waiter exists for: +// - every call to After that has not yet signaled its channel. +// - every call to AfterFunc that has not yet called its callback. +// - every timer created with NewTimer which is currently ticking. +// - every ticker created with NewTicker which is currently ticking. +// - every ticker created with Tick. +func (f *FakeClock) Waiters() int { + f.lock.RLock() + defer f.lock.RUnlock() + return len(f.waiters) +} + +// Sleep is akin to time.Sleep +func (f *FakeClock) Sleep(d time.Duration) { + f.Step(d) +} + +// IntervalClock implements clock.PassiveClock, but each invocation of Now steps the clock forward the specified duration. +// IntervalClock technically implements the other methods of clock.Clock, but each implementation is just a panic. +// +// Deprecated: See SimpleIntervalClock for an alternative that only has the methods of PassiveClock. +type IntervalClock struct { + Time time.Time + Duration time.Duration +} + +// Now returns i's time. +func (i *IntervalClock) Now() time.Time { + i.Time = i.Time.Add(i.Duration) + return i.Time +} + +// Since returns time since the time in i. +func (i *IntervalClock) Since(ts time.Time) time.Duration { + return i.Time.Sub(ts) +} + +// After is unimplemented, will panic. +// TODO: make interval clock use FakeClock so this can be implemented. +func (*IntervalClock) After(d time.Duration) <-chan time.Time { + panic("IntervalClock doesn't implement After") +} + +// NewTimer is unimplemented, will panic. +// TODO: make interval clock use FakeClock so this can be implemented. +func (*IntervalClock) NewTimer(d time.Duration) clock.Timer { + panic("IntervalClock doesn't implement NewTimer") +} + +// AfterFunc is unimplemented, will panic. +// TODO: make interval clock use FakeClock so this can be implemented. +func (*IntervalClock) AfterFunc(d time.Duration, f func()) clock.Timer { + panic("IntervalClock doesn't implement AfterFunc") +} + +// Tick is unimplemented, will panic. +// TODO: make interval clock use FakeClock so this can be implemented. +func (*IntervalClock) Tick(d time.Duration) <-chan time.Time { + panic("IntervalClock doesn't implement Tick") +} + +// NewTicker has no implementation yet and is omitted. +// TODO: make interval clock use FakeClock so this can be implemented. +func (*IntervalClock) NewTicker(d time.Duration) clock.Ticker { + panic("IntervalClock doesn't implement NewTicker") +} + +// Sleep is unimplemented, will panic. +func (*IntervalClock) Sleep(d time.Duration) { + panic("IntervalClock doesn't implement Sleep") +} + +var _ = clock.Timer(&fakeTimer{}) + +// fakeTimer implements clock.Timer based on a FakeClock. +type fakeTimer struct { + fakeClock *FakeClock + waiter fakeClockWaiter +} + +// C returns the channel that notifies when this timer has fired. +func (f *fakeTimer) C() <-chan time.Time { + return f.waiter.destChan +} + +// Stop prevents the Timer from firing. It returns true if the call stops the +// timer, false if the timer has already expired or been stopped. +func (f *fakeTimer) Stop() bool { + f.fakeClock.lock.Lock() + defer f.fakeClock.lock.Unlock() + + active := false + newWaiters := make([]*fakeClockWaiter, 0, len(f.fakeClock.waiters)) + for i := range f.fakeClock.waiters { + w := f.fakeClock.waiters[i] + if w != &f.waiter { + newWaiters = append(newWaiters, w) + continue + } + // If timer is found, it has not been fired yet. + active = true + } + + f.fakeClock.waiters = newWaiters + + return active +} + +// Reset changes the timer to expire after duration d. It returns true if the +// timer had been active, false if the timer had expired or been stopped. +func (f *fakeTimer) Reset(d time.Duration) bool { + f.fakeClock.lock.Lock() + defer f.fakeClock.lock.Unlock() + + active := false + + f.waiter.targetTime = f.fakeClock.time.Add(d) + + for i := range f.fakeClock.waiters { + w := f.fakeClock.waiters[i] + if w == &f.waiter { + // If timer is found, it has not been fired yet. + active = true + break + } + } + if !active { + f.fakeClock.waiters = append(f.fakeClock.waiters, &f.waiter) + } + + return active +} + +type fakeTicker struct { + c <-chan time.Time +} + +func (t *fakeTicker) C() <-chan time.Time { + return t.c +} + +func (t *fakeTicker) Stop() { +} diff --git a/vendor/k8s.io/utils/clock/testing/simple_interval_clock.go b/vendor/k8s.io/utils/clock/testing/simple_interval_clock.go new file mode 100644 index 00000000..951ca4d1 --- /dev/null +++ b/vendor/k8s.io/utils/clock/testing/simple_interval_clock.go @@ -0,0 +1,44 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "time" + + "k8s.io/utils/clock" +) + +var ( + _ = clock.PassiveClock(&SimpleIntervalClock{}) +) + +// SimpleIntervalClock implements clock.PassiveClock, but each invocation of Now steps the clock forward the specified duration +type SimpleIntervalClock struct { + Time time.Time + Duration time.Duration +} + +// Now returns i's time. +func (i *SimpleIntervalClock) Now() time.Time { + i.Time = i.Time.Add(i.Duration) + return i.Time +} + +// Since returns time since the time in i. +func (i *SimpleIntervalClock) Since(ts time.Time) time.Duration { + return i.Time.Sub(ts) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7236e087..de4edb50 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1582,6 +1582,7 @@ k8s.io/kubernetes/pkg/apis/core ## explicit; go 1.18 k8s.io/utils/buffer k8s.io/utils/clock +k8s.io/utils/clock/testing k8s.io/utils/integer k8s.io/utils/internal/third_party/forked/golang/golang-lru k8s.io/utils/internal/third_party/forked/golang/net From f55b505803f1432607ffdd5ed4ae28c1b870e21c Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Fri, 26 Jun 2026 14:50:33 +0600 Subject: [PATCH 3/3] Run unit-tests in CI make ci was check-license lint build with unit-tests/cover/verify commented out, so CI ran no tests at all. Now that the controller suite is re-enabled and race-clean under -race, add unit-tests to the ci target so the suite actually guards changes. (cover/verify remain opt-in.) Signed-off-by: Tamal Saha --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c81af8c7..d65a7cb3 100644 --- a/Makefile +++ b/Makefile @@ -493,7 +493,7 @@ check-license: ltag -t "./hack/license" --excludes "vendor contrib bin" --check -v .PHONY: ci -ci: check-license lint build #unit-tests cover verify +ci: check-license lint build unit-tests #cover verify .PHONY: qa qa: