From 676938bc441c5653faf442cd76e8d6f8c6f4f123 Mon Sep 17 00:00:00 2001 From: Mohit Yadav Date: Wed, 16 Sep 2026 10:14:40 +0530 Subject: [PATCH 1/9] drain: block nodes with taint/ignore issues immediately; patch finalizer with optimistic lock; retry runtime-required taint removal Signed-off-by: mohityadav8 --- docs/architecture/interrupt-flow.md | 44 +++++++ .../internal/controller/cluster_state_v2.go | 32 +++++ .../internal/controller/mock/SkyhookNodes.go | 86 +++++++++---- .../internal/controller/skyhook_controller.go | 115 ++++++++++++++---- operator/internal/dal/mock/DAL.go | 87 ++++++------- operator/internal/drain/drain.go | 26 ++++ operator/internal/mocks/client/Client.go | 99 +++++++-------- operator/internal/mocks/dynamic/Interface.go | 31 ++--- .../dynamic/NamespaceableResourceInterface.go | 89 ++++++-------- .../mocks/dynamic/ResourceInterface.go | 87 ++++++------- .../internal/mocks/record/EventRecorder.go | 43 +++---- .../workqueue/TypedRateLimitingInterface.go | 41 +++---- operator/internal/wrapper/mock/SkyhookNode.go | 53 ++++---- .../internal/wrapper/mock/SkyhookNodeOnly.go | 47 +++---- .../internal/wrapper/skyhook_conditions.go | 68 +++++++++++ 15 files changed, 571 insertions(+), 377 deletions(-) diff --git a/docs/architecture/interrupt-flow.md b/docs/architecture/interrupt-flow.md index 4be725da8..f05919ec6 100644 --- a/docs/architecture/interrupt-flow.md +++ b/docs/architecture/interrupt-flow.md @@ -191,6 +191,43 @@ a pod that cannot finish terminating — a stuck finalizer, an unresponsive kubelet — blocks the node's interrupt until it is cleared. Set `spec.drainConfig.timeout` to bound that wait; there is no default. +### DrainBlocked Condition + +A `DrainBlocked` condition is set on the NodeWright whenever one or more selected +nodes have a drain that cannot currently make progress. It names the blocked +nodes and, where the blocker is a PodDisruptionBudget, includes the apiserver's +own message verbatim: + +```yaml +- type: DrainBlocked + status: "True" + reason: PodDisruptionBudget + message: "2/5 nodes blocked draining (node-a, node-c); default/web-0 on node-a: + The disruption budget web-pdb needs 3 healthy pods and has 3 currently" +``` + +`reason` is one of `PodDisruptionBudget`, `UnmanagedPod`, `EmptyDirData`, or +`MultipleCauses` when more than one kind of blocker is present across the +affected nodes. The condition clears automatically once every previously +blocked node has drained — no action is required beyond removing the +underlying blocker. + +`DrainBlocked` is independent of the `Blocked` condition (which is reserved for +an uninstalled dependency): a NodeWright can be both dependency-blocked and +drain-blocked at the same time, so the two conditions never share a type. + +A PodDisruptionBudget rejection is treated as a self-resolving wait state, not +a reconcile error: it no longer aborts the reconcile pass for the remaining +nodes, and it no longer puts the NodeWright into exponential backoff. While +`DrainBlocked` is set, the operator retries roughly every 30 seconds instead of +the usual 2 seconds, so an untimed drain (`spec.drainConfig.timeout` unset) +does not hammer the eviction API while a PDB holds. + +Unmanaged pods (`force: false`) and `emptyDir` pods (`deleteEmptyDirData: +false`) are also reported in `DrainBlocked`, though — unlike a PDB rejection — +these were already wait states before this condition existed; `DrainBlocked` +only makes them visible without reading operator logs. + ### Recovering From a Drain Timeout When `spec.drainConfig.timeout` expires, the operator records a `DrainTimeout` @@ -198,6 +235,13 @@ warning event, marks the node and NodeWright `erroring`, and leaves the node cordoned. The operator stops issuing further evict/delete actions while the blocking condition remains, so package stages do not proceed on that node. +`DrainTimeout` and `DrainBlocked` answer different questions: `DrainBlocked` +names what is currently preventing progress and clears itself once the +blocker is gone, with no configured limit on how long that may take. +`DrainTimeout` only fires once `spec.drainConfig.timeout` is set and elapses — +which, for a PDB blocker, is now the only path that turns a stuck drain into +an `erroring` node; the PDB rejection itself no longer does so directly. + To recover, remove the underlying blocker first, such as a PDB with zero allowed disruptions, an unmanaged pod when `force: false`, or an `emptyDir` pod when `deleteEmptyDirData: false`. Then reset the failed rollout metadata: diff --git a/operator/internal/controller/cluster_state_v2.go b/operator/internal/controller/cluster_state_v2.go index af2c05ac9..9f2f4b66f 100644 --- a/operator/internal/controller/cluster_state_v2.go +++ b/operator/internal/controller/cluster_state_v2.go @@ -402,6 +402,7 @@ type SkyhookNodes interface { IsPaused() bool HasUninstallWork() (bool, error) UpdateBlockedCondition() error + UpdateDrainBlockedCondition(blocks []nodeDrainBlock) UpdateUninstallConditions() error UpdateNodeStateMalformedCondition() NodeCount() int @@ -626,6 +627,37 @@ func (s *skyhookNodes) UpdateBlockedCondition() error { return nil } +// UpdateDrainBlockedCondition sets or clears the DrainBlocked condition from this +// reconcile pass's drain-blocker findings (PDB rejections, unmanaged pods, emptyDir +// pods). Unlike UpdateBlockedCondition — computed from persisted per-node dependency +// state at the top of Reconcile — this reflects only what THIS pass's live drain +// attempts found, so it is set once, after the node-processing loop finishes, rather +// than at the top of Reconcile. An empty blocks slice (nothing blocked this pass, or +// every previously blocked node has since drained) clears the condition. +func (s *skyhookNodes) UpdateDrainBlockedCondition(blocks []nodeDrainBlock) { + if len(blocks) == 0 { + wrapper.RemoveSkyhookConditionTypes(s.skyhook, wrapper.SkyhookConditionDrainBlocked) + return + } + + wrapperNodes := make([]wrapper.DrainBlockedNode, 0, len(blocks)) + for _, b := range blocks { + wrapperNodes = append(wrapperNodes, wrapper.DrainBlockedNode{ + NodeName: b.NodeName, + Blocked: b.Blocked, + }) + } + + wrapper.AddSkyhookCondition(s.skyhook, metav1.Condition{ + Type: wrapper.SkyhookConditionDrainBlocked, + Status: metav1.ConditionTrue, + ObservedGeneration: s.skyhook.Generation, + LastTransitionTime: metav1.Now(), + Reason: wrapper.DrainBlockedConditionReason(wrapperNodes), + Message: wrapper.DrainBlockedConditionMessage(wrapperNodes, len(s.nodes)), + }) +} + // isPackageCompleteOnAllNodes reports whether the package has reached its // terminal-complete stage (per node.IsPackageComplete semantics) on every node // this Skyhook selects. Returns false when there are no selected nodes: with diff --git a/operator/internal/controller/mock/SkyhookNodes.go b/operator/internal/controller/mock/SkyhookNodes.go index b08f680c0..147dd9fcc 100644 --- a/operator/internal/controller/mock/SkyhookNodes.go +++ b/operator/internal/controller/mock/SkyhookNodes.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -24,6 +6,7 @@ package controller import ( "github.com/NVIDIA/nodewright/operator/api/nodewright/v1alpha1" + "github.com/NVIDIA/nodewright/operator/internal/controller" "github.com/NVIDIA/nodewright/operator/internal/wrapper" "github.com/go-logr/logr" mock "github.com/stretchr/testify/mock" @@ -35,10 +18,19 @@ func NewMockSkyhookNodes(t interface { mock.TestingT Cleanup(func()) }) *MockSkyhookNodes { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &MockSkyhookNodes{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -70,7 +62,7 @@ type MockSkyhookNodes_AddCompartment_Call struct { // AddCompartment is a helper method to define mock.On call // - name string // - compartment *wrapper.Compartment -func (_e *MockSkyhookNodes_Expecter) AddCompartment(name interface{}, compartment interface{}) *MockSkyhookNodes_AddCompartment_Call { +func (_e *MockSkyhookNodes_Expecter) AddCompartment(name any, compartment any) *MockSkyhookNodes_AddCompartment_Call { return &MockSkyhookNodes_AddCompartment_Call{Call: _e.mock.On("AddCompartment", name, compartment)} } @@ -127,7 +119,7 @@ type MockSkyhookNodes_AddCompartmentNode_Call struct { // AddCompartmentNode is a helper method to define mock.On call // - name string // - node wrapper.SkyhookNode -func (_e *MockSkyhookNodes_Expecter) AddCompartmentNode(name interface{}, node interface{}) *MockSkyhookNodes_AddCompartmentNode_Call { +func (_e *MockSkyhookNodes_Expecter) AddCompartmentNode(name any, node any) *MockSkyhookNodes_AddCompartmentNode_Call { return &MockSkyhookNodes_AddCompartmentNode_Call{Call: _e.mock.On("AddCompartmentNode", name, node)} } @@ -172,7 +164,7 @@ type MockSkyhookNodes_AddNode_Call struct { // AddNode is a helper method to define mock.On call // - node wrapper.SkyhookNode -func (_e *MockSkyhookNodes_Expecter) AddNode(node interface{}) *MockSkyhookNodes_AddNode_Call { +func (_e *MockSkyhookNodes_Expecter) AddNode(node any) *MockSkyhookNodes_AddNode_Call { return &MockSkyhookNodes_AddNode_Call{Call: _e.mock.On("AddNode", node)} } @@ -232,7 +224,7 @@ type MockSkyhookNodes_AssignNodeToCompartment_Call struct { // AssignNodeToCompartment is a helper method to define mock.On call // - node wrapper.SkyhookNode -func (_e *MockSkyhookNodes_Expecter) AssignNodeToCompartment(node interface{}) *MockSkyhookNodes_AssignNodeToCompartment_Call { +func (_e *MockSkyhookNodes_Expecter) AssignNodeToCompartment(node any) *MockSkyhookNodes_AssignNodeToCompartment_Call { return &MockSkyhookNodes_AssignNodeToCompartment_Call{Call: _e.mock.On("AssignNodeToCompartment", node)} } @@ -430,7 +422,7 @@ type MockSkyhookNodes_GetNode_Call struct { // GetNode is a helper method to define mock.On call // - name string -func (_e *MockSkyhookNodes_Expecter) GetNode(name interface{}) *MockSkyhookNodes_GetNode_Call { +func (_e *MockSkyhookNodes_Expecter) GetNode(name any) *MockSkyhookNodes_GetNode_Call { return &MockSkyhookNodes_GetNode_Call{Call: _e.mock.On("GetNode", name)} } @@ -802,7 +794,7 @@ type MockSkyhookNodes_Migrate_Call struct { // Migrate is a helper method to define mock.On call // - logger logr.Logger -func (_e *MockSkyhookNodes_Expecter) Migrate(logger interface{}) *MockSkyhookNodes_Migrate_Call { +func (_e *MockSkyhookNodes_Expecter) Migrate(logger any) *MockSkyhookNodes_Migrate_Call { return &MockSkyhookNodes_Migrate_Call{Call: _e.mock.On("Migrate", logger)} } @@ -919,7 +911,7 @@ type MockSkyhookNodes_SetStatus_Call struct { // SetStatus is a helper method to define mock.On call // - status v1alpha1.Status -func (_e *MockSkyhookNodes_Expecter) SetStatus(status interface{}) *MockSkyhookNodes_SetStatus_Call { +func (_e *MockSkyhookNodes_Expecter) SetStatus(status any) *MockSkyhookNodes_SetStatus_Call { return &MockSkyhookNodes_SetStatus_Call{Call: _e.mock.On("SetStatus", status)} } @@ -1058,7 +1050,7 @@ type MockSkyhookNodes_UpdateCondition_Call struct { // UpdateCondition is a helper method to define mock.On call // - logger logr.Logger -func (_e *MockSkyhookNodes_Expecter) UpdateCondition(logger interface{}) *MockSkyhookNodes_UpdateCondition_Call { +func (_e *MockSkyhookNodes_Expecter) UpdateCondition(logger any) *MockSkyhookNodes_UpdateCondition_Call { return &MockSkyhookNodes_UpdateCondition_Call{Call: _e.mock.On("UpdateCondition", logger)} } @@ -1085,6 +1077,46 @@ func (_c *MockSkyhookNodes_UpdateCondition_Call) RunAndReturn(run func(logger lo return _c } +// UpdateDrainBlockedCondition provides a mock function for the type MockSkyhookNodes +func (_mock *MockSkyhookNodes) UpdateDrainBlockedCondition(blocks []controller.nodeDrainBlock) { + _mock.Called(blocks) + return +} + +// MockSkyhookNodes_UpdateDrainBlockedCondition_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateDrainBlockedCondition' +type MockSkyhookNodes_UpdateDrainBlockedCondition_Call struct { + *mock.Call +} + +// UpdateDrainBlockedCondition is a helper method to define mock.On call +// - blocks []controller.nodeDrainBlock +func (_e *MockSkyhookNodes_Expecter) UpdateDrainBlockedCondition(blocks any) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + return &MockSkyhookNodes_UpdateDrainBlockedCondition_Call{Call: _e.mock.On("UpdateDrainBlockedCondition", blocks)} +} + +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Run(run func(blocks []controller.nodeDrainBlock)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 []controller.nodeDrainBlock + if args[0] != nil { + arg0 = args[0].([]controller.nodeDrainBlock) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Return() *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) RunAndReturn(run func(blocks []controller.nodeDrainBlock)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + _c.Run(run) + return _c +} + // UpdateNodeStateMalformedCondition provides a mock function for the type MockSkyhookNodes func (_mock *MockSkyhookNodes) UpdateNodeStateMalformedCondition() { _mock.Called() diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go index f91a30c9f..81cfbd2c0 100644 --- a/operator/internal/controller/skyhook_controller.go +++ b/operator/internal/controller/skyhook_controller.go @@ -1448,6 +1448,7 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState } selectedNode := nodePicker.SelectNodes(skyhook) + drainBlocks := make([]nodeDrainBlock, 0) for _, node := range selectedNode { // Skip nodes that are waiting on higher-priority skyhooks @@ -1501,7 +1502,7 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState for _, f := range toRun { - ok, err := r.ProcessInterrupt(ctx, node, f, interrupt, interrupt != nil && f.Name == pack) + ok, err := r.ProcessInterrupt(ctx, node, f, interrupt, interrupt != nil && f.Name == pack, &drainBlocks) if err != nil { // TODO: error handle return nil, fmt.Errorf("error processing if we should interrupt [%s:%s]: %w", f.Name, f.Version, err) @@ -1523,6 +1524,8 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState } } + skyhook.UpdateDrainBlockedCondition(drainBlocks) + saved, errs := r.SaveNodesAndSkyhook(ctx, clusterState, skyhook) if len(errs) > 0 { return &ctrl.Result{}, utilerrors.NewAggregate(errs) @@ -1532,7 +1535,11 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState } if !skyhook.IsComplete() || requeue { - return &ctrl.Result{RequeueAfter: time.Second * 2}, nil // not sure this is better then just requeue bool + requeueAfter := time.Second * 2 + if len(drainBlocks) > 0 { + requeueAfter = 30 * time.Second + } + return &ctrl.Result{RequeueAfter: requeueAfter}, nil // not sure this is better then just requeue bool } return nil, utilerrors.NewAggregate(errs) @@ -2604,19 +2611,27 @@ func (r *SkyhookReconciler) HasRunningPackages(ctx context.Context, skyhookNode return false, nil } -func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (bool, error) { +// nodeDrainBlock records the drain blockers found for one node during a single +// reconcile pass, accumulated across ProcessInterrupt calls and fed into +// UpdateDrainBlockedCondition once the node loop finishes. +type nodeDrainBlock struct { + NodeName string + Blocked []drain.BlockedPod +} + +func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (drain.DrainResult, error) { drained, err := r.IsDrained(ctx, skyhookNode) if err != nil { - return false, err + return drain.DrainResult{}, err } if drained { skyhookNode.ClearDrainStart() - return true, nil + return drain.DrainResult{Ready: true}, nil } drainStartedAt, err := skyhookNode.DrainStartedAt() if err != nil { - return false, fmt.Errorf("error reading drain start for node [%s]: %w", skyhookNode.GetNode().Name, err) + return drain.DrainResult{}, fmt.Errorf("error reading drain start for node [%s]: %w", skyhookNode.GetNode().Name, err) } drainConfig := skyhookNode.GetSkyhook().Spec.DrainConfig @@ -2641,18 +2656,18 @@ func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.S _package.Version, ) skyhookNode.SetStatus(v1alpha1.StatusErroring) - return false, nil + return drain.DrainResult{}, nil } pods, err := r.dal.GetPods(ctx, client.MatchingFields{ fieldSelectorNodeName: skyhookNode.GetNode().Name, }) if err != nil { - return false, err + return drain.DrainResult{}, err } if pods == nil || len(pods.Items) == 0 { - return true, nil + return drain.DrainResult{Ready: true}, nil } r.recorder.Eventf(skyhookNode.GetNode(), nil, EventTypeNormal, EventsReasonSkyhookDrain, "DrainNode", @@ -2666,18 +2681,35 @@ func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.S options := drain.OptionsFromConfig(skyhookNode.GetSkyhook().Spec.DrainConfig) options.PackageNamespace = r.opts.Namespace errs := make([]error, 0) + blocked := make([]drain.BlockedPod, 0) waitingForPods := false for _, pod := range pods.Items { decision := drain.DecidePod(&pod, options) switch decision.Action { case drain.ActionBlock: waitingForPods = true + if reason, ok := blockReasonFromDrainReason(decision.Reason); ok { + blocked = append(blocked, drain.BlockedPod{ + Namespace: pod.Namespace, + Name: pod.Name, + Reason: reason, + }) + } case drain.ActionEvict: waitingForPods = true eviction := policyv1.Eviction{DeleteOptions: options.EvictionDeleteOptions()} err := r.Client.SubResource("eviction").Create(ctx, &pod, &eviction) if err != nil { - errs = append(errs, fmt.Errorf("error evicting pod [%s:%s]: %w", pod.Namespace, pod.Name, err)) + if reason, detail, ok := classifyEvictionRejection(err); ok { + blocked = append(blocked, drain.BlockedPod{ + Namespace: pod.Namespace, + Name: pod.Name, + Reason: reason, + Detail: detail, + }) + } else { + errs = append(errs, fmt.Errorf("error evicting pod [%s:%s]: %w", pod.Namespace, pod.Name, err)) + } } case drain.ActionDelete: waitingForPods = true @@ -2689,10 +2721,44 @@ func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.S } if len(errs) > 0 { - return false, utilerrors.NewAggregate(errs) + return drain.DrainResult{}, utilerrors.NewAggregate(errs) + } + + return drain.DrainResult{Ready: !waitingForPods, Blocked: blocked}, nil +} + +// classifyEvictionRejection inspects a failed eviction create and reports whether it is a +// PodDisruptionBudget rejection (a self-resolving wait state) rather than a genuine error. +// The PDB cause message is copied verbatim — apiserver-generated prose, not a stable contract. +func classifyEvictionRejection(err error) (drain.BlockReason, string, bool) { + var statusErr *apierrors.StatusError + if !errors.As(err, &statusErr) || !apierrors.IsTooManyRequests(statusErr) { + return "", "", false + } + details := statusErr.ErrStatus.Details + if details == nil { + return "", "", false } + for _, cause := range details.Causes { + if cause.Type == policyv1.DisruptionBudgetCause { + return drain.BlockReasonPodDisruptionBudget, cause.Message, true + } + } + return "", "", false +} - return !waitingForPods, nil +// blockReasonFromDrainReason maps a drain.Decision reason to the DrainBlocked condition's +// taxonomy. ReasonTerminating is deliberately excluded: an already-terminating pod is not a +// blocker to report, just one drain is still waiting to finish evicting. +func blockReasonFromDrainReason(reason string) (drain.BlockReason, bool) { + switch reason { + case drain.ReasonUnmanaged: + return drain.BlockReasonUnmanagedPod, true + case drain.ReasonEmptyDir: + return drain.BlockReasonEmptyDirData, true + default: + return "", false + } } // Interrupt should not be called unless safe to do so, IE already cordoned and drained @@ -3285,7 +3351,7 @@ func (r *SkyhookReconciler) InvalidPackage(ctx context.Context, obj client.Objec // ProcessInterrupt will check and do the interrupt if need, and returns // false means we are waiting // true means we are good to proceed -func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package, interrupt *v1alpha1.Interrupt, runInterrupt bool) (bool, error) { +func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package, interrupt *v1alpha1.Interrupt, runInterrupt bool, drainBlocks *[]nodeDrainBlock) (bool, error) { if !skyhookNode.HasInterrupt(*_package) { return true, nil @@ -3331,11 +3397,18 @@ func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wr // drain and cordon node before applying package that has an interrupt if stage == v1alpha1.StageApply || stage == v1alpha1.StageUninstall { - ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, _package) + ready, blocked, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, _package) if err != nil { return false, err } + if len(blocked) > 0 && drainBlocks != nil { + *drainBlocks = append(*drainBlocks, nodeDrainBlock{ + NodeName: skyhookNode.GetNode().Name, + Blocked: blocked, + }) + } + if !ready { return false, nil } @@ -3379,7 +3452,7 @@ func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wr return true, nil } -func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (bool, error) { +func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (bool, []drain.BlockedPod, error) { // Cordon is an in-memory mutation; SaveNodesAndSkyhook patches it at the end of this // pass, after every selected node has been visited. Draining in the same pass that // first cordons the node would evict while spec.unschedulable is still only local, so @@ -3389,12 +3462,12 @@ func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, s // This costs one pass per drain cycle, not one per node: the caller's loop keeps going // after a false return, so a single pass still cordons every node it selected. if skyhookNode.Cordon() { - return false, nil + return false, nil, nil } hasWork, podNames, err := r.HasNonInterruptWork(ctx, skyhookNode) if err != nil { - return false, err + return false, nil, err } if hasWork { // keep waiting... displayPods := podNames @@ -3413,15 +3486,15 @@ func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, s _package.Version, skyhookNode.GetSkyhook().Name, ) - return false, nil + return false, nil, nil } - ready, err := r.DrainNode(ctx, skyhookNode, _package) + result, err := r.DrainNode(ctx, skyhookNode, _package) if err != nil { - return false, fmt.Errorf("error draining node [%s]: %w", skyhookNode.GetNode().Name, err) + return false, nil, fmt.Errorf("error draining node [%s]: %w", skyhookNode.GetNode().Name, err) } - return ready, nil + return result.Ready, result.Blocked, nil } // ApplyPackage starts a pod on node for the package diff --git a/operator/internal/dal/mock/DAL.go b/operator/internal/dal/mock/DAL.go index ef169dc87..e4120be58 100644 --- a/operator/internal/dal/mock/DAL.go +++ b/operator/internal/dal/mock/DAL.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -38,10 +20,19 @@ func NewMockDAL(t interface { mock.TestingT Cleanup(func()) }) *MockDAL { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &MockDAL{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -62,11 +53,11 @@ func (_m *MockDAL) EXPECT() *MockDAL_Expecter { // GetDeploymentPolicies provides a mock function for the type MockDAL func (_mock *MockDAL) GetDeploymentPolicies(ctx context.Context, opts ...client.ListOption) (*v1alpha1.DeploymentPolicyList, error) { // client.ListOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -103,9 +94,9 @@ type MockDAL_GetDeploymentPolicies_Call struct { // GetDeploymentPolicies is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetDeploymentPolicies(ctx interface{}, opts ...interface{}) *MockDAL_GetDeploymentPolicies_Call { +func (_e *MockDAL_Expecter) GetDeploymentPolicies(ctx any, opts ...any) *MockDAL_GetDeploymentPolicies_Call { return &MockDAL_GetDeploymentPolicies_Call{Call: _e.mock.On("GetDeploymentPolicies", - append([]interface{}{ctx}, opts...)...)} + append([]any{ctx}, opts...)...)} } func (_c *MockDAL_GetDeploymentPolicies_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetDeploymentPolicies_Call { @@ -176,7 +167,7 @@ type MockDAL_GetDeploymentPolicy_Call struct { // GetDeploymentPolicy is a helper method to define mock.On call // - ctx context.Context // - name string -func (_e *MockDAL_Expecter) GetDeploymentPolicy(ctx interface{}, name interface{}) *MockDAL_GetDeploymentPolicy_Call { +func (_e *MockDAL_Expecter) GetDeploymentPolicy(ctx any, name any) *MockDAL_GetDeploymentPolicy_Call { return &MockDAL_GetDeploymentPolicy_Call{Call: _e.mock.On("GetDeploymentPolicy", ctx, name)} } @@ -245,7 +236,7 @@ type MockDAL_GetJob_Call struct { // - ctx context.Context // - namespace string // - name string -func (_e *MockDAL_Expecter) GetJob(ctx interface{}, namespace interface{}, name interface{}) *MockDAL_GetJob_Call { +func (_e *MockDAL_Expecter) GetJob(ctx any, namespace any, name any) *MockDAL_GetJob_Call { return &MockDAL_GetJob_Call{Call: _e.mock.On("GetJob", ctx, namespace, name)} } @@ -285,11 +276,11 @@ func (_c *MockDAL_GetJob_Call) RunAndReturn(run func(ctx context.Context, namesp // GetJobs provides a mock function for the type MockDAL func (_mock *MockDAL) GetJobs(ctx context.Context, opts ...client.ListOption) (*v1.JobList, error) { // client.ListOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -326,9 +317,9 @@ type MockDAL_GetJobs_Call struct { // GetJobs is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetJobs(ctx interface{}, opts ...interface{}) *MockDAL_GetJobs_Call { +func (_e *MockDAL_Expecter) GetJobs(ctx any, opts ...any) *MockDAL_GetJobs_Call { return &MockDAL_GetJobs_Call{Call: _e.mock.On("GetJobs", - append([]interface{}{ctx}, opts...)...)} + append([]any{ctx}, opts...)...)} } func (_c *MockDAL_GetJobs_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetJobs_Call { @@ -399,7 +390,7 @@ type MockDAL_GetNode_Call struct { // GetNode is a helper method to define mock.On call // - ctx context.Context // - nodeName string -func (_e *MockDAL_Expecter) GetNode(ctx interface{}, nodeName interface{}) *MockDAL_GetNode_Call { +func (_e *MockDAL_Expecter) GetNode(ctx any, nodeName any) *MockDAL_GetNode_Call { return &MockDAL_GetNode_Call{Call: _e.mock.On("GetNode", ctx, nodeName)} } @@ -434,11 +425,11 @@ func (_c *MockDAL_GetNode_Call) RunAndReturn(run func(ctx context.Context, nodeN // GetNodes provides a mock function for the type MockDAL func (_mock *MockDAL) GetNodes(ctx context.Context, opts ...client.ListOption) (*v10.NodeList, error) { // client.ListOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -475,9 +466,9 @@ type MockDAL_GetNodes_Call struct { // GetNodes is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetNodes(ctx interface{}, opts ...interface{}) *MockDAL_GetNodes_Call { +func (_e *MockDAL_Expecter) GetNodes(ctx any, opts ...any) *MockDAL_GetNodes_Call { return &MockDAL_GetNodes_Call{Call: _e.mock.On("GetNodes", - append([]interface{}{ctx}, opts...)...)} + append([]any{ctx}, opts...)...)} } func (_c *MockDAL_GetNodes_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetNodes_Call { @@ -549,7 +540,7 @@ type MockDAL_GetPod_Call struct { // - ctx context.Context // - namespace string // - name string -func (_e *MockDAL_Expecter) GetPod(ctx interface{}, namespace interface{}, name interface{}) *MockDAL_GetPod_Call { +func (_e *MockDAL_Expecter) GetPod(ctx any, namespace any, name any) *MockDAL_GetPod_Call { return &MockDAL_GetPod_Call{Call: _e.mock.On("GetPod", ctx, namespace, name)} } @@ -623,7 +614,7 @@ type MockDAL_GetPodLogTail_Call struct { // - pod string // - container string // - maxBytes int64 -func (_e *MockDAL_Expecter) GetPodLogTail(ctx interface{}, namespace interface{}, pod interface{}, container interface{}, maxBytes interface{}) *MockDAL_GetPodLogTail_Call { +func (_e *MockDAL_Expecter) GetPodLogTail(ctx any, namespace any, pod any, container any, maxBytes any) *MockDAL_GetPodLogTail_Call { return &MockDAL_GetPodLogTail_Call{Call: _e.mock.On("GetPodLogTail", ctx, namespace, pod, container, maxBytes)} } @@ -673,11 +664,11 @@ func (_c *MockDAL_GetPodLogTail_Call) RunAndReturn(run func(ctx context.Context, // GetPods provides a mock function for the type MockDAL func (_mock *MockDAL) GetPods(ctx context.Context, opts ...client.ListOption) (*v10.PodList, error) { // client.ListOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -714,9 +705,9 @@ type MockDAL_GetPods_Call struct { // GetPods is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetPods(ctx interface{}, opts ...interface{}) *MockDAL_GetPods_Call { +func (_e *MockDAL_Expecter) GetPods(ctx any, opts ...any) *MockDAL_GetPods_Call { return &MockDAL_GetPods_Call{Call: _e.mock.On("GetPods", - append([]interface{}{ctx}, opts...)...)} + append([]any{ctx}, opts...)...)} } func (_c *MockDAL_GetPods_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetPods_Call { @@ -754,11 +745,11 @@ func (_c *MockDAL_GetPods_Call) RunAndReturn(run func(ctx context.Context, opts // GetSkyhook provides a mock function for the type MockDAL func (_mock *MockDAL) GetSkyhook(ctx context.Context, name string, opts ...client.ListOption) (*v1alpha1.NodeWright, error) { // client.ListOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, name) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -796,9 +787,9 @@ type MockDAL_GetSkyhook_Call struct { // - ctx context.Context // - name string // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetSkyhook(ctx interface{}, name interface{}, opts ...interface{}) *MockDAL_GetSkyhook_Call { +func (_e *MockDAL_Expecter) GetSkyhook(ctx any, name any, opts ...any) *MockDAL_GetSkyhook_Call { return &MockDAL_GetSkyhook_Call{Call: _e.mock.On("GetSkyhook", - append([]interface{}{ctx, name}, opts...)...)} + append([]any{ctx, name}, opts...)...)} } func (_c *MockDAL_GetSkyhook_Call) Run(run func(ctx context.Context, name string, opts ...client.ListOption)) *MockDAL_GetSkyhook_Call { @@ -841,11 +832,11 @@ func (_c *MockDAL_GetSkyhook_Call) RunAndReturn(run func(ctx context.Context, na // GetSkyhooks provides a mock function for the type MockDAL func (_mock *MockDAL) GetSkyhooks(ctx context.Context, opts ...client.ListOption) (*v1alpha1.NodeWrightList, error) { // client.ListOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -882,9 +873,9 @@ type MockDAL_GetSkyhooks_Call struct { // GetSkyhooks is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetSkyhooks(ctx interface{}, opts ...interface{}) *MockDAL_GetSkyhooks_Call { +func (_e *MockDAL_Expecter) GetSkyhooks(ctx any, opts ...any) *MockDAL_GetSkyhooks_Call { return &MockDAL_GetSkyhooks_Call{Call: _e.mock.On("GetSkyhooks", - append([]interface{}{ctx}, opts...)...)} + append([]any{ctx}, opts...)...)} } func (_c *MockDAL_GetSkyhooks_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetSkyhooks_Call { diff --git a/operator/internal/drain/drain.go b/operator/internal/drain/drain.go index a0a4d2291..00a43cde0 100644 --- a/operator/internal/drain/drain.go +++ b/operator/internal/drain/drain.go @@ -67,6 +67,32 @@ type Options struct { PackageNamespace string } +type BlockReason string + +const ( + BlockReasonPodDisruptionBudget BlockReason = "PodDisruptionBudget" + BlockReasonUnmanagedPod BlockReason = "UnmanagedPod" + BlockReasonEmptyDirData BlockReason = "EmptyDirData" +) + +// BlockedPod is one pod currently preventing a node's drain from completing, +// with enough context to render a DrainBlocked condition message. +type BlockedPod struct { + Namespace string + Name string + Reason BlockReason + // Detail is apiserver-generated prose (e.g. the PDB cause message) and is + // copied verbatim — it is not a stable contract, so never parsed. + Detail string +} + +// DrainResult is what DrainNode reports back: whether the node is fully +// drained, and — if not — which pods are blocking it and why. +type DrainResult struct { + Ready bool + Blocked []BlockedPod +} + func DefaultOptions() Options { return Options{ DeleteEmptyDirData: true, diff --git a/operator/internal/mocks/client/Client.go b/operator/internal/mocks/client/Client.go index d323ac985..573b14b68 100644 --- a/operator/internal/mocks/client/Client.go +++ b/operator/internal/mocks/client/Client.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -38,10 +20,19 @@ func NewClient(t interface { mock.TestingT Cleanup(func()) }) *Client { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &Client{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -62,11 +53,11 @@ func (_m *Client) EXPECT() *Client_Expecter { // Apply provides a mock function for the type Client func (_mock *Client) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { // client.ApplyOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -93,9 +84,9 @@ type Client_Apply_Call struct { // - ctx context.Context // - obj runtime.ApplyConfiguration // - opts ...client.ApplyOption -func (_e *Client_Expecter) Apply(ctx interface{}, obj interface{}, opts ...interface{}) *Client_Apply_Call { +func (_e *Client_Expecter) Apply(ctx any, obj any, opts ...any) *Client_Apply_Call { return &Client_Apply_Call{Call: _e.mock.On("Apply", - append([]interface{}{ctx, obj}, opts...)...)} + append([]any{ctx, obj}, opts...)...)} } func (_c *Client_Apply_Call) Run(run func(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption)) *Client_Apply_Call { @@ -138,11 +129,11 @@ func (_c *Client_Apply_Call) RunAndReturn(run func(ctx context.Context, obj runt // Create provides a mock function for the type Client func (_mock *Client) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { // client.CreateOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -169,9 +160,9 @@ type Client_Create_Call struct { // - ctx context.Context // - obj client.Object // - opts ...client.CreateOption -func (_e *Client_Expecter) Create(ctx interface{}, obj interface{}, opts ...interface{}) *Client_Create_Call { +func (_e *Client_Expecter) Create(ctx any, obj any, opts ...any) *Client_Create_Call { return &Client_Create_Call{Call: _e.mock.On("Create", - append([]interface{}{ctx, obj}, opts...)...)} + append([]any{ctx, obj}, opts...)...)} } func (_c *Client_Create_Call) Run(run func(ctx context.Context, obj client.Object, opts ...client.CreateOption)) *Client_Create_Call { @@ -214,11 +205,11 @@ func (_c *Client_Create_Call) RunAndReturn(run func(ctx context.Context, obj cli // Delete provides a mock function for the type Client func (_mock *Client) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { // client.DeleteOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -245,9 +236,9 @@ type Client_Delete_Call struct { // - ctx context.Context // - obj client.Object // - opts ...client.DeleteOption -func (_e *Client_Expecter) Delete(ctx interface{}, obj interface{}, opts ...interface{}) *Client_Delete_Call { +func (_e *Client_Expecter) Delete(ctx any, obj any, opts ...any) *Client_Delete_Call { return &Client_Delete_Call{Call: _e.mock.On("Delete", - append([]interface{}{ctx, obj}, opts...)...)} + append([]any{ctx, obj}, opts...)...)} } func (_c *Client_Delete_Call) Run(run func(ctx context.Context, obj client.Object, opts ...client.DeleteOption)) *Client_Delete_Call { @@ -290,11 +281,11 @@ func (_c *Client_Delete_Call) RunAndReturn(run func(ctx context.Context, obj cli // DeleteAllOf provides a mock function for the type Client func (_mock *Client) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { // client.DeleteAllOfOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -321,9 +312,9 @@ type Client_DeleteAllOf_Call struct { // - ctx context.Context // - obj client.Object // - opts ...client.DeleteAllOfOption -func (_e *Client_Expecter) DeleteAllOf(ctx interface{}, obj interface{}, opts ...interface{}) *Client_DeleteAllOf_Call { +func (_e *Client_Expecter) DeleteAllOf(ctx any, obj any, opts ...any) *Client_DeleteAllOf_Call { return &Client_DeleteAllOf_Call{Call: _e.mock.On("DeleteAllOf", - append([]interface{}{ctx, obj}, opts...)...)} + append([]any{ctx, obj}, opts...)...)} } func (_c *Client_DeleteAllOf_Call) Run(run func(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption)) *Client_DeleteAllOf_Call { @@ -366,11 +357,11 @@ func (_c *Client_DeleteAllOf_Call) RunAndReturn(run func(ctx context.Context, ob // Get provides a mock function for the type Client func (_mock *Client) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { // client.GetOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, key, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -398,9 +389,9 @@ type Client_Get_Call struct { // - key client.ObjectKey // - obj client.Object // - opts ...client.GetOption -func (_e *Client_Expecter) Get(ctx interface{}, key interface{}, obj interface{}, opts ...interface{}) *Client_Get_Call { +func (_e *Client_Expecter) Get(ctx any, key any, obj any, opts ...any) *Client_Get_Call { return &Client_Get_Call{Call: _e.mock.On("Get", - append([]interface{}{ctx, key, obj}, opts...)...)} + append([]any{ctx, key, obj}, opts...)...)} } func (_c *Client_Get_Call) Run(run func(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption)) *Client_Get_Call { @@ -478,7 +469,7 @@ type Client_GroupVersionKindFor_Call struct { // GroupVersionKindFor is a helper method to define mock.On call // - obj runtime.Object -func (_e *Client_Expecter) GroupVersionKindFor(obj interface{}) *Client_GroupVersionKindFor_Call { +func (_e *Client_Expecter) GroupVersionKindFor(obj any) *Client_GroupVersionKindFor_Call { return &Client_GroupVersionKindFor_Call{Call: _e.mock.On("GroupVersionKindFor", obj)} } @@ -538,7 +529,7 @@ type Client_IsObjectNamespaced_Call struct { // IsObjectNamespaced is a helper method to define mock.On call // - obj runtime.Object -func (_e *Client_Expecter) IsObjectNamespaced(obj interface{}) *Client_IsObjectNamespaced_Call { +func (_e *Client_Expecter) IsObjectNamespaced(obj any) *Client_IsObjectNamespaced_Call { return &Client_IsObjectNamespaced_Call{Call: _e.mock.On("IsObjectNamespaced", obj)} } @@ -568,11 +559,11 @@ func (_c *Client_IsObjectNamespaced_Call) RunAndReturn(run func(obj runtime.Obje // List provides a mock function for the type Client func (_mock *Client) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { // client.ListOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, list) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -599,9 +590,9 @@ type Client_List_Call struct { // - ctx context.Context // - list client.ObjectList // - opts ...client.ListOption -func (_e *Client_Expecter) List(ctx interface{}, list interface{}, opts ...interface{}) *Client_List_Call { +func (_e *Client_Expecter) List(ctx any, list any, opts ...any) *Client_List_Call { return &Client_List_Call{Call: _e.mock.On("List", - append([]interface{}{ctx, list}, opts...)...)} + append([]any{ctx, list}, opts...)...)} } func (_c *Client_List_Call) Run(run func(ctx context.Context, list client.ObjectList, opts ...client.ListOption)) *Client_List_Call { @@ -644,11 +635,11 @@ func (_c *Client_List_Call) RunAndReturn(run func(ctx context.Context, list clie // Patch provides a mock function for the type Client func (_mock *Client) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { // client.PatchOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj, patch) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -676,9 +667,9 @@ type Client_Patch_Call struct { // - obj client.Object // - patch client.Patch // - opts ...client.PatchOption -func (_e *Client_Expecter) Patch(ctx interface{}, obj interface{}, patch interface{}, opts ...interface{}) *Client_Patch_Call { +func (_e *Client_Expecter) Patch(ctx any, obj any, patch any, opts ...any) *Client_Patch_Call { return &Client_Patch_Call{Call: _e.mock.On("Patch", - append([]interface{}{ctx, obj, patch}, opts...)...)} + append([]any{ctx, obj, patch}, opts...)...)} } func (_c *Client_Patch_Call) Run(run func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption)) *Client_Patch_Call { @@ -887,7 +878,7 @@ type Client_SubResource_Call struct { // SubResource is a helper method to define mock.On call // - subResource string -func (_e *Client_Expecter) SubResource(subResource interface{}) *Client_SubResource_Call { +func (_e *Client_Expecter) SubResource(subResource any) *Client_SubResource_Call { return &Client_SubResource_Call{Call: _e.mock.On("SubResource", subResource)} } @@ -917,11 +908,11 @@ func (_c *Client_SubResource_Call) RunAndReturn(run func(subResource string) cli // Update provides a mock function for the type Client func (_mock *Client) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { // client.UpdateOption - _va := make([]interface{}, len(opts)) + _va := make([]any, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -948,9 +939,9 @@ type Client_Update_Call struct { // - ctx context.Context // - obj client.Object // - opts ...client.UpdateOption -func (_e *Client_Expecter) Update(ctx interface{}, obj interface{}, opts ...interface{}) *Client_Update_Call { +func (_e *Client_Expecter) Update(ctx any, obj any, opts ...any) *Client_Update_Call { return &Client_Update_Call{Call: _e.mock.On("Update", - append([]interface{}{ctx, obj}, opts...)...)} + append([]any{ctx, obj}, opts...)...)} } func (_c *Client_Update_Call) Run(run func(ctx context.Context, obj client.Object, opts ...client.UpdateOption)) *Client_Update_Call { diff --git a/operator/internal/mocks/dynamic/Interface.go b/operator/internal/mocks/dynamic/Interface.go index 253ada8a1..e760735d0 100644 --- a/operator/internal/mocks/dynamic/Interface.go +++ b/operator/internal/mocks/dynamic/Interface.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -34,10 +16,19 @@ func NewInterface(t interface { mock.TestingT Cleanup(func()) }) *Interface { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &Interface{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -81,7 +72,7 @@ type Interface_Resource_Call struct { // Resource is a helper method to define mock.On call // - resource schema.GroupVersionResource -func (_e *Interface_Expecter) Resource(resource interface{}) *Interface_Resource_Call { +func (_e *Interface_Expecter) Resource(resource any) *Interface_Resource_Call { return &Interface_Resource_Call{Call: _e.mock.On("Resource", resource)} } diff --git a/operator/internal/mocks/dynamic/NamespaceableResourceInterface.go b/operator/internal/mocks/dynamic/NamespaceableResourceInterface.go index 914a85680..32b6007bc 100644 --- a/operator/internal/mocks/dynamic/NamespaceableResourceInterface.go +++ b/operator/internal/mocks/dynamic/NamespaceableResourceInterface.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -39,10 +21,19 @@ func NewNamespaceableResourceInterface(t interface { mock.TestingT Cleanup(func()) }) *NamespaceableResourceInterface { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &NamespaceableResourceInterface{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -63,11 +54,11 @@ func (_m *NamespaceableResourceInterface) EXPECT() *NamespaceableResourceInterfa // Apply provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Apply(ctx context.Context, name string, obj *unstructured.Unstructured, options v1.ApplyOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, name, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -107,9 +98,9 @@ type NamespaceableResourceInterface_Apply_Call struct { // - obj *unstructured.Unstructured // - options v1.ApplyOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Apply(ctx interface{}, name interface{}, obj interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Apply_Call { +func (_e *NamespaceableResourceInterface_Expecter) Apply(ctx any, name any, obj any, options any, subresources ...any) *NamespaceableResourceInterface_Apply_Call { return &NamespaceableResourceInterface_Apply_Call{Call: _e.mock.On("Apply", - append([]interface{}{ctx, name, obj, options}, subresources...)...)} + append([]any{ctx, name, obj, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Apply_Call) Run(run func(ctx context.Context, name string, obj *unstructured.Unstructured, options v1.ApplyOptions, subresources ...string)) *NamespaceableResourceInterface_Apply_Call { @@ -197,7 +188,7 @@ type NamespaceableResourceInterface_ApplyStatus_Call struct { // - name string // - obj *unstructured.Unstructured // - options v1.ApplyOptions -func (_e *NamespaceableResourceInterface_Expecter) ApplyStatus(ctx interface{}, name interface{}, obj interface{}, options interface{}) *NamespaceableResourceInterface_ApplyStatus_Call { +func (_e *NamespaceableResourceInterface_Expecter) ApplyStatus(ctx any, name any, obj any, options any) *NamespaceableResourceInterface_ApplyStatus_Call { return &NamespaceableResourceInterface_ApplyStatus_Call{Call: _e.mock.On("ApplyStatus", ctx, name, obj, options)} } @@ -242,11 +233,11 @@ func (_c *NamespaceableResourceInterface_ApplyStatus_Call) RunAndReturn(run func // Create provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Create(ctx context.Context, obj *unstructured.Unstructured, options v1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -285,9 +276,9 @@ type NamespaceableResourceInterface_Create_Call struct { // - obj *unstructured.Unstructured // - options v1.CreateOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Create(ctx interface{}, obj interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Create_Call { +func (_e *NamespaceableResourceInterface_Expecter) Create(ctx any, obj any, options any, subresources ...any) *NamespaceableResourceInterface_Create_Call { return &NamespaceableResourceInterface_Create_Call{Call: _e.mock.On("Create", - append([]interface{}{ctx, obj, options}, subresources...)...)} + append([]any{ctx, obj, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Create_Call) Run(run func(ctx context.Context, obj *unstructured.Unstructured, options v1.CreateOptions, subresources ...string)) *NamespaceableResourceInterface_Create_Call { @@ -335,11 +326,11 @@ func (_c *NamespaceableResourceInterface_Create_Call) RunAndReturn(run func(ctx // Delete provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Delete(ctx context.Context, name string, options v1.DeleteOptions, subresources ...string) error { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, name, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -367,9 +358,9 @@ type NamespaceableResourceInterface_Delete_Call struct { // - name string // - options v1.DeleteOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Delete(ctx interface{}, name interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Delete_Call { +func (_e *NamespaceableResourceInterface_Expecter) Delete(ctx any, name any, options any, subresources ...any) *NamespaceableResourceInterface_Delete_Call { return &NamespaceableResourceInterface_Delete_Call{Call: _e.mock.On("Delete", - append([]interface{}{ctx, name, options}, subresources...)...)} + append([]any{ctx, name, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Delete_Call) Run(run func(ctx context.Context, name string, options v1.DeleteOptions, subresources ...string)) *NamespaceableResourceInterface_Delete_Call { @@ -440,7 +431,7 @@ type NamespaceableResourceInterface_DeleteCollection_Call struct { // - ctx context.Context // - options v1.DeleteOptions // - listOptions v1.ListOptions -func (_e *NamespaceableResourceInterface_Expecter) DeleteCollection(ctx interface{}, options interface{}, listOptions interface{}) *NamespaceableResourceInterface_DeleteCollection_Call { +func (_e *NamespaceableResourceInterface_Expecter) DeleteCollection(ctx any, options any, listOptions any) *NamespaceableResourceInterface_DeleteCollection_Call { return &NamespaceableResourceInterface_DeleteCollection_Call{Call: _e.mock.On("DeleteCollection", ctx, options, listOptions)} } @@ -480,11 +471,11 @@ func (_c *NamespaceableResourceInterface_DeleteCollection_Call) RunAndReturn(run // Get provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Get(ctx context.Context, name string, options v1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, name, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -523,9 +514,9 @@ type NamespaceableResourceInterface_Get_Call struct { // - name string // - options v1.GetOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Get(ctx interface{}, name interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Get_Call { +func (_e *NamespaceableResourceInterface_Expecter) Get(ctx any, name any, options any, subresources ...any) *NamespaceableResourceInterface_Get_Call { return &NamespaceableResourceInterface_Get_Call{Call: _e.mock.On("Get", - append([]interface{}{ctx, name, options}, subresources...)...)} + append([]any{ctx, name, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Get_Call) Run(run func(ctx context.Context, name string, options v1.GetOptions, subresources ...string)) *NamespaceableResourceInterface_Get_Call { @@ -606,7 +597,7 @@ type NamespaceableResourceInterface_List_Call struct { // List is a helper method to define mock.On call // - ctx context.Context // - opts v1.ListOptions -func (_e *NamespaceableResourceInterface_Expecter) List(ctx interface{}, opts interface{}) *NamespaceableResourceInterface_List_Call { +func (_e *NamespaceableResourceInterface_Expecter) List(ctx any, opts any) *NamespaceableResourceInterface_List_Call { return &NamespaceableResourceInterface_List_Call{Call: _e.mock.On("List", ctx, opts)} } @@ -664,7 +655,7 @@ type NamespaceableResourceInterface_Namespace_Call struct { // Namespace is a helper method to define mock.On call // - s string -func (_e *NamespaceableResourceInterface_Expecter) Namespace(s interface{}) *NamespaceableResourceInterface_Namespace_Call { +func (_e *NamespaceableResourceInterface_Expecter) Namespace(s any) *NamespaceableResourceInterface_Namespace_Call { return &NamespaceableResourceInterface_Namespace_Call{Call: _e.mock.On("Namespace", s)} } @@ -694,11 +685,11 @@ func (_c *NamespaceableResourceInterface_Namespace_Call) RunAndReturn(run func(s // Patch provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options v1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, name, pt, data, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -739,9 +730,9 @@ type NamespaceableResourceInterface_Patch_Call struct { // - data []byte // - options v1.PatchOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Patch(ctx interface{}, name interface{}, pt interface{}, data interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Patch_Call { +func (_e *NamespaceableResourceInterface_Expecter) Patch(ctx any, name any, pt any, data any, options any, subresources ...any) *NamespaceableResourceInterface_Patch_Call { return &NamespaceableResourceInterface_Patch_Call{Call: _e.mock.On("Patch", - append([]interface{}{ctx, name, pt, data, options}, subresources...)...)} + append([]any{ctx, name, pt, data, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Patch_Call) Run(run func(ctx context.Context, name string, pt types.PatchType, data []byte, options v1.PatchOptions, subresources ...string)) *NamespaceableResourceInterface_Patch_Call { @@ -799,11 +790,11 @@ func (_c *NamespaceableResourceInterface_Patch_Call) RunAndReturn(run func(ctx c // Update provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Update(ctx context.Context, obj *unstructured.Unstructured, options v1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -842,9 +833,9 @@ type NamespaceableResourceInterface_Update_Call struct { // - obj *unstructured.Unstructured // - options v1.UpdateOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Update(ctx interface{}, obj interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Update_Call { +func (_e *NamespaceableResourceInterface_Expecter) Update(ctx any, obj any, options any, subresources ...any) *NamespaceableResourceInterface_Update_Call { return &NamespaceableResourceInterface_Update_Call{Call: _e.mock.On("Update", - append([]interface{}{ctx, obj, options}, subresources...)...)} + append([]any{ctx, obj, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Update_Call) Run(run func(ctx context.Context, obj *unstructured.Unstructured, options v1.UpdateOptions, subresources ...string)) *NamespaceableResourceInterface_Update_Call { @@ -926,7 +917,7 @@ type NamespaceableResourceInterface_UpdateStatus_Call struct { // - ctx context.Context // - obj *unstructured.Unstructured // - options v1.UpdateOptions -func (_e *NamespaceableResourceInterface_Expecter) UpdateStatus(ctx interface{}, obj interface{}, options interface{}) *NamespaceableResourceInterface_UpdateStatus_Call { +func (_e *NamespaceableResourceInterface_Expecter) UpdateStatus(ctx any, obj any, options any) *NamespaceableResourceInterface_UpdateStatus_Call { return &NamespaceableResourceInterface_UpdateStatus_Call{Call: _e.mock.On("UpdateStatus", ctx, obj, options)} } @@ -999,7 +990,7 @@ type NamespaceableResourceInterface_Watch_Call struct { // Watch is a helper method to define mock.On call // - ctx context.Context // - opts v1.ListOptions -func (_e *NamespaceableResourceInterface_Expecter) Watch(ctx interface{}, opts interface{}) *NamespaceableResourceInterface_Watch_Call { +func (_e *NamespaceableResourceInterface_Expecter) Watch(ctx any, opts any) *NamespaceableResourceInterface_Watch_Call { return &NamespaceableResourceInterface_Watch_Call{Call: _e.mock.On("Watch", ctx, opts)} } diff --git a/operator/internal/mocks/dynamic/ResourceInterface.go b/operator/internal/mocks/dynamic/ResourceInterface.go index e1fe53e80..5cf342615 100644 --- a/operator/internal/mocks/dynamic/ResourceInterface.go +++ b/operator/internal/mocks/dynamic/ResourceInterface.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -38,10 +20,19 @@ func NewResourceInterface(t interface { mock.TestingT Cleanup(func()) }) *ResourceInterface { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &ResourceInterface{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -62,11 +53,11 @@ func (_m *ResourceInterface) EXPECT() *ResourceInterface_Expecter { // Apply provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Apply(ctx context.Context, name string, obj *unstructured.Unstructured, options v1.ApplyOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, name, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -106,9 +97,9 @@ type ResourceInterface_Apply_Call struct { // - obj *unstructured.Unstructured // - options v1.ApplyOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Apply(ctx interface{}, name interface{}, obj interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Apply_Call { +func (_e *ResourceInterface_Expecter) Apply(ctx any, name any, obj any, options any, subresources ...any) *ResourceInterface_Apply_Call { return &ResourceInterface_Apply_Call{Call: _e.mock.On("Apply", - append([]interface{}{ctx, name, obj, options}, subresources...)...)} + append([]any{ctx, name, obj, options}, subresources...)...)} } func (_c *ResourceInterface_Apply_Call) Run(run func(ctx context.Context, name string, obj *unstructured.Unstructured, options v1.ApplyOptions, subresources ...string)) *ResourceInterface_Apply_Call { @@ -196,7 +187,7 @@ type ResourceInterface_ApplyStatus_Call struct { // - name string // - obj *unstructured.Unstructured // - options v1.ApplyOptions -func (_e *ResourceInterface_Expecter) ApplyStatus(ctx interface{}, name interface{}, obj interface{}, options interface{}) *ResourceInterface_ApplyStatus_Call { +func (_e *ResourceInterface_Expecter) ApplyStatus(ctx any, name any, obj any, options any) *ResourceInterface_ApplyStatus_Call { return &ResourceInterface_ApplyStatus_Call{Call: _e.mock.On("ApplyStatus", ctx, name, obj, options)} } @@ -241,11 +232,11 @@ func (_c *ResourceInterface_ApplyStatus_Call) RunAndReturn(run func(ctx context. // Create provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Create(ctx context.Context, obj *unstructured.Unstructured, options v1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -284,9 +275,9 @@ type ResourceInterface_Create_Call struct { // - obj *unstructured.Unstructured // - options v1.CreateOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Create(ctx interface{}, obj interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Create_Call { +func (_e *ResourceInterface_Expecter) Create(ctx any, obj any, options any, subresources ...any) *ResourceInterface_Create_Call { return &ResourceInterface_Create_Call{Call: _e.mock.On("Create", - append([]interface{}{ctx, obj, options}, subresources...)...)} + append([]any{ctx, obj, options}, subresources...)...)} } func (_c *ResourceInterface_Create_Call) Run(run func(ctx context.Context, obj *unstructured.Unstructured, options v1.CreateOptions, subresources ...string)) *ResourceInterface_Create_Call { @@ -334,11 +325,11 @@ func (_c *ResourceInterface_Create_Call) RunAndReturn(run func(ctx context.Conte // Delete provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Delete(ctx context.Context, name string, options v1.DeleteOptions, subresources ...string) error { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, name, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -366,9 +357,9 @@ type ResourceInterface_Delete_Call struct { // - name string // - options v1.DeleteOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Delete(ctx interface{}, name interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Delete_Call { +func (_e *ResourceInterface_Expecter) Delete(ctx any, name any, options any, subresources ...any) *ResourceInterface_Delete_Call { return &ResourceInterface_Delete_Call{Call: _e.mock.On("Delete", - append([]interface{}{ctx, name, options}, subresources...)...)} + append([]any{ctx, name, options}, subresources...)...)} } func (_c *ResourceInterface_Delete_Call) Run(run func(ctx context.Context, name string, options v1.DeleteOptions, subresources ...string)) *ResourceInterface_Delete_Call { @@ -439,7 +430,7 @@ type ResourceInterface_DeleteCollection_Call struct { // - ctx context.Context // - options v1.DeleteOptions // - listOptions v1.ListOptions -func (_e *ResourceInterface_Expecter) DeleteCollection(ctx interface{}, options interface{}, listOptions interface{}) *ResourceInterface_DeleteCollection_Call { +func (_e *ResourceInterface_Expecter) DeleteCollection(ctx any, options any, listOptions any) *ResourceInterface_DeleteCollection_Call { return &ResourceInterface_DeleteCollection_Call{Call: _e.mock.On("DeleteCollection", ctx, options, listOptions)} } @@ -479,11 +470,11 @@ func (_c *ResourceInterface_DeleteCollection_Call) RunAndReturn(run func(ctx con // Get provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Get(ctx context.Context, name string, options v1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, name, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -522,9 +513,9 @@ type ResourceInterface_Get_Call struct { // - name string // - options v1.GetOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Get(ctx interface{}, name interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Get_Call { +func (_e *ResourceInterface_Expecter) Get(ctx any, name any, options any, subresources ...any) *ResourceInterface_Get_Call { return &ResourceInterface_Get_Call{Call: _e.mock.On("Get", - append([]interface{}{ctx, name, options}, subresources...)...)} + append([]any{ctx, name, options}, subresources...)...)} } func (_c *ResourceInterface_Get_Call) Run(run func(ctx context.Context, name string, options v1.GetOptions, subresources ...string)) *ResourceInterface_Get_Call { @@ -605,7 +596,7 @@ type ResourceInterface_List_Call struct { // List is a helper method to define mock.On call // - ctx context.Context // - opts v1.ListOptions -func (_e *ResourceInterface_Expecter) List(ctx interface{}, opts interface{}) *ResourceInterface_List_Call { +func (_e *ResourceInterface_Expecter) List(ctx any, opts any) *ResourceInterface_List_Call { return &ResourceInterface_List_Call{Call: _e.mock.On("List", ctx, opts)} } @@ -640,11 +631,11 @@ func (_c *ResourceInterface_List_Call) RunAndReturn(run func(ctx context.Context // Patch provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options v1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, name, pt, data, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -685,9 +676,9 @@ type ResourceInterface_Patch_Call struct { // - data []byte // - options v1.PatchOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Patch(ctx interface{}, name interface{}, pt interface{}, data interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Patch_Call { +func (_e *ResourceInterface_Expecter) Patch(ctx any, name any, pt any, data any, options any, subresources ...any) *ResourceInterface_Patch_Call { return &ResourceInterface_Patch_Call{Call: _e.mock.On("Patch", - append([]interface{}{ctx, name, pt, data, options}, subresources...)...)} + append([]any{ctx, name, pt, data, options}, subresources...)...)} } func (_c *ResourceInterface_Patch_Call) Run(run func(ctx context.Context, name string, pt types.PatchType, data []byte, options v1.PatchOptions, subresources ...string)) *ResourceInterface_Patch_Call { @@ -745,11 +736,11 @@ func (_c *ResourceInterface_Patch_Call) RunAndReturn(run func(ctx context.Contex // Update provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Update(ctx context.Context, obj *unstructured.Unstructured, options v1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]interface{}, len(subresources)) + _va := make([]any, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []interface{} + var _ca []any _ca = append(_ca, ctx, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -788,9 +779,9 @@ type ResourceInterface_Update_Call struct { // - obj *unstructured.Unstructured // - options v1.UpdateOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Update(ctx interface{}, obj interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Update_Call { +func (_e *ResourceInterface_Expecter) Update(ctx any, obj any, options any, subresources ...any) *ResourceInterface_Update_Call { return &ResourceInterface_Update_Call{Call: _e.mock.On("Update", - append([]interface{}{ctx, obj, options}, subresources...)...)} + append([]any{ctx, obj, options}, subresources...)...)} } func (_c *ResourceInterface_Update_Call) Run(run func(ctx context.Context, obj *unstructured.Unstructured, options v1.UpdateOptions, subresources ...string)) *ResourceInterface_Update_Call { @@ -872,7 +863,7 @@ type ResourceInterface_UpdateStatus_Call struct { // - ctx context.Context // - obj *unstructured.Unstructured // - options v1.UpdateOptions -func (_e *ResourceInterface_Expecter) UpdateStatus(ctx interface{}, obj interface{}, options interface{}) *ResourceInterface_UpdateStatus_Call { +func (_e *ResourceInterface_Expecter) UpdateStatus(ctx any, obj any, options any) *ResourceInterface_UpdateStatus_Call { return &ResourceInterface_UpdateStatus_Call{Call: _e.mock.On("UpdateStatus", ctx, obj, options)} } @@ -945,7 +936,7 @@ type ResourceInterface_Watch_Call struct { // Watch is a helper method to define mock.On call // - ctx context.Context // - opts v1.ListOptions -func (_e *ResourceInterface_Expecter) Watch(ctx interface{}, opts interface{}) *ResourceInterface_Watch_Call { +func (_e *ResourceInterface_Expecter) Watch(ctx any, opts any) *ResourceInterface_Watch_Call { return &ResourceInterface_Watch_Call{Call: _e.mock.On("Watch", ctx, opts)} } diff --git a/operator/internal/mocks/record/EventRecorder.go b/operator/internal/mocks/record/EventRecorder.go index 6e088ac20..7132707d9 100644 --- a/operator/internal/mocks/record/EventRecorder.go +++ b/operator/internal/mocks/record/EventRecorder.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -33,10 +15,19 @@ func NewEventRecorder(t interface { mock.TestingT Cleanup(func()) }) *EventRecorder { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &EventRecorder{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -56,7 +47,7 @@ func (_m *EventRecorder) EXPECT() *EventRecorder_Expecter { // AnnotatedEventf provides a mock function for the type EventRecorder func (_mock *EventRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype string, reason string, messageFmt string, args ...interface{}) { - var _ca []interface{} + var _ca []any _ca = append(_ca, object, annotations, eventtype, reason, messageFmt) _ca = append(_ca, args...) _mock.Called(_ca...) @@ -75,9 +66,9 @@ type EventRecorder_AnnotatedEventf_Call struct { // - reason string // - messageFmt string // - args ...interface{} -func (_e *EventRecorder_Expecter) AnnotatedEventf(object interface{}, annotations interface{}, eventtype interface{}, reason interface{}, messageFmt interface{}, args ...interface{}) *EventRecorder_AnnotatedEventf_Call { +func (_e *EventRecorder_Expecter) AnnotatedEventf(object any, annotations any, eventtype any, reason any, messageFmt any, args ...any) *EventRecorder_AnnotatedEventf_Call { return &EventRecorder_AnnotatedEventf_Call{Call: _e.mock.On("AnnotatedEventf", - append([]interface{}{object, annotations, eventtype, reason, messageFmt}, args...)...)} + append([]any{object, annotations, eventtype, reason, messageFmt}, args...)...)} } func (_c *EventRecorder_AnnotatedEventf_Call) Run(run func(object runtime.Object, annotations map[string]string, eventtype string, reason string, messageFmt string, args ...interface{})) *EventRecorder_AnnotatedEventf_Call { @@ -148,7 +139,7 @@ type EventRecorder_Event_Call struct { // - eventtype string // - reason string // - message string -func (_e *EventRecorder_Expecter) Event(object interface{}, eventtype interface{}, reason interface{}, message interface{}) *EventRecorder_Event_Call { +func (_e *EventRecorder_Expecter) Event(object any, eventtype any, reason any, message any) *EventRecorder_Event_Call { return &EventRecorder_Event_Call{Call: _e.mock.On("Event", object, eventtype, reason, message)} } @@ -192,7 +183,7 @@ func (_c *EventRecorder_Event_Call) RunAndReturn(run func(object runtime.Object, // Eventf provides a mock function for the type EventRecorder func (_mock *EventRecorder) Eventf(object runtime.Object, eventtype string, reason string, messageFmt string, args ...interface{}) { - var _ca []interface{} + var _ca []any _ca = append(_ca, object, eventtype, reason, messageFmt) _ca = append(_ca, args...) _mock.Called(_ca...) @@ -210,9 +201,9 @@ type EventRecorder_Eventf_Call struct { // - reason string // - messageFmt string // - args ...interface{} -func (_e *EventRecorder_Expecter) Eventf(object interface{}, eventtype interface{}, reason interface{}, messageFmt interface{}, args ...interface{}) *EventRecorder_Eventf_Call { +func (_e *EventRecorder_Expecter) Eventf(object any, eventtype any, reason any, messageFmt any, args ...any) *EventRecorder_Eventf_Call { return &EventRecorder_Eventf_Call{Call: _e.mock.On("Eventf", - append([]interface{}{object, eventtype, reason, messageFmt}, args...)...)} + append([]any{object, eventtype, reason, messageFmt}, args...)...)} } func (_c *EventRecorder_Eventf_Call) Run(run func(object runtime.Object, eventtype string, reason string, messageFmt string, args ...interface{})) *EventRecorder_Eventf_Call { diff --git a/operator/internal/mocks/workqueue/TypedRateLimitingInterface.go b/operator/internal/mocks/workqueue/TypedRateLimitingInterface.go index 23f9ebaf8..6c509cd92 100644 --- a/operator/internal/mocks/workqueue/TypedRateLimitingInterface.go +++ b/operator/internal/mocks/workqueue/TypedRateLimitingInterface.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -34,10 +16,19 @@ func NewTypedRateLimitingInterface[T comparable](t interface { mock.TestingT Cleanup(func()) }) *TypedRateLimitingInterface[T] { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &TypedRateLimitingInterface[T]{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -68,7 +59,7 @@ type TypedRateLimitingInterface_Add_Call[T comparable] struct { // Add is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) Add(item interface{}) *TypedRateLimitingInterface_Add_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) Add(item any) *TypedRateLimitingInterface_Add_Call[T] { return &TypedRateLimitingInterface_Add_Call[T]{Call: _e.mock.On("Add", item)} } @@ -109,7 +100,7 @@ type TypedRateLimitingInterface_AddAfter_Call[T comparable] struct { // AddAfter is a helper method to define mock.On call // - item T // - duration time.Duration -func (_e *TypedRateLimitingInterface_Expecter[T]) AddAfter(item interface{}, duration interface{}) *TypedRateLimitingInterface_AddAfter_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) AddAfter(item any, duration any) *TypedRateLimitingInterface_AddAfter_Call[T] { return &TypedRateLimitingInterface_AddAfter_Call[T]{Call: _e.mock.On("AddAfter", item, duration)} } @@ -154,7 +145,7 @@ type TypedRateLimitingInterface_AddRateLimited_Call[T comparable] struct { // AddRateLimited is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) AddRateLimited(item interface{}) *TypedRateLimitingInterface_AddRateLimited_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) AddRateLimited(item any) *TypedRateLimitingInterface_AddRateLimited_Call[T] { return &TypedRateLimitingInterface_AddRateLimited_Call[T]{Call: _e.mock.On("AddRateLimited", item)} } @@ -194,7 +185,7 @@ type TypedRateLimitingInterface_Done_Call[T comparable] struct { // Done is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) Done(item interface{}) *TypedRateLimitingInterface_Done_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) Done(item any) *TypedRateLimitingInterface_Done_Call[T] { return &TypedRateLimitingInterface_Done_Call[T]{Call: _e.mock.On("Done", item)} } @@ -234,7 +225,7 @@ type TypedRateLimitingInterface_Forget_Call[T comparable] struct { // Forget is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) Forget(item interface{}) *TypedRateLimitingInterface_Forget_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) Forget(item any) *TypedRateLimitingInterface_Forget_Call[T] { return &TypedRateLimitingInterface_Forget_Call[T]{Call: _e.mock.On("Forget", item)} } @@ -384,7 +375,7 @@ type TypedRateLimitingInterface_NumRequeues_Call[T comparable] struct { // NumRequeues is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) NumRequeues(item interface{}) *TypedRateLimitingInterface_NumRequeues_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) NumRequeues(item any) *TypedRateLimitingInterface_NumRequeues_Call[T] { return &TypedRateLimitingInterface_NumRequeues_Call[T]{Call: _e.mock.On("NumRequeues", item)} } diff --git a/operator/internal/wrapper/mock/SkyhookNode.go b/operator/internal/wrapper/mock/SkyhookNode.go index 4cadb293b..10b4c4800 100644 --- a/operator/internal/wrapper/mock/SkyhookNode.go +++ b/operator/internal/wrapper/mock/SkyhookNode.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -37,10 +19,19 @@ func NewMockSkyhookNode(t interface { mock.TestingT Cleanup(func()) }) *MockSkyhookNode { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &MockSkyhookNode{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -473,7 +464,7 @@ type MockSkyhookNode_HasInterrupt_Call struct { // HasInterrupt is a helper method to define mock.On call // - _package v1alpha1.Package -func (_e *MockSkyhookNode_Expecter) HasInterrupt(_package interface{}) *MockSkyhookNode_HasInterrupt_Call { +func (_e *MockSkyhookNode_Expecter) HasInterrupt(_package any) *MockSkyhookNode_HasInterrupt_Call { return &MockSkyhookNode_HasInterrupt_Call{Call: _e.mock.On("HasInterrupt", _package)} } @@ -612,7 +603,7 @@ type MockSkyhookNode_IsPackageComplete_Call struct { // IsPackageComplete is a helper method to define mock.On call // - _package v1alpha1.Package -func (_e *MockSkyhookNode_Expecter) IsPackageComplete(_package interface{}) *MockSkyhookNode_IsPackageComplete_Call { +func (_e *MockSkyhookNode_Expecter) IsPackageComplete(_package any) *MockSkyhookNode_IsPackageComplete_Call { return &MockSkyhookNode_IsPackageComplete_Call{Call: _e.mock.On("IsPackageComplete", _package)} } @@ -663,7 +654,7 @@ type MockSkyhookNode_Migrate_Call struct { // Migrate is a helper method to define mock.On call // - logger logr.Logger -func (_e *MockSkyhookNode_Expecter) Migrate(logger interface{}) *MockSkyhookNode_Migrate_Call { +func (_e *MockSkyhookNode_Expecter) Migrate(logger any) *MockSkyhookNode_Migrate_Call { return &MockSkyhookNode_Migrate_Call{Call: _e.mock.On("Migrate", logger)} } @@ -716,7 +707,7 @@ type MockSkyhookNode_NextStage_Call struct { // NextStage is a helper method to define mock.On call // - _package *v1alpha1.Package -func (_e *MockSkyhookNode_Expecter) NextStage(_package interface{}) *MockSkyhookNode_NextStage_Call { +func (_e *MockSkyhookNode_Expecter) NextStage(_package any) *MockSkyhookNode_NextStage_Call { return &MockSkyhookNode_NextStage_Call{Call: _e.mock.On("NextStage", _package)} } @@ -778,7 +769,7 @@ type MockSkyhookNode_PackageStatus_Call struct { // PackageStatus is a helper method to define mock.On call // - name string -func (_e *MockSkyhookNode_Expecter) PackageStatus(name interface{}) *MockSkyhookNode_PackageStatus_Call { +func (_e *MockSkyhookNode_Expecter) PackageStatus(name any) *MockSkyhookNode_PackageStatus_Call { return &MockSkyhookNode_PackageStatus_Call{Call: _e.mock.On("PackageStatus", name)} } @@ -961,7 +952,7 @@ type MockSkyhookNode_RemoveState_Call struct { // RemoveState is a helper method to define mock.On call // - _package v1alpha1.PackageRef -func (_e *MockSkyhookNode_Expecter) RemoveState(_package interface{}) *MockSkyhookNode_RemoveState_Call { +func (_e *MockSkyhookNode_Expecter) RemoveState(_package any) *MockSkyhookNode_RemoveState_Call { return &MockSkyhookNode_RemoveState_Call{Call: _e.mock.On("RemoveState", _package)} } @@ -1001,7 +992,7 @@ type MockSkyhookNode_RemoveTaint_Call struct { // RemoveTaint is a helper method to define mock.On call // - key string -func (_e *MockSkyhookNode_Expecter) RemoveTaint(key interface{}) *MockSkyhookNode_RemoveTaint_Call { +func (_e *MockSkyhookNode_Expecter) RemoveTaint(key any) *MockSkyhookNode_RemoveTaint_Call { return &MockSkyhookNode_RemoveTaint_Call{Call: _e.mock.On("RemoveTaint", key)} } @@ -1140,7 +1131,7 @@ type MockSkyhookNode_SetState_Call struct { // SetState is a helper method to define mock.On call // - state v1alpha1.NodeState -func (_e *MockSkyhookNode_Expecter) SetState(state interface{}) *MockSkyhookNode_SetState_Call { +func (_e *MockSkyhookNode_Expecter) SetState(state any) *MockSkyhookNode_SetState_Call { return &MockSkyhookNode_SetState_Call{Call: _e.mock.On("SetState", state)} } @@ -1180,7 +1171,7 @@ type MockSkyhookNode_SetStatus_Call struct { // SetStatus is a helper method to define mock.On call // - status v1alpha1.Status -func (_e *MockSkyhookNode_Expecter) SetStatus(status interface{}) *MockSkyhookNode_SetStatus_Call { +func (_e *MockSkyhookNode_Expecter) SetStatus(status any) *MockSkyhookNode_SetStatus_Call { return &MockSkyhookNode_SetStatus_Call{Call: _e.mock.On("SetStatus", status)} } @@ -1253,7 +1244,7 @@ type MockSkyhookNode_StartDrain_Call struct { // StartDrain is a helper method to define mock.On call // - startedAt v1.Time -func (_e *MockSkyhookNode_Expecter) StartDrain(startedAt interface{}) *MockSkyhookNode_StartDrain_Call { +func (_e *MockSkyhookNode_Expecter) StartDrain(startedAt any) *MockSkyhookNode_StartDrain_Call { return &MockSkyhookNode_StartDrain_Call{Call: _e.mock.On("StartDrain", startedAt)} } @@ -1392,7 +1383,7 @@ type MockSkyhookNode_Taint_Call struct { // Taint is a helper method to define mock.On call // - key string -func (_e *MockSkyhookNode_Expecter) Taint(key interface{}) *MockSkyhookNode_Taint_Call { +func (_e *MockSkyhookNode_Expecter) Taint(key any) *MockSkyhookNode_Taint_Call { return &MockSkyhookNode_Taint_Call{Call: _e.mock.On("Taint", key)} } @@ -1514,7 +1505,7 @@ type MockSkyhookNode_Upsert_Call struct { // - stage v1alpha1.Stage // - restarts int32 // - containerSHA string -func (_e *MockSkyhookNode_Expecter) Upsert(_package interface{}, image interface{}, state interface{}, stage interface{}, restarts interface{}, containerSHA interface{}) *MockSkyhookNode_Upsert_Call { +func (_e *MockSkyhookNode_Expecter) Upsert(_package any, image any, state any, stage any, restarts any, containerSHA any) *MockSkyhookNode_Upsert_Call { return &MockSkyhookNode_Upsert_Call{Call: _e.mock.On("Upsert", _package, image, state, stage, restarts, containerSHA)} } diff --git a/operator/internal/wrapper/mock/SkyhookNodeOnly.go b/operator/internal/wrapper/mock/SkyhookNodeOnly.go index 22344f013..ea392e03e 100644 --- a/operator/internal/wrapper/mock/SkyhookNodeOnly.go +++ b/operator/internal/wrapper/mock/SkyhookNodeOnly.go @@ -1,21 +1,3 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -36,10 +18,19 @@ func NewMockSkyhookNodeOnly(t interface { mock.TestingT Cleanup(func()) }) *MockSkyhookNodeOnly { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock := &MockSkyhookNodeOnly{} mock.Mock.Test(t) - t.Cleanup(func() { mock.AssertExpectations(t) }) + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) return mock } @@ -347,7 +338,7 @@ type MockSkyhookNodeOnly_Migrate_Call struct { // Migrate is a helper method to define mock.On call // - logger logr.Logger -func (_e *MockSkyhookNodeOnly_Expecter) Migrate(logger interface{}) *MockSkyhookNodeOnly_Migrate_Call { +func (_e *MockSkyhookNodeOnly_Expecter) Migrate(logger any) *MockSkyhookNodeOnly_Migrate_Call { return &MockSkyhookNodeOnly_Migrate_Call{Call: _e.mock.On("Migrate", logger)} } @@ -409,7 +400,7 @@ type MockSkyhookNodeOnly_PackageStatus_Call struct { // PackageStatus is a helper method to define mock.On call // - name string -func (_e *MockSkyhookNodeOnly_Expecter) PackageStatus(name interface{}) *MockSkyhookNodeOnly_PackageStatus_Call { +func (_e *MockSkyhookNodeOnly_Expecter) PackageStatus(name any) *MockSkyhookNodeOnly_PackageStatus_Call { return &MockSkyhookNodeOnly_PackageStatus_Call{Call: _e.mock.On("PackageStatus", name)} } @@ -548,7 +539,7 @@ type MockSkyhookNodeOnly_RemoveState_Call struct { // RemoveState is a helper method to define mock.On call // - _package v1alpha1.PackageRef -func (_e *MockSkyhookNodeOnly_Expecter) RemoveState(_package interface{}) *MockSkyhookNodeOnly_RemoveState_Call { +func (_e *MockSkyhookNodeOnly_Expecter) RemoveState(_package any) *MockSkyhookNodeOnly_RemoveState_Call { return &MockSkyhookNodeOnly_RemoveState_Call{Call: _e.mock.On("RemoveState", _package)} } @@ -588,7 +579,7 @@ type MockSkyhookNodeOnly_RemoveTaint_Call struct { // RemoveTaint is a helper method to define mock.On call // - key string -func (_e *MockSkyhookNodeOnly_Expecter) RemoveTaint(key interface{}) *MockSkyhookNodeOnly_RemoveTaint_Call { +func (_e *MockSkyhookNodeOnly_Expecter) RemoveTaint(key any) *MockSkyhookNodeOnly_RemoveTaint_Call { return &MockSkyhookNodeOnly_RemoveTaint_Call{Call: _e.mock.On("RemoveTaint", key)} } @@ -672,7 +663,7 @@ type MockSkyhookNodeOnly_SetState_Call struct { // SetState is a helper method to define mock.On call // - state v1alpha1.NodeState -func (_e *MockSkyhookNodeOnly_Expecter) SetState(state interface{}) *MockSkyhookNodeOnly_SetState_Call { +func (_e *MockSkyhookNodeOnly_Expecter) SetState(state any) *MockSkyhookNodeOnly_SetState_Call { return &MockSkyhookNodeOnly_SetState_Call{Call: _e.mock.On("SetState", state)} } @@ -712,7 +703,7 @@ type MockSkyhookNodeOnly_SetStatus_Call struct { // SetStatus is a helper method to define mock.On call // - status v1alpha1.Status -func (_e *MockSkyhookNodeOnly_Expecter) SetStatus(status interface{}) *MockSkyhookNodeOnly_SetStatus_Call { +func (_e *MockSkyhookNodeOnly_Expecter) SetStatus(status any) *MockSkyhookNodeOnly_SetStatus_Call { return &MockSkyhookNodeOnly_SetStatus_Call{Call: _e.mock.On("SetStatus", status)} } @@ -785,7 +776,7 @@ type MockSkyhookNodeOnly_StartDrain_Call struct { // StartDrain is a helper method to define mock.On call // - startedAt v1.Time -func (_e *MockSkyhookNodeOnly_Expecter) StartDrain(startedAt interface{}) *MockSkyhookNodeOnly_StartDrain_Call { +func (_e *MockSkyhookNodeOnly_Expecter) StartDrain(startedAt any) *MockSkyhookNodeOnly_StartDrain_Call { return &MockSkyhookNodeOnly_StartDrain_Call{Call: _e.mock.On("StartDrain", startedAt)} } @@ -924,7 +915,7 @@ type MockSkyhookNodeOnly_Taint_Call struct { // Taint is a helper method to define mock.On call // - key string -func (_e *MockSkyhookNodeOnly_Expecter) Taint(key interface{}) *MockSkyhookNodeOnly_Taint_Call { +func (_e *MockSkyhookNodeOnly_Expecter) Taint(key any) *MockSkyhookNodeOnly_Taint_Call { return &MockSkyhookNodeOnly_Taint_Call{Call: _e.mock.On("Taint", key)} } @@ -1013,7 +1004,7 @@ type MockSkyhookNodeOnly_Upsert_Call struct { // - stage v1alpha1.Stage // - restarts int32 // - containerSHA string -func (_e *MockSkyhookNodeOnly_Expecter) Upsert(_package interface{}, image interface{}, state interface{}, stage interface{}, restarts interface{}, containerSHA interface{}) *MockSkyhookNodeOnly_Upsert_Call { +func (_e *MockSkyhookNodeOnly_Expecter) Upsert(_package any, image any, state any, stage any, restarts any, containerSHA any) *MockSkyhookNodeOnly_Upsert_Call { return &MockSkyhookNodeOnly_Upsert_Call{Call: _e.mock.On("Upsert", _package, image, state, stage, restarts, containerSHA)} } diff --git a/operator/internal/wrapper/skyhook_conditions.go b/operator/internal/wrapper/skyhook_conditions.go index 9509edc8a..48e1cbeac 100644 --- a/operator/internal/wrapper/skyhook_conditions.go +++ b/operator/internal/wrapper/skyhook_conditions.go @@ -20,9 +20,11 @@ package wrapper import ( "fmt" + "sort" "strings" "github.com/NVIDIA/nodewright/operator/api/nodewright/v1alpha1" + "github.com/NVIDIA/nodewright/operator/internal/drain" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -40,6 +42,12 @@ const ( SkyhookConditionUninstallFailed = "UninstallFailed" SkyhookConditionNodeStateMalformed = "NodeStateMalformed" SkyhookConditionDeletionBlocked = "DeletionBlocked" + SkyhookConditionDrainBlocked = "DrainBlocked" + + drainBlockedReasonPDB = "PodDisruptionBudget" + drainBlockedReasonUnmanagedPod = "UnmanagedPod" + drainBlockedReasonEmptyDir = "EmptyDirData" + drainBlockedReasonMultiple = "MultipleCauses" SkyhookReasonNonInterruptPodsRunning = "NonInterruptPodsRunning" @@ -321,3 +329,63 @@ func FormatNodeList(nodes []string) string { } return fmt.Sprintf(" (%s)", strings.Join(nodes, ", ")) } + +// DrainBlockedNode is one node's drain blockers for the DrainBlocked condition +// message builder below — kept independent of the controller package's +// nodeDrainBlock so wrapper has no import cycle back to controller. +type DrainBlockedNode struct { + NodeName string + Blocked []drain.BlockedPod +} + +// DrainBlockedConditionReason picks the condition Reason from the set of block +// reasons observed this pass. MultipleCauses covers both "one node has two kinds +// of blocker" and "different nodes are blocked for different reasons". +func DrainBlockedConditionReason(nodes []DrainBlockedNode) string { + seen := make(map[drain.BlockReason]struct{}) + for _, n := range nodes { + for _, b := range n.Blocked { + seen[b.Reason] = struct{}{} + } + } + if len(seen) != 1 { + return drainBlockedReasonMultiple + } + for reason := range seen { + switch reason { + case drain.BlockReasonPodDisruptionBudget: + return drainBlockedReasonPDB + case drain.BlockReasonUnmanagedPod: + return drainBlockedReasonUnmanagedPod + case drain.BlockReasonEmptyDirData: + return drainBlockedReasonEmptyDir + } + } + return drainBlockedReasonMultiple +} + +// DrainBlockedConditionMessage renders the aggregate DrainBlocked message: a +// "N/total nodes blocked draining (names)" summary line — following the same +// truncation idiom as the Ready condition — followed by one "/ on +// : " line per blocked pod that carries a Detail (PDB +// cases only; Detail is apiserver prose and is never altered). +func DrainBlockedConditionMessage(nodes []DrainBlockedNode, totalSelected int) string { + names := make([]string, 0, len(nodes)) + for _, n := range nodes { + names = append(names, n.NodeName) + } + sort.Strings(names) + + lines := []string{fmt.Sprintf("%d/%d nodes blocked draining%s", len(nodes), totalSelected, formatNodeList(names))} + + for _, n := range nodes { + for _, b := range n.Blocked { + if b.Detail == "" { + continue + } + lines = append(lines, fmt.Sprintf("%s/%s on %s: %s", b.Namespace, b.Name, n.NodeName, b.Detail)) + } + } + + return strings.Join(lines, "; ") +} From bacfc77097d87edffbabae7dd69174552b9ca4a1 Mon Sep 17 00:00:00 2001 From: mohityadav8 Date: Wed, 16 Sep 2026 12:18:04 +0530 Subject: [PATCH 2/9] fix: use exported wrapper.DrainBlockedNode instead of unexported nodeDrainBlock; fix test call sites for updated DrainNode/EnsureNodeIsReadyForInterrupt/ProcessInterrupt signatures Signed-off-by: mohityadav8 --- .../internal/controller/cluster_state_v2.go | 16 ++----- .../internal/controller/mock/SkyhookNodes.go | 13 +++--- .../internal/controller/skyhook_controller.go | 14 ++---- .../controller/skyhook_controller_test.go | 44 +++++++++---------- 4 files changed, 35 insertions(+), 52 deletions(-) diff --git a/operator/internal/controller/cluster_state_v2.go b/operator/internal/controller/cluster_state_v2.go index 9f2f4b66f..6e0d5ecad 100644 --- a/operator/internal/controller/cluster_state_v2.go +++ b/operator/internal/controller/cluster_state_v2.go @@ -402,7 +402,7 @@ type SkyhookNodes interface { IsPaused() bool HasUninstallWork() (bool, error) UpdateBlockedCondition() error - UpdateDrainBlockedCondition(blocks []nodeDrainBlock) + UpdateDrainBlockedCondition(blocks []wrapper.DrainBlockedNode) UpdateUninstallConditions() error UpdateNodeStateMalformedCondition() NodeCount() int @@ -634,27 +634,19 @@ func (s *skyhookNodes) UpdateBlockedCondition() error { // attempts found, so it is set once, after the node-processing loop finishes, rather // than at the top of Reconcile. An empty blocks slice (nothing blocked this pass, or // every previously blocked node has since drained) clears the condition. -func (s *skyhookNodes) UpdateDrainBlockedCondition(blocks []nodeDrainBlock) { +func (s *skyhookNodes) UpdateDrainBlockedCondition(blocks []wrapper.DrainBlockedNode) { if len(blocks) == 0 { wrapper.RemoveSkyhookConditionTypes(s.skyhook, wrapper.SkyhookConditionDrainBlocked) return } - wrapperNodes := make([]wrapper.DrainBlockedNode, 0, len(blocks)) - for _, b := range blocks { - wrapperNodes = append(wrapperNodes, wrapper.DrainBlockedNode{ - NodeName: b.NodeName, - Blocked: b.Blocked, - }) - } - wrapper.AddSkyhookCondition(s.skyhook, metav1.Condition{ Type: wrapper.SkyhookConditionDrainBlocked, Status: metav1.ConditionTrue, ObservedGeneration: s.skyhook.Generation, LastTransitionTime: metav1.Now(), - Reason: wrapper.DrainBlockedConditionReason(wrapperNodes), - Message: wrapper.DrainBlockedConditionMessage(wrapperNodes, len(s.nodes)), + Reason: wrapper.DrainBlockedConditionReason(blocks), + Message: wrapper.DrainBlockedConditionMessage(blocks, len(s.nodes)), }) } diff --git a/operator/internal/controller/mock/SkyhookNodes.go b/operator/internal/controller/mock/SkyhookNodes.go index 147dd9fcc..1b5f55e03 100644 --- a/operator/internal/controller/mock/SkyhookNodes.go +++ b/operator/internal/controller/mock/SkyhookNodes.go @@ -6,7 +6,6 @@ package controller import ( "github.com/NVIDIA/nodewright/operator/api/nodewright/v1alpha1" - "github.com/NVIDIA/nodewright/operator/internal/controller" "github.com/NVIDIA/nodewright/operator/internal/wrapper" "github.com/go-logr/logr" mock "github.com/stretchr/testify/mock" @@ -1078,7 +1077,7 @@ func (_c *MockSkyhookNodes_UpdateCondition_Call) RunAndReturn(run func(logger lo } // UpdateDrainBlockedCondition provides a mock function for the type MockSkyhookNodes -func (_mock *MockSkyhookNodes) UpdateDrainBlockedCondition(blocks []controller.nodeDrainBlock) { +func (_mock *MockSkyhookNodes) UpdateDrainBlockedCondition(blocks []wrapper.DrainBlockedNode) { _mock.Called(blocks) return } @@ -1089,16 +1088,16 @@ type MockSkyhookNodes_UpdateDrainBlockedCondition_Call struct { } // UpdateDrainBlockedCondition is a helper method to define mock.On call -// - blocks []controller.nodeDrainBlock +// - blocks []wrapper.DrainBlockedNode func (_e *MockSkyhookNodes_Expecter) UpdateDrainBlockedCondition(blocks any) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { return &MockSkyhookNodes_UpdateDrainBlockedCondition_Call{Call: _e.mock.On("UpdateDrainBlockedCondition", blocks)} } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Run(run func(blocks []controller.nodeDrainBlock)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Run(run func(blocks []wrapper.DrainBlockedNode)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { _c.Call.Run(func(args mock.Arguments) { - var arg0 []controller.nodeDrainBlock + var arg0 []wrapper.DrainBlockedNode if args[0] != nil { - arg0 = args[0].([]controller.nodeDrainBlock) + arg0 = args[0].([]wrapper.DrainBlockedNode) } run( arg0, @@ -1112,7 +1111,7 @@ func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Return() *MockSkyho return _c } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) RunAndReturn(run func(blocks []controller.nodeDrainBlock)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) RunAndReturn(run func(blocks []wrapper.DrainBlockedNode)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { _c.Run(run) return _c } diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go index 81cfbd2c0..e62a0e78a 100644 --- a/operator/internal/controller/skyhook_controller.go +++ b/operator/internal/controller/skyhook_controller.go @@ -1448,7 +1448,7 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState } selectedNode := nodePicker.SelectNodes(skyhook) - drainBlocks := make([]nodeDrainBlock, 0) + drainBlocks := make([]wrapper.DrainBlockedNode, 0) for _, node := range selectedNode { // Skip nodes that are waiting on higher-priority skyhooks @@ -2611,14 +2611,6 @@ func (r *SkyhookReconciler) HasRunningPackages(ctx context.Context, skyhookNode return false, nil } -// nodeDrainBlock records the drain blockers found for one node during a single -// reconcile pass, accumulated across ProcessInterrupt calls and fed into -// UpdateDrainBlockedCondition once the node loop finishes. -type nodeDrainBlock struct { - NodeName string - Blocked []drain.BlockedPod -} - func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (drain.DrainResult, error) { drained, err := r.IsDrained(ctx, skyhookNode) if err != nil { @@ -3351,7 +3343,7 @@ func (r *SkyhookReconciler) InvalidPackage(ctx context.Context, obj client.Objec // ProcessInterrupt will check and do the interrupt if need, and returns // false means we are waiting // true means we are good to proceed -func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package, interrupt *v1alpha1.Interrupt, runInterrupt bool, drainBlocks *[]nodeDrainBlock) (bool, error) { +func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package, interrupt *v1alpha1.Interrupt, runInterrupt bool, drainBlocks *[]wrapper.DrainBlockedNode) (bool, error) { if !skyhookNode.HasInterrupt(*_package) { return true, nil @@ -3403,7 +3395,7 @@ func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wr } if len(blocked) > 0 && drainBlocks != nil { - *drainBlocks = append(*drainBlocks, nodeDrainBlock{ + *drainBlocks = append(*drainBlocks, wrapper.DrainBlockedNode{ NodeName: skyhookNode.GetNode().Name, Blocked: blocked, }) diff --git a/operator/internal/controller/skyhook_controller_test.go b/operator/internal/controller/skyhook_controller_test.go index 9745d9b2a..42979f342 100644 --- a/operator/internal/controller/skyhook_controller_test.go +++ b/operator/internal/controller/skyhook_controller_test.go @@ -603,22 +603,22 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) Expect(err).ToNot(HaveOccurred()) - drained, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + result, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) - Expect(drained).To(BeFalse()) + Expect(result.Ready).To(BeFalse()) Expect(gracePeriodSeconds).To(Equal(int64(7))) deletedPod := &corev1.Pod{} err = testClient.Get(ctx, types.NamespacedName{Namespace: "default", Name: "workload"}, deletedPod) Expect(apierrors.IsNotFound(err)).To(BeTrue()) - drained, err = r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + result, err = r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) - Expect(drained).To(BeTrue()) + Expect(result.Ready).To(BeTrue()) }) It("should not report drained while an evicted pod is still terminating", func() { @@ -675,9 +675,9 @@ var _ = Describe("skyhook controller tests", func() { PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, } - drained, err := r.DrainNode(ctx, skyhookNode, _package) + result, err := r.DrainNode(ctx, skyhookNode, _package) Expect(err).ToNot(HaveOccurred()) - Expect(drained).To(BeFalse()) + Expect(result.Ready).To(BeFalse()) Expect(deleteCount).To(Equal(1)) terminating := &corev1.Pod{} @@ -688,18 +688,18 @@ var _ = Describe("skyhook controller tests", func() { Expect(err).ToNot(HaveOccurred()) Expect(isDrained).To(BeFalse()) - drained, err = r.DrainNode(ctx, skyhookNode, _package) + result, err = r.DrainNode(ctx, skyhookNode, _package) Expect(err).ToNot(HaveOccurred()) - Expect(drained).To(BeFalse()) + Expect(result.Ready).To(BeFalse()) Expect(deleteCount).To(Equal(1)) terminating.Finalizers = nil Expect(testClient.Update(ctx, terminating)).To(Succeed()) Expect(apierrors.IsNotFound(testClient.Get(ctx, types.NamespacedName{Namespace: "default", Name: "workload"}, &corev1.Pod{}))).To(BeTrue()) - drained, err = r.DrainNode(ctx, skyhookNode, _package) + result, err = r.DrainNode(ctx, skyhookNode, _package) Expect(err).ToNot(HaveOccurred()) - Expect(drained).To(BeTrue()) + Expect(result.Ready).To(BeTrue()) }) It("should wait without deleting unmanaged pods when force is false", func() { @@ -748,11 +748,11 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) Expect(err).ToNot(HaveOccurred()) - drained, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + result, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) - Expect(drained).To(BeFalse()) + Expect(result.Ready).To(BeFalse()) Expect(deleteCalled).To(BeFalse()) Expect(evictCalled).To(BeFalse()) Expect(skyhookNode.Status()).To(Equal(v1alpha1.StatusInProgress)) @@ -835,7 +835,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -1097,11 +1097,11 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) Expect(err).ToNot(HaveOccurred()) - drained, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + result, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) - Expect(drained).To(BeFalse()) + Expect(result.Ready).To(BeFalse()) Expect(deleteCalled).To(BeFalse()) Expect(skyhookNode.Status()).To(Equal(v1alpha1.StatusErroring)) }) @@ -1161,11 +1161,11 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) Expect(err).ToNot(HaveOccurred()) - drained, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + result, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) - Expect(drained).To(BeFalse()) + Expect(result.Ready).To(BeFalse()) Expect(skyhookNode.Status()).To(Equal(v1alpha1.StatusErroring)) Eventually(recorder.Events).Should(Receive(ContainSubstring("Warning Drain drain timed out after [1s] for node [node-a] package [pkg:1.0.0] from [nodewright:drain-timeout]"))) Eventually(recorder.Events).Should(Receive(ContainSubstring("Warning Drain drain timed out after [1s] for node [node-a] package [pkg:1.0.0]"))) @@ -1215,7 +1215,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}}, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -1252,7 +1252,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(cordoned, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -1280,7 +1280,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}}, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -4550,7 +4550,7 @@ var _ = Describe("ProcessInterrupt skipped-package promotion", func() { pkg := sn.GetSkyhook().Spec.Packages["baxter"] r := &SkyhookReconciler{} - proceed, err := r.ProcessInterrupt(context.Background(), sn, &pkg, pkg.Interrupt, true) + proceed, err := r.ProcessInterrupt(context.Background(), sn, &pkg, pkg.Interrupt, true, nil) Expect(err).NotTo(HaveOccurred()) Expect(proceed).To(BeFalse()) @@ -4565,7 +4565,7 @@ var _ = Describe("ProcessInterrupt skipped-package promotion", func() { pkg := sn.GetSkyhook().Spec.Packages["baxter"] r := &SkyhookReconciler{} - proceed, err := r.ProcessInterrupt(context.Background(), sn, &pkg, pkg.Interrupt, false) + proceed, err := r.ProcessInterrupt(context.Background(), sn, &pkg, pkg.Interrupt, false, nil) Expect(err).NotTo(HaveOccurred()) Expect(proceed).To(BeFalse()) From 4fa7fe3931125a3f51126223842ec2ee455d86e1 Mon Sep 17 00:00:00 2001 From: mohityadav8 Date: Thu, 17 Sep 2026 18:42:20 +0530 Subject: [PATCH 3/9] fix: surface drain blockers (PDB, unmanaged pods, emptyDir) as a DrainBlocked condition Signed-off-by: mohityadav8 --- docs/architecture/operator-status.md | 1 + .../internal/controller/skyhook_controller.go | 47 +++++++++++------ .../controller/skyhook_controller_test.go | 18 +++++++ .../internal/wrapper/skyhook_conditions.go | 50 ++++++++++++++++--- 4 files changed, 93 insertions(+), 23 deletions(-) diff --git a/docs/architecture/operator-status.md b/docs/architecture/operator-status.md index a12926ebc..9b99fc63e 100644 --- a/docs/architecture/operator-status.md +++ b/docs/architecture/operator-status.md @@ -90,6 +90,7 @@ The operator also sets additional condition types that may be useful for trouble - `NodesIgnored`: selected nodes are skipped because they have the ignore label set - `ApplyPackage`: the controller is applying a package to a node - `DeploymentPolicyNotFound`: the referenced `DeploymentPolicy` is missing at reconcile time +- `DrainBlocked`: one or more selected nodes have a drain that cannot currently make progress. Reasons: `PodDisruptionBudget`, `UnmanagedPod`, `EmptyDirData`, or `MultipleCauses` when more than one kind of blocker is present. Independent of `Blocked` — a NodeWright can be both dependency-blocked and drain-blocked at once. These conditions complement, rather than replace, `.status.status` and `Ready`. diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go index e62a0e78a..e2d603eaf 100644 --- a/operator/internal/controller/skyhook_controller.go +++ b/operator/internal/controller/skyhook_controller.go @@ -1449,6 +1449,7 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState selectedNode := nodePicker.SelectNodes(skyhook) drainBlocks := make([]wrapper.DrainBlockedNode, 0) + serialStop := false for _, node := range selectedNode { // Skip nodes that are waiting on higher-priority skyhooks @@ -1519,9 +1520,14 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState // process one package at a time if skyhook.GetSkyhook().Spec.Serial { - return &ctrl.Result{RequeueAfter: time.Second * 2}, nil + serialStop = true + break } } + + if serialStop { + break + } } skyhook.UpdateDrainBlockedCondition(drainBlocks) @@ -1534,12 +1540,8 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState requeue = true } - if !skyhook.IsComplete() || requeue { - requeueAfter := time.Second * 2 - if len(drainBlocks) > 0 { - requeueAfter = 30 * time.Second - } - return &ctrl.Result{RequeueAfter: requeueAfter}, nil // not sure this is better then just requeue bool + if serialStop || !skyhook.IsComplete() || requeue { + return &ctrl.Result{RequeueAfter: time.Second * 2}, nil // not sure this is better then just requeue bool } return nil, utilerrors.NewAggregate(errs) @@ -2713,7 +2715,7 @@ func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.S } if len(errs) > 0 { - return drain.DrainResult{}, utilerrors.NewAggregate(errs) + return drain.DrainResult{Blocked: blocked}, utilerrors.NewAggregate(errs) } return drain.DrainResult{Ready: !waitingForPods, Blocked: blocked}, nil @@ -3390,17 +3392,13 @@ func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wr // drain and cordon node before applying package that has an interrupt if stage == v1alpha1.StageApply || stage == v1alpha1.StageUninstall { ready, blocked, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, _package) + if len(blocked) > 0 && drainBlocks != nil { + mergeDrainBlockedNode(drainBlocks, skyhookNode.GetNode().Name, blocked) + } if err != nil { return false, err } - if len(blocked) > 0 && drainBlocks != nil { - *drainBlocks = append(*drainBlocks, wrapper.DrainBlockedNode{ - NodeName: skyhookNode.GetNode().Name, - Blocked: blocked, - }) - } - if !ready { return false, nil } @@ -3444,6 +3442,23 @@ func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wr return true, nil } +// mergeDrainBlockedNode records blocked pods for a node in drainBlocks, merging into an +// existing entry for that node rather than appending a duplicate. ProcessInterrupt runs once +// per runnable package on a node in a single reconcile pass, so a node with two interrupt- +// bearing packages must contribute to one DrainBlockedNode entry, not two. +func mergeDrainBlockedNode(drainBlocks *[]wrapper.DrainBlockedNode, nodeName string, blocked []drain.BlockedPod) { + for i := range *drainBlocks { + if (*drainBlocks)[i].NodeName == nodeName { + (*drainBlocks)[i].Blocked = append((*drainBlocks)[i].Blocked, blocked...) + return + } + } + *drainBlocks = append(*drainBlocks, wrapper.DrainBlockedNode{ + NodeName: nodeName, + Blocked: blocked, + }) +} + func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (bool, []drain.BlockedPod, error) { // Cordon is an in-memory mutation; SaveNodesAndSkyhook patches it at the end of this // pass, after every selected node has been visited. Draining in the same pass that @@ -3483,7 +3498,7 @@ func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, s result, err := r.DrainNode(ctx, skyhookNode, _package) if err != nil { - return false, nil, fmt.Errorf("error draining node [%s]: %w", skyhookNode.GetNode().Name, err) + return false, result.Blocked, fmt.Errorf("error draining node [%s]: %w", skyhookNode.GetNode().Name, err) } return result.Ready, result.Blocked, nil diff --git a/operator/internal/controller/skyhook_controller_test.go b/operator/internal/controller/skyhook_controller_test.go index 42979f342..4b7378cc3 100644 --- a/operator/internal/controller/skyhook_controller_test.go +++ b/operator/internal/controller/skyhook_controller_test.go @@ -30,6 +30,7 @@ import ( skyhookNodesMock "github.com/NVIDIA/nodewright/operator/internal/controller/mock" "github.com/NVIDIA/nodewright/operator/internal/dal" dalMock "github.com/NVIDIA/nodewright/operator/internal/dal/mock" + "github.com/NVIDIA/nodewright/operator/internal/drain" "github.com/NVIDIA/nodewright/operator/internal/wrapper" wrapperMock "github.com/NVIDIA/nodewright/operator/internal/wrapper/mock" . "github.com/onsi/ginkgo/v2" @@ -1293,6 +1294,23 @@ var _ = Describe("skyhook controller tests", func() { }) }) + Describe("mergeDrainBlockedNode", func() { + It("merges blocked pods into an existing node entry instead of duplicating it", func() { + drainBlocks := []wrapper.DrainBlockedNode{} + mergeDrainBlockedNode(&drainBlocks, "node-a", []drain.BlockedPod{{Name: "pod-1"}}) + mergeDrainBlockedNode(&drainBlocks, "node-a", []drain.BlockedPod{{Name: "pod-2"}}) + Expect(drainBlocks).To(HaveLen(1)) + Expect(drainBlocks[0].Blocked).To(HaveLen(2)) + }) + + It("creates separate entries for different nodes", func() { + drainBlocks := []wrapper.DrainBlockedNode{} + mergeDrainBlockedNode(&drainBlocks, "node-a", []drain.BlockedPod{{Name: "pod-1"}}) + mergeDrainBlockedNode(&drainBlocks, "node-b", []drain.BlockedPod{{Name: "pod-2"}}) + Expect(drainBlocks).To(HaveLen(2)) + }) + }) + It("should set monotonic SKYHOOK_NODE_ORDER across nodes and batches", func() { now := time.Now() testSkyhook := wrapper.NewSkyhookWrapper(&v1alpha1.NodeWright{ diff --git a/operator/internal/wrapper/skyhook_conditions.go b/operator/internal/wrapper/skyhook_conditions.go index 48e1cbeac..12b48fb12 100644 --- a/operator/internal/wrapper/skyhook_conditions.go +++ b/operator/internal/wrapper/skyhook_conditions.go @@ -331,8 +331,7 @@ func FormatNodeList(nodes []string) string { } // DrainBlockedNode is one node's drain blockers for the DrainBlocked condition -// message builder below — kept independent of the controller package's -// nodeDrainBlock so wrapper has no import cycle back to controller. +// message builder below. type DrainBlockedNode struct { NodeName string Blocked []drain.BlockedPod @@ -364,28 +363,65 @@ func DrainBlockedConditionReason(nodes []DrainBlockedNode) string { return drainBlockedReasonMultiple } +// drainBlockedDetailLineLimit caps the number of per-pod detail lines rendered into the +// DrainBlocked message. .status.conditions[].message has a 32768-byte apiserver limit; +// without a cap, a large enough blocked set (each PDB detail line carries apiserver prose +// of unbounded length) can fail the status update outright — precisely when the cluster is +// most blocked. The full set is always available from nodes[].Blocked for logging by the +// caller; this function stays pure and does not log. +const drainBlockedDetailLineLimit = 10 + // DrainBlockedConditionMessage renders the aggregate DrainBlocked message: a // "N/total nodes blocked draining (names)" summary line — following the same // truncation idiom as the Ready condition — followed by one "/ on // : " line per blocked pod that carries a Detail (PDB // cases only; Detail is apiserver prose and is never altered). +// +// Node and pod order are sorted rather than taken from nodes/nodes[].Blocked as given: +// node order there comes from a compartment map and pod order from the informer store, +// neither of which is stable between otherwise-identical reconcile passes. An unsorted +// message reshuffles every pass and triggers a spurious status write each time — the same +// reason the Ready condition's node lists are sorted. func DrainBlockedConditionMessage(nodes []DrainBlockedNode, totalSelected int) string { - names := make([]string, 0, len(nodes)) - for _, n := range nodes { - names = append(names, n.NodeName) + sorted := make([]DrainBlockedNode, len(nodes)) + copy(sorted, nodes) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].NodeName < sorted[j].NodeName }) + + names := make([]string, 0, len(sorted)) + for i := range sorted { + names = append(names, sorted[i].NodeName) + blocked := make([]drain.BlockedPod, len(sorted[i].Blocked)) + copy(blocked, sorted[i].Blocked) + sort.Slice(blocked, func(a, b int) bool { + if blocked[a].Namespace != blocked[b].Namespace { + return blocked[a].Namespace < blocked[b].Namespace + } + return blocked[a].Name < blocked[b].Name + }) + sorted[i].Blocked = blocked } sort.Strings(names) - lines := []string{fmt.Sprintf("%d/%d nodes blocked draining%s", len(nodes), totalSelected, formatNodeList(names))} + lines := []string{fmt.Sprintf("%d/%d nodes blocked draining%s", len(sorted), totalSelected, formatNodeList(names))} - for _, n := range nodes { + detailLines := 0 + truncated := false + for _, n := range sorted { for _, b := range n.Blocked { if b.Detail == "" { continue } + if detailLines >= drainBlockedDetailLineLimit { + truncated = true + continue + } lines = append(lines, fmt.Sprintf("%s/%s on %s: %s", b.Namespace, b.Name, n.NodeName, b.Detail)) + detailLines++ } } + if truncated { + lines = append(lines, "(additional detail truncated; see controller logs)") + } return strings.Join(lines, "; ") } From b3f6963eb4d4a2287a5a9b781dec576d60d9807f Mon Sep 17 00:00:00 2001 From: mohityadav8 Date: Tue, 22 Sep 2026 00:11:16 +0530 Subject: [PATCH 4/9] fix: resolve merge conflicts with upstream/main; fix FormatNodeList casing and stale EnsureNodeIsReadyForInterrupt call sites Signed-off-by: mohityadav8 --- .../internal/controller/skyhook_controller_test.go | 12 ++++++------ operator/internal/wrapper/skyhook_conditions.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/operator/internal/controller/skyhook_controller_test.go b/operator/internal/controller/skyhook_controller_test.go index 4b7378cc3..4ecdc35f5 100644 --- a/operator/internal/controller/skyhook_controller_test.go +++ b/operator/internal/controller/skyhook_controller_test.go @@ -853,14 +853,14 @@ var _ = Describe("skyhook controller tests", func() { Expect(testClient.Delete(ctx, goldenPod)).To(Succeed()) - ready, err = r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, _, err = r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) Expect(ready).To(BeFalse(), "evictable pod is still being drained") Expect(deleteCalled).To(BeTrue(), "DrainNode should delete evictable pod") - ready, err = r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, _, err = r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -926,7 +926,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -5385,12 +5385,12 @@ var _ = Describe("drain blocked by non-interrupt pods multi-node reconcile", fun Expect(nodeWrapperA).ToNot(BeNil()) Expect(nodeWrapperB).ToNot(BeNil()) - readyA, err := r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperA, &pkg) + readyA, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperA, &pkg) Expect(err).ToNot(HaveOccurred()) Expect(readyA).To(BeFalse()) Eventually(recorder.Events).Should(Receive(ContainSubstring(fmt.Sprintf("Warning Drain drain blocked by non-interrupt pods [default/%s] for package [pkg:1.0.0] from [nodewright:%s]", podName, skyhookName)))) - readyB, err := r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperB, &pkg) + readyB, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperB, &pkg) Expect(err).ToNot(HaveOccurred()) Expect(readyB).To(BeTrue(), "node-b has no non-interrupt work and should be ready for interrupt") @@ -5437,7 +5437,7 @@ var _ = Describe("drain blocked by non-interrupt pods multi-node reconcile", fun cond = meta.FindStatusCondition(sn.GetSkyhook().Status.Conditions, wrapper.SkyhookConditionBlocked) Expect(cond).To(BeNil(), "Blocked condition must be removed when no nodes are blocked") - readyA, err = r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperA, &pkg) + readyA, _, err = r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperA, &pkg) Expect(err).ToNot(HaveOccurred()) Expect(readyA).To(BeTrue(), "node-a should be ready for interrupt once non-interrupt pods are gone") }) diff --git a/operator/internal/wrapper/skyhook_conditions.go b/operator/internal/wrapper/skyhook_conditions.go index 12b48fb12..0e21dd6bf 100644 --- a/operator/internal/wrapper/skyhook_conditions.go +++ b/operator/internal/wrapper/skyhook_conditions.go @@ -402,7 +402,7 @@ func DrainBlockedConditionMessage(nodes []DrainBlockedNode, totalSelected int) s } sort.Strings(names) - lines := []string{fmt.Sprintf("%d/%d nodes blocked draining%s", len(sorted), totalSelected, formatNodeList(names))} + lines := []string{fmt.Sprintf("%d/%d nodes blocked draining%s", len(sorted), totalSelected, FormatNodeList(names))} detailLines := 0 truncated := false From c56de8e27d688c0f47daed3962b9902353b18659 Mon Sep 17 00:00:00 2001 From: mohityadav8 Date: Tue, 22 Sep 2026 00:28:16 +0530 Subject: [PATCH 5/9] fix: deduplicate blocked pods per node and render Detail fallback for unmanaged/emptyDir blockers Signed-off-by: mohityadav8 --- .../internal/controller/skyhook_controller.go | 24 +++++++++++++++++-- .../internal/wrapper/skyhook_conditions.go | 11 ++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go index e2d603eaf..85701031e 100644 --- a/operator/internal/controller/skyhook_controller.go +++ b/operator/internal/controller/skyhook_controller.go @@ -3449,16 +3449,36 @@ func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wr func mergeDrainBlockedNode(drainBlocks *[]wrapper.DrainBlockedNode, nodeName string, blocked []drain.BlockedPod) { for i := range *drainBlocks { if (*drainBlocks)[i].NodeName == nodeName { - (*drainBlocks)[i].Blocked = append((*drainBlocks)[i].Blocked, blocked...) + (*drainBlocks)[i].Blocked = appendUniqueBlockedPods((*drainBlocks)[i].Blocked, blocked) return } } *drainBlocks = append(*drainBlocks, wrapper.DrainBlockedNode{ NodeName: nodeName, - Blocked: blocked, + Blocked: appendUniqueBlockedPods(nil, blocked), }) } +// appendUniqueBlockedPods appends new to existing, skipping any (namespace, name, reason) already +// present. DrainNode reclassifies every pod on the node on each call, so ProcessInterrupt calling +// it once per interrupt-bearing package can otherwise report the same blocker more than once, +// wasting the message builder's per-node detail-line budget on duplicates. +func appendUniqueBlockedPods(existing, new []drain.BlockedPod) []drain.BlockedPod { + seen := make(map[[3]string]struct{}, len(existing)) + for _, b := range existing { + seen[[3]string{b.Namespace, b.Name, string(b.Reason)}] = struct{}{} + } + for _, b := range new { + key := [3]string{b.Namespace, b.Name, string(b.Reason)} + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + existing = append(existing, b) + } + return existing +} + func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (bool, []drain.BlockedPod, error) { // Cordon is an in-memory mutation; SaveNodesAndSkyhook patches it at the end of this // pass, after every selected node has been visited. Draining in the same pass that diff --git a/operator/internal/wrapper/skyhook_conditions.go b/operator/internal/wrapper/skyhook_conditions.go index 0e21dd6bf..9ebc5d6f6 100644 --- a/operator/internal/wrapper/skyhook_conditions.go +++ b/operator/internal/wrapper/skyhook_conditions.go @@ -408,14 +408,19 @@ func DrainBlockedConditionMessage(nodes []DrainBlockedNode, totalSelected int) s truncated := false for _, n := range sorted { for _, b := range n.Blocked { - if b.Detail == "" { - continue + detail := b.Detail + if detail == "" { + // DrainNode creates unmanaged/emptyDir blockers with a Reason but no + // apiserver-generated Detail (that's PDB-only). Fall back to the reason + // so these blockers still surface which pod is holding drain, instead + // of being silently dropped from the message. + detail = string(b.Reason) } if detailLines >= drainBlockedDetailLineLimit { truncated = true continue } - lines = append(lines, fmt.Sprintf("%s/%s on %s: %s", b.Namespace, b.Name, n.NodeName, b.Detail)) + lines = append(lines, fmt.Sprintf("%s/%s on %s: %s", b.Namespace, b.Name, n.NodeName, detail)) detailLines++ } } From a9e1288773283ab10798e58126e124c25940fbc9 Mon Sep 17 00:00:00 2001 From: mohityadav8 Date: Tue, 22 Sep 2026 15:08:25 +0530 Subject: [PATCH 6/9] fix: make DrainBlocked condition level-triggered off persisted per-node state Signed-off-by: mohityadav8 --- .../internal/controller/cluster_state_v2.go | 41 +++++-- .../internal/controller/mock/SkyhookNodes.go | 40 ++++--- .../internal/controller/skyhook_controller.go | 49 +++++++- operator/internal/wrapper/mock/SkyhookNode.go | 109 +++++++++++++++++- .../internal/wrapper/mock/SkyhookNodeOnly.go | 109 +++++++++++++++++- operator/internal/wrapper/node.go | 77 +++++++++++++ .../internal/wrapper/skyhook_conditions.go | 19 +-- 7 files changed, 405 insertions(+), 39 deletions(-) diff --git a/operator/internal/controller/cluster_state_v2.go b/operator/internal/controller/cluster_state_v2.go index 6e0d5ecad..72378ac7f 100644 --- a/operator/internal/controller/cluster_state_v2.go +++ b/operator/internal/controller/cluster_state_v2.go @@ -402,7 +402,7 @@ type SkyhookNodes interface { IsPaused() bool HasUninstallWork() (bool, error) UpdateBlockedCondition() error - UpdateDrainBlockedCondition(blocks []wrapper.DrainBlockedNode) + UpdateDrainBlockedCondition() error UpdateUninstallConditions() error UpdateNodeStateMalformedCondition() NodeCount() int @@ -627,17 +627,37 @@ func (s *skyhookNodes) UpdateBlockedCondition() error { return nil } -// UpdateDrainBlockedCondition sets or clears the DrainBlocked condition from this -// reconcile pass's drain-blocker findings (PDB rejections, unmanaged pods, emptyDir -// pods). Unlike UpdateBlockedCondition — computed from persisted per-node dependency -// state at the top of Reconcile — this reflects only what THIS pass's live drain -// attempts found, so it is set once, after the node-processing loop finishes, rather -// than at the top of Reconcile. An empty blocks slice (nothing blocked this pass, or -// every previously blocked node has since drained) clears the condition. -func (s *skyhookNodes) UpdateDrainBlockedCondition(blocks []wrapper.DrainBlockedNode) { +// UpdateDrainBlockedCondition rebuilds the DrainBlocked condition from each node's +// persisted drain-blocker annotation (SkyhookNode.DrainBlocked), rather than from a +// single pass's live findings. This makes the condition level-triggered, matching +// UpdateBlockedCondition: it is correct from persisted state alone regardless of +// whether this reconcile pass actually ran RunSkyhookPackages for this Skyhook (paused, +// disabled, complete Skyhooks skip it) or returned from it early (an error, or +// spec.serial stopping after the first node) — those nodes simply keep whatever was +// last recorded for them, rather than being wrongly treated as unblocked. +// +// A node whose annotation fails to parse is skipped for this computation, the same +// tolerance UpdateBlockedCondition applies to unreadable nodeState — the parse failure +// is a NodeStateMalformed-adjacent concern, not something this condition should error on. +func (s *skyhookNodes) UpdateDrainBlockedCondition() error { + blocks := make([]wrapper.DrainBlockedNode, 0, len(s.nodes)) + for _, node := range s.nodes { + blocked, err := node.DrainBlocked() + if err != nil { + continue + } + if len(blocked) == 0 { + continue + } + blocks = append(blocks, wrapper.DrainBlockedNode{ + NodeName: node.GetNode().Name, + Blocked: blocked, + }) + } + if len(blocks) == 0 { wrapper.RemoveSkyhookConditionTypes(s.skyhook, wrapper.SkyhookConditionDrainBlocked) - return + return nil } wrapper.AddSkyhookCondition(s.skyhook, metav1.Condition{ @@ -648,6 +668,7 @@ func (s *skyhookNodes) UpdateDrainBlockedCondition(blocks []wrapper.DrainBlocked Reason: wrapper.DrainBlockedConditionReason(blocks), Message: wrapper.DrainBlockedConditionMessage(blocks, len(s.nodes)), }) + return nil } // isPackageCompleteOnAllNodes reports whether the package has reached its diff --git a/operator/internal/controller/mock/SkyhookNodes.go b/operator/internal/controller/mock/SkyhookNodes.go index 1b5f55e03..9ed407441 100644 --- a/operator/internal/controller/mock/SkyhookNodes.go +++ b/operator/internal/controller/mock/SkyhookNodes.go @@ -1077,9 +1077,20 @@ func (_c *MockSkyhookNodes_UpdateCondition_Call) RunAndReturn(run func(logger lo } // UpdateDrainBlockedCondition provides a mock function for the type MockSkyhookNodes -func (_mock *MockSkyhookNodes) UpdateDrainBlockedCondition(blocks []wrapper.DrainBlockedNode) { - _mock.Called(blocks) - return +func (_mock *MockSkyhookNodes) UpdateDrainBlockedCondition() error { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for UpdateDrainBlockedCondition") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func() error); ok { + r0 = returnFunc() + } else { + r0 = ret.Error(0) + } + return r0 } // MockSkyhookNodes_UpdateDrainBlockedCondition_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateDrainBlockedCondition' @@ -1088,31 +1099,24 @@ type MockSkyhookNodes_UpdateDrainBlockedCondition_Call struct { } // UpdateDrainBlockedCondition is a helper method to define mock.On call -// - blocks []wrapper.DrainBlockedNode -func (_e *MockSkyhookNodes_Expecter) UpdateDrainBlockedCondition(blocks any) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { - return &MockSkyhookNodes_UpdateDrainBlockedCondition_Call{Call: _e.mock.On("UpdateDrainBlockedCondition", blocks)} +func (_e *MockSkyhookNodes_Expecter) UpdateDrainBlockedCondition() *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + return &MockSkyhookNodes_UpdateDrainBlockedCondition_Call{Call: _e.mock.On("UpdateDrainBlockedCondition")} } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Run(run func(blocks []wrapper.DrainBlockedNode)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Run(run func()) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { _c.Call.Run(func(args mock.Arguments) { - var arg0 []wrapper.DrainBlockedNode - if args[0] != nil { - arg0 = args[0].([]wrapper.DrainBlockedNode) - } - run( - arg0, - ) + run() }) return _c } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Return() *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { - _c.Call.Return() +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Return(err error) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + _c.Call.Return(err) return _c } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) RunAndReturn(run func(blocks []wrapper.DrainBlockedNode)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { - _c.Run(run) +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) RunAndReturn(run func() error) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + _c.Call.Return(run) return _c } diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go index 85701031e..ed89705e7 100644 --- a/operator/internal/controller/skyhook_controller.go +++ b/operator/internal/controller/skyhook_controller.go @@ -604,6 +604,14 @@ func (r *SkyhookReconciler) refreshSkyhookConditions(ctx context.Context, cluste if err := skyhook.UpdateBlockedCondition(); err != nil { return fmt.Errorf("error updating blocked condition: %w", err) } + // DrainBlocked (PDB/unmanaged-pod/emptyDir drain blockers). Distinct from the + // NonInterruptPodsRunning-flavored Blocked condition r.updateDrainBlockedCondition + // below maintains — same name prefix, different condition type. Rebuilt from + // persisted per-node state so it stays correct on paused/disabled/complete/error/ + // serial-partial passes; see cluster_state_v2.go's UpdateDrainBlockedCondition. + if err := skyhook.UpdateDrainBlockedCondition(); err != nil { + return fmt.Errorf("error rebuilding drain blocked condition: %w", err) + } if err := r.updateDrainBlockedCondition(ctx, skyhook); err != nil { return fmt.Errorf("error updating drain blocked condition: %w", err) } @@ -1530,7 +1538,21 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState } } - skyhook.UpdateDrainBlockedCondition(drainBlocks) + // The message builder truncates detail lines past drainBlockedDetailLineLimit and points + // the reader at controller logs for the rest — so log the full set here, mirroring + // updateTaintToleranceCondition's log-before-truncate pattern, or that pointer is a lie. + // drainBlocks is only this pass's local findings, used for this diagnostic log line; + // the condition itself is rebuilt from persisted per-node state (see below), which is + // what makes it level-triggered rather than dependent on reaching this line. + if totalBlockedPods := countBlockedPods(drainBlocks); totalBlockedPods > wrapper.ReadyConditionNodeListLimit { + logger.Info("DrainBlocked condition message truncated; full blocked set", "nodewright", skyhook.GetSkyhook().Name, "drainBlocks", drainBlocks) + } + // Re-run here too (also done unconditionally in refreshSkyhookConditions) so the + // condition reflects this pass's fresh findings immediately rather than waiting one + // more reconcile for refreshSkyhookConditions to pick them up. + if err := skyhook.UpdateDrainBlockedCondition(); err != nil { + return &ctrl.Result{}, fmt.Errorf("error rebuilding drain blocked condition: %w", err) + } saved, errs := r.SaveNodesAndSkyhook(ctx, clusterState, skyhook) if len(errs) > 0 { @@ -2650,7 +2672,14 @@ func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.S _package.Version, ) skyhookNode.SetStatus(v1alpha1.StatusErroring) - return drain.DrainResult{}, nil + // Preserve whatever blockers were last recorded rather than clearing them: this is + // the moment a stuck drain becomes a user-visible DrainTimeout error, so the + // condition should keep explaining what was blocking it, not go silent. + lastBlocked, blockedErr := skyhookNode.DrainBlocked() + if blockedErr != nil { + return drain.DrainResult{}, blockedErr + } + return drain.DrainResult{Blocked: lastBlocked}, nil } pods, err := r.dal.GetPods(ctx, client.MatchingFields{ @@ -3479,6 +3508,16 @@ func appendUniqueBlockedPods(existing, new []drain.BlockedPod) []drain.BlockedPo return existing } +// countBlockedPods sums Blocked across every node, for deciding whether the DrainBlocked +// message will be truncated and the full set needs logging. +func countBlockedPods(blocks []wrapper.DrainBlockedNode) int { + total := 0 + for _, b := range blocks { + total += len(b.Blocked) + } + return total +} + func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (bool, []drain.BlockedPod, error) { // Cordon is an in-memory mutation; SaveNodesAndSkyhook patches it at the end of this // pass, after every selected node has been visited. Draining in the same pass that @@ -3517,6 +3556,12 @@ func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, s } result, err := r.DrainNode(ctx, skyhookNode, _package) + // Persist regardless of err: this is what makes DrainBlocked level-triggered rather + // than dependent on this pass reaching UpdateDrainBlockedCondition later. See + // SetDrainBlocked's doc comment. + if setErr := skyhookNode.SetDrainBlocked(result.Blocked); setErr != nil && err == nil { + err = setErr + } if err != nil { return false, result.Blocked, fmt.Errorf("error draining node [%s]: %w", skyhookNode.GetNode().Name, err) } diff --git a/operator/internal/wrapper/mock/SkyhookNode.go b/operator/internal/wrapper/mock/SkyhookNode.go index 10b4c4800..343a82123 100644 --- a/operator/internal/wrapper/mock/SkyhookNode.go +++ b/operator/internal/wrapper/mock/SkyhookNode.go @@ -6,11 +6,12 @@ package wrapper import ( "github.com/NVIDIA/nodewright/operator/api/nodewright/v1alpha1" + "github.com/NVIDIA/nodewright/operator/internal/drain" "github.com/NVIDIA/nodewright/operator/internal/wrapper" "github.com/go-logr/logr" mock "github.com/stretchr/testify/mock" v10 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // NewMockSkyhookNode creates a new instance of MockSkyhookNode. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. @@ -203,6 +204,61 @@ func (_c *MockSkyhookNode_Cordon_Call) RunAndReturn(run func() bool) *MockSkyhoo return _c } +// DrainBlocked provides a mock function for the type MockSkyhookNode +func (_mock *MockSkyhookNode) DrainBlocked() ([]drain.BlockedPod, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for DrainBlocked") + } + + var r0 []drain.BlockedPod + var r1 error + if returnFunc, ok := ret.Get(0).(func() ([]drain.BlockedPod, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() []drain.BlockedPod); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]drain.BlockedPod) + } + } + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockSkyhookNode_DrainBlocked_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DrainBlocked' +type MockSkyhookNode_DrainBlocked_Call struct { + *mock.Call +} + +// DrainBlocked is a helper method to define mock.On call +func (_e *MockSkyhookNode_Expecter) DrainBlocked() *MockSkyhookNode_DrainBlocked_Call { + return &MockSkyhookNode_DrainBlocked_Call{Call: _e.mock.On("DrainBlocked")} +} + +func (_c *MockSkyhookNode_DrainBlocked_Call) Run(run func()) *MockSkyhookNode_DrainBlocked_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockSkyhookNode_DrainBlocked_Call) Return(blockedPods []drain.BlockedPod, err error) *MockSkyhookNode_DrainBlocked_Call { + _c.Call.Return(blockedPods, err) + return _c +} + +func (_c *MockSkyhookNode_DrainBlocked_Call) RunAndReturn(run func() ([]drain.BlockedPod, error)) *MockSkyhookNode_DrainBlocked_Call { + _c.Call.Return(run) + return _c +} + // DrainStartedAt provides a mock function for the type MockSkyhookNode func (_mock *MockSkyhookNode) DrainStartedAt() (*v1.Time, error) { ret := _mock.Called() @@ -1107,6 +1163,57 @@ func (_c *MockSkyhookNode_RunNext_Call) RunAndReturn(run func() ([]*v1alpha1.Pac return _c } +// SetDrainBlocked provides a mock function for the type MockSkyhookNode +func (_mock *MockSkyhookNode) SetDrainBlocked(blocked []drain.BlockedPod) error { + ret := _mock.Called(blocked) + + if len(ret) == 0 { + panic("no return value specified for SetDrainBlocked") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func([]drain.BlockedPod) error); ok { + r0 = returnFunc(blocked) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockSkyhookNode_SetDrainBlocked_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetDrainBlocked' +type MockSkyhookNode_SetDrainBlocked_Call struct { + *mock.Call +} + +// SetDrainBlocked is a helper method to define mock.On call +// - blocked []drain.BlockedPod +func (_e *MockSkyhookNode_Expecter) SetDrainBlocked(blocked any) *MockSkyhookNode_SetDrainBlocked_Call { + return &MockSkyhookNode_SetDrainBlocked_Call{Call: _e.mock.On("SetDrainBlocked", blocked)} +} + +func (_c *MockSkyhookNode_SetDrainBlocked_Call) Run(run func(blocked []drain.BlockedPod)) *MockSkyhookNode_SetDrainBlocked_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 []drain.BlockedPod + if args[0] != nil { + arg0 = args[0].([]drain.BlockedPod) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockSkyhookNode_SetDrainBlocked_Call) Return(err error) *MockSkyhookNode_SetDrainBlocked_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockSkyhookNode_SetDrainBlocked_Call) RunAndReturn(run func([]drain.BlockedPod) error) *MockSkyhookNode_SetDrainBlocked_Call { + _c.Call.Return(run) + return _c +} + // SetState provides a mock function for the type MockSkyhookNode func (_mock *MockSkyhookNode) SetState(state v1alpha1.NodeState) error { ret := _mock.Called(state) diff --git a/operator/internal/wrapper/mock/SkyhookNodeOnly.go b/operator/internal/wrapper/mock/SkyhookNodeOnly.go index ea392e03e..689e5c894 100644 --- a/operator/internal/wrapper/mock/SkyhookNodeOnly.go +++ b/operator/internal/wrapper/mock/SkyhookNodeOnly.go @@ -6,10 +6,11 @@ package wrapper import ( "github.com/NVIDIA/nodewright/operator/api/nodewright/v1alpha1" + "github.com/NVIDIA/nodewright/operator/internal/drain" "github.com/go-logr/logr" mock "github.com/stretchr/testify/mock" v10 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // NewMockSkyhookNodeOnly creates a new instance of MockSkyhookNodeOnly. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. @@ -169,6 +170,61 @@ func (_c *MockSkyhookNodeOnly_Cordon_Call) RunAndReturn(run func() bool) *MockSk return _c } +// DrainBlocked provides a mock function for the type MockSkyhookNodeOnly +func (_mock *MockSkyhookNodeOnly) DrainBlocked() ([]drain.BlockedPod, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for DrainBlocked") + } + + var r0 []drain.BlockedPod + var r1 error + if returnFunc, ok := ret.Get(0).(func() ([]drain.BlockedPod, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() []drain.BlockedPod); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]drain.BlockedPod) + } + } + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockSkyhookNodeOnly_DrainBlocked_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DrainBlocked' +type MockSkyhookNodeOnly_DrainBlocked_Call struct { + *mock.Call +} + +// DrainBlocked is a helper method to define mock.On call +func (_e *MockSkyhookNodeOnly_Expecter) DrainBlocked() *MockSkyhookNodeOnly_DrainBlocked_Call { + return &MockSkyhookNodeOnly_DrainBlocked_Call{Call: _e.mock.On("DrainBlocked")} +} + +func (_c *MockSkyhookNodeOnly_DrainBlocked_Call) Run(run func()) *MockSkyhookNodeOnly_DrainBlocked_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockSkyhookNodeOnly_DrainBlocked_Call) Return(blockedPods []drain.BlockedPod, err error) *MockSkyhookNodeOnly_DrainBlocked_Call { + _c.Call.Return(blockedPods, err) + return _c +} + +func (_c *MockSkyhookNodeOnly_DrainBlocked_Call) RunAndReturn(run func() ([]drain.BlockedPod, error)) *MockSkyhookNodeOnly_DrainBlocked_Call { + _c.Call.Return(run) + return _c +} + // DrainStartedAt provides a mock function for the type MockSkyhookNodeOnly func (_mock *MockSkyhookNodeOnly) DrainStartedAt() (*v1.Time, error) { ret := _mock.Called() @@ -639,6 +695,57 @@ func (_c *MockSkyhookNodeOnly_Reset_Call) RunAndReturn(run func()) *MockSkyhookN return _c } +// SetDrainBlocked provides a mock function for the type MockSkyhookNodeOnly +func (_mock *MockSkyhookNodeOnly) SetDrainBlocked(blocked []drain.BlockedPod) error { + ret := _mock.Called(blocked) + + if len(ret) == 0 { + panic("no return value specified for SetDrainBlocked") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func([]drain.BlockedPod) error); ok { + r0 = returnFunc(blocked) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockSkyhookNodeOnly_SetDrainBlocked_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetDrainBlocked' +type MockSkyhookNodeOnly_SetDrainBlocked_Call struct { + *mock.Call +} + +// SetDrainBlocked is a helper method to define mock.On call +// - blocked []drain.BlockedPod +func (_e *MockSkyhookNodeOnly_Expecter) SetDrainBlocked(blocked any) *MockSkyhookNodeOnly_SetDrainBlocked_Call { + return &MockSkyhookNodeOnly_SetDrainBlocked_Call{Call: _e.mock.On("SetDrainBlocked", blocked)} +} + +func (_c *MockSkyhookNodeOnly_SetDrainBlocked_Call) Run(run func(blocked []drain.BlockedPod)) *MockSkyhookNodeOnly_SetDrainBlocked_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 []drain.BlockedPod + if args[0] != nil { + arg0 = args[0].([]drain.BlockedPod) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockSkyhookNodeOnly_SetDrainBlocked_Call) Return(err error) *MockSkyhookNodeOnly_SetDrainBlocked_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockSkyhookNodeOnly_SetDrainBlocked_Call) RunAndReturn(run func([]drain.BlockedPod) error) *MockSkyhookNodeOnly_SetDrainBlocked_Call { + _c.Call.Return(run) + return _c +} + // SetState provides a mock function for the type MockSkyhookNodeOnly func (_mock *MockSkyhookNodeOnly) SetState(state v1alpha1.NodeState) error { ret := _mock.Called(state) diff --git a/operator/internal/wrapper/node.go b/operator/internal/wrapper/node.go index 155f11e98..eb33df810 100644 --- a/operator/internal/wrapper/node.go +++ b/operator/internal/wrapper/node.go @@ -26,6 +26,7 @@ import ( "time" "github.com/NVIDIA/nodewright/operator/api/nodewright/v1alpha1" + "github.com/NVIDIA/nodewright/operator/internal/drain" "github.com/NVIDIA/nodewright/operator/internal/graph" "github.com/NVIDIA/nodewright/operator/internal/version" "github.com/go-logr/logr" @@ -113,6 +114,12 @@ type SkyhookNodeOnly interface { DrainStartedAt() (*metav1.Time, error) // ClearDrainStart removes the drain start marker for this Skyhook on this node. ClearDrainStart() + // SetDrainBlocked persists this pass's drain-blocker findings for this Skyhook on + // this node, replacing any previously recorded findings. Nil or empty clears it. + SetDrainBlocked(blocked []drain.BlockedPod) error + // DrainBlocked returns the drain-blocker findings last persisted for this Skyhook on + // this node. + DrainBlocked() ([]drain.BlockedPod, error) // Uncordon marks the node schedulable and removes this Skyhook's cordon annotation if present. Uncordon() // Reset clears Skyhook-related state and annotations so the node can be reconfigured from scratch. @@ -198,6 +205,10 @@ func (node *skyhookNode) drainStartAnnotationKey() string { return fmt.Sprintf("%s/drainStart_%s", v1alpha1.METADATA_PREFIX, node.skyhookName) } +func (node *skyhookNode) drainBlockedAnnotationKey() string { + return fmt.Sprintf("%s/drainBlocked_%s", v1alpha1.METADATA_PREFIX, node.skyhookName) +} + // GetSkyhook returns the Skyhook associated with this node, or nil if only a name was set. func (node *skyhookNode) GetSkyhook() *Skyhook { return node.skyhook @@ -602,6 +613,71 @@ func (node *skyhookNode) ClearDrainStart() { node.updated = true } +// SetDrainBlocked persists this pass's drain-blocker findings (PDB rejections, unmanaged +// pods, emptyDir pods) for this Skyhook on this node, replacing whatever was recorded +// previously. Nil or empty blocked clears the annotation. Unlike the per-package +// nodeState annotation, this always sets the whole value: there is exactly one DrainNode +// call's findings to record per pass, never several to merge. +// +// Persisting here — at the moment the caller learns the result — rather than accumulating +// in a caller-local slice is what lets the DrainBlocked condition be level-triggered: +// UpdateDrainBlockedCondition rebuilds it from this annotation on every reconcile pass, +// including passes that skip RunSkyhookPackages entirely (paused, disabled, complete +// Skyhooks) or that return from it early (an error, or spec.serial stopping after the +// first node). Without persisting immediately, all three of those paths would either +// never update the condition or wrongly clear it for nodes the pass never reached. +func (node *skyhookNode) SetDrainBlocked(blocked []drain.BlockedPod) error { + key := node.drainBlockedAnnotationKey() + + if len(blocked) == 0 { + if node.Annotations == nil { + return nil + } + if _, ok := node.Annotations[key]; !ok { + return nil + } + delete(node.Annotations, key) + node.updated = true + return nil + } + + data, err := json.Marshal(blocked) + if err != nil { + return fmt.Errorf("error marshalling drain blocked pods: %w", err) + } + + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + + if existing, ok := node.Annotations[key]; !ok || existing != string(data) { + node.Annotations[key] = string(data) + node.updated = true + } + + return nil +} + +// DrainBlocked returns the drain-blocker findings last persisted for this Skyhook on this +// node by SetDrainBlocked, or nil if none are recorded. +func (node *skyhookNode) DrainBlocked() ([]drain.BlockedPod, error) { + if node.Annotations == nil { + return nil, nil + } + + value, ok := node.Annotations[node.drainBlockedAnnotationKey()] + if !ok { + return nil, nil + } + + var blocked []drain.BlockedPod + if err := json.Unmarshal([]byte(value), &blocked); err != nil { + return nil, fmt.Errorf("error unmarshalling drain blocked pods: %w", err) + } + + return blocked, nil +} + // Uncordon marks the node schedulable and removes this Skyhook's cordon annotation if present. func (node *skyhookNode) Uncordon() { @@ -647,6 +723,7 @@ func (node *skyhookNode) Reset() { delete(node.Annotations, cordonAnnotationKey(node.skyhookName)) delete(node.Annotations, node.drainStartAnnotationKey()) + delete(node.Annotations, node.drainBlockedAnnotationKey()) delete(node.Annotations, fmt.Sprintf("%s/nodeState_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)) delete(node.Annotations, fmt.Sprintf("%s/status_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)) delete(node.Annotations, fmt.Sprintf("%s/version_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)) diff --git a/operator/internal/wrapper/skyhook_conditions.go b/operator/internal/wrapper/skyhook_conditions.go index 9ebc5d6f6..ee293a1c3 100644 --- a/operator/internal/wrapper/skyhook_conditions.go +++ b/operator/internal/wrapper/skyhook_conditions.go @@ -30,6 +30,8 @@ import ( const ( // ReadyConditionNodeListLimit caps condition message fan-out to avoid etcd object bloat and excess watch bandwidth on large rollouts. + // Also used as drainBlockedDetailLineLimit's value: the two independent condition-message + // fan-out caps must move together, or a future tuning pass silently desyncs them. ReadyConditionNodeListLimit = 10 SkyhookConditionReady = "Ready" @@ -364,12 +366,14 @@ func DrainBlockedConditionReason(nodes []DrainBlockedNode) string { } // drainBlockedDetailLineLimit caps the number of per-pod detail lines rendered into the -// DrainBlocked message. .status.conditions[].message has a 32768-byte apiserver limit; -// without a cap, a large enough blocked set (each PDB detail line carries apiserver prose -// of unbounded length) can fail the status update outright — precisely when the cluster is -// most blocked. The full set is always available from nodes[].Blocked for logging by the -// caller; this function stays pure and does not log. -const drainBlockedDetailLineLimit = 10 +// DrainBlocked message, so it stays well inside .status.conditions[].message's 32768-byte +// apiserver limit even though each Detail carries apiserver-generated prose of unbounded +// length; a genuinely pathological Detail could still overrun this line-count cap, but PDB +// cause messages are short in practice. Shares ReadyConditionNodeListLimit's value rather +// than redeclaring it, since both exist to bound condition-message fan-out for the same +// reason. The full set is always available from nodes[].Blocked for logging by the caller; +// this function stays pure and does not log. +const drainBlockedDetailLineLimit = ReadyConditionNodeListLimit // DrainBlockedConditionMessage renders the aggregate DrainBlocked message: a // "N/total nodes blocked draining (names)" summary line — following the same @@ -400,7 +404,8 @@ func DrainBlockedConditionMessage(nodes []DrainBlockedNode, totalSelected int) s }) sorted[i].Blocked = blocked } - sort.Strings(names) + // names is already in NodeName order: it's appended while iterating sorted, which was + // sorted by NodeName above. Re-sorting here would just prove that twice. lines := []string{fmt.Sprintf("%d/%d nodes blocked draining%s", len(sorted), totalSelected, FormatNodeList(names))} From 95e8cf34140a5bbe8d8b6714e29b1ffcbd0a3bfa Mon Sep 17 00:00:00 2001 From: mohityadav8 Date: Thu, 24 Sep 2026 14:51:06 +0530 Subject: [PATCH 7/9] fix: address ayuskauskas review round 2 on DrainBlocked condition Signed-off-by: mohityadav8 --- docs/architecture/interrupt-flow.md | 11 +- .../internal/controller/cluster_state_v2.go | 31 ++++- .../internal/controller/mock/SkyhookNodes.go | 40 +++--- .../internal/controller/skyhook_controller.go | 128 +++++++----------- .../controller/skyhook_controller_test.go | 42 ++---- operator/internal/drain/drain.go | 8 +- .../internal/wrapper/skyhook_conditions.go | 19 +-- 7 files changed, 123 insertions(+), 156 deletions(-) diff --git a/docs/architecture/interrupt-flow.md b/docs/architecture/interrupt-flow.md index f05919ec6..fa1d8307e 100644 --- a/docs/architecture/interrupt-flow.md +++ b/docs/architecture/interrupt-flow.md @@ -218,10 +218,13 @@ drain-blocked at the same time, so the two conditions never share a type. A PodDisruptionBudget rejection is treated as a self-resolving wait state, not a reconcile error: it no longer aborts the reconcile pass for the remaining -nodes, and it no longer puts the NodeWright into exponential backoff. While -`DrainBlocked` is set, the operator retries roughly every 30 seconds instead of -the usual 2 seconds, so an untimed drain (`spec.drainConfig.timeout` unset) -does not hammer the eviction API while a PDB holds. +nodes. However, this also means it no longer puts the NodeWright into +controller-runtime's exponential backoff. With a PDB at zero allowed +disruptions and `spec.drainConfig.timeout` unset, the operator currently +retries the eviction every 2 seconds indefinitely, where it previously backed +off toward roughly 1000 seconds — a real increase in eviction-API load. +Per-node throttling for the drain-blocked case specifically is tracked in +[#632](https://github.com/NVIDIA/nodewright/issues/632) and not yet shipped. Unmanaged pods (`force: false`) and `emptyDir` pods (`deleteEmptyDirData: false`) are also reported in `DrainBlocked`, though — unlike a PDB rejection — diff --git a/operator/internal/controller/cluster_state_v2.go b/operator/internal/controller/cluster_state_v2.go index 72378ac7f..839942576 100644 --- a/operator/internal/controller/cluster_state_v2.go +++ b/operator/internal/controller/cluster_state_v2.go @@ -402,7 +402,7 @@ type SkyhookNodes interface { IsPaused() bool HasUninstallWork() (bool, error) UpdateBlockedCondition() error - UpdateDrainBlockedCondition() error + UpdateDrainBlockedCondition(logger logr.Logger) UpdateUninstallConditions() error UpdateNodeStateMalformedCondition() NodeCount() int @@ -637,9 +637,11 @@ func (s *skyhookNodes) UpdateBlockedCondition() error { // last recorded for them, rather than being wrongly treated as unblocked. // // A node whose annotation fails to parse is skipped for this computation, the same -// tolerance UpdateBlockedCondition applies to unreadable nodeState — the parse failure -// is a NodeStateMalformed-adjacent concern, not something this condition should error on. -func (s *skyhookNodes) UpdateDrainBlockedCondition() error { +// tolerance UpdateBlockedCondition applies to unreadable nodeState — this is a +// NodeStateMalformed-adjacent concern, but has no dedicated user-visible signal of +// its own; a node dropping out of this aggregate silently is an accepted limitation +// rather than a deliberate design, tracked for follow-up. +func (s *skyhookNodes) UpdateDrainBlockedCondition(logger logr.Logger) { blocks := make([]wrapper.DrainBlockedNode, 0, len(s.nodes)) for _, node := range s.nodes { blocked, err := node.DrainBlocked() @@ -657,7 +659,15 @@ func (s *skyhookNodes) UpdateDrainBlockedCondition() error { if len(blocks) == 0 { wrapper.RemoveSkyhookConditionTypes(s.skyhook, wrapper.SkyhookConditionDrainBlocked) - return nil + return + } + + // The message builder truncates detail lines past drainBlockedDetailLineLimit and + // points the reader at controller logs for the rest, mirroring + // updateTaintToleranceCondition's log-before-truncate pattern — so log the same + // persisted set the message is built from here, not a caller-local subset. + if totalBlockedPods := countBlockedPods(blocks); totalBlockedPods > wrapper.ReadyConditionNodeListLimit { + logger.Info("DrainBlocked condition message truncated; full blocked set", "nodewright", s.skyhook.Name, "drainBlocks", blocks) } wrapper.AddSkyhookCondition(s.skyhook, metav1.Condition{ @@ -668,7 +678,16 @@ func (s *skyhookNodes) UpdateDrainBlockedCondition() error { Reason: wrapper.DrainBlockedConditionReason(blocks), Message: wrapper.DrainBlockedConditionMessage(blocks, len(s.nodes)), }) - return nil +} + +// countBlockedPods sums Blocked across every node, for deciding whether the DrainBlocked +// message will be truncated and the full set needs logging. +func countBlockedPods(blocks []wrapper.DrainBlockedNode) int { + total := 0 + for _, b := range blocks { + total += len(b.Blocked) + } + return total } // isPackageCompleteOnAllNodes reports whether the package has reached its diff --git a/operator/internal/controller/mock/SkyhookNodes.go b/operator/internal/controller/mock/SkyhookNodes.go index 9ed407441..e6b772eba 100644 --- a/operator/internal/controller/mock/SkyhookNodes.go +++ b/operator/internal/controller/mock/SkyhookNodes.go @@ -1077,20 +1077,9 @@ func (_c *MockSkyhookNodes_UpdateCondition_Call) RunAndReturn(run func(logger lo } // UpdateDrainBlockedCondition provides a mock function for the type MockSkyhookNodes -func (_mock *MockSkyhookNodes) UpdateDrainBlockedCondition() error { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for UpdateDrainBlockedCondition") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func() error); ok { - r0 = returnFunc() - } else { - r0 = ret.Error(0) - } - return r0 +func (_mock *MockSkyhookNodes) UpdateDrainBlockedCondition(logger logr.Logger) { + _mock.Called(logger) + return } // MockSkyhookNodes_UpdateDrainBlockedCondition_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateDrainBlockedCondition' @@ -1099,24 +1088,31 @@ type MockSkyhookNodes_UpdateDrainBlockedCondition_Call struct { } // UpdateDrainBlockedCondition is a helper method to define mock.On call -func (_e *MockSkyhookNodes_Expecter) UpdateDrainBlockedCondition() *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { - return &MockSkyhookNodes_UpdateDrainBlockedCondition_Call{Call: _e.mock.On("UpdateDrainBlockedCondition")} +// - logger logr.Logger +func (_e *MockSkyhookNodes_Expecter) UpdateDrainBlockedCondition(logger any) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + return &MockSkyhookNodes_UpdateDrainBlockedCondition_Call{Call: _e.mock.On("UpdateDrainBlockedCondition", logger)} } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Run(run func()) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Run(run func(logger logr.Logger)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { _c.Call.Run(func(args mock.Arguments) { - run() + var arg0 logr.Logger + if args[0] != nil { + arg0 = args[0].(logr.Logger) + } + run( + arg0, + ) }) return _c } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Return(err error) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { - _c.Call.Return(err) +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Return() *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + _c.Call.Return() return _c } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) RunAndReturn(run func() error) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { - _c.Call.Return(run) +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) RunAndReturn(run func(logr.Logger)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + _c.Run(run) return _c } diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go index ed89705e7..062ba3ee2 100644 --- a/operator/internal/controller/skyhook_controller.go +++ b/operator/internal/controller/skyhook_controller.go @@ -609,9 +609,7 @@ func (r *SkyhookReconciler) refreshSkyhookConditions(ctx context.Context, cluste // below maintains — same name prefix, different condition type. Rebuilt from // persisted per-node state so it stays correct on paused/disabled/complete/error/ // serial-partial passes; see cluster_state_v2.go's UpdateDrainBlockedCondition. - if err := skyhook.UpdateDrainBlockedCondition(); err != nil { - return fmt.Errorf("error rebuilding drain blocked condition: %w", err) - } + skyhook.UpdateDrainBlockedCondition(log.FromContext(ctx)) if err := r.updateDrainBlockedCondition(ctx, skyhook); err != nil { return fmt.Errorf("error updating drain blocked condition: %w", err) } @@ -1456,7 +1454,6 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState } selectedNode := nodePicker.SelectNodes(skyhook) - drainBlocks := make([]wrapper.DrainBlockedNode, 0) serialStop := false for _, node := range selectedNode { @@ -1470,6 +1467,18 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState continue } + // A node with no runnable, interrupt-requiring package this pass will never reach + // EnsureNodeIsReadyForInterrupt below, so nothing will refresh — or clear — its + // persisted drain-blocker annotation this pass. Clear it proactively: a package + // whose interrupt requirement was removed, or a node that has simply finished + // draining, must not leave DrainBlocked reporting a blocker with no bearing on + // this node's current state. + if !nodeNeedsInterruptDrain(ctx, node) { + if err := node.SetDrainBlocked(nil); err != nil { + return nil, fmt.Errorf("clearing stale drain blocked state for node [%s]: %w", node.GetNode().Name, err) + } + } + toRun, err := node.RunNext() if err != nil { return nil, fmt.Errorf("error getting next packages to run: %w", err) @@ -1511,9 +1520,17 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState for _, f := range toRun { - ok, err := r.ProcessInterrupt(ctx, node, f, interrupt, interrupt != nil && f.Name == pack, &drainBlocks) + ok, err := r.ProcessInterrupt(ctx, node, f, interrupt, interrupt != nil && f.Name == pack) if err != nil { - // TODO: error handle + // ProcessInterrupt may have already mutated node.Annotations in memory via + // SetDrainBlocked even though it is now returning an error (e.g. one pod's + // PDB rejection classified correctly, another pod's Delete genuinely failed). + // Save now so that in-memory mutation reaches the apiserver instead of being + // silently discarded when this function returns early and the next pass + // rebuilds cluster state fresh, with no trace of what this pass learned. + if _, saveErrs := r.SaveNodesAndSkyhook(ctx, clusterState, skyhook); len(saveErrs) > 0 { + return nil, utilerrors.NewAggregate(append(saveErrs, fmt.Errorf("error processing if we should interrupt [%s:%s]: %w", f.Name, f.Version, err))) + } return nil, fmt.Errorf("error processing if we should interrupt [%s:%s]: %w", f.Name, f.Version, err) } if !ok { @@ -1523,6 +1540,9 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState err = r.ApplyPackage(ctx, logger, clusterState, node, f, interrupt != nil && f.Name == pack) if err != nil { + if _, saveErrs := r.SaveNodesAndSkyhook(ctx, clusterState, skyhook); len(saveErrs) > 0 { + return nil, utilerrors.NewAggregate(append(saveErrs, fmt.Errorf("error applying package [%s:%s]: %w", f.Name, f.Version, err))) + } return nil, fmt.Errorf("error applying package [%s:%s]: %w", f.Name, f.Version, err) } @@ -1538,21 +1558,13 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState } } - // The message builder truncates detail lines past drainBlockedDetailLineLimit and points - // the reader at controller logs for the rest — so log the full set here, mirroring - // updateTaintToleranceCondition's log-before-truncate pattern, or that pointer is a lie. - // drainBlocks is only this pass's local findings, used for this diagnostic log line; - // the condition itself is rebuilt from persisted per-node state (see below), which is - // what makes it level-triggered rather than dependent on reaching this line. - if totalBlockedPods := countBlockedPods(drainBlocks); totalBlockedPods > wrapper.ReadyConditionNodeListLimit { - logger.Info("DrainBlocked condition message truncated; full blocked set", "nodewright", skyhook.GetSkyhook().Name, "drainBlocks", drainBlocks) - } // Re-run here too (also done unconditionally in refreshSkyhookConditions) so the // condition reflects this pass's fresh findings immediately rather than waiting one - // more reconcile for refreshSkyhookConditions to pick them up. - if err := skyhook.UpdateDrainBlockedCondition(); err != nil { - return &ctrl.Result{}, fmt.Errorf("error rebuilding drain blocked condition: %w", err) - } + // more reconcile for refreshSkyhookConditions to pick them up. Rebuilt from persisted + // per-node state (see UpdateDrainBlockedCondition), including its own truncation-log + // line, rather than from a local slice — that is what keeps it correct for nodes this + // pass skipped or never reached. + skyhook.UpdateDrainBlockedCondition(logger) saved, errs := r.SaveNodesAndSkyhook(ctx, clusterState, skyhook) if len(errs) > 0 { @@ -3374,7 +3386,7 @@ func (r *SkyhookReconciler) InvalidPackage(ctx context.Context, obj client.Objec // ProcessInterrupt will check and do the interrupt if need, and returns // false means we are waiting // true means we are good to proceed -func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package, interrupt *v1alpha1.Interrupt, runInterrupt bool, drainBlocks *[]wrapper.DrainBlockedNode) (bool, error) { +func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package, interrupt *v1alpha1.Interrupt, runInterrupt bool) (bool, error) { if !skyhookNode.HasInterrupt(*_package) { return true, nil @@ -3420,10 +3432,7 @@ func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wr // drain and cordon node before applying package that has an interrupt if stage == v1alpha1.StageApply || stage == v1alpha1.StageUninstall { - ready, blocked, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, _package) - if len(blocked) > 0 && drainBlocks != nil { - mergeDrainBlockedNode(drainBlocks, skyhookNode.GetNode().Name, blocked) - } + ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, _package) if err != nil { return false, err } @@ -3471,54 +3480,7 @@ func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wr return true, nil } -// mergeDrainBlockedNode records blocked pods for a node in drainBlocks, merging into an -// existing entry for that node rather than appending a duplicate. ProcessInterrupt runs once -// per runnable package on a node in a single reconcile pass, so a node with two interrupt- -// bearing packages must contribute to one DrainBlockedNode entry, not two. -func mergeDrainBlockedNode(drainBlocks *[]wrapper.DrainBlockedNode, nodeName string, blocked []drain.BlockedPod) { - for i := range *drainBlocks { - if (*drainBlocks)[i].NodeName == nodeName { - (*drainBlocks)[i].Blocked = appendUniqueBlockedPods((*drainBlocks)[i].Blocked, blocked) - return - } - } - *drainBlocks = append(*drainBlocks, wrapper.DrainBlockedNode{ - NodeName: nodeName, - Blocked: appendUniqueBlockedPods(nil, blocked), - }) -} - -// appendUniqueBlockedPods appends new to existing, skipping any (namespace, name, reason) already -// present. DrainNode reclassifies every pod on the node on each call, so ProcessInterrupt calling -// it once per interrupt-bearing package can otherwise report the same blocker more than once, -// wasting the message builder's per-node detail-line budget on duplicates. -func appendUniqueBlockedPods(existing, new []drain.BlockedPod) []drain.BlockedPod { - seen := make(map[[3]string]struct{}, len(existing)) - for _, b := range existing { - seen[[3]string{b.Namespace, b.Name, string(b.Reason)}] = struct{}{} - } - for _, b := range new { - key := [3]string{b.Namespace, b.Name, string(b.Reason)} - if _, dup := seen[key]; dup { - continue - } - seen[key] = struct{}{} - existing = append(existing, b) - } - return existing -} - -// countBlockedPods sums Blocked across every node, for deciding whether the DrainBlocked -// message will be truncated and the full set needs logging. -func countBlockedPods(blocks []wrapper.DrainBlockedNode) int { - total := 0 - for _, b := range blocks { - total += len(b.Blocked) - } - return total -} - -func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (bool, []drain.BlockedPod, error) { +func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, skyhookNode wrapper.SkyhookNode, _package *v1alpha1.Package) (bool, error) { // Cordon is an in-memory mutation; SaveNodesAndSkyhook patches it at the end of this // pass, after every selected node has been visited. Draining in the same pass that // first cordons the node would evict while spec.unschedulable is still only local, so @@ -3528,12 +3490,12 @@ func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, s // This costs one pass per drain cycle, not one per node: the caller's loop keeps going // after a false return, so a single pass still cordons every node it selected. if skyhookNode.Cordon() { - return false, nil, nil + return false, nil } hasWork, podNames, err := r.HasNonInterruptWork(ctx, skyhookNode) if err != nil { - return false, nil, err + return false, err } if hasWork { // keep waiting... displayPods := podNames @@ -3552,21 +3514,31 @@ func (r *SkyhookReconciler) EnsureNodeIsReadyForInterrupt(ctx context.Context, s _package.Version, skyhookNode.GetSkyhook().Name, ) - return false, nil, nil + // We have not reached DrainNode this pass, so we don't know whether any + // previously-recorded PDB/unmanaged/emptyDir blockers still apply. A stale + // blocker naming a pod that no longer holds the drain is worse than reporting + // none — non-interrupt work has its own accurate signal in + // updateDrainBlockedCondition's Blocked/NonInterruptPodsRunning condition. + if err := skyhookNode.SetDrainBlocked(nil); err != nil { + return false, err + } + return false, nil } result, err := r.DrainNode(ctx, skyhookNode, _package) // Persist regardless of err: this is what makes DrainBlocked level-triggered rather // than dependent on this pass reaching UpdateDrainBlockedCondition later. See - // SetDrainBlocked's doc comment. + // SetDrainBlocked's doc comment. The mutation only reaches the apiserver once + // SaveNodesAndSkyhook runs — see RunSkyhookPackages' error-path handling for why + // callers here must not simply return before that happens. if setErr := skyhookNode.SetDrainBlocked(result.Blocked); setErr != nil && err == nil { err = setErr } if err != nil { - return false, result.Blocked, fmt.Errorf("error draining node [%s]: %w", skyhookNode.GetNode().Name, err) + return false, fmt.Errorf("error draining node [%s]: %w", skyhookNode.GetNode().Name, err) } - return result.Ready, result.Blocked, nil + return result.Ready, nil } // ApplyPackage starts a pod on node for the package diff --git a/operator/internal/controller/skyhook_controller_test.go b/operator/internal/controller/skyhook_controller_test.go index 4ecdc35f5..e7d4c8160 100644 --- a/operator/internal/controller/skyhook_controller_test.go +++ b/operator/internal/controller/skyhook_controller_test.go @@ -30,7 +30,6 @@ import ( skyhookNodesMock "github.com/NVIDIA/nodewright/operator/internal/controller/mock" "github.com/NVIDIA/nodewright/operator/internal/dal" dalMock "github.com/NVIDIA/nodewright/operator/internal/dal/mock" - "github.com/NVIDIA/nodewright/operator/internal/drain" "github.com/NVIDIA/nodewright/operator/internal/wrapper" wrapperMock "github.com/NVIDIA/nodewright/operator/internal/wrapper/mock" . "github.com/onsi/ginkgo/v2" @@ -836,7 +835,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -853,14 +852,14 @@ var _ = Describe("skyhook controller tests", func() { Expect(testClient.Delete(ctx, goldenPod)).To(Succeed()) - ready, _, err = r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, err = r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) Expect(ready).To(BeFalse(), "evictable pod is still being drained") Expect(deleteCalled).To(BeTrue(), "DrainNode should delete evictable pod") - ready, _, err = r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, err = r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -926,7 +925,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -1216,7 +1215,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}}, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -1253,7 +1252,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(cordoned, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -1281,7 +1280,7 @@ var _ = Describe("skyhook controller tests", func() { skyhookNode, err := wrapper.NewSkyhookNode(&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}}, skyhook) Expect(err).ToNot(HaveOccurred()) - ready, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, }) Expect(err).ToNot(HaveOccurred()) @@ -1294,23 +1293,6 @@ var _ = Describe("skyhook controller tests", func() { }) }) - Describe("mergeDrainBlockedNode", func() { - It("merges blocked pods into an existing node entry instead of duplicating it", func() { - drainBlocks := []wrapper.DrainBlockedNode{} - mergeDrainBlockedNode(&drainBlocks, "node-a", []drain.BlockedPod{{Name: "pod-1"}}) - mergeDrainBlockedNode(&drainBlocks, "node-a", []drain.BlockedPod{{Name: "pod-2"}}) - Expect(drainBlocks).To(HaveLen(1)) - Expect(drainBlocks[0].Blocked).To(HaveLen(2)) - }) - - It("creates separate entries for different nodes", func() { - drainBlocks := []wrapper.DrainBlockedNode{} - mergeDrainBlockedNode(&drainBlocks, "node-a", []drain.BlockedPod{{Name: "pod-1"}}) - mergeDrainBlockedNode(&drainBlocks, "node-b", []drain.BlockedPod{{Name: "pod-2"}}) - Expect(drainBlocks).To(HaveLen(2)) - }) - }) - It("should set monotonic SKYHOOK_NODE_ORDER across nodes and batches", func() { now := time.Now() testSkyhook := wrapper.NewSkyhookWrapper(&v1alpha1.NodeWright{ @@ -4568,7 +4550,7 @@ var _ = Describe("ProcessInterrupt skipped-package promotion", func() { pkg := sn.GetSkyhook().Spec.Packages["baxter"] r := &SkyhookReconciler{} - proceed, err := r.ProcessInterrupt(context.Background(), sn, &pkg, pkg.Interrupt, true, nil) + proceed, err := r.ProcessInterrupt(context.Background(), sn, &pkg, pkg.Interrupt, true) Expect(err).NotTo(HaveOccurred()) Expect(proceed).To(BeFalse()) @@ -4583,7 +4565,7 @@ var _ = Describe("ProcessInterrupt skipped-package promotion", func() { pkg := sn.GetSkyhook().Spec.Packages["baxter"] r := &SkyhookReconciler{} - proceed, err := r.ProcessInterrupt(context.Background(), sn, &pkg, pkg.Interrupt, false, nil) + proceed, err := r.ProcessInterrupt(context.Background(), sn, &pkg, pkg.Interrupt, false) Expect(err).NotTo(HaveOccurred()) Expect(proceed).To(BeFalse()) @@ -5385,12 +5367,12 @@ var _ = Describe("drain blocked by non-interrupt pods multi-node reconcile", fun Expect(nodeWrapperA).ToNot(BeNil()) Expect(nodeWrapperB).ToNot(BeNil()) - readyA, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperA, &pkg) + readyA, err := r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperA, &pkg) Expect(err).ToNot(HaveOccurred()) Expect(readyA).To(BeFalse()) Eventually(recorder.Events).Should(Receive(ContainSubstring(fmt.Sprintf("Warning Drain drain blocked by non-interrupt pods [default/%s] for package [pkg:1.0.0] from [nodewright:%s]", podName, skyhookName)))) - readyB, _, err := r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperB, &pkg) + readyB, err := r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperB, &pkg) Expect(err).ToNot(HaveOccurred()) Expect(readyB).To(BeTrue(), "node-b has no non-interrupt work and should be ready for interrupt") @@ -5437,7 +5419,7 @@ var _ = Describe("drain blocked by non-interrupt pods multi-node reconcile", fun cond = meta.FindStatusCondition(sn.GetSkyhook().Status.Conditions, wrapper.SkyhookConditionBlocked) Expect(cond).To(BeNil(), "Blocked condition must be removed when no nodes are blocked") - readyA, _, err = r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperA, &pkg) + readyA, err = r.EnsureNodeIsReadyForInterrupt(ctx, nodeWrapperA, &pkg) Expect(err).ToNot(HaveOccurred()) Expect(readyA).To(BeTrue(), "node-a should be ready for interrupt once non-interrupt pods are gone") }) diff --git a/operator/internal/drain/drain.go b/operator/internal/drain/drain.go index 00a43cde0..7e5b3d7b0 100644 --- a/operator/internal/drain/drain.go +++ b/operator/internal/drain/drain.go @@ -78,12 +78,12 @@ const ( // BlockedPod is one pod currently preventing a node's drain from completing, // with enough context to render a DrainBlocked condition message. type BlockedPod struct { - Namespace string - Name string - Reason BlockReason + Namespace string `json:"namespace"` + Name string `json:"name"` + Reason BlockReason `json:"reason"` // Detail is apiserver-generated prose (e.g. the PDB cause message) and is // copied verbatim — it is not a stable contract, so never parsed. - Detail string + Detail string `json:"detail,omitempty"` } // DrainResult is what DrainNode reports back: whether the node is fully diff --git a/operator/internal/wrapper/skyhook_conditions.go b/operator/internal/wrapper/skyhook_conditions.go index ee293a1c3..9413e2b40 100644 --- a/operator/internal/wrapper/skyhook_conditions.go +++ b/operator/internal/wrapper/skyhook_conditions.go @@ -46,10 +46,7 @@ const ( SkyhookConditionDeletionBlocked = "DeletionBlocked" SkyhookConditionDrainBlocked = "DrainBlocked" - drainBlockedReasonPDB = "PodDisruptionBudget" - drainBlockedReasonUnmanagedPod = "UnmanagedPod" - drainBlockedReasonEmptyDir = "EmptyDirData" - drainBlockedReasonMultiple = "MultipleCauses" + drainBlockedReasonMultiple = "MultipleCauses" SkyhookReasonNonInterruptPodsRunning = "NonInterruptPodsRunning" @@ -352,15 +349,13 @@ func DrainBlockedConditionReason(nodes []DrainBlockedNode) string { if len(seen) != 1 { return drainBlockedReasonMultiple } + // Exactly one reason observed: return it directly rather than hand-mapping + // against drain.BlockReason's constants. The old switch fell through to + // MultipleCauses for any drain.BlockReason it didn't explicitly list — so a + // new reason added to that type would silently mislabel a single-cause block + // as "more than one kind of blocker present," with no compiler error to catch it. for reason := range seen { - switch reason { - case drain.BlockReasonPodDisruptionBudget: - return drainBlockedReasonPDB - case drain.BlockReasonUnmanagedPod: - return drainBlockedReasonUnmanagedPod - case drain.BlockReasonEmptyDirData: - return drainBlockedReasonEmptyDir - } + return string(reason) } return drainBlockedReasonMultiple } From ed0728dcf426b0ef11a064cc3653cc10a26704ec Mon Sep 17 00:00:00 2001 From: mohityadav8 Date: Tue, 29 Sep 2026 00:27:40 +0530 Subject: [PATCH 8/9] refactor: reduce RunSkyhookPackages complexity; restore mock license headers Signed-off-by: mohityadav8 --- .../internal/controller/mock/SkyhookNodes.go | 18 ++++ .../internal/controller/skyhook_controller.go | 52 +++++----- operator/internal/dal/mock/DAL.go | 87 ++++++++-------- operator/internal/mocks/client/Client.go | 99 ++++++++++--------- operator/internal/mocks/dynamic/Interface.go | 31 +++--- .../dynamic/NamespaceableResourceInterface.go | 89 +++++++++-------- .../mocks/dynamic/ResourceInterface.go | 87 ++++++++-------- .../internal/mocks/record/EventRecorder.go | 43 ++++---- .../workqueue/TypedRateLimitingInterface.go | 41 +++++--- operator/internal/wrapper/mock/SkyhookNode.go | 18 ++++ .../internal/wrapper/mock/SkyhookNodeOnly.go | 18 ++++ 11 files changed, 354 insertions(+), 229 deletions(-) diff --git a/operator/internal/controller/mock/SkyhookNodes.go b/operator/internal/controller/mock/SkyhookNodes.go index e6b772eba..70e1e5870 100644 --- a/operator/internal/controller/mock/SkyhookNodes.go +++ b/operator/internal/controller/mock/SkyhookNodes.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go index 062ba3ee2..36b5521a6 100644 --- a/operator/internal/controller/skyhook_controller.go +++ b/operator/internal/controller/skyhook_controller.go @@ -1420,6 +1420,29 @@ func (r *SkyhookReconciler) TrackReboots(ctx context.Context, clusterState *clus return updates, utilerrors.NewAggregate(errs) } +// saveThenWrap persists any in-memory node mutations (for example SetDrainBlocked) before an +// early error return, so they reach the apiserver instead of being dropped when the next +// pass rebuilds cluster state. Save failures are aggregated with the original error. +func (r *SkyhookReconciler) saveThenWrap(ctx context.Context, clusterState *clusterState, skyhook SkyhookNodes, err error) error { + if _, saveErrs := r.SaveNodesAndSkyhook(ctx, clusterState, skyhook); len(saveErrs) > 0 { + return utilerrors.NewAggregate(append(saveErrs, err)) + } + return err +} + +// filterApplicablePackages drops packages whose uninstall is in progress or already +// completed on this node. See shouldSkipApplyForUninstall for the exact rule. +func filterApplicablePackages(toRun []*v1alpha1.Package, nodeState v1alpha1.NodeState, beingDeleted bool) []*v1alpha1.Package { + filtered := make([]*v1alpha1.Package, 0, len(toRun)) + for _, pkg := range toRun { + if shouldSkipApplyForUninstall(pkg, nodeState, beingDeleted) { + continue + } + filtered = append(filtered, pkg) + } + return filtered +} + // RunSkyhookPackages runs all skyhook packages then saves and requeues if changes were made func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState *clusterState, nodePicker *NodePicker, skyhook SkyhookNodes) (*ctrl.Result, error) { @@ -1501,14 +1524,7 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState return nil, fmt.Errorf("node %s: reading state while filtering runnable packages: %w", node.GetNode().Name, err) } - filtered := make([]*v1alpha1.Package, 0, len(toRun)) - for _, pkg := range toRun { - if shouldSkipApplyForUninstall(pkg, nodeState, beingDeleted) { - continue - } - filtered = append(filtered, pkg) - } - toRun = filtered + toRun = filterApplicablePackages(toRun, nodeState, beingDeleted) // prepend the uninstall packages so they are ran first. // filterUninstallForNode drops entries that aren't in this node's @@ -1522,16 +1538,10 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState ok, err := r.ProcessInterrupt(ctx, node, f, interrupt, interrupt != nil && f.Name == pack) if err != nil { - // ProcessInterrupt may have already mutated node.Annotations in memory via - // SetDrainBlocked even though it is now returning an error (e.g. one pod's - // PDB rejection classified correctly, another pod's Delete genuinely failed). - // Save now so that in-memory mutation reaches the apiserver instead of being - // silently discarded when this function returns early and the next pass - // rebuilds cluster state fresh, with no trace of what this pass learned. - if _, saveErrs := r.SaveNodesAndSkyhook(ctx, clusterState, skyhook); len(saveErrs) > 0 { - return nil, utilerrors.NewAggregate(append(saveErrs, fmt.Errorf("error processing if we should interrupt [%s:%s]: %w", f.Name, f.Version, err))) - } - return nil, fmt.Errorf("error processing if we should interrupt [%s:%s]: %w", f.Name, f.Version, err) + // ProcessInterrupt may have already recorded drain blockers in memory before + // erroring; saveThenWrap persists them so they are not lost. + return nil, r.saveThenWrap(ctx, clusterState, skyhook, + fmt.Errorf("error processing if we should interrupt [%s:%s]: %w", f.Name, f.Version, err)) } if !ok { requeue = true @@ -1540,10 +1550,8 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState err = r.ApplyPackage(ctx, logger, clusterState, node, f, interrupt != nil && f.Name == pack) if err != nil { - if _, saveErrs := r.SaveNodesAndSkyhook(ctx, clusterState, skyhook); len(saveErrs) > 0 { - return nil, utilerrors.NewAggregate(append(saveErrs, fmt.Errorf("error applying package [%s:%s]: %w", f.Name, f.Version, err))) - } - return nil, fmt.Errorf("error applying package [%s:%s]: %w", f.Name, f.Version, err) + return nil, r.saveThenWrap(ctx, clusterState, skyhook, + fmt.Errorf("error applying package [%s:%s]: %w", f.Name, f.Version, err)) } // process one package at a time diff --git a/operator/internal/dal/mock/DAL.go b/operator/internal/dal/mock/DAL.go index e4120be58..ef169dc87 100644 --- a/operator/internal/dal/mock/DAL.go +++ b/operator/internal/dal/mock/DAL.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -20,19 +38,10 @@ func NewMockDAL(t interface { mock.TestingT Cleanup(func()) }) *MockDAL { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock := &MockDAL{} mock.Mock.Test(t) - t.Cleanup(func() { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock.AssertExpectations(t) - }) + t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } @@ -53,11 +62,11 @@ func (_m *MockDAL) EXPECT() *MockDAL_Expecter { // GetDeploymentPolicies provides a mock function for the type MockDAL func (_mock *MockDAL) GetDeploymentPolicies(ctx context.Context, opts ...client.ListOption) (*v1alpha1.DeploymentPolicyList, error) { // client.ListOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -94,9 +103,9 @@ type MockDAL_GetDeploymentPolicies_Call struct { // GetDeploymentPolicies is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetDeploymentPolicies(ctx any, opts ...any) *MockDAL_GetDeploymentPolicies_Call { +func (_e *MockDAL_Expecter) GetDeploymentPolicies(ctx interface{}, opts ...interface{}) *MockDAL_GetDeploymentPolicies_Call { return &MockDAL_GetDeploymentPolicies_Call{Call: _e.mock.On("GetDeploymentPolicies", - append([]any{ctx}, opts...)...)} + append([]interface{}{ctx}, opts...)...)} } func (_c *MockDAL_GetDeploymentPolicies_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetDeploymentPolicies_Call { @@ -167,7 +176,7 @@ type MockDAL_GetDeploymentPolicy_Call struct { // GetDeploymentPolicy is a helper method to define mock.On call // - ctx context.Context // - name string -func (_e *MockDAL_Expecter) GetDeploymentPolicy(ctx any, name any) *MockDAL_GetDeploymentPolicy_Call { +func (_e *MockDAL_Expecter) GetDeploymentPolicy(ctx interface{}, name interface{}) *MockDAL_GetDeploymentPolicy_Call { return &MockDAL_GetDeploymentPolicy_Call{Call: _e.mock.On("GetDeploymentPolicy", ctx, name)} } @@ -236,7 +245,7 @@ type MockDAL_GetJob_Call struct { // - ctx context.Context // - namespace string // - name string -func (_e *MockDAL_Expecter) GetJob(ctx any, namespace any, name any) *MockDAL_GetJob_Call { +func (_e *MockDAL_Expecter) GetJob(ctx interface{}, namespace interface{}, name interface{}) *MockDAL_GetJob_Call { return &MockDAL_GetJob_Call{Call: _e.mock.On("GetJob", ctx, namespace, name)} } @@ -276,11 +285,11 @@ func (_c *MockDAL_GetJob_Call) RunAndReturn(run func(ctx context.Context, namesp // GetJobs provides a mock function for the type MockDAL func (_mock *MockDAL) GetJobs(ctx context.Context, opts ...client.ListOption) (*v1.JobList, error) { // client.ListOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -317,9 +326,9 @@ type MockDAL_GetJobs_Call struct { // GetJobs is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetJobs(ctx any, opts ...any) *MockDAL_GetJobs_Call { +func (_e *MockDAL_Expecter) GetJobs(ctx interface{}, opts ...interface{}) *MockDAL_GetJobs_Call { return &MockDAL_GetJobs_Call{Call: _e.mock.On("GetJobs", - append([]any{ctx}, opts...)...)} + append([]interface{}{ctx}, opts...)...)} } func (_c *MockDAL_GetJobs_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetJobs_Call { @@ -390,7 +399,7 @@ type MockDAL_GetNode_Call struct { // GetNode is a helper method to define mock.On call // - ctx context.Context // - nodeName string -func (_e *MockDAL_Expecter) GetNode(ctx any, nodeName any) *MockDAL_GetNode_Call { +func (_e *MockDAL_Expecter) GetNode(ctx interface{}, nodeName interface{}) *MockDAL_GetNode_Call { return &MockDAL_GetNode_Call{Call: _e.mock.On("GetNode", ctx, nodeName)} } @@ -425,11 +434,11 @@ func (_c *MockDAL_GetNode_Call) RunAndReturn(run func(ctx context.Context, nodeN // GetNodes provides a mock function for the type MockDAL func (_mock *MockDAL) GetNodes(ctx context.Context, opts ...client.ListOption) (*v10.NodeList, error) { // client.ListOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -466,9 +475,9 @@ type MockDAL_GetNodes_Call struct { // GetNodes is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetNodes(ctx any, opts ...any) *MockDAL_GetNodes_Call { +func (_e *MockDAL_Expecter) GetNodes(ctx interface{}, opts ...interface{}) *MockDAL_GetNodes_Call { return &MockDAL_GetNodes_Call{Call: _e.mock.On("GetNodes", - append([]any{ctx}, opts...)...)} + append([]interface{}{ctx}, opts...)...)} } func (_c *MockDAL_GetNodes_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetNodes_Call { @@ -540,7 +549,7 @@ type MockDAL_GetPod_Call struct { // - ctx context.Context // - namespace string // - name string -func (_e *MockDAL_Expecter) GetPod(ctx any, namespace any, name any) *MockDAL_GetPod_Call { +func (_e *MockDAL_Expecter) GetPod(ctx interface{}, namespace interface{}, name interface{}) *MockDAL_GetPod_Call { return &MockDAL_GetPod_Call{Call: _e.mock.On("GetPod", ctx, namespace, name)} } @@ -614,7 +623,7 @@ type MockDAL_GetPodLogTail_Call struct { // - pod string // - container string // - maxBytes int64 -func (_e *MockDAL_Expecter) GetPodLogTail(ctx any, namespace any, pod any, container any, maxBytes any) *MockDAL_GetPodLogTail_Call { +func (_e *MockDAL_Expecter) GetPodLogTail(ctx interface{}, namespace interface{}, pod interface{}, container interface{}, maxBytes interface{}) *MockDAL_GetPodLogTail_Call { return &MockDAL_GetPodLogTail_Call{Call: _e.mock.On("GetPodLogTail", ctx, namespace, pod, container, maxBytes)} } @@ -664,11 +673,11 @@ func (_c *MockDAL_GetPodLogTail_Call) RunAndReturn(run func(ctx context.Context, // GetPods provides a mock function for the type MockDAL func (_mock *MockDAL) GetPods(ctx context.Context, opts ...client.ListOption) (*v10.PodList, error) { // client.ListOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -705,9 +714,9 @@ type MockDAL_GetPods_Call struct { // GetPods is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetPods(ctx any, opts ...any) *MockDAL_GetPods_Call { +func (_e *MockDAL_Expecter) GetPods(ctx interface{}, opts ...interface{}) *MockDAL_GetPods_Call { return &MockDAL_GetPods_Call{Call: _e.mock.On("GetPods", - append([]any{ctx}, opts...)...)} + append([]interface{}{ctx}, opts...)...)} } func (_c *MockDAL_GetPods_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetPods_Call { @@ -745,11 +754,11 @@ func (_c *MockDAL_GetPods_Call) RunAndReturn(run func(ctx context.Context, opts // GetSkyhook provides a mock function for the type MockDAL func (_mock *MockDAL) GetSkyhook(ctx context.Context, name string, opts ...client.ListOption) (*v1alpha1.NodeWright, error) { // client.ListOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, name) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -787,9 +796,9 @@ type MockDAL_GetSkyhook_Call struct { // - ctx context.Context // - name string // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetSkyhook(ctx any, name any, opts ...any) *MockDAL_GetSkyhook_Call { +func (_e *MockDAL_Expecter) GetSkyhook(ctx interface{}, name interface{}, opts ...interface{}) *MockDAL_GetSkyhook_Call { return &MockDAL_GetSkyhook_Call{Call: _e.mock.On("GetSkyhook", - append([]any{ctx, name}, opts...)...)} + append([]interface{}{ctx, name}, opts...)...)} } func (_c *MockDAL_GetSkyhook_Call) Run(run func(ctx context.Context, name string, opts ...client.ListOption)) *MockDAL_GetSkyhook_Call { @@ -832,11 +841,11 @@ func (_c *MockDAL_GetSkyhook_Call) RunAndReturn(run func(ctx context.Context, na // GetSkyhooks provides a mock function for the type MockDAL func (_mock *MockDAL) GetSkyhooks(ctx context.Context, opts ...client.ListOption) (*v1alpha1.NodeWrightList, error) { // client.ListOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -873,9 +882,9 @@ type MockDAL_GetSkyhooks_Call struct { // GetSkyhooks is a helper method to define mock.On call // - ctx context.Context // - opts ...client.ListOption -func (_e *MockDAL_Expecter) GetSkyhooks(ctx any, opts ...any) *MockDAL_GetSkyhooks_Call { +func (_e *MockDAL_Expecter) GetSkyhooks(ctx interface{}, opts ...interface{}) *MockDAL_GetSkyhooks_Call { return &MockDAL_GetSkyhooks_Call{Call: _e.mock.On("GetSkyhooks", - append([]any{ctx}, opts...)...)} + append([]interface{}{ctx}, opts...)...)} } func (_c *MockDAL_GetSkyhooks_Call) Run(run func(ctx context.Context, opts ...client.ListOption)) *MockDAL_GetSkyhooks_Call { diff --git a/operator/internal/mocks/client/Client.go b/operator/internal/mocks/client/Client.go index 573b14b68..d323ac985 100644 --- a/operator/internal/mocks/client/Client.go +++ b/operator/internal/mocks/client/Client.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -20,19 +38,10 @@ func NewClient(t interface { mock.TestingT Cleanup(func()) }) *Client { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock := &Client{} mock.Mock.Test(t) - t.Cleanup(func() { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock.AssertExpectations(t) - }) + t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } @@ -53,11 +62,11 @@ func (_m *Client) EXPECT() *Client_Expecter { // Apply provides a mock function for the type Client func (_mock *Client) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { // client.ApplyOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -84,9 +93,9 @@ type Client_Apply_Call struct { // - ctx context.Context // - obj runtime.ApplyConfiguration // - opts ...client.ApplyOption -func (_e *Client_Expecter) Apply(ctx any, obj any, opts ...any) *Client_Apply_Call { +func (_e *Client_Expecter) Apply(ctx interface{}, obj interface{}, opts ...interface{}) *Client_Apply_Call { return &Client_Apply_Call{Call: _e.mock.On("Apply", - append([]any{ctx, obj}, opts...)...)} + append([]interface{}{ctx, obj}, opts...)...)} } func (_c *Client_Apply_Call) Run(run func(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption)) *Client_Apply_Call { @@ -129,11 +138,11 @@ func (_c *Client_Apply_Call) RunAndReturn(run func(ctx context.Context, obj runt // Create provides a mock function for the type Client func (_mock *Client) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { // client.CreateOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -160,9 +169,9 @@ type Client_Create_Call struct { // - ctx context.Context // - obj client.Object // - opts ...client.CreateOption -func (_e *Client_Expecter) Create(ctx any, obj any, opts ...any) *Client_Create_Call { +func (_e *Client_Expecter) Create(ctx interface{}, obj interface{}, opts ...interface{}) *Client_Create_Call { return &Client_Create_Call{Call: _e.mock.On("Create", - append([]any{ctx, obj}, opts...)...)} + append([]interface{}{ctx, obj}, opts...)...)} } func (_c *Client_Create_Call) Run(run func(ctx context.Context, obj client.Object, opts ...client.CreateOption)) *Client_Create_Call { @@ -205,11 +214,11 @@ func (_c *Client_Create_Call) RunAndReturn(run func(ctx context.Context, obj cli // Delete provides a mock function for the type Client func (_mock *Client) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { // client.DeleteOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -236,9 +245,9 @@ type Client_Delete_Call struct { // - ctx context.Context // - obj client.Object // - opts ...client.DeleteOption -func (_e *Client_Expecter) Delete(ctx any, obj any, opts ...any) *Client_Delete_Call { +func (_e *Client_Expecter) Delete(ctx interface{}, obj interface{}, opts ...interface{}) *Client_Delete_Call { return &Client_Delete_Call{Call: _e.mock.On("Delete", - append([]any{ctx, obj}, opts...)...)} + append([]interface{}{ctx, obj}, opts...)...)} } func (_c *Client_Delete_Call) Run(run func(ctx context.Context, obj client.Object, opts ...client.DeleteOption)) *Client_Delete_Call { @@ -281,11 +290,11 @@ func (_c *Client_Delete_Call) RunAndReturn(run func(ctx context.Context, obj cli // DeleteAllOf provides a mock function for the type Client func (_mock *Client) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { // client.DeleteAllOfOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -312,9 +321,9 @@ type Client_DeleteAllOf_Call struct { // - ctx context.Context // - obj client.Object // - opts ...client.DeleteAllOfOption -func (_e *Client_Expecter) DeleteAllOf(ctx any, obj any, opts ...any) *Client_DeleteAllOf_Call { +func (_e *Client_Expecter) DeleteAllOf(ctx interface{}, obj interface{}, opts ...interface{}) *Client_DeleteAllOf_Call { return &Client_DeleteAllOf_Call{Call: _e.mock.On("DeleteAllOf", - append([]any{ctx, obj}, opts...)...)} + append([]interface{}{ctx, obj}, opts...)...)} } func (_c *Client_DeleteAllOf_Call) Run(run func(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption)) *Client_DeleteAllOf_Call { @@ -357,11 +366,11 @@ func (_c *Client_DeleteAllOf_Call) RunAndReturn(run func(ctx context.Context, ob // Get provides a mock function for the type Client func (_mock *Client) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { // client.GetOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, key, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -389,9 +398,9 @@ type Client_Get_Call struct { // - key client.ObjectKey // - obj client.Object // - opts ...client.GetOption -func (_e *Client_Expecter) Get(ctx any, key any, obj any, opts ...any) *Client_Get_Call { +func (_e *Client_Expecter) Get(ctx interface{}, key interface{}, obj interface{}, opts ...interface{}) *Client_Get_Call { return &Client_Get_Call{Call: _e.mock.On("Get", - append([]any{ctx, key, obj}, opts...)...)} + append([]interface{}{ctx, key, obj}, opts...)...)} } func (_c *Client_Get_Call) Run(run func(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption)) *Client_Get_Call { @@ -469,7 +478,7 @@ type Client_GroupVersionKindFor_Call struct { // GroupVersionKindFor is a helper method to define mock.On call // - obj runtime.Object -func (_e *Client_Expecter) GroupVersionKindFor(obj any) *Client_GroupVersionKindFor_Call { +func (_e *Client_Expecter) GroupVersionKindFor(obj interface{}) *Client_GroupVersionKindFor_Call { return &Client_GroupVersionKindFor_Call{Call: _e.mock.On("GroupVersionKindFor", obj)} } @@ -529,7 +538,7 @@ type Client_IsObjectNamespaced_Call struct { // IsObjectNamespaced is a helper method to define mock.On call // - obj runtime.Object -func (_e *Client_Expecter) IsObjectNamespaced(obj any) *Client_IsObjectNamespaced_Call { +func (_e *Client_Expecter) IsObjectNamespaced(obj interface{}) *Client_IsObjectNamespaced_Call { return &Client_IsObjectNamespaced_Call{Call: _e.mock.On("IsObjectNamespaced", obj)} } @@ -559,11 +568,11 @@ func (_c *Client_IsObjectNamespaced_Call) RunAndReturn(run func(obj runtime.Obje // List provides a mock function for the type Client func (_mock *Client) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { // client.ListOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, list) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -590,9 +599,9 @@ type Client_List_Call struct { // - ctx context.Context // - list client.ObjectList // - opts ...client.ListOption -func (_e *Client_Expecter) List(ctx any, list any, opts ...any) *Client_List_Call { +func (_e *Client_Expecter) List(ctx interface{}, list interface{}, opts ...interface{}) *Client_List_Call { return &Client_List_Call{Call: _e.mock.On("List", - append([]any{ctx, list}, opts...)...)} + append([]interface{}{ctx, list}, opts...)...)} } func (_c *Client_List_Call) Run(run func(ctx context.Context, list client.ObjectList, opts ...client.ListOption)) *Client_List_Call { @@ -635,11 +644,11 @@ func (_c *Client_List_Call) RunAndReturn(run func(ctx context.Context, list clie // Patch provides a mock function for the type Client func (_mock *Client) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { // client.PatchOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj, patch) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -667,9 +676,9 @@ type Client_Patch_Call struct { // - obj client.Object // - patch client.Patch // - opts ...client.PatchOption -func (_e *Client_Expecter) Patch(ctx any, obj any, patch any, opts ...any) *Client_Patch_Call { +func (_e *Client_Expecter) Patch(ctx interface{}, obj interface{}, patch interface{}, opts ...interface{}) *Client_Patch_Call { return &Client_Patch_Call{Call: _e.mock.On("Patch", - append([]any{ctx, obj, patch}, opts...)...)} + append([]interface{}{ctx, obj, patch}, opts...)...)} } func (_c *Client_Patch_Call) Run(run func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption)) *Client_Patch_Call { @@ -878,7 +887,7 @@ type Client_SubResource_Call struct { // SubResource is a helper method to define mock.On call // - subResource string -func (_e *Client_Expecter) SubResource(subResource any) *Client_SubResource_Call { +func (_e *Client_Expecter) SubResource(subResource interface{}) *Client_SubResource_Call { return &Client_SubResource_Call{Call: _e.mock.On("SubResource", subResource)} } @@ -908,11 +917,11 @@ func (_c *Client_SubResource_Call) RunAndReturn(run func(subResource string) cli // Update provides a mock function for the type Client func (_mock *Client) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { // client.UpdateOption - _va := make([]any, len(opts)) + _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -939,9 +948,9 @@ type Client_Update_Call struct { // - ctx context.Context // - obj client.Object // - opts ...client.UpdateOption -func (_e *Client_Expecter) Update(ctx any, obj any, opts ...any) *Client_Update_Call { +func (_e *Client_Expecter) Update(ctx interface{}, obj interface{}, opts ...interface{}) *Client_Update_Call { return &Client_Update_Call{Call: _e.mock.On("Update", - append([]any{ctx, obj}, opts...)...)} + append([]interface{}{ctx, obj}, opts...)...)} } func (_c *Client_Update_Call) Run(run func(ctx context.Context, obj client.Object, opts ...client.UpdateOption)) *Client_Update_Call { diff --git a/operator/internal/mocks/dynamic/Interface.go b/operator/internal/mocks/dynamic/Interface.go index e760735d0..253ada8a1 100644 --- a/operator/internal/mocks/dynamic/Interface.go +++ b/operator/internal/mocks/dynamic/Interface.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -16,19 +34,10 @@ func NewInterface(t interface { mock.TestingT Cleanup(func()) }) *Interface { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock := &Interface{} mock.Mock.Test(t) - t.Cleanup(func() { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock.AssertExpectations(t) - }) + t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } @@ -72,7 +81,7 @@ type Interface_Resource_Call struct { // Resource is a helper method to define mock.On call // - resource schema.GroupVersionResource -func (_e *Interface_Expecter) Resource(resource any) *Interface_Resource_Call { +func (_e *Interface_Expecter) Resource(resource interface{}) *Interface_Resource_Call { return &Interface_Resource_Call{Call: _e.mock.On("Resource", resource)} } diff --git a/operator/internal/mocks/dynamic/NamespaceableResourceInterface.go b/operator/internal/mocks/dynamic/NamespaceableResourceInterface.go index 32b6007bc..914a85680 100644 --- a/operator/internal/mocks/dynamic/NamespaceableResourceInterface.go +++ b/operator/internal/mocks/dynamic/NamespaceableResourceInterface.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -21,19 +39,10 @@ func NewNamespaceableResourceInterface(t interface { mock.TestingT Cleanup(func()) }) *NamespaceableResourceInterface { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock := &NamespaceableResourceInterface{} mock.Mock.Test(t) - t.Cleanup(func() { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock.AssertExpectations(t) - }) + t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } @@ -54,11 +63,11 @@ func (_m *NamespaceableResourceInterface) EXPECT() *NamespaceableResourceInterfa // Apply provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Apply(ctx context.Context, name string, obj *unstructured.Unstructured, options v1.ApplyOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, name, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -98,9 +107,9 @@ type NamespaceableResourceInterface_Apply_Call struct { // - obj *unstructured.Unstructured // - options v1.ApplyOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Apply(ctx any, name any, obj any, options any, subresources ...any) *NamespaceableResourceInterface_Apply_Call { +func (_e *NamespaceableResourceInterface_Expecter) Apply(ctx interface{}, name interface{}, obj interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Apply_Call { return &NamespaceableResourceInterface_Apply_Call{Call: _e.mock.On("Apply", - append([]any{ctx, name, obj, options}, subresources...)...)} + append([]interface{}{ctx, name, obj, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Apply_Call) Run(run func(ctx context.Context, name string, obj *unstructured.Unstructured, options v1.ApplyOptions, subresources ...string)) *NamespaceableResourceInterface_Apply_Call { @@ -188,7 +197,7 @@ type NamespaceableResourceInterface_ApplyStatus_Call struct { // - name string // - obj *unstructured.Unstructured // - options v1.ApplyOptions -func (_e *NamespaceableResourceInterface_Expecter) ApplyStatus(ctx any, name any, obj any, options any) *NamespaceableResourceInterface_ApplyStatus_Call { +func (_e *NamespaceableResourceInterface_Expecter) ApplyStatus(ctx interface{}, name interface{}, obj interface{}, options interface{}) *NamespaceableResourceInterface_ApplyStatus_Call { return &NamespaceableResourceInterface_ApplyStatus_Call{Call: _e.mock.On("ApplyStatus", ctx, name, obj, options)} } @@ -233,11 +242,11 @@ func (_c *NamespaceableResourceInterface_ApplyStatus_Call) RunAndReturn(run func // Create provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Create(ctx context.Context, obj *unstructured.Unstructured, options v1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -276,9 +285,9 @@ type NamespaceableResourceInterface_Create_Call struct { // - obj *unstructured.Unstructured // - options v1.CreateOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Create(ctx any, obj any, options any, subresources ...any) *NamespaceableResourceInterface_Create_Call { +func (_e *NamespaceableResourceInterface_Expecter) Create(ctx interface{}, obj interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Create_Call { return &NamespaceableResourceInterface_Create_Call{Call: _e.mock.On("Create", - append([]any{ctx, obj, options}, subresources...)...)} + append([]interface{}{ctx, obj, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Create_Call) Run(run func(ctx context.Context, obj *unstructured.Unstructured, options v1.CreateOptions, subresources ...string)) *NamespaceableResourceInterface_Create_Call { @@ -326,11 +335,11 @@ func (_c *NamespaceableResourceInterface_Create_Call) RunAndReturn(run func(ctx // Delete provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Delete(ctx context.Context, name string, options v1.DeleteOptions, subresources ...string) error { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, name, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -358,9 +367,9 @@ type NamespaceableResourceInterface_Delete_Call struct { // - name string // - options v1.DeleteOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Delete(ctx any, name any, options any, subresources ...any) *NamespaceableResourceInterface_Delete_Call { +func (_e *NamespaceableResourceInterface_Expecter) Delete(ctx interface{}, name interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Delete_Call { return &NamespaceableResourceInterface_Delete_Call{Call: _e.mock.On("Delete", - append([]any{ctx, name, options}, subresources...)...)} + append([]interface{}{ctx, name, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Delete_Call) Run(run func(ctx context.Context, name string, options v1.DeleteOptions, subresources ...string)) *NamespaceableResourceInterface_Delete_Call { @@ -431,7 +440,7 @@ type NamespaceableResourceInterface_DeleteCollection_Call struct { // - ctx context.Context // - options v1.DeleteOptions // - listOptions v1.ListOptions -func (_e *NamespaceableResourceInterface_Expecter) DeleteCollection(ctx any, options any, listOptions any) *NamespaceableResourceInterface_DeleteCollection_Call { +func (_e *NamespaceableResourceInterface_Expecter) DeleteCollection(ctx interface{}, options interface{}, listOptions interface{}) *NamespaceableResourceInterface_DeleteCollection_Call { return &NamespaceableResourceInterface_DeleteCollection_Call{Call: _e.mock.On("DeleteCollection", ctx, options, listOptions)} } @@ -471,11 +480,11 @@ func (_c *NamespaceableResourceInterface_DeleteCollection_Call) RunAndReturn(run // Get provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Get(ctx context.Context, name string, options v1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, name, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -514,9 +523,9 @@ type NamespaceableResourceInterface_Get_Call struct { // - name string // - options v1.GetOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Get(ctx any, name any, options any, subresources ...any) *NamespaceableResourceInterface_Get_Call { +func (_e *NamespaceableResourceInterface_Expecter) Get(ctx interface{}, name interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Get_Call { return &NamespaceableResourceInterface_Get_Call{Call: _e.mock.On("Get", - append([]any{ctx, name, options}, subresources...)...)} + append([]interface{}{ctx, name, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Get_Call) Run(run func(ctx context.Context, name string, options v1.GetOptions, subresources ...string)) *NamespaceableResourceInterface_Get_Call { @@ -597,7 +606,7 @@ type NamespaceableResourceInterface_List_Call struct { // List is a helper method to define mock.On call // - ctx context.Context // - opts v1.ListOptions -func (_e *NamespaceableResourceInterface_Expecter) List(ctx any, opts any) *NamespaceableResourceInterface_List_Call { +func (_e *NamespaceableResourceInterface_Expecter) List(ctx interface{}, opts interface{}) *NamespaceableResourceInterface_List_Call { return &NamespaceableResourceInterface_List_Call{Call: _e.mock.On("List", ctx, opts)} } @@ -655,7 +664,7 @@ type NamespaceableResourceInterface_Namespace_Call struct { // Namespace is a helper method to define mock.On call // - s string -func (_e *NamespaceableResourceInterface_Expecter) Namespace(s any) *NamespaceableResourceInterface_Namespace_Call { +func (_e *NamespaceableResourceInterface_Expecter) Namespace(s interface{}) *NamespaceableResourceInterface_Namespace_Call { return &NamespaceableResourceInterface_Namespace_Call{Call: _e.mock.On("Namespace", s)} } @@ -685,11 +694,11 @@ func (_c *NamespaceableResourceInterface_Namespace_Call) RunAndReturn(run func(s // Patch provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options v1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, name, pt, data, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -730,9 +739,9 @@ type NamespaceableResourceInterface_Patch_Call struct { // - data []byte // - options v1.PatchOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Patch(ctx any, name any, pt any, data any, options any, subresources ...any) *NamespaceableResourceInterface_Patch_Call { +func (_e *NamespaceableResourceInterface_Expecter) Patch(ctx interface{}, name interface{}, pt interface{}, data interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Patch_Call { return &NamespaceableResourceInterface_Patch_Call{Call: _e.mock.On("Patch", - append([]any{ctx, name, pt, data, options}, subresources...)...)} + append([]interface{}{ctx, name, pt, data, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Patch_Call) Run(run func(ctx context.Context, name string, pt types.PatchType, data []byte, options v1.PatchOptions, subresources ...string)) *NamespaceableResourceInterface_Patch_Call { @@ -790,11 +799,11 @@ func (_c *NamespaceableResourceInterface_Patch_Call) RunAndReturn(run func(ctx c // Update provides a mock function for the type NamespaceableResourceInterface func (_mock *NamespaceableResourceInterface) Update(ctx context.Context, obj *unstructured.Unstructured, options v1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -833,9 +842,9 @@ type NamespaceableResourceInterface_Update_Call struct { // - obj *unstructured.Unstructured // - options v1.UpdateOptions // - subresources ...string -func (_e *NamespaceableResourceInterface_Expecter) Update(ctx any, obj any, options any, subresources ...any) *NamespaceableResourceInterface_Update_Call { +func (_e *NamespaceableResourceInterface_Expecter) Update(ctx interface{}, obj interface{}, options interface{}, subresources ...interface{}) *NamespaceableResourceInterface_Update_Call { return &NamespaceableResourceInterface_Update_Call{Call: _e.mock.On("Update", - append([]any{ctx, obj, options}, subresources...)...)} + append([]interface{}{ctx, obj, options}, subresources...)...)} } func (_c *NamespaceableResourceInterface_Update_Call) Run(run func(ctx context.Context, obj *unstructured.Unstructured, options v1.UpdateOptions, subresources ...string)) *NamespaceableResourceInterface_Update_Call { @@ -917,7 +926,7 @@ type NamespaceableResourceInterface_UpdateStatus_Call struct { // - ctx context.Context // - obj *unstructured.Unstructured // - options v1.UpdateOptions -func (_e *NamespaceableResourceInterface_Expecter) UpdateStatus(ctx any, obj any, options any) *NamespaceableResourceInterface_UpdateStatus_Call { +func (_e *NamespaceableResourceInterface_Expecter) UpdateStatus(ctx interface{}, obj interface{}, options interface{}) *NamespaceableResourceInterface_UpdateStatus_Call { return &NamespaceableResourceInterface_UpdateStatus_Call{Call: _e.mock.On("UpdateStatus", ctx, obj, options)} } @@ -990,7 +999,7 @@ type NamespaceableResourceInterface_Watch_Call struct { // Watch is a helper method to define mock.On call // - ctx context.Context // - opts v1.ListOptions -func (_e *NamespaceableResourceInterface_Expecter) Watch(ctx any, opts any) *NamespaceableResourceInterface_Watch_Call { +func (_e *NamespaceableResourceInterface_Expecter) Watch(ctx interface{}, opts interface{}) *NamespaceableResourceInterface_Watch_Call { return &NamespaceableResourceInterface_Watch_Call{Call: _e.mock.On("Watch", ctx, opts)} } diff --git a/operator/internal/mocks/dynamic/ResourceInterface.go b/operator/internal/mocks/dynamic/ResourceInterface.go index 5cf342615..e1fe53e80 100644 --- a/operator/internal/mocks/dynamic/ResourceInterface.go +++ b/operator/internal/mocks/dynamic/ResourceInterface.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -20,19 +38,10 @@ func NewResourceInterface(t interface { mock.TestingT Cleanup(func()) }) *ResourceInterface { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock := &ResourceInterface{} mock.Mock.Test(t) - t.Cleanup(func() { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock.AssertExpectations(t) - }) + t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } @@ -53,11 +62,11 @@ func (_m *ResourceInterface) EXPECT() *ResourceInterface_Expecter { // Apply provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Apply(ctx context.Context, name string, obj *unstructured.Unstructured, options v1.ApplyOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, name, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -97,9 +106,9 @@ type ResourceInterface_Apply_Call struct { // - obj *unstructured.Unstructured // - options v1.ApplyOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Apply(ctx any, name any, obj any, options any, subresources ...any) *ResourceInterface_Apply_Call { +func (_e *ResourceInterface_Expecter) Apply(ctx interface{}, name interface{}, obj interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Apply_Call { return &ResourceInterface_Apply_Call{Call: _e.mock.On("Apply", - append([]any{ctx, name, obj, options}, subresources...)...)} + append([]interface{}{ctx, name, obj, options}, subresources...)...)} } func (_c *ResourceInterface_Apply_Call) Run(run func(ctx context.Context, name string, obj *unstructured.Unstructured, options v1.ApplyOptions, subresources ...string)) *ResourceInterface_Apply_Call { @@ -187,7 +196,7 @@ type ResourceInterface_ApplyStatus_Call struct { // - name string // - obj *unstructured.Unstructured // - options v1.ApplyOptions -func (_e *ResourceInterface_Expecter) ApplyStatus(ctx any, name any, obj any, options any) *ResourceInterface_ApplyStatus_Call { +func (_e *ResourceInterface_Expecter) ApplyStatus(ctx interface{}, name interface{}, obj interface{}, options interface{}) *ResourceInterface_ApplyStatus_Call { return &ResourceInterface_ApplyStatus_Call{Call: _e.mock.On("ApplyStatus", ctx, name, obj, options)} } @@ -232,11 +241,11 @@ func (_c *ResourceInterface_ApplyStatus_Call) RunAndReturn(run func(ctx context. // Create provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Create(ctx context.Context, obj *unstructured.Unstructured, options v1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -275,9 +284,9 @@ type ResourceInterface_Create_Call struct { // - obj *unstructured.Unstructured // - options v1.CreateOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Create(ctx any, obj any, options any, subresources ...any) *ResourceInterface_Create_Call { +func (_e *ResourceInterface_Expecter) Create(ctx interface{}, obj interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Create_Call { return &ResourceInterface_Create_Call{Call: _e.mock.On("Create", - append([]any{ctx, obj, options}, subresources...)...)} + append([]interface{}{ctx, obj, options}, subresources...)...)} } func (_c *ResourceInterface_Create_Call) Run(run func(ctx context.Context, obj *unstructured.Unstructured, options v1.CreateOptions, subresources ...string)) *ResourceInterface_Create_Call { @@ -325,11 +334,11 @@ func (_c *ResourceInterface_Create_Call) RunAndReturn(run func(ctx context.Conte // Delete provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Delete(ctx context.Context, name string, options v1.DeleteOptions, subresources ...string) error { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, name, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -357,9 +366,9 @@ type ResourceInterface_Delete_Call struct { // - name string // - options v1.DeleteOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Delete(ctx any, name any, options any, subresources ...any) *ResourceInterface_Delete_Call { +func (_e *ResourceInterface_Expecter) Delete(ctx interface{}, name interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Delete_Call { return &ResourceInterface_Delete_Call{Call: _e.mock.On("Delete", - append([]any{ctx, name, options}, subresources...)...)} + append([]interface{}{ctx, name, options}, subresources...)...)} } func (_c *ResourceInterface_Delete_Call) Run(run func(ctx context.Context, name string, options v1.DeleteOptions, subresources ...string)) *ResourceInterface_Delete_Call { @@ -430,7 +439,7 @@ type ResourceInterface_DeleteCollection_Call struct { // - ctx context.Context // - options v1.DeleteOptions // - listOptions v1.ListOptions -func (_e *ResourceInterface_Expecter) DeleteCollection(ctx any, options any, listOptions any) *ResourceInterface_DeleteCollection_Call { +func (_e *ResourceInterface_Expecter) DeleteCollection(ctx interface{}, options interface{}, listOptions interface{}) *ResourceInterface_DeleteCollection_Call { return &ResourceInterface_DeleteCollection_Call{Call: _e.mock.On("DeleteCollection", ctx, options, listOptions)} } @@ -470,11 +479,11 @@ func (_c *ResourceInterface_DeleteCollection_Call) RunAndReturn(run func(ctx con // Get provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Get(ctx context.Context, name string, options v1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, name, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -513,9 +522,9 @@ type ResourceInterface_Get_Call struct { // - name string // - options v1.GetOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Get(ctx any, name any, options any, subresources ...any) *ResourceInterface_Get_Call { +func (_e *ResourceInterface_Expecter) Get(ctx interface{}, name interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Get_Call { return &ResourceInterface_Get_Call{Call: _e.mock.On("Get", - append([]any{ctx, name, options}, subresources...)...)} + append([]interface{}{ctx, name, options}, subresources...)...)} } func (_c *ResourceInterface_Get_Call) Run(run func(ctx context.Context, name string, options v1.GetOptions, subresources ...string)) *ResourceInterface_Get_Call { @@ -596,7 +605,7 @@ type ResourceInterface_List_Call struct { // List is a helper method to define mock.On call // - ctx context.Context // - opts v1.ListOptions -func (_e *ResourceInterface_Expecter) List(ctx any, opts any) *ResourceInterface_List_Call { +func (_e *ResourceInterface_Expecter) List(ctx interface{}, opts interface{}) *ResourceInterface_List_Call { return &ResourceInterface_List_Call{Call: _e.mock.On("List", ctx, opts)} } @@ -631,11 +640,11 @@ func (_c *ResourceInterface_List_Call) RunAndReturn(run func(ctx context.Context // Patch provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options v1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, name, pt, data, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -676,9 +685,9 @@ type ResourceInterface_Patch_Call struct { // - data []byte // - options v1.PatchOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Patch(ctx any, name any, pt any, data any, options any, subresources ...any) *ResourceInterface_Patch_Call { +func (_e *ResourceInterface_Expecter) Patch(ctx interface{}, name interface{}, pt interface{}, data interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Patch_Call { return &ResourceInterface_Patch_Call{Call: _e.mock.On("Patch", - append([]any{ctx, name, pt, data, options}, subresources...)...)} + append([]interface{}{ctx, name, pt, data, options}, subresources...)...)} } func (_c *ResourceInterface_Patch_Call) Run(run func(ctx context.Context, name string, pt types.PatchType, data []byte, options v1.PatchOptions, subresources ...string)) *ResourceInterface_Patch_Call { @@ -736,11 +745,11 @@ func (_c *ResourceInterface_Patch_Call) RunAndReturn(run func(ctx context.Contex // Update provides a mock function for the type ResourceInterface func (_mock *ResourceInterface) Update(ctx context.Context, obj *unstructured.Unstructured, options v1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) { // string - _va := make([]any, len(subresources)) + _va := make([]interface{}, len(subresources)) for _i := range subresources { _va[_i] = subresources[_i] } - var _ca []any + var _ca []interface{} _ca = append(_ca, ctx, obj, options) _ca = append(_ca, _va...) ret := _mock.Called(_ca...) @@ -779,9 +788,9 @@ type ResourceInterface_Update_Call struct { // - obj *unstructured.Unstructured // - options v1.UpdateOptions // - subresources ...string -func (_e *ResourceInterface_Expecter) Update(ctx any, obj any, options any, subresources ...any) *ResourceInterface_Update_Call { +func (_e *ResourceInterface_Expecter) Update(ctx interface{}, obj interface{}, options interface{}, subresources ...interface{}) *ResourceInterface_Update_Call { return &ResourceInterface_Update_Call{Call: _e.mock.On("Update", - append([]any{ctx, obj, options}, subresources...)...)} + append([]interface{}{ctx, obj, options}, subresources...)...)} } func (_c *ResourceInterface_Update_Call) Run(run func(ctx context.Context, obj *unstructured.Unstructured, options v1.UpdateOptions, subresources ...string)) *ResourceInterface_Update_Call { @@ -863,7 +872,7 @@ type ResourceInterface_UpdateStatus_Call struct { // - ctx context.Context // - obj *unstructured.Unstructured // - options v1.UpdateOptions -func (_e *ResourceInterface_Expecter) UpdateStatus(ctx any, obj any, options any) *ResourceInterface_UpdateStatus_Call { +func (_e *ResourceInterface_Expecter) UpdateStatus(ctx interface{}, obj interface{}, options interface{}) *ResourceInterface_UpdateStatus_Call { return &ResourceInterface_UpdateStatus_Call{Call: _e.mock.On("UpdateStatus", ctx, obj, options)} } @@ -936,7 +945,7 @@ type ResourceInterface_Watch_Call struct { // Watch is a helper method to define mock.On call // - ctx context.Context // - opts v1.ListOptions -func (_e *ResourceInterface_Expecter) Watch(ctx any, opts any) *ResourceInterface_Watch_Call { +func (_e *ResourceInterface_Expecter) Watch(ctx interface{}, opts interface{}) *ResourceInterface_Watch_Call { return &ResourceInterface_Watch_Call{Call: _e.mock.On("Watch", ctx, opts)} } diff --git a/operator/internal/mocks/record/EventRecorder.go b/operator/internal/mocks/record/EventRecorder.go index 7132707d9..6e088ac20 100644 --- a/operator/internal/mocks/record/EventRecorder.go +++ b/operator/internal/mocks/record/EventRecorder.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -15,19 +33,10 @@ func NewEventRecorder(t interface { mock.TestingT Cleanup(func()) }) *EventRecorder { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock := &EventRecorder{} mock.Mock.Test(t) - t.Cleanup(func() { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock.AssertExpectations(t) - }) + t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } @@ -47,7 +56,7 @@ func (_m *EventRecorder) EXPECT() *EventRecorder_Expecter { // AnnotatedEventf provides a mock function for the type EventRecorder func (_mock *EventRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype string, reason string, messageFmt string, args ...interface{}) { - var _ca []any + var _ca []interface{} _ca = append(_ca, object, annotations, eventtype, reason, messageFmt) _ca = append(_ca, args...) _mock.Called(_ca...) @@ -66,9 +75,9 @@ type EventRecorder_AnnotatedEventf_Call struct { // - reason string // - messageFmt string // - args ...interface{} -func (_e *EventRecorder_Expecter) AnnotatedEventf(object any, annotations any, eventtype any, reason any, messageFmt any, args ...any) *EventRecorder_AnnotatedEventf_Call { +func (_e *EventRecorder_Expecter) AnnotatedEventf(object interface{}, annotations interface{}, eventtype interface{}, reason interface{}, messageFmt interface{}, args ...interface{}) *EventRecorder_AnnotatedEventf_Call { return &EventRecorder_AnnotatedEventf_Call{Call: _e.mock.On("AnnotatedEventf", - append([]any{object, annotations, eventtype, reason, messageFmt}, args...)...)} + append([]interface{}{object, annotations, eventtype, reason, messageFmt}, args...)...)} } func (_c *EventRecorder_AnnotatedEventf_Call) Run(run func(object runtime.Object, annotations map[string]string, eventtype string, reason string, messageFmt string, args ...interface{})) *EventRecorder_AnnotatedEventf_Call { @@ -139,7 +148,7 @@ type EventRecorder_Event_Call struct { // - eventtype string // - reason string // - message string -func (_e *EventRecorder_Expecter) Event(object any, eventtype any, reason any, message any) *EventRecorder_Event_Call { +func (_e *EventRecorder_Expecter) Event(object interface{}, eventtype interface{}, reason interface{}, message interface{}) *EventRecorder_Event_Call { return &EventRecorder_Event_Call{Call: _e.mock.On("Event", object, eventtype, reason, message)} } @@ -183,7 +192,7 @@ func (_c *EventRecorder_Event_Call) RunAndReturn(run func(object runtime.Object, // Eventf provides a mock function for the type EventRecorder func (_mock *EventRecorder) Eventf(object runtime.Object, eventtype string, reason string, messageFmt string, args ...interface{}) { - var _ca []any + var _ca []interface{} _ca = append(_ca, object, eventtype, reason, messageFmt) _ca = append(_ca, args...) _mock.Called(_ca...) @@ -201,9 +210,9 @@ type EventRecorder_Eventf_Call struct { // - reason string // - messageFmt string // - args ...interface{} -func (_e *EventRecorder_Expecter) Eventf(object any, eventtype any, reason any, messageFmt any, args ...any) *EventRecorder_Eventf_Call { +func (_e *EventRecorder_Expecter) Eventf(object interface{}, eventtype interface{}, reason interface{}, messageFmt interface{}, args ...interface{}) *EventRecorder_Eventf_Call { return &EventRecorder_Eventf_Call{Call: _e.mock.On("Eventf", - append([]any{object, eventtype, reason, messageFmt}, args...)...)} + append([]interface{}{object, eventtype, reason, messageFmt}, args...)...)} } func (_c *EventRecorder_Eventf_Call) Run(run func(object runtime.Object, eventtype string, reason string, messageFmt string, args ...interface{})) *EventRecorder_Eventf_Call { diff --git a/operator/internal/mocks/workqueue/TypedRateLimitingInterface.go b/operator/internal/mocks/workqueue/TypedRateLimitingInterface.go index 6c509cd92..23f9ebaf8 100644 --- a/operator/internal/mocks/workqueue/TypedRateLimitingInterface.go +++ b/operator/internal/mocks/workqueue/TypedRateLimitingInterface.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify @@ -16,19 +34,10 @@ func NewTypedRateLimitingInterface[T comparable](t interface { mock.TestingT Cleanup(func()) }) *TypedRateLimitingInterface[T] { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock := &TypedRateLimitingInterface[T]{} mock.Mock.Test(t) - t.Cleanup(func() { - if helper, ok := t.(interface{ Helper() }); ok { - helper.Helper() - } - mock.AssertExpectations(t) - }) + t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } @@ -59,7 +68,7 @@ type TypedRateLimitingInterface_Add_Call[T comparable] struct { // Add is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) Add(item any) *TypedRateLimitingInterface_Add_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) Add(item interface{}) *TypedRateLimitingInterface_Add_Call[T] { return &TypedRateLimitingInterface_Add_Call[T]{Call: _e.mock.On("Add", item)} } @@ -100,7 +109,7 @@ type TypedRateLimitingInterface_AddAfter_Call[T comparable] struct { // AddAfter is a helper method to define mock.On call // - item T // - duration time.Duration -func (_e *TypedRateLimitingInterface_Expecter[T]) AddAfter(item any, duration any) *TypedRateLimitingInterface_AddAfter_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) AddAfter(item interface{}, duration interface{}) *TypedRateLimitingInterface_AddAfter_Call[T] { return &TypedRateLimitingInterface_AddAfter_Call[T]{Call: _e.mock.On("AddAfter", item, duration)} } @@ -145,7 +154,7 @@ type TypedRateLimitingInterface_AddRateLimited_Call[T comparable] struct { // AddRateLimited is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) AddRateLimited(item any) *TypedRateLimitingInterface_AddRateLimited_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) AddRateLimited(item interface{}) *TypedRateLimitingInterface_AddRateLimited_Call[T] { return &TypedRateLimitingInterface_AddRateLimited_Call[T]{Call: _e.mock.On("AddRateLimited", item)} } @@ -185,7 +194,7 @@ type TypedRateLimitingInterface_Done_Call[T comparable] struct { // Done is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) Done(item any) *TypedRateLimitingInterface_Done_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) Done(item interface{}) *TypedRateLimitingInterface_Done_Call[T] { return &TypedRateLimitingInterface_Done_Call[T]{Call: _e.mock.On("Done", item)} } @@ -225,7 +234,7 @@ type TypedRateLimitingInterface_Forget_Call[T comparable] struct { // Forget is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) Forget(item any) *TypedRateLimitingInterface_Forget_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) Forget(item interface{}) *TypedRateLimitingInterface_Forget_Call[T] { return &TypedRateLimitingInterface_Forget_Call[T]{Call: _e.mock.On("Forget", item)} } @@ -375,7 +384,7 @@ type TypedRateLimitingInterface_NumRequeues_Call[T comparable] struct { // NumRequeues is a helper method to define mock.On call // - item T -func (_e *TypedRateLimitingInterface_Expecter[T]) NumRequeues(item any) *TypedRateLimitingInterface_NumRequeues_Call[T] { +func (_e *TypedRateLimitingInterface_Expecter[T]) NumRequeues(item interface{}) *TypedRateLimitingInterface_NumRequeues_Call[T] { return &TypedRateLimitingInterface_NumRequeues_Call[T]{Call: _e.mock.On("NumRequeues", item)} } diff --git a/operator/internal/wrapper/mock/SkyhookNode.go b/operator/internal/wrapper/mock/SkyhookNode.go index 343a82123..b7472980a 100644 --- a/operator/internal/wrapper/mock/SkyhookNode.go +++ b/operator/internal/wrapper/mock/SkyhookNode.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify diff --git a/operator/internal/wrapper/mock/SkyhookNodeOnly.go b/operator/internal/wrapper/mock/SkyhookNodeOnly.go index 689e5c894..6fa78cae8 100644 --- a/operator/internal/wrapper/mock/SkyhookNodeOnly.go +++ b/operator/internal/wrapper/mock/SkyhookNodeOnly.go @@ -1,3 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify From 9926d1b7394b80990d999757f9a35a0bdfaf2e19 Mon Sep 17 00:00:00 2001 From: mohityadav8 Date: Wed, 30 Sep 2026 02:47:31 +0530 Subject: [PATCH 9/9] fix: only persist DrainBlocked when DrainNode actually evaluated pods; clear stale annotations on every pass Signed-off-by: mohityadav8 --- .../internal/controller/cluster_state_v2.go | 17 +++++++++++-- .../internal/controller/mock/SkyhookNodes.go | 24 ++++++++++++------- .../internal/controller/skyhook_controller.go | 19 +++++---------- operator/internal/drain/drain.go | 10 ++++++++ 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/operator/internal/controller/cluster_state_v2.go b/operator/internal/controller/cluster_state_v2.go index 839942576..8a10a41e2 100644 --- a/operator/internal/controller/cluster_state_v2.go +++ b/operator/internal/controller/cluster_state_v2.go @@ -19,6 +19,7 @@ package controller import ( + "context" "fmt" "sort" "strings" @@ -402,7 +403,7 @@ type SkyhookNodes interface { IsPaused() bool HasUninstallWork() (bool, error) UpdateBlockedCondition() error - UpdateDrainBlockedCondition(logger logr.Logger) + UpdateDrainBlockedCondition(ctx context.Context, logger logr.Logger) UpdateUninstallConditions() error UpdateNodeStateMalformedCondition() NodeCount() int @@ -641,9 +642,21 @@ func (s *skyhookNodes) UpdateBlockedCondition() error { // NodeStateMalformed-adjacent concern, but has no dedicated user-visible signal of // its own; a node dropping out of this aggregate silently is an accepted limitation // rather than a deliberate design, tracked for follow-up. -func (s *skyhookNodes) UpdateDrainBlockedCondition(logger logr.Logger) { +func (s *skyhookNodes) UpdateDrainBlockedCondition(ctx context.Context, logger logr.Logger) { blocks := make([]wrapper.DrainBlockedNode, 0, len(s.nodes)) for _, node := range s.nodes { + // A node with no runnable, interrupt-requiring package this pass will never + // reach EnsureNodeIsReadyForInterrupt, so nothing else clears its persisted + // drain-blocker annotation. Clear it here instead: this function is the one + // thing that runs on every pass regardless of paused/disabled/complete state + // or an error elsewhere in the reconcile, which is what keeps a removed or + // finished drain from reporting a blocker that no longer exists. + if !nodeNeedsInterruptDrain(ctx, node) { + if err := node.SetDrainBlocked(nil); err != nil { + logger.Error(err, "clearing stale drain blocked state", "node", node.GetNode().Name) + } + } + blocked, err := node.DrainBlocked() if err != nil { continue diff --git a/operator/internal/controller/mock/SkyhookNodes.go b/operator/internal/controller/mock/SkyhookNodes.go index 70e1e5870..754f02eeb 100644 --- a/operator/internal/controller/mock/SkyhookNodes.go +++ b/operator/internal/controller/mock/SkyhookNodes.go @@ -23,6 +23,8 @@ package controller import ( + "context" + "github.com/NVIDIA/nodewright/operator/api/nodewright/v1alpha1" "github.com/NVIDIA/nodewright/operator/internal/wrapper" "github.com/go-logr/logr" @@ -1095,8 +1097,8 @@ func (_c *MockSkyhookNodes_UpdateCondition_Call) RunAndReturn(run func(logger lo } // UpdateDrainBlockedCondition provides a mock function for the type MockSkyhookNodes -func (_mock *MockSkyhookNodes) UpdateDrainBlockedCondition(logger logr.Logger) { - _mock.Called(logger) +func (_mock *MockSkyhookNodes) UpdateDrainBlockedCondition(ctx context.Context, logger logr.Logger) { + _mock.Called(ctx, logger) return } @@ -1106,19 +1108,25 @@ type MockSkyhookNodes_UpdateDrainBlockedCondition_Call struct { } // UpdateDrainBlockedCondition is a helper method to define mock.On call +// - ctx context.Context // - logger logr.Logger -func (_e *MockSkyhookNodes_Expecter) UpdateDrainBlockedCondition(logger any) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { - return &MockSkyhookNodes_UpdateDrainBlockedCondition_Call{Call: _e.mock.On("UpdateDrainBlockedCondition", logger)} +func (_e *MockSkyhookNodes_Expecter) UpdateDrainBlockedCondition(ctx any, logger any) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { + return &MockSkyhookNodes_UpdateDrainBlockedCondition_Call{Call: _e.mock.On("UpdateDrainBlockedCondition", ctx, logger)} } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Run(run func(logger logr.Logger)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Run(run func(ctx context.Context, logger logr.Logger)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { _c.Call.Run(func(args mock.Arguments) { - var arg0 logr.Logger + var arg0 context.Context if args[0] != nil { - arg0 = args[0].(logr.Logger) + arg0 = args[0].(context.Context) + } + var arg1 logr.Logger + if args[1] != nil { + arg1 = args[1].(logr.Logger) } run( arg0, + arg1, ) }) return _c @@ -1129,7 +1137,7 @@ func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) Return() *MockSkyho return _c } -func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) RunAndReturn(run func(logr.Logger)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { +func (_c *MockSkyhookNodes_UpdateDrainBlockedCondition_Call) RunAndReturn(run func(context.Context, logr.Logger)) *MockSkyhookNodes_UpdateDrainBlockedCondition_Call { _c.Run(run) return _c } diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go index 36b5521a6..58cd3b8d5 100644 --- a/operator/internal/controller/skyhook_controller.go +++ b/operator/internal/controller/skyhook_controller.go @@ -609,7 +609,7 @@ func (r *SkyhookReconciler) refreshSkyhookConditions(ctx context.Context, cluste // below maintains — same name prefix, different condition type. Rebuilt from // persisted per-node state so it stays correct on paused/disabled/complete/error/ // serial-partial passes; see cluster_state_v2.go's UpdateDrainBlockedCondition. - skyhook.UpdateDrainBlockedCondition(log.FromContext(ctx)) + skyhook.UpdateDrainBlockedCondition(ctx, log.FromContext(ctx)) if err := r.updateDrainBlockedCondition(ctx, skyhook); err != nil { return fmt.Errorf("error updating drain blocked condition: %w", err) } @@ -1490,17 +1490,10 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState continue } - // A node with no runnable, interrupt-requiring package this pass will never reach - // EnsureNodeIsReadyForInterrupt below, so nothing will refresh — or clear — its - // persisted drain-blocker annotation this pass. Clear it proactively: a package - // whose interrupt requirement was removed, or a node that has simply finished - // draining, must not leave DrainBlocked reporting a blocker with no bearing on - // this node's current state. - if !nodeNeedsInterruptDrain(ctx, node) { - if err := node.SetDrainBlocked(nil); err != nil { - return nil, fmt.Errorf("clearing stale drain blocked state for node [%s]: %w", node.GetNode().Name, err) - } - } + // The stale-annotation clear for nodes with no runnable interrupt-requiring + // package now lives in UpdateDrainBlockedCondition (see cluster_state_v2.go), + // which runs on every pass — paused, disabled, complete, and error exits + // included — rather than only the passes that reach this loop. toRun, err := node.RunNext() if err != nil { @@ -1572,7 +1565,7 @@ func (r *SkyhookReconciler) RunSkyhookPackages(ctx context.Context, clusterState // per-node state (see UpdateDrainBlockedCondition), including its own truncation-log // line, rather than from a local slice — that is what keeps it correct for nodes this // pass skipped or never reached. - skyhook.UpdateDrainBlockedCondition(logger) + skyhook.UpdateDrainBlockedCondition(ctx, logger) saved, errs := r.SaveNodesAndSkyhook(ctx, clusterState, skyhook) if len(errs) > 0 { diff --git a/operator/internal/drain/drain.go b/operator/internal/drain/drain.go index 7e5b3d7b0..e9d18bb8a 100644 --- a/operator/internal/drain/drain.go +++ b/operator/internal/drain/drain.go @@ -93,6 +93,16 @@ type DrainResult struct { Blocked []BlockedPod } +// IsZero reports whether the result carries no information: DrainNode returns this +// alongside an error on paths that never got far enough to evaluate any pod (a +// GetPods failure, for instance), as distinct from a result that legitimately found +// nothing blocking. The caller uses this to avoid persisting a false "nothing +// blocking" over a real, previously-recorded blocker just because this pass +// couldn't look. +func (r DrainResult) IsZero() bool { + return !r.Ready && len(r.Blocked) == 0 +} + func DefaultOptions() Options { return Options{ DeleteEmptyDirData: true,