Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions frontend/pkg/frontend/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,32 @@ func (f *Frontend) newClusterAdmissionContext(ctx context.Context, op operation.
OriginalCluster: originalCluster.DeepCopy(),
}

if op.Type == operation.Create {
subscriptionID := originalCluster.ID.SubscriptionID
clusterIterator, err := f.resourcesDBClient.HCPClusters(subscriptionID, "").List(ctx, nil)
if err != nil {
return nil, fmt.Errorf("cannot list clusters for cluster admission: %w", err)
}
for _, cluster := range clusterIterator.Items(ctx) {
admissionContext.SubscriptionClusters = append(admissionContext.SubscriptionClusters, cluster)

nodePoolIterator, err := f.resourcesDBClient.HCPClusters(subscriptionID, cluster.ID.ResourceGroupName).NodePools(cluster.ID.Name).List(ctx, nil)
if err != nil {
return nil, fmt.Errorf("cannot list node pools for cluster admission: %w", err)
}
for _, nodePool := range nodePoolIterator.Items(ctx) {
admissionContext.SubscriptionNodePools = append(admissionContext.SubscriptionNodePools, nodePool)
}
if err := nodePoolIterator.GetError(); err != nil {
return nil, fmt.Errorf("cannot list node pools for cluster admission: %w", err)
}
}
if err := clusterIterator.GetError(); err != nil {
return nil, fmt.Errorf("cannot list clusters for cluster admission: %w", err)
}
return admissionContext, nil
}

if op.Type != operation.Update {
return admissionContext, nil
}
Expand Down
113 changes: 113 additions & 0 deletions internal/admission/admit_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,15 @@ type ClusterAdmissionContext struct {
// ClusterNodePools is the list of node pools belonging to the cluster, used
// for minor-version skew checks against the desired cluster version.
ClusterNodePools []ClusterAdmissionNodePool
// SubscriptionClusters lists cluster documents in the same subscription, used

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Document whether it includes the cluster being processed itself or not

// for cross-cluster platform resource uniqueness on CREATE.
// The list is empty on UPDATE.
SubscriptionClusters []*api.HCPOpenShiftCluster
// SubscriptionNodePools lists node pool documents under SubscriptionClusters,
// used to ensure a cluster subnet is not already assigned to another cluster's
// node pool on CREATE.
// The list is empty on UPDATE.
SubscriptionNodePools []*api.HCPOpenShiftClusterNodePool

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to check this? don't we check that for nodepools the subnet must be part of the VNet of the cluster?

}

// ClusterAdmissionNodePool is a single node pool plus its prefetched service
Expand Down Expand Up @@ -222,6 +231,110 @@ func admitClusterCustomerProperties(ctx context.Context, admissionContext *Clust
errs := field.ErrorList{}

errs = append(errs, admitClusterVersionProfile(ctx, admissionContext, op, fldPath.Child("version"), &newObj.Version, safe.Field(oldObj, validation.ToClusterCustomerPropertiesVersion))...)
platformPath := fldPath.Child("platform")

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Move all of this to admitClusterPlatform

errs = append(errs, admitClusterManagedResourceGroupName(ctx, admissionContext, op, platformPath, &newObj.Platform)...)
errs = append(errs, admitClusterSubnetResourceID(ctx, admissionContext, op, platformPath, &newObj.Platform)...)
errs = append(errs, admitClusterNetworkSecurityGroupResourceID(ctx, admissionContext, op, platformPath, &newObj.Platform)...)

return errs
}

// admitClusterManagedResourceGroupName ensures the managed resource group name
// is unique within the subscription on CREATE.
//
// Best-effort only: compares against SubscriptionClusters prefetched before
// admission runs. Concurrent creates with the same MRG name can both succeed.
func admitClusterManagedResourceGroupName(_ context.Context, admissionContext *ClusterAdmissionContext, op operation.Operation, fldPath *field.Path, newObj *api.CustomerPlatformProfile) field.ErrorList {
if op.Type != operation.Create || len(newObj.ManagedResourceGroup) == 0 {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can managedresourcegroup be nil at this point? if not, remove the check

return nil
}

subscriptionID := admissionContext.OriginalCluster.ID.SubscriptionID
var errs field.ErrorList

for _, existing := range admissionContext.SubscriptionClusters {
if strings.EqualFold(newObj.ManagedResourceGroup, existing.CustomerProperties.Platform.ManagedResourceGroup) {
errs = append(errs, field.Invalid(
fldPath.Child("managedResourceGroup"),
newObj.ManagedResourceGroup,
fmt.Sprintf("Cluster with managed resource group name '%s' in subscription '%s' "+
"already exists, please provide a unique managed resource group name",
newObj.ManagedResourceGroup, subscriptionID),
))
break
}
}

return errs
}

// admitClusterSubnetResourceID ensures that the subnet ID is not already in use by any other
// cluster or node pool within the same subscription when creating a new cluster.
//
// Best-effort only: compares against SubscriptionClusters and SubscriptionNodePools
// prefetched before admission runs. Concurrent creates (or a create racing with a
// node pool create) using the same subnet can both succeed.
func admitClusterSubnetResourceID(_ context.Context, admissionContext *ClusterAdmissionContext, op operation.Operation, fldPath *field.Path, newObj *api.CustomerPlatformProfile) field.ErrorList {
if op.Type != operation.Create || newObj.SubnetID == nil {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can newObj.SubnetID be nil at this point? if not, remove the check

return nil
}

subnetPath := fldPath.Child("subnetId")
subnetID := newObj.SubnetID.String()
var errs field.ErrorList

for _, existing := range admissionContext.SubscriptionClusters {
existingSubnet := existing.CustomerProperties.Platform.SubnetID
if strings.EqualFold(subnetID, existingSubnet.String()) {
errs = append(errs, field.Invalid(
subnetPath,
subnetID,
fmt.Sprintf("Subnet '%s' is already in use by another cluster", subnetID),
))
break
}
}
Comment on lines +286 to +296

for _, nodePool := range admissionContext.SubscriptionNodePools {
nodePoolSubnet := nodePool.Properties.Platform.SubnetID
if strings.EqualFold(subnetID, nodePoolSubnet.String()) {
errs = append(errs, field.Invalid(
subnetPath,
subnetID,
fmt.Sprintf("Subnet '%s' is already in use by another cluster", subnetID),
))
break
}
}
Comment on lines +298 to +308

return errs
}

// admitClusterNetworkSecurityGroupResourceID ensures that the network security group ID is not already in use by any other
// cluster within the same subscription when creating a new cluster.
//
// Best-effort only: compares against SubscriptionClusters prefetched before
// admission runs. Concurrent creates with the same NSG can both succeed.
func admitClusterNetworkSecurityGroupResourceID(_ context.Context, admissionContext *ClusterAdmissionContext, op operation.Operation, fldPath *field.Path, newObj *api.CustomerPlatformProfile) field.ErrorList {
if op.Type != operation.Create || newObj.NetworkSecurityGroupID == nil {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can newObj.SubnetID be nil at this point? if not, remove the check

return nil
}

nsgPath := fldPath.Child("networkSecurityGroupId")
nsgID := newObj.NetworkSecurityGroupID.String()
var errs field.ErrorList

for _, existing := range admissionContext.SubscriptionClusters {
existingNSG := existing.CustomerProperties.Platform.NetworkSecurityGroupID
if strings.EqualFold(nsgID, existingNSG.String()) {
errs = append(errs, field.Invalid(
nsgPath,
nsgID,
fmt.Sprintf("Network Security Group '%s' is already in use by another cluster", nsgID),
))
break
}
}
Comment on lines +327 to +337

return errs
}
Expand Down
129 changes: 129 additions & 0 deletions internal/admission/admit_cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -684,3 +684,132 @@ func TestAdmitCluster_Update(t *testing.T) {
})
}
}

func TestAdmitCluster_PlatformResourceIDs(t *testing.T) {
t.Parallel()

ctx := context.Background()

const subscriptionID = "6b690bec-0c16-4ecb-8f67-781caf40bba7"

subnetID := api.Must(azcorearm.ParseResourceID(
"/subscriptions/" + subscriptionID + "/resourceGroups/customer/providers/Microsoft.Network/virtualNetworks/vnet/subnets/cluster-subnet"))
otherSubnetID := api.Must(azcorearm.ParseResourceID(
"/subscriptions/" + subscriptionID + "/resourceGroups/other/providers/Microsoft.Network/virtualNetworks/vnet/subnets/other"))
nsgID := api.Must(azcorearm.ParseResourceID(
"/subscriptions/" + subscriptionID + "/resourceGroups/customer/providers/Microsoft.Network/networkSecurityGroups/cluster-nsg"))
otherNsgID := api.Must(azcorearm.ParseResourceID(
"/subscriptions/" + subscriptionID + "/resourceGroups/other/providers/Microsoft.Network/networkSecurityGroups/other-nsg"))

makeCluster := func(name, managedResourceGroup string, subnet, nsg *azcorearm.ResourceID) *api.HCPOpenShiftCluster {
resourceID := api.Must(azcorearm.ParseResourceID(fmt.Sprintf(
"/subscriptions/%s/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/%s",
subscriptionID, name)))
return &api.HCPOpenShiftCluster{
TrackedResource: arm.NewTrackedResource(resourceID, "eastus"),
CustomerProperties: api.HCPOpenShiftClusterCustomerProperties{
Platform: api.CustomerPlatformProfile{
ManagedResourceGroup: managedResourceGroup,
SubnetID: subnet,
NetworkSecurityGroupID: nsg,
},
},
}
}

makeNodePool := func(clusterName, nodePoolName string, subnet *azcorearm.ResourceID) *api.HCPOpenShiftClusterNodePool {
resourceID := api.Must(azcorearm.ParseResourceID(fmt.Sprintf(
"/subscriptions/%s/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/%s/hcpOpenShiftClusterNodePools/%s",
subscriptionID, clusterName, nodePoolName)))
return &api.HCPOpenShiftClusterNodePool{
TrackedResource: arm.NewTrackedResource(resourceID, "eastus"),
Properties: api.HCPOpenShiftClusterNodePoolProperties{
Platform: api.NodePoolPlatformProfile{
SubnetID: subnet,
},
},
}
}

tests := []struct {
name string
subscriptionClusters []*api.HCPOpenShiftCluster
subscriptionNodePools []*api.HCPOpenShiftClusterNodePool
newCluster *api.HCPOpenShiftCluster
expectErrors []utils.ExpectedError
}{
{
name: "create with empty subscription clusters",
subscriptionClusters: nil,
newCluster: makeCluster("new-cluster", "mrg-new", subnetID, nsgID),
expectErrors: []utils.ExpectedError{},
},
{
name: "create rejects duplicate subnet",
subscriptionClusters: []*api.HCPOpenShiftCluster{
makeCluster("existing-cluster", "mrg-existing", subnetID, nsgID),
},
newCluster: makeCluster("new-cluster", "mrg-new", subnetID, otherNsgID),
expectErrors: []utils.ExpectedError{
{FieldPath: "properties.platform.subnetId", Message: "already in use by another cluster"},
},
},
{
name: "create rejects duplicate network security group",
subscriptionClusters: []*api.HCPOpenShiftCluster{
makeCluster("existing-cluster", "mrg-existing", subnetID, nsgID),
},
newCluster: makeCluster("new-cluster", "mrg-new", otherSubnetID, nsgID),
expectErrors: []utils.ExpectedError{
{FieldPath: "properties.platform.networkSecurityGroupId", Message: "already in use by another cluster"},
},
},
{
name: "create rejects duplicate managed resource group",
subscriptionClusters: []*api.HCPOpenShiftCluster{
makeCluster("existing-cluster", "shared-mrg", subnetID, nsgID),
},
newCluster: makeCluster("new-cluster", "shared-mrg", otherSubnetID, otherNsgID),
expectErrors: []utils.ExpectedError{
{FieldPath: "properties.platform.managedResourceGroup", Message: "please provide a unique managed resource group name"},
},
},
{
name: "create rejects duplicate subnet used by node pool",
subscriptionClusters: []*api.HCPOpenShiftCluster{
makeCluster("existing-cluster", "mrg-existing", otherSubnetID, nsgID),
},
subscriptionNodePools: []*api.HCPOpenShiftClusterNodePool{
makeNodePool("existing-cluster", "workers", subnetID),
},
newCluster: makeCluster("new-cluster", "mrg-new", subnetID, otherNsgID),
expectErrors: []utils.ExpectedError{
{FieldPath: "properties.platform.subnetId", Message: "already in use by another cluster"},
},
},
{
name: "create allows distinct platform values",
subscriptionClusters: []*api.HCPOpenShiftCluster{
makeCluster("existing-cluster", "mrg-existing", subnetID, nsgID),
},
newCluster: makeCluster("new-cluster", "mrg-new", otherSubnetID, otherNsgID),
expectErrors: []utils.ExpectedError{},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()

admissionContext := &ClusterAdmissionContext{
OriginalCluster: tt.newCluster.DeepCopy(),
SubscriptionClusters: tt.subscriptionClusters,
SubscriptionNodePools: tt.subscriptionNodePools,
}

errs := AdmitCluster(ctx, admissionContext, operation.Operation{Type: operation.Create}, tt.newCluster, nil)

utils.VerifyErrorsMatch(t, tt.expectErrors, errs)
})
}
}
1 change: 1 addition & 0 deletions internal/validation/hcpopenshiftcluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -806,6 +806,7 @@ func TestClusterValidate(t *testing.T) {
// Use a different resource group name to avoid a subnet ID error.
SubnetID: api.Must(azcorearm.ParseResourceID(path.Join("/subscriptions", api.TestSubscriptionID, "resourceGroups", "anotherResourceGroup", "providers", "Microsoft.Network", "virtualNetworks", api.TestVirtualNetworkName, "subnets", api.TestSubnetName))),
VnetIntegrationSubnetID: api.Must(azcorearm.ParseResourceID(path.Join("/subscriptions", api.TestSubscriptionID, "resourceGroups", "anotherResourceGroup", "providers", "Microsoft.Network", "virtualNetworks", api.TestVirtualNetworkName, "subnets", api.TestVnetIntegrationSubnetName))),
NetworkSecurityGroupID: api.Must(azcorearm.ParseResourceID(path.Join("/subscriptions", api.TestSubscriptionID, "resourceGroups", "anotherResourceGroup", "providers", "Microsoft.Network", "networkSecurityGroups", api.TestNetworkSecurityGroupName))),
},
},
},
Expand Down
5 changes: 5 additions & 0 deletions internal/validation/validate_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,8 @@ func validateResourceIDsAgainstClusterID(ctx context.Context, op operation.Opera
errs = append(errs, DifferentResourceGroupName(ctx, op, field.NewPath("customerProperties", "platform", "managedResourceGroup"), &newCluster.CustomerProperties.Platform.ManagedResourceGroup, nil, newCluster.ID.ResourceGroupName)...)
errs = append(errs, SameSubscription(ctx, op, field.NewPath("customerProperties", "platform", "subnetId"), newCluster.CustomerProperties.Platform.SubnetID, nil, newCluster.ID.SubscriptionID)...)
errs = append(errs, DifferentResourceGroupNameFromResourceID(ctx, op, field.NewPath("customerProperties", "platform", "subnetId"), newCluster.CustomerProperties.Platform.SubnetID, nil, newCluster.CustomerProperties.Platform.ManagedResourceGroup)...)
errs = append(errs, SameSubscription(ctx, op, field.NewPath("customerProperties", "platform", "networkSecurityGroupId"), newCluster.CustomerProperties.Platform.NetworkSecurityGroupID, nil, newCluster.ID.SubscriptionID)...)
errs = append(errs, DifferentResourceGroupNameFromResourceID(ctx, op, field.NewPath("customerProperties", "platform", "networkSecurityGroupId"), newCluster.CustomerProperties.Platform.NetworkSecurityGroupID, nil, newCluster.CustomerProperties.Platform.ManagedResourceGroup)...)
errs = append(errs, SameSubscription(ctx, op, field.NewPath("customerProperties", "platform", "vnetIntegrationSubnetId"), newCluster.CustomerProperties.Platform.VnetIntegrationSubnetID, nil, newCluster.ID.SubscriptionID)...)

for operatorName, operatorIdentity := range newCluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators {
Expand Down Expand Up @@ -633,6 +635,9 @@ func validateCustomerPlatformProfile(ctx context.Context, op operation.Operation
errs = append(errs, validate.RequiredPointer(ctx, op, fldPath.Child("networkSecurityGroupId"), newObj.NetworkSecurityGroupID, safe.Field(oldObj, toPlatformNetworkSecurityGroupID))...)
errs = append(errs, immutableByReflect(ctx, op, fldPath.Child("networkSecurityGroupId"), newObj.NetworkSecurityGroupID, safe.Field(oldObj, toPlatformNetworkSecurityGroupID))...)
errs = append(errs, RestrictedResourceIDWithResourceGroup(ctx, op, fldPath.Child("networkSecurityGroupId"), newObj.NetworkSecurityGroupID, safe.Field(oldObj, toPlatformNetworkSecurityGroupID), "Microsoft.Network/networkSecurityGroups")...)
// Note: SameSubscription and DifferentResourceGroupNameFromResourceID for
// networkSecurityGroupId are performed at the cluster peer-field level in
// validateResourceIDsAgainstClusterID.

//OperatorsAuthentication OperatorsAuthenticationProfile `json:"operatorsAuthentication,omitempty"`
errs = append(errs, immutableByReflect(ctx, op, fldPath.Child("operatorsAuthentication"), &newObj.OperatorsAuthentication, safe.Field(oldObj, toPlatformOperatorsAuthentication))...)
Expand Down
25 changes: 24 additions & 1 deletion internal/validation/validate_cluster_comprehensive_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@ func TestValidateClusterCreate(t *testing.T) {
name: "wrong NSG resource type - create",
cluster: func() *api.HCPOpenShiftCluster {
c := createValidCluster()
c.CustomerProperties.Platform.NetworkSecurityGroupID = api.Must(azcorearm.ParseResourceID("/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet"))
c.CustomerProperties.Platform.NetworkSecurityGroupID = api.Must(azcorearm.ParseResourceID("/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet"))
return c
}(),
expectErrors: []utils.ExpectedError{
Expand Down Expand Up @@ -744,6 +744,7 @@ func TestValidateClusterCreate(t *testing.T) {
}(),
expectErrors: []utils.ExpectedError{
{Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.subnetId"},
{Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.networkSecurityGroupId"},
{Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.vnetIntegrationSubnetId"},
{Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.managedResourceGroup"},
{Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.operatorsAuthentication.userAssignedIdentities.controlPlaneOperators[test-operator]"},
Expand All @@ -761,6 +762,28 @@ func TestValidateClusterCreate(t *testing.T) {
{Message: "must be in the same Azure subscription", FieldPath: "customerProperties.platform.subnetId"},
},
},
{
name: "network security group in different subscription - create",
cluster: func() *api.HCPOpenShiftCluster {
c := createValidCluster()
c.CustomerProperties.Platform.NetworkSecurityGroupID = api.Must(azcorearm.ParseResourceID("/subscriptions/different-sub/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg"))
return c
}(),
expectErrors: []utils.ExpectedError{
{Message: "must be in the same Azure subscription", FieldPath: "customerProperties.platform.networkSecurityGroupId"},
},
},
{
name: "network security group in managed resource group - create",
cluster: func() *api.HCPOpenShiftCluster {
c := createValidCluster()
c.CustomerProperties.Platform.NetworkSecurityGroupID = api.Must(azcorearm.ParseResourceID("/subscriptions/0465bc32-c654-41b8-8d87-9815d7abe8f6/resourceGroups/managed-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg"))
return c
}(),
expectErrors: []utils.ExpectedError{
{Message: "must not be the same resource group name", FieldPath: "customerProperties.platform.networkSecurityGroupId"},
},
},
{
name: "vnet integration subnet is optional - create",
cluster: func() *api.HCPOpenShiftCluster {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@
"serviceCidr": "172.30.0.0/16"
},
"platform": {
"managedResourceGroup": "managed-resource-group-name",
"networkSecurityGroupId": "/subscriptions/a1b2c3d4-0000-0000-0000-afec00000000/resourceGroups/bar/providers/Microsoft.Network/networkSecurityGroups/nsg",
"managedResourceGroup": "managed-resource-group-create-with-419",
"networkSecurityGroupId": "/subscriptions/a1b2c3d4-0000-0000-0000-afec00000000/resourceGroups/bar/providers/Microsoft.Network/networkSecurityGroups/nsg-create-with-419",
"outboundType": "LoadBalancer",
"subnetId": "/subscriptions/a1b2c3d4-0000-0000-0000-afec00000000/resourceGroups/bar/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet"
"subnetId": "/subscriptions/a1b2c3d4-0000-0000-0000-afec00000000/resourceGroups/bar/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet-create-with-419"
},
"version": {
"channelGroup": "stable",
Expand Down