From f61336905715b990b8c8383d72dd76fcc20d62e2 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Mon, 12 Oct 2026 19:18:53 -0400 Subject: [PATCH 1/2] feat(api): Refuse IPv4 NetworkService VIPs until galactic serves them galactic's gateway datapath cannot serve an IPv4 VIP yet (datum-cloud/galactic#705), but vipAddresses admitted one, so a service could carry an address that never answers. A second CEL rule on vipAddresses now refuses any address whose family is not IPv6, using the same CEL IP library as the existing isIP rule. An IPv4-mapped IPv6 address is already refused by isIP. Existing objects keep their IPv4 VIPs while vipAddresses is unchanged, because the API server ratchets validation of an unchanged field (Kubernetes 1.30 and later). Any edit to the list must drop the IPv4 entry. An envtest covers both cases against a real API server, and a Chainsaw test covers the refusal in kind. The rule is replaced by per-family limits once galactic serves IPv4 VIPs. Co-Authored-By: Claude Opus 5.5 --- .../networkservice_admission_envtest_test.go | 204 ++++++++++++++++++ api/v1alpha1/networkservice_types.go | 11 +- ...network.datumapis.com_networkservices.yaml | 13 +- docs/api/gateway.md | 2 +- .../chainsaw-test.yaml | 72 +++++++ 5 files changed, 297 insertions(+), 5 deletions(-) create mode 100644 api/v1alpha1/networkservice_admission_envtest_test.go create mode 100644 test/e2e/tests/networkservice-vip-family/chainsaw-test.yaml diff --git a/api/v1alpha1/networkservice_admission_envtest_test.go b/api/v1alpha1/networkservice_admission_envtest_test.go new file mode 100644 index 0000000..f76c69d --- /dev/null +++ b/api/v1alpha1/networkservice_admission_envtest_test.go @@ -0,0 +1,204 @@ +// Copyright 2026 Datum Cloud, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later + +package v1alpha1_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/yaml" + + networkv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// ipv4VIPRuleMarker identifies the vipAddresses rule that refuses IPv4 VIPs +// until galactic serves them. +const ipv4VIPRuleMarker = "ip(v).family() == 6" + +// TestNetworkServiceVIPFamilyAdmission covers the IPv4 VIP stopgap against a +// real API server: a new IPv4 VIP is refused and an IPv6 one admitted, and a +// rule that already held an IPv4 VIP before the rule was installed survives +// edits that leave vipAddresses alone, but not an edit to the list. +func TestNetworkServiceVIPFamilyAdmission(t *testing.T) { + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("set KUBEBUILDER_ASSETS for isolated API admission tests") + } + data, err := os.ReadFile(filepath.Join("..", "..", "config", "crd", "network.datumapis.com_networkservices.yaml")) + if err != nil { + t.Fatal(err) + } + full := &apiextensionsv1.CustomResourceDefinition{} + if err := yaml.Unmarshal(data, full); err != nil { + t.Fatal(err) + } + // The CRD as it was before the rule, to create an object the current + // schema would refuse. + legacy := full.DeepCopy() + if !dropVIPFamilyRule(legacy) { + t.Fatalf("CRD has no vipAddresses rule containing %q", ipv4VIPRuleMarker) + } + + environment := &envtest.Environment{CRDs: []*apiextensionsv1.CustomResourceDefinition{legacy}} + cfg, err := environment.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Errorf("stop API: %v", err) + } + }) + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = apiextensionsv1.AddToScheme(scheme) + _ = networkv1alpha1.AddToScheme(scheme) + c, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + const ns = "vip-family" + if err := c.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}); err != nil { + t.Fatal(err) + } + rule := func(name string, vips ...string) *networkv1alpha1.NetworkService { + return &networkv1alpha1.NetworkService{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: networkv1alpha1.NetworkServiceSpec{ + VPCRef: "vpc-a", + VIPAddresses: vips, + Protocol: networkv1alpha1.NetworkServiceProtocolTCP, + Port: 443, + BackendSelector: metav1.LabelSelector{MatchLabels: map[string]string{"app": "web"}}, + BackendPort: 8443, + }, + } + } + + existing := rule("existing", "2001:db8:1::10", "203.0.113.10") + if err := c.Create(ctx, existing); err != nil { + t.Fatalf("create IPv4 rule under the legacy schema: %v", err) + } + installSchema(ctx, t, c, full) + + for _, tt := range []struct { + name string + vips []string + wantError string + }{ + {"ipv6", []string{"2001:db8:1::11"}, ""}, + {"ipv4", []string{"203.0.113.11"}, "galactic#705"}, + {"dual-stack", []string{"2001:db8:1::12", "203.0.113.12"}, "galactic#705"}, + // isIP refuses an IPv4-mapped IPv6 address, so the first rule does. + {"ipv4-mapped", []string{"::ffff:203.0.113.13"}, "valid IPv4 or IPv6"}, + } { + t.Run(tt.name, func(t *testing.T) { + err := c.Create(ctx, rule(tt.name, tt.vips...)) + if tt.wantError == "" { + if err != nil { + t.Fatalf("create: %v", err) + } + return + } + if !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), tt.wantError) { + t.Fatalf("create error = %v, want a schema rejection naming %q", err, tt.wantError) + } + }) + } + + t.Run("existing rule keeps an unchanged IPv4 VIP", func(t *testing.T) { + current := &networkv1alpha1.NetworkService{} + if err := c.Get(ctx, client.ObjectKeyFromObject(existing), current); err != nil { + t.Fatal(err) + } + current.Labels = map[string]string{"edited": "true"} + current.Spec.Port = 8443 + if err := c.Update(ctx, current); err != nil { + t.Fatalf("update leaving vipAddresses unchanged: %v", err) + } + }) + + t.Run("existing rule cannot edit a list holding an IPv4 VIP", func(t *testing.T) { + current := &networkv1alpha1.NetworkService{} + if err := c.Get(ctx, client.ObjectKeyFromObject(existing), current); err != nil { + t.Fatal(err) + } + current.Spec.VIPAddresses = append(current.Spec.VIPAddresses, "2001:db8:1::13") + err := c.Update(ctx, current) + if !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), "galactic#705") { + t.Fatalf("update adding a VIP beside an IPv4 VIP: err = %v, want the IPv4 VIP rule's rejection", err) + } + current.Spec.VIPAddresses = []string{"2001:db8:1::10"} + if err := c.Update(ctx, current); err != nil { + t.Fatalf("update dropping the IPv4 VIP: %v", err) + } + }) +} + +// dropVIPFamilyRule removes the IPv4 VIP rule from crd's vipAddresses schema, +// reporting whether it was there. +func dropVIPFamilyRule(crd *apiextensionsv1.CustomResourceDefinition) bool { + field := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"].Properties["vipAddresses"] + kept := field.XValidations[:0] + for _, v := range field.XValidations { + if !strings.Contains(v.Rule, ipv4VIPRuleMarker) { + kept = append(kept, v) + } + } + dropped := len(kept) != len(field.XValidations) + field.XValidations = kept + crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"].Properties["vipAddresses"] = field + return dropped +} + +// installSchema replaces the served NetworkService schema with want's and waits +// until the API server enforces it, which a fresh create of an IPv4 VIP +// shows. +func installSchema(ctx context.Context, t *testing.T, c client.Client, want *apiextensionsv1.CustomResourceDefinition) { + t.Helper() + current := &apiextensionsv1.CustomResourceDefinition{} + if err := c.Get(ctx, client.ObjectKey{Name: want.Name}, current); err != nil { + t.Fatal(err) + } + current.Spec.Versions = want.Spec.Versions + if err := c.Update(ctx, current); err != nil { + t.Fatalf("install the current NetworkService schema: %v", err) + } + probe := &networkv1alpha1.NetworkService{ + ObjectMeta: metav1.ObjectMeta{Name: "schema-probe", Namespace: "vip-family"}, + Spec: networkv1alpha1.NetworkServiceSpec{ + VPCRef: "vpc-a", + VIPAddresses: []string{"198.51.100.1"}, + Protocol: networkv1alpha1.NetworkServiceProtocolTCP, + Port: 443, + BackendSelector: metav1.LabelSelector{MatchLabels: map[string]string{"app": "web"}}, + BackendPort: 8443, + }, + } + var lastErr error + err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 30*time.Second, true, + func(ctx context.Context) (bool, error) { + p := probe.DeepCopy() + lastErr = c.Create(ctx, p) + if lastErr == nil { + return false, c.Delete(ctx, p) + } + return apierrors.IsInvalid(lastErr) && strings.Contains(lastErr.Error(), "galactic#705"), nil + }) + if err != nil { + t.Fatalf("API server never enforced the current NetworkService schema: %v (last create: %v)", err, lastErr) + } +} diff --git a/api/v1alpha1/networkservice_types.go b/api/v1alpha1/networkservice_types.go index 20e9cc4..c706e8a 100644 --- a/api/v1alpha1/networkservice_types.go +++ b/api/v1alpha1/networkservice_types.go @@ -82,13 +82,20 @@ type NetworkServiceSpec struct { // +kubebuilder:validation:MinLength=1 VPCRef string `json:"vpcRef"` - // VIPAddresses is the list of ingress VIP addresses (IPv4 and/or IPv6) - // this service provisions on the assigned gateway node(s). + // VIPAddresses is the list of ingress VIP addresses this service provisions + // on the assigned gateway node(s). + // + // Only IPv6 addresses are admitted until the gateway datapath serves IPv4 + // VIPs (datum-cloud/galactic#705). A service created with an IPv4 VIP before + // this restriction keeps it while the list is unchanged, since the API + // server ratchets validation of an unchanged field; any edit to the list + // must drop it. The gateways do not load or advertise an IPv4 VIP. // +kubebuilder:validation:Required // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=8 // +kubebuilder:validation:items:MaxLength=45 // +kubebuilder:validation:XValidation:rule="self.all(v, isIP(v))",message="vipAddresses must all be valid IPv4 or IPv6 addresses" + // +kubebuilder:validation:XValidation:rule="self.all(v, !isIP(v) || ip(v).family() == 6)",message="vipAddresses must be IPv6: IPv4 VIPs are not served yet (datum-cloud/galactic#705)" // +listType=set VIPAddresses []string `json:"vipAddresses"` diff --git a/config/crd/network.datumapis.com_networkservices.yaml b/config/crd/network.datumapis.com_networkservices.yaml index ef17865..67a6691 100644 --- a/config/crd/network.datumapis.com_networkservices.yaml +++ b/config/crd/network.datumapis.com_networkservices.yaml @@ -160,8 +160,14 @@ spec: type: string vipAddresses: description: |- - VIPAddresses is the list of ingress VIP addresses (IPv4 and/or IPv6) - this service provisions on the assigned gateway node(s). + VIPAddresses is the list of ingress VIP addresses this service provisions + on the assigned gateway node(s). + + Only IPv6 addresses are admitted until the gateway datapath serves IPv4 + VIPs (datum-cloud/galactic#705). A service created with an IPv4 VIP before + this restriction keeps it while the list is unchanged, since the API + server ratchets validation of an unchanged field; any edit to the list + must drop it. The gateways do not load or advertise an IPv4 VIP. items: maxLength: 45 type: string @@ -172,6 +178,9 @@ spec: x-kubernetes-validations: - message: vipAddresses must all be valid IPv4 or IPv6 addresses rule: self.all(v, isIP(v)) + - message: 'vipAddresses must be IPv6: IPv4 VIPs are not served yet + (datum-cloud/galactic#705)' + rule: self.all(v, !isIP(v) || ip(v).family() == 6) vpcRef: description: |- VPCRef is the opaque identifier of the target VPC this service applies diff --git a/docs/api/gateway.md b/docs/api/gateway.md index 6d088aa..cec66d6 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -432,7 +432,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `vpcRef` _string_ | VPCRef is the opaque identifier of the target VPC this service applies
to. This repo does not own the VPC API and does not validate the
identifier beyond non-emptiness; the admission webhook of the
consuming controller is responsible for verifying the requester is
authorized for this VPC before the service is accepted. | | MinLength: 1
Required: \{\}
| -| `vipAddresses` _string array_ | VIPAddresses is the list of ingress VIP addresses (IPv4 and/or IPv6)
this service provisions on the assigned gateway node(s). | | MaxItems: 8
MinItems: 1
Required: \{\}
items:MaxLength: 45
| +| `vipAddresses` _string array_ | VIPAddresses is the list of ingress VIP addresses this service provisions
on the assigned gateway node(s).
Only IPv6 addresses are admitted until the gateway datapath serves IPv4
VIPs (datum-cloud/galactic#705). A service created with an IPv4 VIP before
this restriction keeps it while the list is unchanged, since the API
server ratchets validation of an unchanged field; any edit to the list
must drop it. The gateways do not load or advertise an IPv4 VIP. | | MaxItems: 8
MinItems: 1
Required: \{\}
items:MaxLength: 45
| | `protocol` _[NetworkServiceProtocol](#networkserviceprotocol)_ | Protocol is the transport protocol matched by VIPAddresses/Port. | | Enum: [tcp udp]
Required: \{\}
| | `port` _integer_ | Port is the ingress port on VIPAddresses that this service load-balances. | | Maximum: 65535
Minimum: 1
Required: \{\}
| | `backendSelector` _[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#labelselector-v1-meta)_ | BackendSelector selects the VPCAttachments (cloud.datumapis.com/v1alpha1,
in any namespace) that serve this service, matched against each
VPCAttachment object's own labels. Only attachments whose status.vpc
equals VPCRef are candidates, whatever their labels, so a selector cannot
reach into another tenant's VPC. Each selected attachment contributes the
IPv6 addresses in its spec.interface.addresses as backends, on
BackendPort; the DSR datapath carries IPv6 backends only. An attachment
with no status.node or no IPv6 address is not a backend until it has
both. An empty selector is rejected rather than read as "every attachment
in the VPC". | | Required: \{\}
| diff --git a/test/e2e/tests/networkservice-vip-family/chainsaw-test.yaml b/test/e2e/tests/networkservice-vip-family/chainsaw-test.yaml new file mode 100644 index 0000000..3c41b80 --- /dev/null +++ b/test/e2e/tests/networkservice-vip-family/chainsaw-test.yaml @@ -0,0 +1,72 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkservice-vip-family +spec: + description: > + Verify that a NetworkService admits IPv6 VIPs and refuses IPv4 ones, alone or + beside an IPv6 VIP, until the gateway datapath serves IPv4 VIPs + (datum-cloud/galactic#705). Does not require any operator to be running; + the rule is enforced by the API server alone. + steps: + - name: accept-ipv6-vip + try: + - apply: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: NetworkService + metadata: + name: e2e-service-ipv6-vip + spec: + vpcRef: vpc-a + vipAddresses: + - 2001:db8:1::20 + protocol: tcp + port: 443 + backendSelector: + matchLabels: + app: web + backendPort: 8443 + - assert: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: NetworkService + metadata: + name: e2e-service-ipv6-vip + spec: + vipAddresses: + - 2001:db8:1::20 + + - name: reject-ipv4-vips + try: + - script: + content: | + set +e + for VIPS in '["203.0.113.20"]' '["2001:db8:1::21", "203.0.113.21"]'; do + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 < Date: Mon, 12 Oct 2026 19:27:16 -0400 Subject: [PATCH 2/2] test(api): Name the VIP family envtest after what it covers PR #34 adds rule_admission_envtest_test.go too, so the two would conflict. Also state on the field that ratcheting needs Kubernetes 1.30 or later, and what an older API server does instead. Co-Authored-By: Claude Opus 5.5 --- api/v1alpha1/networkservice_types.go | 6 ++++-- ...est_test.go => networkservice_vipfamily_envtest_test.go} | 0 config/crd/network.datumapis.com_networkservices.yaml | 6 ++++-- docs/api/gateway.md | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) rename api/v1alpha1/{networkservice_admission_envtest_test.go => networkservice_vipfamily_envtest_test.go} (100%) diff --git a/api/v1alpha1/networkservice_types.go b/api/v1alpha1/networkservice_types.go index c706e8a..e61c843 100644 --- a/api/v1alpha1/networkservice_types.go +++ b/api/v1alpha1/networkservice_types.go @@ -88,8 +88,10 @@ type NetworkServiceSpec struct { // Only IPv6 addresses are admitted until the gateway datapath serves IPv4 // VIPs (datum-cloud/galactic#705). A service created with an IPv4 VIP before // this restriction keeps it while the list is unchanged, since the API - // server ratchets validation of an unchanged field; any edit to the list - // must drop it. The gateways do not load or advertise an IPv4 VIP. + // server ratchets validation of an unchanged field on Kubernetes 1.30 and + // later; any edit to the list must drop it, and on an older API server any + // update to such a service is refused. The gateways do not load or advertise + // an IPv4 VIP. // +kubebuilder:validation:Required // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=8 diff --git a/api/v1alpha1/networkservice_admission_envtest_test.go b/api/v1alpha1/networkservice_vipfamily_envtest_test.go similarity index 100% rename from api/v1alpha1/networkservice_admission_envtest_test.go rename to api/v1alpha1/networkservice_vipfamily_envtest_test.go diff --git a/config/crd/network.datumapis.com_networkservices.yaml b/config/crd/network.datumapis.com_networkservices.yaml index 67a6691..7b60d3c 100644 --- a/config/crd/network.datumapis.com_networkservices.yaml +++ b/config/crd/network.datumapis.com_networkservices.yaml @@ -166,8 +166,10 @@ spec: Only IPv6 addresses are admitted until the gateway datapath serves IPv4 VIPs (datum-cloud/galactic#705). A service created with an IPv4 VIP before this restriction keeps it while the list is unchanged, since the API - server ratchets validation of an unchanged field; any edit to the list - must drop it. The gateways do not load or advertise an IPv4 VIP. + server ratchets validation of an unchanged field on Kubernetes 1.30 and + later; any edit to the list must drop it, and on an older API server any + update to such a service is refused. The gateways do not load or advertise + an IPv4 VIP. items: maxLength: 45 type: string diff --git a/docs/api/gateway.md b/docs/api/gateway.md index cec66d6..8d1eadb 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -432,7 +432,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `vpcRef` _string_ | VPCRef is the opaque identifier of the target VPC this service applies
to. This repo does not own the VPC API and does not validate the
identifier beyond non-emptiness; the admission webhook of the
consuming controller is responsible for verifying the requester is
authorized for this VPC before the service is accepted. | | MinLength: 1
Required: \{\}
| -| `vipAddresses` _string array_ | VIPAddresses is the list of ingress VIP addresses this service provisions
on the assigned gateway node(s).
Only IPv6 addresses are admitted until the gateway datapath serves IPv4
VIPs (datum-cloud/galactic#705). A service created with an IPv4 VIP before
this restriction keeps it while the list is unchanged, since the API
server ratchets validation of an unchanged field; any edit to the list
must drop it. The gateways do not load or advertise an IPv4 VIP. | | MaxItems: 8
MinItems: 1
Required: \{\}
items:MaxLength: 45
| +| `vipAddresses` _string array_ | VIPAddresses is the list of ingress VIP addresses this service provisions
on the assigned gateway node(s).
Only IPv6 addresses are admitted until the gateway datapath serves IPv4
VIPs (datum-cloud/galactic#705). A service created with an IPv4 VIP before
this restriction keeps it while the list is unchanged, since the API
server ratchets validation of an unchanged field on Kubernetes 1.30 and
later; any edit to the list must drop it, and on an older API server any
update to such a service is refused. The gateways do not load or advertise
an IPv4 VIP. | | MaxItems: 8
MinItems: 1
Required: \{\}
items:MaxLength: 45
| | `protocol` _[NetworkServiceProtocol](#networkserviceprotocol)_ | Protocol is the transport protocol matched by VIPAddresses/Port. | | Enum: [tcp udp]
Required: \{\}
| | `port` _integer_ | Port is the ingress port on VIPAddresses that this service load-balances. | | Maximum: 65535
Minimum: 1
Required: \{\}
| | `backendSelector` _[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#labelselector-v1-meta)_ | BackendSelector selects the VPCAttachments (cloud.datumapis.com/v1alpha1,
in any namespace) that serve this service, matched against each
VPCAttachment object's own labels. Only attachments whose status.vpc
equals VPCRef are candidates, whatever their labels, so a selector cannot
reach into another tenant's VPC. Each selected attachment contributes the
IPv6 addresses in its spec.interface.addresses as backends, on
BackendPort; the DSR datapath carries IPv6 backends only. An attachment
with no status.node or no IPv6 address is not a backend until it has
both. An empty selector is rejected rather than read as "every attachment
in the VPC". | | Required: \{\}
|