From 49276abb1a2619d38a7f5bd87e98a9647b2d324f Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Sun, 4 Oct 2026 07:39:45 -0400 Subject: [PATCH] feat: Let an egress shard translate the NAT64 WKP Public DNS64 resolvers synthesize into the RFC 6052 Well-Known Prefix 64:ff9b::/96, but a shard translates only its Network-Specific Prefix, so clients using those resolvers blackhole. nat64Prefix is write-once, so add a separate opt-in rather than changing it. Key changes: - Add spec.translateWellKnownPrefix, a mutable boolean that additionally translates the WKP from the shard's IPv4 address; a CEL rule requires nat64Prefix when it is true - Add status.translatesWellKnownPrefix reporting what the datapath programs - Document that the WKP must not map non-global IPv4 (RFC 6052 3.1) and that disabling the flag blackholes already-resolved WKP destinations until DNS TTLs expire - Cover the rule and the toggle in the e2e CEL test and the JSON keys in unit tests --- api/v1alpha1/egressshard_types.go | 26 ++++++++- api/v1alpha1/egressshard_types_test.go | 5 ++ .../network.datumapis.com_egressshards.yaml | 28 ++++++++- docs/api/bgp.md | 4 +- docs/api/gateway.md | 4 +- .../chainsaw-test.yaml | 57 ++++++++++++++++++- 6 files changed, 116 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/egressshard_types.go b/api/v1alpha1/egressshard_types.go index cc1b78e..c0e5dec 100644 --- a/api/v1alpha1/egressshard_types.go +++ b/api/v1alpha1/egressshard_types.go @@ -212,6 +212,7 @@ type AddressClaimRef struct { // +kubebuilder:validation:XValidation:rule="!(has(oldSelf.nat64Prefix) && size(oldSelf.nat64Prefix) > 0) || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="nat64Prefix cannot be unassigned once assigned" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.shardAddressIPv6ClaimRef) || has(self.shardAddressIPv6ClaimRef)",message="shardAddressIPv6ClaimRef cannot be unassigned once assigned" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.shardAddressIPv4ClaimRef) || has(self.shardAddressIPv4ClaimRef)",message="shardAddressIPv4ClaimRef cannot be unassigned once assigned" +// +kubebuilder:validation:XValidation:rule="!(has(self.translateWellKnownPrefix) && self.translateWellKnownPrefix) || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="translateWellKnownPrefix requires nat64Prefix" type EgressShardSpec struct { // TargetRef identifies the Node this shard executes on. // +kubebuilder:validation:Required @@ -283,8 +284,9 @@ type EgressShardSpec struct { ShardAddressIPv4ClaimRef *AddressClaimRef `json:"shardAddressIPv4ClaimRef,omitempty"` // NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard - // translates to IPv4 — one Datum-operated Network-Specific Prefix, shared - // fabric-wide, never per-tenant. It must be the prefix the resolver + // translates to IPv4 — a Datum-operated Network-Specific Prefix, shared + // fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052 + // Well-Known Prefix alongside it. It must be the prefix the resolver // synthesizes into; a shard translating for a different one is a // blackhole with no symptom on either side. // @@ -297,6 +299,20 @@ type EgressShardSpec struct { // +kubebuilder:validation:XValidation:rule="self == '' || isCIDR(self)",message="nat64Prefix must be a valid CIDR" // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="nat64Prefix is immutable once assigned" NAT64Prefix string `json:"nat64Prefix,omitempty"` + + // TranslateWellKnownPrefix additionally translates destinations inside the + // RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so + // clients using a public DNS64 resolver reach IPv4 destinations. Requires + // NAT64Prefix. + // + // RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global + // IPv4 addresses; the implementation must refuse such destinations. + // + // Mutable, unlike NAT64Prefix. Disabling it blackholes every destination + // clients already resolved into the Well-Known Prefix until their DNS TTLs + // expire. + // +optional + TranslateWellKnownPrefix bool `json:"translateWellKnownPrefix,omitempty"` } // EgressShardStatus defines the observed state of an EgressShard. @@ -345,6 +361,12 @@ type EgressShardStatus struct { // +kubebuilder:validation:XValidation:rule="self == '' || isCIDR(self)",message="nat64Prefix must be a valid CIDR" NAT64Prefix string `json:"nat64Prefix,omitempty"` + // TranslatesWellKnownPrefix reports whether this shard's datapath is + // programmed to translate the RFC 6052 Well-Known Prefix 64:ff9b::/96, from + // Spec.TranslateWellKnownPrefix. False whenever ShardAddressIPv4 is empty. + // +optional + TranslatesWellKnownPrefix bool `json:"translatesWellKnownPrefix,omitempty"` + // Conditions contains the standard conditions for this resource, // including Programmed (see ConditionTypeProgrammed). // diff --git a/api/v1alpha1/egressshard_types_test.go b/api/v1alpha1/egressshard_types_test.go index 4deddcb..36da54d 100644 --- a/api/v1alpha1/egressshard_types_test.go +++ b/api/v1alpha1/egressshard_types_test.go @@ -142,6 +142,7 @@ func TestEgressShardSpecFieldNames(t *testing.T) { orig := newTestEgressShard() orig.Spec.ShardAddressIPv4 = "198.51.100.7" orig.Spec.NAT64Prefix = "64:ff9b::/96" + orig.Spec.TranslateWellKnownPrefix = true data, err := json.Marshal(orig.Spec) if err != nil { @@ -164,6 +165,9 @@ func TestEgressShardSpecFieldNames(t *testing.T) { t.Errorf("%s: got %v, want %q", key, raw, value) } } + if raw, ok := m["translateWellKnownPrefix"]; !ok || raw != true { + t.Errorf("translateWellKnownPrefix: got %v, want true", raw) + } if _, ok := m["targetRef"]; !ok { t.Error("expected \"targetRef\" key to be present") } @@ -205,6 +209,7 @@ func TestEgressShardSpecOmitEmpty(t *testing.T) { for _, key := range []string{ "shardSID", "shardAddressIPv6", "shardAddressIPv4", "nat64Prefix", "shardAddressIPv6ClaimRef", "shardAddressIPv4ClaimRef", + "translateWellKnownPrefix", } { if _, ok := m[key]; ok { t.Errorf("expected %q key to be absent when empty", key) diff --git a/config/crd/network.datumapis.com_egressshards.yaml b/config/crd/network.datumapis.com_egressshards.yaml index f9d2e3a..60adfcd 100644 --- a/config/crd/network.datumapis.com_egressshards.yaml +++ b/config/crd/network.datumapis.com_egressshards.yaml @@ -123,8 +123,9 @@ spec: nat64Prefix: description: |- NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard - translates to IPv4 — one Datum-operated Network-Specific Prefix, shared - fabric-wide, never per-tenant. It must be the prefix the resolver + translates to IPv4 — a Datum-operated Network-Specific Prefix, shared + fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052 + Well-Known Prefix alongside it. It must be the prefix the resolver synthesizes into; a shard translating for a different one is a blackhole with no symptom on either side. @@ -300,6 +301,20 @@ spec: - kind - name type: object + translateWellKnownPrefix: + description: |- + TranslateWellKnownPrefix additionally translates destinations inside the + RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so + clients using a public DNS64 resolver reach IPv4 destinations. Requires + NAT64Prefix. + + RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global + IPv4 addresses; the implementation must refuse such destinations. + + Mutable, unlike NAT64Prefix. Disabling it blackholes every destination + clients already resolved into the Well-Known Prefix until their DNS TTLs + expire. + type: boolean required: - targetRef type: object @@ -325,6 +340,9 @@ spec: rule: '!has(oldSelf.shardAddressIPv6ClaimRef) || has(self.shardAddressIPv6ClaimRef)' - message: shardAddressIPv4ClaimRef cannot be unassigned once assigned rule: '!has(oldSelf.shardAddressIPv4ClaimRef) || has(self.shardAddressIPv4ClaimRef)' + - message: translateWellKnownPrefix requires nat64Prefix + rule: '!(has(self.translateWellKnownPrefix) && self.translateWellKnownPrefix) + || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)' status: description: |- EgressShardStatus defines the observed state of an EgressShard. @@ -444,6 +462,12 @@ spec: x-kubernetes-validations: - message: shardSID must be a valid IPv6 address rule: self == '' || (isIP(self) && ip(self).family() == 6) + translatesWellKnownPrefix: + description: |- + TranslatesWellKnownPrefix reports whether this shard's datapath is + programmed to translate the RFC 6052 Well-Known Prefix 64:ff9b::/96, from + Spec.TranslateWellKnownPrefix. False whenever ShardAddressIPv4 is empty. + type: boolean type: object type: object served: true diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 07d24b9..d8345bb 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -1063,7 +1063,8 @@ _Appears in:_ | `shardAddressIPv6ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv6ClaimRef records the addressing-service claim
ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | | `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim
ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | -| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — a Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052
Well-Known Prefix alongside it. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | +| `translateWellKnownPrefix` _boolean_ | TranslateWellKnownPrefix additionally translates destinations inside the
RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so
clients using a public DNS64 resolver reach IPv4 destinations. Requires
NAT64Prefix.
RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global
IPv4 addresses; the implementation must refuse such destinations.
Mutable, unlike NAT64Prefix. Disabling it blackholes every destination
clients already resolved into the Well-Known Prefix until their DNS TTLs
expire. | | | #### EgressShardStatus @@ -1089,6 +1090,7 @@ _Appears in:_ | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is
programmed with. Empty means it translates no IPv6 flow. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is
programmed with. Empty means it performs no NAT64. Publishing it is also
what makes the underlay reachability prerequisite in
Spec.ShardAddressIPv4 checkable. | | | | `nat64Prefix` _string_ | NAT64Prefix is the prefix this shard's datapath is programmed to
translate. Empty whenever ShardAddressIPv4 is empty. | | | +| `translatesWellKnownPrefix` _boolean_ | TranslatesWellKnownPrefix reports whether this shard's datapath is
programmed to translate the RFC 6052 Well-Known Prefix 64:ff9b::/96, from
Spec.TranslateWellKnownPrefix. False whenever ShardAddressIPv4 is empty. | | | | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource,
including Programmed (see ConditionTypeProgrammed). | | | diff --git a/docs/api/gateway.md b/docs/api/gateway.md index cff36ee..7cda32a 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -164,7 +164,8 @@ _Appears in:_ | `shardAddressIPv6ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv6ClaimRef records the addressing-service claim
ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | | `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim
ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | -| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — a Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052
Well-Known Prefix alongside it. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | +| `translateWellKnownPrefix` _boolean_ | TranslateWellKnownPrefix additionally translates destinations inside the
RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so
clients using a public DNS64 resolver reach IPv4 destinations. Requires
NAT64Prefix.
RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global
IPv4 addresses; the implementation must refuse such destinations.
Mutable, unlike NAT64Prefix. Disabling it blackholes every destination
clients already resolved into the Well-Known Prefix until their DNS TTLs
expire. | | | #### EgressShardStatus @@ -190,6 +191,7 @@ _Appears in:_ | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is
programmed with. Empty means it translates no IPv6 flow. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is
programmed with. Empty means it performs no NAT64. Publishing it is also
what makes the underlay reachability prerequisite in
Spec.ShardAddressIPv4 checkable. | | | | `nat64Prefix` _string_ | NAT64Prefix is the prefix this shard's datapath is programmed to
translate. Empty whenever ShardAddressIPv4 is empty. | | | +| `translatesWellKnownPrefix` _boolean_ | TranslatesWellKnownPrefix reports whether this shard's datapath is
programmed to translate the RFC 6052 Well-Known Prefix 64:ff9b::/96, from
Spec.TranslateWellKnownPrefix. False whenever ShardAddressIPv4 is empty. | | | | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource,
including Programmed (see ConditionTypeProgrammed). | | | diff --git a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml index 40c61cf..6452818 100644 --- a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml +++ b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml @@ -10,7 +10,8 @@ spec: value is accepted, and changing or unassigning an assigned value is rejected. Also verifies that a claim reference defaults its apiGroup and kind, and that an address and a claim reference are independently - optional. Also verifies that status remains writable, + optional. Also verifies that translateWellKnownPrefix requires nat64Prefix + and, unlike it, can be turned off again. Also verifies that status remains writable, since status reports what the datapath actually programmed and must still converge. Does not require any operator to be running — every rule under test is enforced by the API server alone. @@ -353,6 +354,57 @@ spec: kubectl delete egressshard e2e-egress-shard-pending -n "$NAMESPACE" echo "OK: a claim reference whose allocation has not resolved yet is accepted" + - name: reject-well-known-prefix-without-nat64 + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-wkp-only + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv6: 2001:db8:f00d::101 + translateWellKnownPrefix: true + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of translateWellKnownPrefix with no nat64Prefix but the resource was accepted" + kubectl delete egressshard e2e-egress-shard-wkp-only -n "$NAMESPACE" 2>/dev/null || true + exit 1 + fi + case "$OUTPUT" in + *"translateWellKnownPrefix requires nat64Prefix"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: translateWellKnownPrefix with no nat64Prefix correctly rejected" + echo "Server response: $OUTPUT" + + - name: toggle-well-known-prefix + try: + - script: + content: | + set -e + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"translateWellKnownPrefix":true}}' + ON=$(kubectl get egressshard e2e-egress-shard -n "$NAMESPACE" \ + -o jsonpath='{.spec.translateWellKnownPrefix}') + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"translateWellKnownPrefix":false}}' + OFF=$(kubectl get egressshard e2e-egress-shard -n "$NAMESPACE" \ + -o jsonpath='{.spec.translateWellKnownPrefix}') + if [ "$ON" != "true" ] || [ "$OFF" != "false" ]; then + echo "ERROR: translateWellKnownPrefix toggled to $ON then $OFF, want true then false" + exit 1 + fi + echo "OK: translateWellKnownPrefix enabled with nat64Prefix set, then disabled" + - name: accept-status-changes try: - script: @@ -363,7 +415,7 @@ spec: {"status":{"shardAddressIPv6":"2001:db8:f00d::100","shardSID":"2001:db8:ff01::"}}' kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" \ --subresource=status --type=merge -p ' - {"status":{"shardAddressIPv6":"2001:db8:f00d::200"}}' + {"status":{"shardAddressIPv6":"2001:db8:f00d::200","translatesWellKnownPrefix":true}}' echo "OK: status reports what the datapath programmed and stays mutable" - assert: resource: @@ -373,6 +425,7 @@ spec: name: e2e-egress-shard status: shardAddressIPv6: 2001:db8:f00d::200 + translatesWellKnownPrefix: true finally: - delete: