Enable native Pods to use a VPC as their primary network

cmd/main.go

@@ -408,6 +408,24 @@ func main() {
 	}
 
 	if enableCellControllers {
+		if features.FeatureGate.Enabled(features.PodNetworkAttachment) {
+			if !features.FeatureGate.Enabled(features.NetworkingIntegration) {
+				setupLog.Error(nil, "Pod network attachment requires NetworkingIntegration")
+				os.Exit(1)
+			}
+			if serverConfig.WebhookServer == nil {
+				setupLog.Error(nil, "Pod primary network admission requires a cell webhook server")
+				os.Exit(1)
+			}
+			mgr.GetLocalManager().GetWebhookServer().Register(
+				computewebhook.PodNetworkWebhookPath,
+				computewebhook.NewPodNetworkAttachmentHandler(scheme),
+			)
+			if err = (&controller.PodNetworkAttachmentReconciler{}).SetupWithManager(mgr.GetLocalManager()); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "PodNetworkAttachment")
+				os.Exit(1)
+			}
+		}
 		clusterNameForProject := func(_ string) multicluster.ClusterName {
 			return multicluster.ClusterName(singleClusterName)
 		}

config/components/controller_rbac/role.yaml

@@ -4,6 +4,15 @@ kind: ClusterRole
 metadata:
   name: compute
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
 - apiGroups:
   - ""
   resources:

config/components/pod-network-attachment/README.md

@@ -0,0 +1,35 @@
+# Native Pod primary VPC attachment
+
+Use `config/overlays/cell-pod-network` in a **cell** only when NSO, Cloud, Multus,
+Galactic, the networking CRDs, and a working Compute webhook serving certificate
+are present. The component enables `NetworkingIntegration` and
+`PodNetworkAttachment`; merge additional feature gates into the Deployment as
+needed. Configure the cert-manager CSI issuer on the mounted volume and inject
+the webhook configuration's CA bundle in the installing infrastructure. The
+webhook service already exists in the manager base.
+
+Opt in a Pod or a DaemonSet's Pod template with both:
+
+```yaml
+metadata:
+  labels:
+    compute.datumapis.com/primary-network-enabled: "true"
+  annotations:
+    compute.datumapis.com/primary-network: galactic
+```
+
+The network must exist in the Pod namespace. Admission adds a scheduling gate.
+Compute creates a UID-scoped `NetworkInterfaceClaim` owned by the Pod. Once
+Bound, Allocated, and Prepared are true, it writes the namespace-qualified
+Cloud NAD as `v1.multus-cni.io/default-network`, then removes the gate in a
+separate API write. Kubernetes garbage collection deletes the claim with the
+Pod. Do not use a network probe from the host until kubelet reachability to the
+VPC address is proven; an exec readiness probe avoids that dependency.
+
+Current Cloud code names its NAD after the VPCAttachment, and publishes the
+VPCAttachment name through `NetworkInterface.status.attachmentRef`. This name
+equality is a temporary provider contract. A provider-neutral prepared NAD
+reference on the interface would remove that dependency. The flow still needs
+an end-to-end cell test proving that Multus observes the late annotation after
+the gate is removed, and that delete/recreate waits for CNI detach before
+releasing an address. Neither is established by unit tests.

config/components/pod-network-attachment/kustomization.yaml

@@ -0,0 +1,43 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+resources:
+  - webhook.yaml
+
+patches:
+  - target:
+      kind: ConfigMap
+      name: compute-config
+    patch: |-
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: compute-config
+      data:
+        config.yaml: |
+          apiVersion: apiserver.config.datumapis.com/v1alpha1
+          kind: WorkloadOperator
+          metricsServer:
+            bindAddress: "0"
+          webhookServer: {}
+          discovery:
+            quotaKubeconfigPath: /etc/quota-credentials/kubeconfig
+  - target:
+      kind: Deployment
+      name: compute-manager
+    patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: compute-manager
+      spec:
+        template:
+          spec:
+            containers:
+              - name: manager
+                env:
+                  - name: FEATURE_GATES
+                    value: NetworkingIntegration=true,PodNetworkAttachment=true
+
+configurations:
+  - webhook-kustomizeconfig.yaml

config/components/pod-network-attachment/webhook-kustomizeconfig.yaml

@@ -0,0 +1,10 @@
+nameReference:
+  - kind: Service
+    version: v1
+    fieldSpecs:
+      - kind: MutatingWebhookConfiguration
+        path: webhooks/clientConfig/service/name
+namespace:
+  - kind: MutatingWebhookConfiguration
+    path: webhooks/clientConfig/service/namespace
+    create: true

config/components/pod-network-attachment/webhook.yaml

@@ -0,0 +1,24 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: compute-pod-network-attachment
+webhooks:
+  - name: pod-network-attachment.compute.datumapis.com
+    admissionReviewVersions: [v1]
+    sideEffects: None
+    failurePolicy: Fail
+    reinvocationPolicy: Never
+    clientConfig:
+      service:
+        name: webhook-service
+        namespace: compute-system
+        path: /mutate-v1-pod-primary-network
+    rules:
+      - operations: [CREATE]
+        apiGroups: [""]
+        apiVersions: [v1]
+        resources: [pods]
+        scope: Namespaced
+    objectSelector:
+      matchLabels:
+        compute.datumapis.com/primary-network-enabled: "true"

config/overlays/cell-pod-network/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: compute-system
+resources:
+  - ../cell
+components:
+  - ../../components/pod-network-attachment
+  - ../../components/csi-webhook-cert

docs/enhancements/native-pod-vpc-attachment.md

@@ -0,0 +1,66 @@
+# Native Pods on a VPC
+
+**Status:** Draft
+
+## Summary
+
+Platform teams need to run a service on every eligible node while giving each
+Pod an address in a chosen VPC. Internal DNS is the first use case: one DNS Pod
+on each compute and Connect gateway node, with Galactic as its primary network.
+The same capability can place telemetry endpoints in consumer VPCs and forward
+their traffic to a shared collector.
+
+Compute adds an opt-in attachment path for native Kubernetes Pods, including
+DaemonSet Pods. The workload keeps Kubernetes scheduling and rollout behavior.
+Each Pod receives its own VPC interface before it starts, and that interface is
+its primary `eth0` rather than an additional network.
+
+## User experience
+
+An operator selects a VPC network on a Pod template and opts in to primary VPC
+networking. The network must be available in the Pod's namespace. A DaemonSet
+can then target the desired node classes using ordinary node selectors,
+affinity, and tolerations. New Pods wait for an address and attachment before
+they are scheduled; a failed attachment leaves the Pod pending rather than
+starting it on the cluster network.
+
+For the DNS rollout, operators can deploy one DaemonSet across compute and
+Connect gateway nodes and observe each Pod's claim and conditions when it is
+waiting. The same opt-in works for other node services without adding a
+DNS-specific workload API.
+
+## Proposed behavior
+
+1. Admission adds a scheduling gate to each opted-in Pod.
+2. Compute requests a VPC interface through a Pod-owned
+   `NetworkInterfaceClaim`.
+3. NSO allocates the address; Cloud prepares a VPC attachment and a unique
+   Multus network definition for that Pod.
+4. Once the claim is bound, allocated, and prepared, Compute sets the prepared
+   definition as the Pod's primary network and removes the scheduling gate.
+5. Multus invokes Galactic CNI when Kubernetes starts the Pod. Kubernetes
+   garbage collection removes the claim when the Pod is deleted.
+
+The interface is scoped to the Pod UID, so replacement Pods receive distinct
+attachments. Compute does not wait for the interface to be programmed before
+releasing the gate: programming happens when the Pod sandbox starts.
+
+## Acceptance criteria
+
+- A DaemonSet Pod on each selected compute and Connect gateway node starts with
+  a Galactic VPC address on `eth0`.
+- An attachment that cannot be prepared leaves its Pod pending, with a visible
+  claim condition; it never falls back to the cluster's primary network.
+- Replacing a Pod creates a distinct attachment and releases the old claim.
+- Ordinary Pods without the opt-in keep their existing admission and network
+  behavior.
+
+## Rollout dependencies
+
+This first implementation is feature gated and intended for a cell pilot. Before
+enabling the DNS DaemonSet, we need a stable provider-published reference to the
+prepared Multus definition, plus an end-to-end check that Multus sees the
+annotation added before gate removal. We also need to verify CNI detach and
+address release ordering during Pod replacement. Host-originated network probes
+need a reachable management path; an in-container exec probe can be used for
+the pilot. These checks determine when the capability is ready for general use.

internal/controller/pod_network_attachment.go

@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package controller
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha"
+)
+
+const (
+	PodPrimaryNetworkEnabledLabel  = "compute.datumapis.com/primary-network-enabled"
+	PodPrimaryNetworkEnabledValue  = "true"
+	PodPrimaryNetworkAnnotation    = "compute.datumapis.com/primary-network"
+	PodNetworkSchedulingGate       = "compute.datumapis.com/network-prepared"
+	MultusDefaultNetworkAnnotation = "v1.multus-cni.io/default-network"
+)
+
+// PodNetworkAttachmentReconciler attaches opted-in native Pods, including
+// DaemonSet Pods, to a network before the scheduler is allowed to bind them.
+// It runs only against the cell cluster, not project control planes.
+type PodNetworkAttachmentReconciler struct {
+	client.Client
+}
+
+// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;patch
+// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims,verbs=get;list;watch;create;delete
+// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces,verbs=get
+
+func (r *PodNetworkAttachmentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	var pod corev1.Pod
+	if err := r.Get(ctx, req.NamespacedName, &pod); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+	if pod.Labels[PodPrimaryNetworkEnabledLabel] != PodPrimaryNetworkEnabledValue || pod.Annotations[PodPrimaryNetworkAnnotation] == "" || !pod.DeletionTimestamp.IsZero() {
+		return ctrl.Result{}, nil
+	}
+	if !hasPodNetworkGate(&pod) {
+		// Never change a running Pod's network annotation. Admission must have
+		// installed the gate before this controller can act.
+		return ctrl.Result{}, nil
+	}
+	if pod.UID == "" {
+		return ctrl.Result{}, nil
+	}
+
+	claimName := podNetworkClaimName(pod.UID)
+	var claim networkingv1alpha.NetworkInterfaceClaim
+	key := types.NamespacedName{Namespace: pod.Namespace, Name: claimName}
+	if err := r.Get(ctx, key, &claim); apierrors.IsNotFound(err) {
+		claim = networkingv1alpha.NetworkInterfaceClaim{
+			ObjectMeta: metav1.ObjectMeta{Namespace: pod.Namespace, Name: claimName},
+			Spec: networkingv1alpha.NetworkInterfaceClaimSpec{
+				Network:        networkingv1alpha.LocalNetworkRef{Name: pod.Annotations[PodPrimaryNetworkAnnotation]},
+				InterfaceName:  defaultInterfaceName,
+				AttachmentMode: networkingv1alpha.NetworkInterfaceAttachmentModeNetns,
+			},
+		}
+		if err := controllerutil.SetControllerReference(&pod, &claim, r.Scheme()); err != nil {
+			return ctrl.Result{}, err
+		}
+		if err := r.Create(ctx, &claim); err != nil && !apierrors.IsAlreadyExists(err) {
+			return ctrl.Result{}, fmt.Errorf("create Pod network claim: %w", err)
+		}
+		return ctrl.Result{RequeueAfter: 2 * time.Second}, nil
+	} else if err != nil {
+		return ctrl.Result{}, err
+	}
+	if !metav1.IsControlledBy(&claim, &pod) || claim.Spec.Network.Name != pod.Annotations[PodPrimaryNetworkAnnotation] {
+		return ctrl.Result{}, fmt.Errorf("network claim %s is not owned by Pod %s or requests a different network", claim.Name, pod.Name)
+	}
+	if !networkInterfaceClaimSatisfied(&claim) || claim.Status.NetworkInterfaceRef == nil {
+		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+	}
+
+	var networkInterface networkingv1alpha.NetworkInterface
+	if err := r.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: claim.Status.NetworkInterfaceRef.Name}, &networkInterface); err != nil {
+		if apierrors.IsNotFound(err) {
+			// The claim status can arrive before the interface appears in this
+			// controller's cache. There is no interface watch to wake this Pod.
+			return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+		}
+		return ctrl.Result{}, err
+	}
+	// Cloud currently names the prepared NAD after the VPCAttachment. Keep
+	// the readiness check tied to the provider-published attachment ref;
+	// a future provider-neutral NAD ref should replace this name contract.
+	ref := networkInterface.Status.AttachmentRef
+	if ref == nil || ref.Name == "" || ref.Kind != "VPCAttachment" || ref.APIGroup != "cloud.datumapis.com" {
+		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+	}
+
+	before := pod.DeepCopy()
+	if pod.Annotations == nil {
+		pod.Annotations = map[string]string{}
+	}
+	// Explicit namespace/name works even when the NAD and Multus namespace
+	// differ. The named NAD is unique to this Pod's UID-backed claim.
+	defaultNetwork := pod.Namespace + "/" + ref.Name
+	if existing := pod.Annotations[MultusDefaultNetworkAnnotation]; existing != "" && existing != defaultNetwork {
+		return ctrl.Result{}, fmt.Errorf("pod %s already specifies a different Multus default network", pod.Name)
+	}
+	// Persist the annotation while the Pod is still gated, then remove the gate
+	// in a separate API write so the scheduler cannot bind it between steps.
+	if pod.Annotations[MultusDefaultNetworkAnnotation] != defaultNetwork {
+		pod.Annotations[MultusDefaultNetworkAnnotation] = defaultNetwork
+		if err := r.Patch(ctx, &pod, client.MergeFrom(before)); err != nil {
+			return ctrl.Result{}, fmt.Errorf("set Pod default network: %w", err)
+		}
+		return ctrl.Result{RequeueAfter: time.Second}, nil
+	}
+	filtered := pod.Spec.SchedulingGates[:0]
+	for _, gate := range pod.Spec.SchedulingGates {
+		if gate.Name != PodNetworkSchedulingGate {
+			filtered = append(filtered, gate)
+		}
+	}
+	pod.Spec.SchedulingGates = filtered
+	if err := r.Patch(ctx, &pod, client.MergeFrom(before)); err != nil {
+		return ctrl.Result{}, fmt.Errorf("release Pod network scheduling gate: %w", err)
+	}
+	return ctrl.Result{}, nil
+}
+
+func podNetworkClaimName(uid types.UID) string { return "pod-" + string(uid) + "-eth0" }
+
+func hasPodNetworkGate(pod *corev1.Pod) bool {
+	for _, gate := range pod.Spec.SchedulingGates {
+		if gate.Name == PodNetworkSchedulingGate {
+			return true
+		}
+	}
+	return false
+}
+
+func (r *PodNetworkAttachmentReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	r.Client = mgr.GetClient()
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Pod{}).
+		Watches(&networkingv1alpha.NetworkInterfaceClaim{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []reconcile.Request {
+			for _, owner := range obj.GetOwnerReferences() {
+				if owner.Kind == "Pod" && owner.APIVersion == "v1" {
+					return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: obj.GetNamespace(), Name: owner.Name}}}
+				}
+			}
+			return nil
+		})).
+		Complete(r)
+}