feat: Report private service readiness from each node

docs/router/private-service-routes.md

@@ -80,3 +80,17 @@ Policies claiming the same frontend address, protocol, and port on one consumer
 attachment are rejected together. Separate VPC attachments can reuse that tuple.
 The publisher owns authorization renewal and service health; Galactic enforces
 network access and forwards packets. It does not interpret DNS records or zones.
+
+## Node programming reports
+
+`ServiceRoutePolicy.status.nodes` reports whether each participating node
+programmed the current path. A report identifies the policy UID, generation,
+and input digest. It expires within 60 seconds or when authorization expires,
+whichever comes first. Consumer and producer nodes report independently for
+remote delivery.
+
+Controllers using these reports must require the current topology digest and
+all required nodes. `Accepted=True` means the policy is valid; it does not prove
+that forwarding is installed. Reports cover network programming, not the
+health of the producer application. Startup withdraws previous local readiness
+before rebuilding the node's state.

internal/controller/serviceroute_controller.go

@@ -112,6 +112,15 @@ func (r *ServiceRoutePolicyReconciler) Reconcile(ctx context.Context, req ctrl.R
 		logger.Error(err, "get ServiceRoutePolicy")
 		return ctrl.Result{}, err
 	}
+	inputDigest := ""
+	programmed := false
+	if policy.Spec.Frontend != nil {
+		defer func() {
+			if err := r.setNodeProgramming(ctx, policy, inputDigest, programmed); err != nil {
+				logger.Error(err, "publish node programming acknowledgment")
+			}
+		}()
+	}
 	if policy.DeletionTimestamp != nil {
 		return ctrl.Result{}, r.removePolicyLocked(req.NamespacedName)
 	}
@@ -163,6 +172,9 @@ func (r *ServiceRoutePolicyReconciler) Reconcile(ctx context.Context, req ctrl.R
 	for i := range attachments.Items {
 		all = append(all, &attachments.Items[i])
 	}
+	if policy.Spec.Frontend != nil {
+		inputDigest = serviceProgrammingDigest(policy, endpoint, all, vpcIdentity)
+	}
 	conflicts, conflictErr := r.conflictingPolicies(ctx, all)
 	if conflictErr != nil {
 		return ctrl.Result{}, errors.Join(conflictErr, r.removePolicyLocked(req.NamespacedName))
@@ -220,6 +232,7 @@ func (r *ServiceRoutePolicyReconciler) Reconcile(ctx context.Context, req ctrl.R
 		logger.Error(err, "program service eBPF policy", "endpoint", endpoint.Name)
 		return ctrl.Result{}, err
 	}
+	programmed = len(intents) != 0 && inputDigest != ""
 	if err := r.setAccepted(ctx, policy, metav1.ConditionTrue, "Accepted", "policy is valid"); err != nil {
 		resultLabel = reconcileResultError
 		logger.Error(err, "update ServiceRoutePolicy acceptance status")
@@ -339,6 +352,12 @@ func (r *ServiceRoutePolicyReconciler) syncAllPolicies(ctx context.Context) erro
 		policy := &policies.Items[index]
 		key := types.NamespacedName{Namespace: policy.Namespace, Name: policy.Name}
 		currentPolicies[key] = struct{}{}
+		if policy.Spec.Frontend != nil {
+			if err := r.setNodeProgramming(ctx, policy, "", false); err != nil {
+				return startupSyncError(err, true)
+			}
+		}
+
 		if (policy.Spec.Frontend != nil && !r.FrontendEnabled) || r.validateAuthorization(policy) != nil || conflicts[key] {
 			compiled = append(compiled, compiledPolicy{key: key})
 			continue

internal/controller/serviceroute_programming.go

@@ -0,0 +1,75 @@
+// Copyright 2026 Datum Cloud, Inc.
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package controller
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
+
+	cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1"
+	"go.datum.net/galactic/internal/serviceroute"
+	networkv1alpha1 "go.datum.net/network/api/v1alpha1"
+)
+
+const serviceProgrammingAckDuration = 60 * time.Second
+
+func (r *ServiceRoutePolicyReconciler) setNodeProgramming(
+	ctx context.Context, policy *networkv1alpha1.ServiceRoutePolicy, digest string, ready bool,
+) error {
+	deadline := r.now().Add(serviceProgrammingAckDuration)
+	if policy.Spec.Authorization != nil && policy.Spec.Authorization.ValidUntil.Before(&metav1.Time{Time: deadline}) {
+		deadline = policy.Spec.Authorization.ValidUntil.Time
+	}
+	report := networkv1alpha1.ServiceRouteNodeStatus{
+		NodeName:           r.NodeName,
+		PolicyUID:          string(policy.UID),
+		ObservedGeneration: policy.Generation,
+		InputDigest:        digest,
+		Ready:              ready,
+		ValidUntil:         metav1.NewTime(deadline),
+	}
+	return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		current := &networkv1alpha1.ServiceRoutePolicy{}
+		key := types.NamespacedName{Namespace: policy.Namespace, Name: policy.Name}
+		if err := r.Get(ctx, key, current); err != nil {
+			return err
+		}
+		if current.UID != policy.UID || current.Generation != policy.Generation {
+			return errors.New("policy changed before programming acknowledgment")
+		}
+		found := false
+		for i := range current.Status.Nodes {
+			if current.Status.Nodes[i].NodeName == r.NodeName {
+				current.Status.Nodes[i] = report
+				found = true
+			}
+		}
+		if !found && !ready {
+			return nil
+		}
+		if !found {
+			current.Status.Nodes = append(current.Status.Nodes, report)
+		}
+		sort.Slice(current.Status.Nodes, func(i, j int) bool {
+			return current.Status.Nodes[i].NodeName < current.Status.Nodes[j].NodeName
+		})
+		return r.Status().Update(ctx, current)
+	})
+}
+
+func serviceProgrammingDigest(policy *networkv1alpha1.ServiceRoutePolicy, endpoint *networkv1alpha1.ServiceEndpoint,
+	attachments []*cloudv1alpha1.VPCAttachment, vpcIdentity string,
+) string {
+	plan, err := serviceroute.ProgrammingPlan(policy, endpoint, attachments, vpcIdentity)
+	if err != nil {
+		return ""
+	}
+	return plan.InputDigest
+}

internal/controller/serviceroute_programming_test.go

@@ -0,0 +1,161 @@
+// Copyright 2026 Datum Cloud, Inc.
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package controller
+
+import (
+	"context"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1"
+	"go.datum.net/galactic/internal/crdnames"
+	networkv1alpha1 "go.datum.net/network/api/v1alpha1"
+)
+
+const testPSCRemoteNode = "psc-node-b"
+
+func TestProgrammingAcknowledgmentFencesAndWithdrawal(t *testing.T) {
+	r, p, _ := pscFixture(t)
+	ctx := context.Background()
+	key := types.NamespacedName{Namespace: p.Namespace, Name: p.Name}
+	if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil {
+		t.Fatal(err)
+	}
+	current := &networkv1alpha1.ServiceRoutePolicy{}
+	_ = r.Get(ctx, key, current)
+	if len(current.Status.Nodes) != 1 || !current.Status.Nodes[0].Ready || current.Status.Nodes[0].InputDigest == "" {
+		t.Fatalf("nodes=%+v", current.Status.Nodes)
+	}
+	old := current.DeepCopy()
+	current.Generation++
+	if err := r.Update(ctx, current); err != nil {
+		t.Fatal(err)
+	}
+	if err := r.setNodeProgramming(ctx, old, "old", true); err == nil {
+		t.Fatal("stale generation acknowledged")
+	}
+	current.Spec.ConsumerVPCRef.UID = "wrong"
+	if err := r.Update(ctx, current); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil {
+		t.Fatal(err)
+	}
+	_ = r.Get(ctx, key, current)
+	if current.Status.Nodes[0].Ready || len(r.Applied) != 0 {
+		t.Fatalf("invalid pin retained state: %+v", current.Status.Nodes)
+	}
+}
+
+func TestRemoteProgrammingReportsBothPathHalves(t *testing.T) {
+	r, p, _ := pscFixture(t)
+	ctx := context.Background()
+	producer := &cloudv1alpha1.VPCAttachment{}
+	producerKey := types.NamespacedName{Namespace: "service", Name: "server"}
+	if err := r.Get(ctx, producerKey, producer); err != nil {
+		t.Fatal(err)
+	}
+	producer.Status.Node = testPSCRemoteNode
+	if err := r.Update(ctx, producer); err != nil {
+		t.Fatal(err)
+	}
+	endpoint := &networkv1alpha1.ServiceEndpoint{}
+	key := types.NamespacedName{Namespace: p.Namespace, Name: p.Name}
+	_ = r.Get(ctx, key, endpoint)
+	endpoint.Spec.DeliveryMode = networkv1alpha1.ServiceEndpointDeliveryModePreferNodeLocal
+	if err := r.Update(ctx, endpoint); err != nil {
+		t.Fatal(err)
+	}
+	for index, node := range []string{r.NodeName, testPSCRemoteNode} {
+		router := &networkv1alpha1.BGPRouter{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "galactic-system",
+				Name:      node,
+			},
+			Spec: networkv1alpha1.BGPRouterSpec{
+				TargetRef: networkv1alpha1.TargetRef{
+					Name: node,
+				},
+				SRv6Locator: "fd00:1234::/48",
+				NodeID:      int32(index + 1),
+			},
+		}
+		if err := r.Create(ctx, router); err != nil {
+			t.Fatal(err)
+		}
+		vpc := map[string]string{r.NodeName: "vpc-identity", testPSCRemoteNode: "service-vpc"}[node]
+		vrf := &networkv1alpha1.BGPVRFInstance{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "galactic-system",
+				Name:      crdnames.BGPVRFInstanceName(vpc, node),
+			},
+			Spec: networkv1alpha1.BGPVRFInstanceSpec{
+				RouterTarget: networkv1alpha1.RouterTarget{
+					RouterRef: &networkv1alpha1.RouterRef{
+						Name: node,
+					},
+				},
+				VRFID: int32(index + 1),
+			},
+		}
+		if err := r.Create(ctx, vrf); err != nil {
+			t.Fatal(err)
+		}
+	}
+	second := &ServiceRoutePolicyReconciler{
+		Client:          r.Client,
+		NodeName:        testPSCRemoteNode,
+		FrontendEnabled: true,
+		Programmer:      &retryRouteProgrammer{},
+		Now:             r.Now,
+	}
+	for _, node := range []*ServiceRoutePolicyReconciler{r, second} {
+		if _, err := node.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	current := &networkv1alpha1.ServiceRoutePolicy{}
+	_ = r.Get(ctx, key, current)
+	if len(current.Status.Nodes) != 2 || !current.Status.Nodes[0].Ready || !current.Status.Nodes[1].Ready ||
+		current.Status.Nodes[0].InputDigest != current.Status.Nodes[1].InputDigest {
+		t.Fatalf("remote reports=%+v", current.Status.Nodes)
+	}
+	if len(r.Applied[key]) != 1 || len(second.Applied[key]) != 1 {
+		t.Fatal("remote halves not independently programmed")
+	}
+	producer.Status.Conditions[0].Status = metav1.ConditionFalse
+	if err := r.Update(ctx, producer); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := second.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil {
+		t.Fatal(err)
+	}
+	_ = r.Get(ctx, key, current)
+	if current.Status.Nodes[1].Ready || len(second.Applied) != 0 {
+		t.Fatalf("withdrawn producer retained readiness=%+v", current.Status.Nodes)
+	}
+}
+
+func TestDisablingFrontendWithdrawsNodeAcknowledgment(t *testing.T) {
+	r, p, _ := pscFixture(t)
+	ctx := context.Background()
+	key := types.NamespacedName{Namespace: p.Namespace, Name: p.Name}
+	if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil {
+		t.Fatal(err)
+	}
+	r.FrontendEnabled = false
+	if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil {
+		t.Fatal(err)
+	}
+	current := &networkv1alpha1.ServiceRoutePolicy{}
+	if err := r.Get(ctx, key, current); err != nil {
+		t.Fatal(err)
+	}
+	if current.Status.Nodes[0].Ready || len(r.Applied) != 0 {
+		t.Fatal("disabled node retained programming proof")
+	}
+}