diff --git a/docs/router/private-service-routes.md b/docs/router/private-service-routes.md index 914714d0..166ccf0b 100644 --- a/docs/router/private-service-routes.md +++ b/docs/router/private-service-routes.md @@ -80,3 +80,17 @@ Policies claiming the same frontend address, protocol, and port on one consumer attachment are rejected together. Separate VPC attachments can reuse that tuple. The publisher owns authorization renewal and service health; Galactic enforces network access and forwards packets. It does not interpret DNS records or zones. + +## Node programming reports + +`ServiceRoutePolicy.status.nodes` reports whether each participating node +programmed the current path. A report identifies the policy UID, generation, +and input digest. It expires within 60 seconds or when authorization expires, +whichever comes first. Consumer and producer nodes report independently for +remote delivery. + +Controllers using these reports must require the current topology digest and +all required nodes. `Accepted=True` means the policy is valid; it does not prove +that forwarding is installed. Reports cover network programming, not the +health of the producer application. Startup withdraws previous local readiness +before rebuilding the node's state. diff --git a/internal/controller/serviceroute_controller.go b/internal/controller/serviceroute_controller.go index 8b4331e1..0640f5d4 100644 --- a/internal/controller/serviceroute_controller.go +++ b/internal/controller/serviceroute_controller.go @@ -112,6 +112,15 @@ func (r *ServiceRoutePolicyReconciler) Reconcile(ctx context.Context, req ctrl.R logger.Error(err, "get ServiceRoutePolicy") return ctrl.Result{}, err } + inputDigest := "" + programmed := false + if policy.Spec.Frontend != nil { + defer func() { + if err := r.setNodeProgramming(ctx, policy, inputDigest, programmed); err != nil { + logger.Error(err, "publish node programming acknowledgment") + } + }() + } if policy.DeletionTimestamp != nil { return ctrl.Result{}, r.removePolicyLocked(req.NamespacedName) } @@ -163,6 +172,9 @@ func (r *ServiceRoutePolicyReconciler) Reconcile(ctx context.Context, req ctrl.R for i := range attachments.Items { all = append(all, &attachments.Items[i]) } + if policy.Spec.Frontend != nil { + inputDigest = serviceProgrammingDigest(policy, endpoint, all, vpcIdentity) + } conflicts, conflictErr := r.conflictingPolicies(ctx, all) if conflictErr != nil { return ctrl.Result{}, errors.Join(conflictErr, r.removePolicyLocked(req.NamespacedName)) @@ -220,6 +232,7 @@ func (r *ServiceRoutePolicyReconciler) Reconcile(ctx context.Context, req ctrl.R logger.Error(err, "program service eBPF policy", "endpoint", endpoint.Name) return ctrl.Result{}, err } + programmed = len(intents) != 0 && inputDigest != "" if err := r.setAccepted(ctx, policy, metav1.ConditionTrue, "Accepted", "policy is valid"); err != nil { resultLabel = reconcileResultError logger.Error(err, "update ServiceRoutePolicy acceptance status") @@ -339,6 +352,12 @@ func (r *ServiceRoutePolicyReconciler) syncAllPolicies(ctx context.Context) erro policy := &policies.Items[index] key := types.NamespacedName{Namespace: policy.Namespace, Name: policy.Name} currentPolicies[key] = struct{}{} + if policy.Spec.Frontend != nil { + if err := r.setNodeProgramming(ctx, policy, "", false); err != nil { + return startupSyncError(err, true) + } + } + if (policy.Spec.Frontend != nil && !r.FrontendEnabled) || r.validateAuthorization(policy) != nil || conflicts[key] { compiled = append(compiled, compiledPolicy{key: key}) continue diff --git a/internal/controller/serviceroute_programming.go b/internal/controller/serviceroute_programming.go new file mode 100644 index 00000000..67167426 --- /dev/null +++ b/internal/controller/serviceroute_programming.go @@ -0,0 +1,75 @@ +// Copyright 2026 Datum Cloud, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controller + +import ( + "context" + "errors" + "sort" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + "go.datum.net/galactic/internal/serviceroute" + networkv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +const serviceProgrammingAckDuration = 60 * time.Second + +func (r *ServiceRoutePolicyReconciler) setNodeProgramming( + ctx context.Context, policy *networkv1alpha1.ServiceRoutePolicy, digest string, ready bool, +) error { + deadline := r.now().Add(serviceProgrammingAckDuration) + if policy.Spec.Authorization != nil && policy.Spec.Authorization.ValidUntil.Before(&metav1.Time{Time: deadline}) { + deadline = policy.Spec.Authorization.ValidUntil.Time + } + report := networkv1alpha1.ServiceRouteNodeStatus{ + NodeName: r.NodeName, + PolicyUID: string(policy.UID), + ObservedGeneration: policy.Generation, + InputDigest: digest, + Ready: ready, + ValidUntil: metav1.NewTime(deadline), + } + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + current := &networkv1alpha1.ServiceRoutePolicy{} + key := types.NamespacedName{Namespace: policy.Namespace, Name: policy.Name} + if err := r.Get(ctx, key, current); err != nil { + return err + } + if current.UID != policy.UID || current.Generation != policy.Generation { + return errors.New("policy changed before programming acknowledgment") + } + found := false + for i := range current.Status.Nodes { + if current.Status.Nodes[i].NodeName == r.NodeName { + current.Status.Nodes[i] = report + found = true + } + } + if !found && !ready { + return nil + } + if !found { + current.Status.Nodes = append(current.Status.Nodes, report) + } + sort.Slice(current.Status.Nodes, func(i, j int) bool { + return current.Status.Nodes[i].NodeName < current.Status.Nodes[j].NodeName + }) + return r.Status().Update(ctx, current) + }) +} + +func serviceProgrammingDigest(policy *networkv1alpha1.ServiceRoutePolicy, endpoint *networkv1alpha1.ServiceEndpoint, + attachments []*cloudv1alpha1.VPCAttachment, vpcIdentity string, +) string { + plan, err := serviceroute.ProgrammingPlan(policy, endpoint, attachments, vpcIdentity) + if err != nil { + return "" + } + return plan.InputDigest +} diff --git a/internal/controller/serviceroute_programming_test.go b/internal/controller/serviceroute_programming_test.go new file mode 100644 index 00000000..c7ba2254 --- /dev/null +++ b/internal/controller/serviceroute_programming_test.go @@ -0,0 +1,161 @@ +// Copyright 2026 Datum Cloud, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controller + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + "go.datum.net/galactic/internal/crdnames" + networkv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +const testPSCRemoteNode = "psc-node-b" + +func TestProgrammingAcknowledgmentFencesAndWithdrawal(t *testing.T) { + r, p, _ := pscFixture(t) + ctx := context.Background() + key := types.NamespacedName{Namespace: p.Namespace, Name: p.Name} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil { + t.Fatal(err) + } + current := &networkv1alpha1.ServiceRoutePolicy{} + _ = r.Get(ctx, key, current) + if len(current.Status.Nodes) != 1 || !current.Status.Nodes[0].Ready || current.Status.Nodes[0].InputDigest == "" { + t.Fatalf("nodes=%+v", current.Status.Nodes) + } + old := current.DeepCopy() + current.Generation++ + if err := r.Update(ctx, current); err != nil { + t.Fatal(err) + } + if err := r.setNodeProgramming(ctx, old, "old", true); err == nil { + t.Fatal("stale generation acknowledged") + } + current.Spec.ConsumerVPCRef.UID = "wrong" + if err := r.Update(ctx, current); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil { + t.Fatal(err) + } + _ = r.Get(ctx, key, current) + if current.Status.Nodes[0].Ready || len(r.Applied) != 0 { + t.Fatalf("invalid pin retained state: %+v", current.Status.Nodes) + } +} + +func TestRemoteProgrammingReportsBothPathHalves(t *testing.T) { + r, p, _ := pscFixture(t) + ctx := context.Background() + producer := &cloudv1alpha1.VPCAttachment{} + producerKey := types.NamespacedName{Namespace: "service", Name: "server"} + if err := r.Get(ctx, producerKey, producer); err != nil { + t.Fatal(err) + } + producer.Status.Node = testPSCRemoteNode + if err := r.Update(ctx, producer); err != nil { + t.Fatal(err) + } + endpoint := &networkv1alpha1.ServiceEndpoint{} + key := types.NamespacedName{Namespace: p.Namespace, Name: p.Name} + _ = r.Get(ctx, key, endpoint) + endpoint.Spec.DeliveryMode = networkv1alpha1.ServiceEndpointDeliveryModePreferNodeLocal + if err := r.Update(ctx, endpoint); err != nil { + t.Fatal(err) + } + for index, node := range []string{r.NodeName, testPSCRemoteNode} { + router := &networkv1alpha1.BGPRouter{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "galactic-system", + Name: node, + }, + Spec: networkv1alpha1.BGPRouterSpec{ + TargetRef: networkv1alpha1.TargetRef{ + Name: node, + }, + SRv6Locator: "fd00:1234::/48", + NodeID: int32(index + 1), + }, + } + if err := r.Create(ctx, router); err != nil { + t.Fatal(err) + } + vpc := map[string]string{r.NodeName: "vpc-identity", testPSCRemoteNode: "service-vpc"}[node] + vrf := &networkv1alpha1.BGPVRFInstance{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "galactic-system", + Name: crdnames.BGPVRFInstanceName(vpc, node), + }, + Spec: networkv1alpha1.BGPVRFInstanceSpec{ + RouterTarget: networkv1alpha1.RouterTarget{ + RouterRef: &networkv1alpha1.RouterRef{ + Name: node, + }, + }, + VRFID: int32(index + 1), + }, + } + if err := r.Create(ctx, vrf); err != nil { + t.Fatal(err) + } + } + second := &ServiceRoutePolicyReconciler{ + Client: r.Client, + NodeName: testPSCRemoteNode, + FrontendEnabled: true, + Programmer: &retryRouteProgrammer{}, + Now: r.Now, + } + for _, node := range []*ServiceRoutePolicyReconciler{r, second} { + if _, err := node.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil { + t.Fatal(err) + } + } + current := &networkv1alpha1.ServiceRoutePolicy{} + _ = r.Get(ctx, key, current) + if len(current.Status.Nodes) != 2 || !current.Status.Nodes[0].Ready || !current.Status.Nodes[1].Ready || + current.Status.Nodes[0].InputDigest != current.Status.Nodes[1].InputDigest { + t.Fatalf("remote reports=%+v", current.Status.Nodes) + } + if len(r.Applied[key]) != 1 || len(second.Applied[key]) != 1 { + t.Fatal("remote halves not independently programmed") + } + producer.Status.Conditions[0].Status = metav1.ConditionFalse + if err := r.Update(ctx, producer); err != nil { + t.Fatal(err) + } + if _, err := second.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil { + t.Fatal(err) + } + _ = r.Get(ctx, key, current) + if current.Status.Nodes[1].Ready || len(second.Applied) != 0 { + t.Fatalf("withdrawn producer retained readiness=%+v", current.Status.Nodes) + } +} + +func TestDisablingFrontendWithdrawsNodeAcknowledgment(t *testing.T) { + r, p, _ := pscFixture(t) + ctx := context.Background() + key := types.NamespacedName{Namespace: p.Namespace, Name: p.Name} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil { + t.Fatal(err) + } + r.FrontendEnabled = false + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}); err != nil { + t.Fatal(err) + } + current := &networkv1alpha1.ServiceRoutePolicy{} + if err := r.Get(ctx, key, current); err != nil { + t.Fatal(err) + } + if current.Status.Nodes[0].Ready || len(r.Applied) != 0 { + t.Fatal("disabled node retained programming proof") + } +} diff --git a/internal/serviceroute/programming.go b/internal/serviceroute/programming.go new file mode 100644 index 00000000..5e54ee9d --- /dev/null +++ b/internal/serviceroute/programming.go @@ -0,0 +1,133 @@ +// Copyright 2026 Datum Cloud, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later + +package serviceroute + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + networkv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// ProgrammingRequirements fences acknowledgments to a complete topology snapshot. +type ProgrammingRequirements struct { + InputDigest string + Nodes []string +} + +// ProgrammingPlan identifies every consumer and selected producer node required +// to serve a policy. Attachment health establishes network eligibility only. +func ProgrammingPlan(policy *networkv1alpha1.ServiceRoutePolicy, endpoint *networkv1alpha1.ServiceEndpoint, + attachments []*cloudv1alpha1.VPCAttachment, consumerVPCIdentity string, +) (ProgrammingRequirements, error) { + plan := ProgrammingRequirements{} + if policy == nil || endpoint == nil { + return plan, errors.New("policy and endpoint are required") + } + if policy.Spec.Frontend != nil && (policy.UID == "" || endpoint.UID == "" || consumerVPCIdentity == "") { + return plan, errors.New("translated programming requires current policy, endpoint, and VPC identities") + } + selector, err := metav1.LabelSelectorAsSelector(&policy.Spec.AttachmentSelector) + if err != nil { + return plan, err + } + producers, err := producerCandidates(endpoint, attachments) + if err != nil { + return plan, err + } + if err := validateProducerTopology(endpoint, producers); err != nil { + return plan, err + } + ready := map[types.NamespacedName]bool{} + for _, a := range readyAttachments(attachments) { + ready[types.NamespacedName{Namespace: a.Namespace, Name: a.Name}] = a.Status.HostInterface != "" + } + relevant := map[types.NamespacedName]bool{} + for _, p := range producers { + relevant[types.NamespacedName{Namespace: p.Namespace, Name: p.Name}] = true + } + nodes := map[string]bool{} + selected := 0 + sorted := make([]*cloudv1alpha1.VPCAttachment, 0, len(attachments)) + for _, a := range attachments { + if a != nil { + sorted = append(sorted, a) + } + } + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].Namespace+"/"+sorted[i].Name < sorted[j].Namespace+"/"+sorted[j].Name + }) + for _, a := range sorted { + if !selector.Matches(labels.Set(a.Labels)) { + continue + } + if !consumerMatchesVPC(policy, a, consumerVPCIdentity) { + continue + } + relevant[types.NamespacedName{Namespace: a.Namespace, Name: a.Name}] = true + selected++ + if a.DeletionTimestamp != nil || !ready[types.NamespacedName{Namespace: a.Namespace, Name: a.Name}] { + return plan, fmt.Errorf("consumer attachment %s/%s is not ready", a.Namespace, a.Name) + } + p := chooseProducer(endpoint.Spec.DeliveryMode, a.Status.Node, producers) + if p == nil { + return plan, fmt.Errorf("consumer attachment %s/%s has no eligible producer", a.Namespace, a.Name) + } + if policy.Spec.Frontend != nil && (a.UID == "" || p.UID == "") { + return plan, errors.New("translated programming requires current attachment UIDs") + } + nodes[a.Status.Node], nodes[p.Status.Node] = true, true + } + if selected == 0 { + return plan, errors.New("no eligible consumer attachments") + } + // Resource versions and node reports are excluded: heartbeat writes do not + // invalidate the desired topology. UIDs, generations, specs, and statuses do. + snapshot := struct { + PolicyUID types.UID + PolicyGeneration int64 + PolicySpec networkv1alpha1.ServiceRoutePolicySpec + EndpointUID types.UID + EndpointGeneration int64 + EndpointSpec networkv1alpha1.ServiceEndpointSpec + VPCIdentity string + Attachments []any + }{PolicyUID: policy.UID, PolicyGeneration: policy.Generation, PolicySpec: policy.Spec, + EndpointUID: endpoint.UID, EndpointGeneration: endpoint.Generation, EndpointSpec: endpoint.Spec, + VPCIdentity: consumerVPCIdentity} + for _, a := range sorted { + if !relevant[types.NamespacedName{Namespace: a.Namespace, Name: a.Name}] { + continue + } + snapshot.Attachments = append(snapshot.Attachments, struct { + Namespace, Name string + UID types.UID + Generation int64 + Labels map[string]string + Deleted bool + Spec cloudv1alpha1.VPCAttachmentSpec + Status cloudv1alpha1.VPCAttachmentStatus + }{a.Namespace, a.Name, a.UID, a.Generation, a.Labels, a.DeletionTimestamp != nil, a.Spec, a.Status}) + } + encoded, err := json.Marshal(snapshot) + if err != nil { + return plan, fmt.Errorf("encode service programming snapshot: %w", err) + } + sum := sha256.Sum256(encoded) + plan.InputDigest = hex.EncodeToString(sum[:]) + for node := range nodes { + plan.Nodes = append(plan.Nodes, node) + } + sort.Strings(plan.Nodes) + return plan, nil +} diff --git a/internal/serviceroute/programming_test.go b/internal/serviceroute/programming_test.go new file mode 100644 index 00000000..df80ca6c --- /dev/null +++ b/internal/serviceroute/programming_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Datum Cloud, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later + +package serviceroute + +import ( + "testing" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + networkv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +func TestProgrammingPlanFencesTopologyAndIgnoresUnrelatedVPC(t *testing.T) { + policy, endpoint := testPolicyAndEndpoint(networkv1alpha1.ServiceEndpointDeliveryModePreferNodeLocal) + consumer := readyAttachment("tenant", "client", "client", testNodeName, "client0", testLabels(testAccessLabel, "yes")) + producer := readyAttachment("service", "server", "server", testRemoteNodeName, "server0", + testLabels(testProducerLabel, "yes")) + topology := []*cloudv1alpha1.VPCAttachment{consumer, producer} + first, err := ProgrammingPlan(policy, endpoint, topology, "") + if err != nil { + t.Fatal(err) + } + if len(first.Nodes) != 2 { + t.Fatalf("nodes=%v", first.Nodes) + } + unrelated := readyAttachment("other", "other", "other", testNodeName, "other0", nil) + second, err := ProgrammingPlan(policy, endpoint, append(topology, unrelated), "") + if err != nil { + t.Fatal(err) + } + if first.InputDigest != second.InputDigest { + t.Fatal("unrelated VPC invalidated programming") + } + producer.UID = "replacement" + second, err = ProgrammingPlan(policy, endpoint, topology, "") + if err != nil { + t.Fatal(err) + } + if first.InputDigest == second.InputDigest { + t.Fatal("recreated producer retained acknowledgment digest") + } + endpoint.Generation++ + third, err := ProgrammingPlan(policy, endpoint, topology, "") + if err != nil { + t.Fatal(err) + } + if second.InputDigest == third.InputDigest { + t.Fatal("endpoint generation retained acknowledgment digest") + } + producer.Status.Conditions[0].Status = "False" + if _, err := ProgrammingPlan(policy, endpoint, topology, ""); err == nil { + t.Fatal("unhealthy producer considered programmed") + } +}