fix: Stop the gateway and NAT shard running without forwarding

cmd/galactic-gateway/gateway.go

@@ -114,8 +114,9 @@ func setupGatewayDatapath(
 	// ingress interface, meaning whichever resolved target the packet
 	// arrived on, so this must be applied to every one of them: once the
 	// named interface is a bond, its slaves rather than the master are
-	// what the kernel reports as ingress. Best-effort and non-fatal,
-	// matching how sysctls are configured elsewhere here.
+	// what the kernel reports as ingress. A target left without forwarding
+	// fails startup: the datapath would drop every packet arriving there
+	// while the process reported healthy.
 	for _, target := range targets {
 		if err := sysctl.ConfigureFIBLookupUplinkSysctls(target); err != nil {
 			return nil, fmt.Errorf("configure IPv6 forwarding on public interface %q: %w", target, err)

cmd/galactic-gateway/root.go

@@ -28,6 +28,7 @@ import (
 	"go.datum.net/galactic/internal/gateway"
 	"go.datum.net/galactic/internal/metadata"
 	"go.datum.net/galactic/internal/plumbing/ebpf/xdpdispatch"
+	"go.datum.net/galactic/internal/plumbing/sysctl"
 	bgpv1alpha1 "go.datum.net/network/api/v1alpha1"
 )
 
@@ -135,6 +136,9 @@ func runCmd(cfg *config.GatewayConfig) error {
 	})
 	var gwDatapath gateway.Datapath = gateway.NoopDatapath{}
 	if cfg.DatapathEnabled {
+		if err := sysctl.SetProcSysPath(cfg.ProcSysPath); err != nil {
+			return err
+		}
 		gwDatapath, err = setupGatewayDatapath(ctx, cfg.PublicInterface, cfg.InternalInterfaces, cfg.SRv6Address,
 			cfg.XDPAttach, ctrlmetrics.Registry, coverage)
 		if err != nil {
@@ -260,6 +264,9 @@ func newRootCommand() *cobra.Command {
 	cmd.Flags().Bool("gateway-datapath-enabled", true,
 		"Run the edge gateway datapath; false keeps the process up but attaches nothing and withdraws "+
 			"this node's VIP advertisements")
+	cmd.Flags().String("gateway-proc-sys-path", config.DefaultProcSysPath,
+		"Procfs sysctl root to write the datapath's forwarding sysctls under, such as a writable mount "+
+			"of the host's /proc/sys/net at <path>/net")
 	cmd.Flags().Bool("build-info", false, "Print build information and exit")
 	cmd.Flags().BoolP("version", "V", false, "Print version and exit")
 	return cmd

cmd/galactic-nat/root.go

@@ -29,6 +29,7 @@ import (
 	"go.datum.net/galactic/internal/controller"
 	"go.datum.net/galactic/internal/metadata"
 	"go.datum.net/galactic/internal/plumbing/ebpf/xdpdispatch"
+	"go.datum.net/galactic/internal/plumbing/sysctl"
 	bgpv1alpha1 "go.datum.net/network/api/v1alpha1"
 )
 
@@ -126,6 +127,9 @@ func runCmd(cfg *config.NATConfig) error {
 	coverageChanged := make(chan struct{}, 1)
 	var datapath controller.EgressDatapath
 	if cfg.DatapathEnabled {
+		if err := sysctl.SetProcSysPath(cfg.ProcSysPath); err != nil {
+			return err
+		}
 		datapath, err = setupNatDatapath(ctx, cfg, ctrlmetrics.Registry, func() {
 			select {
 			case coverageChanged <- struct{}{}:
@@ -252,6 +256,9 @@ func newRootCommand() *cobra.Command {
 	cmd.Flags().Bool("nat-echo-responder", false,
 		"Answer ICMP and ICMPv6 Echo Requests addressed to this shard's own masquerade addresses, "+
 			"rate-limited, instead of dropping them")
+	cmd.Flags().String("nat-proc-sys-path", config.DefaultProcSysPath,
+		"Procfs sysctl root to write the datapath's forwarding sysctls under, such as a writable mount "+
+			"of the host's /proc/sys/net at <path>/net")
 	cmd.Flags().Bool("build-info", false, "Print build information and exit")
 	cmd.Flags().BoolP("version", "V", false, "Print version and exit")
 	return cmd

config/galactic-gateway/base/daemonset.yaml

@@ -86,6 +86,14 @@ spec:
                   fieldPath: spec.nodeName
             - name: GALACTIC_GATEWAY_GRPC_HEALTH_PORT
               value: "5181"
+            # An unprivileged container gets /proc/sys read-only, so the
+            # forwarding sysctls the datapath's FIB lookup needs are written
+            # through the host's /proc/sys/net, mounted below. hostNetwork
+            # makes those writes land in the node's own network namespace.
+            # A sysctl that still does not read 1 fails startup rather than
+            # leaving a datapath that drops every packet (#586).
+            - name: GALACTIC_GATEWAY_PROC_SYS_PATH
+              value: /host/proc/sys
             # GALACTIC_GATEWAY_INTERNAL_INTERFACES is also left unset,
             # for the same deployment-specific reason. It names the
             # compute-facing interfaces, comma-separated, whose
@@ -187,8 +195,17 @@ spec:
             # container on first pin.
             - name: bpf-fs
               mountPath: /sys/fs/bpf
+            # The host's net sysctls, writable, for the forwarding settings
+            # galactic-gateway sets. Only net.*: the rest of /proc/sys stays
+            # read-only.
+            - name: host-proc-sys-net
+              mountPath: /host/proc/sys/net
       volumes:
         - name: bpf-fs
           hostPath:
             path: /sys/fs/bpf
             type: Directory
+        - name: host-proc-sys-net
+          hostPath:
+            path: /proc/sys/net
+            type: Directory

config/galactic-nat/base/daemonset.yaml

@@ -102,6 +102,14 @@ spec:
             # internal/config/nat.go's EnvNATEchoResponder.
             - name: GALACTIC_NAT_ECHO_RESPONDER
               value: "false"
+            # An unprivileged container gets /proc/sys read-only, so the
+            # forwarding sysctls the datapath's FIB lookup needs are written
+            # through the host's /proc/sys/net, mounted below. hostNetwork
+            # makes those writes land in the node's own network namespace.
+            # A sysctl that still does not read 1 fails startup rather than
+            # leaving a datapath that drops every packet (#586).
+            - name: GALACTIC_NAT_PROC_SYS_PATH
+              value: /host/proc/sys
             # GALACTIC_NAT_XDP_ATTACH is left at its default, "direct": the
             # shard attaches its own program to the uplinks. "dispatch" runs
             # it from the node's shared, pinned XDP dispatcher instead, so a
@@ -213,8 +221,17 @@ spec:
             # galactic-xdp/, the shared XDP dispatcher's, in dispatch mode.
             - name: bpf-fs
               mountPath: /sys/fs/bpf
+            # The host's net sysctls, writable, for the forwarding settings
+            # galactic-nat sets. Only net.*: the rest of /proc/sys stays
+            # read-only.
+            - name: host-proc-sys-net
+              mountPath: /host/proc/sys/net
       volumes:
         - name: bpf-fs
           hostPath:
             path: /sys/fs/bpf
             type: Directory
+        - name: host-proc-sys-net
+          hostPath:
+            path: /proc/sys/net
+            type: Directory

docs/agents/ARCHITECTURE-GATEWAY.md

@@ -462,8 +462,14 @@ pre-existing containerlab veth/XDP_TX blocker — the sysctl gap, not
 [Known Constraints](#known-constraints) below for the related, lab-only
 `XDP_TX` observability quirk this investigation also turned up, and for
 why the sysctl targets the resolved bond slave rather than
-`PublicInterface` itself when the two differ). It then loads
-`edgeprog.EdgedsrObjects` (`edgeattach.Load`), attaches it to every
+`PublicInterface` itself when the two differ). The helper reads each
+sysctl back and returns an error unless it reads `1`, which fails startup:
+before #586 it logged the write failure and returned `nil`, so a pod with
+a read-only `/proc/sys` ran a datapath that dropped every packet. The
+DaemonSet mounts the host's `/proc/sys/net` at `/host/proc/sys/net` and
+points `GALACTIC_GATEWAY_PROC_SYS_PATH` at `/host/proc/sys` so the write
+can succeed without `privileged`. It then loads `edgeprog.EdgedsrObjects`
+(`edgeattach.Load`), attaches it to every
 resolved target (`edgeattach.Attach` — native XDP driver mode only, no
 generic/SKB-mode fallback, one `link.Link` per target), and constructs a
 `gateway.KernelDatapath` over it, writing this node's SRv6 encap-source

docs/gateway/configuration.md

@@ -156,16 +156,17 @@ flags, or a combination of both (CLI flags take precedence), with the
 `GALACTIC_GATEWAY` env prefix — the same three-tier precedence pattern
 `galactic-router` uses (see [docs/router/configuration.md](../router/configuration.md)).
 
-| Option              | Environment Variable                   | CLI Flag                        | Default    | Required          |
-| ------------------- | -------------------------------------- | ------------------------------- | ---------- | ----------------- |
-| Node name           | `GALACTIC_GATEWAY_NODE_NAME`           | `--node-name`, `-n`             | —          | Yes               |
-| Public interface    | `GALACTIC_GATEWAY_PUBLIC_INTERFACE`    | `--gateway-public-interface`    | —          | While enabled     |
-| SRv6 address        | `GALACTIC_GATEWAY_SRV6_ADDRESS`        | `--gateway-srv6-address`        | —          | While enabled     |
-| Internal interfaces | `GALACTIC_GATEWAY_INTERNAL_INTERFACES` | `--gateway-internal-interfaces` | —          | No                |
-| XDP attach mode     | `GALACTIC_GATEWAY_XDP_ATTACH`          | `--gateway-xdp-attach`          | `dispatch` | No                |
-| Datapath enabled    | `GALACTIC_GATEWAY_DATAPATH_ENABLED`    | `--gateway-datapath-enabled`    | `true`     | No                |
-| Metrics port        | `GALACTIC_GATEWAY_METRICS_PORT`        | `--metrics-port`                | `8081`     | No                |
-| gRPC health port    | `GALACTIC_GATEWAY_GRPC_HEALTH_PORT`    | `--grpc-health-port`            | `5181`     | No                |
+| Option              | Environment Variable                   | CLI Flag                        | Default     | Required      |
+| ------------------- | -------------------------------------- | ------------------------------- | ----------- | ------------- |
+| Node name           | `GALACTIC_GATEWAY_NODE_NAME`           | `--node-name`, `-n`             | —           | Yes           |
+| Public interface    | `GALACTIC_GATEWAY_PUBLIC_INTERFACE`    | `--gateway-public-interface`    | —           | While enabled |
+| SRv6 address        | `GALACTIC_GATEWAY_SRV6_ADDRESS`        | `--gateway-srv6-address`        | —           | While enabled |
+| Internal interfaces | `GALACTIC_GATEWAY_INTERNAL_INTERFACES` | `--gateway-internal-interfaces` | —           | No            |
+| XDP attach mode     | `GALACTIC_GATEWAY_XDP_ATTACH`          | `--gateway-xdp-attach`          | `dispatch`  | No            |
+| Datapath enabled    | `GALACTIC_GATEWAY_DATAPATH_ENABLED`    | `--gateway-datapath-enabled`    | `true`      | No            |
+| Procfs sysctl root  | `GALACTIC_GATEWAY_PROC_SYS_PATH`       | `--gateway-proc-sys-path`       | `/proc/sys` | No            |
+| Metrics port        | `GALACTIC_GATEWAY_METRICS_PORT`        | `--metrics-port`                | `8081`      | No            |
+| gRPC health port    | `GALACTIC_GATEWAY_GRPC_HEALTH_PORT`    | `--grpc-health-port`            | `5181`      | No            |
 
 `GALACTIC_GATEWAY_XDP_ATTACH` is `dispatch` or `direct`. `dispatch` runs
 `edge_lb` and `edge_return` from the gateway slots of the node's shared,
@@ -182,6 +183,17 @@ withdraws every one of this node's VIP advertisements. The NetworkGateway's
 `Ready` reads `False` with reason `DatapathDisabled`. The public interface
 and SRv6 address are not required while it is off.
 
+`GALACTIC_GATEWAY_PROC_SYS_PATH` is the procfs root the datapath writes its
+forwarding sysctls under (`net.ipv6.conf.<iface>.forwarding` and
+`net.ipv6.conf.all.forwarding`). `bpf_fib_lookup` refuses every lookup on an
+interface with forwarding off, so a sysctl that does not read `1` after the
+write stops the gateway at startup, and leaves an interface found later
+without the datapath until it does. A pod that is not privileged gets
+`/proc/sys` read-only, so `config/galactic-gateway/base/daemonset.yaml`
+mounts the host's `/proc/sys/net` at `/host/proc/sys/net` and sets this to
+`/host/proc/sys`. A node where something else already turned forwarding on
+passes with the default too. SELinux policy on an enforcing node can still
+refuse the write, and the gateway then says so at startup.
 
 `GALACTIC_GATEWAY_INTERNAL_INTERFACES` is a comma-separated list of this
 node's compute-facing interfaces, and it is what puts the return path in

docs/nat/configuration.md

@@ -41,15 +41,16 @@ flags, or a combination of both (CLI flags take precedence), with the
 `galactic-router` and `galactic-gateway` use (see
 [docs/router/configuration.md](../router/configuration.md)).
 
-| Option            | Environment Variable             | CLI Flag                  | Default       | Required |
-| ----------------- | -------------------------------- | ------------------------- | ------------- | -------- |
-| Node name         | `GALACTIC_NAT_NODE_NAME`         | `--node-name`             | —             | Yes      |
-| Uplink interfaces | `GALACTIC_NAT_UPLINK_INTERFACES` | `--nat-uplink-interfaces` | auto-detected | No       |
-| XDP attach mode   | `GALACTIC_NAT_XDP_ATTACH`        | `--nat-xdp-attach`        | `direct`      | No       |
-| Datapath enabled  | `GALACTIC_NAT_DATAPATH_ENABLED`  | `--nat-datapath-enabled`  | `true`        | No       |
-| Echo responder    | `GALACTIC_NAT_ECHO_RESPONDER`    | `--nat-echo-responder`    | `false`       | No       |
-| Metrics port      | `GALACTIC_NAT_METRICS_PORT`      | `--metrics-port`          | `9182`        | No       |
-| gRPC health port  | `GALACTIC_NAT_GRPC_HEALTH_PORT`  | `--grpc-health-port`      | `5182`        | No       |
+| Option             | Environment Variable             | CLI Flag                  | Default       | Required |
+| ------------------ | -------------------------------- | ------------------------- | ------------- | -------- |
+| Node name          | `GALACTIC_NAT_NODE_NAME`         | `--node-name`             | —             | Yes      |
+| Uplink interfaces  | `GALACTIC_NAT_UPLINK_INTERFACES` | `--nat-uplink-interfaces` | auto-detected | No       |
+| XDP attach mode    | `GALACTIC_NAT_XDP_ATTACH`        | `--nat-xdp-attach`        | `direct`      | No       |
+| Datapath enabled   | `GALACTIC_NAT_DATAPATH_ENABLED`  | `--nat-datapath-enabled`  | `true`        | No       |
+| Echo responder     | `GALACTIC_NAT_ECHO_RESPONDER`    | `--nat-echo-responder`    | `false`       | No       |
+| Procfs sysctl root | `GALACTIC_NAT_PROC_SYS_PATH`     | `--nat-proc-sys-path`     | `/proc/sys`   | No       |
+| Metrics port       | `GALACTIC_NAT_METRICS_PORT`      | `--metrics-port`          | `9182`        | No       |
+| gRPC health port   | `GALACTIC_NAT_GRPC_HEALTH_PORT`  | `--grpc-health-port`      | `5182`        | No       |
 
 `config/galactic-nat/base/daemonset.yaml` leaves `GALACTIC_NAT_XDP_ATTACH`
 at the binary's default, `direct`. `dispatch` is the mode to move to: it is
@@ -201,6 +202,19 @@ reply counts as `icmp_rate_limited`. The setting is written into the datapath
 whenever the shard programs its identity, so changing it takes a pod restart
 and nothing else.
 
+**`--nat-proc-sys-path` / `GALACTIC_NAT_PROC_SYS_PATH`**
+The procfs root the datapath writes its forwarding sysctls under:
+`net.ipv6.conf.<iface>.forwarding` and `net.ipv6.conf.all.forwarding`, plus
+the IPv4 ones once the shard serves NAT64. `bpf_fib_lookup` refuses every
+lookup on an interface with forwarding off, so a sysctl that does not read
+`1` after the write stops the shard at startup, and leaves an uplink found
+later uncovered until it does. A pod that is not privileged gets `/proc/sys`
+read-only, so `config/galactic-nat/base/daemonset.yaml` mounts the host's
+`/proc/sys/net` at `/host/proc/sys/net` and sets this to `/host/proc/sys`. A
+node where something else already turned forwarding on passes with the
+default too. SELinux policy on an enforcing node can still refuse the write,
+and the shard then says so at startup.
+
 ### Capabilities and host requirements
 
 `galactic-nat` runs `hostNetwork: true` and needs:

internal/config/config.go

@@ -22,6 +22,10 @@ const (
 	DefaultLogFile    = "/var/log/galactic/galactic-cni.log"
 	DefaultLogLevel   = "info"
 
+	// DefaultProcSysPath is the procfs sysctl root galactic-gateway and
+	// galactic-nat write forwarding sysctls under unless configured otherwise.
+	DefaultProcSysPath = "/proc/sys"
+
 	LogLevelDebug   = "debug"
 	LogLevelWarn    = "warn"
 	LogLevelWarning = "warning"

internal/config/gateway.go

@@ -75,6 +75,13 @@ const (
 	// load-balance it. Every other datapath on the node keeps running. The
 	// public interface and SRv6 address are not required while it is off.
 	EnvGatewayDatapathEnabled = "GALACTIC_GATEWAY_DATAPATH_ENABLED"
+
+	// EnvGatewayProcSysPath is the procfs sysctl root the datapath writes its
+	// interfaces' forwarding sysctls under. Optional, defaulting to
+	// DefaultProcSysPath. An unprivileged pod gets /proc/sys read-only, so the
+	// DaemonSet mounts the host's /proc/sys/net at /host/proc/sys/net and sets
+	// this to /host/proc/sys.
+	EnvGatewayProcSysPath = "GALACTIC_GATEWAY_PROC_SYS_PATH"
 )
 
 // EnvGatewayXDPAttach's values.
@@ -113,6 +120,9 @@ type GatewayConfig struct {
 
 	// DatapathEnabled is EnvGatewayDatapathEnabled.
 	DatapathEnabled bool
+
+	// ProcSysPath is EnvGatewayProcSysPath.
+	ProcSysPath string
 }
 
 // NewGatewayConfig creates a config resolver reading the GALACTIC_GATEWAY
@@ -130,6 +140,7 @@ func NewGatewayConfig() *GatewayConfig {
 	v.SetDefault("srv6_address", "")
 	v.SetDefault("xdp_attach", GatewayXDPAttachDispatch)
 	v.SetDefault("datapath_enabled", true)
+	v.SetDefault("proc_sys_path", DefaultProcSysPath)
 
 	cfg := &GatewayConfig{
 		v:      v,
@@ -154,6 +165,7 @@ func (c *GatewayConfig) BindFlags(flags *pflag.FlagSet) {
 		{"gateway-srv6-address", "srv6_address"},
 		{"gateway-xdp-attach", "xdp_attach"},
 		{"gateway-datapath-enabled", "datapath_enabled"},
+		{"gateway-proc-sys-path", "proc_sys_path"},
 	}
 	for _, b := range bindings {
 		if flags.Changed(b.flag) {
@@ -176,6 +188,7 @@ func (c *GatewayConfig) readFields() {
 	c.SRv6Address = c.v.GetString("srv6_address")
 	c.XDPAttach = c.v.GetString("xdp_attach")
 	c.DatapathEnabled = c.v.GetBool("datapath_enabled")
+	c.ProcSysPath = c.v.GetString("proc_sys_path")
 }
 
 // splitCommaList parses a comma-separated list, such as interface names or BMP

internal/config/gateway_test.go

@@ -207,3 +207,13 @@ func TestGatewayConfigDisabledNeedsNoInterface(t *testing.T) {
 		t.Error("Validate() = nil for an enabled datapath with no public interface")
 	}
 }
+
+func TestGatewayConfigProcSysPath(t *testing.T) {
+	if got := NewGatewayConfig().ProcSysPath; got != DefaultProcSysPath {
+		t.Errorf("default ProcSysPath = %q, want %q", got, DefaultProcSysPath)
+	}
+	t.Setenv(EnvGatewayProcSysPath, testHostProcSys)
+	if got := NewGatewayConfig().ProcSysPath; got != testHostProcSys {
+		t.Errorf("ProcSysPath = %q with %s set, want %s", got, EnvGatewayProcSysPath, testHostProcSys)
+	}
+}

internal/config/nat.go

@@ -90,6 +90,13 @@ const (
 	// are rate-limited in the datapath by a token bucket per CPU, shared with
 	// any other ICMP the shard emits on its own behalf.
 	EnvNATEchoResponder = "GALACTIC_NAT_ECHO_RESPONDER"
+
+	// EnvNATProcSysPath is the procfs sysctl root the datapath writes its
+	// uplinks' forwarding sysctls under. Optional, defaulting to
+	// DefaultProcSysPath. An unprivileged pod gets /proc/sys read-only, so the
+	// DaemonSet mounts the host's /proc/sys/net at /host/proc/sys/net and sets
+	// this to /host/proc/sys.
+	EnvNATProcSysPath = "GALACTIC_NAT_PROC_SYS_PATH"
 )
 
 // EnvNATXDPAttach's values. NATXDPAttachChain is deprecated and read as
@@ -132,6 +139,9 @@ type NATConfig struct {
 
 	// EchoResponder is EnvNATEchoResponder.
 	EchoResponder bool
+
+	// ProcSysPath is EnvNATProcSysPath.
+	ProcSysPath string
 }
 
 // NewNATConfig creates a config resolver reading the GALACTIC_NAT
@@ -149,6 +159,7 @@ func NewNATConfig() *NATConfig {
 	v.SetDefault("xdp_attach", NATXDPAttachDirect)
 	v.SetDefault("datapath_enabled", true)
 	v.SetDefault("echo_responder", false)
+	v.SetDefault("proc_sys_path", DefaultProcSysPath)
 
 	cfg := &NATConfig{
 		v:      v,
@@ -172,6 +183,7 @@ func (c *NATConfig) BindFlags(flags *pflag.FlagSet) {
 		{"nat-xdp-attach", "xdp_attach"},
 		{"nat-datapath-enabled", "datapath_enabled"},
 		{"nat-echo-responder", "echo_responder"},
+		{"nat-proc-sys-path", "proc_sys_path"},
 	}
 	for _, b := range bindings {
 		if flags.Changed(b.flag) {
@@ -197,6 +209,7 @@ func (c *NATConfig) readFields() {
 	}
 	c.DatapathEnabled = c.v.GetBool("datapath_enabled")
 	c.EchoResponder = c.v.GetBool("echo_responder")
+	c.ProcSysPath = c.v.GetString("proc_sys_path")
 }
 
 // Validate checks that the required configuration fields are set.