diff --git a/internal/plumbing/ebpf/attach/attach.go b/internal/plumbing/ebpf/attach/attach.go index 0eb97219..5ee0283a 100644 --- a/internal/plumbing/ebpf/attach/attach.go +++ b/internal/plumbing/ebpf/attach/attach.go @@ -250,10 +250,31 @@ func Load(pinDir string) (objs *prog.UsidObjects, err error) { err = fmt.Errorf("attach: pin usid_service_egress program: %w", pinErr) return nil, err } + removeObsoletePins(pinDir) return &loaded, nil } +// obsoletePins are pins of maps no compiled program declares any more, left +// by an earlier build. egress_shard_group_table and egress_shard_table held +// the egress shard groups as ordinary arrays updated in place, before +// egress_shard_groups replaced them with snapshots swapped in whole; nothing +// reads them, so they are only memory. +var obsoletePins = []string{"egress_shard_group_table", "egress_shard_table"} + +// removeObsoletePins removes the obsoletePins under pinDir. A failure is +// logged: the pin costs memory, not correctness. +func removeObsoletePins(pinDir string) { + for _, name := range obsoletePins { + path := filepath.Join(pinDir, name) + if err := os.Remove(path); err == nil { + slog.Info("attach: removed obsolete eBPF map pin", "pin", path) + } else if !os.IsNotExist(err) { + slog.Warn("attach: could not remove obsolete eBPF map pin", "pin", path, "err", err) + } + } +} + // UsidEgressPinName is the bpffs filename usid_egress is pinned under, distinct // from every map name in the same directory so the two can never collide. const UsidEgressPinName = "usid_egress_prog" diff --git a/internal/plumbing/ebpf/attachreg/egressgroups.go b/internal/plumbing/ebpf/attachreg/egressgroups.go new file mode 100644 index 00000000..bc89b53b --- /dev/null +++ b/internal/plumbing/ebpf/attachreg/egressgroups.go @@ -0,0 +1,170 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package attachreg + +import ( + "errors" + "fmt" + "io" + "log/slog" + "net" + "os" + + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/plumbing/srv6" +) + +// shardGroupReader is what installing a VRF's egress routes needs from the +// egress shard groups. *egressroutemap.ShardGroupTable implements it. +type shardGroupReader interface { + AnyOpen() (bool, error) + FindClass(class egressroutemap.TranslationClass) (uint32, egressroutemap.GroupStatus, bool, error) +} + +// openShardGroupsFn opens the shard groups pinned under pinDir. A datapath +// older than the shard group map has no such pin, which reads as nil: ordered +// mode, as before. A variable so tests can state the groups without a pinned +// map. +var openShardGroupsFn = func(pinDir string) (shardGroupReader, io.Closer, error) { + groups, closer, err := egressroutemap.OpenPinnedShardGroupTable(pinDir) + if errors.Is(err, os.ErrNotExist) { + return nil, io.NopCloser(nil), nil + } + if err != nil { + return nil, nil, fmt.Errorf("open pinned egress shard groups: %w", err) + } + return groups, closer, nil +} + +// egressGroupRouteAddFn is a variable so tests can observe group route +// installs without a pinned map. +var egressGroupRouteAddFn = func( + table *egressroutemap.EgressRouteTable, vrfTableID uint32, prefix *net.IPNet, groupID uint32, argument uint16, +) error { + return table.RegisterGroup(vrfTableID, prefix, groupID, argument) +} + +// egressPrefixes returns the routes a VRF's egress needs: ::/0, then one per +// NAT64 prefix in nat64Raw. +func egressPrefixes(nat64Raw string) ([]*net.IPNet, error) { + prefixes, err := config.ParseNAT64Prefixes(nat64Raw) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", config.EnvCNINAT64Prefix, err) + } + return append([]*net.IPNet{egressroutemap.DefaultPrefix}, prefixes...), nil +} + +// installVRFEgress installs vrfTableID's egress routes, in hashed mode when +// any shard group is open and in ordered mode otherwise. exists, when not nil, +// skips routes already present. +// +// In hashed mode each route names the group serving its translation class: +// NAT66 for ::/0, the matching NAT64 prefix for the others. A class no group +// serves, or whose group has no eligible shard, gets no route, as an empty +// static list gives none. A class whose shards are all unreachable or +// draining fails the ADD, as an ordered list with no resolvable shard does. +// +// Once the routes are written the groups are read again. galactic-cni marks +// every group closing before it moves the cluster back to ordered mode, and +// moves routes off a group only after that; a route written by an ADD that +// read the groups before they closed, and is written after galactic-cni's last +// scan, would otherwise outlive the group. If the groups closed meanwhile, +// this ADD rewrites its routes in ordered mode itself. +func installVRFEgress( + groups shardGroupReader, table *egressroutemap.EgressRouteTable, vrfTableID uint32, argument uint16, + cfg EgressConfig, exists routeExistsFn, +) (int, error) { + open := false + if groups != nil { + var err error + if open, err = groups.AnyOpen(); err != nil { + return 0, fmt.Errorf("read egress shard groups: %w", err) + } + } + if !open { + return installOrderedEgress(table, vrfTableID, argument, cfg, exists) + } + + n, err := installGroupRoutesIn(groups, table, vrfTableID, argument, cfg.NAT64Prefix, exists) + if err != nil { + return n, err + } + stillOpen, err := groups.AnyOpen() + if err != nil { + return n, fmt.Errorf("re-read egress shard groups: %w", err) + } + if stillOpen { + return n, nil + } + slog.Info("egress shard groups closed while installing this VRF's routes; installing them in ordered mode", + "vrfTable", vrfTableID) + m, err := installOrderedEgress(table, vrfTableID, argument, cfg, nil) + if err != nil { + return n, fmt.Errorf("egress shard groups closed during install, and ordered install failed: %w", err) + } + return m, nil +} + +// installOrderedEgress installs vrfTableID's routes toward the first +// resolvable shard of cfg's static list. An empty list installs nothing. +func installOrderedEgress( + table *egressroutemap.EgressRouteTable, vrfTableID uint32, argument uint16, cfg EgressConfig, + exists routeExistsFn, +) (int, error) { + tenantSIDs, err := tenantShardSIDs(cfg, argument) + if err != nil || len(tenantSIDs) == 0 { + return 0, err + } + return installEgressRoutesIn(table, vrfTableID, tenantSIDs, cfg.NAT64Prefix, exists) +} + +// installGroupRoutesIn writes, for each of the VRF's egress prefixes, a +// sentinel naming the open group that serves the prefix's class, carrying +// argument. Nothing is resolved here: the group's shards carry their own next +// hops. +func installGroupRoutesIn( + groups shardGroupReader, table *egressroutemap.EgressRouteTable, vrfTableID uint32, argument uint16, + nat64Raw string, exists routeExistsFn, +) (int, error) { + prefixes, err := egressPrefixes(nat64Raw) + if err != nil { + return 0, err + } + written := 0 + for _, prefix := range prefixes { + if exists != nil { + found, err := exists(prefix) + if err != nil { + return written, err + } + if found { + continue + } + } + class, ok := egressroutemap.ClassForRoute(prefix) + if !ok { + continue + } + groupID, status, found, err := groups.FindClass(class) + if err != nil { + return written, fmt.Errorf("find the %s egress shard group: %w", class, err) + } + if !found || !status.Open() || status.Candidates == 0 { + slog.Warn("no egress shard in the cluster translates this route's class; installing no route for it", + "vrfTable", vrfTableID, "prefix", prefix, "class", class.String()) + continue + } + if status.Active == 0 { + return written, fmt.Errorf("%w yet for %s: the %s egress shard group has %d candidates and none is "+ + "reachable and accepting", srv6.ErrNoShardResolvable, prefix, class, status.Candidates) + } + if err := egressGroupRouteAddFn(table, vrfTableID, prefix, groupID, argument); err != nil { + return written, fmt.Errorf("install %s group route for %s: %w", class, prefix, err) + } + written++ + } + return written, nil +} diff --git a/internal/plumbing/ebpf/attachreg/fill.go b/internal/plumbing/ebpf/attachreg/fill.go index 636c41e2..853a5ebf 100644 --- a/internal/plumbing/ebpf/attachreg/fill.go +++ b/internal/plumbing/ebpf/attachreg/fill.go @@ -34,6 +34,9 @@ type Maps struct { EgressRoute *egressroutemap.EgressRouteTable NodeSource *egressroutemap.NodeSourceAddress PublicUplink *egressroutemap.PublicUplink + // ShardGroups is nil on a datapath older than the shard group maps, + // which runs in ordered mode. + ShardGroups *egressroutemap.ShardGroupTable // routePrefixes is egress_route_table's exact prefixes by VRF table, // read on first use and updated with every route this Maps writes. @@ -134,6 +137,15 @@ func OpenPinnedMaps(pinDir string) (*Maps, io.Closer, error) { } m.PublicUplink, cs = uplink, append(cs, uplinkCloser) + groups, groupsCloser, err := egressroutemap.OpenPinnedShardGroupTable(pinDir) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + return fail(fmt.Errorf("open pinned egress shard group: %w", err)) + default: + m.ShardGroups, cs = groups, append(cs, groupsCloser) + } + return &m, cs, nil } @@ -340,19 +352,10 @@ func (m *Maps) FillVRF(a Attachment, egress EgressConfig) (VRFRows, error) { present[prefix.String()] = struct{}{} return ok, nil } - tenantSIDs, err := tenantShardSIDs(egress, a.Argument) - switch { - case err != nil: + n, err := m.fillShardRoutes(a, egress, routeExists) + written.ShardEgressRoutes += n + if err != nil { errs = append(errs, err) - case len(tenantSIDs) > 0 && !m.shardsUnresolvable: - n, err := installEgressRoutesIn(m.EgressRoute, a.VRFTableID, tenantSIDs, egress.NAT64Prefix, routeExists) - written.ShardEgressRoutes += n - if errors.Is(err, srv6.ErrNoShardResolvable) { - m.shardsUnresolvable = true - } - if err != nil { - errs = append(errs, fmt.Errorf("install shard egress routes: %w", err)) - } } local, err := registerLocalEgressRoutesIn(m.EgressRoute, a.VRFTableID, a.LocalPrefixes, routeExists) @@ -387,6 +390,26 @@ func (m *Maps) FillVRF(a Attachment, egress EgressConfig) (VRFRows, error) { return written, errors.Join(errs...) } +// fillShardRoutes writes a's VRF's shard routes for FillVRF exactly as CNI ADD +// would: see installVRFEgress. +func (m *Maps) fillShardRoutes(a Attachment, egress EgressConfig, routeExists routeExistsFn) (int, error) { + if m.shardsUnresolvable { + return 0, nil + } + var groups shardGroupReader + if m.ShardGroups != nil { + groups = m.ShardGroups + } + n, err := installVRFEgress(groups, m.EgressRoute, a.VRFTableID, a.Argument, egress, routeExists) + if errors.Is(err, srv6.ErrNoShardResolvable) { + m.shardsUnresolvable = true + } + if err != nil { + return n, fmt.Errorf("install shard egress routes: %w", err) + } + return n, nil +} + // RemoveVRFRows removes rows FillVRF wrote for a, as written records them, for // an attachment whose interface went away while they were written. Local // routes in keepPrefixes, which another attachment in the same VRF table diff --git a/internal/plumbing/ebpf/attachreg/group_test.go b/internal/plumbing/ebpf/attachreg/group_test.go new file mode 100644 index 00000000..8df1b4f5 --- /dev/null +++ b/internal/plumbing/ebpf/attachreg/group_test.go @@ -0,0 +1,303 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package attachreg + +import ( + "errors" + "net" + "net/netip" + "testing" + + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/srv6" +) + +// testShardSID is a static shard list of one entry. +const testShardSID = "2001:db8:ff01:9:e001::" + +// fakeGroups is a shardGroupReader over fixed group statuses. openCalls lists +// what successive AnyOpen calls answer; the last answer repeats. +type fakeGroups struct { + groups map[uint32]egressroutemap.GroupStatus + openCalls []bool + calls int +} + +func (f *fakeGroups) AnyOpen() (bool, error) { + if len(f.openCalls) == 0 { + for _, s := range f.groups { + if s.Open() { + return true, nil + } + } + return false, nil + } + i := min(f.calls, len(f.openCalls)-1) + f.calls++ + return f.openCalls[i], nil +} + +func (f *fakeGroups) FindClass( + class egressroutemap.TranslationClass, +) (uint32, egressroutemap.GroupStatus, bool, error) { + for id, s := range f.groups { + if s.Class == class { + return id, s, true, nil + } + } + return 0, egressroutemap.GroupStatus{}, false, nil +} + +func openStatus(class egressroutemap.TranslationClass, candidates, active int) egressroutemap.GroupStatus { + return egressroutemap.GroupStatus{ + Present: true, Enabled: true, Class: class, Candidates: candidates, Active: active, + } +} + +var testNAT64Class = egressroutemap.NAT64Class(netip.MustParsePrefix(testNAT64Prefix)) + +// groupCall is one egressGroupRouteAddFn invocation. +type groupCall struct { + group uint32 + argument uint16 +} + +// recordGroupAdds replaces egressGroupRouteAddFn with a recorder, keyed by +// prefix. +func recordGroupAdds(t *testing.T) *map[string]groupCall { + t.Helper() + original := egressGroupRouteAddFn + t.Cleanup(func() { egressGroupRouteAddFn = original }) + calls := map[string]groupCall{} + egressGroupRouteAddFn = func( + _ *egressroutemap.EgressRouteTable, _ uint32, prefix *net.IPNet, group uint32, argument uint16, + ) error { + calls[prefix.String()] = groupCall{group, argument} + return nil + } + return &calls +} + +func refuseOrderedRoutes(t *testing.T) { + t.Helper() + original := egressPrefixRouteAddFn + t.Cleanup(func() { egressPrefixRouteAddFn = original }) + egressPrefixRouteAddFn = func(*egressroutemap.EgressRouteTable, uint32, *net.IPNet, []net.IP) error { + t.Error("ordered shard route written in hashed mode") + return nil + } +} + +// TestInstallVRFEgress_EachRouteGoesToItsClassGroup checks that ::/0 names the +// NAT66 group and the NAT64 prefix names its own group, each carrying the +// attachment's Argument, and that the static list is not read. +func TestInstallVRFEgress_EachRouteGoesToItsClassGroup(t *testing.T) { + calls := recordGroupAdds(t) + refuseOrderedRoutes(t) + groups := &fakeGroups{groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 2, 2), + 1: openStatus(testNAT64Class, 1, 1), + }} + n, err := installVRFEgress(groups, nil, 7, 0x21, + EgressConfig{ShardSIDs: testShardSID, NAT64Prefix: testNAT64Prefix}, nil) + if err != nil || n != 2 { + t.Fatalf("installVRFEgress = %d, %v, want 2 routes", n, err) + } + if c := (*calls)[defaultPrefix]; c != (groupCall{0, 0x21}) { + t.Errorf("::/0 -> %+v, want group 0 argument 0x21", c) + } + if c := (*calls)[testNAT64Prefix]; c != (groupCall{1, 0x21}) { + t.Errorf("%s -> %+v, want group 1 argument 0x21", testNAT64Prefix, c) + } +} + +// TestInstallVRFEgress_SingleFamilyDeployments checks a cluster whose shards +// serve one family only, and one whose shards translate a different NAT64 +// prefix: a route goes only to a group that can translate it, and a route no +// group can serve is not installed rather than pointed at shards that would +// drop it. +func TestInstallVRFEgress_SingleFamilyDeployments(t *testing.T) { + cfg := EgressConfig{NAT64Prefix: testNAT64Prefix} + tests := []struct { + name string + groups map[uint32]egressroutemap.GroupStatus + want []string + absent []string + }{ + { + name: "NAT66Only", + groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 2, 2), + 1: openStatus(testNAT64Class, 0, 0), // every shard ineligible + }, + want: []string{defaultPrefix}, absent: []string{testNAT64Prefix}, + }, + { + name: "NAT64Only", + groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 0, 0), + 1: openStatus(testNAT64Class, 2, 2), + }, + want: []string{testNAT64Prefix}, absent: []string{defaultPrefix}, + }, + { + name: "DualCapable", + groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 2, 2), + 1: openStatus(testNAT64Class, 2, 2), + }, + want: []string{defaultPrefix, testNAT64Prefix}, + }, + { + name: "MismatchedPrefix", + groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 1, 1), + 1: openStatus(egressroutemap.NAT64Class(netip.MustParsePrefix("2001:db8:65::/96")), 1, 1), + }, + want: []string{defaultPrefix}, absent: []string{testNAT64Prefix}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + calls := recordGroupAdds(t) + if _, err := installVRFEgress(&fakeGroups{groups: tt.groups}, nil, 7, 0x21, cfg, nil); err != nil { + t.Fatal(err) + } + for _, p := range tt.want { + if _, ok := (*calls)[p]; !ok { + t.Errorf("no route installed for %s", p) + } + } + for _, p := range tt.absent { + if c, ok := (*calls)[p]; ok { + t.Errorf("route for %s installed toward group %d, which cannot translate it", p, c.group) + } + } + }) + } +} + +// TestInstallVRFEgress_UnreachableClassFails checks a class with candidates but +// none reachable fails the ADD, as an ordered list with no resolvable shard +// does. +func TestInstallVRFEgress_UnreachableClassFails(t *testing.T) { + recordGroupAdds(t) + groups := &fakeGroups{groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 3, 0), + }} + _, err := installVRFEgress(groups, nil, 7, 0x21, EgressConfig{}, nil) + if !errors.Is(err, srv6.ErrNoShardResolvable) { + t.Errorf("installVRFEgress = %v, want ErrNoShardResolvable", err) + } +} + +// TestInstallVRFEgress_ClosedGroupsUseTheStaticList checks that closing groups +// send ADD down the ordered path. +func TestInstallVRFEgress_ClosedGroupsUseTheStaticList(t *testing.T) { + ordered := recordRouteAdds(t, nil) + group := recordGroupAdds(t) + closing := openStatus(egressroutemap.NAT66Class, 1, 1) + closing.Closing = true + groups := &fakeGroups{groups: map[uint32]egressroutemap.GroupStatus{0: closing}} + if _, err := installVRFEgress(groups, nil, 7, 0x21, EgressConfig{ShardSIDs: testShardSID}, nil); err != nil { + t.Fatal(err) + } + if len(*ordered) != 1 || len(*group) != 0 { + t.Errorf("ordered routes %d, group routes %d, want 1 and 0", len(*ordered), len(*group)) + } +} + +// TestInstallVRFEgress_GroupsCloseDuringInstall covers the F01 race from the +// ADD side: an ADD that saw the groups open, but finds them closed once its +// routes are written, rewrites them in ordered mode, so no route naming a +// group outlives galactic-cni's last scan. +func TestInstallVRFEgress_GroupsCloseDuringInstall(t *testing.T) { + ordered := recordRouteAdds(t, nil) + recordGroupAdds(t) + groups := &fakeGroups{ + groups: map[uint32]egressroutemap.GroupStatus{0: openStatus(egressroutemap.NAT66Class, 1, 1)}, + openCalls: []bool{true, false}, + } + if _, err := installVRFEgress(groups, nil, 7, 0x21, EgressConfig{ShardSIDs: testShardSID}, nil); err != nil { + t.Fatal(err) + } + if len(*ordered) != 1 || (*ordered)[0].prefix != defaultPrefix { + t.Errorf("ordered rewrite = %+v, want ::/0 rewritten toward the static shard", *ordered) + } +} + +// TestInstallVRFEgress_GroupsCloseAndOrderedFails checks the ADD reports a +// failed ordered rewrite instead of succeeding with a route on a closing group. +func TestInstallVRFEgress_GroupsCloseAndOrderedFails(t *testing.T) { + recordRouteAdds(t, srv6.ErrNoShardResolvable) + recordGroupAdds(t) + groups := &fakeGroups{ + groups: map[uint32]egressroutemap.GroupStatus{0: openStatus(egressroutemap.NAT66Class, 1, 1)}, + openCalls: []bool{true, false}, + } + _, err := installVRFEgress(groups, nil, 7, 0x21, EgressConfig{ShardSIDs: testShardSID}, nil) + if !errors.Is(err, srv6.ErrNoShardResolvable) { + t.Errorf("installVRFEgress = %v, want the ordered failure reported", err) + } +} + +// valueStore is a GroupStore over fixed snapshots. +type valueStore map[uint32]prog.UsidEgressShardGroupValue + +func (s valueStore) Load(id uint32) (prog.UsidEgressShardGroupValue, bool, error) { + v, ok := s[id] + return v, ok, nil +} + +func (s valueStore) Publish(id uint32, v prog.UsidEgressShardGroupValue) error { + s[id] = v + return nil +} + +// TestFillVRF_HashedModeUsesTheGroups checks the datapath repair path goes +// through the same installer, against a real ShardGroupTable. +func TestFillVRF_HashedModeUsesTheGroups(t *testing.T) { + calls := recordGroupAdds(t) + refuseOrderedRoutes(t) + nat64 := prog.UsidEgressShardGroupValue{Flags: 1, Candidates: 1, Active: 1, ClassKind: 2, + ClassPrefix: netip.MustParsePrefix(testNAT64Prefix).Addr().As16()} + m := newMemMaps(true) + m.ShardGroups = egressroutemap.NewShardGroupTable(valueStore{ + 0: {Flags: 1, Candidates: 1, Active: 1, ClassKind: 1}, + 1: nat64, + }) + written, err := m.FillVRF(testAttachment(), EgressConfig{NAT64Prefix: testNAT64Prefix}) + if err != nil || written.ShardEgressRoutes != 2 || len(*calls) != 2 { + t.Errorf("FillVRF = %+v, %v with %d group routes, want 2", written.Counts, err, len(*calls)) + } +} + +// TestFillVRF_DisabledGroupUsesTheStaticList checks ordered mode is unchanged +// when the group map exists but no group is open. +func TestFillVRF_DisabledGroupUsesTheStaticList(t *testing.T) { + calls := recordRouteAdds(t, nil) + group := recordGroupAdds(t) + m := newMemMaps(true) + m.ShardGroups = egressroutemap.NewShardGroupTable(valueStore{0: {ClassKind: 1}}) + if _, err := m.FillVRF(testAttachment(), EgressConfig{ShardSIDs: testShardSID}); err != nil { + t.Fatalf("FillVRF: %v", err) + } + if len(*calls) != 1 || len(*group) != 0 { + t.Errorf("ordered routes %d, group routes %d, want 1 and 0", len(*calls), len(*group)) + } +} + +// TestOpenShardGroups_MissingPinIsOrderedMode checks that a datapath without +// the shard group map reads as no groups. +func TestOpenShardGroups_MissingPinIsOrderedMode(t *testing.T) { + groups, closer, err := openShardGroupsFn(t.TempDir()) + if err != nil || groups != nil { + t.Errorf("openShardGroupsFn = %v, %v, want no groups and no error", groups, err) + } + if err := closer.Close(); err != nil { + t.Error(err) + } +} diff --git a/internal/plumbing/ebpf/attachreg/register.go b/internal/plumbing/ebpf/attachreg/register.go index 70fe795e..e92b627a 100644 --- a/internal/plumbing/ebpf/attachreg/register.go +++ b/internal/plumbing/ebpf/attachreg/register.go @@ -386,18 +386,29 @@ func registerLocalEgressRoutesIn( // // No shard configured is not an error and installs nothing. A shard SID that // is invalid, or that has no reachable route yet, is an error. +// +// Where the installer runs the cluster's shards as groups (hashed mode), the +// routes name the group serving each route's class instead of one shard, and +// the static shard list is not read; see installVRFEgress. func installEgressRoutes(pinDir string, vrfTableID uint32, argument uint16, cfg EgressConfig) error { - tenantSIDs, err := tenantShardSIDs(cfg, argument) - if err != nil || len(tenantSIDs) == 0 { + groups, groupsCloser, err := openShardGroupsFn(pinDir) + if err != nil { return err } + defer func() { _ = groupsCloser.Close() }() + + if groups == nil { + if tenantSIDs, err := tenantShardSIDs(cfg, argument); err != nil || len(tenantSIDs) == 0 { + return err + } + } table, closer, err := egressroutemap.OpenPinnedEgressRouteTable(pinDir) if err != nil { return fmt.Errorf("open pinned egress_route_table: %w", err) } defer func() { _ = closer.Close() }() - _, err = installEgressRoutesIn(table, vrfTableID, tenantSIDs, cfg.NAT64Prefix, nil) + _, err = installVRFEgress(groups, table, vrfTableID, argument, cfg, nil) return err } @@ -431,6 +442,15 @@ func installEgressRoutesIn( table *egressroutemap.EgressRouteTable, vrfTableID uint32, tenantSIDs []net.IP, nat64Raw string, exists routeExistsFn, ) (int, error) { + return installEgressPrefixes(func(prefix *net.IPNet) error { + return egressPrefixRouteAddFn(table, vrfTableID, prefix, tenantSIDs) + }, nat64Raw, exists) +} + +// installEgressPrefixes writes ::/0 and then each NAT64 prefix in nat64Raw +// with write, skipping any prefix exists reports present, and returns how +// many it wrote. +func installEgressPrefixes(write func(prefix *net.IPNet) error, nat64Raw string, exists routeExistsFn) (int, error) { written := 0 add := func(prefix *net.IPNet) error { if exists != nil { @@ -439,7 +459,7 @@ func installEgressRoutesIn( return err } } - if err := egressPrefixRouteAddFn(table, vrfTableID, prefix, tenantSIDs); err != nil { + if err := write(prefix); err != nil { return err } written++ diff --git a/internal/plumbing/ebpf/egressroutemap/egressroute.go b/internal/plumbing/ebpf/egressroutemap/egressroute.go index adac30af..ae8aeb4c 100644 --- a/internal/plumbing/ebpf/egressroutemap/egressroute.go +++ b/internal/plumbing/ebpf/egressroutemap/egressroute.go @@ -123,6 +123,18 @@ func sidTo16(sid net.IP) ([16]byte, error) { // given is a fake. var resolveLinkAndL2Fn = resolveLinkAndL2 +// OverrideResolverForTest replaces the next-hop resolver Register, Refresh and +// the shard groups use, for tests in other packages that drive them against +// fake maps, and returns a function restoring it. Never call it outside a +// test. +func OverrideResolverForTest( + fn func(sid net.IP) (linkIndex int, dmac, smac net.HardwareAddr, err error), +) (restore func()) { + prev := resolveLinkAndL2Fn + resolveLinkAndL2Fn = fn + return func() { resolveLinkAndL2Fn = prev } +} + // uplinkIndexesFn is an override point so tests can state which links count as // SRv6 uplinks without a host that has any. Production always leaves it at // attach.UplinkIndexes. @@ -355,12 +367,7 @@ func (t *EgressRouteTable) Prefixes() (map[uint32]map[string]struct{}, error) { ) iter := t.table.Iterate() for iter.Next(&key, &value) { - ones := int(key.Prefixlen) - egressRouteKeyFixedBits - prefix := &net.IPNet{IP: net.IP(append([]byte(nil), key.Addr[:]...)), Mask: net.CIDRMask(ones, 128)} - if key.Family == egressRouteFamilyINET4 { - prefix = &net.IPNet{IP: net.IP(append([]byte(nil), key.Addr[:4]...)), Mask: net.CIDRMask(ones, 32)} - } - prefix.IP = prefix.IP.Mask(prefix.Mask) + prefix := keyPrefix(key) if byTable[key.TableId] == nil { byTable[key.TableId] = map[string]struct{}{} } @@ -522,6 +529,62 @@ type RefreshResult struct { // them, counted separately from the pass-through entries that have nothing // to resolve. Skipped int + // Grouped is the shard-bound entries rewritten to the shard group + // sentinel, in hashed mode. + Grouped int + // Ungrouped is the shard group sentinels rewritten back to a single shard, + // in ordered mode. + Ungrouped int + // InGroup is the shard group sentinels left in place: in hashed mode + // because they already name their class's group, or because no group + // serves their class; in ordered mode because no configured shard + // resolves for them yet. + InGroup int + // GroupRoutes counts, by group ID, the sentinels egress_route_table holds + // once the sweep's writes are done: those left in place and those just + // written. Valid only when Refresh returns no error. + GroupRoutes map[uint32]int +} + +// SentinelRoutes is the total of GroupRoutes: every route still naming a +// shard group. +func (r RefreshResult) SentinelRoutes() int { + n := 0 + for _, c := range r.GroupRoutes { + n += c + } + return n +} + +// RefreshOption adjusts one Refresh sweep. +type RefreshOption func(*refreshConfig) + +type refreshConfig struct { + groupFor func(prefix *net.IPNet) (uint32, bool) +} + +// WithShardGroups runs the sweep in hashed mode. groupFor names the group +// serving a route's prefix, or reports false when no group serves it. An entry +// bound to one of the sweep's shards whose prefix has a group is rewritten to +// a sentinel naming that group, keeping its Argument; a sentinel naming +// another group is moved to the right one; a sentinel whose prefix has no +// group is left alone. Without it, the sweep runs in ordered mode and rewrites +// every sentinel back to the first shard that resolves. +func WithShardGroups(groupFor func(prefix *net.IPNet) (uint32, bool)) RefreshOption { + return func(c *refreshConfig) { + c.groupFor = groupFor + } +} + +// keyPrefix returns the prefix an egress_route_table key covers. +func keyPrefix(key prog.UsidEgressRouteKey) *net.IPNet { + ones := int(key.Prefixlen) - egressRouteKeyFixedBits + prefix := &net.IPNet{IP: net.IP(append([]byte(nil), key.Addr[:]...)), Mask: net.CIDRMask(ones, 128)} + if key.Family == egressRouteFamilyINET4 { + prefix = &net.IPNet{IP: net.IP(append([]byte(nil), key.Addr[:4]...)), Mask: net.CIDRMask(ones, 32)} + } + prefix.IP = prefix.IP.Mask(prefix.Mask) + return prefix } // Refresh re-resolves the encapsulating entries this caller's network namespace @@ -579,8 +642,16 @@ type RefreshResult struct { // inert -- an entry keyed on a routing table no packet reaches any more, since // the VRF it belonged to is being torn down -- so it is not worth a // compare-and-swap the map API does not offer. -func (t *EgressRouteTable) Refresh(foreignTableIDs map[uint32]struct{}, shardSIDs []net.IP) (RefreshResult, error) { - var result RefreshResult +func (t *EgressRouteTable) Refresh( + foreignTableIDs map[uint32]struct{}, shardSIDs []net.IP, opts ...RefreshOption, +) (RefreshResult, error) { + var ( + result = RefreshResult{GroupRoutes: map[uint32]int{}} + cfg refreshConfig + ) + for _, opt := range opts { + opt(&cfg) + } shards := newShardSelector(shardSIDs) @@ -602,12 +673,40 @@ func (t *EgressRouteTable) Refresh(foreignTableIDs map[uint32]struct{}, shardSID } if _, foreign := foreignTableIDs[key.TableId]; foreign { result.Skipped++ + if isGroupSentinel(value) { + // Not ours to move, but a group it names must not be retired + // under it. + groupID, _ := sentinelGroup(value) + result.InGroup++ + result.GroupRoutes[groupID]++ + } continue // resolved in another network namespace; not ours to rewrite } + if isGroupSentinel(value) { + update, ok := t.refreshSentinel(key, value, cfg, shards, &result) + if ok { + updates = append(updates, pending{key: key, value: update}) + } + continue + } + sid := make(net.IP, 16) copy(sid, value.Sid[:]) + if cfg.groupFor != nil { + if fields, err := decodeSID(sid); err == nil && shards.isShard(fields) { + if groupID, ok := cfg.groupFor(keyPrefix(key)); ok { + updates = append(updates, pending{key: key, value: groupSentinel(groupID, fields.Argument)}) + result.Grouped++ + result.GroupRoutes[groupID]++ + slog.Info("egressroutemap: refresh: egress route moved onto an egress shard group", + "table", key.TableId, "fromSid", sid, "group", groupID) + continue + } + } + } + target, next, err := shards.resolve(sid) if err != nil { result.Unresolved++ @@ -654,6 +753,47 @@ func (t *EgressRouteTable) Refresh(foreignTableIDs map[uint32]struct{}, shardSID return result, nil } +// refreshSentinel decides what one shard group sentinel becomes. In hashed +// mode a sentinel naming the sweep's group stays, and one naming another +// group is moved to it. In ordered mode it goes back to the first configured +// shard that resolves, with the sentinel's Argument; when none resolves it is +// counted unresolved and left alone, as any other entry would be. +func (t *EgressRouteTable) refreshSentinel( + key prog.UsidEgressRouteKey, value prog.UsidEgressRouteValue, cfg refreshConfig, shards *shardSelector, + result *RefreshResult, +) (prog.UsidEgressRouteValue, bool) { + groupID, argument := sentinelGroup(value) + if cfg.groupFor != nil { + target, ok := cfg.groupFor(keyPrefix(key)) + if !ok || target == groupID { + result.InGroup++ + result.GroupRoutes[groupID]++ + return value, false + } + result.Grouped++ + result.GroupRoutes[target]++ + return groupSentinel(target, argument), true + } + + target, next, err := shards.resolveArgument(argument) + if err != nil { + result.Unresolved++ + result.InGroup++ + result.GroupRoutes[groupID]++ + slog.Warn("egressroutemap: refresh: leaving egress route on its shard group, no configured shard resolves", + "table", key.TableId, "group", groupID, "err", err) + return value, false + } + updated := prog.UsidEgressRouteValue{LinkIfindex: uint32(next.link)} //nolint:gosec // a kernel ifindex + copy(updated.Sid[:], target.To16()) + copy(updated.Dmac[:], next.dmac) + copy(updated.Smac[:], next.smac) + result.Ungrouped++ + slog.Info("egressroutemap: refresh: egress route moved off the egress shard group onto one shard", + "table", key.TableId, "group", groupID, "toSid", target, "toLink", next.link) + return updated, true +} + // nextHop is one resolveLinkAndL2Fn answer. type nextHop struct { link int @@ -710,10 +850,15 @@ func (s *shardSelector) resolve(sid net.IP) (net.IP, nextHop, error) { link, dmac, smac, err := resolveLinkAndL2Fn(sid) return sid, nextHop{link: link, dmac: dmac, smac: smac}, err } + return s.resolveArgument(fields.Argument) +} +// resolveArgument returns the first configured shard, in preference order, +// that resolves, with argument written into its SID, and its next hop. +func (s *shardSelector) resolveArgument(argument uint16) (net.IP, nextHop, error) { var errs []error for _, shard := range s.shards { - shard.Argument = fields.Argument + shard.Argument = argument addr, err := uformat.Encode(shard) if err != nil { errs = append(errs, err) diff --git a/internal/plumbing/ebpf/egressroutemap/groupstore.go b/internal/plumbing/ebpf/egressroutemap/groupstore.go new file mode 100644 index 00000000..45421d03 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/groupstore.go @@ -0,0 +1,108 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "encoding/binary" + "errors" + "fmt" + "os" + + "github.com/cilium/ebpf" + + "go.datum.net/galactic/internal/plumbing/ebpf/prog" +) + +// GroupSnapshotSpec is the spec of one egress_shard_groups inner map, which +// must match the compiled map's inner template exactly: the kernel refuses to +// store an inner map of any other shape. +var GroupSnapshotSpec = ebpf.MapSpec{ + Name: "egress_shard_gr", + Type: ebpf.Array, + KeySize: 4, + ValueSize: uint32(binary.Size(prog.UsidEgressShardGroupValue{})), //nolint:gosec // a fixed struct size + MaxEntries: 1, +} + +// snapshotLookupAttempts bounds Load's retries when an inner map is replaced +// and freed between reading its ID and opening it. +const snapshotLookupAttempts = 5 + +// KernelGroupStore publishes group snapshots into egress_shard_groups, an +// ARRAY_OF_MAPS. Each publication creates a fresh inner map, fills it, and +// swaps it into the outer slot with one update. The swap replaces an +// RCU-protected pointer: a packet sees the old snapshot or the new one, never +// a mix, and the kernel frees the old inner map once no program can still be +// reading it. A published inner map is never written again. +// +// The three steps fail independently: creating the inner map, writing its +// value, and the swap. A failure before the swap publishes nothing. +type KernelGroupStore struct { + outer *ebpf.Map + + // The three publication steps, each a test override point so a fault can + // be injected at every boundary. + newMap func(*ebpf.MapSpec) (*ebpf.Map, error) + writeInner func(inner *ebpf.Map, value *prog.UsidEgressShardGroupValue) error + swap func(outer *ebpf.Map, groupID uint32, inner *ebpf.Map) error +} + +// NewKernelGroupStore wraps a loaded or pinned egress_shard_groups. +func NewKernelGroupStore(outer *ebpf.Map) *KernelGroupStore { + return &KernelGroupStore{ + outer: outer, + newMap: ebpf.NewMap, + writeInner: func(inner *ebpf.Map, value *prog.UsidEgressShardGroupValue) error { + return inner.Put(uint32(0), value) + }, + swap: func(outer *ebpf.Map, groupID uint32, inner *ebpf.Map) error { + return outer.Put(groupID, inner) + }, + } +} + +// Load implements GroupStore. +func (k *KernelGroupStore) Load(groupID uint32) (prog.UsidEgressShardGroupValue, bool, error) { + var value prog.UsidEgressShardGroupValue + for range snapshotLookupAttempts { + var inner *ebpf.Map + err := k.outer.Lookup(groupID, &inner) + if errors.Is(err, ebpf.ErrKeyNotExist) { + return value, false, nil + } + if errors.Is(err, os.ErrNotExist) { + // Replaced and freed between reading its ID and opening it. + continue + } + if err != nil { + return value, false, fmt.Errorf("look up egress_shard_groups[%d]: %w", groupID, err) + } + err = inner.Lookup(uint32(0), &value) + _ = inner.Close() + if err != nil { + return value, false, fmt.Errorf("read egress_shard_groups[%d] snapshot: %w", groupID, err) + } + return value, true, nil + } + return value, false, fmt.Errorf("read egress_shard_groups[%d]: snapshot replaced %d times while reading", + groupID, snapshotLookupAttempts) +} + +// Publish implements GroupStore. +func (k *KernelGroupStore) Publish(groupID uint32, value prog.UsidEgressShardGroupValue) error { + spec := GroupSnapshotSpec + inner, err := k.newMap(&spec) + if err != nil { + return fmt.Errorf("create snapshot map: %w", err) + } + defer func() { _ = inner.Close() }() + if err := k.writeInner(inner, &value); err != nil { + return fmt.Errorf("write snapshot: %w", err) + } + if err := k.swap(k.outer, groupID, inner); err != nil { + return fmt.Errorf("swap snapshot into egress_shard_groups[%d]: %w", groupID, err) + } + return nil +} diff --git a/internal/plumbing/ebpf/egressroutemap/pin.go b/internal/plumbing/ebpf/egressroutemap/pin.go index ae4cb09d..1f6c4a04 100644 --- a/internal/plumbing/ebpf/egressroutemap/pin.go +++ b/internal/plumbing/ebpf/egressroutemap/pin.go @@ -54,3 +54,18 @@ func OpenPinnedPublicUplink(pinDir string) (*PublicUplink, io.Closer, error) { } return NewPublicUplink(usidmap.KernelTable{Map: m}), m, nil } + +// OpenPinnedShardGroupTable opens egress_shard_groups from its pinned path +// under pinDir and returns a table over it. See OpenPinnedEgressRouteTable for +// the pinning convention and close contract. +// +// A datapath older than the shard group map has no such pin, and the error +// then matches os.ErrNotExist. +func OpenPinnedShardGroupTable(pinDir string) (*ShardGroupTable, io.Closer, error) { + outer, err := ebpf.LoadPinnedMap(filepath.Join(pinDir, prog.UsidMapEgressShardGroups), nil) + if err != nil { + return nil, nil, fmt.Errorf("egressroutemap: open pinned map %q under %q: %w", + prog.UsidMapEgressShardGroups, pinDir, err) + } + return NewShardGroupTable(NewKernelGroupStore(outer)), outer, nil +} diff --git a/internal/plumbing/ebpf/egressroutemap/refresh_group_test.go b/internal/plumbing/ebpf/egressroutemap/refresh_group_test.go new file mode 100644 index 00000000..93ddd69e --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/refresh_group_test.go @@ -0,0 +1,231 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "errors" + "net" + "net/netip" + "testing" + + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" +) + +// faultyTable wraps fakeTable with injected failures: iterErr ends every +// iteration with an error after its entries, and putFailAfter, when +// non-negative, fails every Put after that many have succeeded. +type faultyTable struct { + *fakeTable + iterErr error + putFailAfter int + puts int +} + +func newFaultyTable() *faultyTable { + return &faultyTable{fakeTable: newFakeTable(), putFailAfter: -1} +} + +var errInjectedPut = errors.New("injected map write failure") + +func (f *faultyTable) Put(key, value any) error { + if f.putFailAfter >= 0 && f.puts >= f.putFailAfter { + return errInjectedPut + } + f.puts++ + return f.fakeTable.Put(key, value) +} + +func (f *faultyTable) Iterate() usidmap.Iterator { + return &faultyIterator{Iterator: f.fakeTable.Iterate(), err: f.iterErr} +} + +type faultyIterator struct { + usidmap.Iterator + err error +} + +func (it *faultyIterator) Err() error { return it.err } + +// tenantSID is shard's SID carrying argument. +func tenantSID(shard netip.Addr, argument uint16) net.IP { + b := shard.As16() + b[8] = b[8]&0xF0 | byte(argument>>8)&0x0F + b[9] = byte(argument) + return net.IP(b[:]) +} + +var nsp = netip.MustParsePrefix("2001:db8:64::/96") + +func nspNet() *net.IPNet { + return &net.IPNet{IP: net.IP(nsp.Addr().AsSlice()), Mask: net.CIDRMask(96, 128)} +} + +// byClass is a WithShardGroups classifier: NAT66 to group 0, the NSP to 1. +func byClass(prefix *net.IPNet) (uint32, bool) { + class, ok := ClassForRoute(prefix) + switch { + case !ok: + return 0, false + case class == NAT66Class: + return 0, true + case class == NAT64Class(nsp): + return 1, true + } + return 0, false +} + +// TestRefreshMovesShardEntriesOntoTheirClassGroups checks hashed mode's sweep: +// each shard-bound route becomes a sentinel for its class's group, keeping its +// Argument; a sentinel on the wrong group moves; and the sentinel count comes +// back per group. +func TestRefreshMovesShardEntriesOntoTheirClassGroups(t *testing.T) { + stubResolveLinkAndL2(t) + tbl := NewEgressRouteTable(newFakeTable()) + shard := shardSID(0x10) + if err := tbl.Register(1, DefaultPrefix, tenantSID(shard, 0x123)); err != nil { + t.Fatal(err) + } + if err := tbl.Register(1, nspNet(), tenantSID(shard, 0x123)); err != nil { + t.Fatal(err) + } + if err := tbl.RegisterGroup(2, nspNet(), 3, 0x456); err != nil { // wrong group + t.Fatal(err) + } + + result, err := tbl.Refresh(nil, []net.IP{net.IP(shard.AsSlice())}, WithShardGroups(byClass)) + if err != nil { + t.Fatal(err) + } + if result.Grouped != 3 || result.Refreshed != 3 || result.SentinelRoutes() != 3 || + result.GroupRoutes[0] != 1 || result.GroupRoutes[1] != 2 { + t.Errorf("result = %+v, want 3 grouped, group 0 with 1 route and group 1 with 2", result) + } + for _, c := range []struct { + table uint32 + prefix *net.IPNet + group uint32 + arg uint16 + }{{1, DefaultPrefix, 0, 0x123}, {1, nspNet(), 1, 0x123}, {2, nspNet(), 1, 0x456}} { + key, _ := buildKey(c.table, c.prefix) + value, _ := lookupRaw(tbl, key) + if g, a := sentinelGroup(value); !isGroupSentinel(value) || g != c.group || a != c.arg { + t.Errorf("table %d %s = group %d arg %#x, want group %d arg %#x", c.table, c.prefix, g, a, c.group, c.arg) + } + } +} + +// TestRefreshLeavesAnUnservedClassAlone checks that a route whose class no +// group serves keeps its shard, and a sentinel for it stays counted. +func TestRefreshLeavesAnUnservedClassAlone(t *testing.T) { + stubResolveLinkAndL2(t) + tbl := NewEgressRouteTable(newFakeTable()) + shard := shardSID(0x10) + wkp := &net.IPNet{IP: net.IP(WellKnownPrefix.Addr().AsSlice()), Mask: net.CIDRMask(96, 128)} + if err := tbl.Register(1, wkp, tenantSID(shard, 0x123)); err != nil { + t.Fatal(err) + } + if err := tbl.RegisterGroup(2, wkp, 2, 0x456); err != nil { + t.Fatal(err) + } + result, err := tbl.Refresh(nil, []net.IP{net.IP(shard.AsSlice())}, WithShardGroups(byClass)) + if err != nil { + t.Fatal(err) + } + if result.Grouped != 0 || result.InGroup != 1 || result.GroupRoutes[2] != 1 { + t.Errorf("result = %+v, want nothing grouped and the stray sentinel counted", result) + } +} + +// TestRefreshOrderedModeReturnsErrors covers F01 at its source: a sweep whose +// iteration or writes fail returns the error, so its caller cannot mistake a +// partial conversion for a complete one. A retry once the fault clears +// converts every sentinel. +func TestRefreshOrderedModeReturnsErrors(t *testing.T) { + shard := shardSID(0x11) + configured := []net.IP{net.IP(shard.AsSlice())} + reachable(t, shard) + // The resolver stub keys on the exact SID, so the tenant copies resolve too. + prev := resolveLinkAndL2Fn + resolveLinkAndL2Fn = func(sid net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + return 1, dpShardMAC, dpUplinkMAC, nil + } + t.Cleanup(func() { resolveLinkAndL2Fn = prev }) + + setup := func() (*faultyTable, *EgressRouteTable) { + raw := newFaultyTable() + tbl := NewEgressRouteTable(raw) + for table := uint32(1); table <= 4; table++ { + if err := tbl.RegisterGroup(table, DefaultPrefix, 0, uint16(0x100+table)); err != nil { //nolint:gosec // test + t.Fatal(err) + } + } + return raw, tbl + } + sentinels := func(tbl *EgressRouteTable) int { + n := 0 + for table := uint32(1); table <= 4; table++ { + k, _ := buildKey(table, DefaultPrefix) + v, _ := lookupRaw(tbl, k) + if isGroupSentinel(v) { + n++ + } + } + return n + } + + t.Run("IterationError", func(t *testing.T) { + raw, tbl := setup() + raw.iterErr = errors.New("injected iteration failure") + if _, err := tbl.Refresh(nil, configured); err == nil { + t.Fatal("Refresh with a failing iterator = nil, want the error") + } + if n := sentinels(tbl); n != 4 { + t.Errorf("%d sentinels after a failed iteration, want all 4 left untouched", n) + } + }) + t.Run("PartialWrite", func(t *testing.T) { + raw, tbl := setup() + raw.putFailAfter = raw.puts + 2 + result, err := tbl.Refresh(nil, configured) + if !errors.Is(err, errInjectedPut) { + t.Fatalf("Refresh with a failing write = %v, want the injected error", err) + } + if n := sentinels(tbl); n != 2 || result.Refreshed != 2 { + t.Errorf("%d sentinels left and %d refreshed after the partial write, want 2 and 2", n, result.Refreshed) + } + + raw.putFailAfter = -1 + result, err = tbl.Refresh(nil, configured) + if err != nil || result.SentinelRoutes() != 0 || sentinels(tbl) != 0 { + t.Errorf("retry = %+v, %v; want every sentinel converted and none counted", result, err) + } + }) + t.Run("Success", func(t *testing.T) { + _, tbl := setup() + result, err := tbl.Refresh(nil, configured) + if err != nil || result.Ungrouped != 4 || result.SentinelRoutes() != 0 { + t.Errorf("Refresh = %+v, %v; want 4 ungrouped and none left", result, err) + } + sid, _, _ := tbl.Lookup(2, DefaultPrefix) + if !sid.Equal(tenantSID(shard, 0x102)) { + t.Errorf("table 2's route = %s, want %s: the static shard with the sentinel's Argument", + sid, tenantSID(shard, 0x102)) + } + }) +} + +// TestRefreshCountsForeignSentinels checks that a sentinel in a table another +// namespace owns is left alone but still counted, so ordered mode never +// retires a group a foreign route still names. +func TestRefreshCountsForeignSentinels(t *testing.T) { + tbl := NewEgressRouteTable(newFakeTable()) + if err := tbl.RegisterGroup(9, DefaultPrefix, 0, 0x10); err != nil { + t.Fatal(err) + } + result, err := tbl.Refresh(map[uint32]struct{}{9: {}}, nil) + if err != nil || result.SentinelRoutes() != 1 || result.Skipped != 1 { + t.Errorf("Refresh = %+v, %v; want the foreign sentinel skipped and counted", result, err) + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/sentinel.go b/internal/plumbing/ebpf/egressroutemap/sentinel.go new file mode 100644 index 00000000..1fb96c83 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/sentinel.go @@ -0,0 +1,114 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "bytes" + "encoding/binary" + "fmt" + "hash/fnv" + "net" + "net/netip" + + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" +) + +// RegisterGroup installs or replaces egress_route_table's entry for prefix in +// Linux VRF table tableID as a shard group sentinel: usid_egress picks one of +// groupID's shards per packet and writes argument, the tenant's VRFID, into +// its SID. Nothing is resolved here; the group's shards carry their own next +// hops. +func (t *EgressRouteTable) RegisterGroup(tableID uint32, prefix *net.IPNet, groupID uint32, argument uint16) error { + key, err := buildKey(tableID, prefix) + if err != nil { + return fmt.Errorf("egressroutemap: egress_route_table: register group: %w", err) + } + if groupID >= MaxShardGroups { + return fmt.Errorf("egressroutemap: egress_route_table: register group: group %d out of range", groupID) + } + if argument < uformat.ArgumentMin || argument > uformat.ArgumentMax { + return fmt.Errorf("egressroutemap: egress_route_table: register group: %w", uformat.ValidateArgument(argument)) + } + if err := t.table.Put(key, groupSentinel(groupID, argument)); err != nil { + return fmt.Errorf("egressroutemap: egress_route_table: register group table=%d prefix=%s: %w", + tableID, prefix, err) + } + return nil +} + +// groupSentinel builds the sentinel value for groupID and argument. See struct +// egress_route_value's sentinel comment in usid.c for the layout. +func groupSentinel(groupID uint32, argument uint16) prog.UsidEgressRouteValue { + value := prog.UsidEgressRouteValue{LinkIfindex: groupSentinelIfindex} + binary.NativeEndian.PutUint32(value.Sid[0:4], groupID) + value.Sid[8] = byte(argument>>8) & 0x0F + value.Sid[9] = byte(argument) + return value +} + +// isGroupSentinel reports whether value names a shard group. +func isGroupSentinel(value prog.UsidEgressRouteValue) bool { + return value.LinkIfindex == groupSentinelIfindex +} + +// sentinelGroup returns the group and tenant Argument a sentinel carries. +func sentinelGroup(value prog.UsidEgressRouteValue) (groupID uint32, argument uint16) { + return binary.NativeEndian.Uint32(value.Sid[0:4]), uint16(value.Sid[8]&0x0F)<<8 | uint16(value.Sid[9]) +} + +// ShardPinKey is the input the datapath hashes to place a packet, and the key +// it pins the result under. It mirrors struct egress_shard_pin_key. +type ShardPinKey struct { + TableID uint32 + GroupID uint32 + // Src and Dst are the packet's inner addresses. Dst, Protocol and the + // ports are read only in flow mode. + Src, Dst netip.Addr + Protocol uint8 + // SrcPort and DstPort are zero for a fragment and for any protocol but + // TCP and UDP, as the datapath leaves them. + SrcPort, DstPort uint16 +} + +// Bytes returns k laid out as struct egress_shard_pin_key for mode, with the +// fields mode does not read zeroed as the datapath zeroes them. +func (k ShardPinKey) Bytes(mode HashMode) []byte { + key := prog.UsidEgressShardPinKey{TableId: k.TableID, Group: uint8(k.GroupID)} //nolint:gosec // < MaxShardGroups + if k.Src.Is4() { + key.Family = egressRouteFamilyINET4 + v4 := k.Src.As4() + copy(key.Saddr[:4], v4[:]) + } else { + key.Family = egressRouteFamilyINET6 + key.Saddr = k.Src.As16() + } + if mode == HashFlow { + key.Protocol = k.Protocol + if k.Dst.Is4() { + v4 := k.Dst.As4() + copy(key.Daddr[:4], v4[:]) + } else if k.Dst.IsValid() { + key.Daddr = k.Dst.As16() + } + // The datapath copies the ports straight out of the header, so + // they sit in the key in network byte order. + var ports [4]byte + binary.BigEndian.PutUint16(ports[0:2], k.SrcPort) + binary.BigEndian.PutUint16(ports[2:4], k.DstPort) + key.Sport = binary.NativeEndian.Uint16(ports[0:2]) + key.Dport = binary.NativeEndian.Uint16(ports[2:4]) + } + var buf bytes.Buffer + _ = binary.Write(&buf, binary.NativeEndian, key) // a fixed-size struct cannot fail to encode + return buf.Bytes() +} + +// ShardHash is the hash usid_egress computes over key: 32-bit FNV-1a. +func ShardHash(key []byte) uint32 { + h := fnv.New32a() + _, _ = h.Write(key) + return h.Sum32() +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup.go b/internal/plumbing/ebpf/egressroutemap/shardgroup.go new file mode 100644 index 00000000..6c972901 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup.go @@ -0,0 +1,674 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "fmt" + "math" + "net" + "net/netip" + "sort" + "sync" + "time" + + "go.datum.net/galactic/internal/maglev" + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" +) + +// The shard group layout, mirroring usid.c's USID_EGRESS_* constants. bpf2go +// generates no symbol for a #define, so each value is named once here. +const ( + // NAT66ShardGroup is the group serving the ::/0 route. NAT64 prefixes + // take the groups after it, in the order they are configured. + NAT66ShardGroup uint32 = 0 + + // MaxShardGroups is how many groups egress_shard_groups holds: NAT66 and + // up to MaxNAT64Groups NAT64 prefixes. + MaxShardGroups = 4 + + // MaxNAT64Groups is how many NAT64 prefixes hashed mode can serve. + MaxNAT64Groups = MaxShardGroups - 1 + + // MaxShardsPerGroup is how many shards one group holds. + MaxShardsPerGroup = 32 + + // ShardMaglevTableSize is the number of slots in a group's Maglev table. + ShardMaglevTableSize = 1021 + + // groupSentinelIfindex marks an egress_route_table entry that names a + // shard group rather than one shard. + groupSentinelIfindex uint32 = math.MaxUint32 + + // shardSlotNone marks a Maglev slot with no shard. + shardSlotNone uint8 = 0xFF + + groupFlagEnabled uint32 = 1 << 0 + groupFlagHashFlow uint32 = 1 << 1 + groupFlagClosing uint32 = 1 << 2 + + shardFlagAlive uint32 = 1 << 0 + shardFlagDraining uint32 = 1 << 1 + + classKindNAT66 uint8 = 1 + classKindNAT64 uint8 = 2 +) + +// DefaultPinIdle is how long a tenant address stays pinned to its shard after +// its last packet: the egress shard's longest idle timeout, the 2 h 4 min +// RFC 5382 asks for an established TCP session. A pin then never outlives the +// shard session it protects. +const DefaultPinIdle = 2*time.Hour + 4*time.Minute + +// WellKnownPrefix is the RFC 6052 NAT64 Well-Known Prefix, which a shard +// translates when its status says translatesWellKnownPrefix. +var WellKnownPrefix = netip.MustParsePrefix("64:ff9b::/96") + +// HashMode selects what a shard group hashes to place a packet. +type HashMode uint8 + +const ( + // HashSource hashes the tenant's source address, so every connection + // from one address uses one shard and one public address (RFC 4787 + // REQ-2, paired pooling). The default. + HashSource HashMode = iota + + // HashFlow hashes the 5-tuple, spreading one heavy tenant address over + // every shard. It gives up paired pooling: one address appears from + // several public addresses. + HashFlow +) + +// String returns the mode's configuration spelling. +func (m HashMode) String() string { + if m == HashFlow { + return "flow" + } + return "source" +} + +// TranslationClass is what a group translates: NAT66, for the ::/0 route, or +// NAT64 toward one prefix, for that prefix's route. A group's members are only +// the shards that can translate its class. +type TranslationClass struct { + // NAT64 is false for NAT66. + NAT64 bool + // Prefix is the NAT64 /96. Unset for NAT66. + Prefix netip.Prefix +} + +// NAT66Class is the class of the ::/0 route. +var NAT66Class = TranslationClass{} + +// NAT64Class is the class of the route toward prefix. +func NAT64Class(prefix netip.Prefix) TranslationClass { + return TranslationClass{NAT64: true, Prefix: prefix.Masked()} +} + +// String returns the class as a metric label: nat66, or nat64:. +func (c TranslationClass) String() string { + if !c.NAT64 { + return "nat66" + } + return "nat64:" + c.Prefix.String() +} + +func (c TranslationClass) encode(value *prog.UsidEgressShardGroupValue) { + value.ClassKind = classKindNAT66 + value.ClassPrefix = [16]byte{} + if c.NAT64 { + value.ClassKind = classKindNAT64 + value.ClassPrefix = c.Prefix.Addr().As16() + } +} + +func decodeClass(value prog.UsidEgressShardGroupValue) (TranslationClass, bool) { + switch value.ClassKind { + case classKindNAT66: + return NAT66Class, true + case classKindNAT64: + return NAT64Class(netip.PrefixFrom(netip.AddrFrom16(value.ClassPrefix), 96)), true + } + return TranslationClass{}, false +} + +// ClassForRoute returns the class an egress route toward prefix needs: NAT66 +// for the IPv6 default, NAT64 for any other IPv6 prefix. +func ClassForRoute(prefix *net.IPNet) (TranslationClass, bool) { + addr, ok := netip.AddrFromSlice(prefix.IP) + if !ok { + return TranslationClass{}, false + } + ones, bits := prefix.Mask.Size() + if bits != 128 || addr.Unmap().Is4() { + return TranslationClass{}, false + } + if ones == 0 { + return NAT66Class, true + } + return NAT64Class(netip.PrefixFrom(addr, ones)), true +} + +// GroupPolicy is how a shard group places tenants and what it translates. +type GroupPolicy struct { + Class TranslationClass + Hash HashMode + // PinIdle is how long a pin outlives its last packet. Zero turns pins + // off, leaving placement to the Maglev table alone. + PinIdle time.Duration + // Closing marks the group as being retired; see ShardGroupTable.Close. + Closing bool + // Ineligible is how many of the cluster's shards cannot translate Class. + // Recorded for the metrics only. + Ineligible int +} + +// ShardCandidate is one shard a group may use. SID's Argument is ignored: the +// group stores each shard's base SID and the datapath writes each tenant's +// Argument into it per packet. +type ShardCandidate struct { + SID net.IP + // Draining keeps the shard out of the Maglev table, so it takes no new + // tenant address, while its pins keep working. + Draining bool +} + +// ShardState reports one group member. +type ShardState struct { + // SID is the shard's base SID. + SID netip.Addr + // Slot is the shard's index within its group. + Slot int + // Generation identifies the shard's tenure of its slot. + Generation uint32 + // Alive reports that the SID resolved to an uplink neighbor. + Alive bool + // Draining echoes the candidate's Draining. + Draining bool + // Err is why the SID did not resolve, when it did not. + Err error + // Assigned reports that this Apply gave the shard its slot, so nothing + // counted in the slot before belongs to it. + Assigned bool +} + +// Active reports whether the shard takes new tenant addresses. +func (s ShardState) Active() bool { return s.Alive && !s.Draining } + +// GroupResult reports what Apply found and published. +type GroupResult struct { + // Changed reports that Apply published a new snapshot. + Changed bool + // Generation is the group's publication count after Apply. + Generation uint32 + // Shards is every candidate, ordered by base SID. + Shards []ShardState +} + +// Active counts the shards taking new tenant addresses. +func (r GroupResult) Active() int { + n := 0 + for _, s := range r.Shards { + if s.Active() { + n++ + } + } + return n +} + +// GroupStatus is a group's published state, as CNI ADD and the metrics read +// it. +type GroupStatus struct { + // Present reports that a snapshot has ever been published for the group. + Present bool + // Enabled reports that the group carries traffic. + Enabled bool + // Closing reports that the cluster is moving back to ordered mode: no + // new route should name the group. + Closing bool + // Class is what the group translates. + Class TranslationClass + // Candidates counts the shards eligible for Class. + Candidates int + // Active counts the shards taking new tenant addresses. + Active int + // Ineligible counts the cluster's shards that cannot translate Class. + Ineligible int + // Generation is the publication count. + Generation uint32 +} + +// Open reports whether a new route may name the group. +func (s GroupStatus) Open() bool { return s.Enabled && !s.Closing } + +// GroupStore reads and publishes whole group snapshots. Publish must make the +// new snapshot visible to the datapath in one step, or not at all. +type GroupStore interface { + // Load returns groupID's published snapshot, and false when none has + // been published. + Load(groupID uint32) (prog.UsidEgressShardGroupValue, bool, error) + // Publish replaces groupID's snapshot with value. + Publish(groupID uint32, value prog.UsidEgressShardGroupValue) error +} + +// ShardGroupTable builds and publishes the egress shard groups. galactic-cni's +// installer is the only writer; CNI ADD and the metrics only read. +// +// Slot generations. Every member slot carries a generation identifying the +// shard's tenure of it, and a pin is honoured only while the slot still +// carries the generation it recorded. A generation is allocated from the +// group's next_slot_generation, which is published in the snapshot, and from +// highWater, which also counts generations allocated for snapshots this +// process built but failed to publish. Neither ever goes down: disabling a +// group keeps next_slot_generation, and a restarted process starts from the +// published value. A generation the published value does not cover was only +// ever in a snapshot that was never published, so no packet saw it and no pin +// can hold it. The counter is 32 bits and skips 0 on wrap, which takes four +// billion slot assignments; a pin lives at most PinIdle, so a wrapped value +// cannot meet a pin still holding it. +type ShardGroupTable struct { + store GroupStore + + mu sync.Mutex + highWater [MaxShardGroups]uint32 +} + +// NewShardGroupTable wraps store. Production callers use +// OpenPinnedShardGroupTable; tests pass a fake. +func NewShardGroupTable(store GroupStore) *ShardGroupTable { + return &ShardGroupTable{store: store} +} + +// BaseShardSID returns sid with its Argument and Slot cleared: the identity a +// group stores a shard under. An address that is not a well-formed uSID is an +// error. +func BaseShardSID(sid net.IP) (netip.Addr, error) { + fields, err := decodeSID(sid) + if err != nil { + return netip.Addr{}, fmt.Errorf("egress shard SID %s: %w", sid, err) + } + fields.Argument, fields.Slot = 0, 0 + base, err := uformat.Encode(fields) + if err != nil { + return netip.Addr{}, fmt.Errorf("egress shard SID %s: %w", sid, err) + } + return base, nil +} + +func checkGroup(groupID uint32) error { + if groupID >= MaxShardGroups { + return fmt.Errorf("egressroutemap: shard group %d out of range", groupID) + } + return nil +} + +// load reads groupID's snapshot, an absent one reading as the zero value. +func (g *ShardGroupTable) load(groupID uint32) (prog.UsidEgressShardGroupValue, bool, error) { + if err := checkGroup(groupID); err != nil { + return prog.UsidEgressShardGroupValue{}, false, err + } + value, ok, err := g.store.Load(groupID) + if err != nil { + return prog.UsidEgressShardGroupValue{}, false, fmt.Errorf("egressroutemap: read shard group %d: %w", groupID, err) + } + return value, ok, nil +} + +// Status reads groupID's published state. +func (g *ShardGroupTable) Status(groupID uint32) (GroupStatus, error) { + value, ok, err := g.load(groupID) + if err != nil || !ok { + return GroupStatus{}, err + } + class, _ := decodeClass(value) + return GroupStatus{ + Present: true, + Enabled: value.Flags&groupFlagEnabled != 0, + Closing: value.Flags&groupFlagClosing != 0, + Class: class, + Candidates: int(value.Candidates), + Active: int(value.Active), + Ineligible: int(value.Ineligible), + Generation: value.Generation, + }, nil +} + +// FindClass returns the group whose published class is class. +func (g *ShardGroupTable) FindClass(class TranslationClass) (uint32, GroupStatus, bool, error) { + for id := range uint32(MaxShardGroups) { + status, err := g.Status(id) + if err != nil { + return 0, GroupStatus{}, false, err + } + if status.Present && status.Class == class { + return id, status, true, nil + } + } + return 0, GroupStatus{}, false, nil +} + +// AnyOpen reports whether any group may take new routes: hashed mode is in +// force on this node. +func (g *ShardGroupTable) AnyOpen() (bool, error) { + for id := range uint32(MaxShardGroups) { + status, err := g.Status(id) + if err != nil { + return false, err + } + if status.Open() { + return true, nil + } + } + return false, nil +} + +// Members returns groupID's occupied slots, ordered by slot. +func (g *ShardGroupTable) Members(groupID uint32) ([]ShardState, error) { + value, _, err := g.load(groupID) + if err != nil { + return nil, err + } + var members []ShardState + for i, s := range value.Shards { + if !slotOccupied(s) { + continue + } + members = append(members, ShardState{ + SID: netip.AddrFrom16(s.Sid), + Slot: i, + Generation: s.Generation, + Alive: s.Flags&shardFlagAlive != 0, + Draining: s.Flags&shardFlagDraining != 0, + }) + } + return members, nil +} + +func slotOccupied(s prog.UsidEgressShardMember) bool { + return s.Sid != [16]byte{} +} + +// shardBackend is one active shard as a Maglev backend. +type shardBackend struct { + sid netip.Addr + slot int +} + +func (b shardBackend) Key() string { return b.sid.String() } + +// allocator hands out slot generations for one build: from the larger of the +// published counter and this process's high-water mark, never 0. +type allocator struct{ next uint32 } + +func (a *allocator) take() uint32 { + if a.next == 0 { + a.next = 1 + } + g := a.next + a.next++ + return g +} + +// Apply makes groupID serve exactly candidates under policy and publishes the +// result as one snapshot. +// +// Every candidate's SID is resolved to an uplink next hop here, as Register +// does for a single shard, and only a shard that resolves is alive. The Maglev +// table spreads tenant addresses over the alive shards that are not draining. +// Every node computes the same table from the same set, whatever slots the +// shards occupy locally. +// +// A shard keeps its slot, and the slot keeps its generation, for as long as +// the shard stays a candidate of the same class, so a rebuild leaves its pins +// intact. A new shard takes the lowest free slot under a fresh generation. +// Nothing is published when nothing changed. +func (g *ShardGroupTable) Apply(groupID uint32, policy GroupPolicy, candidates []ShardCandidate) (GroupResult, error) { + wanted, err := canonicalCandidates(candidates) + if err != nil { + return GroupResult{}, err + } + if len(wanted) > MaxShardsPerGroup { + return GroupResult{}, fmt.Errorf("egressroutemap: %d egress shards exceed the group limit of %d", + len(wanted), MaxShardsPerGroup) + } + + g.mu.Lock() + defer g.mu.Unlock() + + current, _, err := g.load(groupID) + if err != nil { + return GroupResult{}, err + } + previous := current.Shards + if class, ok := decodeClass(current); ok && class.String() != policy.Class.String() { + // A group that changes class starts afresh: no member keeps its + // slot, so no pin made for the old class survives. + previous = [MaxShardsPerGroup]prog.UsidEgressShardMember{} + } + alloc := allocator{next: max(current.NextSlotGeneration, g.highWater[groupID])} + + assignment := assignSlots(previous, wanted) + resolved := resolveShards(wanted) + + var ( + value prog.UsidEgressShardGroupValue + result = GroupResult{Generation: current.Generation} + backends []maglev.Backend + ) + for _, c := range wanted { + slot := assignment[c.sid] + hop := resolved[c.sid] + member := prog.UsidEgressShardMember{Sid: c.sid.As16(), Generation: previous[slot].Generation} + assigned := !slotOccupied(previous[slot]) || netip.AddrFrom16(previous[slot].Sid) != c.sid || + member.Generation == 0 + if assigned { + member.Generation = alloc.take() + } + if hop.err == nil { + member.Flags |= shardFlagAlive + member.LinkIfindex = uint32(hop.hop.link) //nolint:gosec // a kernel ifindex + copy(member.Dmac[:], hop.hop.dmac) + copy(member.Smac[:], hop.hop.smac) + } + if c.draining { + member.Flags |= shardFlagDraining + } + value.Shards[slot] = member + + state := ShardState{ + SID: c.sid, Slot: slot, Generation: member.Generation, Alive: hop.err == nil, + Draining: c.draining, Err: hop.err, Assigned: assigned, + } + result.Shards = append(result.Shards, state) + if state.Active() { + backends = append(backends, shardBackend{sid: c.sid, slot: slot}) + } + } + // The high-water mark moves before publishing, so a failed publication + // never hands the same generation out twice. + g.highWater[groupID] = alloc.next + + if err := fillGroupValue(&value, policy, len(wanted), backends); err != nil { + return GroupResult{}, err + } + value.NextSlotGeneration = alloc.next + value.Generation = current.Generation + if value == current { + return result, nil + } + value.Generation = current.Generation + 1 + if err := g.store.Publish(groupID, value); err != nil { + return result, fmt.Errorf("egressroutemap: publish shard group %d: %w", groupID, err) + } + result.Generation = value.Generation + result.Changed = true + return result, nil +} + +// Close marks groupID as being retired, for the move back to ordered mode. The +// datapath keeps forwarding through it, with its members' next hops +// re-resolved, so a route still naming it works; CNI ADD sees it closing and +// writes no new route naming it. A group that is not enabled is left alone. +// It reports whether the group is closing afterwards. +func (g *ShardGroupTable) Close(groupID uint32) (bool, error) { + value, ok, err := g.load(groupID) + if err != nil || !ok || value.Flags&groupFlagEnabled == 0 { + return false, err + } + class, _ := decodeClass(value) + policy := GroupPolicy{ + Class: class, + PinIdle: time.Duration(value.PinIdleSec) * time.Second, + Closing: true, + Ineligible: int(value.Ineligible), + } + if value.Flags&groupFlagHashFlow != 0 { + policy.Hash = HashFlow + } + var candidates []ShardCandidate + for _, s := range value.Shards { + if slotOccupied(s) { + candidates = append(candidates, ShardCandidate{ + SID: net.IP(netip.AddrFrom16(s.Sid).AsSlice()), Draining: s.Flags&shardFlagDraining != 0, + }) + } + } + if _, err := g.Apply(groupID, policy, candidates); err != nil { + return false, err + } + return true, nil +} + +// Disable publishes groupID with no members and traffic off, which is what +// ordered mode leaves behind. A route still naming it drops until Refresh +// moves it back onto a single shard, so callers disable a group only once no +// route names it. The class and the generation counter are kept. It reports +// whether anything changed. +func (g *ShardGroupTable) Disable(groupID uint32) (bool, error) { + g.mu.Lock() + defer g.mu.Unlock() + + current, ok, err := g.load(groupID) + if err != nil || !ok { + return false, err + } + disabled := prog.UsidEgressShardGroupValue{ + Generation: current.Generation, + NextSlotGeneration: max(current.NextSlotGeneration, g.highWater[groupID]), + ClassKind: current.ClassKind, + ClassPrefix: current.ClassPrefix, + } + for i := range disabled.Maglev { + disabled.Maglev[i] = shardSlotNone + } + if disabled == current { + return false, nil + } + disabled.Generation = current.Generation + 1 + if err := g.store.Publish(groupID, disabled); err != nil { + return false, fmt.Errorf("egressroutemap: disable shard group %d: %w", groupID, err) + } + return true, nil +} + +// candidate is a ShardCandidate reduced to its base SID. +type candidate struct { + sid netip.Addr + draining bool +} + +// canonicalCandidates reduces candidates to base SIDs, ordered by SID. A +// shard listed twice is draining if either listing says so. +func canonicalCandidates(candidates []ShardCandidate) ([]candidate, error) { + bySID := map[netip.Addr]bool{} + for _, c := range candidates { + base, err := BaseShardSID(c.SID) + if err != nil { + return nil, fmt.Errorf("egressroutemap: %w", err) + } + bySID[base] = bySID[base] || c.Draining + } + out := make([]candidate, 0, len(bySID)) + for sid, draining := range bySID { + out = append(out, candidate{sid: sid, draining: draining}) + } + sort.Slice(out, func(i, j int) bool { return out[i].sid.Less(out[j].sid) }) + return out, nil +} + +// assignSlots keeps every wanted shard already in a slot there, and gives each +// new one the lowest slot no wanted shard holds. +func assignSlots(slots [MaxShardsPerGroup]prog.UsidEgressShardMember, wanted []candidate) map[netip.Addr]int { + assignment := make(map[netip.Addr]int, len(wanted)) + taken := map[int]bool{} + isWanted := make(map[netip.Addr]bool, len(wanted)) + for _, c := range wanted { + isWanted[c.sid] = true + } + for i, s := range slots { + sid := netip.AddrFrom16(s.Sid) + if slotOccupied(s) && isWanted[sid] { + if _, dup := assignment[sid]; !dup { + assignment[sid] = i + taken[i] = true + } + } + } + next := 0 + for _, c := range wanted { + if _, ok := assignment[c.sid]; ok { + continue + } + for taken[next] { + next++ + } + assignment[c.sid] = next + taken[next] = true + } + return assignment +} + +// resolveShards resolves every wanted shard's next hop. +func resolveShards(wanted []candidate) map[netip.Addr]resolution { + out := make(map[netip.Addr]resolution, len(wanted)) + for _, c := range wanted { + link, dmac, smac, err := resolveLinkAndL2Fn(net.IP(c.sid.AsSlice())) + out[c.sid] = resolution{hop: nextHop{link: link, dmac: dmac, smac: smac}, err: err} + } + return out +} + +// fillGroupValue fills an enabled group's header and Maglev table over +// backends. With no backend every slot is shardSlotNone. +func fillGroupValue( + value *prog.UsidEgressShardGroupValue, policy GroupPolicy, candidates int, backends []maglev.Backend, +) error { + value.Flags = groupFlagEnabled + if policy.Hash == HashFlow { + value.Flags |= groupFlagHashFlow + } + if policy.Closing { + value.Flags |= groupFlagClosing + } + policy.Class.encode(value) + value.Candidates = uint16(candidates) //nolint:gosec // bounded by MaxShardsPerGroup + value.Active = uint16(len(backends)) //nolint:gosec // bounded by MaxShardsPerGroup + value.Ineligible = uint16(min(policy.Ineligible, math.MaxUint16)) //nolint:gosec // clamped + value.PinIdleSec = uint32(policy.PinIdle / time.Second) //nolint:gosec // a configured duration + for i := range value.Maglev { + value.Maglev[i] = shardSlotNone + } + if len(backends) == 0 { + return nil + } + table, err := maglev.New(backends, ShardMaglevTableSize) + if err != nil { + return fmt.Errorf("egressroutemap: build shard Maglev table: %w", err) + } + for i := range value.Maglev { + value.Maglev[i] = uint8(table.Lookup(uint64(i)).(shardBackend).slot) //nolint:gosec // slot < MaxShardsPerGroup + } + return nil +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go new file mode 100644 index 00000000..00fc04eb --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go @@ -0,0 +1,522 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "encoding/binary" + "errors" + "net" + "net/netip" + "testing" + "time" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/rlimit" + + "go.datum.net/galactic/internal/maglev" + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" +) + +// These tests run usid_egress under BPF_PROG_RUN against maps written by the +// Go group API, so the datapath's hash, Maglev lookup, pin handling and SID +// completion are checked against what the control plane wrote rather than +// against hand-built map values. + +const ( + tcActShot = 2 + tcActRedirect = 7 + + dpTableID = 42 + dpBlock = 0x0102030405AA + dpArgument = 0x2A5 + ethLen = 14 + ip6Len = 40 +) + +var ( + dpShardMAC = net.HardwareAddr{0xAA, 0xBB, 0xCC, 0x00, 0x00, 0x01} + dpUplinkMAC = net.HardwareAddr{0xAA, 0xBB, 0xCC, 0x00, 0x00, 0x02} +) + +// datapath is one loaded usid program with an attachment on ifindex 1, the +// skb->ifindex BPF_PROG_RUN uses, whose VRF's ::/0 names the cluster group. +type datapath struct { + objs *prog.UsidObjects + store *KernelGroupStore + groups *ShardGroupTable + routes *EgressRouteTable +} + +func loadDatapath(t *testing.T) *datapath { + t.Helper() + requireRoot(t) + if err := rlimit.RemoveMemlock(); err != nil { + t.Fatalf("rlimit.RemoveMemlock: %v", err) + } + + var objs prog.UsidObjects + if err := prog.LoadUsidObjects(&objs, nil); err != nil { + var ve *ebpf.VerifierError + if errors.As(err, &ve) { + t.Fatalf("load objects: verifier rejected program:\n%+v", ve) + } + t.Fatalf("load objects: %v", err) + } + t.Cleanup(func() { _ = objs.Close() }) + + attachment := prog.UsidIfindexVrfValue{Block: dpBlock, Argument: dpArgument} + if err := objs.IfindexVrfTable.Put(uint32(1), attachment); err != nil { + t.Fatalf("populate ifindex_vrf_table: %v", err) + } + vrfKey, err := uformat.NewVRFKey(dpBlock, dpArgument) + if err != nil { + t.Fatalf("NewVRFKey: %v", err) + } + if err := objs.VrfTable.Put(uint64(vrfKey), prog.UsidVrfValue{VrfTableId: dpTableID}); err != nil { + t.Fatalf("populate vrf_table: %v", err) + } + nodeSrc := netip.MustParseAddr("2001:db8:ff01:1:e000::").As16() + if err := objs.NodeSrcAddrTable.Put(uint32(0), nodeSrc); err != nil { + t.Fatalf("populate node_src_addr_table: %v", err) + } + + dp := &datapath{ + objs: &objs, + store: NewKernelGroupStore(objs.EgressShardGroups), + routes: NewEgressRouteTable(usidmap.KernelTable{Map: objs.EgressRouteTable}), + } + dp.groups = NewShardGroupTable(dp.store) + if err := dp.routes.RegisterGroup(dpTableID, DefaultPrefix, NAT66ShardGroup, dpArgument); err != nil { + t.Fatalf("RegisterGroup(::/0): %v", err) + } + _, v4Default, _ := net.ParseCIDR("0.0.0.0/0") + if err := dp.routes.RegisterGroup(dpTableID, v4Default, NAT66ShardGroup, dpArgument); err != nil { + t.Fatalf("RegisterGroup(0.0.0.0/0): %v", err) + } + return dp +} + +// reachable stubs the resolver so every SID in alive resolves out ifindex 1 +// and every other SID fails. +func reachable(t *testing.T, alive ...netip.Addr) { + t.Helper() + ok := map[netip.Addr]bool{} + for _, a := range alive { + ok[a] = true + } + prev := resolveLinkAndL2Fn + resolveLinkAndL2Fn = func(sid net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + addr, _ := netip.AddrFromSlice(sid) + if !ok[addr.Unmap()] { + return 0, nil, nil, errors.New("unreachable in this test") + } + return 1, dpShardMAC, dpUplinkMAC, nil + } + t.Cleanup(func() { resolveLinkAndL2Fn = prev }) +} + +func shardSID(nodeID uint16) netip.Addr { + addr, err := uformat.Encode(uformat.Fields{Block: 0x20010db8ff09, NodeID: nodeID, Function: uformat.FunctionEndDT46}) + if err != nil { + panic(err) + } + return addr +} + +func candidates(draining map[netip.Addr]bool, sids ...netip.Addr) []ShardCandidate { + out := make([]ShardCandidate, 0, len(sids)) + for _, s := range sids { + out = append(out, ShardCandidate{SID: net.IP(s.AsSlice()), Draining: draining[s]}) + } + return out +} + +func (dp *datapath) apply(t *testing.T, policy GroupPolicy, c []ShardCandidate) GroupResult { + t.Helper() + result, err := dp.groups.Apply(NAT66ShardGroup, policy, c) + if err != nil { + t.Fatalf("Apply: %v", err) + } + return result +} + +// udp6 builds Ethernet + IPv6 + UDP from src:sport to dst:dport. +func udp6(src, dst netip.Addr, sport, dport uint16) []byte { + pkt := make([]byte, 0, ethLen+ip6Len+8) + pkt = append(pkt, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB, 0x86, 0xDD) + pkt = append(pkt, 0x60, 0, 0, 0, 0, 8, 17, 64) + s, d := src.As16(), dst.As16() + pkt = append(pkt, s[:]...) + pkt = append(pkt, d[:]...) + pkt = binary.BigEndian.AppendUint16(pkt, sport) + pkt = binary.BigEndian.AppendUint16(pkt, dport) + pkt = append(pkt, 0, 8, 0, 0) + return pkt +} + +// udp4 builds Ethernet + IPv4 + UDP, with fragOff as the raw frag_off field. +func udp4(src, dst netip.Addr, sport, dport, fragOff uint16) []byte { + pkt := make([]byte, 0, ethLen+20+8) + pkt = append(pkt, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB, 0x08, 0x00) + pkt = append(pkt, 0x45, 0, 0, 28, 0, 0) + pkt = binary.BigEndian.AppendUint16(pkt, fragOff) + pkt = append(pkt, 64, 17, 0, 0) + s, d := src.As4(), dst.As4() + pkt = append(pkt, s[:]...) + pkt = append(pkt, d[:]...) + pkt = binary.BigEndian.AppendUint16(pkt, sport) + pkt = binary.BigEndian.AppendUint16(pkt, dport) + pkt = append(pkt, 0, 8, 0, 0) + return pkt +} + +// send runs pkt through usid_egress and returns the verdict and, for a +// redirect, the outer destination with its Argument cleared, plus the +// Argument it carried. +func (dp *datapath) send(t *testing.T, pkt []byte) (verdict uint32, shard netip.Addr, argument uint16) { + t.Helper() + ret, out, err := dp.objs.UsidEgress.Test(pkt) + if err != nil { + t.Fatalf("program test-run: %v", err) + } + if ret != tcActRedirect { + return ret, netip.Addr{}, 0 + } + if len(out) != len(pkt)+ip6Len { + t.Fatalf("output length %d, want %d", len(out), len(pkt)+ip6Len) + } + if got := net.HardwareAddr(out[0:6]); got.String() != dpShardMAC.String() { + t.Errorf("outer h_dest = %s, want the shard's resolved %s", got, dpShardMAC) + } + dst, _ := netip.AddrFromSlice(out[ethLen+24 : ethLen+40]) + fields, err := uformat.Decode(dst) + if err != nil { + t.Fatalf("outer destination %s is not a uSID: %v", dst, err) + } + argument = fields.Argument + fields.Argument = 0 + base, _ := uformat.Encode(fields) + return ret, base, argument +} + +// predict is the shard the Go side expects the datapath to choose for key +// with no pin: the Maglev table built over active, indexed by ShardHash. +func predict(t *testing.T, key ShardPinKey, mode HashMode, active ...netip.Addr) netip.Addr { + t.Helper() + backends := make([]maglev.Backend, 0, len(active)) + for i, a := range active { + backends = append(backends, shardBackend{sid: a, slot: i}) + } + table, err := maglev.New(backends, ShardMaglevTableSize) + if err != nil { + t.Fatalf("maglev.New: %v", err) + } + return table.Lookup(uint64(ShardHash(key.Bytes(mode)))).(shardBackend).sid +} + +func tenant(i int) netip.Addr { + a := netip.MustParseAddr("2001:db8:aaaa::").As16() + binary.BigEndian.PutUint16(a[14:16], uint16(i)) //nolint:gosec // test index + return netip.AddrFrom16(a) +} + +var internet = netip.MustParseAddr("2001:db8:ffff::1") + +func stat(t *testing.T, m *ebpf.Map, index uint32) uint64 { + t.Helper() + var perCPU []uint64 + if err := m.Lookup(index, &perCPU); err != nil { + t.Fatalf("lookup stat %d: %v", index, err) + } + var total uint64 + for _, v := range perCPU { + total += v + } + return total +} + +// TestGroupDatapath_SourceHashMatchesGoAndSpreadsEvenly checks that every +// tenant address goes where the Go side predicts, that the tenant's Argument +// is written into the chosen shard's SID, and that three shards each carry +// close to a third of 600 addresses. +func TestGroupDatapath_SourceHashMatchesGoAndSpreadsEvenly(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + dp.apply(t, GroupPolicy{}, candidates(nil, a, b, c)) + + share := map[netip.Addr]int{} + for i := range 600 { + src := tenant(i) + verdict, shard, argument := dp.send(t, udp6(src, internet, uint16(1000+i), 443)) //nolint:gosec // test index + if verdict != tcActRedirect { + t.Fatalf("tenant %s: verdict %d, want redirect", src, verdict) + } + if argument != dpArgument { + t.Fatalf("tenant %s: outer SID Argument %#x, want the VRF's %#x", src, argument, dpArgument) + } + want := predict(t, ShardPinKey{TableID: dpTableID, Src: src}, HashSource, a, b, c) + if shard != want { + t.Fatalf("tenant %s: datapath chose %s, Go predicts %s", src, shard, want) + } + share[shard]++ + } + for _, s := range []netip.Addr{a, b, c} { + if share[s] < 150 || share[s] > 250 { + t.Errorf("shard %s carried %d of 600 tenant addresses, want close to 200 (shares %v)", s, share[s], share) + } + } + + // The same address keeps its shard whatever its ports: paired pooling. + for port := range uint16(20) { + _, shard, _ := dp.send(t, udp6(tenant(7), internet, 2000+port, 80+port)) + if want := predict(t, ShardPinKey{TableID: dpTableID, Src: tenant(7)}, HashSource, a, b, c); shard != want { + t.Fatalf("tenant %s port %d moved to %s, want %s for every flow", tenant(7), 2000+port, shard, want) + } + } + + var counters []prog.UsidEgressShardCounter + var counted uint64 + for slot := range 3 { + if err := dp.objs.EgressShardCounters.Lookup(uint32(slot), &counters); err != nil { //nolint:gosec // < 3 + t.Fatalf("lookup egress_shard_counters[%d]: %v", slot, err) + } + for _, v := range counters { + counted += v.Packets + } + } + if counted != 620 { + t.Errorf("egress_shard_counters total %d packets, want 620", counted) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatMaglev); got != 620 { + t.Errorf("egress_shard_stats[maglev] = %d, want 620", got) + } +} + +// TestGroupDatapath_FailureMovesOnlyTheFailedShardsTenants checks Maglev's +// stability through the whole stack: when one of three shards stops +// resolving, only the addresses it carried move. +func TestGroupDatapath_FailureMovesOnlyTheFailedShardsTenants(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + dp.apply(t, GroupPolicy{}, candidates(nil, a, b, c)) + + before := map[int]netip.Addr{} + for i := range 300 { + _, before[i], _ = dp.send(t, udp6(tenant(i), internet, 5000, 443)) + } + + reachable(t, a, c) // b's route is withdrawn + result := dp.apply(t, GroupPolicy{}, candidates(nil, a, b, c)) + if result.Active() != 2 { + t.Fatalf("Apply active = %d, want 2", result.Active()) + } + // Maglev bounds disruption rather than eliminating it: a few slots of + // the surviving shards are reassigned as the table refills. Allow 5%. + var survivors, moved int + for i := range 300 { + verdict, after, _ := dp.send(t, udp6(tenant(i), internet, 5000, 443)) + if verdict != tcActRedirect { + t.Fatalf("tenant %d: verdict %d after b failed, want redirect", i, verdict) + } + if after == b { + t.Fatalf("tenant %d still sent to unreachable shard %s", i, b) + } + if before[i] != b { + survivors++ + if after != before[i] { + moved++ + } + } + } + if moved*20 > survivors { + t.Errorf("%d of %d tenants on surviving shards moved, want at most 5%%", moved, survivors) + } +} + +// TestGroupDatapath_PinsHoldThroughAddAndDrain checks Phase 3: a tenant +// address pinned to a shard stays there when a new shard joins and when its +// shard drains, and moves only once its shard becomes unreachable. +func TestGroupDatapath_PinsHoldThroughAddAndDrain(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + policy := GroupPolicy{PinIdle: time.Hour} + dp.apply(t, policy, candidates(nil, a, b)) + + // Find a tenant that Maglev would move to c once c joins. + var mover netip.Addr + var pinnedTo netip.Addr + for i := range 1000 { + key := ShardPinKey{TableID: dpTableID, Src: tenant(i)} + if predict(t, key, HashSource, a, b, c) == c { + mover, pinnedTo = tenant(i), predict(t, key, HashSource, a, b) + break + } + } + if !mover.IsValid() { + t.Fatal("no tenant address in 1000 hashes to the new shard") + } + if _, got, _ := dp.send(t, udp6(mover, internet, 1, 1)); got != pinnedTo { + t.Fatalf("first packet went to %s, want %s", got, pinnedTo) + } + + dp.apply(t, policy, candidates(nil, a, b, c)) + if _, got, _ := dp.send(t, udp6(mover, internet, 2, 2)); got != pinnedTo { + t.Errorf("after c joined, pinned tenant went to %s, want its pinned %s", got, pinnedTo) + } + // A new tenant address that hashes to c does go there. + if _, got, _ := dp.send(t, udp6(mover.Next(), internet, 1, 1)); got != predict(t, + ShardPinKey{TableID: dpTableID, Src: mover.Next()}, HashSource, a, b, c) { + t.Errorf("an unpinned tenant was not placed by the new table") + } + + dp.apply(t, policy, candidates(map[netip.Addr]bool{pinnedTo: true}, a, b, c)) + if _, got, _ := dp.send(t, udp6(mover, internet, 3, 3)); got != pinnedTo { + t.Errorf("after %s began draining, pinned tenant went to %s, want it kept there", pinnedTo, got) + } + // Nothing new is placed on the draining shard. + for i := 1000; i < 1300; i++ { + if _, got, _ := dp.send(t, udp6(tenant(i), internet, 1, 1)); got == pinnedTo { + t.Fatalf("new tenant %s placed on draining shard %s", tenant(i), pinnedTo) + } + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinHit); got != 2 { + t.Errorf("egress_shard_stats[pin_hit] = %d, want 2", got) + } + + var others []netip.Addr + for _, s := range []netip.Addr{a, b, c} { + if s != pinnedTo { + others = append(others, s) + } + } + reachable(t, others...) + dp.apply(t, policy, candidates(map[netip.Addr]bool{pinnedTo: true}, a, b, c)) + if _, got, _ := dp.send(t, udp6(mover, internet, 4, 4)); got == pinnedTo || !got.IsValid() { + t.Errorf("after %s became unreachable, pinned tenant went to %s, want another shard", pinnedTo, got) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinStale); got != 1 { + t.Errorf("egress_shard_stats[pin_stale] = %d, want 1", got) + } +} + +// TestGroupDatapath_SlotReuseDoesNotInheritPins checks that a slot handed to +// a different shard does not honour the previous occupant's pins. +func TestGroupDatapath_SlotReuseDoesNotInheritPins(t *testing.T) { + dp := loadDatapath(t) + a, b := shardSID(0x10), shardSID(0x11) + reachable(t, a, b) + policy := GroupPolicy{PinIdle: time.Hour} + dp.apply(t, policy, candidates(nil, a)) + if _, got, _ := dp.send(t, udp6(tenant(1), internet, 1, 1)); got != a { + t.Fatalf("only shard is %s, packet went to %s", a, got) + } + + // a leaves and b takes slot 0. + result := dp.apply(t, policy, candidates(nil, b)) + if result.Shards[0].Slot != 0 || !result.Shards[0].Assigned { + t.Fatalf("b got slot %d (assigned %v), want slot 0 newly assigned", result.Shards[0].Slot, result.Shards[0].Assigned) + } + if _, got, _ := dp.send(t, udp6(tenant(1), internet, 1, 1)); got != b { + t.Errorf("packet went to %s, want %s", got, b) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinStale); got != 1 { + t.Errorf("egress_shard_stats[pin_stale] = %d, want 1: the old pin must be rejected by generation", got) + } +} + +// TestGroupDatapath_EmptyOrDisabledGroupDrops checks that a sentinel never +// fails open. +func TestGroupDatapath_EmptyOrDisabledGroupDrops(t *testing.T) { + dp := loadDatapath(t) + a := shardSID(0x10) + pkt := udp6(tenant(1), internet, 1, 1) + + if verdict, _, _ := dp.send(t, pkt); verdict != tcActShot { + t.Errorf("never-applied group: verdict %d, want TC_ACT_SHOT", verdict) + } + + reachable(t) // nothing resolves + dp.apply(t, GroupPolicy{}, candidates(nil, a)) + if verdict, _, _ := dp.send(t, pkt); verdict != tcActShot { + t.Errorf("group with no reachable shard: verdict %d, want TC_ACT_SHOT", verdict) + } + + reachable(t, a) + dp.apply(t, GroupPolicy{}, candidates(nil, a)) + if verdict, _, _ := dp.send(t, pkt); verdict != tcActRedirect { + t.Errorf("group with a reachable shard: verdict %d, want redirect", verdict) + } + + if _, err := dp.groups.Disable(NAT66ShardGroup); err != nil { + t.Fatalf("Disable: %v", err) + } + if verdict, _, _ := dp.send(t, pkt); verdict != tcActShot { + t.Errorf("disabled group: verdict %d, want TC_ACT_SHOT", verdict) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatGroupEmpty); got != 3 { + t.Errorf("egress_shard_stats[group_empty] = %d, want 3", got) + } +} + +// TestGroupDatapath_FlowHashSpreadsOneAddress checks Phase 4's flow mode: one +// tenant address reaches more than one shard, every packet of one flow +// reaches one, and IPv4 fragments of one datagram stay together. +func TestGroupDatapath_FlowHashSpreadsOneAddress(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + dp.apply(t, GroupPolicy{Hash: HashFlow}, candidates(nil, a, b, c)) + + used := map[netip.Addr]bool{} + for port := range uint16(60) { + src := tenant(1) + _, got, _ := dp.send(t, udp6(src, internet, 3000+port, 443)) + want := predict(t, ShardPinKey{ + TableID: dpTableID, Src: src, Dst: internet, Protocol: 17, SrcPort: 3000 + port, DstPort: 443, + }, HashFlow, a, b, c) + if got != want { + t.Fatalf("flow from port %d went to %s, Go predicts %s", 3000+port, got, want) + } + used[got] = true + } + if len(used) != 3 { + t.Errorf("60 flows from one address used %d shards, want all 3", len(used)) + } + + src4, dst4 := netip.MustParseAddr("10.0.0.7"), netip.MustParseAddr("192.0.2.9") + const moreFragments, offset = 0x2000, 0x00B9 + _, first, _ := dp.send(t, udp4(src4, dst4, 4000, 53, moreFragments)) + _, later, _ := dp.send(t, udp4(src4, dst4, 0, 0, offset)) + if first != later { + t.Errorf("IPv4 fragments of one datagram went to %s and %s, want one shard", first, later) + } + want := predict(t, ShardPinKey{TableID: dpTableID, Src: src4, Dst: dst4, Protocol: 17}, HashFlow, a, b, c) + if first != want { + t.Errorf("first fragment went to %s, Go predicts %s (ports ignored for a fragment)", first, want) + } +} + +// TestGroupDatapath_IPv4SourceHash checks the IPv4 key layout matches. +func TestGroupDatapath_IPv4SourceHash(t *testing.T) { + dp := loadDatapath(t) + a, b := shardSID(0x10), shardSID(0x11) + reachable(t, a, b) + dp.apply(t, GroupPolicy{}, candidates(nil, a, b)) + for i := range 100 { + src := netip.AddrFrom4([4]byte{10, 1, byte(i >> 8), byte(i)}) //nolint:gosec // test index + _, got, argument := dp.send(t, udp4(src, netip.MustParseAddr("192.0.2.1"), 1, 53, 0)) + if want := predict(t, ShardPinKey{TableID: dpTableID, Src: src}, HashSource, a, b); got != want { + t.Fatalf("IPv4 tenant %s went to %s, Go predicts %s", src, got, want) + } + if argument != dpArgument { + t.Fatalf("IPv4 tenant %s: Argument %#x, want %#x", src, argument, dpArgument) + } + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_publication_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_publication_test.go new file mode 100644 index 00000000..5207125f --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_publication_test.go @@ -0,0 +1,246 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "errors" + "net" + "net/netip" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cilium/ebpf" + + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" +) + +// These run usid_egress under BPF_PROG_RUN against snapshots published through +// the real egress_shard_groups ARRAY_OF_MAPS. + +var errBoundary = errors.New("injected failure at a publication boundary") + +// failAt makes dp's store fail one publication at boundary ("create", "write" +// or "swap"), then behave normally again. +func (dp *datapath) failAt(boundary string) { + real := NewKernelGroupStore(dp.objs.EgressShardGroups) + once := true + pass := func() bool { + if once { + once = false + return false + } + return true + } + switch boundary { + case "create": + dp.store.newMap = func(spec *ebpf.MapSpec) (*ebpf.Map, error) { + if !pass() { + return nil, errBoundary + } + return real.newMap(spec) + } + case "write": + dp.store.writeInner = func(inner *ebpf.Map, v *prog.UsidEgressShardGroupValue) error { + if !pass() { + return errBoundary + } + return real.writeInner(inner, v) + } + case "swap": + dp.store.swap = func(outer *ebpf.Map, id uint32, inner *ebpf.Map) error { + if !pass() { + return errBoundary + } + return real.swap(outer, id, inner) + } + } +} + +// TestGroupPublication_FailedPublicationIsInvisible reproduces the review's +// F05 sequence against the kernel at every publication boundary. A occupies +// slot 0 and a tenant is pinned to it. Replacing A with B fails, so packets +// keep seeing A and the pin keeps holding. C then replaces B and is published: +// the pin must be refused, since C's generation is neither A's nor the one B +// was allocated, and the tenant is placed on C afresh. +func TestGroupPublication_FailedPublicationIsInvisible(t *testing.T) { + for _, boundary := range []string{"create", "write", "swap"} { + t.Run(boundary, func(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + policy := GroupPolicy{PinIdle: time.Hour} + pkt := udp6(tenant(1), internet, 1, 1) + + dp.apply(t, policy, candidates(nil, a)) + if _, got, _ := dp.send(t, pkt); got != a { + t.Fatalf("pinned to %s, want %s", got, a) + } + + dp.failAt(boundary) + rb, err := dp.groups.Apply(NAT66ShardGroup, policy, candidates(nil, b)) + if !errors.Is(err, errBoundary) { + t.Fatalf("Apply(B) = %v, want the injected %s failure", err, boundary) + } + if _, got, _ := dp.send(t, pkt); got != a { + t.Errorf("after the failed publication the tenant went to %s, want A, still published", got) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinHit); got != 1 { + t.Errorf("pin_hit = %d, want 1: the pin to A holds while A stays published", got) + } + + rc := dp.apply(t, policy, candidates(nil, c)) + if slotOf(t, rc, c).Generation == slotOf(t, rb, b).Generation { + t.Fatalf("C reused B's allocated generation %d", slotOf(t, rc, c).Generation) + } + if _, got, _ := dp.send(t, pkt); got != c { + t.Errorf("after C was published the tenant went to %s, want C", got) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinStale); got != 1 { + t.Errorf("pin_stale = %d, want 1: A's pin must be refused for C", got) + } + }) + } +} + +// TestGroupPublication_RestartKeepsPins checks that a galactic-cni restart, +// which rebuilds the group from the published snapshot, keeps every member's +// generation and so every pin. +func TestGroupPublication_RestartKeepsPins(t *testing.T) { + dp := loadDatapath(t) + a, b := shardSID(0x10), shardSID(0x11) + reachable(t, a, b) + policy := GroupPolicy{PinIdle: time.Hour} + dp.apply(t, policy, candidates(nil, a, b)) + pkt := udp6(tenant(3), internet, 1, 1) + _, first, _ := dp.send(t, pkt) + + restarted := NewShardGroupTable(NewKernelGroupStore(dp.objs.EgressShardGroups)) + result, err := restarted.Apply(NAT66ShardGroup, policy, candidates(nil, a, b)) + if err != nil { + t.Fatal(err) + } + if result.Changed { + t.Errorf("rebuild after restart published a new snapshot: %+v", result.Shards) + } + if _, again, _ := dp.send(t, pkt); again != first { + t.Errorf("after restart the tenant went to %s, want %s", again, first) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinHit); got != 1 { + t.Errorf("pin_hit = %d, want 1", got) + } +} + +// TestGroupPublication_ClassesIsolateTranslation checks that a route's group +// decides which shards can be chosen: NAT64 traffic is placed only on the +// NAT64 group's members, never on a shard that serves only NAT66. +func TestGroupPublication_ClassesIsolateTranslation(t *testing.T) { + dp := loadDatapath(t) + nat66Only, dual := shardSID(0x10), shardSID(0x11) + reachable(t, nat66Only, dual) + if _, err := dp.groups.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, nat66Only, dual)); err != nil { + t.Fatal(err) + } + if _, err := dp.groups.Apply(1, GroupPolicy{Class: NAT64Class(nsp), Ineligible: 1}, + candidates(nil, dual)); err != nil { + t.Fatal(err) + } + if err := dp.routes.RegisterGroup(dpTableID, nspNet(), 1, dpArgument); err != nil { + t.Fatal(err) + } + + sawNAT66Only := false + for i := range 200 { + dst := netip.AddrFrom16(nsp.Addr().As16()) + b := dst.As16() + b[15] = byte(i) + _, got, _ := dp.send(t, udp6(tenant(i), netip.AddrFrom16(b), 1, 53)) + if got != dual { + t.Fatalf("NAT64 traffic from %s went to %s, want only %s", tenant(i), got, dual) + } + if _, g66, _ := dp.send(t, udp6(tenant(i), internet, 1, 53)); g66 == nat66Only { + sawNAT66Only = true + } + } + if !sawNAT66Only { + t.Error("NAT66 traffic never used the NAT66-only shard, want it in the NAT66 group") + } +} + +// TestGroupPublication_ConcurrentSwapsStayCoherent publishes two snapshots +// alternately, replacing the only member of slot 0 each time with a shard +// whose next hop differs, while packets run through usid_egress on other +// CPUs. Every packet must leave with one snapshot's SID and that same +// snapshot's MAC addresses. An in-place update of an ordinary array value can +// mix the two; a swapped immutable snapshot cannot. +func TestGroupPublication_ConcurrentSwapsStayCoherent(t *testing.T) { + dp := loadDatapath(t) + a, b := shardSID(0x21), shardSID(0x22) + // Each shard's next hop names it: its MACs end in its Node-ID byte. + prev := resolveLinkAndL2Fn + resolveLinkAndL2Fn = func(sid net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + addr, _ := netip.AddrFromSlice(sid) + id, _ := uformat.NodeID(addr.Unmap()) + return 1, net.HardwareAddr{0xAA, 0, 0, 0, 0, byte(id)}, net.HardwareAddr{0xBB, 0, 0, 0, 0, byte(id)}, nil + } + t.Cleanup(func() { resolveLinkAndL2Fn = prev }) + dp.apply(t, GroupPolicy{}, candidates(nil, a)) + + const swaps = 400 + var ( + stop atomic.Bool + wg sync.WaitGroup + mixed, seenA, seenB, all atomic.Int64 + ) + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + pkt := udp6(tenant(5), internet, 1, 1) + for !stop.Load() { + ret, out, err := dp.objs.UsidEgress.Test(pkt) + if err != nil || ret != tcActRedirect || len(out) < ethLen+ip6Len { + continue + } + all.Add(1) + dst, _ := netip.AddrFromSlice(out[ethLen+24 : ethLen+40]) + id, _ := uformat.NodeID(dst) + switch { + case out[5] != byte(id) || out[11] != byte(id): + mixed.Add(1) + case id == 0x21: + seenA.Add(1) + case id == 0x22: + seenB.Add(1) + } + } + }() + } + for i := range swaps { + next := a + if i%2 == 0 { + next = b + } + if _, err := dp.groups.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, next)); err != nil { + stop.Store(true) + wg.Wait() + t.Fatal(err) + } + } + stop.Store(true) + wg.Wait() + + t.Logf("%d packets across %d swaps: %d on A, %d on B, %d mixed", all.Load(), swaps, seenA.Load(), seenB.Load(), + mixed.Load()) + if mixed.Load() != 0 { + t.Errorf("%d packets left with one shard's SID and another's MACs", mixed.Load()) + } + if seenA.Load() == 0 || seenB.Load() == 0 { + t.Errorf("packets saw A %d times and B %d times: the swaps did not interleave with packets, the test proved "+ + "nothing", seenA.Load(), seenB.Load()) + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_test.go new file mode 100644 index 00000000..80bcf191 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_test.go @@ -0,0 +1,538 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "encoding/binary" + "errors" + "net" + "net/netip" + "testing" + "time" + + "go.datum.net/galactic/internal/plumbing/ebpf/prog" +) + +// fakeGroupStore is an in-memory GroupStore. Publish is atomic, as the kernel +// store's swap is: a failed publication changes nothing. fail, when set, +// decides whether a publication fails. +type fakeGroupStore struct { + published map[uint32]prog.UsidEgressShardGroupValue + attempts []prog.UsidEgressShardGroupValue + fail func(groupID uint32, value prog.UsidEgressShardGroupValue) error +} + +func newFakeGroupStore() *fakeGroupStore { + return &fakeGroupStore{published: map[uint32]prog.UsidEgressShardGroupValue{}} +} + +func (f *fakeGroupStore) Load(groupID uint32) (prog.UsidEgressShardGroupValue, bool, error) { + v, ok := f.published[groupID] + return v, ok, nil +} + +func (f *fakeGroupStore) Publish(groupID uint32, value prog.UsidEgressShardGroupValue) error { + f.attempts = append(f.attempts, value) + if f.fail != nil { + if err := f.fail(groupID, value); err != nil { + return err + } + } + f.published[groupID] = value + return nil +} + +var errInjected = errors.New("injected publication failure") + +func applyOK(t *testing.T, g *ShardGroupTable, policy GroupPolicy, c []ShardCandidate) GroupResult { + t.Helper() + result, err := g.Apply(NAT66ShardGroup, policy, c) + if err != nil { + t.Fatalf("Apply: %v", err) + } + return result +} + +func published(t *testing.T, store *fakeGroupStore) prog.UsidEgressShardGroupValue { + t.Helper() + v, ok := store.published[NAT66ShardGroup] + if !ok { + t.Fatal("no snapshot published") + } + return v +} + +func slotOf(t *testing.T, r GroupResult, sid netip.Addr) ShardState { + t.Helper() + for _, s := range r.Shards { + if s.SID == sid { + return s + } + } + t.Fatalf("shard %s not in result %+v", sid, r.Shards) + return ShardState{} +} + +// TestApplyPublishesOneSnapshot checks that Apply strips the Argument, +// resolves next hops, fills every Maglev slot with an active shard's index, +// records the class, and publishes it all as one value. +func TestApplyPublishesOneSnapshot(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a, b := shardSID(0x10), shardSID(0x11) + reachable(t, a, b) + + withArgument := a.As16() + withArgument[9] = 0x42 + result := applyOK(t, g, GroupPolicy{PinIdle: time.Minute, Ineligible: 1}, []ShardCandidate{ + {SID: net.IP(withArgument[:])}, {SID: net.IP(b.AsSlice())}, + }) + if !result.Changed || result.Generation != 1 || result.Active() != 2 || len(store.attempts) != 1 { + t.Fatalf("result = %+v after %d publications, want one publication, generation 1, 2 active", + result, len(store.attempts)) + } + v := published(t, store) + if v.Flags&groupFlagEnabled == 0 || v.Candidates != 2 || v.Active != 2 || v.PinIdleSec != 60 || + v.Ineligible != 1 || v.ClassKind != classKindNAT66 { + t.Errorf("snapshot header = %+v, want enabled NAT66, 2 candidates, 2 active, 1 ineligible, pin 60s", v) + } + counts := map[uint8]int{} + for _, slot := range v.Maglev { + counts[slot]++ + } + if len(counts) != 2 || counts[shardSlotNone] != 0 { + t.Errorf("Maglev slots use %v, want exactly the two shard slots", counts) + } + stored := v.Shards[slotOf(t, result, a).Slot] + if netip.AddrFrom16(stored.Sid) != a || stored.Flags != shardFlagAlive || stored.LinkIfindex != 1 || + net.HardwareAddr(stored.Dmac[:]).String() != dpShardMAC.String() || stored.Generation == 0 { + t.Errorf("member = %+v, want base SID %s, alive, resolved next hop, nonzero generation", stored, a) + } + + again := applyOK(t, g, GroupPolicy{PinIdle: time.Minute, Ineligible: 1}, + candidates(nil, b, a)) + if again.Changed || len(store.attempts) != 1 { + t.Errorf("unchanged rebuild published again (changed %v, %d publications)", again.Changed, len(store.attempts)) + } +} + +// TestApplyKeepsSlotsAndGenerations checks that a shard keeps its slot and +// generation across rebuilds and a new shard takes a fresh generation, and +// that a freed slot reused by another shard never reuses a generation. +func TestApplyKeepsSlotsAndGenerations(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a, b, c, d := shardSID(0x10), shardSID(0x11), shardSID(0x12), shardSID(0x13) + reachable(t, a, b, c, d) + + first := applyOK(t, g, GroupPolicy{}, candidates(nil, a, b)) + third := applyOK(t, g, GroupPolicy{}, candidates(nil, a, b, c)) + if slotOf(t, third, a) != withAssigned(slotOf(t, first, a), false) || + slotOf(t, third, b).Generation != slotOf(t, first, b).Generation { + t.Errorf("existing shards changed slot or generation: before %+v, after %+v", first.Shards, third.Shards) + } + if !slotOf(t, third, c).Assigned || slotOf(t, third, a).Assigned { + t.Errorf("Assigned wrong: a %v c %v", slotOf(t, third, a).Assigned, slotOf(t, third, c).Assigned) + } + + applyOK(t, g, GroupPolicy{}, candidates(nil, b, c)) + if published(t, store).Shards[slotOf(t, first, a).Slot].Sid != [16]byte{} { + t.Error("slot still holds a after it left") + } + fifth := applyOK(t, g, GroupPolicy{}, candidates(nil, b, c, d)) + if slotOf(t, fifth, d).Slot != slotOf(t, first, a).Slot { + t.Errorf("d took slot %d, want the freed slot %d", slotOf(t, fifth, d).Slot, slotOf(t, first, a).Slot) + } + seen := map[uint32]netip.Addr{} + for _, r := range []GroupResult{first, third, fifth} { + for _, s := range r.Shards { + if other, dup := seen[s.Generation]; dup && other != s.SID { + t.Errorf("generation %d given to both %s and %s", s.Generation, other, s.SID) + } + seen[s.Generation] = s.SID + } + } +} + +func withAssigned(s ShardState, assigned bool) ShardState { + s.Assigned = assigned + return s +} + +// TestApplyGenerationsSurvivePartialPublication reproduces the review's F05 +// sequence. A occupies slot 0. B replaces A, but publishing B fails, so the +// published snapshot still holds A. C replaces B before any successful +// publication. C's generation must differ from both A's and the generation B +// was allocated, so no pin made for either can follow C. +func TestApplyGenerationsSurvivePartialPublication(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + + ra := applyOK(t, g, GroupPolicy{}, candidates(nil, a)) + genA := slotOf(t, ra, a).Generation + + store.fail = func(uint32, prog.UsidEgressShardGroupValue) error { return errInjected } + rb, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, b)) + if !errors.Is(err, errInjected) { + t.Fatalf("Apply(B) error = %v, want the injected failure", err) + } + genB := slotOf(t, rb, b).Generation + if v := published(t, store); netip.AddrFrom16(v.Shards[0].Sid) != a || v.Shards[0].Generation != genA { + t.Fatalf("published slot 0 = %s gen %d after the failed publication, want A gen %d", + netip.AddrFrom16(v.Shards[0].Sid), v.Shards[0].Generation, genA) + } + + store.fail = nil + rc := applyOK(t, g, GroupPolicy{}, candidates(nil, c)) + genC := slotOf(t, rc, c).Generation + if genC == genA || genC == genB || genB == genA { + t.Errorf("generations A=%d B=%d C=%d, want all distinct", genA, genB, genC) + } + if v := published(t, store); v.NextSlotGeneration <= genC { + t.Errorf("published next_slot_generation %d, want above every handed-out generation (C=%d)", + v.NextSlotGeneration, genC) + } +} + +// TestApplyGenerationsAcrossFailuresRetriesAndRestart drives a long sequence +// of replacements with failures injected at random publications, a retry +// after each, a disable and re-enable, and a restart from the published +// snapshot, and checks no generation is ever handed to two shards. +func TestApplyGenerationsAcrossFailuresRetriesAndRestart(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + sids := make([]netip.Addr, 0, 40) + for i := range 40 { + sids = append(sids, shardSID(uint16(0x100+i))) //nolint:gosec // test index + } + reachable(t, sids...) + + owner := map[uint32]netip.Addr{} + check := func(r GroupResult) { + for _, s := range r.Shards { + if prev, ok := owner[s.Generation]; ok && prev != s.SID { + t.Fatalf("generation %d handed to %s after %s", s.Generation, s.SID, prev) + } + owner[s.Generation] = s.SID + } + } + for i := range 39 { + n := 0 + store.fail = func(uint32, prog.UsidEgressShardGroupValue) error { + n++ + if i%3 == 0 && n == 1 { + return errInjected + } + return nil + } + set := candidates(nil, sids[i], sids[i+1]) + r, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, set) + check(r) + if err != nil { + r, err = g.Apply(NAT66ShardGroup, GroupPolicy{}, set) // retry + if err != nil { + t.Fatalf("retry %d: %v", i, err) + } + check(r) + } + if i == 20 { + if _, err := g.Disable(NAT66ShardGroup); err != nil { + t.Fatal(err) + } + } + if i == 30 { + g = NewShardGroupTable(store) // a restarted galactic-cni + } + } +} + +// TestApplyClassChangeStartsAfresh checks a group that changes class keeps no +// slot, so no pin made for the old class is honoured. +func TestApplyClassChangeStartsAfresh(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a := shardSID(0x10) + reachable(t, a) + first := applyOK(t, g, GroupPolicy{Class: NAT64Class(netip.MustParsePrefix("2001:db8:64::/96"))}, + candidates(nil, a)) + second := applyOK(t, g, GroupPolicy{Class: NAT64Class(WellKnownPrefix)}, candidates(nil, a)) + if !slotOf(t, second, a).Assigned || slotOf(t, second, a).Generation == slotOf(t, first, a).Generation { + t.Errorf("class change kept the member's tenure: before %+v, after %+v", first.Shards, second.Shards) + } + status, _ := g.Status(NAT66ShardGroup) + if status.Class != NAT64Class(WellKnownPrefix) { + t.Errorf("published class %s, want nat64:64:ff9b::/96", status.Class) + } +} + +// TestApplyExcludesDrainingAndUnreachableFromTheTable checks that only alive, +// non-draining shards appear in the Maglev table, while both other kinds stay +// candidates. +func TestApplyExcludesDrainingAndUnreachableFromTheTable(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b) // c does not resolve + + result := applyOK(t, g, GroupPolicy{}, candidates(map[netip.Addr]bool{b: true}, a, b, c)) + if result.Active() != 1 { + t.Fatalf("active = %d, want 1", result.Active()) + } + if s := slotOf(t, result, c); s.Alive || s.Err == nil { + t.Errorf("unreachable shard state %+v, want not alive with an error", s) + } + v := published(t, store) + if v.Candidates != 3 || v.Active != 1 { + t.Errorf("snapshot candidates %d active %d, want 3 and 1", v.Candidates, v.Active) + } + onlyA := uint8(slotOf(t, result, a).Slot) //nolint:gosec // < 32 + for i, slot := range v.Maglev { + if slot != onlyA { + t.Fatalf("Maglev slot %d = %d, want every slot on %s (slot %d)", i, slot, a, onlyA) + } + } + if f := v.Shards[slotOf(t, result, b).Slot].Flags; f != shardFlagAlive|shardFlagDraining { + t.Errorf("draining shard flags %#x, want alive|draining", f) + } +} + +// TestApplyWithNoActiveShardEmptiesTheTable checks that a group whose shards +// are all unreachable stays enabled with every slot empty, which the datapath +// drops on. +func TestApplyWithNoActiveShardEmptiesTheTable(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + reachable(t) + applyOK(t, g, GroupPolicy{}, candidates(nil, shardSID(0x10))) + status, err := g.Status(NAT66ShardGroup) + if err != nil { + t.Fatal(err) + } + if !status.Open() || status.Candidates != 1 || status.Active != 0 { + t.Errorf("status = %+v, want open with 1 candidate and none active", status) + } + for i, slot := range published(t, store).Maglev { + if slot != shardSlotNone { + t.Fatalf("Maglev slot %d = %d, want none", i, slot) + } + } +} + +// TestApplyRejectsTooManyShardsAndBadSIDs checks Apply's input validation. +func TestApplyRejectsTooManyShardsAndBadSIDs(t *testing.T) { + g := NewShardGroupTable(newFakeGroupStore()) + reachable(t) + many := make([]netip.Addr, 0, MaxShardsPerGroup+1) + for i := range MaxShardsPerGroup + 1 { + many = append(many, shardSID(uint16(0x100+i))) //nolint:gosec // test index + } + if _, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, many...)); err == nil { + t.Error("Apply with 33 shards = nil, want an error") + } + padded := []ShardCandidate{{SID: net.ParseIP("2001:db8::1:2:3")}} + if _, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, padded); err == nil { + t.Error("Apply with a SID whose padding is set = nil, want an error") + } + if _, err := g.Apply(MaxShardGroups, GroupPolicy{}, nil); err == nil { + t.Error("Apply to an out-of-range group = nil, want an error") + } +} + +// TestCloseKeepsForwardingAndRefusesNewRoutes checks Close: the group stays +// enabled with its members and their generations, and reads as no longer open. +func TestCloseKeepsForwardingAndRefusesNewRoutes(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a := shardSID(0x10) + reachable(t, a) + before := applyOK(t, g, GroupPolicy{PinIdle: time.Hour}, candidates(nil, a)) + + closing, err := g.Close(NAT66ShardGroup) + if err != nil || !closing { + t.Fatalf("Close = %v, %v, want closing", closing, err) + } + status, _ := g.Status(NAT66ShardGroup) + if !status.Enabled || !status.Closing || status.Open() || status.Active != 1 { + t.Errorf("status after Close = %+v, want enabled, closing, not open, 1 active", status) + } + members, _ := g.Members(NAT66ShardGroup) + if len(members) != 1 || members[0].Generation != slotOf(t, before, a).Generation { + t.Errorf("members after Close = %+v, want a with its generation kept", members) + } + if v := published(t, store); v.PinIdleSec != 3600 { + t.Errorf("pin idle after Close = %d, want the group's own 3600", v.PinIdleSec) + } + + if closing, _ := NewShardGroupTable(newFakeGroupStore()).Close(NAT66ShardGroup); closing { + t.Error("Close of a group never published reported closing") + } +} + +// TestDisableKeepsTheGenerationCounter checks Disable turns the group off, +// clears every slot, is a no-op the second time, and that a re-enabled group +// hands out generations above everything before it. +func TestDisableKeepsTheGenerationCounter(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a := shardSID(0x10) + reachable(t, a) + before := applyOK(t, g, GroupPolicy{}, candidates(nil, a)) + + changed, err := g.Disable(NAT66ShardGroup) + if err != nil || !changed { + t.Fatalf("Disable = %v, %v, want changed", changed, err) + } + status, _ := g.Status(NAT66ShardGroup) + if status.Enabled || status.Candidates != 0 { + t.Errorf("status after Disable = %+v, want disabled and empty", status) + } + if members, _ := g.Members(NAT66ShardGroup); len(members) != 0 { + t.Errorf("members after Disable = %v, want none", members) + } + if changed, err := g.Disable(NAT66ShardGroup); err != nil || changed { + t.Errorf("second Disable = %v, %v, want unchanged", changed, err) + } + after := applyOK(t, NewShardGroupTable(store), GroupPolicy{}, candidates(nil, a)) + if slotOf(t, after, a).Generation <= slotOf(t, before, a).Generation { + t.Errorf("re-enabled generation %d, want above %d", slotOf(t, after, a).Generation, + slotOf(t, before, a).Generation) + } +} + +// TestFindClassAndAnyOpen checks the lookups CNI ADD makes. +func TestFindClassAndAnyOpen(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a := shardSID(0x10) + reachable(t, a) + nsp := NAT64Class(netip.MustParsePrefix("2001:db8:64::/96")) + if _, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, a)); err != nil { + t.Fatal(err) + } + if _, err := g.Apply(2, GroupPolicy{Class: nsp}, candidates(nil, a)); err != nil { + t.Fatal(err) + } + if id, _, ok, _ := g.FindClass(nsp); !ok || id != 2 { + t.Errorf("FindClass(%s) = %d, %v, want group 2", nsp, id, ok) + } + if _, _, ok, _ := g.FindClass(NAT64Class(WellKnownPrefix)); ok { + t.Error("FindClass found a group for a class nothing serves") + } + if open, _ := g.AnyOpen(); !open { + t.Error("AnyOpen = false with two open groups") + } + for _, id := range []uint32{0, 2} { + if _, err := g.Close(id); err != nil { + t.Fatal(err) + } + } + if open, _ := g.AnyOpen(); open { + t.Error("AnyOpen = true with every group closing") + } +} + +// TestClassForRoute checks which class each egress route needs. +func TestClassForRoute(t *testing.T) { + cidr := func(s string) *net.IPNet { + _, n, err := net.ParseCIDR(s) + if err != nil { + t.Fatal(err) + } + return n + } + if c, ok := ClassForRoute(cidr("::/0")); !ok || c != NAT66Class { + t.Errorf("::/0 -> %v %v, want nat66", c, ok) + } + if c, ok := ClassForRoute(cidr("64:ff9b::/96")); !ok || c != NAT64Class(WellKnownPrefix) { + t.Errorf("64:ff9b::/96 -> %v %v, want nat64", c, ok) + } + if _, ok := ClassForRoute(cidr("10.0.0.0/8")); ok { + t.Error("an IPv4 route has a translation class") + } +} + +// TestRegisterGroupWritesTheSentinel checks the sentinel layout usid.c reads. +func TestRegisterGroupWritesTheSentinel(t *testing.T) { + tbl := NewEgressRouteTable(newFakeTable()) + if err := tbl.RegisterGroup(7, DefaultPrefix, 2, 0x2A5); err != nil { + t.Fatal(err) + } + key, _ := buildKey(7, DefaultPrefix) + value, err := lookupRaw(tbl, key) + if err != nil { + t.Fatal(err) + } + if value.LinkIfindex != 0xFFFFFFFF { + t.Errorf("link_ifindex = %#x, want the group sentinel", value.LinkIfindex) + } + if got := binary.NativeEndian.Uint32(value.Sid[0:4]); got != 2 { + t.Errorf("group id = %d, want 2", got) + } + if value.Sid[8] != 0x02 || value.Sid[9] != 0xA5 { + t.Errorf("argument bytes = %#x %#x, want 0x02 0xa5", value.Sid[8], value.Sid[9]) + } + if err := tbl.RegisterGroup(7, DefaultPrefix, MaxShardGroups, 1); err == nil { + t.Error("RegisterGroup with an out-of-range group = nil, want an error") + } + if err := tbl.RegisterGroup(7, DefaultPrefix, 0, 0); err == nil { + t.Error("RegisterGroup with Argument 0 = nil, want an error") + } +} + +// TestShardPinKeyBytesMatchesTheCLayout checks the key's size and the byte +// order of the fields the datapath copies raw from the packet. +func TestShardPinKeyBytesMatchesTheCLayout(t *testing.T) { + key := ShardPinKey{ + TableID: 0x01020304, GroupID: 1, + Src: netip.MustParseAddr("2001:db8::1"), Dst: netip.MustParseAddr("2001:db8::2"), + Protocol: 6, SrcPort: 0x1234, DstPort: 0x0050, + } + flow := key.Bytes(HashFlow) + if len(flow) != 44 { + t.Fatalf("key is %d bytes, want 44 (struct egress_shard_pin_key)", len(flow)) + } + if flow[4] != egressRouteFamilyINET6 || flow[5] != 6 || flow[6] != 1 { + t.Errorf("family/protocol/group = %d/%d/%d, want 0/6/1", flow[4], flow[5], flow[6]) + } + if flow[8] != 0x12 || flow[9] != 0x34 || flow[10] != 0x00 || flow[11] != 0x50 { + t.Errorf("ports = % x, want network byte order 12 34 00 50", flow[8:12]) + } + source := key.Bytes(HashSource) + for i, b := range source[8:12] { + if b != 0 { + t.Errorf("source mode port byte %d = %#x, want 0", i, b) + } + } + for i, b := range source[28:44] { + if b != 0 { + t.Errorf("source mode daddr byte %d = %#x, want 0", i, b) + } + } + v4 := ShardPinKey{Src: netip.MustParseAddr("10.1.2.3")}.Bytes(HashSource) + if v4[4] != egressRouteFamilyINET4 || v4[12] != 10 || v4[15] != 3 || v4[16] != 0 { + t.Errorf("IPv4 key = % x, want family 1 and the address in the first 4 saddr bytes", v4) + } +} + +// TestGroupSnapshotSpecMatchesTheCompiledInnerMap guards the hand-written inner +// map spec against the compiled one: the kernel refuses an inner map of any +// other shape, so a drift fails every publication. +func TestGroupSnapshotSpecMatchesTheCompiledInnerMap(t *testing.T) { + spec, err := prog.LoadUsid() + if err != nil { + t.Fatalf("LoadUsid: %v", err) + } + inner := spec.Maps[prog.UsidMapEgressShardGroups].InnerMap + if inner == nil { + t.Fatal("egress_shard_groups has no inner map spec") + } + if inner.Type != GroupSnapshotSpec.Type || inner.KeySize != GroupSnapshotSpec.KeySize || + inner.ValueSize != GroupSnapshotSpec.ValueSize || inner.MaxEntries != GroupSnapshotSpec.MaxEntries || + inner.Flags != GroupSnapshotSpec.Flags { + t.Errorf("compiled inner map %+v, GroupSnapshotSpec %+v", inner, GroupSnapshotSpec) + } +} diff --git a/internal/plumbing/ebpf/prog/doc.go b/internal/plumbing/ebpf/prog/doc.go index debe2dd3..6b178e7f 100644 --- a/internal/plumbing/ebpf/prog/doc.go +++ b/internal/plumbing/ebpf/prog/doc.go @@ -40,4 +40,4 @@ package prog // -cc is deliberately omitted: bpf2go defaults to $BPF2GO_CC or "clang", so CI // can pin an exact version without this directive changing. // -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cflags "-O2 -g -Wall -idirafter /usr/include/x86_64-linux-gnu -idirafter /usr/include/aarch64-linux-gnu" -target bpfel,bpfeb -type locator_value -type function_value -type vrf_value -type ifindex_vrf_value -type nptv6_value -type vip_xlat_key -type vip_xlat_value -type egress_route_key -type egress_route_value -type service_route_key -type service_route_value -type service_access_key -type service_access_value -type service_deny_key -type service_deny_value -type service_reverse_key -type service_reverse_value -type service_remote_grant_key -type service_remote_grant_value -type service_policy_state_value -type public_uplink_value -type mss_clamp_value -type tenant_gw_value -type vpc_attribution_value Usid usid.c +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cflags "-O2 -g -Wall -idirafter /usr/include/x86_64-linux-gnu -idirafter /usr/include/aarch64-linux-gnu" -target bpfel,bpfeb -type locator_value -type function_value -type vrf_value -type ifindex_vrf_value -type nptv6_value -type vip_xlat_key -type vip_xlat_value -type egress_route_key -type egress_route_value -type service_route_key -type service_route_value -type service_access_key -type service_access_value -type service_deny_key -type service_deny_value -type service_reverse_key -type service_reverse_value -type service_remote_grant_key -type service_remote_grant_value -type service_policy_state_value -type public_uplink_value -type egress_shard_group_value -type egress_shard_member -type egress_shard_pin_key -type egress_shard_pin_value -type egress_shard_counter -type mss_clamp_value -type tenant_gw_value -type vpc_attribution_value Usid usid.c diff --git a/internal/plumbing/ebpf/prog/egressshardstat.go b/internal/plumbing/ebpf/prog/egressshardstat.go new file mode 100644 index 00000000..5b820d32 --- /dev/null +++ b/internal/plumbing/ebpf/prog/egressshardstat.go @@ -0,0 +1,29 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package prog + +// Egress shard stat indices into the egress_shard_stats map, mirroring the +// datapath's enum egress_shard_stat. Hand-kept in sync with the C source, for +// the same reason as the drop reason indices in dropreason.go. +const ( + EgressShardStatGroupEmpty uint32 = 0 + EgressShardStatPinStale uint32 = 1 + EgressShardStatPinHit uint32 = 2 + EgressShardStatMaglev uint32 = 3 + EgressShardStatShardDead uint32 = 4 + EgressShardStatParseFailed uint32 = 5 + EgressShardStatCount uint32 = 6 +) + +// EgressShardStatNames maps each index to a short, stable, metrics-friendly +// name. +var EgressShardStatNames = map[uint32]string{ + EgressShardStatGroupEmpty: "group_empty", + EgressShardStatPinStale: "pin_stale", + EgressShardStatPinHit: "pin_hit", + EgressShardStatMaglev: "maglev", + EgressShardStatShardDead: "shard_dead", + EgressShardStatParseFailed: "parse_failed", +} diff --git a/internal/plumbing/ebpf/prog/usid.c b/internal/plumbing/ebpf/prog/usid.c index 89d2a82b..5b6c0e38 100644 --- a/internal/plumbing/ebpf/prog/usid.c +++ b/internal/plumbing/ebpf/prog/usid.c @@ -74,6 +74,7 @@ #define SEC(name) __attribute__((section(name), used)) #define __uint(name, val) int (*name)[val] #define __type(name, val) typeof(val) *name +#define __array(name, val) typeof(val) *name[] #define USID_ALWAYS_INLINE inline __attribute__((always_inline)) #define USID_NOINLINE __attribute__((noinline)) @@ -592,6 +593,219 @@ struct egress_route_value { __u8 smac[6]; } __attribute__((packed)); +// --------------------------------------------------------------------- +// Egress shard groups: active/active egress across a cluster's shards. +// +// An egress_route_table entry whose link_ifindex is +// USID_EGRESS_GROUP_SENTINEL does not name one shard. It names a shard group, +// and usid_egress picks a member per packet: from a pin in egress_shard_pin +// when the tenant address already has a live one, and otherwise from the +// group's Maglev table, indexed by a hash of the tenant's source. Every compute +// node builds the same table from the same member set. The hash includes the +// node-local VRF table ID, so a given tenant address keeps one shard, and one +// public address, on the node it lives on: the paired pooling RFC 4787 REQ-2 +// asks for. The same address on another node may land elsewhere. +// +// A group serves one translation class: NAT66 (the ::/0 route) or NAT64 +// toward one prefix (that prefix's route). Its members are only the shards +// that can translate that class, so a packet can never be placed on a shard +// that would drop it for want of an address family or a prefix. +// +// The sentinel value's sid carries the group ID in its first four bytes, in +// host byte order, and the tenant's Argument in the low 12 bits of bytes 8 and +// 9, exactly where a shard SID carries it. Every other byte is zero. The +// datapath completes the chosen shard's base SID with it, the same splice it +// already does for the outer source. +// +// Like link_ifindex == 0 for pass-through, the sentinel can never collide with +// a real registration: no interface has ifindex 0xFFFFFFFF. +// +// Publication. A group is one immutable snapshot: its Maglev table and every +// member's SID, next hop, flags and slot generation in a single value, held in +// a one-entry inner array map. galactic-cni builds a new inner map for every +// change and swaps it into egress_shard_groups, an ARRAY_OF_MAPS, in one +// update. The swap replaces an RCU-protected pointer, so a packet sees either +// the old snapshot or the new one, never a mix; and the kernel frees the old +// inner map only after every program that could still be reading it has +// finished. No published snapshot is ever written again. An ordinary ARRAY +// value, by contrast, is updated in place, so a packet reading it during an +// update can see half of each record. +// +// None of this changes a map that predates it, so a node upgrading keeps every +// pinned row it already has (see attach.Load). +// --------------------------------------------------------------------- + +#define USID_EGRESS_GROUP_SENTINEL 0xFFFFFFFFu + +// USID_EGRESS_MAX_GROUPS bounds the group ID: one NAT66 group and up to three +// NAT64 prefixes. +#define USID_EGRESS_MAX_GROUPS 4 + +// USID_EGRESS_MAX_SHARDS is a power of two so a shard index read out of a map +// value can be masked into range for the verifier, as the gateway does with +// EDGE_MAX_BACKENDS. +#define USID_EGRESS_MAX_SHARDS 32 + +// USID_EGRESS_MAGLEV_SIZE is prime, as Maglev requires, and the gateway's +// table size. 1021 slots keep the disruption bound for up to ten shards. +#define USID_EGRESS_MAGLEV_SIZE 1021 + +// USID_EGRESS_SHARD_NONE marks a Maglev slot with no shard, which a table built +// from an empty member set is made of. +#define USID_EGRESS_SHARD_NONE 0xFF + +// egress_shard_group_value.flags. +// +// ENABLED is set while the group may carry traffic. A sentinel pointing at a +// group that is missing or not enabled drops rather than guessing. +// +// HASH_FLOW selects the 5-tuple hash, which spreads one heavy tenant address +// over every shard at the cost of paired pooling. Unset is the source-address +// hash. +// +// CLOSING is control-plane state the datapath ignores: the cluster is moving +// back to ordered mode, so CNI ADD writes no new sentinel, while sentinels +// already written keep forwarding until galactic-cni has moved every one of +// them off the group. +#define USID_EGRESS_GROUP_ENABLED (1u << 0) +#define USID_EGRESS_GROUP_HASH_FLOW (1u << 1) +#define USID_EGRESS_GROUP_CLOSING (1u << 2) + +// egress_shard_group_value.class_kind: the translation the group serves. +#define USID_EGRESS_CLASS_NAT66 1 +#define USID_EGRESS_CLASS_NAT64 2 + +// egress_shard_member.flags. +// +// ALIVE means the shard's SID resolved to an uplink neighbor when the snapshot +// was built. Only an alive shard is encapsulated toward, whether chosen by a +// pin or by the Maglev table. +// +// DRAINING means the shard takes no new tenant addresses: it is absent from the +// Maglev table but keeps serving the pins it already holds. +#define USID_EGRESS_SHARD_ALIVE (1u << 0) +#define USID_EGRESS_SHARD_DRAINING (1u << 1) + +// struct egress_shard_member is one group member, with the link and L2 +// addresses resolved ahead of time for the same reason egress_route_value +// carries them. sid is the shard's base SID, with no tenant Argument. +// +// generation identifies the shard's tenure of this slot. It is allocated from +// the group's next_slot_generation when a shard takes the slot, never reused, +// and never zero for an occupied slot. A pin records it, and is honoured only +// while the slot still carries the same value, so a slot handed to a new shard +// cannot inherit the old one's pins. +struct egress_shard_member { + __u8 sid[16]; + __u32 link_ifindex; + __u8 dmac[6]; + __u8 smac[6]; + __u32 flags; + __u32 generation; +}; + +// struct egress_shard_group_value is one group snapshot, the only value of an +// egress_shard_groups inner map. +// +// candidates counts the shards eligible for the group's class and active the +// ones the Maglev table spreads new tenants across; a draining or unreachable +// shard is a candidate that is not active. ineligible counts the cluster's +// shards that cannot translate this class. CNI ADD and the metrics read these; +// the datapath does not: a slot holding USID_EGRESS_SHARD_NONE is its empty +// signal. +// +// pin_idle_sec is how long a pin outlives its last packet. Zero turns pins +// off, leaving selection to the Maglev table alone. +// +// generation counts publications, for observability. next_slot_generation is +// the next member generation to hand out; it only grows, and survives +// disabling and re-enabling the group. +struct egress_shard_group_value { + __u32 flags; + __u32 generation; + __u32 next_slot_generation; + __u16 candidates; + __u16 active; + __u16 ineligible; + __u8 class_kind; + __u8 pad0; + __u32 pin_idle_sec; + __u8 class_prefix[16]; + __u8 maglev[USID_EGRESS_MAGLEV_SIZE]; + __u8 pad1[3]; + struct egress_shard_member shards[USID_EGRESS_MAX_SHARDS]; +}; + +// struct egress_shard_pin_key is both the input to the shard hash and +// egress_shard_pin's key. In source-hash mode only table_id, family and saddr +// are set, so every flow from one tenant address shares one pin. In flow-hash +// mode protocol, the ports and daddr are set too; the ports stay zero for a +// fragment, so every fragment of a datagram, the first included, hashes alike. +struct egress_shard_pin_key { + __u32 table_id; + __u8 family; + __u8 protocol; + __u8 group; + __u8 pad; + __be16 sport; + __be16 dport; + __u8 saddr[16]; + __u8 daddr[16]; +}; + +// struct egress_shard_pin_value records which slot a tenant address was sent +// to, the slot's generation at the time, and when the pin last carried a +// packet, on bpf_ktime_get_boot_ns's clock. +struct egress_shard_pin_value { + __u64 last_seen_ns; + __u32 generation; + __u8 shard; + __u8 pad[3]; +}; + +// struct egress_shard_counter is one shard slot's packet and byte count, per +// CPU, for the share each shard carries. +struct egress_shard_counter { + __u64 packets; + __u64 bytes; +}; + +// struct egress_shard_scratch_value holds what resolve_egress_group builds for +// one packet: the pin key it hashes and looks up, and the route it resolves +// the sentinel to, which usid_egress then encapsulates from exactly as it does +// from an ordinary egress_route_table entry. Per CPU, for the same reason as +// vip_xlat_key_scratch: neither fits on the stack alongside send_too_big's. +struct egress_shard_scratch_value { + struct egress_shard_pin_key key; + struct egress_route_value route; +}; + +// enum egress_shard_stat indexes egress_shard_stats. Its own map rather than +// drop_reasons slots, for the reason mss_clamp_stats is. +enum egress_shard_stat { + // A sentinel named a group that is disabled, missing or has no shard in + // the slot its hash chose. The packet is dropped: failing open would send + // it untranslated into the node's main table. + EGRESS_SHARD_STAT_GROUP_EMPTY = 0, + // A pin named a shard that is no longer alive in that slot, or has idled + // out. The packet was re-placed by the Maglev table. + EGRESS_SHARD_STAT_PIN_STALE = 1, + // The packet followed a live pin. + EGRESS_SHARD_STAT_PIN_HIT = 2, + // The packet was placed by the Maglev table. + EGRESS_SHARD_STAT_MAGLEV = 3, + // The Maglev slot named a shard that is not alive. Dropped: the installer + // has not rebuilt the table since the shard went away. + EGRESS_SHARD_STAT_SHARD_DEAD = 4, + // Reading the tenant's addresses out of the packet failed. Dropped. + EGRESS_SHARD_STAT_PARSE_FAILED = 5, + __EGRESS_SHARD_STAT_COUNT, +}; + +// egress_shard_stats is sized above __EGRESS_SHARD_STAT_COUNT, as +// mss_clamp_stats is. +#define USID_EGRESS_SHARD_STATS_SLOTS 16 + // service_route_key scopes a private-service address to the exact consumer // attachment that policy selected. Using the ingress ifindex instead of the // shared VRF table prevents one selected attachment from granting every other @@ -1031,6 +1245,68 @@ struct { __type(value, struct egress_route_value); } egress_route_table SEC(".maps"); +// egress_shard_group_inner describes one egress_shard_groups inner map: a +// single immutable struct egress_shard_group_value. galactic-cni creates a new +// one for every publication; nothing ever writes a published one. +struct egress_shard_group_inner { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct egress_shard_group_value); +}; + +// egress_shard_group_template is one instance of the inner map type, never +// read or written. It exists so the compiler emits the full BTF of struct +// egress_shard_group_value: clang before 21 emits a type reached only through +// an ARRAY_OF_MAPS inner definition as a forward declaration, which the loader +// cannot size. libbpf's own map-in-map tests declare their inner maps the same +// way. It costs one 2.4 KiB array. +struct egress_shard_group_inner egress_shard_group_template SEC(".maps"); + +// egress_shard_groups: one slot per group ID, each holding the group's current +// snapshot. See the publication note above struct egress_shard_member. An +// empty slot reads as a missing group, which drops. +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, USID_EGRESS_MAX_GROUPS); + __type(key, __u32); + __array(values, struct egress_shard_group_inner); +} egress_shard_groups SEC(".maps"); + +// egress_shard_pin: see struct egress_shard_pin_key. An LRU, so a pin is +// evicted under pressure rather than refusing a new one. An evicted pin falls +// back to the Maglev table, which usually gives the same answer anyway. Sized +// like the shard's own nat_conn_table. +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 65536); + __type(key, struct egress_shard_pin_key); + __type(value, struct egress_shard_pin_value); +} egress_shard_pin SEC(".maps"); + +// egress_shard_counters: see struct egress_shard_counter. Indexed by +// group * USID_EGRESS_MAX_SHARDS + slot. +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, USID_EGRESS_MAX_GROUPS * USID_EGRESS_MAX_SHARDS); + __type(key, __u32); + __type(value, struct egress_shard_counter); +} egress_shard_counters SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, USID_EGRESS_SHARD_STATS_SLOTS); + __type(key, __u32); + __type(value, __u64); +} egress_shard_stats SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct egress_shard_scratch_value); +} egress_shard_scratch SEC(".maps"); + // Node-local private-service routes are exact attachment/service matches. They // remain separate from fabric routes so policy never creates a Linux route. struct { @@ -3303,6 +3579,235 @@ static USID_NOINLINE long service_path(struct __sk_buff *skb, __be16 h_proto, __ return rc; } +static USID_ALWAYS_INLINE void count_egress_shard_stat(__u32 stat) +{ + __u64 *count = bpf_map_lookup_elem(&egress_shard_stats, &stat); + + if (count) + *count += 1; +} + +// egress_shard_hash is 32-bit FNV-1a over every byte of key, the hash the +// gateway's fnv1a_flow uses. load_egress_shard_key zeroes the key first, so +// fields the hash mode does not use, and the padding, hash as zeros. +// internal/plumbing/ebpf/egressroutemap rebuilds the same bytes to predict a +// placement. +static USID_ALWAYS_INLINE __u32 egress_shard_hash(const struct egress_shard_pin_key *key) +{ + const __u8 *p = (const __u8 *) key; + __u32 h = 2166136261u; + +#pragma unroll + for (int i = 0; i < (int) sizeof(*key); i++) { + h ^= p[i]; + h *= 16777619u; + } + return h; +} + +// load_egress_shard_key fills key from the packet about to be encapsulated. +// It reads with bpf_skb_load_bytes rather than through packet pointers, so +// nothing the caller derived from skb->data is invalidated. +// +// Source mode reads only the source address. Flow mode adds the destination, +// the protocol and, for an unfragmented TCP or UDP packet, both ports. An IPv6 +// packet whose first next header is an extension header, a Fragment header +// included, hashes on its addresses and that header's type, so every fragment +// of a datagram goes to one shard. +// +// Returns 0 on success and -1 when the packet is too short to read. +static USID_ALWAYS_INLINE int load_egress_shard_key(struct __sk_buff *skb, struct egress_shard_pin_key *key, + __u32 table_id, __u8 family, __u32 group_id, int flow) +{ + __builtin_memset(key, 0, sizeof(*key)); + key->table_id = table_id; + key->family = family; + key->group = (__u8) group_id; + + if (family == USID_EGRESS_ROUTE_FAMILY_INET6) { + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + USID_OFFSETOF(struct usid_ip6hdr, saddr), key->saddr, 16)) + return -1; + if (!flow) + return 0; + + __u8 nexthdr; + + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + USID_OFFSETOF(struct usid_ip6hdr, nexthdr), &nexthdr, 1)) + return -1; + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + USID_OFFSETOF(struct usid_ip6hdr, daddr), key->daddr, 16)) + return -1; + key->protocol = nexthdr; + if (nexthdr == USID_IPPROTO_TCP || nexthdr == USID_IPPROTO_UDP) { + // sport and dport are adjacent in the key, as in the header. + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + (__u32) sizeof(struct usid_ip6hdr), &key->sport, 4)) + return -1; + } + return 0; + } + + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + USID_OFFSETOF(struct usid_iphdr, saddr), key->saddr, 4)) + return -1; + if (!flow) + return 0; + + struct usid_iphdr ip4; + + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET, &ip4, sizeof(ip4))) + return -1; + __builtin_memcpy(key->daddr, ip4.daddr, sizeof(ip4.daddr)); + key->protocol = ip4.protocol; + + __u16 frag = __builtin_bswap16(ip4.frag_off) & (USID_IPV4_FRAG_OFFSET_MASK | USID_IPV4_MORE_FRAGMENTS); + + if (frag == 0 && (ip4.protocol == USID_IPPROTO_TCP || ip4.protocol == USID_IPPROTO_UDP)) { + __u32 ihl = (__u32) (ip4.ver_ihl & 0x0F) * 4; + + if (ihl < USID_IPV4_MIN_HDR_LEN) + return -1; + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + ihl, &key->sport, 4)) + return -1; + } + return 0; +} + +// resolve_egress_group picks the shard a packet matching a group sentinel goes +// to, and writes the route to encapsulate it with into egress_shard_scratch's +// route: the shard's base SID completed with function_argument, the +// sentinel's bytes 8 and 9, and the shard's link and L2 addresses. +// +// Everything it reads about the group comes from one published snapshot, so +// the Maglev slot, the member's identity, its liveness and its next hop always +// belong together: a snapshot is never written once published. +// +// A live pin wins. A pin is live while its slot is alive and still carries the +// generation the pin recorded, and its last packet is no older than the +// group's pin_idle_sec; a draining shard keeps its pins. Otherwise the Maglev +// table chooses, and with pins on, the choice is pinned. +// +// Returns 0 when the route is ready and -1 when the packet must be dropped, +// already counted in egress_shard_stats. Not inlined: usid_egress is already +// close to the verifier's per-function stack and complexity budgets. +static USID_NOINLINE int resolve_egress_group(struct __sk_buff *skb, __u32 table_id, __u32 family, __u32 group_id, + __u32 function_argument) +{ + __u32 zero = 0; + struct egress_shard_scratch_value *s = bpf_map_lookup_elem(&egress_shard_scratch, &zero); + + if (!s) + return -1; + if (group_id >= USID_EGRESS_MAX_GROUPS) { + count_egress_shard_stat(EGRESS_SHARD_STAT_GROUP_EMPTY); + return -1; + } + + void *snapshot = bpf_map_lookup_elem(&egress_shard_groups, &group_id); + + if (!snapshot) { + count_egress_shard_stat(EGRESS_SHARD_STAT_GROUP_EMPTY); + return -1; + } + + struct egress_shard_group_value *g = bpf_map_lookup_elem(snapshot, &zero); + + if (!g || !(g->flags & USID_EGRESS_GROUP_ENABLED)) { + count_egress_shard_stat(EGRESS_SHARD_STAT_GROUP_EMPTY); + return -1; + } + if (load_egress_shard_key(skb, &s->key, table_id, (__u8) family, group_id, + (g->flags & USID_EGRESS_GROUP_HASH_FLOW) != 0)) { + count_egress_shard_stat(EGRESS_SHARD_STAT_PARSE_FAILED); + return -1; + } + + __u64 now = bpf_ktime_get_boot_ns(); + __u64 idle_ns = (__u64) g->pin_idle_sec * 1000000000ULL; + __u32 index = 0; + int placed = 0; + + if (idle_ns) { + struct egress_shard_pin_value *pin = bpf_map_lookup_elem(&egress_shard_pin, &s->key); + + if (pin) { + __u32 pinned = pin->shard; + + USID_BARRIER_VAR(pinned); + pinned &= USID_EGRESS_MAX_SHARDS - 1; + + struct egress_shard_member *m = &g->shards[pinned]; + __u64 last = pin->last_seen_ns; + // Another CPU may have stamped the pin after now was read. + int fresh = last > now || now - last <= idle_ns; + + if ((m->flags & USID_EGRESS_SHARD_ALIVE) && m->generation != 0 && + m->generation == pin->generation && fresh) { + index = pinned; + placed = 1; + pin->last_seen_ns = now; + count_egress_shard_stat(EGRESS_SHARD_STAT_PIN_HIT); + } else { + count_egress_shard_stat(EGRESS_SHARD_STAT_PIN_STALE); + } + } + } + + if (!placed) { + __u32 slot = egress_shard_hash(&s->key) % USID_EGRESS_MAGLEV_SIZE; + + USID_BARRIER_VAR(slot); + if (slot >= USID_EGRESS_MAGLEV_SIZE) + slot = USID_EGRESS_MAGLEV_SIZE - 1; + + __u8 member = g->maglev[slot]; + + if (member == USID_EGRESS_SHARD_NONE) { + count_egress_shard_stat(EGRESS_SHARD_STAT_GROUP_EMPTY); + return -1; + } + // member is a plain byte read from a map value. The mask is the range + // check; the barrier keeps clang from dropping it as redundant. + index = member; + USID_BARRIER_VAR(index); + index &= USID_EGRESS_MAX_SHARDS - 1; + if (!(g->shards[index].flags & USID_EGRESS_SHARD_ALIVE)) { + count_egress_shard_stat(EGRESS_SHARD_STAT_SHARD_DEAD); + return -1; + } + count_egress_shard_stat(EGRESS_SHARD_STAT_MAGLEV); + + if (idle_ns) { + struct egress_shard_pin_value pin_value = { + .last_seen_ns = now, + .generation = g->shards[index].generation, + .shard = (__u8) index, + }; + // A failed insert costs nothing but the pin: the next packet asks + // the Maglev table again and almost always gets this answer. + bpf_map_update_elem(&egress_shard_pin, &s->key, &pin_value, 0 /* BPF_ANY */); + } + } + + USID_BARRIER_VAR(index); + index &= USID_EGRESS_MAX_SHARDS - 1; + + struct egress_shard_member *shard = &g->shards[index]; + + __builtin_memcpy(s->route.sid, shard->sid, sizeof(s->route.sid)); + s->route.sid[8] = (__u8) ((s->route.sid[8] & 0xF0) | ((function_argument >> 8) & 0x0F)); + s->route.sid[9] = (__u8) (function_argument & 0xFF); + s->route.link_ifindex = shard->link_ifindex; + __builtin_memcpy(s->route.dmac, shard->dmac, sizeof(s->route.dmac)); + __builtin_memcpy(s->route.smac, shard->smac, sizeof(s->route.smac)); + + __u32 counter_key = group_id * USID_EGRESS_MAX_SHARDS + index; + struct egress_shard_counter *counter = bpf_map_lookup_elem(&egress_shard_counters, &counter_key); + + if (counter) { + counter->packets += 1; + counter->bytes += skb->len; + } + return 0; +} + SEC("tc") int usid_egress(struct __sk_buff *skb) { @@ -3576,6 +4081,28 @@ int usid_egress(struct __sk_buff *skb) if (src_or == 0) return TC_ACT_UNSPEC; // this node's own source SID isn't registered yet -- fail open rather than encapsulate with an all-zero source + // A shard group sentinel: pick this packet's shard and carry on from the + // route resolve_egress_group built, exactly as from an ordinary entry. + // Placed after the fail-open checks so a packet that is not going to be + // encapsulated neither moves a pin nor counts toward a shard's share. + if (rv->link_ifindex == USID_EGRESS_GROUP_SENTINEL) { + __u32 group_id; + + __builtin_memcpy(&group_id, rv->sid, sizeof(group_id)); + __u32 function_argument = ((__u32) rv->sid[8] << 8) | rv->sid[9]; + + if (resolve_egress_group(skb, vrf->vrf_table_id, route_family, group_id, function_argument)) + return TC_ACT_SHOT; + + __u32 shard_scratch_key = 0; + struct egress_shard_scratch_value *shard_scratch = + bpf_map_lookup_elem(&egress_shard_scratch, &shard_scratch_key); + + if (!shard_scratch) + return TC_ACT_SHOT; + rv = &shard_scratch->route; + } + // Clamp the MSS of a SYN about to be encapsulated: past the last fail-open // check above, so only traffic that really crosses the fabric is clamped, // and before the header push below, which re-derives every packet pointer