From 719929f6d11cf09badfe5efe8f284e86fda488e4 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 09:15:16 +0000 Subject: [PATCH 01/78] tailcfg: add SCION service protocol and node attribute for SCION preference --- tailcfg/tailcfg.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index b49791be6fb39..e63d9d5e50a84 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -751,13 +751,14 @@ const ( PeerAPI4 = ServiceProto("peerapi4") PeerAPI6 = ServiceProto("peerapi6") PeerAPIDNS = ServiceProto("peerapi-dns-proxy") + SCION = ServiceProto("scion") ) // IsKnownServiceProto checks whether sp represents a known-valid value of // ServiceProto. func IsKnownServiceProto(sp ServiceProto) bool { switch sp { - case TCP, UDP, PeerAPI4, PeerAPI6, PeerAPIDNS, ServiceProto("egg"): + case TCP, UDP, PeerAPI4, PeerAPI6, PeerAPIDNS, SCION, ServiceProto("egg"): return true } return false @@ -2755,6 +2756,12 @@ const ( // See https://github.com/tailscale/tailscale/issues/15404. // TODO(bradfitz): remove this a few releases after 2026-02-16. NodeAttrForceRegisterMagicDNSIPv4Only NodeCapability = "force-register-magicdns-ipv4-only" + + // NodeAttrSCIONPrefer indicates that the node should prefer SCION paths + // when communicating with other SCION-capable peers that also have this + // attribute. Both self and peer must have this attribute for SCION to be + // preferred over direct UDP. + NodeAttrSCIONPrefer NodeCapability = "scion-prefer" ) // SetDNSRequest is a request to add a DNS record. From 36425f854f26b25b2f6fef72ec1eb42dc32faabe Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 09:15:46 +0000 Subject: [PATCH 02/78] wgengine/magicsock: enhance endpoint structure with SCION support --- wgengine/magicsock/endpoint.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 1f99f57ec2d16..9a46f7aff4236 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -99,6 +99,9 @@ type endpoint struct { expired bool // whether the node has expired isWireguardOnly bool // whether the endpoint is WireGuard only relayCapable bool // whether the node is capable of speaking via a [tailscale.com/net/udprelay.Server] + + scionState *scionEndpointState // nil if peer has no SCION address + scionPreferred bool // true if both self and peer have NodeAttrSCIONPrefer } // udpRelayEndpointReady determines whether the given relay [addrQuality] should @@ -1794,19 +1797,28 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd } // epAddr is a [netip.AddrPort] with an optional Geneve header (RFC8926) -// [packet.VirtualNetworkID]. +// [packet.VirtualNetworkID] or SCION path key. type epAddr struct { - ap netip.AddrPort // if ap == tailcfg.DerpMagicIPAddr then vni is never set - vni packet.VirtualNetworkID // vni.IsSet() indicates if this [epAddr] involves a Geneve header + ap netip.AddrPort // if ap == tailcfg.DerpMagicIPAddr then vni is never set + vni packet.VirtualNetworkID // vni.IsSet() indicates if this [epAddr] involves a Geneve header + scionKey scionPathKey // non-zero if this is a SCION endpoint } // isDirect returns true if e.ap is valid and not tailcfg.DerpMagicIPAddr, -// and a VNI is not set. +// and neither a VNI nor a SCION key is set. func (e epAddr) isDirect() bool { - return e.ap.IsValid() && e.ap.Addr() != tailcfg.DerpMagicIPAddr && !e.vni.IsSet() + return e.ap.IsValid() && e.ap.Addr() != tailcfg.DerpMagicIPAddr && !e.vni.IsSet() && !e.scionKey.IsSet() +} + +// isSCION reports whether this address represents a SCION path. +func (e epAddr) isSCION() bool { + return e.scionKey.IsSet() } func (e epAddr) String() string { + if e.scionKey.IsSet() { + return fmt.Sprintf("%v:scion:%d", e.ap.String(), e.scionKey) + } if !e.vni.IsSet() { return e.ap.String() } From beeed31865fce7773f31d29969ff492008933ad2 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 09:20:29 +0000 Subject: [PATCH 03/78] wgengine/magicsock: implement SCION connection support --- go.mod | 41 ++- go.sum | 117 ++++--- wgengine/magicsock/magicsock.go | 9 + wgengine/magicsock/magicsock_scion.go | 446 ++++++++++++++++++++++++++ 4 files changed, 559 insertions(+), 54 deletions(-) create mode 100644 wgengine/magicsock/magicsock_scion.go diff --git a/go.mod b/go.mod index caa58b60833bc..fe6fce7011d7d 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/bradfitz/go-tool-cache v0.0.0-20260216153636-9e5201344fe5 github.com/bradfitz/monogok v0.0.0-20260208031948-2219c393d032 github.com/bramvdbogaerde/go-scp v1.4.0 - github.com/cilium/ebpf v0.16.0 + github.com/cilium/ebpf v0.18.0 github.com/coder/websocket v1.8.12 github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf @@ -80,12 +80,13 @@ require ( github.com/peterbourgon/ff/v3 v3.4.0 github.com/pires/go-proxyproto v0.8.1 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.6 + github.com/pkg/sftp v1.13.7 github.com/prometheus-community/pro-bing v0.4.0 github.com/prometheus/client_golang v1.23.0 github.com/prometheus/common v0.65.0 github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff github.com/safchain/ethtool v0.3.0 + github.com/scionproto/scion v0.14.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/studio-b12/gowebdav v0.9.0 github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e @@ -169,6 +170,7 @@ require ( github.com/containerd/platforms v1.0.0-rc.2 // indirect github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/dchest/cmac v1.0.0 // indirect github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect github.com/docker/go-connections v0.5.0 // indirect @@ -193,10 +195,14 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/renameio/v2 v2.0.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/gopacket/gopacket v1.3.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-metrics v0.5.4 // indirect @@ -224,6 +230,9 @@ require ( github.com/moby/term v0.5.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect + github.com/olekukonko/ll v0.0.8 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect @@ -231,9 +240,13 @@ require ( github.com/rtr7/dhcp4 v0.0.0-20220302171438-18c84d089b46 // indirect github.com/rubenv/sql-migrate v1.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect github.com/stacklok/frizbee v0.1.7 // indirect + github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect + github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/vishvananda/netlink v1.3.1-0.20240922070040-084abd93d350 // indirect github.com/xen0n/gosmopolitan v1.2.2 // indirect github.com/xlab/treeprint v1.2.0 // indirect @@ -246,6 +259,7 @@ require ( go.opentelemetry.io/otel v1.39.0 // indirect go.opentelemetry.io/otel/metric v1.39.0 // indirect go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect @@ -368,7 +382,6 @@ require ( github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -395,17 +408,15 @@ require ( github.com/ldez/tagliatelle v0.5.0 // indirect github.com/leonklingele/grouper v1.1.1 // indirect github.com/lufeee/execinquery v1.2.1 // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/maratori/testableexamples v1.0.0 // indirect github.com/maratori/testpackage v1.1.1 // indirect github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/mdlayher/socket v0.5.0 + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mdlayher/socket v0.5.1 github.com/mgechev/revive v1.3.7 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect @@ -416,10 +427,10 @@ require ( github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.16.1 // indirect - github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/olekukonko/tablewriter v1.0.7 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e // indirect @@ -431,7 +442,7 @@ require ( github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/ryancurrah/gomodguard v1.3.1 // indirect github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect @@ -448,17 +459,16 @@ require ( github.com/skeema/knownhosts v1.3.1 // indirect github.com/sonatard/noctx v0.0.2 // indirect github.com/sourcegraph/go-diff v0.7.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect github.com/spf13/cobra v1.10.2 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.9 // indirect - github.com/spf13/viper v1.16.0 // indirect + github.com/spf13/viper v1.20.1 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/stretchr/testify v1.11.1 - github.com/subosito/gotenv v1.4.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c // indirect github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 github.com/tdakkota/asciicheck v0.2.0 // indirect @@ -486,7 +496,6 @@ require ( gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 1f8195e47fff6..7f44077484c8d 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,8 @@ github.com/Djarvur/go-err113 v0.1.0 h1:uCRZZOdMQ0TZPHYTdYpoC0bLYJKPEHPUJ8MeAa51l github.com/Djarvur/go-err113 v0.1.0/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0 h1:sATXp1x6/axKxz2Gjxv8MALP0bXaNRfQinEwyfMcx8c= github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0/go.mod h1:Nl76DrGNJTA1KJ0LePKBw/vznBX1EHbAZX8mwjR82nI= +github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Kodeworks/golang-image-ico v0.0.0-20141118225523-73f0f4cfade9 h1:1ltqoej5GtaWF8jaiA49HwsZD459jqm9YFz9ZtMFpQA= github.com/Kodeworks/golang-image-ico v0.0.0-20141118225523-73f0f4cfade9/go.mod h1:7uhhqiBaR4CpN0k9rMjOtjpcfGd6DG2m04zQxKnWQ0I= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -187,6 +189,7 @@ github.com/axiomhq/hyperloglog v0.0.0-20240319100328-84253e514e02/go.mod h1:k08r github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -246,8 +249,8 @@ github.com/chavacava/garif v0.1.0/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+U github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= -github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= +github.com/cilium/ebpf v0.18.0 h1:OsSwqS4y+gQHxaKgg2U/+Fev834kdnsQbtzRnbVC6Gs= +github.com/cilium/ebpf v0.18.0/go.mod h1:vmsAT73y4lW2b4peE+qcOqw6MxvWQdC+LiU5gd/xyo4= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/ckaznocha/intrange v0.1.0 h1:ZiGBhvrdsKpoEfzh9CjBfDSZof6QB0ORY5tXasUtiew= @@ -301,6 +304,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa h1:h8TfIT1xc8FWbwwpmHn1J5i43Y0uZP97GqasGCzSRJk= github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa/go.mod h1:Nx87SkVqTKd8UtT+xu7sM/l+LgXs6c0aHrlKusR+2EQ= +github.com/dchest/cmac v1.0.0 h1:Vaorm9FVpO2P+YmRdH0RVCUB1XF3Ge1yg9scPvJphyk= +github.com/dchest/cmac v1.0.0/go.mod h1:0zViPqHm8iZwwMl1cuK3HqK7Tu4Q7DV4EuMIOUwBVQ0= github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= github.com/deckarep/golang-set/v2 v2.8.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= @@ -436,8 +441,8 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= -github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -509,6 +514,8 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= +github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -606,6 +613,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopacket/gopacket v1.3.1 h1:ZppWyLrOJNZPe5XkdjLbtuTkfQoxQ0xyMJzQCqtqaPU= +github.com/gopacket/gopacket v1.3.1/go.mod h1:3I13qcqSpB2R9fFQg866OOgzylYkZxLTmkvcXhvf6qg= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gordonklaus/ineffassign v0.1.0 h1:y2Gd/9I7MdY1oEIt+n+rowjBNDcLQq3RsH5hwJd0f9s= @@ -642,9 +651,15 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -674,12 +689,10 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw= -github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU= +github.com/hashicorp/golang-lru/arc/v2 v2.0.7 h1:QxkVTxwColcduO+LP7eJO56r2hFiG8zEbfAAzRv52KQ= +github.com/hashicorp/golang-lru/arc/v2 v2.0.7/go.mod h1:Pe7gBlGdc8clY5LJ0LpJXMt5AmgmWNH1g+oFFVUHOEc= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/raft v1.7.2 h1:pyvxhfJ4R8VIAlHKvLoKQWElZspsCVT6YWuxVxsPAgc= github.com/hashicorp/raft v1.7.2/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ= github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKcZ+qp2VlaaZsYZfLOmIiuJNpEi48Rl8u9cQ= @@ -811,8 +824,6 @@ github.com/lufeee/execinquery v1.2.1 h1:hf0Ems4SHcUGBxpGN7Jz78z1ppVkP/837ZlETPCE github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= github.com/macabu/inamedparam v0.1.3 h1:2tk/phHkMlEL/1GNe/Yf6kkR/hkcUdAEY3L0hjYV1Mk= github.com/macabu/inamedparam v0.1.3/go.mod h1:93FLICAIk/quk7eaPPQvbzihUdn/QkGDwIZEoLtpH6I= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maratori/testableexamples v1.0.0 h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s93SLMxb2vI= @@ -832,9 +843,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -847,8 +857,8 @@ github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU github.com/mdlayher/raw v0.0.0-20190303161257-764d452d77af/go.mod h1:rC/yE65s/DoHB6BzVOUBNYBGTg772JVytyAytffIZkY= github.com/mdlayher/sdnotify v1.0.0 h1:Ma9XeLVN/l0qpyx1tNeMSeTjCPH6NtuD6/N9XdTlQ3c= github.com/mdlayher/sdnotify v1.0.0/go.mod h1:HQUmpM4XgYkhDLtd+Uad8ZFK1T9D5+pNxnXQjCeJlGE= -github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI= -github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI= +github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= +github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= github.com/mdlayher/watchdog v0.0.0-20221003142519-49be0df7b3b5 h1:80FAK3TW5lVymfHu3kvB1QvTZvy9Kmx1lx6sT5Ep16s= github.com/mdlayher/watchdog v0.0.0-20221003142519-49be0df7b3b5/go.mod h1:z0QjVpjpK4jksEkffQwS3+abQ3XFTm1bnimyDzWyUk0= github.com/mgechev/revive v1.3.7 h1:502QY0vQGe9KtYJ9FpxMz9rL+Fc/P13CI5POL4uHCcE= @@ -863,8 +873,6 @@ github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/buildkit v0.20.2 h1:qIeR47eQ1tzI1rwz0on3Xx2enRw/1CKjFhoONVcTlMA= @@ -911,8 +919,12 @@ github.com/nunnatsa/ginkgolinter v0.16.1 h1:uDIPSxgVHZ7PgbJElRDGzymkXH+JaF7mjew+ github.com/nunnatsa/ginkgolinter v0.16.1/go.mod h1:4tWRinDN1FeJgU+iJANW/kz7xKN5nYRAOfJDQUS9dOQ= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 h1:r3FaAI0NZK3hSmtTDrBVREhKULp8oUeqLT5Eyl2mSPo= +github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= +github.com/olekukonko/ll v0.0.8 h1:sbGZ1Fx4QxJXEqL/6IG8GEFnYojUSQ45dJVwN2FH2fc= +github.com/olekukonko/ll v0.0.8/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g= +github.com/olekukonko/tablewriter v1.0.7 h1:HCC2e3MM+2g72M81ZcJU11uciw6z/p82aEnm4/ySDGw= +github.com/olekukonko/tablewriter v1.0.7/go.mod h1:H428M+HzoUXC6JU2Abj9IT9ooRmdq9CxuDmKMtrOCMs= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= @@ -923,6 +935,9 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= @@ -934,8 +949,8 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= -github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= @@ -955,8 +970,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= -github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= +github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM= +github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -1019,8 +1034,8 @@ github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnA github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -1038,6 +1053,8 @@ github.com/ryanrolds/sqlclosecheck v0.5.1 h1:dibWW826u0P8jNLsLN+En7+RqWWTYrjCB9f github.com/ryanrolds/sqlclosecheck v0.5.1/go.mod h1:2g3dUjoS6AL4huFdv6wn55WpLIDjY7ZgUR4J8HOO/XQ= github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/sanposhiho/wastedassign/v2 v2.0.7 h1:J+6nrY4VW+gC9xFzUc+XjPD3g3wF3je/NsJFwFK7Uxc= github.com/sanposhiho/wastedassign/v2 v2.0.7/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= @@ -1048,6 +1065,8 @@ github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tM github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= github.com/sashamelentyev/usestdlibvars v1.25.0 h1:IK8SI2QyFzy/2OD2PYnhy84dpfNo9qADrRt6LH8vSzU= github.com/sashamelentyev/usestdlibvars v1.25.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7aRoS4j6EBye3YBhmAIRF8= +github.com/scionproto/scion v0.14.0 h1:aoSM4f/klmhO/RsXG2RJ7KbaNZ6cujxe9APfqFby0Lw= +github.com/scionproto/scion v0.14.0/go.mod h1:gCXIVztXV7HMe9P/ymVk4U4oSZOYaNnhkeskYxl2h60= github.com/securego/gosec/v2 v2.19.0 h1:gl5xMkOI0/E6Hxx0XCY2XujA3V7SNSefA8sC+3f1gnk= github.com/securego/gosec/v2 v2.19.0/go.mod h1:hOkDcHz9J/XIgIlPDXalxjeVYsHxoWUc5zJSHxcB8YM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1079,21 +1098,21 @@ github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= github.com/sonatard/noctx v0.0.2 h1:L7Dz4De2zDQhW8S0t+KUjY0MAQJd6SgVwhzNIc4ok00= github.com/sonatard/noctx v0.0.2/go.mod h1:kzFz+CzWSjQ2OzIm46uJZoXuBpa2+0y3T36U18dWqIo= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= -github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stacklok/frizbee v0.1.7 h1:IgrZy8dqKy+vBxNWrZTbDoctnV0doQKrFC6bNbWP5ho= @@ -1117,13 +1136,12 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/studio-b12/gowebdav v0.9.0 h1:1j1sc9gQnNxbXXM4M/CebPOX4aXYtr7MojAVcN4dHjU= github.com/studio-b12/gowebdav v0.9.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= -github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= -github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c h1:+aPplBwWcHBo6q9xrfWdMrT9o4kltkmmvpemgIjep/8= github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c/go.mod h1:SbErYREK7xXdsRiigaQiQkI9McGRzYMvlKYaP3Nimdk= github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e h1:PtWT87weP5LWHEY//SWsYkSO3RWRZo4OSWagh3YD2vQ= @@ -1187,6 +1205,10 @@ github.com/u-root/u-root v0.14.0 h1:Ka4T10EEML7dQ5XDvO9c3MBN8z4nuSnGjcd1jmU2ivg= github.com/u-root/u-root v0.14.0/go.mod h1:hAyZorapJe4qzbLWlAkmSVCJGbfoU9Pu4jpJ1WMluqE= github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM= github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA= +github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= +github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ultraware/funlen v0.1.0 h1:BuqclbkY6pO+cvxoq7OsktIXZpgBSkYTQtmwhAK81vI= @@ -1289,12 +1311,18 @@ go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6 go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= @@ -1315,6 +1343,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/crypto/x509roots/fallback v0.0.0-20260113154411-7d0074ccc6f1 h1:EBHQuS9qI8xJ96+YRgVV2ahFLUYbWpt1rf3wPfXN2wQ= @@ -1407,6 +1436,7 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1481,6 +1511,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1496,7 +1527,9 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 h1:E2/AqCUMZGgd73TQkxUMcMla25GB9i/5HOdLr+uH7Vo= @@ -1507,6 +1540,8 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1519,6 +1554,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1608,6 +1645,8 @@ golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1651,6 +1690,7 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= @@ -1677,6 +1717,8 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/grpc/examples v0.0.0-20240321213419-eb5828bae753 h1:crPucDOfTtZF6lBfOiv4ex+5g+TFoNjyiSrSDJUpYPc= +google.golang.org/grpc/examples v0.0.0-20240321213419-eb5828bae753/go.mod h1:fYxPglWChrD7bqbWtDwno019ra5SPuE1c3i+4YAvado= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1704,8 +1746,6 @@ gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -1722,6 +1762,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 169369f4bb472..097e493cc9f19 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -187,6 +187,9 @@ type Conn struct { pconn4 RebindingUDPConn pconn6 RebindingUDPConn + // pconnSCION is the SCION connection, nil if SCION is not available. + pconnSCION *scionConn + receiveBatchPool sync.Pool // closeDisco4 and closeDisco6 are io.Closers to shut down the raw @@ -408,6 +411,12 @@ type Conn struct { // homeDERPGauge is the usermetric gauge for the home DERP region ID. // This can be nil when [Options.Metrics] are not enabled. homeDERPGauge *usermetric.Gauge + + // scionPaths is the registry of SCION path information, keyed by + // scionPathKey. Each entry holds the full SCION address and path + // data for a peer. + scionPaths map[scionPathKey]*scionPathInfo + scionPathSeq atomic.Uint32 // monotonic key generator for scionPaths } // SetDebugLoggingEnabled controls whether spammy debug logging is enabled. diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go new file mode 100644 index 0000000000000..22ec70231ee5e --- /dev/null +++ b/wgengine/magicsock/magicsock_scion.go @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: BSD-3-Clause + +package magicsock + +import ( + "context" + "fmt" + "net" + "net/netip" + "os" + "strings" + "sync" + "time" + + "github.com/scionproto/scion/pkg/addr" + "github.com/scionproto/scion/pkg/daemon" + "github.com/scionproto/scion/pkg/snet" + wgconn "github.com/tailscale/wireguard-go/conn" + "tailscale.com/tailcfg" + "tailscale.com/tstime/mono" + "tailscale.com/types/key" +) + +// scionPathKey is a compact index into the Conn-level scionPaths registry. +// This keeps epAddr small and comparable (snet.UDPAddr contains slices). +// A zero value means "not a SCION path." +type scionPathKey uint32 + +// IsSet reports whether k refers to a valid SCION path entry. +func (k scionPathKey) IsSet() bool { return k != 0 } + +// scionPathInfo holds the full SCION path information for a peer, indexed by +// scionPathKey. The actual SCION address and path data live here rather than +// in epAddr to keep epAddr comparable and small. +type scionPathInfo struct { + peerIA addr.IA + hostAddr netip.AddrPort // peer's SCION host IP:port + path snet.Path // current best SCION path to this peer + expiry time.Time // path expiration from path metadata + mu sync.Mutex +} + +// scionEndpointState tracks SCION-specific per-peer state on an endpoint. +type scionEndpointState struct { + peerIA addr.IA // peer's ISD-AS from Services advertisement + hostAddr netip.AddrPort // peer's SCION host IP:port + pathKey scionPathKey // key into Conn.scionPaths +} + +// scionConn wraps a SCION connection for use by magicsock. +type scionConn struct { + conn *snet.Conn // from SCIONNetwork.Listen() + localIA addr.IA // our ISD-AS + daemon daemon.Connector // for path queries + topo snet.Topology // local topology +} + +// close shuts down the SCION connection and daemon connector. +func (sc *scionConn) close() error { + if sc.conn != nil { + sc.conn.Close() + } + if sc.daemon != nil { + sc.daemon.Close() + } + return nil +} + +// writeTo sends b to a peer identified by the given scionPathInfo. +func (sc *scionConn) writeTo(b []byte, pi *scionPathInfo) (int, error) { + pi.mu.Lock() + path := pi.path + hostAddr := pi.hostAddr + peerIA := pi.peerIA + pi.mu.Unlock() + + dst := &snet.UDPAddr{ + IA: peerIA, + Host: &net.UDPAddr{ + IP: hostAddr.Addr().AsSlice(), + Port: int(hostAddr.Port()), + }, + } + if path != nil { + dst.Path = path.Dataplane() + dst.NextHop = path.UnderlayNextHop() + } + + return sc.conn.WriteTo(b, dst) +} + +// readFrom reads a packet from the SCION connection, returning the data, the +// source SCION address, and any error. +func (sc *scionConn) readFrom(b []byte) (int, *snet.UDPAddr, error) { + n, srcAddr, err := sc.conn.ReadFrom(b) + if err != nil { + return 0, nil, err + } + src, ok := srcAddr.(*snet.UDPAddr) + if !ok { + return 0, nil, fmt.Errorf("unexpected source address type: %T", srcAddr) + } + return n, src, nil +} + +// scionDaemonAddr returns the SCION daemon address to use, checking the +// environment variable first, then falling back to the default socket. +func scionDaemonAddr() string { + if a := os.Getenv("SCION_DAEMON_ADDRESS"); a != "" { + return a + } + return daemon.DefaultAPIAddress +} + +// trySCIONConnect attempts to connect to the local SCION daemon and set up a +// SCION listener. Returns nil if SCION is not available. +func trySCIONConnect(ctx context.Context, localPort uint16) (*scionConn, error) { + daemonAddr := scionDaemonAddr() + svc := daemon.Service{Address: daemonAddr} + conn, err := svc.Connect(ctx) + if err != nil { + return nil, fmt.Errorf("connecting to SCION daemon at %s: %w", daemonAddr, err) + } + + topo, err := daemon.LoadTopology(ctx, conn) + if err != nil { + conn.Close() + return nil, fmt.Errorf("loading SCION topology: %w", err) + } + + network := &snet.SCIONNetwork{ + Topology: topo, + } + + listenAddr := &net.UDPAddr{ + IP: net.IPv4zero, + Port: int(localPort), + } + sconn, err := network.Listen(ctx, "udp", listenAddr) + if err != nil { + conn.Close() + return nil, fmt.Errorf("listening on SCION: %w", err) + } + + return &scionConn{ + conn: sconn, + localIA: topo.LocalIA, + daemon: conn, + topo: topo, + }, nil +} + +// parseSCIONServiceAddr parses a SCION service description string of the form +// "ISD-AS,host-IP" and returns the IA and host address. The port comes from the +// Service.Port field. +func parseSCIONServiceAddr(description string, port uint16) (ia addr.IA, hostAddr netip.AddrPort, err error) { + parts := strings.SplitN(description, ",", 2) + if len(parts) != 2 { + return 0, netip.AddrPort{}, fmt.Errorf("invalid SCION service description %q: want ISD-AS,host-IP", description) + } + + ia, err = addr.ParseIA(parts[0]) + if err != nil { + return 0, netip.AddrPort{}, fmt.Errorf("parsing SCION IA %q: %w", parts[0], err) + } + + hostIP, err := netip.ParseAddr(parts[1]) + if err != nil { + return 0, netip.AddrPort{}, fmt.Errorf("parsing SCION host IP %q: %w", parts[1], err) + } + + return ia, netip.AddrPortFrom(hostIP, port), nil +} + +// sendSCIONBatch sends a batch of WireGuard packets over the SCION connection. +// It looks up the full path info from the Conn's scionPaths registry using the +// scionPathKey from the epAddr. +func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent bool, err error) { + sc := c.pconnSCION + if sc == nil { + return false, errNoSCION + } + + pi := c.lookupSCIONPath(addr.scionKey) + if pi == nil { + return false, fmt.Errorf("no SCION path info for key %d", addr.scionKey) + } + + for _, buf := range buffs { + _, err = sc.writeTo(buf[offset:], pi) + if err != nil { + return false, err + } + } + return true, nil +} + +// sendSCION sends a single packet over SCION, used for disco messages. +func (c *Conn) sendSCION(sk scionPathKey, b []byte) (bool, error) { + sc := c.pconnSCION + if sc == nil { + return false, errNoSCION + } + pi := c.lookupSCIONPath(sk) + if pi == nil { + return false, fmt.Errorf("no SCION path info for key %d", sk) + } + _, err := sc.writeTo(b, pi) + if err != nil { + return false, err + } + return true, nil +} + +// lookupSCIONPath returns the scionPathInfo for the given key, or nil if not found. +func (c *Conn) lookupSCIONPath(k scionPathKey) *scionPathInfo { + c.mu.Lock() + defer c.mu.Unlock() + return c.scionPaths[k] +} + +// registerSCIONPath stores a scionPathInfo and returns a key for it. +func (c *Conn) registerSCIONPath(pi *scionPathInfo) scionPathKey { + k := scionPathKey(c.scionPathSeq.Add(1)) + c.mu.Lock() + defer c.mu.Unlock() + if c.scionPaths == nil { + c.scionPaths = make(map[scionPathKey]*scionPathInfo) + } + c.scionPaths[k] = pi + return k +} + +// unregisterSCIONPath removes a SCION path entry. +func (c *Conn) unregisterSCIONPath(k scionPathKey) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.scionPaths, k) +} + +// receiveSCION is the conn.ReceiveFunc for SCION packets. It reads from the +// SCION connection and dispatches disco or WireGuard packets. +func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { + sc := c.pconnSCION + if sc == nil { + // Block until the Conn is closed if SCION is not available. + <-c.donec + return 0, net.ErrClosed + } + + for { + n, srcAddr, err := sc.readFrom(buffs[0]) + if err != nil { + return 0, err + } + if n == 0 { + continue + } + + b := buffs[0][:n] + + // Build an epAddr for this SCION source. We use the host IP:port + // from the SCION address. The scionKey on the epAddr is not set + // here since we're on the receive path — the peerMap lookup uses + // the host addr portion. + srcHostAddr := srcAddr.Host.AddrPort() + srcEpAddr := epAddr{ap: srcHostAddr} + + // Check for disco packets (same as receiveIP does). + pt, _ := packetLooksLike(b) + if pt == packetLooksLikeDisco { + c.handleDiscoMessage(b, srcEpAddr, false, key.NodePublic{}, discoRXPathSCION) + continue + } + + // WireGuard packet — look up the endpoint by source address. + c.mu.Lock() + ep, ok := c.peerMap.endpointForEpAddr(srcEpAddr) + c.mu.Unlock() + if !ok { + // Try looking up without SCION key since the receive side + // may not have the scionKey set in peerMap. + continue + } + + ep.noteRecvActivity(srcEpAddr, mono.Now()) + sizes[0] = n + eps[0] = ep + return 1, nil + } +} + +// discoverSCIONPaths queries the SCION daemon for paths to the given peer IA, +// selects the best one, and stores it in the path registry. Returns the +// scionPathKey for the path. +func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr netip.AddrPort) (scionPathKey, error) { + sc := c.pconnSCION + if sc == nil { + return 0, errNoSCION + } + + paths, err := sc.daemon.Paths(ctx, peerIA, sc.localIA, daemon.PathReqFlags{Refresh: false}) + if err != nil { + return 0, fmt.Errorf("querying SCION paths to %s: %w", peerIA, err) + } + if len(paths) == 0 { + return 0, fmt.Errorf("no SCION paths to %s", peerIA) + } + + // Pick the path with lowest total latency. + best := paths[0] + bestLatency := totalPathLatency(best) + for _, p := range paths[1:] { + lat := totalPathLatency(p) + if lat < bestLatency { + best = p + bestLatency = lat + } + } + + var expiry time.Time + if md := best.Metadata(); md != nil { + expiry = md.Expiry + } + + pi := &scionPathInfo{ + peerIA: peerIA, + hostAddr: hostAddr, + path: best, + expiry: expiry, + } + return c.registerSCIONPath(pi), nil +} + +// totalPathLatency returns the sum of all hop latencies for a SCION path. +// Returns a large value if latency information is unavailable. +func totalPathLatency(p snet.Path) time.Duration { + md := p.Metadata() + if md == nil || len(md.Latency) == 0 { + return time.Hour // large sentinel for unknown latency + } + var total time.Duration + for _, l := range md.Latency { + if l < 0 { + // LatencyUnset — treat as unknown + total += 10 * time.Millisecond + } else { + total += l + } + } + return total +} + +// refreshSCIONPaths runs in a background goroutine, periodically refreshing +// SCION paths before they expire. +func (c *Conn) refreshSCIONPaths() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-c.donec: + return + case <-ticker.C: + c.refreshSCIONPathsOnce() + } + } +} + +func (c *Conn) refreshSCIONPathsOnce() { + sc := c.pconnSCION + if sc == nil { + return + } + + c.mu.Lock() + // Snapshot the current paths under lock. + pathsCopy := make(map[scionPathKey]*scionPathInfo, len(c.scionPaths)) + for k, v := range c.scionPaths { + pathsCopy[k] = v + } + c.mu.Unlock() + + ctx, cancel := context.WithTimeout(c.connCtx, 10*time.Second) + defer cancel() + + now := time.Now() + for _, pi := range pathsCopy { + pi.mu.Lock() + needsRefresh := !pi.expiry.IsZero() && now.After(pi.expiry.Add(-1*time.Minute)) + peerIA := pi.peerIA + pi.mu.Unlock() + + if !needsRefresh { + continue + } + + paths, err := sc.daemon.Paths(ctx, peerIA, sc.localIA, daemon.PathReqFlags{Refresh: true}) + if err != nil || len(paths) == 0 { + c.logf("magicsock: SCION path refresh for %s failed: %v", peerIA, err) + continue + } + + best := paths[0] + bestLatency := totalPathLatency(best) + for _, p := range paths[1:] { + lat := totalPathLatency(p) + if lat < bestLatency { + best = p + bestLatency = lat + } + } + + pi.mu.Lock() + pi.path = best + if md := best.Metadata(); md != nil { + pi.expiry = md.Expiry + } + pi.mu.Unlock() + } +} + +// scionServiceFromPeer extracts SCION service info from a peer node's Services. +func scionServiceFromPeer(n tailcfg.NodeView) (ia addr.IA, hostAddr netip.AddrPort, ok bool) { + hi := n.Hostinfo() + if !hi.Valid() { + return 0, netip.AddrPort{}, false + } + services := hi.Services() + for i := range services.Len() { + svc := services.At(i) + if svc.Proto != tailcfg.SCION { + continue + } + parsedIA, parsedAddr, err := parseSCIONServiceAddr(svc.Description, svc.Port) + if err != nil { + continue + } + return parsedIA, parsedAddr, true + } + return 0, netip.AddrPort{}, false +} + +var errNoSCION = fmt.Errorf("SCION not available") + +const discoRXPathSCION discoRXPath = "SCION" From 3d7692ca11aef9473a0b5e9e9c9946c14b8a7e49 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 09:43:53 +0000 Subject: [PATCH 04/78] wgengine/magicsock: enhance SCION support with improved endpoint handling and address quality evaluation --- wgengine/magicsock/endpoint.go | 35 ++++++++++++++++++++++++++++++++- wgengine/magicsock/magicsock.go | 19 ++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 9a46f7aff4236..0c7ffb2d16144 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1074,7 +1074,12 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { } } var err error - if udpAddr.ap.IsValid() { + if udpAddr.isSCION() { + _, err = de.c.sendSCIONBatch(udpAddr, buffs, offset) + if err != nil { + de.noteBadEndpoint(udpAddr) + } + } else if udpAddr.ap.IsValid() { _, err = de.c.sendUDPBatch(udpAddr, buffs, offset) // If the error is known to indicate that the endpoint is no longer @@ -1832,6 +1837,7 @@ type addrQuality struct { relayServerDisco key.DiscoPublic // only relevant if epAddr.vni.isSet(), otherwise zero value latency time.Duration wireMTU tstun.WireMTU + scionPreferred bool // true if both self and peer have NodeAttrSCIONPrefer } func (a addrQuality) String() string { @@ -1868,6 +1874,23 @@ func betterAddr(a, b addrQuality) bool { return false } + // SCION paths are preferred over relay but not over direct UDP. + // Direct UDP > SCION > UDP Relay > DERP + if a.scionKey.IsSet() != b.scionKey.IsSet() { + if a.scionKey.IsSet() { + // a is SCION + if b.isDirect() { + return false // direct wins over SCION + } + return true // SCION wins over relay/DERP + } + // b is SCION + if a.isDirect() { + return true // direct wins over SCION + } + return false // SCION wins over relay/DERP + } + // Each address starts with a set of points (from 0 to 100) that // represents how much faster they are than the highest-latency // endpoint. For example, if a has latency 200ms and b has latency @@ -1912,6 +1935,16 @@ func betterAddr(a, b addrQuality) bool { bPoints += 10 } + // When both self and peer have NodeAttrSCIONPrefer, give SCION paths + // a large bonus so they're preferred over direct UDP unless direct UDP + // is dramatically faster. + if a.scionPreferred && a.scionKey.IsSet() { + aPoints += 40 + } + if b.scionPreferred && b.scionKey.IsSet() { + bPoints += 40 + } + // Don't change anything if the latency improvement is less than 1%; we // want a bit of "stickiness" (a.k.a. hysteresis) to avoid flapping if // there's two roughly-equivalent endpoints. diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 097e493cc9f19..f223cad3f6167 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3271,6 +3271,9 @@ func (c *connBind) Open(ignoredPort uint16) ([]conn.ReceiveFunc, uint16, error) if runtime.GOOS == "js" { fns = []conn.ReceiveFunc{c.receiveDERP} } + if c.pconnSCION != nil { + fns = append(fns, c.receiveSCION) + } // TODO: Combine receiveIPv4 and receiveIPv6 and receiveIP into a single // closure that closes over a *RebindingUDPConn? return fns, c.LocalPort(), nil @@ -3353,6 +3356,9 @@ func (c *Conn) Close() error { // They will frequently have been closed already by a call to connBind.Close. c.pconn6.Close() c.pconn4.Close() + if c.pconnSCION != nil { + c.pconnSCION.close() + } if c.closeDisco4 != nil { c.closeDisco4.Close() } @@ -3600,6 +3606,19 @@ func (c *Conn) rebind(curPortFate currentPortFate) error { c.portMapper.SetLocalPort(c.LocalPort()) } c.UpdatePMTUD() + + // Try to set up SCION if not already connected. + if c.pconnSCION == nil { + sc, err := trySCIONConnect(c.connCtx, c.LocalPort()) + if err != nil { + c.logf("magicsock: SCION not available: %v", err) + } else { + c.logf("magicsock: SCION available, local IA: %s", sc.localIA) + c.pconnSCION = sc + go c.refreshSCIONPaths() + } + } + return nil } From 859022e213a0c225d95035926a57566581166d61 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 10:40:24 +0000 Subject: [PATCH 05/78] wgengine/magicsock: integrate SCION service advertisement and enhance endpoint discovery --- ipn/ipnlocal/local.go | 6 ++++ wgengine/magicsock/endpoint.go | 50 ++++++++++++++++++++++++--- wgengine/magicsock/magicsock.go | 6 +++- wgengine/magicsock/magicsock_scion.go | 20 +++++++++++ 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index bae1e66393a4b..950c16ebfd4af 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4924,6 +4924,12 @@ func (b *LocalBackend) hostInfoWithServicesLocked() *tailcfg.Hostinfo { // the slice with no free capacity. c := len(hi.Services) hi.Services = append(hi.Services[:c:c], peerAPIServices...) + + // Advertise SCION service if available. + if scionSvc, ok := b.MagicConn().SCIONService(); ok { + hi.Services = append(hi.Services, scionSvc) + } + hi.PushDeviceToken = b.pushDeviceToken.Load() // Compare the expected ports from peerAPIServices to the actual ports in hi.Services. diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 0c7ffb2d16144..6c8e5778b6d33 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1295,7 +1295,7 @@ func (de *endpoint) startDiscoPingLocked(ep epAddr, now mono.Time, purpose disco if runtime.GOOS == "js" { return } - if debugNeverDirectUDP() && !ep.vni.IsSet() && ep.ap.Addr() != tailcfg.DerpMagicIPAddr { + if debugNeverDirectUDP() && !ep.vni.IsSet() && !ep.scionKey.IsSet() && ep.ap.Addr() != tailcfg.DerpMagicIPAddr { return } epDisco := de.disco.Load() @@ -1303,7 +1303,7 @@ func (de *endpoint) startDiscoPingLocked(ep epAddr, now mono.Time, purpose disco return } if purpose != pingCLI && - !ep.vni.IsSet() { // de.endpointState is only relevant for direct/non-vni epAddr's + !ep.vni.IsSet() && !ep.scionKey.IsSet() { // de.endpointState is only relevant for direct/non-vni/non-SCION epAddr's st, ok := de.endpointState[ep.ap] if !ok { // Shouldn't happen. But don't ping an endpoint that's @@ -1382,6 +1382,16 @@ func (de *endpoint) sendDiscoPingsLocked(now mono.Time, sendCallMeMaybe bool) { de.startDiscoPingLocked(epAddr{ap: ep}, now, pingDiscovery, 0, nil) } + // Also ping over SCION if available for this peer. + if de.scionState != nil && de.c.pconnSCION != nil { + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: de.scionState.pathKey, + } + de.startDiscoPingLocked(scionEp, now, pingDiscovery, 0, nil) + sentAny = true + } + derpAddr := de.derpAddr if sentAny && sendCallMeMaybe && derpAddr.IsValid() { // Have our magicsock.Conn figure out its STUN endpoint (if @@ -1533,6 +1543,35 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p de.setEndpointsLocked(n.Endpoints()) de.relayCapable = capVerIsRelayCapable(n.Cap()) + + // Check for SCION service advertisement from this peer. + if peerIA, hostAddr, ok := scionServiceFromPeer(n); ok { + if de.scionState == nil || de.scionState.peerIA != peerIA || de.scionState.hostAddr != hostAddr { + // New or changed SCION address — discover paths. + if de.c.pconnSCION != nil { + pathKey, err := de.c.discoverSCIONPaths(de.c.connCtx, peerIA, hostAddr) + if err != nil { + de.c.logf("magicsock: SCION path discovery for %s failed: %v", peerIA, err) + } else { + de.scionState = &scionEndpointState{ + peerIA: peerIA, + hostAddr: hostAddr, + pathKey: pathKey, + } + de.c.logf("[v1] magicsock: discovered SCION path to %s (key=%d)", peerIA, pathKey) + } + } + } + } else if de.scionState != nil { + // Peer no longer advertises SCION. + de.c.unregisterSCIONPath(de.scionState.pathKey) + de.scionState = nil + } + + // Check if SCION should be preferred for this peer. + peerSCIONPrefer := n.CapMap().Contains(tailcfg.NodeAttrSCIONPrefer) + selfSCIONPrefer := de.c.self.Valid() && de.c.self.CapMap().Contains(tailcfg.NodeAttrSCIONPrefer) + de.scionPreferred = peerSCIONPrefer && selfSCIONPrefer && de.scionState != nil } func (de *endpoint) setEndpointsLocked(eps interface { @@ -1767,9 +1806,10 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd // TODO(bradfitz): decide how latency vs. preference order affects decision if !isDerp { thisPong := addrQuality{ - epAddr: sp.to, - latency: latency, - wireMTU: pingSizeToPktLen(sp.size, sp.to), + epAddr: sp.to, + latency: latency, + wireMTU: pingSizeToPktLen(sp.size, sp.to), + scionPreferred: de.scionPreferred, } // TODO(jwhited): consider checking de.trustBestAddrUntil as well. If // de.bestAddr is untrusted we may want to clear it, otherwise we could diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index f223cad3f6167..4c2a3bc6496f5 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -1971,7 +1971,11 @@ func (c *Conn) sendDiscoMessage(dst epAddr, dstKey key.NodePublic, dstDisco key. box := di.sharedKey.Seal(m.AppendMarshal(nil)) pkt = append(pkt, box...) const isDisco = true - sent, err = c.sendAddr(dst.ap, dstKey, pkt, isDisco, dst.vni.IsSet()) + if dst.scionKey.IsSet() { + sent, err = c.sendSCION(dst.scionKey, pkt) + } else { + sent, err = c.sendAddr(dst.ap, dstKey, pkt, isDisco, dst.vni.IsSet()) + } if sent { if logLevel == discoLog || (logLevel == discoVerboseLog && debugDisco()) { node := "?" diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 22ec70231ee5e..1cb6aabcc707a 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -441,6 +441,26 @@ func scionServiceFromPeer(n tailcfg.NodeView) (ia addr.IA, hostAddr netip.AddrPo return 0, netip.AddrPort{}, false } +// SCIONService returns the SCION service entry to advertise in Hostinfo, +// or ok=false if SCION is not available. +func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { + sc := c.pconnSCION + if sc == nil { + return tailcfg.Service{}, false + } + // The local host IP comes from the SCION connection's local address. + localAddr := sc.conn.LocalAddr() + hostIP := "0.0.0.0" + if ua, uaOk := localAddr.(*net.UDPAddr); uaOk && ua.IP != nil { + hostIP = ua.IP.String() + } + return tailcfg.Service{ + Proto: tailcfg.SCION, + Port: c.LocalPort(), + Description: fmt.Sprintf("%s,%s", sc.localIA, hostIP), + }, true +} + var errNoSCION = fmt.Errorf("SCION not available") const discoRXPathSCION discoRXPath = "SCION" From 3bf65012906779c8b792fdac711fc317b7d233e2 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 11:04:03 +0000 Subject: [PATCH 06/78] wgengine/magicsock: refine address quality evaluation for SCION paths with preference handling --- wgengine/magicsock/endpoint.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 6c8e5778b6d33..48f7020e792df 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1914,9 +1914,10 @@ func betterAddr(a, b addrQuality) bool { return false } - // SCION paths are preferred over relay but not over direct UDP. - // Direct UDP > SCION > UDP Relay > DERP - if a.scionKey.IsSet() != b.scionKey.IsSet() { + // SCION paths are preferred over relay but not over direct UDP, + // unless scionPreferred is set (both peers have NodeAttrSCIONPrefer), + // in which case the points system with the +40 bonus handles it. + if a.scionKey.IsSet() != b.scionKey.IsSet() && !a.scionPreferred && !b.scionPreferred { if a.scionKey.IsSet() { // a is SCION if b.isDirect() { From ac3ac6d01a751d6f08025b372df02c08433fd026 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 11:09:45 +0000 Subject: [PATCH 07/78] wgengine/magicsock: add SCION path tests and enhance address handling with scion mock dependencies --- go.mod | 1 + go.sum | 1 + wgengine/magicsock/magicsock_scion_test.go | 915 +++++++++++++++++++++ 3 files changed, 917 insertions(+) create mode 100644 wgengine/magicsock/magicsock_scion_test.go diff --git a/go.mod b/go.mod index fe6fce7011d7d..d1c329cd92070 100644 --- a/go.mod +++ b/go.mod @@ -189,6 +189,7 @@ require ( github.com/gokrazy/gokapi v0.0.0-20250222071133-506fdb322775 // indirect github.com/gokrazy/internal v0.0.0-20251208203110-3c1aa9087c82 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/golang/mock v1.7.0-rc.1 // indirect github.com/golangci/plugin-module-register v0.1.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-github/v66 v66.0.0 // indirect diff --git a/go.sum b/go.sum index 7f44077484c8d..5f6b993727d08 100644 --- a/go.sum +++ b/go.sum @@ -1619,6 +1619,7 @@ golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0t golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go new file mode 100644 index 0000000000000..6b67d5ce1346a --- /dev/null +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -0,0 +1,915 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package magicsock + +import ( + "context" + "fmt" + "net" + "net/netip" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/scionproto/scion/pkg/addr" + "github.com/scionproto/scion/pkg/daemon" + "github.com/scionproto/scion/pkg/daemon/mock_daemon" + "github.com/scionproto/scion/pkg/snet" + "github.com/scionproto/scion/pkg/snet/mock_snet" + "tailscale.com/net/packet" + "tailscale.com/net/tstun" + "tailscale.com/syncs" + "tailscale.com/tailcfg" + "tailscale.com/types/key" +) + +func TestScionPathKeyIsSet(t *testing.T) { + var zero scionPathKey + if zero.IsSet() { + t.Error("zero scionPathKey should not be set") + } + k := scionPathKey(1) + if !k.IsSet() { + t.Error("non-zero scionPathKey should be set") + } + k = scionPathKey(42) + if !k.IsSet() { + t.Error("scionPathKey(42) should be set") + } +} + +func TestEpAddrIsSCION(t *testing.T) { + tests := []struct { + name string + addr epAddr + isSCION bool + isDirect bool + }{ + { + name: "plain UDP", + addr: epAddr{ap: netip.MustParseAddrPort("192.0.2.1:7")}, + isSCION: false, + isDirect: true, + }, + { + name: "with VNI", + addr: func() epAddr { + e := epAddr{ap: netip.MustParseAddrPort("192.0.2.1:7")} + e.vni.Set(7) + return e + }(), + isSCION: false, + isDirect: false, + }, + { + name: "with scionKey", + addr: epAddr{ap: netip.MustParseAddrPort("192.0.2.1:7"), scionKey: 1}, + isSCION: true, + isDirect: false, + }, + { + name: "DERP magic addr", + addr: epAddr{ap: netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, 1)}, + isSCION: false, + isDirect: false, + }, + { + name: "zero epAddr", + addr: epAddr{}, + isSCION: false, + isDirect: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.addr.isSCION(); got != tt.isSCION { + t.Errorf("isSCION() = %v, want %v", got, tt.isSCION) + } + if got := tt.addr.isDirect(); got != tt.isDirect { + t.Errorf("isDirect() = %v, want %v", got, tt.isDirect) + } + }) + } +} + +func TestEpAddrStringWithSCION(t *testing.T) { + e := epAddr{ap: netip.MustParseAddrPort("10.0.0.1:41641"), scionKey: 5} + got := e.String() + want := "10.0.0.1:41641:scion:5" + if got != want { + t.Errorf("String() = %q, want %q", got, want) + } + + // Non-SCION should not include scion label. + e2 := epAddr{ap: netip.MustParseAddrPort("10.0.0.1:41641")} + got2 := e2.String() + want2 := "10.0.0.1:41641" + if got2 != want2 { + t.Errorf("String() = %q, want %q", got2, want2) + } +} + +func TestParseSCIONServiceAddr(t *testing.T) { + tests := []struct { + name string + description string + port uint16 + wantIA addr.IA + wantAddr netip.AddrPort + wantErr bool + }{ + { + name: "valid IPv4", + description: "1-ff00:0:110,192.0.2.1", + port: 41641, + wantIA: addr.MustParseIA("1-ff00:0:110"), + wantAddr: netip.MustParseAddrPort("192.0.2.1:41641"), + }, + { + name: "valid IPv6", + description: "1-ff00:0:110,2001:db8::1", + port: 12345, + wantIA: addr.MustParseIA("1-ff00:0:110"), + wantAddr: netip.MustParseAddrPort("[2001:db8::1]:12345"), + }, + { + name: "missing comma", + description: "1-ff00:0:110", + port: 41641, + wantErr: true, + }, + { + name: "bad IA", + description: "not-an-ia,192.0.2.1", + port: 41641, + wantErr: true, + }, + { + name: "bad IP", + description: "1-ff00:0:110,not-an-ip", + port: 41641, + wantErr: true, + }, + { + name: "empty string", + description: "", + port: 41641, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ia, hostAddr, err := parseSCIONServiceAddr(tt.description, tt.port) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got ia=%v hostAddr=%v", ia, hostAddr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ia != tt.wantIA { + t.Errorf("IA = %v, want %v", ia, tt.wantIA) + } + if hostAddr != tt.wantAddr { + t.Errorf("hostAddr = %v, want %v", hostAddr, tt.wantAddr) + } + }) + } +} + +func TestSCIONPathRegistry(t *testing.T) { + c := &Conn{} + c.mu = syncs.Mutex{} + + // Register a path. + pi := &scionPathInfo{ + peerIA: addr.MustParseIA("1-ff00:0:111"), + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + } + k := c.registerSCIONPath(pi) + if !k.IsSet() { + t.Fatal("registered key should be set") + } + + // Look it up. + got := c.lookupSCIONPath(k) + if got != pi { + t.Fatalf("lookupSCIONPath(%d) returned wrong path info", k) + } + + // Register another. + pi2 := &scionPathInfo{ + peerIA: addr.MustParseIA("1-ff00:0:112"), + hostAddr: netip.MustParseAddrPort("10.0.0.2:41641"), + } + k2 := c.registerSCIONPath(pi2) + if k2 == k { + t.Fatal("second key should differ from first") + } + if c.lookupSCIONPath(k2) != pi2 { + t.Fatal("second path not found") + } + + // Unregister the first. + c.unregisterSCIONPath(k) + if c.lookupSCIONPath(k) != nil { + t.Fatal("unregistered path should return nil") + } + // Second should still be there. + if c.lookupSCIONPath(k2) != pi2 { + t.Fatal("second path should still be present after unregistering first") + } + + // Lookup of non-existent key. + if c.lookupSCIONPath(scionPathKey(9999)) != nil { + t.Fatal("non-existent key should return nil") + } +} + +func TestBetterAddrSCION(t *testing.T) { + const ms = time.Millisecond + + al := func(ipps string, d time.Duration) addrQuality { + return addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort(ipps)}, latency: d} + } + alSCION := func(ipps string, sk scionPathKey, d time.Duration) addrQuality { + return addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort(ipps), scionKey: sk}, + latency: d, + } + } + alSCIONPref := func(ipps string, sk scionPathKey, d time.Duration) addrQuality { + return addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort(ipps), scionKey: sk}, + latency: d, + scionPreferred: true, + } + } + avl := func(ipps string, vni uint32, d time.Duration) addrQuality { + q := al(ipps, d) + q.vni.Set(vni) + return q + } + + const ( + publicV4 = "1.2.3.4:555" + publicV4_2 = "5.6.7.8:999" + ) + + tests := []struct { + name string + a, b addrQuality + want bool + }{ + // Direct UDP wins over SCION at equal latency. + { + name: "direct beats SCION same latency", + a: al(publicV4, 100*ms), + b: alSCION(publicV4_2, 1, 100*ms), + want: true, + }, + { + name: "SCION loses to direct same latency", + a: alSCION(publicV4_2, 1, 100*ms), + b: al(publicV4, 100*ms), + want: false, + }, + // SCION wins over relay (VNI). + { + name: "SCION beats relay same latency", + a: alSCION(publicV4, 1, 100*ms), + b: avl(publicV4_2, 1, 100*ms), + want: true, + }, + { + name: "relay loses to SCION same latency", + a: avl(publicV4_2, 1, 100*ms), + b: alSCION(publicV4, 1, 100*ms), + want: false, + }, + // With scionPreferred, SCION gets 40-point bonus and can beat + // direct UDP at similar latency. + { + name: "scionPreferred SCION beats direct at similar latency", + a: alSCIONPref(publicV4_2, 1, 100*ms), + b: al(publicV4, 100*ms), + want: true, + }, + // But direct UDP still wins if it's dramatically faster. + { + name: "direct beats scionPreferred SCION when much faster", + a: al(publicV4, 10*ms), + b: alSCIONPref(publicV4_2, 1, 100*ms), + want: true, + }, + // Two SCION paths: lower latency wins. + { + name: "faster SCION beats slower SCION", + a: alSCION(publicV4, 1, 50*ms), + b: alSCION(publicV4_2, 2, 100*ms), + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := betterAddr(tt.a, tt.b) + if got != tt.want { + t.Errorf("betterAddr(%+v, %+v) = %v; want %v", tt.a, tt.b, got, tt.want) + } + }) + } +} + +// newMockPathWithMetadata creates a mock snet.Path that returns the given metadata. +func newMockPathWithMetadata(ctrl *gomock.Controller, md *snet.PathMetadata) *mock_snet.MockPath { + p := mock_snet.NewMockPath(ctrl) + p.EXPECT().Metadata().Return(md).AnyTimes() + p.EXPECT().UnderlayNextHop().Return(&net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 30041}).AnyTimes() + p.EXPECT().Dataplane().Return(nil).AnyTimes() + p.EXPECT().Source().Return(addr.IA(0)).AnyTimes() + p.EXPECT().Destination().Return(addr.IA(0)).AnyTimes() + return p +} + +func TestTotalPathLatency(t *testing.T) { + ctrl := gomock.NewController(t) + + tests := []struct { + name string + path snet.Path + want time.Duration + }{ + { + name: "nil metadata", + path: newMockPathWithMetadata(ctrl, nil), + want: time.Hour, + }, + { + name: "empty latency slice", + path: newMockPathWithMetadata(ctrl, &snet.PathMetadata{Latency: nil}), + want: time.Hour, + }, + { + name: "single hop", + path: newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{5 * time.Millisecond}, + }), + want: 5 * time.Millisecond, + }, + { + name: "multiple hops", + path: newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{ + 5 * time.Millisecond, + 10 * time.Millisecond, + 3 * time.Millisecond, + }, + }), + want: 18 * time.Millisecond, + }, + { + name: "with unset latency", + path: newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{ + 5 * time.Millisecond, + -1, // LatencyUnset + 3 * time.Millisecond, + }, + }), + want: 5*time.Millisecond + 10*time.Millisecond + 3*time.Millisecond, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := totalPathLatency(tt.path) + if got != tt.want { + t.Errorf("totalPathLatency() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestScionServiceFromPeer(t *testing.T) { + tests := []struct { + name string + node *tailcfg.Node + wantIA addr.IA + wantAddr netip.AddrPort + wantOk bool + }{ + { + name: "peer with SCION service", + node: &tailcfg.Node{ + ID: 1, + Key: testNodeKey(), + Hostinfo: (&tailcfg.Hostinfo{ + Services: []tailcfg.Service{ + {Proto: tailcfg.TCP, Port: 80}, + {Proto: tailcfg.SCION, Port: 41641, Description: "1-ff00:0:110,192.0.2.1"}, + }, + }).View(), + }, + wantIA: addr.MustParseIA("1-ff00:0:110"), + wantAddr: netip.MustParseAddrPort("192.0.2.1:41641"), + wantOk: true, + }, + { + name: "peer without SCION service", + node: &tailcfg.Node{ + ID: 2, + Key: testNodeKey(), + Hostinfo: (&tailcfg.Hostinfo{ + Services: []tailcfg.Service{ + {Proto: tailcfg.TCP, Port: 80}, + }, + }).View(), + }, + wantOk: false, + }, + { + name: "peer with invalid SCION description", + node: &tailcfg.Node{ + ID: 3, + Key: testNodeKey(), + Hostinfo: (&tailcfg.Hostinfo{ + Services: []tailcfg.Service{ + {Proto: tailcfg.SCION, Port: 41641, Description: "bad-desc"}, + }, + }).View(), + }, + wantOk: false, + }, + { + name: "peer with no services", + node: &tailcfg.Node{ + ID: 4, + Key: testNodeKey(), + Hostinfo: (&tailcfg.Hostinfo{}).View(), + }, + wantOk: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nv := tt.node.View() + ia, hostAddr, ok := scionServiceFromPeer(nv) + if ok != tt.wantOk { + t.Fatalf("ok = %v, want %v", ok, tt.wantOk) + } + if !tt.wantOk { + return + } + if ia != tt.wantIA { + t.Errorf("IA = %v, want %v", ia, tt.wantIA) + } + if hostAddr != tt.wantAddr { + t.Errorf("hostAddr = %v, want %v", hostAddr, tt.wantAddr) + } + }) + } +} + +func TestIsKnownServiceProtoSCION(t *testing.T) { + if !tailcfg.IsKnownServiceProto(tailcfg.SCION) { + t.Error("SCION should be a known service proto") + } +} + +func TestEpAddrComparability(t *testing.T) { + // Verify that epAddr with scionKey is still comparable (usable as map key). + a := epAddr{ap: netip.MustParseAddrPort("10.0.0.1:41641"), scionKey: 1} + b := epAddr{ap: netip.MustParseAddrPort("10.0.0.1:41641"), scionKey: 1} + c := epAddr{ap: netip.MustParseAddrPort("10.0.0.1:41641"), scionKey: 2} + + if a != b { + t.Error("identical epAddr values should be equal") + } + if a == c { + t.Error("epAddr values with different scionKey should not be equal") + } + + // Verify usable as map key. + m := map[epAddr]bool{a: true} + if !m[b] { + t.Error("identical epAddr should be found in map") + } + if m[c] { + t.Error("different scionKey epAddr should not be found in map") + } +} + +func TestBetterAddrSCIONWithExistingCases(t *testing.T) { + // Verify that adding SCION support doesn't break existing betterAddr + // behavior for non-SCION addresses. These are a subset of cases from + // the existing TestBetterAddr. + const ms = time.Millisecond + al := func(ipps string, d time.Duration) addrQuality { + return addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort(ipps)}, latency: d} + } + almtu := func(ipps string, d time.Duration, mtu tstun.WireMTU) addrQuality { + return addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort(ipps)}, latency: d, wireMTU: mtu} + } + avl := func(ipps string, vni uint32, d time.Duration) addrQuality { + q := al(ipps, d) + q.vni.Set(vni) + return q + } + zero := addrQuality{} + + tests := []struct { + a, b addrQuality + want bool + }{ + {a: zero, b: zero, want: false}, + {a: al("1.2.3.4:555", 5*ms), b: zero, want: true}, + {a: zero, b: al("1.2.3.4:555", 5*ms), want: false}, + {a: al("1.2.3.4:555", 5*ms), b: al("5.6.7.8:999", 10*ms), want: true}, + // Private IP preference still works. + {a: al("10.0.0.2:123", 100*ms), b: al("1.2.3.4:555", 91*ms), want: true}, + // Geneve preference still works. + {a: al("1.2.3.4:555", 100*ms), b: avl("1.2.3.4:555", 1, 100*ms), want: true}, + {a: avl("1.2.3.4:555", 1, 100*ms), b: al("1.2.3.4:555", 100*ms), want: false}, + // MTU preference for same address still works. + {a: almtu("1.2.3.4:555", 30*ms, 1500), b: almtu("1.2.3.4:555", 30*ms, 0), want: true}, + } + for i, tt := range tests { + got := betterAddr(tt.a, tt.b) + if got != tt.want { + t.Errorf("[%d] betterAddr(%+v, %+v) = %v; want %v", i, tt.a, tt.b, got, tt.want) + } + } +} + +func TestSCIONEndpointState(t *testing.T) { + ia := addr.MustParseIA("1-ff00:0:110") + hostAddr := netip.MustParseAddrPort("192.0.2.1:41641") + + st := &scionEndpointState{ + peerIA: ia, + hostAddr: hostAddr, + pathKey: scionPathKey(5), + } + + if st.peerIA != ia { + t.Errorf("peerIA = %v, want %v", st.peerIA, ia) + } + if st.hostAddr != hostAddr { + t.Errorf("hostAddr = %v, want %v", st.hostAddr, hostAddr) + } + if !st.pathKey.IsSet() { + t.Error("pathKey should be set") + } +} + +func TestSendSCIONBatchNoConn(t *testing.T) { + c := &Conn{} + + ep := epAddr{ + ap: netip.MustParseAddrPort("10.0.0.1:41641"), + scionKey: 1, + } + _, err := c.sendSCIONBatch(ep, [][]byte{{0x01}}, 0) + if err != errNoSCION { + t.Errorf("sendSCIONBatch with nil pconnSCION: got err=%v, want %v", err, errNoSCION) + } +} + +func TestSendSCIONNoConn(t *testing.T) { + c := &Conn{} + + _, err := c.sendSCION(scionPathKey(1), []byte{0x01}) + if err != errNoSCION { + t.Errorf("sendSCION with nil pconnSCION: got err=%v, want %v", err, errNoSCION) + } +} + +func TestSCIONPathInfoMutexSafety(t *testing.T) { + pi := &scionPathInfo{ + peerIA: addr.MustParseIA("1-ff00:0:110"), + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + expiry: time.Now().Add(time.Hour), + } + + // Verify concurrent access is safe. + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < 100; i++ { + pi.mu.Lock() + _ = pi.peerIA + _ = pi.hostAddr + _ = pi.expiry + pi.mu.Unlock() + } + }() + for i := 0; i < 100; i++ { + pi.mu.Lock() + pi.expiry = time.Now().Add(time.Duration(i) * time.Minute) + pi.mu.Unlock() + } + <-done +} + +func TestSCIONDiscoRXPath(t *testing.T) { + if discoRXPathSCION != "SCION" { + t.Errorf("discoRXPathSCION = %q, want %q", discoRXPathSCION, "SCION") + } +} + +// testNodeKey returns a new NodePublic for test node construction. +func testNodeKey() key.NodePublic { return key.NewNode().Public() } + +func TestDiscoverSCIONPaths(t *testing.T) { + ctrl := gomock.NewController(t) + mockDaemon := mock_daemon.NewMockConnector(ctrl) + + localIA := addr.MustParseIA("1-ff00:0:110") + peerIA := addr.MustParseIA("1-ff00:0:111") + hostAddr := netip.MustParseAddrPort("10.0.0.1:41641") + + t.Run("picks lowest latency path", func(t *testing.T) { + // Create three mock paths with different latencies. + slowPath := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{50 * time.Millisecond, 50 * time.Millisecond}, + Expiry: time.Now().Add(time.Hour), + }) + fastPath := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{5 * time.Millisecond}, + Expiry: time.Now().Add(time.Hour), + }) + mediumPath := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{20 * time.Millisecond, 10 * time.Millisecond}, + Expiry: time.Now().Add(time.Hour), + }) + + mockDaemon.EXPECT().Paths(gomock.Any(), peerIA, localIA, daemon.PathReqFlags{Refresh: false}). + Return([]snet.Path{slowPath, fastPath, mediumPath}, nil) + + c := &Conn{} + c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + + k, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !k.IsSet() { + t.Fatal("returned key should be set") + } + + pi := c.lookupSCIONPath(k) + if pi == nil { + t.Fatal("path info not found in registry") + } + if pi.peerIA != peerIA { + t.Errorf("peerIA = %v, want %v", pi.peerIA, peerIA) + } + if pi.hostAddr != hostAddr { + t.Errorf("hostAddr = %v, want %v", pi.hostAddr, hostAddr) + } + // The selected path should be the fast one (5ms). + if pi.path != fastPath { + t.Error("should have selected the lowest-latency path") + } + }) + + t.Run("no paths available", func(t *testing.T) { + mockDaemon.EXPECT().Paths(gomock.Any(), peerIA, localIA, daemon.PathReqFlags{Refresh: false}). + Return(nil, nil) + + c := &Conn{} + c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + + _, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) + if err == nil { + t.Fatal("expected error for no paths") + } + }) + + t.Run("daemon error", func(t *testing.T) { + mockDaemon.EXPECT().Paths(gomock.Any(), peerIA, localIA, daemon.PathReqFlags{Refresh: false}). + Return(nil, fmt.Errorf("daemon unavailable")) + + c := &Conn{} + c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + + _, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) + if err == nil { + t.Fatal("expected error for daemon failure") + } + }) + + t.Run("nil pconnSCION", func(t *testing.T) { + c := &Conn{} + _, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) + if err != errNoSCION { + t.Errorf("expected errNoSCION, got %v", err) + } + }) + + t.Run("single path with no metadata", func(t *testing.T) { + noMetaPath := newMockPathWithMetadata(ctrl, nil) + mockDaemon.EXPECT().Paths(gomock.Any(), peerIA, localIA, daemon.PathReqFlags{Refresh: false}). + Return([]snet.Path{noMetaPath}, nil) + + c := &Conn{} + c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + + k, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + pi := c.lookupSCIONPath(k) + if pi == nil { + t.Fatal("path info not found") + } + if !pi.expiry.IsZero() { + t.Errorf("expiry should be zero for nil metadata, got %v", pi.expiry) + } + }) +} + +func TestRefreshSCIONPathsOnce(t *testing.T) { + ctrl := gomock.NewController(t) + mockDaemon := mock_daemon.NewMockConnector(ctrl) + + localIA := addr.MustParseIA("1-ff00:0:110") + peerIA := addr.MustParseIA("1-ff00:0:111") + + t.Run("refreshes expiring path", func(t *testing.T) { + newExpiry := time.Now().Add(2 * time.Hour) + newPath := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{3 * time.Millisecond}, + Expiry: newExpiry, + }) + + mockDaemon.EXPECT().Paths(gomock.Any(), peerIA, localIA, daemon.PathReqFlags{Refresh: true}). + Return([]snet.Path{newPath}, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := &Conn{ + connCtx: ctx, + } + c.logf = t.Logf + c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + + // Register a path that's about to expire (30s from now, within the 1-min refresh window). + pi := &scionPathInfo{ + peerIA: peerIA, + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + expiry: time.Now().Add(30 * time.Second), + } + k := c.registerSCIONPath(pi) + + c.refreshSCIONPathsOnce() + + // Verify the path was updated. + got := c.lookupSCIONPath(k) + got.mu.Lock() + gotPath := got.path + gotExpiry := got.expiry + got.mu.Unlock() + + if gotPath != newPath { + t.Error("path should have been refreshed to new path") + } + if !gotExpiry.Equal(newExpiry) { + t.Errorf("expiry = %v, want %v", gotExpiry, newExpiry) + } + }) + + t.Run("skips non-expiring path", func(t *testing.T) { + // No daemon calls expected — the path doesn't need refresh. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := &Conn{ + connCtx: ctx, + } + c.logf = t.Logf + c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + + // Register a path that's far from expiry. + pi := &scionPathInfo{ + peerIA: peerIA, + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + expiry: time.Now().Add(2 * time.Hour), + } + c.registerSCIONPath(pi) + + // Should not call daemon.Paths since path doesn't need refresh. + c.refreshSCIONPathsOnce() + }) + + t.Run("handles daemon failure gracefully", func(t *testing.T) { + mockDaemon.EXPECT().Paths(gomock.Any(), peerIA, localIA, daemon.PathReqFlags{Refresh: true}). + Return(nil, fmt.Errorf("daemon unreachable")) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := &Conn{ + connCtx: ctx, + } + c.logf = t.Logf + c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + + oldPath := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{10 * time.Millisecond}, + }) + + // Register an expiring path. + pi := &scionPathInfo{ + peerIA: peerIA, + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + path: oldPath, + expiry: time.Now().Add(30 * time.Second), + } + k := c.registerSCIONPath(pi) + + c.refreshSCIONPathsOnce() + + // Path should remain unchanged after daemon failure. + got := c.lookupSCIONPath(k) + got.mu.Lock() + gotPath := got.path + got.mu.Unlock() + + if gotPath != oldPath { + t.Error("path should not have changed after daemon failure") + } + }) + + t.Run("picks best path among refreshed results", func(t *testing.T) { + slowPath := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{100 * time.Millisecond}, + Expiry: time.Now().Add(2 * time.Hour), + }) + fastPath := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{2 * time.Millisecond}, + Expiry: time.Now().Add(2 * time.Hour), + }) + + mockDaemon.EXPECT().Paths(gomock.Any(), peerIA, localIA, daemon.PathReqFlags{Refresh: true}). + Return([]snet.Path{slowPath, fastPath}, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := &Conn{ + connCtx: ctx, + } + c.logf = t.Logf + c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + + pi := &scionPathInfo{ + peerIA: peerIA, + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + expiry: time.Now().Add(30 * time.Second), // about to expire + } + k := c.registerSCIONPath(pi) + + c.refreshSCIONPathsOnce() + + got := c.lookupSCIONPath(k) + got.mu.Lock() + gotPath := got.path + got.mu.Unlock() + + if gotPath != fastPath { + t.Error("should have selected lowest-latency path during refresh") + } + }) +} + +// Verify that the scionPathKey field doesn't break epAddr's use in +// packet.VirtualNetworkID interactions. +func TestEpAddrSCIONAndVNIMutualExclusion(t *testing.T) { + // SCION and VNI shouldn't be set simultaneously in practice, + // but verify the type behavior is correct. + var vni packet.VirtualNetworkID + vni.Set(42) + + both := epAddr{ + ap: netip.MustParseAddrPort("1.2.3.4:555"), + vni: vni, + scionKey: 1, + } + // With both set, it's neither direct nor SCION-only. + if both.isDirect() { + t.Error("epAddr with both VNI and scionKey should not be direct") + } + if !both.isSCION() { + t.Error("epAddr with scionKey should report isSCION") + } + // String should show SCION since scionKey takes precedence in String(). + got := both.String() + if got != "1.2.3.4:555:scion:1" { + t.Errorf("String() = %q, want SCION format", got) + } +} From 33c056c3a69afa66cc19affcc0d1aefd81606836 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 12:43:26 +0000 Subject: [PATCH 08/78] wgengine/magicsock: implement SCION port handling and validation with new tests --- wgengine/magicsock/magicsock.go | 2 +- wgengine/magicsock/magicsock_scion.go | 41 +++++++++++++++++++--- wgengine/magicsock/magicsock_scion_test.go | 36 +++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 4c2a3bc6496f5..3fcebef23cfae 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3613,7 +3613,7 @@ func (c *Conn) rebind(curPortFate currentPortFate) error { // Try to set up SCION if not already connected. if c.pconnSCION == nil { - sc, err := trySCIONConnect(c.connCtx, c.LocalPort()) + sc, err := trySCIONConnect(c.connCtx) if err != nil { c.logf("magicsock: SCION not available: %v", err) } else { diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 1cb6aabcc707a..154dc587043b3 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -47,6 +47,14 @@ type scionEndpointState struct { pathKey scionPathKey // key into Conn.scionPaths } +// SCION dispatcher port range. The SCION dispatcher only forwards packets +// addressed to ports within this range. +const ( + scionDispatchedPortMin = 30000 + scionDispatchedPortMax = 32767 + scionDefaultPort = 31000 +) + // scionConn wraps a SCION connection for use by magicsock. type scionConn struct { conn *snet.Conn // from SCIONNetwork.Listen() @@ -112,9 +120,25 @@ func scionDaemonAddr() string { return daemon.DefaultAPIAddress } +// scionListenPort returns the SCION port to use, checking the TS_SCION_PORT +// environment variable first, then falling back to the default. The port must +// be within the SCION dispatched port range (30000-32767). +func scionListenPort() uint16 { + if p := os.Getenv("TS_SCION_PORT"); p != "" { + var v int + if _, err := fmt.Sscanf(p, "%d", &v); err == nil { + if v >= scionDispatchedPortMin && v <= scionDispatchedPortMax { + return uint16(v) + } + } + } + return scionDefaultPort +} + // trySCIONConnect attempts to connect to the local SCION daemon and set up a -// SCION listener. Returns nil if SCION is not available. -func trySCIONConnect(ctx context.Context, localPort uint16) (*scionConn, error) { +// SCION listener on a port within the SCION dispatched port range (30000-32767). +// Returns nil if SCION is not available. +func trySCIONConnect(ctx context.Context) (*scionConn, error) { daemonAddr := scionDaemonAddr() svc := daemon.Service{Address: daemonAddr} conn, err := svc.Connect(ctx) @@ -132,14 +156,15 @@ func trySCIONConnect(ctx context.Context, localPort uint16) (*scionConn, error) Topology: topo, } + listenPort := scionListenPort() listenAddr := &net.UDPAddr{ IP: net.IPv4zero, - Port: int(localPort), + Port: int(listenPort), } sconn, err := network.Listen(ctx, "udp", listenAddr) if err != nil { conn.Close() - return nil, fmt.Errorf("listening on SCION: %w", err) + return nil, fmt.Errorf("listening on SCION port %d: %w", listenPort, err) } return &scionConn{ @@ -454,9 +479,15 @@ func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { if ua, uaOk := localAddr.(*net.UDPAddr); uaOk && ua.IP != nil { hostIP = ua.IP.String() } + // Advertise the actual SCION listen port (within dispatched range), + // not the magicsock/WireGuard port. + var scionPort uint16 + if ua, uaOk := localAddr.(*net.UDPAddr); uaOk { + scionPort = uint16(ua.Port) + } return tailcfg.Service{ Proto: tailcfg.SCION, - Port: c.LocalPort(), + Port: scionPort, Description: fmt.Sprintf("%s,%s", sc.localIA, hostIP), }, true } diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 6b67d5ce1346a..d05d29b155aed 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -614,6 +614,42 @@ func TestSCIONPathInfoMutexSafety(t *testing.T) { <-done } +func TestScionListenPort(t *testing.T) { + tests := []struct { + name string + envVal string + want uint16 + }{ + {"default", "", scionDefaultPort}, + {"valid port", "31337", 31337}, + {"min port", "30000", 30000}, + {"max port", "32767", 32767}, + {"below range", "29999", scionDefaultPort}, + {"above range", "32768", scionDefaultPort}, + {"non-numeric", "abc", scionDefaultPort}, + {"wireguard port", "41641", scionDefaultPort}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.envVal != "" { + t.Setenv("TS_SCION_PORT", tt.envVal) + } + got := scionListenPort() + if got != tt.want { + t.Errorf("scionListenPort() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestSCIONDispatchedPortRange(t *testing.T) { + // Verify the default port is within the dispatched range. + if scionDefaultPort < scionDispatchedPortMin || scionDefaultPort > scionDispatchedPortMax { + t.Errorf("scionDefaultPort %d is outside dispatched range [%d, %d]", + scionDefaultPort, scionDispatchedPortMin, scionDispatchedPortMax) + } +} + func TestSCIONDiscoRXPath(t *testing.T) { if discoRXPathSCION != "SCION" { t.Errorf("discoRXPathSCION = %q, want %q", discoRXPathSCION, "SCION") From 3d78bfda55d1ce12fef66b1cf57850b39c7613ff Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 14:10:53 +0000 Subject: [PATCH 09/78] wgengine/magicsock: enhance SCION service handling with piggybacking and improve path registration locking - Added support for piggybacking SCION service information in the peerapi4 Description field. - Updated path registration and lookup methods to ensure thread safety with locking. - Enhanced tests to validate new SCION service extraction logic and path registration behavior. --- go.mod | 13 +- go.sum | 42 +++++- ipn/ipnlocal/local.go | 13 +- wgengine/magicsock/endpoint.go | 45 +++--- wgengine/magicsock/magicsock_scion.go | 155 ++++++++++++++++----- wgengine/magicsock/magicsock_scion_test.go | 98 +++++++------ 6 files changed, 249 insertions(+), 117 deletions(-) diff --git a/go.mod b/go.mod index d1c329cd92070..96e4c1a803353 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( github.com/gokrazy/gokrazy v0.0.0-20260123094004-294c93fa173c github.com/gokrazy/serial-busybox v0.0.0-20250119153030-ac58ba7574e7 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 + github.com/golang/mock v1.7.0-rc.1 github.com/golang/snappy v0.0.4 github.com/golangci/golangci-lint v1.57.1 github.com/google/go-cmp v0.7.0 @@ -86,7 +87,7 @@ require ( github.com/prometheus/common v0.65.0 github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff github.com/safchain/ethtool v0.3.0 - github.com/scionproto/scion v0.14.0 + github.com/scionproto/scion v0.12.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/studio-b12/gowebdav v0.9.0 github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e @@ -176,6 +177,7 @@ require ( github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-events v0.0.0-20250808211157-605354379745 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -189,14 +191,12 @@ require ( github.com/gokrazy/gokapi v0.0.0-20250222071133-506fdb322775 // indirect github.com/gokrazy/internal v0.0.0-20251208203110-3c1aa9087c82 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect - github.com/golang/mock v1.7.0-rc.1 // indirect github.com/golangci/plugin-module-register v0.1.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-github/v66 v66.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/renameio/v2 v2.0.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/gopacket/gopacket v1.3.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosuri/uitable v0.0.4 // indirect @@ -222,6 +222,7 @@ require ( github.com/lib/pq v1.10.9 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/macabu/inamedparam v0.1.3 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mdlayher/packet v1.1.2 // indirect github.com/mdlayher/watchdog v0.0.0-20221003142519-49be0df7b3b5 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect @@ -231,6 +232,7 @@ require ( github.com/moby/term v0.5.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect github.com/olekukonko/ll v0.0.8 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect @@ -238,6 +240,7 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/puzpuzpuz/xsync v1.5.2 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rtr7/dhcp4 v0.0.0-20220302171438-18c84d089b46 // indirect github.com/rubenv/sql-migrate v1.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -275,6 +278,10 @@ require ( k8s.io/cli-runtime v0.34.0 // indirect k8s.io/component-base v0.34.0 // indirect k8s.io/kubectl v0.34.0 // indirect + modernc.org/libc v1.66.3 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.38.2 // indirect oras.land/oras-go/v2 v2.6.0 // indirect sigs.k8s.io/kustomize/api v0.20.1 // indirect sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect diff --git a/go.sum b/go.sum index 5f6b993727d08..b3837107c7df1 100644 --- a/go.sum +++ b/go.sum @@ -342,6 +342,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI= github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/crd-ref-docs v0.0.12 h1:F3seyncbzUz3rT3d+caeYWhumb5ojYQ6Bl0Z+zOp16M= github.com/elastic/crd-ref-docs v0.0.12/go.mod h1:X83mMBdJt05heJUYiS3T0yJ/JkCuliuhSUNav5Gjo/U= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= @@ -600,8 +602,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= @@ -613,8 +615,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopacket/gopacket v1.3.1 h1:ZppWyLrOJNZPe5XkdjLbtuTkfQoxQ0xyMJzQCqtqaPU= -github.com/gopacket/gopacket v1.3.1/go.mod h1:3I13qcqSpB2R9fFQg866OOgzylYkZxLTmkvcXhvf6qg= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gordonklaus/ineffassign v0.1.0 h1:y2Gd/9I7MdY1oEIt+n+rowjBNDcLQq3RsH5hwJd0f9s= @@ -909,6 +909,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= @@ -1033,6 +1035,8 @@ github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnAfVjZNvfJTYfPetfZk5yoSTLaQ= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -1065,8 +1069,8 @@ github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tM github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= github.com/sashamelentyev/usestdlibvars v1.25.0 h1:IK8SI2QyFzy/2OD2PYnhy84dpfNo9qADrRt6LH8vSzU= github.com/sashamelentyev/usestdlibvars v1.25.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7aRoS4j6EBye3YBhmAIRF8= -github.com/scionproto/scion v0.14.0 h1:aoSM4f/klmhO/RsXG2RJ7KbaNZ6cujxe9APfqFby0Lw= -github.com/scionproto/scion v0.14.0/go.mod h1:gCXIVztXV7HMe9P/ymVk4U4oSZOYaNnhkeskYxl2h60= +github.com/scionproto/scion v0.12.0 h1:NbBa1HAxWOXr40C8YuanGhJ3g5hYlJetR5YevKtnHGQ= +github.com/scionproto/scion v0.12.0/go.mod h1:jOmbOiLREf4zn6cNrFqto35rP3eH6RhDJEmrjmJIUUI= github.com/securego/gosec/v2 v2.19.0 h1:gl5xMkOI0/E6Hxx0XCY2XujA3V7SNSefA8sC+3f1gnk= github.com/securego/gosec/v2 v2.19.0/go.mod h1:hOkDcHz9J/XIgIlPDXalxjeVYsHxoWUc5zJSHxcB8YM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1805,6 +1809,32 @@ k8s.io/kubectl v0.34.0 h1:NcXz4TPTaUwhiX4LU+6r6udrlm0NsVnSkP3R9t0dmxs= k8s.io/kubectl v0.34.0/go.mod h1:bmd0W5i+HuG7/p5sqicr0Li0rR2iIhXL0oUyLF3OjR4= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= +modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= +modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= +modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek= +modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= mvdan.cc/gofumpt v0.6.0 h1:G3QvahNDmpD+Aek/bNOLrFR2XC6ZAdo62dZu65gmwGo= mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA= mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14 h1:zCr3iRRgdk5eIikZNDphGcM6KGVTx3Yu+/Uu9Es254w= diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 950c16ebfd4af..35784bf5622b0 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4925,9 +4925,20 @@ func (b *LocalBackend) hostInfoWithServicesLocked() *tailcfg.Hostinfo { c := len(hi.Services) hi.Services = append(hi.Services[:c:c], peerAPIServices...) - // Advertise SCION service if available. + // Advertise SCION service if available. Since the coord server only + // relays peerapi4/peerapi6 services to peers, we encode the SCION info + // directly into the peerapi4 service's Description field so it reaches + // peers without coord server changes. if scionSvc, ok := b.MagicConn().SCIONService(); ok { + // Still include the standalone SCION service for the coord server. hi.Services = append(hi.Services, scionSvc) + // Also piggyback on peerapi4's Description field. + for i := range hi.Services { + if hi.Services[i].Proto == tailcfg.PeerAPI4 { + hi.Services[i].Description = fmt.Sprintf("scion=%s:%d", scionSvc.Description, scionSvc.Port) + break + } + } } hi.PushDeviceToken = b.pushDeviceToken.Load() diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 48f7020e792df..4f8a881e1797f 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1547,19 +1547,13 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p // Check for SCION service advertisement from this peer. if peerIA, hostAddr, ok := scionServiceFromPeer(n); ok { if de.scionState == nil || de.scionState.peerIA != peerIA || de.scionState.hostAddr != hostAddr { - // New or changed SCION address — discover paths. + // New or changed SCION address — discover paths asynchronously + // to avoid blocking updateFromNode (which holds the endpoint lock). if de.c.pconnSCION != nil { - pathKey, err := de.c.discoverSCIONPaths(de.c.connCtx, peerIA, hostAddr) - if err != nil { - de.c.logf("magicsock: SCION path discovery for %s failed: %v", peerIA, err) - } else { - de.scionState = &scionEndpointState{ - peerIA: peerIA, - hostAddr: hostAddr, - pathKey: pathKey, - } - de.c.logf("[v1] magicsock: discovered SCION path to %s (key=%d)", peerIA, pathKey) - } + de.c.logf("magicsock: SCION peer %s at %s, discovering paths...", peerIA, hostAddr) + go de.discoverSCIONPathAsync(peerIA, hostAddr) + } else { + de.c.logf("magicsock: peer has SCION (%s) but local SCION not available", peerIA) } } } else if de.scionState != nil { @@ -1767,9 +1761,10 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd now := mono.Now() latency := now.Sub(sp.at) - if !isDerp && !src.vni.IsSet() { - // Note: we check vni.isSet() as relay [epAddr]'s are not stored in - // endpointState, they are either de.bestAddr or not. + if !isDerp && !src.vni.IsSet() && !src.scionKey.IsSet() { + // Note: we check vni.IsSet() and scionKey.IsSet() as relay and + // SCION epAddr's are not stored in endpointState; they are either + // de.bestAddr or not. st, ok := de.endpointState[sp.to.ap] if !ok { // This is no longer an endpoint we care about. @@ -1914,22 +1909,14 @@ func betterAddr(a, b addrQuality) bool { return false } - // SCION paths are preferred over relay but not over direct UDP, - // unless scionPreferred is set (both peers have NodeAttrSCIONPrefer), - // in which case the points system with the +40 bonus handles it. - if a.scionKey.IsSet() != b.scionKey.IsSet() && !a.scionPreferred && !b.scionPreferred { + // SCION paths are preferred over direct UDP and relay. + // TODO(scion): revert to "SCION < direct" once data plane is validated; + // this is temporarily inverted for testing. + if a.scionKey.IsSet() != b.scionKey.IsSet() { if a.scionKey.IsSet() { - // a is SCION - if b.isDirect() { - return false // direct wins over SCION - } - return true // SCION wins over relay/DERP - } - // b is SCION - if a.isDirect() { - return true // direct wins over SCION + return true // SCION wins over direct/relay/DERP } - return false // SCION wins over relay/DERP + return false // SCION wins over direct/relay/DERP } // Each address starts with a set of points (from 0 to 100) that diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 154dc587043b3..e25c214848fb0 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -47,12 +47,11 @@ type scionEndpointState struct { pathKey scionPathKey // key into Conn.scionPaths } -// SCION dispatcher port range. The SCION dispatcher only forwards packets -// addressed to ports within this range. +// SCION dispatched port range. The SCION endhost stack only directly dispatches +// packets addressed to ports within this range. Port 0 means auto-select. const ( scionDispatchedPortMin = 30000 scionDispatchedPortMax = 32767 - scionDefaultPort = 31000 ) // scionConn wraps a SCION connection for use by magicsock. @@ -121,8 +120,9 @@ func scionDaemonAddr() string { } // scionListenPort returns the SCION port to use, checking the TS_SCION_PORT -// environment variable first, then falling back to the default. The port must -// be within the SCION dispatched port range (30000-32767). +// environment variable first, then falling back to 0 (auto-select from the +// topology's dispatched port range). If set, the port must be within the SCION +// dispatched port range. func scionListenPort() uint16 { if p := os.Getenv("TS_SCION_PORT"); p != "" { var v int @@ -132,11 +132,12 @@ func scionListenPort() uint16 { } } } - return scionDefaultPort + return 0 // let snet auto-select from topology port range } // trySCIONConnect attempts to connect to the local SCION daemon and set up a -// SCION listener on a port within the SCION dispatched port range (30000-32767). +// SCION listener. The listener binds to 127.0.0.1 (required by snet, which +// rejects unspecified addresses) on a port within the dispatched range. // Returns nil if SCION is not available. func trySCIONConnect(ctx context.Context) (*scionConn, error) { daemonAddr := scionDaemonAddr() @@ -146,32 +147,33 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { return nil, fmt.Errorf("connecting to SCION daemon at %s: %w", daemonAddr, err) } - topo, err := daemon.LoadTopology(ctx, conn) + localIA, err := conn.LocalIA(ctx) if err != nil { conn.Close() - return nil, fmt.Errorf("loading SCION topology: %w", err) + return nil, fmt.Errorf("querying local IA: %w", err) } + // In scion v0.12.0, daemon.Connector satisfies snet.Topology. network := &snet.SCIONNetwork{ - Topology: topo, + Topology: conn, } listenPort := scionListenPort() listenAddr := &net.UDPAddr{ - IP: net.IPv4zero, + IP: net.IPv4(127, 0, 0, 1), Port: int(listenPort), } sconn, err := network.Listen(ctx, "udp", listenAddr) if err != nil { conn.Close() - return nil, fmt.Errorf("listening on SCION port %d: %w", listenPort, err) + return nil, fmt.Errorf("listening on SCION %s: %w", listenAddr, err) } return &scionConn{ conn: sconn, - localIA: topo.LocalIA, + localIA: localIA, daemon: conn, - topo: topo, + topo: conn, }, nil } @@ -206,7 +208,7 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo return false, errNoSCION } - pi := c.lookupSCIONPath(addr.scionKey) + pi := c.lookupSCIONPathLocking(addr.scionKey) if pi == nil { return false, fmt.Errorf("no SCION path info for key %d", addr.scionKey) } @@ -226,7 +228,7 @@ func (c *Conn) sendSCION(sk scionPathKey, b []byte) (bool, error) { if sc == nil { return false, errNoSCION } - pi := c.lookupSCIONPath(sk) + pi := c.lookupSCIONPathLocking(sk) if pi == nil { return false, fmt.Errorf("no SCION path info for key %d", sk) } @@ -238,14 +240,32 @@ func (c *Conn) sendSCION(sk scionPathKey, b []byte) (bool, error) { } // lookupSCIONPath returns the scionPathInfo for the given key, or nil if not found. +// c.mu must be held. func (c *Conn) lookupSCIONPath(k scionPathKey) *scionPathInfo { + return c.scionPaths[k] +} + +// lookupSCIONPathLocking returns the scionPathInfo for the given key, acquiring c.mu. +func (c *Conn) lookupSCIONPathLocking(k scionPathKey) *scionPathInfo { c.mu.Lock() defer c.mu.Unlock() return c.scionPaths[k] } // registerSCIONPath stores a scionPathInfo and returns a key for it. +// c.mu must be held. func (c *Conn) registerSCIONPath(pi *scionPathInfo) scionPathKey { + k := scionPathKey(c.scionPathSeq.Add(1)) + if c.scionPaths == nil { + c.scionPaths = make(map[scionPathKey]*scionPathInfo) + } + c.scionPaths[k] = pi + return k +} + +// registerSCIONPathLocking stores a scionPathInfo, acquiring c.mu, and returns +// a key for it. +func (c *Conn) registerSCIONPathLocking(pi *scionPathInfo) scionPathKey { k := scionPathKey(c.scionPathSeq.Add(1)) c.mu.Lock() defer c.mu.Unlock() @@ -257,9 +277,8 @@ func (c *Conn) registerSCIONPath(pi *scionPathInfo) scionPathKey { } // unregisterSCIONPath removes a SCION path entry. +// c.mu must be held. func (c *Conn) unregisterSCIONPath(k scionPathKey) { - c.mu.Lock() - defer c.mu.Unlock() delete(c.scionPaths, k) } @@ -284,12 +303,13 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) b := buffs[0][:n] - // Build an epAddr for this SCION source. We use the host IP:port - // from the SCION address. The scionKey on the epAddr is not set - // here since we're on the receive path — the peerMap lookup uses - // the host addr portion. + // Build an epAddr for this SCION source. Look up the scionKey + // so that pong replies are routed back over SCION. srcHostAddr := srcAddr.Host.AddrPort() srcEpAddr := epAddr{ap: srcHostAddr} + if sk := c.scionKeyForAddr(srcAddr.IA, srcHostAddr); sk.IsSet() { + srcEpAddr.scionKey = sk + } // Check for disco packets (same as receiveIP does). pt, _ := packetLooksLike(b) @@ -354,7 +374,7 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr path: best, expiry: expiry, } - return c.registerSCIONPath(pi), nil + return c.registerSCIONPathLocking(pi), nil } // totalPathLatency returns the sum of all hop latencies for a SCION path. @@ -446,6 +466,9 @@ func (c *Conn) refreshSCIONPathsOnce() { } // scionServiceFromPeer extracts SCION service info from a peer node's Services. +// It checks for a dedicated SCION service entry first, then falls back to +// checking the peerapi4 Description field (which is used to piggyback SCION +// info through coord servers that only relay peerapi services). func scionServiceFromPeer(n tailcfg.NodeView) (ia addr.IA, hostAddr netip.AddrPort, ok bool) { hi := n.Hostinfo() if !hi.Valid() { @@ -454,14 +477,34 @@ func scionServiceFromPeer(n tailcfg.NodeView) (ia addr.IA, hostAddr netip.AddrPo services := hi.Services() for i := range services.Len() { svc := services.At(i) - if svc.Proto != tailcfg.SCION { - continue + // Direct SCION service entry. + if svc.Proto == tailcfg.SCION { + parsedIA, parsedAddr, err := parseSCIONServiceAddr(svc.Description, svc.Port) + if err != nil { + continue + } + return parsedIA, parsedAddr, true } - parsedIA, parsedAddr, err := parseSCIONServiceAddr(svc.Description, svc.Port) - if err != nil { - continue + // Piggyback: SCION info in peerapi4's Description field. + // Format: "scion=ISD-AS,host-IP:port" + if svc.Proto == tailcfg.PeerAPI4 && strings.HasPrefix(svc.Description, "scion=") { + scionDesc := svc.Description[len("scion="):] + // Parse "ISD-AS,host-IP:port" + lastColon := strings.LastIndex(scionDesc, ":") + if lastColon < 0 { + continue + } + addrPart := scionDesc[:lastColon] + var port uint16 + if _, err := fmt.Sscanf(scionDesc[lastColon+1:], "%d", &port); err != nil { + continue + } + parsedIA, parsedAddr, err := parseSCIONServiceAddr(addrPart, port) + if err != nil { + continue + } + return parsedIA, parsedAddr, true } - return parsedIA, parsedAddr, true } return 0, netip.AddrPort{}, false } @@ -473,17 +516,15 @@ func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { if sc == nil { return tailcfg.Service{}, false } - // The local host IP comes from the SCION connection's local address. + // snet.Conn.LocalAddr() returns an *snet.UDPAddr; extract host IP and port. localAddr := sc.conn.LocalAddr() - hostIP := "0.0.0.0" - if ua, uaOk := localAddr.(*net.UDPAddr); uaOk && ua.IP != nil { - hostIP = ua.IP.String() - } - // Advertise the actual SCION listen port (within dispatched range), - // not the magicsock/WireGuard port. + hostIP := "127.0.0.1" var scionPort uint16 - if ua, uaOk := localAddr.(*net.UDPAddr); uaOk { - scionPort = uint16(ua.Port) + if sa, saOk := localAddr.(*snet.UDPAddr); saOk && sa.Host != nil { + if sa.Host.IP != nil { + hostIP = sa.Host.IP.String() + } + scionPort = uint16(sa.Host.Port) } return tailcfg.Service{ Proto: tailcfg.SCION, @@ -492,6 +533,44 @@ func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { }, true } +// discoverSCIONPathAsync runs SCION path discovery in a goroutine, +// avoiding blocking updateFromNode which holds the endpoint lock. +func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPort) { + ctx, cancel := context.WithTimeout(de.c.connCtx, 10*time.Second) + defer cancel() + + pathKey, err := de.c.discoverSCIONPaths(ctx, peerIA, hostAddr) + if err != nil { + de.c.logf("magicsock: SCION path discovery for %s failed: %v", peerIA, err) + return + } + + de.mu.Lock() + de.scionState = &scionEndpointState{ + peerIA: peerIA, + hostAddr: hostAddr, + pathKey: pathKey, + } + de.mu.Unlock() + de.c.logf("magicsock: discovered SCION path to %s (key=%d)", peerIA, pathKey) +} + +// scionKeyForAddr returns the scionPathKey for the given peer IA and host +// address, or a zero key if not found. +func (c *Conn) scionKeyForAddr(peerIA addr.IA, hostAddr netip.AddrPort) scionPathKey { + c.mu.Lock() + defer c.mu.Unlock() + for k, pi := range c.scionPaths { + pi.mu.Lock() + match := pi.peerIA == peerIA && pi.hostAddr == hostAddr + pi.mu.Unlock() + if match { + return k + } + } + return 0 +} + var errNoSCION = fmt.Errorf("SCION not available") const discoRXPathSCION discoRXPath = "SCION" diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index d05d29b155aed..fcca456654c56 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -19,7 +19,6 @@ import ( "github.com/scionproto/scion/pkg/snet/mock_snet" "tailscale.com/net/packet" "tailscale.com/net/tstun" - "tailscale.com/syncs" "tailscale.com/tailcfg" "tailscale.com/types/key" ) @@ -182,22 +181,20 @@ func TestParseSCIONServiceAddr(t *testing.T) { func TestSCIONPathRegistry(t *testing.T) { c := &Conn{} - c.mu = syncs.Mutex{} - // Register a path. + // Test locking versions (used by callers outside c.mu). pi := &scionPathInfo{ peerIA: addr.MustParseIA("1-ff00:0:111"), hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), } - k := c.registerSCIONPath(pi) + k := c.registerSCIONPathLocking(pi) if !k.IsSet() { t.Fatal("registered key should be set") } - // Look it up. - got := c.lookupSCIONPath(k) + got := c.lookupSCIONPathLocking(k) if got != pi { - t.Fatalf("lookupSCIONPath(%d) returned wrong path info", k) + t.Fatalf("lookupSCIONPathLocking(%d) returned wrong path info", k) } // Register another. @@ -205,26 +202,27 @@ func TestSCIONPathRegistry(t *testing.T) { peerIA: addr.MustParseIA("1-ff00:0:112"), hostAddr: netip.MustParseAddrPort("10.0.0.2:41641"), } - k2 := c.registerSCIONPath(pi2) + k2 := c.registerSCIONPathLocking(pi2) if k2 == k { t.Fatal("second key should differ from first") } - if c.lookupSCIONPath(k2) != pi2 { + if c.lookupSCIONPathLocking(k2) != pi2 { t.Fatal("second path not found") } - // Unregister the first. + // Unregister the first (non-locking, must hold c.mu). + c.mu.Lock() c.unregisterSCIONPath(k) - if c.lookupSCIONPath(k) != nil { + c.mu.Unlock() + + if c.lookupSCIONPathLocking(k) != nil { t.Fatal("unregistered path should return nil") } - // Second should still be there. - if c.lookupSCIONPath(k2) != pi2 { + if c.lookupSCIONPathLocking(k2) != pi2 { t.Fatal("second path should still be present after unregistering first") } - // Lookup of non-existent key. - if c.lookupSCIONPath(scionPathKey(9999)) != nil { + if c.lookupSCIONPathLocking(scionPathKey(9999)) != nil { t.Fatal("non-existent key should return nil") } } @@ -452,6 +450,34 @@ func TestScionServiceFromPeer(t *testing.T) { }, wantOk: false, }, + { + name: "peer with SCION in peerapi4 description (piggyback)", + node: &tailcfg.Node{ + ID: 5, + Key: testNodeKey(), + Hostinfo: (&tailcfg.Hostinfo{ + Services: []tailcfg.Service{ + {Proto: tailcfg.PeerAPI4, Port: 12345, Description: "scion=1-ff00:0:110,192.0.2.1:32766"}, + }, + }).View(), + }, + wantIA: addr.MustParseIA("1-ff00:0:110"), + wantAddr: netip.MustParseAddrPort("192.0.2.1:32766"), + wantOk: true, + }, + { + name: "peer with bad SCION piggyback", + node: &tailcfg.Node{ + ID: 6, + Key: testNodeKey(), + Hostinfo: (&tailcfg.Hostinfo{ + Services: []tailcfg.Service{ + {Proto: tailcfg.PeerAPI4, Port: 12345, Description: "scion=bad-data"}, + }, + }).View(), + }, + wantOk: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -616,18 +642,18 @@ func TestSCIONPathInfoMutexSafety(t *testing.T) { func TestScionListenPort(t *testing.T) { tests := []struct { - name string - envVal string - want uint16 + name string + envVal string + want uint16 }{ - {"default", "", scionDefaultPort}, + {"default auto-select", "", 0}, {"valid port", "31337", 31337}, {"min port", "30000", 30000}, {"max port", "32767", 32767}, - {"below range", "29999", scionDefaultPort}, - {"above range", "32768", scionDefaultPort}, - {"non-numeric", "abc", scionDefaultPort}, - {"wireguard port", "41641", scionDefaultPort}, + {"below range", "29999", 0}, + {"above range", "32768", 0}, + {"non-numeric", "abc", 0}, + {"wireguard port", "41641", 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -642,14 +668,6 @@ func TestScionListenPort(t *testing.T) { } } -func TestSCIONDispatchedPortRange(t *testing.T) { - // Verify the default port is within the dispatched range. - if scionDefaultPort < scionDispatchedPortMin || scionDefaultPort > scionDispatchedPortMax { - t.Errorf("scionDefaultPort %d is outside dispatched range [%d, %d]", - scionDefaultPort, scionDispatchedPortMin, scionDispatchedPortMax) - } -} - func TestSCIONDiscoRXPath(t *testing.T) { if discoRXPathSCION != "SCION" { t.Errorf("discoRXPathSCION = %q, want %q", discoRXPathSCION, "SCION") @@ -696,7 +714,7 @@ func TestDiscoverSCIONPaths(t *testing.T) { t.Fatal("returned key should be set") } - pi := c.lookupSCIONPath(k) + pi := c.lookupSCIONPathLocking(k) if pi == nil { t.Fatal("path info not found in registry") } @@ -758,7 +776,7 @@ func TestDiscoverSCIONPaths(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - pi := c.lookupSCIONPath(k) + pi := c.lookupSCIONPathLocking(k) if pi == nil { t.Fatal("path info not found") } @@ -800,12 +818,12 @@ func TestRefreshSCIONPathsOnce(t *testing.T) { hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), expiry: time.Now().Add(30 * time.Second), } - k := c.registerSCIONPath(pi) + k := c.registerSCIONPathLocking(pi) c.refreshSCIONPathsOnce() // Verify the path was updated. - got := c.lookupSCIONPath(k) + got := c.lookupSCIONPathLocking(k) got.mu.Lock() gotPath := got.path gotExpiry := got.expiry @@ -836,7 +854,7 @@ func TestRefreshSCIONPathsOnce(t *testing.T) { hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), expiry: time.Now().Add(2 * time.Hour), } - c.registerSCIONPath(pi) + c.registerSCIONPathLocking(pi) // Should not call daemon.Paths since path doesn't need refresh. c.refreshSCIONPathsOnce() @@ -866,12 +884,12 @@ func TestRefreshSCIONPathsOnce(t *testing.T) { path: oldPath, expiry: time.Now().Add(30 * time.Second), } - k := c.registerSCIONPath(pi) + k := c.registerSCIONPathLocking(pi) c.refreshSCIONPathsOnce() // Path should remain unchanged after daemon failure. - got := c.lookupSCIONPath(k) + got := c.lookupSCIONPathLocking(k) got.mu.Lock() gotPath := got.path got.mu.Unlock() @@ -908,11 +926,11 @@ func TestRefreshSCIONPathsOnce(t *testing.T) { hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), expiry: time.Now().Add(30 * time.Second), // about to expire } - k := c.registerSCIONPath(pi) + k := c.registerSCIONPathLocking(pi) c.refreshSCIONPathsOnce() - got := c.lookupSCIONPath(k) + got := c.lookupSCIONPathLocking(k) got.mu.Lock() gotPath := got.path got.mu.Unlock() From b2bb2f0134e37a042705a2aee9f83d96e04614ee Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 14:56:24 +0000 Subject: [PATCH 10/78] wgengine/magicsock: improve SCION path handling and address evaluation - Enhanced logic for determining when to send full pings and disco pings based on SCION path characteristics. - Updated MTU probing logic to account for SCION paths, ensuring proper handling of payload sizes. - Refined address quality evaluation in tests to reflect the new preference for SCION over direct UDP connections. - Improved logging to include MTU information when discovering SCION paths. --- wgengine/magicsock/endpoint.go | 15 ++++- wgengine/magicsock/magicsock.go | 6 ++ wgengine/magicsock/magicsock_scion.go | 68 ++++++++++++++++++---- wgengine/magicsock/magicsock_scion_test.go | 26 ++++----- 4 files changed, 87 insertions(+), 28 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 4f8a881e1797f..5692684163f25 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -936,7 +936,7 @@ func (de *endpoint) wantFullPingLocked(now mono.Time) bool { if runtime.GOOS == "js" { return false } - if !de.bestAddr.isDirect() || de.lastFullPing.IsZero() { + if (!de.bestAddr.isDirect() && !de.bestAddr.isSCION()) || de.lastFullPing.IsZero() { return true } if now.After(de.trustBestAddrUntil) { @@ -1052,7 +1052,7 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { if startWGPing { de.sendWireGuardOnlyPingsLocked(now) } - } else if !udpAddr.isDirect() || now.After(de.trustBestAddrUntil) { + } else if (!udpAddr.isDirect() && !udpAddr.isSCION()) || now.After(de.trustBestAddrUntil) { de.sendDiscoPingsLocked(now, true) if de.wantUDPRelayPathDiscoveryLocked(now) { de.discoverUDPRelayPathsLocked(now) @@ -1322,7 +1322,9 @@ func (de *endpoint) startDiscoPingLocked(ep epAddr, now mono.Time, purpose disco sizes := []int{size} if de.c.PeerMTUEnabled() { isDerp := ep.ap.Addr() == tailcfg.DerpMagicIPAddr - if !isDerp && ((purpose == pingDiscovery) || (purpose == pingCLI && size == 0)) { + // Skip MTU probing for SCION paths — the SCION path MTU is known + // from metadata and oversized probes won't fit through the path. + if !isDerp && !ep.scionKey.IsSet() && ((purpose == pingDiscovery) || (purpose == pingCLI && size == 0)) { de.c.dlogf("[v1] magicsock: starting MTU probe") sizes = mtuProbePingSizesV4 if ep.ap.Addr().Is6() { @@ -1701,6 +1703,13 @@ func pingSizeToPktLen(size int, udpAddr epAddr) tstun.WireMTU { if size == 0 { return tstun.SafeWireMTU() } + if udpAddr.scionKey.IsSet() { + // For SCION paths, the snet library handles all SCION header + // serialization internally. The payload we pass to WriteTo is the + // WireGuard packet. Since we skip MTU probing for SCION (the path + // MTU is known from metadata), just return the safe wire MTU. + return tstun.SafeWireMTU() + } headerLen := ipv4.HeaderLen if udpAddr.ap.Addr().Is6() { headerLen = ipv6.HeaderLen diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 3fcebef23cfae..8c6bcc60eb6eb 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2515,6 +2515,12 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpN if nk, ok := c.unambiguousNodeKeyOfPingLocked(dm, di.discoKey, derpNodeSrc); ok { if !isDerp { c.peerMap.setNodeKeyForEpAddr(src, nk) + // For SCION sources, also register the plain host addr + // so WireGuard data packets (which don't carry a scionKey) + // can be looked up in the peerMap. + if src.scionKey.IsSet() { + c.peerMap.setNodeKeyForEpAddr(epAddr{ap: src.ap}, nk) + } } } diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index e25c214848fb0..b06d70a7f676c 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -37,9 +37,28 @@ type scionPathInfo struct { hostAddr netip.AddrPort // peer's SCION host IP:port path snet.Path // current best SCION path to this peer expiry time.Time // path expiration from path metadata + mtu uint16 // SCION payload MTU from path metadata mu sync.Mutex } +// scionHeaderOverhead is the fixed overhead added by SCION encapsulation, +// excluding the variable-length path header: +// - Underlay IPv4+UDP: 20 + 8 = 28 bytes +// - SCION common header: 12 bytes +// - Address header (IPv4, 2x ISD-AS + 2x IPv4): 2*8 + 2*4 = 24 bytes +// - SCION/UDP L4 header: 8 bytes +// +// Total fixed: 72 bytes. The path header is variable (depends on hop count). +// Rather than parsing the path to determine the exact overhead, we use the +// path MTU from metadata directly: the SCION daemon reports the maximum +// *payload* size that can traverse the path (i.e., MTU already accounts for +// all SCION headers). So the effective wire MTU for WireGuard is simply the +// SCION path MTU. +// +// However, when no path MTU is available, we use a conservative estimate: +// 1280 bytes (minimum IPv6-compatible MTU). +const scionFallbackPayloadMTU = 1280 + // scionEndpointState tracks SCION-specific per-peer state on an endpoint. type scionEndpointState struct { peerIA addr.IA // peer's ISD-AS from Services advertisement @@ -303,32 +322,45 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) b := buffs[0][:n] - // Build an epAddr for this SCION source. Look up the scionKey - // so that pong replies are routed back over SCION. srcHostAddr := srcAddr.Host.AddrPort() - srcEpAddr := epAddr{ap: srcHostAddr} - if sk := c.scionKeyForAddr(srcAddr.IA, srcHostAddr); sk.IsSet() { - srcEpAddr.scionKey = sk - } // Check for disco packets (same as receiveIP does). pt, _ := packetLooksLike(b) if pt == packetLooksLikeDisco { + // For disco messages, include the scionKey so pong replies + // are routed back over SCION. + srcEpAddr := epAddr{ap: srcHostAddr} + if sk := c.scionKeyForAddr(srcAddr.IA, srcHostAddr); sk.IsSet() { + srcEpAddr.scionKey = sk + } c.handleDiscoMessage(b, srcEpAddr, false, key.NodePublic{}, discoRXPathSCION) continue } - // WireGuard packet — look up the endpoint by source address. + if !c.havePrivateKey.Load() { + // No private key means we're logged out. Don't pass WireGuard + // packets up; wireguard-go will just complain. + continue + } + + // WireGuard packet — look up the endpoint by host addr only + // (peerMap is keyed by netip.AddrPort, not scionKey). + srcEpAddr := epAddr{ap: srcHostAddr} c.mu.Lock() ep, ok := c.peerMap.endpointForEpAddr(srcEpAddr) c.mu.Unlock() if !ok { - // Try looking up without SCION key since the receive side - // may not have the scionKey set in peerMap. - continue + // No peerMap entry yet. Return a lazyEndpoint so WireGuard + // can identify the peer from the encrypted packet and + // register it, matching the behavior of receiveIP. + sizes[0] = n + eps[0] = &lazyEndpoint{c: c, src: srcEpAddr} + return 1, nil } - ep.noteRecvActivity(srcEpAddr, mono.Now()) + now := mono.Now() + ep.lastRecvUDPAny.StoreAtomic(now) + ep.noteRecvActivity(srcEpAddr, now) sizes[0] = n eps[0] = ep return 1, nil @@ -364,8 +396,10 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr } var expiry time.Time + var mtu uint16 if md := best.Metadata(); md != nil { expiry = md.Expiry + mtu = md.MTU } pi := &scionPathInfo{ @@ -373,6 +407,7 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr hostAddr: hostAddr, path: best, expiry: expiry, + mtu: mtu, } return c.registerSCIONPathLocking(pi), nil } @@ -460,6 +495,7 @@ func (c *Conn) refreshSCIONPathsOnce() { pi.path = best if md := best.Metadata(); md != nil { pi.expiry = md.Expiry + pi.mtu = md.MTU } pi.mu.Unlock() } @@ -552,7 +588,15 @@ func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPo pathKey: pathKey, } de.mu.Unlock() - de.c.logf("magicsock: discovered SCION path to %s (key=%d)", peerIA, pathKey) + + pi := de.c.lookupSCIONPathLocking(pathKey) + var mtu uint16 + if pi != nil { + pi.mu.Lock() + mtu = pi.mtu + pi.mu.Unlock() + } + de.c.logf("magicsock: discovered SCION path to %s (key=%d, mtu=%d)", peerIA, pathKey, mtu) } // scionKeyForAddr returns the scionPathKey for the given peer IA and host diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index fcca456654c56..a3dbd252725ef 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -262,17 +262,18 @@ func TestBetterAddrSCION(t *testing.T) { a, b addrQuality want bool }{ - // Direct UDP wins over SCION at equal latency. + // SCION wins over direct UDP at equal latency. + // TODO(scion): revert when SCION preference is behind NodeAttrSCIONPrefer. { - name: "direct beats SCION same latency", - a: al(publicV4, 100*ms), - b: alSCION(publicV4_2, 1, 100*ms), + name: "SCION beats direct same latency", + a: alSCION(publicV4_2, 1, 100*ms), + b: al(publicV4, 100*ms), want: true, }, { - name: "SCION loses to direct same latency", - a: alSCION(publicV4_2, 1, 100*ms), - b: al(publicV4, 100*ms), + name: "direct loses to SCION same latency", + a: al(publicV4, 100*ms), + b: alSCION(publicV4_2, 1, 100*ms), want: false, }, // SCION wins over relay (VNI). @@ -288,19 +289,18 @@ func TestBetterAddrSCION(t *testing.T) { b: alSCION(publicV4, 1, 100*ms), want: false, }, - // With scionPreferred, SCION gets 40-point bonus and can beat - // direct UDP at similar latency. + // scionPreferred bonus is additive with SCION preference. { name: "scionPreferred SCION beats direct at similar latency", a: alSCIONPref(publicV4_2, 1, 100*ms), b: al(publicV4, 100*ms), want: true, }, - // But direct UDP still wins if it's dramatically faster. + // SCION always wins over direct (current preference order). { - name: "direct beats scionPreferred SCION when much faster", - a: al(publicV4, 10*ms), - b: alSCIONPref(publicV4_2, 1, 100*ms), + name: "SCION still wins over much faster direct", + a: alSCION(publicV4_2, 1, 100*ms), + b: al(publicV4, 10*ms), want: true, }, // Two SCION paths: lower latency wins. From ec4cc210f3aed53852fe67d26aa6a91fb45ac946 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 9 Mar 2026 16:08:22 +0000 Subject: [PATCH 11/78] wgengine/magicsock: improve SCION path management and metrics tracking --- wgengine/magicsock/endpoint.go | 57 ++++++--- wgengine/magicsock/magicsock.go | 31 ++++- wgengine/magicsock/magicsock_scion.go | 120 +++++++++++++++--- wgengine/magicsock/magicsock_scion_test.go | 139 +++++++++++++++++++-- 4 files changed, 307 insertions(+), 40 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 5692684163f25..86eef8359cf02 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -519,7 +519,8 @@ func (de *endpoint) noteRecvActivity(src epAddr, now mono.Time) bool { // kick off discovery disco pings every trustUDPAddrDuration and mirror // to DERP. de.mu.Lock() - if de.heartbeatDisabled && de.bestAddr.epAddr == src { + if de.heartbeatDisabled && (de.bestAddr.epAddr == src || + (de.bestAddr.isSCION() && de.bestAddr.ap == src.ap)) { de.trustBestAddrUntil = now.Add(trustUDPAddrDuration) } de.mu.Unlock() @@ -1078,6 +1079,13 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { _, err = de.c.sendSCIONBatch(udpAddr, buffs, offset) if err != nil { de.noteBadEndpoint(udpAddr) + } else if de.c.metrics != nil { + var txBytes int + for _, b := range buffs { + txBytes += len(b[offset:]) + } + de.c.metrics.outboundPacketsSCIONTotal.Add(int64(len(buffs))) + de.c.metrics.outboundBytesSCIONTotal.Add(int64(txBytes)) } } else if udpAddr.ap.IsValid() { _, err = de.c.sendUDPBatch(udpAddr, buffs, offset) @@ -1918,14 +1926,12 @@ func betterAddr(a, b addrQuality) bool { return false } - // SCION paths are preferred over direct UDP and relay. - // TODO(scion): revert to "SCION < direct" once data plane is validated; - // this is temporarily inverted for testing. - if a.scionKey.IsSet() != b.scionKey.IsSet() { - if a.scionKey.IsSet() { - return true // SCION wins over direct/relay/DERP - } - return false // SCION wins over direct/relay/DERP + // SCION beats relay (Geneve) unconditionally. + if a.scionKey.IsSet() && !b.scionKey.IsSet() && b.vni.IsSet() { + return true + } + if b.scionKey.IsSet() && !a.scionKey.IsSet() && a.vni.IsSet() { + return false } // Each address starts with a set of points (from 0 to 100) that @@ -1972,14 +1978,20 @@ func betterAddr(a, b addrQuality) bool { bPoints += 10 } - // When both self and peer have NodeAttrSCIONPrefer, give SCION paths - // a large bonus so they're preferred over direct UDP unless direct UDP - // is dramatically faster. + // SCION paths get a configurable bonus (default +15) so they win at + // similar latency. NodeAttrSCIONPrefer adds +25 more for a strong + // admin preference (total +40 at default). + if a.scionKey.IsSet() { + aPoints += scionPreferenceBonus() + } + if b.scionKey.IsSet() { + bPoints += scionPreferenceBonus() + } if a.scionPreferred && a.scionKey.IsSet() { - aPoints += 40 + aPoints += 25 } if b.scionPreferred && b.scionKey.IsSet() { - bPoints += 40 + bPoints += 25 } // Don't change anything if the latency improvement is less than 1%; we @@ -2113,7 +2125,14 @@ func (de *endpoint) populatePeerStatus(ps *ipnstate.PeerStatus) { func (de *endpoint) stopAndReset() { atomic.AddInt64(&de.numStopAndResetAtomic, 1) de.mu.Lock() - defer de.mu.Unlock() + + // Extract scionPathKey before releasing de.mu so we can clean it up + // under c.mu afterward (lock order: c.mu before de.mu). + var scionKey scionPathKey + if de.scionState != nil { + scionKey = de.scionState.pathKey + de.scionState = nil + } if closing := de.c.closing.Load(); !closing { if de.isWireguardOnly { @@ -2132,6 +2151,14 @@ func (de *endpoint) stopAndReset() { de.heartBeatTimer.Stop() de.heartBeatTimer = nil } + de.mu.Unlock() + + // Clean up SCION path outside de.mu (lock order: c.mu before de.mu). + if scionKey.IsSet() { + de.c.mu.Lock() + de.c.unregisterSCIONPath(scionKey) + de.c.mu.Unlock() + } } // resetLocked clears all the endpoint's p2p state, reverting it to a diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 8c6bcc60eb6eb..bd3c61d77eac1 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -96,6 +96,7 @@ const ( PathDERP Path = "derp" PathPeerRelayIPv4 Path = "peer_relay_ipv4" PathPeerRelayIPv6 Path = "peer_relay_ipv6" + PathSCION Path = "scion" ) type pathLabel struct { @@ -146,6 +147,12 @@ type metrics struct { outboundBytesPeerRelayIPv4Total expvar.Int outboundBytesPeerRelayIPv6Total expvar.Int + // SCION path counters. + inboundPacketsSCIONTotal expvar.Int + inboundBytesSCIONTotal expvar.Int + outboundPacketsSCIONTotal expvar.Int + outboundBytesSCIONTotal expvar.Int + // outboundPacketsDroppedErrors is the total number of outbound packets // dropped due to errors. outboundPacketsDroppedErrors expvar.Int @@ -415,8 +422,9 @@ type Conn struct { // scionPaths is the registry of SCION path information, keyed by // scionPathKey. Each entry holds the full SCION address and path // data for a peer. - scionPaths map[scionPathKey]*scionPathInfo - scionPathSeq atomic.Uint32 // monotonic key generator for scionPaths + scionPaths map[scionPathKey]*scionPathInfo + scionPathsByAddr map[scionAddrKey]scionPathKey // reverse index for O(1) lookup + scionPathSeq atomic.Uint32 // monotonic key generator for scionPaths } // SetDebugLoggingEnabled controls whether spammy debug logging is enabled. @@ -722,6 +730,7 @@ func registerMetrics(reg *usermetric.Registry) *metrics { pathDERP := pathLabel{Path: PathDERP} pathPeerRelayV4 := pathLabel{Path: PathPeerRelayIPv4} pathPeerRelayV6 := pathLabel{Path: PathPeerRelayIPv6} + pathSCION := pathLabel{Path: PathSCION} inboundPacketsTotal := usermetric.NewMultiLabelMapWithRegistry[pathLabel]( reg, "tailscaled_inbound_packets_total", @@ -776,30 +785,38 @@ func registerMetrics(reg *usermetric.Registry) *metrics { metricSendDERP.Register(&m.outboundPacketsDERPTotal) metricSendPeerRelay.Register(&m.outboundPacketsPeerRelayIPv4Total) metricSendPeerRelay.Register(&m.outboundPacketsPeerRelayIPv6Total) + metricRecvDataPacketsSCION.Register(&m.inboundPacketsSCIONTotal) + metricRecvDataBytesSCION.Register(&m.inboundBytesSCIONTotal) + metricSendDataPacketsSCION.Register(&m.outboundPacketsSCIONTotal) + metricSendDataBytesSCION.Register(&m.outboundBytesSCIONTotal) inboundPacketsTotal.Set(pathDirectV4, &m.inboundPacketsIPv4Total) inboundPacketsTotal.Set(pathDirectV6, &m.inboundPacketsIPv6Total) inboundPacketsTotal.Set(pathDERP, &m.inboundPacketsDERPTotal) inboundPacketsTotal.Set(pathPeerRelayV4, &m.inboundPacketsPeerRelayIPv4Total) inboundPacketsTotal.Set(pathPeerRelayV6, &m.inboundPacketsPeerRelayIPv6Total) + inboundPacketsTotal.Set(pathSCION, &m.inboundPacketsSCIONTotal) inboundBytesTotal.Set(pathDirectV4, &m.inboundBytesIPv4Total) inboundBytesTotal.Set(pathDirectV6, &m.inboundBytesIPv6Total) inboundBytesTotal.Set(pathDERP, &m.inboundBytesDERPTotal) inboundBytesTotal.Set(pathPeerRelayV4, &m.inboundBytesPeerRelayIPv4Total) inboundBytesTotal.Set(pathPeerRelayV6, &m.inboundBytesPeerRelayIPv6Total) + inboundBytesTotal.Set(pathSCION, &m.inboundBytesSCIONTotal) outboundPacketsTotal.Set(pathDirectV4, &m.outboundPacketsIPv4Total) outboundPacketsTotal.Set(pathDirectV6, &m.outboundPacketsIPv6Total) outboundPacketsTotal.Set(pathDERP, &m.outboundPacketsDERPTotal) outboundPacketsTotal.Set(pathPeerRelayV4, &m.outboundPacketsPeerRelayIPv4Total) outboundPacketsTotal.Set(pathPeerRelayV6, &m.outboundPacketsPeerRelayIPv6Total) + outboundPacketsTotal.Set(pathSCION, &m.outboundPacketsSCIONTotal) outboundBytesTotal.Set(pathDirectV4, &m.outboundBytesIPv4Total) outboundBytesTotal.Set(pathDirectV6, &m.outboundBytesIPv6Total) outboundBytesTotal.Set(pathDERP, &m.outboundBytesDERPTotal) outboundBytesTotal.Set(pathPeerRelayV4, &m.outboundBytesPeerRelayIPv4Total) outboundBytesTotal.Set(pathPeerRelayV6, &m.outboundBytesPeerRelayIPv6Total) + outboundBytesTotal.Set(pathSCION, &m.outboundBytesSCIONTotal) outboundPacketsDroppedErrors.Set(usermetric.DropLabels{Reason: usermetric.ReasonError}, &m.outboundPacketsDroppedErrors) @@ -832,6 +849,10 @@ func deregisterMetrics() { metricSendUDP.UnregisterAll() metricSendDERP.UnregisterAll() metricSendPeerRelay.UnregisterAll() + metricRecvDataPacketsSCION.UnregisterAll() + metricRecvDataBytesSCION.UnregisterAll() + metricSendDataPacketsSCION.UnregisterAll() + metricSendDataBytesSCION.UnregisterAll() } // InstallCaptureHook installs a callback which is called to @@ -4032,6 +4053,12 @@ var ( metricSendDataBytesPeerRelayIPv4 = clientmetric.NewAggregateCounter("magicsock_send_data_bytes_peer_relay_ipv4") metricSendDataBytesPeerRelayIPv6 = clientmetric.NewAggregateCounter("magicsock_send_data_bytes_peer_relay_ipv6") + // SCION data packets and bytes + metricRecvDataPacketsSCION = clientmetric.NewAggregateCounter("magicsock_recv_data_scion") + metricRecvDataBytesSCION = clientmetric.NewAggregateCounter("magicsock_recv_data_bytes_scion") + metricSendDataPacketsSCION = clientmetric.NewAggregateCounter("magicsock_send_data_scion") + metricSendDataBytesSCION = clientmetric.NewAggregateCounter("magicsock_send_data_bytes_scion") + // Disco packets metricSendDiscoUDP = clientmetric.NewCounter("magicsock_disco_send_udp") metricSendDiscoDERP = clientmetric.NewCounter("magicsock_disco_send_derp") diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index b06d70a7f676c..21d02fb7445f2 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -16,11 +16,28 @@ import ( "github.com/scionproto/scion/pkg/daemon" "github.com/scionproto/scion/pkg/snet" wgconn "github.com/tailscale/wireguard-go/conn" + "tailscale.com/envknob" "tailscale.com/tailcfg" "tailscale.com/tstime/mono" "tailscale.com/types/key" ) +// debugSCIONPreference is the TS_SCION_PREFERENCE envknob controlling the +// betterAddr points bonus for SCION paths. Default 15; set to 0 to disable. +var debugSCIONPreference = envknob.RegisterInt("TS_SCION_PREFERENCE") + +// scionPreferenceBonus returns the betterAddr points bonus for SCION paths. +// Returns the value of TS_SCION_PREFERENCE if set, otherwise defaults to 15. +func scionPreferenceBonus() int { + if v := debugSCIONPreference(); v != 0 { + return v + } + if v, ok := envknob.LookupInt("TS_SCION_PREFERENCE"); ok { + return v // allow explicit 0 + } + return 15 +} + // scionPathKey is a compact index into the Conn-level scionPaths registry. // This keeps epAddr small and comparable (snet.UDPAddr contains slices). // A zero value means "not a SCION path." @@ -29,6 +46,13 @@ type scionPathKey uint32 // IsSet reports whether k refers to a valid SCION path entry. func (k scionPathKey) IsSet() bool { return k != 0 } +// scionAddrKey is a comparable key for the reverse index from (IA, host:port) +// to scionPathKey, enabling O(1) lookup in receiveSCION. +type scionAddrKey struct { + ia addr.IA + addr netip.AddrPort +} + // scionPathInfo holds the full SCION path information for a peer, indexed by // scionPathKey. The actual SCION address and path data live here rather than // in epAddr to keep epAddr comparable and small. @@ -232,8 +256,29 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo return false, fmt.Errorf("no SCION path info for key %d", addr.scionKey) } + // Read path info once for the entire batch to avoid repeated locking. + pi.mu.Lock() + path, hostAddr, peerIA := pi.path, pi.hostAddr, pi.peerIA + expired := !pi.expiry.IsZero() && time.Now().After(pi.expiry) + pi.mu.Unlock() + if expired { + return false, fmt.Errorf("SCION path expired for key %d", addr.scionKey) + } + + dst := &snet.UDPAddr{ + IA: peerIA, + Host: &net.UDPAddr{ + IP: hostAddr.Addr().AsSlice(), + Port: int(hostAddr.Port()), + }, + } + if path != nil { + dst.Path = path.Dataplane() + dst.NextHop = path.UnderlayNextHop() + } + for _, buf := range buffs { - _, err = sc.writeTo(buf[offset:], pi) + _, err = sc.conn.WriteTo(buf[offset:], dst) if err != nil { return false, err } @@ -251,6 +296,12 @@ func (c *Conn) sendSCION(sk scionPathKey, b []byte) (bool, error) { if pi == nil { return false, fmt.Errorf("no SCION path info for key %d", sk) } + pi.mu.Lock() + expired := !pi.expiry.IsZero() && time.Now().After(pi.expiry) + pi.mu.Unlock() + if expired { + return false, fmt.Errorf("SCION path expired for key %d", sk) + } _, err := sc.writeTo(b, pi) if err != nil { return false, err @@ -278,7 +329,11 @@ func (c *Conn) registerSCIONPath(pi *scionPathInfo) scionPathKey { if c.scionPaths == nil { c.scionPaths = make(map[scionPathKey]*scionPathInfo) } + if c.scionPathsByAddr == nil { + c.scionPathsByAddr = make(map[scionAddrKey]scionPathKey) + } c.scionPaths[k] = pi + c.scionPathsByAddr[scionAddrKey{ia: pi.peerIA, addr: pi.hostAddr}] = k return k } @@ -291,13 +346,20 @@ func (c *Conn) registerSCIONPathLocking(pi *scionPathInfo) scionPathKey { if c.scionPaths == nil { c.scionPaths = make(map[scionPathKey]*scionPathInfo) } + if c.scionPathsByAddr == nil { + c.scionPathsByAddr = make(map[scionAddrKey]scionPathKey) + } c.scionPaths[k] = pi + c.scionPathsByAddr[scionAddrKey{ia: pi.peerIA, addr: pi.hostAddr}] = k return k } // unregisterSCIONPath removes a SCION path entry. // c.mu must be held. func (c *Conn) unregisterSCIONPath(k scionPathKey) { + if pi, ok := c.scionPaths[k]; ok { + delete(c.scionPathsByAddr, scionAddrKey{ia: pi.peerIA, addr: pi.hostAddr}) + } delete(c.scionPaths, k) } @@ -361,6 +423,10 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) now := mono.Now() ep.lastRecvUDPAny.StoreAtomic(now) ep.noteRecvActivity(srcEpAddr, now) + if c.metrics != nil { + c.metrics.inboundPacketsSCIONTotal.Add(1) + c.metrics.inboundBytesSCIONTotal.Add(int64(n)) + } sizes[0] = n eps[0] = ep return 1, nil @@ -432,25 +498,48 @@ func totalPathLatency(p snet.Path) time.Duration { } // refreshSCIONPaths runs in a background goroutine, periodically refreshing -// SCION paths before they expire. +// SCION paths before they expire. It uses exponential backoff when the SCION +// daemon is unreachable. func (c *Conn) refreshSCIONPaths() { - ticker := time.NewTicker(30 * time.Second) + const ( + baseInterval = 30 * time.Second + maxBackoff = 10 * time.Minute + ) + ticker := time.NewTicker(baseInterval) defer ticker.Stop() + var consecutiveFailures int for { select { case <-c.donec: return case <-ticker.C: - c.refreshSCIONPathsOnce() + if consecutiveFailures > 0 { + backoff := baseInterval * time.Duration(1< maxBackoff { + backoff = maxBackoff + } + ticker.Reset(backoff) + } + if err := c.refreshSCIONPathsOnce(); err != nil { + consecutiveFailures++ + if consecutiveFailures == 5 { + c.logf("magicsock: SCION path refresh failing repeatedly (%d consecutive failures)", consecutiveFailures) + } + } else { + if consecutiveFailures > 0 { + ticker.Reset(baseInterval) + } + consecutiveFailures = 0 + } } } } -func (c *Conn) refreshSCIONPathsOnce() { +func (c *Conn) refreshSCIONPathsOnce() error { sc := c.pconnSCION if sc == nil { - return + return nil } c.mu.Lock() @@ -465,6 +554,7 @@ func (c *Conn) refreshSCIONPathsOnce() { defer cancel() now := time.Now() + var lastErr error for _, pi := range pathsCopy { pi.mu.Lock() needsRefresh := !pi.expiry.IsZero() && now.After(pi.expiry.Add(-1*time.Minute)) @@ -478,6 +568,11 @@ func (c *Conn) refreshSCIONPathsOnce() { paths, err := sc.daemon.Paths(ctx, peerIA, sc.localIA, daemon.PathReqFlags{Refresh: true}) if err != nil || len(paths) == 0 { c.logf("magicsock: SCION path refresh for %s failed: %v", peerIA, err) + if err != nil { + lastErr = err + } else { + lastErr = fmt.Errorf("no paths to %s", peerIA) + } continue } @@ -499,6 +594,7 @@ func (c *Conn) refreshSCIONPathsOnce() { } pi.mu.Unlock() } + return lastErr } // scionServiceFromPeer extracts SCION service info from a peer node's Services. @@ -600,19 +696,11 @@ func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPo } // scionKeyForAddr returns the scionPathKey for the given peer IA and host -// address, or a zero key if not found. +// address, or a zero key if not found. O(1) via reverse index. func (c *Conn) scionKeyForAddr(peerIA addr.IA, hostAddr netip.AddrPort) scionPathKey { c.mu.Lock() defer c.mu.Unlock() - for k, pi := range c.scionPaths { - pi.mu.Lock() - match := pi.peerIA == peerIA && pi.hostAddr == hostAddr - pi.mu.Unlock() - if match { - return k - } - } - return 0 + return c.scionPathsByAddr[scionAddrKey{ia: peerIA, addr: hostAddr}] } var errNoSCION = fmt.Errorf("SCION not available") diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index a3dbd252725ef..f8180962d7d65 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "net/netip" + "strings" "testing" "time" @@ -20,6 +21,7 @@ import ( "tailscale.com/net/packet" "tailscale.com/net/tstun" "tailscale.com/tailcfg" + "tailscale.com/tstime/mono" "tailscale.com/types/key" ) @@ -262,8 +264,7 @@ func TestBetterAddrSCION(t *testing.T) { a, b addrQuality want bool }{ - // SCION wins over direct UDP at equal latency. - // TODO(scion): revert when SCION preference is behind NodeAttrSCIONPrefer. + // SCION beats direct at equal latency (default +15 bonus). { name: "SCION beats direct same latency", a: alSCION(publicV4_2, 1, 100*ms), @@ -276,7 +277,7 @@ func TestBetterAddrSCION(t *testing.T) { b: alSCION(publicV4_2, 1, 100*ms), want: false, }, - // SCION wins over relay (VNI). + // SCION wins over relay (VNI) unconditionally. { name: "SCION beats relay same latency", a: alSCION(publicV4, 1, 100*ms), @@ -289,19 +290,19 @@ func TestBetterAddrSCION(t *testing.T) { b: alSCION(publicV4, 1, 100*ms), want: false, }, - // scionPreferred bonus is additive with SCION preference. + // scionPreferred bonus (+25 on top of +15) beats direct. { name: "scionPreferred SCION beats direct at similar latency", a: alSCIONPref(publicV4_2, 1, 100*ms), b: al(publicV4, 100*ms), want: true, }, - // SCION always wins over direct (current preference order). + // Direct wins when significantly faster (SCION only has +15 bonus). { - name: "SCION still wins over much faster direct", + name: "much faster direct beats SCION", a: alSCION(publicV4_2, 1, 100*ms), b: al(publicV4, 10*ms), - want: true, + want: false, }, // Two SCION paths: lower latency wins. { @@ -570,6 +571,38 @@ func TestBetterAddrSCIONWithExistingCases(t *testing.T) { } } +func TestSCIONPathRegistryReverseIndex(t *testing.T) { + c := &Conn{} + + pi := &scionPathInfo{ + peerIA: addr.MustParseIA("1-ff00:0:111"), + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + } + k := c.registerSCIONPathLocking(pi) + + // Reverse lookup should find the key. + got := c.scionKeyForAddr(pi.peerIA, pi.hostAddr) + if got != k { + t.Errorf("scionKeyForAddr returned %d, want %d", got, k) + } + + // Different address should not match. + got2 := c.scionKeyForAddr(pi.peerIA, netip.MustParseAddrPort("10.0.0.2:41641")) + if got2.IsSet() { + t.Error("scionKeyForAddr should return zero for unknown address") + } + + // Unregister should remove from reverse index. + c.mu.Lock() + c.unregisterSCIONPath(k) + c.mu.Unlock() + + got3 := c.scionKeyForAddr(pi.peerIA, pi.hostAddr) + if got3.IsSet() { + t.Error("scionKeyForAddr should return zero after unregister") + } +} + func TestSCIONEndpointState(t *testing.T) { ia := addr.MustParseIA("1-ff00:0:110") hostAddr := netip.MustParseAddrPort("192.0.2.1:41641") @@ -967,3 +1000,95 @@ func TestEpAddrSCIONAndVNIMutualExclusion(t *testing.T) { t.Errorf("String() = %q, want SCION format", got) } } + +func TestStopAndResetCleansSCIONPath(t *testing.T) { + c := &Conn{} + c.logf = t.Logf + + pi := &scionPathInfo{ + peerIA: addr.MustParseIA("1-ff00:0:111"), + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + } + k := c.registerSCIONPathLocking(pi) + + de := &endpoint{c: c} + de.scionState = &scionEndpointState{ + peerIA: pi.peerIA, + hostAddr: pi.hostAddr, + pathKey: k, + } + + de.stopAndReset() + + if de.scionState != nil { + t.Error("scionState should be nil after stopAndReset") + } + if c.lookupSCIONPathLocking(k) != nil { + t.Error("SCION path should be removed from registry after stopAndReset") + } +} + +func TestNoteRecvActivitySCIONTrustRefresh(t *testing.T) { + c := &Conn{} + de := &endpoint{c: c} + de.heartbeatDisabled = true + + scionAddr := epAddr{ap: netip.MustParseAddrPort("127.0.0.1:32766"), scionKey: 2} + plainAddr := epAddr{ap: netip.MustParseAddrPort("127.0.0.1:32766")} + + now := mono.Now() + de.bestAddr.epAddr = scionAddr + de.bestAddrAt = now + + // WireGuard data arrives with plain addr (no scionKey). + de.noteRecvActivity(plainAddr, now) + + de.mu.Lock() + trust := de.trustBestAddrUntil + de.mu.Unlock() + + if trust == 0 { + t.Error("trustBestAddrUntil should be extended for SCION bestAddr when receiving plain addr data") + } +} + +func TestSendSCIONBatchExpiredPath(t *testing.T) { + c := &Conn{} + c.pconnSCION = &scionConn{} + + pi := &scionPathInfo{ + peerIA: addr.MustParseIA("1-ff00:0:111"), + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + expiry: time.Now().Add(-1 * time.Hour), // expired + } + k := c.registerSCIONPathLocking(pi) + + ep := epAddr{ap: netip.MustParseAddrPort("10.0.0.1:41641"), scionKey: k} + _, err := c.sendSCIONBatch(ep, [][]byte{{0x01}}, 0) + if err == nil { + t.Fatal("expected error for expired path") + } + if !strings.Contains(err.Error(), "expired") { + t.Errorf("error should mention 'expired', got: %v", err) + } +} + +func TestSendSCIONExpiredPath(t *testing.T) { + c := &Conn{} + c.pconnSCION = &scionConn{} + + pi := &scionPathInfo{ + peerIA: addr.MustParseIA("1-ff00:0:111"), + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + expiry: time.Now().Add(-1 * time.Hour), // expired + } + k := c.registerSCIONPathLocking(pi) + + _, err := c.sendSCION(k, []byte{0x01}) + if err == nil { + t.Fatal("expected error for expired path") + } + if !strings.Contains(err.Error(), "expired") { + t.Errorf("error should mention 'expired', got: %v", err) + } +} From ad5ede374ede682b07359b8deef2bd991a17d638 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Tue, 10 Mar 2026 07:32:02 +0000 Subject: [PATCH 12/78] wgengine/magicsock: enhance SCION path discovery and cleanup logic - Implemented throttled re-discovery for SCION paths to improve responsiveness when paths expire. - Added cleanup of old SCION path entries outside of critical sections to prevent deadlocks. - Introduced a constant for assumed per-hop latency when SCION reports LatencyUnset, improving path latency calculations. - Updated metrics tracking for SCION disco messages to better reflect usage patterns. --- wgengine/magicsock/endpoint.go | 23 ++++++++++- wgengine/magicsock/magicsock.go | 6 +++ wgengine/magicsock/magicsock_scion.go | 48 ++++++++++++++++++---- wgengine/magicsock/magicsock_scion_test.go | 2 +- 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 86eef8359cf02..eba635e8d4b6b 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1079,6 +1079,15 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { _, err = de.c.sendSCIONBatch(udpAddr, buffs, offset) if err != nil { de.noteBadEndpoint(udpAddr) + // Trigger throttled re-discovery so we don't wait up to 30s + // for the periodic refreshSCIONPaths to fix an expired path. + de.mu.Lock() + st := de.scionState + shouldRediscover := st != nil && time.Since(st.lastDiscoveryAt) > 5*time.Second + de.mu.Unlock() + if shouldRediscover { + go de.discoverSCIONPathAsync(st.peerIA, st.hostAddr) + } } else if de.c.metrics != nil { var txBytes int for _, b := range buffs { @@ -1500,7 +1509,6 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p panic("nil node when updating endpoint") } de.mu.Lock() - defer de.mu.Unlock() de.heartbeatDisabled = heartbeatDisabled if probeUDPLifetimeEnabled { @@ -1555,6 +1563,8 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p de.relayCapable = capVerIsRelayCapable(n.Cap()) // Check for SCION service advertisement from this peer. + // Extract old SCION key for cleanup outside de.mu (lock order: c.mu before de.mu). + var oldSCIONKey scionPathKey if peerIA, hostAddr, ok := scionServiceFromPeer(n); ok { if de.scionState == nil || de.scionState.peerIA != peerIA || de.scionState.hostAddr != hostAddr { // New or changed SCION address — discover paths asynchronously @@ -1568,7 +1578,7 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p } } else if de.scionState != nil { // Peer no longer advertises SCION. - de.c.unregisterSCIONPath(de.scionState.pathKey) + oldSCIONKey = de.scionState.pathKey de.scionState = nil } @@ -1576,6 +1586,15 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p peerSCIONPrefer := n.CapMap().Contains(tailcfg.NodeAttrSCIONPrefer) selfSCIONPrefer := de.c.self.Valid() && de.c.self.CapMap().Contains(tailcfg.NodeAttrSCIONPrefer) de.scionPreferred = peerSCIONPrefer && selfSCIONPrefer && de.scionState != nil + + de.mu.Unlock() + + // Clean up SCION path outside de.mu (lock order: c.mu before de.mu). + if oldSCIONKey.IsSet() { + de.c.mu.Lock() + de.c.unregisterSCIONPath(oldSCIONKey) + de.c.mu.Unlock() + } } func (de *endpoint) setEndpointsLocked(eps interface { diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index bd3c61d77eac1..dcacea5f3ab0a 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -1985,6 +1985,8 @@ func (c *Conn) sendDiscoMessage(dst epAddr, dstKey key.NodePublic, dstDisco key. if isDERP { metricSendDiscoDERP.Add(1) + } else if dst.scionKey.IsSet() { + metricSendDiscoSCION.Add(1) } else { metricSendDiscoUDP.Add(1) } @@ -2007,6 +2009,8 @@ func (c *Conn) sendDiscoMessage(dst epAddr, dstKey key.NodePublic, dstDisco key. } if isDERP { metricSentDiscoDERP.Add(1) + } else if dst.scionKey.IsSet() { + metricSentDiscoSCION.Add(1) } else { metricSentDiscoUDP.Add(1) } @@ -4062,8 +4066,10 @@ var ( // Disco packets metricSendDiscoUDP = clientmetric.NewCounter("magicsock_disco_send_udp") metricSendDiscoDERP = clientmetric.NewCounter("magicsock_disco_send_derp") + metricSendDiscoSCION = clientmetric.NewCounter("magicsock_disco_send_scion") metricSentDiscoUDP = clientmetric.NewCounter("magicsock_disco_sent_udp") metricSentDiscoDERP = clientmetric.NewCounter("magicsock_disco_sent_derp") + metricSentDiscoSCION = clientmetric.NewCounter("magicsock_disco_sent_scion") metricSentDiscoPing = clientmetric.NewCounter("magicsock_disco_sent_ping") metricSentDiscoPong = clientmetric.NewCounter("magicsock_disco_sent_pong") metricSentDiscoPeerMTUProbes = clientmetric.NewCounter("magicsock_disco_sent_peer_mtu_probes") diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 21d02fb7445f2..89991e156a00b 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -1,3 +1,4 @@ +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package magicsock @@ -83,11 +84,16 @@ type scionPathInfo struct { // 1280 bytes (minimum IPv6-compatible MTU). const scionFallbackPayloadMTU = 1280 +// scionUnsetHopLatency is the assumed per-hop latency when the SCION daemon +// reports LatencyUnset for a hop. Conservative estimate for path selection. +const scionUnsetHopLatency = 10 * time.Millisecond + // scionEndpointState tracks SCION-specific per-peer state on an endpoint. type scionEndpointState struct { - peerIA addr.IA // peer's ISD-AS from Services advertisement - hostAddr netip.AddrPort // peer's SCION host IP:port - pathKey scionPathKey // key into Conn.scionPaths + peerIA addr.IA // peer's ISD-AS from Services advertisement + hostAddr netip.AddrPort // peer's SCION host IP:port + pathKey scionPathKey // key into Conn.scionPaths + lastDiscoveryAt time.Time // when path discovery last started (throttle) } // SCION dispatched port range. The SCION endhost stack only directly dispatches @@ -489,7 +495,7 @@ func totalPathLatency(p snet.Path) time.Duration { for _, l := range md.Latency { if l < 0 { // LatencyUnset — treat as unknown - total += 10 * time.Millisecond + total += scionUnsetHopLatency } else { total += l } @@ -523,8 +529,9 @@ func (c *Conn) refreshSCIONPaths() { } if err := c.refreshSCIONPathsOnce(); err != nil { consecutiveFailures++ - if consecutiveFailures == 5 { - c.logf("magicsock: SCION path refresh failing repeatedly (%d consecutive failures)", consecutiveFailures) + if consecutiveFailures == 1 || consecutiveFailures&(consecutiveFailures-1) == 0 { + c.logf("magicsock: SCION path refresh failed (%d consecutive): %v", + consecutiveFailures, err) } } else { if consecutiveFailures > 0 { @@ -668,20 +675,43 @@ func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { // discoverSCIONPathAsync runs SCION path discovery in a goroutine, // avoiding blocking updateFromNode which holds the endpoint lock. func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPort) { + // Record discovery start time for throttling. + de.mu.Lock() + if de.scionState != nil { + de.scionState.lastDiscoveryAt = time.Now() + } + de.mu.Unlock() + ctx, cancel := context.WithTimeout(de.c.connCtx, 10*time.Second) defer cancel() + // Capture old key before discovering new path. + de.mu.Lock() + var oldKey scionPathKey + if de.scionState != nil { + oldKey = de.scionState.pathKey + } + de.mu.Unlock() + pathKey, err := de.c.discoverSCIONPaths(ctx, peerIA, hostAddr) if err != nil { de.c.logf("magicsock: SCION path discovery for %s failed: %v", peerIA, err) return } + // Clean up old path entry if the key changed. + if oldKey.IsSet() && oldKey != pathKey { + de.c.mu.Lock() + de.c.unregisterSCIONPath(oldKey) + de.c.mu.Unlock() + } + de.mu.Lock() de.scionState = &scionEndpointState{ - peerIA: peerIA, - hostAddr: hostAddr, - pathKey: pathKey, + peerIA: peerIA, + hostAddr: hostAddr, + pathKey: pathKey, + lastDiscoveryAt: time.Now(), } de.mu.Unlock() diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index f8180962d7d65..9878d20554399 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -379,7 +379,7 @@ func TestTotalPathLatency(t *testing.T) { 3 * time.Millisecond, }, }), - want: 5*time.Millisecond + 10*time.Millisecond + 3*time.Millisecond, + want: 5*time.Millisecond + scionUnsetHopLatency + 3*time.Millisecond, }, } for _, tt := range tests { From e56a2468f3d0f57c2bc458d26ec67ab41cbb7120 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Tue, 10 Mar 2026 12:16:30 +0000 Subject: [PATCH 13/78] fix deadlock --- wgengine/magicsock/endpoint.go | 31 ++++--- wgengine/magicsock/magicsock_scion.go | 98 ++++++++++++++++------ wgengine/magicsock/magicsock_scion_test.go | 3 + 3 files changed, 97 insertions(+), 35 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index eba635e8d4b6b..244cf61b67ae9 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1079,13 +1079,13 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { _, err = de.c.sendSCIONBatch(udpAddr, buffs, offset) if err != nil { de.noteBadEndpoint(udpAddr) - // Trigger throttled re-discovery so we don't wait up to 30s - // for the periodic refreshSCIONPaths to fix an expired path. + // Trigger re-discovery so we don't wait up to 30s for the + // periodic refreshSCIONPaths to fix an expired path. + // discoverSCIONPathAsync self-throttles to once per 5s. de.mu.Lock() st := de.scionState - shouldRediscover := st != nil && time.Since(st.lastDiscoveryAt) > 5*time.Second de.mu.Unlock() - if shouldRediscover { + if st != nil { go de.discoverSCIONPathAsync(st.peerIA, st.hostAddr) } } else if de.c.metrics != nil { @@ -1589,11 +1589,10 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p de.mu.Unlock() - // Clean up SCION path outside de.mu (lock order: c.mu before de.mu). + // Clean up SCION path outside de.mu. c.mu is held by caller (updateNodes), + // so call unregisterSCIONPath directly without re-locking. if oldSCIONKey.IsSet() { - de.c.mu.Lock() de.c.unregisterSCIONPath(oldSCIONKey) - de.c.mu.Unlock() } } @@ -1953,6 +1952,16 @@ func betterAddr(a, b addrQuality) bool { return false } + // When TS_PREFER_SCION=1, SCION beats everything unconditionally. + if preferSCION() { + if a.scionKey.IsSet() && !b.scionKey.IsSet() { + return true + } + if b.scionKey.IsSet() && !a.scionKey.IsSet() { + return false + } + } + // Each address starts with a set of points (from 0 to 100) that // represents how much faster they are than the highest-latency // endpoint. For example, if a has latency 200ms and b has latency @@ -2140,7 +2149,8 @@ func (de *endpoint) populatePeerStatus(ps *ipnstate.PeerStatus) { // stopAndReset stops timers associated with de and resets its state back to zero. // It's called when a discovery endpoint is no longer present in the // NetworkMap, or when magicsock is transitioning from running to -// stopped state (via SetPrivateKey(zero)) +// stopped state (via SetPrivateKey(zero)). +// c.mu must be held. func (de *endpoint) stopAndReset() { atomic.AddInt64(&de.numStopAndResetAtomic, 1) de.mu.Lock() @@ -2172,11 +2182,10 @@ func (de *endpoint) stopAndReset() { } de.mu.Unlock() - // Clean up SCION path outside de.mu (lock order: c.mu before de.mu). + // Clean up SCION path outside de.mu. c.mu is held by caller + // (updateNodes, SetPrivateKey, Close), so call directly. if scionKey.IsSet() { - de.c.mu.Lock() de.c.unregisterSCIONPath(scionKey) - de.c.mu.Unlock() } } diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 89991e156a00b..bd688641ffe71 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -27,6 +27,11 @@ import ( // betterAddr points bonus for SCION paths. Default 15; set to 0 to disable. var debugSCIONPreference = envknob.RegisterInt("TS_SCION_PREFERENCE") +// preferSCION reports whether TS_PREFER_SCION=1 is set, which makes SCION +// paths unconditionally preferred over all other path types (direct, relay). +// Other paths are only used if no SCION path is available. +var preferSCION = envknob.RegisterBool("TS_PREFER_SCION") + // scionPreferenceBonus returns the betterAddr points bonus for SCION paths. // Returns the value of TS_SCION_PREFERENCE if set, otherwise defaults to 15. func scionPreferenceBonus() int { @@ -58,12 +63,13 @@ type scionAddrKey struct { // scionPathKey. The actual SCION address and path data live here rather than // in epAddr to keep epAddr comparable and small. type scionPathInfo struct { - peerIA addr.IA - hostAddr netip.AddrPort // peer's SCION host IP:port - path snet.Path // current best SCION path to this peer - expiry time.Time // path expiration from path metadata - mtu uint16 // SCION payload MTU from path metadata - mu sync.Mutex + peerIA addr.IA + hostAddr netip.AddrPort // peer's SCION host IP:port + path snet.Path // current best SCION path to this peer + replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) + expiry time.Time // path expiration from path metadata + mtu uint16 // SCION payload MTU from path metadata + mu sync.Mutex } // scionHeaderOverhead is the fixed overhead added by SCION encapsulation, @@ -96,13 +102,6 @@ type scionEndpointState struct { lastDiscoveryAt time.Time // when path discovery last started (throttle) } -// SCION dispatched port range. The SCION endhost stack only directly dispatches -// packets addressed to ports within this range. Port 0 means auto-select. -const ( - scionDispatchedPortMin = 30000 - scionDispatchedPortMax = 32767 -) - // scionConn wraps a SCION connection for use by magicsock. type scionConn struct { conn *snet.Conn // from SCIONNetwork.Listen() @@ -126,10 +125,17 @@ func (sc *scionConn) close() error { func (sc *scionConn) writeTo(b []byte, pi *scionPathInfo) (int, error) { pi.mu.Lock() path := pi.path + replyPath := pi.replyPath hostAddr := pi.hostAddr peerIA := pi.peerIA pi.mu.Unlock() + // If we have a replyPath (bootstrapped from an incoming packet), + // use it directly — it's already reversed by snet's ReplyPather. + if path == nil && replyPath != nil { + return sc.conn.WriteTo(b, replyPath) + } + dst := &snet.UDPAddr{ IA: peerIA, Host: &net.UDPAddr{ @@ -170,15 +176,12 @@ func scionDaemonAddr() string { // scionListenPort returns the SCION port to use, checking the TS_SCION_PORT // environment variable first, then falling back to 0 (auto-select from the -// topology's dispatched port range). If set, the port must be within the SCION -// dispatched port range. +// topology's dispatched port range). func scionListenPort() uint16 { if p := os.Getenv("TS_SCION_PORT"); p != "" { var v int - if _, err := fmt.Sscanf(p, "%d", &v); err == nil { - if v >= scionDispatchedPortMin && v <= scionDispatchedPortMax { - return uint16(v) - } + if _, err := fmt.Sscanf(p, "%d", &v); err == nil && v > 0 && v <= 65535 { + return uint16(v) } } return 0 // let snet auto-select from topology port range @@ -208,6 +211,19 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { } listenPort := scionListenPort() + if listenPort != 0 { + // Validate the configured port against the daemon's dispatched range. + portMin, portMax, err := conn.PortRange(ctx) + if err != nil { + conn.Close() + return nil, fmt.Errorf("querying SCION port range: %w", err) + } + if listenPort < portMin || listenPort > portMax { + conn.Close() + return nil, fmt.Errorf("TS_SCION_PORT=%d outside dispatched range [%d, %d]", listenPort, portMin, portMax) + } + } + listenAddr := &net.UDPAddr{ IP: net.IPv4(127, 0, 0, 1), Port: int(listenPort), @@ -264,13 +280,25 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo // Read path info once for the entire batch to avoid repeated locking. pi.mu.Lock() - path, hostAddr, peerIA := pi.path, pi.hostAddr, pi.peerIA + path, replyPath, hostAddr, peerIA := pi.path, pi.replyPath, pi.hostAddr, pi.peerIA expired := !pi.expiry.IsZero() && time.Now().After(pi.expiry) pi.mu.Unlock() if expired { return false, fmt.Errorf("SCION path expired for key %d", addr.scionKey) } + // If no discovered path, fall back to replyPath (bootstrapped from an + // incoming packet before path discovery completes). + if path == nil && replyPath != nil { + for _, buf := range buffs { + _, err = sc.conn.WriteTo(buf[offset:], replyPath) + if err != nil { + return false, err + } + } + return true, nil + } + dst := &snet.UDPAddr{ IA: peerIA, Host: &net.UDPAddr{ @@ -396,11 +424,25 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) pt, _ := packetLooksLike(b) if pt == packetLooksLikeDisco { // For disco messages, include the scionKey so pong replies - // are routed back over SCION. + // are routed back over SCION. Use a single critical section + // for the lookup+register to avoid a TOCTOU race where a + // concurrent discoverSCIONPaths could register between our + // check and our register, creating orphaned entries. srcEpAddr := epAddr{ap: srcHostAddr} - if sk := c.scionKeyForAddr(srcAddr.IA, srcHostAddr); sk.IsSet() { - srcEpAddr.scionKey = sk + c.mu.Lock() + sk := c.scionPathsByAddr[scionAddrKey{ia: srcAddr.IA, addr: srcHostAddr}] + if !sk.IsSet() { + // First disco packet from this SCION peer — bootstrap a + // reverse path entry so the pong can go back over SCION. + // ReadFrom returns a pre-reversed path suitable for replies. + sk = c.registerSCIONPath(&scionPathInfo{ + peerIA: srcAddr.IA, + hostAddr: srcHostAddr, + replyPath: srcAddr, + }) } + c.mu.Unlock() + srcEpAddr.scionKey = sk c.handleDiscoMessage(b, srcEpAddr, false, key.NodePublic{}, discoRXPathSCION) continue } @@ -674,9 +716,17 @@ func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { // discoverSCIONPathAsync runs SCION path discovery in a goroutine, // avoiding blocking updateFromNode which holds the endpoint lock. +// It self-throttles to at most once every 5 seconds to prevent concurrent +// launches (from updateFromNode and send error paths) from creating +// orphaned path entries. func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPort) { - // Record discovery start time for throttling. + // Throttle: skip if discovery ran recently. This prevents concurrent + // launches from orphaning path entries in the registry. de.mu.Lock() + if de.scionState != nil && time.Since(de.scionState.lastDiscoveryAt) < 5*time.Second { + de.mu.Unlock() + return + } if de.scionState != nil { de.scionState.lastDiscoveryAt = time.Now() } diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 9878d20554399..66e4e9648e585 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -1018,7 +1018,10 @@ func TestStopAndResetCleansSCIONPath(t *testing.T) { pathKey: k, } + // stopAndReset requires c.mu to be held (all production callers hold it). + c.mu.Lock() de.stopAndReset() + c.mu.Unlock() if de.scionState != nil { t.Error("scionState should be nil after stopAndReset") From 979f4be49f0de3e7f8737cbcfbbe1afcaf9b33da Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Tue, 10 Mar 2026 12:35:46 +0000 Subject: [PATCH 14/78] wgengine/magicsock: update SCION port test cases for accurate validation --- wgengine/magicsock/magicsock_scion_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 66e4e9648e585..e987ed5684f1d 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -683,10 +683,10 @@ func TestScionListenPort(t *testing.T) { {"valid port", "31337", 31337}, {"min port", "30000", 30000}, {"max port", "32767", 32767}, - {"below range", "29999", 0}, - {"above range", "32768", 0}, + {"below range", "29999", 29999}, // scionListenPort only parses; range validation is in trySCIONConnect + {"above range", "32768", 32768}, // same: validated against daemon port range later {"non-numeric", "abc", 0}, - {"wireguard port", "41641", 0}, + {"wireguard port", "41641", 41641}, // any valid port is accepted at parse time } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From a6874a590d45b0137e2acf3eb1493f0149b4eb6e Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Tue, 10 Mar 2026 12:59:46 +0000 Subject: [PATCH 15/78] wgengine/magicsock: optimize SCION connection setup by adjusting socket buffer sizes - Replaced the use of Listen with OpenRaw to allow setting custom UDP socket buffer sizes. - Increased the read and write buffer sizes to 7 MB to prevent packet drops at high throughput. - Wrapped the raw connection with NewCookedConn for enhanced SCION connection management. --- wgengine/magicsock/magicsock_scion.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index bd688641ffe71..f04fb8c7750f8 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -228,12 +228,34 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { IP: net.IPv4(127, 0, 0, 1), Port: int(listenPort), } - sconn, err := network.Listen(ctx, "udp", listenAddr) + + // Use OpenRaw + NewCookedConn instead of Listen so we can set socket + // buffer sizes on the underlying UDP connection before wrapping it. + pconn, err := network.OpenRaw(ctx, listenAddr) if err != nil { conn.Close() return nil, fmt.Errorf("listening on SCION %s: %w", listenAddr, err) } + // Increase underlay UDP socket buffers to match the regular magicsock + // UDP sockets (7 MB). The default kernel buffer (~212 KB) overflows + // easily at high throughput, causing packet drops and TCP retransmissions. + if pc, ok := pconn.(*snet.SCIONPacketConn); ok { + if err := pc.SetReadBuffer(socketBufferSize); err != nil { + fmt.Fprintf(os.Stderr, "magicsock: SCION: failed to set read buffer to %d: %v\n", socketBufferSize, err) + } + if err := pc.SetWriteBuffer(socketBufferSize); err != nil { + fmt.Fprintf(os.Stderr, "magicsock: SCION: failed to set write buffer to %d: %v\n", socketBufferSize, err) + } + } + + sconn, err := snet.NewCookedConn(pconn, conn) + if err != nil { + pconn.Close() + conn.Close() + return nil, fmt.Errorf("creating SCION conn: %w", err) + } + return &scionConn{ conn: sconn, localIA: localIA, From 74929ec68669c4f7aca247945fa99d9d8e4f09c7 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Tue, 10 Mar 2026 13:22:48 +0000 Subject: [PATCH 16/78] wgengine/magicsock: enhance SCION ping logic in heartbeat and discoPing methods - Added logic to ensure SCION paths are pinged during heartbeat even when a low-latency direct path is preferred. - Updated discoPing method to include SCION pings for peers when available, improving path competition and responsiveness. --- wgengine/magicsock/endpoint.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 244cf61b67ae9..617bdb65ccdfc 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -866,6 +866,16 @@ func (de *endpoint) heartbeat() { if de.wantFullPingLocked(now) { de.sendDiscoPingsLocked(now, true) + } else if de.scionState != nil && de.c.pconnSCION != nil && !de.bestAddr.isSCION() { + // Even when the current best path is "good enough" to skip a full ping + // round, heartbeat the SCION path so it can compete via betterAddr. + // Without this, SCION never gets pinged once a low-latency direct path + // suppresses wantFullPingLocked. + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: de.scionState.pathKey, + } + de.startDiscoPingLocked(scionEp, now, pingHeartbeat, 0, nil) } if de.wantUDPRelayPathDiscoveryLocked(now) { @@ -1027,6 +1037,14 @@ func (de *endpoint) discoPing(res *ipnstate.PingResult, size int, cb func(*ipnst for ep := range de.endpointState { de.startDiscoPingLocked(epAddr{ap: ep}, now, pingCLI, size, resCB) } + // Also ping over SCION if available for this peer. + if de.scionState != nil && de.c.pconnSCION != nil { + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: de.scionState.pathKey, + } + de.startDiscoPingLocked(scionEp, now, pingCLI, size, resCB) + } if de.wantUDPRelayPathDiscoveryLocked(now) { de.discoverUDPRelayPathsLocked(now) } From 270f5d0b69f677c4d6e37ce647a2853c639685fb Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Tue, 10 Mar 2026 15:25:05 +0000 Subject: [PATCH 17/78] wgengine/magicsock: improve SCION socket management and reconnection logic - Introduced mechanisms to detect dead SCION sockets and trigger reconnections based on packet reception time. - Added constants for read deadlines and reconnection thresholds to enhance socket reliability. - Enhanced the receiveSCION function to handle read timeouts and errors gracefully without propagating them to WireGuard. - Implemented path re-discovery for active SCION peers upon reconnection to ensure updated routing. --- wgengine/magicsock/endpoint.go | 30 ++-- wgengine/magicsock/magicsock.go | 9 ++ wgengine/magicsock/magicsock_scion.go | 190 +++++++++++++++++++++++++- 3 files changed, 208 insertions(+), 21 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 617bdb65ccdfc..6d3a3c90ecf34 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1859,20 +1859,22 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd wireMTU: pingSizeToPktLen(sp.size, sp.to), scionPreferred: de.scionPreferred, } - // TODO(jwhited): consider checking de.trustBestAddrUntil as well. If - // de.bestAddr is untrusted we may want to clear it, otherwise we could - // get stuck with a forever untrusted bestAddr that blackholes, since - // we don't clear direct UDP paths on disco ping timeout (see - // discoPingTimeout). - if betterAddr(thisPong, de.bestAddr) { - de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v tx=%x", de.publicKey.ShortString(), de.discoShort(), sp.to, thisPong.wireMTU, m.TxID[:6]) - de.debugUpdates.Add(EndpointChange{ - When: time.Now(), - What: "handlePongConnLocked-bestAddr-update", - From: de.bestAddr, - To: thisPong, - }) - de.setBestAddrLocked(thisPong) + // If the current bestAddr is untrusted (no recent pong confirming + // it works), allow any fresh pong to replace it. This prevents + // getting stuck with a dead bestAddr that betterAddr() refuses to + // demote due to preference rules (e.g., TS_PREFER_SCION=1). + curBestUntrusted := de.bestAddr.ap.IsValid() && now.After(de.trustBestAddrUntil) + if curBestUntrusted || betterAddr(thisPong, de.bestAddr) { + if thisPong.epAddr != de.bestAddr.epAddr { + de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v tx=%x", de.publicKey.ShortString(), de.discoShort(), sp.to, thisPong.wireMTU, m.TxID[:6]) + de.debugUpdates.Add(EndpointChange{ + When: time.Now(), + What: "handlePongConnLocked-bestAddr-update", + From: de.bestAddr, + To: thisPong, + }) + de.setBestAddrLocked(thisPong) + } } if de.bestAddr.epAddr == thisPong.epAddr { de.debugUpdates.Add(EndpointChange{ diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index dcacea5f3ab0a..103160d2479df 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -425,6 +425,10 @@ type Conn struct { scionPaths map[scionPathKey]*scionPathInfo scionPathsByAddr map[scionAddrKey]scionPathKey // reverse index for O(1) lookup scionPathSeq atomic.Uint32 // monotonic key generator for scionPaths + + // lastSCIONRecv is the last time we received any SCION packet. + // Used by receiveSCION to detect a dead socket and trigger reconnection. + lastSCIONRecv atomic.Int64 // unix nanos } // SetDebugLoggingEnabled controls whether spammy debug logging is enabled. @@ -3341,6 +3345,11 @@ func (c *connBind) Close() error { if c.closeDisco6 != nil { c.closeDisco6.Close() } + if c.pconnSCION != nil { + // Set an immediate read deadline to unblock receiveSCION. + // We don't close the SCION socket here; Conn.Close handles that. + c.pconnSCION.conn.SetReadDeadline(time.Now()) + } // Send an empty read result to unblock receiveDERP, // which will then check connBind.Closed. // connBind.Closed takes c.mu, but c.derpRecvCh is buffered. diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index f04fb8c7750f8..ea29b0e9b52b4 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -5,6 +5,7 @@ package magicsock import ( "context" + "errors" "fmt" "net" "net/netip" @@ -94,6 +95,18 @@ const scionFallbackPayloadMTU = 1280 // reports LatencyUnset for a hop. Conservative estimate for path selection. const scionUnsetHopLatency = 10 * time.Millisecond +// scionReadDeadline is the read deadline set on the SCION socket. +// If no packet is received within this duration, we check whether the +// socket is still alive. This must be long enough to avoid spurious +// reconnections during idle periods, but short enough to detect a dead +// socket promptly. +const scionReadDeadline = 30 * time.Second + +// scionReconnectThreshold is the maximum time without receiving any SCION +// packet before we consider the socket dead and attempt to reconnect. +// This is only checked when there are active SCION peers. +const scionReconnectThreshold = 30 * time.Second + // scionEndpointState tracks SCION-specific per-peer state on an endpoint. type scionEndpointState struct { peerIA addr.IA // peer's ISD-AS from Services advertisement @@ -421,23 +434,85 @@ func (c *Conn) unregisterSCIONPath(k scionPathKey) { // receiveSCION is the conn.ReceiveFunc for SCION packets. It reads from the // SCION connection and dispatches disco or WireGuard packets. +// +// Unlike receiveIP, this function handles read timeouts internally and never +// propagates them to WireGuard. This is critical because WireGuard's +// RoutineReceiveIncoming exits the goroutine permanently after 10 consecutive +// temporary errors, and we need to survive SCION socket death + reconnection. +// +// The function uses SetReadDeadline to periodically wake up and check whether +// the socket is still alive. If no packets are received for +// scionReconnectThreshold while active SCION peers exist, we close the old +// socket and reconnect. func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { sc := c.pconnSCION if sc == nil { - // Block until the Conn is closed if SCION is not available. <-c.donec return 0, net.ErrClosed } + // Initialize lastSCIONRecv so we don't trigger reconnection on startup. + c.lastSCIONRecv.CompareAndSwap(0, time.Now().UnixNano()) + for { + // Check for graceful shutdown. + select { + case <-c.donec: + return 0, net.ErrClosed + default: + } + + // Re-read pconnSCION — it may have been swapped by reconnectSCION. + sc = c.pconnSCION + if sc == nil { + // Socket was closed and reconnection failed. Retry. + select { + case <-c.donec: + return 0, net.ErrClosed + case <-time.After(5 * time.Second): + } + c.retrySCIONConnect() + continue + } + + // Set a read deadline so we wake up periodically even if the socket + // is silently dead (SCION router lost our port registration). + sc.conn.SetReadDeadline(time.Now().Add(scionReadDeadline)) + n, srcAddr, err := sc.readFrom(buffs[0]) if err != nil { - return 0, err + // Graceful shutdown: Conn is closing. + select { + case <-c.donec: + return 0, net.ErrClosed + default: + } + + // Timeout: check if we need to reconnect. + if isTimeoutError(err) { + if c.shouldReconnectSCION() { + c.reconnectSCION() + } + continue + } + + // Socket closed (by reconnectSCION or externally): re-read + // pconnSCION on next iteration. + if errors.Is(err, net.ErrClosed) { + continue + } + + // Other errors: log and continue. Never propagate to WireGuard. + c.logf("magicsock: SCION read error: %v", err) + continue } if n == 0 { continue } + // Got a packet — record receive time. + c.lastSCIONRecv.Store(time.Now().UnixNano()) + b := buffs[0][:n] srcHostAddr := srcAddr.Host.AddrPort() @@ -470,8 +545,6 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } if !c.havePrivateKey.Load() { - // No private key means we're logged out. Don't pass WireGuard - // packets up; wireguard-go will just complain. continue } @@ -482,9 +555,6 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) ep, ok := c.peerMap.endpointForEpAddr(srcEpAddr) c.mu.Unlock() if !ok { - // No peerMap entry yet. Return a lazyEndpoint so WireGuard - // can identify the peer from the encrypted packet and - // register it, matching the behavior of receiveIP. sizes[0] = n eps[0] = &lazyEndpoint{c: c, src: srcEpAddr} return 1, nil @@ -503,6 +573,112 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } } +// isTimeoutError reports whether err is a network timeout (from SetReadDeadline). +func isTimeoutError(err error) bool { + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +// shouldReconnectSCION reports whether the SCION socket appears dead and +// should be reconnected. The socket is considered dead when: +// 1. No SCION packet has been received for scionReconnectThreshold, AND +// 2. There are active SCION peers (otherwise silence is expected). +func (c *Conn) shouldReconnectSCION() bool { + lastRecv := time.Unix(0, c.lastSCIONRecv.Load()) + if time.Since(lastRecv) < scionReconnectThreshold { + return false + } + + // Check if any endpoint has SCION state (active SCION peers). + c.mu.Lock() + hasSCIONPeers := len(c.scionPaths) > 0 + c.mu.Unlock() + return hasSCIONPeers +} + +// reconnectSCION closes the current SCION socket and creates a new one. +// The receiveSCION loop will pick up the new socket on the next iteration. +func (c *Conn) reconnectSCION() { + c.logf("magicsock: SCION socket appears dead (no recv for %v), reconnecting...", scionReconnectThreshold) + + oldSC := c.pconnSCION + + // Close old connection first — we must release the port before binding + // the new socket. When TS_SCION_PORT is set, both sockets would try + // to bind the same port. This means there's a brief window where + // pconnSCION is nil and sends will fail, but that's acceptable — + // the endpoint was already dead anyway. + if oldSC != nil { + oldSC.close() + } + c.pconnSCION = nil + + newSC, err := trySCIONConnect(c.connCtx) + if err != nil { + c.logf("magicsock: SCION reconnect failed: %v", err) + // Reset the receive timestamp so we retry after scionReconnectThreshold. + c.lastSCIONRecv.Store(time.Now().UnixNano()) + return + } + + // Swap in the new connection. + c.pconnSCION = newSC + + // Reset the receive timestamp so we don't immediately re-trigger. + c.lastSCIONRecv.Store(time.Now().UnixNano()) + + c.logf("magicsock: SCION reconnected successfully, local IA: %s", newSC.localIA) + + // Re-discover paths for all SCION peers. We need fresh paths that + // use the new socket's local address. + c.rediscoverAllSCIONPaths() +} + +// retrySCIONConnect attempts to re-establish a SCION connection when +// pconnSCION is nil (previous reconnect attempt failed). +func (c *Conn) retrySCIONConnect() { + if c.pconnSCION != nil { + return // another goroutine beat us to it + } + newSC, err := trySCIONConnect(c.connCtx) + if err != nil { + c.logf("magicsock: SCION reconnect retry failed: %v", err) + return + } + c.pconnSCION = newSC + c.lastSCIONRecv.Store(time.Now().UnixNano()) + c.logf("magicsock: SCION reconnect retry succeeded, local IA: %s", newSC.localIA) + c.rediscoverAllSCIONPaths() +} + +// rediscoverAllSCIONPaths triggers path re-discovery for all endpoints that +// have SCION state. This is called after reconnecting the SCION socket to +// ensure paths reference the new connection. +func (c *Conn) rediscoverAllSCIONPaths() { + c.mu.Lock() + var peers []struct { + ep *endpoint + peerIA addr.IA + hostAddr netip.AddrPort + } + c.peerMap.forEachEndpoint(func(ep *endpoint) { + ep.mu.Lock() + if ep.scionState != nil { + peers = append(peers, struct { + ep *endpoint + peerIA addr.IA + hostAddr netip.AddrPort + }{ep, ep.scionState.peerIA, ep.scionState.hostAddr}) + } + ep.mu.Unlock() + }) + c.mu.Unlock() + + for _, p := range peers { + go p.ep.discoverSCIONPathAsync(p.peerIA, p.hostAddr) + } +} + // discoverSCIONPaths queries the SCION daemon for paths to the given peer IA, // selects the best one, and stores it in the path registry. Returns the // scionPathKey for the path. From 12cdd7dcf9adecd62338cdda45dba9631029c64b Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Tue, 10 Mar 2026 17:07:55 +0000 Subject: [PATCH 18/78] wgengine/magicsock: refactor SCION mulitple path management --- wgengine/magicsock/endpoint.go | 89 +++-- wgengine/magicsock/magicsock_scion.go | 397 ++++++++++++++++----- wgengine/magicsock/magicsock_scion_test.go | 52 ++- 3 files changed, 401 insertions(+), 137 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 6d3a3c90ecf34..cd096b904deea 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -868,14 +868,20 @@ func (de *endpoint) heartbeat() { de.sendDiscoPingsLocked(now, true) } else if de.scionState != nil && de.c.pconnSCION != nil && !de.bestAddr.isSCION() { // Even when the current best path is "good enough" to skip a full ping - // round, heartbeat the SCION path so it can compete via betterAddr. + // round, heartbeat all SCION paths so they can compete via betterAddr. // Without this, SCION never gets pinged once a low-latency direct path // suppresses wantFullPingLocked. - scionEp := epAddr{ - ap: de.scionState.hostAddr, - scionKey: de.scionState.pathKey, + for pk, ps := range de.scionState.paths { + if !ps.lastPing.IsZero() && now.Sub(ps.lastPing) < discoPingInterval { + continue + } + ps.lastPing = now + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: pk, + } + de.startDiscoPingLocked(scionEp, now, pingHeartbeat, 0, nil) } - de.startDiscoPingLocked(scionEp, now, pingHeartbeat, 0, nil) } if de.wantUDPRelayPathDiscoveryLocked(now) { @@ -1037,13 +1043,15 @@ func (de *endpoint) discoPing(res *ipnstate.PingResult, size int, cb func(*ipnst for ep := range de.endpointState { de.startDiscoPingLocked(epAddr{ap: ep}, now, pingCLI, size, resCB) } - // Also ping over SCION if available for this peer. + // Also ping all SCION paths if available for this peer. if de.scionState != nil && de.c.pconnSCION != nil { - scionEp := epAddr{ - ap: de.scionState.hostAddr, - scionKey: de.scionState.pathKey, + for pk := range de.scionState.paths { + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: pk, + } + de.startDiscoPingLocked(scionEp, now, pingCLI, size, resCB) } - de.startDiscoPingLocked(scionEp, now, pingCLI, size, resCB) } if de.wantUDPRelayPathDiscoveryLocked(now) { de.discoverUDPRelayPathsLocked(now) @@ -1419,13 +1427,19 @@ func (de *endpoint) sendDiscoPingsLocked(now mono.Time, sendCallMeMaybe bool) { de.startDiscoPingLocked(epAddr{ap: ep}, now, pingDiscovery, 0, nil) } - // Also ping over SCION if available for this peer. + // Also ping all SCION paths if available for this peer. if de.scionState != nil && de.c.pconnSCION != nil { - scionEp := epAddr{ - ap: de.scionState.hostAddr, - scionKey: de.scionState.pathKey, + for pk, ps := range de.scionState.paths { + if !ps.lastPing.IsZero() && now.Sub(ps.lastPing) < discoPingInterval { + continue + } + ps.lastPing = now + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: pk, + } + de.startDiscoPingLocked(scionEp, now, pingDiscovery, 0, nil) } - de.startDiscoPingLocked(scionEp, now, pingDiscovery, 0, nil) sentAny = true } @@ -1581,8 +1595,8 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p de.relayCapable = capVerIsRelayCapable(n.Cap()) // Check for SCION service advertisement from this peer. - // Extract old SCION key for cleanup outside de.mu (lock order: c.mu before de.mu). - var oldSCIONKey scionPathKey + // Extract old SCION keys for cleanup outside de.mu (lock order: c.mu before de.mu). + var oldSCIONKeys []scionPathKey if peerIA, hostAddr, ok := scionServiceFromPeer(n); ok { if de.scionState == nil || de.scionState.peerIA != peerIA || de.scionState.hostAddr != hostAddr { // New or changed SCION address — discover paths asynchronously @@ -1596,7 +1610,9 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p } } else if de.scionState != nil { // Peer no longer advertises SCION. - oldSCIONKey = de.scionState.pathKey + for k := range de.scionState.paths { + oldSCIONKeys = append(oldSCIONKeys, k) + } de.scionState = nil } @@ -1607,10 +1623,10 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p de.mu.Unlock() - // Clean up SCION path outside de.mu. c.mu is held by caller (updateNodes), + // Clean up SCION paths outside de.mu. c.mu is held by caller (updateNodes), // so call unregisterSCIONPath directly without re-locking. - if oldSCIONKey.IsSet() { - de.c.unregisterSCIONPath(oldSCIONKey) + for _, k := range oldSCIONKeys { + de.c.unregisterSCIONPath(k) } } @@ -1834,6 +1850,18 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd }) } + // Record latency for SCION paths in per-path probe state. + if !isDerp && src.scionKey.IsSet() { + if de.scionState != nil { + if ps, ok := de.scionState.paths[src.scionKey]; ok { + ps.addPongReply(scionPongReply{ + latency: latency, + pongAt: now, + }) + } + } + } + if sp.purpose != pingHeartbeat && sp.purpose != pingHeartbeatForUDPLifetime { de.c.dlogf("[v1] magicsock: disco: %v<-%v (%v, %v) got pong tx=%x latency=%v pktlen=%v pong.src=%v%v", de.c.discoAtomic.Short(), de.discoShort(), de.publicKey.ShortString(), src, m.TxID[:6], latency.Round(time.Millisecond), pktLen, m.Src, logger.ArgWriter(func(bw *bufio.Writer) { if sp.to != src { @@ -1874,6 +1902,11 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd To: thisPong, }) de.setBestAddrLocked(thisPong) + // Update activePath when switching to a SCION path. + if thisPong.epAddr.scionKey.IsSet() && de.scionState != nil { + de.scionState.activePath = thisPong.epAddr.scionKey + go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, thisPong.epAddr.scionKey) + } } } if de.bestAddr.epAddr == thisPong.epAddr { @@ -2175,11 +2208,13 @@ func (de *endpoint) stopAndReset() { atomic.AddInt64(&de.numStopAndResetAtomic, 1) de.mu.Lock() - // Extract scionPathKey before releasing de.mu so we can clean it up + // Extract scionPathKeys before releasing de.mu so we can clean them up // under c.mu afterward (lock order: c.mu before de.mu). - var scionKey scionPathKey + var scionKeys []scionPathKey if de.scionState != nil { - scionKey = de.scionState.pathKey + for k := range de.scionState.paths { + scionKeys = append(scionKeys, k) + } de.scionState = nil } @@ -2202,10 +2237,10 @@ func (de *endpoint) stopAndReset() { } de.mu.Unlock() - // Clean up SCION path outside de.mu. c.mu is held by caller + // Clean up SCION paths outside de.mu. c.mu is held by caller // (updateNodes, SetPrivateKey, Close), so call directly. - if scionKey.IsSet() { - de.c.unregisterSCIONPath(scionKey) + for _, k := range scionKeys { + de.c.unregisterSCIONPath(k) } } diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index ea29b0e9b52b4..fdf8ed24e23ca 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -10,6 +10,7 @@ import ( "net" "net/netip" "os" + "sort" "strings" "sync" "time" @@ -64,13 +65,14 @@ type scionAddrKey struct { // scionPathKey. The actual SCION address and path data live here rather than // in epAddr to keep epAddr comparable and small. type scionPathInfo struct { - peerIA addr.IA - hostAddr netip.AddrPort // peer's SCION host IP:port - path snet.Path // current best SCION path to this peer - replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) - expiry time.Time // path expiration from path metadata - mtu uint16 // SCION payload MTU from path metadata - mu sync.Mutex + peerIA addr.IA + hostAddr netip.AddrPort // peer's SCION host IP:port + fingerprint snet.PathFingerprint // SHA256 of interface sequence; for matching across refreshes + path snet.Path // current best SCION path to this peer + replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) + expiry time.Time // path expiration from path metadata + mtu uint16 // SCION payload MTU from path metadata + mu sync.Mutex } // scionHeaderOverhead is the fixed overhead added by SCION encapsulation, @@ -107,12 +109,60 @@ const scionReadDeadline = 30 * time.Second // This is only checked when there are active SCION peers. const scionReconnectThreshold = 30 * time.Second +// defaultSCIONProbePaths is the default number of SCION paths to probe per peer. +const defaultSCIONProbePaths = 5 + +// scionPongHistoryCount is the ring buffer size for per-path pong latency tracking. +const scionPongHistoryCount = 8 + +// scionMaxProbePaths returns the max number of SCION paths to probe per peer. +// Defaults to 5, overridable via TS_SCION_MAX_PROBE_PATHS. +func scionMaxProbePaths() int { + if v, ok := envknob.LookupInt("TS_SCION_MAX_PROBE_PATHS"); ok && v > 0 { + return v + } + return defaultSCIONProbePaths +} + // scionEndpointState tracks SCION-specific per-peer state on an endpoint. type scionEndpointState struct { - peerIA addr.IA // peer's ISD-AS from Services advertisement - hostAddr netip.AddrPort // peer's SCION host IP:port - pathKey scionPathKey // key into Conn.scionPaths - lastDiscoveryAt time.Time // when path discovery last started (throttle) + peerIA addr.IA // peer's ISD-AS from Services advertisement + hostAddr netip.AddrPort // peer's SCION host IP:port + paths map[scionPathKey]*scionPathProbeState // probed paths (up to scionMaxProbePaths) + activePath scionPathKey // currently selected best path for data + lastDiscoveryAt time.Time // when path discovery last started (throttle) +} + +// scionPathProbeState tracks disco probing state for one SCION path. +type scionPathProbeState struct { + fingerprint snet.PathFingerprint + lastPing mono.Time + recentPongs [scionPongHistoryCount]scionPongReply // ring buffer + recentPong uint16 // index of most recent entry + pongCount uint16 // total pongs received (capped at ring size) +} + +// scionPongReply records one pong measurement for a SCION path. +type scionPongReply struct { + latency time.Duration + pongAt mono.Time +} + +// addPongReply records a pong measurement in the ring buffer. +func (ps *scionPathProbeState) addPongReply(r scionPongReply) { + ps.recentPong = (ps.recentPong + 1) % scionPongHistoryCount + ps.recentPongs[ps.recentPong] = r + if ps.pongCount < scionPongHistoryCount { + ps.pongCount++ + } +} + +// latency returns the most recent pong latency, or time.Hour if no pongs received. +func (ps *scionPathProbeState) latency() time.Duration { + if ps.pongCount == 0 { + return time.Hour + } + return ps.recentPongs[ps.recentPong].latency } // scionConn wraps a SCION connection for use by magicsock. @@ -398,40 +448,56 @@ func (c *Conn) registerSCIONPath(pi *scionPathInfo) scionPathKey { if c.scionPaths == nil { c.scionPaths = make(map[scionPathKey]*scionPathInfo) } - if c.scionPathsByAddr == nil { - c.scionPathsByAddr = make(map[scionAddrKey]scionPathKey) - } c.scionPaths[k] = pi - c.scionPathsByAddr[scionAddrKey{ia: pi.peerIA, addr: pi.hostAddr}] = k + // Don't unconditionally overwrite scionPathsByAddr here — with multi-path, + // multiple keys share the same (IA, hostAddr). The caller is responsible + // for setting the active path via setActiveSCIONPath. return k } // registerSCIONPathLocking stores a scionPathInfo, acquiring c.mu, and returns // a key for it. func (c *Conn) registerSCIONPathLocking(pi *scionPathInfo) scionPathKey { - k := scionPathKey(c.scionPathSeq.Add(1)) c.mu.Lock() defer c.mu.Unlock() - if c.scionPaths == nil { - c.scionPaths = make(map[scionPathKey]*scionPathInfo) - } - if c.scionPathsByAddr == nil { - c.scionPathsByAddr = make(map[scionAddrKey]scionPathKey) - } - c.scionPaths[k] = pi - c.scionPathsByAddr[scionAddrKey{ia: pi.peerIA, addr: pi.hostAddr}] = k - return k + return c.registerSCIONPath(pi) } -// unregisterSCIONPath removes a SCION path entry. +// unregisterSCIONPath removes a SCION path entry and its peerMap entry. // c.mu must be held. func (c *Conn) unregisterSCIONPath(k scionPathKey) { if pi, ok := c.scionPaths[k]; ok { - delete(c.scionPathsByAddr, scionAddrKey{ia: pi.peerIA, addr: pi.hostAddr}) + // Only remove reverse index if it points to this key. + ak := scionAddrKey{ia: pi.peerIA, addr: pi.hostAddr} + if c.scionPathsByAddr[ak] == k { + delete(c.scionPathsByAddr, ak) + } + // Remove stale peerMap entry for this scionKey. + scionEp := epAddr{ap: pi.hostAddr, scionKey: k} + if peerInf := c.peerMap.byEpAddr[scionEp]; peerInf != nil { + delete(peerInf.epAddrs, scionEp) + delete(c.peerMap.byEpAddr, scionEp) + } } delete(c.scionPaths, k) } +// setActiveSCIONPath updates the reverse index to point to the given key. +// c.mu must be held. +func (c *Conn) setActiveSCIONPath(peerIA addr.IA, hostAddr netip.AddrPort, k scionPathKey) { + if c.scionPathsByAddr == nil { + c.scionPathsByAddr = make(map[scionAddrKey]scionPathKey) + } + c.scionPathsByAddr[scionAddrKey{ia: peerIA, addr: hostAddr}] = k +} + +// updateActiveSCIONPathLocking updates the reverse index, acquiring c.mu. +func (c *Conn) updateActiveSCIONPathLocking(peerIA addr.IA, hostAddr netip.AddrPort, k scionPathKey) { + c.mu.Lock() + defer c.mu.Unlock() + c.setActiveSCIONPath(peerIA, hostAddr, k) +} + // receiveSCION is the conn.ReceiveFunc for SCION packets. It reads from the // SCION connection and dispatches disco or WireGuard packets. // @@ -537,6 +603,7 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) hostAddr: srcHostAddr, replyPath: srcAddr, }) + c.setActiveSCIONPath(srcAddr.IA, srcHostAddr, sk) } c.mu.Unlock() srcEpAddr.scionKey = sk @@ -680,48 +747,83 @@ func (c *Conn) rediscoverAllSCIONPaths() { } // discoverSCIONPaths queries the SCION daemon for paths to the given peer IA, -// selects the best one, and stores it in the path registry. Returns the -// scionPathKey for the path. -func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr netip.AddrPort) (scionPathKey, error) { +// deduplicates by fingerprint, selects the top N by latency, and stores them +// in the path registry. Returns the scionPathKeys for the registered paths +// (first element is the lowest-latency path). +func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr netip.AddrPort) ([]scionPathKey, error) { sc := c.pconnSCION if sc == nil { - return 0, errNoSCION + return nil, errNoSCION } paths, err := sc.daemon.Paths(ctx, peerIA, sc.localIA, daemon.PathReqFlags{Refresh: false}) if err != nil { - return 0, fmt.Errorf("querying SCION paths to %s: %w", peerIA, err) + return nil, fmt.Errorf("querying SCION paths to %s: %w", peerIA, err) } if len(paths) == 0 { - return 0, fmt.Errorf("no SCION paths to %s", peerIA) + return nil, fmt.Errorf("no SCION paths to %s", peerIA) } - // Pick the path with lowest total latency. - best := paths[0] - bestLatency := totalPathLatency(best) - for _, p := range paths[1:] { - lat := totalPathLatency(p) - if lat < bestLatency { - best = p - bestLatency = lat + // Deduplicate by fingerprint (topologically identical paths). + type pathWithMeta struct { + path snet.Path + fingerprint snet.PathFingerprint + latency time.Duration + } + seen := make(map[snet.PathFingerprint]bool) + var unique []pathWithMeta + for _, p := range paths { + fp := snet.Fingerprint(p) + if fp != "" && seen[fp] { + continue + } + if fp != "" { + seen[fp] = true } + unique = append(unique, pathWithMeta{ + path: p, + fingerprint: fp, + latency: totalPathLatency(p), + }) } - var expiry time.Time - var mtu uint16 - if md := best.Metadata(); md != nil { - expiry = md.Expiry - mtu = md.MTU + // Sort by latency ascending. + sort.Slice(unique, func(i, j int) bool { + return unique[i].latency < unique[j].latency + }) + + // Take top N. + maxPaths := scionMaxProbePaths() + if len(unique) > maxPaths { + unique = unique[:maxPaths] } - pi := &scionPathInfo{ - peerIA: peerIA, - hostAddr: hostAddr, - path: best, - expiry: expiry, - mtu: mtu, + // Register each path. + c.mu.Lock() + defer c.mu.Unlock() + keys := make([]scionPathKey, 0, len(unique)) + for _, u := range unique { + var expiry time.Time + var mtu uint16 + if md := u.path.Metadata(); md != nil { + expiry = md.Expiry + mtu = md.MTU + } + pi := &scionPathInfo{ + peerIA: peerIA, + hostAddr: hostAddr, + fingerprint: u.fingerprint, + path: u.path, + expiry: expiry, + mtu: mtu, + } + keys = append(keys, c.registerSCIONPath(pi)) } - return c.registerSCIONPathLocking(pi), nil + // Set the first (lowest-latency) path as active for the reverse index. + if len(keys) > 0 { + c.setActiveSCIONPath(peerIA, hostAddr, keys[0]) + } + return keys, nil } // totalPathLatency returns the sum of all hop latencies for a SCION path. @@ -797,49 +899,115 @@ func (c *Conn) refreshSCIONPathsOnce() error { } c.mu.Unlock() - ctx, cancel := context.WithTimeout(c.connCtx, 10*time.Second) - defer cancel() - + // Group paths by peerIA so we query the daemon once per peer. + type peerGroup struct { + peerIA addr.IA + needRefresh bool + keys []scionPathKey + infos []*scionPathInfo + } + groups := make(map[addr.IA]*peerGroup) now := time.Now() - var lastErr error - for _, pi := range pathsCopy { + for k, pi := range pathsCopy { pi.mu.Lock() - needsRefresh := !pi.expiry.IsZero() && now.After(pi.expiry.Add(-1*time.Minute)) peerIA := pi.peerIA + needsRefresh := !pi.expiry.IsZero() && now.After(pi.expiry.Add(-1*time.Minute)) pi.mu.Unlock() - if !needsRefresh { + g := groups[peerIA] + if g == nil { + g = &peerGroup{peerIA: peerIA} + groups[peerIA] = g + } + g.keys = append(g.keys, k) + g.infos = append(g.infos, pi) + if needsRefresh { + g.needRefresh = true + } + } + + ctx, cancel := context.WithTimeout(c.connCtx, 10*time.Second) + defer cancel() + + var lastErr error + for _, g := range groups { + if !g.needRefresh { continue } - paths, err := sc.daemon.Paths(ctx, peerIA, sc.localIA, daemon.PathReqFlags{Refresh: true}) - if err != nil || len(paths) == 0 { - c.logf("magicsock: SCION path refresh for %s failed: %v", peerIA, err) + daemonPaths, err := sc.daemon.Paths(ctx, g.peerIA, sc.localIA, daemon.PathReqFlags{Refresh: true}) + if err != nil || len(daemonPaths) == 0 { + c.logf("magicsock: SCION path refresh for %s failed: %v", g.peerIA, err) if err != nil { lastErr = err } else { - lastErr = fmt.Errorf("no paths to %s", peerIA) + lastErr = fmt.Errorf("no paths to %s", g.peerIA) } continue } - best := paths[0] - bestLatency := totalPathLatency(best) - for _, p := range paths[1:] { + // Index daemon paths by fingerprint for matching. + type daemonPathEntry struct { + path snet.Path + fp snet.PathFingerprint + } + var daemonByFP []daemonPathEntry + for _, dp := range daemonPaths { + daemonByFP = append(daemonByFP, daemonPathEntry{ + path: dp, + fp: snet.Fingerprint(dp), + }) + } + + // Find the best daemon path for fallback use. + bestDaemon := daemonPaths[0] + bestDaemonLat := totalPathLatency(bestDaemon) + for _, p := range daemonPaths[1:] { lat := totalPathLatency(p) - if lat < bestLatency { - best = p - bestLatency = lat + if lat < bestDaemonLat { + bestDaemon = p + bestDaemonLat = lat } } - pi.mu.Lock() - pi.path = best - if md := best.Metadata(); md != nil { - pi.expiry = md.Expiry - pi.mtu = md.MTU + // Match existing registered paths to daemon paths by fingerprint. + // Unmatched paths with known fingerprints (disappeared from daemon) + // are left unchanged — they'll be replaced on the next + // discoverSCIONPathAsync cycle. Paths with empty fingerprints + // (no metadata) get the best daemon path as fallback. + for _, pi := range g.infos { + pi.mu.Lock() + fp := pi.fingerprint + pi.mu.Unlock() + + var matched snet.Path + if fp != "" { + for _, d := range daemonByFP { + if d.fp == fp { + matched = d.path + break + } + } + if matched == nil { + // Known fingerprint disappeared from daemon results. + // Skip — don't overwrite with a different topology. + continue + } + } else { + // No fingerprint (missing metadata). Use best daemon path. + matched = bestDaemon + } + + pi.mu.Lock() + pi.path = matched + newFP := snet.Fingerprint(matched) + pi.fingerprint = newFP + if md := matched.Metadata(); md != nil { + pi.expiry = md.Expiry + pi.mtu = md.MTU + } + pi.mu.Unlock() } - pi.mu.Unlock() } return lastErr } @@ -933,44 +1101,85 @@ func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPo ctx, cancel := context.WithTimeout(de.c.connCtx, 10*time.Second) defer cancel() - // Capture old key before discovering new path. + // Capture old keys before discovering new paths. de.mu.Lock() - var oldKey scionPathKey + var oldKeys []scionPathKey if de.scionState != nil { - oldKey = de.scionState.pathKey + for k := range de.scionState.paths { + oldKeys = append(oldKeys, k) + } } de.mu.Unlock() - pathKey, err := de.c.discoverSCIONPaths(ctx, peerIA, hostAddr) + newKeys, err := de.c.discoverSCIONPaths(ctx, peerIA, hostAddr) if err != nil { de.c.logf("magicsock: SCION path discovery for %s failed: %v", peerIA, err) return } - // Clean up old path entry if the key changed. - if oldKey.IsSet() && oldKey != pathKey { - de.c.mu.Lock() - de.c.unregisterSCIONPath(oldKey) - de.c.mu.Unlock() + // Build set of new keys for fast lookup. + newKeySet := make(map[scionPathKey]bool, len(newKeys)) + for _, k := range newKeys { + newKeySet[k] = true + } + + // Extract fingerprints under c.mu (must be acquired before de.mu per lock ordering). + de.c.mu.Lock() + fpByKey := make(map[scionPathKey]snet.PathFingerprint, len(newKeys)) + for _, k := range newKeys { + if pi := de.c.lookupSCIONPath(k); pi != nil { + fpByKey[k] = pi.fingerprint + } + } + // Clean up old keys that aren't in the new set. + for _, k := range oldKeys { + if !newKeySet[k] { + de.c.unregisterSCIONPath(k) + } } + de.c.mu.Unlock() + // Build probe state map, preserving history for surviving paths by fingerprint. de.mu.Lock() + var oldProbeByFP map[snet.PathFingerprint]*scionPathProbeState + if de.scionState != nil { + oldProbeByFP = make(map[snet.PathFingerprint]*scionPathProbeState, len(de.scionState.paths)) + for _, ps := range de.scionState.paths { + if ps.fingerprint != "" { + oldProbeByFP[ps.fingerprint] = ps + } + } + } + + newPaths := make(map[scionPathKey]*scionPathProbeState, len(newKeys)) + for _, k := range newKeys { + fp := fpByKey[k] + // Preserve existing probe history if the fingerprint matches. + if fp != "" && oldProbeByFP != nil { + if old, ok := oldProbeByFP[fp]; ok { + old.fingerprint = fp // ensure set + newPaths[k] = old + continue + } + } + newPaths[k] = &scionPathProbeState{fingerprint: fp} + } + + activePath := scionPathKey(0) + if len(newKeys) > 0 { + activePath = newKeys[0] + } + de.scionState = &scionEndpointState{ peerIA: peerIA, hostAddr: hostAddr, - pathKey: pathKey, + paths: newPaths, + activePath: activePath, lastDiscoveryAt: time.Now(), } de.mu.Unlock() - pi := de.c.lookupSCIONPathLocking(pathKey) - var mtu uint16 - if pi != nil { - pi.mu.Lock() - mtu = pi.mtu - pi.mu.Unlock() - } - de.c.logf("magicsock: discovered SCION path to %s (key=%d, mtu=%d)", peerIA, pathKey, mtu) + de.c.logf("magicsock: discovered %d SCION paths to %s (active key=%d)", len(newKeys), peerIA, activePath) } // scionKeyForAddr returns the scionPathKey for the given peer IA and host diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index e987ed5684f1d..939f89cda688f 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -580,6 +580,11 @@ func TestSCIONPathRegistryReverseIndex(t *testing.T) { } k := c.registerSCIONPathLocking(pi) + // Set as active path so the reverse index is populated. + c.mu.Lock() + c.setActiveSCIONPath(pi.peerIA, pi.hostAddr, k) + c.mu.Unlock() + // Reverse lookup should find the key. got := c.scionKeyForAddr(pi.peerIA, pi.hostAddr) if got != k { @@ -607,10 +612,12 @@ func TestSCIONEndpointState(t *testing.T) { ia := addr.MustParseIA("1-ff00:0:110") hostAddr := netip.MustParseAddrPort("192.0.2.1:41641") + pk := scionPathKey(5) st := &scionEndpointState{ - peerIA: ia, - hostAddr: hostAddr, - pathKey: scionPathKey(5), + peerIA: ia, + hostAddr: hostAddr, + paths: map[scionPathKey]*scionPathProbeState{pk: {}}, + activePath: pk, } if st.peerIA != ia { @@ -619,8 +626,11 @@ func TestSCIONEndpointState(t *testing.T) { if st.hostAddr != hostAddr { t.Errorf("hostAddr = %v, want %v", st.hostAddr, hostAddr) } - if !st.pathKey.IsSet() { - t.Error("pathKey should be set") + if !st.activePath.IsSet() { + t.Error("activePath should be set") + } + if len(st.paths) != 1 { + t.Errorf("paths count = %d, want 1", len(st.paths)) } } @@ -739,15 +749,21 @@ func TestDiscoverSCIONPaths(t *testing.T) { c := &Conn{} c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} - k, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) + keys, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) if err != nil { t.Fatalf("unexpected error: %v", err) } - if !k.IsSet() { - t.Fatal("returned key should be set") + if len(keys) == 0 { + t.Fatal("returned keys should not be empty") + } + // All 3 paths should be registered (deduped by fingerprint, but mock + // paths have no metadata for fingerprinting so they're all unique). + if len(keys) != 3 { + t.Fatalf("expected 3 keys, got %d", len(keys)) } - pi := c.lookupSCIONPathLocking(k) + // First key should be the lowest-latency path (fast one, 5ms). + pi := c.lookupSCIONPathLocking(keys[0]) if pi == nil { t.Fatal("path info not found in registry") } @@ -757,9 +773,9 @@ func TestDiscoverSCIONPaths(t *testing.T) { if pi.hostAddr != hostAddr { t.Errorf("hostAddr = %v, want %v", pi.hostAddr, hostAddr) } - // The selected path should be the fast one (5ms). + // The first (active) path should be the fast one (5ms). if pi.path != fastPath { - t.Error("should have selected the lowest-latency path") + t.Error("first key should be the lowest-latency path") } }) @@ -805,11 +821,14 @@ func TestDiscoverSCIONPaths(t *testing.T) { c := &Conn{} c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} - k, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) + keys, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) if err != nil { t.Fatalf("unexpected error: %v", err) } - pi := c.lookupSCIONPathLocking(k) + if len(keys) == 0 { + t.Fatal("returned keys should not be empty") + } + pi := c.lookupSCIONPathLocking(keys[0]) if pi == nil { t.Fatal("path info not found") } @@ -1013,9 +1032,10 @@ func TestStopAndResetCleansSCIONPath(t *testing.T) { de := &endpoint{c: c} de.scionState = &scionEndpointState{ - peerIA: pi.peerIA, - hostAddr: pi.hostAddr, - pathKey: k, + peerIA: pi.peerIA, + hostAddr: pi.hostAddr, + paths: map[scionPathKey]*scionPathProbeState{k: {}}, + activePath: k, } // stopAndReset requires c.mu to be held (all production callers hold it). From 1f7c97954b54cf61fcbaab4671288942599de661 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Tue, 10 Mar 2026 21:21:51 +0000 Subject: [PATCH 19/78] wgengine/magicsock: enhance SCION path handling with cached destination addresses - Introduced a cached destination address in scionPathInfo to optimize path resolution. - Updated writeTo and sendSCIONBatch methods to utilize cached destination for improved performance. - Refactored lastSCIONRecv to use monotonic time for better performance - Ensured buildCachedDst is called during path updates to maintain cache consistency. --- wgengine/magicsock/magicsock.go | 4 +- wgengine/magicsock/magicsock_scion.go | 88 +++++++++++++-------------- 2 files changed, 44 insertions(+), 48 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 103160d2479df..e9c9a5bb8850d 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -426,9 +426,9 @@ type Conn struct { scionPathsByAddr map[scionAddrKey]scionPathKey // reverse index for O(1) lookup scionPathSeq atomic.Uint32 // monotonic key generator for scionPaths - // lastSCIONRecv is the last time we received any SCION packet. + // lastSCIONRecv is the last time we received any SCION packet (monotonic). // Used by receiveSCION to detect a dead socket and trigger reconnection. - lastSCIONRecv atomic.Int64 // unix nanos + lastSCIONRecv mono.Time } // SetDebugLoggingEnabled controls whether spammy debug logging is enabled. diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index fdf8ed24e23ca..d9d6e9fac32b4 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -70,11 +70,29 @@ type scionPathInfo struct { fingerprint snet.PathFingerprint // SHA256 of interface sequence; for matching across refreshes path snet.Path // current best SCION path to this peer replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) + cachedDst *snet.UDPAddr // pre-built destination addr; rebuilt when path changes expiry time.Time // path expiration from path metadata mtu uint16 // SCION payload MTU from path metadata mu sync.Mutex } +// buildCachedDst constructs the cached destination address from the current +// path info. Must be called with pi.mu held (or before the info is shared). +func (pi *scionPathInfo) buildCachedDst() { + dst := &snet.UDPAddr{ + IA: pi.peerIA, + Host: &net.UDPAddr{ + IP: pi.hostAddr.Addr().AsSlice(), + Port: int(pi.hostAddr.Port()), + }, + } + if pi.path != nil { + dst.Path = pi.path.Dataplane() + dst.NextHop = pi.path.UnderlayNextHop() + } + pi.cachedDst = dst +} + // scionHeaderOverhead is the fixed overhead added by SCION encapsulation, // excluding the variable-length path header: // - Underlay IPv4+UDP: 20 + 8 = 28 bytes @@ -187,30 +205,17 @@ func (sc *scionConn) close() error { // writeTo sends b to a peer identified by the given scionPathInfo. func (sc *scionConn) writeTo(b []byte, pi *scionPathInfo) (int, error) { pi.mu.Lock() - path := pi.path replyPath := pi.replyPath - hostAddr := pi.hostAddr - peerIA := pi.peerIA + cachedDst := pi.cachedDst pi.mu.Unlock() - // If we have a replyPath (bootstrapped from an incoming packet), - // use it directly — it's already reversed by snet's ReplyPather. - if path == nil && replyPath != nil { - return sc.conn.WriteTo(b, replyPath) - } - - dst := &snet.UDPAddr{ - IA: peerIA, - Host: &net.UDPAddr{ - IP: hostAddr.Addr().AsSlice(), - Port: int(hostAddr.Port()), - }, + dst := cachedDst + if dst == nil && replyPath != nil { + dst = replyPath } - if path != nil { - dst.Path = path.Dataplane() - dst.NextHop = path.UnderlayNextHop() + if dst == nil { + return 0, fmt.Errorf("no SCION destination") } - return sc.conn.WriteTo(b, dst) } @@ -365,7 +370,8 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo // Read path info once for the entire batch to avoid repeated locking. pi.mu.Lock() - path, replyPath, hostAddr, peerIA := pi.path, pi.replyPath, pi.hostAddr, pi.peerIA + replyPath := pi.replyPath + cachedDst := pi.cachedDst expired := !pi.expiry.IsZero() && time.Now().After(pi.expiry) pi.mu.Unlock() if expired { @@ -374,26 +380,12 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo // If no discovered path, fall back to replyPath (bootstrapped from an // incoming packet before path discovery completes). - if path == nil && replyPath != nil { - for _, buf := range buffs { - _, err = sc.conn.WriteTo(buf[offset:], replyPath) - if err != nil { - return false, err - } - } - return true, nil + dst := cachedDst + if dst == nil && replyPath != nil { + dst = replyPath } - - dst := &snet.UDPAddr{ - IA: peerIA, - Host: &net.UDPAddr{ - IP: hostAddr.Addr().AsSlice(), - Port: int(hostAddr.Port()), - }, - } - if path != nil { - dst.Path = path.Dataplane() - dst.NextHop = path.UnderlayNextHop() + if dst == nil { + return false, fmt.Errorf("no SCION destination for key %d", addr.scionKey) } for _, buf := range buffs { @@ -518,7 +510,9 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } // Initialize lastSCIONRecv so we don't trigger reconnection on startup. - c.lastSCIONRecv.CompareAndSwap(0, time.Now().UnixNano()) + if c.lastSCIONRecv.LoadAtomic() == 0 { + c.lastSCIONRecv.StoreAtomic(mono.Now()) + } for { // Check for graceful shutdown. @@ -577,7 +571,7 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } // Got a packet — record receive time. - c.lastSCIONRecv.Store(time.Now().UnixNano()) + c.lastSCIONRecv.StoreAtomic(mono.Now()) b := buffs[0][:n] @@ -651,8 +645,8 @@ func isTimeoutError(err error) bool { // 1. No SCION packet has been received for scionReconnectThreshold, AND // 2. There are active SCION peers (otherwise silence is expected). func (c *Conn) shouldReconnectSCION() bool { - lastRecv := time.Unix(0, c.lastSCIONRecv.Load()) - if time.Since(lastRecv) < scionReconnectThreshold { + lastRecv := c.lastSCIONRecv.LoadAtomic() + if mono.Since(lastRecv) < scionReconnectThreshold { return false } @@ -684,7 +678,7 @@ func (c *Conn) reconnectSCION() { if err != nil { c.logf("magicsock: SCION reconnect failed: %v", err) // Reset the receive timestamp so we retry after scionReconnectThreshold. - c.lastSCIONRecv.Store(time.Now().UnixNano()) + c.lastSCIONRecv.StoreAtomic(mono.Now()) return } @@ -692,7 +686,7 @@ func (c *Conn) reconnectSCION() { c.pconnSCION = newSC // Reset the receive timestamp so we don't immediately re-trigger. - c.lastSCIONRecv.Store(time.Now().UnixNano()) + c.lastSCIONRecv.StoreAtomic(mono.Now()) c.logf("magicsock: SCION reconnected successfully, local IA: %s", newSC.localIA) @@ -713,7 +707,7 @@ func (c *Conn) retrySCIONConnect() { return } c.pconnSCION = newSC - c.lastSCIONRecv.Store(time.Now().UnixNano()) + c.lastSCIONRecv.StoreAtomic(mono.Now()) c.logf("magicsock: SCION reconnect retry succeeded, local IA: %s", newSC.localIA) c.rediscoverAllSCIONPaths() } @@ -817,6 +811,7 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr expiry: expiry, mtu: mtu, } + pi.buildCachedDst() keys = append(keys, c.registerSCIONPath(pi)) } // Set the first (lowest-latency) path as active for the reverse index. @@ -1006,6 +1001,7 @@ func (c *Conn) refreshSCIONPathsOnce() error { pi.expiry = md.Expiry pi.mtu = md.MTU } + pi.buildCachedDst() pi.mu.Unlock() } } From ca1fe18e63daafd417928369a7e3fce8009320e5 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Tue, 10 Mar 2026 22:25:44 +0000 Subject: [PATCH 20/78] wgengine/magicsock: implement SCION fast-path checksum and header serialization - Added functions for computing the SCION pseudo-header checksum and finishing the checksum for SCION/UDP packets. - Introduced a pre-serialized header template for fast-path sends to optimize performance by bypassing standard serialization. - Enhanced the scionConn structure to support fast-path operations, including adjustments to the underlay connection handling. - Updated tests to validate the correctness of the new checksum computations and fast-path functionality. --- wgengine/magicsock/magicsock_scion.go | 299 ++++++++++++++++- wgengine/magicsock/magicsock_scion_test.go | 370 +++++++++++++++++++++ 2 files changed, 656 insertions(+), 13 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index d9d6e9fac32b4..261a487d6b3eb 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -5,6 +5,7 @@ package magicsock import ( "context" + "encoding/binary" "errors" "fmt" "net" @@ -19,6 +20,7 @@ import ( "github.com/scionproto/scion/pkg/daemon" "github.com/scionproto/scion/pkg/snet" wgconn "github.com/tailscale/wireguard-go/conn" + "golang.org/x/net/ipv4" "tailscale.com/envknob" "tailscale.com/tailcfg" "tailscale.com/tstime/mono" @@ -71,6 +73,7 @@ type scionPathInfo struct { path snet.Path // current best SCION path to this peer replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) cachedDst *snet.UDPAddr // pre-built destination addr; rebuilt when path changes + fastPath *scionFastPath // pre-serialized header template for fast sends expiry time.Time // path expiration from path metadata mtu uint16 // SCION payload MTU from path metadata mu sync.Mutex @@ -183,12 +186,182 @@ func (ps *scionPathProbeState) latency() time.Duration { return ps.recentPongs[ps.recentPong].latency } +// scionFastPath holds a pre-serialized SCION+UDP header template for a +// specific path. At send time, the template is copied, per-packet fields +// (PayloadLen, UDP Length, UDP Checksum) are patched, payload is appended, +// and the result is sent directly on the underlay UDP socket — bypassing +// snet.Conn and gopacket serialization entirely. +type scionFastPath struct { + hdr []byte // [SCION header][UDP header], no payload + udpOffset int // byte offset of UDP header within hdr + nextHop *net.UDPAddr // underlay next-hop for this path + pseudoCsum uint32 // constant part of SCION pseudo-header checksum +} + +// scionMaxBatchSize is the max number of packets in a single sendmmsg call. +const scionMaxBatchSize = 64 + +// scionSendBatch is a reusable set of buffers for sendSCIONBatchFast. +type scionSendBatch struct { + bufs [][]byte + msgs []ipv4.Message +} + +var scionSendBatchPool = sync.Pool{ + New: func() any { + b := &scionSendBatch{ + bufs: make([][]byte, scionMaxBatchSize), + msgs: make([]ipv4.Message, scionMaxBatchSize), + } + for i := range b.bufs { + b.bufs[i] = make([]byte, 1500) + } + for i := range b.msgs { + b.msgs[i].Buffers = make([][]byte, 1) + } + return b + }, +} + +// scionPseudoHeaderPartial computes the constant part of the SCION +// pseudo-header checksum: srcIA + dstIA + srcAddr + dstAddr + protocol(17). +// The per-packet upper-layer length and data are added at send time. +func scionPseudoHeaderPartial(srcIA, dstIA addr.IA, srcIP, dstIP netip.Addr) uint32 { + var csum uint32 + var buf [8]byte + + // Source IA (8 bytes) + binary.BigEndian.PutUint64(buf[:], uint64(srcIA)) + for i := 0; i < 8; i += 2 { + csum += uint32(buf[i]) << 8 + csum += uint32(buf[i+1]) + } + + // Destination IA (8 bytes) + binary.BigEndian.PutUint64(buf[:], uint64(dstIA)) + for i := 0; i < 8; i += 2 { + csum += uint32(buf[i]) << 8 + csum += uint32(buf[i+1]) + } + + // Source address + if srcIP.Is4() { + b4 := srcIP.As4() + csum += uint32(b4[0])<<8 + uint32(b4[1]) + csum += uint32(b4[2])<<8 + uint32(b4[3]) + } else { + b16 := srcIP.As16() + for i := 0; i < 16; i += 2 { + csum += uint32(b16[i])<<8 + uint32(b16[i+1]) + } + } + + // Destination address + if dstIP.Is4() { + b4 := dstIP.As4() + csum += uint32(b4[0])<<8 + uint32(b4[1]) + csum += uint32(b4[2])<<8 + uint32(b4[3]) + } else { + b16 := dstIP.As16() + for i := 0; i < 16; i += 2 { + csum += uint32(b16[i])<<8 + uint32(b16[i+1]) + } + } + + // Protocol: L4UDP = 17 + csum += 17 + + return csum +} + +// scionFinishChecksum completes the SCION/UDP checksum by adding the +// upper-layer length and bytes to the pre-computed partial checksum, +// then folding and complementing. +func scionFinishChecksum(partialCsum uint32, upperLayer []byte) uint16 { + csum := partialCsum + + // Add upper-layer length + l := uint32(len(upperLayer)) + csum += (l >> 16) + (l & 0xffff) + + // Sum upper-layer bytes in 16-bit words + n := len(upperLayer) + for i := 0; i+1 < n; i += 2 { + csum += uint32(upperLayer[i]) << 8 + csum += uint32(upperLayer[i+1]) + } + if n%2 == 1 { + csum += uint32(upperLayer[n-1]) << 8 + } + + // Fold to 16 bits + for csum > 0xffff { + csum = (csum >> 16) + (csum & 0xffff) + } + return ^uint16(csum) +} + +// buildSCIONFastPath creates a pre-serialized header template for fast-path +// sends. Must be called with pi.mu held (or before pi is shared). +// Returns nil if the fast path cannot be built (e.g. no discovered path). +func buildSCIONFastPath(sc *scionConn, pi *scionPathInfo) *scionFastPath { + if sc.underlayConn == nil { + return nil + } + dst := pi.cachedDst + if dst == nil || dst.Path == nil || dst.NextHop == nil { + return nil + } + + dstIP, ok := netip.AddrFromSlice(dst.Host.IP) + if !ok { + return nil + } + srcIP := sc.localHostIP + + // Use snet.Packet.Serialize() with empty payload to get a correctly + // encoded SCION+UDP header template. + pkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: pi.peerIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: sc.localIA, Host: addr.HostIP(srcIP)}, + Path: dst.Path, + Payload: snet.UDPPayload{ + SrcPort: sc.localPort, + DstPort: uint16(dst.Host.Port), + Payload: nil, // empty payload → headers only + }, + }, + } + if err := pkt.Serialize(); err != nil { + return nil + } + + // pkt.Bytes is now [SCION header][8-byte UDP header] + hdr := make([]byte, len(pkt.Bytes)) + copy(hdr, pkt.Bytes) + udpOffset := len(hdr) - 8 + + pseudoCsum := scionPseudoHeaderPartial(sc.localIA, pi.peerIA, srcIP, dstIP) + + return &scionFastPath{ + hdr: hdr, + udpOffset: udpOffset, + nextHop: dst.NextHop, + pseudoCsum: pseudoCsum, + } +} + // scionConn wraps a SCION connection for use by magicsock. type scionConn struct { - conn *snet.Conn // from SCIONNetwork.Listen() - localIA addr.IA // our ISD-AS - daemon daemon.Connector // for path queries - topo snet.Topology // local topology + conn *snet.Conn // from SCIONNetwork.Listen() + underlayConn *net.UDPConn // raw underlay for fast-path sends (owned by conn) + underlayXPC *ipv4.PacketConn // for WriteBatch / sendmmsg + localIA addr.IA // our ISD-AS + localHostIP netip.Addr // local host IP (e.g. 127.0.0.1) + localPort uint16 // local SCION/UDP port + daemon daemon.Connector // for path queries + topo snet.Topology // local topology } // close shuts down the SCION connection and daemon connector. @@ -305,10 +478,11 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { return nil, fmt.Errorf("listening on SCION %s: %w", listenAddr, err) } - // Increase underlay UDP socket buffers to match the regular magicsock - // UDP sockets (7 MB). The default kernel buffer (~212 KB) overflows - // easily at high throughput, causing packet drops and TCP retransmissions. + // Extract the underlay *net.UDPConn for fast-path sends that bypass + // snet.Conn serialization. Also increase socket buffer sizes. + var underlayConn *net.UDPConn if pc, ok := pconn.(*snet.SCIONPacketConn); ok { + underlayConn = pc.Conn if err := pc.SetReadBuffer(socketBufferSize); err != nil { fmt.Fprintf(os.Stderr, "magicsock: SCION: failed to set read buffer to %d: %v\n", socketBufferSize, err) } @@ -324,11 +498,31 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { return nil, fmt.Errorf("creating SCION conn: %w", err) } + // Extract local address info for fast-path header templates. + var localHostIP netip.Addr + var localPort uint16 + if sa, saOk := sconn.LocalAddr().(*snet.UDPAddr); saOk && sa.Host != nil { + if ip, ipOk := netip.AddrFromSlice(sa.Host.IP); ipOk { + localHostIP = ip + } + localPort = uint16(sa.Host.Port) + } + + // Wrap underlay conn for sendmmsg batching. + var underlayXPC *ipv4.PacketConn + if underlayConn != nil { + underlayXPC = ipv4.NewPacketConn(underlayConn) + } + return &scionConn{ - conn: sconn, - localIA: localIA, - daemon: conn, - topo: conn, + conn: sconn, + underlayConn: underlayConn, + underlayXPC: underlayXPC, + localIA: localIA, + localHostIP: localHostIP, + localPort: localPort, + daemon: conn, + topo: conn, }, nil } @@ -357,6 +551,11 @@ func parseSCIONServiceAddr(description string, port uint16) (ia addr.IA, hostAdd // sendSCIONBatch sends a batch of WireGuard packets over the SCION connection. // It looks up the full path info from the Conn's scionPaths registry using the // scionPathKey from the epAddr. +// +// When a fast-path template is available (pre-serialized headers + underlay +// socket), packets are serialized by patching a header template and sent via +// sendmmsg in a single syscall. Otherwise, falls back to snet.Conn.WriteTo +// per packet. func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent bool, err error) { sc := c.pconnSCION if sc == nil { @@ -372,14 +571,20 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo pi.mu.Lock() replyPath := pi.replyPath cachedDst := pi.cachedDst + fastPath := pi.fastPath expired := !pi.expiry.IsZero() && time.Now().After(pi.expiry) pi.mu.Unlock() if expired { return false, fmt.Errorf("SCION path expired for key %d", addr.scionKey) } - // If no discovered path, fall back to replyPath (bootstrapped from an - // incoming packet before path discovery completes). + // Fast path: pre-serialized headers + sendmmsg. + if fastPath != nil && sc.underlayXPC != nil { + err = c.sendSCIONBatchFast(sc, fastPath, buffs, offset) + return err == nil, err + } + + // Slow path: snet.Conn.WriteTo per packet. dst := cachedDst if dst == nil && replyPath != nil { dst = replyPath @@ -397,6 +602,70 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo return true, nil } +// sendSCIONBatchFast sends a batch of packets using pre-serialized SCION +// headers and sendmmsg on the underlay UDP socket. Each packet is built by +// copying the header template, patching per-packet fields (PayloadLen, UDP +// Length, UDP Checksum), and appending the WireGuard payload. +func (c *Conn) sendSCIONBatchFast(sc *scionConn, fp *scionFastPath, buffs [][]byte, offset int) error { + batch := scionSendBatchPool.Get().(*scionSendBatch) + defer scionSendBatchPool.Put(batch) + + hdrLen := len(fp.hdr) + n := len(buffs) + if n > scionMaxBatchSize { + n = scionMaxBatchSize + } + + for i := 0; i < n; i++ { + payload := buffs[i][offset:] + pktLen := hdrLen + len(payload) + + // Grow buffer if needed. + buf := batch.bufs[i] + if cap(buf) < pktLen { + buf = make([]byte, pktLen) + batch.bufs[i] = buf + } else { + buf = buf[:pktLen] + } + + // Copy header template and append payload. + copy(buf, fp.hdr) + copy(buf[hdrLen:], payload) + + // Patch SCION PayloadLen (bytes 6:8) = UDP header (8) + payload. + udpTotalLen := uint16(8 + len(payload)) + binary.BigEndian.PutUint16(buf[6:], udpTotalLen) + + // Patch UDP Length (udpOffset+4:+6). + binary.BigEndian.PutUint16(buf[fp.udpOffset+4:], udpTotalLen) + + // Zero checksum, compute over full upper layer, set result. + buf[fp.udpOffset+6] = 0 + buf[fp.udpOffset+7] = 0 + upperLayer := buf[fp.udpOffset:pktLen] + csum := scionFinishChecksum(fp.pseudoCsum, upperLayer) + binary.BigEndian.PutUint16(buf[fp.udpOffset+6:], csum) + + batch.msgs[i].Buffers[0] = buf[:pktLen] + batch.msgs[i].Addr = fp.nextHop + } + + // WriteBatch uses sendmmsg on Linux for batched sends. + msgs := batch.msgs[:n] + var head int + for { + written, err := sc.underlayXPC.WriteBatch(msgs[head:], 0) + if err != nil { + return err + } + head += written + if head >= n { + return nil + } + } +} + // sendSCION sends a single packet over SCION, used for disco messages. func (c *Conn) sendSCION(sk scionPathKey, b []byte) (bool, error) { sc := c.pconnSCION @@ -812,6 +1081,9 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr mtu: mtu, } pi.buildCachedDst() + if sc := c.pconnSCION; sc != nil { + pi.fastPath = buildSCIONFastPath(sc, pi) + } keys = append(keys, c.registerSCIONPath(pi)) } // Set the first (lowest-latency) path as active for the reverse index. @@ -1002,6 +1274,7 @@ func (c *Conn) refreshSCIONPathsOnce() error { pi.mtu = md.MTU } pi.buildCachedDst() + pi.fastPath = buildSCIONFastPath(sc, pi) pi.mu.Unlock() } } diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 939f89cda688f..1533f952bdcb9 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -5,6 +5,7 @@ package magicsock import ( "context" + "encoding/binary" "fmt" "net" "net/netip" @@ -18,6 +19,7 @@ import ( "github.com/scionproto/scion/pkg/daemon/mock_daemon" "github.com/scionproto/scion/pkg/snet" "github.com/scionproto/scion/pkg/snet/mock_snet" + snetpath "github.com/scionproto/scion/pkg/snet/path" "tailscale.com/net/packet" "tailscale.com/net/tstun" "tailscale.com/tailcfg" @@ -1115,3 +1117,371 @@ func TestSendSCIONExpiredPath(t *testing.T) { t.Errorf("error should mention 'expired', got: %v", err) } } + +// TestSCIONPseudoHeaderPartial verifies the partial checksum computation +// matches the reference SCION implementation for known inputs. +func TestSCIONPseudoHeaderPartial(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + + // Verify by computing the same checksum manually: + // srcIA = 0x0001ff0000000110, dstIA = 0x0001ff0000000111 + // srcAddr = 127.0.0.1 = [0x7f, 0x00, 0x00, 0x01] + // dstAddr = 127.0.0.1 = [0x7f, 0x00, 0x00, 0x01] + // protocol = 17 + + var expected uint32 + // srcIA bytes: 00 01 ff 00 00 00 01 10 + expected += 0x0001 + 0xff00 + 0x0000 + 0x0110 + // dstIA bytes: 00 01 ff 00 00 00 01 11 + expected += 0x0001 + 0xff00 + 0x0000 + 0x0111 + // srcAddr: 7f 00 00 01 + expected += 0x7f00 + 0x0001 + // dstAddr: 7f 00 00 01 + expected += 0x7f00 + 0x0001 + // protocol + expected += 17 + + if partial != expected { + t.Errorf("scionPseudoHeaderPartial = %d, want %d", partial, expected) + } +} + +// TestSCIONPseudoHeaderPartialIPv6 verifies checksum with IPv6 addresses. +func TestSCIONPseudoHeaderPartialIPv6(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("::1") + dstIP := netip.MustParseAddr("fd00::1") + + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + if partial == 0 { + t.Fatal("checksum should not be zero") + } + + // Verify IPv6 addrs are 16 bytes each. + // ::1 = 00...01, fd00::1 = fd 00 00...01 + var expected uint32 + // IAs + expected += 0x0001 + 0xff00 + 0x0000 + 0x0110 + expected += 0x0001 + 0xff00 + 0x0000 + 0x0111 + // srcIP ::1 = all zeros except last byte + expected += 0x0001 + // dstIP fd00::1 + expected += 0xfd00 + 0x0001 + expected += 17 + + if partial != expected { + t.Errorf("scionPseudoHeaderPartial(IPv6) = %d, want %d", partial, expected) + } +} + +// TestSCIONFinishChecksum verifies the full checksum computation matches +// the reference SCION implementation by comparing against a packet +// serialized with snet.Packet.Serialize(). +func TestSCIONFinishChecksum(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + srcPort := uint16(32766) + dstPort := uint16(32766) + payload := []byte("Hello, SCION fast path!") + + // Build the packet using snet's reference serializer. + pkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: srcPort, + DstPort: dstPort, + Payload: payload, + }, + }, + } + if err := pkt.Serialize(); err != nil { + t.Fatalf("snet.Packet.Serialize: %v", err) + } + + // Extract the reference checksum from the serialized packet. + // The UDP header is the last 8 bytes before the payload. + udpOffset := len(pkt.Bytes) - 8 - len(payload) + refChecksum := binary.BigEndian.Uint16(pkt.Bytes[udpOffset+6:]) + + // Now compute it using our fast-path functions. + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + + // Build the upper layer: UDP header (8 bytes) + payload + upperLayer := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint16(upperLayer[0:], srcPort) + binary.BigEndian.PutUint16(upperLayer[2:], dstPort) + binary.BigEndian.PutUint16(upperLayer[4:], uint16(8+len(payload))) + // checksum field = 0 for computation + copy(upperLayer[8:], payload) + + fastChecksum := scionFinishChecksum(partial, upperLayer) + + if fastChecksum != refChecksum { + t.Errorf("fast-path checksum = 0x%04x, reference = 0x%04x", fastChecksum, refChecksum) + } +} + +// TestSCIONFinishChecksumEmptyPayload verifies checksum with empty payload. +func TestSCIONFinishChecksumEmptyPayload(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + + pkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: 1000, + DstPort: 2000, + Payload: nil, + }, + }, + } + if err := pkt.Serialize(); err != nil { + t.Fatalf("snet.Packet.Serialize: %v", err) + } + + // UDP header is the last 8 bytes (no payload). + udpOffset := len(pkt.Bytes) - 8 + refChecksum := binary.BigEndian.Uint16(pkt.Bytes[udpOffset+6:]) + + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + upperLayer := make([]byte, 8) + binary.BigEndian.PutUint16(upperLayer[0:], 1000) + binary.BigEndian.PutUint16(upperLayer[2:], 2000) + binary.BigEndian.PutUint16(upperLayer[4:], 8) + + fastChecksum := scionFinishChecksum(partial, upperLayer) + + if fastChecksum != refChecksum { + t.Errorf("fast-path checksum (empty) = 0x%04x, reference = 0x%04x", fastChecksum, refChecksum) + } +} + +// TestSCIONFinishChecksumOddPayload verifies correct handling of odd-length payloads. +func TestSCIONFinishChecksumOddPayload(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("10.0.0.1") + payload := []byte("ABC") // 3 bytes, odd + + pkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: 5000, + DstPort: 6000, + Payload: payload, + }, + }, + } + if err := pkt.Serialize(); err != nil { + t.Fatalf("snet.Packet.Serialize: %v", err) + } + + udpOffset := len(pkt.Bytes) - 8 - len(payload) + refChecksum := binary.BigEndian.Uint16(pkt.Bytes[udpOffset+6:]) + + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + upperLayer := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint16(upperLayer[0:], 5000) + binary.BigEndian.PutUint16(upperLayer[2:], 6000) + binary.BigEndian.PutUint16(upperLayer[4:], uint16(8+len(payload))) + copy(upperLayer[8:], payload) + + fastChecksum := scionFinishChecksum(partial, upperLayer) + + if fastChecksum != refChecksum { + t.Errorf("fast-path checksum (odd) = 0x%04x, reference = 0x%04x", fastChecksum, refChecksum) + } +} + +// TestBuildSCIONFastPath verifies that buildSCIONFastPath produces a template +// that matches the reference serializer output for the same parameters. +func TestBuildSCIONFastPath(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + srcPort := uint16(32766) + dstPort := uint16(32766) + + sc := &scionConn{ + underlayConn: &net.UDPConn{}, // non-nil to enable fast path + localIA: srcIA, + localHostIP: srcIP, + localPort: srcPort, + } + + pi := &scionPathInfo{ + peerIA: dstIA, + hostAddr: netip.MustParseAddrPort("127.0.0.1:32766"), + cachedDst: &snet.UDPAddr{ + IA: dstIA, + Host: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(dstPort)}, + Path: snetpath.Empty{}, + NextHop: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 30041}, + }, + } + + fp := buildSCIONFastPath(sc, pi) + if fp == nil { + t.Fatal("buildSCIONFastPath returned nil") + } + + // The template should match a reference packet with empty payload. + refPkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: srcPort, + DstPort: dstPort, + Payload: nil, + }, + }, + } + if err := refPkt.Serialize(); err != nil { + t.Fatalf("reference Serialize: %v", err) + } + + if len(fp.hdr) != len(refPkt.Bytes) { + t.Fatalf("fast-path header len = %d, reference = %d", len(fp.hdr), len(refPkt.Bytes)) + } + + // Compare header bytes (everything except checksum which may differ + // due to computation order, but should be the same for empty payload). + for i := range fp.hdr { + if fp.hdr[i] != refPkt.Bytes[i] { + t.Errorf("byte %d: fast-path=0x%02x, reference=0x%02x", i, fp.hdr[i], refPkt.Bytes[i]) + } + } + + if fp.udpOffset != len(fp.hdr)-8 { + t.Errorf("udpOffset = %d, expected %d", fp.udpOffset, len(fp.hdr)-8) + } + + if fp.nextHop == nil { + t.Error("nextHop should not be nil") + } +} + +// TestSCIONFastPathPacketMatchesReference verifies that a packet built with +// the fast-path template+patching produces identical bytes to one built with +// snet.Packet.Serialize(). +func TestSCIONFastPathPacketMatchesReference(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + srcPort := uint16(32766) + dstPort := uint16(32766) + payload := []byte("WireGuard test payload data for SCION fast path verification") + + sc := &scionConn{ + underlayConn: &net.UDPConn{}, + localIA: srcIA, + localHostIP: srcIP, + localPort: srcPort, + } + + pi := &scionPathInfo{ + peerIA: dstIA, + hostAddr: netip.MustParseAddrPort("127.0.0.1:32766"), + cachedDst: &snet.UDPAddr{ + IA: dstIA, + Host: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(dstPort)}, + Path: snetpath.Empty{}, + NextHop: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 30041}, + }, + } + + fp := buildSCIONFastPath(sc, pi) + if fp == nil { + t.Fatal("buildSCIONFastPath returned nil") + } + + // Build packet using fast-path template + patching. + hdrLen := len(fp.hdr) + pktLen := hdrLen + len(payload) + buf := make([]byte, pktLen) + copy(buf, fp.hdr) + copy(buf[hdrLen:], payload) + + udpTotalLen := uint16(8 + len(payload)) + binary.BigEndian.PutUint16(buf[6:], udpTotalLen) + binary.BigEndian.PutUint16(buf[fp.udpOffset+4:], udpTotalLen) + buf[fp.udpOffset+6] = 0 + buf[fp.udpOffset+7] = 0 + upperLayer := buf[fp.udpOffset:pktLen] + csum := scionFinishChecksum(fp.pseudoCsum, upperLayer) + binary.BigEndian.PutUint16(buf[fp.udpOffset+6:], csum) + + // Build reference packet using snet. + refPkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: srcPort, + DstPort: dstPort, + Payload: payload, + }, + }, + } + if err := refPkt.Serialize(); err != nil { + t.Fatalf("reference Serialize: %v", err) + } + + if len(buf) != len(refPkt.Bytes) { + t.Fatalf("fast-path pkt len = %d, reference = %d", len(buf), len(refPkt.Bytes)) + } + + for i := range buf { + if buf[i] != refPkt.Bytes[i] { + t.Errorf("byte %d: fast-path=0x%02x, reference=0x%02x", i, buf[i], refPkt.Bytes[i]) + } + } +} + +// TestSCIONSendBatchPool verifies the pool returns usable batches. +func TestSCIONSendBatchPool(t *testing.T) { + batch := scionSendBatchPool.Get().(*scionSendBatch) + defer scionSendBatchPool.Put(batch) + + if len(batch.bufs) != scionMaxBatchSize { + t.Errorf("batch.bufs len = %d, want %d", len(batch.bufs), scionMaxBatchSize) + } + if len(batch.msgs) != scionMaxBatchSize { + t.Errorf("batch.msgs len = %d, want %d", len(batch.msgs), scionMaxBatchSize) + } + for i, buf := range batch.bufs { + if cap(buf) < 1500 { + t.Errorf("batch.bufs[%d] cap = %d, want >= 1500", i, cap(buf)) + } + } + for i, msg := range batch.msgs { + if len(msg.Buffers) != 1 { + t.Errorf("batch.msgs[%d].Buffers len = %d, want 1", i, len(msg.Buffers)) + } + } +} From 95a135ead10db7378d7cf22aa21deba7e5ce1bc7 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 11 Mar 2026 10:25:57 +0000 Subject: [PATCH 21/78] cmd/{containerboot,k8s-operator}: reissue auth keys for broken proxies (#16450) Adds logic for containerboot to signal that it can't auth, so the operator can reissue a new auth key. This only applies when running with a config file and with a kube state store. If the operator sees reissue_authkey in a state Secret, it will create a new auth key iff the config has no auth key or its auth key matches the value of reissue_authkey from the state Secret. This is to ensure we don't reissue auth keys in a tight loop if the proxy is slow to start or failing for some other reason. The reissue logic also uses a burstable rate limiter to ensure there's no way a terminally misconfigured or buggy operator can automatically generate new auth keys in a tight loop. Additional implementation details (ChaosInTheCRD): - Added `ipn.NotifyInitialHealthState` to ipn watcher, to ensure that `n.Health` is populated when notify's are returned. - on auth failure, containerboot: - Disconnects from control server - Sets reissue_authkey marker in state Secret with the failing key - Polls config file for new auth key (10 minute timeout) - Restarts after receiving new key to apply it - modified operator's reissue logic slightly: - Deletes old device from tailnet before creating new key - Rate limiting: 1 key per 30s with initial burst equal to replica count - In-flight tracking (authKeyReissuing map) prevents duplicate API calls across reconcile loops Updates #14080 Change-Id: I6982f8e741932a6891f2f48a2936f7f6a455317f (cherry picked from commit 969927c47c3d4de05e90f5b26a6d8d931c5ceed4) Signed-off-by: Tom Proctor Co-authored-by: chaosinthecrd --- cmd/containerboot/kube.go | 148 ++++++++++++-- cmd/containerboot/kube_test.go | 72 ++++++- cmd/containerboot/main.go | 66 ++++++- cmd/containerboot/main_test.go | 189 ++++++++++++++++-- cmd/k8s-operator/operator.go | 3 + cmd/k8s-operator/proxygroup.go | 216 +++++++++++++++----- cmd/k8s-operator/proxygroup_test.go | 297 ++++++++++++++++++++++++---- cmd/k8s-operator/sts.go | 14 +- cmd/k8s-operator/testutils_test.go | 4 +- cmd/k8s-operator/tsrecorder_test.go | 2 +- kube/kubetypes/types.go | 16 +- 11 files changed, 873 insertions(+), 154 deletions(-) diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index 4943bddba7ad4..73f5819b406db 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -14,9 +14,12 @@ import ( "net/http" "net/netip" "os" + "path/filepath" "strings" "time" + "github.com/fsnotify/fsnotify" + "tailscale.com/client/local" "tailscale.com/ipn" "tailscale.com/kube/egressservices" "tailscale.com/kube/ingressservices" @@ -26,9 +29,11 @@ import ( "tailscale.com/tailcfg" "tailscale.com/types/logger" "tailscale.com/util/backoff" - "tailscale.com/util/set" ) +const fieldManager = "tailscale-container" +const kubeletMountedConfigLn = "..data" + // kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use // this rather than any of the upstream Kubernetes client libaries to avoid extra imports. type kubeClient struct { @@ -46,7 +51,7 @@ func newKubeClient(root string, stateSecret string) (*kubeClient, error) { var err error kc, err := kubeclient.New("tailscale-container") if err != nil { - return nil, fmt.Errorf("Error creating kube client: %w", err) + return nil, fmt.Errorf("error creating kube client: %w", err) } if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" { // Derive the API server address from the environment variables @@ -63,7 +68,7 @@ func (kc *kubeClient) storeDeviceID(ctx context.Context, deviceID tailcfg.Stable kubetypes.KeyDeviceID: []byte(deviceID), }, } - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) } // storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields 'device_ips', 'device_fqdn' of client's @@ -84,7 +89,7 @@ func (kc *kubeClient) storeDeviceEndpoints(ctx context.Context, fqdn string, add kubetypes.KeyDeviceIPs: deviceIPs, }, } - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) } // storeHTTPSEndpoint writes an HTTPS endpoint exposed by this device via 'tailscale serve' to the client's state @@ -96,7 +101,7 @@ func (kc *kubeClient) storeHTTPSEndpoint(ctx context.Context, ep string) error { kubetypes.KeyHTTPSEndpoint: []byte(ep), }, } - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) } // deleteAuthKey deletes the 'authkey' field of the given kube @@ -122,7 +127,7 @@ func (kc *kubeClient) deleteAuthKey(ctx context.Context) error { // resetContainerbootState resets state from previous runs of containerboot to // ensure the operator doesn't use stale state when a Pod is first recreated. -func (kc *kubeClient) resetContainerbootState(ctx context.Context, podUID string) error { +func (kc *kubeClient) resetContainerbootState(ctx context.Context, podUID string, tailscaledConfigAuthkey string) error { existingSecret, err := kc.GetSecret(ctx, kc.stateSecret) switch { case kubeclient.IsNotFoundErr(err): @@ -131,32 +136,135 @@ func (kc *kubeClient) resetContainerbootState(ctx context.Context, podUID string case err != nil: return fmt.Errorf("failed to read state Secret %q to reset state: %w", kc.stateSecret, err) } + s := &kubeapi.Secret{ Data: map[string][]byte{ kubetypes.KeyCapVer: fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion), + + // TODO(tomhjp): Perhaps shouldn't clear device ID and use a different signal, as this could leak tailnet devices. + kubetypes.KeyDeviceID: nil, + kubetypes.KeyDeviceFQDN: nil, + kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, }, } if podUID != "" { s.Data[kubetypes.KeyPodUID] = []byte(podUID) } - toClear := set.SetOf([]string{ - kubetypes.KeyDeviceID, - kubetypes.KeyDeviceFQDN, - kubetypes.KeyDeviceIPs, - kubetypes.KeyHTTPSEndpoint, - egressservices.KeyEgressServices, - ingressservices.IngressConfigKey, - }) - for key := range existingSecret.Data { - if toClear.Contains(key) { - // It's fine to leave the key in place as a debugging breadcrumb, - // it should get a new value soon. - s.Data[key] = nil + // Only clear reissue_authkey if the operator has actioned it. + brokenAuthkey, ok := existingSecret.Data[kubetypes.KeyReissueAuthkey] + if ok && tailscaledConfigAuthkey != "" && string(brokenAuthkey) != tailscaledConfigAuthkey { + s.Data[kubetypes.KeyReissueAuthkey] = nil + } + + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) +} + +func (kc *kubeClient) setAndWaitForAuthKeyReissue(ctx context.Context, client *local.Client, cfg *settings, tailscaledConfigAuthKey string) error { + err := client.DisconnectControl(ctx) + if err != nil { + return fmt.Errorf("error disconnecting from control: %w", err) + } + + err = kc.setReissueAuthKey(ctx, tailscaledConfigAuthKey) + if err != nil { + return fmt.Errorf("failed to set reissue_authkey in Kubernetes Secret: %w", err) + } + + err = kc.waitForAuthKeyReissue(ctx, cfg.TailscaledConfigFilePath, tailscaledConfigAuthKey, 10*time.Minute) + if err != nil { + return fmt.Errorf("failed to receive new auth key: %w", err) + } + + return nil +} + +func (kc *kubeClient) setReissueAuthKey(ctx context.Context, authKey string) error { + s := &kubeapi.Secret{ + Data: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte(authKey), + }, + } + + log.Printf("Requesting a new auth key from operator") + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) +} + +func (kc *kubeClient) waitForAuthKeyReissue(ctx context.Context, configPath string, oldAuthKey string, maxWait time.Duration) error { + log.Printf("Waiting for operator to provide new auth key (max wait: %v)", maxWait) + + ctx, cancel := context.WithTimeout(ctx, maxWait) + defer cancel() + + tailscaledCfgDir := filepath.Dir(configPath) + toWatch := filepath.Join(tailscaledCfgDir, kubeletMountedConfigLn) + + var ( + pollTicker <-chan time.Time + eventChan <-chan fsnotify.Event + ) + + pollInterval := 5 * time.Second + + // Try to use fsnotify for faster notification + if w, err := fsnotify.NewWatcher(); err != nil { + log.Printf("auth key reissue: fsnotify unavailable, using polling: %v", err) + } else if err := w.Add(tailscaledCfgDir); err != nil { + w.Close() + log.Printf("auth key reissue: fsnotify watch failed, using polling: %v", err) + } else { + defer w.Close() + log.Printf("auth key reissue: watching for config changes via fsnotify") + eventChan = w.Events + } + + // still keep polling if using fsnotify, for logging and in case fsnotify fails + pt := time.NewTicker(pollInterval) + defer pt.Stop() + pollTicker = pt.C + + start := time.Now() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for auth key reissue after %v", maxWait) + case <-pollTicker: // Waits for polling tick, continues when received + case event := <-eventChan: + if event.Name != toWatch { + continue + } + } + + newAuthKey := authkeyFromTailscaledConfig(configPath) + if newAuthKey != "" && newAuthKey != oldAuthKey { + log.Printf("New auth key received from operator after %v", time.Since(start).Round(time.Second)) + + if err := kc.clearReissueAuthKeyRequest(ctx); err != nil { + log.Printf("Warning: failed to clear reissue request: %v", err) + } + + return nil + } + + if eventChan == nil && pollTicker != nil { + log.Printf("Waiting for new auth key from operator (%v elapsed)", time.Since(start).Round(time.Second)) } } +} - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") +// clearReissueAuthKeyRequest removes the reissue_authkey marker from the Secret +// to signal to the operator that we've successfully received the new key. +func (kc *kubeClient) clearReissueAuthKeyRequest(ctx context.Context) error { + s := &kubeapi.Secret{ + Data: map[string][]byte{ + kubetypes.KeyReissueAuthkey: nil, + }, + } + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) } // waitForConsistentState waits for tailscaled to finish writing state if it diff --git a/cmd/containerboot/kube_test.go b/cmd/containerboot/kube_test.go index bc80e9cdf2cb3..6acaa60e1588e 100644 --- a/cmd/containerboot/kube_test.go +++ b/cmd/containerboot/kube_test.go @@ -248,25 +248,42 @@ func TestResetContainerbootState(t *testing.T) { capver := fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion) for name, tc := range map[string]struct { podUID string + authkey string initial map[string][]byte expected map[string][]byte }{ "empty_initial": { podUID: "1234", + authkey: "new-authkey", initial: map[string][]byte{}, expected: map[string][]byte{ kubetypes.KeyCapVer: capver, kubetypes.KeyPodUID: []byte("1234"), + // Cleared keys. + kubetypes.KeyDeviceID: nil, + kubetypes.KeyDeviceFQDN: nil, + kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, }, }, "empty_initial_no_pod_uid": { initial: map[string][]byte{}, expected: map[string][]byte{ kubetypes.KeyCapVer: capver, + // Cleared keys. + kubetypes.KeyDeviceID: nil, + kubetypes.KeyDeviceFQDN: nil, + kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, }, }, "only_relevant_keys_updated": { - podUID: "1234", + podUID: "1234", + authkey: "new-authkey", initial: map[string][]byte{ kubetypes.KeyCapVer: []byte("1"), kubetypes.KeyPodUID: []byte("5678"), @@ -295,6 +312,57 @@ func TestResetContainerbootState(t *testing.T) { // Tailscaled keys not included in patch. }, }, + "new_authkey_issued": { + initial: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-authkey"), + }, + authkey: "new-authkey", + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyReissueAuthkey: nil, + // Cleared keys. + kubetypes.KeyDeviceID: nil, + kubetypes.KeyDeviceFQDN: nil, + kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + }, + }, + "authkey_not_yet_updated": { + initial: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-authkey"), + }, + authkey: "old-authkey", + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + // reissue_authkey not cleared. + // Cleared keys. + kubetypes.KeyDeviceID: nil, + kubetypes.KeyDeviceFQDN: nil, + kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + }, + }, + "authkey_deleted_from_config": { + initial: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-authkey"), + }, + authkey: "", + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + // reissue_authkey not cleared. + // Cleared keys. + kubetypes.KeyDeviceID: nil, + kubetypes.KeyDeviceFQDN: nil, + kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + }, + }, } { t.Run(name, func(t *testing.T) { var actual map[string][]byte @@ -309,7 +377,7 @@ func TestResetContainerbootState(t *testing.T) { return nil }, }} - if err := kc.resetContainerbootState(context.Background(), tc.podUID); err != nil { + if err := kc.resetContainerbootState(context.Background(), tc.podUID, tc.authkey); err != nil { t.Fatalf("resetContainerbootState() error = %v", err) } if diff := cmp.Diff(tc.expected, actual); diff != "" { diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index ba47111fd797f..76c6e910a9dbc 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -137,7 +137,9 @@ import ( "golang.org/x/sys/unix" "tailscale.com/client/tailscale" + "tailscale.com/health" "tailscale.com/ipn" + "tailscale.com/ipn/conffile" kubeutils "tailscale.com/k8s-operator" healthz "tailscale.com/kube/health" "tailscale.com/kube/kubetypes" @@ -206,6 +208,11 @@ func run() error { bootCtx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() + var tailscaledConfigAuthkey string + if isOneStepConfig(cfg) { + tailscaledConfigAuthkey = authkeyFromTailscaledConfig(cfg.TailscaledConfigFilePath) + } + var kc *kubeClient if cfg.KubeSecret != "" { kc, err = newKubeClient(cfg.Root, cfg.KubeSecret) @@ -219,7 +226,7 @@ func run() error { // hasKubeStateStore because although we know we're in kube, that // doesn't guarantee the state store is properly configured. if hasKubeStateStore(cfg) { - if err := kc.resetContainerbootState(bootCtx, cfg.PodUID); err != nil { + if err := kc.resetContainerbootState(bootCtx, cfg.PodUID, tailscaledConfigAuthkey); err != nil { return fmt.Errorf("error clearing previous state from Secret: %w", err) } } @@ -299,7 +306,7 @@ func run() error { } } - w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState) + w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState|ipn.NotifyInitialHealthState) if err != nil { return fmt.Errorf("failed to watch tailscaled for updates: %w", err) } @@ -365,8 +372,23 @@ authLoop: if isOneStepConfig(cfg) { // This could happen if this is the first time tailscaled was run for this // device and the auth key was not passed via the configfile. - return fmt.Errorf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.") + if hasKubeStateStore(cfg) { + log.Printf("Auth key missing or invalid (NeedsLogin state), disconnecting from control and requesting new key from operator") + + err := kc.setAndWaitForAuthKeyReissue(bootCtx, client, cfg, tailscaledConfigAuthkey) + if err != nil { + return fmt.Errorf("failed to get a reissued authkey: %w", err) + } + + log.Printf("Successfully received new auth key, restarting to apply configuration") + + // we don't return an error here since we have handled the reissue gracefully. + return nil + } + + return errors.New("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file") } + if err := authTailscale(); err != nil { return fmt.Errorf("failed to auth tailscale: %w", err) } @@ -384,6 +406,27 @@ authLoop: log.Printf("tailscaled in state %q, waiting", *n.State) } } + + if n.Health != nil { + // This can happen if the config has an auth key but it's invalid, + // for example if it was single-use and already got used, but the + // device state was lost. + if _, ok := n.Health.Warnings[health.LoginStateWarnable.Code]; ok { + if isOneStepConfig(cfg) && hasKubeStateStore(cfg) { + log.Printf("Auth key failed to authenticate (may be expired or single-use), disconnecting from control and requesting new key from operator") + + err := kc.setAndWaitForAuthKeyReissue(bootCtx, client, cfg, tailscaledConfigAuthkey) + if err != nil { + return fmt.Errorf("failed to get a reissued authkey: %w", err) + } + + // we don't return an error here since we have handled the reissue gracefully. + log.Printf("Successfully received new auth key, restarting to apply configuration") + + return nil + } + } + } } w.Close() @@ -409,9 +452,9 @@ authLoop: // We were told to only auth once, so any secret-bound // authkey is no longer needed. We don't strictly need to // wipe it, but it's good hygiene. - log.Printf("Deleting authkey from kube secret") + log.Printf("Deleting authkey from Kubernetes Secret") if err := kc.deleteAuthKey(ctx); err != nil { - return fmt.Errorf("deleting authkey from kube secret: %w", err) + return fmt.Errorf("deleting authkey from Kubernetes Secret: %w", err) } } @@ -422,8 +465,10 @@ authLoop: // If tailscaled config was read from a mounted file, watch the file for updates and reload. cfgWatchErrChan := make(chan error) + cfgWatchCtx, cfgWatchCancel := context.WithCancel(ctx) + defer cfgWatchCancel() if cfg.TailscaledConfigFilePath != "" { - go watchTailscaledConfigChanges(ctx, cfg.TailscaledConfigFilePath, client, cfgWatchErrChan) + go watchTailscaledConfigChanges(cfgWatchCtx, cfg.TailscaledConfigFilePath, client, cfgWatchErrChan) } var ( @@ -523,6 +568,7 @@ runLoop: case err := <-cfgWatchErrChan: return fmt.Errorf("failed to watch tailscaled config: %w", err) case n := <-notifyChan: + // TODO: (ChaosInTheCRD) Add node removed check when supported by ipn if n.State != nil && *n.State != ipn.Running { // Something's gone wrong and we've left the authenticated state. // Our container image never recovered gracefully from this, and the @@ -979,3 +1025,11 @@ func serviceIPsFromNetMap(nm *netmap.NetworkMap, fqdn dnsname.FQDN) []netip.Pref return prefixes } + +func authkeyFromTailscaledConfig(path string) string { + if cfg, err := conffile.Load(path); err == nil && cfg.Parsed.AuthKey != nil { + return *cfg.Parsed.AuthKey + } + + return "" +} diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index 365cf218424de..5ea402f6678c9 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -32,6 +32,7 @@ import ( "github.com/google/go-cmp/cmp" "golang.org/x/sys/unix" + "tailscale.com/health" "tailscale.com/ipn" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubeclient" @@ -41,6 +42,8 @@ import ( "tailscale.com/types/netmap" ) +const configFileAuthKey = "some-auth-key" + func TestContainerBoot(t *testing.T) { boot := filepath.Join(t.TempDir(), "containerboot") if err := exec.Command("go", "build", "-ldflags", "-X main.testSleepDuration=1ms", "-o", boot, "tailscale.com/cmd/containerboot").Run(); err != nil { @@ -77,6 +80,10 @@ func TestContainerBoot(t *testing.T) { // phase (simulates our fake tailscaled doing it). UpdateKubeSecret map[string]string + // Update files with these paths/contents at the beginning of the phase + // (simulates the operator updating mounted config files). + UpdateFiles map[string]string + // WantFiles files that should exist in the container and their // contents. WantFiles map[string]string @@ -781,6 +788,127 @@ func TestContainerBoot(t *testing.T) { }, } }, + "sets_reissue_authkey_if_needs_login": func(env *testEnv) testCase { + newAuthKey := "new-reissued-auth-key" + return testCase{ + Env: map[string]string{ + "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), + "KUBERNETES_SERVICE_HOST": env.kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, + }, + Phases: []phase{ + { + UpdateFiles: map[string]string{ + "etc/tailscaled/..data": "", + }, + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + }, + }, { + Notify: &ipn.Notify{ + State: new(ipn.NeedsLogin), + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyReissueAuthkey: configFileAuthKey, + }, + WantLog: "watching for config changes via fsnotify", + }, { + UpdateFiles: map[string]string{ + "etc/tailscaled/cap-95.hujson": fmt.Sprintf(`{"Version":"alpha0","AuthKey":"%s"}`, newAuthKey), + "etc/tailscaled/..data": "updated", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + }, + WantExitCode: new(0), + WantLog: "Successfully received new auth key, restarting to apply configuration", + }, + }, + } + }, + "sets_reissue_authkey_if_auth_fails": func(env *testEnv) testCase { + newAuthKey := "new-reissued-auth-key" + return testCase{ + Env: map[string]string{ + "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), + "KUBERNETES_SERVICE_HOST": env.kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, + }, + Phases: []phase{ + { + UpdateFiles: map[string]string{ + "etc/tailscaled/..data": "", + }, + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + }, + }, { + Notify: &ipn.Notify{ + Health: &health.State{ + Warnings: map[health.WarnableCode]health.UnhealthyState{ + health.LoginStateWarnable.Code: {}, + }, + }, + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyReissueAuthkey: configFileAuthKey, + }, + WantLog: "watching for config changes via fsnotify", + }, { + UpdateFiles: map[string]string{ + "etc/tailscaled/cap-95.hujson": fmt.Sprintf(`{"Version":"alpha0","AuthKey":"%s"}`, newAuthKey), + "etc/tailscaled/..data": "updated", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + }, + WantExitCode: new(0), + WantLog: "Successfully received new auth key, restarting to apply configuration", + }, + }, + } + }, + "clears_reissue_authkey_on_change": func(env *testEnv) testCase { + return testCase{ + Env: map[string]string{ + "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), + "KUBERNETES_SERVICE_HOST": env.kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, + }, + KubeSecret: map[string]string{ + kubetypes.KeyReissueAuthkey: "some-older-authkey", + "foo": "bar", // Check not everything is cleared. + }, + Phases: []phase{ + { + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + "foo": "bar", + }, + }, { + Notify: runningNotify, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + "foo": "bar", + kubetypes.KeyDeviceFQDN: "test-node.test.ts.net.", + kubetypes.KeyDeviceID: "myID", + kubetypes.KeyDeviceIPs: `["100.64.0.1"]`, + }, + }, + }, + } + }, "metrics_enabled": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ @@ -1134,19 +1262,22 @@ func TestContainerBoot(t *testing.T) { for k, v := range p.UpdateKubeSecret { env.kube.SetSecret(k, v) } + for path, content := range p.UpdateFiles { + fullPath := filepath.Join(env.d, path) + if err := os.WriteFile(fullPath, []byte(content), 0700); err != nil { + t.Fatalf("phase %d: updating file %q: %v", i, path, err) + } + // Explicitly update mtime to ensure fsnotify detects the change. + // Without this, file operations can be buffered and fsnotify events may not trigger. + now := time.Now() + if err := os.Chtimes(fullPath, now, now); err != nil { + t.Fatalf("phase %d: updating mtime for %q: %v", i, path, err) + } + } env.lapi.Notify(p.Notify) if p.Signal != nil { cmd.Process.Signal(*p.Signal) } - if p.WantLog != "" { - err := tstest.WaitFor(2*time.Second, func() error { - waitLogLine(t, time.Second, cbOut, p.WantLog) - return nil - }) - if err != nil { - t.Fatal(err) - } - } if p.WantExitCode != nil { state, err := cmd.Process.Wait() @@ -1156,14 +1287,19 @@ func TestContainerBoot(t *testing.T) { if state.ExitCode() != *p.WantExitCode { t.Fatalf("phase %d: want exit code %d, got %d", i, *p.WantExitCode, state.ExitCode()) } + } - // Early test return, we don't expect the successful startup log message. - return + if p.WantLog != "" { + err := tstest.WaitFor(5*time.Second, func() error { + waitLogLine(t, 5*time.Second, cbOut, p.WantLog) + return nil + }) + if err != nil { + t.Fatal(err) + } } - wantCmds = append(wantCmds, p.WantCmds...) - waitArgs(t, 2*time.Second, env.d, env.argFile, strings.Join(wantCmds, "\n")) - err := tstest.WaitFor(2*time.Second, func() error { + err := tstest.WaitFor(5*time.Second, func() error { if p.WantKubeSecret != nil { got := env.kube.Secret() if diff := cmp.Diff(got, p.WantKubeSecret); diff != "" { @@ -1180,6 +1316,16 @@ func TestContainerBoot(t *testing.T) { if err != nil { t.Fatalf("test: %q phase %d: %v", name, i, err) } + + // if we provide a wanted exit code, we expect that the process is finished, + // so should return from the test. + if p.WantExitCode != nil { + return + } + + wantCmds = append(wantCmds, p.WantCmds...) + waitArgs(t, 2*time.Second, env.d, env.argFile, strings.Join(wantCmds, "\n")) + err = tstest.WaitFor(2*time.Second, func() error { for path, want := range p.WantFiles { gotBs, err := os.ReadFile(filepath.Join(env.d, path)) @@ -1393,6 +1539,13 @@ func (lc *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: panic(fmt.Sprintf("unsupported method %q", r.Method)) } + // In the localAPI ServeHTTP method + case "/localapi/v0/disconnect-control": + if r.Method != "POST" { + panic(fmt.Sprintf("unsupported method %q", r.Method)) + } + w.WriteHeader(http.StatusOK) + return default: panic(fmt.Sprintf("unsupported path %q", r.URL.Path)) } @@ -1591,7 +1744,11 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) } for key, val := range req.Data { - k.secret[key] = string(val) + if val == nil { + delete(k.secret, key) + } else { + k.secret[key] = string(val) + } } default: panic(fmt.Sprintf("unknown content type %q", r.Header.Get("Content-Type"))) @@ -1659,7 +1816,7 @@ func newTestEnv(t *testing.T) testEnv { kube.Start(t) t.Cleanup(kube.Close) - tailscaledConf := &ipn.ConfigVAlpha{AuthKey: new("foo"), Version: "alpha0"} + tailscaledConf := &ipn.ConfigVAlpha{AuthKey: new(configFileAuthKey), Version: "alpha0"} serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}} serveConfWithServices := ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index ef55d27481266..1060c6f3da9e7 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -20,6 +20,7 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" @@ -723,6 +724,8 @@ func runReconcilers(opts reconcilerOpts) { tsFirewallMode: opts.proxyFirewallMode, defaultProxyClass: opts.defaultProxyClass, loginServer: opts.tsServer.ControlURL, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), }) if err != nil { startlog.Fatalf("could not create ProxyGroup reconciler: %v", err) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 538933f14dbe1..4d5a795d79796 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -16,10 +16,12 @@ import ( "sort" "strings" "sync" + "time" dockerref "github.com/distribution/reference" "go.uber.org/zap" xslices "golang.org/x/exp/slices" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -94,10 +96,12 @@ type ProxyGroupReconciler struct { defaultProxyClass string loginServer string - mu sync.Mutex // protects following - egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge - ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge - apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge + mu sync.Mutex // protects following + egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge + ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge + apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge + authKeyRateLimits map[string]*rate.Limiter // per-ProxyGroup rate limiters for auth key re-issuance. + authKeyReissuing map[string]bool } func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger { @@ -294,7 +298,7 @@ func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGrou func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tailscaleClient tsClient, loginUrl string, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) { logger := r.logger(pg.Name) r.mu.Lock() - r.ensureAddedToGaugeForProxyGroup(pg) + r.ensureStateAddedForProxyGroup(pg) r.mu.Unlock() svcToNodePorts := make(map[string]uint16) @@ -629,13 +633,13 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, tai } for _, m := range metadata { - if m.ordinal+1 <= int(pgReplicas(pg)) { + if m.ordinal+1 <= pgReplicas(pg) { continue } // Dangling resource, delete the config + state Secrets, as well as // deleting the device from the tailnet. - if err := r.deleteTailnetDevice(ctx, tailscaleClient, m.tsID, logger); err != nil { + if err := r.ensureDeviceDeleted(ctx, tailscaleClient, m.tsID, logger); err != nil { return err } if err := r.Delete(ctx, m.stateSecret); err != nil && !apierrors.IsNotFound(err) { @@ -687,7 +691,7 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, tailscaleClient } for _, m := range metadata { - if err := r.deleteTailnetDevice(ctx, tailscaleClient, m.tsID, logger); err != nil { + if err := r.ensureDeviceDeleted(ctx, tailscaleClient, m.tsID, logger); err != nil { return false, err } } @@ -703,12 +707,12 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, tailscaleClient logger.Infof("cleaned up ProxyGroup resources") r.mu.Lock() - r.ensureRemovedFromGaugeForProxyGroup(pg) + r.ensureStateRemovedForProxyGroup(pg) r.mu.Unlock() return true, nil } -func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, tailscaleClient tsClient, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error { +func (r *ProxyGroupReconciler) ensureDeviceDeleted(ctx context.Context, tailscaleClient tsClient, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error { logger.Debugf("deleting device %s from control", string(id)) if err := tailscaleClient.DeleteDevice(ctx, string(id)); err != nil { if errResp, ok := errors.AsType[tailscale.ErrResponse](err); ok && errResp.Status == http.StatusNotFound { @@ -734,6 +738,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated( logger := r.logger(pg.Name) endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) // keyed by Service name. for i := range pgReplicas(pg) { + logger = logger.With("Pod", fmt.Sprintf("%s-%d", pg.Name, i)) cfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName(pg.Name, i), @@ -751,38 +756,9 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated( return nil, err } - var authKey *string - if existingCfgSecret == nil { - logger.Debugf("Creating authkey for new ProxyGroup proxy") - tags := pg.Spec.Tags.Stringify() - if len(tags) == 0 { - tags = r.defaultTags - } - key, err := newAuthKey(ctx, tailscaleClient, tags) - if err != nil { - return nil, err - } - authKey = &key - } - - if authKey == nil { - // Get state Secret to check if it's already authed. - stateSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: pgStateSecretName(pg.Name, i), - Namespace: r.tsNamespace, - }, - } - if err = r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) { - return nil, err - } - - if shouldRetainAuthKey(stateSecret) && existingCfgSecret != nil { - authKey, err = authKeyFromSecret(existingCfgSecret) - if err != nil { - return nil, fmt.Errorf("error retrieving auth key from existing config Secret: %w", err) - } - } + authKey, err := r.getAuthKey(ctx, tailscaleClient, pg, existingCfgSecret, i, logger) + if err != nil { + return nil, err } nodePortSvcName := pgNodePortServiceName(pg.Name, i) @@ -918,11 +894,137 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated( return nil, err } } + } return endpoints, nil } +// getAuthKey returns an auth key for the proxy, or nil if none is needed. +// A new key is created if the config Secret doesn't exist yet, or if the +// proxy has requested a reissue via its state Secret. An existing key is +// retained while the device hasn't authed or a reissue is in progress. +func (r *ProxyGroupReconciler) getAuthKey(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup, existingCfgSecret *corev1.Secret, ordinal int32, logger *zap.SugaredLogger) (*string, error) { + // Get state Secret to check if it's already authed or has requested + // a fresh auth key. + stateSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pg.Name, ordinal), + Namespace: r.tsNamespace, + }, + } + if err := r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + + var createAuthKey bool + var cfgAuthKey *string + if existingCfgSecret == nil { + createAuthKey = true + } else { + var err error + cfgAuthKey, err = authKeyFromSecret(existingCfgSecret) + if err != nil { + return nil, fmt.Errorf("error retrieving auth key from existing config Secret: %w", err) + } + } + + if !createAuthKey { + var err error + createAuthKey, err = r.shouldReissueAuthKey(ctx, tailscaleClient, pg, stateSecret, cfgAuthKey) + if err != nil { + return nil, err + } + } + + var authKey *string + if createAuthKey { + logger.Debugf("creating auth key for ProxyGroup proxy %q", stateSecret.Name) + + tags := pg.Spec.Tags.Stringify() + if len(tags) == 0 { + tags = r.defaultTags + } + key, err := newAuthKey(ctx, tailscaleClient, tags) + if err != nil { + return nil, err + } + authKey = &key + } else { + // Retain auth key if the device hasn't authed yet, or if a + // reissue is in progress (device_id is stale during reissue). + _, reissueRequested := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !deviceAuthed(stateSecret) || reissueRequested { + authKey = cfgAuthKey + } + } + + return authKey, nil +} + +// shouldReissueAuthKey returns true if the proxy needs a new auth key. It +// tracks in-flight reissues via authKeyReissuing to avoid duplicate API calls +// across reconciles. +func (r *ProxyGroupReconciler) shouldReissueAuthKey(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup, stateSecret *corev1.Secret, cfgAuthKey *string) (shouldReissue bool, err error) { + r.mu.Lock() + reissuing := r.authKeyReissuing[stateSecret.Name] + r.mu.Unlock() + + if reissuing { + // Check if reissue is complete by seeing if request was cleared + _, requestStillPresent := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !requestStillPresent { + // Containerboot cleared the request, reissue is complete + r.mu.Lock() + r.authKeyReissuing[stateSecret.Name] = false + r.mu.Unlock() + r.log.Debugf("auth key reissue completed for %q", stateSecret.Name) + return false, nil + } + + // Reissue still in-flight; waiting for containerboot to pick up new key + r.log.Debugf("auth key already in process of re-issuance, waiting for secret to be updated") + return false, nil + } + + defer func() { + r.mu.Lock() + r.authKeyReissuing[stateSecret.Name] = shouldReissue + r.mu.Unlock() + }() + + brokenAuthkey, ok := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !ok { + // reissue hasn't been requested since the key in the secret hasn't been populated + return false, nil + } + + empty := cfgAuthKey == nil || *cfgAuthKey == "" + broken := cfgAuthKey != nil && *cfgAuthKey == string(brokenAuthkey) + + // A new key has been written but the proxy hasn't picked it up yet. + if !empty && !broken { + return false, nil + } + + lim := r.authKeyRateLimits[pg.Name] + if !lim.Allow() { + r.log.Debugf("auth key re-issuance rate limit exceeded, limit: %.2f, burst: %d, tokens: %.2f", + lim.Limit(), lim.Burst(), lim.Tokens()) + return false, fmt.Errorf("auth key re-issuance rate limit exceeded for ProxyGroup %q, will retry with backoff", pg.Name) + } + + r.log.Infof("Proxy failing to auth; attempting cleanup and new key") + if tsID := stateSecret.Data[kubetypes.KeyDeviceID]; len(tsID) > 0 { + id := tailcfg.StableNodeID(tsID) + if err := r.ensureDeviceDeleted(ctx, tailscaleClient, id, r.log); err != nil { + return false, err + } + } + + return true, nil +} + type FindStaticEndpointErr struct { msg string } @@ -1016,9 +1118,9 @@ func getStaticEndpointAddress(a *corev1.NodeAddress, port uint16) *netip.AddrPor return new(netip.AddrPortFrom(addr, port)) } -// ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup -// is created. r.mu must be held. -func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGroup) { +// ensureStateAddedForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup +// is created, and initialises per-ProxyGroup rate limits on re-issuing auth keys. r.mu must be held. +func (r *ProxyGroupReconciler) ensureStateAddedForProxyGroup(pg *tsapi.ProxyGroup) { switch pg.Spec.Type { case tsapi.ProxyGroupTypeEgress: r.egressProxyGroups.Add(pg.UID) @@ -1030,11 +1132,24 @@ func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGr gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) + + if _, ok := r.authKeyRateLimits[pg.Name]; !ok { + // Allow every replica to have its auth key re-issued quickly the first + // time, but with an overall limit of 1 every 30s after a burst. + r.authKeyRateLimits[pg.Name] = rate.NewLimiter(rate.Every(30*time.Second), int(pgReplicas(pg))) + } + + for i := range pgReplicas(pg) { + rep := pgStateSecretName(pg.Name, i) + if _, ok := r.authKeyReissuing[rep]; !ok { + r.authKeyReissuing[rep] = false + } + } } -// ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the -// ProxyGroup is deleted. r.mu must be held. -func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.ProxyGroup) { +// ensureStateRemovedForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the +// ProxyGroup is deleted, and deletes the per-ProxyGroup rate limiter to free memory. r.mu must be held. +func (r *ProxyGroupReconciler) ensureStateRemovedForProxyGroup(pg *tsapi.ProxyGroup) { switch pg.Spec.Type { case tsapi.ProxyGroupTypeEgress: r.egressProxyGroups.Remove(pg.UID) @@ -1046,6 +1161,7 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) + delete(r.authKeyRateLimits, pg.Name) } func pgTailscaledConfig(pg *tsapi.ProxyGroup, loginServer string, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string) (tailscaledConfigs, error) { @@ -1106,7 +1222,7 @@ func getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup, cl client.Client return nil, fmt.Errorf("failed to list state Secrets: %w", err) } for _, secret := range secrets.Items { - var ordinal int + var ordinal int32 if _, err := fmt.Sscanf(secret.Name, pg.Name+"-%d", &ordinal); err != nil { return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err) } @@ -1213,7 +1329,7 @@ func (r *ProxyGroupReconciler) getClientAndLoginURL(ctx context.Context, tailnet } type nodeMetadata struct { - ordinal int + ordinal int32 stateSecret *corev1.Secret podUID string // or empty if the Pod no longer exists. tsID tailcfg.StableNodeID diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 9b3ee0e0fd30f..1a50ee1f05f44 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -6,15 +6,19 @@ package main import ( + "context" "encoding/json" "fmt" "net/netip" + "reflect" "slices" + "strings" "testing" "time" "github.com/google/go-cmp/cmp" "go.uber.org/zap" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -28,7 +32,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "tailscale.com/client/tailscale" "tailscale.com/ipn" - kube "tailscale.com/k8s-operator" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/k8s-proxy/conf" @@ -637,10 +640,12 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { tsFirewallMode: "auto", defaultProxyClass: "default-pc", - Client: fc, - tsClient: tsClient, - recorder: fr, - clock: cl, + Client: fc, + tsClient: tsClient, + recorder: fr, + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } for i, r := range tt.reconciles { @@ -780,11 +785,13 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { tsFirewallMode: "auto", defaultProxyClass: "default-pc", - Client: fc, - tsClient: tsClient, - recorder: fr, - log: zl.Sugar().With("TestName", tt.name).With("Reconcile", "cleanup"), - clock: cl, + Client: fc, + tsClient: tsClient, + recorder: fr, + log: zl.Sugar().With("TestName", tt.name).With("Reconcile", "cleanup"), + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } if err := fc.Delete(t.Context(), pg); err != nil { @@ -841,12 +848,15 @@ func TestProxyGroup(t *testing.T) { tsFirewallMode: "auto", defaultProxyClass: "default-pc", - Client: fc, - tsClient: tsClient, - recorder: fr, - log: zl.Sugar(), - clock: cl, + Client: fc, + tsClient: tsClient, + recorder: fr, + log: zl.Sugar(), + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } + crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}} opts := configOpts{ proxyType: "proxygroup", @@ -863,7 +873,7 @@ func TestProxyGroup(t *testing.T) { tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 1, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, false, pc) - if kube.ProxyGroupAvailable(pg) { + if tsoperator.ProxyGroupAvailable(pg) { t.Fatal("expected ProxyGroup to not be available") } }) @@ -891,7 +901,7 @@ func TestProxyGroup(t *testing.T) { tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) - if kube.ProxyGroupAvailable(pg) { + if tsoperator.ProxyGroupAvailable(pg) { t.Fatal("expected ProxyGroup to not be available") } if expected := 1; reconciler.egressProxyGroups.Len() != expected { @@ -935,7 +945,7 @@ func TestProxyGroup(t *testing.T) { tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) - if !kube.ProxyGroupAvailable(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { t.Fatal("expected ProxyGroup to be available") } }) @@ -1045,12 +1055,14 @@ func TestProxyGroupTypes(t *testing.T) { zl, _ := zap.NewDevelopment() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - tsProxyImage: testProxyImage, - Client: fc, - log: zl.Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + log: zl.Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } t.Run("egress_type", func(t *testing.T) { @@ -1285,12 +1297,14 @@ func TestKubeAPIServerStatusConditionFlow(t *testing.T) { WithStatusSubresource(pg). Build() r := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - tsProxyImage: testProxyImage, - Client: fc, - log: zap.Must(zap.NewDevelopment()).Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + log: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } expectReconciled(t, r, "", pg.Name) @@ -1338,12 +1352,14 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { Build() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - tsProxyImage: testProxyImage, - Client: fc, - log: zap.Must(zap.NewDevelopment()).Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + log: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } pg := &tsapi.ProxyGroup{ @@ -1367,7 +1383,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { cfg := conf.VersionedConfig{ Version: "v1alpha1", ConfigV1Alpha1: &conf.ConfigV1Alpha1{ - AuthKey: new("secret-authkey"), + AuthKey: new("new-authkey"), State: new(fmt.Sprintf("kube:%s", pgPodName(pg.Name, 0))), App: new(kubetypes.AppProxyGroupKubeAPIServer), LogLevel: new("debug"), @@ -1423,12 +1439,14 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { WithStatusSubresource(&tsapi.ProxyGroup{}). Build() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - tsProxyImage: testProxyImage, - Client: fc, - log: zap.Must(zap.NewDevelopment()).Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + log: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } existingServices := []string{"svc1", "svc2"} @@ -1653,6 +1671,197 @@ func TestValidateProxyGroup(t *testing.T) { } } +func TestProxyGroupGetAuthKey(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeEgress, + Replicas: new(int32(1)), + }, + } + tsClient := &fakeTSClient{} + + // Variables to reference in test cases. + existingAuthKey := new("existing-auth-key") + newAuthKey := new("new-authkey") + configWith := func(authKey *string) map[string][]byte { + value := []byte("{}") + if authKey != nil { + value = fmt.Appendf(nil, `{"AuthKey": "%s"}`, *authKey) + } + return map[string][]byte{ + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): value, + } + } + + initTest := func() (*ProxyGroupReconciler, client.WithWatch) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg). + WithStatusSubresource(pg). + Build() + zl, _ := zap.NewDevelopment() + fr := record.NewFakeRecorder(1) + cl := tstest.NewClock(tstest.ClockOpts{}) + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + defaultTags: []string{"tag:test-tag"}, + tsFirewallMode: "auto", + + Client: fc, + tsClient: tsClient, + recorder: fr, + log: zl.Sugar(), + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), + } + reconciler.ensureStateAddedForProxyGroup(pg) + + return reconciler, fc + } + + // Config Secret: exists or not, has key or not. + // State Secret: has device ID or not, requested reissue or not. + for name, tc := range map[string]struct { + configData map[string][]byte + stateData map[string][]byte + expectedAuthKey *string + expectReissue bool + }{ + "no_secrets_needs_new": { + expectedAuthKey: newAuthKey, // New ProxyGroup or manually cleared Pod. + }, + "no_config_secret_state_authed_ok": { + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + }, + expectedAuthKey: newAuthKey, // Always create an auth key if we're creating the config Secret. + }, + "config_secret_without_key_state_authed_with_reissue_needs_new": { + configData: configWith(nil), + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + kubetypes.KeyReissueAuthkey: []byte(""), + }, + expectedAuthKey: newAuthKey, + expectReissue: true, // Device is authed but reissue was requested. + }, + "config_secret_with_key_state_with_reissue_stale_ok": { + configData: configWith(existingAuthKey), + stateData: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("some-older-authkey"), + }, + expectedAuthKey: existingAuthKey, // Config's auth key is different from the one marked for reissue. + }, + "config_secret_with_key_state_with_reissue_existing_key_needs_new": { + configData: configWith(existingAuthKey), + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + kubetypes.KeyReissueAuthkey: []byte(*existingAuthKey), + }, + expectedAuthKey: newAuthKey, + expectReissue: true, // Current config's auth key is marked for reissue. + }, + "config_secret_without_key_no_state_ok": { + configData: configWith(nil), + expectedAuthKey: nil, // Proxy will set reissue_authkey and then next reconcile will reissue. + }, + "config_secret_without_key_state_authed_ok": { + configData: configWith(nil), + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + }, + expectedAuthKey: nil, // Device is already authed. + }, + "config_secret_with_key_state_authed_ok": { + configData: configWith(existingAuthKey), + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + }, + expectedAuthKey: nil, // Auth key getting removed because device is authed. + }, + "config_secret_with_key_no_state_keeps_existing": { + configData: configWith(existingAuthKey), + expectedAuthKey: existingAuthKey, // No state, waiting for containerboot to try the auth key. + }, + } { + t.Run(name, func(t *testing.T) { + tsClient.deleted = tsClient.deleted[:0] // Reset deleted devices for each test case. + reconciler, fc := initTest() + var cfgSecret *corev1.Secret + if tc.configData != nil { + cfgSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgConfigSecretName(pg.Name, 0), + Namespace: tsNamespace, + }, + Data: tc.configData, + } + } + if tc.stateData != nil { + mustCreate(t, fc, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pg.Name, 0), + Namespace: tsNamespace, + }, + Data: tc.stateData, + }) + } + + authKey, err := reconciler.getAuthKey(t.Context(), tsClient, pg, cfgSecret, 0, reconciler.log.With("TestName", t.Name())) + if err != nil { + t.Fatalf("unexpected error getting auth key: %v", err) + } + if !reflect.DeepEqual(authKey, tc.expectedAuthKey) { + deref := func(s *string) string { + if s == nil { + return "" + } + return *s + } + t.Errorf("expected auth key %v, got %v", deref(tc.expectedAuthKey), deref(authKey)) + } + + // Use the device deletion as a proxy for the fact the new auth key + // was due to a reissue. + switch { + case tc.expectReissue && len(tsClient.deleted) != 1: + t.Errorf("expected 1 deleted device, got %v", tsClient.deleted) + case !tc.expectReissue && len(tsClient.deleted) != 0: + t.Errorf("expected no deleted devices, got %v", tsClient.deleted) + } + + if tc.expectReissue { + // Trigger the rate limit in a tight loop. Up to 100 iterations + // to allow for CI that is extremely slow, but should happen on + // first try for any reasonable machine. + stateSecretName := pgStateSecretName(pg.Name, 0) + for range 100 { + //NOTE: (ChaosInTheCRD) we added some protection here to avoid + // trying to reissue when already reissung. This overrides it. + reconciler.mu.Lock() + reconciler.authKeyReissuing[stateSecretName] = false + reconciler.mu.Unlock() + _, err := reconciler.getAuthKey(context.Background(), tsClient, pg, cfgSecret, 0, + reconciler.log.With("TestName", t.Name())) + if err != nil { + if !strings.Contains(err.Error(), "rate limit exceeded") { + t.Fatalf("unexpected error getting auth key: %v", err) + } + return // Expected rate limit error. + } + } + t.Fatal("expected rate limit error, but got none") + } + }) + } +} + func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) { pcLEStaging := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ @@ -1903,6 +2112,8 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { tsClient: &fakeTSClient{}, log: zl.Sugar(), clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } expectReconciled(t, reconciler, "", pg.Name) diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 5f33a94905785..519f81fe0db29 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -1111,7 +1111,7 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, loginUrl string, newAuthkey stri if newAuthkey != "" { conf.AuthKey = &newAuthkey - } else if shouldRetainAuthKey(oldSecret) { + } else if !deviceAuthed(oldSecret) { key, err := authKeyFromSecret(oldSecret) if err != nil { return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err) @@ -1164,6 +1164,8 @@ func latestConfigFromSecret(s *corev1.Secret) (*ipn.ConfigVAlpha, error) { return conf, nil } +// authKeyFromSecret returns the auth key from the latest config version if +// found, or else nil. func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { conf, err := latestConfigFromSecret(s) if err != nil { @@ -1180,13 +1182,13 @@ func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { return key, nil } -// shouldRetainAuthKey returns true if the state stored in a proxy's state Secret suggests that auth key should be -// retained (because the proxy has not yet successfully authenticated). -func shouldRetainAuthKey(s *corev1.Secret) bool { +// deviceAuthed returns true if the state stored in a proxy's state Secret +// suggests that the proxy has successfully authenticated. +func deviceAuthed(s *corev1.Secret) bool { if s == nil { - return false // nothing to retain here + return false // No state Secret means no device state. } - return len(s.Data["device_id"]) == 0 // proxy has not authed yet + return len(s.Data["device_id"]) > 0 } func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool { diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index d418f01284b95..36b608ef6f4fd 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -529,7 +529,7 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec AcceptDNS: "false", Hostname: &opts.hostname, Locked: "false", - AuthKey: new("secret-authkey"), + AuthKey: new("new-authkey"), AcceptRoutes: "false", AppConnector: &ipn.AppConnectorPrefs{Advertise: false}, NoStatefulFiltering: "true", @@ -859,7 +859,7 @@ func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabili Created: time.Now(), Capabilities: caps, } - return "secret-authkey", k, nil + return "new-authkey", k, nil } func (c *fakeTSClient) Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) { diff --git a/cmd/k8s-operator/tsrecorder_test.go b/cmd/k8s-operator/tsrecorder_test.go index 0e1641243c937..d3ebc3bd5eaa8 100644 --- a/cmd/k8s-operator/tsrecorder_test.go +++ b/cmd/k8s-operator/tsrecorder_test.go @@ -284,7 +284,7 @@ func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recor } for replica := range replicas { - auth := tsrAuthSecret(tsr, tsNamespace, "secret-authkey", replica) + auth := tsrAuthSecret(tsr, tsNamespace, "new-authkey", replica) state := tsrStateSecret(tsr, tsNamespace, replica) if shouldExist { diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index 187f54f3481f8..9f1b29064acca 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -38,17 +38,17 @@ const ( // Keys that containerboot writes to state file that can be used to determine its state. // fields set in Tailscale state Secret. These are mostly used by the Tailscale Kubernetes operator to determine // the state of this tailscale device. - KeyDeviceID string = "device_id" // node stable ID of the device - KeyDeviceFQDN string = "device_fqdn" // device's tailnet hostname - KeyDeviceIPs string = "device_ips" // device's tailnet IPs - KeyPodUID string = "pod_uid" // Pod UID - // KeyCapVer contains Tailscale capability version of this proxy instance. - KeyCapVer string = "tailscale_capver" + KeyDeviceID = "device_id" // node stable ID of the device + KeyDeviceFQDN = "device_fqdn" // device's tailnet hostname + KeyDeviceIPs = "device_ips" // device's tailnet IPs + KeyPodUID = "pod_uid" // Pod UID + KeyCapVer = "tailscale_capver" // tailcfg.CurrentCapabilityVersion of this proxy instance. + KeyReissueAuthkey = "reissue_authkey" // Proxies will set this to the authkey that failed, or "no-authkey", if they can't log in. // KeyHTTPSEndpoint is a name of a field that can be set to the value of any HTTPS endpoint currently exposed by // this device to the tailnet. This is used by the Kubernetes operator Ingress proxy to communicate to the operator // that cluster workloads behind the Ingress can now be accessed via the given DNS name over HTTPS. - KeyHTTPSEndpoint string = "https_endpoint" - ValueNoHTTPS string = "no-https" + KeyHTTPSEndpoint = "https_endpoint" + ValueNoHTTPS = "no-https" // Pod's IPv4 address header key as returned by containerboot health check endpoint. PodIPv4Header string = "Pod-IPv4" From 70de1113948dd03e72e10277be60411faf72d459 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 9 Mar 2026 21:26:36 +0000 Subject: [PATCH 22/78] wgengine/magicsock: fix three race conditions in TestTwoDevicePing Fix three independent flake sources, at least as debugged by Claude, though empirically no longer flaking as it was before: 1. Poll for connection counter data instead of reading immediately. The conncount callback fires asynchronously on received WireGuard traffic, so after counts.Reset() there is no guarantee the counter has been repopulated before checkStats reads it. Use tstest.WaitFor with a 5s timeout to retry until a matching connection appears. 2. Replace the *2 symmetry assumption in global metric assertions. metricSendUDP and friends are AggregateCounters that sum per-conn expvars from both magicsock instances. The old assertion assumed both instances had identical packet counts, which breaks under asymmetric background WireGuard activity (handshake retries, etc). The new assertGlobalMetricsMatchPerConn computes the actual sum of both conns' expvars and compares against the AggregateCounter value. 3. Tolerate physical stats being 0 when user metrics are non-zero. A rebind event replaces the socket mid-measurement, resetting the physical connection counter while user metrics still reflect packets processed before the rebind. Log instead of failing in this case. Also move counts.Reset() after metric reads and reorder the reset sequence (counts before metrics) to minimize the race window. Fixes tailscale/tailscale#13420 Change-Id: I7b090a4dc229a862c1a52161b3f2547ec1d1f23f Signed-off-by: Brad Fitzpatrick --- wgengine/magicsock/magicsock_test.go | 103 ++++++++++++++++++--------- 1 file changed, 71 insertions(+), 32 deletions(-) diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 7a8a6374cd1bc..4ecea8b18a586 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -1191,15 +1191,19 @@ func testTwoDevicePing(t *testing.T, d *devices) { m2.conn.SetConnectionCounter(m2.counts.Add) checkStats := func(t *testing.T, m *magicStack, wantConns []netlogtype.Connection) { + t.Helper() defer m.counts.Reset() - counts := m.counts.Clone() - for _, conn := range wantConns { - if _, ok := counts[conn]; ok { - return + if err := tstest.WaitFor(5*time.Second, func() error { + counts := m.counts.Clone() + for _, conn := range wantConns { + if _, ok := counts[conn]; ok { + return nil + } } + return fmt.Errorf("missing any connection to %s from %s", wantConns, slicesx.MapKeys(counts)) + }); err != nil { + t.Error(err) } - t.Helper() - t.Errorf("missing any connection to %s from %s", wantConns, slicesx.MapKeys(counts)) } addrPort := netip.MustParseAddrPort @@ -1261,15 +1265,16 @@ func testTwoDevicePing(t *testing.T, d *devices) { t.Run("compare-metrics-stats", func(t *testing.T) { setT(t) defer setT(outerT) - m1.conn.resetMetricsForTest() m1.counts.Reset() - m2.conn.resetMetricsForTest() m2.counts.Reset() + m1.conn.resetMetricsForTest() + m2.conn.resetMetricsForTest() t.Logf("Metrics before: %s\n", m1.metrics.String()) ping1(t) ping2(t) assertConnStatsAndUserMetricsEqual(t, m1) assertConnStatsAndUserMetricsEqual(t, m2) + assertGlobalMetricsMatchPerConn(t, m1, m2) t.Logf("Metrics after: %s\n", m1.metrics.String()) }) } @@ -1290,6 +1295,7 @@ func (c *Conn) resetMetricsForTest() { } func assertConnStatsAndUserMetricsEqual(t *testing.T, ms *magicStack) { + t.Helper() physIPv4RxBytes := int64(0) physIPv4TxBytes := int64(0) physDERPRxBytes := int64(0) @@ -1312,7 +1318,6 @@ func assertConnStatsAndUserMetricsEqual(t *testing.T, ms *magicStack) { physIPv4TxPackets += int64(count.TxPackets) } } - ms.counts.Reset() metricIPv4RxBytes := ms.conn.metrics.inboundBytesIPv4Total.Value() metricIPv4RxPackets := ms.conn.metrics.inboundPacketsIPv4Total.Value() @@ -1324,30 +1329,64 @@ func assertConnStatsAndUserMetricsEqual(t *testing.T, ms *magicStack) { metricDERPTxBytes := ms.conn.metrics.outboundBytesDERPTotal.Value() metricDERPTxPackets := ms.conn.metrics.outboundPacketsDERPTotal.Value() + // Reset counts after reading all values to minimize the window where a + // background packet could increment metrics but miss the cloned counts. + ms.counts.Reset() + + // Compare physical connection stats with per-conn user metrics. + // A rebind during the measurement window can reset the physical connection + // counter, causing physical stats to show 0 while user metrics recorded + // packets normally. Tolerate this by logging instead of failing. + checkPhysVsMetric := func(phys, metric int64, name string) { + if phys == metric { + return + } + if phys == 0 && metric > 0 { + t.Logf("%s: physical counter is 0 but metric is %d (possible rebind during measurement)", name, metric) + return + } + t.Errorf("%s: physical=%d, metric=%d", name, phys, metric) + } + checkPhysVsMetric(physDERPRxBytes, metricDERPRxBytes, "DERPRxBytes") + checkPhysVsMetric(physDERPTxBytes, metricDERPTxBytes, "DERPTxBytes") + checkPhysVsMetric(physIPv4RxBytes, metricIPv4RxBytes, "IPv4RxBytes") + checkPhysVsMetric(physIPv4TxBytes, metricIPv4TxBytes, "IPv4TxBytes") + checkPhysVsMetric(physDERPRxPackets, metricDERPRxPackets, "DERPRxPackets") + checkPhysVsMetric(physDERPTxPackets, metricDERPTxPackets, "DERPTxPackets") + checkPhysVsMetric(physIPv4RxPackets, metricIPv4RxPackets, "IPv4RxPackets") + checkPhysVsMetric(physIPv4TxPackets, metricIPv4TxPackets, "IPv4TxPackets") +} + +// assertGlobalMetricsMatchPerConn validates that the global clientmetric +// AggregateCounters match the sum of per-conn user metrics from both magicsock +// instances. This tests the metric registration wiring rather than assuming +// symmetric traffic between the two instances. +func assertGlobalMetricsMatchPerConn(t *testing.T, m1, m2 *magicStack) { + t.Helper() c := qt.New(t) - c.Assert(physDERPRxBytes, qt.Equals, metricDERPRxBytes) - c.Assert(physDERPTxBytes, qt.Equals, metricDERPTxBytes) - c.Assert(physIPv4RxBytes, qt.Equals, metricIPv4RxBytes) - c.Assert(physIPv4TxBytes, qt.Equals, metricIPv4TxBytes) - c.Assert(physDERPRxPackets, qt.Equals, metricDERPRxPackets) - c.Assert(physDERPTxPackets, qt.Equals, metricDERPTxPackets) - c.Assert(physIPv4RxPackets, qt.Equals, metricIPv4RxPackets) - c.Assert(physIPv4TxPackets, qt.Equals, metricIPv4TxPackets) - - // Validate that the usermetrics and clientmetrics are in sync - // Note: the clientmetrics are global, this means that when they are registering with the - // wgengine, multiple in-process nodes used by this test will be updating the same metrics. This is why we need to multiply - // the metrics by 2 to get the expected value. - // TODO(kradalby): https://github.com/tailscale/tailscale/issues/13420 - c.Assert(metricSendUDP.Value(), qt.Equals, metricIPv4TxPackets*2) - c.Assert(metricSendDataPacketsIPv4.Value(), qt.Equals, metricIPv4TxPackets*2) - c.Assert(metricSendDataPacketsDERP.Value(), qt.Equals, metricDERPTxPackets*2) - c.Assert(metricSendDataBytesIPv4.Value(), qt.Equals, metricIPv4TxBytes*2) - c.Assert(metricSendDataBytesDERP.Value(), qt.Equals, metricDERPTxBytes*2) - c.Assert(metricRecvDataPacketsIPv4.Value(), qt.Equals, metricIPv4RxPackets*2) - c.Assert(metricRecvDataPacketsDERP.Value(), qt.Equals, metricDERPRxPackets*2) - c.Assert(metricRecvDataBytesIPv4.Value(), qt.Equals, metricIPv4RxBytes*2) - c.Assert(metricRecvDataBytesDERP.Value(), qt.Equals, metricDERPRxBytes*2) + m1m := m1.conn.metrics + m2m := m2.conn.metrics + + // metricSendUDP aggregates outboundPacketsIPv4Total + outboundPacketsIPv6Total + c.Assert(metricSendUDP.Value(), qt.Equals, + m1m.outboundPacketsIPv4Total.Value()+m1m.outboundPacketsIPv6Total.Value()+ + m2m.outboundPacketsIPv4Total.Value()+m2m.outboundPacketsIPv6Total.Value()) + c.Assert(metricSendDataPacketsIPv4.Value(), qt.Equals, + m1m.outboundPacketsIPv4Total.Value()+m2m.outboundPacketsIPv4Total.Value()) + c.Assert(metricSendDataPacketsDERP.Value(), qt.Equals, + m1m.outboundPacketsDERPTotal.Value()+m2m.outboundPacketsDERPTotal.Value()) + c.Assert(metricSendDataBytesIPv4.Value(), qt.Equals, + m1m.outboundBytesIPv4Total.Value()+m2m.outboundBytesIPv4Total.Value()) + c.Assert(metricSendDataBytesDERP.Value(), qt.Equals, + m1m.outboundBytesDERPTotal.Value()+m2m.outboundBytesDERPTotal.Value()) + c.Assert(metricRecvDataPacketsIPv4.Value(), qt.Equals, + m1m.inboundPacketsIPv4Total.Value()+m2m.inboundPacketsIPv4Total.Value()) + c.Assert(metricRecvDataPacketsDERP.Value(), qt.Equals, + m1m.inboundPacketsDERPTotal.Value()+m2m.inboundPacketsDERPTotal.Value()) + c.Assert(metricRecvDataBytesIPv4.Value(), qt.Equals, + m1m.inboundBytesIPv4Total.Value()+m2m.inboundBytesIPv4Total.Value()) + c.Assert(metricRecvDataBytesDERP.Value(), qt.Equals, + m1m.inboundBytesDERPTotal.Value()+m2m.inboundBytesDERPTotal.Value()) } // tests that having a endpoint.String prevents wireguard-go's From 607d01cdaee918d801157ac110530b4b92d3d11c Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 11 Mar 2026 10:13:49 -0700 Subject: [PATCH 23/78] net/batching: clarify & simplify single packet read limitations ReadFromUDPAddrPort worked if UDP GRO was unsupported, but we don't actually want attempted usage, nor does any exist today. Future work on tailscale/corp#37679 would have required more complexity in this method, vs clarifying the API intents. Updates tailscale/corp#37679 Signed-off-by: Jordan Whited --- net/batching/conn.go | 12 +++++++++++- net/batching/conn_linux.go | 11 +---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/net/batching/conn.go b/net/batching/conn.go index 1631c33cfe448..1843a2cfced5a 100644 --- a/net/batching/conn.go +++ b/net/batching/conn.go @@ -19,14 +19,24 @@ var ( _ ipv6.Message = ipv4.Message{} ) -// Conn is a nettype.PacketConn that provides batched i/o using +// Conn is a [nettype.PacketConn] that provides batched i/o using // platform-specific optimizations, e.g. {recv,send}mmsg & UDP GSO/GRO. // +// Conn does not support single packet reads (see ReadFromUDPAddrPort docs). It +// is the caller's responsibility to use the appropriate read API where a +// [nettype.PacketConn] has been upgraded to support batched i/o. +// // Conn originated from (and is still used by) magicsock where its API was // strongly influenced by [wireguard-go/conn.Bind] constraints, namely // wireguard-go's ownership of packet memory. type Conn interface { nettype.PacketConn + // ReadFromUDPAddrPort always returns an error, as UDP GRO is incompatible + // with single packet reads. A single datagram may be multiple, coalesced + // datagrams, and this API lacks the ability to pass that context. + // + // TODO: consider detaching Conn from [nettype.PacketConn] + ReadFromUDPAddrPort([]byte) (int, netip.AddrPort, error) // ReadBatch reads messages from [Conn] into msgs. It returns the number of // messages the caller should evaluate for nonzero len, as a zero len // message may fall on either side of a nonzero. diff --git a/net/batching/conn_linux.go b/net/batching/conn_linux.go index 373625b772738..70f91cfb6847c 100644 --- a/net/batching/conn_linux.go +++ b/net/batching/conn_linux.go @@ -61,16 +61,7 @@ type linuxBatchingConn struct { } func (c *linuxBatchingConn) ReadFromUDPAddrPort(p []byte) (n int, addr netip.AddrPort, err error) { - if c.rxOffload { - // UDP_GRO is opt-in on Linux via setsockopt(). Once enabled you may - // receive a "monster datagram" from any read call. The ReadFrom() API - // does not support passing the GSO size and is unsafe to use in such a - // case. Other platforms may vary in behavior, but we go with the most - // conservative approach to prevent this from becoming a footgun in the - // future. - return 0, netip.AddrPort{}, errors.New("rx UDP offload is enabled on this socket, single packet reads are unavailable") - } - return c.pc.ReadFromUDPAddrPort(p) + return 0, netip.AddrPort{}, errors.New("single packet reads are unsupported") } func (c *linuxBatchingConn) SetDeadline(t time.Time) error { From dd1da0b38921de5a8091f4e9e83845ac997d2c83 Mon Sep 17 00:00:00 2001 From: kari-ts <135075563+kari-ts@users.noreply.github.com> Date: Wed, 11 Mar 2026 12:21:50 -0700 Subject: [PATCH 24/78] wgengine: search randomly for unused port instead of in contiguous range (#18974) In TestUserspaceEnginePortReconfig, when selecting a port, use a random offset rather than searching in a continguous range in case there is a range that is blocked Updates tailscale/tailscale#2855 Signed-off-by: kari-ts --- wgengine/userspace_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wgengine/userspace_test.go b/wgengine/userspace_test.go index b06ea527b27ba..18d870af1e6dc 100644 --- a/wgengine/userspace_test.go +++ b/wgengine/userspace_test.go @@ -5,6 +5,7 @@ package wgengine import ( "fmt" + "math/rand" "net/netip" "os" "reflect" @@ -175,8 +176,8 @@ func TestUserspaceEnginePortReconfig(t *testing.T) { var ue *userspaceEngine ht := health.NewTracker(bus) reg := new(usermetric.Registry) - for i := range 100 { - attempt := uint16(defaultPort + i) + for range 100 { + attempt := uint16(defaultPort + rand.Intn(1000)) e, err := NewFakeUserspaceEngine(t.Logf, attempt, &knobs, ht, reg, bus) if err != nil { t.Fatal(err) From 4c7c1091ba5c623031df289affe2337d26585fcc Mon Sep 17 00:00:00 2001 From: kari-ts <135075563+kari-ts@users.noreply.github.com> Date: Wed, 11 Mar 2026 12:28:28 -0700 Subject: [PATCH 25/78] netns: add Android callback to bind socket to network (#18915) After switching from cellular to wifi without ipv6, ForeachInterface still sees rmnet prefixes, so HaveV6 stays true, and magicsock keeps attempting ipv6 connections that either route through cellular or time out for users on wifi without ipv6 This: -Adds SetAndroidBindToNetworkFunc, a callback to bind the socket to the selected Android Network object Updates tailscale/tailscale#6152 Signed-off-by: kari-ts --- ipn/ipnlocal/local.go | 3 +++ net/netns/netns.go | 12 ++++++++++++ net/netns/netns_android.go | 39 +++++++++++++++++++++++++++++++++++--- tailcfg/tailcfg.go | 9 ++++++++- 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index ea5af0897a54a..da126ed0f8ca0 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -6299,6 +6299,9 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { // See the netns package for documentation on what these capability do. netns.SetBindToInterfaceByRoute(b.logf, nm.HasCap(tailcfg.CapabilityBindToInterfaceByRoute)) + if runtime.GOOS == "android" { + netns.SetDisableAndroidBindToActiveNetwork(b.logf, nm.HasCap(tailcfg.NodeAttrDisableAndroidBindToActiveNetwork)) + } netns.SetDisableBindConnToInterface(b.logf, nm.HasCap(tailcfg.CapabilityDebugDisableBindConnToInterface)) netns.SetDisableBindConnToInterfaceAppleExt(b.logf, nm.HasCap(tailcfg.CapabilityDebugDisableBindConnToInterfaceAppleExt)) diff --git a/net/netns/netns.go b/net/netns/netns.go index 5d692c787eae8..fe7ff4dcbadd8 100644 --- a/net/netns/netns.go +++ b/net/netns/netns.go @@ -46,6 +46,18 @@ func SetBindToInterfaceByRoute(logf logger.Logf, v bool) { } } +// When true, disableAndroidBindToActiveNetwork skips binding sockets to the currently +// active network on Android. +var disableAndroidBindToActiveNetwork atomic.Bool + +// SetDisableAndroidBindToActiveNetwork disables the default behavior of binding +// sockets to the currently active network on Android. +func SetDisableAndroidBindToActiveNetwork(logf logger.Logf, v bool) { + if runtime.GOOS == "android" && disableAndroidBindToActiveNetwork.Swap(v) != v { + logf("netns: disableAndroidBindToActiveNetwork changed to %v", v) + } +} + var disableBindConnToInterface atomic.Bool // SetDisableBindConnToInterface disables the (normal) behavior of binding diff --git a/net/netns/netns_android.go b/net/netns/netns_android.go index e747f61f40e50..7c5fe3214dcbf 100644 --- a/net/netns/netns_android.go +++ b/net/netns/netns_android.go @@ -17,6 +17,9 @@ import ( var ( androidProtectFuncMu sync.Mutex androidProtectFunc func(fd int) error + + androidBindToNetworkFuncMu sync.Mutex + androidBindToNetworkFunc func(fd int) error ) // UseSocketMark reports whether SO_MARK is in use. Android does not use SO_MARK. @@ -50,6 +53,14 @@ func SetAndroidProtectFunc(f func(fd int) error) { androidProtectFunc = f } +// SetAndroidBindToNetworkFunc registers a func provided by Android that binds +// the socket FD to the currently selected underlying network. +func SetAndroidBindToNetworkFunc(f func(fd int) error) { + androidBindToNetworkFuncMu.Lock() + defer androidBindToNetworkFuncMu.Unlock() + androidBindToNetworkFunc = f +} + func control(logger.Logf, *netmon.Monitor) func(network, address string, c syscall.RawConn) error { return controlC } @@ -60,14 +71,36 @@ func control(logger.Logf, *netmon.Monitor) func(network, address string, c sysca // and net.ListenConfig.Control. func controlC(network, address string, c syscall.RawConn) error { var sockErr error + err := c.Control(func(fd uintptr) { + fdInt := int(fd) + + // Protect from VPN loops androidProtectFuncMu.Lock() - f := androidProtectFunc + pf := androidProtectFunc androidProtectFuncMu.Unlock() - if f != nil { - sockErr = f(int(fd)) + if pf != nil { + if err := pf(fdInt); err != nil { + sockErr = err + return + } + } + + if disableAndroidBindToActiveNetwork.Load() { + return + } + + androidBindToNetworkFuncMu.Lock() + bf := androidBindToNetworkFunc + androidBindToNetworkFuncMu.Unlock() + if bf != nil { + if err := bf(fdInt); err != nil { + sockErr = err + return + } } }) + if err != nil { return fmt.Errorf("RawConn.Control on %T: %w", c, err) } diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 1efa6c959214e..04389fabaded8 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -180,7 +180,8 @@ type CapabilityVersion int // - 131: 2025-11-25: client respects [NodeAttrDefaultAutoUpdate] // - 132: 2026-02-13: client respects [NodeAttrDisableHostsFileUpdates] // - 133: 2026-02-17: client understands [NodeAttrForceRegisterMagicDNSIPv4Only]; MagicDNS IPv6 registered w/ OS by default -const CurrentCapabilityVersion CapabilityVersion = 133 +// - 134: 2026-03-09: Client understands [NodeAttrDisableAndroidBindToActiveNetwork] +const CurrentCapabilityVersion CapabilityVersion = 134 // ID is an integer ID for a user, node, or login allocated by the // control plane. @@ -2463,6 +2464,12 @@ const ( // details on the behaviour of this capability. CapabilityBindToInterfaceByRoute NodeCapability = "https://tailscale.com/cap/bind-to-interface-by-route" + // NodeAttrDisableAndroidBindToActiveNetwork disables binding sockets to the + // currently active network on Android, which is enabled by default. + // This allows the control plane to turn off the behavior if it causes + // problems. + NodeAttrDisableAndroidBindToActiveNetwork NodeCapability = "disable-android-bind-to-active-network" + // CapabilityDebugDisableAlternateDefaultRouteInterface changes how Darwin // nodes get the default interface. There is an optional hook (used by the // macOS and iOS clients) to override the default interface, this capability From 073a9a8c9ed449c1a620106084e43b0d38d1c5cb Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 11 Mar 2026 13:29:06 +0000 Subject: [PATCH 26/78] wgengine{,/magicsock}: add DERP hooks for filtering+sending packets Add two small APIs to support out-of-tree projects to exchange custom signaling messages over DERP without requiring disco protocol extensions: - OnDERPRecv callback on magicsock.Options / wgengine.Config: called for every non-disco DERP packet before the peer map lookup, allowing callers to intercept packets from unknown peers that would otherwise be dropped. - SendDERPPacketTo method on magicsock.Conn: sends arbitrary bytes to a node key via a DERP region, creating the connection if needed. Thin wrapper around the existing internal sendAddr. Also allow netstack.Start to accept a nil LocalBackend for use cases that wire up TCP/UDP handlers directly without a full LocalBackend. Updates tailscale/corp#24454 Change-Id: I99a523ef281625b8c0024a963f5f5bf5d8792c17 Signed-off-by: Brad Fitzpatrick --- wgengine/magicsock/derp.go | 13 +++++++++++++ wgengine/magicsock/magicsock.go | 17 +++++++++++++---- wgengine/netstack/netstack.go | 24 +++++++++++++++++------- wgengine/userspace.go | 8 ++++++++ 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/wgengine/magicsock/derp.go b/wgengine/magicsock/derp.go index f9e5050705b31..17e3cfa82ebe6 100644 --- a/wgengine/magicsock/derp.go +++ b/wgengine/magicsock/derp.go @@ -725,6 +725,10 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep *en return 0, nil } + if c.onDERPRecv != nil && c.onDERPRecv(regionID, dm.src, b[:n]) { + return 0, nil + } + var ok bool c.mu.Lock() ep, ok = c.peerMap.endpointForNodeKey(dm.src) @@ -745,6 +749,15 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep *en return n, ep } +// SendDERPPacketTo sends an arbitrary packet to the given node key via +// the DERP relay for the given region. It creates the DERP connection +// to the region if one doesn't already exist. +func (c *Conn) SendDERPPacketTo(dstKey key.NodePublic, regionID int, pkt []byte) (sent bool, err error) { + return c.sendAddr( + netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, uint16(regionID)), + dstKey, pkt, false, false) +} + // SetOnlyTCP443 set whether the magicsock connection is restricted // to only using TCP port 443 outbound. If true, no UDP is allowed, // no STUN checks are performend, etc. diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index f61e85b37fcec..5c16750f7e8a0 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -163,10 +163,11 @@ type Conn struct { derpActiveFunc func() idleFunc func() time.Duration // nil means unknown testOnlyPacketListener nettype.PacketListener - noteRecvActivity func(key.NodePublic) // or nil, see Options.NoteRecvActivity - netMon *netmon.Monitor // must be non-nil - health *health.Tracker // or nil - controlKnobs *controlknobs.Knobs // or nil + noteRecvActivity func(key.NodePublic) // or nil, see Options.NoteRecvActivity + onDERPRecv func(int, key.NodePublic, []byte) bool // or nil, see Options.OnDERPRecv + netMon *netmon.Monitor // must be non-nil + health *health.Tracker // or nil + controlKnobs *controlknobs.Knobs // or nil // ================================================================ // No locking required to access these fields, either because @@ -502,6 +503,13 @@ type Options struct { // leave it zero, in which case a new disco key is generated per // Tailscale start and kept only in memory. ForceDiscoKey key.DiscoPrivate + + // OnDERPRecv, if non-nil, is called for every non-disco packet + // received from DERP before the peer map lookup. If it returns + // true, the packet is considered handled and is not passed to + // WireGuard. The pkt slice is borrowed and must be copied if + // the callee needs to retain it. + OnDERPRecv func(regionID int, src key.NodePublic, pkt []byte) bool } func (o *Options) logf() logger.Logf { @@ -640,6 +648,7 @@ func NewConn(opts Options) (*Conn, error) { c.idleFunc = opts.IdleFunc c.testOnlyPacketListener = opts.TestOnlyPacketListener c.noteRecvActivity = opts.NoteRecvActivity + c.onDERPRecv = opts.OnDERPRecv // Set up publishers and subscribers. Subscribe calls must return before // NewConn otherwise published events can be missed. diff --git a/wgengine/netstack/netstack.go b/wgengine/netstack/netstack.go index 59c2613451fa5..ae77a1dac1787 100644 --- a/wgengine/netstack/netstack.go +++ b/wgengine/netstack/netstack.go @@ -603,15 +603,25 @@ type LocalBackend = any // Start sets up all the handlers so netstack can start working. Implements // wgengine.FakeImpl. +// +// The provided LocalBackend interface can be either nil, for special case users +// of netstack that don't have a LocalBackend, or a non-nil +// *ipnlocal.LocalBackend. Any other type will cause Start to panic. +// +// Start currently (2026-03-11) never returns a non-nil error, but maybe it did +// in the past and maybe it will in the future. func (ns *Impl) Start(b LocalBackend) error { - if b == nil { - panic("nil LocalBackend interface") - } - lb := b.(*ipnlocal.LocalBackend) - if lb == nil { - panic("nil LocalBackend") + switch b := b.(type) { + case nil: + // No backend, so just continue with ns.lb unset. + case *ipnlocal.LocalBackend: + if b == nil { + panic("nil LocalBackend") + } + ns.lb = b + default: + panic(fmt.Sprintf("unexpected type for LocalBackend: %T", b)) } - ns.lb = lb tcpFwd := tcp.NewForwarder(ns.ipstack, tcpRXBufDefSize, maxInFlightConnectionAttempts(), ns.acceptTCP) udpFwd := udp.NewForwarder(ns.ipstack, ns.acceptUDPNoICMP) ns.ipstack.SetTransportProtocolHandler(tcp.ProtocolNumber, ns.wrapTCPProtocolHandler(tcpFwd.HandlePacket)) diff --git a/wgengine/userspace.go b/wgengine/userspace.go index 705555d4446a6..ecf3c22983aa4 100644 --- a/wgengine/userspace.go +++ b/wgengine/userspace.go @@ -272,6 +272,13 @@ type Config struct { // leave it zero, in which case a new disco key is generated per // Tailscale start and kept only in memory. ForceDiscoKey key.DiscoPrivate + + // OnDERPRecv, if non-nil, is called for every non-disco packet + // received from DERP before the peer map lookup. If it returns + // true, the packet is considered handled and is not passed to + // WireGuard. The pkt slice is borrowed and must be copied if + // the callee needs to retain it. + OnDERPRecv func(regionID int, src key.NodePublic, pkt []byte) (handled bool) } // NewFakeUserspaceEngine returns a new userspace engine for testing. @@ -441,6 +448,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error) ControlKnobs: conf.ControlKnobs, PeerByKeyFunc: e.PeerByKey, ForceDiscoKey: conf.ForceDiscoKey, + OnDERPRecv: conf.OnDERPRecv, } if buildfeatures.HasLazyWG { magicsockOpts.NoteRecvActivity = e.noteRecvActivity From 0c53cf7ad9bc065b55e62bd45fda454a665e0726 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 12:01:14 +0000 Subject: [PATCH 27/78] .github: Bump actions/upload-artifact from 6.0.0 to 7.0.0 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 6.0.0 to 7.0.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/b7c566a772e6b6bfb58ed0dc250532a479d7789f...bbbca2ddaa5d8feaa63e36b76fdaad77386f024f) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cigocacher.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cigocacher.yml b/.github/workflows/cigocacher.yml index 15aec8af90904..8a13474f2d92c 100644 --- a/.github/workflows/cigocacher.yml +++ b/.github/workflows/cigocacher.yml @@ -24,7 +24,7 @@ jobs: ./tool/go build -o "${OUT}" ./cmd/cigocacher/ tar -zcf cigocacher-${{ matrix.GOOS }}-${{ matrix.GOARCH }}.tar.gz "${OUT}" - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: cigocacher-${{ matrix.GOOS }}-${{ matrix.GOARCH }} path: cigocacher-${{ matrix.GOOS }}-${{ matrix.GOARCH }}.tar.gz diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f6068e6e33cd..317052229676e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -741,7 +741,7 @@ jobs: run: | echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV - name: upload crash - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: steps.run.outcome != 'success' && steps.build.outcome == 'success' with: name: artifacts From 224305b57710bb6131afe8f469bdc3f91c0b570a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 12:01:20 +0000 Subject: [PATCH 28/78] .github: Bump actions/download-artifact from 7.0.0 to 8.0.0 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 7.0.0 to 8.0.0. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/37930b1c2abaa49bbe596cd826c3c89aef350131...70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: 8.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cigocacher.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cigocacher.yml b/.github/workflows/cigocacher.yml index 8a13474f2d92c..fea1f6a0dc988 100644 --- a/.github/workflows/cigocacher.yml +++ b/.github/workflows/cigocacher.yml @@ -36,7 +36,7 @@ jobs: contents: write steps: - name: Download all artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: pattern: 'cigocacher-*' merge-multiple: true From 0a4e0e2940c0938697305d1c87a38f53b5aefefd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:47:36 +0000 Subject: [PATCH 29/78] .github: Bump github/codeql-action from 4.32.5 to 4.32.6 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.32.5 to 4.32.6. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/c793b717bc78562f491db7b0e93a3a178b099162...0d579ffd059c29b07949a3cce3983f0780820c98) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.32.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 51bae5a068df5..9e1e518f666fc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -55,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@c793b717bc78562f491db7b0e93a3a178b099162 # v4.32.5 + uses: github/codeql-action/init@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@c793b717bc78562f491db7b0e93a3a178b099162 # v4.32.5 + uses: github/codeql-action/autobuild@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@c793b717bc78562f491db7b0e93a3a178b099162 # v4.32.5 + uses: github/codeql-action/analyze@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 From becc5be78a8d4bac9e754eec287c4cf3ad08fd58 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Thu, 12 Mar 2026 09:35:11 +0000 Subject: [PATCH 30/78] wgengine/magicsock: implement SCION batch receive and parsing enhancements - Introduced scionRecvBatch for efficient batch processing of SCION packets, utilizing a sync.Pool for buffer reuse. - Added parseSCIONPacket function to extract source address and payload from raw SCION packets, improving packet handling. - Enhanced receiveSCION method to support batch reading from the underlay socket, optimizing performance during packet reception. - Updated logic for handling disco packets to leverage the new batch processing capabilities. --- wgengine/magicsock/magicsock_scion.go | 283 +++++++++++++++++++++++--- 1 file changed, 260 insertions(+), 23 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 261a487d6b3eb..fea81a78573f1 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -16,8 +16,12 @@ import ( "sync" "time" + "github.com/google/gopacket" "github.com/scionproto/scion/pkg/addr" "github.com/scionproto/scion/pkg/daemon" + "github.com/scionproto/scion/pkg/slayers" + scionpath "github.com/scionproto/scion/pkg/slayers/path/scion" + snetpath "github.com/scionproto/scion/pkg/snet/path" "github.com/scionproto/scion/pkg/snet" wgconn "github.com/tailscale/wireguard-go/conn" "golang.org/x/net/ipv4" @@ -223,6 +227,40 @@ var scionSendBatchPool = sync.Pool{ }, } +// scionRecvBatch is a reusable set of buffers for receiveSCIONBatch. +type scionRecvBatch struct { + msgs []ipv4.Message + bufs [][]byte + scn slayers.SCION // reusable SCION header parser (with RecyclePaths) +} + +var scionRecvBatchPool = sync.Pool{ + New: func() any { + b := &scionRecvBatch{ + msgs: make([]ipv4.Message, scionMaxBatchSize), + bufs: make([][]byte, scionMaxBatchSize), + } + b.scn.RecyclePaths() + for i := range b.bufs { + b.bufs[i] = make([]byte, 1500) + } + for i := range b.msgs { + b.msgs[i].Buffers = [][]byte{b.bufs[i]} + } + return b + }, +} + +// putScionRecvBatch resets batch state and returns it to the pool. +func putScionRecvBatch(batch *scionRecvBatch) { + for i := range batch.msgs { + batch.msgs[i].N = 0 + batch.msgs[i].Addr = nil + batch.msgs[i].Buffers[0] = batch.bufs[i] + } + scionRecvBatchPool.Put(batch) +} + // scionPseudoHeaderPartial computes the constant part of the SCION // pseudo-header checksum: srcIA + dstIA + srcAddr + dstAddr + protocol(17). // The per-packet upper-layer length and data are added at send time. @@ -352,6 +390,89 @@ func buildSCIONFastPath(sc *scionConn, pi *scionPathInfo) *scionFastPath { } } +// parseSCIONPacket parses a raw SCION packet from the underlay, extracting +// the source address info and UDP payload. scn is a reusable slayers.SCION +// (with RecyclePaths enabled). Returns srcIA, srcAddr, payload, rawPath, ok. +func parseSCIONPacket(data []byte, scn *slayers.SCION) ( + srcIA addr.IA, srcAddr netip.AddrPort, payload []byte, rawPathBytes []byte, ok bool, +) { + if err := scn.DecodeFromBytes(data, gopacket.NilDecodeFeedback); err != nil { + return 0, netip.AddrPort{}, nil, nil, false + } + if scn.NextHdr != slayers.L4UDP { + return 0, netip.AddrPort{}, nil, nil, false + } + + srcHost, err := scn.SrcAddr() + if err != nil { + return 0, netip.AddrPort{}, nil, nil, false + } + srcIP := srcHost.IP() + srcIA = scn.SrcIA + + // L4 payload starts at HdrLen * 4 bytes (SCION header is HdrLen + // 4-byte words). The first 8 bytes are the UDP header. + hdrBytes := int(scn.HdrLen) * 4 + if len(data) < hdrBytes+8 { + return 0, netip.AddrPort{}, nil, nil, false + } + // Extract UDP source port from the first 2 bytes of the L4 header. + srcPort := binary.BigEndian.Uint16(data[hdrBytes:]) + srcAddr = netip.AddrPortFrom(srcIP, srcPort) + payload = data[hdrBytes+8:] + + // Extract raw path bytes for potential reversal (disco first-contact). + if scn.Path != nil { + pathLen := scn.Path.Len() + // The path sits between the address header and the L4 header + // in the SCION common+address+path header region. + addrHdrLen := scn.AddrHdrLen() + // Common header is 12 bytes, then address header, then path. + pathStart := 12 + addrHdrLen + pathEnd := pathStart + pathLen + if pathEnd <= hdrBytes && pathLen > 0 { + rawPathBytes = data[pathStart:pathEnd] + } + } + + return srcIA, srcAddr, payload, rawPathBytes, true +} + +// buildSCIONReplyAddr builds an *snet.UDPAddr with reversed path for disco +// reply routing from raw path bytes extracted during receive. +func buildSCIONReplyAddr(srcIA addr.IA, srcHostAddr netip.AddrPort, rawPathBytes []byte) *snet.UDPAddr { + if len(rawPathBytes) == 0 { + return nil + } + // Copy path bytes since DecodeFromBytes references the slice. + pathCopy := make([]byte, len(rawPathBytes)) + copy(pathCopy, rawPathBytes) + + var raw scionpath.Raw + if err := raw.DecodeFromBytes(pathCopy); err != nil { + return nil + } + reversed, err := raw.Reverse() + if err != nil { + return nil + } + // Serialize the reversed path to raw bytes and wrap in snetpath.SCION + // which implements snet.DataplanePath. + revBytes := make([]byte, reversed.Len()) + if err := reversed.SerializeTo(revBytes); err != nil { + return nil + } + + return &snet.UDPAddr{ + IA: srcIA, + Host: &net.UDPAddr{ + IP: srcHostAddr.Addr().AsSlice(), + Port: int(srcHostAddr.Port()), + }, + Path: snetpath.SCION{Raw: revBytes}, + } +} + // scionConn wraps a SCION connection for use by magicsock. type scionConn struct { conn *snet.Conn // from SCIONNetwork.Listen() @@ -771,6 +892,10 @@ func (c *Conn) updateActiveSCIONPathLocking(peerIA addr.IA, hostAddr netip.AddrP // the socket is still alive. If no packets are received for // scionReconnectThreshold while active SCION peers exist, we close the old // socket and reconnect. +// +// When the underlay socket is available, packets are read in batches via +// recvmmsg and parsed with lightweight slayers.SCION decoding. Otherwise, +// falls back to single-packet snet.Conn.ReadFrom. func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { sc := c.pconnSCION if sc == nil { @@ -804,34 +929,55 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) continue } - // Set a read deadline so we wake up periodically even if the socket - // is silently dead (SCION router lost our port registration). + // Fast path: batch read from underlay via recvmmsg. + if sc.underlayXPC != nil { + sc.underlayConn.SetReadDeadline(time.Now().Add(scionReadDeadline)) + + n, err := c.receiveSCIONBatch(sc, buffs, sizes, eps) + if n > 0 { + return n, nil + } + if err != nil { + select { + case <-c.donec: + return 0, net.ErrClosed + default: + } + if isTimeoutError(err) { + if c.shouldReconnectSCION() { + c.reconnectSCION() + } + continue + } + if errors.Is(err, net.ErrClosed) { + continue + } + c.logf("magicsock: SCION read error: %v", err) + continue + } + // n == 0 and no error means all packets were disco/filtered. + continue + } + + // Slow path: single-packet snet.Conn.ReadFrom. sc.conn.SetReadDeadline(time.Now().Add(scionReadDeadline)) n, srcAddr, err := sc.readFrom(buffs[0]) if err != nil { - // Graceful shutdown: Conn is closing. select { case <-c.donec: return 0, net.ErrClosed default: } - - // Timeout: check if we need to reconnect. if isTimeoutError(err) { if c.shouldReconnectSCION() { c.reconnectSCION() } continue } - - // Socket closed (by reconnectSCION or externally): re-read - // pconnSCION on next iteration. if errors.Is(err, net.ErrClosed) { continue } - - // Other errors: log and continue. Never propagate to WireGuard. c.logf("magicsock: SCION read error: %v", err) continue } @@ -839,28 +985,19 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) continue } - // Got a packet — record receive time. c.lastSCIONRecv.StoreAtomic(mono.Now()) b := buffs[0][:n] - srcHostAddr := srcAddr.Host.AddrPort() - // Check for disco packets (same as receiveIP does). pt, _ := packetLooksLike(b) if pt == packetLooksLikeDisco { - // For disco messages, include the scionKey so pong replies - // are routed back over SCION. Use a single critical section - // for the lookup+register to avoid a TOCTOU race where a - // concurrent discoverSCIONPaths could register between our - // check and our register, creating orphaned entries. + // Slow path disco: snet.Conn.ReadFrom returns a pre-reversed + // path suitable for replies, so use srcAddr directly. srcEpAddr := epAddr{ap: srcHostAddr} c.mu.Lock() sk := c.scionPathsByAddr[scionAddrKey{ia: srcAddr.IA, addr: srcHostAddr}] if !sk.IsSet() { - // First disco packet from this SCION peer — bootstrap a - // reverse path entry so the pong can go back over SCION. - // ReadFrom returns a pre-reversed path suitable for replies. sk = c.registerSCIONPath(&scionPathInfo{ peerIA: srcAddr.IA, hostAddr: srcHostAddr, @@ -878,8 +1015,6 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) continue } - // WireGuard packet — look up the endpoint by host addr only - // (peerMap is keyed by netip.AddrPort, not scionKey). srcEpAddr := epAddr{ap: srcHostAddr} c.mu.Lock() ep, ok := c.peerMap.endpointForEpAddr(srcEpAddr) @@ -903,6 +1038,108 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } } +// receiveSCIONBatch reads a batch of raw SCION packets from the underlay +// socket via recvmmsg, parses SCION+UDP headers with slayers, and copies +// payloads into WireGuard's buffs. Disco packets are handled inline and +// not reported to the caller. +func (c *Conn) receiveSCIONBatch(sc *scionConn, buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { + batch := scionRecvBatchPool.Get().(*scionRecvBatch) + defer putScionRecvBatch(batch) + + n := len(buffs) + if n > scionMaxBatchSize { + n = scionMaxBatchSize + } + + numMsgs, err := sc.underlayXPC.ReadBatch(batch.msgs[:n], 0) + if err != nil { + return 0, err + } + + reportToCaller := false + count := 0 + for i := 0; i < numMsgs; i++ { + msg := &batch.msgs[i] + if msg.N == 0 { + sizes[count] = 0 + continue + } + + srcIA, srcHostAddr, payload, rawPath, ok := parseSCIONPacket( + msg.Buffers[0][:msg.N], &batch.scn) + if !ok || len(payload) == 0 { + continue + } + + // Copy payload into WireGuard's buffer. + pn := copy(buffs[count], payload) + + c.lastSCIONRecv.StoreAtomic(mono.Now()) + + pt, _ := packetLooksLike(buffs[count][:pn]) + if pt == packetLooksLikeDisco { + c.handleSCIONDisco(buffs[count][:pn], srcIA, srcHostAddr, rawPath) + continue + } + + if !c.havePrivateKey.Load() { + continue + } + + srcEpAddr := epAddr{ap: srcHostAddr} + c.mu.Lock() + ep, ok := c.peerMap.endpointForEpAddr(srcEpAddr) + c.mu.Unlock() + if !ok { + sizes[count] = pn + eps[count] = &lazyEndpoint{c: c, src: srcEpAddr} + count++ + reportToCaller = true + continue + } + + now := mono.Now() + ep.lastRecvUDPAny.StoreAtomic(now) + ep.noteRecvActivity(srcEpAddr, now) + if c.metrics != nil { + c.metrics.inboundPacketsSCIONTotal.Add(1) + c.metrics.inboundBytesSCIONTotal.Add(int64(pn)) + } + sizes[count] = pn + eps[count] = ep + count++ + reportToCaller = true + } + + if reportToCaller { + return count, nil + } + return 0, nil +} + +// handleSCIONDisco handles a disco packet received on the batch path. +// It looks up or registers a SCION path entry and dispatches to handleDiscoMessage. +// For first-contact, the raw path bytes are reversed to build a reply path. +func (c *Conn) handleSCIONDisco(b []byte, srcIA addr.IA, srcHostAddr netip.AddrPort, rawPath []byte) { + srcEpAddr := epAddr{ap: srcHostAddr} + c.mu.Lock() + sk := c.scionPathsByAddr[scionAddrKey{ia: srcIA, addr: srcHostAddr}] + if !sk.IsSet() { + // First disco packet from this SCION peer — build a reply path + // by reversing the raw SCION path from the incoming packet. + replyAddr := buildSCIONReplyAddr(srcIA, srcHostAddr, rawPath) + sk = c.registerSCIONPath(&scionPathInfo{ + peerIA: srcIA, + hostAddr: srcHostAddr, + replyPath: replyAddr, + }) + c.setActiveSCIONPath(srcIA, srcHostAddr, sk) + } + c.mu.Unlock() + srcEpAddr.scionKey = sk + c.handleDiscoMessage(b, srcEpAddr, false, key.NodePublic{}, discoRXPathSCION) +} + // isTimeoutError reports whether err is a network timeout (from SetReadDeadline). func isTimeoutError(err error) bool { var netErr net.Error From 0865a92e7c0bd58758a048b23075675432ae4c06 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Thu, 12 Mar 2026 10:09:17 +0000 Subject: [PATCH 31/78] wgengine/magicsock: enhance SCION underlay support for IPv6 - Added support for IPv6 in the SCION connection handling, allowing for batch I/O operations with both IPv4 and IPv6. - Updated scionListenAddr to allow overriding the listen address via the TS_SCION_LISTEN_ADDR environment variable, supporting IPv6 localhost. - Refactored scionConn to use a common interface for underlay connections, improving flexibility for packet handling. - Enhanced documentation to clarify the behavior of the listen address and its default settings. --- wgengine/magicsock/magicsock_scion.go | 56 ++++++++++++++++++++------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index fea81a78573f1..31d9bb413234c 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -25,6 +25,7 @@ import ( "github.com/scionproto/scion/pkg/snet" wgconn "github.com/tailscale/wireguard-go/conn" "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" "tailscale.com/envknob" "tailscale.com/tailcfg" "tailscale.com/tstime/mono" @@ -102,7 +103,7 @@ func (pi *scionPathInfo) buildCachedDst() { // scionHeaderOverhead is the fixed overhead added by SCION encapsulation, // excluding the variable-length path header: -// - Underlay IPv4+UDP: 20 + 8 = 28 bytes +// - Underlay IPv4+UDP: 20 + 8 = 28 bytes (or IPv6+UDP: 40 + 8 = 48 bytes) // - SCION common header: 12 bytes // - Address header (IPv4, 2x ISD-AS + 2x IPv4): 2*8 + 2*4 = 24 bytes // - SCION/UDP L4 header: 8 bytes @@ -473,11 +474,19 @@ func buildSCIONReplyAddr(srcIA addr.IA, srcHostAddr netip.AddrPort, rawPathBytes } } +// scionBatchRW abstracts ipv4.PacketConn and ipv6.PacketConn for +// batch I/O. Both have identical ReadBatch/WriteBatch signatures +// since ipv4.Message and ipv6.Message are the same type (socket.Message). +type scionBatchRW interface { + ReadBatch([]ipv4.Message, int) (int, error) + WriteBatch([]ipv4.Message, int) (int, error) +} + // scionConn wraps a SCION connection for use by magicsock. type scionConn struct { conn *snet.Conn // from SCIONNetwork.Listen() underlayConn *net.UDPConn // raw underlay for fast-path sends (owned by conn) - underlayXPC *ipv4.PacketConn // for WriteBatch / sendmmsg + underlayXPC scionBatchRW // for WriteBatch / sendmmsg (ipv4 or ipv6) localIA addr.IA // our ISD-AS localHostIP netip.Addr // local host IP (e.g. 127.0.0.1) localPort uint16 // local SCION/UDP port @@ -549,9 +558,26 @@ func scionListenPort() uint16 { return 0 // let snet auto-select from topology port range } +// scionListenAddr returns the listen address for the SCION underlay socket. +// TS_SCION_LISTEN_ADDR can override the IP (e.g. "::1" for IPv6 localhost). +// Defaults to 127.0.0.1 (matches current behavior and snet requirement that +// the address not be unspecified). +func scionListenAddr() *net.UDPAddr { + port := scionListenPort() + if a := os.Getenv("TS_SCION_LISTEN_ADDR"); a != "" { + ip := net.ParseIP(a) + if ip != nil { + return &net.UDPAddr{IP: ip, Port: int(port)} + } + } + return &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(port)} +} + // trySCIONConnect attempts to connect to the local SCION daemon and set up a -// SCION listener. The listener binds to 127.0.0.1 (required by snet, which -// rejects unspecified addresses) on a port within the dispatched range. +// SCION listener. The listener binds to a localhost address (required by snet, +// which rejects unspecified addresses) on a port within the dispatched range. +// The listen IP defaults to 127.0.0.1 but can be overridden via +// TS_SCION_LISTEN_ADDR (e.g. "::1" for IPv6 underlay). // Returns nil if SCION is not available. func trySCIONConnect(ctx context.Context) (*scionConn, error) { daemonAddr := scionDaemonAddr() @@ -572,25 +598,21 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { Topology: conn, } - listenPort := scionListenPort() - if listenPort != 0 { + listenAddr := scionListenAddr() + if listenAddr.Port != 0 { // Validate the configured port against the daemon's dispatched range. portMin, portMax, err := conn.PortRange(ctx) if err != nil { conn.Close() return nil, fmt.Errorf("querying SCION port range: %w", err) } + listenPort := uint16(listenAddr.Port) if listenPort < portMin || listenPort > portMax { conn.Close() return nil, fmt.Errorf("TS_SCION_PORT=%d outside dispatched range [%d, %d]", listenPort, portMin, portMax) } } - listenAddr := &net.UDPAddr{ - IP: net.IPv4(127, 0, 0, 1), - Port: int(listenPort), - } - // Use OpenRaw + NewCookedConn instead of Listen so we can set socket // buffer sizes on the underlying UDP connection before wrapping it. pconn, err := network.OpenRaw(ctx, listenAddr) @@ -629,10 +651,16 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { localPort = uint16(sa.Host.Port) } - // Wrap underlay conn for sendmmsg batching. - var underlayXPC *ipv4.PacketConn + // Wrap underlay conn for sendmmsg batching, selecting the correct + // address family based on the local address. + var underlayXPC scionBatchRW if underlayConn != nil { - underlayXPC = ipv4.NewPacketConn(underlayConn) + local := underlayConn.LocalAddr().(*net.UDPAddr) + if local.IP.To4() != nil { + underlayXPC = ipv4.NewPacketConn(underlayConn) + } else { + underlayXPC = ipv6.NewPacketConn(underlayConn) + } } return &scionConn{ From be62e6dc68650c986c76c69df2d75ad204034e14 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 11 Mar 2026 20:17:05 +0000 Subject: [PATCH 32/78] tsnet: make tsnet fallback to control url from environment This commit adds a "fallback" mechanism to tsnet to allow the consumer to set "TS_CONTROL_URL" to override the control server. This allows tsnet applications to gain support for an alternative control server by just updating without explicitly exposing the ControlURL option. Updates #16934 Signed-off-by: Kristoffer Dalby --- tsnet/tsnet.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index 4a116cf3467f7..38ea865998d14 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -151,6 +151,8 @@ type Server struct { // ControlURL optionally specifies the coordination server URL. // If empty, the Tailscale default is used. + // If empty, it defaults to the TS_CONTROL_URL environment variable. + // If that is also empty, the Tailscale default is used. ControlURL string // RunWebClient, if true, runs a client for managing this node over @@ -568,6 +570,13 @@ func (s *Server) getAuthKey() string { return os.Getenv("TS_AUTH_KEY") } +func (s *Server) getControlURL() string { + if v := s.ControlURL; v != "" { + return v + } + return os.Getenv("TS_CONTROL_URL") +} + func (s *Server) getClientSecret() string { if v := s.ClientSecret; v != "" { return v @@ -769,7 +778,7 @@ func (s *Server) start() (reterr error) { prefs := ipn.NewPrefs() prefs.Hostname = s.hostname prefs.WantRunning = true - prefs.ControlURL = s.ControlURL + prefs.ControlURL = s.getControlURL() prefs.RunWebClient = s.RunWebClient prefs.AdvertiseTags = s.AdvertiseTags authKey, err := s.resolveAuthKey() @@ -858,7 +867,7 @@ func (s *Server) resolveAuthKey() (string, error) { return "", fmt.Errorf("audience for workload identity federation found, but client ID is empty") } } - authKey, err = resolveViaWIF(s.shutdownCtx, s.ControlURL, clientID, idToken, audience, s.AdvertiseTags) + authKey, err = resolveViaWIF(s.shutdownCtx, s.getControlURL(), clientID, idToken, audience, s.AdvertiseTags) if err != nil { return "", err } From 7412fc00acbd3434c57f20f685a3273e8fe75e57 Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Thu, 12 Mar 2026 10:42:41 -0400 Subject: [PATCH 33/78] flake.nix: update build to use buildGo126Module (#18977) Updates #fixup Signed-off-by: Mike O'Driscoll --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 5ac0726dab25c..3f65fb5fe5a60 100644 --- a/flake.nix +++ b/flake.nix @@ -87,7 +87,7 @@ # you're an end user you should be prepared for this flake to not # build periodically. packages = eachSystem (pkgs: rec { - default = pkgs.buildGo125Module { + default = pkgs.buildGo126Module { name = "tailscale"; pname = "tailscale"; src = ./.; From a1933d2496100222b98a5666833aa373d728c889 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Thu, 12 Mar 2026 14:53:13 +0000 Subject: [PATCH 34/78] wgengine/magicsock: implement embedded SCION connection handling - Introduced an embedded SCION connector to eliminate the need for an external SCION daemon, enhancing flexibility in SCION connection management. - Added support for a cascading fallback strategy in SCION connection setup, allowing for external daemon, embedded topology, and bootstrap server discovery. - Implemented functions for bootstrapping SCION topology and TRCs from a server, improving the robustness of the connection setup. - Enhanced error handling and logging for SCION connection attempts, providing clearer feedback on connection failures. - Refactored existing SCION connection logic to integrate with the new embedded connector, ensuring compatibility and performance improvements. --- go.mod | 6 +- go.sum | 12 ++ wgengine/magicsock/magicsock_scion.go | 96 ++++++++-- wgengine/magicsock/scion_bootstrap.go | 211 +++++++++++++++++++++ wgengine/magicsock/scion_embedded.go | 263 ++++++++++++++++++++++++++ 5 files changed, 569 insertions(+), 19 deletions(-) create mode 100644 wgengine/magicsock/scion_bootstrap.go create mode 100644 wgengine/magicsock/scion_embedded.go diff --git a/go.mod b/go.mod index 96e4c1a803353..e60a9ba7067b3 100644 --- a/go.mod +++ b/go.mod @@ -124,6 +124,7 @@ require ( golang.org/x/tools v0.39.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 golang.zx2c4.com/wireguard/windows v0.5.3 + google.golang.org/grpc v1.78.0 gopkg.in/square/go-jose.v2 v2.6.0 gvisor.dev/gvisor v0.0.0-20260224225140-573d5e7127a8 helm.sh/helm/v3 v3.19.0 @@ -191,6 +192,7 @@ require ( github.com/gokrazy/gokapi v0.0.0-20250222071133-506fdb322775 // indirect github.com/gokrazy/internal v0.0.0-20251208203110-3c1aa9087c82 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golangci/plugin-module-register v0.1.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-github/v66 v66.0.0 // indirect @@ -210,6 +212,7 @@ require ( github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect + github.com/iancoleman/strcase v0.3.0 // indirect github.com/jjti/go-spancheck v0.5.3 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect github.com/josharian/native v1.1.0 // indirect @@ -236,6 +239,7 @@ require ( github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect github.com/olekukonko/ll v0.0.8 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/patrickmn/go-cache v2.1.1-0.20180815053127-5633e0862627+incompatible // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect @@ -273,7 +277,6 @@ require ( golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect - google.golang.org/grpc v1.78.0 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect k8s.io/cli-runtime v0.34.0 // indirect k8s.io/component-base v0.34.0 // indirect @@ -287,6 +290,7 @@ require ( sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + zgo.at/zcache/v2 v2.1.0 // indirect ) require ( diff --git a/go.sum b/go.sum index b3837107c7df1..c6eb6f577b7dd 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-proxyproto v0.0.0-20210323213023-7e956b284f0a/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU= @@ -709,6 +711,8 @@ github.com/hugelgupf/vmtest v0.0.0-20240216064925-0561770280a1 h1:jWoR2Yqg8tzM0v github.com/hugelgupf/vmtest v0.0.0-20240216064925-0561770280a1/go.mod h1:B63hDJMhTupLWCHwopAyEo7wRFowx9kOc8m8j1sfOqE= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/illarion/gonotify/v3 v3.0.2 h1:O7S6vcopHexutmpObkeWsnzMJt/r1hONIEogeVNmJMk= github.com/illarion/gonotify/v3 v3.0.2/go.mod h1:HWGPdPe817GfvY3w7cx6zkbzNZfi3QjcBm/wgVvEL1U= @@ -949,6 +953,8 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.1-0.20180815053127-5633e0862627+incompatible h1:MUIwjEiAMYk8zkXXUQeb5itrXF+HpS2pfxNsA2a7AiY= +github.com/patrickmn/go-cache v2.1.1-0.20180815053127-5633e0862627+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= @@ -1029,6 +1035,8 @@ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= +github.com/quic-go/quic-go v0.43.1 h1:fLiMNfQVe9q2JvSsiXo4fXOEguXHGGl9+6gLp4RPeZQ= +github.com/quic-go/quic-go v0.43.1/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= @@ -1323,6 +1331,8 @@ go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnw go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -1864,3 +1874,5 @@ sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= +zgo.at/zcache/v2 v2.1.0 h1:USo+ubK+R4vtjw4viGzTe/zjXyPw6R7SK/RL3epBBxs= +zgo.at/zcache/v2 v2.1.0/go.mod h1:gyCeoLVo01QjDZynjime8xUGHHMbsLiPyUTBpDGd4Gk= diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 31d9bb413234c..0186a9bc02e91 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -11,6 +11,7 @@ import ( "net" "net/netip" "os" + "path/filepath" "sort" "strings" "sync" @@ -573,13 +574,66 @@ func scionListenAddr() *net.UDPAddr { return &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(port)} } -// trySCIONConnect attempts to connect to the local SCION daemon and set up a -// SCION listener. The listener binds to a localhost address (required by snet, -// which rejects unspecified addresses) on a port within the dispatched range. -// The listen IP defaults to 127.0.0.1 but can be overridden via -// TS_SCION_LISTEN_ADDR (e.g. "::1" for IPv6 underlay). -// Returns nil if SCION is not available. +// forceEmbeddedSCION is the TS_SCION_EMBEDDED envknob. When set to "1", +// the external daemon attempt is skipped and only the embedded connector is tried. +var forceEmbeddedSCION = envknob.RegisterBool("TS_SCION_EMBEDDED") + +// trySCIONConnect attempts to set up a SCION connection using a cascading +// fallback strategy: +// 1. External daemon (existing behavior, quick check) — skipped if TS_SCION_EMBEDDED=1 +// 2. Embedded with existing local topology file (TS_SCION_TOPOLOGY or /etc/scion/topology.json) +// 3. Bootstrap from configured URL (TS_SCION_BOOTSTRAP_URL / TS_SCION_BOOTSTRAP_URLS) +// 4. DNS-based discovery (SRV for _sciondiscovery._tcp) +// 5. Hardcoded bootstrap URLs (if any) +// +// Returns nil if SCION is not available via any method. func trySCIONConnect(ctx context.Context) (*scionConn, error) { + var externalErr error + + // Step 1: Try external daemon (unless forced embedded). + if !forceEmbeddedSCION() { + sc, err := tryExternalDaemon(ctx) + if err == nil { + return sc, nil + } + externalErr = err + } + + // Step 2: Try embedded with existing local topology file. + topoPath := scionTopologyPath() + if _, err := os.Stat(topoPath); err == nil { + sc, err := tryEmbeddedDaemon(ctx, topoPath) + if err == nil { + return sc, nil + } + // Fall through to bootstrap attempts. + } + + // Steps 3-5: Try bootstrap from URLs (explicit, DNS-discovered, hardcoded). + stateDir := scionStateDir() + for _, url := range bootstrapURLs(ctx) { + if err := bootstrapSCION(ctx, url, stateDir); err != nil { + continue + } + bootstrappedTopo := filepath.Join(stateDir, "topology.json") + if _, err := os.Stat(bootstrappedTopo); err != nil { + continue + } + sc, err := tryEmbeddedDaemon(ctx, bootstrappedTopo) + if err == nil { + return sc, nil + } + } + + if externalErr != nil { + return nil, fmt.Errorf("external daemon: %w; embedded: no topology available", externalErr) + } + return nil, fmt.Errorf("SCION not available: no external daemon, no topology file, no bootstrap server") +} + +// tryExternalDaemon attempts to connect to an external SCION daemon and set up +// a SCION listener. This is the original trySCIONConnect behavior. +func tryExternalDaemon(ctx context.Context) (*scionConn, error) { daemonAddr := scionDaemonAddr() svc := daemon.Service{Address: daemonAddr} conn, err := svc.Connect(ctx) @@ -587,28 +641,36 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { return nil, fmt.Errorf("connecting to SCION daemon at %s: %w", daemonAddr, err) } - localIA, err := conn.LocalIA(ctx) + sc, err := finishSCIONConnect(ctx, conn, conn) if err != nil { conn.Close() + return nil, err + } + return sc, nil +} + +// finishSCIONConnect completes the SCION connection setup given a +// daemon.Connector (for path queries) and snet.Topology (for local info). +// This is shared between the external daemon and embedded connector paths. +func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo snet.Topology) (*scionConn, error) { + localIA, err := connector.LocalIA(ctx) + if err != nil { return nil, fmt.Errorf("querying local IA: %w", err) } - // In scion v0.12.0, daemon.Connector satisfies snet.Topology. network := &snet.SCIONNetwork{ - Topology: conn, + Topology: topo, } listenAddr := scionListenAddr() if listenAddr.Port != 0 { - // Validate the configured port against the daemon's dispatched range. - portMin, portMax, err := conn.PortRange(ctx) + // Validate the configured port against the dispatched range. + portMin, portMax, err := connector.PortRange(ctx) if err != nil { - conn.Close() return nil, fmt.Errorf("querying SCION port range: %w", err) } listenPort := uint16(listenAddr.Port) if listenPort < portMin || listenPort > portMax { - conn.Close() return nil, fmt.Errorf("TS_SCION_PORT=%d outside dispatched range [%d, %d]", listenPort, portMin, portMax) } } @@ -617,7 +679,6 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { // buffer sizes on the underlying UDP connection before wrapping it. pconn, err := network.OpenRaw(ctx, listenAddr) if err != nil { - conn.Close() return nil, fmt.Errorf("listening on SCION %s: %w", listenAddr, err) } @@ -634,10 +695,9 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { } } - sconn, err := snet.NewCookedConn(pconn, conn) + sconn, err := snet.NewCookedConn(pconn, connector) if err != nil { pconn.Close() - conn.Close() return nil, fmt.Errorf("creating SCION conn: %w", err) } @@ -670,8 +730,8 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { localIA: localIA, localHostIP: localHostIP, localPort: localPort, - daemon: conn, - topo: conn, + daemon: connector, + topo: topo, }, nil } diff --git a/wgengine/magicsock/scion_bootstrap.go b/wgengine/magicsock/scion_bootstrap.go new file mode 100644 index 0000000000000..b5abd4916ccfa --- /dev/null +++ b/wgengine/magicsock/scion_bootstrap.go @@ -0,0 +1,211 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package magicsock + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "tailscale.com/net/dns/resolvconffile" +) + +const ( + // bootstrapHTTPTimeout is the timeout for HTTP requests to the bootstrap server. + bootstrapHTTPTimeout = 10 * time.Second + + // defaultBootstrapPort is the default port for SCION discovery servers. + defaultBootstrapPort = "8041" + + // scionDiscoverySRV is the SRV record name for SCION discovery. + scionDiscoverySRV = "_sciondiscovery._tcp" +) + +// defaultBootstrapURLs contains well-known bootstrap server URLs for major +// SCION deployments. Populated as deployments are identified; DNS discovery +// is the primary automatic mechanism. +var defaultBootstrapURLs []string + +// bootstrapSCION fetches topology.json and TRCs from a bootstrap server, +// saving them to destDir. +func bootstrapSCION(ctx context.Context, serverURL string, destDir string) error { + if err := os.MkdirAll(destDir, 0o700); err != nil { + return fmt.Errorf("creating bootstrap directory %s: %w", destDir, err) + } + + client := &http.Client{Timeout: bootstrapHTTPTimeout} + + // Fetch topology. + topoURL := strings.TrimRight(serverURL, "/") + "/topology" + topoData, err := httpGet(ctx, client, topoURL) + if err != nil { + return fmt.Errorf("fetching topology from %s: %w", topoURL, err) + } + topoPath := filepath.Join(destDir, "topology.json") + if err := os.WriteFile(topoPath, topoData, 0o644); err != nil { + return fmt.Errorf("writing topology to %s: %w", topoPath, err) + } + + // Fetch TRC index. + trcsURL := strings.TrimRight(serverURL, "/") + "/trcs" + trcsData, err := httpGet(ctx, client, trcsURL) + if err != nil { + // TRCs are optional for Phase 1 (accept-all verification). + return nil + } + + certsDir := filepath.Join(destDir, "certs") + if err := os.MkdirAll(certsDir, 0o700); err != nil { + return fmt.Errorf("creating certs directory %s: %w", certsDir, err) + } + + // Parse TRC index and fetch each TRC blob. + var trcIndex []trcEntry + if err := json.Unmarshal(trcsData, &trcIndex); err != nil { + // Non-fatal: TRC index may not be JSON array on all servers. + return nil + } + + for _, entry := range trcIndex { + blobURL := strings.TrimRight(serverURL, "/") + "/trcs/" + entry.ID + "/blob" + blob, err := httpGet(ctx, client, blobURL) + if err != nil { + continue // Best-effort TRC download. + } + trcPath := filepath.Join(certsDir, entry.ID+".trc") + _ = os.WriteFile(trcPath, blob, 0o644) + } + + return nil +} + +// trcEntry represents an entry in the TRC index returned by the bootstrap server. +type trcEntry struct { + ID string `json:"id"` +} + +// discoverBootstrapURL attempts DNS-based discovery of a SCION bootstrap server. +// It follows the JPAN discovery chain: +// 1. SRV lookup for _sciondiscovery._tcp. +// 2. TXT lookup for _sciondiscovery._tcp. for port override +// 3. Fallback to port 8041 +func discoverBootstrapURL(ctx context.Context) (string, error) { + // Determine local search domain from system resolver. + domain, err := localSearchDomain() + if err != nil { + return "", fmt.Errorf("determining search domain: %w", err) + } + if domain == "" { + return "", fmt.Errorf("no search domain found") + } + + r := &net.Resolver{} + + // Try SRV lookup. + _, addrs, err := r.LookupSRV(ctx, "sciondiscovery", "tcp", domain) + if err == nil && len(addrs) > 0 { + host := strings.TrimRight(addrs[0].Target, ".") + port := fmt.Sprintf("%d", addrs[0].Port) + + // Check for TXT record port override. + if txtPort, err := lookupDiscoveryPort(ctx, r, domain); err == nil && txtPort != "" { + port = txtPort + } + + return fmt.Sprintf("http://%s:%s", host, port), nil + } + + // Fallback: try the domain itself on the default port. + return fmt.Sprintf("http://%s:%s", domain, defaultBootstrapPort), nil +} + +// lookupDiscoveryPort queries TXT records for the discovery port override. +func lookupDiscoveryPort(ctx context.Context, r *net.Resolver, domain string) (string, error) { + name := scionDiscoverySRV + "." + domain + txts, err := r.LookupTXT(ctx, name) + if err != nil { + return "", err + } + for _, txt := range txts { + if strings.HasPrefix(txt, "x-sciondiscovery=") { + return strings.TrimPrefix(txt, "x-sciondiscovery="), nil + } + } + return "", fmt.Errorf("no x-sciondiscovery TXT record found") +} + +// localSearchDomain returns the first search domain from the system's DNS +// configuration, using Tailscale's cross-platform resolv.conf parser. +func localSearchDomain() (string, error) { + cfg, err := resolvconffile.ParseFile(resolvconffile.Path) + if err != nil { + return "", err + } + if len(cfg.SearchDomains) > 0 { + return cfg.SearchDomains[0].WithoutTrailingDot(), nil + } + return "", nil +} + +// httpGet performs an HTTP GET request and returns the response body. +func httpGet(ctx context.Context, client *http.Client, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d from %s", resp.StatusCode, url) + } + + // Limit response body to 10MB to prevent excessive memory usage. + body, err := io.ReadAll(io.LimitReader(resp.Body, 10<<20)) + if err != nil { + return nil, err + } + return body, nil +} + +// bootstrapURLs returns the list of bootstrap URLs to try, from explicit +// configuration, DNS discovery, and hardcoded defaults. +func bootstrapURLs(ctx context.Context) []string { + var urls []string + + // Explicit URL from environment. + if u := os.Getenv("TS_SCION_BOOTSTRAP_URL"); u != "" { + urls = append(urls, u) + } + + // Comma-separated list from environment. + if u := os.Getenv("TS_SCION_BOOTSTRAP_URLS"); u != "" { + for _, url := range strings.Split(u, ",") { + url = strings.TrimSpace(url) + if url != "" { + urls = append(urls, url) + } + } + } + + // DNS-discovered URL. + if discovered, err := discoverBootstrapURL(ctx); err == nil { + urls = append(urls, discovered) + } + + // Hardcoded defaults. + urls = append(urls, defaultBootstrapURLs...) + + return urls +} diff --git a/wgengine/magicsock/scion_embedded.go b/wgengine/magicsock/scion_embedded.go new file mode 100644 index 0000000000000..4c46fb95a39b5 --- /dev/null +++ b/wgengine/magicsock/scion_embedded.go @@ -0,0 +1,263 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package magicsock + +import ( + "context" + "fmt" + "net" + "net/netip" + "os" + "path/filepath" + + "github.com/scionproto/scion/daemon/config" + "github.com/scionproto/scion/daemon/fetcher" + "github.com/scionproto/scion/pkg/addr" + "github.com/scionproto/scion/pkg/daemon" + "github.com/scionproto/scion/pkg/drkey" + libgrpc "github.com/scionproto/scion/pkg/grpc" + "github.com/scionproto/scion/pkg/private/ctrl/path_mgmt" + "github.com/scionproto/scion/pkg/private/serrors" + cryptopb "github.com/scionproto/scion/pkg/proto/crypto" + "github.com/scionproto/scion/pkg/scrypto/cppki" + "github.com/scionproto/scion/pkg/scrypto/signed" + "github.com/scionproto/scion/pkg/snet" + segfetchergrpc "github.com/scionproto/scion/private/segment/segfetcher/grpc" + infra "github.com/scionproto/scion/private/segment/verifier" + "github.com/scionproto/scion/private/revcache" + "github.com/scionproto/scion/private/storage" + "github.com/scionproto/scion/private/topology" + "github.com/scionproto/scion/private/trust" + "google.golang.org/grpc/resolver" + "tailscale.com/paths" +) + +// embeddedConnector implements daemon.Connector and snet.Topology using an +// embedded topology loader and path fetcher, eliminating the need for an +// external SCION daemon process. +type embeddedConnector struct { + topo *topology.Loader + fetcher fetcher.Fetcher + pathDB storage.PathDB + revCache revcache.RevCache + cancel context.CancelFunc // cancels the topology loader goroutine +} + +// Compile-time interface checks. +var ( + _ daemon.Connector = (*embeddedConnector)(nil) + _ snet.Topology = (*embeddedConnector)(nil) +) + +// LocalIA returns the local ISD-AS from the topology. +func (ec *embeddedConnector) LocalIA(_ context.Context) (addr.IA, error) { + return ec.topo.IA(), nil +} + +// PortRange returns the endhost port range from the topology. +func (ec *embeddedConnector) PortRange(_ context.Context) (uint16, uint16, error) { + min, max := ec.topo.PortRange() + return min, max, nil +} + +// Interfaces returns the interface-to-underlay address map from the topology. +func (ec *embeddedConnector) Interfaces(_ context.Context) (map[uint16]netip.AddrPort, error) { + ifInfoMap := ec.topo.InterfaceInfoMap() + result := make(map[uint16]netip.AddrPort, len(ifInfoMap)) + for id, info := range ifInfoMap { + result[uint16(id)] = info.InternalAddr + } + return result, nil +} + +// Paths resolves end-to-end paths using the embedded fetcher (segment fetch + combination). +func (ec *embeddedConnector) Paths(ctx context.Context, dst, src addr.IA, f daemon.PathReqFlags) ([]snet.Path, error) { + return ec.fetcher.GetPaths(ctx, src, dst, f.Refresh) +} + +// ASInfo is not supported by the embedded connector. +func (ec *embeddedConnector) ASInfo(_ context.Context, _ addr.IA) (daemon.ASInfo, error) { + return daemon.ASInfo{}, serrors.New("not supported by embedded connector") +} + +// SVCInfo is not supported by the embedded connector. +func (ec *embeddedConnector) SVCInfo(_ context.Context, _ []addr.SVC) (map[addr.SVC][]string, error) { + return nil, serrors.New("not supported by embedded connector") +} + +// RevNotification is not supported by the embedded connector. +func (ec *embeddedConnector) RevNotification(_ context.Context, _ *path_mgmt.RevInfo) error { + return serrors.New("not supported by embedded connector") +} + +// DRKeyGetASHostKey is not supported by the embedded connector. +func (ec *embeddedConnector) DRKeyGetASHostKey(_ context.Context, _ drkey.ASHostMeta) (drkey.ASHostKey, error) { + return drkey.ASHostKey{}, serrors.New("not supported by embedded connector") +} + +// DRKeyGetHostASKey is not supported by the embedded connector. +func (ec *embeddedConnector) DRKeyGetHostASKey(_ context.Context, _ drkey.HostASMeta) (drkey.HostASKey, error) { + return drkey.HostASKey{}, serrors.New("not supported by embedded connector") +} + +// DRKeyGetHostHostKey is not supported by the embedded connector. +func (ec *embeddedConnector) DRKeyGetHostHostKey(_ context.Context, _ drkey.HostHostMeta) (drkey.HostHostKey, error) { + return drkey.HostHostKey{}, serrors.New("not supported by embedded connector") +} + +// Close shuts down the embedded connector, stopping the topology loader +// and closing storage backends. +func (ec *embeddedConnector) Close() error { + if ec.cancel != nil { + ec.cancel() + } + if ec.pathDB != nil { + ec.pathDB.Close() + } + if ec.revCache != nil { + ec.revCache.Close() + } + return nil +} + +// newEmbeddedConnector creates a new embeddedConnector from a topology file. +// It wires up the path fetcher pipeline following the daemon's own assembly +// (daemon/cmd/daemon/main.go), but without trust verification (Phase 1). +func newEmbeddedConnector(ctx context.Context, topoPath, stateDir string) (*embeddedConnector, error) { + // 1. Load topology. + topo, err := topology.NewLoader(topology.LoaderCfg{ + File: topoPath, + Validator: &topology.DefaultValidator{}, + }) + if err != nil { + return nil, fmt.Errorf("loading topology from %s: %w", topoPath, err) + } + + // Start the topology loader in a background goroutine for reload support. + topoCtx, topoCancel := context.WithCancel(ctx) + go func() { + _ = topo.Run(topoCtx) + }() + + // 2. Create storage backends. + if err := os.MkdirAll(stateDir, 0o700); err != nil { + topoCancel() + return nil, fmt.Errorf("creating state directory %s: %w", stateDir, err) + } + + dbPath := filepath.Join(stateDir, "scion-pathdb.sqlite") + pathDB, err := storage.NewPathStorage(storage.DBConfig{Connection: dbPath}) + if err != nil { + topoCancel() + return nil, fmt.Errorf("creating path storage at %s: %w", dbPath, err) + } + + revCache := storage.NewRevocationStorage() + + // 3. Create gRPC dialer that resolves CS addresses from the topology. + dialer := &libgrpc.TCPDialer{ + SvcResolver: func(dst addr.SVC) []resolver.Address { + targets := []resolver.Address{} + for _, entry := range topo.ControlServiceAddresses() { + targets = append(targets, resolver.Address{Addr: entry.String()}) + } + return targets + }, + } + + // 4. Create the segment fetcher requester (gRPC to local CS). + requester := &segfetchergrpc.Requester{ + Dialer: dialer, + } + + // 5. Create the path fetcher with accept-all verification (Phase 1). + sdCfg := config.SDConfig{} + sdCfg.InitDefaults() + + f := fetcher.NewFetcher(fetcher.FetcherConfig{ + IA: topo.IA(), + MTU: topo.MTU(), + Core: topo.Core(), + NextHopper: topo, + RPC: requester, + PathDB: pathDB, + Inspector: endHostInspector{}, + Verifier: acceptAllVerifier{}, + RevCache: revCache, + Cfg: sdCfg, + }) + + return &embeddedConnector{ + topo: topo, + fetcher: f, + pathDB: pathDB, + revCache: revCache, + cancel: topoCancel, + }, nil +} + +// acceptAllVerifier skips segment verification. This matches the daemon's own +// behavior when DisableSegVerification is set (daemon/cmd/daemon/main.go:359-377). +type acceptAllVerifier struct{} + +func (acceptAllVerifier) Verify(_ context.Context, _ *cryptopb.SignedMessage, + _ ...[]byte) (*signed.Message, error) { + return nil, nil +} + +func (v acceptAllVerifier) WithServer(_ net.Addr) infra.Verifier { + return v +} + +func (v acceptAllVerifier) WithIA(_ addr.IA) infra.Verifier { + return v +} + +func (v acceptAllVerifier) WithValidity(_ cppki.Validity) infra.Verifier { + return v +} + +// endHostInspector is a minimal trust.Inspector for non-core endhosts. +// It always reports no attributes, which is correct for endhost path resolution. +type endHostInspector struct{} + +func (endHostInspector) ByAttributes(_ context.Context, _ addr.ISD, _ trust.Attribute) ([]addr.IA, error) { + return nil, nil +} + +func (endHostInspector) HasAttributes(_ context.Context, _ addr.IA, _ trust.Attribute) (bool, error) { + return false, nil +} + +// scionTopologyPath returns the path to the SCION topology file, checking +// TS_SCION_TOPOLOGY first, then falling back to /etc/scion/topology.json. +func scionTopologyPath() string { + if p := os.Getenv("TS_SCION_TOPOLOGY"); p != "" { + return p + } + return "/etc/scion/topology.json" +} + +// scionStateDir returns the directory for SCION state (PathDB, etc.), +// checking TS_SCION_STATE_DIR first, then falling back to a "scion" +// subdirectory under the platform's default tailscaled state directory. +func scionStateDir() string { + if d := os.Getenv("TS_SCION_STATE_DIR"); d != "" { + return d + } + return filepath.Join(paths.DefaultTailscaledStateDir(), "scion") +} + +// tryEmbeddedDaemon attempts to set up a SCION connection using the embedded +// connector with the given topology file. This mirrors trySCIONConnect but +// uses the embedded connector instead of an external daemon. +func tryEmbeddedDaemon(ctx context.Context, topoPath string) (*scionConn, error) { + stateDir := scionStateDir() + ec, err := newEmbeddedConnector(ctx, topoPath, stateDir) + if err != nil { + return nil, fmt.Errorf("creating embedded connector: %w", err) + } + + return finishSCIONConnect(ctx, ec, ec) +} From 632e14fb39587b5393d3e18ba7d5d6a4a6bf2f88 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Thu, 12 Mar 2026 15:38:27 +0000 Subject: [PATCH 35/78] wgengine/magicsock: enhance SCION topology path resolution - Updated scionTopologyPath function to implement a cascading fallback strategy for locating the SCION topology file. - Added checks for the standard SCION installation path and a bootstrapped topology under the tailscaled state directory. --- wgengine/magicsock/scion_embedded.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/wgengine/magicsock/scion_embedded.go b/wgengine/magicsock/scion_embedded.go index 4c46fb95a39b5..7018c23edcebf 100644 --- a/wgengine/magicsock/scion_embedded.go +++ b/wgengine/magicsock/scion_embedded.go @@ -231,12 +231,20 @@ func (endHostInspector) HasAttributes(_ context.Context, _ addr.IA, _ trust.Attr } // scionTopologyPath returns the path to the SCION topology file, checking -// TS_SCION_TOPOLOGY first, then falling back to /etc/scion/topology.json. +// TS_SCION_TOPOLOGY first, then the platform's SCION config directory +// (/etc/scion/ on Linux), then a "scion" subdirectory under the tailscaled +// state directory (for bootstrapped topologies). func scionTopologyPath() string { if p := os.Getenv("TS_SCION_TOPOLOGY"); p != "" { return p } - return "/etc/scion/topology.json" + // Standard SCION installation path (Linux/Unix convention from scionproto). + const defaultSCIONTopology = "/etc/scion/topology.json" + if _, err := os.Stat(defaultSCIONTopology); err == nil { + return defaultSCIONTopology + } + // Bootstrapped topology under the tailscaled state directory. + return filepath.Join(paths.DefaultTailscaledStateDir(), "scion", "topology.json") } // scionStateDir returns the directory for SCION state (PathDB, etc.), From dd480f0fb931bb971f01336c820f960aa858171f Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 10 Mar 2026 22:52:49 +0000 Subject: [PATCH 36/78] gokrazy: fix busybox breakglass support, add test Updates #1866 Change-Id: Ica73ae8268b08a04ae97bc570869a04180585e75 Signed-off-by: Brad Fitzpatrick --- flake.nix | 2 +- go.mod | 2 +- go.mod.sri | 2 +- go.sum | 4 +- gokrazy/gokrazy_test.go | 286 ++++++++++++++++++++++++++++++++++++++++ shell.nix | 2 +- 6 files changed, 292 insertions(+), 6 deletions(-) create mode 100644 gokrazy/gokrazy_test.go diff --git a/flake.nix b/flake.nix index 3f65fb5fe5a60..e32cf3866a28e 100644 --- a/flake.nix +++ b/flake.nix @@ -151,4 +151,4 @@ }); }; } -# nix-direnv cache busting line: sha256-dx+SJyDx+eZptFaMatoyM6w1E3nJKY+hKs7nuR997bE= +# nix-direnv cache busting line: sha256-V4vJ1MonIWbuL+R5fUiO7hV7f+k/Iqoz+EFWnOJwZAs= diff --git a/go.mod b/go.mod index 533ef04489cc6..d2c7d6dae0438 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssm v1.44.7 github.com/axiomhq/hyperloglog v0.0.0-20240319100328-84253e514e02 github.com/bradfitz/go-tool-cache v0.0.0-20260216153636-9e5201344fe5 - github.com/bradfitz/monogok v0.0.0-20260208031948-2219c393d032 + github.com/bradfitz/monogok v0.0.0-20260310223834-65a3d9465088 github.com/bramvdbogaerde/go-scp v1.4.0 github.com/cilium/ebpf v0.16.0 github.com/coder/websocket v1.8.12 diff --git a/go.mod.sri b/go.mod.sri index 0e0a6fdece5ee..ab47b01f02f07 100644 --- a/go.mod.sri +++ b/go.mod.sri @@ -1 +1 @@ -sha256-dx+SJyDx+eZptFaMatoyM6w1E3nJKY+hKs7nuR997bE= +sha256-V4vJ1MonIWbuL+R5fUiO7hV7f+k/Iqoz+EFWnOJwZAs= diff --git a/go.sum b/go.sum index 48b1e9379006f..3bd1d887c2551 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,8 @@ github.com/bombsimon/wsl/v4 v4.2.1 h1:Cxg6u+XDWff75SIFFmNsqnIOgob+Q9hG6y/ioKbRFi github.com/bombsimon/wsl/v4 v4.2.1/go.mod h1:Xu/kDxGZTofQcDGCtQe9KCzhHphIe0fDuyWTxER9Feo= github.com/bradfitz/go-tool-cache v0.0.0-20260216153636-9e5201344fe5 h1:0sG3c7afYdBNlc3QyhckvZ4bV9iqlfqCQM1i+mWm0eE= github.com/bradfitz/go-tool-cache v0.0.0-20260216153636-9e5201344fe5/go.mod h1:78ZLITnBUCDJeU01+wYYJKaPYYgsDzJPRfxeI8qFh5g= -github.com/bradfitz/monogok v0.0.0-20260208031948-2219c393d032 h1:xDomVqO85ss/98Ky5zxM/g86bXDNBLebM2I9G/fu6uA= -github.com/bradfitz/monogok v0.0.0-20260208031948-2219c393d032/go.mod h1:TG1HbU9fRVDnNgXncVkKz9GdvjIvqquXjH6QZSEVmY4= +github.com/bradfitz/monogok v0.0.0-20260310223834-65a3d9465088 h1:dDVY5cJ+7bQQll29aeWGx1Ima4RIGy/f1fXVs+HlIxo= +github.com/bradfitz/monogok v0.0.0-20260310223834-65a3d9465088/go.mod h1:TG1HbU9fRVDnNgXncVkKz9GdvjIvqquXjH6QZSEVmY4= github.com/bramvdbogaerde/go-scp v1.4.0 h1:jKMwpwCbcX1KyvDbm/PDJuXcMuNVlLGi0Q0reuzjyKY= github.com/bramvdbogaerde/go-scp v1.4.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ= github.com/breml/bidichk v0.2.7 h1:dAkKQPLl/Qrk7hnP6P+E0xOodrq8Us7+U0o4UBOAlQY= diff --git a/gokrazy/gokrazy_test.go b/gokrazy/gokrazy_test.go new file mode 100644 index 0000000000000..76398d49bf594 --- /dev/null +++ b/gokrazy/gokrazy_test.go @@ -0,0 +1,286 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "hash/fnv" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/mod/modfile" +) + +var runVMTests = flag.Bool("run-vm-tests", false, "run tests that require a VM") + +func findKernelPath(t *testing.T) string { + t.Helper() + goModPath := filepath.Join("..", "go.mod") + b, err := os.ReadFile(goModPath) + if err != nil { + t.Fatalf("reading go.mod: %v", err) + } + mf, err := modfile.Parse("go.mod", b, nil) + if err != nil { + t.Fatalf("parsing go.mod: %v", err) + } + goModB, err := exec.Command("go", "env", "GOMODCACHE").CombinedOutput() + if err != nil { + t.Fatalf("go env GOMODCACHE: %v", err) + } + for _, r := range mf.Require { + if r.Mod.Path == "github.com/tailscale/gokrazy-kernel" { + return strings.TrimSpace(string(goModB)) + "/" + r.Mod.String() + "/vmlinuz" + } + } + t.Fatal("failed to find gokrazy-kernel in go.mod") + return "" +} + +// gptPartuuid returns the GPT PARTUUID for a gokrazy appliance partition, +// matching the scheme used by monogok: fnv32a(hostname) formatted into +// the gokrazy GUID prefix. +func gptPartuuid(hostname string, partition uint16) string { + h := fnv.New32a() + h.Write([]byte(hostname)) + return fmt.Sprintf("60c24cc1-f3f9-427a-8199-%08x00%02x", h.Sum32(), partition) +} + +func buildTsappImage(t *testing.T) string { + t.Helper() + imgPath, err := filepath.Abs("tsapp.img") + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(imgPath); err == nil { + t.Logf("using existing tsapp.img: %s", imgPath) + return imgPath + } + + t.Logf("building tsapp.img...") + cmd := exec.Command("make", "image") + cmd.Dir, _ = os.Getwd() + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + t.Fatalf("make image: %v", err) + } + if _, err := os.Stat(imgPath); err != nil { + t.Fatalf("tsapp.img not found after build: %v", err) + } + return imgPath +} + +// serialLog collects serial console output in a thread-safe manner. +type serialLog struct { + mu sync.Mutex + lines []string +} + +func (sl *serialLog) add(line string) { + sl.mu.Lock() + defer sl.mu.Unlock() + sl.lines = append(sl.lines, line) +} + +func (sl *serialLog) lastN(n int) []string { + sl.mu.Lock() + defer sl.mu.Unlock() + if len(sl.lines) <= n { + cp := make([]string, len(sl.lines)) + copy(cp, sl.lines) + return cp + } + cp := make([]string, n) + copy(cp, sl.lines[len(sl.lines)-n:]) + return cp +} + +func (sl *serialLog) findLine(pred func(string) bool) bool { + sl.mu.Lock() + defer sl.mu.Unlock() + for _, line := range sl.lines { + if pred(line) { + return true + } + } + return false +} + +// TestBusyboxInTsapp boots the tsapp image in QEMU and verifies that +// busybox is accessible via the serial console shell. This validates +// that the serial-busybox package's extra files (the busybox binary) +// are properly included in the image by monogok. +func TestBusyboxInTsapp(t *testing.T) { + if !*runVMTests { + t.Skip("skipping VM test; set --run-vm-tests to run") + } + + kernel := findKernelPath(t) + if _, err := os.Stat(kernel); err != nil { + t.Skipf("kernel not found at %s: %v", kernel, err) + } + t.Logf("kernel: %s", kernel) + + // Read the hostname from config.json to compute the GPT PARTUUID. + cfgBytes, err := os.ReadFile("tsapp/config.json") + if err != nil { + t.Fatalf("reading tsapp/config.json: %v", err) + } + var cfg struct { + Hostname string + } + if err := json.Unmarshal(cfgBytes, &cfg); err != nil { + t.Fatalf("parsing config.json: %v", err) + } + rootParam := fmt.Sprintf("root=PARTUUID=%s/PARTNROFF=1", gptPartuuid(cfg.Hostname, 1)) + t.Logf("root param: %s", rootParam) + + imgPath := buildTsappImage(t) + + // Create a temporary qcow2 overlay so we don't modify the original image. + tmpDir := t.TempDir() + disk := filepath.Join(tmpDir, "tsapp-test.qcow2") + out, err := exec.Command("qemu-img", "create", + "-f", "qcow2", + "-F", "raw", + "-b", imgPath, + disk).CombinedOutput() + if err != nil { + t.Fatalf("qemu-img create: %v, %s", err, out) + } + + // Set up a Unix socket for the serial console. + sockPath := filepath.Join(tmpDir, "serial.sock") + ln, err := net.Listen("unix", sockPath) + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + + // Boot QEMU with microvm, explicit kernel, and serial via virtconsole + // connected to our Unix socket. The kernel sees hvc0 as the console + // device, and gokrazy uses it for the serial shell. + cmd := exec.Command("qemu-system-x86_64", + "-M", "microvm,isa-serial=off", + "-m", "1G", + "-nodefaults", "-no-user-config", "-nographic", + "-kernel", kernel, + "-append", "console=hvc0 "+rootParam+" ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet", + "-drive", "id=blk0,file="+disk+",format=qcow2", + "-device", "virtio-blk-device,drive=blk0", + "-device", "virtio-rng-device", + "-device", "virtio-serial-device", + "-chardev", "socket,id=virtiocon0,path="+sockPath+",server=off", + "-device", "virtconsole,chardev=virtiocon0", + "-netdev", "user,id=net0", + "-device", "virtio-net-device,netdev=net0", + ) + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatalf("qemu start: %v", err) + } + t.Cleanup(func() { + cmd.Process.Kill() + cmd.Wait() + }) + + // Accept the serial console connection from QEMU. + ln.(*net.UnixListener).SetDeadline(time.Now().Add(30 * time.Second)) + conn, err := ln.Accept() + if err != nil { + t.Fatalf("accept serial connection: %v", err) + } + defer conn.Close() + + // Read serial output in a goroutine. + slog := &serialLog{} + bootDone := make(chan struct{}) + go func() { + buf := make([]byte, 4096) + var partial string + for { + n, err := conn.Read(buf) + if n > 0 { + partial += string(buf[:n]) + for { + idx := strings.IndexByte(partial, '\n') + if idx < 0 { + break + } + line := strings.TrimRight(partial[:idx], "\r") + partial = partial[idx+1:] + slog.add(line) + t.Logf("serial: %s", line) + // gokrazy logs socket listener info when boot is done. + if strings.Contains(line, "listening on") { + select { + case <-bootDone: + default: + close(bootDone) + } + } + } + } + if err != nil { + if err != io.EOF { + t.Logf("serial read error: %v", err) + } + return + } + } + }() + + // Wait for boot to complete (up to 120 seconds). + select { + case <-bootDone: + t.Logf("boot complete") + case <-time.After(120 * time.Second): + t.Fatalf("timeout waiting for boot; last lines:\n%s", + strings.Join(slog.lastN(20), "\n")) + } + + // Small delay to let services fully initialize. + time.Sleep(2 * time.Second) + + // Send a newline to trigger the serial shell. + // gokrazy's init reads stdin and calls tryStartShell() on any input. + fmt.Fprintf(conn, "\n") + time.Sleep(2 * time.Second) + + // Send a command to test busybox. The echo command is a busybox builtin, + // so if busybox is working, we'll see our marker in the output. + marker := "BUSYBOX_TEST_OK_12345" + fmt.Fprintf(conn, "echo %s\n", marker) + + // Wait for our marker in the output (not on the echo command line itself). + deadline := time.After(15 * time.Second) + for { + select { + case <-deadline: + t.Fatalf("timeout waiting for busybox echo response; busybox binary is likely missing from the image.\n"+ + "This indicates monogok is not copying _gokrazy/extrafiles from serial-busybox.\n"+ + "Last serial lines:\n%s", + strings.Join(slog.lastN(30), "\n")) + default: + } + time.Sleep(200 * time.Millisecond) + // Look for the marker on a line by itself (the echo output, not the command). + if slog.findLine(func(line string) bool { + return strings.TrimSpace(line) == marker + }) { + t.Logf("busybox shell is working: got echo response") + return // success + } + } +} diff --git a/shell.nix b/shell.nix index 7e965bb11082a..17ad795876a58 100644 --- a/shell.nix +++ b/shell.nix @@ -16,4 +16,4 @@ ) { src = ./.; }).shellNix -# nix-direnv cache busting line: sha256-dx+SJyDx+eZptFaMatoyM6w1E3nJKY+hKs7nuR997bE= +# nix-direnv cache busting line: sha256-V4vJ1MonIWbuL+R5fUiO7hV7f+k/Iqoz+EFWnOJwZAs= From 0e99825c079e5a2e73d01c5266fb765ed9fb138a Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Fri, 13 Mar 2026 07:35:26 +0000 Subject: [PATCH 37/78] wgengine/magicsock: implement local IP resolution for SCION underlay - Added scionResolveLocalIP function to determine the local IP for SCION underlay sockets based on the topology's border routers. - Updated scionListenAddr to utilize the new local IP resolution, enhancing flexibility in address handling. - Improved error handling by logging warnings when multiple border routers resolve to different local IPs, guiding users to set TS_SCION_LISTEN_ADDR explicitly if needed. --- wgengine/magicsock/magicsock_scion.go | 55 ++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 0186a9bc02e91..dbdc8d910d54a 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -24,6 +24,7 @@ import ( scionpath "github.com/scionproto/scion/pkg/slayers/path/scion" snetpath "github.com/scionproto/scion/pkg/snet/path" "github.com/scionproto/scion/pkg/snet" + "github.com/scionproto/scion/pkg/snet/addrutil" wgconn "github.com/tailscale/wireguard-go/conn" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" @@ -559,11 +560,54 @@ func scionListenPort() uint16 { return 0 // let snet auto-select from topology port range } +// scionResolveLocalIP determines the local IP for the SCION underlay socket +// by checking what source IP the OS would use to reach the border routers' +// internal addresses from the topology. This mirrors how `scion address` works +// (via addrutil.ResolveLocal). +// +// With multiple BRs, if all resolve to the same local IP, that IP is used. +// If they disagree, the first resolved IP is used and a warning is logged — +// the user should set TS_SCION_LISTEN_ADDR explicitly. +// +// Falls back to 127.0.0.1 if no interfaces or resolution fails. +func scionResolveLocalIP(ctx context.Context, topo snet.Topology) netip.Addr { + ifMap, err := topo.Interfaces(ctx) + if err != nil || len(ifMap) == 0 { + return netip.AddrFrom4([4]byte{127, 0, 0, 1}) + } + + var first netip.Addr + allSame := true + for _, ap := range ifMap { + resolved, err := addrutil.ResolveLocal(ap.Addr().AsSlice()) + if err != nil { + continue + } + ip, ok := netip.AddrFromSlice(resolved) + if !ok { + continue + } + ip = ip.Unmap() + if !first.IsValid() { + first = ip + } else if first != ip { + allSame = false + } + } + + if !first.IsValid() { + return netip.AddrFrom4([4]byte{127, 0, 0, 1}) + } + if !allSame { + fmt.Fprintf(os.Stderr, "magicsock: SCION: multiple BRs resolve to different local IPs; using %s, set TS_SCION_LISTEN_ADDR to override\n", first) + } + return first +} + // scionListenAddr returns the listen address for the SCION underlay socket. // TS_SCION_LISTEN_ADDR can override the IP (e.g. "::1" for IPv6 localhost). -// Defaults to 127.0.0.1 (matches current behavior and snet requirement that -// the address not be unspecified). -func scionListenAddr() *net.UDPAddr { +// Otherwise resolves the local IP from the topology's BR internal addresses. +func scionListenAddr(ctx context.Context, topo snet.Topology) *net.UDPAddr { port := scionListenPort() if a := os.Getenv("TS_SCION_LISTEN_ADDR"); a != "" { ip := net.ParseIP(a) @@ -571,7 +615,8 @@ func scionListenAddr() *net.UDPAddr { return &net.UDPAddr{IP: ip, Port: int(port)} } } - return &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(port)} + ip := scionResolveLocalIP(ctx, topo) + return &net.UDPAddr{IP: ip.AsSlice(), Port: int(port)} } // forceEmbeddedSCION is the TS_SCION_EMBEDDED envknob. When set to "1", @@ -662,7 +707,7 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn Topology: topo, } - listenAddr := scionListenAddr() + listenAddr := scionListenAddr(ctx, topo) if listenAddr.Port != 0 { // Validate the configured port against the dispatched range. portMin, portMax, err := connector.PortRange(ctx) From 904088d6c321dbfd399303eb2af3ba2ade15060e Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Fri, 13 Mar 2026 08:11:47 +0000 Subject: [PATCH 38/78] wgengine/magicsock: update SCION dependencies and enhance topology handling - Upgraded the SCION library from v0.12.0 to v0.14.0 to incorporate the latest features and fixes. - Refactored functions to utilize the new daemon.Connector interface for improved topology management and error handling. --- go.mod | 8 +-- go.sum | 18 ++--- wgengine/magicsock/magicsock_scion.go | 98 ++++++++++++++++++++++++--- wgengine/magicsock/scion_embedded.go | 36 +++++++--- 4 files changed, 128 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index e60a9ba7067b3..18dc421ea7bc6 100644 --- a/go.mod +++ b/go.mod @@ -87,7 +87,7 @@ require ( github.com/prometheus/common v0.65.0 github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff github.com/safchain/ethtool v0.3.0 - github.com/scionproto/scion v0.12.0 + github.com/scionproto/scion v0.14.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/studio-b12/gowebdav v0.9.0 github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e @@ -154,6 +154,7 @@ require ( github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect github.com/alecthomas/go-check-sumtype v0.1.4 // indirect github.com/alexkohler/nakedret/v2 v2.0.4 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beevik/ntp v0.3.0 // indirect @@ -192,13 +193,13 @@ require ( github.com/gokrazy/gokapi v0.0.0-20250222071133-506fdb322775 // indirect github.com/gokrazy/internal v0.0.0-20251208203110-3c1aa9087c82 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/golangci/plugin-module-register v0.1.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-github/v66 v66.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/renameio/v2 v2.0.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/gopacket/gopacket v1.5.0 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosuri/uitable v0.0.4 // indirect @@ -225,7 +226,6 @@ require ( github.com/lib/pq v1.10.9 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/macabu/inamedparam v0.1.3 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mdlayher/packet v1.1.2 // indirect github.com/mdlayher/watchdog v0.0.0-20221003142519-49be0df7b3b5 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect @@ -284,7 +284,7 @@ require ( modernc.org/libc v1.66.3 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.38.2 // indirect + modernc.org/sqlite v1.39.0 // indirect oras.land/oras-go/v2 v2.6.0 // indirect sigs.k8s.io/kustomize/api v0.20.1 // indirect sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect diff --git a/go.sum b/go.sum index c6eb6f577b7dd..33031cd0ab0bc 100644 --- a/go.sum +++ b/go.sum @@ -617,6 +617,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopacket/gopacket v1.5.0 h1:9s9fcSUVKFlRV97B77Bq9XNV3ly2gvvsneFMQUGjc+M= +github.com/gopacket/gopacket v1.5.0/go.mod h1:i3NaGaqfoWKAr1+g7qxEdWsmfT+MXuWkAe9+THv8LME= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gordonklaus/ineffassign v0.1.0 h1:y2Gd/9I7MdY1oEIt+n+rowjBNDcLQq3RsH5hwJd0f9s= @@ -1035,8 +1037,8 @@ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= -github.com/quic-go/quic-go v0.43.1 h1:fLiMNfQVe9q2JvSsiXo4fXOEguXHGGl9+6gLp4RPeZQ= -github.com/quic-go/quic-go v0.43.1/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= +github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= +github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= @@ -1077,8 +1079,8 @@ github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tM github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= github.com/sashamelentyev/usestdlibvars v1.25.0 h1:IK8SI2QyFzy/2OD2PYnhy84dpfNo9qADrRt6LH8vSzU= github.com/sashamelentyev/usestdlibvars v1.25.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7aRoS4j6EBye3YBhmAIRF8= -github.com/scionproto/scion v0.12.0 h1:NbBa1HAxWOXr40C8YuanGhJ3g5hYlJetR5YevKtnHGQ= -github.com/scionproto/scion v0.12.0/go.mod h1:jOmbOiLREf4zn6cNrFqto35rP3eH6RhDJEmrjmJIUUI= +github.com/scionproto/scion v0.14.0 h1:aoSM4f/klmhO/RsXG2RJ7KbaNZ6cujxe9APfqFby0Lw= +github.com/scionproto/scion v0.14.0/go.mod h1:gCXIVztXV7HMe9P/ymVk4U4oSZOYaNnhkeskYxl2h60= github.com/securego/gosec/v2 v2.19.0 h1:gl5xMkOI0/E6Hxx0XCY2XujA3V7SNSefA8sC+3f1gnk= github.com/securego/gosec/v2 v2.19.0/go.mod h1:hOkDcHz9J/XIgIlPDXalxjeVYsHxoWUc5zJSHxcB8YM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1331,8 +1333,8 @@ go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnw go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= -go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -1839,8 +1841,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek= -modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY= +modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index dbdc8d910d54a..9ddec0fd86638 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -6,6 +6,7 @@ package magicsock import ( "context" "encoding/binary" + "encoding/json" "errors" "fmt" "net" @@ -17,7 +18,7 @@ import ( "sync" "time" - "github.com/google/gopacket" + "github.com/gopacket/gopacket" "github.com/scionproto/scion/pkg/addr" "github.com/scionproto/scion/pkg/daemon" "github.com/scionproto/scion/pkg/slayers" @@ -570,8 +571,8 @@ func scionListenPort() uint16 { // the user should set TS_SCION_LISTEN_ADDR explicitly. // // Falls back to 127.0.0.1 if no interfaces or resolution fails. -func scionResolveLocalIP(ctx context.Context, topo snet.Topology) netip.Addr { - ifMap, err := topo.Interfaces(ctx) +func scionResolveLocalIP(ctx context.Context, connector daemon.Connector) netip.Addr { + ifMap, err := connector.Interfaces(ctx) if err != nil || len(ifMap) == 0 { return netip.AddrFrom4([4]byte{127, 0, 0, 1}) } @@ -607,7 +608,7 @@ func scionResolveLocalIP(ctx context.Context, topo snet.Topology) netip.Addr { // scionListenAddr returns the listen address for the SCION underlay socket. // TS_SCION_LISTEN_ADDR can override the IP (e.g. "::1" for IPv6 localhost). // Otherwise resolves the local IP from the topology's BR internal addresses. -func scionListenAddr(ctx context.Context, topo snet.Topology) *net.UDPAddr { +func scionListenAddr(ctx context.Context, connector daemon.Connector) *net.UDPAddr { port := scionListenPort() if a := os.Getenv("TS_SCION_LISTEN_ADDR"); a != "" { ip := net.ParseIP(a) @@ -615,7 +616,7 @@ func scionListenAddr(ctx context.Context, topo snet.Topology) *net.UDPAddr { return &net.UDPAddr{IP: ip, Port: int(port)} } } - ip := scionResolveLocalIP(ctx, topo) + ip := scionResolveLocalIP(ctx, connector) return &net.UDPAddr{IP: ip.AsSlice(), Port: int(port)} } @@ -686,7 +687,27 @@ func tryExternalDaemon(ctx context.Context) (*scionConn, error) { return nil, fmt.Errorf("connecting to SCION daemon at %s: %w", daemonAddr, err) } - sc, err := finishSCIONConnect(ctx, conn, conn) + topo, err := snetTopologyFromConnector(ctx, conn) + if err != nil { + conn.Close() + return nil, fmt.Errorf("building topology from daemon: %w", err) + } + + // Probe Paths() to detect wire-format incompatibility with older + // daemons (e.g. v0.12 daemon vs v0.14 client). Simple RPCs like + // LocalIA/Interfaces/ASInfo use compatible proto types, but Paths + // responses with real hop data trigger unmarshal failures. + // We need a reachable remote IA to get a non-empty response; + // parse the topology file for a neighbor AS. + if neighborIA, ok := neighborIAFromTopology(scionTopologyPath()); ok { + localIA, _ := conn.LocalIA(ctx) + if _, err := conn.Paths(ctx, neighborIA, localIA, daemon.PathReqFlags{}); err != nil { + conn.Close() + return nil, fmt.Errorf("daemon path probe failed (version mismatch?): %w", err) + } + } + + sc, err := finishSCIONConnect(ctx, conn, topo) if err != nil { conn.Close() return nil, err @@ -694,6 +715,61 @@ func tryExternalDaemon(ctx context.Context) (*scionConn, error) { return sc, nil } +// snetTopologyFromConnector builds an snet.Topology struct by querying +// a daemon.Connector for local topology information. +func snetTopologyFromConnector(ctx context.Context, conn daemon.Connector) (snet.Topology, error) { + localIA, err := conn.LocalIA(ctx) + if err != nil { + return snet.Topology{}, fmt.Errorf("querying local IA: %w", err) + } + portStart, portEnd, err := conn.PortRange(ctx) + if err != nil { + return snet.Topology{}, fmt.Errorf("querying port range: %w", err) + } + ifMap, err := conn.Interfaces(ctx) + if err != nil { + return snet.Topology{}, fmt.Errorf("querying interfaces: %w", err) + } + return snet.Topology{ + LocalIA: localIA, + PortRange: snet.TopologyPortRange{Start: portStart, End: portEnd}, + Interface: func(id uint16) (netip.AddrPort, bool) { + ap, ok := ifMap[id] + return ap, ok + }, + }, nil +} + +// neighborIAFromTopology parses the SCION topology JSON file and returns +// the IA of the first neighbor AS found in the border router interfaces. +// This is used to probe the daemon with a Paths() call that returns real +// path data, detecting proto wire-format incompatibilities. +func neighborIAFromTopology(topoPath string) (addr.IA, bool) { + data, err := os.ReadFile(topoPath) + if err != nil { + return 0, false + } + var topo struct { + BorderRouters map[string]struct { + Interfaces map[string]struct { + ISDAS string `json:"isd_as"` + } `json:"interfaces"` + } `json:"border_routers"` + } + if err := json.Unmarshal(data, &topo); err != nil { + return 0, false + } + for _, br := range topo.BorderRouters { + for _, iface := range br.Interfaces { + ia, err := addr.ParseIA(iface.ISDAS) + if err == nil { + return ia, true + } + } + } + return 0, false +} + // finishSCIONConnect completes the SCION connection setup given a // daemon.Connector (for path queries) and snet.Topology (for local info). // This is shared between the external daemon and embedded connector paths. @@ -707,7 +783,7 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn Topology: topo, } - listenAddr := scionListenAddr(ctx, topo) + listenAddr := scionListenAddr(ctx, connector) if listenAddr.Port != 0 { // Validate the configured port against the dispatched range. portMin, portMax, err := connector.PortRange(ctx) @@ -740,7 +816,7 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn } } - sconn, err := snet.NewCookedConn(pconn, connector) + sconn, err := snet.NewCookedConn(pconn, topo) if err != nil { pconn.Close() return nil, fmt.Errorf("creating SCION conn: %w", err) @@ -1406,7 +1482,7 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr seen := make(map[snet.PathFingerprint]bool) var unique []pathWithMeta for _, p := range paths { - fp := snet.Fingerprint(p) + fp := p.Metadata().Fingerprint() if fp != "" && seen[fp] { continue } @@ -1592,7 +1668,7 @@ func (c *Conn) refreshSCIONPathsOnce() error { for _, dp := range daemonPaths { daemonByFP = append(daemonByFP, daemonPathEntry{ path: dp, - fp: snet.Fingerprint(dp), + fp: dp.Metadata().Fingerprint(), }) } @@ -1637,7 +1713,7 @@ func (c *Conn) refreshSCIONPathsOnce() error { pi.mu.Lock() pi.path = matched - newFP := snet.Fingerprint(matched) + newFP := matched.Metadata().Fingerprint() pi.fingerprint = newFP if md := matched.Metadata(); md != nil { pi.expiry = md.Expiry diff --git a/wgengine/magicsock/scion_embedded.go b/wgengine/magicsock/scion_embedded.go index 7018c23edcebf..e9491c9fe27ea 100644 --- a/wgengine/magicsock/scion_embedded.go +++ b/wgengine/magicsock/scion_embedded.go @@ -22,6 +22,7 @@ import ( cryptopb "github.com/scionproto/scion/pkg/proto/crypto" "github.com/scionproto/scion/pkg/scrypto/cppki" "github.com/scionproto/scion/pkg/scrypto/signed" + "github.com/scionproto/scion/pkg/segment/iface" "github.com/scionproto/scion/pkg/snet" segfetchergrpc "github.com/scionproto/scion/private/segment/segfetcher/grpc" infra "github.com/scionproto/scion/private/segment/verifier" @@ -33,9 +34,9 @@ import ( "tailscale.com/paths" ) -// embeddedConnector implements daemon.Connector and snet.Topology using an -// embedded topology loader and path fetcher, eliminating the need for an -// external SCION daemon process. +// embeddedConnector implements daemon.Connector using an embedded topology +// loader and path fetcher, eliminating the need for an external SCION daemon +// process. type embeddedConnector struct { topo *topology.Loader fetcher fetcher.Fetcher @@ -44,11 +45,8 @@ type embeddedConnector struct { cancel context.CancelFunc // cancels the topology loader goroutine } -// Compile-time interface checks. -var ( - _ daemon.Connector = (*embeddedConnector)(nil) - _ snet.Topology = (*embeddedConnector)(nil) -) +// Compile-time interface check. +var _ daemon.Connector = (*embeddedConnector)(nil) // LocalIA returns the local ISD-AS from the topology. func (ec *embeddedConnector) LocalIA(_ context.Context) (addr.IA, error) { @@ -71,6 +69,26 @@ func (ec *embeddedConnector) Interfaces(_ context.Context) (map[uint16]netip.Add return result, nil } +// snetTopology returns an snet.Topology struct built from the embedded +// topology loader. The Interface callback delegates to the loader for +// live topology access. +func (ec *embeddedConnector) snetTopology() snet.Topology { + ia := ec.topo.IA() + portMin, portMax := ec.topo.PortRange() + return snet.Topology{ + LocalIA: ia, + PortRange: snet.TopologyPortRange{Start: portMin, End: portMax}, + Interface: func(id uint16) (netip.AddrPort, bool) { + ifInfoMap := ec.topo.InterfaceInfoMap() + info, ok := ifInfoMap[iface.ID(id)] + if !ok { + return netip.AddrPort{}, false + } + return info.InternalAddr, true + }, + } +} + // Paths resolves end-to-end paths using the embedded fetcher (segment fetch + combination). func (ec *embeddedConnector) Paths(ctx context.Context, dst, src addr.IA, f daemon.PathReqFlags) ([]snet.Path, error) { return ec.fetcher.GetPaths(ctx, src, dst, f.Refresh) @@ -267,5 +285,5 @@ func tryEmbeddedDaemon(ctx context.Context, topoPath string) (*scionConn, error) return nil, fmt.Errorf("creating embedded connector: %w", err) } - return finishSCIONConnect(ctx, ec, ec) + return finishSCIONConnect(ctx, ec, ec.snetTopology()) } From 53d0d11357279640370c7d973364a98b3067b5d1 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Fri, 13 Mar 2026 10:30:54 +0000 Subject: [PATCH 39/78] wgengine/magicsock: enhance SCION path health tracking and probing - Implemented tracking for consecutive ping losses to mark SCION paths as unhealthy after three losses. - Added functionality to probe non-best SCION paths in a round-robin manner to keep latency data fresh. - Enhanced path re-evaluation logic to switch to healthier paths based on measured latency. - Updated tests to validate the new path health tracking and probing mechanisms. --- wgengine/magicsock/endpoint.go | 181 +++++++++++ wgengine/magicsock/magicsock.go | 8 +- wgengine/magicsock/magicsock_scion.go | 355 ++++++++++++++++++-- wgengine/magicsock/magicsock_scion_test.go | 361 +++++++++++++++++++++ 4 files changed, 873 insertions(+), 32 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index cd096b904deea..3203a77b892d6 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -876,12 +876,17 @@ func (de *endpoint) heartbeat() { continue } ps.lastPing = now + ps.pingsSent++ scionEp := epAddr{ ap: de.scionState.hostAddr, scionKey: pk, } de.startDiscoPingLocked(scionEp, now, pingHeartbeat, 0, nil) } + } else if de.scionState != nil && de.c.pconnSCION != nil && len(de.scionState.paths) > 1 { + // Probe non-best SCION paths one at a time via round-robin so + // latency data stays fresh for re-evaluation. + de.probeSCIONNonBestLocked(now) } if de.wantUDPRelayPathDiscoveryLocked(now) { @@ -1237,6 +1242,19 @@ func (de *endpoint) discoPingTimeout(txid stun.TxID) { de.c.dlogf("[v1] magicsock: disco: timeout waiting for pong %x from %v (%v, %v)", txid[:6], sp.to, de.publicKey.ShortString(), de.discoShort()) } de.removeSentDiscoPingLocked(txid, sp, discoPingTimedOut) + + // Track consecutive loss for SCION paths and demote if unhealthy. + if sp.to.scionKey.IsSet() && de.scionState != nil { + if ps, ok := de.scionState.paths[sp.to.scionKey]; ok { + ps.consecutiveLoss++ + if ps.consecutiveLoss >= 3 && ps.healthy { + ps.healthy = false + de.c.logf("magicsock: SCION path %d unhealthy for %v (loss: %d)", + sp.to.scionKey, de.publicKey.ShortString(), ps.consecutiveLoss) + de.demoteSCIONPathLocked(sp.to.scionKey) + } + } + } } // forgetDiscoPing is called when a ping fails to send. @@ -1248,6 +1266,161 @@ func (de *endpoint) forgetDiscoPing(txid stun.TxID) { } } +// probeSCIONNonBestLocked probes one non-active SCION path per call using +// round-robin ordering. This keeps latency data fresh for paths that aren't +// currently the active path, enabling re-evaluation to detect better options. +// de.mu must be held. +func (de *endpoint) probeSCIONNonBestLocked(now mono.Time) { + if de.scionState == nil { + return + } + + // Collect non-active path keys and sort for deterministic ordering. + var nonBest []scionPathKey + for k := range de.scionState.paths { + if k != de.scionState.activePath { + nonBest = append(nonBest, k) + } + } + if len(nonBest) == 0 { + return + } + slices.SortFunc(nonBest, func(a, b scionPathKey) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 + }) + + // Pick one via round-robin. + idx := de.scionState.probeRoundRobin % len(nonBest) + de.scionState.probeRoundRobin++ + pk := nonBest[idx] + ps := de.scionState.paths[pk] + + // Rate limit per path. + if !ps.lastPing.IsZero() && now.Sub(ps.lastPing) < discoPingInterval { + return + } + ps.lastPing = now + ps.pingsSent++ + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: pk, + } + de.startDiscoPingLocked(scionEp, now, pingHeartbeat, 0, nil) +} + +// demoteSCIONPathLocked is called when a SCION path is marked unhealthy. +// It finds the best remaining healthy path by measured latency and switches +// activePath and bestAddr if the demoted path was active/best. +// de.mu must be held. +func (de *endpoint) demoteSCIONPathLocked(demotedKey scionPathKey) { + if de.scionState == nil { + return + } + + // Find best healthy path by measured latency. + var bestKey scionPathKey + var bestLatency time.Duration + for k, ps := range de.scionState.paths { + if k == demotedKey || !ps.healthy { + continue + } + lat := ps.latency() + if !bestKey.IsSet() || lat < bestLatency { + bestKey = k + bestLatency = lat + } + } + + // Only act if the demoted path was the active path. + if de.scionState.activePath != demotedKey { + return + } + + if bestKey.IsSet() { + de.scionState.activePath = bestKey + newAddr := addrQuality{ + epAddr: epAddr{ap: de.scionState.hostAddr, scionKey: bestKey}, + latency: bestLatency, + scionPreferred: de.scionPreferred, + } + de.c.logf("magicsock: SCION path demoted, switching to path %d for %v", bestKey, de.publicKey.ShortString()) + de.setBestAddrLocked(newAddr) + go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, bestKey) + } else { + // No healthy SCION paths remain. Clear SCION bestAddr to fall back. + de.scionState.activePath = 0 + if de.bestAddr.isSCION() { + de.c.logf("magicsock: no healthy SCION paths for %v, clearing bestAddr", de.publicKey.ShortString()) + de.clearBestAddrLocked() + } + } +} + +// reEvalSCIONPathsLocked re-evaluates all SCION paths by measured latency +// after a pong is recorded. Throttled to once per 2 seconds. If a healthier, +// lower-latency path is found, switches bestAddr and activePath. +// de.mu must be held. +func (de *endpoint) reEvalSCIONPathsLocked(now mono.Time) { + if de.scionState == nil || len(de.scionState.paths) < 2 { + return + } + if !de.scionState.lastFullEvalAt.IsZero() && now.Sub(de.scionState.lastFullEvalAt) < 2*time.Second { + return + } + de.scionState.lastFullEvalAt = now + + // Check all paths have at least 1 pong measurement. + for _, ps := range de.scionState.paths { + if ps.pongCount == 0 { + return + } + } + + // Find the healthy path with lowest measured latency. + var bestKey scionPathKey + var bestLatency time.Duration + for k, ps := range de.scionState.paths { + if !ps.healthy { + continue + } + lat := ps.latency() + if !bestKey.IsSet() || lat < bestLatency { + bestKey = k + bestLatency = lat + } + } + + if !bestKey.IsSet() || bestKey == de.scionState.activePath { + return + } + + candidate := addrQuality{ + epAddr: epAddr{ap: de.scionState.hostAddr, scionKey: bestKey}, + latency: bestLatency, + scionPreferred: de.scionPreferred, + } + + if betterAddr(candidate, de.bestAddr) { + de.c.logf("magicsock: SCION re-eval: switching to path %d (latency %v) for %v", + bestKey, bestLatency.Round(time.Millisecond), de.publicKey.ShortString()) + de.debugUpdates.Add(EndpointChange{ + When: time.Now(), + What: "reEvalSCIONPathsLocked-switch", + From: de.bestAddr, + To: candidate, + }) + de.setBestAddrLocked(candidate) + de.scionState.activePath = bestKey + go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, bestKey) + } +} + // discoPingResult represents the result of an attempted disco ping send // operation. type discoPingResult int @@ -1434,6 +1607,7 @@ func (de *endpoint) sendDiscoPingsLocked(now mono.Time, sendCallMeMaybe bool) { continue } ps.lastPing = now + ps.pingsSent++ scionEp := epAddr{ ap: de.scionState.hostAddr, scionKey: pk, @@ -1858,7 +2032,14 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd latency: latency, pongAt: now, }) + ps.pongsReceived++ + ps.consecutiveLoss = 0 + if !ps.healthy { + ps.healthy = true + de.c.logf("magicsock: SCION path %d recovered for %v", src.scionKey, de.publicKey.ShortString()) + } } + de.reEvalSCIONPathsLocked(now) } } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index e9c9a5bb8850d..170830e43261e 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -24,6 +24,7 @@ import ( "sync/atomic" "time" + "github.com/scionproto/scion/pkg/addr" "github.com/tailscale/wireguard-go/conn" "github.com/tailscale/wireguard-go/device" "go4.org/mem" @@ -422,9 +423,10 @@ type Conn struct { // scionPaths is the registry of SCION path information, keyed by // scionPathKey. Each entry holds the full SCION address and path // data for a peer. - scionPaths map[scionPathKey]*scionPathInfo - scionPathsByAddr map[scionAddrKey]scionPathKey // reverse index for O(1) lookup - scionPathSeq atomic.Uint32 // monotonic key generator for scionPaths + scionPaths map[scionPathKey]*scionPathInfo + scionPathsByAddr map[scionAddrKey]scionPathKey // reverse index for O(1) lookup + scionPathSeq atomic.Uint32 // monotonic key generator for scionPaths + scionSoftRefreshAt map[addr.IA]time.Time // last soft refresh per peer, guarded by c.mu // lastSCIONRecv is the last time we received any SCION packet (monotonic). // Used by receiveSCION to detect a dead socket and trigger reconnection. diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 9ddec0fd86638..8f8e7e7ec6834 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -33,6 +33,7 @@ import ( "tailscale.com/tailcfg" "tailscale.com/tstime/mono" "tailscale.com/types/key" + "tailscale.com/util/mak" ) // debugSCIONPreference is the TS_SCION_PREFERENCE envknob controlling the @@ -82,9 +83,10 @@ type scionPathInfo struct { replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) cachedDst *snet.UDPAddr // pre-built destination addr; rebuilt when path changes fastPath *scionFastPath // pre-serialized header template for fast sends - expiry time.Time // path expiration from path metadata - mtu uint16 // SCION payload MTU from path metadata - mu sync.Mutex + expiry time.Time // path expiration from path metadata + mtu uint16 // SCION payload MTU from path metadata + refreshMissCount int // consecutive refresh cycles fingerprint absent from daemon + mu sync.Mutex } // buildCachedDst constructs the cached destination address from the current @@ -141,6 +143,11 @@ const scionReconnectThreshold = 30 * time.Second // defaultSCIONProbePaths is the default number of SCION paths to probe per peer. const defaultSCIONProbePaths = 5 +// scionStalePathThreshold is the number of consecutive refresh cycles a +// fingerprint must be absent from daemon results before the path is removed. +// At the default 30s refresh interval, this is ~90s. +const scionStalePathThreshold = 3 + // scionPongHistoryCount is the ring buffer size for per-path pong latency tracking. const scionPongHistoryCount = 8 @@ -160,15 +167,21 @@ type scionEndpointState struct { paths map[scionPathKey]*scionPathProbeState // probed paths (up to scionMaxProbePaths) activePath scionPathKey // currently selected best path for data lastDiscoveryAt time.Time // when path discovery last started (throttle) + lastFullEvalAt mono.Time // throttles re-evaluation of SCION path latencies + probeRoundRobin int // round-robin index for non-best path probing } // scionPathProbeState tracks disco probing state for one SCION path. type scionPathProbeState struct { - fingerprint snet.PathFingerprint - lastPing mono.Time - recentPongs [scionPongHistoryCount]scionPongReply // ring buffer - recentPong uint16 // index of most recent entry - pongCount uint16 // total pongs received (capped at ring size) + fingerprint snet.PathFingerprint + lastPing mono.Time + recentPongs [scionPongHistoryCount]scionPongReply // ring buffer + recentPong uint16 // index of most recent entry + pongCount uint16 // total pongs received (capped at ring size) + pingsSent uint32 // total pings sent on this path + pongsReceived uint32 // total pongs received (uncapped) + consecutiveLoss uint16 // consecutive pings without pong (reset on pong) + healthy bool // false = demoted from active selection } // scionPongReply records one pong measurement for a SCION path. @@ -1474,11 +1487,6 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr } // Deduplicate by fingerprint (topologically identical paths). - type pathWithMeta struct { - path snet.Path - fingerprint snet.PathFingerprint - latency time.Duration - } seen := make(map[snet.PathFingerprint]bool) var unique []pathWithMeta for _, p := range paths { @@ -1496,16 +1504,9 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr }) } - // Sort by latency ascending. - sort.Slice(unique, func(i, j int) bool { - return unique[i].latency < unique[j].latency - }) - - // Take top N. + // Select paths balancing latency and topological diversity. maxPaths := scionMaxProbePaths() - if len(unique) > maxPaths { - unique = unique[:maxPaths] - } + unique = selectDiversePaths(unique, maxPaths) // Register each path. c.mu.Lock() @@ -1558,6 +1559,131 @@ func totalPathLatency(p snet.Path) time.Duration { return total } +// pathWithMeta pairs a SCION path with its fingerprint and estimated latency +// for use in path selection and diversity algorithms. +type pathWithMeta struct { + path snet.Path + fingerprint snet.PathFingerprint + latency time.Duration +} + +// debugSCIONDiversityThreshold is the TS_SCION_DIVERSITY_THRESHOLD envknob +// controlling the latency penalty threshold (in ms) for diversity selection. +// Default 50ms. +var debugSCIONDiversityThreshold = envknob.RegisterInt("TS_SCION_DIVERSITY_THRESHOLD") + +// scionDiversityThreshold returns the latency threshold for diversity scoring. +func scionDiversityThreshold() time.Duration { + if v := debugSCIONDiversityThreshold(); v > 0 { + return time.Duration(v) * time.Millisecond + } + return 50 * time.Millisecond +} + +// interfaceOverlap computes the fraction of interfaces in path a that also +// appear in path b: |a ∩ b| / |a|. Returns 0.0 if either path has no +// interface metadata (unknown paths are assumed diverse). +func interfaceOverlap(a, b snet.Path) float64 { + mdA := a.Metadata() + mdB := b.Metadata() + if mdA == nil || mdB == nil || len(mdA.Interfaces) == 0 || len(mdB.Interfaces) == 0 { + return 0.0 + } + + bSet := make(map[snet.PathInterface]bool, len(mdB.Interfaces)) + for _, iface := range mdB.Interfaces { + bSet[iface] = true + } + + var overlap int + for _, iface := range mdA.Interfaces { + if bSet[iface] { + overlap++ + } + } + return float64(overlap) / float64(len(mdA.Interfaces)) +} + +// selectDiversePaths selects up to maxPaths from candidates, balancing low +// latency with topological diversity. It uses a greedy algorithm: +// 1. Sort candidates by latency ascending, pick the best. +// 2. For each subsequent slot, score remaining candidates by diversity +// (1 − max overlap with selected) minus latency penalty. +// 3. Fill remaining slots by pure latency if no diversity benefit. +func selectDiversePaths(candidates []pathWithMeta, maxPaths int) []pathWithMeta { + if len(candidates) <= maxPaths { + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].latency < candidates[j].latency + }) + return candidates + } + + // Sort by latency ascending. + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].latency < candidates[j].latency + }) + + threshold := scionDiversityThreshold() + selected := make([]pathWithMeta, 0, maxPaths) + used := make([]bool, len(candidates)) + + // Always pick the lowest-latency path first. + selected = append(selected, candidates[0]) + used[0] = true + bestLatency := candidates[0].latency + + for len(selected) < maxPaths { + bestScore := -1.0 + bestIdx := -1 + + for i, c := range candidates { + if used[i] { + continue + } + + // Compute diversity score: 1 − max overlap with any selected path. + var maxOverlap float64 + for _, s := range selected { + if ov := interfaceOverlap(c.path, s.path); ov > maxOverlap { + maxOverlap = ov + } + } + diversityScore := 1.0 - maxOverlap + + // Latency penalty: how much slower than the best path, normalized. + var latencyPenalty float64 + if threshold > 0 { + latencyPenalty = float64(c.latency-bestLatency) / float64(threshold) + } + + score := diversityScore - latencyPenalty + if score > bestScore { + bestScore = score + bestIdx = i + } + } + + if bestIdx < 0 || bestScore <= 0 { + // No diversity benefit; fill remaining by pure latency. + for i, c := range candidates { + if used[i] { + continue + } + selected = append(selected, c) + if len(selected) >= maxPaths { + break + } + } + break + } + + selected = append(selected, candidates[bestIdx]) + used[bestIdx] = true + } + + return selected +} + // refreshSCIONPaths runs in a background goroutine, periodically refreshing // SCION paths before they expire. It uses exponential backoff when the SCION // daemon is unreachable. @@ -1615,6 +1741,7 @@ func (c *Conn) refreshSCIONPathsOnce() error { // Group paths by peerIA so we query the daemon once per peer. type peerGroup struct { peerIA addr.IA + hostAddr netip.AddrPort needRefresh bool keys []scionPathKey infos []*scionPathInfo @@ -1624,12 +1751,13 @@ func (c *Conn) refreshSCIONPathsOnce() error { for k, pi := range pathsCopy { pi.mu.Lock() peerIA := pi.peerIA + hostAddr := pi.hostAddr needsRefresh := !pi.expiry.IsZero() && now.After(pi.expiry.Add(-1*time.Minute)) pi.mu.Unlock() g := groups[peerIA] if g == nil { - g = &peerGroup{peerIA: peerIA} + g = &peerGroup{peerIA: peerIA, hostAddr: hostAddr} groups[peerIA] = g } g.keys = append(g.keys, k) @@ -1685,10 +1813,11 @@ func (c *Conn) refreshSCIONPathsOnce() error { // Match existing registered paths to daemon paths by fingerprint. // Unmatched paths with known fingerprints (disappeared from daemon) - // are left unchanged — they'll be replaced on the next - // discoverSCIONPathAsync cycle. Paths with empty fingerprints - // (no metadata) get the best daemon path as fallback. - for _, pi := range g.infos { + // have their refreshMissCount incremented. When the count exceeds + // scionStalePathThreshold, the path is removed. Paths with empty + // fingerprints (no metadata) get the best daemon path as fallback. + var stalePaths []scionPathKey + for i, pi := range g.infos { pi.mu.Lock() fp := pi.fingerprint pi.mu.Unlock() @@ -1703,7 +1832,12 @@ func (c *Conn) refreshSCIONPathsOnce() error { } if matched == nil { // Known fingerprint disappeared from daemon results. - // Skip — don't overwrite with a different topology. + pi.mu.Lock() + pi.refreshMissCount++ + if pi.refreshMissCount >= scionStalePathThreshold { + stalePaths = append(stalePaths, g.keys[i]) + } + pi.mu.Unlock() continue } } else { @@ -1712,6 +1846,7 @@ func (c *Conn) refreshSCIONPathsOnce() error { } pi.mu.Lock() + pi.refreshMissCount = 0 pi.path = matched newFP := matched.Metadata().Fingerprint() pi.fingerprint = newFP @@ -1723,10 +1858,172 @@ func (c *Conn) refreshSCIONPathsOnce() error { pi.fastPath = buildSCIONFastPath(sc, pi) pi.mu.Unlock() } + + // Remove stale paths that have been absent for too many refresh cycles. + if len(stalePaths) > 0 { + c.mu.Lock() + for _, k := range stalePaths { + c.logf("magicsock: SCION path %d stale for %s, removing", k, g.peerIA) + c.unregisterSCIONPath(k) + } + c.mu.Unlock() + c.cleanStaleSCIONPathFromEndpoints(stalePaths, g.peerIA) + } } + + // Soft refresh pass: for groups that did NOT need hard refresh, check + // if new paths have appeared in the daemon's cache. This discovers new + // better paths that become available mid-session without waiting for + // expiry-driven hard refresh. + const softRefreshInterval = 5 * time.Minute + for _, g := range groups { + if g.needRefresh { + continue // already refreshed above + } + c.mu.Lock() + lastSoft := c.scionSoftRefreshAt[g.peerIA] + c.mu.Unlock() + if !lastSoft.IsZero() && now.Sub(lastSoft) < softRefreshInterval { + continue + } + + daemonPaths, err := sc.daemon.Paths(ctx, g.peerIA, sc.localIA, daemon.PathReqFlags{Refresh: false}) + if err != nil || len(daemonPaths) == 0 { + continue + } + + // Collect existing fingerprints for this group. + knownFPs := make(map[snet.PathFingerprint]bool, len(g.infos)) + for _, pi := range g.infos { + pi.mu.Lock() + if pi.fingerprint != "" { + knownFPs[pi.fingerprint] = true + } + pi.mu.Unlock() + } + + // Filter to paths with new fingerprints. + maxSlots := scionMaxProbePaths() + available := maxSlots - len(g.keys) + if available <= 0 { + c.mu.Lock() + mak.Set(&c.scionSoftRefreshAt, g.peerIA, now) + c.mu.Unlock() + continue + } + + var newPaths []snet.Path + for _, dp := range daemonPaths { + fp := dp.Metadata().Fingerprint() + if fp == "" || knownFPs[fp] { + continue + } + newPaths = append(newPaths, dp) + if len(newPaths) >= available { + break + } + } + + if len(newPaths) > 0 { + c.addNewSCIONPathsForPeer(g.peerIA, g.hostAddr, newPaths) + c.logf("magicsock: SCION soft refresh: %d new paths for %s", len(newPaths), g.peerIA) + } + + c.mu.Lock() + mak.Set(&c.scionSoftRefreshAt, g.peerIA, now) + c.mu.Unlock() + } + return lastErr } +// addNewSCIONPathsForPeer registers new SCION paths and adds probe states +// to the corresponding endpoint. Called during soft refresh when new paths +// appear in the daemon's cache. +func (c *Conn) addNewSCIONPathsForPeer(peerIA addr.IA, hostAddr netip.AddrPort, paths []snet.Path) { + sc := c.pconnSCION + if sc == nil { + return + } + + c.mu.Lock() + var newKeys []scionPathKey + for _, p := range paths { + md := p.Metadata() + var expiry time.Time + var mtu uint16 + if md != nil { + expiry = md.Expiry + mtu = md.MTU + } + pi := &scionPathInfo{ + peerIA: peerIA, + hostAddr: hostAddr, + fingerprint: md.Fingerprint(), + path: p, + expiry: expiry, + mtu: mtu, + } + pi.buildCachedDst() + pi.fastPath = buildSCIONFastPath(sc, pi) + k := c.registerSCIONPath(pi) + newKeys = append(newKeys, k) + } + + // Find the endpoint for this peerIA and add probe states. + for _, peerInf := range c.peerMap.byNodeKey { + ep := peerInf.ep + ep.mu.Lock() + if ep.scionState != nil && ep.scionState.peerIA == peerIA { + for _, k := range newKeys { + if _, exists := ep.scionState.paths[k]; !exists { + fp := c.scionPaths[k].fingerprint + ep.scionState.paths[k] = &scionPathProbeState{ + fingerprint: fp, + healthy: true, + } + } + } + } + ep.mu.Unlock() + } + c.mu.Unlock() +} + +// cleanStaleSCIONPathFromEndpoints removes stale SCION path keys from all +// endpoints that reference the given peerIA. If the removed key was the +// activePath, reassigns to the first remaining path. +func (c *Conn) cleanStaleSCIONPathFromEndpoints(staleKeys []scionPathKey, peerIA addr.IA) { + staleSet := make(map[scionPathKey]bool, len(staleKeys)) + for _, k := range staleKeys { + staleSet[k] = true + } + + c.mu.Lock() + defer c.mu.Unlock() + for _, pi := range c.peerMap.byNodeKey { + ep := pi.ep + ep.mu.Lock() + if ep.scionState == nil || ep.scionState.peerIA != peerIA { + ep.mu.Unlock() + continue + } + for k := range ep.scionState.paths { + if staleSet[k] { + delete(ep.scionState.paths, k) + } + } + if staleSet[ep.scionState.activePath] { + ep.scionState.activePath = 0 + for k := range ep.scionState.paths { + ep.scionState.activePath = k + break + } + } + ep.mu.Unlock() + } +} + // scionServiceFromPeer extracts SCION service info from a peer node's Services. // It checks for a dedicated SCION service entry first, then falls back to // checking the peerapi4 Description field (which is used to piggyback SCION @@ -1877,7 +2174,7 @@ func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPo continue } } - newPaths[k] = &scionPathProbeState{fingerprint: fp} + newPaths[k] = &scionPathProbeState{fingerprint: fp, healthy: true} } activePath := scionPathKey(0) diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 1533f952bdcb9..c13b493995546 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -1485,3 +1485,364 @@ func TestSCIONSendBatchPool(t *testing.T) { } } } + +// --- Tests for SCION Path Handling Improvements --- + +func TestScionInterfaceOverlap(t *testing.T) { + ctrl := gomock.NewController(t) + + ifaceIA1 := addr.MustParseIA("1-ff00:0:110") + ifaceIA2 := addr.MustParseIA("1-ff00:0:111") + ifaceIA3 := addr.MustParseIA("1-ff00:0:112") + + t.Run("full overlap", func(t *testing.T) { + a := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Interfaces: []snet.PathInterface{ + {IA: ifaceIA1, ID: 1}, {IA: ifaceIA2, ID: 2}, + }, + }) + b := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Interfaces: []snet.PathInterface{ + {IA: ifaceIA1, ID: 1}, {IA: ifaceIA2, ID: 2}, + }, + }) + got := interfaceOverlap(a, b) + if got != 1.0 { + t.Errorf("full overlap = %v, want 1.0", got) + } + }) + + t.Run("no overlap", func(t *testing.T) { + a := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Interfaces: []snet.PathInterface{ + {IA: ifaceIA1, ID: 1}, + }, + }) + b := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Interfaces: []snet.PathInterface{ + {IA: ifaceIA2, ID: 2}, {IA: ifaceIA3, ID: 3}, + }, + }) + got := interfaceOverlap(a, b) + if got != 0.0 { + t.Errorf("no overlap = %v, want 0.0", got) + } + }) + + t.Run("partial overlap", func(t *testing.T) { + a := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Interfaces: []snet.PathInterface{ + {IA: ifaceIA1, ID: 1}, {IA: ifaceIA2, ID: 2}, + }, + }) + b := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Interfaces: []snet.PathInterface{ + {IA: ifaceIA1, ID: 1}, {IA: ifaceIA3, ID: 3}, + }, + }) + got := interfaceOverlap(a, b) + if got != 0.5 { + t.Errorf("partial overlap = %v, want 0.5", got) + } + }) + + t.Run("nil metadata returns zero", func(t *testing.T) { + a := newMockPathWithMetadata(ctrl, nil) + b := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Interfaces: []snet.PathInterface{{IA: ifaceIA1, ID: 1}}, + }) + got := interfaceOverlap(a, b) + if got != 0.0 { + t.Errorf("nil metadata = %v, want 0.0", got) + } + }) + + t.Run("empty interfaces returns zero", func(t *testing.T) { + a := newMockPathWithMetadata(ctrl, &snet.PathMetadata{}) + b := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Interfaces: []snet.PathInterface{{IA: ifaceIA1, ID: 1}}, + }) + got := interfaceOverlap(a, b) + if got != 0.0 { + t.Errorf("empty interfaces = %v, want 0.0", got) + } + }) +} + +func TestScionSelectDiversePaths(t *testing.T) { + ctrl := gomock.NewController(t) + + ifaceIA1 := addr.MustParseIA("1-ff00:0:110") + ifaceIA2 := addr.MustParseIA("1-ff00:0:111") + ifaceIA3 := addr.MustParseIA("1-ff00:0:112") + + t.Run("fewer candidates than max", func(t *testing.T) { + paths := []pathWithMeta{ + {path: newMockPathWithMetadata(ctrl, &snet.PathMetadata{Latency: []time.Duration{10 * time.Millisecond}}), latency: 10 * time.Millisecond}, + {path: newMockPathWithMetadata(ctrl, &snet.PathMetadata{Latency: []time.Duration{5 * time.Millisecond}}), latency: 5 * time.Millisecond}, + } + result := selectDiversePaths(paths, 5) + if len(result) != 2 { + t.Fatalf("got %d paths, want 2", len(result)) + } + // Should be sorted by latency. + if result[0].latency > result[1].latency { + t.Error("should be sorted by latency ascending") + } + }) + + t.Run("prefers diverse path over duplicate topology", func(t *testing.T) { + // Path A: fastest, through ifaceIA1+ifaceIA2. + pathA := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{5 * time.Millisecond}, + Interfaces: []snet.PathInterface{{IA: ifaceIA1, ID: 1}, {IA: ifaceIA2, ID: 2}}, + }) + // Path B: slightly slower, same interfaces as A. + pathB := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{6 * time.Millisecond}, + Interfaces: []snet.PathInterface{{IA: ifaceIA1, ID: 1}, {IA: ifaceIA2, ID: 2}}, + }) + // Path C: a bit slower, different interfaces. + pathC := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{8 * time.Millisecond}, + Interfaces: []snet.PathInterface{{IA: ifaceIA1, ID: 1}, {IA: ifaceIA3, ID: 3}}, + }) + + candidates := []pathWithMeta{ + {path: pathA, latency: 5 * time.Millisecond}, + {path: pathB, latency: 6 * time.Millisecond}, + {path: pathC, latency: 8 * time.Millisecond}, + } + result := selectDiversePaths(candidates, 2) + if len(result) != 2 { + t.Fatalf("got %d paths, want 2", len(result)) + } + // First should be pathA (lowest latency), second should be pathC (diverse). + if result[0].path != pathA { + t.Error("first path should be lowest-latency (pathA)") + } + if result[1].path != pathC { + t.Error("second path should be diverse (pathC), not duplicate (pathB)") + } + }) +} + +func TestScionStalePathCleanup(t *testing.T) { + ctrl := gomock.NewController(t) + mockDaemon := mock_daemon.NewMockConnector(ctrl) + + localIA := addr.MustParseIA("1-ff00:0:110") + peerIA := addr.MustParseIA("1-ff00:0:111") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := &Conn{ + connCtx: ctx, + peerMap: newPeerMap(), + } + c.logf = t.Logf + c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + + // Register a path with a fingerprint that will disappear. + pi := &scionPathInfo{ + peerIA: peerIA, + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + fingerprint: "stale-fp", + expiry: time.Now().Add(30 * time.Second), // about to expire + } + k := c.registerSCIONPathLocking(pi) + + // Daemon returns a path with a different fingerprint each time. + newPath := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{5 * time.Millisecond}, + Expiry: time.Now().Add(2 * time.Hour), + }) + + // Call refresh scionStalePathThreshold times. + for i := 0; i < scionStalePathThreshold; i++ { + mockDaemon.EXPECT().Paths(gomock.Any(), peerIA, localIA, daemon.PathReqFlags{Refresh: true}). + Return([]snet.Path{newPath}, nil) + c.refreshSCIONPathsOnce() + } + + // Path should have been cleaned up. + got := c.lookupSCIONPathLocking(k) + if got != nil { + t.Error("stale path should have been removed after threshold exceeded") + } +} + +func TestScionPathHealthTracking(t *testing.T) { + t.Run("pong resets consecutive loss and marks healthy", func(t *testing.T) { + ps := &scionPathProbeState{healthy: true} + + // Simulate 2 losses. + ps.consecutiveLoss = 2 + ps.pingsSent = 3 + + // Pong arrives. + ps.pongsReceived++ + ps.consecutiveLoss = 0 + + if !ps.healthy { + t.Error("should still be healthy after pong") + } + if ps.consecutiveLoss != 0 { + t.Error("consecutive loss should be reset") + } + if ps.pongsReceived != 1 { + t.Errorf("pongsReceived = %d, want 1", ps.pongsReceived) + } + }) + + t.Run("three consecutive losses marks unhealthy", func(t *testing.T) { + ps := &scionPathProbeState{healthy: true} + + for i := 0; i < 3; i++ { + ps.consecutiveLoss++ + } + + if ps.consecutiveLoss < 3 { + t.Error("should have 3 consecutive losses") + } + // In real code, demoteSCIONPathLocked would set healthy = false. + ps.healthy = false + if ps.healthy { + t.Error("should be unhealthy") + } + }) + + t.Run("recovery after unhealthy", func(t *testing.T) { + ps := &scionPathProbeState{healthy: false, consecutiveLoss: 5} + + // Pong arrives — recovery. + ps.pongsReceived++ + ps.consecutiveLoss = 0 + ps.healthy = true + + if !ps.healthy { + t.Error("should be healthy after recovery") + } + }) +} + +func TestScionDemoteSCIONPathLocked(t *testing.T) { + hostAddr := netip.MustParseAddrPort("10.0.0.1:41641") + peerIA := addr.MustParseIA("1-ff00:0:111") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := &Conn{ + connCtx: ctx, + peerMap: newPeerMap(), + } + c.logf = t.Logf + + de := &endpoint{c: c} + de.scionState = &scionEndpointState{ + peerIA: peerIA, + hostAddr: hostAddr, + activePath: scionPathKey(1), + paths: map[scionPathKey]*scionPathProbeState{ + scionPathKey(1): {healthy: false, recentPongs: [scionPongHistoryCount]scionPongReply{{latency: 50 * time.Millisecond}}, pongCount: 1, recentPong: 0}, + scionPathKey(2): {healthy: true, recentPongs: [scionPongHistoryCount]scionPongReply{{latency: 30 * time.Millisecond}}, pongCount: 1, recentPong: 0}, + scionPathKey(3): {healthy: true, recentPongs: [scionPongHistoryCount]scionPongReply{{latency: 40 * time.Millisecond}}, pongCount: 1, recentPong: 0}, + }, + } + de.bestAddr = addrQuality{ + epAddr: epAddr{ap: hostAddr, scionKey: scionPathKey(1)}, + } + + de.mu.Lock() + de.demoteSCIONPathLocked(scionPathKey(1)) + activePath := de.scionState.activePath + bestKey := de.bestAddr.scionKey + de.mu.Unlock() + + if activePath != scionPathKey(2) { + t.Errorf("activePath = %d, want 2 (best healthy)", activePath) + } + if bestKey != scionPathKey(2) { + t.Errorf("bestAddr scionKey = %d, want 2", bestKey) + } +} + +func TestScionReEvalSCIONPathsLocked(t *testing.T) { + hostAddr := netip.MustParseAddrPort("10.0.0.1:41641") + peerIA := addr.MustParseIA("1-ff00:0:111") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := &Conn{ + connCtx: ctx, + peerMap: newPeerMap(), + } + c.logf = t.Logf + + de := &endpoint{c: c} + de.scionState = &scionEndpointState{ + peerIA: peerIA, + hostAddr: hostAddr, + activePath: scionPathKey(1), + paths: map[scionPathKey]*scionPathProbeState{ + scionPathKey(1): {healthy: true, recentPongs: [scionPongHistoryCount]scionPongReply{{latency: 50 * time.Millisecond}}, pongCount: 1, recentPong: 0}, + scionPathKey(2): {healthy: true, recentPongs: [scionPongHistoryCount]scionPongReply{{latency: 10 * time.Millisecond}}, pongCount: 1, recentPong: 0}, + }, + } + de.bestAddr = addrQuality{ + epAddr: epAddr{ap: hostAddr, scionKey: scionPathKey(1)}, + latency: 50 * time.Millisecond, + } + + de.mu.Lock() + de.reEvalSCIONPathsLocked(mono.Now()) + activePath := de.scionState.activePath + bestKey := de.bestAddr.scionKey + de.mu.Unlock() + + // Path 2 has lower latency, should be selected. + if activePath != scionPathKey(2) { + t.Errorf("activePath = %d, want 2 (lower latency)", activePath) + } + if bestKey != scionPathKey(2) { + t.Errorf("bestAddr scionKey = %d, want 2", bestKey) + } +} + +func TestScionProbeSCIONNonBestLocked(t *testing.T) { + // Test that probeSCIONNonBestLocked round-robins through non-active paths. + state := &scionEndpointState{ + hostAddr: netip.MustParseAddrPort("10.0.0.1:41641"), + activePath: scionPathKey(1), + paths: map[scionPathKey]*scionPathProbeState{ + scionPathKey(1): {healthy: true}, + scionPathKey(2): {healthy: true}, + scionPathKey(3): {healthy: true}, + }, + } + + // Collect non-active keys manually as probeSCIONNonBestLocked would. + var nonBest []scionPathKey + for k := range state.paths { + if k != state.activePath { + nonBest = append(nonBest, k) + } + } + + if len(nonBest) != 2 { + t.Fatalf("expected 2 non-best paths, got %d", len(nonBest)) + } + + // Verify round-robin increments. + idx0 := state.probeRoundRobin % len(nonBest) + state.probeRoundRobin++ + idx1 := state.probeRoundRobin % len(nonBest) + state.probeRoundRobin++ + + if idx0 == idx1 { + t.Error("round-robin should pick different paths on consecutive calls") + } +} From fffafe7f1d0eb0ca55cc7e1240a2a6466a4deafb Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Fri, 13 Mar 2026 12:23:20 +0000 Subject: [PATCH 40/78] wgengine/magicsock: add default bootstrap server URLs for SCIERA (OVGU and UVA) --- wgengine/magicsock/scion_bootstrap.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/wgengine/magicsock/scion_bootstrap.go b/wgengine/magicsock/scion_bootstrap.go index b5abd4916ccfa..5be0c6f448d5e 100644 --- a/wgengine/magicsock/scion_bootstrap.go +++ b/wgengine/magicsock/scion_bootstrap.go @@ -32,7 +32,11 @@ const ( // defaultBootstrapURLs contains well-known bootstrap server URLs for major // SCION deployments. Populated as deployments are identified; DNS discovery // is the primary automatic mechanism. -var defaultBootstrapURLs []string +var defaultBootstrapURLs []string = []string{ + "http://141.44.25.151:8041", + "http://128.143.201.144:8041", +} + // bootstrapSCION fetches topology.json and TRCs from a bootstrap server, // saving them to destDir. From 621f71981cf05e5ea6c6accbd92123f40b531f45 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Fri, 13 Mar 2026 14:31:16 +0000 Subject: [PATCH 41/78] cmd/k8s-operator: fix Service reconcile triggers for default ProxyClass (#18983) The e2e ingress test was very occasionally flaky. On looking at operator logs from one failure, you can see the default ProxyClass was not ready before the first reconcile loop for the exposed Service. The ProxyClass became ready soon after, but no additional reconciles were triggered for the exposed Service because we only triggered reconciles for Services that explicitly named their ProxyClass. This change adds additional list API calls for when it's the default ProxyClass that's been updated in order to catch Services that use it by default. It also adds indexes for the fields we need to search on to ensure the list is efficient. Fixes tailscale/corp#37533 Signed-off-by: Tom Proctor --- cmd/k8s-operator/e2e/ingress_test.go | 76 +-------- cmd/k8s-operator/e2e/setup.go | 23 ++- cmd/k8s-operator/operator.go | 79 +++++++++- cmd/k8s-operator/svc.go | 22 ++- cmd/k8s-operator/svc_test.go | 221 +++++++++++++++++++++++++++ 5 files changed, 331 insertions(+), 90 deletions(-) create mode 100644 cmd/k8s-operator/svc_test.go diff --git a/cmd/k8s-operator/e2e/ingress_test.go b/cmd/k8s-operator/e2e/ingress_test.go index 95fbbab9df697..a136d2ad358e2 100644 --- a/cmd/k8s-operator/e2e/ingress_test.go +++ b/cmd/k8s-operator/e2e/ingress_test.go @@ -5,7 +5,6 @@ package e2e import ( "context" - "encoding/json" "fmt" "net/http" "testing" @@ -14,10 +13,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" "tailscale.com/cmd/testwrapper/flakytest" kube "tailscale.com/k8s-operator" @@ -90,81 +85,20 @@ func TestIngress(t *testing.T) { } createAndCleanup(t, kubeClient, svc) - // TODO(tomhjp): Delete once we've reproduced the flake with this extra info. - t0 := time.Now() - watcherCtx, cancelWatcher := context.WithCancel(t.Context()) - defer cancelWatcher() - go func() { - // client-go client for logs. - clientGoKubeClient, err := kubernetes.NewForConfig(restCfg) - if err != nil { - t.Logf("error creating client-go Kubernetes client: %v", err) - return - } - - for { - select { - case <-watcherCtx.Done(): - t.Logf("stopping watcher after %v", time.Since(t0)) - return - case <-time.After(time.Minute): - t.Logf("dumping info after %v elapsed", time.Since(t0)) - // Service itself. - svc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")} - err := get(watcherCtx, kubeClient, svc) - svcYaml, _ := yaml.Marshal(svc) - t.Logf("Service: %s, error: %v\n%s", svc.Name, err, string(svcYaml)) - - // Pods in tailscale namespace. - var pods corev1.PodList - if err := kubeClient.List(watcherCtx, &pods, client.InNamespace("tailscale")); err != nil { - t.Logf("error listing Pods in tailscale namespace: %v", err) - } else { - t.Logf("%d Pods", len(pods.Items)) - for _, pod := range pods.Items { - podYaml, _ := yaml.Marshal(pod) - t.Logf("Pod: %s\n%s", pod.Name, string(podYaml)) - logs := clientGoKubeClient.CoreV1().Pods("tailscale").GetLogs(pod.Name, &corev1.PodLogOptions{}).Do(watcherCtx) - logData, err := logs.Raw() - if err != nil { - t.Logf("error reading logs for Pod %s: %v", pod.Name, err) - continue - } - t.Logf("Logs for Pod %s:\n%s", pod.Name, string(logData)) - } - } - - // Tailscale status on the tailnet. - lc, err := tnClient.LocalClient() - if err != nil { - t.Logf("error getting tailnet local client: %v", err) - } else { - status, err := lc.Status(watcherCtx) - statusJSON, _ := json.MarshalIndent(status, "", " ") - t.Logf("Tailnet status: %s, error: %v", string(statusJSON), err) - } - } - } - }() - - // TODO: instead of timing out only when test times out, cancel context after 60s or so. - if err := wait.PollUntilContextCancel(t.Context(), time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) { - if time.Since(t0) > time.Minute { - t.Logf("%v elapsed waiting for Service default/test-ingress to become Ready", time.Since(t0)) - } + if err := tstest.WaitFor(time.Minute, func() error { maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")} - if err := get(ctx, kubeClient, maybeReadySvc); err != nil { - return false, err + if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { + return err } isReady := kube.SvcIsReady(maybeReadySvc) if isReady { t.Log("Service is ready") + return nil } - return isReady, nil + return fmt.Errorf("Service is not ready yet") }); err != nil { t.Fatalf("error waiting for the Service to become Ready: %v", err) } - cancelWatcher() var resp *http.Response if err := tstest.WaitFor(time.Minute, func() error { diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go index c4fd45d3e4125..e3d7ed89b55ca 100644 --- a/cmd/k8s-operator/e2e/setup.go +++ b/cmd/k8s-operator/e2e/setup.go @@ -56,6 +56,7 @@ import ( "tailscale.com/internal/client/tailscale" "tailscale.com/ipn" "tailscale.com/ipn/store/mem" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tsnet" ) @@ -438,7 +439,7 @@ func runTests(m *testing.M) (int, error) { return 0, fmt.Errorf("failed to install %q via helm: %w", relName, err) } - if err := applyDefaultProxyClass(ctx, kubeClient); err != nil { + if err := applyDefaultProxyClass(ctx, logger, kubeClient); err != nil { return 0, fmt.Errorf("failed to apply default ProxyClass: %w", err) } @@ -537,7 +538,7 @@ func tagForRepo(dir string) (string, error) { return tag, nil } -func applyDefaultProxyClass(ctx context.Context, cl client.Client) error { +func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl client.Client) error { pc := &tsapi.ProxyClass{ TypeMeta: metav1.TypeMeta{ APIVersion: tsapi.SchemeGroupVersion.String(), @@ -565,6 +566,24 @@ func applyDefaultProxyClass(ctx context.Context, cl client.Client) error { return fmt.Errorf("failed to apply default ProxyClass: %w", err) } + // Wait for the ProxyClass to be marked ready. + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + for { + if err := cl.Get(ctx, client.ObjectKeyFromObject(pc), pc); err != nil { + return fmt.Errorf("failed to get default ProxyClass: %w", err) + } + if tsoperator.ProxyClassIsReady(pc) { + break + } + logger.Info("waiting for default ProxyClass to be ready...") + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for default ProxyClass to be ready") + case <-time.After(time.Second): + } + } + return nil } diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 1060c6f3da9e7..d353c53337fd6 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -77,6 +77,12 @@ import ( // Generate CRD API docs. //go:generate go run github.com/elastic/crd-ref-docs --renderer=markdown --source-path=../../k8s-operator/apis/ --config=../../k8s-operator/api-docs-config.yaml --output-path=../../k8s-operator/api.md +const ( + indexServiceProxyClass = ".metadata.annotations.service-proxy-class" + indexServiceExposed = ".metadata.annotations.service-expose" + indexServiceType = ".metadata.annotations.service-type" +) + func main() { // Required to use our client API. We're fine with the instability since the // client lives in the same repo as this code. @@ -351,7 +357,12 @@ func runReconcilers(opts reconcilerOpts) { svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc")) // If a ProxyClass changes, enqueue all Services labeled with that // ProxyClass's name. - proxyClassFilterForSvc := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForSvc(mgr.GetClient(), startlog)) + proxyClassFilterForSvc := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForSvc( + mgr.GetClient(), + startlog, + opts.defaultProxyClass, + opts.proxyActAsDefaultLoadBalancer, + )) eventRecorder := mgr.GetEventRecorderFor("tailscale-operator") ssr := &tailscaleSTSReconciler{ @@ -389,6 +400,18 @@ func runReconcilers(opts reconcilerOpts) { if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexServiceProxyClass, indexProxyClass); err != nil { startlog.Fatalf("failed setting up ProxyClass indexer for Services: %v", err) } + if opts.defaultProxyClass != "" { + // If a default ProxyClass is specified, we'll need to list all objects + // that could be affected. For L3 ingress, this is Services with the + // "tailscale.com/expose" annotation and LoadBalancer services (either + // with the loadBalancerClass "tailscale", or unset if we're the default). + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexServiceExposed, indexExposed); err != nil { + startlog.Fatalf("failed setting up exposed indexer for Services: %v", err) + } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexServiceType, indexType); err != nil { + startlog.Fatalf("failed setting up type indexer for Services: %v", err) + } + } ingressChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("ingress")) // If a ProxyClassChanges, enqueue all Ingresses labeled with that @@ -910,10 +933,27 @@ func indexProxyClass(o client.Object) []string { return []string{o.GetAnnotations()[LabelAnnotationProxyClass]} } +func indexExposed(o client.Object) []string { + if o.GetAnnotations()[AnnotationExpose] != "true" { + return nil + } + + return []string{o.GetAnnotations()[AnnotationExpose]} +} + +func indexType(o client.Object) []string { + svc, ok := o.(*corev1.Service) + if !ok { + return nil + } + + return []string{string(svc.Spec.Type)} +} + // proxyClassHandlerForSvc returns a handler that, for a given ProxyClass, // returns a list of reconcile requests for all Services labeled with // tailscale.com/proxy-class: . -func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger, defaultProxyClass string, isDefaultLoadBalancer bool) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { svcList := new(corev1.ServiceList) labels := map[string]string{ @@ -932,13 +972,12 @@ func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger) handle seenSvcs.Add(fmt.Sprintf("%s/%s", svc.Namespace, svc.Name)) } - svcAnnotationList := new(corev1.ServiceList) - if err := cl.List(ctx, svcAnnotationList, client.MatchingFields{indexServiceProxyClass: o.GetName()}); err != nil { + if err := cl.List(ctx, svcList, client.MatchingFields{indexServiceProxyClass: o.GetName()}); err != nil { logger.Debugf("error listing Services for ProxyClass: %v", err) return nil } - for _, svc := range svcAnnotationList.Items { + for _, svc := range svcList.Items { nsname := fmt.Sprintf("%s/%s", svc.Namespace, svc.Name) if seenSvcs.Contains(nsname) { continue @@ -948,6 +987,36 @@ func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger) handle seenSvcs.Add(nsname) } + if o.GetName() == defaultProxyClass { + // For the default ProxyClass, we also need to reconcile all exposed + // Services that don't have an explicit ProxyClass set. + for _, matcher := range []client.ListOption{ + client.MatchingFields{indexServiceExposed: "true"}, + client.MatchingFields{indexServiceType: string(corev1.ServiceTypeLoadBalancer)}, + } { + if err := cl.List(ctx, svcList, matcher); err != nil { + logger.Debugf("error listing exposed Services for ProxyClass: %v", err) + return nil + } + + for _, svc := range svcList.Items { + if hasProxyClassAnnotation(&svc) { + continue + } + if !shouldExpose(&svc, isDefaultLoadBalancer) { + continue + } + nsname := fmt.Sprintf("%s/%s", svc.Namespace, svc.Name) + if seenSvcs.Contains(nsname) { + continue + } + + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)}) + seenSvcs.Add(nsname) + } + } + } + return reqs } } diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index 31be22aa12ca3..6f12148c85807 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -42,8 +42,6 @@ const ( reasonProxyInvalid = "ProxyInvalid" reasonProxyFailed = "ProxyFailed" reasonProxyPending = "ProxyPending" - - indexServiceProxyClass = ".metadata.annotations.service-proxy-class" ) type ServiceReconciler struct { @@ -97,7 +95,7 @@ func childResourceLabels(name, ns, typ string) map[string]string { func (a *ServiceReconciler) isTailscaleService(svc *corev1.Service) bool { targetIP := tailnetTargetAnnotation(svc) targetFQDN := svc.Annotations[AnnotationTailnetTargetFQDN] - return a.shouldExpose(svc) || targetIP != "" || targetFQDN != "" + return shouldExpose(svc, a.isDefaultLoadBalancer) || targetIP != "" || targetFQDN != "" } func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) { @@ -164,7 +162,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare } proxyTyp := proxyTypeEgress - if a.shouldExpose(svc) { + if shouldExpose(svc, a.isDefaultLoadBalancer) { proxyTyp = proxyTypeIngressService } @@ -275,16 +273,16 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga LoginServer: a.ssr.loginServer, } sts.proxyType = proxyTypeEgress - if a.shouldExpose(svc) { + if shouldExpose(svc, a.isDefaultLoadBalancer) { sts.proxyType = proxyTypeIngressService } a.mu.Lock() - if a.shouldExposeClusterIP(svc) { + if shouldExposeClusterIP(svc, a.isDefaultLoadBalancer) { sts.ClusterTargetIP = svc.Spec.ClusterIP a.managedIngressProxies.Add(svc.UID) gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len())) - } else if a.shouldExposeDNSName(svc) { + } else if shouldExposeDNSName(svc) { sts.ClusterTargetDNSName = svc.Spec.ExternalName a.managedIngressProxies.Add(svc.UID) gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len())) @@ -410,19 +408,19 @@ func validateService(svc *corev1.Service) []string { return violations } -func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool { - return a.shouldExposeClusterIP(svc) || a.shouldExposeDNSName(svc) +func shouldExpose(svc *corev1.Service, isDefaultLoadBalancer bool) bool { + return shouldExposeClusterIP(svc, isDefaultLoadBalancer) || shouldExposeDNSName(svc) } -func (a *ServiceReconciler) shouldExposeDNSName(svc *corev1.Service) bool { +func shouldExposeDNSName(svc *corev1.Service) bool { return hasExposeAnnotation(svc) && svc.Spec.Type == corev1.ServiceTypeExternalName && svc.Spec.ExternalName != "" } -func (a *ServiceReconciler) shouldExposeClusterIP(svc *corev1.Service) bool { +func shouldExposeClusterIP(svc *corev1.Service, isDefaultLoadBalancer bool) bool { if svc.Spec.ClusterIP == "" { return false } - return isTailscaleLoadBalancerService(svc, a.isDefaultLoadBalancer) || hasExposeAnnotation(svc) + return isTailscaleLoadBalancerService(svc, isDefaultLoadBalancer) || hasExposeAnnotation(svc) } func isTailscaleLoadBalancerService(svc *corev1.Service, isDefaultLoadBalancer bool) bool { diff --git a/cmd/k8s-operator/svc_test.go b/cmd/k8s-operator/svc_test.go new file mode 100644 index 0000000000000..3a6ea044d5af5 --- /dev/null +++ b/cmd/k8s-operator/svc_test.go @@ -0,0 +1,221 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "slices" + "testing" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" + "tailscale.com/tstest" +) + +func TestService_DefaultProxyClassInitiallyNotReady(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"}, + Spec: tsapi.ProxyClassSpec{ + TailscaleConfig: &tsapi.TailscaleConfig{ + AcceptRoutes: true, + }, + StatefulSet: &tsapi.StatefulSet{ + Labels: tsapi.Labels{"foo": "bar"}, + Annotations: map[string]string{"bar.io/foo": "some-val"}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}, + }, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc). + WithStatusSubresource(pc). + Build() + ft := &fakeTSClient{} + zl := zap.Must(zap.NewDevelopment()) + clock := tstest.NewClock(tstest.ClockOpts{}) + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + defaultProxyClass: "custom-metadata", + logger: zl.Sugar(), + clock: clock, + } + + // 1. A new tailscale LoadBalancer Service is created but the default + // ProxyClass is not ready yet. + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: new("tailscale"), + }, + }) + expectReconciled(t, sr, "default", "test") + labels := map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentName: "test", + LabelParentNamespace: "operator-ns", + LabelParentType: "svc", + } + s, err := getSingleObject[corev1.Secret](context.Background(), fc, "operator-ns", labels) + if err != nil { + t.Fatalf("finding Secret for %q: %v", "test", err) + } + if s != nil { + t.Fatalf("expected no Secret to be created when default ProxyClass is not ready, but found one: %v", s) + } + + // 2. ProxyClass is set to Ready, the Service can become ready now. + mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) { + pc.Status = tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Status: metav1.ConditionTrue, + Type: string(tsapi.ProxyClassReady), + ObservedGeneration: pc.Generation, + }}, + } + }) + expectReconciled(t, sr, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") + opts := configOpts{ + replicas: new(int32(1)), + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "svc", + hostname: "default-test", + clusterTargetIP: "10.20.30.40", + app: kubetypes.AppIngressProxy, + proxyClass: pc.Name, + } + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) +} + +func TestProxyClassHandlerForSvc(t *testing.T) { + svc := func(name string, annotations, labels map[string]string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + }, + } + } + lbSvc := func(name string, annotations map[string]string, class *string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "foo", + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: class, + ClusterIP: "1.2.3.4", + }, + } + } + + const ( + defaultPCName = "default-proxyclass" + otherPCName = "other-proxyclass" + unreferencedPCName = "unreferenced-proxyclass" + ) + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithIndex(&corev1.Service{}, indexServiceProxyClass, indexProxyClass). + WithIndex(&corev1.Service{}, indexServiceExposed, indexExposed). + WithIndex(&corev1.Service{}, indexServiceType, indexType). + WithObjects( + svc("not-exposed", nil, nil), + svc("exposed-default", map[string]string{AnnotationExpose: "true"}, nil), + svc("exposed-other", map[string]string{AnnotationExpose: "true", LabelAnnotationProxyClass: otherPCName}, nil), + svc("annotated", map[string]string{LabelAnnotationProxyClass: defaultPCName}, nil), + svc("labelled", nil, map[string]string{LabelAnnotationProxyClass: defaultPCName}), + lbSvc("lb-svc", nil, new("tailscale")), + lbSvc("lb-svc-no-class", nil, nil), + lbSvc("lb-svc-other-class", nil, new("other")), + lbSvc("lb-svc-other-pc", map[string]string{LabelAnnotationProxyClass: otherPCName}, nil), + ). + Build() + + zl := zap.Must(zap.NewDevelopment()) + mapFunc := proxyClassHandlerForSvc(fc, zl.Sugar(), defaultPCName, true) + + for _, tc := range []struct { + name string + proxyClassName string + expected []reconcile.Request + }{ + { + name: "default_ProxyClass", + proxyClassName: defaultPCName, + expected: []reconcile.Request{ + {NamespacedName: types.NamespacedName{Namespace: "default", Name: "exposed-default"}}, + {NamespacedName: types.NamespacedName{Namespace: "default", Name: "annotated"}}, + {NamespacedName: types.NamespacedName{Namespace: "default", Name: "labelled"}}, + {NamespacedName: types.NamespacedName{Namespace: "foo", Name: "lb-svc"}}, + {NamespacedName: types.NamespacedName{Namespace: "foo", Name: "lb-svc-no-class"}}, + }, + }, + { + name: "other_ProxyClass", + proxyClassName: otherPCName, + expected: []reconcile.Request{ + {NamespacedName: types.NamespacedName{Namespace: "default", Name: "exposed-other"}}, + {NamespacedName: types.NamespacedName{Namespace: "foo", Name: "lb-svc-other-pc"}}, + }, + }, + { + name: "unreferenced_ProxyClass", + proxyClassName: unreferencedPCName, + expected: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + reqs := mapFunc(t.Context(), &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: tc.proxyClassName, + }, + }) + if len(reqs) != len(tc.expected) { + t.Fatalf("expected %d requests, got %d: %v", len(tc.expected), len(reqs), reqs) + } + for _, expected := range tc.expected { + if !slices.Contains(reqs, expected) { + t.Errorf("expected request for Service %q not found in results: %v", expected.Name, reqs) + } + } + }) + } +} From 5dc32b1213bee7c8ac58c69fda6bddfa56b9afc2 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Fri, 13 Mar 2026 15:11:58 +0000 Subject: [PATCH 42/78] wgengine/magicsock: implement SCION feature support and refactor - Added support for SCION network integration with new feature flags for enabling/disabling SCION in builds. - Introduced new files for SCION-enabled and SCION-disabled configurations, allowing for conditional compilation based on build tags. - Enhanced the SCION connection handling with improved initialization and cleanup methods. - Refactored existing SCION path handling logic to streamline operations and improve maintainability. - Updated feature metadata to include SCION integration details. --- .../buildfeatures/feature_scion_disabled.go | 13 + .../buildfeatures/feature_scion_enabled.go | 13 + feature/featuretags/featuretags.go | 1 + wgengine/magicsock/endpoint.go | 297 +------------- wgengine/magicsock/endpoint_scion.go | 362 ++++++++++++++++++ wgengine/magicsock/magicsock.go | 22 +- wgengine/magicsock/magicsock_scion.go | 53 ++- wgengine/magicsock/magicsock_scion_conn.go | 47 +++ wgengine/magicsock/magicsock_scion_omit.go | 60 +++ wgengine/magicsock/magicsock_scion_test.go | 2 + wgengine/magicsock/scion_bootstrap.go | 21 + wgengine/magicsock/scion_embedded.go | 83 +++- 12 files changed, 647 insertions(+), 327 deletions(-) create mode 100644 feature/buildfeatures/feature_scion_disabled.go create mode 100644 feature/buildfeatures/feature_scion_enabled.go create mode 100644 wgengine/magicsock/endpoint_scion.go create mode 100644 wgengine/magicsock/magicsock_scion_conn.go create mode 100644 wgengine/magicsock/magicsock_scion_omit.go diff --git a/feature/buildfeatures/feature_scion_disabled.go b/feature/buildfeatures/feature_scion_disabled.go new file mode 100644 index 0000000000000..a6cbf2502e01d --- /dev/null +++ b/feature/buildfeatures/feature_scion_disabled.go @@ -0,0 +1,13 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Code generated by gen.go; DO NOT EDIT. + +//go:build ts_omit_scion + +package buildfeatures + +// HasSCION is whether the binary was built with support for modular feature "SCION network integration". +// Specifically, it's whether the binary was NOT built with the "ts_omit_scion" build tag. +// It's a const so it can be used for dead code elimination. +const HasSCION = false diff --git a/feature/buildfeatures/feature_scion_enabled.go b/feature/buildfeatures/feature_scion_enabled.go new file mode 100644 index 0000000000000..24be474ad49ca --- /dev/null +++ b/feature/buildfeatures/feature_scion_enabled.go @@ -0,0 +1,13 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Code generated by gen.go; DO NOT EDIT. + +//go:build !ts_omit_scion + +package buildfeatures + +// HasSCION is whether the binary was built with support for modular feature "SCION network integration". +// Specifically, it's whether the binary was NOT built with the "ts_omit_scion" build tag. +// It's a const so it can be used for dead code elimination. +const HasSCION = true diff --git a/feature/featuretags/featuretags.go b/feature/featuretags/featuretags.go index 4220c02b75fa2..a3b916b0a5724 100644 --- a/feature/featuretags/featuretags.go +++ b/feature/featuretags/featuretags.go @@ -235,6 +235,7 @@ var Features = map[FeatureTag]FeatureMeta{ Desc: "Linux systemd-resolved integration", Deps: []FeatureTag{"dbus"}, }, + "scion": {Sym: "SCION", Desc: "SCION network integration"}, "sdnotify": { Sym: "SDNotify", Desc: "systemd notification support", diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 3203a77b892d6..ec101ba6eb0d5 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -866,27 +866,8 @@ func (de *endpoint) heartbeat() { if de.wantFullPingLocked(now) { de.sendDiscoPingsLocked(now, true) - } else if de.scionState != nil && de.c.pconnSCION != nil && !de.bestAddr.isSCION() { - // Even when the current best path is "good enough" to skip a full ping - // round, heartbeat all SCION paths so they can compete via betterAddr. - // Without this, SCION never gets pinged once a low-latency direct path - // suppresses wantFullPingLocked. - for pk, ps := range de.scionState.paths { - if !ps.lastPing.IsZero() && now.Sub(ps.lastPing) < discoPingInterval { - continue - } - ps.lastPing = now - ps.pingsSent++ - scionEp := epAddr{ - ap: de.scionState.hostAddr, - scionKey: pk, - } - de.startDiscoPingLocked(scionEp, now, pingHeartbeat, 0, nil) - } - } else if de.scionState != nil && de.c.pconnSCION != nil && len(de.scionState.paths) > 1 { - // Probe non-best SCION paths one at a time via round-robin so - // latency data stays fresh for re-evaluation. - de.probeSCIONNonBestLocked(now) + } else { + de.heartbeatSCIONLocked(now) } if de.wantUDPRelayPathDiscoveryLocked(now) { @@ -1048,16 +1029,7 @@ func (de *endpoint) discoPing(res *ipnstate.PingResult, size int, cb func(*ipnst for ep := range de.endpointState { de.startDiscoPingLocked(epAddr{ap: ep}, now, pingCLI, size, resCB) } - // Also ping all SCION paths if available for this peer. - if de.scionState != nil && de.c.pconnSCION != nil { - for pk := range de.scionState.paths { - scionEp := epAddr{ - ap: de.scionState.hostAddr, - scionKey: pk, - } - de.startDiscoPingLocked(scionEp, now, pingCLI, size, resCB) - } - } + de.cliPingSCIONLocked(now, size, resCB) if de.wantUDPRelayPathDiscoveryLocked(now) { de.discoverUDPRelayPathsLocked(now) } @@ -1107,26 +1079,7 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { } var err error if udpAddr.isSCION() { - _, err = de.c.sendSCIONBatch(udpAddr, buffs, offset) - if err != nil { - de.noteBadEndpoint(udpAddr) - // Trigger re-discovery so we don't wait up to 30s for the - // periodic refreshSCIONPaths to fix an expired path. - // discoverSCIONPathAsync self-throttles to once per 5s. - de.mu.Lock() - st := de.scionState - de.mu.Unlock() - if st != nil { - go de.discoverSCIONPathAsync(st.peerIA, st.hostAddr) - } - } else if de.c.metrics != nil { - var txBytes int - for _, b := range buffs { - txBytes += len(b[offset:]) - } - de.c.metrics.outboundPacketsSCIONTotal.Add(int64(len(buffs))) - de.c.metrics.outboundBytesSCIONTotal.Add(int64(txBytes)) - } + err = de.sendSCIONData(udpAddr, buffs, offset) } else if udpAddr.ap.IsValid() { _, err = de.c.sendUDPBatch(udpAddr, buffs, offset) @@ -1243,18 +1196,7 @@ func (de *endpoint) discoPingTimeout(txid stun.TxID) { } de.removeSentDiscoPingLocked(txid, sp, discoPingTimedOut) - // Track consecutive loss for SCION paths and demote if unhealthy. - if sp.to.scionKey.IsSet() && de.scionState != nil { - if ps, ok := de.scionState.paths[sp.to.scionKey]; ok { - ps.consecutiveLoss++ - if ps.consecutiveLoss >= 3 && ps.healthy { - ps.healthy = false - de.c.logf("magicsock: SCION path %d unhealthy for %v (loss: %d)", - sp.to.scionKey, de.publicKey.ShortString(), ps.consecutiveLoss) - de.demoteSCIONPathLocked(sp.to.scionKey) - } - } - } + de.discoPingTimeoutSCIONLocked(sp) } // forgetDiscoPing is called when a ping fails to send. @@ -1266,161 +1208,6 @@ func (de *endpoint) forgetDiscoPing(txid stun.TxID) { } } -// probeSCIONNonBestLocked probes one non-active SCION path per call using -// round-robin ordering. This keeps latency data fresh for paths that aren't -// currently the active path, enabling re-evaluation to detect better options. -// de.mu must be held. -func (de *endpoint) probeSCIONNonBestLocked(now mono.Time) { - if de.scionState == nil { - return - } - - // Collect non-active path keys and sort for deterministic ordering. - var nonBest []scionPathKey - for k := range de.scionState.paths { - if k != de.scionState.activePath { - nonBest = append(nonBest, k) - } - } - if len(nonBest) == 0 { - return - } - slices.SortFunc(nonBest, func(a, b scionPathKey) int { - if a < b { - return -1 - } - if a > b { - return 1 - } - return 0 - }) - - // Pick one via round-robin. - idx := de.scionState.probeRoundRobin % len(nonBest) - de.scionState.probeRoundRobin++ - pk := nonBest[idx] - ps := de.scionState.paths[pk] - - // Rate limit per path. - if !ps.lastPing.IsZero() && now.Sub(ps.lastPing) < discoPingInterval { - return - } - ps.lastPing = now - ps.pingsSent++ - scionEp := epAddr{ - ap: de.scionState.hostAddr, - scionKey: pk, - } - de.startDiscoPingLocked(scionEp, now, pingHeartbeat, 0, nil) -} - -// demoteSCIONPathLocked is called when a SCION path is marked unhealthy. -// It finds the best remaining healthy path by measured latency and switches -// activePath and bestAddr if the demoted path was active/best. -// de.mu must be held. -func (de *endpoint) demoteSCIONPathLocked(demotedKey scionPathKey) { - if de.scionState == nil { - return - } - - // Find best healthy path by measured latency. - var bestKey scionPathKey - var bestLatency time.Duration - for k, ps := range de.scionState.paths { - if k == demotedKey || !ps.healthy { - continue - } - lat := ps.latency() - if !bestKey.IsSet() || lat < bestLatency { - bestKey = k - bestLatency = lat - } - } - - // Only act if the demoted path was the active path. - if de.scionState.activePath != demotedKey { - return - } - - if bestKey.IsSet() { - de.scionState.activePath = bestKey - newAddr := addrQuality{ - epAddr: epAddr{ap: de.scionState.hostAddr, scionKey: bestKey}, - latency: bestLatency, - scionPreferred: de.scionPreferred, - } - de.c.logf("magicsock: SCION path demoted, switching to path %d for %v", bestKey, de.publicKey.ShortString()) - de.setBestAddrLocked(newAddr) - go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, bestKey) - } else { - // No healthy SCION paths remain. Clear SCION bestAddr to fall back. - de.scionState.activePath = 0 - if de.bestAddr.isSCION() { - de.c.logf("magicsock: no healthy SCION paths for %v, clearing bestAddr", de.publicKey.ShortString()) - de.clearBestAddrLocked() - } - } -} - -// reEvalSCIONPathsLocked re-evaluates all SCION paths by measured latency -// after a pong is recorded. Throttled to once per 2 seconds. If a healthier, -// lower-latency path is found, switches bestAddr and activePath. -// de.mu must be held. -func (de *endpoint) reEvalSCIONPathsLocked(now mono.Time) { - if de.scionState == nil || len(de.scionState.paths) < 2 { - return - } - if !de.scionState.lastFullEvalAt.IsZero() && now.Sub(de.scionState.lastFullEvalAt) < 2*time.Second { - return - } - de.scionState.lastFullEvalAt = now - - // Check all paths have at least 1 pong measurement. - for _, ps := range de.scionState.paths { - if ps.pongCount == 0 { - return - } - } - - // Find the healthy path with lowest measured latency. - var bestKey scionPathKey - var bestLatency time.Duration - for k, ps := range de.scionState.paths { - if !ps.healthy { - continue - } - lat := ps.latency() - if !bestKey.IsSet() || lat < bestLatency { - bestKey = k - bestLatency = lat - } - } - - if !bestKey.IsSet() || bestKey == de.scionState.activePath { - return - } - - candidate := addrQuality{ - epAddr: epAddr{ap: de.scionState.hostAddr, scionKey: bestKey}, - latency: bestLatency, - scionPreferred: de.scionPreferred, - } - - if betterAddr(candidate, de.bestAddr) { - de.c.logf("magicsock: SCION re-eval: switching to path %d (latency %v) for %v", - bestKey, bestLatency.Round(time.Millisecond), de.publicKey.ShortString()) - de.debugUpdates.Add(EndpointChange{ - When: time.Now(), - What: "reEvalSCIONPathsLocked-switch", - From: de.bestAddr, - To: candidate, - }) - de.setBestAddrLocked(candidate) - de.scionState.activePath = bestKey - go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, bestKey) - } -} - // discoPingResult represents the result of an attempted disco ping send // operation. type discoPingResult int @@ -1600,20 +1387,7 @@ func (de *endpoint) sendDiscoPingsLocked(now mono.Time, sendCallMeMaybe bool) { de.startDiscoPingLocked(epAddr{ap: ep}, now, pingDiscovery, 0, nil) } - // Also ping all SCION paths if available for this peer. - if de.scionState != nil && de.c.pconnSCION != nil { - for pk, ps := range de.scionState.paths { - if !ps.lastPing.IsZero() && now.Sub(ps.lastPing) < discoPingInterval { - continue - } - ps.lastPing = now - ps.pingsSent++ - scionEp := epAddr{ - ap: de.scionState.hostAddr, - scionKey: pk, - } - de.startDiscoPingLocked(scionEp, now, pingDiscovery, 0, nil) - } + if de.sendDiscoPingsSCIONLocked(now) { sentAny = true } @@ -1768,32 +1542,7 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p de.relayCapable = capVerIsRelayCapable(n.Cap()) - // Check for SCION service advertisement from this peer. - // Extract old SCION keys for cleanup outside de.mu (lock order: c.mu before de.mu). - var oldSCIONKeys []scionPathKey - if peerIA, hostAddr, ok := scionServiceFromPeer(n); ok { - if de.scionState == nil || de.scionState.peerIA != peerIA || de.scionState.hostAddr != hostAddr { - // New or changed SCION address — discover paths asynchronously - // to avoid blocking updateFromNode (which holds the endpoint lock). - if de.c.pconnSCION != nil { - de.c.logf("magicsock: SCION peer %s at %s, discovering paths...", peerIA, hostAddr) - go de.discoverSCIONPathAsync(peerIA, hostAddr) - } else { - de.c.logf("magicsock: peer has SCION (%s) but local SCION not available", peerIA) - } - } - } else if de.scionState != nil { - // Peer no longer advertises SCION. - for k := range de.scionState.paths { - oldSCIONKeys = append(oldSCIONKeys, k) - } - de.scionState = nil - } - - // Check if SCION should be preferred for this peer. - peerSCIONPrefer := n.CapMap().Contains(tailcfg.NodeAttrSCIONPrefer) - selfSCIONPrefer := de.c.self.Valid() && de.c.self.CapMap().Contains(tailcfg.NodeAttrSCIONPrefer) - de.scionPreferred = peerSCIONPrefer && selfSCIONPrefer && de.scionState != nil + oldSCIONKeys := de.updateFromNodeSCIONLocked(n) de.mu.Unlock() @@ -2025,22 +1774,8 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd } // Record latency for SCION paths in per-path probe state. - if !isDerp && src.scionKey.IsSet() { - if de.scionState != nil { - if ps, ok := de.scionState.paths[src.scionKey]; ok { - ps.addPongReply(scionPongReply{ - latency: latency, - pongAt: now, - }) - ps.pongsReceived++ - ps.consecutiveLoss = 0 - if !ps.healthy { - ps.healthy = true - de.c.logf("magicsock: SCION path %d recovered for %v", src.scionKey, de.publicKey.ShortString()) - } - } - de.reEvalSCIONPathsLocked(now) - } + if !isDerp { + de.handlePongSCIONLocked(src, latency, now) } if sp.purpose != pingHeartbeat && sp.purpose != pingHeartbeatForUDPLifetime { @@ -2083,11 +1818,7 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd To: thisPong, }) de.setBestAddrLocked(thisPong) - // Update activePath when switching to a SCION path. - if thisPong.epAddr.scionKey.IsSet() && de.scionState != nil { - de.scionState.activePath = thisPong.epAddr.scionKey - go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, thisPong.epAddr.scionKey) - } + de.handlePongPromoteSCIONLocked(thisPong) } } if de.bestAddr.epAddr == thisPong.epAddr { @@ -2391,13 +2122,7 @@ func (de *endpoint) stopAndReset() { // Extract scionPathKeys before releasing de.mu so we can clean them up // under c.mu afterward (lock order: c.mu before de.mu). - var scionKeys []scionPathKey - if de.scionState != nil { - for k := range de.scionState.paths { - scionKeys = append(scionKeys, k) - } - de.scionState = nil - } + scionKeys := de.stopAndResetSCIONLocked() if closing := de.c.closing.Load(); !closing { if de.isWireguardOnly { diff --git a/wgengine/magicsock/endpoint_scion.go b/wgengine/magicsock/endpoint_scion.go new file mode 100644 index 0000000000000..b57311ab2897d --- /dev/null +++ b/wgengine/magicsock/endpoint_scion.go @@ -0,0 +1,362 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_scion + +package magicsock + +import ( + "cmp" + "slices" + "time" + + "tailscale.com/tailcfg" + "tailscale.com/tstime/mono" +) + +// heartbeatSCIONLocked handles SCION-specific heartbeat logic. +// When the best address is not SCION, it heartbeats all SCION paths so they +// can compete via betterAddr. When the best IS SCION and there are multiple +// paths, it probes non-best paths via round-robin. +// de.mu must be held. +func (de *endpoint) heartbeatSCIONLocked(now mono.Time) { + if de.scionState == nil || de.c.pconnSCION == nil { + return + } + if !de.bestAddr.isSCION() { + // Even when the current best path is "good enough" to skip a full ping + // round, heartbeat all SCION paths so they can compete via betterAddr. + // Without this, SCION never gets pinged once a low-latency direct path + // suppresses wantFullPingLocked. + for pk, ps := range de.scionState.paths { + if !ps.lastPing.IsZero() && now.Sub(ps.lastPing) < discoPingInterval { + continue + } + ps.lastPing = now + ps.pingsSent++ + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: pk, + } + de.startDiscoPingLocked(scionEp, now, pingHeartbeat, 0, nil) + } + } else if len(de.scionState.paths) > 1 { + // Probe non-best SCION paths one at a time via round-robin so + // latency data stays fresh for re-evaluation. + de.probeSCIONNonBestLocked(now) + } +} + +// sendDiscoPingsSCIONLocked pings all SCION paths for this peer during a +// full discovery round. Returns true if SCION is available for this peer. +// de.mu must be held. +func (de *endpoint) sendDiscoPingsSCIONLocked(now mono.Time) bool { + if de.scionState == nil || de.c.pconnSCION == nil { + return false + } + for pk, ps := range de.scionState.paths { + if !ps.lastPing.IsZero() && now.Sub(ps.lastPing) < discoPingInterval { + continue + } + ps.lastPing = now + ps.pingsSent++ + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: pk, + } + de.startDiscoPingLocked(scionEp, now, pingDiscovery, 0, nil) + } + return true +} + +// cliPingSCIONLocked pings all SCION paths when the user runs "tailscale ping". +// de.mu must be held. +func (de *endpoint) cliPingSCIONLocked(now mono.Time, size int, resCB *pingResultAndCallback) { + if de.scionState == nil || de.c.pconnSCION == nil { + return + } + for pk := range de.scionState.paths { + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: pk, + } + de.startDiscoPingLocked(scionEp, now, pingCLI, size, resCB) + } +} + +// discoPingTimeoutSCIONLocked handles disco ping timeout for SCION paths, +// tracking consecutive loss and demoting unhealthy paths. +// de.mu must be held. +func (de *endpoint) discoPingTimeoutSCIONLocked(sp sentPing) { + if !sp.to.scionKey.IsSet() || de.scionState == nil { + return + } + ps, ok := de.scionState.paths[sp.to.scionKey] + if !ok { + return + } + ps.consecutiveLoss++ + if ps.consecutiveLoss >= 3 && ps.healthy { + ps.healthy = false + de.c.logf("magicsock: SCION path %d unhealthy for %v (loss: %d)", + sp.to.scionKey, de.publicKey.ShortString(), ps.consecutiveLoss) + de.demoteSCIONPathLocked(sp.to.scionKey) + } +} + +// handlePongSCIONLocked records a pong measurement for a SCION path and +// triggers re-evaluation of path latencies. +// de.mu must be held. +func (de *endpoint) handlePongSCIONLocked(src epAddr, latency time.Duration, now mono.Time) { + if !src.scionKey.IsSet() || de.scionState == nil { + return + } + if ps, ok := de.scionState.paths[src.scionKey]; ok { + ps.addPongReply(scionPongReply{ + latency: latency, + pongAt: now, + }) + ps.pongsReceived++ + ps.consecutiveLoss = 0 + if !ps.healthy { + ps.healthy = true + de.c.logf("magicsock: SCION path %d recovered for %v", src.scionKey, de.publicKey.ShortString()) + } + } + de.reEvalSCIONPathsLocked(now) +} + +// handlePongPromoteSCIONLocked updates the SCION activePath after bestAddr +// switches to a SCION path via pong promotion. +// de.mu must be held. +func (de *endpoint) handlePongPromoteSCIONLocked(thisPong addrQuality) { + if !thisPong.epAddr.scionKey.IsSet() || de.scionState == nil { + return + } + de.scionState.activePath = thisPong.epAddr.scionKey + go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, thisPong.epAddr.scionKey) +} + +// updateFromNodeSCIONLocked handles the SCION-specific parts of updateFromNode: +// detects new/changed SCION service advertisements, triggers path discovery, +// and computes SCION preference. Returns old SCION path keys that need cleanup +// outside de.mu (lock order: c.mu before de.mu). +// de.mu must be held. +func (de *endpoint) updateFromNodeSCIONLocked(n tailcfg.NodeView) []scionPathKey { + var oldSCIONKeys []scionPathKey + if peerIA, hostAddr, ok := scionServiceFromPeer(n); ok { + if de.scionState == nil || de.scionState.peerIA != peerIA || de.scionState.hostAddr != hostAddr { + // New or changed SCION address — discover paths asynchronously + // to avoid blocking updateFromNode (which holds the endpoint lock). + if de.c.pconnSCION != nil { + de.c.logf("magicsock: SCION peer %s at %s, discovering paths...", peerIA, hostAddr) + go de.discoverSCIONPathAsync(peerIA, hostAddr) + } else { + de.c.logf("magicsock: peer has SCION (%s) but local SCION not available", peerIA) + } + } + } else if de.scionState != nil { + // Peer no longer advertises SCION. + for k := range de.scionState.paths { + oldSCIONKeys = append(oldSCIONKeys, k) + } + de.scionState = nil + } + + // Check if SCION should be preferred for this peer. + peerSCIONPrefer := n.CapMap().Contains(tailcfg.NodeAttrSCIONPrefer) + selfSCIONPrefer := de.c.self.Valid() && de.c.self.CapMap().Contains(tailcfg.NodeAttrSCIONPrefer) + de.scionPreferred = peerSCIONPrefer && selfSCIONPrefer && de.scionState != nil + + return oldSCIONKeys +} + +// stopAndResetSCIONLocked extracts SCION path keys for cleanup before the +// endpoint state is cleared. Returns keys that need cleanup outside de.mu. +// de.mu must be held. +func (de *endpoint) stopAndResetSCIONLocked() []scionPathKey { + if de.scionState == nil { + return nil + } + var keys []scionPathKey + for k := range de.scionState.paths { + keys = append(keys, k) + } + de.scionState = nil + return keys +} + +// sendSCIONData sends WireGuard data over a SCION path, handling error +// recovery (re-discovery) and metrics. Called from send() after de.mu is released. +func (de *endpoint) sendSCIONData(udpAddr epAddr, buffs [][]byte, offset int) error { + _, err := de.c.sendSCIONBatch(udpAddr, buffs, offset) + if err != nil { + de.noteBadEndpoint(udpAddr) + // Trigger re-discovery so we don't wait up to 30s for the + // periodic refreshSCIONPaths to fix an expired path. + // discoverSCIONPathAsync self-throttles to once per 5s. + de.mu.Lock() + st := de.scionState + de.mu.Unlock() + if st != nil { + go de.discoverSCIONPathAsync(st.peerIA, st.hostAddr) + } + } else if de.c.metrics != nil { + var txBytes int + for _, b := range buffs { + txBytes += len(b[offset:]) + } + de.c.metrics.outboundPacketsSCIONTotal.Add(int64(len(buffs))) + de.c.metrics.outboundBytesSCIONTotal.Add(int64(txBytes)) + } + return err +} + +// probeSCIONNonBestLocked probes one non-active SCION path per call using +// round-robin ordering. This keeps latency data fresh for paths that aren't +// currently the active path, enabling re-evaluation to detect better options. +// de.mu must be held. +func (de *endpoint) probeSCIONNonBestLocked(now mono.Time) { + if de.scionState == nil { + return + } + + // Collect non-active path keys and sort for deterministic ordering. + var nonBest []scionPathKey + for k := range de.scionState.paths { + if k != de.scionState.activePath { + nonBest = append(nonBest, k) + } + } + if len(nonBest) == 0 { + return + } + slices.SortFunc(nonBest, func(a, b scionPathKey) int { + return cmp.Compare(a, b) + }) + + // Pick one via round-robin. + idx := de.scionState.probeRoundRobin % len(nonBest) + de.scionState.probeRoundRobin++ + pk := nonBest[idx] + ps := de.scionState.paths[pk] + + // Rate limit per path. + if !ps.lastPing.IsZero() && now.Sub(ps.lastPing) < discoPingInterval { + return + } + ps.lastPing = now + ps.pingsSent++ + scionEp := epAddr{ + ap: de.scionState.hostAddr, + scionKey: pk, + } + de.startDiscoPingLocked(scionEp, now, pingHeartbeat, 0, nil) +} + +// demoteSCIONPathLocked is called when a SCION path is marked unhealthy. +// It finds the best remaining healthy path by measured latency and switches +// activePath and bestAddr if the demoted path was active/best. +// de.mu must be held. +func (de *endpoint) demoteSCIONPathLocked(demotedKey scionPathKey) { + if de.scionState == nil { + return + } + + // Find best healthy path by measured latency. + var bestKey scionPathKey + var bestLatency time.Duration + for k, ps := range de.scionState.paths { + if k == demotedKey || !ps.healthy { + continue + } + lat := ps.latency() + if !bestKey.IsSet() || lat < bestLatency { + bestKey = k + bestLatency = lat + } + } + + // Only act if the demoted path was the active path. + if de.scionState.activePath != demotedKey { + return + } + + if bestKey.IsSet() { + de.scionState.activePath = bestKey + newAddr := addrQuality{ + epAddr: epAddr{ap: de.scionState.hostAddr, scionKey: bestKey}, + latency: bestLatency, + scionPreferred: de.scionPreferred, + } + de.c.logf("magicsock: SCION path demoted, switching to path %d for %v", bestKey, de.publicKey.ShortString()) + de.setBestAddrLocked(newAddr) + go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, bestKey) + } else { + // No healthy SCION paths remain. Clear SCION bestAddr to fall back. + de.scionState.activePath = 0 + if de.bestAddr.isSCION() { + de.c.logf("magicsock: no healthy SCION paths for %v, clearing bestAddr", de.publicKey.ShortString()) + de.clearBestAddrLocked() + } + } +} + +// reEvalSCIONPathsLocked re-evaluates all SCION paths by measured latency +// after a pong is recorded. Throttled to once per 2 seconds. If a healthier, +// lower-latency path is found, switches bestAddr and activePath. +// de.mu must be held. +func (de *endpoint) reEvalSCIONPathsLocked(now mono.Time) { + if de.scionState == nil || len(de.scionState.paths) < 2 { + return + } + if !de.scionState.lastFullEvalAt.IsZero() && now.Sub(de.scionState.lastFullEvalAt) < 2*time.Second { + return + } + de.scionState.lastFullEvalAt = now + + // Check all paths have at least 1 pong measurement. + for _, ps := range de.scionState.paths { + if ps.pongCount == 0 { + return + } + } + + // Find the healthy path with lowest measured latency. + var bestKey scionPathKey + var bestLatency time.Duration + for k, ps := range de.scionState.paths { + if !ps.healthy { + continue + } + lat := ps.latency() + if !bestKey.IsSet() || lat < bestLatency { + bestKey = k + bestLatency = lat + } + } + + if !bestKey.IsSet() || bestKey == de.scionState.activePath { + return + } + + candidate := addrQuality{ + epAddr: epAddr{ap: de.scionState.hostAddr, scionKey: bestKey}, + latency: bestLatency, + scionPreferred: de.scionPreferred, + } + + if betterAddr(candidate, de.bestAddr) { + de.c.logf("magicsock: SCION re-eval: switching to path %d (latency %v) for %v", + bestKey, bestLatency.Round(time.Millisecond), de.publicKey.ShortString()) + de.debugUpdates.Add(EndpointChange{ + When: time.Now(), + What: "reEvalSCIONPathsLocked-switch", + From: de.bestAddr, + To: candidate, + }) + de.setBestAddrLocked(candidate) + de.scionState.activePath = bestKey + go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, bestKey) + } +} diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 170830e43261e..07f7b248040b7 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3347,11 +3347,7 @@ func (c *connBind) Close() error { if c.closeDisco6 != nil { c.closeDisco6.Close() } - if c.pconnSCION != nil { - // Set an immediate read deadline to unblock receiveSCION. - // We don't close the SCION socket here; Conn.Close handles that. - c.pconnSCION.conn.SetReadDeadline(time.Now()) - } + c.closeSCIONBindLocked() // Send an empty read result to unblock receiveDERP, // which will then check connBind.Closed. // connBind.Closed takes c.mu, but c.derpRecvCh is buffered. @@ -3402,9 +3398,7 @@ func (c *Conn) Close() error { // They will frequently have been closed already by a call to connBind.Close. c.pconn6.Close() c.pconn4.Close() - if c.pconnSCION != nil { - c.pconnSCION.close() - } + c.closeSCIONLocked() if c.closeDisco4 != nil { c.closeDisco4.Close() } @@ -3653,17 +3647,7 @@ func (c *Conn) rebind(curPortFate currentPortFate) error { } c.UpdatePMTUD() - // Try to set up SCION if not already connected. - if c.pconnSCION == nil { - sc, err := trySCIONConnect(c.connCtx) - if err != nil { - c.logf("magicsock: SCION not available: %v", err) - } else { - c.logf("magicsock: SCION available, local IA: %s", sc.localIA) - c.pconnSCION = sc - go c.refreshSCIONPaths() - } - } + c.initSCIONLocked(c.connCtx) return nil } diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 8f8e7e7ec6834..386d6d1b0336b 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -1,6 +1,8 @@ // Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_scion + package magicsock import ( @@ -30,9 +32,12 @@ import ( "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" "tailscale.com/envknob" + "tailscale.com/net/netmon" + "tailscale.com/net/netns" "tailscale.com/tailcfg" "tailscale.com/tstime/mono" "tailscale.com/types/key" + "tailscale.com/types/logger" "tailscale.com/util/mak" ) @@ -493,6 +498,8 @@ func buildSCIONReplyAddr(srcIA addr.IA, srcHostAddr netip.AddrPort, rawPathBytes // scionBatchRW abstracts ipv4.PacketConn and ipv6.PacketConn for // batch I/O. Both have identical ReadBatch/WriteBatch signatures // since ipv4.Message and ipv6.Message are the same type (socket.Message). +// On non-Linux platforms, ReadBatch/WriteBatch fall back to per-message +// sendto/recvfrom (golang.org/x/net handles this internally). type scionBatchRW interface { ReadBatch([]ipv4.Message, int) (int, error) WriteBatch([]ipv4.Message, int) (int, error) @@ -646,12 +653,12 @@ var forceEmbeddedSCION = envknob.RegisterBool("TS_SCION_EMBEDDED") // 5. Hardcoded bootstrap URLs (if any) // // Returns nil if SCION is not available via any method. -func trySCIONConnect(ctx context.Context) (*scionConn, error) { +func trySCIONConnect(ctx context.Context, logf logger.Logf, netMon *netmon.Monitor) (*scionConn, error) { var externalErr error // Step 1: Try external daemon (unless forced embedded). if !forceEmbeddedSCION() { - sc, err := tryExternalDaemon(ctx) + sc, err := tryExternalDaemon(ctx, logf, netMon) if err == nil { return sc, nil } @@ -661,7 +668,7 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { // Step 2: Try embedded with existing local topology file. topoPath := scionTopologyPath() if _, err := os.Stat(topoPath); err == nil { - sc, err := tryEmbeddedDaemon(ctx, topoPath) + sc, err := tryEmbeddedDaemon(ctx, topoPath, logf, netMon) if err == nil { return sc, nil } @@ -670,6 +677,12 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { // Steps 3-5: Try bootstrap from URLs (explicit, DNS-discovered, hardcoded). stateDir := scionStateDir() + if stateDir == "" { + if externalErr != nil { + return nil, fmt.Errorf("external daemon: %w; embedded: no state directory available", externalErr) + } + return nil, fmt.Errorf("SCION not available: no external daemon, no topology file, no state directory for bootstrap") + } for _, url := range bootstrapURLs(ctx) { if err := bootstrapSCION(ctx, url, stateDir); err != nil { continue @@ -678,7 +691,7 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { if _, err := os.Stat(bootstrappedTopo); err != nil { continue } - sc, err := tryEmbeddedDaemon(ctx, bootstrappedTopo) + sc, err := tryEmbeddedDaemon(ctx, bootstrappedTopo, logf, netMon) if err == nil { return sc, nil } @@ -692,7 +705,7 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { // tryExternalDaemon attempts to connect to an external SCION daemon and set up // a SCION listener. This is the original trySCIONConnect behavior. -func tryExternalDaemon(ctx context.Context) (*scionConn, error) { +func tryExternalDaemon(ctx context.Context, logf logger.Logf, netMon *netmon.Monitor) (*scionConn, error) { daemonAddr := scionDaemonAddr() svc := daemon.Service{Address: daemonAddr} conn, err := svc.Connect(ctx) @@ -720,7 +733,7 @@ func tryExternalDaemon(ctx context.Context) (*scionConn, error) { } } - sc, err := finishSCIONConnect(ctx, conn, topo) + sc, err := finishSCIONConnect(ctx, conn, topo, logf, netMon) if err != nil { conn.Close() return nil, err @@ -786,7 +799,7 @@ func neighborIAFromTopology(topoPath string) (addr.IA, bool) { // finishSCIONConnect completes the SCION connection setup given a // daemon.Connector (for path queries) and snet.Topology (for local info). // This is shared between the external daemon and embedded connector paths. -func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo snet.Topology) (*scionConn, error) { +func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo snet.Topology, logf logger.Logf, netMon *netmon.Monitor) (*scionConn, error) { localIA, err := connector.LocalIA(ctx) if err != nil { return nil, fmt.Errorf("querying local IA: %w", err) @@ -822,10 +835,28 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn if pc, ok := pconn.(*snet.SCIONPacketConn); ok { underlayConn = pc.Conn if err := pc.SetReadBuffer(socketBufferSize); err != nil { - fmt.Fprintf(os.Stderr, "magicsock: SCION: failed to set read buffer to %d: %v\n", socketBufferSize, err) + logf("magicsock: SCION: failed to set read buffer to %d: %v", socketBufferSize, err) } if err := pc.SetWriteBuffer(socketBufferSize); err != nil { - fmt.Fprintf(os.Stderr, "magicsock: SCION: failed to set write buffer to %d: %v\n", socketBufferSize, err) + logf("magicsock: SCION: failed to set write buffer to %d: %v", socketBufferSize, err) + } + } + + // Apply platform-specific socket options (SO_MARK on Linux, + // VpnService.protect on Android, IP_BOUND_IF on macOS) to + // prevent the SCION underlay socket from routing through the + // VPN tunnel, which would cause loops. + if underlayConn != nil { + rawConn, err := underlayConn.SyscallConn() + if err == nil { + lc := netns.Listener(logf, netMon) + if lc.Control != nil { + if err := lc.Control("udp", underlayConn.LocalAddr().String(), rawConn); err != nil { + logf("magicsock: SCION: netns control: %v", err) + } + } + } else { + logf("magicsock: SCION: SyscallConn: %v", err) } } @@ -1402,7 +1433,7 @@ func (c *Conn) reconnectSCION() { } c.pconnSCION = nil - newSC, err := trySCIONConnect(c.connCtx) + newSC, err := trySCIONConnect(c.connCtx, c.logf, c.netMon) if err != nil { c.logf("magicsock: SCION reconnect failed: %v", err) // Reset the receive timestamp so we retry after scionReconnectThreshold. @@ -1429,7 +1460,7 @@ func (c *Conn) retrySCIONConnect() { if c.pconnSCION != nil { return // another goroutine beat us to it } - newSC, err := trySCIONConnect(c.connCtx) + newSC, err := trySCIONConnect(c.connCtx, c.logf, c.netMon) if err != nil { c.logf("magicsock: SCION reconnect retry failed: %v", err) return diff --git a/wgengine/magicsock/magicsock_scion_conn.go b/wgengine/magicsock/magicsock_scion_conn.go new file mode 100644 index 0000000000000..2e99b4205a6c5 --- /dev/null +++ b/wgengine/magicsock/magicsock_scion_conn.go @@ -0,0 +1,47 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_scion + +package magicsock + +import ( + "context" + "time" +) + +// initSCIONLocked tries to set up a SCION connection if not already connected. +// On success, stores the scionConn and starts the background path refresher. +// c.mu must be held. +func (c *Conn) initSCIONLocked(ctx context.Context) { + if c.pconnSCION != nil { + return + } + sc, err := trySCIONConnect(ctx, c.logf, c.netMon) + if err != nil { + c.logf("magicsock: SCION not available: %v", err) + return + } + c.logf("magicsock: SCION available, local IA: %s", sc.localIA) + c.pconnSCION = sc + go c.refreshSCIONPaths() +} + +// closeSCIONLocked closes the SCION connection if open. +// c.mu must be held. +func (c *Conn) closeSCIONLocked() { + if c.pconnSCION != nil { + c.pconnSCION.close() + } +} + +// closeSCIONBindLocked sets an immediate read deadline on the SCION socket +// to unblock receiveSCION, without closing it. Called from connBind.Close. +// c.mu must be held (via connBind.mu). +func (c *Conn) closeSCIONBindLocked() { + if c.pconnSCION != nil { + // Set an immediate read deadline to unblock receiveSCION. + // We don't close the SCION socket here; Conn.Close handles that. + c.pconnSCION.conn.SetReadDeadline(time.Now()) + } +} diff --git a/wgengine/magicsock/magicsock_scion_omit.go b/wgengine/magicsock/magicsock_scion_omit.go new file mode 100644 index 0000000000000..8ffabdc28bf4a --- /dev/null +++ b/wgengine/magicsock/magicsock_scion_omit.go @@ -0,0 +1,60 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build ts_omit_scion + +package magicsock + +import ( + "context" + "time" + + wgconn "github.com/tailscale/wireguard-go/conn" + "tailscale.com/tailcfg" + "tailscale.com/tstime/mono" +) + +// Stub types for ts_omit_scion builds. + +type scionPathKey uint32 + +func (k scionPathKey) IsSet() bool { return false } + +type scionConn struct{} + +func (sc *scionConn) close() error { return nil } + +type scionPathInfo struct{} +type scionAddrKey struct{} +type scionEndpointState struct{} + +// Stub Conn methods. + +func (c *Conn) initSCIONLocked(_ context.Context) {} +func (c *Conn) closeSCIONLocked() {} +func (c *Conn) closeSCIONBindLocked() {} +func (c *Conn) receiveSCION(_ [][]byte, _ []int, _ []wgconn.Endpoint) (int, error) { return 0, nil } +func (c *Conn) sendSCION(_ scionPathKey, _ []byte) (bool, error) { return false, nil } +func (c *Conn) unregisterSCIONPath(_ scionPathKey) {} + +// Stub endpoint methods. + +func (de *endpoint) heartbeatSCIONLocked(_ mono.Time) {} +func (de *endpoint) sendDiscoPingsSCIONLocked(_ mono.Time) bool { return false } +func (de *endpoint) cliPingSCIONLocked(_ mono.Time, _ int, _ *pingResultAndCallback) {} +func (de *endpoint) discoPingTimeoutSCIONLocked(_ sentPing) {} +func (de *endpoint) handlePongSCIONLocked(_ epAddr, _ time.Duration, _ mono.Time) {} +func (de *endpoint) handlePongPromoteSCIONLocked(_ addrQuality) {} +func (de *endpoint) updateFromNodeSCIONLocked(_ tailcfg.NodeView) []scionPathKey { return nil } +func (de *endpoint) stopAndResetSCIONLocked() []scionPathKey { return nil } +func (de *endpoint) sendSCIONData(_ epAddr, _ [][]byte, _ int) error { return nil } + +// SCIONService returns false when SCION is omitted. +func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { return tailcfg.Service{}, false } + +// Stub standalone functions used by betterAddr in endpoint.go. + +var preferSCION = func() bool { return false } + +func scionPreferenceBonus() int { return 0 } +func scionDiversityThreshold() time.Duration { return 0 } diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index c13b493995546..d3c426da99169 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -1,6 +1,8 @@ // Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_scion + package magicsock import ( diff --git a/wgengine/magicsock/scion_bootstrap.go b/wgengine/magicsock/scion_bootstrap.go index 5be0c6f448d5e..e3fd3d34c671d 100644 --- a/wgengine/magicsock/scion_bootstrap.go +++ b/wgengine/magicsock/scion_bootstrap.go @@ -1,6 +1,8 @@ // Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_scion + package magicsock import ( @@ -12,6 +14,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "strings" "time" @@ -149,6 +152,9 @@ func lookupDiscoveryPort(ctx context.Context, r *net.Resolver, domain string) (s // localSearchDomain returns the first search domain from the system's DNS // configuration, using Tailscale's cross-platform resolv.conf parser. func localSearchDomain() (string, error) { + if runtime.GOOS == "windows" || runtime.GOOS == "android" { + return localSearchDomainFromHostname() + } cfg, err := resolvconffile.ParseFile(resolvconffile.Path) if err != nil { return "", err @@ -159,6 +165,21 @@ func localSearchDomain() (string, error) { return "", nil } +// localSearchDomainFromHostname infers the search domain from the +// system hostname. Used on platforms without resolv.conf. +// Note: on Windows, os.Hostname() typically returns a short NetBIOS name +// without a domain suffix, so this will usually return an error. +func localSearchDomainFromHostname() (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", err + } + if i := strings.IndexByte(hostname, '.'); i >= 0 { + return hostname[i+1:], nil + } + return "", fmt.Errorf("no search domain found") +} + // httpGet performs an HTTP GET request and returns the response body. func httpGet(ctx context.Context, client *http.Client, url string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) diff --git a/wgengine/magicsock/scion_embedded.go b/wgengine/magicsock/scion_embedded.go index e9491c9fe27ea..0262b288f9ea3 100644 --- a/wgengine/magicsock/scion_embedded.go +++ b/wgengine/magicsock/scion_embedded.go @@ -1,6 +1,8 @@ // Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_scion + package magicsock import ( @@ -10,6 +12,7 @@ import ( "net/netip" "os" "path/filepath" + "runtime" "github.com/scionproto/scion/daemon/config" "github.com/scionproto/scion/daemon/fetcher" @@ -30,8 +33,14 @@ import ( "github.com/scionproto/scion/private/storage" "github.com/scionproto/scion/private/topology" "github.com/scionproto/scion/private/trust" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "tailscale.com/net/netmon" + "tailscale.com/net/netns" "tailscale.com/paths" + "tailscale.com/types/logger" ) // embeddedConnector implements daemon.Connector using an embedded topology @@ -142,7 +151,7 @@ func (ec *embeddedConnector) Close() error { // newEmbeddedConnector creates a new embeddedConnector from a topology file. // It wires up the path fetcher pipeline following the daemon's own assembly // (daemon/cmd/daemon/main.go), but without trust verification (Phase 1). -func newEmbeddedConnector(ctx context.Context, topoPath, stateDir string) (*embeddedConnector, error) { +func newEmbeddedConnector(ctx context.Context, topoPath, stateDir string, logf logger.Logf, netMon *netmon.Monitor) (*embeddedConnector, error) { // 1. Load topology. topo, err := topology.NewLoader(topology.LoaderCfg{ File: topoPath, @@ -173,8 +182,10 @@ func newEmbeddedConnector(ctx context.Context, topoPath, stateDir string) (*embe revCache := storage.NewRevocationStorage() - // 3. Create gRPC dialer that resolves CS addresses from the topology. - dialer := &libgrpc.TCPDialer{ + // 3. Create gRPC dialer that resolves CS addresses from the topology, + // using netns-aware TCP connections for cross-platform compatibility + // (SO_MARK on Linux, VpnService.protect on Android, IP_BOUND_IF on macOS). + dialer := &netnsTCPDialer{ SvcResolver: func(dst addr.SVC) []resolver.Address { targets := []resolver.Address{} for _, entry := range topo.ControlServiceAddresses() { @@ -182,6 +193,7 @@ func newEmbeddedConnector(ctx context.Context, topoPath, stateDir string) (*embe } return targets }, + NetDialer: netns.NewDialer(logf, netMon).DialContext, } // 4. Create the segment fetcher requester (gRPC to local CS). @@ -248,6 +260,45 @@ func (endHostInspector) HasAttributes(_ context.Context, _ addr.IA, _ trust.Attr return false, nil } +// netnsTCPDialer implements libgrpc.Dialer with netns-aware TCP connections +// for cross-platform socket control (SO_MARK, VpnService.protect, IP_BOUND_IF). +type netnsTCPDialer struct { + SvcResolver func(addr.SVC) []resolver.Address + NetDialer func(ctx context.Context, network, address string) (net.Conn, error) +} + +// Compile-time interface check. +var _ libgrpc.Dialer = (*netnsTCPDialer)(nil) + +func (d *netnsTCPDialer) Dial(ctx context.Context, dst net.Addr) (*grpc.ClientConn, error) { + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return d.NetDialer(ctx, "tcp", addr) + }), + libgrpc.UnaryClientInterceptor(), + libgrpc.StreamClientInterceptor(), + } + + if v, ok := dst.(*snet.SVCAddr); ok { + targets := d.SvcResolver(v.SVC) + if len(targets) == 0 { + return nil, serrors.New("could not resolve", "svc", v.SVC) + } + r := manual.NewBuilderWithScheme("svc") + r.InitialState(resolver.State{Addresses: targets}) + opts = append(opts, + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + grpc.WithResolvers(r), + ) + //nolint:staticcheck // grpc.DialContext is used by scionproto v0.14.0 + return grpc.DialContext(ctx, r.Scheme()+":///"+v.SVC.BaseString(), opts...) + } + + //nolint:staticcheck // grpc.DialContext is used by scionproto v0.14.0 + return grpc.DialContext(ctx, dst.String(), opts...) +} + // scionTopologyPath returns the path to the SCION topology file, checking // TS_SCION_TOPOLOGY first, then the platform's SCION config directory // (/etc/scion/ on Linux), then a "scion" subdirectory under the tailscaled @@ -256,10 +307,11 @@ func scionTopologyPath() string { if p := os.Getenv("TS_SCION_TOPOLOGY"); p != "" { return p } - // Standard SCION installation path (Linux/Unix convention from scionproto). - const defaultSCIONTopology = "/etc/scion/topology.json" - if _, err := os.Stat(defaultSCIONTopology); err == nil { - return defaultSCIONTopology + if runtime.GOOS == "linux" { + const defaultSCIONTopology = "/etc/scion/topology.json" + if _, err := os.Stat(defaultSCIONTopology); err == nil { + return defaultSCIONTopology + } } // Bootstrapped topology under the tailscaled state directory. return filepath.Join(paths.DefaultTailscaledStateDir(), "scion", "topology.json") @@ -272,18 +324,27 @@ func scionStateDir() string { if d := os.Getenv("TS_SCION_STATE_DIR"); d != "" { return d } - return filepath.Join(paths.DefaultTailscaledStateDir(), "scion") + base := paths.DefaultTailscaledStateDir() + if base == "" || base == "." { + if appDir := paths.AppSharedDir.Load(); appDir != "" { + base = appDir + } + } + if base == "" || base == "." { + return "" + } + return filepath.Join(base, "scion") } // tryEmbeddedDaemon attempts to set up a SCION connection using the embedded // connector with the given topology file. This mirrors trySCIONConnect but // uses the embedded connector instead of an external daemon. -func tryEmbeddedDaemon(ctx context.Context, topoPath string) (*scionConn, error) { +func tryEmbeddedDaemon(ctx context.Context, topoPath string, logf logger.Logf, netMon *netmon.Monitor) (*scionConn, error) { stateDir := scionStateDir() - ec, err := newEmbeddedConnector(ctx, topoPath, stateDir) + ec, err := newEmbeddedConnector(ctx, topoPath, stateDir, logf, netMon) if err != nil { return nil, fmt.Errorf("creating embedded connector: %w", err) } - return finishSCIONConnect(ctx, ec, ec.snetTopology()) + return finishSCIONConnect(ctx, ec, ec.snetTopology(), logf, netMon) } From e1fc208a140cad701079e7ef96d11b37bb0601a4 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Fri, 13 Mar 2026 17:21:08 +0000 Subject: [PATCH 43/78] wgengine/magicsock: refactor SCION environment variable handling - migrate to envknob --- wgengine/magicsock/magicsock_scion.go | 17 ++++++++++++----- wgengine/magicsock/scion_embedded.go | 10 ++++++++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 386d6d1b0336b..f354d44ec462e 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -50,6 +50,12 @@ var debugSCIONPreference = envknob.RegisterInt("TS_SCION_PREFERENCE") // Other paths are only used if no SCION path is available. var preferSCION = envknob.RegisterBool("TS_PREFER_SCION") +var ( + scionDaemonAddress = envknob.RegisterString("SCION_DAEMON_ADDRESS") + scionPort = envknob.RegisterString("TS_SCION_PORT") + scionListenAddrEnv = envknob.RegisterString("TS_SCION_LISTEN_ADDR") +) + // scionPreferenceBonus returns the betterAddr points bonus for SCION paths. // Returns the value of TS_SCION_PREFERENCE if set, otherwise defaults to 15. func scionPreferenceBonus() int { @@ -562,7 +568,7 @@ func (sc *scionConn) readFrom(b []byte) (int, *snet.UDPAddr, error) { // scionDaemonAddr returns the SCION daemon address to use, checking the // environment variable first, then falling back to the default socket. func scionDaemonAddr() string { - if a := os.Getenv("SCION_DAEMON_ADDRESS"); a != "" { + if a := scionDaemonAddress(); a != "" { return a } return daemon.DefaultAPIAddress @@ -572,7 +578,7 @@ func scionDaemonAddr() string { // environment variable first, then falling back to 0 (auto-select from the // topology's dispatched port range). func scionListenPort() uint16 { - if p := os.Getenv("TS_SCION_PORT"); p != "" { + if p := scionPort(); p != "" { var v int if _, err := fmt.Sscanf(p, "%d", &v); err == nil && v > 0 && v <= 65535 { return uint16(v) @@ -630,7 +636,7 @@ func scionResolveLocalIP(ctx context.Context, connector daemon.Connector) netip. // Otherwise resolves the local IP from the topology's BR internal addresses. func scionListenAddr(ctx context.Context, connector daemon.Connector) *net.UDPAddr { port := scionListenPort() - if a := os.Getenv("TS_SCION_LISTEN_ADDR"); a != "" { + if a := scionListenAddrEnv(); a != "" { ip := net.ParseIP(a) if ip != nil { return &net.UDPAddr{IP: ip, Port: int(port)} @@ -683,8 +689,9 @@ func trySCIONConnect(ctx context.Context, logf logger.Logf, netMon *netmon.Monit } return nil, fmt.Errorf("SCION not available: no external daemon, no topology file, no state directory for bootstrap") } - for _, url := range bootstrapURLs(ctx) { - if err := bootstrapSCION(ctx, url, stateDir); err != nil { + for _, url := range bootstrapURLs(ctx, logf) { + if err := bootstrapSCION(ctx, logf, url, stateDir); err != nil { + logf("scion: bootstrap from %s failed: %v", url, err) continue } bootstrappedTopo := filepath.Join(stateDir, "topology.json") diff --git a/wgengine/magicsock/scion_embedded.go b/wgengine/magicsock/scion_embedded.go index 0262b288f9ea3..e8863130a762a 100644 --- a/wgengine/magicsock/scion_embedded.go +++ b/wgengine/magicsock/scion_embedded.go @@ -37,12 +37,18 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" + "tailscale.com/envknob" "tailscale.com/net/netmon" "tailscale.com/net/netns" "tailscale.com/paths" "tailscale.com/types/logger" ) +var ( + scionTopology = envknob.RegisterString("TS_SCION_TOPOLOGY") + scionStateDirEnv = envknob.RegisterString("TS_SCION_STATE_DIR") +) + // embeddedConnector implements daemon.Connector using an embedded topology // loader and path fetcher, eliminating the need for an external SCION daemon // process. @@ -304,7 +310,7 @@ func (d *netnsTCPDialer) Dial(ctx context.Context, dst net.Addr) (*grpc.ClientCo // (/etc/scion/ on Linux), then a "scion" subdirectory under the tailscaled // state directory (for bootstrapped topologies). func scionTopologyPath() string { - if p := os.Getenv("TS_SCION_TOPOLOGY"); p != "" { + if p := scionTopology(); p != "" { return p } if runtime.GOOS == "linux" { @@ -321,7 +327,7 @@ func scionTopologyPath() string { // checking TS_SCION_STATE_DIR first, then falling back to a "scion" // subdirectory under the platform's default tailscaled state directory. func scionStateDir() string { - if d := os.Getenv("TS_SCION_STATE_DIR"); d != "" { + if d := scionStateDirEnv(); d != "" { return d } base := paths.DefaultTailscaledStateDir() From 660a4608d2ccb4508af71d888454b333e5eb501f Mon Sep 17 00:00:00 2001 From: George Jones Date: Fri, 13 Mar 2026 13:26:08 -0400 Subject: [PATCH 44/78] feature/conn25: Update ConnectorTransitIPRequest handling (#18979) Changed the mapping to store the transit IPs to be indexed by peer IP rather than NodeID because the data path only has access to the peer's IP. This change means that IPv4 transit IPs need to be indexed by the peer's IPv4 address, and IPv6 transit IPs need to be indexed by the peer's IPv6 address. It is an error if the peer does not have an address of the same family as the transit IP. It is also an error if the transit and destination IP families do not match. Added a check to ensure that the TransitIPRequest.App matches a configured app on the connector. Added additional TransitIPResponse codes to identify the new errors and change the exsting use of the Other code to use it's own specific code. Added logging for the error cases, since they generally indicate that a peer has constructed a bad request or that there is a config mismatch between the peer and the local netmap. Added a test framework for handleConnectorTransitIPRequest and moved the existing tests into the framework and added new tests. Fixes tailscale/corp#37143 Signed-off-by: George Jones --- feature/conn25/conn25.go | 97 +++++-- feature/conn25/conn25_test.go | 489 +++++++++++++++++++++++----------- 2 files changed, 409 insertions(+), 177 deletions(-) diff --git a/feature/conn25/conn25.go b/feature/conn25/conn25.go index b5d0dc9dfe155..5318d2bdde3ed 100644 --- a/feature/conn25/conn25.go +++ b/feature/conn25/conn25.go @@ -26,6 +26,7 @@ import ( "tailscale.com/ipn/ipnext" "tailscale.com/ipn/ipnlocal" "tailscale.com/net/dns" + "tailscale.com/net/tsaddr" "tailscale.com/tailcfg" "tailscale.com/types/appctype" "tailscale.com/types/logger" @@ -131,7 +132,7 @@ func (e *extension) handleConnectorTransitIP(h ipnlocal.PeerAPIHandler, w http.R http.Error(w, "Error decoding JSON", http.StatusBadRequest) return } - resp := e.conn25.handleConnectorTransitIPRequest(h.Peer().ID(), req) + resp := e.conn25.handleConnectorTransitIPRequest(h.Peer(), req) bs, err := json.Marshal(resp) if err != nil { http.Error(w, "Error encoding JSON", http.StatusInternalServerError) @@ -248,47 +249,94 @@ func (c *Conn25) mapDNSResponse(buf []byte) []byte { } const dupeTransitIPMessage = "Duplicate transit address in ConnectorTransitIPRequest" +const noMatchingPeerIPFamilyMessage = "No peer IP found with matching IP family" +const addrFamilyMismatchMessage = "Transit and Destination addresses must have matching IP family" +const unknownAppNameMessage = "The App name in the request does not match a configured App" + +// handleConnectorTransitIPRequest creates a ConnectorTransitIPResponse in response +// to a ConnectorTransitIPRequest. It updates the connectors mapping of +// TransitIP->DestinationIP per peer (using the Peer's IP that matches the address +// family of the transitIP). If a peer has stored this mapping in the connector, +// Conn25 will route traffic to TransitIPs to DestinationIPs for that peer. +func (c *Conn25) handleConnectorTransitIPRequest(n tailcfg.NodeView, ctipr ConnectorTransitIPRequest) ConnectorTransitIPResponse { + var peerIPv4, peerIPv6 netip.Addr + for _, ip := range n.Addresses().All() { + if !ip.IsSingleIP() || !tsaddr.IsTailscaleIP(ip.Addr()) { + continue + } + if ip.Addr().Is4() && !peerIPv4.IsValid() { + peerIPv4 = ip.Addr() + } else if ip.Addr().Is6() && !peerIPv6.IsValid() { + peerIPv6 = ip.Addr() + } + } -// handleConnectorTransitIPRequest creates a ConnectorTransitIPResponse in response to a ConnectorTransitIPRequest. -// It updates the connectors mapping of TransitIP->DestinationIP per peer (tailcfg.NodeID). -// If a peer has stored this mapping in the connector Conn25 will route traffic to TransitIPs to DestinationIPs for that peer. -func (c *Conn25) handleConnectorTransitIPRequest(nid tailcfg.NodeID, ctipr ConnectorTransitIPRequest) ConnectorTransitIPResponse { resp := ConnectorTransitIPResponse{} seen := map[netip.Addr]bool{} for _, each := range ctipr.TransitIPs { if seen[each.TransitIP] { resp.TransitIPs = append(resp.TransitIPs, TransitIPResponse{ - Code: OtherFailure, + Code: DuplicateTransitIP, Message: dupeTransitIPMessage, }) + c.connector.logf("[Unexpected] peer attempt to map a transit IP reused a transitIP: node: %s, IP: %v", + n.StableID(), each.TransitIP) continue } - tipresp := c.connector.handleTransitIPRequest(nid, each) + tipresp := c.connector.handleTransitIPRequest(n, peerIPv4, peerIPv6, each) seen[each.TransitIP] = true resp.TransitIPs = append(resp.TransitIPs, tipresp) } return resp } -func (s *connector) handleTransitIPRequest(nid tailcfg.NodeID, tipr TransitIPRequest) TransitIPResponse { +func (s *connector) handleTransitIPRequest(n tailcfg.NodeView, peerV4 netip.Addr, peerV6 netip.Addr, tipr TransitIPRequest) TransitIPResponse { + if tipr.TransitIP.Is4() != tipr.DestinationIP.Is4() { + s.logf("[Unexpected] peer attempt to map a transit IP to dest IP did not have matching families: node: %s, tIPv4: %v dIPv4: %v", + n.StableID(), tipr.TransitIP.Is4(), tipr.DestinationIP.Is4()) + return TransitIPResponse{Code: AddrFamilyMismatch, Message: addrFamilyMismatchMessage} + } + + // Datapath lookups only have access to the peer IP, and that will match the family + // of the transit IP, so we need to store v4 and v6 mappings separately. + var peerAddr netip.Addr + if tipr.TransitIP.Is4() { + peerAddr = peerV4 + } else { + peerAddr = peerV6 + } + + // If we couldn't find a matching family, return an error. + if !peerAddr.IsValid() { + s.logf("[Unexpected] peer attempt to map a transit IP did not have a matching address family: node: %s, IPv4: %v", + n.StableID(), tipr.TransitIP.Is4()) + return TransitIPResponse{NoMatchingPeerIPFamily, noMatchingPeerIPFamilyMessage} + } + s.mu.Lock() defer s.mu.Unlock() + if _, ok := s.config.appsByName[tipr.App]; !ok { + s.logf("[Unexpected] peer attempt to map a transit IP referenced unknown app: node: %s, app: %q", + n.StableID(), tipr.App) + return TransitIPResponse{Code: UnknownAppName, Message: unknownAppNameMessage} + } + if s.transitIPs == nil { - s.transitIPs = make(map[tailcfg.NodeID]map[netip.Addr]appAddr) + s.transitIPs = make(map[netip.Addr]map[netip.Addr]appAddr) } - peerMap, ok := s.transitIPs[nid] + peerMap, ok := s.transitIPs[peerAddr] if !ok { peerMap = make(map[netip.Addr]appAddr) - s.transitIPs[nid] = peerMap + s.transitIPs[peerAddr] = peerMap } peerMap[tipr.TransitIP] = appAddr{addr: tipr.DestinationIP, app: tipr.App} return TransitIPResponse{} } -func (s *connector) transitIPTarget(nid tailcfg.NodeID, tip netip.Addr) netip.Addr { +func (s *connector) transitIPTarget(peerIP, tip netip.Addr) netip.Addr { s.mu.Lock() defer s.mu.Unlock() - return s.transitIPs[nid][tip].addr + return s.transitIPs[peerIP][tip].addr } // TransitIPRequest details a single TransitIP allocation request from a client to a @@ -322,8 +370,24 @@ const ( OK TransitIPResponseCode = 0 // OtherFailure indicates that the mapping failed for a reason that does not have - // another relevant [TransitIPResponsecode]. + // another relevant [TransitIPResponseCode]. OtherFailure TransitIPResponseCode = 1 + + // DuplicateTransitIP indicates that the same transit address appeared more than + // once in a [ConnectorTransitIPRequest]. + DuplicateTransitIP TransitIPResponseCode = 2 + + // NoMatchingPeerIPFamily indicates that the peer did not have an associated + // IP with the same family as transit IP being registered. + NoMatchingPeerIPFamily = 3 + + // AddrFamilyMismatch indicates that the transit IP and destination IP addresses + // do not belong to the same IP family. + AddrFamilyMismatch = 4 + + // UnknownAppName indicates that the connector is not configured to handle requests + // for the App name that was specified in the request. + UnknownAppName = 5 ) // TransitIPResponse is the response to a TransitIPRequest @@ -651,8 +715,9 @@ type connector struct { logf logger.Logf mu sync.Mutex // protects the fields below - // transitIPs is a map of connector client peer NodeID -> client transitIPs that we update as connector client peers instruct us to, and then use to route traffic to its destination on behalf of connector clients. - transitIPs map[tailcfg.NodeID]map[netip.Addr]appAddr + // transitIPs is a map of connector client peer IP -> client transitIPs that we update as connector client peers instruct us to, and then use to route traffic to its destination on behalf of connector clients. + // Note that each peer could potentially have two maps: one for its IPv4 address, and one for its IPv6 address. The transit IPs map for a given peer IP will contain transit IPs of the same family as the peer's IP. + transitIPs map[netip.Addr]map[netip.Addr]appAddr config config } diff --git a/feature/conn25/conn25_test.go b/feature/conn25/conn25_test.go index 97a22c50017df..574320af8db78 100644 --- a/feature/conn25/conn25_test.go +++ b/feature/conn25/conn25_test.go @@ -38,180 +38,347 @@ func mustIPSetFromPrefix(s string) *netipx.IPSet { return set } -// TestHandleConnectorTransitIPRequestZeroLength tests that if sent a -// ConnectorTransitIPRequest with 0 TransitIPRequests, we respond with a -// ConnectorTransitIPResponse with 0 TransitIPResponses. -func TestHandleConnectorTransitIPRequestZeroLength(t *testing.T) { - c := newConn25(logger.Discard) - req := ConnectorTransitIPRequest{} - nid := tailcfg.NodeID(1) +// TestHandleConnectorTransitIPRequest tests that if sent a +// request with a transit addr and a destination addr we store that mapping +// and can retrieve it. +func TestHandleConnectorTransitIPRequest(t *testing.T) { - resp := c.handleConnectorTransitIPRequest(nid, req) - if len(resp.TransitIPs) != 0 { - t.Fatalf("n TransitIPs in response: %d, want 0", len(resp.TransitIPs)) - } -} + const appName = "TestApp" -// TestHandleConnectorTransitIPRequestStoresAddr tests that if sent a -// request with a transit addr and a destination addr we store that mapping -// and can retrieve it. If sent another req with a different dst for that transit addr -// we store that instead. -func TestHandleConnectorTransitIPRequestStoresAddr(t *testing.T) { - c := newConn25(logger.Discard) - nid := tailcfg.NodeID(1) - tip := netip.MustParseAddr("0.0.0.1") - dip := netip.MustParseAddr("1.2.3.4") - dip2 := netip.MustParseAddr("1.2.3.5") - mr := func(t, d netip.Addr) ConnectorTransitIPRequest { - return ConnectorTransitIPRequest{ - TransitIPs: []TransitIPRequest{ - {TransitIP: t, DestinationIP: d}, - }, - } - } + // Peer IPs + pipV4_1 := netip.MustParseAddr("100.101.101.101") + pipV4_2 := netip.MustParseAddr("100.101.101.102") - resp := c.handleConnectorTransitIPRequest(nid, mr(tip, dip)) - if len(resp.TransitIPs) != 1 { - t.Fatalf("n TransitIPs in response: %d, want 1", len(resp.TransitIPs)) - } - got := resp.TransitIPs[0].Code - if got != TransitIPResponseCode(0) { - t.Fatalf("TransitIP Code: %d, want 0", got) - } - gotAddr := c.connector.transitIPTarget(nid, tip) - if gotAddr != dip { - t.Fatalf("Connector stored destination for tip: %v, want %v", gotAddr, dip) - } + pipV6_1 := netip.MustParseAddr("fd7a:115c:a1e0::101") + pipV6_3 := netip.MustParseAddr("fd7a:115c:a1e0::103") - // mapping can be overwritten - resp2 := c.handleConnectorTransitIPRequest(nid, mr(tip, dip2)) - if len(resp2.TransitIPs) != 1 { - t.Fatalf("n TransitIPs in response: %d, want 1", len(resp2.TransitIPs)) - } - got2 := resp.TransitIPs[0].Code - if got2 != TransitIPResponseCode(0) { - t.Fatalf("TransitIP Code: %d, want 0", got2) - } - gotAddr2 := c.connector.transitIPTarget(nid, tip) - if gotAddr2 != dip2 { - t.Fatalf("Connector stored destination for tip: %v, want %v", gotAddr, dip2) - } -} + // Transit IPs + tipV4_1 := netip.MustParseAddr("0.0.0.1") + tipV4_2 := netip.MustParseAddr("0.0.0.2") -// TestHandleConnectorTransitIPRequestMultipleTIP tests that we can -// get a req with multiple mappings and we store them all. Including -// multiple transit addrs for the same destination. -func TestHandleConnectorTransitIPRequestMultipleTIP(t *testing.T) { - c := newConn25(logger.Discard) - nid := tailcfg.NodeID(1) - tip := netip.MustParseAddr("0.0.0.1") - tip2 := netip.MustParseAddr("0.0.0.2") - tip3 := netip.MustParseAddr("0.0.0.3") - dip := netip.MustParseAddr("1.2.3.4") - dip2 := netip.MustParseAddr("1.2.3.5") - req := ConnectorTransitIPRequest{ - TransitIPs: []TransitIPRequest{ - {TransitIP: tip, DestinationIP: dip}, - {TransitIP: tip2, DestinationIP: dip2}, - // can store same dst addr for multiple transit addrs - {TransitIP: tip3, DestinationIP: dip}, - }, - } - resp := c.handleConnectorTransitIPRequest(nid, req) - if len(resp.TransitIPs) != 3 { - t.Fatalf("n TransitIPs in response: %d, want 3", len(resp.TransitIPs)) - } + tipV6_1 := netip.MustParseAddr("FE80::1") - for i := range 3 { - got := resp.TransitIPs[i].Code - if got != TransitIPResponseCode(0) { - t.Fatalf("i=%d TransitIP Code: %d, want 0", i, got) - } - } - gotAddr1 := c.connector.transitIPTarget(nid, tip) - if gotAddr1 != dip { - t.Fatalf("Connector stored destination for tip(%v): %v, want %v", tip, gotAddr1, dip) - } - gotAddr2 := c.connector.transitIPTarget(nid, tip2) - if gotAddr2 != dip2 { - t.Fatalf("Connector stored destination for tip(%v): %v, want %v", tip2, gotAddr2, dip2) - } - gotAddr3 := c.connector.transitIPTarget(nid, tip3) - if gotAddr3 != dip { - t.Fatalf("Connector stored destination for tip(%v): %v, want %v", tip3, gotAddr3, dip) - } -} + // Destination IPs + dipV4_1 := netip.MustParseAddr("10.0.0.1") + dipV4_2 := netip.MustParseAddr("10.0.0.2") + dipV4_3 := netip.MustParseAddr("10.0.0.3") -// TestHandleConnectorTransitIPRequestSameTIP tests that if we get -// a req that has more than one TransitIPRequest for the same transit addr -// only the first is stored, and the subsequent ones get an error code and -// message in the response. -func TestHandleConnectorTransitIPRequestSameTIP(t *testing.T) { - c := newConn25(logger.Discard) - nid := tailcfg.NodeID(1) - tip := netip.MustParseAddr("0.0.0.1") - tip2 := netip.MustParseAddr("0.0.0.2") - dip := netip.MustParseAddr("1.2.3.4") - dip2 := netip.MustParseAddr("1.2.3.5") - dip3 := netip.MustParseAddr("1.2.3.6") - req := ConnectorTransitIPRequest{ - TransitIPs: []TransitIPRequest{ - {TransitIP: tip, DestinationIP: dip}, - // cannot have dupe TransitIPs in one ConnectorTransitIPRequest - {TransitIP: tip, DestinationIP: dip2}, - {TransitIP: tip2, DestinationIP: dip3}, + dipV6_1 := netip.MustParseAddr("fc00::1") + + // Peer nodes + peerV4V6 := (&tailcfg.Node{ + ID: tailcfg.NodeID(1), + Addresses: []netip.Prefix{netip.PrefixFrom(pipV4_1, 32), netip.PrefixFrom(pipV6_1, 128)}, + }).View() + + peerV4Only := (&tailcfg.Node{ + ID: tailcfg.NodeID(2), + Addresses: []netip.Prefix{netip.PrefixFrom(pipV4_2, 32)}, + }).View() + + peerV6Only := (&tailcfg.Node{ + ID: tailcfg.NodeID(3), + Addresses: []netip.Prefix{netip.PrefixFrom(pipV6_3, 128)}, + }).View() + + tests := []struct { + name string + ctipReqPeers []tailcfg.NodeView // One entry per request and the other + ctipReqs []ConnectorTransitIPRequest // arrays in this struct must have the same + wants []ConnectorTransitIPResponse // cardinality + // For checking lookups: + // The outer array needs to correspond to the number of requests, + // can be nil if no lookups need to be done after the request is processed. + // + // The middle array is the set of lookups for the corresponding request. + // + // The inner array is a tuple of (PeerIP, TransitIP, ExpectedDestinationIP) + wantLookups [][][]netip.Addr + }{ + // Single peer, single request with success ipV4 + { + name: "one-peer-one-req-ipv4", + ctipReqPeers: []tailcfg.NodeView{peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}}}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_2, tipV4_1, dipV4_1}}, + }, + }, + // Single peer, single request with success ipV6 + { + name: "one-peer-one-req-ipv6", + ctipReqPeers: []tailcfg.NodeView{peerV6Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{{TransitIP: tipV6_1, DestinationIP: dipV6_1, App: appName}}}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV6_3, tipV6_1, dipV6_1}}, + }, + }, + // Single peer, multi request with success, ipV4 + { + name: "one-peer-multi-req-ipv4", + ctipReqPeers: []tailcfg.NodeView{peerV4Only, peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}}}, + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_2, DestinationIP: dipV4_2, App: appName}}}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_2, tipV4_1, dipV4_1}}, + {{pipV4_2, tipV4_2, dipV4_2}}, + }, + }, + // Single peer, multi request remap tip, ipV4 + { + name: "one-peer-remap-tip", + ctipReqPeers: []tailcfg.NodeView{peerV4Only, peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}}}, + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_1, DestinationIP: dipV4_2, App: appName}}}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_2, tipV4_1, dipV4_1}}, + {{pipV4_2, tipV4_1, dipV4_2}}, + }, + }, + // Single peer, multi request with success, ipV4 and ipV6 + { + name: "one-peer-multi-req-ipv4-ipv6", + ctipReqPeers: []tailcfg.NodeView{peerV4V6, peerV4V6}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}}}, + {TransitIPs: []TransitIPRequest{{TransitIP: tipV6_1, DestinationIP: dipV6_1, App: appName}}}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_1, tipV4_1, dipV4_1}}, + {{pipV4_1, tipV4_1, dipV4_1}, {pipV6_1, tipV6_1, dipV6_1}, {pipV4_1, tipV6_1, netip.Addr{}}}, + }, + }, + // Single peer, multi map with success, ipV4 + { + name: "one-peer-multi-map-ipv4", + ctipReqPeers: []tailcfg.NodeView{peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{ + {TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}, + {TransitIP: tipV4_2, DestinationIP: dipV4_2, App: appName}, + }}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}, {Code: OK, Message: ""}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_2, tipV4_1, dipV4_1}, {pipV4_2, tipV4_2, dipV4_2}}, + }, + }, + // Single peer, error reuse same tip in one request, ensure all non-dup requests are processed + { + name: "one-peer-multi-map-duplicate-tip", + ctipReqPeers: []tailcfg.NodeView{peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{ + {TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}, + {TransitIP: tipV4_1, DestinationIP: dipV4_2, App: appName}, + {TransitIP: tipV4_2, DestinationIP: dipV4_3, App: appName}, + }}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{ + {Code: OK, Message: ""}, + {Code: DuplicateTransitIP, Message: dupeTransitIPMessage}, + {Code: OK, Message: ""}}, + }, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_2, tipV4_1, dipV4_1}, {pipV4_2, tipV4_2, dipV4_3}}, + }, + }, + // Multi peer, success reuse same tip in one request + { + name: "multi-peer-duplicate-tip", + ctipReqPeers: []tailcfg.NodeView{peerV4V6, peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}}}, + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_1, DestinationIP: dipV4_2, App: appName}}}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_1, tipV4_1, dipV4_1}}, + {{pipV4_1, tipV4_1, dipV4_1}, {pipV4_2, tipV4_1, dipV4_2}}, + }, + }, + // Single peer, multi map, multiple tip to same dip + { + name: "one-peer-multi-map-multi-tip-to-dip", + ctipReqPeers: []tailcfg.NodeView{peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{ + {TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}, + {TransitIP: tipV4_2, DestinationIP: dipV4_1, App: appName}, + }}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: OK, Message: ""}, {Code: OK, Message: ""}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_2, tipV4_1, dipV4_1}, {pipV4_2, tipV4_2, dipV4_1}}, + }, + }, + // Single peer, ipv4 tip, no ipv4 pip, but ipv6 tip works + { + name: "one-peer-missing-ipv4-family", + ctipReqPeers: []tailcfg.NodeView{peerV6Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{ + {TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}, + {TransitIP: tipV6_1, DestinationIP: dipV6_1, App: appName}, + }}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{ + {Code: NoMatchingPeerIPFamily, Message: noMatchingPeerIPFamilyMessage}, + {Code: OK, Message: ""}, + }}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV6_3, tipV4_1, netip.Addr{}}, {pipV6_3, tipV6_1, dipV6_1}}, + }, + }, + // Single peer, ipv6 tip, no ipv6 pip, but ipv4 tip works + { + name: "one-peer-missing-ipv6-family", + ctipReqPeers: []tailcfg.NodeView{peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{ + {TransitIP: tipV6_1, DestinationIP: dipV6_1, App: appName}, + {TransitIP: tipV4_1, DestinationIP: dipV4_1, App: appName}, + }}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{ + {Code: NoMatchingPeerIPFamily, Message: noMatchingPeerIPFamilyMessage}, + {Code: OK, Message: ""}, + }}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_2, tipV6_1, netip.Addr{}}, {pipV4_2, tipV4_1, dipV4_1}}, + }, + }, + // Single peer, mismatched transit and destination ips + { + name: "one-peer-mismatched-tip-dip", + ctipReqPeers: []tailcfg.NodeView{peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_1, DestinationIP: dipV6_1, App: appName}}}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: AddrFamilyMismatch, Message: addrFamilyMismatchMessage}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_2, tipV4_1, netip.Addr{}}}, + }, + }, + // Single peer, invalid app name + { + name: "one-peer-invalid-app", + ctipReqPeers: []tailcfg.NodeView{peerV4Only}, + ctipReqs: []ConnectorTransitIPRequest{ + {TransitIPs: []TransitIPRequest{{TransitIP: tipV4_1, DestinationIP: dipV4_1, App: "Unknown App"}}}, + }, + wants: []ConnectorTransitIPResponse{ + {TransitIPs: []TransitIPResponse{{Code: UnknownAppName, Message: unknownAppNameMessage}}}, + }, + wantLookups: [][][]netip.Addr{ + {{pipV4_2, tipV4_1, netip.Addr{}}}, + }, }, } - resp := c.handleConnectorTransitIPRequest(nid, req) - if len(resp.TransitIPs) != 3 { - t.Fatalf("n TransitIPs in response: %d, want 3", len(resp.TransitIPs)) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + switch { + case len(tt.ctipReqPeers) != len(tt.ctipReqs): + t.Fatalf("error in test setup: ctipReqPeers has length %d does not match ctipReqs length %d", + len(tt.ctipReqPeers), len(tt.ctipReqs)) + case len(tt.ctipReqPeers) != len(tt.wants): + t.Fatalf("error in test setup: ctipReqPeers has length %d does not match wants length %d", + len(tt.ctipReqPeers), len(tt.wants)) + case len(tt.ctipReqPeers) != len(tt.wantLookups): + t.Fatalf("error in test setup: ctipReqPeers has length %d does not match wantLookups length %d", + len(tt.ctipReqPeers), len(tt.wantLookups)) + } - got := resp.TransitIPs[0].Code - if got != TransitIPResponseCode(0) { - t.Fatalf("i=0 TransitIP Code: %d, want 0", got) - } - msg := resp.TransitIPs[0].Message - if msg != "" { - t.Fatalf("i=0 TransitIP Message: \"%s\", want \"%s\"", msg, "") - } - got1 := resp.TransitIPs[1].Code - if got1 != TransitIPResponseCode(1) { - t.Fatalf("i=1 TransitIP Code: %d, want 1", got1) - } - msg1 := resp.TransitIPs[1].Message - if msg1 != dupeTransitIPMessage { - t.Fatalf("i=1 TransitIP Message: \"%s\", want \"%s\"", msg1, dupeTransitIPMessage) - } - got2 := resp.TransitIPs[2].Code - if got2 != TransitIPResponseCode(0) { - t.Fatalf("i=2 TransitIP Code: %d, want 0", got2) - } - msg2 := resp.TransitIPs[2].Message - if msg2 != "" { - t.Fatalf("i=2 TransitIP Message: \"%s\", want \"%s\"", msg, "") - } + // Use the same Conn25 for each request in the test and seed it with a test app name. + c := newConn25(logger.Discard) + c.connector.config = config{ + appsByName: map[string]appctype.Conn25Attr{appName: {}}, + } - gotAddr1 := c.connector.transitIPTarget(nid, tip) - if gotAddr1 != dip { - t.Fatalf("Connector stored destination for tip(%v): %v, want %v", tip, gotAddr1, dip) - } - gotAddr2 := c.connector.transitIPTarget(nid, tip2) - if gotAddr2 != dip3 { - t.Fatalf("Connector stored destination for tip(%v): %v, want %v", tip2, gotAddr2, dip3) - } -} + for i, peer := range tt.ctipReqPeers { + req := tt.ctipReqs[i] + want := tt.wants[i] -// TestGetDstIPUnknownTIP tests that unknown transit addresses can be looked up without problem. -func TestTransitIPTargetUnknownTIP(t *testing.T) { - c := newConn25(logger.Discard) - nid := tailcfg.NodeID(1) - tip := netip.MustParseAddr("0.0.0.1") - got := c.connector.transitIPTarget(nid, tip) - want := netip.Addr{} - if got != want { - t.Fatalf("Unknown transit addr, want: %v, got %v", want, got) + resp := c.handleConnectorTransitIPRequest(peer, req) + + // Ensure that we have the expected number of responses + if len(resp.TransitIPs) != len(want.TransitIPs) { + t.Fatalf("wrong number of TransitIPs in response %d: got %d, want %d", + i, len(resp.TransitIPs), len(want.TransitIPs)) + } + + // Validate the contents of each response + for j, tipResp := range resp.TransitIPs { + wantResp := want.TransitIPs[j] + if tipResp.Code != wantResp.Code { + t.Errorf("transitIP.Code mismatch in response %d, tipresp %d: got %d, want %d", + i, j, tipResp.Code, wantResp.Code) + } + if tipResp.Message != wantResp.Message { + t.Errorf("transitIP.Message mismatch in response %d, tipresp %d: got %q, want %q", + i, j, tipResp.Message, wantResp.Message) + } + } + + // Validate the state of the transitIP map after each request + if tt.wantLookups[i] != nil { + for j, wantLookup := range tt.wantLookups[i] { + if len(wantLookup) != 3 { + t.Fatalf("test setup error: wantLookup for request %d lookup %d contains %d IPs, expected 3", + i, j, len(wantLookup)) + } + pip, tip, wantDip := wantLookup[0], wantLookup[1], wantLookup[2] + gotDip := c.connector.transitIPTarget(pip, tip) + if gotDip != wantDip { + t.Errorf("wrong result on lookup[%d][%d] ([%v], [%v]): got [%v] expected [%v]", + i, j, pip, tip, gotDip, wantDip) + } + } + } + } + }) } } From 2df95175cc297f5aa67de08f4630f6c70ad10427 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Fri, 13 Mar 2026 17:26:43 +0000 Subject: [PATCH 45/78] wgengine/magicsock: add SCION bootstrap support for multiple platforms - Introduced platform-specific implementations of localSearchDomain for Android, Unix, and Windows to retrieve DNS search domains. - Added tests for SCION bootstrap functionality, including TRC ID parsing and bootstrap server interactions. - Enhanced bootstrapSCION function to handle topology and TRC fetching with improved error logging and handling. - Refactored existing code to streamline SCION integration and improve maintainability. --- wgengine/magicsock/scion_bootstrap.go | 106 +++++----- wgengine/magicsock/scion_bootstrap_other.go | 12 ++ wgengine/magicsock/scion_bootstrap_test.go | 189 ++++++++++++++++++ wgengine/magicsock/scion_bootstrap_unix.go | 23 +++ wgengine/magicsock/scion_bootstrap_windows.go | 63 ++++++ 5 files changed, 346 insertions(+), 47 deletions(-) create mode 100644 wgengine/magicsock/scion_bootstrap_other.go create mode 100644 wgengine/magicsock/scion_bootstrap_test.go create mode 100644 wgengine/magicsock/scion_bootstrap_unix.go create mode 100644 wgengine/magicsock/scion_bootstrap_windows.go diff --git a/wgengine/magicsock/scion_bootstrap.go b/wgengine/magicsock/scion_bootstrap.go index e3fd3d34c671d..ba0653ea606a1 100644 --- a/wgengine/magicsock/scion_bootstrap.go +++ b/wgengine/magicsock/scion_bootstrap.go @@ -14,20 +14,18 @@ import ( "net/http" "os" "path/filepath" - "runtime" "strings" "time" - "tailscale.com/net/dns/resolvconffile" + "tailscale.com/atomicfile" + "tailscale.com/envknob" + "tailscale.com/types/logger" ) const ( // bootstrapHTTPTimeout is the timeout for HTTP requests to the bootstrap server. bootstrapHTTPTimeout = 10 * time.Second - // defaultBootstrapPort is the default port for SCION discovery servers. - defaultBootstrapPort = "8041" - // scionDiscoverySRV is the SRV record name for SCION discovery. scionDiscoverySRV = "_sciondiscovery._tcp" ) @@ -40,10 +38,14 @@ var defaultBootstrapURLs []string = []string{ "http://128.143.201.144:8041", } +var ( + scionBootstrapURL = envknob.RegisterString("TS_SCION_BOOTSTRAP_URL") + scionBootstrapURLs = envknob.RegisterString("TS_SCION_BOOTSTRAP_URLS") +) // bootstrapSCION fetches topology.json and TRCs from a bootstrap server, // saving them to destDir. -func bootstrapSCION(ctx context.Context, serverURL string, destDir string) error { +func bootstrapSCION(ctx context.Context, logf logger.Logf, serverURL string, destDir string) error { if err := os.MkdirAll(destDir, 0o700); err != nil { return fmt.Errorf("creating bootstrap directory %s: %w", destDir, err) } @@ -57,15 +59,17 @@ func bootstrapSCION(ctx context.Context, serverURL string, destDir string) error return fmt.Errorf("fetching topology from %s: %w", topoURL, err) } topoPath := filepath.Join(destDir, "topology.json") - if err := os.WriteFile(topoPath, topoData, 0o644); err != nil { + if err := atomicfile.WriteFile(topoPath, topoData, 0o644); err != nil { return fmt.Errorf("writing topology to %s: %w", topoPath, err) } + logf("scion: bootstrap: fetched topology from %s", serverURL) // Fetch TRC index. trcsURL := strings.TrimRight(serverURL, "/") + "/trcs" trcsData, err := httpGet(ctx, client, trcsURL) if err != nil { // TRCs are optional for Phase 1 (accept-all verification). + logf("scion: bootstrap: TRC index not available from %s: %v", serverURL, err) return nil } @@ -78,33 +82,56 @@ func bootstrapSCION(ctx context.Context, serverURL string, destDir string) error var trcIndex []trcEntry if err := json.Unmarshal(trcsData, &trcIndex); err != nil { // Non-fatal: TRC index may not be JSON array on all servers. + logf("scion: bootstrap: failed to parse TRC index: %v", err) return nil } + fetched := 0 for _, entry := range trcIndex { - blobURL := strings.TrimRight(serverURL, "/") + "/trcs/" + entry.ID + "/blob" + if entry.ID.ISD == 0 { + continue // skip unparseable entries + } + idStr := entry.ID.String() + blobURL := strings.TrimRight(serverURL, "/") + "/trcs/" + idStr + "/blob" blob, err := httpGet(ctx, client, blobURL) if err != nil { continue // Best-effort TRC download. } - trcPath := filepath.Join(certsDir, entry.ID+".trc") - _ = os.WriteFile(trcPath, blob, 0o644) + trcPath := filepath.Join(certsDir, idStr+".trc") + if err := atomicfile.WriteFile(trcPath, blob, 0o644); err != nil { + continue + } + fetched++ } + logf("scion: bootstrap: fetched %d/%d TRCs from %s", fetched, len(trcIndex), serverURL) return nil } // trcEntry represents an entry in the TRC index returned by the bootstrap server. +// The server returns {"id": {"isd": 19, "base_number": 1, "serial_number": 1}}. type trcEntry struct { - ID string `json:"id"` + ID trcID `json:"id"` +} + +// trcID represents the composite identifier for a TRC. +type trcID struct { + ISD int `json:"isd"` + BaseNumber int `json:"base_number"` + SerialNumber int `json:"serial_number"` +} + +// String returns a filesystem-safe representation of the TRC ID, +// e.g. "isd19-b1-s1". +func (id trcID) String() string { + return fmt.Sprintf("isd%d-b%d-s%d", id.ISD, id.BaseNumber, id.SerialNumber) } // discoverBootstrapURL attempts DNS-based discovery of a SCION bootstrap server. // It follows the JPAN discovery chain: // 1. SRV lookup for _sciondiscovery._tcp. // 2. TXT lookup for _sciondiscovery._tcp. for port override -// 3. Fallback to port 8041 -func discoverBootstrapURL(ctx context.Context) (string, error) { +func discoverBootstrapURL(ctx context.Context, logf logger.Logf) (string, error) { // Determine local search domain from system resolver. domain, err := localSearchDomain() if err != nil { @@ -118,20 +145,21 @@ func discoverBootstrapURL(ctx context.Context) (string, error) { // Try SRV lookup. _, addrs, err := r.LookupSRV(ctx, "sciondiscovery", "tcp", domain) - if err == nil && len(addrs) > 0 { - host := strings.TrimRight(addrs[0].Target, ".") - port := fmt.Sprintf("%d", addrs[0].Port) + if err != nil || len(addrs) == 0 { + return "", fmt.Errorf("SRV lookup for %s.%s failed: %w", scionDiscoverySRV, domain, err) + } - // Check for TXT record port override. - if txtPort, err := lookupDiscoveryPort(ctx, r, domain); err == nil && txtPort != "" { - port = txtPort - } + host := strings.TrimRight(addrs[0].Target, ".") + port := fmt.Sprintf("%d", addrs[0].Port) - return fmt.Sprintf("http://%s:%s", host, port), nil + // Check for TXT record port override. + if txtPort, err := lookupDiscoveryPort(ctx, r, domain); err == nil && txtPort != "" { + port = txtPort } - // Fallback: try the domain itself on the default port. - return fmt.Sprintf("http://%s:%s", domain, defaultBootstrapPort), nil + url := fmt.Sprintf("http://%s:%s", host, port) + logf("scion: bootstrap: discovered %s via DNS SRV for %s", url, domain) + return url, nil } // lookupDiscoveryPort queries TXT records for the discovery port override. @@ -149,26 +177,9 @@ func lookupDiscoveryPort(ctx context.Context, r *net.Resolver, domain string) (s return "", fmt.Errorf("no x-sciondiscovery TXT record found") } -// localSearchDomain returns the first search domain from the system's DNS -// configuration, using Tailscale's cross-platform resolv.conf parser. -func localSearchDomain() (string, error) { - if runtime.GOOS == "windows" || runtime.GOOS == "android" { - return localSearchDomainFromHostname() - } - cfg, err := resolvconffile.ParseFile(resolvconffile.Path) - if err != nil { - return "", err - } - if len(cfg.SearchDomains) > 0 { - return cfg.SearchDomains[0].WithoutTrailingDot(), nil - } - return "", nil -} - // localSearchDomainFromHostname infers the search domain from the -// system hostname. Used on platforms without resolv.conf. -// Note: on Windows, os.Hostname() typically returns a short NetBIOS name -// without a domain suffix, so this will usually return an error. +// system hostname. Used as a fallback on platforms where the primary +// DNS discovery method fails. func localSearchDomainFromHostname() (string, error) { hostname, err := os.Hostname() if err != nil { @@ -177,7 +188,7 @@ func localSearchDomainFromHostname() (string, error) { if i := strings.IndexByte(hostname, '.'); i >= 0 { return hostname[i+1:], nil } - return "", fmt.Errorf("no search domain found") + return "", fmt.Errorf("no domain suffix in hostname %q", hostname) } // httpGet performs an HTTP GET request and returns the response body. @@ -206,16 +217,16 @@ func httpGet(ctx context.Context, client *http.Client, url string) ([]byte, erro // bootstrapURLs returns the list of bootstrap URLs to try, from explicit // configuration, DNS discovery, and hardcoded defaults. -func bootstrapURLs(ctx context.Context) []string { +func bootstrapURLs(ctx context.Context, logf logger.Logf) []string { var urls []string // Explicit URL from environment. - if u := os.Getenv("TS_SCION_BOOTSTRAP_URL"); u != "" { + if u := scionBootstrapURL(); u != "" { urls = append(urls, u) } // Comma-separated list from environment. - if u := os.Getenv("TS_SCION_BOOTSTRAP_URLS"); u != "" { + if u := scionBootstrapURLs(); u != "" { for _, url := range strings.Split(u, ",") { url = strings.TrimSpace(url) if url != "" { @@ -225,12 +236,13 @@ func bootstrapURLs(ctx context.Context) []string { } // DNS-discovered URL. - if discovered, err := discoverBootstrapURL(ctx); err == nil { + if discovered, err := discoverBootstrapURL(ctx, logf); err == nil { urls = append(urls, discovered) } // Hardcoded defaults. urls = append(urls, defaultBootstrapURLs...) + logf("scion: bootstrap: %d URLs to try", len(urls)) return urls } diff --git a/wgengine/magicsock/scion_bootstrap_other.go b/wgengine/magicsock/scion_bootstrap_other.go new file mode 100644 index 0000000000000..13540c1ece40d --- /dev/null +++ b/wgengine/magicsock/scion_bootstrap_other.go @@ -0,0 +1,12 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_scion && !(linux || darwin || freebsd || openbsd || netbsd || windows) + +package magicsock + +// localSearchDomain returns the search domain on platforms without +// resolv.conf or winipcfg (e.g. Android). Falls back to hostname parsing. +func localSearchDomain() (string, error) { + return localSearchDomainFromHostname() +} diff --git a/wgengine/magicsock/scion_bootstrap_test.go b/wgengine/magicsock/scion_bootstrap_test.go new file mode 100644 index 0000000000000..bdaac825fd1be --- /dev/null +++ b/wgengine/magicsock/scion_bootstrap_test.go @@ -0,0 +1,189 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_scion + +package magicsock + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "tailscale.com/envknob" + "tailscale.com/types/logger" +) + +func TestTrcIDString(t *testing.T) { + tests := []struct { + id trcID + want string + }{ + {trcID{ISD: 19, BaseNumber: 1, SerialNumber: 1}, "isd19-b1-s1"}, + {trcID{ISD: 1, BaseNumber: 2, SerialNumber: 3}, "isd1-b2-s3"}, + {trcID{ISD: 0, BaseNumber: 0, SerialNumber: 0}, "isd0-b0-s0"}, + } + for _, tt := range tests { + got := tt.id.String() + if got != tt.want { + t.Errorf("trcID%+v.String() = %q, want %q", tt.id, got, tt.want) + } + } +} + +func TestTrcIndexParsing(t *testing.T) { + // Real bootstrap server JSON format. + raw := `[{"id":{"isd":19,"base_number":1,"serial_number":1}},{"id":{"isd":19,"base_number":1,"serial_number":2}}]` + var entries []trcEntry + if err := json.Unmarshal([]byte(raw), &entries); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if len(entries) != 2 { + t.Fatalf("got %d entries, want 2", len(entries)) + } + if got := entries[0].ID.ISD; got != 19 { + t.Errorf("entries[0].ID.ISD = %d, want 19", got) + } + if got := entries[0].ID.String(); got != "isd19-b1-s1" { + t.Errorf("entries[0].ID.String() = %q, want %q", got, "isd19-b1-s1") + } + if got := entries[1].ID.String(); got != "isd19-b1-s2" { + t.Errorf("entries[1].ID.String() = %q, want %q", got, "isd19-b1-s2") + } +} + +func TestTrcIndexParsingOldFormat(t *testing.T) { + // The old flat string format ({"id": "ISD19-B1-S1"}) is incompatible + // with the nested struct. json.Unmarshal should return an error, + // and bootstrapSCION handles this gracefully (non-fatal). + raw := `[{"id":"ISD19-B1-S1"}]` + var entries []trcEntry + if err := json.Unmarshal([]byte(raw), &entries); err == nil { + t.Fatal("expected Unmarshal error for old string format, got nil") + } +} + +func TestBootstrapSCION(t *testing.T) { + topoJSON := `{"isd_as":"19-ffaa:1:eba","mtu":1472}` + trcBlob := []byte("fake-trc-blob") + trcIndex := `[{"id":{"isd":19,"base_number":1,"serial_number":1}}]` + + mux := http.NewServeMux() + mux.HandleFunc("/topology", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(topoJSON)) + }) + mux.HandleFunc("/trcs", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(trcIndex)) + }) + mux.HandleFunc("/trcs/isd19-b1-s1/blob", func(w http.ResponseWriter, r *http.Request) { + w.Write(trcBlob) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + destDir := t.TempDir() + logf := logger.WithPrefix(t.Logf, "test: ") + + if err := bootstrapSCION(context.Background(), logf, srv.URL, destDir); err != nil { + t.Fatalf("bootstrapSCION: %v", err) + } + + // Verify topology file. + topoPath := filepath.Join(destDir, "topology.json") + data, err := os.ReadFile(topoPath) + if err != nil { + t.Fatalf("reading topology: %v", err) + } + if string(data) != topoJSON { + t.Errorf("topology content = %q, want %q", data, topoJSON) + } + + // Verify TRC file. + trcPath := filepath.Join(destDir, "certs", "isd19-b1-s1.trc") + data, err = os.ReadFile(trcPath) + if err != nil { + t.Fatalf("reading TRC: %v", err) + } + if string(data) != string(trcBlob) { + t.Errorf("TRC content = %q, want %q", data, trcBlob) + } +} + +func TestBootstrapSCIONTopologyOnly(t *testing.T) { + // Server that returns topology but 404 on TRCs — should succeed. + topoJSON := `{"isd_as":"19-ffaa:1:eba"}` + + mux := http.NewServeMux() + mux.HandleFunc("/topology", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(topoJSON)) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + destDir := t.TempDir() + logf := logger.WithPrefix(t.Logf, "test: ") + + if err := bootstrapSCION(context.Background(), logf, srv.URL, destDir); err != nil { + t.Fatalf("bootstrapSCION: %v", err) + } + + data, err := os.ReadFile(filepath.Join(destDir, "topology.json")) + if err != nil { + t.Fatalf("reading topology: %v", err) + } + if string(data) != topoJSON { + t.Errorf("topology content = %q, want %q", data, topoJSON) + } +} + +func TestBootstrapURLs(t *testing.T) { + logf := logger.WithPrefix(t.Logf, "test: ") + + // Use envknob.Setenv so the registered knob functions see the values. + envknob.Setenv("TS_SCION_BOOTSTRAP_URL", "http://explicit:8041") + t.Cleanup(func() { envknob.Setenv("TS_SCION_BOOTSTRAP_URL", "") }) + envknob.Setenv("TS_SCION_BOOTSTRAP_URLS", "http://list1:8041, http://list2:8041") + t.Cleanup(func() { envknob.Setenv("TS_SCION_BOOTSTRAP_URLS", "") }) + + urls := bootstrapURLs(context.Background(), logf) + + if len(urls) < 4 { + t.Fatalf("expected at least 4 URLs, got %d: %v", len(urls), urls) + } + if urls[0] != "http://explicit:8041" { + t.Errorf("urls[0] = %q, want explicit URL", urls[0]) + } + if urls[1] != "http://list1:8041" { + t.Errorf("urls[1] = %q, want list1", urls[1]) + } + if urls[2] != "http://list2:8041" { + t.Errorf("urls[2] = %q, want list2", urls[2]) + } + + // Hardcoded defaults should be at the end. + tail := urls[len(urls)-len(defaultBootstrapURLs):] + for i, want := range defaultBootstrapURLs { + if tail[i] != want { + t.Errorf("tail[%d] = %q, want %q", i, tail[i], want) + } + } +} + +func TestLocalSearchDomainFromHostname(t *testing.T) { + tests := []struct { + name string + want string + wantErr bool + }{ + // Note: we can't easily override os.Hostname() in tests, + // so we test the parsing logic via the function contract. + } + _ = tests + + // At minimum, verify the function doesn't panic. + _, _ = localSearchDomainFromHostname() +} diff --git a/wgengine/magicsock/scion_bootstrap_unix.go b/wgengine/magicsock/scion_bootstrap_unix.go new file mode 100644 index 0000000000000..795ee1631e062 --- /dev/null +++ b/wgengine/magicsock/scion_bootstrap_unix.go @@ -0,0 +1,23 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_scion && (linux || darwin || freebsd || openbsd || netbsd) + +package magicsock + +import ( + "tailscale.com/net/dns/resolvconffile" +) + +// localSearchDomain returns the first search domain from the system's DNS +// configuration, using Tailscale's resolv.conf parser. +func localSearchDomain() (string, error) { + cfg, err := resolvconffile.ParseFile(resolvconffile.Path) + if err != nil { + return localSearchDomainFromHostname() + } + if len(cfg.SearchDomains) > 0 { + return cfg.SearchDomains[0].WithoutTrailingDot(), nil + } + return localSearchDomainFromHostname() +} diff --git a/wgengine/magicsock/scion_bootstrap_windows.go b/wgengine/magicsock/scion_bootstrap_windows.go new file mode 100644 index 0000000000000..1986468265597 --- /dev/null +++ b/wgengine/magicsock/scion_bootstrap_windows.go @@ -0,0 +1,63 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_scion && windows + +package magicsock + +import ( + "golang.org/x/sys/windows" + "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" +) + +// localSearchDomain returns the DNS suffix from the default network adapter +// on Windows, using winipcfg.GetAdaptersAddresses. Falls back to hostname +// parsing if no adapter suffix is found. +func localSearchDomain() (string, error) { + iface, err := getWindowsDefaultAdapter() + if err == nil && iface != nil { + if suffix := iface.DNSSuffix(); suffix != "" { + return suffix, nil + } + } + return localSearchDomainFromHostname() +} + +// getWindowsDefaultAdapter returns the default IPv4 network adapter. +func getWindowsDefaultAdapter() (*winipcfg.IPAdapterAddresses, error) { + ifs, err := winipcfg.GetAdaptersAddresses(windows.AF_INET, winipcfg.GAAFlagIncludeAllInterfaces) + if err != nil { + return nil, err + } + + routes, err := winipcfg.GetIPForwardTable2(windows.AF_INET) + if err != nil { + return nil, err + } + + // Index adapters by LUID, filtering to operational non-loopback interfaces. + byLUID := make(map[winipcfg.LUID]*winipcfg.IPAdapterAddresses) + for _, iface := range ifs { + if iface.OperStatus == winipcfg.IfOperStatusUp && iface.IfType != winipcfg.IfTypeSoftwareLoopback { + byLUID[iface.LUID] = iface + } + } + + // Find the default route (prefix length 0) with the lowest metric. + bestMetric := ^uint32(0) + var best *winipcfg.IPAdapterAddresses + for _, route := range routes { + if route.DestinationPrefix.PrefixLength != 0 { + continue + } + iface := byLUID[route.InterfaceLUID] + if iface == nil { + continue + } + if route.Metric < bestMetric { + bestMetric = route.Metric + best = iface + } + } + return best, nil +} From f4c2fdfe7adee82f66eb44f2646e758b5939b752 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Fri, 13 Mar 2026 21:02:39 +0000 Subject: [PATCH 46/78] wgengine/magicsock: add forceBootstrapSCION environment variable and update SCION bootstrap URLs --- wgengine/magicsock/magicsock_scion.go | 17 +++++++++++------ wgengine/magicsock/scion_bootstrap.go | 5 +++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index f354d44ec462e..ae3349a373060 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -649,6 +649,9 @@ func scionListenAddr(ctx context.Context, connector daemon.Connector) *net.UDPAd // forceEmbeddedSCION is the TS_SCION_EMBEDDED envknob. When set to "1", // the external daemon attempt is skipped and only the embedded connector is tried. var forceEmbeddedSCION = envknob.RegisterBool("TS_SCION_EMBEDDED") +// forceBootstrapSCION is the TS_SCION_FORCE_BOOTSTRAP envknob. When set to "1", +// the local topology file attempt is skipped and only the bootstrap attempt is tried. +var forceBootstrapSCION = envknob.RegisterBool("TS_SCION_FORCE_BOOTSTRAP") // trySCIONConnect attempts to set up a SCION connection using a cascading // fallback strategy: @@ -672,13 +675,15 @@ func trySCIONConnect(ctx context.Context, logf logger.Logf, netMon *netmon.Monit } // Step 2: Try embedded with existing local topology file. - topoPath := scionTopologyPath() - if _, err := os.Stat(topoPath); err == nil { - sc, err := tryEmbeddedDaemon(ctx, topoPath, logf, netMon) - if err == nil { - return sc, nil + if !forceEmbeddedSCION() { + topoPath := scionTopologyPath() + if _, err := os.Stat(topoPath); err == nil { + sc, err := tryEmbeddedDaemon(ctx, topoPath, logf, netMon) + if err == nil { + return sc, nil + } + // Fall through to bootstrap attempts. } - // Fall through to bootstrap attempts. } // Steps 3-5: Try bootstrap from URLs (explicit, DNS-discovered, hardcoded). diff --git a/wgengine/magicsock/scion_bootstrap.go b/wgengine/magicsock/scion_bootstrap.go index ba0653ea606a1..a2fef787844b6 100644 --- a/wgengine/magicsock/scion_bootstrap.go +++ b/wgengine/magicsock/scion_bootstrap.go @@ -34,8 +34,9 @@ const ( // SCION deployments. Populated as deployments are identified; DNS discovery // is the primary automatic mechanism. var defaultBootstrapURLs []string = []string{ - "http://141.44.25.151:8041", - "http://128.143.201.144:8041", + "http://141.44.25.151:8041", // ovgu.de + "http://128.143.201.144:8041", // uva + "http://netsec-w37w3w.inf.ethz.ch:8041", // ethz.ch } var ( From 27d6955e2de90b69c6c0fecf64845ce2794dd30d Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Fri, 13 Mar 2026 22:35:22 +0000 Subject: [PATCH 47/78] wgengine/magicsock: refactor SCION connection handling and update environment variable usage - Reorganized SCION-related constants and types for improved clarity and maintainability. - Introduced closeSocket method to handle SCION socket closure while preserving the daemon connector. - Updated the logic for SCION bootstrap to utilize the forceBootstrapSCION environment variable. - Enhanced comments for better understanding of SCION connection management. --- wgengine/magicsock/magicsock_scion.go | 218 +++++++++++++++----------- 1 file changed, 130 insertions(+), 88 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index ae3349a373060..b775088d89c99 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -25,9 +25,9 @@ import ( "github.com/scionproto/scion/pkg/daemon" "github.com/scionproto/scion/pkg/slayers" scionpath "github.com/scionproto/scion/pkg/slayers/path/scion" - snetpath "github.com/scionproto/scion/pkg/snet/path" "github.com/scionproto/scion/pkg/snet" "github.com/scionproto/scion/pkg/snet/addrutil" + snetpath "github.com/scionproto/scion/pkg/snet/path" wgconn "github.com/tailscale/wireguard-go/conn" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" @@ -87,13 +87,13 @@ type scionAddrKey struct { // scionPathKey. The actual SCION address and path data live here rather than // in epAddr to keep epAddr comparable and small. type scionPathInfo struct { - peerIA addr.IA - hostAddr netip.AddrPort // peer's SCION host IP:port - fingerprint snet.PathFingerprint // SHA256 of interface sequence; for matching across refreshes - path snet.Path // current best SCION path to this peer - replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) - cachedDst *snet.UDPAddr // pre-built destination addr; rebuilt when path changes - fastPath *scionFastPath // pre-serialized header template for fast sends + peerIA addr.IA + hostAddr netip.AddrPort // peer's SCION host IP:port + fingerprint snet.PathFingerprint // SHA256 of interface sequence; for matching across refreshes + path snet.Path // current best SCION path to this peer + replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) + cachedDst *snet.UDPAddr // pre-built destination addr; rebuilt when path changes + fastPath *scionFastPath // pre-serialized header template for fast sends expiry time.Time // path expiration from path metadata mtu uint16 // SCION payload MTU from path metadata refreshMissCount int // consecutive refresh cycles fingerprint absent from daemon @@ -139,17 +139,9 @@ const scionFallbackPayloadMTU = 1280 // reports LatencyUnset for a hop. Conservative estimate for path selection. const scionUnsetHopLatency = 10 * time.Millisecond -// scionReadDeadline is the read deadline set on the SCION socket. -// If no packet is received within this duration, we check whether the -// socket is still alive. This must be long enough to avoid spurious -// reconnections during idle periods, but short enough to detect a dead -// socket promptly. -const scionReadDeadline = 30 * time.Second - -// scionReconnectThreshold is the maximum time without receiving any SCION -// packet before we consider the socket dead and attempt to reconnect. -// This is only checked when there are active SCION peers. -const scionReconnectThreshold = 30 * time.Second +// scionDaemonProbeTimeout is the timeout for probing the SCION daemon +// connector to check if it's still alive (used for tiered reconnection). +const scionDaemonProbeTimeout = 5 * time.Second // defaultSCIONProbePaths is the default number of SCION paths to probe per peer. const defaultSCIONProbePaths = 5 @@ -173,13 +165,13 @@ func scionMaxProbePaths() int { // scionEndpointState tracks SCION-specific per-peer state on an endpoint. type scionEndpointState struct { - peerIA addr.IA // peer's ISD-AS from Services advertisement - hostAddr netip.AddrPort // peer's SCION host IP:port - paths map[scionPathKey]*scionPathProbeState // probed paths (up to scionMaxProbePaths) - activePath scionPathKey // currently selected best path for data - lastDiscoveryAt time.Time // when path discovery last started (throttle) - lastFullEvalAt mono.Time // throttles re-evaluation of SCION path latencies - probeRoundRobin int // round-robin index for non-best path probing + peerIA addr.IA // peer's ISD-AS from Services advertisement + hostAddr netip.AddrPort // peer's SCION host IP:port + paths map[scionPathKey]*scionPathProbeState // probed paths (up to scionMaxProbePaths) + activePath scionPathKey // currently selected best path for data + lastDiscoveryAt time.Time // when path discovery last started (throttle) + lastFullEvalAt mono.Time // throttles re-evaluation of SCION path latencies + probeRoundRobin int // round-robin index for non-best path probing } // scionPathProbeState tracks disco probing state for one SCION path. @@ -513,14 +505,14 @@ type scionBatchRW interface { // scionConn wraps a SCION connection for use by magicsock. type scionConn struct { - conn *snet.Conn // from SCIONNetwork.Listen() - underlayConn *net.UDPConn // raw underlay for fast-path sends (owned by conn) - underlayXPC scionBatchRW // for WriteBatch / sendmmsg (ipv4 or ipv6) - localIA addr.IA // our ISD-AS - localHostIP netip.Addr // local host IP (e.g. 127.0.0.1) - localPort uint16 // local SCION/UDP port - daemon daemon.Connector // for path queries - topo snet.Topology // local topology + conn *snet.Conn // from SCIONNetwork.Listen() + underlayConn *net.UDPConn // raw underlay for fast-path sends (owned by conn) + underlayXPC scionBatchRW // for WriteBatch / sendmmsg (ipv4 or ipv6) + localIA addr.IA // our ISD-AS + localHostIP netip.Addr // local host IP (e.g. 127.0.0.1) + localPort uint16 // local SCION/UDP port + daemon daemon.Connector // for path queries + topo snet.Topology // local topology } // close shuts down the SCION connection and daemon connector. @@ -534,6 +526,17 @@ func (sc *scionConn) close() error { return nil } +// closeSocket closes only the SCION socket (conn, underlayConn, underlayXPC), +// preserving the daemon connector and topology for socket-only reconnection. +func (sc *scionConn) closeSocket() { + if sc.conn != nil { + sc.conn.Close() + } + sc.conn = nil + sc.underlayConn = nil + sc.underlayXPC = nil +} + // writeTo sends b to a peer identified by the given scionPathInfo. func (sc *scionConn) writeTo(b []byte, pi *scionPathInfo) (int, error) { pi.mu.Lock() @@ -649,6 +652,7 @@ func scionListenAddr(ctx context.Context, connector daemon.Connector) *net.UDPAd // forceEmbeddedSCION is the TS_SCION_EMBEDDED envknob. When set to "1", // the external daemon attempt is skipped and only the embedded connector is tried. var forceEmbeddedSCION = envknob.RegisterBool("TS_SCION_EMBEDDED") + // forceBootstrapSCION is the TS_SCION_FORCE_BOOTSTRAP envknob. When set to "1", // the local topology file attempt is skipped and only the bootstrap attempt is tried. var forceBootstrapSCION = envknob.RegisterBool("TS_SCION_FORCE_BOOTSTRAP") @@ -675,7 +679,7 @@ func trySCIONConnect(ctx context.Context, logf logger.Logf, netMon *netmon.Monit } // Step 2: Try embedded with existing local topology file. - if !forceEmbeddedSCION() { + if !forceBootstrapSCION() { topoPath := scionTopologyPath() if _, err := os.Stat(topoPath); err == nil { sc, err := tryEmbeddedDaemon(ctx, topoPath, logf, netMon) @@ -967,6 +971,9 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo // Fast path: pre-serialized headers + sendmmsg. if fastPath != nil && sc.underlayXPC != nil { err = c.sendSCIONBatchFast(sc, fastPath, buffs, offset) + if err != nil { + c.handleSCIONSendError(err) + } return err == nil, err } @@ -982,6 +989,7 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo for _, buf := range buffs { _, err = sc.conn.WriteTo(buf[offset:], dst) if err != nil { + c.handleSCIONSendError(err) return false, err } } @@ -1070,11 +1078,29 @@ func (c *Conn) sendSCION(sk scionPathKey, b []byte) (bool, error) { } _, err := sc.writeTo(b, pi) if err != nil { + c.handleSCIONSendError(err) return false, err } return true, nil } +// handleSCIONSendError triggers SCION reconnection when a send fails with +// a socket error. This is the primary reconnection mechanism — rather than +// polling for liveness on the receive side, we reconnect when sends actually +// fail. The receive loop picks up the new socket automatically because the +// old socket's close unblocks its read with net.ErrClosed. +func (c *Conn) handleSCIONSendError(err error) { + if err == nil { + return + } + // Don't reconnect for logical errors (nil path, expired path, no SCION). + if errors.Is(err, errNoSCION) { + return + } + c.logf("magicsock: SCION send failed: %v, triggering reconnect", err) + go c.reconnectSCION() +} + // lookupSCIONPath returns the scionPathInfo for the given key, or nil if not found. // c.mu must be held. func (c *Conn) lookupSCIONPath(k scionPathKey) *scionPathInfo { @@ -1148,15 +1174,16 @@ func (c *Conn) updateActiveSCIONPathLocking(peerIA addr.IA, hostAddr netip.AddrP // receiveSCION is the conn.ReceiveFunc for SCION packets. It reads from the // SCION connection and dispatches disco or WireGuard packets. // -// Unlike receiveIP, this function handles read timeouts internally and never +// Unlike receiveIP, this function handles read errors internally and never // propagates them to WireGuard. This is critical because WireGuard's // RoutineReceiveIncoming exits the goroutine permanently after 10 consecutive // temporary errors, and we need to survive SCION socket death + reconnection. // -// The function uses SetReadDeadline to periodically wake up and check whether -// the socket is still alive. If no packets are received for -// scionReconnectThreshold while active SCION peers exist, we close the old -// socket and reconnect. +// The read blocks indefinitely (like IPv4/IPv6 sockets). On shutdown, +// closeSCIONBindLocked sets an immediate deadline to unblock the read. +// On socket swap (reconnection from the send path), the old socket is +// closed which unblocks the read with net.ErrClosed; the loop then +// re-reads c.pconnSCION to pick up the new socket. // // When the underlay socket is available, packets are read in batches via // recvmmsg and parsed with lightweight slayers.SCION decoding. Otherwise, @@ -1168,11 +1195,6 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) return 0, net.ErrClosed } - // Initialize lastSCIONRecv so we don't trigger reconnection on startup. - if c.lastSCIONRecv.LoadAtomic() == 0 { - c.lastSCIONRecv.StoreAtomic(mono.Now()) - } - for { // Check for graceful shutdown. select { @@ -1195,9 +1217,10 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } // Fast path: batch read from underlay via recvmmsg. + // No read deadline — blocks indefinitely like IPv4/IPv6 sockets. + // On shutdown, closeSCIONBindLocked sets an immediate deadline. + // On reconnection, the old socket is closed → net.ErrClosed. if sc.underlayXPC != nil { - sc.underlayConn.SetReadDeadline(time.Now().Add(scionReadDeadline)) - n, err := c.receiveSCIONBatch(sc, buffs, sizes, eps) if n > 0 { return n, nil @@ -1208,13 +1231,9 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) return 0, net.ErrClosed default: } - if isTimeoutError(err) { - if c.shouldReconnectSCION() { - c.reconnectSCION() - } - continue - } - if errors.Is(err, net.ErrClosed) { + if errors.Is(err, net.ErrClosed) || isTimeoutError(err) { + // Socket closed (reconnection or shutdown) or + // deadline set by closeSCIONBindLocked — re-check. continue } c.logf("magicsock: SCION read error: %v", err) @@ -1225,8 +1244,7 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } // Slow path: single-packet snet.Conn.ReadFrom. - sc.conn.SetReadDeadline(time.Now().Add(scionReadDeadline)) - + // No read deadline — blocks indefinitely. n, srcAddr, err := sc.readFrom(buffs[0]) if err != nil { select { @@ -1234,13 +1252,7 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) return 0, net.ErrClosed default: } - if isTimeoutError(err) { - if c.shouldReconnectSCION() { - c.reconnectSCION() - } - continue - } - if errors.Is(err, net.ErrClosed) { + if errors.Is(err, net.ErrClosed) || isTimeoutError(err) { continue } c.logf("magicsock: SCION read error: %v", err) @@ -1411,28 +1423,68 @@ func isTimeoutError(err error) bool { return errors.As(err, &netErr) && netErr.Timeout() } -// shouldReconnectSCION reports whether the SCION socket appears dead and -// should be reconnected. The socket is considered dead when: -// 1. No SCION packet has been received for scionReconnectThreshold, AND -// 2. There are active SCION peers (otherwise silence is expected). -func (c *Conn) shouldReconnectSCION() bool { - lastRecv := c.lastSCIONRecv.LoadAtomic() - if mono.Since(lastRecv) < scionReconnectThreshold { +// scionDaemonAlive probes the SCION daemon connector to check if it's +// still responsive. For the embedded connector this is trivial (field read); +// for external daemons it's a gRPC call confirming the process is alive. +func (c *Conn) scionDaemonAlive() bool { + sc := c.pconnSCION + if sc == nil || sc.daemon == nil { return false } + ctx, cancel := context.WithTimeout(c.connCtx, scionDaemonProbeTimeout) + defer cancel() + _, err := sc.daemon.LocalIA(ctx) + return err == nil +} - // Check if any endpoint has SCION state (active SCION peers). - c.mu.Lock() - hasSCIONPeers := len(c.scionPaths) > 0 - c.mu.Unlock() - return hasSCIONPeers +// reconnectSCIONSocket attempts a socket-only reconnection: close the +// socket but keep the daemon connector and topology, then call +// finishSCIONConnect to create a new socket with the existing connector. +// Returns true on success. +func (c *Conn) reconnectSCIONSocket() bool { + sc := c.pconnSCION + if sc == nil { + return false + } + + savedDaemon := sc.daemon + savedTopo := sc.topo + + // Close socket, release the port for rebinding. + sc.closeSocket() + c.pconnSCION = nil + + newSC, err := finishSCIONConnect(c.connCtx, savedDaemon, savedTopo, c.logf, c.netMon) + if err != nil { + c.logf("magicsock: SCION socket-only reconnect failed: %v", err) + return false + } + + c.pconnSCION = newSC + c.logf("magicsock: SCION socket-only reconnect succeeded, local IA: %s", newSC.localIA) + return true } -// reconnectSCION closes the current SCION socket and creates a new one. -// The receiveSCION loop will pick up the new socket on the next iteration. +// reconnectSCION performs tiered SCION reconnection: +// - Tier 1: If the daemon connector is alive, do a socket-only reconnect +// (avoids expensive bootstrap: DNS SRV, topology fetch, TRCs, etc.) +// - Tier 2: Full bootstrap — close everything, trySCIONConnect from scratch +// +// The receiveSCION loop picks up the new socket on the next iteration +// because the old socket's close unblocks the read with net.ErrClosed. func (c *Conn) reconnectSCION() { - c.logf("magicsock: SCION socket appears dead (no recv for %v), reconnecting...", scionReconnectThreshold) + // Tier 1: socket-only reconnect if daemon is alive. + if c.scionDaemonAlive() { + c.logf("magicsock: SCION daemon alive, trying socket-only reconnect") + if c.reconnectSCIONSocket() { + c.rediscoverAllSCIONPaths() + return + } + c.logf("magicsock: SCION socket-only reconnect failed, falling through to full bootstrap") + } + // Tier 2: full bootstrap. + c.logf("magicsock: SCION doing full bootstrap reconnect") oldSC := c.pconnSCION // Close old connection first — we must release the port before binding @@ -1448,21 +1500,11 @@ func (c *Conn) reconnectSCION() { newSC, err := trySCIONConnect(c.connCtx, c.logf, c.netMon) if err != nil { c.logf("magicsock: SCION reconnect failed: %v", err) - // Reset the receive timestamp so we retry after scionReconnectThreshold. - c.lastSCIONRecv.StoreAtomic(mono.Now()) return } - // Swap in the new connection. c.pconnSCION = newSC - - // Reset the receive timestamp so we don't immediately re-trigger. - c.lastSCIONRecv.StoreAtomic(mono.Now()) - c.logf("magicsock: SCION reconnected successfully, local IA: %s", newSC.localIA) - - // Re-discover paths for all SCION peers. We need fresh paths that - // use the new socket's local address. c.rediscoverAllSCIONPaths() } From f80e9a5fe8d42f5ab46e5e2c963c7f809288b018 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Sat, 14 Mar 2026 01:11:49 +0000 Subject: [PATCH 48/78] wgengine/magicsock: implement SCION dispatcher shim for legacy support - Added a new receive function for the SCION dispatcher shim to handle packets on the legacy port 30041. - Introduced logic to bind a receive-only UDP socket for the dispatcher shim, allowing it to process SCION packets similarly to the main socket. - Updated connection handling to gracefully manage scenarios where the dispatcher port is already in use. - Enhanced tests to validate the behavior of the dispatcher shim under various conditions. --- wgengine/magicsock/magicsock.go | 3 + wgengine/magicsock/magicsock_scion.go | 157 ++++++++++++++++++++- wgengine/magicsock/magicsock_scion_conn.go | 4 + wgengine/magicsock/magicsock_scion_omit.go | 1 + wgengine/magicsock/magicsock_scion_test.go | 58 ++++++++ 5 files changed, 216 insertions(+), 7 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 251a75aaf8fe3..a85b5ba6e942d 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3332,6 +3332,9 @@ func (c *connBind) Open(ignoredPort uint16) ([]conn.ReceiveFunc, uint16, error) } if c.pconnSCION != nil { fns = append(fns, c.receiveSCION) + if c.pconnSCION.shimXPC != nil { + fns = append(fns, c.receiveSCIONShim) + } } // TODO: Combine receiveIPv4 and receiveIPv6 and receiveIP into a single // closure that closes over a *RebindingUDPConn? diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index b775088d89c99..27d4f3f69a93c 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -50,10 +50,16 @@ var debugSCIONPreference = envknob.RegisterInt("TS_SCION_PREFERENCE") // Other paths are only used if no SCION path is available. var preferSCION = envknob.RegisterBool("TS_PREFER_SCION") +// scionDispatcherPort is the legacy SCION dispatcher port. Older deployments +// redirect all SCION traffic to this port instead of delivering to application +// ports directly. +const scionDispatcherPort = 30041 + var ( scionDaemonAddress = envknob.RegisterString("SCION_DAEMON_ADDRESS") scionPort = envknob.RegisterString("TS_SCION_PORT") scionListenAddrEnv = envknob.RegisterString("TS_SCION_LISTEN_ADDR") + noDispatcherShim = envknob.RegisterBool("TS_SCION_NO_DISPATCHER_SHIM") ) // scionPreferenceBonus returns the betterAddr points bonus for SCION paths. @@ -513,10 +519,15 @@ type scionConn struct { localPort uint16 // local SCION/UDP port daemon daemon.Connector // for path queries topo snet.Topology // local topology + shimConn *net.UDPConn // receive-only socket on port 30041; nil if unavailable + shimXPC scionBatchRW // batch reader for shim socket } // close shuts down the SCION connection and daemon connector. func (sc *scionConn) close() error { + if sc.shimConn != nil { + sc.shimConn.Close() + } if sc.conn != nil { sc.conn.Close() } @@ -526,9 +537,15 @@ func (sc *scionConn) close() error { return nil } -// closeSocket closes only the SCION socket (conn, underlayConn, underlayXPC), -// preserving the daemon connector and topology for socket-only reconnection. +// closeSocket closes only the SCION socket (conn, underlayConn, underlayXPC) +// and the dispatcher shim, preserving the daemon connector and topology for +// socket-only reconnection. func (sc *scionConn) closeSocket() { + if sc.shimConn != nil { + sc.shimConn.Close() + } + sc.shimConn = nil + sc.shimXPC = nil if sc.conn != nil { sc.conn.Close() } @@ -904,7 +921,7 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn } } - return &scionConn{ + sc := &scionConn{ conn: sconn, underlayConn: underlayConn, underlayXPC: underlayXPC, @@ -913,7 +930,69 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn localPort: localPort, daemon: connector, topo: topo, - }, nil + } + openDispatcherShim(sc, logf, netMon) + return sc, nil +} + +// openDispatcherShim tries to bind a receive-only UDP socket on the legacy +// dispatcher port (30041). In older SCION deployments, border routers send all +// packets to this port instead of directly to the application's endhost port. +// If binding succeeds (no dispatcher running), the shim socket receives packets +// identically to the main socket. If binding fails (EADDRINUSE), we log and +// continue — the real dispatcher handles forwarding. +func openDispatcherShim(sc *scionConn, logf logger.Logf, netMon *netmon.Monitor) { + if noDispatcherShim() { + logf("magicsock: SCION dispatcher shim disabled via TS_SCION_NO_DISPATCHER_SHIM") + return + } + if sc.localPort == scionDispatcherPort { + logf("magicsock: SCION main socket already on dispatcher port %d, skipping shim", scionDispatcherPort) + return + } + + shimAddr := &net.UDPAddr{ + IP: sc.localHostIP.AsSlice(), + Port: scionDispatcherPort, + } + shimConn, err := net.ListenUDP("udp", shimAddr) + if err != nil { + logf("magicsock: SCION dispatcher shim on :%d: %v (continuing without shim)", scionDispatcherPort, err) + return + } + + if err := shimConn.SetReadBuffer(socketBufferSize); err != nil { + logf("magicsock: SCION shim: failed to set read buffer to %d: %v", socketBufferSize, err) + } + + // Apply platform-specific socket options (SO_MARK, VPN isolation) + // to prevent the shim socket from routing through the VPN tunnel. + if netMon != nil { + rawConn, err := shimConn.SyscallConn() + if err == nil { + lc := netns.Listener(logf, netMon) + if lc.Control != nil { + if err := lc.Control("udp", shimConn.LocalAddr().String(), rawConn); err != nil { + logf("magicsock: SCION shim: netns control: %v", err) + } + } + } else { + logf("magicsock: SCION shim: SyscallConn: %v", err) + } + } + + // Wrap for batch I/O, selecting address family based on local address. + var xpc scionBatchRW + local := shimConn.LocalAddr().(*net.UDPAddr) + if local.IP.To4() != nil { + xpc = ipv4.NewPacketConn(shimConn) + } else { + xpc = ipv6.NewPacketConn(shimConn) + } + + sc.shimConn = shimConn + sc.shimXPC = xpc + logf("magicsock: SCION dispatcher shim listening on %s", shimConn.LocalAddr()) } // parseSCIONServiceAddr parses a SCION service description string of the form @@ -1221,7 +1300,7 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) // On shutdown, closeSCIONBindLocked sets an immediate deadline. // On reconnection, the old socket is closed → net.ErrClosed. if sc.underlayXPC != nil { - n, err := c.receiveSCIONBatch(sc, buffs, sizes, eps) + n, err := c.receiveSCIONBatch(sc.underlayXPC, buffs, sizes, eps) if n > 0 { return n, nil } @@ -1315,11 +1394,75 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } } +// receiveSCIONShim is the conn.ReceiveFunc for the legacy dispatcher shim +// socket (port 30041). It reads SCION packets identically to the main socket's +// batch path, reusing receiveSCIONBatch for all parsing and disco handling. +// +// Unlike receiveSCION, this function does not trigger reconnections (that is +// the main socket's responsibility) and has no slow-path fallback (the shim +// is always a raw *net.UDPConn). +func (c *Conn) receiveSCIONShim(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { + sc := c.pconnSCION + if sc == nil || sc.shimXPC == nil { + <-c.donec + return 0, net.ErrClosed + } + + for { + select { + case <-c.donec: + return 0, net.ErrClosed + default: + } + + // Re-read pconnSCION — it may have been swapped by reconnectSCION. + sc = c.pconnSCION + if sc == nil { + // Main socket reconnection in progress. Wait and retry; + // the reconnect may or may not rebind port 30041. + select { + case <-c.donec: + return 0, net.ErrClosed + case <-time.After(5 * time.Second): + } + continue + } + if sc.shimXPC == nil { + // Shim was not rebound after reconnection. Wait and retry. + select { + case <-c.donec: + return 0, net.ErrClosed + case <-time.After(5 * time.Second): + } + continue + } + + n, err := c.receiveSCIONBatch(sc.shimXPC, buffs, sizes, eps) + if n > 0 { + return n, nil + } + if err != nil { + select { + case <-c.donec: + return 0, net.ErrClosed + default: + } + if errors.Is(err, net.ErrClosed) || isTimeoutError(err) { + continue + } + c.logf("magicsock: SCION shim read error: %v", err) + continue + } + // n == 0 and no error means all packets were disco/filtered. + continue + } +} + // receiveSCIONBatch reads a batch of raw SCION packets from the underlay // socket via recvmmsg, parses SCION+UDP headers with slayers, and copies // payloads into WireGuard's buffs. Disco packets are handled inline and // not reported to the caller. -func (c *Conn) receiveSCIONBatch(sc *scionConn, buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { +func (c *Conn) receiveSCIONBatch(xpc scionBatchRW, buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { batch := scionRecvBatchPool.Get().(*scionRecvBatch) defer putScionRecvBatch(batch) @@ -1328,7 +1471,7 @@ func (c *Conn) receiveSCIONBatch(sc *scionConn, buffs [][]byte, sizes []int, eps n = scionMaxBatchSize } - numMsgs, err := sc.underlayXPC.ReadBatch(batch.msgs[:n], 0) + numMsgs, err := xpc.ReadBatch(batch.msgs[:n], 0) if err != nil { return 0, err } diff --git a/wgengine/magicsock/magicsock_scion_conn.go b/wgengine/magicsock/magicsock_scion_conn.go index 2e99b4205a6c5..c71e1466e7bc0 100644 --- a/wgengine/magicsock/magicsock_scion_conn.go +++ b/wgengine/magicsock/magicsock_scion_conn.go @@ -43,5 +43,9 @@ func (c *Conn) closeSCIONBindLocked() { // Set an immediate read deadline to unblock receiveSCION. // We don't close the SCION socket here; Conn.Close handles that. c.pconnSCION.conn.SetReadDeadline(time.Now()) + // Also unblock the dispatcher shim's ReadBatch if present. + if c.pconnSCION.shimConn != nil { + c.pconnSCION.shimConn.SetReadDeadline(time.Now()) + } } } diff --git a/wgengine/magicsock/magicsock_scion_omit.go b/wgengine/magicsock/magicsock_scion_omit.go index 8ffabdc28bf4a..edcff8eb8ec76 100644 --- a/wgengine/magicsock/magicsock_scion_omit.go +++ b/wgengine/magicsock/magicsock_scion_omit.go @@ -34,6 +34,7 @@ func (c *Conn) initSCIONLocked(_ context.Context) func (c *Conn) closeSCIONLocked() {} func (c *Conn) closeSCIONBindLocked() {} func (c *Conn) receiveSCION(_ [][]byte, _ []int, _ []wgconn.Endpoint) (int, error) { return 0, nil } +func (c *Conn) receiveSCIONShim(_ [][]byte, _ []int, _ []wgconn.Endpoint) (int, error) { return 0, nil } func (c *Conn) sendSCION(_ scionPathKey, _ []byte) (bool, error) { return false, nil } func (c *Conn) unregisterSCIONPath(_ scionPathKey) {} diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index d3c426da99169..360651b8756cb 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -1848,3 +1848,61 @@ func TestScionProbeSCIONNonBestLocked(t *testing.T) { t.Error("round-robin should pick different paths on consecutive calls") } } + +func TestDispatcherShim(t *testing.T) { + t.Run("binds_when_port_available", func(t *testing.T) { + sc := &scionConn{ + localHostIP: netip.MustParseAddr("127.0.0.1"), + localPort: 32766, + } + openDispatcherShim(sc, t.Logf, nil) + if sc.shimConn == nil { + t.Fatal("expected shimConn to be set when port 30041 is available") + } + defer sc.shimConn.Close() + if sc.shimXPC == nil { + t.Fatal("expected shimXPC to be set") + } + addr := sc.shimConn.LocalAddr().(*net.UDPAddr) + if addr.Port != scionDispatcherPort { + t.Errorf("shimConn port = %d, want %d", addr.Port, scionDispatcherPort) + } + }) + + t.Run("graceful_on_EADDRINUSE", func(t *testing.T) { + // Occupy port 30041 first. + blocker, err := net.ListenUDP("udp", &net.UDPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: scionDispatcherPort, + }) + if err != nil { + t.Skipf("cannot bind port %d for test: %v", scionDispatcherPort, err) + } + defer blocker.Close() + + sc := &scionConn{ + localHostIP: netip.MustParseAddr("127.0.0.1"), + localPort: 32766, + } + openDispatcherShim(sc, t.Logf, nil) + if sc.shimConn != nil { + sc.shimConn.Close() + t.Fatal("expected shimConn to be nil when port is already in use") + } + if sc.shimXPC != nil { + t.Fatal("expected shimXPC to be nil when port is already in use") + } + }) + + t.Run("skipped_when_main_on_dispatcher_port", func(t *testing.T) { + sc := &scionConn{ + localHostIP: netip.MustParseAddr("127.0.0.1"), + localPort: scionDispatcherPort, + } + openDispatcherShim(sc, t.Logf, nil) + if sc.shimConn != nil { + sc.shimConn.Close() + t.Fatal("expected shimConn to be nil when main socket is on dispatcher port") + } + }) +} From 5d7eddd306b07e4d66dea8433a05259a867f74d5 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Sat, 14 Mar 2026 16:06:39 +0000 Subject: [PATCH 49/78] wgengine/magicsock: update SCION path handling and add new tests - Added wireMTU field to addrQuality struct for SCION path management. - Refactored pingSizeToPktLen to return a fixed wire MTU for SCION paths. - Enhanced SCION path info string representation for better debugging. - Introduced new tests for SCION path formatting and display string generation. - Updated go.mod to include gopacket as a direct dependency. --- go.mod | 2 +- wgengine/magicsock/endpoint.go | 37 +++-- wgengine/magicsock/endpoint_scion.go | 2 + wgengine/magicsock/magicsock.go | 8 +- wgengine/magicsock/magicsock_scion.go | 154 +++++++++++++++++---- wgengine/magicsock/magicsock_scion_test.go | 104 ++++++++++++++ 6 files changed, 268 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index a602e4658715e..7f5066e134eaf 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/google/gopacket v1.1.19 github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806 github.com/google/uuid v1.6.0 + github.com/gopacket/gopacket v1.5.0 github.com/goreleaser/nfpm/v2 v2.33.1 github.com/hashicorp/go-hclog v1.6.2 github.com/hashicorp/raft v1.7.2 @@ -199,7 +200,6 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/renameio/v2 v2.0.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/gopacket/gopacket v1.5.0 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosuri/uitable v0.0.4 // indirect diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index b091502ad4139..8bbdf889ef8c4 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1683,14 +1683,14 @@ func (de *endpoint) noteConnectivityChange() { // udpAddr. size is the length of the entire disco message including // disco headers. If size is zero, assume it is the safe wire MTU. func pingSizeToPktLen(size int, udpAddr epAddr) tstun.WireMTU { - if size == 0 { - return tstun.SafeWireMTU() - } if udpAddr.scionKey.IsSet() { - // For SCION paths, the snet library handles all SCION header - // serialization internally. The payload we pass to WriteTo is the - // WireGuard packet. Since we skip MTU probing for SCION (the path - // MTU is known from metadata), just return the safe wire MTU. + // SCION wire MTU is fixed regardless of ping size: the WireGuard + // packet must fit inside the SCION path's payload budget (path MTU + // minus variable SCION headers). We use a conservative value + // rather than computing per-path overhead from the hop count. + return scionWireMTU + } + if size == 0 { return tstun.SafeWireMTU() } headerLen := ipv4.HeaderLen @@ -1808,7 +1808,18 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd // getting stuck with a dead bestAddr that betterAddr() refuses to // demote due to preference rules (e.g., TS_PREFER_SCION=1). curBestUntrusted := de.bestAddr.ap.IsValid() && now.After(de.trustBestAddrUntil) - if curBestUntrusted || betterAddr(thisPong, de.bestAddr) { + + // When the current bestAddr is a trusted SCION path and this pong + // is from a different SCION path on the same host, skip generic + // betterAddr promotion. The SCION-aware reEvalSCIONPathsLocked + // (throttled to 2s) handles multi-path switching to avoid flapping + // between paths with similar latency. + scionToScion := thisPong.epAddr.isSCION() && de.bestAddr.isSCION() && + thisPong.epAddr.ap == de.bestAddr.ap && + thisPong.epAddr.scionKey != de.bestAddr.scionKey + skipPromotion := scionToScion && !curBestUntrusted + + if !skipPromotion && (curBestUntrusted || betterAddr(thisPong, de.bestAddr)) { if thisPong.epAddr != de.bestAddr.epAddr { de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v tx=%x", de.publicKey.ShortString(), de.discoShort(), sp.to, thisPong.wireMTU, m.TxID[:6]) de.debugUpdates.Add(EndpointChange{ @@ -2103,7 +2114,15 @@ func (de *endpoint) populatePeerStatus(ps *ipnstate.PeerStatus) { ps.Active = now.Sub(de.lastSendExt) < sessionActiveTimeout if udpAddr, derpAddr, _ := de.addrForSendLocked(now); udpAddr.ap.IsValid() && !derpAddr.IsValid() { - if udpAddr.vni.IsSet() { + if udpAddr.isSCION() { + // Access c.scionPaths directly — c.mu is already held by + // our caller (Conn.UpdateStatus). + if pi, ok := de.c.scionPaths[udpAddr.scionKey]; ok { + ps.CurAddr = pi.String() + } else { + ps.CurAddr = udpAddr.String() + } + } else if udpAddr.vni.IsSet() { ps.PeerRelay = udpAddr.String() } else { ps.CurAddr = udpAddr.String() diff --git a/wgengine/magicsock/endpoint_scion.go b/wgengine/magicsock/endpoint_scion.go index b57311ab2897d..4132a958eba3e 100644 --- a/wgengine/magicsock/endpoint_scion.go +++ b/wgengine/magicsock/endpoint_scion.go @@ -287,6 +287,7 @@ func (de *endpoint) demoteSCIONPathLocked(demotedKey scionPathKey) { newAddr := addrQuality{ epAddr: epAddr{ap: de.scionState.hostAddr, scionKey: bestKey}, latency: bestLatency, + wireMTU: scionWireMTU, scionPreferred: de.scionPreferred, } de.c.logf("magicsock: SCION path demoted, switching to path %d for %v", bestKey, de.publicKey.ShortString()) @@ -343,6 +344,7 @@ func (de *endpoint) reEvalSCIONPathsLocked(now mono.Time) { candidate := addrQuality{ epAddr: epAddr{ap: de.scionState.hostAddr, scionKey: bestKey}, latency: bestLatency, + wireMTU: scionWireMTU, scionPreferred: de.scionPreferred, } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index a85b5ba6e942d..f010b3bad7bce 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -1201,7 +1201,13 @@ func (c *Conn) Ping(peer tailcfg.NodeView, res *ipnstate.PingResult, size int, c func (c *Conn) populateCLIPingResponseLocked(res *ipnstate.PingResult, latency time.Duration, ep epAddr) { res.LatencySeconds = latency.Seconds() if ep.ap.Addr() != tailcfg.DerpMagicIPAddr { - if ep.vni.IsSet() { + if ep.isSCION() { + if pi, ok := c.scionPaths[ep.scionKey]; ok { + res.Endpoint = pi.String() + } else { + res.Endpoint = ep.String() + } + } else if ep.vni.IsSet() { res.PeerRelay = ep.String() } else { res.Endpoint = ep.String() diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 27d4f3f69a93c..d49b324566421 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -34,6 +34,7 @@ import ( "tailscale.com/envknob" "tailscale.com/net/netmon" "tailscale.com/net/netns" + "tailscale.com/net/tstun" "tailscale.com/tailcfg" "tailscale.com/tstime/mono" "tailscale.com/types/key" @@ -60,6 +61,7 @@ var ( scionPort = envknob.RegisterString("TS_SCION_PORT") scionListenAddrEnv = envknob.RegisterString("TS_SCION_LISTEN_ADDR") noDispatcherShim = envknob.RegisterBool("TS_SCION_NO_DISPATCHER_SHIM") + scionNoFastPath = envknob.RegisterBool("TS_SCION_NO_FAST_PATH") ) // scionPreferenceBonus returns the betterAddr points bonus for SCION paths. @@ -103,9 +105,50 @@ type scionPathInfo struct { expiry time.Time // path expiration from path metadata mtu uint16 // SCION payload MTU from path metadata refreshMissCount int // consecutive refresh cycles fingerprint absent from daemon + displayStr string // pre-computed human-readable path string mu sync.Mutex } +// String returns the pre-computed human-readable display string for this path. +// Format: scion:[srcIA ifid>ifid transitIA ... dstIA]:[host]:port +func (pi *scionPathInfo) String() string { + return pi.displayStr +} + +// buildDisplayStr pre-computes the human-readable display string from the +// current path metadata and host address. Must be called with pi.mu held +// (or before the info is shared), and whenever the path is updated. +func (pi *scionPathInfo) buildDisplayStr() { + hops := "?" + if pi.path != nil { + if md := pi.path.Metadata(); md != nil && len(md.Interfaces) > 0 { + hops = formatSCIONHops(md.Interfaces) + } + } + pi.displayStr = fmt.Sprintf("scion:[%s]:[%s]:%d", + hops, pi.hostAddr.Addr(), pi.hostAddr.Port()) +} + +// formatSCIONHops formats SCION path interfaces into standard hop notation. +// Produces: "19-ffaa:1:eba 2>2 19-ffaa:1:bf5" for a 2-hop path. +// Mirrors the format used by snet/path.fmtInterfaces and `scion showpaths`. +func formatSCIONHops(ifaces []snet.PathInterface) string { + if len(ifaces) == 0 { + return "?" + } + var sb strings.Builder + // First interface: srcIA ifid + fmt.Fprintf(&sb, "%s %d", ifaces[0].IA, ifaces[0].ID) + // Middle interfaces come in pairs: entry-ifid transitIA exit-ifid + for i := 1; i < len(ifaces)-1; i += 2 { + fmt.Fprintf(&sb, ">%d %s %d", ifaces[i].ID, ifaces[i].IA, ifaces[i+1].ID) + } + // Last interface: ifid dstIA + last := ifaces[len(ifaces)-1] + fmt.Fprintf(&sb, ">%d %s", last.ID, last.IA) + return sb.String() +} + // buildCachedDst constructs the cached destination address from the current // path info. Must be called with pi.mu held (or before the info is shared). func (pi *scionPathInfo) buildCachedDst() { @@ -123,23 +166,20 @@ func (pi *scionPathInfo) buildCachedDst() { pi.cachedDst = dst } -// scionHeaderOverhead is the fixed overhead added by SCION encapsulation, -// excluding the variable-length path header: -// - Underlay IPv4+UDP: 20 + 8 = 28 bytes (or IPv6+UDP: 40 + 8 = 48 bytes) +// The SCION path metadata MTU is the maximum SCION packet size that can +// traverse the path (including all SCION headers but excluding underlay +// IP+UDP). The actual payload budget depends on the variable-length path +// header, which grows with hop count: // - SCION common header: 12 bytes -// - Address header (IPv4, 2x ISD-AS + 2x IPv4): 2*8 + 2*4 = 24 bytes +// - Address header (IPv4, 2x ISD-AS + 2x IPv4): 24 bytes +// - Path header: ~36 bytes (2 hops) to ~96 bytes (6+ hops) // - SCION/UDP L4 header: 8 bytes // -// Total fixed: 72 bytes. The path header is variable (depends on hop count). -// Rather than parsing the path to determine the exact overhead, we use the -// path MTU from metadata directly: the SCION daemon reports the maximum -// *payload* size that can traverse the path (i.e., MTU already accounts for -// all SCION headers). So the effective wire MTU for WireGuard is simply the -// SCION path MTU. -// -// However, when no path MTU is available, we use a conservative estimate: -// 1280 bytes (minimum IPv6-compatible MTU). -const scionFallbackPayloadMTU = 1280 +// Rather than computing exact overhead per path, we use a conservative +// wire MTU of 1280 bytes (the minimum IPv6 link MTU). This guarantees +// WireGuard packets fit within any SCION path's payload budget regardless +// of hop count. +const scionWireMTU = tstun.WireMTU(1280) // scionUnsetHopLatency is the assumed per-hop latency when the SCION daemon // reports LatencyUnset for a hop. Conservative estimate for path selection. @@ -465,8 +505,10 @@ func parseSCIONPacket(data []byte, scn *slayers.SCION) ( } // buildSCIONReplyAddr builds an *snet.UDPAddr with reversed path for disco -// reply routing from raw path bytes extracted during receive. -func buildSCIONReplyAddr(srcIA addr.IA, srcHostAddr netip.AddrPort, rawPathBytes []byte) *snet.UDPAddr { +// reply routing from raw path bytes extracted during receive. nextHop is the +// underlay border router address from the incoming packet (msg.Addr from +// recvmmsg); it is required for the reply to be routable. +func buildSCIONReplyAddr(srcIA addr.IA, srcHostAddr netip.AddrPort, rawPathBytes []byte, nextHop *net.UDPAddr) *snet.UDPAddr { if len(rawPathBytes) == 0 { return nil } @@ -495,7 +537,8 @@ func buildSCIONReplyAddr(srcIA addr.IA, srcHostAddr netip.AddrPort, rawPathBytes IP: srcHostAddr.Addr().AsSlice(), Port: int(srcHostAddr.Port()), }, - Path: snetpath.SCION{Raw: revBytes}, + Path: snetpath.SCION{Raw: revBytes}, + NextHop: nextHop, } } @@ -1048,7 +1091,7 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo } // Fast path: pre-serialized headers + sendmmsg. - if fastPath != nil && sc.underlayXPC != nil { + if fastPath != nil && sc.underlayXPC != nil && !scionNoFastPath() { err = c.sendSCIONBatchFast(sc, fastPath, buffs, offset) if err != nil { c.handleSCIONSendError(err) @@ -1250,6 +1293,21 @@ func (c *Conn) updateActiveSCIONPathLocking(peerIA addr.IA, hostAddr netip.AddrP c.setActiveSCIONPath(peerIA, hostAddr, k) } +// scionPathString returns the human-readable display string for a SCION path +// key. Returns "scion:" as fallback if the key is not found in the +// registry. Acquires c.mu. +func (c *Conn) scionPathString(key scionPathKey) string { + if !key.IsSet() { + return "" + } + c.mu.Lock() + defer c.mu.Unlock() + if pi, ok := c.scionPaths[key]; ok { + return pi.String() + } + return fmt.Sprintf("scion:%d", key) +} + // receiveSCION is the conn.ReceiveFunc for SCION packets. It reads from the // SCION connection and dispatches disco or WireGuard packets. // @@ -1498,7 +1556,12 @@ func (c *Conn) receiveSCIONBatch(xpc scionBatchRW, buffs [][]byte, sizes []int, pt, _ := packetLooksLike(buffs[count][:pn]) if pt == packetLooksLikeDisco { - c.handleSCIONDisco(buffs[count][:pn], srcIA, srcHostAddr, rawPath) + // Extract underlay source address (border router) for reply NextHop. + var nextHop *net.UDPAddr + if ua, ok := msg.Addr.(*net.UDPAddr); ok { + nextHop = ua + } + c.handleSCIONDisco(buffs[count][:pn], srcIA, srcHostAddr, rawPath, nextHop) continue } @@ -1540,14 +1603,15 @@ func (c *Conn) receiveSCIONBatch(xpc scionBatchRW, buffs [][]byte, sizes []int, // handleSCIONDisco handles a disco packet received on the batch path. // It looks up or registers a SCION path entry and dispatches to handleDiscoMessage. // For first-contact, the raw path bytes are reversed to build a reply path. -func (c *Conn) handleSCIONDisco(b []byte, srcIA addr.IA, srcHostAddr netip.AddrPort, rawPath []byte) { +// nextHop is the underlay border router address from the incoming packet. +func (c *Conn) handleSCIONDisco(b []byte, srcIA addr.IA, srcHostAddr netip.AddrPort, rawPath []byte, nextHop *net.UDPAddr) { srcEpAddr := epAddr{ap: srcHostAddr} c.mu.Lock() sk := c.scionPathsByAddr[scionAddrKey{ia: srcIA, addr: srcHostAddr}] if !sk.IsSet() { // First disco packet from this SCION peer — build a reply path // by reversing the raw SCION path from the incoming packet. - replyAddr := buildSCIONReplyAddr(srcIA, srcHostAddr, rawPath) + replyAddr := buildSCIONReplyAddr(srcIA, srcHostAddr, rawPath, nextHop) sk = c.registerSCIONPath(&scionPathInfo{ peerIA: srcIA, hostAddr: srcHostAddr, @@ -1756,6 +1820,7 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr mtu: mtu, } pi.buildCachedDst() + pi.buildDisplayStr() if sc := c.pconnSCION; sc != nil { pi.fastPath = buildSCIONFastPath(sc, pi) } @@ -2083,6 +2148,7 @@ func (c *Conn) refreshSCIONPathsOnce() error { pi.mtu = md.MTU } pi.buildCachedDst() + pi.buildDisplayStr() pi.fastPath = buildSCIONFastPath(sc, pi) pi.mu.Unlock() } @@ -2091,7 +2157,11 @@ func (c *Conn) refreshSCIONPathsOnce() error { if len(stalePaths) > 0 { c.mu.Lock() for _, k := range stalePaths { - c.logf("magicsock: SCION path %d stale for %s, removing", k, g.peerIA) + pathStr := fmt.Sprintf("scion:%d", k) + if pi, ok := c.scionPaths[k]; ok { + pathStr = pi.String() + } + c.logf("magicsock: SCION path stale for %s, removing %s", g.peerIA, pathStr) c.unregisterSCIONPath(k) } c.mu.Unlock() @@ -2153,8 +2223,10 @@ func (c *Conn) refreshSCIONPathsOnce() error { } if len(newPaths) > 0 { - c.addNewSCIONPathsForPeer(g.peerIA, g.hostAddr, newPaths) - c.logf("magicsock: SCION soft refresh: %d new paths for %s", len(newPaths), g.peerIA) + newKeys := c.addNewSCIONPathsForPeer(g.peerIA, g.hostAddr, newPaths) + for i, k := range newKeys { + c.logf("magicsock: SCION soft refresh for %s: [%d] %s", g.peerIA, i, c.scionPathString(k)) + } } c.mu.Lock() @@ -2167,11 +2239,11 @@ func (c *Conn) refreshSCIONPathsOnce() error { // addNewSCIONPathsForPeer registers new SCION paths and adds probe states // to the corresponding endpoint. Called during soft refresh when new paths -// appear in the daemon's cache. -func (c *Conn) addNewSCIONPathsForPeer(peerIA addr.IA, hostAddr netip.AddrPort, paths []snet.Path) { +// appear in the daemon's cache. Returns the registered path keys. +func (c *Conn) addNewSCIONPathsForPeer(peerIA addr.IA, hostAddr netip.AddrPort, paths []snet.Path) []scionPathKey { sc := c.pconnSCION if sc == nil { - return + return nil } c.mu.Lock() @@ -2193,6 +2265,7 @@ func (c *Conn) addNewSCIONPathsForPeer(peerIA addr.IA, hostAddr netip.AddrPort, mtu: mtu, } pi.buildCachedDst() + pi.buildDisplayStr() pi.fastPath = buildSCIONFastPath(sc, pi) k := c.registerSCIONPath(pi) newKeys = append(newKeys, k) @@ -2216,6 +2289,7 @@ func (c *Conn) addNewSCIONPathsForPeer(peerIA addr.IA, hostAddr netip.AddrPort, ep.mu.Unlock() } c.mu.Unlock() + return newKeys } // cleanStaleSCIONPathFromEndpoints removes stale SCION path keys from all @@ -2419,7 +2493,31 @@ func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPo } de.mu.Unlock() - de.c.logf("magicsock: discovered %d SCION paths to %s (active key=%d)", len(newKeys), peerIA, activePath) + for i, k := range newKeys { + active := "" + if k == activePath { + active = " (active)" + } + var mtuInfo string + de.c.mu.Lock() + if pi, ok := de.c.scionPaths[k]; ok { + hdrLen := 0 + if pi.fastPath != nil { + hdrLen = len(pi.fastPath.hdr) + } + maxWG := int(pi.mtu) - hdrLen + mtuInfo = fmt.Sprintf(" pathMTU=%d hdr=%d maxWG=%d", pi.mtu, hdrLen, maxWG) + // WG overhead: 4 type + 4 receiver + 8 counter + 16 tag = 32 bytes. + // Max TUN packet that fits: maxWG - 32. + const wgOverhead = 32 + if pi.mtu > 0 && hdrLen > 0 && maxWG < 1280+wgOverhead { + de.c.logf("magicsock: WARNING: SCION path MTU %d too small for TUN 1280 (need %d, have %d for WG payload)", + pi.mtu, 1280+wgOverhead+hdrLen, pi.mtu) + } + } + de.c.mu.Unlock() + de.c.logf("magicsock: SCION path to %s: [%d] %s%s%s", peerIA, i, de.c.scionPathString(k), mtuInfo, active) + } } // scionKeyForAddr returns the scionPathKey for the given peer IA and host diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 360651b8756cb..bee0e460be68f 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -1906,3 +1906,107 @@ func TestDispatcherShim(t *testing.T) { } }) } + +func TestFormatSCIONHops(t *testing.T) { + mustIA := func(s string) addr.IA { + ia, err := addr.ParseIA(s) + if err != nil { + t.Fatalf("invalid IA %q: %v", s, err) + } + return ia + } + + tests := []struct { + name string + ifaces []snet.PathInterface + want string + }{ + { + name: "empty", + ifaces: nil, + want: "?", + }, + { + name: "2-hop direct", + ifaces: []snet.PathInterface{ + {IA: mustIA("19-ffaa:1:eba"), ID: 2}, + {IA: mustIA("19-ffaa:1:bf5"), ID: 2}, + }, + want: "19-ffaa:1:eba 2>2 19-ffaa:1:bf5", + }, + { + name: "3-hop via transit", + ifaces: []snet.PathInterface{ + {IA: mustIA("19-ffaa:1:eba"), ID: 1}, + {IA: mustIA("19-ffaa:0:1303"), ID: 62}, + {IA: mustIA("19-ffaa:0:1303"), ID: 104}, + {IA: mustIA("19-ffaa:1:bf5"), ID: 1}, + }, + want: "19-ffaa:1:eba 1>62 19-ffaa:0:1303 104>1 19-ffaa:1:bf5", + }, + { + name: "4-hop two transits", + ifaces: []snet.PathInterface{ + {IA: mustIA("19-ffaa:1:eba"), ID: 1}, + {IA: mustIA("19-ffaa:0:1"), ID: 3}, + {IA: mustIA("19-ffaa:0:1"), ID: 4}, + {IA: mustIA("19-ffaa:0:2"), ID: 5}, + {IA: mustIA("19-ffaa:0:2"), ID: 6}, + {IA: mustIA("19-ffaa:1:bf5"), ID: 1}, + }, + want: "19-ffaa:1:eba 1>3 19-ffaa:0:1 4>5 19-ffaa:0:2 6>1 19-ffaa:1:bf5", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := formatSCIONHops(tt.ifaces) + if got != tt.want { + t.Errorf("formatSCIONHops() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestScionPathInfoString(t *testing.T) { + mustIA := func(s string) addr.IA { + ia, err := addr.ParseIA(s) + if err != nil { + t.Fatalf("invalid IA %q: %v", s, err) + } + return ia + } + + pi := &scionPathInfo{ + peerIA: mustIA("19-ffaa:1:bf5"), + hostAddr: netip.MustParseAddrPort("127.0.0.1:32766"), + path: snetpath.Path{ + Src: mustIA("19-ffaa:1:eba"), + Dst: mustIA("19-ffaa:1:bf5"), + Meta: snet.PathMetadata{ + Interfaces: []snet.PathInterface{ + {IA: mustIA("19-ffaa:1:eba"), ID: 2}, + {IA: mustIA("19-ffaa:1:bf5"), ID: 2}, + }, + MTU: 1472, + }, + }, + } + pi.buildDisplayStr() + + want := "scion:[19-ffaa:1:eba 2>2 19-ffaa:1:bf5]:[127.0.0.1]:32766" + if got := pi.String(); got != want { + t.Errorf("scionPathInfo.String() = %q, want %q", got, want) + } + + // Test with no metadata + piNoMeta := &scionPathInfo{ + peerIA: mustIA("19-ffaa:1:bf5"), + hostAddr: netip.MustParseAddrPort("127.0.0.1:32766"), + } + piNoMeta.buildDisplayStr() + + wantNoMeta := "scion:[?]:[127.0.0.1]:32766" + if got := piNoMeta.String(); got != wantNoMeta { + t.Errorf("scionPathInfo.String() no metadata = %q, want %q", got, wantNoMeta) + } +} From 46d1db132da526824d1bcfeb0ab2af76a0397e0e Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Sat, 14 Mar 2026 16:27:28 +0000 Subject: [PATCH 50/78] wgengine/magicsock: enhance SCION path formatting and handling - Added support for single interface cases in SCION path formatting, ensuring graceful handling even when only one interface is present. - Updated the formatSCIONHops function to return a formatted string for single interfaces. - Improved the display string generation for SCION path information to enhance debugging capabilities. - Added a new test case to validate the formatting of SCION hops with a single interface. --- wgengine/magicsock/magicsock_scion.go | 44 ++++++++++++++++------ wgengine/magicsock/magicsock_scion_test.go | 7 ++++ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index d49b324566421..5f98d6755cfe1 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -136,6 +136,11 @@ func formatSCIONHops(ifaces []snet.PathInterface) string { if len(ifaces) == 0 { return "?" } + if len(ifaces) == 1 { + // Single interface shouldn't occur in valid SCION paths + // (interfaces always come in pairs), but handle gracefully. + return fmt.Sprintf("%s %d", ifaces[0].IA, ifaces[0].ID) + } var sb strings.Builder // First interface: srcIA ifid fmt.Fprintf(&sb, "%s %d", ifaces[0].IA, ifaces[0].ID) @@ -1412,11 +1417,13 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) c.mu.Lock() sk := c.scionPathsByAddr[scionAddrKey{ia: srcAddr.IA, addr: srcHostAddr}] if !sk.IsSet() { - sk = c.registerSCIONPath(&scionPathInfo{ + pi := &scionPathInfo{ peerIA: srcAddr.IA, hostAddr: srcHostAddr, replyPath: srcAddr, - }) + } + pi.buildDisplayStr() + sk = c.registerSCIONPath(pi) c.setActiveSCIONPath(srcAddr.IA, srcHostAddr, sk) } c.mu.Unlock() @@ -1612,11 +1619,13 @@ func (c *Conn) handleSCIONDisco(b []byte, srcIA addr.IA, srcHostAddr netip.AddrP // First disco packet from this SCION peer — build a reply path // by reversing the raw SCION path from the incoming packet. replyAddr := buildSCIONReplyAddr(srcIA, srcHostAddr, rawPath, nextHop) - sk = c.registerSCIONPath(&scionPathInfo{ + pi := &scionPathInfo{ peerIA: srcIA, hostAddr: srcHostAddr, replyPath: replyAddr, - }) + } + pi.buildDisplayStr() + sk = c.registerSCIONPath(pi) c.setActiveSCIONPath(srcIA, srcHostAddr, sk) } c.mu.Unlock() @@ -1782,7 +1791,10 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr seen := make(map[snet.PathFingerprint]bool) var unique []pathWithMeta for _, p := range paths { - fp := p.Metadata().Fingerprint() + var fp snet.PathFingerprint + if md := p.Metadata(); md != nil { + fp = md.Fingerprint() + } if fp != "" && seen[fp] { continue } @@ -2087,9 +2099,13 @@ func (c *Conn) refreshSCIONPathsOnce() error { } var daemonByFP []daemonPathEntry for _, dp := range daemonPaths { + var fp snet.PathFingerprint + if md := dp.Metadata(); md != nil { + fp = md.Fingerprint() + } daemonByFP = append(daemonByFP, daemonPathEntry{ path: dp, - fp: dp.Metadata().Fingerprint(), + fp: fp, }) } @@ -2141,9 +2157,8 @@ func (c *Conn) refreshSCIONPathsOnce() error { pi.mu.Lock() pi.refreshMissCount = 0 pi.path = matched - newFP := matched.Metadata().Fingerprint() - pi.fingerprint = newFP if md := matched.Metadata(); md != nil { + pi.fingerprint = md.Fingerprint() pi.expiry = md.Expiry pi.mtu = md.MTU } @@ -2212,7 +2227,10 @@ func (c *Conn) refreshSCIONPathsOnce() error { var newPaths []snet.Path for _, dp := range daemonPaths { - fp := dp.Metadata().Fingerprint() + var fp snet.PathFingerprint + if md := dp.Metadata(); md != nil { + fp = md.Fingerprint() + } if fp == "" || knownFPs[fp] { continue } @@ -2256,10 +2274,14 @@ func (c *Conn) addNewSCIONPathsForPeer(peerIA addr.IA, hostAddr netip.AddrPort, expiry = md.Expiry mtu = md.MTU } + var fp snet.PathFingerprint + if md != nil { + fp = md.Fingerprint() + } pi := &scionPathInfo{ peerIA: peerIA, hostAddr: hostAddr, - fingerprint: md.Fingerprint(), + fingerprint: fp, path: p, expiry: expiry, mtu: mtu, @@ -2512,7 +2534,7 @@ func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPo const wgOverhead = 32 if pi.mtu > 0 && hdrLen > 0 && maxWG < 1280+wgOverhead { de.c.logf("magicsock: WARNING: SCION path MTU %d too small for TUN 1280 (need %d, have %d for WG payload)", - pi.mtu, 1280+wgOverhead+hdrLen, pi.mtu) + pi.mtu, 1280+wgOverhead+hdrLen, maxWG) } } de.c.mu.Unlock() diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index bee0e460be68f..070539fadb9d4 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -1926,6 +1926,13 @@ func TestFormatSCIONHops(t *testing.T) { ifaces: nil, want: "?", }, + { + name: "single interface", + ifaces: []snet.PathInterface{ + {IA: mustIA("19-ffaa:1:eba"), ID: 2}, + }, + want: "19-ffaa:1:eba 2", + }, { name: "2-hop direct", ifaces: []snet.PathInterface{ From 2e1c0340eaae7f9ab641cee7fa50c12b2329e728 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Sat, 14 Mar 2026 16:52:30 +0000 Subject: [PATCH 51/78] wgengine/magicsock: enhance SCION path re-evaluation logic and add tests - Introduced a constant for the minimum interval between SCION path re-evaluations to improve code clarity. - Updated the re-evaluation logic to prevent flapping between paths with similar latencies, requiring a minimum improvement threshold. - Refactored the latency calculation to return the median latency from recorded pongs, enhancing stability against outliers. - Added comprehensive tests for latency calculation and anti-flap behavior to ensure robust path management. --- wgengine/magicsock/endpoint_scion.go | 24 ++++- wgengine/magicsock/magicsock_scion.go | 17 ++- wgengine/magicsock/magicsock_scion_test.go | 116 +++++++++++++++++++++ 3 files changed, 152 insertions(+), 5 deletions(-) diff --git a/wgengine/magicsock/endpoint_scion.go b/wgengine/magicsock/endpoint_scion.go index 4132a958eba3e..4f132c5b5a1b9 100644 --- a/wgengine/magicsock/endpoint_scion.go +++ b/wgengine/magicsock/endpoint_scion.go @@ -303,15 +303,19 @@ func (de *endpoint) demoteSCIONPathLocked(demotedKey scionPathKey) { } } +// scionReEvalInterval is the minimum time between SCION path re-evaluations. +const scionReEvalInterval = 2 * time.Second + // reEvalSCIONPathsLocked re-evaluates all SCION paths by measured latency -// after a pong is recorded. Throttled to once per 2 seconds. If a healthier, -// lower-latency path is found, switches bestAddr and activePath. +// after a pong is recorded. Throttled to scionReEvalInterval. If a healthier, +// lower-latency path is found, switches bestAddr and activePath. Incumbent +// bias prevents flapping between paths with similar latency. // de.mu must be held. func (de *endpoint) reEvalSCIONPathsLocked(now mono.Time) { if de.scionState == nil || len(de.scionState.paths) < 2 { return } - if !de.scionState.lastFullEvalAt.IsZero() && now.Sub(de.scionState.lastFullEvalAt) < 2*time.Second { + if !de.scionState.lastFullEvalAt.IsZero() && now.Sub(de.scionState.lastFullEvalAt) < scionReEvalInterval { return } de.scionState.lastFullEvalAt = now @@ -341,6 +345,20 @@ func (de *endpoint) reEvalSCIONPathsLocked(now mono.Time) { return } + // Require meaningful improvement over active path to avoid flapping + // between paths with similar latency. The candidate must be ≥20% faster + // or ≥2ms faster (whichever threshold is smaller). + if activePS, ok := de.scionState.paths[de.scionState.activePath]; ok && activePS.healthy { + activeLat := activePS.latency() + threshold := activeLat / 5 // 20% + if minThreshold := 2 * time.Millisecond; threshold < minThreshold { + threshold = minThreshold + } + if activeLat-bestLatency < threshold { + return + } + } + candidate := addrQuality{ epAddr: epAddr{ap: de.scionState.hostAddr, scionKey: bestKey}, latency: bestLatency, diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 5f98d6755cfe1..a83bd2089a29b 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -15,6 +15,7 @@ import ( "net/netip" "os" "path/filepath" + "slices" "sort" "strings" "sync" @@ -253,12 +254,24 @@ func (ps *scionPathProbeState) addPongReply(r scionPongReply) { } } -// latency returns the most recent pong latency, or time.Hour if no pongs received. +// latency returns the median pong latency from available measurements, +// or time.Hour if no pongs received. The median is robust to single-sample +// outliers and provides stable path comparison for anti-flap logic. func (ps *scionPathProbeState) latency() time.Duration { if ps.pongCount == 0 { return time.Hour } - return ps.recentPongs[ps.recentPong].latency + n := int(ps.pongCount) + if n == 1 { + return ps.recentPongs[ps.recentPong].latency + } + samples := make([]time.Duration, n) + for i := range n { + idx := (int(ps.recentPong) - i + scionPongHistoryCount) % scionPongHistoryCount + samples[i] = ps.recentPongs[idx].latency + } + slices.Sort(samples) + return samples[n/2] } // scionFastPath holds a pre-serialized SCION+UDP header template for a diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 070539fadb9d4..2d5961ec492b5 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -2017,3 +2017,119 @@ func TestScionPathInfoString(t *testing.T) { t.Errorf("scionPathInfo.String() no metadata = %q, want %q", got, wantNoMeta) } } + +func TestScionLatencyMedian(t *testing.T) { + tests := []struct { + name string + samples []time.Duration + want time.Duration + }{ + { + name: "no samples", + samples: nil, + want: time.Hour, + }, + { + name: "single sample", + samples: []time.Duration{10 * time.Millisecond}, + want: 10 * time.Millisecond, + }, + { + name: "two samples returns higher (index 1)", + samples: []time.Duration{8 * time.Millisecond, 12 * time.Millisecond}, + want: 12 * time.Millisecond, + }, + { + name: "four samples returns median", + samples: []time.Duration{10, 20, 30, 40}, + want: 30, // samples[4/2] = samples[2] + }, + { + name: "eight samples median", + samples: []time.Duration{5, 10, 15, 20, 25, 30, 35, 40}, + want: 25, // samples[8/2] = samples[4] + }, + { + name: "outlier resistance", + samples: []time.Duration{10, 11, 12, 10, 11, 12, 10, 500}, + want: 11, // median ignores the 500 outlier + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ps := &scionPathProbeState{} + for _, s := range tt.samples { + ps.addPongReply(scionPongReply{latency: s}) + } + if got := ps.latency(); got != tt.want { + t.Errorf("latency() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestScionReEvalAntiFlap(t *testing.T) { + hostAddr := netip.MustParseAddrPort("10.0.0.1:41641") + peerIA := addr.MustParseIA("1-ff00:0:111") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := &Conn{ + connCtx: ctx, + peerMap: newPeerMap(), + } + c.logf = t.Logf + + // Two paths with very similar latency (10ms vs 11ms). + // Path 1 is the incumbent active path. + ps1 := &scionPathProbeState{healthy: true} + ps1.addPongReply(scionPongReply{latency: 10 * time.Millisecond}) + + ps2 := &scionPathProbeState{healthy: true} + ps2.addPongReply(scionPongReply{latency: 11 * time.Millisecond}) + + de := &endpoint{c: c} + de.scionState = &scionEndpointState{ + peerIA: peerIA, + hostAddr: hostAddr, + activePath: scionPathKey(2), // path 2 is active at 11ms + paths: map[scionPathKey]*scionPathProbeState{ + scionPathKey(1): ps1, + scionPathKey(2): ps2, + }, + } + de.bestAddr = addrQuality{ + epAddr: epAddr{ap: hostAddr, scionKey: scionPathKey(2)}, + latency: 11 * time.Millisecond, + } + + // Re-eval should NOT switch — 1ms improvement is within 2ms minimum threshold. + de.mu.Lock() + de.reEvalSCIONPathsLocked(mono.Now()) + activePath := de.scionState.activePath + de.mu.Unlock() + + if activePath != scionPathKey(2) { + t.Errorf("anti-flap: activePath = %d, want 2 (should not switch for 1ms difference)", activePath) + } + + // Now make path 1 significantly worse (50ms) and path 2 stays at 11ms. + // Then flip: make path 2 the slow one (50ms) and path 1 = 10ms. + // The 40ms improvement exceeds the 20% threshold (20% of 50ms = 10ms). + ps2Slow := &scionPathProbeState{healthy: true} + ps2Slow.addPongReply(scionPongReply{latency: 50 * time.Millisecond}) + + de.mu.Lock() + de.scionState.paths[scionPathKey(2)] = ps2Slow + de.scionState.lastFullEvalAt = 0 // reset throttle + de.bestAddr.latency = 50 * time.Millisecond + de.reEvalSCIONPathsLocked(mono.Now()) + activePath = de.scionState.activePath + de.mu.Unlock() + + if activePath != scionPathKey(1) { + t.Errorf("genuine degradation: activePath = %d, want 1 (should switch for 40ms improvement)", activePath) + } +} From 3b05dc7065ed811975d10692aa2ba74ed9fce289 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Sat, 14 Mar 2026 22:24:59 +0000 Subject: [PATCH 52/78] wgengine/magicsock: improve SCION address logging and enhance path state management - Updated logging statements to use a human-readable format for SCION addresses, improving clarity in logs. - Introduced a new method, scionAddrStr, to generate display strings for SCION endpoints, enhancing logging consistency. - Enhanced scionPathProbeState to include a display string for better lock-safe logging of SCION paths. - Refactored path management to ensure display strings are correctly assigned during path discovery and updates. --- wgengine/magicsock/endpoint.go | 4 ++-- wgengine/magicsock/endpoint_scion.go | 19 ++++++++++++--- wgengine/magicsock/magicsock_scion.go | 27 ++++++++++++++-------- wgengine/magicsock/magicsock_scion_omit.go | 1 + 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 8bbdf889ef8c4..6fa76c60002b5 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -127,7 +127,7 @@ func (de *endpoint) udpRelayEndpointReady(maybeBest addrQuality) { // // TODO(jwhited): add observability around !curBestAddrTrusted and sameRelayServer // TODO(jwhited): collapse path change logging with endpoint.handlePongConnLocked() - de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v", de.publicKey.ShortString(), de.discoShort(), maybeBest.epAddr, maybeBest.wireMTU) + de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v", de.publicKey.ShortString(), de.discoShort(), de.scionAddrStr(maybeBest.epAddr), maybeBest.wireMTU) de.setBestAddrLocked(maybeBest) de.trustBestAddrUntil = now.Add(trustUDPAddrDuration) } @@ -1821,7 +1821,7 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd if !skipPromotion && (curBestUntrusted || betterAddr(thisPong, de.bestAddr)) { if thisPong.epAddr != de.bestAddr.epAddr { - de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v tx=%x", de.publicKey.ShortString(), de.discoShort(), sp.to, thisPong.wireMTU, m.TxID[:6]) + de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v tx=%x", de.publicKey.ShortString(), de.discoShort(), de.scionAddrStr(sp.to), thisPong.wireMTU, m.TxID[:6]) de.debugUpdates.Add(EndpointChange{ When: time.Now(), What: "handlePongConnLocked-bestAddr-update", diff --git a/wgengine/magicsock/endpoint_scion.go b/wgengine/magicsock/endpoint_scion.go index 4f132c5b5a1b9..fcfc6897373cd 100644 --- a/wgengine/magicsock/endpoint_scion.go +++ b/wgengine/magicsock/endpoint_scion.go @@ -290,7 +290,7 @@ func (de *endpoint) demoteSCIONPathLocked(demotedKey scionPathKey) { wireMTU: scionWireMTU, scionPreferred: de.scionPreferred, } - de.c.logf("magicsock: SCION path demoted, switching to path %d for %v", bestKey, de.publicKey.ShortString()) + de.c.logf("magicsock: SCION path demoted, switching to %s for %v", de.scionAddrStr(newAddr.epAddr), de.publicKey.ShortString()) de.setBestAddrLocked(newAddr) go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, bestKey) } else { @@ -367,8 +367,8 @@ func (de *endpoint) reEvalSCIONPathsLocked(now mono.Time) { } if betterAddr(candidate, de.bestAddr) { - de.c.logf("magicsock: SCION re-eval: switching to path %d (latency %v) for %v", - bestKey, bestLatency.Round(time.Millisecond), de.publicKey.ShortString()) + de.c.logf("magicsock: SCION re-eval: switching to %s (latency %v) for %v", + de.scionAddrStr(candidate.epAddr), bestLatency.Round(time.Millisecond), de.publicKey.ShortString()) de.debugUpdates.Add(EndpointChange{ When: time.Now(), What: "reEvalSCIONPathsLocked-switch", @@ -380,3 +380,16 @@ func (de *endpoint) reEvalSCIONPathsLocked(now mono.Time) { go de.c.updateActiveSCIONPathLocking(de.scionState.peerIA, de.scionState.hostAddr, bestKey) } } + +// scionAddrStr returns a human-readable string for a SCION epAddr using +// cached path info from de.scionState. Falls back to e.String(). +// de.mu must be held. +func (de *endpoint) scionAddrStr(e epAddr) string { + if !e.scionKey.IsSet() || de.scionState == nil { + return e.String() + } + if ps, ok := de.scionState.paths[e.scionKey]; ok && ps.displayStr != "" { + return ps.displayStr + } + return e.String() +} diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index a83bd2089a29b..9991d0a1fcc11 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -229,6 +229,7 @@ type scionEndpointState struct { // scionPathProbeState tracks disco probing state for one SCION path. type scionPathProbeState struct { fingerprint snet.PathFingerprint + displayStr string // cached from scionPathInfo.displayStr for lock-safe logging lastPing mono.Time recentPongs [scionPongHistoryCount]scionPongReply // ring buffer recentPong uint16 // index of most recent entry @@ -2313,9 +2314,10 @@ func (c *Conn) addNewSCIONPathsForPeer(peerIA addr.IA, hostAddr netip.AddrPort, if ep.scionState != nil && ep.scionState.peerIA == peerIA { for _, k := range newKeys { if _, exists := ep.scionState.paths[k]; !exists { - fp := c.scionPaths[k].fingerprint + pi := c.scionPaths[k] ep.scionState.paths[k] = &scionPathProbeState{ - fingerprint: fp, + fingerprint: pi.fingerprint, + displayStr: pi.displayStr, healthy: true, } } @@ -2472,12 +2474,16 @@ func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPo newKeySet[k] = true } - // Extract fingerprints under c.mu (must be acquired before de.mu per lock ordering). + // Extract fingerprints and display strings under c.mu (must be acquired before de.mu per lock ordering). + type pathSnapshot struct { + fingerprint snet.PathFingerprint + displayStr string + } de.c.mu.Lock() - fpByKey := make(map[scionPathKey]snet.PathFingerprint, len(newKeys)) + snapByKey := make(map[scionPathKey]pathSnapshot, len(newKeys)) for _, k := range newKeys { if pi := de.c.lookupSCIONPath(k); pi != nil { - fpByKey[k] = pi.fingerprint + snapByKey[k] = pathSnapshot{fingerprint: pi.fingerprint, displayStr: pi.displayStr} } } // Clean up old keys that aren't in the new set. @@ -2502,16 +2508,17 @@ func (de *endpoint) discoverSCIONPathAsync(peerIA addr.IA, hostAddr netip.AddrPo newPaths := make(map[scionPathKey]*scionPathProbeState, len(newKeys)) for _, k := range newKeys { - fp := fpByKey[k] + snap := snapByKey[k] // Preserve existing probe history if the fingerprint matches. - if fp != "" && oldProbeByFP != nil { - if old, ok := oldProbeByFP[fp]; ok { - old.fingerprint = fp // ensure set + if snap.fingerprint != "" && oldProbeByFP != nil { + if old, ok := oldProbeByFP[snap.fingerprint]; ok { + old.fingerprint = snap.fingerprint // ensure set + old.displayStr = snap.displayStr newPaths[k] = old continue } } - newPaths[k] = &scionPathProbeState{fingerprint: fp, healthy: true} + newPaths[k] = &scionPathProbeState{fingerprint: snap.fingerprint, displayStr: snap.displayStr, healthy: true} } activePath := scionPathKey(0) diff --git a/wgengine/magicsock/magicsock_scion_omit.go b/wgengine/magicsock/magicsock_scion_omit.go index edcff8eb8ec76..309d213f14b1c 100644 --- a/wgengine/magicsock/magicsock_scion_omit.go +++ b/wgengine/magicsock/magicsock_scion_omit.go @@ -49,6 +49,7 @@ func (de *endpoint) handlePongPromoteSCIONLocked(_ addrQuality) func (de *endpoint) updateFromNodeSCIONLocked(_ tailcfg.NodeView) []scionPathKey { return nil } func (de *endpoint) stopAndResetSCIONLocked() []scionPathKey { return nil } func (de *endpoint) sendSCIONData(_ epAddr, _ [][]byte, _ int) error { return nil } +func (de *endpoint) scionAddrStr(e epAddr) string { return e.String() } // SCIONService returns false when SCION is omitted. func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { return tailcfg.Service{}, false } From a622a1f9654e8ddcf2f217335d50b29162d383aa Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Sat, 14 Mar 2026 22:50:45 +0000 Subject: [PATCH 53/78] wgengine/magicsock: implement SCION path recovery logic and add corresponding tests - Enhanced the addNewSCIONPathsForPeer method to initialize scionState for endpoints when initial path discovery fails. - Implemented logic to register new SCION paths and ensure proper recovery of scionState, allowing for effective disco probing. - Added a new test, TestScionAddNewPathsRecovery, to verify the correct initialization of scionState and path management during recovery scenarios. - Improved overall robustness of SCION path handling in the presence of failed initial discoveries. --- wgengine/magicsock/magicsock_scion.go | 36 ++++++++ wgengine/magicsock/magicsock_scion_test.go | 97 ++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 9991d0a1fcc11..9d51e00878368 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -2325,6 +2325,42 @@ func (c *Conn) addNewSCIONPathsForPeer(peerIA addr.IA, hostAddr netip.AddrPort, } ep.mu.Unlock() } + + // Recovery: if no endpoint had scionState for this peerIA, the initial + // discoverSCIONPathAsync may have failed. Find the endpoint via peerMap + // (the plain hostAddr was registered by handlePingLocked from incoming + // SCION disco) and initialize scionState so disco probing can begin. + if len(newKeys) > 0 { + if ep, ok := c.peerMap.endpointForEpAddr(epAddr{ap: hostAddr}); ok { + ep.mu.Lock() + if ep.scionState == nil { + paths := make(map[scionPathKey]*scionPathProbeState, len(newKeys)) + var activePath scionPathKey + for i, k := range newKeys { + pi := c.scionPaths[k] + paths[k] = &scionPathProbeState{ + fingerprint: pi.fingerprint, + displayStr: pi.displayStr, + healthy: true, + } + if i == 0 { + activePath = k + } + } + ep.scionState = &scionEndpointState{ + peerIA: peerIA, + hostAddr: hostAddr, + paths: paths, + activePath: activePath, + lastDiscoveryAt: time.Now(), + } + c.setActiveSCIONPath(peerIA, hostAddr, activePath) + c.logf("magicsock: SCION recovery: initialized scionState for %s with %d paths", peerIA, len(newKeys)) + } + ep.mu.Unlock() + } + } + c.mu.Unlock() return newKeys } diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 2d5961ec492b5..27fc4b4f99a77 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -2133,3 +2133,100 @@ func TestScionReEvalAntiFlap(t *testing.T) { t.Errorf("genuine degradation: activePath = %d, want 1 (should switch for 40ms improvement)", activePath) } } + +// TestScionAddNewPathsRecovery verifies that addNewSCIONPathsForPeer +// initializes scionState on an endpoint when the initial path discovery +// failed (scionState == nil) but incoming SCION disco registered the +// endpoint in the peerMap via handlePingLocked. +func TestScionAddNewPathsRecovery(t *testing.T) { + ctrl := gomock.NewController(t) + + localIA := addr.MustParseIA("1-ff00:0:110") + peerIA := addr.MustParseIA("1-ff00:0:111") + hostAddr := netip.MustParseAddrPort("10.0.0.1:41641") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := &Conn{ + connCtx: ctx, + peerMap: newPeerMap(), + } + c.logf = t.Logf + c.pconnSCION = &scionConn{daemon: mock_daemon.NewMockConnector(ctrl), localIA: localIA} + + // Create an endpoint and register it in the peerMap at the plain + // hostAddr — this is what handlePingLocked does for incoming SCION disco. + ep := &endpoint{c: c, nodeID: 1} + ep.publicKey = key.NewNode().Public() + ep.disco.Store(&endpointDisco{key: key.NewDisco().Public()}) + c.peerMap.upsertEndpoint(ep, key.DiscoPublic{}) + c.peerMap.setNodeKeyForEpAddr(epAddr{ap: hostAddr}, ep.publicKey) + + // Simulate failed initial discovery: ep.scionState is nil. + if ep.scionState != nil { + t.Fatal("precondition: scionState should be nil") + } + + // Register a reply-path in c.scionPaths (simulates handleSCIONDisco + // creating a path entry from an incoming disco ping). + replyPI := &scionPathInfo{ + peerIA: peerIA, + hostAddr: hostAddr, + fingerprint: "reply-fp", + expiry: time.Now().Add(1 * time.Hour), + } + replyPI.buildDisplayStr() + c.registerSCIONPathLocking(replyPI) + + // Now soft refresh finds new paths from the daemon. Call + // addNewSCIONPathsForPeer with two mock paths. + p1 := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{5 * time.Millisecond}, + Expiry: time.Now().Add(2 * time.Hour), + }) + p2 := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ + Latency: []time.Duration{10 * time.Millisecond}, + Expiry: time.Now().Add(2 * time.Hour), + }) + + newKeys := c.addNewSCIONPathsForPeer(peerIA, hostAddr, []snet.Path{p1, p2}) + if len(newKeys) != 2 { + t.Fatalf("addNewSCIONPathsForPeer returned %d keys, want 2", len(newKeys)) + } + + // Verify recovery initialized scionState. + ep.mu.Lock() + defer ep.mu.Unlock() + + if ep.scionState == nil { + t.Fatal("scionState should have been initialized by recovery") + } + if ep.scionState.peerIA != peerIA { + t.Errorf("peerIA = %s, want %s", ep.scionState.peerIA, peerIA) + } + if ep.scionState.hostAddr != hostAddr { + t.Errorf("hostAddr = %s, want %s", ep.scionState.hostAddr, hostAddr) + } + if len(ep.scionState.paths) != 2 { + t.Errorf("paths count = %d, want 2", len(ep.scionState.paths)) + } + if !ep.scionState.activePath.IsSet() { + t.Error("activePath should be set") + } + // activePath should be one of the new keys. + if ep.scionState.activePath != newKeys[0] { + t.Errorf("activePath = %d, want %d (first new key)", ep.scionState.activePath, newKeys[0]) + } + // Each probe state should be healthy. + for _, k := range newKeys { + ps, ok := ep.scionState.paths[k] + if !ok { + t.Errorf("missing probe state for key %d", k) + continue + } + if !ps.healthy { + t.Errorf("probe state for key %d should be healthy", k) + } + } +} From 24b492c6384e4caf09256320889ba0e5531464a9 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Sat, 14 Mar 2026 22:52:20 +0000 Subject: [PATCH 54/78] wgengine/magicsock: update SCION test environment variable handling - Replaced direct use of t.Setenv with envknob.Setenv for setting the TS_SCION_PORT environment variable in tests. - Added cleanup logic to reset the environment variable after each test, ensuring isolation between test cases. --- wgengine/magicsock/magicsock_scion_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 27fc4b4f99a77..2aa4b7f9b5b98 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -22,6 +22,7 @@ import ( "github.com/scionproto/scion/pkg/snet" "github.com/scionproto/scion/pkg/snet/mock_snet" snetpath "github.com/scionproto/scion/pkg/snet/path" + "tailscale.com/envknob" "tailscale.com/net/packet" "tailscale.com/net/tstun" "tailscale.com/tailcfg" @@ -705,7 +706,8 @@ func TestScionListenPort(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.envVal != "" { - t.Setenv("TS_SCION_PORT", tt.envVal) + envknob.Setenv("TS_SCION_PORT", tt.envVal) + t.Cleanup(func() { envknob.Setenv("TS_SCION_PORT", "") }) } got := scionListenPort() if got != tt.want { From f030503a9b794b542bee9517f821cde1310f5146 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Sat, 14 Mar 2026 23:48:49 +0000 Subject: [PATCH 55/78] wgengine/magicsock: refactor SCION connection types and introduce new constants to make omit scion work --- wgengine/magicsock/magicsock.go | 3 +-- wgengine/magicsock/magicsock_scion.go | 4 ++++ wgengine/magicsock/magicsock_scion_omit.go | 13 ++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index f010b3bad7bce..984f83285933c 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -24,7 +24,6 @@ import ( "sync/atomic" "time" - "github.com/scionproto/scion/pkg/addr" "github.com/tailscale/wireguard-go/conn" "github.com/tailscale/wireguard-go/device" "go4.org/mem" @@ -427,7 +426,7 @@ type Conn struct { scionPaths map[scionPathKey]*scionPathInfo scionPathsByAddr map[scionAddrKey]scionPathKey // reverse index for O(1) lookup scionPathSeq atomic.Uint32 // monotonic key generator for scionPaths - scionSoftRefreshAt map[addr.IA]time.Time // last soft refresh per peer, guarded by c.mu + scionSoftRefreshAt map[scionIAKey]time.Time // last soft refresh per peer, guarded by c.mu // lastSCIONRecv is the last time we received any SCION packet (monotonic). // Used by receiveSCION to detect a dead socket and trigger reconnection. diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 9d51e00878368..b45f8ed8dbb10 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -77,6 +77,10 @@ func scionPreferenceBonus() int { return 15 } +// scionIAKey is a type alias for addr.IA, used in Conn fields shared with +// the ts_omit_scion omit file (which defines scionIAKey = uint64). +type scionIAKey = addr.IA + // scionPathKey is a compact index into the Conn-level scionPaths registry. // This keeps epAddr small and comparable (snet.UDPAddr contains slices). // A zero value means "not a SCION path." diff --git a/wgengine/magicsock/magicsock_scion_omit.go b/wgengine/magicsock/magicsock_scion_omit.go index 309d213f14b1c..4e406e42e9652 100644 --- a/wgengine/magicsock/magicsock_scion_omit.go +++ b/wgengine/magicsock/magicsock_scion_omit.go @@ -10,6 +10,7 @@ import ( "time" wgconn "github.com/tailscale/wireguard-go/conn" + "tailscale.com/net/tstun" "tailscale.com/tailcfg" "tailscale.com/tstime/mono" ) @@ -20,13 +21,23 @@ type scionPathKey uint32 func (k scionPathKey) IsSet() bool { return false } -type scionConn struct{} +type scionBatchRW interface{} + +type scionConn struct { + shimXPC scionBatchRW +} func (sc *scionConn) close() error { return nil } type scionPathInfo struct{} + +func (pi *scionPathInfo) String() string { return "" } + type scionAddrKey struct{} type scionEndpointState struct{} +type scionIAKey = uint64 + +const scionWireMTU = tstun.WireMTU(1280) // Stub Conn methods. From a613ce911ac225148d0330fdd1faf11bf2d8f376 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Sun, 15 Mar 2026 00:19:03 +0000 Subject: [PATCH 56/78] wgengine/magicsock: harden SCION code for cross-platform safety --- wgengine/magicsock/magicsock.go | 2 +- wgengine/magicsock/magicsock_scion.go | 22 +++++++++++++++------- wgengine/magicsock/scion_bootstrap_test.go | 13 ++----------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 984f83285933c..6cf87326b9a41 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -426,7 +426,7 @@ type Conn struct { scionPaths map[scionPathKey]*scionPathInfo scionPathsByAddr map[scionAddrKey]scionPathKey // reverse index for O(1) lookup scionPathSeq atomic.Uint32 // monotonic key generator for scionPaths - scionSoftRefreshAt map[scionIAKey]time.Time // last soft refresh per peer, guarded by c.mu + scionSoftRefreshAt map[scionIAKey]time.Time // last soft refresh per peer, guarded by c.mu; bounded by unique peer count // lastSCIONRecv is the last time we received any SCION packet (monotonic). // Used by receiveSCION to detect a dead socket and trigger reconnection. diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index b45f8ed8dbb10..b556dc19a757a 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -683,7 +683,7 @@ func scionListenPort() uint16 { // the user should set TS_SCION_LISTEN_ADDR explicitly. // // Falls back to 127.0.0.1 if no interfaces or resolution fails. -func scionResolveLocalIP(ctx context.Context, connector daemon.Connector) netip.Addr { +func scionResolveLocalIP(ctx context.Context, connector daemon.Connector, logf logger.Logf) netip.Addr { ifMap, err := connector.Interfaces(ctx) if err != nil || len(ifMap) == 0 { return netip.AddrFrom4([4]byte{127, 0, 0, 1}) @@ -712,7 +712,7 @@ func scionResolveLocalIP(ctx context.Context, connector daemon.Connector) netip. return netip.AddrFrom4([4]byte{127, 0, 0, 1}) } if !allSame { - fmt.Fprintf(os.Stderr, "magicsock: SCION: multiple BRs resolve to different local IPs; using %s, set TS_SCION_LISTEN_ADDR to override\n", first) + logf("magicsock: SCION: multiple BRs resolve to different local IPs; using %s, set TS_SCION_LISTEN_ADDR to override", first) } return first } @@ -720,7 +720,7 @@ func scionResolveLocalIP(ctx context.Context, connector daemon.Connector) netip. // scionListenAddr returns the listen address for the SCION underlay socket. // TS_SCION_LISTEN_ADDR can override the IP (e.g. "::1" for IPv6 localhost). // Otherwise resolves the local IP from the topology's BR internal addresses. -func scionListenAddr(ctx context.Context, connector daemon.Connector) *net.UDPAddr { +func scionListenAddr(ctx context.Context, connector daemon.Connector, logf logger.Logf) *net.UDPAddr { port := scionListenPort() if a := scionListenAddrEnv(); a != "" { ip := net.ParseIP(a) @@ -728,7 +728,7 @@ func scionListenAddr(ctx context.Context, connector daemon.Connector) *net.UDPAd return &net.UDPAddr{IP: ip, Port: int(port)} } } - ip := scionResolveLocalIP(ctx, connector) + ip := scionResolveLocalIP(ctx, connector, logf) return &net.UDPAddr{IP: ip.AsSlice(), Port: int(port)} } @@ -908,7 +908,7 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn Topology: topo, } - listenAddr := scionListenAddr(ctx, connector) + listenAddr := scionListenAddr(ctx, connector, logf) if listenAddr.Port != 0 { // Validate the configured port against the dispatched range. portMin, portMax, err := connector.PortRange(ctx) @@ -979,7 +979,10 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn // address family based on the local address. var underlayXPC scionBatchRW if underlayConn != nil { - local := underlayConn.LocalAddr().(*net.UDPAddr) + local, ok := underlayConn.LocalAddr().(*net.UDPAddr) + if !ok { + return nil, fmt.Errorf("unexpected underlay local address type %T", underlayConn.LocalAddr()) + } if local.IP.To4() != nil { underlayXPC = ipv4.NewPacketConn(underlayConn) } else { @@ -1049,7 +1052,12 @@ func openDispatcherShim(sc *scionConn, logf logger.Logf, netMon *netmon.Monitor) // Wrap for batch I/O, selecting address family based on local address. var xpc scionBatchRW - local := shimConn.LocalAddr().(*net.UDPAddr) + local, ok := shimConn.LocalAddr().(*net.UDPAddr) + if !ok { + shimConn.Close() + logf("magicsock: SCION shim: unexpected local address type %T", shimConn.LocalAddr()) + return + } if local.IP.To4() != nil { xpc = ipv4.NewPacketConn(shimConn) } else { diff --git a/wgengine/magicsock/scion_bootstrap_test.go b/wgengine/magicsock/scion_bootstrap_test.go index bdaac825fd1be..25e1cea4b2949 100644 --- a/wgengine/magicsock/scion_bootstrap_test.go +++ b/wgengine/magicsock/scion_bootstrap_test.go @@ -174,16 +174,7 @@ func TestBootstrapURLs(t *testing.T) { } func TestLocalSearchDomainFromHostname(t *testing.T) { - tests := []struct { - name string - want string - wantErr bool - }{ - // Note: we can't easily override os.Hostname() in tests, - // so we test the parsing logic via the function contract. - } - _ = tests - - // At minimum, verify the function doesn't panic. + // We can't easily override os.Hostname() in tests, so just verify + // the function doesn't panic and returns without error on this host. _, _ = localSearchDomainFromHostname() } From 46e68716d4e45600c408ec8be0b57adca7d05b94 Mon Sep 17 00:00:00 2001 From: Tony John Date: Sun, 15 Mar 2026 20:10:21 +0100 Subject: [PATCH 57/78] ipnstate, magicsock: add SCIONPathInfo to PeerStatus Add SCIONPathInfo struct to ipnstate with path description, active status, health, latency, expiry, and MTU fields. Populate it from endpoint's scionState in populatePeerStatus via a new build-tagged helper method populateSCIONPathsLocked. --- ipn/ipnstate/ipnstate.go | 15 ++++++++++ wgengine/magicsock/endpoint.go | 1 + wgengine/magicsock/magicsock_scion.go | 32 ++++++++++++++++++++++ wgengine/magicsock/magicsock_scion_omit.go | 2 ++ 4 files changed, 50 insertions(+) diff --git a/ipn/ipnstate/ipnstate.go b/ipn/ipnstate/ipnstate.go index 17e6ac870bead..f0eb82eb14814 100644 --- a/ipn/ipnstate/ipnstate.go +++ b/ipn/ipnstate/ipnstate.go @@ -176,6 +176,16 @@ type TailnetStatus struct { MagicDNSEnabled bool } +// SCIONPathInfo describes a SCION path to a peer. +type SCIONPathInfo struct { + Path string `json:"Path"` + Active bool `json:"Active"` + Healthy bool `json:"Healthy"` + LatencyMs float64 `json:"LatencyMs"` + ExpiresAt string `json:"ExpiresAt,omitempty"` + MTU int `json:"MTU,omitempty"` +} + // ExitNodeStatus describes the current exit node. type ExitNodeStatus struct { // ID is the exit node's ID. @@ -256,6 +266,8 @@ type PeerStatus struct { Relay string // DERP region PeerRelay string // peer relay address (ip:port:vni) + SCIONPaths []SCIONPathInfo `json:"SCIONPaths,omitempty"` + RxBytes int64 TxBytes int64 Created time.Time // time registered with tailcontrol @@ -545,6 +557,9 @@ func (sb *StatusBuilder) AddPeer(peer key.NodePublic, st *PeerStatus) { if v := st.TaildropTarget; v != TaildropTargetUnknown { e.TaildropTarget = v } + if v := st.SCIONPaths; v != nil { + e.SCIONPaths = v + } e.Location = st.Location } diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 6fa76c60002b5..d9f0498c33d1a 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -2128,6 +2128,7 @@ func (de *endpoint) populatePeerStatus(ps *ipnstate.PeerStatus) { ps.CurAddr = udpAddr.String() } } + de.populateSCIONPathsLocked(ps) } // stopAndReset stops timers associated with de and resets its state back to zero. diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index b556dc19a757a..0a2ad607c1a8b 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -33,6 +33,7 @@ import ( "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" "tailscale.com/envknob" + "tailscale.com/ipn/ipnstate" "tailscale.com/net/netmon" "tailscale.com/net/netns" "tailscale.com/net/tstun" @@ -2621,3 +2622,34 @@ func (c *Conn) scionKeyForAddr(peerIA addr.IA, hostAddr netip.AddrPort) scionPat var errNoSCION = fmt.Errorf("SCION not available") const discoRXPathSCION discoRXPath = "SCION" + +// populateSCIONPathsLocked fills ps.SCIONPaths from de.scionState. +// de.mu must be held. c.mu must be held (caller is Conn.UpdateStatus). +func (de *endpoint) populateSCIONPathsLocked(ps *ipnstate.PeerStatus) { + ss := de.scionState + if ss == nil || len(ss.paths) == 0 { + return + } + ps.SCIONPaths = make([]ipnstate.SCIONPathInfo, 0, len(ss.paths)) + for pk, probe := range ss.paths { + info := ipnstate.SCIONPathInfo{ + Path: probe.displayStr, + Active: pk == ss.activePath, + Healthy: probe.healthy, + } + lat := probe.latency() + if lat < time.Hour { + info.LatencyMs = float64(lat.Microseconds()) / 1000.0 + } + // Look up full path info from Conn-level registry for expiry/MTU. + if pi, ok := de.c.scionPaths[pk]; ok { + if !pi.expiry.IsZero() { + info.ExpiresAt = pi.expiry.UTC().Format(time.RFC3339) + } + if pi.mtu > 0 { + info.MTU = int(pi.mtu) + } + } + ps.SCIONPaths = append(ps.SCIONPaths, info) + } +} diff --git a/wgengine/magicsock/magicsock_scion_omit.go b/wgengine/magicsock/magicsock_scion_omit.go index 4e406e42e9652..882a2569771ac 100644 --- a/wgengine/magicsock/magicsock_scion_omit.go +++ b/wgengine/magicsock/magicsock_scion_omit.go @@ -10,6 +10,7 @@ import ( "time" wgconn "github.com/tailscale/wireguard-go/conn" + "tailscale.com/ipn/ipnstate" "tailscale.com/net/tstun" "tailscale.com/tailcfg" "tailscale.com/tstime/mono" @@ -61,6 +62,7 @@ func (de *endpoint) updateFromNodeSCIONLocked(_ tailcfg.NodeView) []scionPathKey func (de *endpoint) stopAndResetSCIONLocked() []scionPathKey { return nil } func (de *endpoint) sendSCIONData(_ epAddr, _ [][]byte, _ int) error { return nil } func (de *endpoint) scionAddrStr(e epAddr) string { return e.String() } +func (de *endpoint) populateSCIONPathsLocked(_ *ipnstate.PeerStatus) {} // SCIONService returns false when SCION is omitted. func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { return tailcfg.Service{}, false } From 4650bf4a6d151b10517a9b40fae8201b11003967 Mon Sep 17 00:00:00 2001 From: Tony John Date: Sun, 15 Mar 2026 20:18:42 +0100 Subject: [PATCH 58/78] magicsock: add ReconfigureSCION and SCIONStatus for Android runtime config Add SCIONConfig struct and two methods on *Conn: - ReconfigureSCION: updates envknobs and triggers reconnection - SCIONStatus: returns whether SCION is connected and local IA --- wgengine/magicsock/magicsock.go | 7 +++++ wgengine/magicsock/magicsock_scion.go | 33 ++++++++++++++++++++++ wgengine/magicsock/magicsock_scion_omit.go | 3 ++ 3 files changed, 43 insertions(+) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 6cf87326b9a41..336e3c86d3057 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -99,6 +99,13 @@ const ( PathSCION Path = "scion" ) +// SCIONConfig holds runtime-configurable SCION parameters. +type SCIONConfig struct { + Enabled bool + BootstrapURL string + Prefer bool +} + type pathLabel struct { // Path indicates the path that the packet took: // - direct_ipv4 diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 0a2ad607c1a8b..e0a61e378744b 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -2623,6 +2623,39 @@ var errNoSCION = fmt.Errorf("SCION not available") const discoRXPathSCION discoRXPath = "SCION" +// ReconfigureSCION updates SCION configuration at runtime. +// If disabled, closes the current SCION connection. +// If enabled, updates envknobs and triggers a reconnection attempt. +func (c *Conn) ReconfigureSCION(cfg SCIONConfig) { + if !cfg.Enabled { + c.mu.Lock() + c.closeSCIONLocked() + c.mu.Unlock() + return + } + if cfg.BootstrapURL != "" { + envknob.Setenv("TS_SCION_BOOTSTRAP_URL", cfg.BootstrapURL) + } + if cfg.Prefer { + envknob.Setenv("TS_PREFER_SCION", "true") + } else { + envknob.Setenv("TS_PREFER_SCION", "") + } + // Force embedded mode on Android (no external daemon). + envknob.Setenv("TS_SCION_EMBEDDED", "1") + c.retrySCIONConnect() +} + +// SCIONStatus returns whether SCION is currently connected and the local IA. +func (c *Conn) SCIONStatus() (connected bool, localIA string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.pconnSCION == nil { + return false, "" + } + return true, c.pconnSCION.localIA.String() +} + // populateSCIONPathsLocked fills ps.SCIONPaths from de.scionState. // de.mu must be held. c.mu must be held (caller is Conn.UpdateStatus). func (de *endpoint) populateSCIONPathsLocked(ps *ipnstate.PeerStatus) { diff --git a/wgengine/magicsock/magicsock_scion_omit.go b/wgengine/magicsock/magicsock_scion_omit.go index 882a2569771ac..aed6b47f7fddb 100644 --- a/wgengine/magicsock/magicsock_scion_omit.go +++ b/wgengine/magicsock/magicsock_scion_omit.go @@ -67,6 +67,9 @@ func (de *endpoint) populateSCIONPathsLocked(_ *ipnstate.PeerStatus) // SCIONService returns false when SCION is omitted. func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { return tailcfg.Service{}, false } +func (c *Conn) ReconfigureSCION(_ SCIONConfig) {} +func (c *Conn) SCIONStatus() (connected bool, localIA string) { return false, "" } + // Stub standalone functions used by betterAddr in endpoint.go. var preferSCION = func() bool { return false } From eab9c9cf612cf09ac60865911f860ecff0af822c Mon Sep 17 00:00:00 2001 From: Tony John Date: Sun, 15 Mar 2026 20:19:05 +0100 Subject: [PATCH 59/78] localapi: add scion-status endpoint GET /localapi/v0/scion-status returns SCION connection status and local ISD-AS number. Stub omit file for ts_omit_scion builds. --- ipn/localapi/localapi_scion.go | 39 +++++++++++++++++++++++++++++ ipn/localapi/localapi_scion_omit.go | 6 +++++ 2 files changed, 45 insertions(+) create mode 100644 ipn/localapi/localapi_scion.go create mode 100644 ipn/localapi/localapi_scion_omit.go diff --git a/ipn/localapi/localapi_scion.go b/ipn/localapi/localapi_scion.go new file mode 100644 index 0000000000000..7418e1f20997a --- /dev/null +++ b/ipn/localapi/localapi_scion.go @@ -0,0 +1,39 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_scion + +package localapi + +import ( + "encoding/json" + "net/http" +) + +func init() { + Register("scion-status", (*Handler).serveSCIONStatus) +} + +// SCIONStatusResponse is the JSON response for GET /localapi/v0/scion-status. +type SCIONStatusResponse struct { + Connected bool `json:"Connected"` + LocalIA string `json:"LocalIA,omitempty"` +} + +func (h *Handler) serveSCIONStatus(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + mc := h.b.MagicConn() + if mc == nil { + http.Error(w, "not ready", http.StatusServiceUnavailable) + return + } + connected, localIA := mc.SCIONStatus() + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(SCIONStatusResponse{ + Connected: connected, + LocalIA: localIA, + }) +} \ No newline at end of file diff --git a/ipn/localapi/localapi_scion_omit.go b/ipn/localapi/localapi_scion_omit.go new file mode 100644 index 0000000000000..4381bbb99c0a1 --- /dev/null +++ b/ipn/localapi/localapi_scion_omit.go @@ -0,0 +1,6 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build ts_omit_scion + +package localapi \ No newline at end of file From 5d3306aa43866593fbcb5b857879f1fa6d73e81c Mon Sep 17 00:00:00 2001 From: Tony John Date: Sun, 15 Mar 2026 21:12:20 +0100 Subject: [PATCH 60/78] magicsock: force fresh bootstrap in ReconfigureSCION Close existing SCION connection and set TS_SCION_FORCE_BOOTSTRAP before retrying, so that config changes from the Android UI always trigger a real reconnection attempt. --- wgengine/magicsock/magicsock_scion.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index e0a61e378744b..7b80d70a2b65c 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -2625,7 +2625,8 @@ const discoRXPathSCION discoRXPath = "SCION" // ReconfigureSCION updates SCION configuration at runtime. // If disabled, closes the current SCION connection. -// If enabled, updates envknobs and triggers a reconnection attempt. +// If enabled, updates envknobs, closes any existing connection, and +// triggers a fresh reconnection attempt. func (c *Conn) ReconfigureSCION(cfg SCIONConfig) { if !cfg.Enabled { c.mu.Lock() @@ -2641,8 +2642,15 @@ func (c *Conn) ReconfigureSCION(cfg SCIONConfig) { } else { envknob.Setenv("TS_PREFER_SCION", "") } - // Force embedded mode on Android (no external daemon). + // Force embedded mode and fresh bootstrap on Android. envknob.Setenv("TS_SCION_EMBEDDED", "1") + envknob.Setenv("TS_SCION_FORCE_BOOTSTRAP", "1") + + // Close existing connection (if any) so retrySCIONConnect starts fresh. + c.mu.Lock() + c.closeSCIONLocked() + c.mu.Unlock() + c.retrySCIONConnect() } From cb6e59b2a09a96f2cb3b52e9f41d8eafc17428c7 Mon Sep 17 00:00:00 2001 From: Tony John Date: Sun, 15 Mar 2026 21:42:42 +0100 Subject: [PATCH 61/78] magicsock: always register SCION receive funcs for mid-session connect When SCION connects mid-session (e.g. via ReconfigureSCION from the Android UI), the receive goroutines were never started because receiveFuncs() only included SCION functions if pconnSCION was non-nil at Open() time. Fix: always register receiveSCION and receiveSCIONShim in the receive func list. When pconnSCION is nil, they poll every 5 seconds instead of blocking forever on donec. Once SCION connects, they pick up the new connection and start processing packets. --- wgengine/magicsock/magicsock.go | 10 +++---- wgengine/magicsock/magicsock_scion.go | 38 +++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 336e3c86d3057..23d2310044776 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3342,12 +3342,10 @@ func (c *connBind) Open(ignoredPort uint16) ([]conn.ReceiveFunc, uint16, error) if runtime.GOOS == "js" { fns = []conn.ReceiveFunc{c.receiveDERP} } - if c.pconnSCION != nil { - fns = append(fns, c.receiveSCION) - if c.pconnSCION.shimXPC != nil { - fns = append(fns, c.receiveSCIONShim) - } - } + // Always register SCION receive funcs so they're available when + // SCION connects mid-session (e.g. via ReconfigureSCION from Android). + // receiveSCION handles nil pconnSCION by waiting and retrying. + fns = append(fns, c.receiveSCION, c.receiveSCIONShim) // TODO: Combine receiveIPv4 and receiveIPv6 and receiveIP into a single // closure that closes over a *RebindingUDPConn? return fns, c.LocalPort(), nil diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 7b80d70a2b65c..17a59c0500ab0 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -934,12 +934,15 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn var underlayConn *net.UDPConn if pc, ok := pconn.(*snet.SCIONPacketConn); ok { underlayConn = pc.Conn + logf("magicsock: SCION: extracted underlay conn, local=%v", underlayConn.LocalAddr()) if err := pc.SetReadBuffer(socketBufferSize); err != nil { logf("magicsock: SCION: failed to set read buffer to %d: %v", socketBufferSize, err) } if err := pc.SetWriteBuffer(socketBufferSize); err != nil { logf("magicsock: SCION: failed to set write buffer to %d: %v", socketBufferSize, err) } + } else { + logf("magicsock: SCION: WARNING: pconn is %T, not *snet.SCIONPacketConn; cannot extract underlay", pconn) } // Apply platform-specific socket options (SO_MARK on Linux, @@ -951,13 +954,20 @@ func finishSCIONConnect(ctx context.Context, connector daemon.Connector, topo sn if err == nil { lc := netns.Listener(logf, netMon) if lc.Control != nil { + logf("magicsock: SCION: calling netns control (VpnService.protect) on underlay fd") if err := lc.Control("udp", underlayConn.LocalAddr().String(), rawConn); err != nil { - logf("magicsock: SCION: netns control: %v", err) + logf("magicsock: SCION: netns control FAILED: %v", err) + } else { + logf("magicsock: SCION: netns control succeeded on underlay socket") } + } else { + logf("magicsock: SCION: WARNING: netns Listener.Control is nil, socket NOT protected") } } else { logf("magicsock: SCION: SyscallConn: %v", err) } + } else { + logf("magicsock: SCION: WARNING: no underlay conn, socket NOT protected from VPN") } sconn, err := snet.NewCookedConn(pconn, topo) @@ -1360,8 +1370,18 @@ func (c *Conn) scionPathString(key scionPathKey) string { func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { sc := c.pconnSCION if sc == nil { - <-c.donec - return 0, net.ErrClosed + // SCION not connected yet. Wait and retry instead of blocking + // forever, so that mid-session SCION connections (e.g. from + // ReconfigureSCION on Android) can start receiving. + select { + case <-c.donec: + return 0, net.ErrClosed + case <-time.After(5 * time.Second): + } + sc = c.pconnSCION + if sc == nil { + return 0, nil // return zero to let WireGuard call us again + } } for { @@ -1496,8 +1516,16 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) func (c *Conn) receiveSCIONShim(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { sc := c.pconnSCION if sc == nil || sc.shimXPC == nil { - <-c.donec - return 0, net.ErrClosed + // SCION not connected or no shim. Wait and retry. + select { + case <-c.donec: + return 0, net.ErrClosed + case <-time.After(5 * time.Second): + } + sc = c.pconnSCION + if sc == nil || sc.shimXPC == nil { + return 0, nil + } } for { From c1007f7d8dd9772fea48b1269ae7524223f9b3a4 Mon Sep 17 00:00:00 2001 From: Tony John Date: Sun, 15 Mar 2026 22:56:13 +0100 Subject: [PATCH 62/78] magicsock: nil pconnSCION after close to prevent use-after-close closeSCIONLocked was closing the socket but leaving pconnSCION pointing at the closed conn. This caused panics when toggling SCION off then on, as retrySCIONConnect saw a non-nil (but closed) connection and returned early, or receive goroutines tried to read from the closed socket. --- wgengine/magicsock/magicsock_scion_conn.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wgengine/magicsock/magicsock_scion_conn.go b/wgengine/magicsock/magicsock_scion_conn.go index c71e1466e7bc0..7729d54f0024a 100644 --- a/wgengine/magicsock/magicsock_scion_conn.go +++ b/wgengine/magicsock/magicsock_scion_conn.go @@ -27,11 +27,13 @@ func (c *Conn) initSCIONLocked(ctx context.Context) { go c.refreshSCIONPaths() } -// closeSCIONLocked closes the SCION connection if open. +// closeSCIONLocked closes the SCION connection if open and sets pconnSCION +// to nil so that receiveSCION and retrySCIONConnect see it as disconnected. // c.mu must be held. func (c *Conn) closeSCIONLocked() { if c.pconnSCION != nil { c.pconnSCION.close() + c.pconnSCION = nil } } From 71451f43caa089247d3febc2c5b038ccc5c1f001 Mon Sep 17 00:00:00 2001 From: Tony John Date: Mon, 16 Mar 2026 00:09:08 +0100 Subject: [PATCH 63/78] magicsock: don't report SCION paths when disconnected populateSCIONPathsLocked was returning stale path data from scionState even after SCION was disabled and pconnSCION set to nil. Check pconnSCION first and return empty paths when disconnected. --- wgengine/magicsock/magicsock_scion.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 17a59c0500ab0..841c3a9848148 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -2695,6 +2695,10 @@ func (c *Conn) SCIONStatus() (connected bool, localIA string) { // populateSCIONPathsLocked fills ps.SCIONPaths from de.scionState. // de.mu must be held. c.mu must be held (caller is Conn.UpdateStatus). func (de *endpoint) populateSCIONPathsLocked(ps *ipnstate.PeerStatus) { + // Don't report paths if SCION is disconnected - they're stale. + if de.c.pconnSCION == nil { + return + } ss := de.scionState if ss == nil || len(ss.paths) == 0 { return From 0703859d0bf6cbabfc9b6d6eaecd0e61d33ee2f2 Mon Sep 17 00:00:00 2001 From: Tony John Date: Mon, 16 Mar 2026 00:28:33 +0100 Subject: [PATCH 64/78] magicsock: discover SCION peers and re-advertise after mid-session connect After retrySCIONConnect succeeds: 1. discoverNewSCIONPeers: scans all peers for SCION services and triggers path discovery for those without scionState yet. Fixes the case where SetNetworkMap ran before SCION was available. 2. ReSTUN(scion-connected): triggers endpoint re-advertisement so peers receive our SCION address via Hostinfo update. --- wgengine/magicsock/magicsock_scion.go | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 841c3a9848148..c541f7a151e0c 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -1794,6 +1794,8 @@ func (c *Conn) retrySCIONConnect() { c.lastSCIONRecv.StoreAtomic(mono.Now()) c.logf("magicsock: SCION reconnect retry succeeded, local IA: %s", newSC.localIA) c.rediscoverAllSCIONPaths() + c.discoverNewSCIONPeers() + c.ReSTUN("scion-connected") } // rediscoverAllSCIONPaths triggers path re-discovery for all endpoints that @@ -1824,6 +1826,38 @@ func (c *Conn) rediscoverAllSCIONPaths() { } } +// discoverNewSCIONPeers scans all known peers for SCION service advertisements +// and triggers path discovery for any peers that don't yet have scionState. +// Called after a successful SCION connect to handle the case where the netmap +// was processed before SCION was available. +func (c *Conn) discoverNewSCIONPeers() { + c.mu.Lock() + peers := c.peers + c.mu.Unlock() + + for i := range peers.Len() { + peer := peers.At(i) + peerIA, hostAddr, ok := scionServiceFromPeer(peer) + if !ok { + continue + } + c.mu.Lock() + ep, ok := c.peerMap.endpointForNodeID(peer.ID()) + c.mu.Unlock() + if !ok || ep == nil { + continue + } + ep.mu.Lock() + hasScionState := ep.scionState != nil + ep.mu.Unlock() + if hasScionState { + continue // already tracked by rediscoverAllSCIONPaths + } + c.logf("magicsock: SCION peer %s at %s, discovering paths (post-connect)...", peerIA, hostAddr) + go ep.discoverSCIONPathAsync(peerIA, hostAddr) + } +} + // discoverSCIONPaths queries the SCION daemon for paths to the given peer IA, // deduplicates by fingerprint, selects the top N by latency, and stores them // in the path registry. Returns the scionPathKeys for the registered paths From 833b696bf02410f72ecd89fdeef956f73578a043 Mon Sep 17 00:00:00 2001 From: tjohn327 Date: Mon, 16 Mar 2026 00:41:45 +0000 Subject: [PATCH 65/78] magicsock: use atomic.Pointer for pconnSCION to fix data race pconnSCION was declared in the "no locking required" section of Conn but was read and written from multiple goroutines without synchronization: receiveSCION, sendSCION, and sendSCIONBatch read it on the hot path without locks, while closeSCIONLocked, retrySCIONConnect, reconnectSCION, and ReconfigureSCION wrote it (some under c.mu, some without). This mixed-locking pattern is a data race detectable by the Go race detector, and can cause torn pointer reads on ARM (Android). Change pconnSCION from *scionConn to atomic.Pointer[scionConn], matching the RebindingUDPConn.pconnAtomic pattern used for pconn4/pconn6. All reads become .Load() (lock-free, safe on all architectures) and all writes become .Store() (can still be coordinated with c.mu for higher-level operations like close-then-reconnect sequences). SCIONStatus no longer needs c.mu since the atomic load is sufficient for reading the pointer and the immutable localIA field. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: tjohn327 --- wgengine/magicsock/endpoint_scion.go | 8 ++-- wgengine/magicsock/magicsock.go | 4 +- wgengine/magicsock/magicsock_scion.go | 53 +++++++++++----------- wgengine/magicsock/magicsock_scion_conn.go | 18 ++++---- wgengine/magicsock/magicsock_scion_test.go | 24 +++++----- 5 files changed, 54 insertions(+), 53 deletions(-) diff --git a/wgengine/magicsock/endpoint_scion.go b/wgengine/magicsock/endpoint_scion.go index fcfc6897373cd..3fa7f258987a6 100644 --- a/wgengine/magicsock/endpoint_scion.go +++ b/wgengine/magicsock/endpoint_scion.go @@ -20,7 +20,7 @@ import ( // paths, it probes non-best paths via round-robin. // de.mu must be held. func (de *endpoint) heartbeatSCIONLocked(now mono.Time) { - if de.scionState == nil || de.c.pconnSCION == nil { + if de.scionState == nil || de.c.pconnSCION.Load() == nil { return } if !de.bestAddr.isSCION() { @@ -51,7 +51,7 @@ func (de *endpoint) heartbeatSCIONLocked(now mono.Time) { // full discovery round. Returns true if SCION is available for this peer. // de.mu must be held. func (de *endpoint) sendDiscoPingsSCIONLocked(now mono.Time) bool { - if de.scionState == nil || de.c.pconnSCION == nil { + if de.scionState == nil || de.c.pconnSCION.Load() == nil { return false } for pk, ps := range de.scionState.paths { @@ -72,7 +72,7 @@ func (de *endpoint) sendDiscoPingsSCIONLocked(now mono.Time) bool { // cliPingSCIONLocked pings all SCION paths when the user runs "tailscale ping". // de.mu must be held. func (de *endpoint) cliPingSCIONLocked(now mono.Time, size int, resCB *pingResultAndCallback) { - if de.scionState == nil || de.c.pconnSCION == nil { + if de.scionState == nil || de.c.pconnSCION.Load() == nil { return } for pk := range de.scionState.paths { @@ -148,7 +148,7 @@ func (de *endpoint) updateFromNodeSCIONLocked(n tailcfg.NodeView) []scionPathKey if de.scionState == nil || de.scionState.peerIA != peerIA || de.scionState.hostAddr != hostAddr { // New or changed SCION address — discover paths asynchronously // to avoid blocking updateFromNode (which holds the endpoint lock). - if de.c.pconnSCION != nil { + if de.c.pconnSCION.Load() != nil { de.c.logf("magicsock: SCION peer %s at %s, discovering paths...", peerIA, hostAddr) go de.discoverSCIONPathAsync(peerIA, hostAddr) } else { diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 23d2310044776..34797d9ee7497 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -203,7 +203,9 @@ type Conn struct { pconn6 RebindingUDPConn // pconnSCION is the SCION connection, nil if SCION is not available. - pconnSCION *scionConn + // Accessed atomically from hot-path send/receive goroutines; + // writes happen under c.mu for higher-level coordination. + pconnSCION atomic.Pointer[scionConn] receiveBatchPool sync.Pool diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index c541f7a151e0c..d08316bc19671 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -1111,7 +1111,7 @@ func parseSCIONServiceAddr(description string, port uint16) (ia addr.IA, hostAdd // sendmmsg in a single syscall. Otherwise, falls back to snet.Conn.WriteTo // per packet. func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent bool, err error) { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil { return false, errNoSCION } @@ -1226,7 +1226,7 @@ func (c *Conn) sendSCIONBatchFast(sc *scionConn, fp *scionFastPath, buffs [][]by // sendSCION sends a single packet over SCION, used for disco messages. func (c *Conn) sendSCION(sk scionPathKey, b []byte) (bool, error) { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil { return false, errNoSCION } @@ -1368,7 +1368,7 @@ func (c *Conn) scionPathString(key scionPathKey) string { // recvmmsg and parsed with lightweight slayers.SCION decoding. Otherwise, // falls back to single-packet snet.Conn.ReadFrom. func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil { // SCION not connected yet. Wait and retry instead of blocking // forever, so that mid-session SCION connections (e.g. from @@ -1378,7 +1378,7 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) return 0, net.ErrClosed case <-time.After(5 * time.Second): } - sc = c.pconnSCION + sc = c.pconnSCION.Load() if sc == nil { return 0, nil // return zero to let WireGuard call us again } @@ -1393,7 +1393,7 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } // Re-read pconnSCION — it may have been swapped by reconnectSCION. - sc = c.pconnSCION + sc = c.pconnSCION.Load() if sc == nil { // Socket was closed and reconnection failed. Retry. select { @@ -1514,7 +1514,7 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) // the main socket's responsibility) and has no slow-path fallback (the shim // is always a raw *net.UDPConn). func (c *Conn) receiveSCIONShim(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil || sc.shimXPC == nil { // SCION not connected or no shim. Wait and retry. select { @@ -1522,7 +1522,7 @@ func (c *Conn) receiveSCIONShim(buffs [][]byte, sizes []int, eps []wgconn.Endpoi return 0, net.ErrClosed case <-time.After(5 * time.Second): } - sc = c.pconnSCION + sc = c.pconnSCION.Load() if sc == nil || sc.shimXPC == nil { return 0, nil } @@ -1536,7 +1536,7 @@ func (c *Conn) receiveSCIONShim(buffs [][]byte, sizes []int, eps []wgconn.Endpoi } // Re-read pconnSCION — it may have been swapped by reconnectSCION. - sc = c.pconnSCION + sc = c.pconnSCION.Load() if sc == nil { // Main socket reconnection in progress. Wait and retry; // the reconnect may or may not rebind port 30041. @@ -1698,7 +1698,7 @@ func isTimeoutError(err error) bool { // still responsive. For the embedded connector this is trivial (field read); // for external daemons it's a gRPC call confirming the process is alive. func (c *Conn) scionDaemonAlive() bool { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil || sc.daemon == nil { return false } @@ -1713,7 +1713,7 @@ func (c *Conn) scionDaemonAlive() bool { // finishSCIONConnect to create a new socket with the existing connector. // Returns true on success. func (c *Conn) reconnectSCIONSocket() bool { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil { return false } @@ -1723,7 +1723,7 @@ func (c *Conn) reconnectSCIONSocket() bool { // Close socket, release the port for rebinding. sc.closeSocket() - c.pconnSCION = nil + c.pconnSCION.Store(nil) newSC, err := finishSCIONConnect(c.connCtx, savedDaemon, savedTopo, c.logf, c.netMon) if err != nil { @@ -1731,7 +1731,7 @@ func (c *Conn) reconnectSCIONSocket() bool { return false } - c.pconnSCION = newSC + c.pconnSCION.Store(newSC) c.logf("magicsock: SCION socket-only reconnect succeeded, local IA: %s", newSC.localIA) return true } @@ -1756,7 +1756,7 @@ func (c *Conn) reconnectSCION() { // Tier 2: full bootstrap. c.logf("magicsock: SCION doing full bootstrap reconnect") - oldSC := c.pconnSCION + oldSC := c.pconnSCION.Load() // Close old connection first — we must release the port before binding // the new socket. When TS_SCION_PORT is set, both sockets would try @@ -1766,7 +1766,7 @@ func (c *Conn) reconnectSCION() { if oldSC != nil { oldSC.close() } - c.pconnSCION = nil + c.pconnSCION.Store(nil) newSC, err := trySCIONConnect(c.connCtx, c.logf, c.netMon) if err != nil { @@ -1774,7 +1774,7 @@ func (c *Conn) reconnectSCION() { return } - c.pconnSCION = newSC + c.pconnSCION.Store(newSC) c.logf("magicsock: SCION reconnected successfully, local IA: %s", newSC.localIA) c.rediscoverAllSCIONPaths() } @@ -1782,7 +1782,7 @@ func (c *Conn) reconnectSCION() { // retrySCIONConnect attempts to re-establish a SCION connection when // pconnSCION is nil (previous reconnect attempt failed). func (c *Conn) retrySCIONConnect() { - if c.pconnSCION != nil { + if c.pconnSCION.Load() != nil { return // another goroutine beat us to it } newSC, err := trySCIONConnect(c.connCtx, c.logf, c.netMon) @@ -1790,7 +1790,7 @@ func (c *Conn) retrySCIONConnect() { c.logf("magicsock: SCION reconnect retry failed: %v", err) return } - c.pconnSCION = newSC + c.pconnSCION.Store(newSC) c.lastSCIONRecv.StoreAtomic(mono.Now()) c.logf("magicsock: SCION reconnect retry succeeded, local IA: %s", newSC.localIA) c.rediscoverAllSCIONPaths() @@ -1863,7 +1863,7 @@ func (c *Conn) discoverNewSCIONPeers() { // in the path registry. Returns the scionPathKeys for the registered paths // (first element is the lowest-latency path). func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr netip.AddrPort) ([]scionPathKey, error) { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil { return nil, errNoSCION } @@ -1922,7 +1922,7 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr } pi.buildCachedDst() pi.buildDisplayStr() - if sc := c.pconnSCION; sc != nil { + if sc := c.pconnSCION.Load(); sc != nil { pi.fastPath = buildSCIONFastPath(sc, pi) } keys = append(keys, c.registerSCIONPath(pi)) @@ -2119,7 +2119,7 @@ func (c *Conn) refreshSCIONPaths() { } func (c *Conn) refreshSCIONPathsOnce() error { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil { return nil } @@ -2348,7 +2348,7 @@ func (c *Conn) refreshSCIONPathsOnce() error { // to the corresponding endpoint. Called during soft refresh when new paths // appear in the daemon's cache. Returns the registered path keys. func (c *Conn) addNewSCIONPathsForPeer(peerIA addr.IA, hostAddr netip.AddrPort, paths []snet.Path) []scionPathKey { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil { return nil } @@ -2521,7 +2521,7 @@ func scionServiceFromPeer(n tailcfg.NodeView) (ia addr.IA, hostAddr netip.AddrPo // SCIONService returns the SCION service entry to advertise in Hostinfo, // or ok=false if SCION is not available. func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { - sc := c.pconnSCION + sc := c.pconnSCION.Load() if sc == nil { return tailcfg.Service{}, false } @@ -2718,19 +2718,18 @@ func (c *Conn) ReconfigureSCION(cfg SCIONConfig) { // SCIONStatus returns whether SCION is currently connected and the local IA. func (c *Conn) SCIONStatus() (connected bool, localIA string) { - c.mu.Lock() - defer c.mu.Unlock() - if c.pconnSCION == nil { + sc := c.pconnSCION.Load() + if sc == nil { return false, "" } - return true, c.pconnSCION.localIA.String() + return true, sc.localIA.String() } // populateSCIONPathsLocked fills ps.SCIONPaths from de.scionState. // de.mu must be held. c.mu must be held (caller is Conn.UpdateStatus). func (de *endpoint) populateSCIONPathsLocked(ps *ipnstate.PeerStatus) { // Don't report paths if SCION is disconnected - they're stale. - if de.c.pconnSCION == nil { + if de.c.pconnSCION.Load() == nil { return } ss := de.scionState diff --git a/wgengine/magicsock/magicsock_scion_conn.go b/wgengine/magicsock/magicsock_scion_conn.go index 7729d54f0024a..af1f0efe5a85b 100644 --- a/wgengine/magicsock/magicsock_scion_conn.go +++ b/wgengine/magicsock/magicsock_scion_conn.go @@ -14,7 +14,7 @@ import ( // On success, stores the scionConn and starts the background path refresher. // c.mu must be held. func (c *Conn) initSCIONLocked(ctx context.Context) { - if c.pconnSCION != nil { + if c.pconnSCION.Load() != nil { return } sc, err := trySCIONConnect(ctx, c.logf, c.netMon) @@ -23,7 +23,7 @@ func (c *Conn) initSCIONLocked(ctx context.Context) { return } c.logf("magicsock: SCION available, local IA: %s", sc.localIA) - c.pconnSCION = sc + c.pconnSCION.Store(sc) go c.refreshSCIONPaths() } @@ -31,9 +31,9 @@ func (c *Conn) initSCIONLocked(ctx context.Context) { // to nil so that receiveSCION and retrySCIONConnect see it as disconnected. // c.mu must be held. func (c *Conn) closeSCIONLocked() { - if c.pconnSCION != nil { - c.pconnSCION.close() - c.pconnSCION = nil + if sc := c.pconnSCION.Load(); sc != nil { + sc.close() + c.pconnSCION.Store(nil) } } @@ -41,13 +41,13 @@ func (c *Conn) closeSCIONLocked() { // to unblock receiveSCION, without closing it. Called from connBind.Close. // c.mu must be held (via connBind.mu). func (c *Conn) closeSCIONBindLocked() { - if c.pconnSCION != nil { + if sc := c.pconnSCION.Load(); sc != nil { // Set an immediate read deadline to unblock receiveSCION. // We don't close the SCION socket here; Conn.Close handles that. - c.pconnSCION.conn.SetReadDeadline(time.Now()) + sc.conn.SetReadDeadline(time.Now()) // Also unblock the dispatcher shim's ReadBatch if present. - if c.pconnSCION.shimConn != nil { - c.pconnSCION.shimConn.SetReadDeadline(time.Now()) + if sc.shimConn != nil { + sc.shimConn.SetReadDeadline(time.Now()) } } } diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 2aa4b7f9b5b98..ff5c491f7e223 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -753,7 +753,7 @@ func TestDiscoverSCIONPaths(t *testing.T) { Return([]snet.Path{slowPath, fastPath, mediumPath}, nil) c := &Conn{} - c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mockDaemon, localIA: localIA}) keys, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) if err != nil { @@ -790,7 +790,7 @@ func TestDiscoverSCIONPaths(t *testing.T) { Return(nil, nil) c := &Conn{} - c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mockDaemon, localIA: localIA}) _, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) if err == nil { @@ -803,7 +803,7 @@ func TestDiscoverSCIONPaths(t *testing.T) { Return(nil, fmt.Errorf("daemon unavailable")) c := &Conn{} - c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mockDaemon, localIA: localIA}) _, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) if err == nil { @@ -825,7 +825,7 @@ func TestDiscoverSCIONPaths(t *testing.T) { Return([]snet.Path{noMetaPath}, nil) c := &Conn{} - c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mockDaemon, localIA: localIA}) keys, err := c.discoverSCIONPaths(context.Background(), peerIA, hostAddr) if err != nil { @@ -868,7 +868,7 @@ func TestRefreshSCIONPathsOnce(t *testing.T) { connCtx: ctx, } c.logf = t.Logf - c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mockDaemon, localIA: localIA}) // Register a path that's about to expire (30s from now, within the 1-min refresh window). pi := &scionPathInfo{ @@ -904,7 +904,7 @@ func TestRefreshSCIONPathsOnce(t *testing.T) { connCtx: ctx, } c.logf = t.Logf - c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mockDaemon, localIA: localIA}) // Register a path that's far from expiry. pi := &scionPathInfo{ @@ -929,7 +929,7 @@ func TestRefreshSCIONPathsOnce(t *testing.T) { connCtx: ctx, } c.logf = t.Logf - c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mockDaemon, localIA: localIA}) oldPath := newMockPathWithMetadata(ctrl, &snet.PathMetadata{ Latency: []time.Duration{10 * time.Millisecond}, @@ -977,7 +977,7 @@ func TestRefreshSCIONPathsOnce(t *testing.T) { connCtx: ctx, } c.logf = t.Logf - c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mockDaemon, localIA: localIA}) pi := &scionPathInfo{ peerIA: peerIA, @@ -1083,7 +1083,7 @@ func TestNoteRecvActivitySCIONTrustRefresh(t *testing.T) { func TestSendSCIONBatchExpiredPath(t *testing.T) { c := &Conn{} - c.pconnSCION = &scionConn{} + c.pconnSCION.Store(&scionConn{}) pi := &scionPathInfo{ peerIA: addr.MustParseIA("1-ff00:0:111"), @@ -1104,7 +1104,7 @@ func TestSendSCIONBatchExpiredPath(t *testing.T) { func TestSendSCIONExpiredPath(t *testing.T) { c := &Conn{} - c.pconnSCION = &scionConn{} + c.pconnSCION.Store(&scionConn{}) pi := &scionPathInfo{ peerIA: addr.MustParseIA("1-ff00:0:111"), @@ -1646,7 +1646,7 @@ func TestScionStalePathCleanup(t *testing.T) { peerMap: newPeerMap(), } c.logf = t.Logf - c.pconnSCION = &scionConn{daemon: mockDaemon, localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mockDaemon, localIA: localIA}) // Register a path with a fingerprint that will disappear. pi := &scionPathInfo{ @@ -2155,7 +2155,7 @@ func TestScionAddNewPathsRecovery(t *testing.T) { peerMap: newPeerMap(), } c.logf = t.Logf - c.pconnSCION = &scionConn{daemon: mock_daemon.NewMockConnector(ctrl), localIA: localIA} + c.pconnSCION.Store(&scionConn{daemon: mock_daemon.NewMockConnector(ctrl), localIA: localIA}) // Create an endpoint and register it in the peerMap at the plain // hostAddr — this is what handlePingLocked does for incoming SCION disco. From 936038391773db0f804a7241f7344eab81b1eae1 Mon Sep 17 00:00:00 2001 From: Tony John Date: Mon, 16 Mar 2026 10:18:14 +0100 Subject: [PATCH 66/78] magicsock: make retrySCIONConnect async in ReconfigureSCION Run retrySCIONConnect in a goroutine so ReconfigureSCION returns immediately. The bootstrap cascade can block 30-60s on network I/O; making it async prevents blocking the LocalAPI handler and avoids potential ANR on Android. --- wgengine/magicsock/magicsock_scion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index d08316bc19671..05adc8abe4444 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -2713,7 +2713,7 @@ func (c *Conn) ReconfigureSCION(cfg SCIONConfig) { c.closeSCIONLocked() c.mu.Unlock() - c.retrySCIONConnect() + go c.retrySCIONConnect() } // SCIONStatus returns whether SCION is currently connected and the local IA. From 6f205d4b4a3e163c1131c307135a77e9b2c46028 Mon Sep 17 00:00:00 2001 From: Tony John Date: Mon, 16 Mar 2026 10:45:05 +0100 Subject: [PATCH 67/78] magicsock: reduce receiveSCIONShim polling when no shim exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When connected but shimXPC is nil (new infrastructure, no dispatcher), poll every 30s instead of 5s. shimXPC is immutable per scionConn so frequent polling is wasteful — only a full reconnect creating a new scionConn could add a shim. Reduces wakeups from ~17K/day to ~2.9K/day in the common no-shim case. --- wgengine/magicsock/magicsock_scion.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 05adc8abe4444..4c02aa914291e 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -1515,8 +1515,8 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) // is always a raw *net.UDPConn). func (c *Conn) receiveSCIONShim(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { sc := c.pconnSCION.Load() - if sc == nil || sc.shimXPC == nil { - // SCION not connected or no shim. Wait and retry. + if sc == nil { + // SCION not connected yet. Wait for mid-session connect. select { case <-c.donec: return 0, net.ErrClosed @@ -1526,6 +1526,19 @@ func (c *Conn) receiveSCIONShim(buffs [][]byte, sizes []int, eps []wgconn.Endpoi if sc == nil || sc.shimXPC == nil { return 0, nil } + } else if sc.shimXPC == nil { + // Connected but no dispatcher shim. shimXPC is immutable per + // scionConn, so poll infrequently — only a full reconnect + // (new scionConn) could create one. + select { + case <-c.donec: + return 0, net.ErrClosed + case <-time.After(30 * time.Second): + } + sc = c.pconnSCION.Load() + if sc == nil || sc.shimXPC == nil { + return 0, nil + } } for { @@ -1548,11 +1561,12 @@ func (c *Conn) receiveSCIONShim(buffs [][]byte, sizes []int, eps []wgconn.Endpoi continue } if sc.shimXPC == nil { - // Shim was not rebound after reconnection. Wait and retry. + // Shim was not created for this connection (immutable per scionConn). + // Poll infrequently — only a full reconnect could add one. select { case <-c.donec: return 0, net.ErrClosed - case <-time.After(5 * time.Second): + case <-time.After(30 * time.Second): } continue } From 6bb89eaad226728cbb72a00662be9eeeddc4434e Mon Sep 17 00:00:00 2001 From: Tony John <38129229+tjohn327@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:40:02 +0100 Subject: [PATCH 68/78] Add batch read and write support for SCION (#1) * wgengine/magicsock: implement SCION fast-path checksum and header serialization - Added functions for computing the SCION pseudo-header checksum and finishing the checksum for SCION/UDP packets. - Introduced a pre-serialized header template for fast-path sends to optimize performance by bypassing standard serialization. - Enhanced the scionConn structure to support fast-path operations, including adjustments to the underlay connection handling. - Updated tests to validate the correctness of the new checksum computations and fast-path functionality. * wgengine/magicsock: implement SCION batch receive and parsing enhancements - Introduced scionRecvBatch for efficient batch processing of SCION packets, utilizing a sync.Pool for buffer reuse. - Added parseSCIONPacket function to extract source address and payload from raw SCION packets, improving packet handling. - Enhanced receiveSCION method to support batch reading from the underlay socket, optimizing performance during packet reception. - Updated logic for handling disco packets to leverage the new batch processing capabilities. * wgengine/magicsock: enhance SCION underlay support for IPv6 - Added support for IPv6 in the SCION connection handling, allowing for batch I/O operations with both IPv4 and IPv6. - Updated scionListenAddr to allow overriding the listen address via the TS_SCION_LISTEN_ADDR environment variable, supporting IPv6 localhost. - Refactored scionConn to use a common interface for underlay connections, improving flexibility for packet handling. - Enhanced documentation to clarify the behavior of the listen address and its default settings. --- wgengine/magicsock/magicsock_scion.go | 630 +++++++++++++++++++-- wgengine/magicsock/magicsock_scion_test.go | 370 ++++++++++++ 2 files changed, 954 insertions(+), 46 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index d9d6e9fac32b4..31d9bb413234c 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -5,6 +5,7 @@ package magicsock import ( "context" + "encoding/binary" "errors" "fmt" "net" @@ -15,10 +16,16 @@ import ( "sync" "time" + "github.com/google/gopacket" "github.com/scionproto/scion/pkg/addr" "github.com/scionproto/scion/pkg/daemon" + "github.com/scionproto/scion/pkg/slayers" + scionpath "github.com/scionproto/scion/pkg/slayers/path/scion" + snetpath "github.com/scionproto/scion/pkg/snet/path" "github.com/scionproto/scion/pkg/snet" wgconn "github.com/tailscale/wireguard-go/conn" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" "tailscale.com/envknob" "tailscale.com/tailcfg" "tailscale.com/tstime/mono" @@ -71,6 +78,7 @@ type scionPathInfo struct { path snet.Path // current best SCION path to this peer replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) cachedDst *snet.UDPAddr // pre-built destination addr; rebuilt when path changes + fastPath *scionFastPath // pre-serialized header template for fast sends expiry time.Time // path expiration from path metadata mtu uint16 // SCION payload MTU from path metadata mu sync.Mutex @@ -95,7 +103,7 @@ func (pi *scionPathInfo) buildCachedDst() { // scionHeaderOverhead is the fixed overhead added by SCION encapsulation, // excluding the variable-length path header: -// - Underlay IPv4+UDP: 20 + 8 = 28 bytes +// - Underlay IPv4+UDP: 20 + 8 = 28 bytes (or IPv6+UDP: 40 + 8 = 48 bytes) // - SCION common header: 12 bytes // - Address header (IPv4, 2x ISD-AS + 2x IPv4): 2*8 + 2*4 = 24 bytes // - SCION/UDP L4 header: 8 bytes @@ -183,12 +191,307 @@ func (ps *scionPathProbeState) latency() time.Duration { return ps.recentPongs[ps.recentPong].latency } +// scionFastPath holds a pre-serialized SCION+UDP header template for a +// specific path. At send time, the template is copied, per-packet fields +// (PayloadLen, UDP Length, UDP Checksum) are patched, payload is appended, +// and the result is sent directly on the underlay UDP socket — bypassing +// snet.Conn and gopacket serialization entirely. +type scionFastPath struct { + hdr []byte // [SCION header][UDP header], no payload + udpOffset int // byte offset of UDP header within hdr + nextHop *net.UDPAddr // underlay next-hop for this path + pseudoCsum uint32 // constant part of SCION pseudo-header checksum +} + +// scionMaxBatchSize is the max number of packets in a single sendmmsg call. +const scionMaxBatchSize = 64 + +// scionSendBatch is a reusable set of buffers for sendSCIONBatchFast. +type scionSendBatch struct { + bufs [][]byte + msgs []ipv4.Message +} + +var scionSendBatchPool = sync.Pool{ + New: func() any { + b := &scionSendBatch{ + bufs: make([][]byte, scionMaxBatchSize), + msgs: make([]ipv4.Message, scionMaxBatchSize), + } + for i := range b.bufs { + b.bufs[i] = make([]byte, 1500) + } + for i := range b.msgs { + b.msgs[i].Buffers = make([][]byte, 1) + } + return b + }, +} + +// scionRecvBatch is a reusable set of buffers for receiveSCIONBatch. +type scionRecvBatch struct { + msgs []ipv4.Message + bufs [][]byte + scn slayers.SCION // reusable SCION header parser (with RecyclePaths) +} + +var scionRecvBatchPool = sync.Pool{ + New: func() any { + b := &scionRecvBatch{ + msgs: make([]ipv4.Message, scionMaxBatchSize), + bufs: make([][]byte, scionMaxBatchSize), + } + b.scn.RecyclePaths() + for i := range b.bufs { + b.bufs[i] = make([]byte, 1500) + } + for i := range b.msgs { + b.msgs[i].Buffers = [][]byte{b.bufs[i]} + } + return b + }, +} + +// putScionRecvBatch resets batch state and returns it to the pool. +func putScionRecvBatch(batch *scionRecvBatch) { + for i := range batch.msgs { + batch.msgs[i].N = 0 + batch.msgs[i].Addr = nil + batch.msgs[i].Buffers[0] = batch.bufs[i] + } + scionRecvBatchPool.Put(batch) +} + +// scionPseudoHeaderPartial computes the constant part of the SCION +// pseudo-header checksum: srcIA + dstIA + srcAddr + dstAddr + protocol(17). +// The per-packet upper-layer length and data are added at send time. +func scionPseudoHeaderPartial(srcIA, dstIA addr.IA, srcIP, dstIP netip.Addr) uint32 { + var csum uint32 + var buf [8]byte + + // Source IA (8 bytes) + binary.BigEndian.PutUint64(buf[:], uint64(srcIA)) + for i := 0; i < 8; i += 2 { + csum += uint32(buf[i]) << 8 + csum += uint32(buf[i+1]) + } + + // Destination IA (8 bytes) + binary.BigEndian.PutUint64(buf[:], uint64(dstIA)) + for i := 0; i < 8; i += 2 { + csum += uint32(buf[i]) << 8 + csum += uint32(buf[i+1]) + } + + // Source address + if srcIP.Is4() { + b4 := srcIP.As4() + csum += uint32(b4[0])<<8 + uint32(b4[1]) + csum += uint32(b4[2])<<8 + uint32(b4[3]) + } else { + b16 := srcIP.As16() + for i := 0; i < 16; i += 2 { + csum += uint32(b16[i])<<8 + uint32(b16[i+1]) + } + } + + // Destination address + if dstIP.Is4() { + b4 := dstIP.As4() + csum += uint32(b4[0])<<8 + uint32(b4[1]) + csum += uint32(b4[2])<<8 + uint32(b4[3]) + } else { + b16 := dstIP.As16() + for i := 0; i < 16; i += 2 { + csum += uint32(b16[i])<<8 + uint32(b16[i+1]) + } + } + + // Protocol: L4UDP = 17 + csum += 17 + + return csum +} + +// scionFinishChecksum completes the SCION/UDP checksum by adding the +// upper-layer length and bytes to the pre-computed partial checksum, +// then folding and complementing. +func scionFinishChecksum(partialCsum uint32, upperLayer []byte) uint16 { + csum := partialCsum + + // Add upper-layer length + l := uint32(len(upperLayer)) + csum += (l >> 16) + (l & 0xffff) + + // Sum upper-layer bytes in 16-bit words + n := len(upperLayer) + for i := 0; i+1 < n; i += 2 { + csum += uint32(upperLayer[i]) << 8 + csum += uint32(upperLayer[i+1]) + } + if n%2 == 1 { + csum += uint32(upperLayer[n-1]) << 8 + } + + // Fold to 16 bits + for csum > 0xffff { + csum = (csum >> 16) + (csum & 0xffff) + } + return ^uint16(csum) +} + +// buildSCIONFastPath creates a pre-serialized header template for fast-path +// sends. Must be called with pi.mu held (or before pi is shared). +// Returns nil if the fast path cannot be built (e.g. no discovered path). +func buildSCIONFastPath(sc *scionConn, pi *scionPathInfo) *scionFastPath { + if sc.underlayConn == nil { + return nil + } + dst := pi.cachedDst + if dst == nil || dst.Path == nil || dst.NextHop == nil { + return nil + } + + dstIP, ok := netip.AddrFromSlice(dst.Host.IP) + if !ok { + return nil + } + srcIP := sc.localHostIP + + // Use snet.Packet.Serialize() with empty payload to get a correctly + // encoded SCION+UDP header template. + pkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: pi.peerIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: sc.localIA, Host: addr.HostIP(srcIP)}, + Path: dst.Path, + Payload: snet.UDPPayload{ + SrcPort: sc.localPort, + DstPort: uint16(dst.Host.Port), + Payload: nil, // empty payload → headers only + }, + }, + } + if err := pkt.Serialize(); err != nil { + return nil + } + + // pkt.Bytes is now [SCION header][8-byte UDP header] + hdr := make([]byte, len(pkt.Bytes)) + copy(hdr, pkt.Bytes) + udpOffset := len(hdr) - 8 + + pseudoCsum := scionPseudoHeaderPartial(sc.localIA, pi.peerIA, srcIP, dstIP) + + return &scionFastPath{ + hdr: hdr, + udpOffset: udpOffset, + nextHop: dst.NextHop, + pseudoCsum: pseudoCsum, + } +} + +// parseSCIONPacket parses a raw SCION packet from the underlay, extracting +// the source address info and UDP payload. scn is a reusable slayers.SCION +// (with RecyclePaths enabled). Returns srcIA, srcAddr, payload, rawPath, ok. +func parseSCIONPacket(data []byte, scn *slayers.SCION) ( + srcIA addr.IA, srcAddr netip.AddrPort, payload []byte, rawPathBytes []byte, ok bool, +) { + if err := scn.DecodeFromBytes(data, gopacket.NilDecodeFeedback); err != nil { + return 0, netip.AddrPort{}, nil, nil, false + } + if scn.NextHdr != slayers.L4UDP { + return 0, netip.AddrPort{}, nil, nil, false + } + + srcHost, err := scn.SrcAddr() + if err != nil { + return 0, netip.AddrPort{}, nil, nil, false + } + srcIP := srcHost.IP() + srcIA = scn.SrcIA + + // L4 payload starts at HdrLen * 4 bytes (SCION header is HdrLen + // 4-byte words). The first 8 bytes are the UDP header. + hdrBytes := int(scn.HdrLen) * 4 + if len(data) < hdrBytes+8 { + return 0, netip.AddrPort{}, nil, nil, false + } + // Extract UDP source port from the first 2 bytes of the L4 header. + srcPort := binary.BigEndian.Uint16(data[hdrBytes:]) + srcAddr = netip.AddrPortFrom(srcIP, srcPort) + payload = data[hdrBytes+8:] + + // Extract raw path bytes for potential reversal (disco first-contact). + if scn.Path != nil { + pathLen := scn.Path.Len() + // The path sits between the address header and the L4 header + // in the SCION common+address+path header region. + addrHdrLen := scn.AddrHdrLen() + // Common header is 12 bytes, then address header, then path. + pathStart := 12 + addrHdrLen + pathEnd := pathStart + pathLen + if pathEnd <= hdrBytes && pathLen > 0 { + rawPathBytes = data[pathStart:pathEnd] + } + } + + return srcIA, srcAddr, payload, rawPathBytes, true +} + +// buildSCIONReplyAddr builds an *snet.UDPAddr with reversed path for disco +// reply routing from raw path bytes extracted during receive. +func buildSCIONReplyAddr(srcIA addr.IA, srcHostAddr netip.AddrPort, rawPathBytes []byte) *snet.UDPAddr { + if len(rawPathBytes) == 0 { + return nil + } + // Copy path bytes since DecodeFromBytes references the slice. + pathCopy := make([]byte, len(rawPathBytes)) + copy(pathCopy, rawPathBytes) + + var raw scionpath.Raw + if err := raw.DecodeFromBytes(pathCopy); err != nil { + return nil + } + reversed, err := raw.Reverse() + if err != nil { + return nil + } + // Serialize the reversed path to raw bytes and wrap in snetpath.SCION + // which implements snet.DataplanePath. + revBytes := make([]byte, reversed.Len()) + if err := reversed.SerializeTo(revBytes); err != nil { + return nil + } + + return &snet.UDPAddr{ + IA: srcIA, + Host: &net.UDPAddr{ + IP: srcHostAddr.Addr().AsSlice(), + Port: int(srcHostAddr.Port()), + }, + Path: snetpath.SCION{Raw: revBytes}, + } +} + +// scionBatchRW abstracts ipv4.PacketConn and ipv6.PacketConn for +// batch I/O. Both have identical ReadBatch/WriteBatch signatures +// since ipv4.Message and ipv6.Message are the same type (socket.Message). +type scionBatchRW interface { + ReadBatch([]ipv4.Message, int) (int, error) + WriteBatch([]ipv4.Message, int) (int, error) +} + // scionConn wraps a SCION connection for use by magicsock. type scionConn struct { - conn *snet.Conn // from SCIONNetwork.Listen() - localIA addr.IA // our ISD-AS - daemon daemon.Connector // for path queries - topo snet.Topology // local topology + conn *snet.Conn // from SCIONNetwork.Listen() + underlayConn *net.UDPConn // raw underlay for fast-path sends (owned by conn) + underlayXPC scionBatchRW // for WriteBatch / sendmmsg (ipv4 or ipv6) + localIA addr.IA // our ISD-AS + localHostIP netip.Addr // local host IP (e.g. 127.0.0.1) + localPort uint16 // local SCION/UDP port + daemon daemon.Connector // for path queries + topo snet.Topology // local topology } // close shuts down the SCION connection and daemon connector. @@ -255,9 +558,26 @@ func scionListenPort() uint16 { return 0 // let snet auto-select from topology port range } +// scionListenAddr returns the listen address for the SCION underlay socket. +// TS_SCION_LISTEN_ADDR can override the IP (e.g. "::1" for IPv6 localhost). +// Defaults to 127.0.0.1 (matches current behavior and snet requirement that +// the address not be unspecified). +func scionListenAddr() *net.UDPAddr { + port := scionListenPort() + if a := os.Getenv("TS_SCION_LISTEN_ADDR"); a != "" { + ip := net.ParseIP(a) + if ip != nil { + return &net.UDPAddr{IP: ip, Port: int(port)} + } + } + return &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(port)} +} + // trySCIONConnect attempts to connect to the local SCION daemon and set up a -// SCION listener. The listener binds to 127.0.0.1 (required by snet, which -// rejects unspecified addresses) on a port within the dispatched range. +// SCION listener. The listener binds to a localhost address (required by snet, +// which rejects unspecified addresses) on a port within the dispatched range. +// The listen IP defaults to 127.0.0.1 but can be overridden via +// TS_SCION_LISTEN_ADDR (e.g. "::1" for IPv6 underlay). // Returns nil if SCION is not available. func trySCIONConnect(ctx context.Context) (*scionConn, error) { daemonAddr := scionDaemonAddr() @@ -278,25 +598,21 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { Topology: conn, } - listenPort := scionListenPort() - if listenPort != 0 { + listenAddr := scionListenAddr() + if listenAddr.Port != 0 { // Validate the configured port against the daemon's dispatched range. portMin, portMax, err := conn.PortRange(ctx) if err != nil { conn.Close() return nil, fmt.Errorf("querying SCION port range: %w", err) } + listenPort := uint16(listenAddr.Port) if listenPort < portMin || listenPort > portMax { conn.Close() return nil, fmt.Errorf("TS_SCION_PORT=%d outside dispatched range [%d, %d]", listenPort, portMin, portMax) } } - listenAddr := &net.UDPAddr{ - IP: net.IPv4(127, 0, 0, 1), - Port: int(listenPort), - } - // Use OpenRaw + NewCookedConn instead of Listen so we can set socket // buffer sizes on the underlying UDP connection before wrapping it. pconn, err := network.OpenRaw(ctx, listenAddr) @@ -305,10 +621,11 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { return nil, fmt.Errorf("listening on SCION %s: %w", listenAddr, err) } - // Increase underlay UDP socket buffers to match the regular magicsock - // UDP sockets (7 MB). The default kernel buffer (~212 KB) overflows - // easily at high throughput, causing packet drops and TCP retransmissions. + // Extract the underlay *net.UDPConn for fast-path sends that bypass + // snet.Conn serialization. Also increase socket buffer sizes. + var underlayConn *net.UDPConn if pc, ok := pconn.(*snet.SCIONPacketConn); ok { + underlayConn = pc.Conn if err := pc.SetReadBuffer(socketBufferSize); err != nil { fmt.Fprintf(os.Stderr, "magicsock: SCION: failed to set read buffer to %d: %v\n", socketBufferSize, err) } @@ -324,11 +641,37 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { return nil, fmt.Errorf("creating SCION conn: %w", err) } + // Extract local address info for fast-path header templates. + var localHostIP netip.Addr + var localPort uint16 + if sa, saOk := sconn.LocalAddr().(*snet.UDPAddr); saOk && sa.Host != nil { + if ip, ipOk := netip.AddrFromSlice(sa.Host.IP); ipOk { + localHostIP = ip + } + localPort = uint16(sa.Host.Port) + } + + // Wrap underlay conn for sendmmsg batching, selecting the correct + // address family based on the local address. + var underlayXPC scionBatchRW + if underlayConn != nil { + local := underlayConn.LocalAddr().(*net.UDPAddr) + if local.IP.To4() != nil { + underlayXPC = ipv4.NewPacketConn(underlayConn) + } else { + underlayXPC = ipv6.NewPacketConn(underlayConn) + } + } + return &scionConn{ - conn: sconn, - localIA: localIA, - daemon: conn, - topo: conn, + conn: sconn, + underlayConn: underlayConn, + underlayXPC: underlayXPC, + localIA: localIA, + localHostIP: localHostIP, + localPort: localPort, + daemon: conn, + topo: conn, }, nil } @@ -357,6 +700,11 @@ func parseSCIONServiceAddr(description string, port uint16) (ia addr.IA, hostAdd // sendSCIONBatch sends a batch of WireGuard packets over the SCION connection. // It looks up the full path info from the Conn's scionPaths registry using the // scionPathKey from the epAddr. +// +// When a fast-path template is available (pre-serialized headers + underlay +// socket), packets are serialized by patching a header template and sent via +// sendmmsg in a single syscall. Otherwise, falls back to snet.Conn.WriteTo +// per packet. func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent bool, err error) { sc := c.pconnSCION if sc == nil { @@ -372,14 +720,20 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo pi.mu.Lock() replyPath := pi.replyPath cachedDst := pi.cachedDst + fastPath := pi.fastPath expired := !pi.expiry.IsZero() && time.Now().After(pi.expiry) pi.mu.Unlock() if expired { return false, fmt.Errorf("SCION path expired for key %d", addr.scionKey) } - // If no discovered path, fall back to replyPath (bootstrapped from an - // incoming packet before path discovery completes). + // Fast path: pre-serialized headers + sendmmsg. + if fastPath != nil && sc.underlayXPC != nil { + err = c.sendSCIONBatchFast(sc, fastPath, buffs, offset) + return err == nil, err + } + + // Slow path: snet.Conn.WriteTo per packet. dst := cachedDst if dst == nil && replyPath != nil { dst = replyPath @@ -397,6 +751,70 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo return true, nil } +// sendSCIONBatchFast sends a batch of packets using pre-serialized SCION +// headers and sendmmsg on the underlay UDP socket. Each packet is built by +// copying the header template, patching per-packet fields (PayloadLen, UDP +// Length, UDP Checksum), and appending the WireGuard payload. +func (c *Conn) sendSCIONBatchFast(sc *scionConn, fp *scionFastPath, buffs [][]byte, offset int) error { + batch := scionSendBatchPool.Get().(*scionSendBatch) + defer scionSendBatchPool.Put(batch) + + hdrLen := len(fp.hdr) + n := len(buffs) + if n > scionMaxBatchSize { + n = scionMaxBatchSize + } + + for i := 0; i < n; i++ { + payload := buffs[i][offset:] + pktLen := hdrLen + len(payload) + + // Grow buffer if needed. + buf := batch.bufs[i] + if cap(buf) < pktLen { + buf = make([]byte, pktLen) + batch.bufs[i] = buf + } else { + buf = buf[:pktLen] + } + + // Copy header template and append payload. + copy(buf, fp.hdr) + copy(buf[hdrLen:], payload) + + // Patch SCION PayloadLen (bytes 6:8) = UDP header (8) + payload. + udpTotalLen := uint16(8 + len(payload)) + binary.BigEndian.PutUint16(buf[6:], udpTotalLen) + + // Patch UDP Length (udpOffset+4:+6). + binary.BigEndian.PutUint16(buf[fp.udpOffset+4:], udpTotalLen) + + // Zero checksum, compute over full upper layer, set result. + buf[fp.udpOffset+6] = 0 + buf[fp.udpOffset+7] = 0 + upperLayer := buf[fp.udpOffset:pktLen] + csum := scionFinishChecksum(fp.pseudoCsum, upperLayer) + binary.BigEndian.PutUint16(buf[fp.udpOffset+6:], csum) + + batch.msgs[i].Buffers[0] = buf[:pktLen] + batch.msgs[i].Addr = fp.nextHop + } + + // WriteBatch uses sendmmsg on Linux for batched sends. + msgs := batch.msgs[:n] + var head int + for { + written, err := sc.underlayXPC.WriteBatch(msgs[head:], 0) + if err != nil { + return err + } + head += written + if head >= n { + return nil + } + } +} + // sendSCION sends a single packet over SCION, used for disco messages. func (c *Conn) sendSCION(sk scionPathKey, b []byte) (bool, error) { sc := c.pconnSCION @@ -502,6 +920,10 @@ func (c *Conn) updateActiveSCIONPathLocking(peerIA addr.IA, hostAddr netip.AddrP // the socket is still alive. If no packets are received for // scionReconnectThreshold while active SCION peers exist, we close the old // socket and reconnect. +// +// When the underlay socket is available, packets are read in batches via +// recvmmsg and parsed with lightweight slayers.SCION decoding. Otherwise, +// falls back to single-packet snet.Conn.ReadFrom. func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { sc := c.pconnSCION if sc == nil { @@ -535,34 +957,55 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) continue } - // Set a read deadline so we wake up periodically even if the socket - // is silently dead (SCION router lost our port registration). + // Fast path: batch read from underlay via recvmmsg. + if sc.underlayXPC != nil { + sc.underlayConn.SetReadDeadline(time.Now().Add(scionReadDeadline)) + + n, err := c.receiveSCIONBatch(sc, buffs, sizes, eps) + if n > 0 { + return n, nil + } + if err != nil { + select { + case <-c.donec: + return 0, net.ErrClosed + default: + } + if isTimeoutError(err) { + if c.shouldReconnectSCION() { + c.reconnectSCION() + } + continue + } + if errors.Is(err, net.ErrClosed) { + continue + } + c.logf("magicsock: SCION read error: %v", err) + continue + } + // n == 0 and no error means all packets were disco/filtered. + continue + } + + // Slow path: single-packet snet.Conn.ReadFrom. sc.conn.SetReadDeadline(time.Now().Add(scionReadDeadline)) n, srcAddr, err := sc.readFrom(buffs[0]) if err != nil { - // Graceful shutdown: Conn is closing. select { case <-c.donec: return 0, net.ErrClosed default: } - - // Timeout: check if we need to reconnect. if isTimeoutError(err) { if c.shouldReconnectSCION() { c.reconnectSCION() } continue } - - // Socket closed (by reconnectSCION or externally): re-read - // pconnSCION on next iteration. if errors.Is(err, net.ErrClosed) { continue } - - // Other errors: log and continue. Never propagate to WireGuard. c.logf("magicsock: SCION read error: %v", err) continue } @@ -570,28 +1013,19 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) continue } - // Got a packet — record receive time. c.lastSCIONRecv.StoreAtomic(mono.Now()) b := buffs[0][:n] - srcHostAddr := srcAddr.Host.AddrPort() - // Check for disco packets (same as receiveIP does). pt, _ := packetLooksLike(b) if pt == packetLooksLikeDisco { - // For disco messages, include the scionKey so pong replies - // are routed back over SCION. Use a single critical section - // for the lookup+register to avoid a TOCTOU race where a - // concurrent discoverSCIONPaths could register between our - // check and our register, creating orphaned entries. + // Slow path disco: snet.Conn.ReadFrom returns a pre-reversed + // path suitable for replies, so use srcAddr directly. srcEpAddr := epAddr{ap: srcHostAddr} c.mu.Lock() sk := c.scionPathsByAddr[scionAddrKey{ia: srcAddr.IA, addr: srcHostAddr}] if !sk.IsSet() { - // First disco packet from this SCION peer — bootstrap a - // reverse path entry so the pong can go back over SCION. - // ReadFrom returns a pre-reversed path suitable for replies. sk = c.registerSCIONPath(&scionPathInfo{ peerIA: srcAddr.IA, hostAddr: srcHostAddr, @@ -609,8 +1043,6 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) continue } - // WireGuard packet — look up the endpoint by host addr only - // (peerMap is keyed by netip.AddrPort, not scionKey). srcEpAddr := epAddr{ap: srcHostAddr} c.mu.Lock() ep, ok := c.peerMap.endpointForEpAddr(srcEpAddr) @@ -634,6 +1066,108 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } } +// receiveSCIONBatch reads a batch of raw SCION packets from the underlay +// socket via recvmmsg, parses SCION+UDP headers with slayers, and copies +// payloads into WireGuard's buffs. Disco packets are handled inline and +// not reported to the caller. +func (c *Conn) receiveSCIONBatch(sc *scionConn, buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { + batch := scionRecvBatchPool.Get().(*scionRecvBatch) + defer putScionRecvBatch(batch) + + n := len(buffs) + if n > scionMaxBatchSize { + n = scionMaxBatchSize + } + + numMsgs, err := sc.underlayXPC.ReadBatch(batch.msgs[:n], 0) + if err != nil { + return 0, err + } + + reportToCaller := false + count := 0 + for i := 0; i < numMsgs; i++ { + msg := &batch.msgs[i] + if msg.N == 0 { + sizes[count] = 0 + continue + } + + srcIA, srcHostAddr, payload, rawPath, ok := parseSCIONPacket( + msg.Buffers[0][:msg.N], &batch.scn) + if !ok || len(payload) == 0 { + continue + } + + // Copy payload into WireGuard's buffer. + pn := copy(buffs[count], payload) + + c.lastSCIONRecv.StoreAtomic(mono.Now()) + + pt, _ := packetLooksLike(buffs[count][:pn]) + if pt == packetLooksLikeDisco { + c.handleSCIONDisco(buffs[count][:pn], srcIA, srcHostAddr, rawPath) + continue + } + + if !c.havePrivateKey.Load() { + continue + } + + srcEpAddr := epAddr{ap: srcHostAddr} + c.mu.Lock() + ep, ok := c.peerMap.endpointForEpAddr(srcEpAddr) + c.mu.Unlock() + if !ok { + sizes[count] = pn + eps[count] = &lazyEndpoint{c: c, src: srcEpAddr} + count++ + reportToCaller = true + continue + } + + now := mono.Now() + ep.lastRecvUDPAny.StoreAtomic(now) + ep.noteRecvActivity(srcEpAddr, now) + if c.metrics != nil { + c.metrics.inboundPacketsSCIONTotal.Add(1) + c.metrics.inboundBytesSCIONTotal.Add(int64(pn)) + } + sizes[count] = pn + eps[count] = ep + count++ + reportToCaller = true + } + + if reportToCaller { + return count, nil + } + return 0, nil +} + +// handleSCIONDisco handles a disco packet received on the batch path. +// It looks up or registers a SCION path entry and dispatches to handleDiscoMessage. +// For first-contact, the raw path bytes are reversed to build a reply path. +func (c *Conn) handleSCIONDisco(b []byte, srcIA addr.IA, srcHostAddr netip.AddrPort, rawPath []byte) { + srcEpAddr := epAddr{ap: srcHostAddr} + c.mu.Lock() + sk := c.scionPathsByAddr[scionAddrKey{ia: srcIA, addr: srcHostAddr}] + if !sk.IsSet() { + // First disco packet from this SCION peer — build a reply path + // by reversing the raw SCION path from the incoming packet. + replyAddr := buildSCIONReplyAddr(srcIA, srcHostAddr, rawPath) + sk = c.registerSCIONPath(&scionPathInfo{ + peerIA: srcIA, + hostAddr: srcHostAddr, + replyPath: replyAddr, + }) + c.setActiveSCIONPath(srcIA, srcHostAddr, sk) + } + c.mu.Unlock() + srcEpAddr.scionKey = sk + c.handleDiscoMessage(b, srcEpAddr, false, key.NodePublic{}, discoRXPathSCION) +} + // isTimeoutError reports whether err is a network timeout (from SetReadDeadline). func isTimeoutError(err error) bool { var netErr net.Error @@ -812,6 +1346,9 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr mtu: mtu, } pi.buildCachedDst() + if sc := c.pconnSCION; sc != nil { + pi.fastPath = buildSCIONFastPath(sc, pi) + } keys = append(keys, c.registerSCIONPath(pi)) } // Set the first (lowest-latency) path as active for the reverse index. @@ -1002,6 +1539,7 @@ func (c *Conn) refreshSCIONPathsOnce() error { pi.mtu = md.MTU } pi.buildCachedDst() + pi.fastPath = buildSCIONFastPath(sc, pi) pi.mu.Unlock() } } diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 939f89cda688f..1533f952bdcb9 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -5,6 +5,7 @@ package magicsock import ( "context" + "encoding/binary" "fmt" "net" "net/netip" @@ -18,6 +19,7 @@ import ( "github.com/scionproto/scion/pkg/daemon/mock_daemon" "github.com/scionproto/scion/pkg/snet" "github.com/scionproto/scion/pkg/snet/mock_snet" + snetpath "github.com/scionproto/scion/pkg/snet/path" "tailscale.com/net/packet" "tailscale.com/net/tstun" "tailscale.com/tailcfg" @@ -1115,3 +1117,371 @@ func TestSendSCIONExpiredPath(t *testing.T) { t.Errorf("error should mention 'expired', got: %v", err) } } + +// TestSCIONPseudoHeaderPartial verifies the partial checksum computation +// matches the reference SCION implementation for known inputs. +func TestSCIONPseudoHeaderPartial(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + + // Verify by computing the same checksum manually: + // srcIA = 0x0001ff0000000110, dstIA = 0x0001ff0000000111 + // srcAddr = 127.0.0.1 = [0x7f, 0x00, 0x00, 0x01] + // dstAddr = 127.0.0.1 = [0x7f, 0x00, 0x00, 0x01] + // protocol = 17 + + var expected uint32 + // srcIA bytes: 00 01 ff 00 00 00 01 10 + expected += 0x0001 + 0xff00 + 0x0000 + 0x0110 + // dstIA bytes: 00 01 ff 00 00 00 01 11 + expected += 0x0001 + 0xff00 + 0x0000 + 0x0111 + // srcAddr: 7f 00 00 01 + expected += 0x7f00 + 0x0001 + // dstAddr: 7f 00 00 01 + expected += 0x7f00 + 0x0001 + // protocol + expected += 17 + + if partial != expected { + t.Errorf("scionPseudoHeaderPartial = %d, want %d", partial, expected) + } +} + +// TestSCIONPseudoHeaderPartialIPv6 verifies checksum with IPv6 addresses. +func TestSCIONPseudoHeaderPartialIPv6(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("::1") + dstIP := netip.MustParseAddr("fd00::1") + + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + if partial == 0 { + t.Fatal("checksum should not be zero") + } + + // Verify IPv6 addrs are 16 bytes each. + // ::1 = 00...01, fd00::1 = fd 00 00...01 + var expected uint32 + // IAs + expected += 0x0001 + 0xff00 + 0x0000 + 0x0110 + expected += 0x0001 + 0xff00 + 0x0000 + 0x0111 + // srcIP ::1 = all zeros except last byte + expected += 0x0001 + // dstIP fd00::1 + expected += 0xfd00 + 0x0001 + expected += 17 + + if partial != expected { + t.Errorf("scionPseudoHeaderPartial(IPv6) = %d, want %d", partial, expected) + } +} + +// TestSCIONFinishChecksum verifies the full checksum computation matches +// the reference SCION implementation by comparing against a packet +// serialized with snet.Packet.Serialize(). +func TestSCIONFinishChecksum(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + srcPort := uint16(32766) + dstPort := uint16(32766) + payload := []byte("Hello, SCION fast path!") + + // Build the packet using snet's reference serializer. + pkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: srcPort, + DstPort: dstPort, + Payload: payload, + }, + }, + } + if err := pkt.Serialize(); err != nil { + t.Fatalf("snet.Packet.Serialize: %v", err) + } + + // Extract the reference checksum from the serialized packet. + // The UDP header is the last 8 bytes before the payload. + udpOffset := len(pkt.Bytes) - 8 - len(payload) + refChecksum := binary.BigEndian.Uint16(pkt.Bytes[udpOffset+6:]) + + // Now compute it using our fast-path functions. + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + + // Build the upper layer: UDP header (8 bytes) + payload + upperLayer := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint16(upperLayer[0:], srcPort) + binary.BigEndian.PutUint16(upperLayer[2:], dstPort) + binary.BigEndian.PutUint16(upperLayer[4:], uint16(8+len(payload))) + // checksum field = 0 for computation + copy(upperLayer[8:], payload) + + fastChecksum := scionFinishChecksum(partial, upperLayer) + + if fastChecksum != refChecksum { + t.Errorf("fast-path checksum = 0x%04x, reference = 0x%04x", fastChecksum, refChecksum) + } +} + +// TestSCIONFinishChecksumEmptyPayload verifies checksum with empty payload. +func TestSCIONFinishChecksumEmptyPayload(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + + pkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: 1000, + DstPort: 2000, + Payload: nil, + }, + }, + } + if err := pkt.Serialize(); err != nil { + t.Fatalf("snet.Packet.Serialize: %v", err) + } + + // UDP header is the last 8 bytes (no payload). + udpOffset := len(pkt.Bytes) - 8 + refChecksum := binary.BigEndian.Uint16(pkt.Bytes[udpOffset+6:]) + + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + upperLayer := make([]byte, 8) + binary.BigEndian.PutUint16(upperLayer[0:], 1000) + binary.BigEndian.PutUint16(upperLayer[2:], 2000) + binary.BigEndian.PutUint16(upperLayer[4:], 8) + + fastChecksum := scionFinishChecksum(partial, upperLayer) + + if fastChecksum != refChecksum { + t.Errorf("fast-path checksum (empty) = 0x%04x, reference = 0x%04x", fastChecksum, refChecksum) + } +} + +// TestSCIONFinishChecksumOddPayload verifies correct handling of odd-length payloads. +func TestSCIONFinishChecksumOddPayload(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("10.0.0.1") + payload := []byte("ABC") // 3 bytes, odd + + pkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: 5000, + DstPort: 6000, + Payload: payload, + }, + }, + } + if err := pkt.Serialize(); err != nil { + t.Fatalf("snet.Packet.Serialize: %v", err) + } + + udpOffset := len(pkt.Bytes) - 8 - len(payload) + refChecksum := binary.BigEndian.Uint16(pkt.Bytes[udpOffset+6:]) + + partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) + upperLayer := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint16(upperLayer[0:], 5000) + binary.BigEndian.PutUint16(upperLayer[2:], 6000) + binary.BigEndian.PutUint16(upperLayer[4:], uint16(8+len(payload))) + copy(upperLayer[8:], payload) + + fastChecksum := scionFinishChecksum(partial, upperLayer) + + if fastChecksum != refChecksum { + t.Errorf("fast-path checksum (odd) = 0x%04x, reference = 0x%04x", fastChecksum, refChecksum) + } +} + +// TestBuildSCIONFastPath verifies that buildSCIONFastPath produces a template +// that matches the reference serializer output for the same parameters. +func TestBuildSCIONFastPath(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + srcPort := uint16(32766) + dstPort := uint16(32766) + + sc := &scionConn{ + underlayConn: &net.UDPConn{}, // non-nil to enable fast path + localIA: srcIA, + localHostIP: srcIP, + localPort: srcPort, + } + + pi := &scionPathInfo{ + peerIA: dstIA, + hostAddr: netip.MustParseAddrPort("127.0.0.1:32766"), + cachedDst: &snet.UDPAddr{ + IA: dstIA, + Host: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(dstPort)}, + Path: snetpath.Empty{}, + NextHop: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 30041}, + }, + } + + fp := buildSCIONFastPath(sc, pi) + if fp == nil { + t.Fatal("buildSCIONFastPath returned nil") + } + + // The template should match a reference packet with empty payload. + refPkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: srcPort, + DstPort: dstPort, + Payload: nil, + }, + }, + } + if err := refPkt.Serialize(); err != nil { + t.Fatalf("reference Serialize: %v", err) + } + + if len(fp.hdr) != len(refPkt.Bytes) { + t.Fatalf("fast-path header len = %d, reference = %d", len(fp.hdr), len(refPkt.Bytes)) + } + + // Compare header bytes (everything except checksum which may differ + // due to computation order, but should be the same for empty payload). + for i := range fp.hdr { + if fp.hdr[i] != refPkt.Bytes[i] { + t.Errorf("byte %d: fast-path=0x%02x, reference=0x%02x", i, fp.hdr[i], refPkt.Bytes[i]) + } + } + + if fp.udpOffset != len(fp.hdr)-8 { + t.Errorf("udpOffset = %d, expected %d", fp.udpOffset, len(fp.hdr)-8) + } + + if fp.nextHop == nil { + t.Error("nextHop should not be nil") + } +} + +// TestSCIONFastPathPacketMatchesReference verifies that a packet built with +// the fast-path template+patching produces identical bytes to one built with +// snet.Packet.Serialize(). +func TestSCIONFastPathPacketMatchesReference(t *testing.T) { + srcIA := addr.MustParseIA("1-ff00:0:110") + dstIA := addr.MustParseIA("1-ff00:0:111") + srcIP := netip.MustParseAddr("127.0.0.1") + dstIP := netip.MustParseAddr("127.0.0.1") + srcPort := uint16(32766) + dstPort := uint16(32766) + payload := []byte("WireGuard test payload data for SCION fast path verification") + + sc := &scionConn{ + underlayConn: &net.UDPConn{}, + localIA: srcIA, + localHostIP: srcIP, + localPort: srcPort, + } + + pi := &scionPathInfo{ + peerIA: dstIA, + hostAddr: netip.MustParseAddrPort("127.0.0.1:32766"), + cachedDst: &snet.UDPAddr{ + IA: dstIA, + Host: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(dstPort)}, + Path: snetpath.Empty{}, + NextHop: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 30041}, + }, + } + + fp := buildSCIONFastPath(sc, pi) + if fp == nil { + t.Fatal("buildSCIONFastPath returned nil") + } + + // Build packet using fast-path template + patching. + hdrLen := len(fp.hdr) + pktLen := hdrLen + len(payload) + buf := make([]byte, pktLen) + copy(buf, fp.hdr) + copy(buf[hdrLen:], payload) + + udpTotalLen := uint16(8 + len(payload)) + binary.BigEndian.PutUint16(buf[6:], udpTotalLen) + binary.BigEndian.PutUint16(buf[fp.udpOffset+4:], udpTotalLen) + buf[fp.udpOffset+6] = 0 + buf[fp.udpOffset+7] = 0 + upperLayer := buf[fp.udpOffset:pktLen] + csum := scionFinishChecksum(fp.pseudoCsum, upperLayer) + binary.BigEndian.PutUint16(buf[fp.udpOffset+6:], csum) + + // Build reference packet using snet. + refPkt := &snet.Packet{ + PacketInfo: snet.PacketInfo{ + Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, + Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, + Path: snetpath.Empty{}, + Payload: snet.UDPPayload{ + SrcPort: srcPort, + DstPort: dstPort, + Payload: payload, + }, + }, + } + if err := refPkt.Serialize(); err != nil { + t.Fatalf("reference Serialize: %v", err) + } + + if len(buf) != len(refPkt.Bytes) { + t.Fatalf("fast-path pkt len = %d, reference = %d", len(buf), len(refPkt.Bytes)) + } + + for i := range buf { + if buf[i] != refPkt.Bytes[i] { + t.Errorf("byte %d: fast-path=0x%02x, reference=0x%02x", i, buf[i], refPkt.Bytes[i]) + } + } +} + +// TestSCIONSendBatchPool verifies the pool returns usable batches. +func TestSCIONSendBatchPool(t *testing.T) { + batch := scionSendBatchPool.Get().(*scionSendBatch) + defer scionSendBatchPool.Put(batch) + + if len(batch.bufs) != scionMaxBatchSize { + t.Errorf("batch.bufs len = %d, want %d", len(batch.bufs), scionMaxBatchSize) + } + if len(batch.msgs) != scionMaxBatchSize { + t.Errorf("batch.msgs len = %d, want %d", len(batch.msgs), scionMaxBatchSize) + } + for i, buf := range batch.bufs { + if cap(buf) < 1500 { + t.Errorf("batch.bufs[%d] cap = %d, want >= 1500", i, cap(buf)) + } + } + for i, msg := range batch.msgs { + if len(msg.Buffers) != 1 { + t.Errorf("batch.msgs[%d].Buffers len = %d, want 1", i, len(msg.Buffers)) + } + } +} From a59ac2503a21dee7969468714fab6a4d4f3d46c6 Mon Sep 17 00:00:00 2001 From: Tony John <38129229+tjohn327@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:46:29 +0100 Subject: [PATCH 69/78] Revert "Add batch read and write support for SCION (#1)" This reverts commit 6bb89eaad226728cbb72a00662be9eeeddc4434e. --- wgengine/magicsock/magicsock_scion.go | 630 ++------------------- wgengine/magicsock/magicsock_scion_test.go | 370 ------------ 2 files changed, 46 insertions(+), 954 deletions(-) diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 31d9bb413234c..d9d6e9fac32b4 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -5,7 +5,6 @@ package magicsock import ( "context" - "encoding/binary" "errors" "fmt" "net" @@ -16,16 +15,10 @@ import ( "sync" "time" - "github.com/google/gopacket" "github.com/scionproto/scion/pkg/addr" "github.com/scionproto/scion/pkg/daemon" - "github.com/scionproto/scion/pkg/slayers" - scionpath "github.com/scionproto/scion/pkg/slayers/path/scion" - snetpath "github.com/scionproto/scion/pkg/snet/path" "github.com/scionproto/scion/pkg/snet" wgconn "github.com/tailscale/wireguard-go/conn" - "golang.org/x/net/ipv4" - "golang.org/x/net/ipv6" "tailscale.com/envknob" "tailscale.com/tailcfg" "tailscale.com/tstime/mono" @@ -78,7 +71,6 @@ type scionPathInfo struct { path snet.Path // current best SCION path to this peer replyPath *snet.UDPAddr // bootstrapped from incoming packet (pre-reversed) cachedDst *snet.UDPAddr // pre-built destination addr; rebuilt when path changes - fastPath *scionFastPath // pre-serialized header template for fast sends expiry time.Time // path expiration from path metadata mtu uint16 // SCION payload MTU from path metadata mu sync.Mutex @@ -103,7 +95,7 @@ func (pi *scionPathInfo) buildCachedDst() { // scionHeaderOverhead is the fixed overhead added by SCION encapsulation, // excluding the variable-length path header: -// - Underlay IPv4+UDP: 20 + 8 = 28 bytes (or IPv6+UDP: 40 + 8 = 48 bytes) +// - Underlay IPv4+UDP: 20 + 8 = 28 bytes // - SCION common header: 12 bytes // - Address header (IPv4, 2x ISD-AS + 2x IPv4): 2*8 + 2*4 = 24 bytes // - SCION/UDP L4 header: 8 bytes @@ -191,307 +183,12 @@ func (ps *scionPathProbeState) latency() time.Duration { return ps.recentPongs[ps.recentPong].latency } -// scionFastPath holds a pre-serialized SCION+UDP header template for a -// specific path. At send time, the template is copied, per-packet fields -// (PayloadLen, UDP Length, UDP Checksum) are patched, payload is appended, -// and the result is sent directly on the underlay UDP socket — bypassing -// snet.Conn and gopacket serialization entirely. -type scionFastPath struct { - hdr []byte // [SCION header][UDP header], no payload - udpOffset int // byte offset of UDP header within hdr - nextHop *net.UDPAddr // underlay next-hop for this path - pseudoCsum uint32 // constant part of SCION pseudo-header checksum -} - -// scionMaxBatchSize is the max number of packets in a single sendmmsg call. -const scionMaxBatchSize = 64 - -// scionSendBatch is a reusable set of buffers for sendSCIONBatchFast. -type scionSendBatch struct { - bufs [][]byte - msgs []ipv4.Message -} - -var scionSendBatchPool = sync.Pool{ - New: func() any { - b := &scionSendBatch{ - bufs: make([][]byte, scionMaxBatchSize), - msgs: make([]ipv4.Message, scionMaxBatchSize), - } - for i := range b.bufs { - b.bufs[i] = make([]byte, 1500) - } - for i := range b.msgs { - b.msgs[i].Buffers = make([][]byte, 1) - } - return b - }, -} - -// scionRecvBatch is a reusable set of buffers for receiveSCIONBatch. -type scionRecvBatch struct { - msgs []ipv4.Message - bufs [][]byte - scn slayers.SCION // reusable SCION header parser (with RecyclePaths) -} - -var scionRecvBatchPool = sync.Pool{ - New: func() any { - b := &scionRecvBatch{ - msgs: make([]ipv4.Message, scionMaxBatchSize), - bufs: make([][]byte, scionMaxBatchSize), - } - b.scn.RecyclePaths() - for i := range b.bufs { - b.bufs[i] = make([]byte, 1500) - } - for i := range b.msgs { - b.msgs[i].Buffers = [][]byte{b.bufs[i]} - } - return b - }, -} - -// putScionRecvBatch resets batch state and returns it to the pool. -func putScionRecvBatch(batch *scionRecvBatch) { - for i := range batch.msgs { - batch.msgs[i].N = 0 - batch.msgs[i].Addr = nil - batch.msgs[i].Buffers[0] = batch.bufs[i] - } - scionRecvBatchPool.Put(batch) -} - -// scionPseudoHeaderPartial computes the constant part of the SCION -// pseudo-header checksum: srcIA + dstIA + srcAddr + dstAddr + protocol(17). -// The per-packet upper-layer length and data are added at send time. -func scionPseudoHeaderPartial(srcIA, dstIA addr.IA, srcIP, dstIP netip.Addr) uint32 { - var csum uint32 - var buf [8]byte - - // Source IA (8 bytes) - binary.BigEndian.PutUint64(buf[:], uint64(srcIA)) - for i := 0; i < 8; i += 2 { - csum += uint32(buf[i]) << 8 - csum += uint32(buf[i+1]) - } - - // Destination IA (8 bytes) - binary.BigEndian.PutUint64(buf[:], uint64(dstIA)) - for i := 0; i < 8; i += 2 { - csum += uint32(buf[i]) << 8 - csum += uint32(buf[i+1]) - } - - // Source address - if srcIP.Is4() { - b4 := srcIP.As4() - csum += uint32(b4[0])<<8 + uint32(b4[1]) - csum += uint32(b4[2])<<8 + uint32(b4[3]) - } else { - b16 := srcIP.As16() - for i := 0; i < 16; i += 2 { - csum += uint32(b16[i])<<8 + uint32(b16[i+1]) - } - } - - // Destination address - if dstIP.Is4() { - b4 := dstIP.As4() - csum += uint32(b4[0])<<8 + uint32(b4[1]) - csum += uint32(b4[2])<<8 + uint32(b4[3]) - } else { - b16 := dstIP.As16() - for i := 0; i < 16; i += 2 { - csum += uint32(b16[i])<<8 + uint32(b16[i+1]) - } - } - - // Protocol: L4UDP = 17 - csum += 17 - - return csum -} - -// scionFinishChecksum completes the SCION/UDP checksum by adding the -// upper-layer length and bytes to the pre-computed partial checksum, -// then folding and complementing. -func scionFinishChecksum(partialCsum uint32, upperLayer []byte) uint16 { - csum := partialCsum - - // Add upper-layer length - l := uint32(len(upperLayer)) - csum += (l >> 16) + (l & 0xffff) - - // Sum upper-layer bytes in 16-bit words - n := len(upperLayer) - for i := 0; i+1 < n; i += 2 { - csum += uint32(upperLayer[i]) << 8 - csum += uint32(upperLayer[i+1]) - } - if n%2 == 1 { - csum += uint32(upperLayer[n-1]) << 8 - } - - // Fold to 16 bits - for csum > 0xffff { - csum = (csum >> 16) + (csum & 0xffff) - } - return ^uint16(csum) -} - -// buildSCIONFastPath creates a pre-serialized header template for fast-path -// sends. Must be called with pi.mu held (or before pi is shared). -// Returns nil if the fast path cannot be built (e.g. no discovered path). -func buildSCIONFastPath(sc *scionConn, pi *scionPathInfo) *scionFastPath { - if sc.underlayConn == nil { - return nil - } - dst := pi.cachedDst - if dst == nil || dst.Path == nil || dst.NextHop == nil { - return nil - } - - dstIP, ok := netip.AddrFromSlice(dst.Host.IP) - if !ok { - return nil - } - srcIP := sc.localHostIP - - // Use snet.Packet.Serialize() with empty payload to get a correctly - // encoded SCION+UDP header template. - pkt := &snet.Packet{ - PacketInfo: snet.PacketInfo{ - Destination: snet.SCIONAddress{IA: pi.peerIA, Host: addr.HostIP(dstIP)}, - Source: snet.SCIONAddress{IA: sc.localIA, Host: addr.HostIP(srcIP)}, - Path: dst.Path, - Payload: snet.UDPPayload{ - SrcPort: sc.localPort, - DstPort: uint16(dst.Host.Port), - Payload: nil, // empty payload → headers only - }, - }, - } - if err := pkt.Serialize(); err != nil { - return nil - } - - // pkt.Bytes is now [SCION header][8-byte UDP header] - hdr := make([]byte, len(pkt.Bytes)) - copy(hdr, pkt.Bytes) - udpOffset := len(hdr) - 8 - - pseudoCsum := scionPseudoHeaderPartial(sc.localIA, pi.peerIA, srcIP, dstIP) - - return &scionFastPath{ - hdr: hdr, - udpOffset: udpOffset, - nextHop: dst.NextHop, - pseudoCsum: pseudoCsum, - } -} - -// parseSCIONPacket parses a raw SCION packet from the underlay, extracting -// the source address info and UDP payload. scn is a reusable slayers.SCION -// (with RecyclePaths enabled). Returns srcIA, srcAddr, payload, rawPath, ok. -func parseSCIONPacket(data []byte, scn *slayers.SCION) ( - srcIA addr.IA, srcAddr netip.AddrPort, payload []byte, rawPathBytes []byte, ok bool, -) { - if err := scn.DecodeFromBytes(data, gopacket.NilDecodeFeedback); err != nil { - return 0, netip.AddrPort{}, nil, nil, false - } - if scn.NextHdr != slayers.L4UDP { - return 0, netip.AddrPort{}, nil, nil, false - } - - srcHost, err := scn.SrcAddr() - if err != nil { - return 0, netip.AddrPort{}, nil, nil, false - } - srcIP := srcHost.IP() - srcIA = scn.SrcIA - - // L4 payload starts at HdrLen * 4 bytes (SCION header is HdrLen - // 4-byte words). The first 8 bytes are the UDP header. - hdrBytes := int(scn.HdrLen) * 4 - if len(data) < hdrBytes+8 { - return 0, netip.AddrPort{}, nil, nil, false - } - // Extract UDP source port from the first 2 bytes of the L4 header. - srcPort := binary.BigEndian.Uint16(data[hdrBytes:]) - srcAddr = netip.AddrPortFrom(srcIP, srcPort) - payload = data[hdrBytes+8:] - - // Extract raw path bytes for potential reversal (disco first-contact). - if scn.Path != nil { - pathLen := scn.Path.Len() - // The path sits between the address header and the L4 header - // in the SCION common+address+path header region. - addrHdrLen := scn.AddrHdrLen() - // Common header is 12 bytes, then address header, then path. - pathStart := 12 + addrHdrLen - pathEnd := pathStart + pathLen - if pathEnd <= hdrBytes && pathLen > 0 { - rawPathBytes = data[pathStart:pathEnd] - } - } - - return srcIA, srcAddr, payload, rawPathBytes, true -} - -// buildSCIONReplyAddr builds an *snet.UDPAddr with reversed path for disco -// reply routing from raw path bytes extracted during receive. -func buildSCIONReplyAddr(srcIA addr.IA, srcHostAddr netip.AddrPort, rawPathBytes []byte) *snet.UDPAddr { - if len(rawPathBytes) == 0 { - return nil - } - // Copy path bytes since DecodeFromBytes references the slice. - pathCopy := make([]byte, len(rawPathBytes)) - copy(pathCopy, rawPathBytes) - - var raw scionpath.Raw - if err := raw.DecodeFromBytes(pathCopy); err != nil { - return nil - } - reversed, err := raw.Reverse() - if err != nil { - return nil - } - // Serialize the reversed path to raw bytes and wrap in snetpath.SCION - // which implements snet.DataplanePath. - revBytes := make([]byte, reversed.Len()) - if err := reversed.SerializeTo(revBytes); err != nil { - return nil - } - - return &snet.UDPAddr{ - IA: srcIA, - Host: &net.UDPAddr{ - IP: srcHostAddr.Addr().AsSlice(), - Port: int(srcHostAddr.Port()), - }, - Path: snetpath.SCION{Raw: revBytes}, - } -} - -// scionBatchRW abstracts ipv4.PacketConn and ipv6.PacketConn for -// batch I/O. Both have identical ReadBatch/WriteBatch signatures -// since ipv4.Message and ipv6.Message are the same type (socket.Message). -type scionBatchRW interface { - ReadBatch([]ipv4.Message, int) (int, error) - WriteBatch([]ipv4.Message, int) (int, error) -} - // scionConn wraps a SCION connection for use by magicsock. type scionConn struct { - conn *snet.Conn // from SCIONNetwork.Listen() - underlayConn *net.UDPConn // raw underlay for fast-path sends (owned by conn) - underlayXPC scionBatchRW // for WriteBatch / sendmmsg (ipv4 or ipv6) - localIA addr.IA // our ISD-AS - localHostIP netip.Addr // local host IP (e.g. 127.0.0.1) - localPort uint16 // local SCION/UDP port - daemon daemon.Connector // for path queries - topo snet.Topology // local topology + conn *snet.Conn // from SCIONNetwork.Listen() + localIA addr.IA // our ISD-AS + daemon daemon.Connector // for path queries + topo snet.Topology // local topology } // close shuts down the SCION connection and daemon connector. @@ -558,26 +255,9 @@ func scionListenPort() uint16 { return 0 // let snet auto-select from topology port range } -// scionListenAddr returns the listen address for the SCION underlay socket. -// TS_SCION_LISTEN_ADDR can override the IP (e.g. "::1" for IPv6 localhost). -// Defaults to 127.0.0.1 (matches current behavior and snet requirement that -// the address not be unspecified). -func scionListenAddr() *net.UDPAddr { - port := scionListenPort() - if a := os.Getenv("TS_SCION_LISTEN_ADDR"); a != "" { - ip := net.ParseIP(a) - if ip != nil { - return &net.UDPAddr{IP: ip, Port: int(port)} - } - } - return &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(port)} -} - // trySCIONConnect attempts to connect to the local SCION daemon and set up a -// SCION listener. The listener binds to a localhost address (required by snet, -// which rejects unspecified addresses) on a port within the dispatched range. -// The listen IP defaults to 127.0.0.1 but can be overridden via -// TS_SCION_LISTEN_ADDR (e.g. "::1" for IPv6 underlay). +// SCION listener. The listener binds to 127.0.0.1 (required by snet, which +// rejects unspecified addresses) on a port within the dispatched range. // Returns nil if SCION is not available. func trySCIONConnect(ctx context.Context) (*scionConn, error) { daemonAddr := scionDaemonAddr() @@ -598,21 +278,25 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { Topology: conn, } - listenAddr := scionListenAddr() - if listenAddr.Port != 0 { + listenPort := scionListenPort() + if listenPort != 0 { // Validate the configured port against the daemon's dispatched range. portMin, portMax, err := conn.PortRange(ctx) if err != nil { conn.Close() return nil, fmt.Errorf("querying SCION port range: %w", err) } - listenPort := uint16(listenAddr.Port) if listenPort < portMin || listenPort > portMax { conn.Close() return nil, fmt.Errorf("TS_SCION_PORT=%d outside dispatched range [%d, %d]", listenPort, portMin, portMax) } } + listenAddr := &net.UDPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: int(listenPort), + } + // Use OpenRaw + NewCookedConn instead of Listen so we can set socket // buffer sizes on the underlying UDP connection before wrapping it. pconn, err := network.OpenRaw(ctx, listenAddr) @@ -621,11 +305,10 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { return nil, fmt.Errorf("listening on SCION %s: %w", listenAddr, err) } - // Extract the underlay *net.UDPConn for fast-path sends that bypass - // snet.Conn serialization. Also increase socket buffer sizes. - var underlayConn *net.UDPConn + // Increase underlay UDP socket buffers to match the regular magicsock + // UDP sockets (7 MB). The default kernel buffer (~212 KB) overflows + // easily at high throughput, causing packet drops and TCP retransmissions. if pc, ok := pconn.(*snet.SCIONPacketConn); ok { - underlayConn = pc.Conn if err := pc.SetReadBuffer(socketBufferSize); err != nil { fmt.Fprintf(os.Stderr, "magicsock: SCION: failed to set read buffer to %d: %v\n", socketBufferSize, err) } @@ -641,37 +324,11 @@ func trySCIONConnect(ctx context.Context) (*scionConn, error) { return nil, fmt.Errorf("creating SCION conn: %w", err) } - // Extract local address info for fast-path header templates. - var localHostIP netip.Addr - var localPort uint16 - if sa, saOk := sconn.LocalAddr().(*snet.UDPAddr); saOk && sa.Host != nil { - if ip, ipOk := netip.AddrFromSlice(sa.Host.IP); ipOk { - localHostIP = ip - } - localPort = uint16(sa.Host.Port) - } - - // Wrap underlay conn for sendmmsg batching, selecting the correct - // address family based on the local address. - var underlayXPC scionBatchRW - if underlayConn != nil { - local := underlayConn.LocalAddr().(*net.UDPAddr) - if local.IP.To4() != nil { - underlayXPC = ipv4.NewPacketConn(underlayConn) - } else { - underlayXPC = ipv6.NewPacketConn(underlayConn) - } - } - return &scionConn{ - conn: sconn, - underlayConn: underlayConn, - underlayXPC: underlayXPC, - localIA: localIA, - localHostIP: localHostIP, - localPort: localPort, - daemon: conn, - topo: conn, + conn: sconn, + localIA: localIA, + daemon: conn, + topo: conn, }, nil } @@ -700,11 +357,6 @@ func parseSCIONServiceAddr(description string, port uint16) (ia addr.IA, hostAdd // sendSCIONBatch sends a batch of WireGuard packets over the SCION connection. // It looks up the full path info from the Conn's scionPaths registry using the // scionPathKey from the epAddr. -// -// When a fast-path template is available (pre-serialized headers + underlay -// socket), packets are serialized by patching a header template and sent via -// sendmmsg in a single syscall. Otherwise, falls back to snet.Conn.WriteTo -// per packet. func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent bool, err error) { sc := c.pconnSCION if sc == nil { @@ -720,20 +372,14 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo pi.mu.Lock() replyPath := pi.replyPath cachedDst := pi.cachedDst - fastPath := pi.fastPath expired := !pi.expiry.IsZero() && time.Now().After(pi.expiry) pi.mu.Unlock() if expired { return false, fmt.Errorf("SCION path expired for key %d", addr.scionKey) } - // Fast path: pre-serialized headers + sendmmsg. - if fastPath != nil && sc.underlayXPC != nil { - err = c.sendSCIONBatchFast(sc, fastPath, buffs, offset) - return err == nil, err - } - - // Slow path: snet.Conn.WriteTo per packet. + // If no discovered path, fall back to replyPath (bootstrapped from an + // incoming packet before path discovery completes). dst := cachedDst if dst == nil && replyPath != nil { dst = replyPath @@ -751,70 +397,6 @@ func (c *Conn) sendSCIONBatch(addr epAddr, buffs [][]byte, offset int) (sent boo return true, nil } -// sendSCIONBatchFast sends a batch of packets using pre-serialized SCION -// headers and sendmmsg on the underlay UDP socket. Each packet is built by -// copying the header template, patching per-packet fields (PayloadLen, UDP -// Length, UDP Checksum), and appending the WireGuard payload. -func (c *Conn) sendSCIONBatchFast(sc *scionConn, fp *scionFastPath, buffs [][]byte, offset int) error { - batch := scionSendBatchPool.Get().(*scionSendBatch) - defer scionSendBatchPool.Put(batch) - - hdrLen := len(fp.hdr) - n := len(buffs) - if n > scionMaxBatchSize { - n = scionMaxBatchSize - } - - for i := 0; i < n; i++ { - payload := buffs[i][offset:] - pktLen := hdrLen + len(payload) - - // Grow buffer if needed. - buf := batch.bufs[i] - if cap(buf) < pktLen { - buf = make([]byte, pktLen) - batch.bufs[i] = buf - } else { - buf = buf[:pktLen] - } - - // Copy header template and append payload. - copy(buf, fp.hdr) - copy(buf[hdrLen:], payload) - - // Patch SCION PayloadLen (bytes 6:8) = UDP header (8) + payload. - udpTotalLen := uint16(8 + len(payload)) - binary.BigEndian.PutUint16(buf[6:], udpTotalLen) - - // Patch UDP Length (udpOffset+4:+6). - binary.BigEndian.PutUint16(buf[fp.udpOffset+4:], udpTotalLen) - - // Zero checksum, compute over full upper layer, set result. - buf[fp.udpOffset+6] = 0 - buf[fp.udpOffset+7] = 0 - upperLayer := buf[fp.udpOffset:pktLen] - csum := scionFinishChecksum(fp.pseudoCsum, upperLayer) - binary.BigEndian.PutUint16(buf[fp.udpOffset+6:], csum) - - batch.msgs[i].Buffers[0] = buf[:pktLen] - batch.msgs[i].Addr = fp.nextHop - } - - // WriteBatch uses sendmmsg on Linux for batched sends. - msgs := batch.msgs[:n] - var head int - for { - written, err := sc.underlayXPC.WriteBatch(msgs[head:], 0) - if err != nil { - return err - } - head += written - if head >= n { - return nil - } - } -} - // sendSCION sends a single packet over SCION, used for disco messages. func (c *Conn) sendSCION(sk scionPathKey, b []byte) (bool, error) { sc := c.pconnSCION @@ -920,10 +502,6 @@ func (c *Conn) updateActiveSCIONPathLocking(peerIA addr.IA, hostAddr netip.AddrP // the socket is still alive. If no packets are received for // scionReconnectThreshold while active SCION peers exist, we close the old // socket and reconnect. -// -// When the underlay socket is available, packets are read in batches via -// recvmmsg and parsed with lightweight slayers.SCION decoding. Otherwise, -// falls back to single-packet snet.Conn.ReadFrom. func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { sc := c.pconnSCION if sc == nil { @@ -957,55 +535,34 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) continue } - // Fast path: batch read from underlay via recvmmsg. - if sc.underlayXPC != nil { - sc.underlayConn.SetReadDeadline(time.Now().Add(scionReadDeadline)) - - n, err := c.receiveSCIONBatch(sc, buffs, sizes, eps) - if n > 0 { - return n, nil - } - if err != nil { - select { - case <-c.donec: - return 0, net.ErrClosed - default: - } - if isTimeoutError(err) { - if c.shouldReconnectSCION() { - c.reconnectSCION() - } - continue - } - if errors.Is(err, net.ErrClosed) { - continue - } - c.logf("magicsock: SCION read error: %v", err) - continue - } - // n == 0 and no error means all packets were disco/filtered. - continue - } - - // Slow path: single-packet snet.Conn.ReadFrom. + // Set a read deadline so we wake up periodically even if the socket + // is silently dead (SCION router lost our port registration). sc.conn.SetReadDeadline(time.Now().Add(scionReadDeadline)) n, srcAddr, err := sc.readFrom(buffs[0]) if err != nil { + // Graceful shutdown: Conn is closing. select { case <-c.donec: return 0, net.ErrClosed default: } + + // Timeout: check if we need to reconnect. if isTimeoutError(err) { if c.shouldReconnectSCION() { c.reconnectSCION() } continue } + + // Socket closed (by reconnectSCION or externally): re-read + // pconnSCION on next iteration. if errors.Is(err, net.ErrClosed) { continue } + + // Other errors: log and continue. Never propagate to WireGuard. c.logf("magicsock: SCION read error: %v", err) continue } @@ -1013,19 +570,28 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) continue } + // Got a packet — record receive time. c.lastSCIONRecv.StoreAtomic(mono.Now()) b := buffs[0][:n] + srcHostAddr := srcAddr.Host.AddrPort() + // Check for disco packets (same as receiveIP does). pt, _ := packetLooksLike(b) if pt == packetLooksLikeDisco { - // Slow path disco: snet.Conn.ReadFrom returns a pre-reversed - // path suitable for replies, so use srcAddr directly. + // For disco messages, include the scionKey so pong replies + // are routed back over SCION. Use a single critical section + // for the lookup+register to avoid a TOCTOU race where a + // concurrent discoverSCIONPaths could register between our + // check and our register, creating orphaned entries. srcEpAddr := epAddr{ap: srcHostAddr} c.mu.Lock() sk := c.scionPathsByAddr[scionAddrKey{ia: srcAddr.IA, addr: srcHostAddr}] if !sk.IsSet() { + // First disco packet from this SCION peer — bootstrap a + // reverse path entry so the pong can go back over SCION. + // ReadFrom returns a pre-reversed path suitable for replies. sk = c.registerSCIONPath(&scionPathInfo{ peerIA: srcAddr.IA, hostAddr: srcHostAddr, @@ -1043,6 +609,8 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) continue } + // WireGuard packet — look up the endpoint by host addr only + // (peerMap is keyed by netip.AddrPort, not scionKey). srcEpAddr := epAddr{ap: srcHostAddr} c.mu.Lock() ep, ok := c.peerMap.endpointForEpAddr(srcEpAddr) @@ -1066,108 +634,6 @@ func (c *Conn) receiveSCION(buffs [][]byte, sizes []int, eps []wgconn.Endpoint) } } -// receiveSCIONBatch reads a batch of raw SCION packets from the underlay -// socket via recvmmsg, parses SCION+UDP headers with slayers, and copies -// payloads into WireGuard's buffs. Disco packets are handled inline and -// not reported to the caller. -func (c *Conn) receiveSCIONBatch(sc *scionConn, buffs [][]byte, sizes []int, eps []wgconn.Endpoint) (int, error) { - batch := scionRecvBatchPool.Get().(*scionRecvBatch) - defer putScionRecvBatch(batch) - - n := len(buffs) - if n > scionMaxBatchSize { - n = scionMaxBatchSize - } - - numMsgs, err := sc.underlayXPC.ReadBatch(batch.msgs[:n], 0) - if err != nil { - return 0, err - } - - reportToCaller := false - count := 0 - for i := 0; i < numMsgs; i++ { - msg := &batch.msgs[i] - if msg.N == 0 { - sizes[count] = 0 - continue - } - - srcIA, srcHostAddr, payload, rawPath, ok := parseSCIONPacket( - msg.Buffers[0][:msg.N], &batch.scn) - if !ok || len(payload) == 0 { - continue - } - - // Copy payload into WireGuard's buffer. - pn := copy(buffs[count], payload) - - c.lastSCIONRecv.StoreAtomic(mono.Now()) - - pt, _ := packetLooksLike(buffs[count][:pn]) - if pt == packetLooksLikeDisco { - c.handleSCIONDisco(buffs[count][:pn], srcIA, srcHostAddr, rawPath) - continue - } - - if !c.havePrivateKey.Load() { - continue - } - - srcEpAddr := epAddr{ap: srcHostAddr} - c.mu.Lock() - ep, ok := c.peerMap.endpointForEpAddr(srcEpAddr) - c.mu.Unlock() - if !ok { - sizes[count] = pn - eps[count] = &lazyEndpoint{c: c, src: srcEpAddr} - count++ - reportToCaller = true - continue - } - - now := mono.Now() - ep.lastRecvUDPAny.StoreAtomic(now) - ep.noteRecvActivity(srcEpAddr, now) - if c.metrics != nil { - c.metrics.inboundPacketsSCIONTotal.Add(1) - c.metrics.inboundBytesSCIONTotal.Add(int64(pn)) - } - sizes[count] = pn - eps[count] = ep - count++ - reportToCaller = true - } - - if reportToCaller { - return count, nil - } - return 0, nil -} - -// handleSCIONDisco handles a disco packet received on the batch path. -// It looks up or registers a SCION path entry and dispatches to handleDiscoMessage. -// For first-contact, the raw path bytes are reversed to build a reply path. -func (c *Conn) handleSCIONDisco(b []byte, srcIA addr.IA, srcHostAddr netip.AddrPort, rawPath []byte) { - srcEpAddr := epAddr{ap: srcHostAddr} - c.mu.Lock() - sk := c.scionPathsByAddr[scionAddrKey{ia: srcIA, addr: srcHostAddr}] - if !sk.IsSet() { - // First disco packet from this SCION peer — build a reply path - // by reversing the raw SCION path from the incoming packet. - replyAddr := buildSCIONReplyAddr(srcIA, srcHostAddr, rawPath) - sk = c.registerSCIONPath(&scionPathInfo{ - peerIA: srcIA, - hostAddr: srcHostAddr, - replyPath: replyAddr, - }) - c.setActiveSCIONPath(srcIA, srcHostAddr, sk) - } - c.mu.Unlock() - srcEpAddr.scionKey = sk - c.handleDiscoMessage(b, srcEpAddr, false, key.NodePublic{}, discoRXPathSCION) -} - // isTimeoutError reports whether err is a network timeout (from SetReadDeadline). func isTimeoutError(err error) bool { var netErr net.Error @@ -1346,9 +812,6 @@ func (c *Conn) discoverSCIONPaths(ctx context.Context, peerIA addr.IA, hostAddr mtu: mtu, } pi.buildCachedDst() - if sc := c.pconnSCION; sc != nil { - pi.fastPath = buildSCIONFastPath(sc, pi) - } keys = append(keys, c.registerSCIONPath(pi)) } // Set the first (lowest-latency) path as active for the reverse index. @@ -1539,7 +1002,6 @@ func (c *Conn) refreshSCIONPathsOnce() error { pi.mtu = md.MTU } pi.buildCachedDst() - pi.fastPath = buildSCIONFastPath(sc, pi) pi.mu.Unlock() } } diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index 1533f952bdcb9..939f89cda688f 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -5,7 +5,6 @@ package magicsock import ( "context" - "encoding/binary" "fmt" "net" "net/netip" @@ -19,7 +18,6 @@ import ( "github.com/scionproto/scion/pkg/daemon/mock_daemon" "github.com/scionproto/scion/pkg/snet" "github.com/scionproto/scion/pkg/snet/mock_snet" - snetpath "github.com/scionproto/scion/pkg/snet/path" "tailscale.com/net/packet" "tailscale.com/net/tstun" "tailscale.com/tailcfg" @@ -1117,371 +1115,3 @@ func TestSendSCIONExpiredPath(t *testing.T) { t.Errorf("error should mention 'expired', got: %v", err) } } - -// TestSCIONPseudoHeaderPartial verifies the partial checksum computation -// matches the reference SCION implementation for known inputs. -func TestSCIONPseudoHeaderPartial(t *testing.T) { - srcIA := addr.MustParseIA("1-ff00:0:110") - dstIA := addr.MustParseIA("1-ff00:0:111") - srcIP := netip.MustParseAddr("127.0.0.1") - dstIP := netip.MustParseAddr("127.0.0.1") - - partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) - - // Verify by computing the same checksum manually: - // srcIA = 0x0001ff0000000110, dstIA = 0x0001ff0000000111 - // srcAddr = 127.0.0.1 = [0x7f, 0x00, 0x00, 0x01] - // dstAddr = 127.0.0.1 = [0x7f, 0x00, 0x00, 0x01] - // protocol = 17 - - var expected uint32 - // srcIA bytes: 00 01 ff 00 00 00 01 10 - expected += 0x0001 + 0xff00 + 0x0000 + 0x0110 - // dstIA bytes: 00 01 ff 00 00 00 01 11 - expected += 0x0001 + 0xff00 + 0x0000 + 0x0111 - // srcAddr: 7f 00 00 01 - expected += 0x7f00 + 0x0001 - // dstAddr: 7f 00 00 01 - expected += 0x7f00 + 0x0001 - // protocol - expected += 17 - - if partial != expected { - t.Errorf("scionPseudoHeaderPartial = %d, want %d", partial, expected) - } -} - -// TestSCIONPseudoHeaderPartialIPv6 verifies checksum with IPv6 addresses. -func TestSCIONPseudoHeaderPartialIPv6(t *testing.T) { - srcIA := addr.MustParseIA("1-ff00:0:110") - dstIA := addr.MustParseIA("1-ff00:0:111") - srcIP := netip.MustParseAddr("::1") - dstIP := netip.MustParseAddr("fd00::1") - - partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) - if partial == 0 { - t.Fatal("checksum should not be zero") - } - - // Verify IPv6 addrs are 16 bytes each. - // ::1 = 00...01, fd00::1 = fd 00 00...01 - var expected uint32 - // IAs - expected += 0x0001 + 0xff00 + 0x0000 + 0x0110 - expected += 0x0001 + 0xff00 + 0x0000 + 0x0111 - // srcIP ::1 = all zeros except last byte - expected += 0x0001 - // dstIP fd00::1 - expected += 0xfd00 + 0x0001 - expected += 17 - - if partial != expected { - t.Errorf("scionPseudoHeaderPartial(IPv6) = %d, want %d", partial, expected) - } -} - -// TestSCIONFinishChecksum verifies the full checksum computation matches -// the reference SCION implementation by comparing against a packet -// serialized with snet.Packet.Serialize(). -func TestSCIONFinishChecksum(t *testing.T) { - srcIA := addr.MustParseIA("1-ff00:0:110") - dstIA := addr.MustParseIA("1-ff00:0:111") - srcIP := netip.MustParseAddr("127.0.0.1") - dstIP := netip.MustParseAddr("127.0.0.1") - srcPort := uint16(32766) - dstPort := uint16(32766) - payload := []byte("Hello, SCION fast path!") - - // Build the packet using snet's reference serializer. - pkt := &snet.Packet{ - PacketInfo: snet.PacketInfo{ - Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, - Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, - Path: snetpath.Empty{}, - Payload: snet.UDPPayload{ - SrcPort: srcPort, - DstPort: dstPort, - Payload: payload, - }, - }, - } - if err := pkt.Serialize(); err != nil { - t.Fatalf("snet.Packet.Serialize: %v", err) - } - - // Extract the reference checksum from the serialized packet. - // The UDP header is the last 8 bytes before the payload. - udpOffset := len(pkt.Bytes) - 8 - len(payload) - refChecksum := binary.BigEndian.Uint16(pkt.Bytes[udpOffset+6:]) - - // Now compute it using our fast-path functions. - partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) - - // Build the upper layer: UDP header (8 bytes) + payload - upperLayer := make([]byte, 8+len(payload)) - binary.BigEndian.PutUint16(upperLayer[0:], srcPort) - binary.BigEndian.PutUint16(upperLayer[2:], dstPort) - binary.BigEndian.PutUint16(upperLayer[4:], uint16(8+len(payload))) - // checksum field = 0 for computation - copy(upperLayer[8:], payload) - - fastChecksum := scionFinishChecksum(partial, upperLayer) - - if fastChecksum != refChecksum { - t.Errorf("fast-path checksum = 0x%04x, reference = 0x%04x", fastChecksum, refChecksum) - } -} - -// TestSCIONFinishChecksumEmptyPayload verifies checksum with empty payload. -func TestSCIONFinishChecksumEmptyPayload(t *testing.T) { - srcIA := addr.MustParseIA("1-ff00:0:110") - dstIA := addr.MustParseIA("1-ff00:0:111") - srcIP := netip.MustParseAddr("127.0.0.1") - dstIP := netip.MustParseAddr("127.0.0.1") - - pkt := &snet.Packet{ - PacketInfo: snet.PacketInfo{ - Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, - Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, - Path: snetpath.Empty{}, - Payload: snet.UDPPayload{ - SrcPort: 1000, - DstPort: 2000, - Payload: nil, - }, - }, - } - if err := pkt.Serialize(); err != nil { - t.Fatalf("snet.Packet.Serialize: %v", err) - } - - // UDP header is the last 8 bytes (no payload). - udpOffset := len(pkt.Bytes) - 8 - refChecksum := binary.BigEndian.Uint16(pkt.Bytes[udpOffset+6:]) - - partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) - upperLayer := make([]byte, 8) - binary.BigEndian.PutUint16(upperLayer[0:], 1000) - binary.BigEndian.PutUint16(upperLayer[2:], 2000) - binary.BigEndian.PutUint16(upperLayer[4:], 8) - - fastChecksum := scionFinishChecksum(partial, upperLayer) - - if fastChecksum != refChecksum { - t.Errorf("fast-path checksum (empty) = 0x%04x, reference = 0x%04x", fastChecksum, refChecksum) - } -} - -// TestSCIONFinishChecksumOddPayload verifies correct handling of odd-length payloads. -func TestSCIONFinishChecksumOddPayload(t *testing.T) { - srcIA := addr.MustParseIA("1-ff00:0:110") - dstIA := addr.MustParseIA("1-ff00:0:111") - srcIP := netip.MustParseAddr("127.0.0.1") - dstIP := netip.MustParseAddr("10.0.0.1") - payload := []byte("ABC") // 3 bytes, odd - - pkt := &snet.Packet{ - PacketInfo: snet.PacketInfo{ - Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, - Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, - Path: snetpath.Empty{}, - Payload: snet.UDPPayload{ - SrcPort: 5000, - DstPort: 6000, - Payload: payload, - }, - }, - } - if err := pkt.Serialize(); err != nil { - t.Fatalf("snet.Packet.Serialize: %v", err) - } - - udpOffset := len(pkt.Bytes) - 8 - len(payload) - refChecksum := binary.BigEndian.Uint16(pkt.Bytes[udpOffset+6:]) - - partial := scionPseudoHeaderPartial(srcIA, dstIA, srcIP, dstIP) - upperLayer := make([]byte, 8+len(payload)) - binary.BigEndian.PutUint16(upperLayer[0:], 5000) - binary.BigEndian.PutUint16(upperLayer[2:], 6000) - binary.BigEndian.PutUint16(upperLayer[4:], uint16(8+len(payload))) - copy(upperLayer[8:], payload) - - fastChecksum := scionFinishChecksum(partial, upperLayer) - - if fastChecksum != refChecksum { - t.Errorf("fast-path checksum (odd) = 0x%04x, reference = 0x%04x", fastChecksum, refChecksum) - } -} - -// TestBuildSCIONFastPath verifies that buildSCIONFastPath produces a template -// that matches the reference serializer output for the same parameters. -func TestBuildSCIONFastPath(t *testing.T) { - srcIA := addr.MustParseIA("1-ff00:0:110") - dstIA := addr.MustParseIA("1-ff00:0:111") - srcIP := netip.MustParseAddr("127.0.0.1") - dstIP := netip.MustParseAddr("127.0.0.1") - srcPort := uint16(32766) - dstPort := uint16(32766) - - sc := &scionConn{ - underlayConn: &net.UDPConn{}, // non-nil to enable fast path - localIA: srcIA, - localHostIP: srcIP, - localPort: srcPort, - } - - pi := &scionPathInfo{ - peerIA: dstIA, - hostAddr: netip.MustParseAddrPort("127.0.0.1:32766"), - cachedDst: &snet.UDPAddr{ - IA: dstIA, - Host: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(dstPort)}, - Path: snetpath.Empty{}, - NextHop: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 30041}, - }, - } - - fp := buildSCIONFastPath(sc, pi) - if fp == nil { - t.Fatal("buildSCIONFastPath returned nil") - } - - // The template should match a reference packet with empty payload. - refPkt := &snet.Packet{ - PacketInfo: snet.PacketInfo{ - Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, - Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, - Path: snetpath.Empty{}, - Payload: snet.UDPPayload{ - SrcPort: srcPort, - DstPort: dstPort, - Payload: nil, - }, - }, - } - if err := refPkt.Serialize(); err != nil { - t.Fatalf("reference Serialize: %v", err) - } - - if len(fp.hdr) != len(refPkt.Bytes) { - t.Fatalf("fast-path header len = %d, reference = %d", len(fp.hdr), len(refPkt.Bytes)) - } - - // Compare header bytes (everything except checksum which may differ - // due to computation order, but should be the same for empty payload). - for i := range fp.hdr { - if fp.hdr[i] != refPkt.Bytes[i] { - t.Errorf("byte %d: fast-path=0x%02x, reference=0x%02x", i, fp.hdr[i], refPkt.Bytes[i]) - } - } - - if fp.udpOffset != len(fp.hdr)-8 { - t.Errorf("udpOffset = %d, expected %d", fp.udpOffset, len(fp.hdr)-8) - } - - if fp.nextHop == nil { - t.Error("nextHop should not be nil") - } -} - -// TestSCIONFastPathPacketMatchesReference verifies that a packet built with -// the fast-path template+patching produces identical bytes to one built with -// snet.Packet.Serialize(). -func TestSCIONFastPathPacketMatchesReference(t *testing.T) { - srcIA := addr.MustParseIA("1-ff00:0:110") - dstIA := addr.MustParseIA("1-ff00:0:111") - srcIP := netip.MustParseAddr("127.0.0.1") - dstIP := netip.MustParseAddr("127.0.0.1") - srcPort := uint16(32766) - dstPort := uint16(32766) - payload := []byte("WireGuard test payload data for SCION fast path verification") - - sc := &scionConn{ - underlayConn: &net.UDPConn{}, - localIA: srcIA, - localHostIP: srcIP, - localPort: srcPort, - } - - pi := &scionPathInfo{ - peerIA: dstIA, - hostAddr: netip.MustParseAddrPort("127.0.0.1:32766"), - cachedDst: &snet.UDPAddr{ - IA: dstIA, - Host: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(dstPort)}, - Path: snetpath.Empty{}, - NextHop: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 30041}, - }, - } - - fp := buildSCIONFastPath(sc, pi) - if fp == nil { - t.Fatal("buildSCIONFastPath returned nil") - } - - // Build packet using fast-path template + patching. - hdrLen := len(fp.hdr) - pktLen := hdrLen + len(payload) - buf := make([]byte, pktLen) - copy(buf, fp.hdr) - copy(buf[hdrLen:], payload) - - udpTotalLen := uint16(8 + len(payload)) - binary.BigEndian.PutUint16(buf[6:], udpTotalLen) - binary.BigEndian.PutUint16(buf[fp.udpOffset+4:], udpTotalLen) - buf[fp.udpOffset+6] = 0 - buf[fp.udpOffset+7] = 0 - upperLayer := buf[fp.udpOffset:pktLen] - csum := scionFinishChecksum(fp.pseudoCsum, upperLayer) - binary.BigEndian.PutUint16(buf[fp.udpOffset+6:], csum) - - // Build reference packet using snet. - refPkt := &snet.Packet{ - PacketInfo: snet.PacketInfo{ - Destination: snet.SCIONAddress{IA: dstIA, Host: addr.HostIP(dstIP)}, - Source: snet.SCIONAddress{IA: srcIA, Host: addr.HostIP(srcIP)}, - Path: snetpath.Empty{}, - Payload: snet.UDPPayload{ - SrcPort: srcPort, - DstPort: dstPort, - Payload: payload, - }, - }, - } - if err := refPkt.Serialize(); err != nil { - t.Fatalf("reference Serialize: %v", err) - } - - if len(buf) != len(refPkt.Bytes) { - t.Fatalf("fast-path pkt len = %d, reference = %d", len(buf), len(refPkt.Bytes)) - } - - for i := range buf { - if buf[i] != refPkt.Bytes[i] { - t.Errorf("byte %d: fast-path=0x%02x, reference=0x%02x", i, buf[i], refPkt.Bytes[i]) - } - } -} - -// TestSCIONSendBatchPool verifies the pool returns usable batches. -func TestSCIONSendBatchPool(t *testing.T) { - batch := scionSendBatchPool.Get().(*scionSendBatch) - defer scionSendBatchPool.Put(batch) - - if len(batch.bufs) != scionMaxBatchSize { - t.Errorf("batch.bufs len = %d, want %d", len(batch.bufs), scionMaxBatchSize) - } - if len(batch.msgs) != scionMaxBatchSize { - t.Errorf("batch.msgs len = %d, want %d", len(batch.msgs), scionMaxBatchSize) - } - for i, buf := range batch.bufs { - if cap(buf) < 1500 { - t.Errorf("batch.bufs[%d] cap = %d, want >= 1500", i, cap(buf)) - } - } - for i, msg := range batch.msgs { - if len(msg.Buffers) != 1 { - t.Errorf("batch.msgs[%d].Buffers len = %d, want 1", i, len(msg.Buffers)) - } - } -} From 670eeb1c05c959a1c24d2ddf3f6cbbbcd15d409c Mon Sep 17 00:00:00 2001 From: Tony John Date: Mon, 16 Mar 2026 22:01:54 +0100 Subject: [PATCH 70/78] branding: rename packages to tailscale-scion, add NOTICE, fix license - Add NOTICE with Tailscale and SCION attribution - Rename deb/rpm/tgz package to tailscale-scion - Add Conflicts/Replaces for official tailscale package - Fix license from MIT to BSD-3-Clause - Update maintainer, description, homepage for netsys-lab - Add netsys-lab copyright to SCION-specific source files --- NOTICE | 11 +++++++ ipn/localapi/localapi_scion.go | 1 + ipn/localapi/localapi_scion_omit.go | 1 + release/dist/unixpkgs/pkgs.go | 45 +++++++++++------------------ 4 files changed, 30 insertions(+), 28 deletions(-) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000000..7250122885807 --- /dev/null +++ b/NOTICE @@ -0,0 +1,11 @@ +Tailscale (SCION) - Unofficial Tailscale client with SCION path-aware networking + +Based on Tailscale (https://github.com/tailscale/tailscale) +Copyright (c) 2020 Tailscale Inc & contributors. Licensed under BSD-3-Clause. + +SCION networking provided by scionproto/scion +(https://github.com/scionproto/scion), licensed under Apache-2.0. + +Tailscale is a registered trademark of Tailscale Inc. +This project is not affiliated with or endorsed by Tailscale Inc. +WireGuard is a registered trademark of Jason A. Donenfeld. \ No newline at end of file diff --git a/ipn/localapi/localapi_scion.go b/ipn/localapi/localapi_scion.go index 7418e1f20997a..cbb24ca6e697f 100644 --- a/ipn/localapi/localapi_scion.go +++ b/ipn/localapi/localapi_scion.go @@ -1,4 +1,5 @@ // Copyright (c) Tailscale Inc & contributors +// Copyright (c) 2026 netsys-lab // SPDX-License-Identifier: BSD-3-Clause //go:build !ts_omit_scion diff --git a/ipn/localapi/localapi_scion_omit.go b/ipn/localapi/localapi_scion_omit.go index 4381bbb99c0a1..e2d4ac1ee0b6c 100644 --- a/ipn/localapi/localapi_scion_omit.go +++ b/ipn/localapi/localapi_scion_omit.go @@ -1,4 +1,5 @@ // Copyright (c) Tailscale Inc & contributors +// Copyright (c) 2026 netsys-lab // SPDX-License-Identifier: BSD-3-Clause //go:build ts_omit_scion diff --git a/release/dist/unixpkgs/pkgs.go b/release/dist/unixpkgs/pkgs.go index d251ff621f98a..fe9df18cf2bae 100644 --- a/release/dist/unixpkgs/pkgs.go +++ b/release/dist/unixpkgs/pkgs.go @@ -46,7 +46,7 @@ func (t *tgzTarget) Build(b *dist.Build) ([]string, error) { if t.goEnv["GOOS"] == "linux" { // Linux used to be the only tgz architecture, so we didn't put the OS // name in the filename. - filename = fmt.Sprintf("tailscale_%s_%s.tgz", b.Version.Short, t.arch()) + filename = fmt.Sprintf("tailscale-scion_%s_%s.tgz", b.Version.Short, t.arch()) } else { filename = fmt.Sprintf("tailscale_%s_%s_%s.tgz", b.Version.Short, t.os(), t.arch()) } @@ -233,14 +233,14 @@ func (t *debTarget) Build(b *dist.Build) ([]string, error) { return nil, err } info := nfpm.WithDefaults(&nfpm.Info{ - Name: "tailscale", + Name: "tailscale-scion", Arch: arch, Platform: "linux", Version: b.Version.Short, - Maintainer: "Tailscale Inc ", - Description: "The easiest, most secure, cross platform way to use WireGuard + oauth2 + 2FA/SSO", - Homepage: "https://www.tailscale.com", - License: "MIT", + Maintainer: "netsys-lab", + Description: "Tailscale with SCION path-aware networking support", + Homepage: "https://github.com/netsys-lab/tailscale-scion", + License: "BSD-3-Clause", Section: "net", Priority: "extra", Overridables: nfpm.Overridables{ @@ -251,20 +251,9 @@ func (t *debTarget) Build(b *dist.Build) ([]string, error) { PostRemove: filepath.Join(repoDir, "release/deb/debian.postrm.sh"), }, Depends: []string{ - // iptables is almost always required but not strictly needed. - // Even if you can technically run Tailscale without it (by - // manually configuring nftables or userspace mode), we still - // mark this as "Depends" because our previous experiment in - // https://github.com/tailscale/tailscale/issues/9236 of making - // it only Recommends caused too many problems. Until our - // nftables table is more mature, we'd rather err on the side of - // wasting a little disk by including iptables for people who - // might not need it rather than handle reports of it being - // missing. "iptables", }, Recommends: []string{ - "tailscale-archive-keyring (>= 1.35.181)", // The "ip" command isn't needed since 2021-11-01 in // 408b0923a61972ed but kept as an option as of // 2021-11-18 in d24ed3f68e35e802d531371. See @@ -274,8 +263,8 @@ func (t *debTarget) Build(b *dist.Build) ([]string, error) { // we can live without it, so it's not Depends. "iproute2", }, - Replaces: []string{"tailscale-relay"}, - Conflicts: []string{"tailscale-relay"}, + Replaces: []string{"tailscale", "tailscale-relay"}, + Conflicts: []string{"tailscale", "tailscale-relay"}, }, }) pkg, err := nfpm.Get("deb") @@ -283,7 +272,7 @@ func (t *debTarget) Build(b *dist.Build) ([]string, error) { return nil, err } - filename := fmt.Sprintf("tailscale_%s_%s.deb", b.Version.Short, arch) + filename := fmt.Sprintf("tailscale-scion_%s_%s.deb", b.Version.Short, arch) log.Printf("Building %s", filename) f, err := os.Create(filepath.Join(b.Out, filename)) if err != nil { @@ -376,14 +365,14 @@ func (t *rpmTarget) Build(b *dist.Build) ([]string, error) { return nil, err } info := nfpm.WithDefaults(&nfpm.Info{ - Name: "tailscale", + Name: "tailscale-scion", Arch: arch, Platform: "linux", Version: b.Version.Short, - Maintainer: "Tailscale Inc ", - Description: "The easiest, most secure, cross platform way to use WireGuard + oauth2 + 2FA/SSO", - Homepage: "https://www.tailscale.com", - License: "MIT", + Maintainer: "netsys-lab", + Description: "Tailscale with SCION path-aware networking support", + Homepage: "https://github.com/netsys-lab/tailscale-scion", + License: "BSD-3-Clause", Overridables: nfpm.Overridables{ Contents: contents, Scripts: nfpm.Scripts{ @@ -392,8 +381,8 @@ func (t *rpmTarget) Build(b *dist.Build) ([]string, error) { PostRemove: filepath.Join(repoDir, "release/rpm/rpm.postrm.sh"), }, Depends: []string{"iptables", "iproute"}, - Replaces: []string{"tailscale-relay"}, - Conflicts: []string{"tailscale-relay"}, + Replaces: []string{"tailscale", "tailscale-relay"}, + Conflicts: []string{"tailscale", "tailscale-relay"}, RPM: nfpm.RPM{ Group: "Network", Signature: nfpm.RPMSignature{ @@ -409,7 +398,7 @@ func (t *rpmTarget) Build(b *dist.Build) ([]string, error) { return nil, err } - filename := fmt.Sprintf("tailscale_%s_%s.rpm", b.Version.Short, arch) + filename := fmt.Sprintf("tailscale-scion_%s_%s.rpm", b.Version.Short, arch) log.Printf("Building %s", filename) f, err := os.Create(filepath.Join(b.Out, filename)) From ba11956fc8458b0a81bf38f11ba2b840ade8fda5 Mon Sep 17 00:00:00 2001 From: Tony John Date: Mon, 16 Mar 2026 22:19:17 +0100 Subject: [PATCH 71/78] ci: add cross-platform release workflow for GitHub Releases Triggered on tag push (v*-scion.*). Builds Linux deb/rpm/tgz (amd64+arm64), macOS tgz (amd64+arm64), Windows zip (amd64). Publishes all artifacts to GitHub Releases. --- .github/workflows/release.yml | 115 ++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000000..8f1b453c35298 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,115 @@ +name: Release +on: + push: + tags: ['v*-scion.*'] + +jobs: + build-linux: + runs-on: ubuntu-latest + strategy: + matrix: + goarch: [amd64, arm64] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Verify version + run: | + eval $(go run ./cmd/mkversion) + echo "Version: $VERSION_LONG" + - name: Build packages + env: + CGO_ENABLED: 0 + GOARCH: ${{ matrix.goarch }} + run: | + go run ./cmd/dist build \ + "linux/${{ matrix.goarch }}/tgz" \ + "linux/${{ matrix.goarch }}/deb" \ + "linux/${{ matrix.goarch }}/rpm" + - uses: actions/upload-artifact@v4 + with: + name: linux-${{ matrix.goarch }} + path: dist/ + + build-macos: + runs-on: macos-latest + strategy: + matrix: + goarch: [amd64, arm64] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Build binaries + env: + GOOS: darwin + GOARCH: ${{ matrix.goarch }} + CGO_ENABLED: 0 + run: | + eval $(go run ./cmd/mkversion) + mkdir -p dist + ./build_dist.sh -o dist/tailscale ./cmd/tailscale + ./build_dist.sh -o dist/tailscaled ./cmd/tailscaled + cp LICENSE PATENTS NOTICE dist/ 2>/dev/null || true + tar czf "dist/tailscale-scion_${VERSION_SHORT}_macos_${{ matrix.goarch }}.tgz" \ + -C dist tailscale tailscaled LICENSE PATENTS NOTICE + - uses: actions/upload-artifact@v4 + with: + name: macos-${{ matrix.goarch }} + path: dist/*.tgz + + build-windows: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Build binaries + env: + GOOS: windows + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + eval $(go run ./cmd/mkversion) + mkdir -p dist + ./build_dist.sh -o dist/tailscale.exe ./cmd/tailscale + ./build_dist.sh -o dist/tailscaled.exe ./cmd/tailscaled + cp LICENSE PATENTS NOTICE dist/ 2>/dev/null || true + cd dist && zip "tailscale-scion_${VERSION_SHORT}_windows_amd64.zip" \ + tailscale.exe tailscaled.exe LICENSE PATENTS NOTICE + - uses: actions/upload-artifact@v4 + with: + name: windows-amd64 + path: dist/*.zip + + release: + needs: [build-linux, build-macos, build-windows] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: artifacts + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: artifacts/**/* + generate_release_notes: true + body: | + ## Tailscale (SCION) ${{ github.ref_name }} + + Unofficial Tailscale client with SCION path-aware networking. + Based on Tailscale. Not affiliated with Tailscale Inc. + + ### Install + + **Linux (deb):** `sudo dpkg -i tailscale-scion_*.deb && sudo systemctl enable --now tailscaled` + **Linux (rpm):** `sudo rpm -i tailscale-scion_*.rpm && sudo systemctl enable --now tailscaled` + **Linux (tgz):** Extract and copy `tailscale`/`tailscaled` to PATH + **macOS:** Extract tgz, run `sudo ./tailscaled &` then `./tailscale up` + **Windows:** Extract zip, run `tailscaled.exe` then `tailscale.exe up` \ No newline at end of file From 8d98e55a6f91da5c7abf1e27e74443d3b3800b6e Mon Sep 17 00:00:00 2001 From: Tony John Date: Mon, 16 Mar 2026 23:29:02 +0100 Subject: [PATCH 72/78] ci: fix arm64 build - don't set GOARCH on dist tool The dist tool runs on the host (amd64) and cross-compiles internally. Setting GOARCH in the env caused go run to build the dist tool itself for arm64, which can't execute on amd64. --- .github/workflows/release.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f1b453c35298..3abf279380885 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,9 +19,6 @@ jobs: eval $(go run ./cmd/mkversion) echo "Version: $VERSION_LONG" - name: Build packages - env: - CGO_ENABLED: 0 - GOARCH: ${{ matrix.goarch }} run: | go run ./cmd/dist build \ "linux/${{ matrix.goarch }}/tgz" \ From d552ad691e86e202e8681cd5c363f746119bda93 Mon Sep 17 00:00:00 2001 From: Tony John Date: Mon, 16 Mar 2026 23:35:57 +0100 Subject: [PATCH 73/78] ci: use full checkout and TS_USE_TOOLCHAIN=0 for dist build Shallow clone (fetch-depth: 1) caused missing files during package glob. Full checkout ensures all files are available. --- .github/workflows/release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3abf279380885..1ebf62cde77c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,8 @@ jobs: goarch: [amd64, arm64] steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-go@v5 with: go-version-file: go.mod @@ -19,6 +21,8 @@ jobs: eval $(go run ./cmd/mkversion) echo "Version: $VERSION_LONG" - name: Build packages + env: + TS_USE_TOOLCHAIN: 0 run: | go run ./cmd/dist build \ "linux/${{ matrix.goarch }}/tgz" \ From 5c29e3011d14fd067ad1e12e4714773d74db4544 Mon Sep 17 00:00:00 2001 From: Tony John Date: Tue, 17 Mar 2026 00:01:28 +0100 Subject: [PATCH 74/78] dist: disable gocross to fix deb/rpm build in CI gocross was made opt-in in 2025-06-16 but dist.go still forced TS_USE_GOCROSS=1, causing 'no matching files' errors when gocross modified GOOS/GOARCH during go list. Set TS_USE_GOCROSS=0 so the Tailscale Go toolchain is used directly. Also restore deb/rpm targets in the release workflow. --- .github/workflows/release.yml | 4 ---- release/dist/dist.go | 6 ++++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1ebf62cde77c8..3abf279380885 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,6 @@ jobs: goarch: [amd64, arm64] steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - uses: actions/setup-go@v5 with: go-version-file: go.mod @@ -21,8 +19,6 @@ jobs: eval $(go run ./cmd/mkversion) echo "Version: $VERSION_LONG" - name: Build packages - env: - TS_USE_TOOLCHAIN: 0 run: | go run ./cmd/dist build \ "linux/${{ matrix.goarch }}/tgz" \ diff --git a/release/dist/dist.go b/release/dist/dist.go index 094d0a0e04c46..0ab22afd1484a 100644 --- a/release/dist/dist.go +++ b/release/dist/dist.go @@ -323,8 +323,10 @@ func (b *Build) Command(dir, cmd string, args ...string) *Command { ret.Cmd.Stdout = &ret.Output ret.Cmd.Stderr = &ret.Output } - // dist always wants to use gocross if any Go is involved. - ret.Cmd.Env = append(os.Environ(), "TS_USE_GOCROSS=1") + // Don't force gocross — it's opt-in since 2025-06-16 and causes + // "no matching files" errors in CI when building deb/rpm packages. + // The Tailscale Go toolchain (tool/go) works fine without it. + ret.Cmd.Env = append(os.Environ(), "TS_USE_GOCROSS=0") ret.Cmd.Dir = dir return ret } From 18d131d71d888468599f3f1aa9cd906a40764075 Mon Sep 17 00:00:00 2001 From: Tony John Date: Tue, 17 Mar 2026 01:07:54 +0100 Subject: [PATCH 75/78] dist: fix deb/rpm build by suppressing set -x in tool/go In CI (CI=true), gocross-wrapper.sh enables set -x which writes bash traces to stderr. GoPkg() uses CombinedOutput() merging stdout+stderr, so the real path gets mixed with trace output. Add NOBASHDEBUG=true (existing upstream mechanism) to suppress the traces. Restore TS_USE_GOCROSS=1 to match upstream. --- release/dist/dist.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/release/dist/dist.go b/release/dist/dist.go index 0ab22afd1484a..46646c876e0ec 100644 --- a/release/dist/dist.go +++ b/release/dist/dist.go @@ -323,10 +323,11 @@ func (b *Build) Command(dir, cmd string, args ...string) *Command { ret.Cmd.Stdout = &ret.Output ret.Cmd.Stderr = &ret.Output } - // Don't force gocross — it's opt-in since 2025-06-16 and causes - // "no matching files" errors in CI when building deb/rpm packages. - // The Tailscale Go toolchain (tool/go) works fine without it. - ret.Cmd.Env = append(os.Environ(), "TS_USE_GOCROSS=0") + // dist always wants to use gocross if any Go is involved. + // Suppress bash debug traces (set -x) from tool/go in CI, because + // GoPkg() uses CombinedOutput() and set -x pollutes stdout+stderr, + // causing "no matching files" errors when building deb/rpm packages. + ret.Cmd.Env = append(os.Environ(), "TS_USE_GOCROSS=1", "NOBASHDEBUG=true") ret.Cmd.Dir = dir return ret } From dc4cd2eb540066d7a45df7126a112ced5628ca1e Mon Sep 17 00:00:00 2001 From: Tony John Date: Tue, 17 Mar 2026 12:22:01 +0100 Subject: [PATCH 76/78] docs: add SCION architecture docs, README, and fix IPv6 address format Add documentation for the SCION integration: - docs/architecture.md: component overview, connection flow, data flow, key design decisions including peerapi4 piggyback mechanism - README.md: replace upstream README with SCION-specific user guide, env var reference, build instructions Fix SCION service address format to use bracket notation for IPv6 compatibility. The peerapi4 piggyback format changes from "scion=ISD-AS,hostIP:port" to "scion=ISD-AS,[hostIP]:port" so that IPv6 addresses (which contain colons) don't break the port parser. Backward-compatible parsing for unbracketed format is preserved. --- README.md | 132 ++++++++++++--------- docs/architecture.md | 68 +++++++++++ wgengine/magicsock/magicsock_scion.go | 35 ++++-- wgengine/magicsock/magicsock_scion_test.go | 69 ++++++++++- 4 files changed, 228 insertions(+), 76 deletions(-) create mode 100644 docs/architecture.md diff --git a/README.md b/README.md index 70b92d411b9de..51268ad27cf47 100644 --- a/README.md +++ b/README.md @@ -1,87 +1,101 @@ -# Tailscale +# Tailscale (SCION) -https://tailscale.com +Tailscale fork with SCION path-aware transport. -Private WireGuard® networks made easy +## What This Is -## Overview +A fork of [tailscale/tailscale](https://github.com/tailscale/tailscale) that adds [SCION](https://www.scion.org/) as a transport layer alongside WireGuard's existing UDP. Peers on SCION-enabled ASes gets path-aware routing with latency-based path selection. -This repository contains the majority of Tailscale's open source code. -Notably, it includes the `tailscaled` daemon and -the `tailscale` CLI tool. The `tailscaled` daemon runs on Linux, Windows, -[macOS](https://tailscale.com/kb/1065/macos-variants/), and to varying degrees -on FreeBSD and OpenBSD. The Tailscale iOS and Android apps use this repo's -code, but this repo doesn't contain the mobile GUI code. +> **This project is not affiliated with or endorsed by Tailscale Inc.** -Other [Tailscale repos](https://github.com/orgs/tailscale/repositories) of note: +## Status -* the Android app is at https://github.com/tailscale/tailscale-android -* the Synology package is at https://github.com/tailscale/tailscale-synology -* the QNAP package is at https://github.com/tailscale/tailscale-qpkg -* the Chocolatey packaging is at https://github.com/tailscale/tailscale-chocolatey +Experimental. Platforms: Linux, macOS, Windows, FreeBSD, OpenBSD, NetBSD, Android (via [tailscale-android-scion](https://github.com/netsys-lab/tailscale-android-scion)). -For background on which parts of Tailscale are open source and why, -see [https://tailscale.com/opensource/](https://tailscale.com/opensource/). +## Quick Start (Linux) -## Using +```bash +# Build +go install tailscale.com/cmd/tailscale{,d} -We serve packages for a variety of distros and platforms at -[https://pkgs.tailscale.com](https://pkgs.tailscale.com/). +# Run with embedded SCION daemon + bootstrap +TS_SCION_EMBEDDED=1 \ +TS_SCION_BOOTSTRAP_URL=http://your-bootstrap-server:8041 \ + tailscaled -## Other clients +# Verify SCION is connected +curl -s --unix-socket /var/run/tailscale/tailscaled.sock \ + http://local-tailscaled.sock/localapi/v0/scion-status +# {"Connected":true,"LocalIA":"19-ffaa:1:eba"} +``` -The [macOS, iOS, and Windows clients](https://tailscale.com/download) -use the code in this repository but additionally include small GUI -wrappers. The GUI wrappers on non-open source platforms are themselves -not open source. +If you have a local SCION daemon (sciond) running, no environment variables are needed -- Tailscale will connect to it automatically at `127.0.0.1:30255`. -## Building +## Android -We always require the latest Go release, currently Go 1.25. (While we build -releases with our [Go fork](https://github.com/tailscale/go/), its use is not -required.) +See [netsys-lab/tailscale-android-scion](https://github.com/netsys-lab/tailscale-android-scion) for the Android client with SCION settings UI and live path display. -``` -go install tailscale.com/cmd/tailscale{,d} -``` +## Connection Flow -If you're packaging Tailscale for distribution, use `build_dist.sh` -instead, to burn commit IDs and version info into the binaries: +SCION connects using a cascading fallback: -``` -./build_dist.sh tailscale.com/cmd/tailscale -./build_dist.sh tailscale.com/cmd/tailscaled -``` +1. **External daemon** -- connects to sciond at `SCION_DAEMON_ADDRESS`. *Skipped if `TS_SCION_EMBEDDED=1`.* +2. **Embedded daemon** -- loads local topology file (`TS_SCION_TOPOLOGY` or `/etc/scion/topology.json`). *Skipped if `TS_SCION_FORCE_BOOTSTRAP=1`.* +3. **Bootstrap** -- fetches topology from: explicit URL → DNS SRV discovery → hardcoded defaults. Then starts embedded daemon with the fetched topology. -If your distro has conventions that preclude the use of -`build_dist.sh`, please do the equivalent of what it does in your -distro's way, so that bug reports contain useful version information. +See [docs/architecture.md](docs/architecture.md) for details. -## Bugs +## Configuration -Please file any issues about this code or the hosted service on -[the issue tracker](https://github.com/tailscale/tailscale/issues). +### Core -## Contributing +| Variable | Default | Description | +|----------|---------|-------------| +| `SCION_DAEMON_ADDRESS` | `127.0.0.1:30255` | External SCION daemon gRPC address | +| `TS_SCION_EMBEDDED` | `false` | Skip external daemon, use embedded connector only | +| `TS_PREFER_SCION` | `false` | Unconditionally prefer SCION over all other paths | +| `TS_SCION_PREFERENCE` | `15` | betterAddr points bonus for SCION (0 to disable) | +| `TS_SCION_PORT` | (auto) | Local SCION/UDP listen port | +| `TS_SCION_LISTEN_ADDR` | (auto) | Listen address override | -PRs welcome! But please file bugs. Commit messages should [reference -bugs](https://docs.github.com/en/github/writing-on-github/autolinked-references-and-urls). +### Bootstrap & Topology + +| Variable | Default | Description | +|----------|---------|-------------| +| `TS_SCION_TOPOLOGY` | (auto) | Path to `topology.json` (defaults to `/etc/scion/topology.json` on Linux) | +| `TS_SCION_BOOTSTRAP_URL` | (unset) | Single bootstrap server URL | +| `TS_SCION_BOOTSTRAP_URLS` | (unset) | Comma-separated bootstrap server URLs | +| `TS_SCION_FORCE_BOOTSTRAP` | `false` | Skip local topology, go straight to bootstrap | +| `TS_SCION_STATE_DIR` | (auto) | State directory for bootstrap data and PathDB | + +### Advanced + +| Variable | Default | Description | +|----------|---------|-------------| +| `TS_SCION_MAX_PROBE_PATHS` | `5` | Max SCION paths to probe per peer | +| `TS_SCION_DIVERSITY_THRESHOLD` | `50` | Latency penalty threshold (ms) for path diversity | +| `TS_SCION_NO_FAST_PATH` | `false` | Disable pre-serialized fast-path sends | +| `TS_SCION_NO_DISPATCHER_SHIM` | `false` | Disable legacy dispatcher port 30041 shim | + +## Build Tags + +Build without SCION support using the `ts_omit_scion` tag: + +```bash +go install -tags ts_omit_scion tailscale.com/cmd/tailscale{,d} +``` -We require [Developer Certificate of -Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin) -`Signed-off-by` lines in commits. +This compiles out all SCION code, producing a smaller binary with no `scionproto/scion` dependency. -See [commit-messages.md](docs/commit-messages.md) (or skim `git log`) for our commit message style. -## About Us +## Architecture -[Tailscale](https://tailscale.com/) is primarily developed by the -people at https://github.com/orgs/tailscale/people. For other contributors, -see: +See [docs/architecture.md](docs/architecture.md) for component overview, data flow, and design decisions. -* https://github.com/tailscale/tailscale/graphs/contributors -* https://github.com/tailscale/tailscale-android/graphs/contributors +## License -## Legal +BSD-3-Clause. Based on [tailscale/tailscale](https://github.com/tailscale/tailscale). +SCION networking provided by [scionproto/scion](https://github.com/scionproto/scion) (Apache-2.0). -WireGuard is a registered trademark of Jason A. Donenfeld. +This project is not affiliated with or endorsed by Tailscale Inc. +WireGuard is a registered trademark of Jason A. Donenfeld. \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000000000..1830ebb707b39 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,68 @@ +# SCION Integration Architecture + +## Component Overview + +SCION is added as a third transport in `magicsock.Conn`, alongside the existing IPv4/IPv6 UDP and DERP relay transports. The `endpoint.betterAddr` mechanism selects the best path across all three. + + +### Key Files (`wgengine/magicsock/`) + +| File | Role | +|------|------| +| `magicsock_scion.go` | Connection setup, path registry, send/receive, `ReconfigureSCION()` | +| `magicsock_scion_conn.go` | SCION connection lifecycle (init, close, bind) | +| `endpoint_scion.go` | Per-peer SCION state, heartbeat, path probing, pong handling | +| `scion_bootstrap.go` | Topology/TRC fetch from bootstrap servers, DNS SRV discovery | +| `scion_embedded.go` | In-process SCION daemon (no external sciond needed) | +| `magicsock_scion_omit.go` | No-op stubs for `ts_omit_scion` builds | +| `ipn/localapi/localapi_scion.go` | `GET /localapi/v0/scion-status` handler | +| `ipn/ipnlocal/local.go` | SCION service advertisement + peerapi4 piggyback (lines 4892-4906) | + +## Connection Flow + +`trySCIONConnect()` uses a cascading fallback strategy: + +1. **External daemon** -- connects to sciond via gRPC at `SCION_DAEMON_ADDRESS` (default `127.0.0.1:30255`). Probes `Paths()` to detect version mismatch. *Skipped if `TS_SCION_EMBEDDED=1`.* + +2. **Embedded daemon with local topology** -- checks for topology file at: + - `TS_SCION_TOPOLOGY` (explicit path) + - `/etc/scion/topology.json` (Linux default) + - `/scion/topology.json` (from prior bootstrap) + + Creates an in-process `embeddedConnector` with topology loader + segment fetcher (accept-all verification, Phase 1). *Skipped if `TS_SCION_FORCE_BOOTSTRAP=1`.* + +3. **Bootstrap + embedded** -- tries each URL from `bootstrapURLs()` in order: + 1. Explicit `TS_SCION_BOOTSTRAP_URL` + 2. Comma-separated `TS_SCION_BOOTSTRAP_URLS` + 3. DNS SRV: `_sciondiscovery._tcp.` + 4. Hardcoded defaults (ovgu.de, uva, ethz.ch) + + For each URL: fetches `topology.json` + TRCs (TRCs optional) → saves to stateDir → creates embedded daemon with bootstrapped topology. + +**Android**: `ReconfigureSCION()` forces `TS_SCION_EMBEDDED=1` + `TS_SCION_FORCE_BOOTSTRAP=1`, always skipping steps 1-2 and going straight to bootstrap. + +## Data Flow + +**Outbound**: `endpoint.send()` → if bestAddr is SCION → `sendSCION()` → pre-serialized SCION header + WireGuard payload → UDP to first-hop border router. + +**Inbound**: `receiveSCION()` → parse SCION header (slayers) → extract source IA + host → route to endpoint via reverse index → deliver WireGuard payload. + +**Path discovery**: `refreshSCIONPaths()` runs every 30s → queries daemon `Paths()` → discovers up to 5 paths per peer (`TS_SCION_MAX_PROBE_PATHS`) → probes latency via disco pings → `betterAddr` promotes best path (with configurable SCION preference bonus). + +## Key Design Decisions + +- **Third transport, not replacement.** SCION runs alongside IPv4/IPv6 UDP. Fallback is automatic -- if SCION is unavailable, direct or relay paths are used. + +- **Path selection via `betterAddr`.** SCION paths get a configurable preference bonus (`TS_SCION_PREFERENCE`, default 15 points). +25 additional points when both peers have the `NodeAttrSCIONPrefer` capability. Incumbent bias prevents flapping (candidate must be >=20% or >=2ms faster). + +- **Embedded daemon.** `scion_embedded.go` implements `daemon.Connector` with an in-process topology loader and segment fetcher. No external sciond process required -- critical for Android. + +- **Bootstrap discovery.** `scion_bootstrap.go` discovers topology via DNS SRV (`_sciondiscovery._tcp`) or hardcoded fallback URLs. TRC fetch is best-effort (Phase 1: accept-all verification). + +- **Fast-path sends.** Pre-serialized SCION+UDP header templates (`scionFastPath`) avoid per-packet allocation. Batch send via `sendmmsg` where available. Disable with `TS_SCION_NO_FAST_PATH`. + +- **Build tag `ts_omit_scion`.** Compiles out all SCION code via no-op stubs. Feature flag `buildfeatures.HasSCION` set at compile time. Produces smaller binary with no scionproto dependency. + +- **Service advertisement via peerapi4 piggyback.** The Tailscale coordination server only relays `peerapi4`/`peerapi6` services to peers — it drops unknown service types. To work without coord server changes, SCION address info is piggybacked onto the `peerapi4` service's `Description` field as `scion=ISD-AS,[hostIP]:port` (see `ipn/ipnlocal/local.go:4892-4906`). Bracket notation around the IP ensures unambiguous parsing for both IPv4 and IPv6 underlay addresses. A standalone `tailcfg.ServiceProto("scion")` entry is also advertised for future coord server support. On the receiving side, `scionServiceFromPeer()` checks for a dedicated SCION service first, then falls back to parsing the peerapi4 piggyback (with backward compatibility for unbracketed format). + +- **Cross-platform.** Platform-specific DNS search domain resolution in `scion_bootstrap_unix.go` (Linux, macOS, BSDs), `scion_bootstrap_windows.go`, and `scion_bootstrap_other.go` (Android fallback). \ No newline at end of file diff --git a/wgengine/magicsock/magicsock_scion.go b/wgengine/magicsock/magicsock_scion.go index 4c02aa914291e..336ff76628df3 100644 --- a/wgengine/magicsock/magicsock_scion.go +++ b/wgengine/magicsock/magicsock_scion.go @@ -1081,12 +1081,13 @@ func openDispatcherShim(sc *scionConn, logf logger.Logf, netMon *netmon.Monitor) } // parseSCIONServiceAddr parses a SCION service description string of the form -// "ISD-AS,host-IP" and returns the IA and host address. The port comes from the -// Service.Port field. +// "ISD-AS,[host-IP]" and returns the IA and host address. The port comes from +// the Service.Port field. Accepts both bracketed ("[192.0.2.1]", "[2001:db8::1]") +// and unbracketed ("192.0.2.1", "2001:db8::1") IP formats for backward compatibility. func parseSCIONServiceAddr(description string, port uint16) (ia addr.IA, hostAddr netip.AddrPort, err error) { parts := strings.SplitN(description, ",", 2) if len(parts) != 2 { - return 0, netip.AddrPort{}, fmt.Errorf("invalid SCION service description %q: want ISD-AS,host-IP", description) + return 0, netip.AddrPort{}, fmt.Errorf("invalid SCION service description %q: want ISD-AS,[host-IP]", description) } ia, err = addr.ParseIA(parts[0]) @@ -1094,7 +1095,9 @@ func parseSCIONServiceAddr(description string, port uint16) (ia addr.IA, hostAdd return 0, netip.AddrPort{}, fmt.Errorf("parsing SCION IA %q: %w", parts[0], err) } - hostIP, err := netip.ParseAddr(parts[1]) + // Strip brackets if present (e.g., "[192.0.2.1]" or "[2001:db8::1]"). + ipStr := strings.TrimPrefix(strings.TrimSuffix(parts[1], "]"), "[") + hostIP, err := netip.ParseAddr(ipStr) if err != nil { return 0, netip.AddrPort{}, fmt.Errorf("parsing SCION host IP %q: %w", parts[1], err) } @@ -2509,17 +2512,25 @@ func scionServiceFromPeer(n tailcfg.NodeView) (ia addr.IA, hostAddr netip.AddrPo return parsedIA, parsedAddr, true } // Piggyback: SCION info in peerapi4's Description field. - // Format: "scion=ISD-AS,host-IP:port" + // Format: "scion=ISD-AS,[host-IP]:port" if svc.Proto == tailcfg.PeerAPI4 && strings.HasPrefix(svc.Description, "scion=") { scionDesc := svc.Description[len("scion="):] - // Parse "ISD-AS,host-IP:port" - lastColon := strings.LastIndex(scionDesc, ":") - if lastColon < 0 { - continue + var addrPart, portStr string + // Try bracket notation first: "ISD-AS,[hostIP]:port" + if portSep := strings.LastIndex(scionDesc, "]:"); portSep >= 0 { + addrPart = scionDesc[:portSep+1] + portStr = scionDesc[portSep+2:] + } else { + // Backward compat: unbracketed "ISD-AS,hostIP:port" + lastColon := strings.LastIndex(scionDesc, ":") + if lastColon < 0 { + continue + } + addrPart = scionDesc[:lastColon] + portStr = scionDesc[lastColon+1:] } - addrPart := scionDesc[:lastColon] var port uint16 - if _, err := fmt.Sscanf(scionDesc[lastColon+1:], "%d", &port); err != nil { + if _, err := fmt.Sscanf(portStr, "%d", &port); err != nil { continue } parsedIA, parsedAddr, err := parseSCIONServiceAddr(addrPart, port) @@ -2552,7 +2563,7 @@ func (c *Conn) SCIONService() (svc tailcfg.Service, ok bool) { return tailcfg.Service{ Proto: tailcfg.SCION, Port: scionPort, - Description: fmt.Sprintf("%s,%s", sc.localIA, hostIP), + Description: fmt.Sprintf("%s,[%s]", sc.localIA, hostIP), }, true } diff --git a/wgengine/magicsock/magicsock_scion_test.go b/wgengine/magicsock/magicsock_scion_test.go index ff5c491f7e223..bf9cf077e1f1d 100644 --- a/wgengine/magicsock/magicsock_scion_test.go +++ b/wgengine/magicsock/magicsock_scion_test.go @@ -126,14 +126,28 @@ func TestParseSCIONServiceAddr(t *testing.T) { wantErr bool }{ { - name: "valid IPv4", + name: "valid IPv4 bracketed", + description: "1-ff00:0:110,[192.0.2.1]", + port: 41641, + wantIA: addr.MustParseIA("1-ff00:0:110"), + wantAddr: netip.MustParseAddrPort("192.0.2.1:41641"), + }, + { + name: "valid IPv6 bracketed", + description: "1-ff00:0:110,[2001:db8::1]", + port: 12345, + wantIA: addr.MustParseIA("1-ff00:0:110"), + wantAddr: netip.MustParseAddrPort("[2001:db8::1]:12345"), + }, + { + name: "valid IPv4 unbracketed (backward compat)", description: "1-ff00:0:110,192.0.2.1", port: 41641, wantIA: addr.MustParseIA("1-ff00:0:110"), wantAddr: netip.MustParseAddrPort("192.0.2.1:41641"), }, { - name: "valid IPv6", + name: "valid IPv6 unbracketed (backward compat)", description: "1-ff00:0:110,2001:db8::1", port: 12345, wantIA: addr.MustParseIA("1-ff00:0:110"), @@ -406,14 +420,14 @@ func TestScionServiceFromPeer(t *testing.T) { wantOk bool }{ { - name: "peer with SCION service", + name: "peer with SCION service (bracketed IPv4)", node: &tailcfg.Node{ ID: 1, Key: testNodeKey(), Hostinfo: (&tailcfg.Hostinfo{ Services: []tailcfg.Service{ {Proto: tailcfg.TCP, Port: 80}, - {Proto: tailcfg.SCION, Port: 41641, Description: "1-ff00:0:110,192.0.2.1"}, + {Proto: tailcfg.SCION, Port: 41641, Description: "1-ff00:0:110,[192.0.2.1]"}, }, }).View(), }, @@ -421,6 +435,21 @@ func TestScionServiceFromPeer(t *testing.T) { wantAddr: netip.MustParseAddrPort("192.0.2.1:41641"), wantOk: true, }, + { + name: "peer with SCION service (bracketed IPv6)", + node: &tailcfg.Node{ + ID: 1, + Key: testNodeKey(), + Hostinfo: (&tailcfg.Hostinfo{ + Services: []tailcfg.Service{ + {Proto: tailcfg.SCION, Port: 41641, Description: "1-ff00:0:110,[2001:db8::1]"}, + }, + }).View(), + }, + wantIA: addr.MustParseIA("1-ff00:0:110"), + wantAddr: netip.MustParseAddrPort("[2001:db8::1]:41641"), + wantOk: true, + }, { name: "peer without SCION service", node: &tailcfg.Node{ @@ -457,7 +486,37 @@ func TestScionServiceFromPeer(t *testing.T) { wantOk: false, }, { - name: "peer with SCION in peerapi4 description (piggyback)", + name: "peer with SCION in peerapi4 description (piggyback, bracketed IPv4)", + node: &tailcfg.Node{ + ID: 5, + Key: testNodeKey(), + Hostinfo: (&tailcfg.Hostinfo{ + Services: []tailcfg.Service{ + {Proto: tailcfg.PeerAPI4, Port: 12345, Description: "scion=1-ff00:0:110,[192.0.2.1]:32766"}, + }, + }).View(), + }, + wantIA: addr.MustParseIA("1-ff00:0:110"), + wantAddr: netip.MustParseAddrPort("192.0.2.1:32766"), + wantOk: true, + }, + { + name: "peer with SCION piggyback (bracketed IPv6)", + node: &tailcfg.Node{ + ID: 5, + Key: testNodeKey(), + Hostinfo: (&tailcfg.Hostinfo{ + Services: []tailcfg.Service{ + {Proto: tailcfg.PeerAPI4, Port: 12345, Description: "scion=1-ff00:0:110,[2001:db8::1]:32766"}, + }, + }).View(), + }, + wantIA: addr.MustParseIA("1-ff00:0:110"), + wantAddr: netip.MustParseAddrPort("[2001:db8::1]:32766"), + wantOk: true, + }, + { + name: "peer with SCION piggyback (unbracketed IPv4, backward compat)", node: &tailcfg.Node{ ID: 5, Key: testNodeKey(), From 1f05c7ec92f34a0d3c657a1e7d30e53e0dd70f7d Mon Sep 17 00:00:00 2001 From: Tony John Date: Tue, 17 Mar 2026 13:16:18 +0100 Subject: [PATCH 77/78] docs: add Releases section to README and CLI docs link to release notes --- .github/workflows/release.yml | 7 ++++++- README.md | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3abf279380885..e20003c16b13e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -109,4 +109,9 @@ jobs: **Linux (rpm):** `sudo rpm -i tailscale-scion_*.rpm && sudo systemctl enable --now tailscaled` **Linux (tgz):** Extract and copy `tailscale`/`tailscaled` to PATH **macOS:** Extract tgz, run `sudo ./tailscaled &` then `./tailscale up` - **Windows:** Extract zip, run `tailscaled.exe` then `tailscale.exe up` \ No newline at end of file + **Windows:** Extract zip, run `tailscaled.exe` then `tailscale.exe up` + + ### Documentation + + CLI reference: https://tailscale.com/docs/reference/tailscale-cli + SCION configuration: https://github.com/netsys-lab/tailscale-scion#configuration \ No newline at end of file diff --git a/README.md b/README.md index 51268ad27cf47..0a50b3ed3f21c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,12 @@ A fork of [tailscale/tailscale](https://github.com/tailscale/tailscale) that add Experimental. Platforms: Linux, macOS, Windows, FreeBSD, OpenBSD, NetBSD, Android (via [tailscale-android-scion](https://github.com/netsys-lab/tailscale-android-scion)). +## Releases + +Pre-built binaries for Linux (amd64/arm64), macOS, and Windows are available on the [Releases](https://github.com/netsys-lab/tailscale-scion/releases) page. Android APK releases are available from [tailscale-android-scion](https://github.com/netsys-lab/tailscale-android-scion/releases). + +For CLI usage, see the [Tailscale CLI reference](https://tailscale.com/docs/reference/tailscale-cli) — all standard `tailscale` and `tailscaled` commands work the same. + ## Quick Start (Linux) ```bash From 986720525c9a365d535e402cb63f8ca2e9a50c00 Mon Sep 17 00:00:00 2001 From: Tony John Date: Tue, 17 Mar 2026 13:35:15 +0100 Subject: [PATCH 78/78] ci: remove upstream Tailscale workflows that fail on fork These workflows depend on Tailscale-specific infrastructure (self-hosted runners, Azure cigocacher, Slack, FlakeHub, private secrets). Keep only release.yml for our GitHub Releases. --- .github/workflows/checklocks.yml | 34 - .github/workflows/cigocacher.yml | 73 -- .github/workflows/codeql-analysis.yml | 83 -- .github/workflows/docker-base.yml | 29 - .github/workflows/docker-file-build.yml | 13 - .github/workflows/flakehub-publish-tagged.yml | 27 - .github/workflows/golangci-lint.yml | 47 - .github/workflows/govulncheck.yml | 51 - .github/workflows/installer.yml | 143 --- .github/workflows/kubemanifests.yaml | 31 - .github/workflows/natlab-integrationtest.yml | 33 - .github/workflows/pin-github-actions.yml | 29 - .../workflows/request-dataplane-review.yml | 32 - .github/workflows/ssh-integrationtest.yml | 23 - .github/workflows/test.yml | 1007 ----------------- .github/workflows/update-flake.yml | 49 - .../workflows/update-webclient-prebuilt.yml | 50 - .github/workflows/vet.yml | 39 - .github/workflows/webclient.yml | 38 - 19 files changed, 1831 deletions(-) delete mode 100644 .github/workflows/checklocks.yml delete mode 100644 .github/workflows/cigocacher.yml delete mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/docker-base.yml delete mode 100644 .github/workflows/docker-file-build.yml delete mode 100644 .github/workflows/flakehub-publish-tagged.yml delete mode 100644 .github/workflows/golangci-lint.yml delete mode 100644 .github/workflows/govulncheck.yml delete mode 100644 .github/workflows/installer.yml delete mode 100644 .github/workflows/kubemanifests.yaml delete mode 100644 .github/workflows/natlab-integrationtest.yml delete mode 100644 .github/workflows/pin-github-actions.yml delete mode 100644 .github/workflows/request-dataplane-review.yml delete mode 100644 .github/workflows/ssh-integrationtest.yml delete mode 100644 .github/workflows/test.yml delete mode 100644 .github/workflows/update-flake.yml delete mode 100644 .github/workflows/update-webclient-prebuilt.yml delete mode 100644 .github/workflows/vet.yml delete mode 100644 .github/workflows/webclient.yml diff --git a/.github/workflows/checklocks.yml b/.github/workflows/checklocks.yml deleted file mode 100644 index 5768cf05af634..0000000000000 --- a/.github/workflows/checklocks.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: checklocks - -on: - push: - branches: - - main - pull_request: - paths: - - '**/*.go' - - '.github/workflows/checklocks.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - checklocks: - runs-on: [ ubuntu-latest ] - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Build checklocks - run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks - - - name: Run checklocks vet - # TODO(#12625): add more packages as we add annotations - run: |- - ./tool/go vet -vettool=/tmp/checklocks \ - ./envknob \ - ./ipn/store/mem \ - ./net/stun/stuntest \ - ./net/wsconn \ - ./proxymap diff --git a/.github/workflows/cigocacher.yml b/.github/workflows/cigocacher.yml deleted file mode 100644 index fea1f6a0dc988..0000000000000 --- a/.github/workflows/cigocacher.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Build cigocacher - -on: - # Released on-demand. The commit will be used as part of the tag, so generally - # prefer to release from main where the commit is stable in linear history. - workflow_dispatch: - -jobs: - build: - strategy: - matrix: - GOOS: ["linux", "darwin", "windows"] - GOARCH: ["amd64", "arm64"] - runs-on: ubuntu-24.04 - env: - GOOS: "${{ matrix.GOOS }}" - GOARCH: "${{ matrix.GOARCH }}" - CGO_ENABLED: "0" - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Build - run: | - OUT="cigocacher$(./tool/go env GOEXE)" - ./tool/go build -o "${OUT}" ./cmd/cigocacher/ - tar -zcf cigocacher-${{ matrix.GOOS }}-${{ matrix.GOARCH }}.tar.gz "${OUT}" - - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: cigocacher-${{ matrix.GOOS }}-${{ matrix.GOARCH }} - path: cigocacher-${{ matrix.GOOS }}-${{ matrix.GOARCH }}.tar.gz - - release: - runs-on: ubuntu-24.04 - needs: build - permissions: - contents: write - steps: - - name: Download all artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - pattern: 'cigocacher-*' - merge-multiple: true - # This step is a simplified version of actions/create-release and - # actions/upload-release-asset, which are archived and unmaintained. - - name: Create release - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - with: - script: | - const fs = require('fs'); - const path = require('path'); - - const { data: release } = await github.rest.repos.createRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - tag_name: `cmd/cigocacher/${{ github.sha }}`, - name: `cigocacher-${{ github.sha }}`, - draft: false, - prerelease: true, - target_commitish: `${{ github.sha }}` - }); - - const files = fs.readdirSync('.').filter(f => f.endsWith('.tar.gz')); - - for (const file of files) { - await github.rest.repos.uploadReleaseAsset({ - owner: context.repo.owner, - repo: context.repo.repo, - release_id: release.id, - name: file, - data: fs.readFileSync(file) - }); - console.log(`Uploaded ${file}`); - } diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 9e1e518f666fc..0000000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,83 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ main, release-branch/* ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ main ] - merge_group: - branches: [ main ] - schedule: - - cron: '31 14 * * 5' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed - - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - # Install a more recent Go that understands modern go.mod content. - - name: Install Go - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version-file: go.mod - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 diff --git a/.github/workflows/docker-base.yml b/.github/workflows/docker-base.yml deleted file mode 100644 index a3eac2c24e691..0000000000000 --- a/.github/workflows/docker-base.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: "Validate Docker base image" -on: - workflow_dispatch: - pull_request: - paths: - - "Dockerfile.base" - - ".github/workflows/docker-base.yml" -jobs: - build-and-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: "build and test" - run: | - set -e - IMG="test-base:$(head -c 8 /dev/urandom | xxd -p)" - docker build -t "$IMG" -f Dockerfile.base . - - iptables_version=$(docker run --rm "$IMG" iptables --version) - if [[ "$iptables_version" != *"(legacy)"* ]]; then - echo "ERROR: Docker base image should contain legacy iptables; found ${iptables_version}" - exit 1 - fi - - ip6tables_version=$(docker run --rm "$IMG" ip6tables --version) - if [[ "$ip6tables_version" != *"(legacy)"* ]]; then - echo "ERROR: Docker base image should contain legacy ip6tables; found ${ip6tables_version}" - exit 1 - fi diff --git a/.github/workflows/docker-file-build.yml b/.github/workflows/docker-file-build.yml deleted file mode 100644 index 7ee2468682695..0000000000000 --- a/.github/workflows/docker-file-build.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: "Dockerfile build" -on: - push: - branches: - - main - pull_request: -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: "Build Docker image" - run: docker build . diff --git a/.github/workflows/flakehub-publish-tagged.yml b/.github/workflows/flakehub-publish-tagged.yml deleted file mode 100644 index c781e30e5154f..0000000000000 --- a/.github/workflows/flakehub-publish-tagged.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: update-flakehub - -on: - push: - tags: - - "v[0-9]+.*[02468].[0-9]+" - workflow_dispatch: - inputs: - tag: - description: "The existing tag to publish to FlakeHub" - type: "string" - required: true -jobs: - flakehub-publish: - runs-on: "ubuntu-latest" - permissions: - id-token: "write" - contents: "read" - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}" - - uses: DeterminateSystems/nix-installer-action@c5a866b6ab867e88becbed4467b93592bce69f8a # v21 - - uses: DeterminateSystems/flakehub-push@71f57208810a5d299fc6545350981de98fdbc860 # v6 - with: - visibility: "public" - tag: "${{ inputs.tag }}" diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml deleted file mode 100644 index 66b8497e65441..0000000000000 --- a/.github/workflows/golangci-lint.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: golangci-lint -on: - # For now, only lint pull requests, not the main branches. - pull_request: - paths: - - ".github/workflows/golangci-lint.yml" - - "**.go" - - "go.mod" - - "go.sum" - # TODO(andrew): enable for main branch after an initial waiting period. - #push: - # branches: - # - main - - workflow_dispatch: - -permissions: - contents: read - pull-requests: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version-file: go.mod - cache: true - - - name: golangci-lint - uses: golangci/golangci-lint-action@b7bcab6379029e905e3f389a6bf301f1bc220662 # head as of 2026-03-04 - with: - version: v2.10.1 - - # Show only new issues if it's a pull request. - only-new-issues: true - - # Loading packages with a cold cache takes a while: - args: --timeout=10m - diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml deleted file mode 100644 index 2b46aa9b06e57..0000000000000 --- a/.github/workflows/govulncheck.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: govulncheck - -on: - schedule: - - cron: "0 12 * * *" # 8am EST / 10am PST / 12pm UTC - workflow_dispatch: # allow manual trigger for testing - pull_request: - paths: - - ".github/workflows/govulncheck.yml" - -jobs: - source-scan: - runs-on: ubuntu-latest - - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Install govulncheck - run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest - - - name: Scan source code for known vulnerabilities - run: PATH=$PWD/tool/:$PATH "$(./tool/go env GOPATH)/bin/govulncheck" -test ./... - - - name: Post to slack - if: failure() && github.event_name == 'schedule' - uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 - with: - method: chat.postMessage - token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }} - payload: | - { - "channel": "C08FGKZCQTW", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Govulncheck failed in ${{ github.repository }}" - }, - "accessory": { - "type": "button", - "text": { - "type": "plain_text", - "text": "View results" - }, - "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - } - } - ] - } diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml deleted file mode 100644 index 6fc8913c4e19c..0000000000000 --- a/.github/workflows/installer.yml +++ /dev/null @@ -1,143 +0,0 @@ -name: test installer.sh - -on: - schedule: - - cron: '0 15 * * *' # 10am EST (UTC-4/5) - push: - branches: - - "main" - paths: - - scripts/installer.sh - - .github/workflows/installer.yml - pull_request: - paths: - - scripts/installer.sh - - .github/workflows/installer.yml - -jobs: - test: - strategy: - # Don't abort the entire matrix if one element fails. - fail-fast: false - # Don't start all of these at once, which could saturate Github workers. - max-parallel: 4 - matrix: - image: - # This is a list of Docker images against which we test our installer. - # If you find that some of these no longer exist, please feel free - # to remove them from the list. - # When adding new images, please only use official ones. - - "debian:oldstable-slim" - - "debian:stable-slim" - - "debian:testing-slim" - - "debian:sid-slim" - - "ubuntu:20.04" - - "ubuntu:22.04" - - "ubuntu:24.04" - - "elementary/docker:stable" - - "elementary/docker:unstable" - - "parrotsec/core:latest" - - "kalilinux/kali-rolling" - - "kalilinux/kali-dev" - - "oraclelinux:9" - - "oraclelinux:8" - - "fedora:latest" - - "rockylinux:8.7" - - "rockylinux:9" - - "amazonlinux:latest" - - "opensuse/leap:latest" - - "opensuse/tumbleweed:latest" - - "archlinux:latest" - - "alpine:3.21" - - "alpine:latest" - - "alpine:edge" - deps: - # Run all images installing curl as a dependency. - - curl - include: - # Check a few images with wget rather than curl. - - { image: "debian:oldstable-slim", deps: "wget" } - - { image: "debian:sid-slim", deps: "wget" } - - { image: "debian:stable-slim", deps: "curl" } - - { image: "ubuntu:24.04", deps: "curl" } - - { image: "fedora:latest", deps: "curl" } - # Test TAILSCALE_VERSION pinning on a subset of distros. - # Skip Alpine as community repos don't reliably keep old versions. - - { image: "debian:stable-slim", deps: "curl", version: "1.80.0" } - - { image: "ubuntu:24.04", deps: "curl", version: "1.80.0" } - - { image: "fedora:latest", deps: "curl", version: "1.80.0" } - runs-on: ubuntu-latest - container: - image: ${{ matrix.image }} - options: --user root - steps: - - name: install dependencies (pacman) - # Refresh the package databases to ensure that the tailscale package is - # defined. - run: pacman -Sy - if: contains(matrix.image, 'archlinux') - - name: install dependencies (yum) - # tar and gzip are needed by the actions/checkout below. - run: yum install -y --allowerasing tar gzip ${{ matrix.deps }} - if: | - contains(matrix.image, 'centos') || - contains(matrix.image, 'oraclelinux') || - contains(matrix.image, 'fedora') || - contains(matrix.image, 'amazonlinux') - - name: install dependencies (zypper) - # tar and gzip are needed by the actions/checkout below. - run: zypper --non-interactive install tar gzip ${{ matrix.deps }} - if: contains(matrix.image, 'opensuse') - - name: install dependencies (apt-get) - run: | - apt-get update - apt-get install -y ${{ matrix.deps }} - if: | - contains(matrix.image, 'debian') || - contains(matrix.image, 'ubuntu') || - contains(matrix.image, 'elementary') || - contains(matrix.image, 'parrotsec') || - contains(matrix.image, 'kalilinux') - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: run installer - run: scripts/installer.sh - env: - TAILSCALE_VERSION: ${{ matrix.version }} - # Package installation can fail in docker because systemd is not running - # as PID 1, so ignore errors at this step. The real check is the - # `tailscale --version` command below. - continue-on-error: true - - name: check tailscale version - run: | - tailscale --version - if [ -n "${{ matrix.version }}" ]; then - tailscale --version | grep -q "^${{ matrix.version }}" || { echo "Version mismatch!"; exit 1; } - fi - notify-slack: - needs: test - runs-on: ubuntu-latest - steps: - - name: Notify Slack of failure on scheduled runs - if: failure() && github.event_name == 'schedule' - uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 - with: - webhook: ${{ secrets.SLACK_WEBHOOK_URL }} - webhook-type: incoming-webhook - payload: | - { - "attachments": [{ - "title": "Tailscale installer test failed", - "title_link": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", - "text": "One or more OSes in the test matrix failed. See the run for details.", - "fields": [ - { - "title": "Ref", - "value": "${{ github.ref_name }}", - "short": true - } - ], - "footer": "${{ github.workflow }} on schedule", - "color": "danger" - }] - } diff --git a/.github/workflows/kubemanifests.yaml b/.github/workflows/kubemanifests.yaml deleted file mode 100644 index 40734a015dad3..0000000000000 --- a/.github/workflows/kubemanifests.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Kubernetes manifests" -on: - pull_request: - paths: - - 'cmd/k8s-operator/**' - - 'k8s-operator/**' - - '.github/workflows/kubemanifests.yaml' - -# Cancel workflow run if there is a newer push to the same PR for which it is -# running -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - testchart: - runs-on: [ ubuntu-latest ] - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Build and lint Helm chart - run: | - eval `./tool/go run ./cmd/mkversion` - ./tool/helm package --app-version="${VERSION_SHORT}" --version=${VERSION_SHORT} './cmd/k8s-operator/deploy/chart' - ./tool/helm lint "tailscale-operator-${VERSION_SHORT}.tgz" - - name: Verify that static manifests are up to date - run: | - make kube-generate-all - echo - echo - git diff --name-only --exit-code || (echo "Generated files for Tailscale Kubernetes operator are out of date. Please run 'make kube-generate-all' and commit the diff."; exit 1) diff --git a/.github/workflows/natlab-integrationtest.yml b/.github/workflows/natlab-integrationtest.yml deleted file mode 100644 index 162153cb23293..0000000000000 --- a/.github/workflows/natlab-integrationtest.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Run some natlab integration tests. -# See https://github.com/tailscale/tailscale/issues/13038 -name: "natlab-integrationtest" - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - push: - branches: - - "main" - - "release-branch/*" - pull_request: - # all PRs on all branches - merge_group: - branches: - - "main" -jobs: - natlab-integrationtest: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Install qemu - run: | - sudo rm -f /var/lib/man-db/auto-update - sudo apt-get -y update - sudo apt-get -y remove man-db - sudo apt-get install -y qemu-system-x86 qemu-utils - - name: Run natlab integration tests - run: | - ./tool/go test -v -run=^TestEasyEasy$ -timeout=3m -count=1 ./tstest/integration/nat --run-vm-tests diff --git a/.github/workflows/pin-github-actions.yml b/.github/workflows/pin-github-actions.yml deleted file mode 100644 index 836ae46dbfa89..0000000000000 --- a/.github/workflows/pin-github-actions.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Pin images used in github actions to a hash instead of a version tag. -name: pin-github-actions -on: - pull_request: - branches: - - main - paths: - - ".github/workflows/**" - - workflow_dispatch: - -permissions: - contents: read - pull-requests: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - run: - name: pin-github-actions - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: pin - run: make pin-github-actions - - name: check for changed workflow files - run: git diff --no-ext-diff --exit-code .github/workflows || (echo "Some github actions versions need pinning, run make pin-github-actions."; exit 1) diff --git a/.github/workflows/request-dataplane-review.yml b/.github/workflows/request-dataplane-review.yml deleted file mode 100644 index 2b66fc7899428..0000000000000 --- a/.github/workflows/request-dataplane-review.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: request-dataplane-review - -on: - pull_request: - types: [ opened, synchronize, reopened, ready_for_review ] - paths: - - ".github/workflows/request-dataplane-review.yml" - - "**/*derp*" - - "**/derp*/**" - - "!**/depaware.txt" - -jobs: - request-dataplane-review: - if: github.event.pull_request.draft == false - name: Request Dataplane Review - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Get access token - uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 - id: generate-token - with: - # Get token for app: https://github.com/apps/change-visibility-bot - app-id: ${{ secrets.VISIBILITY_BOT_APP_ID }} - private-key: ${{ secrets.VISIBILITY_BOT_APP_PRIVATE_KEY }} - - name: Add reviewers - env: - GH_TOKEN: ${{ steps.generate-token.outputs.token }} - url: ${{ github.event.pull_request.html_url }} - run: | - gh pr edit "$url" --add-reviewer tailscale/dataplane diff --git a/.github/workflows/ssh-integrationtest.yml b/.github/workflows/ssh-integrationtest.yml deleted file mode 100644 index afe2dd2f74683..0000000000000 --- a/.github/workflows/ssh-integrationtest.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Run the ssh integration tests with `make sshintegrationtest`. -# These tests can also be running locally. -name: "ssh-integrationtest" - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - pull_request: - paths: - - "ssh/**" - - "tempfork/gliderlabs/ssh/**" - - ".github/workflows/ssh-integrationtest" -jobs: - ssh-integrationtest: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Run SSH integration tests - run: | - make sshintegrationtest \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 317052229676e..0000000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,1007 +0,0 @@ -# This is our main "CI tests" workflow. It runs everything that should run on -# both PRs and merged commits, and for the latter reports failures to slack. -name: CI - -env: - # Our fuzz job, powered by OSS-Fuzz, fails periodically because we upgrade to - # new Go versions very eagerly. OSS-Fuzz is a little more conservative, and - # ends up being unable to compile our code. - # - # When this happens, we want to disable the fuzz target until OSS-Fuzz catches - # up. However, we also don't want to forget to turn it back on when OSS-Fuzz - # can once again build our code. - # - # This variable toggles the fuzz job between two modes: - # - false: we expect fuzzing to be happy, and should report failure if it's not. - # - true: we expect fuzzing is broken, and should report failure if it start working. - TS_FUZZ_CURRENTLY_BROKEN: false - # GOMODCACHE is the same definition on all OSes. Within the workspace, we use - # toplevel directories "src" (for the checked out source code), and "gomodcache" - # and other caches as siblings to follow. - GOMODCACHE: ${{ github.workspace }}/gomodcache - CMD_GO_USE_GIT_HASH: "true" - -on: - push: - branches: - - "main" - - "release-branch/*" - pull_request: - # all PRs on all branches - merge_group: - branches: - - "main" - -concurrency: - # For PRs, later CI runs preempt previous ones. e.g. a force push on a PR - # cancels running CI jobs and starts all new ones. - # - # For non-PR pushes, concurrency.group needs to be unique for every distinct - # CI run we want to have happen. Use run_id, which in practice means all - # non-PR CI runs will be allowed to run without preempting each other. - group: ${{ github.workflow }}-$${{ github.pull_request.number || github.run_id }} - cancel-in-progress: true - -jobs: - gomod-cache: - runs-on: ubuntu-24.04 - outputs: - cache-key: ${{ steps.hash.outputs.key }} - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Compute cache key from go.{mod,sum} - id: hash - run: echo "key=gomod-cross3-${{ hashFiles('src/go.mod', 'src/go.sum') }}" >> $GITHUB_OUTPUT - # See if the cache entry already exists to avoid downloading it - # and doing the cache write again. - - id: check-cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache # relative to workspace; see env note at top of file - key: ${{ steps.hash.outputs.key }} - lookup-only: true - enableCrossOsArchive: true - - name: Download modules - if: steps.check-cache.outputs.cache-hit != 'true' - working-directory: src - run: go mod download - - name: Cache Go modules - if: steps.check-cache.outputs.cache-hit != 'true' - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache # relative to workspace; see env note at top of file - key: ${{ steps.hash.outputs.key }} - enableCrossOsArchive: true - - race-root-integration: - runs-on: ubuntu-24.04 - needs: gomod-cache - strategy: - fail-fast: false # don't abort the entire matrix if one element fails - matrix: - include: - - shard: '1/4' - - shard: '2/4' - - shard: '3/4' - - shard: '4/4' - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: build test wrapper - working-directory: src - run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper - - name: integration tests as root - working-directory: src - run: PATH=$PWD/tool:$PATH /tmp/testwrapper -exec "sudo -E" -race ./tstest/integration/ - env: - TS_TEST_SHARD: ${{ matrix.shard }} - - test: - strategy: - fail-fast: false # don't abort the entire matrix if one element fails - matrix: - include: - - goarch: amd64 - - goarch: amd64 - buildflags: "-race" - shard: '1/3' - - goarch: amd64 - buildflags: "-race" - shard: '2/3' - - goarch: amd64 - buildflags: "-race" - shard: '3/3' - - goarch: "386" # thanks yaml - runs-on: ubuntu-24.04 - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: Restore Cache - id: restore-cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - # Note: this is only restoring the build cache. Mod cache is shared amongst - # all jobs in the workflow. - path: | - ~/.cache/go-build - ~\AppData\Local\go-build - key: ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - restore-keys: | - ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}-${{ hashFiles('**/go.sum') }}-${{ github.job }}- - ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}- - ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go- - - name: build all - if: matrix.buildflags == '' # skip on race builder - working-directory: src - run: ./tool/go build ${{matrix.buildflags}} ./... - env: - GOARCH: ${{ matrix.goarch }} - - name: build variant CLIs - if: matrix.buildflags == '' # skip on race builder - working-directory: src - run: | - ./build_dist.sh --extra-small ./cmd/tailscaled - ./build_dist.sh --box ./cmd/tailscaled - ./build_dist.sh --extra-small --box ./cmd/tailscaled - rm -f tailscaled - env: - GOARCH: ${{ matrix.goarch }} - - name: get qemu # for tstest/archtest - if: matrix.goarch == 'amd64' && matrix.buildflags == '' - run: | - sudo apt-get -y update - sudo apt-get -y install qemu-user - - name: build test wrapper - working-directory: src - run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper - - name: test all - working-directory: src - run: NOBASHDEBUG=true NOPWSHDEBUG=true PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}} - env: - GOARCH: ${{ matrix.goarch }} - TS_TEST_SHARD: ${{ matrix.shard }} - - name: bench all - working-directory: src - run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ $(for x in $(git grep -l "^func Benchmark" | xargs dirname | sort | uniq); do echo "./$x"; done) - env: - GOARCH: ${{ matrix.goarch }} - - name: check that no tracked files changed - working-directory: src - run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1) - - name: check that no new files were added - working-directory: src - run: | - # Note: The "error: pathspec..." you see below is normal! - # In the success case in which there are no new untracked files, - # git ls-files complains about the pathspec not matching anything. - # That's OK. It's not worth the effort to suppress. Please ignore it. - if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*' - then - echo "Build/test created untracked files in the repo (file names above)." - exit 1 - fi - - name: Tidy cache - working-directory: src - shell: bash - run: | - find $(go env GOCACHE) -type f -mmin +90 -delete - - name: Save Cache - # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. - if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' - uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - # Note: this is only saving the build cache. Mod cache is shared amongst - # all jobs in the workflow. - path: | - ~/.cache/go-build - ~\AppData\Local\go-build - key: ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - - windows: - permissions: - id-token: write # This is required for requesting the GitHub action identity JWT that can auth to cigocached - contents: read # This is required for actions/checkout - # ci-windows-github-1 is a 2022 GitHub-managed runner in our org with 8 cores - # and 32 GB of RAM. It is connected to a private Azure VNet that hosts cigocached. - # https://github.com/organizations/tailscale/settings/actions/github-hosted-runners/5 - runs-on: ci-windows-github-1 - needs: gomod-cache - name: Windows (${{ matrix.name || matrix.shard}}) - strategy: - fail-fast: false # don't abort the entire matrix if one element fails - matrix: - include: - - key: "win-bench" - name: "benchmarks" - - key: "win-shard-1-2" - shard: "1/2" - - key: "win-shard-2-2" - shard: "2/2" - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: ${{ github.workspace }}/src - - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - - name: Set up cigocacher - id: cigocacher-setup - uses: ./src/.github/actions/go-cache - with: - checkout-path: ${{ github.workspace }}/src - cache-dir: ${{ github.workspace }}/cigocacher - cigocached-url: ${{ vars.CIGOCACHED_AZURE_URL }} - cigocached-host: ${{ vars.CIGOCACHED_AZURE_HOST }} - - - name: test - if: matrix.key != 'win-bench' # skip on bench builder - working-directory: src - run: ./tool/go run ./cmd/testwrapper sharded:${{ matrix.shard }} - env: - NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI - - - name: bench all - if: matrix.key == 'win-bench' - working-directory: src - # Don't use -bench=. -benchtime=1x. - # Somewhere in the layers (powershell?) - # the equals signs cause great confusion. - run: ./tool/go test ./... -bench . -benchtime 1x -run "^$" - env: - NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI - - - name: Print stats - shell: pwsh - if: steps.cigocacher-setup.outputs.success == 'true' - env: - GOCACHEPROG: ${{ env.GOCACHEPROG }} - run: | - Invoke-Expression "$env:GOCACHEPROG --stats" | jq . - - macos: - runs-on: macos-latest - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: Restore Cache - id: restore-cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: ~/Library/Caches/go-build - key: ${{ runner.os }}-go-test-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - restore-keys: | - ${{ runner.os }}-go-test-${{ hashFiles('**/go.sum') }}-${{ github.job }}- - ${{ runner.os }}-go-test-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-go-test- - - name: build test wrapper - working-directory: src - run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper - - name: test all - working-directory: src - run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... - - name: check that no tracked files changed - working-directory: src - run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1) - - name: check that no new files were added - working-directory: src - run: | - # Note: The "error: pathspec..." you see below is normal! - # In the success case in which there are no new untracked files, - # git ls-files complains about the pathspec not matching anything. - # That's OK. It's not worth the effort to suppress. Please ignore it. - if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*' - then - echo "Build/test created untracked files in the repo (file names above)." - exit 1 - fi - - name: Tidy cache - working-directory: src - run: | - find $(./tool/go env GOCACHE) -type f -mmin +90 -delete - - name: Save Cache - # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. - if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' - uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: ~/Library/Caches/go-build - key: ${{ runner.os }}-go-test-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - - privileged: - needs: gomod-cache - runs-on: ubuntu-24.04 - container: - image: golang:latest - options: --privileged - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: chown - working-directory: src - run: chown -R $(id -u):$(id -g) $PWD - - name: privileged tests - working-directory: src - run: ./tool/go test ./util/linuxfw ./derp/xdp - - vm: - needs: gomod-cache - runs-on: ["self-hosted", "linux", "vm"] - # VM tests run with some privileges, don't let them run on 3p PRs. - if: github.repository == 'tailscale/tailscale' - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: Run VM tests - working-directory: src - run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2404 - env: - HOME: "/var/lib/ghrunner/home" - TMPDIR: "/tmp" - XDG_CACHE_HOME: "/var/lib/ghrunner/cache" - - cross: # cross-compile checks, build only. - needs: gomod-cache - strategy: - fail-fast: false # don't abort the entire matrix if one element fails - matrix: - include: - # Note: linux/amd64 is not in this matrix, because that goos/goarch is - # tested more exhaustively in the 'test' job above. - - goos: linux - goarch: arm64 - - goos: linux - goarch: "386" # thanks yaml - - goos: linux - goarch: loong64 - - goos: linux - goarch: arm - goarm: "5" - - goos: linux - goarch: arm - goarm: "7" - # macOS - - goos: darwin - goarch: amd64 - - goos: darwin - goarch: arm64 - # Windows - - goos: windows - goarch: amd64 - - goos: windows - goarch: arm64 - # BSDs - - goos: freebsd - goarch: amd64 - - goos: openbsd - goarch: amd64 - - runs-on: ubuntu-24.04 - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: Restore Cache - id: restore-cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - # Note: this is only restoring the build cache. Mod cache is shared amongst - # all jobs in the workflow. - path: | - ~/.cache/go-build - ~\AppData\Local\go-build - key: ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - restore-keys: | - ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}- - ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go- - - name: build all - working-directory: src - run: ./tool/go build ./cmd/... - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - GOARM: ${{ matrix.goarm }} - CGO_ENABLED: "0" - - name: build tests - working-directory: src - run: ./tool/go test -exec=true ./... - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: "0" - - name: Tidy cache - working-directory: src - shell: bash - run: | - find $(go env GOCACHE) -type f -mmin +90 -delete - - name: Save Cache - # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. - if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' - uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - # Note: this is only saving the build cache. Mod cache is shared amongst - # all jobs in the workflow. - path: | - ~/.cache/go-build - ~\AppData\Local\go-build - key: ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - - ios: # similar to cross above, but iOS can't build most of the repo. So, just - # make it build a few smoke packages. - runs-on: ubuntu-24.04 - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: build some - working-directory: src - run: ./tool/go build ./ipn/... ./ssh/tailssh ./wgengine/ ./types/... ./control/controlclient - env: - GOOS: ios - GOARCH: arm64 - - crossmin: # cross-compile for platforms where we only check cmd/tailscale{,d} - needs: gomod-cache - strategy: - fail-fast: false # don't abort the entire matrix if one element fails - matrix: - include: - # Plan9 - - goos: plan9 - goarch: amd64 - # AIX - - goos: aix - goarch: ppc64 - # Solaris - - goos: solaris - goarch: amd64 - # illumos - - goos: illumos - goarch: amd64 - - runs-on: ubuntu-24.04 - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: Restore Cache - id: restore-cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - # Note: this is only restoring the build cache. Mod cache is shared amongst - # all jobs in the workflow. - path: | - ~/.cache/go-build - ~\AppData\Local\go-build - key: ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - restore-keys: | - ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}- - ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go- - - name: build core - working-directory: src - run: ./tool/go build ./cmd/tailscale ./cmd/tailscaled - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - GOARM: ${{ matrix.goarm }} - CGO_ENABLED: "0" - - name: Tidy cache - working-directory: src - shell: bash - run: | - find $(go env GOCACHE) -type f -mmin +90 -delete - - name: Save Cache - # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. - if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' - uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - # Note: this is only saving the build cache. Mod cache is shared amongst - # all jobs in the workflow. - path: | - ~/.cache/go-build - ~\AppData\Local\go-build - key: ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - - android: - # similar to cross above, but android fails to build a few pieces of the - # repo. We should fix those pieces, they're small, but as a stepping stone, - # only test the subset of android that our past smoke test checked. - runs-on: ubuntu-24.04 - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed - # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch - # some Android breakages early. - # TODO(bradfitz): better; see https://github.com/tailscale/tailscale/issues/4482 - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: build some - working-directory: src - run: ./tool/go install ./net/netns ./ipn/ipnlocal ./wgengine/magicsock/ ./wgengine/ ./wgengine/router/ ./wgengine/netstack ./util/dnsname/ ./ipn/ ./net/netmon ./wgengine/router/ ./tailcfg/ ./types/logger/ ./net/dns ./hostinfo ./version ./ssh/tailssh - env: - GOOS: android - GOARCH: arm64 - - wasm: # builds tsconnect, which is the only wasm build we support - runs-on: ubuntu-24.04 - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: Restore Cache - id: restore-cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - # Note: this is only restoring the build cache. Mod cache is shared amongst - # all jobs in the workflow. - path: | - ~/.cache/go-build - ~\AppData\Local\go-build - key: ${{ runner.os }}-js-wasm-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - restore-keys: | - ${{ runner.os }}-js-wasm-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}- - ${{ runner.os }}-js-wasm-go-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-js-wasm-go- - - name: build tsconnect client - working-directory: src - run: ./tool/go build ./cmd/tsconnect/wasm ./cmd/tailscale/cli - env: - GOOS: js - GOARCH: wasm - - name: build tsconnect server - working-directory: src - # Note, no GOOS/GOARCH in env on this build step, we're running a build - # tool that handles the build itself. - run: | - ./tool/go run ./cmd/tsconnect --fast-compression build - ./tool/go run ./cmd/tsconnect --fast-compression build-pkg - - name: Tidy cache - working-directory: src - shell: bash - run: | - find $(go env GOCACHE) -type f -mmin +90 -delete - - name: Save Cache - # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. - if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' - uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - # Note: this is only saving the build cache. Mod cache is shared amongst - # all jobs in the workflow. - path: | - ~/.cache/go-build - ~\AppData\Local\go-build - key: ${{ runner.os }}-js-wasm-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} - - tailscale_go: # Subset of tests that depend on our custom Go toolchain. - runs-on: ubuntu-24.04 - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Set GOMODCACHE env - run: echo "GOMODCACHE=$HOME/.cache/go-mod" >> $GITHUB_ENV - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: test tailscale_go - run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/... - - - fuzz: - # This target periodically breaks (see TS_FUZZ_CURRENTLY_BROKEN at the top - # of the file), so it's more complex than usual: the 'build fuzzers' step - # might fail, and depending on the value of 'TS_FUZZ_CURRENTLY_BROKEN', that - # might or might not be fine. The steps after the build figure out whether - # the success/failure is expected, and appropriately pass/fail the job - # overall accordingly. - # - # Practically, this means that all steps after 'build fuzzers' must have an - # explicit 'if' condition, because the default condition for steps is - # 'success()', meaning "only run this if no previous steps failed". - if: github.event_name == 'pull_request' - runs-on: ubuntu-24.04 - steps: - - name: build fuzzers - id: build - # As of 12 February 2026, this repo doesn't tag releases, so this commit - # hash is just the tip of master. - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@f277aafb36f358582fdb24a41a9a52f2e097a2fd - # continue-on-error makes steps.build.conclusion be 'success' even if - # steps.build.outcome is 'failure'. This means this step does not - # contribute to the job's overall pass/fail evaluation. - continue-on-error: true - with: - oss-fuzz-project-name: 'tailscale' - dry-run: false - language: go - - name: report unexpectedly broken fuzz build - if: steps.build.outcome == 'failure' && env.TS_FUZZ_CURRENTLY_BROKEN != 'true' - run: | - echo "fuzzer build failed, see above for why" - echo "if the failure is due to OSS-Fuzz not being on the latest Go yet," - echo "set TS_FUZZ_CURRENTLY_BROKEN=true in .github/workflows/test.yml" - echo "to temporarily disable fuzzing until OSS-Fuzz works again." - exit 1 - - name: report unexpectedly working fuzz build - if: steps.build.outcome == 'success' && env.TS_FUZZ_CURRENTLY_BROKEN == 'true' - run: | - echo "fuzzer build succeeded, but we expect it to be broken" - echo "please set TS_FUZZ_CURRENTLY_BROKEN=false in .github/workflows/test.yml" - echo "to reenable fuzz testing" - exit 1 - - name: run fuzzers - id: run - # Run the fuzzers whenever they're able to build, even if we're going to - # report a failure because TS_FUZZ_CURRENTLY_BROKEN is set to the wrong - # value. - if: steps.build.outcome == 'success' - # As of 12 February 2026, this repo doesn't tag releases, so this commit - # hash is just the tip of master. - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@f277aafb36f358582fdb24a41a9a52f2e097a2fd - with: - oss-fuzz-project-name: 'tailscale' - fuzz-seconds: 150 - dry-run: false - language: go - - name: Set artifacts_path in env (workaround for actions/upload-artifact#176) - if: steps.run.outcome != 'success' && steps.build.outcome == 'success' - run: | - echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV - - name: upload crash - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - if: steps.run.outcome != 'success' && steps.build.outcome == 'success' - with: - name: artifacts - path: ${{ env.artifacts_path }}/out/artifacts - - depaware: - runs-on: ubuntu-24.04 - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Set GOMODCACHE env - run: echo "GOMODCACHE=$HOME/.cache/go-mod" >> $GITHUB_ENV - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: check depaware - working-directory: src - run: make depaware - - go_generate: - runs-on: ubuntu-24.04 - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: check that 'go generate' is clean - working-directory: src - run: | - pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp') - ./tool/go generate $pkgs - git add -N . # ensure untracked files are noticed - echo - echo - git diff --name-only --exit-code || (echo "The files above need updating. Please run 'go generate'."; exit 1) - - make_tidy: - runs-on: ubuntu-24.04 - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: check that 'make tidy' is clean - working-directory: src - run: | - make tidy - echo - echo - git diff --name-only --exit-code || (echo "Please run 'make tidy'"; exit 1) - - licenses: - runs-on: ubuntu-24.04 - needs: gomod-cache - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: check licenses - working-directory: src - run: | - grep -q TestLicenseHeaders *.go || (echo "Expected a test named TestLicenseHeaders"; exit 1) - ./tool/go test -v -run=TestLicenseHeaders - - staticcheck: - runs-on: ubuntu-24.04 - needs: gomod-cache - name: staticcheck (${{ matrix.name }}) - strategy: - fail-fast: false # don't abort the entire matrix if one element fails - matrix: - include: - - name: "macOS" - goos: "darwin" - goarch: "arm64" - flags: "--with-tags-all=darwin" - - name: "Windows" - goos: "windows" - goarch: "amd64" - flags: "--with-tags-all=windows" - - name: "Linux" - goos: "linux" - goarch: "amd64" - flags: "--with-tags-all=linux" - - name: "Portable (1/4)" - goos: "linux" - goarch: "amd64" - flags: "--without-tags-any=windows,darwin,linux --shard=1/4" - - name: "Portable (2/4)" - goos: "linux" - goarch: "amd64" - flags: "--without-tags-any=windows,darwin,linux --shard=2/4" - - name: "Portable (3/4)" - goos: "linux" - goarch: "amd64" - flags: "--without-tags-any=windows,darwin,linux --shard=3/4" - - name: "Portable (4/4)" - goos: "linux" - goarch: "amd64" - flags: "--without-tags-any=windows,darwin,linux --shard=4/4" - - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: run staticcheck (${{ matrix.name }}) - working-directory: src - run: | - export GOROOT=$(./tool/go env GOROOT) - ./tool/go run -exec \ - "env GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}" \ - honnef.co/go/tools/cmd/staticcheck -- \ - $(./tool/go run ./tool/listpkgs --ignore-3p --goos=${{ matrix.goos }} --goarch=${{ matrix.goarch }} ${{ matrix.flags }} ./...) - - notify_slack: - if: always() - # Any of these jobs failing causes a slack notification. - needs: - - android - - test - - windows - - macos - - vm - - cross - - ios - - wasm - - tailscale_go - - fuzz - - depaware - - go_generate - - make_tidy - - licenses - - staticcheck - runs-on: ubuntu-24.04 - steps: - - name: notify - # Only notify slack for merged commits, not PR failures. - # - # It may be tempting to move this condition into the job's 'if' block, but - # don't: Github only collapses the test list into "everything is OK" if - # all jobs succeeded. A skipped job results in the list staying expanded. - # By having the job always run, but skipping its only step as needed, we - # let the CI output collapse nicely in PRs. - if: failure() && github.event_name == 'push' - uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 - with: - webhook: ${{ secrets.SLACK_WEBHOOK_URL }} - webhook-type: incoming-webhook - payload: | - { - "attachments": [{ - "title": "Failure: ${{ github.workflow }}", - "title_link": "https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks", - "text": "${{ github.repository }}@${{ github.ref_name }}: ", - "fields": [{ "value": ${{ toJson(github.event.head_commit.message) }}, "short": false }], - "footer": "${{ github.event.head_commit.committer.name }} at ${{ github.event.head_commit.timestamp }}", - "color": "danger" - }] - } - - merge_blocker: - if: always() - runs-on: ubuntu-24.04 - needs: - - android - - test - - windows - - macos - - vm - - cross - - ios - - wasm - - tailscale_go - - fuzz - - depaware - - go_generate - - make_tidy - - licenses - - staticcheck - steps: - - name: Decide if change is okay to merge - if: github.event_name != 'push' - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 - with: - jobs: ${{ toJSON(needs) }} - - # This waits on all the jobs which must never fail. Branch protection rules - # enforce these. No flaky tests are allowed in these jobs. (We don't want flaky - # tests anywhere, really, but a flaky test here prevents merging.) - check_mergeability_strict: - if: always() - runs-on: ubuntu-24.04 - needs: - - android - - cross - - crossmin - - ios - - tailscale_go - - depaware - - go_generate - - make_tidy - - licenses - - staticcheck - steps: - - name: Decide if change is okay to merge - if: github.event_name != 'push' - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 - with: - jobs: ${{ toJSON(needs) }} - - check_mergeability: - if: always() - runs-on: ubuntu-24.04 - needs: - - check_mergeability_strict - - test - - windows - - macos - - vm - - wasm - - fuzz - - race-root-integration - - privileged - steps: - - name: Decide if change is okay to merge - if: github.event_name != 'push' - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/update-flake.yml b/.github/workflows/update-flake.yml deleted file mode 100644 index 4c0da7831b5ba..0000000000000 --- a/.github/workflows/update-flake.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: update-flake - -on: - # run action when a change lands in the main branch which updates go.mod. Also - # allow manual triggering. - push: - branches: - - main - paths: - - go.mod - - .github/workflows/update-flake.yml - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - update-flake: - runs-on: ubuntu-latest - - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Run update-flakes - run: ./update-flake.sh - - - name: Get access token - uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 - id: generate-token - with: - # Get token for app: https://github.com/apps/tailscale-code-updater - app-id: ${{ secrets.CODE_UPDATER_APP_ID }} - private-key: ${{ secrets.CODE_UPDATER_APP_PRIVATE_KEY }} - - - name: Send pull request - uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 #v8.1.0 - with: - token: ${{ steps.generate-token.outputs.token }} - author: Flakes Updater - committer: Flakes Updater - branch: flakes - commit-message: "go.mod.sri: update SRI hash for go.mod changes" - title: "go.mod.sri: update SRI hash for go.mod changes" - body: Triggered by ${{ github.repository }}@${{ github.sha }} - signoff: true - delete-branch: true - reviewers: danderson diff --git a/.github/workflows/update-webclient-prebuilt.yml b/.github/workflows/update-webclient-prebuilt.yml deleted file mode 100644 index a3d78e1a5b4a8..0000000000000 --- a/.github/workflows/update-webclient-prebuilt.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: update-webclient-prebuilt - -on: - # manually triggered - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - update-webclient-prebuilt: - runs-on: ubuntu-latest - - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Run go get - run: | - ./tool/go version # build gocross if needed using regular GOPROXY - GOPROXY=direct ./tool/go get github.com/tailscale/web-client-prebuilt - ./tool/go mod tidy - - - name: Get access token - uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 - id: generate-token - with: - # Get token for app: https://github.com/apps/tailscale-code-updater - app-id: ${{ secrets.CODE_UPDATER_APP_ID }} - private-key: ${{ secrets.CODE_UPDATER_APP_PRIVATE_KEY }} - - - name: Send pull request - id: pull-request - uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 #v8.1.0 - with: - token: ${{ steps.generate-token.outputs.token }} - author: OSS Updater - committer: OSS Updater - branch: actions/update-webclient-prebuilt - commit-message: "go.mod: update web-client-prebuilt module" - title: "go.mod: update web-client-prebuilt module" - body: Triggered by ${{ github.repository }}@${{ github.sha }} - signoff: true - delete-branch: true - reviewers: ${{ github.triggering_actor }} - - - name: Summary - if: ${{ steps.pull-request.outputs.pull-request-number }} - run: echo "${{ steps.pull-request.outputs.pull-request-operation}} ${{ steps.pull-request.outputs.pull-request-url }}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/vet.yml b/.github/workflows/vet.yml deleted file mode 100644 index 574852e62beee..0000000000000 --- a/.github/workflows/vet.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: tailscale.com/cmd/vet - -env: - HOME: ${{ github.workspace }} - # GOMODCACHE is the same definition on all OSes. Within the workspace, we use - # toplevel directories "src" (for the checked out source code), and "gomodcache" - # and other caches as siblings to follow. - GOMODCACHE: ${{ github.workspace }}/gomodcache - CMD_GO_USE_GIT_HASH: "true" - -on: - push: - branches: - - main - - "release-branch/*" - paths: - - "**.go" - pull_request: - paths: - - "**.go" - -jobs: - vet: - runs-on: [ self-hosted, linux ] - timeout-minutes: 5 - - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - - name: Build 'go vet' tool - working-directory: src - run: ./tool/go build -o /tmp/vettool tailscale.com/cmd/vet - - - name: Run 'go vet' - working-directory: src - run: ./tool/go vet -vettool=/tmp/vettool tailscale.com/... diff --git a/.github/workflows/webclient.yml b/.github/workflows/webclient.yml deleted file mode 100644 index 1a65eacf56414..0000000000000 --- a/.github/workflows/webclient.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: webclient -on: - workflow_dispatch: - # For now, only run on requests, not the main branches. - pull_request: - paths: - - "client/web/**" - - ".github/workflows/webclient.yml" - - "!**.md" - # TODO(soniaappasamy): enable for main branch after an initial waiting period. - #push: - # branches: - # - main - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - webclient: - runs-on: ubuntu-latest - - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Install deps - run: ./tool/yarn --cwd client/web - - name: Run lint - run: ./tool/yarn --cwd client/web run --silent lint - - name: Run test - run: ./tool/yarn --cwd client/web run --silent test - - name: Run formatter check - run: | - ./tool/yarn --cwd client/web run --silent format-check || ( \ - echo "Run this command on your local device to fix the error:" && \ - echo "" && \ - echo " ./tool/yarn --cwd client/web format" && \ - echo "" && exit 1)