From 6a55e327db140b383abca5d0255c6f4c10a8df00 Mon Sep 17 00:00:00 2001 From: Teo Calin Date: Thu, 1 Oct 2026 17:44:44 +0300 Subject: [PATCH 1/3] daemon: a dial timeout to a peer heard from during the dial is not a transport wedge The rx watchdog's partial-wedge detector trips on consecutiveDialTimeouts >= 4, a single global counter bumped by every dial that exhausts its retry budget. A burst of parallel dials to ONE peer whose SYN limiter drops the excess produces dozens of timeouts with no success in between, so a healthy node logged 'transport wedged ... reason=outbound-dial-wedged dial_timeouts=32', re-registered, and could escalate to the supervisor exit. Count a timeout only when the tunnel decrypted nothing from that peer since the dial started (TunnelManager.LastInboundDecrypt, the per-peer timestamp the path watchdog already uses). Timeouts to silent peers count as before, so the 2026-07-15 partial wedge is still detected. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 ++ pkg/daemon/daemon.go | 14 ++- pkg/daemon/rxwatchdog.go | 5 +- pkg/daemon/zz_rx_watchdog_single_peer_test.go | 104 ++++++++++++++++++ 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 pkg/daemon/zz_rx_watchdog_single_peer_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 88f8f37c..d62ef664 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -439,6 +439,15 @@ Detailed per-release notes are on the one per 5s after that, whatever arrives. An idle node still makes one poll per minute, as before. Nodes and servers that are not updated keep working at the old pace for their part. +- **Dial timeouts against a peer that is answering no longer read as a + wedged transport.** The watchdog's `outbound-dial-wedged` check counted + every dial timeout, so a burst of parallel sends to one peer whose SYN + limiter dropped them (`dial_timeouts=32`) made a healthy node re-register + with the beacon and registry, and could escalate to a restart. A timeout + is now counted only when nothing was received from that peer while the + dial was in progress. Timeouts against peers that stay silent count as + before. Applies to encrypted tunnels (the default); with `-encrypt=false` + every timeout still counts. - **Proxy credential hints no longer send an operator who already set `proxy_cmd` off to set it.** When the daemon re-reads its credentials with a proxy command and the proxy still rejects them (407, or Meta Muse's diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 0128113e..7023e3ca 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -451,6 +451,8 @@ type Daemon struct { // it. consecutiveDialTimeouts counts back-to-back dial timeouts // (reset to 0 on any successful dial); a run of them to distinct // peers means our outbound path is wedged, not that one peer is dead. + // A timeout is not counted when the tunnel decrypted anything from + // that peer during the dial (see dialConnectionLocked). lastDialOKNano atomic.Int64 consecutiveDialTimeouts atomic.Uint64 @@ -3897,6 +3899,7 @@ func (d *Daemon) dialConnectionLocked(ctx context.Context, dstAddr protocol.Addr if err := ctx.Err(); err != nil { return nil, err } + dialStart := time.Now() // Enforce outbound port policy: prevent dialing ports blocked by the network if !d.evaluatePortPolicy(PolicyEventDial, dstAddr.Network, dstPort, dstAddr.Node, 0, "") { @@ -4196,7 +4199,16 @@ func (d *Daemon) dialConnectionLocked(ctx context.Context, dstAddr protocol.Addr // Full direct+relay retry budget exhausted with no SYN-ACK. // Feeds the rx-watchdog's partial-wedge detector: a run of // these to distinct peers means our outbound is wedged. - d.consecutiveDialTimeouts.Add(1) + // Not when the peer was heard from while we dialed: it is + // reachable and declined this SYN (its SYN limiter under a + // burst of our own dials, a full accept path), which says + // nothing about our transport. + if last, ok := d.tunnels.LastInboundDecrypt(dstAddr.Node); ok && last.After(dialStart) { + slog.Debug("dial timed out but the peer was heard from during the dial; not counted as a transport wedge", + "peer_node_id", dstAddr.Node, "dst_port", dstPort) + } else { + d.consecutiveDialTimeouts.Add(1) + } if keyMissing { // Not one SYN left this node: the peer never completed // the key exchange. Say so, so callers can tell "busy diff --git a/pkg/daemon/rxwatchdog.go b/pkg/daemon/rxwatchdog.go index 2f9a6d8d..1f7d6671 100644 --- a/pkg/daemon/rxwatchdog.go +++ b/pkg/daemon/rxwatchdog.go @@ -77,7 +77,10 @@ const ( // dead, not the peers. This is the PARTIAL wedge the rx-silence check // misses: rx keeps trickling (keepalives) so silence never trips, but // no new connection can be opened. 4 = at least four back-to-back - // user-visible dial failures with zero successes in between. + // user-visible dial failures with zero successes in between. A timeout + // against a peer we heard from during that dial is not counted: 32 + // parallel sends to one peer whose SYN limiter dropped them once read + // as a wedge and re-registered a healthy node. dialWedgeThreshold = 4 // rxWatchdogSoftMax is how many consecutive soft recoveries run diff --git a/pkg/daemon/zz_rx_watchdog_single_peer_test.go b/pkg/daemon/zz_rx_watchdog_single_peer_test.go new file mode 100644 index 00000000..4abcb665 --- /dev/null +++ b/pkg/daemon/zz_rx_watchdog_single_peer_test.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package daemon + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/pilot-protocol/common/protocol" +) + +// dialTimeoutBurst runs dialWedgeThreshold concurrent dials to one peer that +// never completes a handshake and returns the daemon once every dial has +// spent its full retry budget. during, if set, runs one second into the +// dials. +func dialTimeoutBurst(t *testing.T, before, during func(d *Daemon, peerNode uint32)) *Daemon { + t.Helper() + d := New(Config{}) + t.Cleanup(func() { d.tunnels.Close() }) + + const peerNode uint32 = 0xCAFEBABE + peerConn := addPeerOnDaemon(t, d, peerNode) + t.Cleanup(func() { peerConn.Close() }) + d.setNodeID_testhelper(0x33330000) + if before != nil { + before(d, peerNode) + } + + dst := protocol.Addr{Network: 0, Node: peerNode} + var wg sync.WaitGroup + for i := 0; i < dialWedgeThreshold; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := d.DialConnectionContext(context.Background(), dst, 1001); !errors.Is(err, protocol.ErrDialTimeout) { + t.Errorf("dial: err = %v, want ErrDialTimeout", err) + } + }() + } + if during != nil { + time.Sleep(time.Second) + during(d, peerNode) + } + wg.Wait() + return d +} + +// TestRxWatchdogDialTimeoutsToAnsweringPeerAreNotAWedge: a burst of dials to +// ONE peer times out while that peer is demonstrably reachable — the tunnel +// keeps decrypting packets from it (it answered the dials that got through +// its SYN limiter and dropped the rest). That says something about the peer, +// not about this node's transport, so the watchdog must not start its +// recovery (beacon + registry re-registration, escalating to a process +// exit). Before the fix every such timeout bumped the global +// consecutiveDialTimeouts and four of them tripped "outbound-dial-wedged". +func TestRxWatchdogDialTimeoutsToAnsweringPeerAreNotAWedge(t *testing.T) { + if testing.Short() { + t.Skip("long retry-budget test") + } + t.Parallel() + + now := time.Now() + d := dialTimeoutBurst(t, nil, func(d *Daemon, peerNode uint32) { + d.tunnels.kx.RecordInboundDecrypt(peerNode) + }) + if c := d.consecutiveDialTimeouts.Load(); c != 0 { + t.Errorf("consecutiveDialTimeouts = %d after timeouts to a peer heard from during the dials, want 0", c) + } + + st := &rxWatchdogState{lastProgress: now, recvSeen: atomic.LoadUint64(&d.tunnels.PktsRecv)} + atomic.AddUint64(&d.tunnels.PktsRecv, 25) + if got := d.rxWatchdogTick(st, now.Add(rxWatchdogTickInterval)); got != rxActionProgress { + t.Fatalf("dial timeouts to an answering peer: action = %q, want %q", got, rxActionProgress) + } +} + +// TestRxWatchdogDialTimeoutsToSilentPeerStillCount keeps the partial-wedge +// detector (2026-07-15) intact: when nothing arrived from the peer during +// the dials — a decrypt from before they started is not evidence — every +// timeout counts and the watchdog recovers even though rx is trickling. +func TestRxWatchdogDialTimeoutsToSilentPeerStillCount(t *testing.T) { + if testing.Short() { + t.Skip("long retry-budget test") + } + t.Parallel() + + now := time.Now() + d := dialTimeoutBurst(t, func(d *Daemon, peerNode uint32) { + d.tunnels.kx.SetLastInboundDecryptForTest(peerNode, time.Now().Add(-time.Minute)) + }, nil) + if c := d.consecutiveDialTimeouts.Load(); c != dialWedgeThreshold { + t.Fatalf("consecutiveDialTimeouts = %d after timeouts to a silent peer, want %d", c, dialWedgeThreshold) + } + + st := &rxWatchdogState{lastProgress: now, recvSeen: atomic.LoadUint64(&d.tunnels.PktsRecv)} + atomic.AddUint64(&d.tunnels.PktsRecv, 3) // keepalive trickle from other peers + if got := d.rxWatchdogTick(st, now.Add(rxWatchdogTickInterval)); got != rxActionSoftRecover { + t.Fatalf("dial timeouts to a silent peer: action = %q, want %q", got, rxActionSoftRecover) + } +} From b08b2d8e2b457ebe85b120700c4182c6f2dd53e7 Mon Sep 17 00:00:00 2001 From: Teo Calin Date: Wed, 7 Oct 2026 16:08:44 +0300 Subject: [PATCH 2/3] daemon: evidence that outbound works is another dial succeeding, not traffic from the peer Review of the previous version found a false negative: it skipped a dial timeout whenever the tunnel had decrypted anything from that peer during the dial, and a node whose outbound path is dead still receives the peer's unsolicited 25s keepalives. With every outbound frame dropped the counter stayed at 0 and the watchdog reported progress; without a beacon it never recovered. A timeout is now skipped only when another dial completed while it ran. That dial's SYN reached its peer and the SYN-ACK came back, so the outbound path worked during this dial; the burst case this fixes (32 parallel dials to one peer, some answered by its SYN limiter) has such successes, a dead outbound path cannot. TestRxWatchdogPeerKeepalivesDoNotHideAWedge pins the reviewed case. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 +++--- pkg/daemon/daemon.go | 19 ++++--- pkg/daemon/rxwatchdog.go | 6 +- pkg/daemon/zz_rx_watchdog_single_peer_test.go | 56 ++++++++++++++----- 4 files changed, 64 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d62ef664..6632da98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -439,15 +439,15 @@ Detailed per-release notes are on the one per 5s after that, whatever arrives. An idle node still makes one poll per minute, as before. Nodes and servers that are not updated keep working at the old pace for their part. -- **Dial timeouts against a peer that is answering no longer read as a - wedged transport.** The watchdog's `outbound-dial-wedged` check counted - every dial timeout, so a burst of parallel sends to one peer whose SYN - limiter dropped them (`dial_timeouts=32`) made a healthy node re-register +- **Dial timeouts while other dials succeed no longer read as a wedged + transport.** The watchdog's `outbound-dial-wedged` check counted every + dial timeout, so a burst of parallel sends to one peer whose SYN limiter + dropped some of them (`dial_timeouts=32`) made a healthy node re-register with the beacon and registry, and could escalate to a restart. A timeout - is now counted only when nothing was received from that peer while the - dial was in progress. Timeouts against peers that stay silent count as - before. Applies to encrypted tunnels (the default); with `-encrypt=false` - every timeout still counts. + is no longer counted when another dial completed while it ran: that dial's + SYN-ACK shows the outbound path was working. Traffic merely received from + the peer does not count as evidence, because a node whose outbound path is + dead still receives its peers' keepalives. - **Proxy credential hints no longer send an operator who already set `proxy_cmd` off to set it.** When the daemon re-reads its credentials with a proxy command and the proxy still rejects them (407, or Meta Muse's diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 7023e3ca..ca8d5c60 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -451,8 +451,8 @@ type Daemon struct { // it. consecutiveDialTimeouts counts back-to-back dial timeouts // (reset to 0 on any successful dial); a run of them to distinct // peers means our outbound path is wedged, not that one peer is dead. - // A timeout is not counted when the tunnel decrypted anything from - // that peer during the dial (see dialConnectionLocked). + // A timeout is not counted when another dial succeeded while it ran + // (see dialConnectionLocked). lastDialOKNano atomic.Int64 consecutiveDialTimeouts atomic.Uint64 @@ -4199,12 +4199,15 @@ func (d *Daemon) dialConnectionLocked(ctx context.Context, dstAddr protocol.Addr // Full direct+relay retry budget exhausted with no SYN-ACK. // Feeds the rx-watchdog's partial-wedge detector: a run of // these to distinct peers means our outbound is wedged. - // Not when the peer was heard from while we dialed: it is - // reachable and declined this SYN (its SYN limiter under a - // burst of our own dials, a full accept path), which says - // nothing about our transport. - if last, ok := d.tunnels.LastInboundDecrypt(dstAddr.Node); ok && last.After(dialStart) { - slog.Debug("dial timed out but the peer was heard from during the dial; not counted as a transport wedge", + // Not when another dial completed while this one ran: that + // SYN reached its peer and the SYN-ACK came back, so our + // outbound path worked during this dial and this one was + // declined (a peer's SYN limiter under a burst of our own + // dials, a full accept path). Traffic merely received from + // the peer is no evidence — its keepalives still arrive + // when it is our outbound path that is dead. + if d.lastDialOKNano.Load() > dialStart.UnixNano() { + slog.Debug("dial timed out while another dial succeeded; not counted as a transport wedge", "peer_node_id", dstAddr.Node, "dst_port", dstPort) } else { d.consecutiveDialTimeouts.Add(1) diff --git a/pkg/daemon/rxwatchdog.go b/pkg/daemon/rxwatchdog.go index 1f7d6671..95f0cc63 100644 --- a/pkg/daemon/rxwatchdog.go +++ b/pkg/daemon/rxwatchdog.go @@ -78,9 +78,9 @@ const ( // misses: rx keeps trickling (keepalives) so silence never trips, but // no new connection can be opened. 4 = at least four back-to-back // user-visible dial failures with zero successes in between. A timeout - // against a peer we heard from during that dial is not counted: 32 - // parallel sends to one peer whose SYN limiter dropped them once read - // as a wedge and re-registered a healthy node. + // during which another dial succeeded is not counted: 32 parallel sends + // to one peer whose SYN limiter dropped some of them once read as a + // wedge and re-registered a healthy node. dialWedgeThreshold = 4 // rxWatchdogSoftMax is how many consecutive soft recoveries run diff --git a/pkg/daemon/zz_rx_watchdog_single_peer_test.go b/pkg/daemon/zz_rx_watchdog_single_peer_test.go index 4abcb665..713ca184 100644 --- a/pkg/daemon/zz_rx_watchdog_single_peer_test.go +++ b/pkg/daemon/zz_rx_watchdog_single_peer_test.go @@ -49,15 +49,16 @@ func dialTimeoutBurst(t *testing.T, before, during func(d *Daemon, peerNode uint return d } -// TestRxWatchdogDialTimeoutsToAnsweringPeerAreNotAWedge: a burst of dials to -// ONE peer times out while that peer is demonstrably reachable — the tunnel -// keeps decrypting packets from it (it answered the dials that got through -// its SYN limiter and dropped the rest). That says something about the peer, -// not about this node's transport, so the watchdog must not start its -// recovery (beacon + registry re-registration, escalating to a process -// exit). Before the fix every such timeout bumped the global -// consecutiveDialTimeouts and four of them tripped "outbound-dial-wedged". -func TestRxWatchdogDialTimeoutsToAnsweringPeerAreNotAWedge(t *testing.T) { +// TestRxWatchdogDialTimeoutsWhileAnotherDialSucceedsAreNotAWedge: a burst of +// dials times out while another dial completes — its SYN reached the peer +// and the SYN-ACK came back, so this node's outbound path demonstrably +// worked (a peer's SYN limiter answered some of our burst and dropped the +// rest). That says something about the peer, not about this node's +// transport, so the watchdog must not start its recovery (beacon and +// registry re-registration, escalating to a process exit). Before the fix +// every such timeout bumped the global consecutiveDialTimeouts and four of +// them tripped "outbound-dial-wedged". +func TestRxWatchdogDialTimeoutsWhileAnotherDialSucceedsAreNotAWedge(t *testing.T) { if testing.Short() { t.Skip("long retry-budget test") } @@ -65,22 +66,49 @@ func TestRxWatchdogDialTimeoutsToAnsweringPeerAreNotAWedge(t *testing.T) { now := time.Now() d := dialTimeoutBurst(t, nil, func(d *Daemon, peerNode uint32) { - d.tunnels.kx.RecordInboundDecrypt(peerNode) + // What a dial that completes does (dialConnectionLocked). + d.lastDialOKNano.Store(time.Now().UnixNano()) + d.consecutiveDialTimeouts.Store(0) }) if c := d.consecutiveDialTimeouts.Load(); c != 0 { - t.Errorf("consecutiveDialTimeouts = %d after timeouts to a peer heard from during the dials, want 0", c) + t.Errorf("consecutiveDialTimeouts = %d after timeouts during which another dial succeeded, want 0", c) } st := &rxWatchdogState{lastProgress: now, recvSeen: atomic.LoadUint64(&d.tunnels.PktsRecv)} atomic.AddUint64(&d.tunnels.PktsRecv, 25) if got := d.rxWatchdogTick(st, now.Add(rxWatchdogTickInterval)); got != rxActionProgress { - t.Fatalf("dial timeouts to an answering peer: action = %q, want %q", got, rxActionProgress) + t.Fatalf("dial timeouts while another dial succeeded: action = %q, want %q", got, rxActionProgress) + } +} + +// TestRxWatchdogPeerKeepalivesDoNotHideAWedge: a node whose outbound path is +// dead still receives its peers' keepalives, so traffic from the dialled peer +// during the dials is not evidence that the transport works. An earlier +// version of this fix treated it as such, and a node with every outbound +// frame dropped never reached the threshold. +func TestRxWatchdogPeerKeepalivesDoNotHideAWedge(t *testing.T) { + if testing.Short() { + t.Skip("long retry-budget test") + } + t.Parallel() + + now := time.Now() + d := dialTimeoutBurst(t, nil, func(d *Daemon, peerNode uint32) { + d.tunnels.kx.RecordInboundDecrypt(peerNode) // the peer's keepalive arrives + }) + if c := d.consecutiveDialTimeouts.Load(); c != dialWedgeThreshold { + t.Fatalf("consecutiveDialTimeouts = %d with only the peer's keepalives arriving, want %d", c, dialWedgeThreshold) + } + + st := &rxWatchdogState{lastProgress: now, recvSeen: atomic.LoadUint64(&d.tunnels.PktsRecv)} + atomic.AddUint64(&d.tunnels.PktsRecv, 3) + if got := d.rxWatchdogTick(st, now.Add(rxWatchdogTickInterval)); got != rxActionSoftRecover { + t.Fatalf("outbound dead, keepalives arriving: action = %q, want %q", got, rxActionSoftRecover) } } // TestRxWatchdogDialTimeoutsToSilentPeerStillCount keeps the partial-wedge -// detector (2026-07-15) intact: when nothing arrived from the peer during -// the dials — a decrypt from before they started is not evidence — every +// detector (2026-07-15) intact: with no dial succeeding meanwhile, every // timeout counts and the watchdog recovers even though rx is trickling. func TestRxWatchdogDialTimeoutsToSilentPeerStillCount(t *testing.T) { if testing.Short() { From 01714069c9ec47a33a020ea2218207737cf376f7 Mon Sep 17 00:00:00 2001 From: Teo Calin Date: Wed, 7 Oct 2026 17:02:39 +0300 Subject: [PATCH 3/3] daemon: count completed dials instead of comparing wall-clock times Review fixes for the dial-wedge rule. - Whether another dial completed while a timed-out dial ran is now decided from a counter of completed dials, read when the dial starts. Comparing UnixNano timestamps followed wall-clock steps: a clock stepped back after a success hid every timeout for the size of the step. - The test for a dial completing during the burst now completes a real dial (its SYN answered with a SYN-ACK) instead of writing the field itself, and a new test checks that a dial completed before the burst does not hide it. Both fail against the matching mutant. - The keepalive-era helper and file name are gone. Co-Authored-By: Claude Opus 5.5 --- pkg/daemon/daemon.go | 10 ++- ...t.go => zz_rx_watchdog_dial_wedge_test.go} | 61 ++++++++++++++++--- 2 files changed, 60 insertions(+), 11 deletions(-) rename pkg/daemon/{zz_rx_watchdog_single_peer_test.go => zz_rx_watchdog_dial_wedge_test.go} (71%) diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index ca8d5c60..f25fa88b 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -452,8 +452,11 @@ type Daemon struct { // (reset to 0 on any successful dial); a run of them to distinct // peers means our outbound path is wedged, not that one peer is dead. // A timeout is not counted when another dial succeeded while it ran - // (see dialConnectionLocked). + // (see dialConnectionLocked): dialOKSeq counts completed dials, and a + // dial compares it with the count it saw when it started. A counter, + // not lastDialOKNano, so a wall-clock step cannot hide timeouts. lastDialOKNano atomic.Int64 + dialOKSeq atomic.Uint64 consecutiveDialTimeouts atomic.Uint64 startTime time.Time @@ -3899,7 +3902,7 @@ func (d *Daemon) dialConnectionLocked(ctx context.Context, dstAddr protocol.Addr if err := ctx.Err(); err != nil { return nil, err } - dialStart := time.Now() + dialsOKAtStart := d.dialOKSeq.Load() // Enforce outbound port policy: prevent dialing ports blocked by the network if !d.evaluatePortPolicy(PolicyEventDial, dstAddr.Network, dstPort, dstAddr.Node, 0, "") { @@ -4091,6 +4094,7 @@ func (d *Daemon) dialConnectionLocked(ctx context.Context, dstAddr protocol.Addr // A completed handshake proves the outbound path is alive — // clears the rx-watchdog's partial-wedge signal. d.lastDialOKNano.Store(time.Now().UnixNano()) + d.dialOKSeq.Add(1) d.consecutiveDialTimeouts.Store(0) return conn, nil, true } @@ -4206,7 +4210,7 @@ func (d *Daemon) dialConnectionLocked(ctx context.Context, dstAddr protocol.Addr // dials, a full accept path). Traffic merely received from // the peer is no evidence — its keepalives still arrive // when it is our outbound path that is dead. - if d.lastDialOKNano.Load() > dialStart.UnixNano() { + if d.dialOKSeq.Load() != dialsOKAtStart { slog.Debug("dial timed out while another dial succeeded; not counted as a transport wedge", "peer_node_id", dstAddr.Node, "dst_port", dstPort) } else { diff --git a/pkg/daemon/zz_rx_watchdog_single_peer_test.go b/pkg/daemon/zz_rx_watchdog_dial_wedge_test.go similarity index 71% rename from pkg/daemon/zz_rx_watchdog_single_peer_test.go rename to pkg/daemon/zz_rx_watchdog_dial_wedge_test.go index 713ca184..beb65498 100644 --- a/pkg/daemon/zz_rx_watchdog_single_peer_test.go +++ b/pkg/daemon/zz_rx_watchdog_dial_wedge_test.go @@ -5,6 +5,7 @@ package daemon import ( "context" "errors" + "net" "sync" "sync/atomic" "testing" @@ -15,8 +16,8 @@ import ( // dialTimeoutBurst runs dialWedgeThreshold concurrent dials to one peer that // never completes a handshake and returns the daemon once every dial has -// spent its full retry budget. during, if set, runs one second into the -// dials. +// spent its full retry budget. before, if set, runs before the dials start; +// during, if set, one second into them. func dialTimeoutBurst(t *testing.T, before, during func(d *Daemon, peerNode uint32)) *Daemon { t.Helper() d := New(Config{}) @@ -66,9 +67,7 @@ func TestRxWatchdogDialTimeoutsWhileAnotherDialSucceedsAreNotAWedge(t *testing.T now := time.Now() d := dialTimeoutBurst(t, nil, func(d *Daemon, peerNode uint32) { - // What a dial that completes does (dialConnectionLocked). - d.lastDialOKNano.Store(time.Now().UnixNano()) - d.consecutiveDialTimeouts.Store(0) + completeDial(t, d, 0xCAFED00D) }) if c := d.consecutiveDialTimeouts.Load(); c != 0 { t.Errorf("consecutiveDialTimeouts = %d after timeouts during which another dial succeeded, want 0", c) @@ -107,6 +106,54 @@ func TestRxWatchdogPeerKeepalivesDoNotHideAWedge(t *testing.T) { } } +// TestRxWatchdogDialSuccessBeforeTheBurstDoesNotHideIt: only a dial that +// completes while a timed-out dial ran says the outbound path worked during +// it. One that completed earlier says nothing about the dials after it. +func TestRxWatchdogDialSuccessBeforeTheBurstDoesNotHideIt(t *testing.T) { + if testing.Short() { + t.Skip("long retry-budget test") + } + t.Parallel() + + d := dialTimeoutBurst(t, func(d *Daemon, peerNode uint32) { + completeDial(t, d, 0xCAFED00E) + }, nil) + if c := d.consecutiveDialTimeouts.Load(); c != dialWedgeThreshold { + t.Fatalf("consecutiveDialTimeouts = %d after timeouts that all started after the last success, want %d", c, dialWedgeThreshold) + } +} + +// completeDial dials a second, answering peer and completes the handshake +// the way the network would: its SYN is answered with a SYN-ACK. +func completeDial(t *testing.T, d *Daemon, peer uint32) { + t.Helper() + pc, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { pc.Close() }) + d.AddTunnelPeer(peer, pc.LocalAddr().(*net.UDPAddr)) + + done := make(chan error, 1) + go func() { + _, err := d.DialConnectionContext(context.Background(), protocol.Addr{Network: 0, Node: peer}, 1001) + done <- err + }() + var syn *protocol.Packet + for deadline := time.Now().Add(5 * time.Second); syn == nil && time.Now().Before(deadline); { + if p := recvPacket(t, pc, time.Second); p != nil && p.Flags&protocol.FlagSYN != 0 { + syn = p + } + } + if syn == nil { + t.Fatal("no SYN reached the answering peer") + } + d.handleStreamPacket(streamPacket(protocol.FlagSYN|protocol.FlagACK, peer, d.NodeID(), syn.DstPort, syn.SrcPort, 999, syn.Seq+1)) + if err := <-done; err != nil { + t.Fatalf("dial to the answering peer: %v", err) + } +} + // TestRxWatchdogDialTimeoutsToSilentPeerStillCount keeps the partial-wedge // detector (2026-07-15) intact: with no dial succeeding meanwhile, every // timeout counts and the watchdog recovers even though rx is trickling. @@ -117,9 +164,7 @@ func TestRxWatchdogDialTimeoutsToSilentPeerStillCount(t *testing.T) { t.Parallel() now := time.Now() - d := dialTimeoutBurst(t, func(d *Daemon, peerNode uint32) { - d.tunnels.kx.SetLastInboundDecryptForTest(peerNode, time.Now().Add(-time.Minute)) - }, nil) + d := dialTimeoutBurst(t, nil, nil) if c := d.consecutiveDialTimeouts.Load(); c != dialWedgeThreshold { t.Fatalf("consecutiveDialTimeouts = %d after timeouts to a silent peer, want %d", c, dialWedgeThreshold) }