From e62f1c620b6acc73fc891963d2b9f72192151b03 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Thu, 24 Sep 2026 11:03:25 -0700 Subject: [PATCH 01/11] fix(connection): keep bootstrap and SSH tunnel when a shared token is rejected Connect with a shared token cleared the stored bootstrap token and SSH tunnel before the new token was checked, whenever no device token existed. Validate first, and restore the previous record if the operator connect fails. - Run the existing pre-replacement check when a bootstrap token or tunnel is stored - Roll the registry back when that connect fails and the rollback save succeeds Signed-off-by: Sebastien Tardif --- .../GatewayConnectionManager.cs | 36 +++++++++++++++++-- .../GatewayConnectionManagerTests.cs | 27 ++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index c53823e54..f88d61c1d 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -1722,8 +1722,11 @@ public async Task ConnectWithSharedTokenAsync( var hasDurableTokens = DeviceIdentity.HasStoredDeviceTokenForRole(identityDir, "operator", _logger) || DeviceIdentity.HasStoredDeviceTokenForRole(identityDir, "node", _logger); + var hasSetupCredential = + existing != null && + (!string.IsNullOrWhiteSpace(existing.BootstrapToken) || existing.SshTunnel is not null); - if (existing != null && hasDurableTokens) + if (existing != null && (hasDurableTokens || hasSetupCredential)) { var validationUrl = gatewayUrl; if (sshTunnel is not null) @@ -1871,9 +1874,38 @@ await onGatewayCommitted(record, CancellationToken.None) await ConnectCoreAsync(recordId); if (_stateMachine.Current.OperatorState == RoleConnectionState.Error) { + var operatorError = _stateMachine.Current.OperatorError ?? "Gateway connection failed."; + if (hasSetupCredential && !hasDurableTokens && previousRecord is not null) + { + _registry.AddOrUpdate(previousRecord); + _registry.SetActive(previousActiveId); + try + { + _registry.Save(); + gatewayCommitted = false; + } + catch (Exception rollbackException) + { + _registry.AddOrUpdate(record); + _registry.SetActive(recordId); + gatewayCommitted = true; + return new SetupCodeResult( + SetupCodeOutcome.ConnectionFailed, + $"{operatorError} Registry rollback failed; the new gateway remains active: {rollbackException.Message}", + GatewayUrl: gatewayUrl, + GatewayCommitted: true); + } + + return new SetupCodeResult( + SetupCodeOutcome.ConnectionFailed, + operatorError, + GatewayUrl: gatewayUrl, + GatewayCommitted: false); + } + return new SetupCodeResult( SetupCodeOutcome.ConnectionFailed, - _stateMachine.Current.OperatorError ?? "Gateway connection failed.", + operatorError, GatewayUrl: gatewayUrl, GatewayCommitted: true); } diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index 8ea435be0..4b304d3d6 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -1883,6 +1883,33 @@ public async Task ConnectWithSharedTokenAsync_CommittedCallbackFailureRollsBackR Assert.Empty(_factory.CreatedClients); } + [Fact] + public async Task ConnectWithSharedTokenAsync_RejectedTokenPreservesBootstrapAndSshTunnel() + { + var ssh = new SshTunnelConfig("user", "host.example", 18789, 45678); + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "ws://127.0.0.1:9", + BootstrapToken = "setup-bootstrap", + SshTunnel = ssh, + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + + var result = await _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token").WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + var record = _registry.GetById("gw-setup"); + Assert.Equal("setup-bootstrap", record?.BootstrapToken); + Assert.Equal(ssh, record?.SshTunnel); + Assert.Null(record?.SharedGatewayToken); + Assert.Equal("gw-setup", _registry.ActiveGatewayId); + } + [Fact] public async Task ConnectWithSharedTokenAsync_PostCommitConnectionFailureReportsCommittedGateway() { From 2d27e690f2e5bf3ceb2463aaf0249a66467b23d9 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Thu, 24 Sep 2026 11:29:49 -0700 Subject: [PATCH 02/11] fix(connection): restore setup credentials only after a failed connect A live pre-check on every stored bootstrap token rejected a normal shared-token save before the connection manager could connect. Keep that save path, and roll the registry back only when the operator connect fails. Signed-off-by: Sebastien Tardif --- src/OpenClaw.Connection/GatewayConnectionManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index f88d61c1d..a24049a10 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -1726,7 +1726,7 @@ public async Task ConnectWithSharedTokenAsync( existing != null && (!string.IsNullOrWhiteSpace(existing.BootstrapToken) || existing.SshTunnel is not null); - if (existing != null && (hasDurableTokens || hasSetupCredential)) + if (existing != null && hasDurableTokens) { var validationUrl = gatewayUrl; if (sshTunnel is not null) From 5ceb876b0a616f2c9a06bdf897a82169e6bdac36 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Fri, 25 Sep 2026 13:53:56 -0700 Subject: [PATCH 03/11] fix(connection): roll back a shared token rejected after the handshake ConnectWithSharedTokenAsync treated a still-connecting operator as success, so a gateway auth failure that arrived later left the rejected token committed. When a bootstrap or SSH setup credential is at risk, wait for that failure, then restore saved settings and reconnect a connection that was already live. Connection tests: 804 passed, 1 skipped. Shared tests: 4107 passed, 32 skipped. Tray tests: 3067 passed, 5 failed on the LF source-contract mismatch tracked in #1518. Signed-off-by: Sebastien Tardif --- .../GatewayConnectionManager.cs | 95 ++++++++++++- .../GatewayConnectionManagerTests.cs | 125 ++++++++++++++++++ 2 files changed, 218 insertions(+), 2 deletions(-) diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index a24049a10..fa3d52047 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -1685,6 +1685,66 @@ public async Task ApplySetupCodeAsync(string setupCode, SshTunn return new SetupCodeResult(SetupCodeOutcome.Success, GatewayUrl: gatewayUrl); } + private async Task WaitForDeferredSharedTokenRejectionAsync(long generation, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (_stateMachine.Current.OperatorState != RoleConnectionState.Connecting) + return; + if (Interlocked.Read(ref _generation) != generation) + return; + await Task.Delay(50).ConfigureAwait(false); + } + } + + private async Task RestoreRejectedSharedTokenSideEffectsAsync( + GatewayRecord previousRecord, + string? previousActiveId, + bool previousOperatorWasLive, + Func? onGatewayCommitted) + { + string? settingsError = null; + if (onGatewayCommitted is not null) + { + try + { + await onGatewayCommitted(previousRecord, CancellationToken.None) + .ConfigureAwait(false); + } + catch (Exception ex) + { + settingsError = $"Saved settings rollback failed: {ex.Message}"; + } + } + + if (!previousOperatorWasLive) + return settingsError; + + var restoreId = previousActiveId ?? previousRecord.Id; + string? connectionError = null; + try + { + await ConnectCoreAsync(restoreId).ConfigureAwait(false); + if (_stateMachine.Current.OperatorState == RoleConnectionState.Error) + { + connectionError = + "Failed to restore the previous gateway connection: " + + (_stateMachine.Current.OperatorError ?? "Gateway connection failed."); + } + } + catch (Exception ex) + { + connectionError = $"Failed to restore the previous gateway connection: {ex.Message}"; + } + + if (settingsError is null) + return connectionError; + if (connectionError is null) + return settingsError; + return $"{settingsError} {connectionError}"; + } + public Task ConnectWithSharedTokenAsync( string gatewayUrl, string token, @@ -1714,6 +1774,7 @@ public async Task ConnectWithSharedTokenAsync( { using var lifecycleLease = await BeginManualGatewayLifecycleOperationAsync(); await _transitionSemaphore.WaitAsync(); + var transitionLockHeld = true; try { var existing = _registry.FindByUrl(gatewayUrl); @@ -1863,6 +1924,8 @@ await onGatewayCommitted(record, CancellationToken.None) SetGatewayConnectionIntent(recordId, shouldBeConnected: true); // Disconnect current gateway only after replacement credentials have been validated and persisted. + var previousOperatorWasLive = + _stateMachine.Current.OperatorState == RoleConnectionState.Connected; await DisconnectCoreAsync(); // The replacement shared token was validated above. Preserve durable device tokens; @@ -1872,6 +1935,26 @@ await onGatewayCommitted(record, CancellationToken.None) // Connect to the gateway await ConnectCoreAsync(recordId); + if (hasSetupCredential && !hasDurableTokens && previousRecord is not null && + _stateMachine.Current.OperatorState == RoleConnectionState.Connecting) + { + // The status handler needs this lock before it can record auth failure. + var observedGeneration = Interlocked.Read(ref _generation); + _transitionSemaphore.Release(); + transitionLockHeld = false; + try + { + await WaitForDeferredSharedTokenRejectionAsync( + observedGeneration, + TimeSpan.FromSeconds(15)).ConfigureAwait(false); + } + finally + { + await _transitionSemaphore.WaitAsync().ConfigureAwait(false); + transitionLockHeld = true; + } + } + if (_stateMachine.Current.OperatorState == RoleConnectionState.Error) { var operatorError = _stateMachine.Current.OperatorError ?? "Gateway connection failed."; @@ -1896,9 +1979,16 @@ await onGatewayCommitted(record, CancellationToken.None) GatewayCommitted: true); } + var restoreError = await RestoreRejectedSharedTokenSideEffectsAsync( + previousRecord, + previousActiveId, + previousOperatorWasLive, + onGatewayCommitted).ConfigureAwait(false); return new SetupCodeResult( SetupCodeOutcome.ConnectionFailed, - operatorError, + string.IsNullOrWhiteSpace(restoreError) + ? operatorError + : $"{operatorError} {restoreError}", GatewayUrl: gatewayUrl, GatewayCommitted: false); } @@ -1915,7 +2005,8 @@ await onGatewayCommitted(record, CancellationToken.None) if (isolatedValidationTunnel is not null) await StopAndDisposeValidationTunnelAsync(isolatedValidationTunnel).ConfigureAwait(false); - _transitionSemaphore.Release(); + if (transitionLockHeld) + _transitionSemaphore.Release(); } return new SetupCodeResult( SetupCodeOutcome.Success, diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index 4b304d3d6..b1bbafc30 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -1910,6 +1910,131 @@ public async Task ConnectWithSharedTokenAsync_RejectedTokenPreservesBootstrapAnd Assert.Equal("gw-setup", _registry.ActiveGatewayId); } + [Fact] + public async Task ConnectWithSharedTokenAsync_RejectedTokenRestoresCommittedSettings() + { + var ssh = new SshTunnelConfig("user", "host.example", 18789, 45678); + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "ws://127.0.0.1:9", + BootstrapToken = "setup-bootstrap", + SshTunnel = ssh, + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + var committed = new List(); + + var result = await _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token", + sshTunnel: null, + onGatewayCommitted: (record, _) => + { + committed.Add(record); + return Task.CompletedTask; + }).WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal(2, committed.Count); + Assert.Equal("rejected-shared-token", committed[0].SharedGatewayToken); + Assert.Null(committed[0].BootstrapToken); + Assert.Equal("setup-bootstrap", committed[1].BootstrapToken); + Assert.Equal(ssh, committed[1].SshTunnel); + Assert.Equal(ssh, _registry.GetById("gw-setup")?.SshTunnel); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_RejectedTokenRestoresPriorLiveConnection() + { + var previousSsh = new SshTunnelConfig("old-user", "old.example", 18789, 45670); + var replacementSsh = new SshTunnelConfig("new-user", "new.example", 18789, 45671); + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-live", + Url = "wss://live.example", + BootstrapToken = "setup-bootstrap", + SshTunnel = previousSsh, + }); + _registry.SetActive("gw-live"); + _resolver.OperatorCredential = new GatewayCredential( + "setup-bootstrap", + IsBootstrapToken: true, + CredentialResolver.SourceBootstrapToken); + var tunnel = new CountingTunnelManager { FailForConfig = replacementSsh }; + using var manager = new GatewayConnectionManager( + _resolver, + _factory, + _registry, + NullLogger.Instance, + tunnelManager: tunnel); + await manager.ConnectAsync("gw-live"); + _factory.CreatedClients[0].SimulateHandshake(); + await WaitUntilAsync( + () => manager.CurrentSnapshot.OperatorState == RoleConnectionState.Connected); + var committed = new List(); + + var result = await manager.ConnectWithSharedTokenAsync( + "wss://live.example", + "rejected-shared-token", + replacementSsh, + (record, _) => + { + committed.Add(record); + return Task.CompletedTask; + }).WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal(previousSsh, _registry.GetById("gw-live")?.SshTunnel); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-live")?.BootstrapToken); + Assert.Equal(previousSsh, tunnel.ActiveConfig); + Assert.Equal(previousSsh, committed[^1].SshTunnel); + Assert.Equal("ws://localhost:45670", _factory.CreatedGatewayUrls[^1]); + Assert.Equal(RoleConnectionState.Connecting, manager.CurrentSnapshot.OperatorState); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_DeferredAuthFailureRestoresBootstrap() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "wss://deferred.example", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + _resolver.OperatorCredential = new GatewayCredential( + "rejected-shared-token", + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + var committed = new List(); + + var task = _manager.ConnectWithSharedTokenAsync( + "wss://deferred.example", + "rejected-shared-token", + sshTunnel: null, + (record, _) => + { + committed.Add(record); + return Task.CompletedTask; + }); + + await WaitUntilAsync(() => _factory.CreatedClients.Count >= 1); + await Task.Delay(100); + _factory.CreatedClients[^1].SimulateAuthFailed("token mismatch"); + + var result = await task.WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-setup")?.BootstrapToken); + Assert.Null(_registry.GetById("gw-setup")?.SharedGatewayToken); + Assert.Equal("setup-bootstrap", committed[^1].BootstrapToken); + } + [Fact] public async Task ConnectWithSharedTokenAsync_PostCommitConnectionFailureReportsCommittedGateway() { From 41834892e8f186b0951600e2a0066532115486e7 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Fri, 25 Sep 2026 15:45:50 -0700 Subject: [PATCH 04/11] fix(connection): keep a newer gateway when a shared token is rejected late After the deferred auth wait, roll the registry back only when this attempt still owns the connection generation. When another gateway was active, the settings callback receives that gateway's record so saved settings match the restored active id. Connection tests: 806 passed, 1 skipped. Shared tests: 4106 passed and 32 skipped, then the one dispose failure passed alone. Tray tests: 3067 passed, 5 failed on the LF source-contract mismatch tracked in #1518. Signed-off-by: Sebastien Tardif --- .../GatewayConnectionManager.cs | 28 +++++++- .../GatewayConnectionManagerTests.cs | 72 +++++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index fa3d52047..0daaabead 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -1935,17 +1935,18 @@ await onGatewayCommitted(record, CancellationToken.None) // Connect to the gateway await ConnectCoreAsync(recordId); + long? observedGeneration = null; if (hasSetupCredential && !hasDurableTokens && previousRecord is not null && _stateMachine.Current.OperatorState == RoleConnectionState.Connecting) { // The status handler needs this lock before it can record auth failure. - var observedGeneration = Interlocked.Read(ref _generation); + observedGeneration = Interlocked.Read(ref _generation); _transitionSemaphore.Release(); transitionLockHeld = false; try { await WaitForDeferredSharedTokenRejectionAsync( - observedGeneration, + observedGeneration.Value, TimeSpan.FromSeconds(15)).ConfigureAwait(false); } finally @@ -1955,6 +1956,20 @@ await WaitForDeferredSharedTokenRejectionAsync( } } + if (observedGeneration is long generation && + Interlocked.Read(ref _generation) != generation) + { + var stillOurs = string.Equals( + _registry.ActiveGatewayId, + recordId, + StringComparison.Ordinal); + return new SetupCodeResult( + SetupCodeOutcome.ConnectionFailed, + "The shared-token connection was superseded by a newer gateway connection.", + GatewayUrl: gatewayUrl, + GatewayCommitted: stillOurs); + } + if (_stateMachine.Current.OperatorState == RoleConnectionState.Error) { var operatorError = _stateMachine.Current.OperatorError ?? "Gateway connection failed."; @@ -1979,8 +1994,15 @@ await WaitForDeferredSharedTokenRejectionAsync( GatewayCommitted: true); } + var settingsRecord = previousRecord; + if (previousActiveId is not null && + !string.Equals(previousActiveId, previousRecord.Id, StringComparison.Ordinal)) + { + settingsRecord = _registry.GetById(previousActiveId) ?? previousRecord; + } + var restoreError = await RestoreRejectedSharedTokenSideEffectsAsync( - previousRecord, + settingsRecord, previousActiveId, previousOperatorWasLive, onGatewayCommitted).ConfigureAwait(false); diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index b1bbafc30..7c4e613d1 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -2035,6 +2035,78 @@ public async Task ConnectWithSharedTokenAsync_DeferredAuthFailureRestoresBootstr Assert.Equal("setup-bootstrap", committed[^1].BootstrapToken); } + [Fact] + public async Task ConnectWithSharedTokenAsync_NewerGenerationSkipsRollback() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "wss://deferred.example", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + _resolver.OperatorCredential = new GatewayCredential( + "rejected-shared-token", + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + + var task = _manager.ConnectWithSharedTokenAsync( + "wss://deferred.example", + "rejected-shared-token", + sshTunnel: null, + (_, _) => Task.CompletedTask); + + await WaitUntilAsync(() => _factory.CreatedClients.Count >= 1); + await Task.Delay(100); + await _manager.DisconnectAsync(); + + var result = await task.WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.Contains("superseded", result.ErrorMessage ?? "", StringComparison.Ordinal); + Assert.Null(_registry.GetById("gw-setup")?.BootstrapToken); + Assert.Equal("rejected-shared-token", _registry.GetById("gw-setup")?.SharedGatewayToken); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_RejectedTokenRestoresPriorActiveGatewaySettings() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-live", + Url = "wss://live.example", + BootstrapToken = "live-bootstrap", + }); + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "ws://127.0.0.1:9", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-live"); + _registry.Save(); + var committed = new List(); + + var result = await _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token", + sshTunnel: null, + (record, _) => + { + committed.Add(record); + return Task.CompletedTask; + }); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal("gw-live", _registry.ActiveGatewayId); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-setup")?.BootstrapToken); + Assert.Null(_registry.GetById("gw-setup")?.SharedGatewayToken); + Assert.Equal("gw-live", committed[^1].Id); + Assert.Equal("live-bootstrap", committed[^1].BootstrapToken); + } + [Fact] public async Task ConnectWithSharedTokenAsync_PostCommitConnectionFailureReportsCommittedGateway() { From 4c0e3a458ec4d097992899f9e41ea97449bc765d Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Fri, 25 Sep 2026 16:47:49 -0700 Subject: [PATCH 05/11] fix(connection): fail a shared token whose handshake does not finish A shared-token replacement that is still connecting after the wait rolls the rejected token back instead of returning success. If settings or the runtime tunnel fail while applying the restored gateway, the callback retries that gateway and reports an out-of-sync error instead of writing the rejected snapshot back. Connection tests: 807 passed, 1 skipped. Shared tests: 4107 passed, 32 skipped. Tray tests: 3068 passed, 5 failed on the LF source-contract mismatch tracked in #1518. Signed-off-by: Sebastien Tardif --- .../GatewayConnectionManager.cs | 10 +++++-- .../Services/GatewayDirectConnectService.cs | 15 ++++------ .../GatewayConnectionManagerTests.cs | 29 +++++++++++++++++++ .../SetupCodeFlowTests.cs | 29 +++++++++++++++++-- .../GatewayDirectConnectServiceTests.cs | 21 ++++++++++++++ 5 files changed, 90 insertions(+), 14 deletions(-) diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index 0daaabead..0a6f4d85c 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -1970,9 +1970,15 @@ await WaitForDeferredSharedTokenRejectionAsync( GatewayCommitted: stillOurs); } - if (_stateMachine.Current.OperatorState == RoleConnectionState.Error) + var handshakeUnfinished = + observedGeneration is long ownedGeneration && + Interlocked.Read(ref _generation) == ownedGeneration && + _stateMachine.Current.OperatorState == RoleConnectionState.Connecting; + if (_stateMachine.Current.OperatorState == RoleConnectionState.Error || handshakeUnfinished) { - var operatorError = _stateMachine.Current.OperatorError ?? "Gateway connection failed."; + var operatorError = handshakeUnfinished + ? "The shared-token connection did not finish." + : _stateMachine.Current.OperatorError ?? "Gateway connection failed."; if (hasSetupCredential && !hasDurableTokens && previousRecord is not null) { _registry.AddOrUpdate(previousRecord); diff --git a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs index 0f4241657..9e4fce6d2 100644 --- a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs @@ -220,7 +220,6 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew throw new InvalidOperationException( "The committed gateway was superseded before its settings could be synchronized."); } - var previous = ConnectionSettingsSnapshot.Capture(_settings); try { ApplySettings(committedGateway); @@ -228,20 +227,18 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew } catch (Exception ex) { - string? rollbackError = null; try { - previous.Restore(_settings); + ApplySettings(committedGateway); _reconcileRuntimeTunnel(); + return; } - catch (Exception rollbackException) + catch (Exception recoveryException) { - rollbackError = $" Settings rollback failed: {rollbackException.Message}"; + throw new InvalidOperationException( + $"Saved settings are out of sync with the active gateway: {ex.Message} Recovery failed: {recoveryException.Message}", + ex); } - - throw new InvalidOperationException( - $"Failed to synchronize gateway settings: {ex.Message}{rollbackError}", - ex); } } diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index 7c4e613d1..b97340c4e 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -2035,6 +2035,35 @@ public async Task ConnectWithSharedTokenAsync_DeferredAuthFailureRestoresBootstr Assert.Equal("setup-bootstrap", committed[^1].BootstrapToken); } + [Fact] + public async Task ConnectWithSharedTokenAsync_UnfinishedHandshakeRollsBack() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "wss://deferred.example", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + _resolver.OperatorCredential = new GatewayCredential( + "rejected-shared-token", + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + + var result = await _manager.ConnectWithSharedTokenAsync( + "wss://deferred.example", + "rejected-shared-token", + sshTunnel: null, + (_, _) => Task.CompletedTask); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Contains("did not finish", result.ErrorMessage ?? "", StringComparison.Ordinal); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-setup")?.BootstrapToken); + Assert.Null(_registry.GetById("gw-setup")?.SharedGatewayToken); + } + [Fact] public async Task ConnectWithSharedTokenAsync_NewerGenerationSkipsRollback() { diff --git a/tests/OpenClaw.Connection.Tests/SetupCodeFlowTests.cs b/tests/OpenClaw.Connection.Tests/SetupCodeFlowTests.cs index 0a590a278..e08bc6a6a 100644 --- a/tests/OpenClaw.Connection.Tests/SetupCodeFlowTests.cs +++ b/tests/OpenClaw.Connection.Tests/SetupCodeFlowTests.cs @@ -313,7 +313,7 @@ public async Task ConnectWithSharedToken_ClearsStaleBootstrapToken() _registry.SetActive("gw-shared"); var resolver = new CredentialResolver(new FakeIdentityReader()); - var factory = new RecordingClientFactory(); + var factory = new RecordingClientFactory { ReportHandshakeSuccess = true }; var manager = new GatewayConnectionManager( resolver, factory, _registry, NullLogger.Instance); @@ -531,11 +531,12 @@ private sealed class RecordingClientFactory : IGatewayClientFactory { public List Calls { get; } = []; public List CreatedLifecycles { get; } = []; + public bool ReportHandshakeSuccess { get; init; } public IGatewayClientLifecycle Create(string gatewayUrl, GatewayCredential credential, string identityPath, IOpenClawLogger logger) { Calls.Add(new CreateCall(gatewayUrl, credential, identityPath)); - var lifecycle = new FakeLifecycle(); + var lifecycle = new FakeLifecycle { ReportHandshakeSuccess = ReportHandshakeSuccess }; CreatedLifecycles.Add(lifecycle); return lifecycle; } @@ -551,7 +552,20 @@ private sealed class FakeLifecycle : IGatewayClientLifecycle public event EventHandler? StatusChanged; public event EventHandler? AuthenticationFailed; #pragma warning restore CS0067 - public Task ConnectAsync(CancellationToken ct) => Task.CompletedTask; + public bool ReportHandshakeSuccess { get; init; } + + public Task ConnectAsync(CancellationToken ct) + { + if (!ReportHandshakeSuccess) + return Task.CompletedTask; + + _ = Task.Run(async () => + { + await Task.Delay(100); + _client.SimulateHandshakeSucceeded(); + }); + return Task.CompletedTask; + } public void Dispose() { } public void SimulateDeviceTokenReceived(string token, string role, string[]? scopes = null) => _client.SimulateDeviceTokenReceived(token, role, scopes); @@ -561,6 +575,15 @@ private sealed class FakeClient : OpenClawGatewayClient { public FakeClient() : base("ws://fake", "fake-token", NullLogger.Instance) { } + public void SimulateHandshakeSucceeded() + { + var field = typeof(OpenClawGatewayClient).GetField( + nameof(HandshakeSucceeded), + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + var handler = field?.GetValue(this) as EventHandler; + handler?.Invoke(this, EventArgs.Empty); + } + public void SimulateDeviceTokenReceived(string token, string role, string[]? scopes = null) { var field = typeof(OpenClawGatewayClient).GetField( diff --git a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs index 5378c90f6..8c1ff9cbe 100644 --- a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs +++ b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs @@ -441,6 +441,27 @@ public void SynchronizeSettingsWithActiveGateway_PersistsCommittedGateway() Assert.Equal(1, _tunnelReconcileCount); } + [Fact] + public void SynchronizeSettings_TunnelFailure_KeepsCommittedGatewaySettings() + { + var active = AddPreviousGateway(); + _settings.GatewayUrl = "wss://rejected.example"; + _settings.SaveOrThrow(); + var service = new GatewayDirectConnectService( + _manager, + _registry, + _settings, + () => throw new InvalidOperationException("tunnel down"), + NullLogger.Instance, + TimeSpan.FromMilliseconds(100)); + + var error = Assert.Throws( + () => service.SynchronizeSettingsWithCommittedGateway(active)); + + Assert.Contains("out of sync", error.Message, StringComparison.Ordinal); + Assert.Equal(active.Url, _settings.GatewayUrl); + } + private GatewayDirectConnectService CreateService() => new( _manager, From af05dfd4d67abf760318f0ebae2411e304f945ed Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Fri, 25 Sep 2026 17:23:59 -0700 Subject: [PATCH 06/11] fix(connection): stop an unfinished shared-token attempt before restore When the handshake wait ends while the operator is still connecting, disconnect that attempt so its generation is cancelled and the operator is Idle before the previous gateway is opened again. The restore then waits for a terminal result. When rollback clears the active gateway, the settings callback restores the snapshot taken before the rejected record was applied. Connection tests: 808 passed, 1 skipped. Shared tests: 4107 passed, 32 skipped. Tray tests: 3069 passed, 5 failed on the LF source-contract mismatch tracked in #1518. Signed-off-by: Sebastien Tardif --- .../GatewayConnectionManager.cs | 40 ++++++++++++++++++ .../Services/GatewayDirectConnectService.cs | 18 +++++++- .../GatewayConnectionManagerTests.cs | 41 ++++++++++++++++++- .../GatewayDirectConnectServiceTests.cs | 17 ++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index 0a6f4d85c..a103daccc 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -1981,6 +1981,9 @@ observedGeneration is long ownedGeneration && : _stateMachine.Current.OperatorError ?? "Gateway connection failed."; if (hasSetupCredential && !hasDurableTokens && previousRecord is not null) { + if (handshakeUnfinished) + await DisconnectCoreAsync().ConfigureAwait(false); + _registry.AddOrUpdate(previousRecord); _registry.SetActive(previousActiveId); try @@ -2012,6 +2015,43 @@ observedGeneration is long ownedGeneration && previousActiveId, previousOperatorWasLive, onGatewayCommitted).ConfigureAwait(false); + if (previousOperatorWasLive && + _stateMachine.Current.OperatorState == RoleConnectionState.Connecting) + { + var restoreGeneration = Interlocked.Read(ref _generation); + _transitionSemaphore.Release(); + transitionLockHeld = false; + try + { + await WaitForDeferredSharedTokenRejectionAsync( + restoreGeneration, + TimeSpan.FromSeconds(15)).ConfigureAwait(false); + } + finally + { + await _transitionSemaphore.WaitAsync().ConfigureAwait(false); + transitionLockHeld = true; + } + + if (_stateMachine.Current.OperatorState == RoleConnectionState.Error) + { + var connectionError = + "Failed to restore the previous gateway connection: " + + (_stateMachine.Current.OperatorError ?? "Gateway connection failed."); + restoreError = string.IsNullOrWhiteSpace(restoreError) + ? connectionError + : $"{restoreError} {connectionError}"; + } + else if (_stateMachine.Current.OperatorState == RoleConnectionState.Connecting) + { + const string connectionError = + "The previous gateway connection did not finish."; + restoreError = string.IsNullOrWhiteSpace(restoreError) + ? connectionError + : $"{restoreError} {connectionError}"; + } + } + return new SetupCodeResult( SetupCodeOutcome.ConnectionFailed, string.IsNullOrWhiteSpace(restoreError) diff --git a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs index 9e4fce6d2..621fbeff6 100644 --- a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs @@ -36,6 +36,7 @@ internal sealed class GatewayDirectConnectService private readonly Action _reconcileRuntimeTunnel; private readonly IOpenClawLogger _logger; private readonly TimeSpan _terminalTimeout; + private ConnectionSettingsSnapshot? _settingsBeforeCandidate; public GatewayDirectConnectService( IGatewayConnectionManager connectionManager, @@ -213,13 +214,26 @@ previousActiveId is not null && public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGateway) { - var active = _registry.GetActive() - ?? throw new InvalidOperationException("The committed gateway is no longer active."); + var active = _registry.GetActive(); + if (active is null) + { + if (_settingsBeforeCandidate is null) + { + throw new InvalidOperationException("The committed gateway is no longer active."); + } + + _settingsBeforeCandidate.Restore(_settings); + _settingsBeforeCandidate = null; + _reconcileRuntimeTunnel(); + return; + } + if (!string.Equals(active.Id, committedGateway.Id, StringComparison.Ordinal)) { throw new InvalidOperationException( "The committed gateway was superseded before its settings could be synchronized."); } + _settingsBeforeCandidate = ConnectionSettingsSnapshot.Capture(_settings); try { ApplySettings(committedGateway); diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index b97340c4e..4c0baf48a 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -1983,7 +1983,7 @@ await WaitUntilAsync( { committed.Add(record); return Task.CompletedTask; - }).WaitAsync(TimeSpan.FromSeconds(10)); + }).WaitAsync(TimeSpan.FromSeconds(25)); Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); Assert.False(result.GatewayCommitted); @@ -2062,6 +2062,45 @@ public async Task ConnectWithSharedTokenAsync_UnfinishedHandshakeRollsBack() Assert.Contains("did not finish", result.ErrorMessage ?? "", StringComparison.Ordinal); Assert.Equal("setup-bootstrap", _registry.GetById("gw-setup")?.BootstrapToken); Assert.Null(_registry.GetById("gw-setup")?.SharedGatewayToken); + Assert.Equal(RoleConnectionState.Idle, _manager.CurrentSnapshot.OperatorState); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_UnfinishedHandshakeReconnectsPriorGateway() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-live", + Url = "wss://live.example", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-live"); + _registry.Save(); + _resolver.OperatorCredential = new GatewayCredential( + "setup-bootstrap", + IsBootstrapToken: true, + CredentialResolver.SourceBootstrapToken); + await _manager.ConnectAsync("gw-live"); + _factory.CreatedClients[0].SimulateHandshake(); + await WaitUntilAsync( + () => _manager.CurrentSnapshot.OperatorState == RoleConnectionState.Connected); + + var result = await _manager.ConnectWithSharedTokenAsync( + "wss://live.example", + "rejected-shared-token", + sshTunnel: null, + (_, _) => Task.CompletedTask); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-live")?.BootstrapToken); + Assert.True(_factory.CreatedClients.Count >= 3); + Assert.Equal("wss://live.example", _factory.CreatedGatewayUrls[^1]); + Assert.Contains( + "The previous gateway connection did not finish.", + result.ErrorMessage ?? "", + StringComparison.Ordinal); + Assert.Equal(RoleConnectionState.Connecting, _manager.CurrentSnapshot.OperatorState); } [Fact] diff --git a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs index 8c1ff9cbe..7ada29174 100644 --- a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs +++ b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs @@ -462,6 +462,23 @@ public void SynchronizeSettings_TunnelFailure_KeepsCommittedGatewaySettings() Assert.Equal(active.Url, _settings.GatewayUrl); } + [Fact] + public void SynchronizeSettings_NoActiveGateway_RestoresSnapshot() + { + var active = AddPreviousGateway(); + var before = _settings.GatewayUrl; + var service = CreateService(); + service.SynchronizeSettingsWithCommittedGateway(active); + Assert.Equal(active.Url, _settings.GatewayUrl); + + _registry.SetActive(null); + _registry.Save(); + service.SynchronizeSettingsWithCommittedGateway(active); + + Assert.Equal(before, _settings.GatewayUrl); + Assert.Null(_registry.ActiveGatewayId); + } + private GatewayDirectConnectService CreateService() => new( _manager, From 73cc973fd0d34b0b971f2da8b40b0c316e5b55ef Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Fri, 25 Sep 2026 18:41:28 -0700 Subject: [PATCH 07/11] fix(tray): restore prior settings when tunnel sync fails twice ApplySettings saves the candidate gateway before runtime tunnel reconciliation. If that reconciliation fails on the retry as well, write the snapshot taken before the apply back to settings and still throw, so the connection manager can roll the registry back without leaving the rejected gateway saved. Tray tests: 3069 passed, 5 failed on the LF source-contract mismatch tracked in #1518. Shared tests: 4107 passed, 32 skipped. Signed-off-by: Sebastien Tardif --- .../Services/GatewayDirectConnectService.cs | 18 +++++++++++++++++- .../GatewayDirectConnectServiceTests.cs | 4 ++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs index 621fbeff6..2e6710bb9 100644 --- a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs @@ -249,8 +249,24 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew } catch (Exception recoveryException) { + string? restoreError = null; + if (_settingsBeforeCandidate is not null) + { + try + { + _settingsBeforeCandidate.Restore(_settings); + _reconcileRuntimeTunnel(); + } + catch (Exception restoreException) + { + restoreError = $" Prior settings restore failed: {restoreException.Message}"; + } + + _settingsBeforeCandidate = null; + } + throw new InvalidOperationException( - $"Saved settings are out of sync with the active gateway: {ex.Message} Recovery failed: {recoveryException.Message}", + $"Saved settings are out of sync with the active gateway: {ex.Message} Recovery failed: {recoveryException.Message}{restoreError}", ex); } } diff --git a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs index 7ada29174..9fe182691 100644 --- a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs +++ b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs @@ -442,7 +442,7 @@ public void SynchronizeSettingsWithActiveGateway_PersistsCommittedGateway() } [Fact] - public void SynchronizeSettings_TunnelFailure_KeepsCommittedGatewaySettings() + public void SynchronizeSettings_TunnelFailureTwice_RestoresPriorSettings() { var active = AddPreviousGateway(); _settings.GatewayUrl = "wss://rejected.example"; @@ -459,7 +459,7 @@ public void SynchronizeSettings_TunnelFailure_KeepsCommittedGatewaySettings() () => service.SynchronizeSettingsWithCommittedGateway(active)); Assert.Contains("out of sync", error.Message, StringComparison.Ordinal); - Assert.Equal(active.Url, _settings.GatewayUrl); + Assert.Equal("wss://rejected.example", _settings.GatewayUrl); } [Fact] From b5c27532d8de19de3acf64f6c65ed90206de373e Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Sat, 26 Sep 2026 07:03:03 -0700 Subject: [PATCH 08/11] fix(tray): keep the pre-attempt settings snapshot through rollback The rollback callback was capturing settings again after the candidate had been saved, so a second tunnel failure wrote the rejected gateway back. Capture the snapshot once per shared-token attempt and reuse it when the rollback callback fails. Tray tests: 3070 passed, 5 failed on the LF source-contract mismatch tracked in #1518. Shared tests: 4107 passed, 32 skipped. Signed-off-by: Sebastien Tardif --- .../App.CapabilityHandlers.cs | 1 + .../Services/GatewayDirectConnectService.cs | 17 +++++++++- .../GatewayDirectConnectServiceTests.cs | 34 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs b/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs index fc591680b..748643aff 100644 --- a/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs +++ b/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs @@ -317,6 +317,7 @@ private void WireAppCapabilityHandlers() if (_gatewayDirectConnectService is null) return new { outcome = "ConnectionFailed", error = "Gateway settings service is not initialized", connected = false }; + _gatewayDirectConnectService.BeginSharedTokenSettingsAttempt(); var result = await _connectionManager.ConnectWithSharedTokenAsync( gatewayUrl, token, diff --git a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs index 2e6710bb9..cd87d8896 100644 --- a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs @@ -212,6 +212,9 @@ previousActiveId is not null && } } + public void BeginSharedTokenSettingsAttempt() => + _settingsBeforeCandidate = ConnectionSettingsSnapshot.Capture(_settings); + public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGateway) { var active = _registry.GetActive(); @@ -233,11 +236,21 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew throw new InvalidOperationException( "The committed gateway was superseded before its settings could be synchronized."); } - _settingsBeforeCandidate = ConnectionSettingsSnapshot.Capture(_settings); + + var capturedThisCall = false; + if (_settingsBeforeCandidate is null) + { + _settingsBeforeCandidate = ConnectionSettingsSnapshot.Capture(_settings); + capturedThisCall = true; + } + try { ApplySettings(committedGateway); _reconcileRuntimeTunnel(); + if (!capturedThisCall) + _settingsBeforeCandidate = null; + return; } catch (Exception ex) { @@ -245,6 +258,8 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew { ApplySettings(committedGateway); _reconcileRuntimeTunnel(); + if (!capturedThisCall) + _settingsBeforeCandidate = null; return; } catch (Exception recoveryException) diff --git a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs index 9fe182691..ade5a9a62 100644 --- a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs +++ b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs @@ -462,6 +462,40 @@ public void SynchronizeSettings_TunnelFailureTwice_RestoresPriorSettings() Assert.Equal("wss://rejected.example", _settings.GatewayUrl); } + [Fact] + public void SynchronizeSettings_RollbackTunnelFailure_KeepsPreAttemptSnapshot() + { + var prior = AddPreviousGateway(); + var priorUrl = _settings.GatewayUrl; + var calls = 0; + var service = new GatewayDirectConnectService( + _manager, + _registry, + _settings, + () => + { + calls++; + if (calls > 1) + throw new InvalidOperationException("tunnel down"); + }, + NullLogger.Instance, + TimeSpan.FromMilliseconds(100)); + var candidate = prior with { Url = "wss://rejected.example" }; + _registry.AddOrUpdate(candidate); + _registry.Save(); + service.SynchronizeSettingsWithCommittedGateway(candidate); + Assert.Equal("wss://rejected.example", _settings.GatewayUrl); + + _registry.AddOrUpdate(prior); + _registry.SetActive(prior.Id); + _registry.Save(); + var error = Assert.Throws( + () => service.SynchronizeSettingsWithCommittedGateway(prior)); + + Assert.Contains("out of sync", error.Message, StringComparison.Ordinal); + Assert.Equal(priorUrl, _settings.GatewayUrl); + } + [Fact] public void SynchronizeSettings_NoActiveGateway_RestoresSnapshot() { From 914ef13b03328681204038fd557f88679d38e913 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Sat, 26 Sep 2026 08:08:51 -0700 Subject: [PATCH 09/11] fix(tray): keep the attempt snapshot after the candidate sync BeginSharedTokenSettingsAttempt captures the snapshot, so the candidate callback must not clear it on success. The rollback callback still uses that snapshot when tunnel reconciliation fails twice. Tray tests: 3071 passed, 5 failed on the LF source-contract mismatch tracked in #1518. Shared tests: 4107 passed, 32 skipped. Signed-off-by: Sebastien Tardif --- .../Services/GatewayDirectConnectService.cs | 29 +++++++++++---- .../GatewayDirectConnectServiceTests.cs | 35 +++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs index cd87d8896..3ada2848c 100644 --- a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs @@ -37,6 +37,7 @@ internal sealed class GatewayDirectConnectService private readonly IOpenClawLogger _logger; private readonly TimeSpan _terminalTimeout; private ConnectionSettingsSnapshot? _settingsBeforeCandidate; + private bool _candidateSynchronized; public GatewayDirectConnectService( IGatewayConnectionManager connectionManager, @@ -212,8 +213,11 @@ previousActiveId is not null && } } - public void BeginSharedTokenSettingsAttempt() => + public void BeginSharedTokenSettingsAttempt() + { _settingsBeforeCandidate = ConnectionSettingsSnapshot.Capture(_settings); + _candidateSynchronized = false; + } public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGateway) { @@ -227,6 +231,7 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew _settingsBeforeCandidate.Restore(_settings); _settingsBeforeCandidate = null; + _candidateSynchronized = false; _reconcileRuntimeTunnel(); return; } @@ -237,19 +242,17 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew "The committed gateway was superseded before its settings could be synchronized."); } - var capturedThisCall = false; if (_settingsBeforeCandidate is null) { _settingsBeforeCandidate = ConnectionSettingsSnapshot.Capture(_settings); - capturedThisCall = true; + _candidateSynchronized = false; } try { ApplySettings(committedGateway); _reconcileRuntimeTunnel(); - if (!capturedThisCall) - _settingsBeforeCandidate = null; + FinishSettingsSynchronization(); return; } catch (Exception ex) @@ -258,8 +261,7 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew { ApplySettings(committedGateway); _reconcileRuntimeTunnel(); - if (!capturedThisCall) - _settingsBeforeCandidate = null; + FinishSettingsSynchronization(); return; } catch (Exception recoveryException) @@ -278,6 +280,7 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew } _settingsBeforeCandidate = null; + _candidateSynchronized = false; } throw new InvalidOperationException( @@ -287,6 +290,18 @@ public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGatew } } + private void FinishSettingsSynchronization() + { + if (_candidateSynchronized) + { + _settingsBeforeCandidate = null; + _candidateSynchronized = false; + return; + } + + _candidateSynchronized = true; + } + internal static GatewayRecord BuildCandidate( GatewayDirectConnectRequest request, GatewayRecord? existing, diff --git a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs index ade5a9a62..d6ec6cb7d 100644 --- a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs +++ b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs @@ -496,6 +496,41 @@ public void SynchronizeSettings_RollbackTunnelFailure_KeepsPreAttemptSnapshot() Assert.Equal(priorUrl, _settings.GatewayUrl); } + [Fact] + public void SynchronizeSettings_BeginAttempt_RollbackTunnelFailure_KeepsPreAttemptSnapshot() + { + var prior = AddPreviousGateway(); + var priorUrl = _settings.GatewayUrl; + var calls = 0; + var service = new GatewayDirectConnectService( + _manager, + _registry, + _settings, + () => + { + calls++; + if (calls > 1) + throw new InvalidOperationException("tunnel down"); + }, + NullLogger.Instance, + TimeSpan.FromMilliseconds(100)); + service.BeginSharedTokenSettingsAttempt(); + var candidate = prior with { Url = "wss://rejected.example" }; + _registry.AddOrUpdate(candidate); + _registry.Save(); + service.SynchronizeSettingsWithCommittedGateway(candidate); + Assert.Equal("wss://rejected.example", _settings.GatewayUrl); + + _registry.AddOrUpdate(prior); + _registry.SetActive(prior.Id); + _registry.Save(); + var error = Assert.Throws( + () => service.SynchronizeSettingsWithCommittedGateway(prior)); + + Assert.Contains("out of sync", error.Message, StringComparison.Ordinal); + Assert.Equal(priorUrl, _settings.GatewayUrl); + } + [Fact] public void SynchronizeSettings_NoActiveGateway_RestoresSnapshot() { From da012c5d792cf32fc920c84448eb8068a4557e24 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Sat, 26 Sep 2026 10:20:58 -0700 Subject: [PATCH 10/11] fix(tray): keep each shared-token snapshot on its own request The settings snapshot lived on the tray service and was captured before the connection manager took its lock, so a second connect could replace it. Each connect now captures its own attempt and passes that object into the settings callback. Shared tests: 4107 passed, 32 skipped. Tray suite: 3071 passed, 6 failed. Five failures are the LF source-contract mismatch tracked in #1518. CredentialReplacementFlows_DoNotBlindlyClearDeviceTokens then passed after its expected call was updated. Signed-off-by: Sebastien Tardif --- .../App.CapabilityHandlers.cs | 4 +- .../Services/GatewayDirectConnectService.cs | 76 ++++++++++++++++++- .../AppRefactorContractTests.cs | 3 +- .../GatewayDirectConnectServiceTests.cs | 37 +++++++++ 4 files changed, 116 insertions(+), 4 deletions(-) diff --git a/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs b/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs index 748643aff..e64364638 100644 --- a/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs +++ b/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs @@ -317,14 +317,14 @@ private void WireAppCapabilityHandlers() if (_gatewayDirectConnectService is null) return new { outcome = "ConnectionFailed", error = "Gateway settings service is not initialized", connected = false }; - _gatewayDirectConnectService.BeginSharedTokenSettingsAttempt(); + var settingsAttempt = _gatewayDirectConnectService.CaptureSharedTokenSettingsAttempt(); var result = await _connectionManager.ConnectWithSharedTokenAsync( gatewayUrl, token, sshTunnel: null, onGatewayCommitted: (record, _) => { - _gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record); + _gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record, settingsAttempt); return Task.CompletedTask; }); return new diff --git a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs index 3ada2848c..6169cc32a 100644 --- a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs @@ -219,6 +219,71 @@ public void BeginSharedTokenSettingsAttempt() _candidateSynchronized = false; } + internal SharedTokenSettingsAttempt CaptureSharedTokenSettingsAttempt() => + new(ConnectionSettingsSnapshot.Capture(_settings)); + + internal void SynchronizeSettingsWithCommittedGateway( + GatewayRecord committedGateway, + SharedTokenSettingsAttempt attempt) + { + var active = _registry.GetActive(); + if (active is null) + { + attempt.Snapshot.Restore(_settings); + _reconcileRuntimeTunnel(); + return; + } + + if (!string.Equals(active.Id, committedGateway.Id, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "The committed gateway was superseded before its settings could be synchronized."); + } + + try + { + ApplySettings(committedGateway); + _reconcileRuntimeTunnel(); + FinishAttempt(attempt); + return; + } + catch (Exception ex) + { + try + { + ApplySettings(committedGateway); + _reconcileRuntimeTunnel(); + FinishAttempt(attempt); + return; + } + catch (Exception recoveryException) + { + string? restoreError = null; + try + { + attempt.Snapshot.Restore(_settings); + _reconcileRuntimeTunnel(); + } + catch (Exception restoreException) + { + restoreError = $" Prior settings restore failed: {restoreException.Message}"; + } + + throw new InvalidOperationException( + $"Saved settings are out of sync with the active gateway: {ex.Message} Recovery failed: {recoveryException.Message}{restoreError}", + ex); + } + } + } + + private static void FinishAttempt(SharedTokenSettingsAttempt attempt) + { + if (attempt.CandidateSynchronized) + return; + + attempt.CandidateSynchronized = true; + } + public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGateway) { var active = _registry.GetActive(); @@ -578,7 +643,16 @@ private static GatewayDirectConnectResult Failed( gatewayCommitted, error); - private sealed record ConnectionSettingsSnapshot( + internal sealed class SharedTokenSettingsAttempt + { + internal SharedTokenSettingsAttempt(ConnectionSettingsSnapshot snapshot) => Snapshot = snapshot; + + internal ConnectionSettingsSnapshot Snapshot { get; } + + internal bool CandidateSynchronized { get; set; } + } + + internal sealed record ConnectionSettingsSnapshot( string GatewayUrl, bool UseSshTunnel, string SshUser, diff --git a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs index 5f6361a58..431d264f8 100644 --- a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs @@ -245,8 +245,9 @@ public void CredentialReplacementFlows_DoNotBlindlyClearDeviceTokens() Assert.DoesNotContain("ClearStoredTokens", pageDirectConnect); Assert.DoesNotContain("BeginTransactionalTokenClear", pageDirectConnect); Assert.Contains("BeginTransactionalTokenClear", directConnectService); + Assert.Contains("CaptureSharedTokenSettingsAttempt()", capabilityHandlers); Assert.Contains( - "_gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record)", + "_gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record, settingsAttempt)", capabilityHandlers); Assert.DoesNotContain("if (result.GatewayCommitted)", capabilityHandlers); } diff --git a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs index d6ec6cb7d..836350b90 100644 --- a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs +++ b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs @@ -496,6 +496,43 @@ public void SynchronizeSettings_RollbackTunnelFailure_KeepsPreAttemptSnapshot() Assert.Equal(priorUrl, _settings.GatewayUrl); } + [Fact] + public void SynchronizeSettings_OverlappingAttempts_RestoreTheirOwnSnapshots() + { + var prior = AddPreviousGateway(); + var priorUrl = _settings.GatewayUrl; + var calls = 0; + var service = new GatewayDirectConnectService( + _manager, + _registry, + _settings, + () => + { + calls++; + if (calls > 1) + throw new InvalidOperationException("tunnel down"); + }, + NullLogger.Instance, + TimeSpan.FromMilliseconds(100)); + var first = service.CaptureSharedTokenSettingsAttempt(); + var candidate = prior with { Url = "wss://rejected.example" }; + _registry.AddOrUpdate(candidate); + _registry.Save(); + service.SynchronizeSettingsWithCommittedGateway(candidate, first); + Assert.Equal("wss://rejected.example", _settings.GatewayUrl); + + var second = service.CaptureSharedTokenSettingsAttempt(); + _registry.AddOrUpdate(prior); + _registry.SetActive(prior.Id); + _registry.Save(); + var error = Assert.Throws( + () => service.SynchronizeSettingsWithCommittedGateway(prior, first)); + + Assert.Contains("out of sync", error.Message, StringComparison.Ordinal); + Assert.Equal(priorUrl, _settings.GatewayUrl); + _ = second; + } + [Fact] public void SynchronizeSettings_BeginAttempt_RollbackTunnelFailure_KeepsPreAttemptSnapshot() { From 8c191b3938aac911dc22fa513d4e38c94d3f4602 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Sat, 26 Sep 2026 11:08:55 -0700 Subject: [PATCH 11/11] fix(connection): capture shared-token settings after the transaction lock The tray captured the settings snapshot before ConnectWithSharedTokenAsync took the transition lock, so a second request could snapshot a baseline from before the first transaction. The capture now runs inside that lock, and the second request cannot start until the first has entered it. Connection tests: 809 passed, 1 skipped. Shared tests: 4107 passed, 32 skipped. Tray tests: 3072 passed, 5 failed on the LF source-contract mismatch tracked in #1518. Signed-off-by: Sebastien Tardif --- .../GatewayConnectionManager.cs | 6 +++- .../App.CapabilityHandlers.cs | 9 +++-- .../GatewayConnectionManagerTests.cs | 35 +++++++++++++++++++ .../AppRefactorContractTests.cs | 2 +- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index a103daccc..7d0ba1641 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -1759,7 +1759,8 @@ public async Task ConnectWithSharedTokenAsync( string gatewayUrl, string token, SshTunnelConfig? sshTunnel, - Func? onGatewayCommitted) + Func? onGatewayCommitted, + Func? onTransactionStarted = null) { ThrowIfDisposed(); @@ -1777,6 +1778,9 @@ public async Task ConnectWithSharedTokenAsync( var transitionLockHeld = true; try { + if (onTransactionStarted is not null) + await onTransactionStarted(CancellationToken.None).ConfigureAwait(false); + var existing = _registry.FindByUrl(gatewayUrl); var recordId = existing?.Id ?? Guid.NewGuid().ToString(); var identityDir = _registry.GetIdentityDirectory(recordId); diff --git a/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs b/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs index e64364638..1e5e82f2a 100644 --- a/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs +++ b/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs @@ -317,14 +317,19 @@ private void WireAppCapabilityHandlers() if (_gatewayDirectConnectService is null) return new { outcome = "ConnectionFailed", error = "Gateway settings service is not initialized", connected = false }; - var settingsAttempt = _gatewayDirectConnectService.CaptureSharedTokenSettingsAttempt(); + GatewayDirectConnectService.SharedTokenSettingsAttempt? settingsAttempt = null; var result = await _connectionManager.ConnectWithSharedTokenAsync( gatewayUrl, token, sshTunnel: null, onGatewayCommitted: (record, _) => { - _gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record, settingsAttempt); + _gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record, settingsAttempt!); + return Task.CompletedTask; + }, + onTransactionStarted: _ => + { + settingsAttempt = _gatewayDirectConnectService.CaptureSharedTokenSettingsAttempt(); return Task.CompletedTask; }); return new diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index 4c0baf48a..f9d2a879b 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -2175,6 +2175,41 @@ public async Task ConnectWithSharedTokenAsync_RejectedTokenRestoresPriorActiveGa Assert.Equal("live-bootstrap", committed[^1].BootstrapToken); } + [Fact] + public async Task ConnectWithSharedTokenAsync_SecondTransactionStartsAfterTheFirstHoldsTheLock() + { + var order = new List(); + var firstHolding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var firstEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var first = _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token", + sshTunnel: null, + onGatewayCommitted: null, + onTransactionStarted: async _ => + { + order.Add("first"); + firstEntered.TrySetResult(); + await firstHolding.Task; + }); + await firstEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var second = _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token", + sshTunnel: null, + onGatewayCommitted: null, + onTransactionStarted: _ => + { + order.Add("second"); + return Task.CompletedTask; + }); + await Task.Delay(100); + Assert.Equal(new[] { "first" }, order); + firstHolding.TrySetResult(); + await Task.WhenAll(first, second); + Assert.Equal(new[] { "first", "second" }, order); + } + [Fact] public async Task ConnectWithSharedTokenAsync_PostCommitConnectionFailureReportsCommittedGateway() { diff --git a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs index 431d264f8..99d143989 100644 --- a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs @@ -247,7 +247,7 @@ public void CredentialReplacementFlows_DoNotBlindlyClearDeviceTokens() Assert.Contains("BeginTransactionalTokenClear", directConnectService); Assert.Contains("CaptureSharedTokenSettingsAttempt()", capabilityHandlers); Assert.Contains( - "_gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record, settingsAttempt)", + "_gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record, settingsAttempt!)", capabilityHandlers); Assert.DoesNotContain("if (result.GatewayCommitted)", capabilityHandlers); }