diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index c53823e54..7d0ba1641 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -1685,6 +1685,66 @@ public async Task ApplySetupCodeAsync(string setupCode, SshTunn return new SetupCodeResult(SetupCodeOutcome.Success, GatewayUrl: gatewayUrl); } + private async Task WaitForDeferredSharedTokenRejectionAsync(long generation, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (_stateMachine.Current.OperatorState != RoleConnectionState.Connecting) + return; + if (Interlocked.Read(ref _generation) != generation) + return; + await Task.Delay(50).ConfigureAwait(false); + } + } + + private async Task RestoreRejectedSharedTokenSideEffectsAsync( + GatewayRecord previousRecord, + string? previousActiveId, + bool previousOperatorWasLive, + Func? onGatewayCommitted) + { + string? settingsError = null; + if (onGatewayCommitted is not null) + { + try + { + await onGatewayCommitted(previousRecord, CancellationToken.None) + .ConfigureAwait(false); + } + catch (Exception ex) + { + settingsError = $"Saved settings rollback failed: {ex.Message}"; + } + } + + if (!previousOperatorWasLive) + return settingsError; + + var restoreId = previousActiveId ?? previousRecord.Id; + string? connectionError = null; + try + { + await ConnectCoreAsync(restoreId).ConfigureAwait(false); + if (_stateMachine.Current.OperatorState == RoleConnectionState.Error) + { + connectionError = + "Failed to restore the previous gateway connection: " + + (_stateMachine.Current.OperatorError ?? "Gateway connection failed."); + } + } + catch (Exception ex) + { + connectionError = $"Failed to restore the previous gateway connection: {ex.Message}"; + } + + if (settingsError is null) + return connectionError; + if (connectionError is null) + return settingsError; + return $"{settingsError} {connectionError}"; + } + public Task ConnectWithSharedTokenAsync( string gatewayUrl, string token, @@ -1699,7 +1759,8 @@ public async Task ConnectWithSharedTokenAsync( string gatewayUrl, string token, SshTunnelConfig? sshTunnel, - Func? onGatewayCommitted) + Func? onGatewayCommitted, + Func? onTransactionStarted = null) { ThrowIfDisposed(); @@ -1714,14 +1775,21 @@ public async Task ConnectWithSharedTokenAsync( { using var lifecycleLease = await BeginManualGatewayLifecycleOperationAsync(); await _transitionSemaphore.WaitAsync(); + var transitionLockHeld = true; try { + if (onTransactionStarted is not null) + await onTransactionStarted(CancellationToken.None).ConfigureAwait(false); + var existing = _registry.FindByUrl(gatewayUrl); var recordId = existing?.Id ?? Guid.NewGuid().ToString(); var identityDir = _registry.GetIdentityDirectory(recordId); var hasDurableTokens = DeviceIdentity.HasStoredDeviceTokenForRole(identityDir, "operator", _logger) || DeviceIdentity.HasStoredDeviceTokenForRole(identityDir, "node", _logger); + var hasSetupCredential = + existing != null && + (!string.IsNullOrWhiteSpace(existing.BootstrapToken) || existing.SshTunnel is not null); if (existing != null && hasDurableTokens) { @@ -1860,6 +1928,8 @@ await onGatewayCommitted(record, CancellationToken.None) SetGatewayConnectionIntent(recordId, shouldBeConnected: true); // Disconnect current gateway only after replacement credentials have been validated and persisted. + var previousOperatorWasLive = + _stateMachine.Current.OperatorState == RoleConnectionState.Connected; await DisconnectCoreAsync(); // The replacement shared token was validated above. Preserve durable device tokens; @@ -1869,11 +1939,135 @@ await onGatewayCommitted(record, CancellationToken.None) // Connect to the gateway await ConnectCoreAsync(recordId); - if (_stateMachine.Current.OperatorState == RoleConnectionState.Error) + long? observedGeneration = null; + if (hasSetupCredential && !hasDurableTokens && previousRecord is not null && + _stateMachine.Current.OperatorState == RoleConnectionState.Connecting) + { + // The status handler needs this lock before it can record auth failure. + observedGeneration = Interlocked.Read(ref _generation); + _transitionSemaphore.Release(); + transitionLockHeld = false; + try + { + await WaitForDeferredSharedTokenRejectionAsync( + observedGeneration.Value, + TimeSpan.FromSeconds(15)).ConfigureAwait(false); + } + finally + { + await _transitionSemaphore.WaitAsync().ConfigureAwait(false); + transitionLockHeld = true; + } + } + + if (observedGeneration is long generation && + Interlocked.Read(ref _generation) != generation) { + var stillOurs = string.Equals( + _registry.ActiveGatewayId, + recordId, + StringComparison.Ordinal); return new SetupCodeResult( SetupCodeOutcome.ConnectionFailed, - _stateMachine.Current.OperatorError ?? "Gateway connection failed.", + "The shared-token connection was superseded by a newer gateway connection.", + GatewayUrl: gatewayUrl, + GatewayCommitted: stillOurs); + } + + var handshakeUnfinished = + observedGeneration is long ownedGeneration && + Interlocked.Read(ref _generation) == ownedGeneration && + _stateMachine.Current.OperatorState == RoleConnectionState.Connecting; + if (_stateMachine.Current.OperatorState == RoleConnectionState.Error || handshakeUnfinished) + { + var operatorError = handshakeUnfinished + ? "The shared-token connection did not finish." + : _stateMachine.Current.OperatorError ?? "Gateway connection failed."; + if (hasSetupCredential && !hasDurableTokens && previousRecord is not null) + { + if (handshakeUnfinished) + await DisconnectCoreAsync().ConfigureAwait(false); + + _registry.AddOrUpdate(previousRecord); + _registry.SetActive(previousActiveId); + try + { + _registry.Save(); + gatewayCommitted = false; + } + catch (Exception rollbackException) + { + _registry.AddOrUpdate(record); + _registry.SetActive(recordId); + gatewayCommitted = true; + return new SetupCodeResult( + SetupCodeOutcome.ConnectionFailed, + $"{operatorError} Registry rollback failed; the new gateway remains active: {rollbackException.Message}", + GatewayUrl: gatewayUrl, + GatewayCommitted: true); + } + + var settingsRecord = previousRecord; + if (previousActiveId is not null && + !string.Equals(previousActiveId, previousRecord.Id, StringComparison.Ordinal)) + { + settingsRecord = _registry.GetById(previousActiveId) ?? previousRecord; + } + + var restoreError = await RestoreRejectedSharedTokenSideEffectsAsync( + settingsRecord, + previousActiveId, + previousOperatorWasLive, + onGatewayCommitted).ConfigureAwait(false); + if (previousOperatorWasLive && + _stateMachine.Current.OperatorState == RoleConnectionState.Connecting) + { + var restoreGeneration = Interlocked.Read(ref _generation); + _transitionSemaphore.Release(); + transitionLockHeld = false; + try + { + await WaitForDeferredSharedTokenRejectionAsync( + restoreGeneration, + TimeSpan.FromSeconds(15)).ConfigureAwait(false); + } + finally + { + await _transitionSemaphore.WaitAsync().ConfigureAwait(false); + transitionLockHeld = true; + } + + if (_stateMachine.Current.OperatorState == RoleConnectionState.Error) + { + var connectionError = + "Failed to restore the previous gateway connection: " + + (_stateMachine.Current.OperatorError ?? "Gateway connection failed."); + restoreError = string.IsNullOrWhiteSpace(restoreError) + ? connectionError + : $"{restoreError} {connectionError}"; + } + else if (_stateMachine.Current.OperatorState == RoleConnectionState.Connecting) + { + const string connectionError = + "The previous gateway connection did not finish."; + restoreError = string.IsNullOrWhiteSpace(restoreError) + ? connectionError + : $"{restoreError} {connectionError}"; + } + } + + return new SetupCodeResult( + SetupCodeOutcome.ConnectionFailed, + string.IsNullOrWhiteSpace(restoreError) + ? operatorError + : $"{operatorError} {restoreError}", + GatewayUrl: gatewayUrl, + GatewayCommitted: false); + } + + return new SetupCodeResult( + SetupCodeOutcome.ConnectionFailed, + operatorError, GatewayUrl: gatewayUrl, GatewayCommitted: true); } @@ -1883,7 +2077,8 @@ await onGatewayCommitted(record, CancellationToken.None) if (isolatedValidationTunnel is not null) await StopAndDisposeValidationTunnelAsync(isolatedValidationTunnel).ConfigureAwait(false); - _transitionSemaphore.Release(); + if (transitionLockHeld) + _transitionSemaphore.Release(); } return new SetupCodeResult( SetupCodeOutcome.Success, diff --git a/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs b/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs index fc591680b..1e5e82f2a 100644 --- a/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs +++ b/src/OpenClaw.Tray.WinUI/App.CapabilityHandlers.cs @@ -317,13 +317,19 @@ private void WireAppCapabilityHandlers() if (_gatewayDirectConnectService is null) return new { outcome = "ConnectionFailed", error = "Gateway settings service is not initialized", connected = false }; + GatewayDirectConnectService.SharedTokenSettingsAttempt? settingsAttempt = null; var result = await _connectionManager.ConnectWithSharedTokenAsync( gatewayUrl, token, sshTunnel: null, onGatewayCommitted: (record, _) => { - _gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record); + _gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record, settingsAttempt!); + return Task.CompletedTask; + }, + onTransactionStarted: _ => + { + settingsAttempt = _gatewayDirectConnectService.CaptureSharedTokenSettingsAttempt(); return Task.CompletedTask; }); return new diff --git a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs index 0f4241657..6169cc32a 100644 --- a/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/GatewayDirectConnectService.cs @@ -36,6 +36,8 @@ internal sealed class GatewayDirectConnectService private readonly Action _reconcileRuntimeTunnel; private readonly IOpenClawLogger _logger; private readonly TimeSpan _terminalTimeout; + private ConnectionSettingsSnapshot? _settingsBeforeCandidate; + private bool _candidateSynchronized; public GatewayDirectConnectService( IGatewayConnectionManager connectionManager, @@ -211,38 +213,158 @@ previousActiveId is not null && } } - public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGateway) + public void BeginSharedTokenSettingsAttempt() + { + _settingsBeforeCandidate = ConnectionSettingsSnapshot.Capture(_settings); + _candidateSynchronized = false; + } + + internal SharedTokenSettingsAttempt CaptureSharedTokenSettingsAttempt() => + new(ConnectionSettingsSnapshot.Capture(_settings)); + + internal void SynchronizeSettingsWithCommittedGateway( + GatewayRecord committedGateway, + SharedTokenSettingsAttempt attempt) { - var active = _registry.GetActive() - ?? throw new InvalidOperationException("The committed gateway is no longer active."); + var active = _registry.GetActive(); + if (active is null) + { + attempt.Snapshot.Restore(_settings); + _reconcileRuntimeTunnel(); + return; + } + if (!string.Equals(active.Id, committedGateway.Id, StringComparison.Ordinal)) { throw new InvalidOperationException( "The committed gateway was superseded before its settings could be synchronized."); } - var previous = ConnectionSettingsSnapshot.Capture(_settings); + try { ApplySettings(committedGateway); _reconcileRuntimeTunnel(); + FinishAttempt(attempt); + return; } catch (Exception ex) { - string? rollbackError = null; try { - previous.Restore(_settings); + ApplySettings(committedGateway); _reconcileRuntimeTunnel(); + FinishAttempt(attempt); + return; } - catch (Exception rollbackException) + catch (Exception recoveryException) { - rollbackError = $" Settings rollback failed: {rollbackException.Message}"; + string? restoreError = null; + try + { + attempt.Snapshot.Restore(_settings); + _reconcileRuntimeTunnel(); + } + catch (Exception restoreException) + { + restoreError = $" Prior settings restore failed: {restoreException.Message}"; + } + + throw new InvalidOperationException( + $"Saved settings are out of sync with the active gateway: {ex.Message} Recovery failed: {recoveryException.Message}{restoreError}", + ex); } + } + } + + private static void FinishAttempt(SharedTokenSettingsAttempt attempt) + { + if (attempt.CandidateSynchronized) + return; + + attempt.CandidateSynchronized = true; + } + + public void SynchronizeSettingsWithCommittedGateway(GatewayRecord committedGateway) + { + var active = _registry.GetActive(); + if (active is null) + { + if (_settingsBeforeCandidate is null) + { + throw new InvalidOperationException("The committed gateway is no longer active."); + } + + _settingsBeforeCandidate.Restore(_settings); + _settingsBeforeCandidate = null; + _candidateSynchronized = false; + _reconcileRuntimeTunnel(); + return; + } + if (!string.Equals(active.Id, committedGateway.Id, StringComparison.Ordinal)) + { throw new InvalidOperationException( - $"Failed to synchronize gateway settings: {ex.Message}{rollbackError}", - ex); + "The committed gateway was superseded before its settings could be synchronized."); + } + + if (_settingsBeforeCandidate is null) + { + _settingsBeforeCandidate = ConnectionSettingsSnapshot.Capture(_settings); + _candidateSynchronized = false; + } + + try + { + ApplySettings(committedGateway); + _reconcileRuntimeTunnel(); + FinishSettingsSynchronization(); + return; } + catch (Exception ex) + { + try + { + ApplySettings(committedGateway); + _reconcileRuntimeTunnel(); + FinishSettingsSynchronization(); + return; + } + catch (Exception recoveryException) + { + string? restoreError = null; + if (_settingsBeforeCandidate is not null) + { + try + { + _settingsBeforeCandidate.Restore(_settings); + _reconcileRuntimeTunnel(); + } + catch (Exception restoreException) + { + restoreError = $" Prior settings restore failed: {restoreException.Message}"; + } + + _settingsBeforeCandidate = null; + _candidateSynchronized = false; + } + + throw new InvalidOperationException( + $"Saved settings are out of sync with the active gateway: {ex.Message} Recovery failed: {recoveryException.Message}{restoreError}", + ex); + } + } + } + + private void FinishSettingsSynchronization() + { + if (_candidateSynchronized) + { + _settingsBeforeCandidate = null; + _candidateSynchronized = false; + return; + } + + _candidateSynchronized = true; } internal static GatewayRecord BuildCandidate( @@ -521,7 +643,16 @@ private static GatewayDirectConnectResult Failed( gatewayCommitted, error); - private sealed record ConnectionSettingsSnapshot( + internal sealed class SharedTokenSettingsAttempt + { + internal SharedTokenSettingsAttempt(ConnectionSettingsSnapshot snapshot) => Snapshot = snapshot; + + internal ConnectionSettingsSnapshot Snapshot { get; } + + internal bool CandidateSynchronized { get; set; } + } + + internal sealed record ConnectionSettingsSnapshot( string GatewayUrl, bool UseSshTunnel, string SshUser, diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index 8ea435be0..f9d2a879b 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -1883,6 +1883,333 @@ public async Task ConnectWithSharedTokenAsync_CommittedCallbackFailureRollsBackR Assert.Empty(_factory.CreatedClients); } + [Fact] + public async Task ConnectWithSharedTokenAsync_RejectedTokenPreservesBootstrapAndSshTunnel() + { + var ssh = new SshTunnelConfig("user", "host.example", 18789, 45678); + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "ws://127.0.0.1:9", + BootstrapToken = "setup-bootstrap", + SshTunnel = ssh, + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + + var result = await _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token").WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + var record = _registry.GetById("gw-setup"); + Assert.Equal("setup-bootstrap", record?.BootstrapToken); + Assert.Equal(ssh, record?.SshTunnel); + Assert.Null(record?.SharedGatewayToken); + Assert.Equal("gw-setup", _registry.ActiveGatewayId); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_RejectedTokenRestoresCommittedSettings() + { + var ssh = new SshTunnelConfig("user", "host.example", 18789, 45678); + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "ws://127.0.0.1:9", + BootstrapToken = "setup-bootstrap", + SshTunnel = ssh, + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + var committed = new List(); + + var result = await _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token", + sshTunnel: null, + onGatewayCommitted: (record, _) => + { + committed.Add(record); + return Task.CompletedTask; + }).WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal(2, committed.Count); + Assert.Equal("rejected-shared-token", committed[0].SharedGatewayToken); + Assert.Null(committed[0].BootstrapToken); + Assert.Equal("setup-bootstrap", committed[1].BootstrapToken); + Assert.Equal(ssh, committed[1].SshTunnel); + Assert.Equal(ssh, _registry.GetById("gw-setup")?.SshTunnel); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_RejectedTokenRestoresPriorLiveConnection() + { + var previousSsh = new SshTunnelConfig("old-user", "old.example", 18789, 45670); + var replacementSsh = new SshTunnelConfig("new-user", "new.example", 18789, 45671); + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-live", + Url = "wss://live.example", + BootstrapToken = "setup-bootstrap", + SshTunnel = previousSsh, + }); + _registry.SetActive("gw-live"); + _resolver.OperatorCredential = new GatewayCredential( + "setup-bootstrap", + IsBootstrapToken: true, + CredentialResolver.SourceBootstrapToken); + var tunnel = new CountingTunnelManager { FailForConfig = replacementSsh }; + using var manager = new GatewayConnectionManager( + _resolver, + _factory, + _registry, + NullLogger.Instance, + tunnelManager: tunnel); + await manager.ConnectAsync("gw-live"); + _factory.CreatedClients[0].SimulateHandshake(); + await WaitUntilAsync( + () => manager.CurrentSnapshot.OperatorState == RoleConnectionState.Connected); + var committed = new List(); + + var result = await manager.ConnectWithSharedTokenAsync( + "wss://live.example", + "rejected-shared-token", + replacementSsh, + (record, _) => + { + committed.Add(record); + return Task.CompletedTask; + }).WaitAsync(TimeSpan.FromSeconds(25)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal(previousSsh, _registry.GetById("gw-live")?.SshTunnel); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-live")?.BootstrapToken); + Assert.Equal(previousSsh, tunnel.ActiveConfig); + Assert.Equal(previousSsh, committed[^1].SshTunnel); + Assert.Equal("ws://localhost:45670", _factory.CreatedGatewayUrls[^1]); + Assert.Equal(RoleConnectionState.Connecting, manager.CurrentSnapshot.OperatorState); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_DeferredAuthFailureRestoresBootstrap() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "wss://deferred.example", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + _resolver.OperatorCredential = new GatewayCredential( + "rejected-shared-token", + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + var committed = new List(); + + var task = _manager.ConnectWithSharedTokenAsync( + "wss://deferred.example", + "rejected-shared-token", + sshTunnel: null, + (record, _) => + { + committed.Add(record); + return Task.CompletedTask; + }); + + await WaitUntilAsync(() => _factory.CreatedClients.Count >= 1); + await Task.Delay(100); + _factory.CreatedClients[^1].SimulateAuthFailed("token mismatch"); + + var result = await task.WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-setup")?.BootstrapToken); + Assert.Null(_registry.GetById("gw-setup")?.SharedGatewayToken); + Assert.Equal("setup-bootstrap", committed[^1].BootstrapToken); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_UnfinishedHandshakeRollsBack() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "wss://deferred.example", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + _resolver.OperatorCredential = new GatewayCredential( + "rejected-shared-token", + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + + var result = await _manager.ConnectWithSharedTokenAsync( + "wss://deferred.example", + "rejected-shared-token", + sshTunnel: null, + (_, _) => Task.CompletedTask); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Contains("did not finish", result.ErrorMessage ?? "", StringComparison.Ordinal); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-setup")?.BootstrapToken); + Assert.Null(_registry.GetById("gw-setup")?.SharedGatewayToken); + Assert.Equal(RoleConnectionState.Idle, _manager.CurrentSnapshot.OperatorState); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_UnfinishedHandshakeReconnectsPriorGateway() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-live", + Url = "wss://live.example", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-live"); + _registry.Save(); + _resolver.OperatorCredential = new GatewayCredential( + "setup-bootstrap", + IsBootstrapToken: true, + CredentialResolver.SourceBootstrapToken); + await _manager.ConnectAsync("gw-live"); + _factory.CreatedClients[0].SimulateHandshake(); + await WaitUntilAsync( + () => _manager.CurrentSnapshot.OperatorState == RoleConnectionState.Connected); + + var result = await _manager.ConnectWithSharedTokenAsync( + "wss://live.example", + "rejected-shared-token", + sshTunnel: null, + (_, _) => Task.CompletedTask); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-live")?.BootstrapToken); + Assert.True(_factory.CreatedClients.Count >= 3); + Assert.Equal("wss://live.example", _factory.CreatedGatewayUrls[^1]); + Assert.Contains( + "The previous gateway connection did not finish.", + result.ErrorMessage ?? "", + StringComparison.Ordinal); + Assert.Equal(RoleConnectionState.Connecting, _manager.CurrentSnapshot.OperatorState); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_NewerGenerationSkipsRollback() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "wss://deferred.example", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-setup"); + _registry.Save(); + _resolver.OperatorCredential = new GatewayCredential( + "rejected-shared-token", + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + + var task = _manager.ConnectWithSharedTokenAsync( + "wss://deferred.example", + "rejected-shared-token", + sshTunnel: null, + (_, _) => Task.CompletedTask); + + await WaitUntilAsync(() => _factory.CreatedClients.Count >= 1); + await Task.Delay(100); + await _manager.DisconnectAsync(); + + var result = await task.WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.Contains("superseded", result.ErrorMessage ?? "", StringComparison.Ordinal); + Assert.Null(_registry.GetById("gw-setup")?.BootstrapToken); + Assert.Equal("rejected-shared-token", _registry.GetById("gw-setup")?.SharedGatewayToken); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_RejectedTokenRestoresPriorActiveGatewaySettings() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-live", + Url = "wss://live.example", + BootstrapToken = "live-bootstrap", + }); + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-setup", + Url = "ws://127.0.0.1:9", + BootstrapToken = "setup-bootstrap", + }); + _registry.SetActive("gw-live"); + _registry.Save(); + var committed = new List(); + + var result = await _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token", + sshTunnel: null, + (record, _) => + { + committed.Add(record); + return Task.CompletedTask; + }); + + Assert.Equal(SetupCodeOutcome.ConnectionFailed, result.Outcome); + Assert.False(result.GatewayCommitted); + Assert.Equal("gw-live", _registry.ActiveGatewayId); + Assert.Equal("setup-bootstrap", _registry.GetById("gw-setup")?.BootstrapToken); + Assert.Null(_registry.GetById("gw-setup")?.SharedGatewayToken); + Assert.Equal("gw-live", committed[^1].Id); + Assert.Equal("live-bootstrap", committed[^1].BootstrapToken); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_SecondTransactionStartsAfterTheFirstHoldsTheLock() + { + var order = new List(); + var firstHolding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var firstEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var first = _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token", + sshTunnel: null, + onGatewayCommitted: null, + onTransactionStarted: async _ => + { + order.Add("first"); + firstEntered.TrySetResult(); + await firstHolding.Task; + }); + await firstEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var second = _manager.ConnectWithSharedTokenAsync( + "ws://127.0.0.1:9", + "rejected-shared-token", + sshTunnel: null, + onGatewayCommitted: null, + onTransactionStarted: _ => + { + order.Add("second"); + return Task.CompletedTask; + }); + await Task.Delay(100); + Assert.Equal(new[] { "first" }, order); + firstHolding.TrySetResult(); + await Task.WhenAll(first, second); + Assert.Equal(new[] { "first", "second" }, order); + } + [Fact] public async Task ConnectWithSharedTokenAsync_PostCommitConnectionFailureReportsCommittedGateway() { diff --git a/tests/OpenClaw.Connection.Tests/SetupCodeFlowTests.cs b/tests/OpenClaw.Connection.Tests/SetupCodeFlowTests.cs index 0a590a278..e08bc6a6a 100644 --- a/tests/OpenClaw.Connection.Tests/SetupCodeFlowTests.cs +++ b/tests/OpenClaw.Connection.Tests/SetupCodeFlowTests.cs @@ -313,7 +313,7 @@ public async Task ConnectWithSharedToken_ClearsStaleBootstrapToken() _registry.SetActive("gw-shared"); var resolver = new CredentialResolver(new FakeIdentityReader()); - var factory = new RecordingClientFactory(); + var factory = new RecordingClientFactory { ReportHandshakeSuccess = true }; var manager = new GatewayConnectionManager( resolver, factory, _registry, NullLogger.Instance); @@ -531,11 +531,12 @@ private sealed class RecordingClientFactory : IGatewayClientFactory { public List Calls { get; } = []; public List CreatedLifecycles { get; } = []; + public bool ReportHandshakeSuccess { get; init; } public IGatewayClientLifecycle Create(string gatewayUrl, GatewayCredential credential, string identityPath, IOpenClawLogger logger) { Calls.Add(new CreateCall(gatewayUrl, credential, identityPath)); - var lifecycle = new FakeLifecycle(); + var lifecycle = new FakeLifecycle { ReportHandshakeSuccess = ReportHandshakeSuccess }; CreatedLifecycles.Add(lifecycle); return lifecycle; } @@ -551,7 +552,20 @@ private sealed class FakeLifecycle : IGatewayClientLifecycle public event EventHandler? StatusChanged; public event EventHandler? AuthenticationFailed; #pragma warning restore CS0067 - public Task ConnectAsync(CancellationToken ct) => Task.CompletedTask; + public bool ReportHandshakeSuccess { get; init; } + + public Task ConnectAsync(CancellationToken ct) + { + if (!ReportHandshakeSuccess) + return Task.CompletedTask; + + _ = Task.Run(async () => + { + await Task.Delay(100); + _client.SimulateHandshakeSucceeded(); + }); + return Task.CompletedTask; + } public void Dispose() { } public void SimulateDeviceTokenReceived(string token, string role, string[]? scopes = null) => _client.SimulateDeviceTokenReceived(token, role, scopes); @@ -561,6 +575,15 @@ private sealed class FakeClient : OpenClawGatewayClient { public FakeClient() : base("ws://fake", "fake-token", NullLogger.Instance) { } + public void SimulateHandshakeSucceeded() + { + var field = typeof(OpenClawGatewayClient).GetField( + nameof(HandshakeSucceeded), + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + var handler = field?.GetValue(this) as EventHandler; + handler?.Invoke(this, EventArgs.Empty); + } + public void SimulateDeviceTokenReceived(string token, string role, string[]? scopes = null) { var field = typeof(OpenClawGatewayClient).GetField( diff --git a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs index 5f6361a58..99d143989 100644 --- a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs @@ -245,8 +245,9 @@ public void CredentialReplacementFlows_DoNotBlindlyClearDeviceTokens() Assert.DoesNotContain("ClearStoredTokens", pageDirectConnect); Assert.DoesNotContain("BeginTransactionalTokenClear", pageDirectConnect); Assert.Contains("BeginTransactionalTokenClear", directConnectService); + Assert.Contains("CaptureSharedTokenSettingsAttempt()", capabilityHandlers); Assert.Contains( - "_gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record)", + "_gatewayDirectConnectService.SynchronizeSettingsWithCommittedGateway(record, settingsAttempt!)", capabilityHandlers); Assert.DoesNotContain("if (result.GatewayCommitted)", capabilityHandlers); } diff --git a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs index 5378c90f6..836350b90 100644 --- a/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs +++ b/tests/OpenClaw.Tray.Tests/GatewayDirectConnectServiceTests.cs @@ -441,6 +441,150 @@ public void SynchronizeSettingsWithActiveGateway_PersistsCommittedGateway() Assert.Equal(1, _tunnelReconcileCount); } + [Fact] + public void SynchronizeSettings_TunnelFailureTwice_RestoresPriorSettings() + { + var active = AddPreviousGateway(); + _settings.GatewayUrl = "wss://rejected.example"; + _settings.SaveOrThrow(); + var service = new GatewayDirectConnectService( + _manager, + _registry, + _settings, + () => throw new InvalidOperationException("tunnel down"), + NullLogger.Instance, + TimeSpan.FromMilliseconds(100)); + + var error = Assert.Throws( + () => service.SynchronizeSettingsWithCommittedGateway(active)); + + Assert.Contains("out of sync", error.Message, StringComparison.Ordinal); + Assert.Equal("wss://rejected.example", _settings.GatewayUrl); + } + + [Fact] + public void SynchronizeSettings_RollbackTunnelFailure_KeepsPreAttemptSnapshot() + { + var prior = AddPreviousGateway(); + var priorUrl = _settings.GatewayUrl; + var calls = 0; + var service = new GatewayDirectConnectService( + _manager, + _registry, + _settings, + () => + { + calls++; + if (calls > 1) + throw new InvalidOperationException("tunnel down"); + }, + NullLogger.Instance, + TimeSpan.FromMilliseconds(100)); + var candidate = prior with { Url = "wss://rejected.example" }; + _registry.AddOrUpdate(candidate); + _registry.Save(); + service.SynchronizeSettingsWithCommittedGateway(candidate); + Assert.Equal("wss://rejected.example", _settings.GatewayUrl); + + _registry.AddOrUpdate(prior); + _registry.SetActive(prior.Id); + _registry.Save(); + var error = Assert.Throws( + () => service.SynchronizeSettingsWithCommittedGateway(prior)); + + Assert.Contains("out of sync", error.Message, StringComparison.Ordinal); + Assert.Equal(priorUrl, _settings.GatewayUrl); + } + + [Fact] + public void SynchronizeSettings_OverlappingAttempts_RestoreTheirOwnSnapshots() + { + var prior = AddPreviousGateway(); + var priorUrl = _settings.GatewayUrl; + var calls = 0; + var service = new GatewayDirectConnectService( + _manager, + _registry, + _settings, + () => + { + calls++; + if (calls > 1) + throw new InvalidOperationException("tunnel down"); + }, + NullLogger.Instance, + TimeSpan.FromMilliseconds(100)); + var first = service.CaptureSharedTokenSettingsAttempt(); + var candidate = prior with { Url = "wss://rejected.example" }; + _registry.AddOrUpdate(candidate); + _registry.Save(); + service.SynchronizeSettingsWithCommittedGateway(candidate, first); + Assert.Equal("wss://rejected.example", _settings.GatewayUrl); + + var second = service.CaptureSharedTokenSettingsAttempt(); + _registry.AddOrUpdate(prior); + _registry.SetActive(prior.Id); + _registry.Save(); + var error = Assert.Throws( + () => service.SynchronizeSettingsWithCommittedGateway(prior, first)); + + Assert.Contains("out of sync", error.Message, StringComparison.Ordinal); + Assert.Equal(priorUrl, _settings.GatewayUrl); + _ = second; + } + + [Fact] + public void SynchronizeSettings_BeginAttempt_RollbackTunnelFailure_KeepsPreAttemptSnapshot() + { + var prior = AddPreviousGateway(); + var priorUrl = _settings.GatewayUrl; + var calls = 0; + var service = new GatewayDirectConnectService( + _manager, + _registry, + _settings, + () => + { + calls++; + if (calls > 1) + throw new InvalidOperationException("tunnel down"); + }, + NullLogger.Instance, + TimeSpan.FromMilliseconds(100)); + service.BeginSharedTokenSettingsAttempt(); + var candidate = prior with { Url = "wss://rejected.example" }; + _registry.AddOrUpdate(candidate); + _registry.Save(); + service.SynchronizeSettingsWithCommittedGateway(candidate); + Assert.Equal("wss://rejected.example", _settings.GatewayUrl); + + _registry.AddOrUpdate(prior); + _registry.SetActive(prior.Id); + _registry.Save(); + var error = Assert.Throws( + () => service.SynchronizeSettingsWithCommittedGateway(prior)); + + Assert.Contains("out of sync", error.Message, StringComparison.Ordinal); + Assert.Equal(priorUrl, _settings.GatewayUrl); + } + + [Fact] + public void SynchronizeSettings_NoActiveGateway_RestoresSnapshot() + { + var active = AddPreviousGateway(); + var before = _settings.GatewayUrl; + var service = CreateService(); + service.SynchronizeSettingsWithCommittedGateway(active); + Assert.Equal(active.Url, _settings.GatewayUrl); + + _registry.SetActive(null); + _registry.Save(); + service.SynchronizeSettingsWithCommittedGateway(active); + + Assert.Equal(before, _settings.GatewayUrl); + Assert.Null(_registry.ActiveGatewayId); + } + private GatewayDirectConnectService CreateService() => new( _manager,