diff --git a/docs/SETUP_ENGINE_REDESIGN.md b/docs/SETUP_ENGINE_REDESIGN.md index 26ed96f5a..bef5cf577 100644 --- a/docs/SETUP_ENGINE_REDESIGN.md +++ b/docs/SETUP_ENGINE_REDESIGN.md @@ -18,15 +18,16 @@ timing explanation, not an established cause of every refusal. Observed service states differ: local diagnostics captured `activating/auto-restart` with no MainPID, while hosted generic refusals captured an `active/running` unit and a live PID. Neither snapshot establishes the admission-time owner-lease predicate. -`SetupWizardRunner` recognizes only the exact -serving-owner refusal, waits for verified managed endpoint ownership using the -existing bounded provenance probe, and retries the normal CLI restart once. +`SetupWizardRunner` recognizes only the exact serving-owner refusal or the +combination of typed state-database coordinator contention and the exact +restart-intent-recording refusal. It waits for verified managed endpoint +ownership using the existing bounded provenance probe, then retries the normal +guarded CLI restart once. The probe allows up to 30 one-second retry delays, plus probe duration, for `NoListener` and `UnknownListener` tagged `ListenerSnapshotChanged`. Other unknown/conflicting listeners, other restart errors, and a repeated refusal still fail setup. Listener provenance does not prove owner-lease or coordinator -readiness; the retried CLI command retains those guards. Restart-intent recording -contention is a separate failure and is not retried here. There is no direct +readiness; the retried CLI command retains those guards. There is no direct systemd restart fallback or ownership bypass. > **Status note (2026-07-06):** Current default setup includes `WindowsNodeBootstrapContextStep`, which injects Windows-node context into the WSL workspace `AGENTS.md` after onboarding. @@ -298,6 +299,25 @@ public abstract class SetupStep public sealed record StepResult(StepOutcome Outcome, string? Message = null, Exception? Exception = null); ``` +### Headless E2E guarded-restart diagnostics + +The disposable `E2ESetupFixture` runs the same CLI entry point with an internal +failure observer. If the wizard step fails specifically at the guarded +post-wizard Gateway restart, the observer awaits a fixed, read-only +`systemctl --user show` probe **before** the normal owned-fixture rollback. +The resulting `gateway-restart-diagnostic.json` contains only an allowlisted +refusal category and coarse unit/state/PID-presence/start-identity-availability +facts, plus allowlisted service result and bounded exit code/status. Failed or +timed-out probes produce an explicit probe status, not raw +command output. The original setup failure and rollback are unchanged. + +The public Gateway CLI does not expose the rejected owner-lease predicate, +so `ownerPredicate=not_exposed_by_gateway_cli` is intentional. An +`unverified` serving owner must not be interpreted as coordinator contention +without the separately observed typed contention error. This fixture-only +diagnostic neither retries the guarded restart nor preserves the distro after +rollback. + --- ## Key Components diff --git a/src/OpenClaw.SetupEngine/GatewayWizardRestartRecoveryPolicy.cs b/src/OpenClaw.SetupEngine/GatewayWizardRestartRecoveryPolicy.cs index f42a2eaec..60d668ccf 100644 --- a/src/OpenClaw.SetupEngine/GatewayWizardRestartRecoveryPolicy.cs +++ b/src/OpenClaw.SetupEngine/GatewayWizardRestartRecoveryPolicy.cs @@ -15,6 +15,12 @@ internal static class GatewayWizardRestartRecoveryPolicy /// public const string HostedWizardTerminationError = "Error: TUI exited from signal SIGTERM"; + public const string RestartIntentCoordinatorContentionError = + "StateDatabaseCoordinatorContentionError: another OpenClaw process owns state-lifecycle"; + public const string RestartIntentRecordingRefusal = + "GATEWAY_RESTART_PREPARATION_REFUSED: Cannot record restart intent for the serving Gateway. Gateway was not signaled."; + public static readonly TimeSpan RestartIntentContentionRetryDelay = + TimeSpan.FromMilliseconds(500); public static bool IsTerminalRestartCandidate( string? gatewayVersion, @@ -93,6 +99,14 @@ public static bool IsRestartLikeDisconnect(Exception exception) public static bool IsRetryableGatewayStartupDisconnect(int? closeStatusCode) => closeStatusCode == 1013; + public static bool IsRestartIntentCoordinatorContention(string? message) => + message?.Contains( + RestartIntentCoordinatorContentionError, + StringComparison.Ordinal) == true && + message.Contains( + RestartIntentRecordingRefusal, + StringComparison.Ordinal); + public static async Task WaitForExpectedManagedGatewayAsync( Func> inspectAsync, int noListenerRetryCount, diff --git a/src/OpenClaw.SetupEngine/Program.cs b/src/OpenClaw.SetupEngine/Program.cs index ea51d5f01..26d665221 100644 --- a/src/OpenClaw.SetupEngine/Program.cs +++ b/src/OpenClaw.SetupEngine/Program.cs @@ -39,6 +39,19 @@ public static class Program ]); public static async Task Main(string[] args) + => await RunAsync(args, null); + + internal static Task RunWithFailureDiagnosticAsync( + string[] args, + Func beforeFailureRollback) + { + ArgumentNullException.ThrowIfNull(beforeFailureRollback); + return RunAsync(args, beforeFailureRollback); + } + + private static async Task RunAsync( + string[] args, + Func? beforeFailureRollback) { Console.WriteLine($"OpenClaw Setup Engine {AppVersionInfo.DisplayVersion}"); Console.WriteLine("─────────────────────────────"); @@ -287,7 +300,7 @@ public static async Task Main(string[] args) steps = BuildSteps(config); } - var pipeline = new SetupPipeline(steps); + var pipeline = new SetupPipeline(steps, rollbackOnFailureOverride: null, beforeFailureRollback); pipeline.StepProgress += (_, e) => { diff --git a/src/OpenClaw.SetupEngine/SetupPipeline.cs b/src/OpenClaw.SetupEngine/SetupPipeline.cs index 8e43789d8..969faaa63 100644 --- a/src/OpenClaw.SetupEngine/SetupPipeline.cs +++ b/src/OpenClaw.SetupEngine/SetupPipeline.cs @@ -136,13 +136,23 @@ public sealed class SetupPipeline private readonly List _steps; private readonly List _completedSteps = new(); private readonly bool? _rollbackOnFailureOverride; + private readonly Func? _beforeFailureRollback; public event EventHandler? StepProgress; public SetupPipeline(IEnumerable steps, bool? rollbackOnFailureOverride = null) + : this(steps, rollbackOnFailureOverride, null) + { + } + + internal SetupPipeline( + IEnumerable steps, + bool? rollbackOnFailureOverride, + Func? beforeFailureRollback) { _steps = steps.ToList(); _rollbackOnFailureOverride = rollbackOnFailureOverride; + _beforeFailureRollback = beforeFailureRollback; } internal static bool ShouldRunTrayArtifactCleanup(PipelineResult result, bool dryRun) @@ -245,6 +255,19 @@ public async Task RunAsync(SetupContext ctx) else ctx.Logger.Warn($"SetupPipeline: Step '{step.Id}' failed: {result.Message}"); + if (_beforeFailureRollback is not null) + { + try + { + await _beforeFailureRollback(ctx, step.Id, result); + } + catch (Exception ex) + { + // A diagnostic must not replace the original failure or prevent owned-resource rollback. + ctx.Logger.Warn($"Pre-rollback diagnostic failed ({ex.GetType().Name}); continuing rollback"); + } + } + if (_rollbackOnFailureOverride ?? ctx.Config.RollbackOnFailure) { await RollbackFailedStep(step, ctx); diff --git a/src/OpenClaw.SetupEngine/SetupWizardRunner.cs b/src/OpenClaw.SetupEngine/SetupWizardRunner.cs index 8fef4b403..ea1b90617 100644 --- a/src/OpenClaw.SetupEngine/SetupWizardRunner.cs +++ b/src/OpenClaw.SetupEngine/SetupWizardRunner.cs @@ -616,13 +616,30 @@ internal async Task RestoreReloadModeAsync() await StartGatewayStep.RestartAndWaitForHealthAsync( _ctx, CancellationToken.None); - if (!restartResult.IsSuccess && - restartResult.Message?.Contains(RestartServingOwnerDiagnostic, StringComparison.Ordinal) == true) + var servingOwnerUnavailable = + !restartResult.IsSuccess && + restartResult.Message?.Contains( + RestartServingOwnerDiagnostic, + StringComparison.Ordinal) == true; + var restartIntentContention = + !restartResult.IsSuccess && + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + restartResult.Message); + if (servingOwnerUnavailable || restartIntentContention) { - // Restoring hybrid reload can initiate a supervisor restart before the CLI - // records its intent. Never bypass that CLI ownership gate or adopt a listener. _ctx.Logger.Warn( - "Gateway restart owner was unavailable after restoring reload. Rechecking managed ownership before one restart retry."); + restartIntentContention + ? "Gateway restart intent was refused by coordinator contention after restoring reload. Rechecking managed ownership before one guarded restart retry." + : "Gateway restart owner was unavailable after restoring reload. Rechecking managed ownership before one guarded restart retry."); + if (restartIntentContention) + { + await _restorationDelayAsync( + GatewayWizardRestartRecoveryPolicy.RestartIntentContentionRetryDelay, + CancellationToken.None); + } + + // Endpoint provenance rejects a foreign listener before retry. The retried + // Gateway CLI command remains responsible for coordinator and owner admission. var retryOwnership = await VerifyExpectedManagedGatewayAsync( "before retrying gateway restart"); if (!retryOwnership.IsSuccess) diff --git a/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs b/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs index 6217303cf..35e72362d 100644 --- a/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs +++ b/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs @@ -165,7 +165,10 @@ private async Task InitializeEnabledAsync() setupArguments.Add(candidatePackage); } - var exitCode = await Program.Main([.. setupArguments]); + var exitCode = await Program.RunWithFailureDiagnosticAsync( + [.. setupArguments], + (ctx, stepId, result) => GatewayRestartFailureDiagnostic.CaptureAsync( + ctx, stepId, result, Path.Combine(ArtifactDir, "gateway-restart-diagnostic.json"))); if (exitCode != 0) { diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs new file mode 100644 index 000000000..04a4e4728 --- /dev/null +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs @@ -0,0 +1,189 @@ +using System.Text.Json; +using OpenClaw.SetupEngine; + +namespace OpenClaw.E2ETests.Setup; + +// This fixture-only probe never prints raw systemctl properties, paths, PIDs, or process arguments. +internal static class GatewayRestartFailureDiagnostic +{ + internal const string ProbeScript = """ + unit=openclaw-gateway.service + if ! properties=$(systemctl --user show "$unit" -p Id -p ActiveState -p SubState -p MainPID -p ExecMainPID -p ExecMainStartTimestampMonotonic -p NRestarts -p Result -p ExecMainCode -p ExecMainStatus 2>/dev/null); then + printf 'probe=unavailable\n' + exit 0 + fi + id= active= sub= pid= exec_pid= started= restarts= service_result= exit_code= exit_status= + while IFS='=' read -r key value; do + case "$key" in + Id) id=$value ;; + ActiveState) active=$value ;; + SubState) sub=$value ;; + MainPID) pid=$value ;; + ExecMainPID) exec_pid=$value ;; + ExecMainStartTimestampMonotonic) started=$value ;; + NRestarts) restarts=$value ;; + Result) service_result=$value ;; + ExecMainCode) exit_code=$value ;; + ExecMainStatus) exit_status=$value ;; + esac + done <<< "$properties" + printf 'probe=ok\nscope=user\n' + if [[ "$id" == "$unit" ]]; then printf 'unit_equal=true\n'; else printf 'unit_equal=false\n'; fi + case "$active" in active|inactive|failed|activating|deactivating|reloading) printf 'active=%s\n' "$active" ;; *) printf 'active=unknown\n' ;; esac + case "$sub" in running|exited|dead|failed|start-pre|start-post|auto-restart|stop-sigterm|stop-post) printf 'sub=%s\n' "$sub" ;; *) printf 'sub=unknown\n' ;; esac + if [[ "$pid" =~ ^[1-9][0-9]*$ ]]; then + printf 'pid_present=true\n' + if kill -0 "$pid" 2>/dev/null; then printf 'pid_live=true\n'; else printf 'pid_live=false\n'; fi + if [[ "$exec_pid" == "$pid" ]]; then printf 'pid_equal=true\n'; else printf 'pid_equal=false\n'; fi + # Redirection order is intentional: silence stderr before opening the racy proc path. + if [[ -r "/proc/$pid/stat" ]] && read -r stat 2>/dev/null < "/proc/$pid/stat"; then + read -r -a fields <<< "${stat##*) }" + if [[ "${fields[19]:-}" =~ ^[1-9][0-9]*$ ]]; then printf 'process_start_available=true\n'; else printf 'process_start_available=false\n'; fi + else + printf 'process_start_available=false\n' + fi + else + printf 'pid_present=false\npid_live=false\npid_equal=unknown\nprocess_start_available=false\n' + fi + if [[ "$started" =~ ^[1-9][0-9]*$ ]]; then printf 'service_start_available=true\n'; else printf 'service_start_available=false\n'; fi + if [[ "$restarts" =~ ^[0-9]{1,6}$ ]]; then printf 'restart_count=%s\n' "$restarts"; else printf 'restart_count=unknown\n'; fi + case "$service_result" in success|exit-code|signal|core-dump|timeout|watchdog|start-limit-hit|resources|protocol|oom-kill) printf 'service_result=%s\n' "$service_result" ;; *) printf 'service_result=unknown\n' ;; esac + if [[ "$exit_code" =~ ^[0-3]$ ]]; then printf 'exit_code=%s\n' "$exit_code"; else printf 'exit_code=unknown\n'; fi + if [[ "$exit_status" =~ ^(0|[1-9][0-9]{0,2})$ ]] && ((exit_status <= 255)); then printf 'exit_status=%s\n' "$exit_status"; else printf 'exit_status=unknown\n'; fi + """; + + internal static string RefusalCategory(string? message) + { + if (message is null) + return "other_restart_failure"; + if (message.Contains("StateDatabaseCoordinatorContentionError", StringComparison.Ordinal)) + return "coordinator_contention"; + if (message.Contains("Cannot record restart intent for the serving Gateway", StringComparison.Ordinal)) + return "restart_intent_refused"; + if (message.Contains("Cannot verify a live serving Gateway owner", StringComparison.Ordinal)) + return "serving_owner_unverified"; + return "other_restart_failure"; + } + + internal static bool IsTargetFailure(string stepId, StepResult result) => + stepId == "run-wizard" && + result.Message?.Contains("Gateway restart after wizard failed:", StringComparison.Ordinal) == true; + + internal static async Task CaptureAsync( + SetupContext ctx, string stepId, StepResult result, string artifactPath) + { + if (!IsTargetFailure(stepId, result)) + return; + + var probe = await ctx.Commands.RunInWslAsync( + ctx.DistroName!, + ProbeScript.Replace("\r\n", "\n", StringComparison.Ordinal), + TimeSpan.FromSeconds(8), + ct: CancellationToken.None, + inputViaStdin: true); + + var snapshot = Parse(probe); + var artifact = new + { + phase = "before_setup_rollback", + refusalCategory = RefusalCategory(result.Message), + ownerPredicate = "not_exposed_by_gateway_cli", + snapshot.Probe, + snapshot.Scope, + snapshot.UnitEqual, + snapshot.Active, + snapshot.Sub, + snapshot.PidPresent, + snapshot.PidLive, + snapshot.PidEqual, + snapshot.ProcessStartAvailable, + snapshot.ServiceStartAvailable, + snapshot.RestartCount, + snapshot.ServiceResult, + snapshot.ExecMainCode, + snapshot.ExecMainStatus + }; + await File.WriteAllTextAsync(artifactPath, JsonSerializer.Serialize(artifact)); + ctx.Logger.Info($"Pre-rollback gateway restart diagnostic: {JsonSerializer.Serialize(artifact)}"); + } + + internal sealed record Snapshot( + string Probe, + string? Scope = null, + bool? UnitEqual = null, + string? Active = null, + string? Sub = null, + bool? PidPresent = null, + bool? PidLive = null, + bool? PidEqual = null, + bool? ProcessStartAvailable = null, + bool? ServiceStartAvailable = null, + int? RestartCount = null, + string? ServiceResult = null, + int? ExecMainCode = null, + int? ExecMainStatus = null); + + internal static Snapshot Parse(CommandResult result) + { + if (result.TimedOut) + return new("timeout"); + if (result.ExitCode != 0) + return new("unavailable"); + + var fields = new Dictionary(StringComparer.Ordinal); + foreach (var line in result.Stdout.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + var parts = line.TrimEnd('\r').Split('=', 2); + if (parts.Length != 2 || !fields.TryAdd(parts[0], parts[1])) + return new("invalid_output"); + } + + if (fields.Count == 1 && fields.GetValueOrDefault("probe") == "unavailable") + return new("unavailable"); + if (fields.Count != 14 || fields.GetValueOrDefault("probe") != "ok" || + fields.GetValueOrDefault("scope") != "user" || + !TryBoolean(fields, "unit_equal", out var unitEqual) || + !TryBoolean(fields, "pid_present", out var pidPresent) || + !TryBoolean(fields, "pid_live", out var pidLive) || + !TryBoolean(fields, "process_start_available", out var processStart) || + !TryBoolean(fields, "service_start_available", out var serviceStart) || + !fields.TryGetValue("pid_equal", out var pidEqualText) || + pidEqualText is not ("true" or "false" or "unknown") || + !fields.TryGetValue("active", out var active) || + active is not ("active" or "inactive" or "failed" or "activating" or "deactivating" or "reloading" or "unknown") || + !fields.TryGetValue("sub", out var sub) || + sub is not ("running" or "exited" or "dead" or "failed" or "start-pre" or "start-post" or "auto-restart" or "stop-sigterm" or "stop-post" or "unknown") || + !fields.TryGetValue("restart_count", out var restartCountText) || + !(restartCountText == "unknown" || + int.TryParse(restartCountText, out var count) && count is >= 0 and <= 999999) || + !fields.TryGetValue("service_result", out var serviceResult) || + serviceResult is not ("success" or "exit-code" or "signal" or "core-dump" or + "timeout" or "watchdog" or "start-limit-hit" or "resources" or + "protocol" or "oom-kill" or "unknown") || + !fields.TryGetValue("exit_code", out var exitCodeText) || + !(exitCodeText == "unknown" || + int.TryParse(exitCodeText, out var exitCode) && exitCode is >= 0 and <= 3) || + !fields.TryGetValue("exit_status", out var exitStatusText) || + !(exitStatusText == "unknown" || + int.TryParse(exitStatusText, out var exitStatus) && exitStatus is >= 0 and <= 255)) + return new("invalid_output"); + + return new Snapshot( + "ok", "user", unitEqual, active, sub, pidPresent, pidLive, + pidEqualText == "unknown" ? null : pidEqualText == "true", + processStart, serviceStart, + restartCountText == "unknown" ? null : int.Parse(restartCountText), + serviceResult, + exitCodeText == "unknown" ? null : int.Parse(exitCodeText), + exitStatusText == "unknown" ? null : int.Parse(exitStatusText)); + } + + private static bool TryBoolean(Dictionary fields, string name, out bool value) + { + value = false; + if (!fields.TryGetValue(name, out var text) || text is not ("true" or "false")) + return false; + value = text == "true"; + return true; + } +} diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs new file mode 100644 index 000000000..293a0b07c --- /dev/null +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs @@ -0,0 +1,130 @@ +using System.Text.Json; +using OpenClaw.SetupEngine; + +namespace OpenClaw.E2ETests.Setup; + +public sealed class GatewayRestartFailureDiagnosticTests +{ + [Fact] + public void SelectsOnlyGuardedPostWizardRestartFailure() + { + Assert.True(GatewayRestartFailureDiagnostic.IsTargetFailure( + "run-wizard", StepResult.Fail("Gateway restart after wizard failed: GATEWAY_RESTART_PREPARATION_REFUSED"))); + Assert.False(GatewayRestartFailureDiagnostic.IsTargetFailure( + "run-wizard", StepResult.Fail("Gateway wizard failed: secret"))); + Assert.False(GatewayRestartFailureDiagnostic.IsTargetFailure( + "start-gateway", StepResult.Fail("Gateway restart after wizard failed: unexpected"))); + } + + [Theory] + [InlineData("StateDatabaseCoordinatorContentionError: another OpenClaw process owns state-lifecycle", "coordinator_contention")] + [InlineData("Cannot record restart intent for the serving Gateway", "restart_intent_refused")] + [InlineData("Cannot verify a live serving Gateway owner", "serving_owner_unverified")] + [InlineData("unrelated sensitive details", "other_restart_failure")] + public void ClassifiesOnlyKnownRefusals(string message, string expected) + { + Assert.Equal(expected, GatewayRestartFailureDiagnostic.RefusalCategory(message)); + } + + [Fact] + public void ParsesOnlyAllowlistedCoarseFacts() + { + var raw = """ + probe=ok + scope=user + unit_equal=true + active=active + sub=running + pid_present=true + pid_live=true + pid_equal=false + process_start_available=true + service_start_available=true + restart_count=2 + service_result=exit-code + exit_code=1 + exit_status=78 + """; + var snapshot = GatewayRestartFailureDiagnostic.Parse(Result(raw)); + Assert.Equal("ok", snapshot.Probe); + Assert.Equal("user", snapshot.Scope); + Assert.True(snapshot.UnitEqual); + Assert.True(snapshot.PidLive); + Assert.False(snapshot.PidEqual); + Assert.Equal(2, snapshot.RestartCount); + Assert.Equal("exit-code", snapshot.ServiceResult); + Assert.Equal(1, snapshot.ExecMainCode); + Assert.Equal(78, snapshot.ExecMainStatus); + Assert.DoesNotContain("home", JsonSerializer.Serialize(snapshot)); + } + + [Theory] + [InlineData("service_result=private-path")] + [InlineData("exit_code=4")] + [InlineData("exit_status=256")] + public void RejectsOutOfRangeServiceExitFacts(string replacement) + { + const string raw = """ + probe=ok + scope=user + unit_equal=true + active=activating + sub=auto-restart + pid_present=false + pid_live=false + pid_equal=unknown + process_start_available=false + service_start_available=true + restart_count=0 + service_result=unknown + exit_code=unknown + exit_status=unknown + """; + var fieldName = replacement.Split('=')[0]; + var output = raw.Replace($"{fieldName}=unknown", replacement, StringComparison.Ordinal); + + var snapshot = GatewayRestartFailureDiagnostic.Parse(Result(output)); + + Assert.Equal("invalid_output", snapshot.Probe); + Assert.DoesNotContain(replacement, JsonSerializer.Serialize(snapshot)); + } + + [Fact] + public void RejectsUnexpectedOrUnvalidatedOutputWithoutPersistingIt() + { + var snapshot = GatewayRestartFailureDiagnostic.Parse( + Result("probe=ok\nscope=user\nsensitive_path=/home/user/.openclaw/secret\n")); + Assert.Equal("invalid_output", snapshot.Probe); + Assert.DoesNotContain("sensitive_path", JsonSerializer.Serialize(snapshot)); + Assert.Equal("timeout", GatewayRestartFailureDiagnostic.Parse(Result("", timedOut: true)).Probe); + Assert.Equal("unavailable", GatewayRestartFailureDiagnostic.Parse(Result("", exitCode: 1)).Probe); + } + + [Fact] + public void ProbeRunsAsUserThroughStdinAndNeverPrintsRawStatus() + { + var script = GatewayRestartFailureDiagnostic.ProbeScript; + Assert.Contains("systemctl --user show", script); + Assert.Contains("-p ExecMainStatus", script); + Assert.Contains("read -r key value", script); + Assert.DoesNotContain("printf '%s\\n' \"$properties\"", script); + Assert.DoesNotContain("openclaw gateway restart", script); + Assert.DoesNotContain("ps ", script); + } + + [Fact] + public void ProbeSuppressesFailedProcReadBeforeCommandLogging() + { + var script = GatewayRestartFailureDiagnostic.ProbeScript; + + Assert.Contains( + """if [[ -r "/proc/$pid/stat" ]] && read -r stat 2>/dev/null < "/proc/$pid/stat"; then""", + script); + Assert.DoesNotContain( + """read -r stat < "/proc/$pid/stat" 2>/dev/null""", + script); + } + + private static CommandResult Result(string stdout, int exitCode = 0, bool timedOut = false) => + new(exitCode, stdout, "", TimeSpan.Zero, timedOut); +} diff --git a/tests/OpenClaw.SetupEngine.Tests/GatewayWizardRestartRecoveryPolicyTests.cs b/tests/OpenClaw.SetupEngine.Tests/GatewayWizardRestartRecoveryPolicyTests.cs index 3fe08efd4..bdb16568b 100644 --- a/tests/OpenClaw.SetupEngine.Tests/GatewayWizardRestartRecoveryPolicyTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/GatewayWizardRestartRecoveryPolicyTests.cs @@ -83,6 +83,27 @@ public void RetryableGatewayStartupDisconnect_IsNarrow( closeStatusCode)); } + [Fact] + public void RestartIntentCoordinatorContention_RequiresBothExactMarkers() + { + var contention = + $"{GatewayWizardRestartRecoveryPolicy.RestartIntentCoordinatorContentionError}. " + + GatewayWizardRestartRecoveryPolicy.RestartIntentRecordingRefusal; + + Assert.True( + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + contention)); + Assert.False( + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + GatewayWizardRestartRecoveryPolicy.RestartIntentCoordinatorContentionError)); + Assert.False( + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + GatewayWizardRestartRecoveryPolicy.RestartIntentRecordingRefusal)); + Assert.False( + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + "another process owns state-lifecycle")); + } + [Theory] [InlineData("done")] [InlineData("Done")] diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index 71b4498d3..24b364804 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -77,6 +77,50 @@ public async Task RunAsync_CompatibilityFailure_PreservesTypedTerminalReason() Assert.Equal(GatewayCompatibilityFailureKind.ProtocolMismatch, result.CompatibilityFailure); } + [Fact] + public async Task RunAsync_FailureDiagnosticRunsBeforeRollback_WithoutChangingFailure() + { + var order = new List(); + var pipeline = new SetupPipeline( + [new MockStep( + "wizard", + (_, _) => Task.FromResult(StepResult.Fail("original failure")), + (_, _) => { order.Add("failed-step-rollback"); return Task.CompletedTask; })], + rollbackOnFailureOverride: true, + (_, stepId, result) => + { + Assert.Equal("wizard", stepId); + Assert.Equal("original failure", result.Message); + order.Add("diagnostic"); + return Task.CompletedTask; + }); + + var result = await pipeline.RunAsync(CreateContext()); + + Assert.Equal(["diagnostic", "failed-step-rollback"], order); + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal("original failure", result.Message); + } + + [Fact] + public async Task RunAsync_FailingDiagnosticStillRollsBackAndPreservesOriginalFailure() + { + var rolledBack = false; + var pipeline = new SetupPipeline( + [new MockStep( + "wizard", + (_, _) => Task.FromResult(StepResult.Fail("original failure")), + (_, _) => { rolledBack = true; return Task.CompletedTask; })], + rollbackOnFailureOverride: true, + (_, _, _) => throw new InvalidOperationException("diagnostic failure")); + + var result = await pipeline.RunAsync(CreateContext()); + + Assert.True(rolledBack); + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal("original failure", result.Message); + } + [Fact] public async Task RunAsync_RestartRequired_PreservesTypedTerminalReason() { diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs index 9c4c88803..0d133e1cf 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs @@ -4309,6 +4309,92 @@ CommandResult Restart() } } + [Fact] + public async Task SetupWizard_RestartIntentContentionReverifiesOwnershipBeforeOneRetry() + { + var restarts = 0; + var inspections = 0; + var delays = new List<(TimeSpan Delay, CancellationToken CancellationToken)>(); + var commands = new FakeCommandRunner( + _ => Ok(), + (_, command, _) => command switch + { + var value when value.Contains("config set gateway.reload.mode") => Ok(), + var value when value.Contains("openclaw gateway restart") => Restart(), + var value when value.Contains("curl -s") => Ok("200"), + _ => Fail($"Unexpected command: {command}"), + }); + var ctx = CreateContext(commands: commands); + ctx.DistroName = "test-distro"; + ctx.EndpointProvenanceProbe = (_, _) => + { + inspections++; + return Task.FromResult(new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + ctx.Config.GatewayPort)); + }; + + var runner = new SetupWizardRunner( + ctx, + (delay, cancellationToken) => + { + delays.Add((delay, cancellationToken)); + return Task.CompletedTask; + }); + + var result = await runner.RestoreReloadModeAsync(); + + Assert.True(result.IsSuccess, result.Message); + Assert.Equal(2, restarts); + Assert.Equal(2, inspections); + var retryDelay = Assert.Single(delays); + Assert.Equal( + GatewayWizardRestartRecoveryPolicy.RestartIntentContentionRetryDelay, + retryDelay.Delay); + Assert.False(retryDelay.CancellationToken.CanBeCanceled); + Assert.DoesNotContain(commands.WslCalls, call => call.Command.Contains("systemctl")); + return; + + CommandResult Restart() + { + if (++restarts == 1) + { + return Fail( + $"{GatewayWizardRestartRecoveryPolicy.RestartIntentCoordinatorContentionError}. " + + GatewayWizardRestartRecoveryPolicy.RestartIntentRecordingRefusal); + } + + Assert.Equal(1, inspections); + return Ok(); + } + } + + [Theory] + [InlineData(GatewayEndpointProvenanceKind.UnknownListener)] + [InlineData(GatewayEndpointProvenanceKind.ConflictingOpenClawGateway)] + public async Task SetupWizard_RestartIntentContentionDoesNotRetryAnUntrustedListener( + GatewayEndpointProvenanceKind kind) + { + var restartFailure = + $"{GatewayWizardRestartRecoveryPolicy.RestartIntentCoordinatorContentionError}. " + + GatewayWizardRestartRecoveryPolicy.RestartIntentRecordingRefusal; + var commands = new FakeCommandRunner( + _ => Ok(), + (_, command, _) => command.Contains("config set gateway.reload.mode") + ? Ok() + : Fail(restartFailure)); + var ctx = CreateContext(commands: commands); + ctx.DistroName = "test-distro"; + ctx.EndpointProvenanceProbe = (_, _) => Task.FromResult( + new GatewayEndpointProvenance(kind, ctx.Config.GatewayPort)); + + var result = await new SetupWizardRunner(ctx).RestoreReloadModeAsync(); + + Assert.False(result.IsSuccess); + Assert.Contains("ownership verification failed", result.Message); + Assert.Single(commands.WslCalls, call => call.Command.Contains("openclaw gateway restart")); + } + [Theory] [InlineData(GatewayEndpointProvenanceKind.UnknownListener)] [InlineData(GatewayEndpointProvenanceKind.ConflictingOpenClawGateway)] @@ -4335,7 +4421,7 @@ public async Task SetupWizard_RestartOwnerGapDoesNotRetryAnUntrustedListener( [Theory] [InlineData(SetupWizardRunner.RestartServingOwnerDiagnostic, 2)] [InlineData("GATEWAY_RESTART_PREPARATION_REFUSED: Cannot verify the selected service command.", 1)] - [InlineData("StateDatabaseCoordinatorContentionError: another OpenClaw process owns state-lifecycle. GATEWAY_RESTART_PREPARATION_REFUSED: Cannot record restart intent for the serving Gateway. Gateway was not signaled.", 1)] + [InlineData("StateDatabaseCoordinatorContentionError: another OpenClaw process owns state-lifecycle. GATEWAY_RESTART_PREPARATION_REFUSED: Cannot record restart intent for the serving Gateway. Gateway was not signaled.", 2)] [InlineData("Unrelated restart failure", 1)] public async Task SetupWizard_RestartRetryRemainsBoundedAndSpecific(string error, int expectedRestarts) {