fix(setup): retry guarded restart intent contention

docs/SETUP_ENGINE_REDESIGN.md

@@ -18,15 +18,16 @@ timing explanation, not an established cause of every refusal. Observed service
 states differ: local diagnostics captured `activating/auto-restart` with no
 MainPID, while hosted generic refusals captured an `active/running` unit and a
 live PID. Neither snapshot establishes the admission-time owner-lease predicate.
-`SetupWizardRunner` recognizes only the exact
-serving-owner refusal, waits for verified managed endpoint ownership using the
-existing bounded provenance probe, and retries the normal CLI restart once.
+`SetupWizardRunner` recognizes only the exact serving-owner refusal or the
+combination of typed state-database coordinator contention and the exact
+restart-intent-recording refusal. It waits for verified managed endpoint
+ownership using the existing bounded provenance probe, then retries the normal
+guarded CLI restart once.
 The probe allows up to 30 one-second retry delays, plus probe duration, for
 `NoListener` and `UnknownListener` tagged `ListenerSnapshotChanged`. Other
 unknown/conflicting listeners, other restart errors, and a repeated refusal still
 fail setup. Listener provenance does not prove owner-lease or coordinator
-readiness; the retried CLI command retains those guards. Restart-intent recording
-contention is a separate failure and is not retried here. There is no direct
+readiness; the retried CLI command retains those guards. There is no direct
 systemd restart fallback or ownership bypass.
 
 > **Status note (2026-07-06):** Current default setup includes `WindowsNodeBootstrapContextStep`, which injects Windows-node context into the WSL workspace `AGENTS.md` after onboarding.
@@ -298,6 +299,25 @@ public abstract class SetupStep
 public sealed record StepResult(StepOutcome Outcome, string? Message = null, Exception? Exception = null);
 ```
 
+### Headless E2E guarded-restart diagnostics
+
+The disposable `E2ESetupFixture` runs the same CLI entry point with an internal
+failure observer. If the wizard step fails specifically at the guarded
+post-wizard Gateway restart, the observer awaits a fixed, read-only
+`systemctl --user show` probe **before** the normal owned-fixture rollback.
+The resulting `gateway-restart-diagnostic.json` contains only an allowlisted
+refusal category and coarse unit/state/PID-presence/start-identity-availability
+facts, plus allowlisted service result and bounded exit code/status. Failed or
+timed-out probes produce an explicit probe status, not raw
+command output. The original setup failure and rollback are unchanged.
+
+The public Gateway CLI does not expose the rejected owner-lease predicate,
+so `ownerPredicate=not_exposed_by_gateway_cli` is intentional. An
+`unverified` serving owner must not be interpreted as coordinator contention
+without the separately observed typed contention error. This fixture-only
+diagnostic neither retries the guarded restart nor preserves the distro after
+rollback.
+
 ---
 
 ## Key Components

src/OpenClaw.SetupEngine/GatewayWizardRestartRecoveryPolicy.cs

@@ -15,6 +15,12 @@ internal static class GatewayWizardRestartRecoveryPolicy
     /// </summary>
     public const string HostedWizardTerminationError =
         "Error: TUI exited from signal SIGTERM";
+    public const string RestartIntentCoordinatorContentionError =
+        "StateDatabaseCoordinatorContentionError: another OpenClaw process owns state-lifecycle";
+    public const string RestartIntentRecordingRefusal =
+        "GATEWAY_RESTART_PREPARATION_REFUSED: Cannot record restart intent for the serving Gateway. Gateway was not signaled.";
+    public static readonly TimeSpan RestartIntentContentionRetryDelay =
+        TimeSpan.FromMilliseconds(500);
 
     public static bool IsTerminalRestartCandidate(
         string? gatewayVersion,
@@ -93,6 +99,14 @@ public static bool IsRestartLikeDisconnect(Exception exception)
     public static bool IsRetryableGatewayStartupDisconnect(int? closeStatusCode) =>
         closeStatusCode == 1013;
 
+    public static bool IsRestartIntentCoordinatorContention(string? message) =>
+        message?.Contains(
+            RestartIntentCoordinatorContentionError,
+            StringComparison.Ordinal) == true &&
+        message.Contains(
+            RestartIntentRecordingRefusal,
+            StringComparison.Ordinal);
+
     public static async Task<GatewayEndpointProvenance> WaitForExpectedManagedGatewayAsync(
         Func<CancellationToken, Task<GatewayEndpointProvenance>> inspectAsync,
         int noListenerRetryCount,

src/OpenClaw.SetupEngine/Program.cs

@@ -39,6 +39,19 @@ public static class Program
     ]);
 
     public static async Task<int> Main(string[] args)
+        => await RunAsync(args, null);
+
+    internal static Task<int> RunWithFailureDiagnosticAsync(
+        string[] args,
+        Func<SetupContext, string, StepResult, Task> beforeFailureRollback)
+    {
+        ArgumentNullException.ThrowIfNull(beforeFailureRollback);
+        return RunAsync(args, beforeFailureRollback);
+    }
+
+    private static async Task<int> RunAsync(
+        string[] args,
+        Func<SetupContext, string, StepResult, Task>? beforeFailureRollback)
     {
         Console.WriteLine($"OpenClaw Setup Engine {AppVersionInfo.DisplayVersion}");
         Console.WriteLine("─────────────────────────────");
@@ -287,7 +300,7 @@ public static async Task<int> Main(string[] args)
             steps = BuildSteps(config);
         }
 
-        var pipeline = new SetupPipeline(steps);
+        var pipeline = new SetupPipeline(steps, rollbackOnFailureOverride: null, beforeFailureRollback);
 
         pipeline.StepProgress += (_, e) =>
         {

src/OpenClaw.SetupEngine/SetupPipeline.cs

@@ -136,13 +136,23 @@ public sealed class SetupPipeline
     private readonly List<SetupStep> _steps;
     private readonly List<SetupStep> _completedSteps = new();
     private readonly bool? _rollbackOnFailureOverride;
+    private readonly Func<SetupContext, string, StepResult, Task>? _beforeFailureRollback;
 
     public event EventHandler<StepProgressEvent>? StepProgress;
 
     public SetupPipeline(IEnumerable<SetupStep> steps, bool? rollbackOnFailureOverride = null)
+        : this(steps, rollbackOnFailureOverride, null)
+    {
+    }
+
+    internal SetupPipeline(
+        IEnumerable<SetupStep> steps,
+        bool? rollbackOnFailureOverride,
+        Func<SetupContext, string, StepResult, Task>? beforeFailureRollback)
     {
         _steps = steps.ToList();
         _rollbackOnFailureOverride = rollbackOnFailureOverride;
+        _beforeFailureRollback = beforeFailureRollback;
     }
 
     internal static bool ShouldRunTrayArtifactCleanup(PipelineResult result, bool dryRun)
@@ -245,6 +255,19 @@ public async Task<PipelineResult> RunAsync(SetupContext ctx)
             else
                 ctx.Logger.Warn($"SetupPipeline: Step '{step.Id}' failed: {result.Message}");
 
+            if (_beforeFailureRollback is not null)
+            {
+                try
+                {
+                    await _beforeFailureRollback(ctx, step.Id, result);
+                }
+                catch (Exception ex)
+                {
+                    // A diagnostic must not replace the original failure or prevent owned-resource rollback.
+                    ctx.Logger.Warn($"Pre-rollback diagnostic failed ({ex.GetType().Name}); continuing rollback");
+                }
+            }
+
             if (_rollbackOnFailureOverride ?? ctx.Config.RollbackOnFailure)
             {
                 await RollbackFailedStep(step, ctx);

src/OpenClaw.SetupEngine/SetupWizardRunner.cs

@@ -616,13 +616,30 @@ internal async Task<StepResult> RestoreReloadModeAsync()
                 await StartGatewayStep.RestartAndWaitForHealthAsync(
                     _ctx,
                     CancellationToken.None);
-            if (!restartResult.IsSuccess &&
-                restartResult.Message?.Contains(RestartServingOwnerDiagnostic, StringComparison.Ordinal) == true)
+            var servingOwnerUnavailable =
+                !restartResult.IsSuccess &&
+                restartResult.Message?.Contains(
+                    RestartServingOwnerDiagnostic,
+                    StringComparison.Ordinal) == true;
+            var restartIntentContention =
+                !restartResult.IsSuccess &&
+                GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention(
+                    restartResult.Message);
+            if (servingOwnerUnavailable || restartIntentContention)
             {
-                // Restoring hybrid reload can initiate a supervisor restart before the CLI
-                // records its intent. Never bypass that CLI ownership gate or adopt a listener.
                 _ctx.Logger.Warn(
-                    "Gateway restart owner was unavailable after restoring reload. Rechecking managed ownership before one restart retry.");
+                    restartIntentContention
+                        ? "Gateway restart intent was refused by coordinator contention after restoring reload. Rechecking managed ownership before one guarded restart retry."
+                        : "Gateway restart owner was unavailable after restoring reload. Rechecking managed ownership before one guarded restart retry.");
+                if (restartIntentContention)
+                {
+                    await _restorationDelayAsync(
+                        GatewayWizardRestartRecoveryPolicy.RestartIntentContentionRetryDelay,
+                        CancellationToken.None);
+                }
+
+                // Endpoint provenance rejects a foreign listener before retry. The retried
+                // Gateway CLI command remains responsible for coordinator and owner admission.
                 var retryOwnership = await VerifyExpectedManagedGatewayAsync(
                     "before retrying gateway restart");
                 if (!retryOwnership.IsSuccess)

tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs

@@ -165,7 +165,10 @@ private async Task InitializeEnabledAsync()
             setupArguments.Add(candidatePackage);
         }
 
-        var exitCode = await Program.Main([.. setupArguments]);
+        var exitCode = await Program.RunWithFailureDiagnosticAsync(
+            [.. setupArguments],
+            (ctx, stepId, result) => GatewayRestartFailureDiagnostic.CaptureAsync(
+                ctx, stepId, result, Path.Combine(ArtifactDir, "gateway-restart-diagnostic.json")));
 
         if (exitCode != 0)
         {

tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs

@@ -0,0 +1,189 @@
+using System.Text.Json;
+using OpenClaw.SetupEngine;
+
+namespace OpenClaw.E2ETests.Setup;
+
+// This fixture-only probe never prints raw systemctl properties, paths, PIDs, or process arguments.
+internal static class GatewayRestartFailureDiagnostic
+{
+    internal const string ProbeScript = """
+        unit=openclaw-gateway.service
+        if ! properties=$(systemctl --user show "$unit" -p Id -p ActiveState -p SubState -p MainPID -p ExecMainPID -p ExecMainStartTimestampMonotonic -p NRestarts -p Result -p ExecMainCode -p ExecMainStatus 2>/dev/null); then
+          printf 'probe=unavailable\n'
+          exit 0
+        fi
+        id= active= sub= pid= exec_pid= started= restarts= service_result= exit_code= exit_status=
+        while IFS='=' read -r key value; do
+          case "$key" in
+            Id) id=$value ;;
+            ActiveState) active=$value ;;
+            SubState) sub=$value ;;
+            MainPID) pid=$value ;;
+            ExecMainPID) exec_pid=$value ;;
+            ExecMainStartTimestampMonotonic) started=$value ;;
+            NRestarts) restarts=$value ;;
+            Result) service_result=$value ;;
+            ExecMainCode) exit_code=$value ;;
+            ExecMainStatus) exit_status=$value ;;
+          esac
+        done <<< "$properties"
+        printf 'probe=ok\nscope=user\n'
+        if [[ "$id" == "$unit" ]]; then printf 'unit_equal=true\n'; else printf 'unit_equal=false\n'; fi
+        case "$active" in active|inactive|failed|activating|deactivating|reloading) printf 'active=%s\n' "$active" ;; *) printf 'active=unknown\n' ;; esac
+        case "$sub" in running|exited|dead|failed|start-pre|start-post|auto-restart|stop-sigterm|stop-post) printf 'sub=%s\n' "$sub" ;; *) printf 'sub=unknown\n' ;; esac
+        if [[ "$pid" =~ ^[1-9][0-9]*$ ]]; then
+          printf 'pid_present=true\n'
+          if kill -0 "$pid" 2>/dev/null; then printf 'pid_live=true\n'; else printf 'pid_live=false\n'; fi
+          if [[ "$exec_pid" == "$pid" ]]; then printf 'pid_equal=true\n'; else printf 'pid_equal=false\n'; fi
+          # Redirection order is intentional: silence stderr before opening the racy proc path.
+          if [[ -r "/proc/$pid/stat" ]] && read -r stat 2>/dev/null < "/proc/$pid/stat"; then
+            read -r -a fields <<< "${stat##*) }"
+            if [[ "${fields[19]:-}" =~ ^[1-9][0-9]*$ ]]; then printf 'process_start_available=true\n'; else printf 'process_start_available=false\n'; fi
+          else
+            printf 'process_start_available=false\n'
+          fi
+        else
+          printf 'pid_present=false\npid_live=false\npid_equal=unknown\nprocess_start_available=false\n'
+        fi
+        if [[ "$started" =~ ^[1-9][0-9]*$ ]]; then printf 'service_start_available=true\n'; else printf 'service_start_available=false\n'; fi
+        if [[ "$restarts" =~ ^[0-9]{1,6}$ ]]; then printf 'restart_count=%s\n' "$restarts"; else printf 'restart_count=unknown\n'; fi
+        case "$service_result" in success|exit-code|signal|core-dump|timeout|watchdog|start-limit-hit|resources|protocol|oom-kill) printf 'service_result=%s\n' "$service_result" ;; *) printf 'service_result=unknown\n' ;; esac
+        if [[ "$exit_code" =~ ^[0-3]$ ]]; then printf 'exit_code=%s\n' "$exit_code"; else printf 'exit_code=unknown\n'; fi
+        if [[ "$exit_status" =~ ^(0|[1-9][0-9]{0,2})$ ]] && ((exit_status <= 255)); then printf 'exit_status=%s\n' "$exit_status"; else printf 'exit_status=unknown\n'; fi
+        """;
+
+    internal static string RefusalCategory(string? message)
+    {
+        if (message is null)
+            return "other_restart_failure";
+        if (message.Contains("StateDatabaseCoordinatorContentionError", StringComparison.Ordinal))
+            return "coordinator_contention";
+        if (message.Contains("Cannot record restart intent for the serving Gateway", StringComparison.Ordinal))
+            return "restart_intent_refused";
+        if (message.Contains("Cannot verify a live serving Gateway owner", StringComparison.Ordinal))
+            return "serving_owner_unverified";
+        return "other_restart_failure";
+    }
+
+    internal static bool IsTargetFailure(string stepId, StepResult result) =>
+        stepId == "run-wizard" &&
+        result.Message?.Contains("Gateway restart after wizard failed:", StringComparison.Ordinal) == true;
+
+    internal static async Task CaptureAsync(
+        SetupContext ctx, string stepId, StepResult result, string artifactPath)
+    {
+        if (!IsTargetFailure(stepId, result))
+            return;
+
+        var probe = await ctx.Commands.RunInWslAsync(
+            ctx.DistroName!,
+            ProbeScript.Replace("\r\n", "\n", StringComparison.Ordinal),
+            TimeSpan.FromSeconds(8),
+            ct: CancellationToken.None,
+            inputViaStdin: true);
+
+        var snapshot = Parse(probe);
+        var artifact = new
+        {
+            phase = "before_setup_rollback",
+            refusalCategory = RefusalCategory(result.Message),
+            ownerPredicate = "not_exposed_by_gateway_cli",
+            snapshot.Probe,
+            snapshot.Scope,
+            snapshot.UnitEqual,
+            snapshot.Active,
+            snapshot.Sub,
+            snapshot.PidPresent,
+            snapshot.PidLive,
+            snapshot.PidEqual,
+            snapshot.ProcessStartAvailable,
+            snapshot.ServiceStartAvailable,
+            snapshot.RestartCount,
+            snapshot.ServiceResult,
+            snapshot.ExecMainCode,
+            snapshot.ExecMainStatus
+        };
+        await File.WriteAllTextAsync(artifactPath, JsonSerializer.Serialize(artifact));
+        ctx.Logger.Info($"Pre-rollback gateway restart diagnostic: {JsonSerializer.Serialize(artifact)}");
+    }
+
+    internal sealed record Snapshot(
+        string Probe,
+        string? Scope = null,
+        bool? UnitEqual = null,
+        string? Active = null,
+        string? Sub = null,
+        bool? PidPresent = null,
+        bool? PidLive = null,
+        bool? PidEqual = null,
+        bool? ProcessStartAvailable = null,
+        bool? ServiceStartAvailable = null,
+        int? RestartCount = null,
+        string? ServiceResult = null,
+        int? ExecMainCode = null,
+        int? ExecMainStatus = null);
+
+    internal static Snapshot Parse(CommandResult result)
+    {
+        if (result.TimedOut)
+            return new("timeout");
+        if (result.ExitCode != 0)
+            return new("unavailable");
+
+        var fields = new Dictionary<string, string>(StringComparer.Ordinal);
+        foreach (var line in result.Stdout.Split('\n', StringSplitOptions.RemoveEmptyEntries))
+        {
+            var parts = line.TrimEnd('\r').Split('=', 2);
+            if (parts.Length != 2 || !fields.TryAdd(parts[0], parts[1]))
+                return new("invalid_output");
+        }
+
+        if (fields.Count == 1 && fields.GetValueOrDefault("probe") == "unavailable")
+            return new("unavailable");
+        if (fields.Count != 14 || fields.GetValueOrDefault("probe") != "ok" ||
+            fields.GetValueOrDefault("scope") != "user" ||
+            !TryBoolean(fields, "unit_equal", out var unitEqual) ||
+            !TryBoolean(fields, "pid_present", out var pidPresent) ||
+            !TryBoolean(fields, "pid_live", out var pidLive) ||
+            !TryBoolean(fields, "process_start_available", out var processStart) ||
+            !TryBoolean(fields, "service_start_available", out var serviceStart) ||
+            !fields.TryGetValue("pid_equal", out var pidEqualText) ||
+            pidEqualText is not ("true" or "false" or "unknown") ||
+            !fields.TryGetValue("active", out var active) ||
+            active is not ("active" or "inactive" or "failed" or "activating" or "deactivating" or "reloading" or "unknown") ||
+            !fields.TryGetValue("sub", out var sub) ||
+            sub is not ("running" or "exited" or "dead" or "failed" or "start-pre" or "start-post" or "auto-restart" or "stop-sigterm" or "stop-post" or "unknown") ||
+            !fields.TryGetValue("restart_count", out var restartCountText) ||
+            !(restartCountText == "unknown" ||
+              int.TryParse(restartCountText, out var count) && count is >= 0 and <= 999999) ||
+            !fields.TryGetValue("service_result", out var serviceResult) ||
+            serviceResult is not ("success" or "exit-code" or "signal" or "core-dump" or
+                "timeout" or "watchdog" or "start-limit-hit" or "resources" or
+                "protocol" or "oom-kill" or "unknown") ||
+            !fields.TryGetValue("exit_code", out var exitCodeText) ||
+            !(exitCodeText == "unknown" ||
+              int.TryParse(exitCodeText, out var exitCode) && exitCode is >= 0 and <= 3) ||
+            !fields.TryGetValue("exit_status", out var exitStatusText) ||
+            !(exitStatusText == "unknown" ||
+              int.TryParse(exitStatusText, out var exitStatus) && exitStatus is >= 0 and <= 255))
+            return new("invalid_output");
+
+        return new Snapshot(
+            "ok", "user", unitEqual, active, sub, pidPresent, pidLive,
+            pidEqualText == "unknown" ? null : pidEqualText == "true",
+            processStart, serviceStart,
+            restartCountText == "unknown" ? null : int.Parse(restartCountText),
+            serviceResult,
+            exitCodeText == "unknown" ? null : int.Parse(exitCodeText),
+            exitStatusText == "unknown" ? null : int.Parse(exitStatusText));
+    }
+
+    private static bool TryBoolean(Dictionary<string, string> fields, string name, out bool value)
+    {
+        value = false;
+        if (!fields.TryGetValue(name, out var text) || text is not ("true" or "false"))
+            return false;
+        value = text == "true";
+        return true;
+    }
+}