From 478a8bdccaec4c0147455eb3afce3b1b91a533fc Mon Sep 17 00:00:00 2001 From: Scott Hanselman Date: Thu, 24 Sep 2026 13:26:00 +0200 Subject: [PATCH 1/5] test(setup): capture coarse Gateway restart state before fixture rollback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1533edb3-7b37-405a-9829-959466a93b54 --- docs/SETUP_ENGINE_REDESIGN.md | 18 ++ src/OpenClaw.SetupEngine/Program.cs | 15 +- src/OpenClaw.SetupEngine/SetupPipeline.cs | 23 +++ .../Setup/E2ESetupFixture.cs | 5 +- .../Setup/GatewayRestartFailureDiagnostic.cs | 164 ++++++++++++++++++ .../GatewayRestartFailureDiagnosticTests.cs | 79 +++++++++ .../SetupPipelineTests.cs | 44 +++++ 7 files changed, 346 insertions(+), 2 deletions(-) create mode 100644 tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs create mode 100644 tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs diff --git a/docs/SETUP_ENGINE_REDESIGN.md b/docs/SETUP_ENGINE_REDESIGN.md index 26ed96f5a..81e4cf79b 100644 --- a/docs/SETUP_ENGINE_REDESIGN.md +++ b/docs/SETUP_ENGINE_REDESIGN.md @@ -298,6 +298,24 @@ public abstract class SetupStep public sealed record StepResult(StepOutcome Outcome, string? Message = null, Exception? Exception = null); ``` +### Headless E2E guarded-restart diagnostics + +The disposable `E2ESetupFixture` runs the same CLI entry point with an internal +failure observer. If the wizard step fails specifically at the guarded +post-wizard Gateway restart, the observer awaits a fixed, read-only +`systemctl --user show` probe **before** the normal owned-fixture rollback. +The resulting `gateway-restart-diagnostic.json` contains only an allowlisted +refusal category and coarse unit/state/PID-presence/start-identity-availability +facts. Failed or timed-out probes produce an explicit probe status, not raw +command output. The original setup failure and rollback are unchanged. + +The public Gateway CLI does not expose the rejected owner-lease predicate, +so `ownerPredicate=not_exposed_by_gateway_cli` is intentional. An +`unverified` serving owner must not be interpreted as coordinator contention +without the separately observed typed contention error. This fixture-only +diagnostic neither retries the guarded restart nor preserves the distro after +rollback. + --- ## Key Components diff --git a/src/OpenClaw.SetupEngine/Program.cs b/src/OpenClaw.SetupEngine/Program.cs index ea51d5f01..26d665221 100644 --- a/src/OpenClaw.SetupEngine/Program.cs +++ b/src/OpenClaw.SetupEngine/Program.cs @@ -39,6 +39,19 @@ public static class Program ]); public static async Task Main(string[] args) + => await RunAsync(args, null); + + internal static Task RunWithFailureDiagnosticAsync( + string[] args, + Func beforeFailureRollback) + { + ArgumentNullException.ThrowIfNull(beforeFailureRollback); + return RunAsync(args, beforeFailureRollback); + } + + private static async Task RunAsync( + string[] args, + Func? beforeFailureRollback) { Console.WriteLine($"OpenClaw Setup Engine {AppVersionInfo.DisplayVersion}"); Console.WriteLine("─────────────────────────────"); @@ -287,7 +300,7 @@ public static async Task Main(string[] args) steps = BuildSteps(config); } - var pipeline = new SetupPipeline(steps); + var pipeline = new SetupPipeline(steps, rollbackOnFailureOverride: null, beforeFailureRollback); pipeline.StepProgress += (_, e) => { diff --git a/src/OpenClaw.SetupEngine/SetupPipeline.cs b/src/OpenClaw.SetupEngine/SetupPipeline.cs index 8e43789d8..969faaa63 100644 --- a/src/OpenClaw.SetupEngine/SetupPipeline.cs +++ b/src/OpenClaw.SetupEngine/SetupPipeline.cs @@ -136,13 +136,23 @@ public sealed class SetupPipeline private readonly List _steps; private readonly List _completedSteps = new(); private readonly bool? _rollbackOnFailureOverride; + private readonly Func? _beforeFailureRollback; public event EventHandler? StepProgress; public SetupPipeline(IEnumerable steps, bool? rollbackOnFailureOverride = null) + : this(steps, rollbackOnFailureOverride, null) + { + } + + internal SetupPipeline( + IEnumerable steps, + bool? rollbackOnFailureOverride, + Func? beforeFailureRollback) { _steps = steps.ToList(); _rollbackOnFailureOverride = rollbackOnFailureOverride; + _beforeFailureRollback = beforeFailureRollback; } internal static bool ShouldRunTrayArtifactCleanup(PipelineResult result, bool dryRun) @@ -245,6 +255,19 @@ public async Task RunAsync(SetupContext ctx) else ctx.Logger.Warn($"SetupPipeline: Step '{step.Id}' failed: {result.Message}"); + if (_beforeFailureRollback is not null) + { + try + { + await _beforeFailureRollback(ctx, step.Id, result); + } + catch (Exception ex) + { + // A diagnostic must not replace the original failure or prevent owned-resource rollback. + ctx.Logger.Warn($"Pre-rollback diagnostic failed ({ex.GetType().Name}); continuing rollback"); + } + } + if (_rollbackOnFailureOverride ?? ctx.Config.RollbackOnFailure) { await RollbackFailedStep(step, ctx); diff --git a/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs b/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs index 6217303cf..35e72362d 100644 --- a/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs +++ b/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs @@ -165,7 +165,10 @@ private async Task InitializeEnabledAsync() setupArguments.Add(candidatePackage); } - var exitCode = await Program.Main([.. setupArguments]); + var exitCode = await Program.RunWithFailureDiagnosticAsync( + [.. setupArguments], + (ctx, stepId, result) => GatewayRestartFailureDiagnostic.CaptureAsync( + ctx, stepId, result, Path.Combine(ArtifactDir, "gateway-restart-diagnostic.json"))); if (exitCode != 0) { diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs new file mode 100644 index 000000000..ede4ffb50 --- /dev/null +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs @@ -0,0 +1,164 @@ +using System.Text.Json; +using OpenClaw.SetupEngine; + +namespace OpenClaw.E2ETests.Setup; + +// This fixture-only probe never prints raw systemctl properties, paths, PIDs, or process arguments. +internal static class GatewayRestartFailureDiagnostic +{ + internal const string ProbeScript = """ + unit=openclaw-gateway.service + if ! properties=$(systemctl --user show "$unit" -p Id -p ActiveState -p SubState -p MainPID -p ExecMainPID -p ExecMainStartTimestampMonotonic -p NRestarts 2>/dev/null); then + printf 'probe=unavailable\n' + exit 0 + fi + id= active= sub= pid= exec_pid= started= restarts= + while IFS='=' read -r key value; do + case "$key" in + Id) id=$value ;; + ActiveState) active=$value ;; + SubState) sub=$value ;; + MainPID) pid=$value ;; + ExecMainPID) exec_pid=$value ;; + ExecMainStartTimestampMonotonic) started=$value ;; + NRestarts) restarts=$value ;; + esac + done <<< "$properties" + printf 'probe=ok\nscope=user\n' + if [[ "$id" == "$unit" ]]; then printf 'unit_equal=true\n'; else printf 'unit_equal=false\n'; fi + case "$active" in active|inactive|failed|activating|deactivating|reloading) printf 'active=%s\n' "$active" ;; *) printf 'active=unknown\n' ;; esac + case "$sub" in running|exited|dead|failed|start-pre|start-post|auto-restart|stop-sigterm|stop-post) printf 'sub=%s\n' "$sub" ;; *) printf 'sub=unknown\n' ;; esac + if [[ "$pid" =~ ^[1-9][0-9]*$ ]]; then + printf 'pid_present=true\n' + if kill -0 "$pid" 2>/dev/null; then printf 'pid_live=true\n'; else printf 'pid_live=false\n'; fi + if [[ "$exec_pid" == "$pid" ]]; then printf 'pid_equal=true\n'; else printf 'pid_equal=false\n'; fi + if [[ -r "/proc/$pid/stat" ]]; then + read -r stat < "/proc/$pid/stat" + read -r -a fields <<< "${stat##*) }" + if [[ "${fields[19]:-}" =~ ^[1-9][0-9]*$ ]]; then printf 'process_start_available=true\n'; else printf 'process_start_available=false\n'; fi + else + printf 'process_start_available=false\n' + fi + else + printf 'pid_present=false\npid_live=false\npid_equal=unknown\nprocess_start_available=false\n' + fi + if [[ "$started" =~ ^[1-9][0-9]*$ ]]; then printf 'service_start_available=true\n'; else printf 'service_start_available=false\n'; fi + if [[ "$restarts" =~ ^[0-9]{1,6}$ ]]; then printf 'restart_count=%s\n' "$restarts"; else printf 'restart_count=unknown\n'; fi + """; + + internal static string RefusalCategory(string? message) + { + if (message is null) + return "other_restart_failure"; + if (message.Contains("StateDatabaseCoordinatorContentionError", StringComparison.Ordinal)) + return "coordinator_contention"; + if (message.Contains("Cannot record restart intent for the serving Gateway", StringComparison.Ordinal)) + return "restart_intent_refused"; + if (message.Contains("Cannot verify a live serving Gateway owner", StringComparison.Ordinal)) + return "serving_owner_unverified"; + return "other_restart_failure"; + } + + internal static bool IsTargetFailure(string stepId, StepResult result) => + stepId == "run-wizard" && + result.Message?.Contains("Gateway restart after wizard failed:", StringComparison.Ordinal) == true; + + internal static async Task CaptureAsync( + SetupContext ctx, string stepId, StepResult result, string artifactPath) + { + if (!IsTargetFailure(stepId, result)) + return; + + var probe = await ctx.Commands.RunInWslAsync( + ctx.DistroName!, + ProbeScript.Replace("\r\n", "\n", StringComparison.Ordinal), + TimeSpan.FromSeconds(8), + ct: CancellationToken.None, + inputViaStdin: true); + + var snapshot = Parse(probe); + var artifact = new + { + phase = "before_setup_rollback", + refusalCategory = RefusalCategory(result.Message), + ownerPredicate = "not_exposed_by_gateway_cli", + snapshot.Probe, + snapshot.Scope, + snapshot.UnitEqual, + snapshot.Active, + snapshot.Sub, + snapshot.PidPresent, + snapshot.PidLive, + snapshot.PidEqual, + snapshot.ProcessStartAvailable, + snapshot.ServiceStartAvailable, + snapshot.RestartCount + }; + await File.WriteAllTextAsync(artifactPath, JsonSerializer.Serialize(artifact)); + ctx.Logger.Info($"Pre-rollback gateway restart diagnostic: {JsonSerializer.Serialize(artifact)}"); + } + + internal sealed record Snapshot( + string Probe, + string? Scope = null, + bool? UnitEqual = null, + string? Active = null, + string? Sub = null, + bool? PidPresent = null, + bool? PidLive = null, + bool? PidEqual = null, + bool? ProcessStartAvailable = null, + bool? ServiceStartAvailable = null, + int? RestartCount = null); + + internal static Snapshot Parse(CommandResult result) + { + if (result.TimedOut) + return new("timeout"); + if (result.ExitCode != 0) + return new("unavailable"); + + var fields = new Dictionary(StringComparer.Ordinal); + foreach (var line in result.Stdout.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + var parts = line.TrimEnd('\r').Split('=', 2); + if (parts.Length != 2 || !fields.TryAdd(parts[0], parts[1])) + return new("invalid_output"); + } + + if (fields.Count == 1 && fields.GetValueOrDefault("probe") == "unavailable") + return new("unavailable"); + if (fields.Count != 11 || fields.GetValueOrDefault("probe") != "ok" || + fields.GetValueOrDefault("scope") != "user" || + !TryBoolean(fields, "unit_equal", out var unitEqual) || + !TryBoolean(fields, "pid_present", out var pidPresent) || + !TryBoolean(fields, "pid_live", out var pidLive) || + !TryBoolean(fields, "process_start_available", out var processStart) || + !TryBoolean(fields, "service_start_available", out var serviceStart) || + !fields.TryGetValue("pid_equal", out var pidEqualText) || + pidEqualText is not ("true" or "false" or "unknown") || + !fields.TryGetValue("active", out var active) || + active is not ("active" or "inactive" or "failed" or "activating" or "deactivating" or "reloading" or "unknown") || + !fields.TryGetValue("sub", out var sub) || + sub is not ("running" or "exited" or "dead" or "failed" or "start-pre" or "start-post" or "auto-restart" or "stop-sigterm" or "stop-post" or "unknown") || + !fields.TryGetValue("restart_count", out var restartCountText) || + !(restartCountText == "unknown" || + int.TryParse(restartCountText, out var count) && count is >= 0 and <= 999999)) + return new("invalid_output"); + + return new Snapshot( + "ok", "user", unitEqual, active, sub, pidPresent, pidLive, + pidEqualText == "unknown" ? null : pidEqualText == "true", + processStart, serviceStart, + restartCountText == "unknown" ? null : int.Parse(restartCountText)); + } + + private static bool TryBoolean(Dictionary fields, string name, out bool value) + { + value = false; + if (!fields.TryGetValue(name, out var text) || text is not ("true" or "false")) + return false; + value = text == "true"; + return true; + } +} diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs new file mode 100644 index 000000000..08afc5f68 --- /dev/null +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs @@ -0,0 +1,79 @@ +using System.Text.Json; +using OpenClaw.SetupEngine; + +namespace OpenClaw.E2ETests.Setup; + +public sealed class GatewayRestartFailureDiagnosticTests +{ + [Fact] + public void SelectsOnlyGuardedPostWizardRestartFailure() + { + Assert.True(GatewayRestartFailureDiagnostic.IsTargetFailure( + "run-wizard", StepResult.Fail("Gateway restart after wizard failed: GATEWAY_RESTART_PREPARATION_REFUSED"))); + Assert.False(GatewayRestartFailureDiagnostic.IsTargetFailure( + "run-wizard", StepResult.Fail("Gateway wizard failed: secret"))); + Assert.False(GatewayRestartFailureDiagnostic.IsTargetFailure( + "start-gateway", StepResult.Fail("Gateway restart after wizard failed: unexpected"))); + } + + [Theory] + [InlineData("StateDatabaseCoordinatorContentionError: another OpenClaw process owns state-lifecycle", "coordinator_contention")] + [InlineData("Cannot record restart intent for the serving Gateway", "restart_intent_refused")] + [InlineData("Cannot verify a live serving Gateway owner", "serving_owner_unverified")] + [InlineData("unrelated sensitive details", "other_restart_failure")] + public void ClassifiesOnlyKnownRefusals(string message, string expected) + { + Assert.Equal(expected, GatewayRestartFailureDiagnostic.RefusalCategory(message)); + } + + [Fact] + public void ParsesOnlyAllowlistedCoarseFacts() + { + var raw = """ + probe=ok + scope=user + unit_equal=true + active=active + sub=running + pid_present=true + pid_live=true + pid_equal=false + process_start_available=true + service_start_available=true + restart_count=2 + """; + var snapshot = GatewayRestartFailureDiagnostic.Parse(Result(raw)); + Assert.Equal("ok", snapshot.Probe); + Assert.Equal("user", snapshot.Scope); + Assert.True(snapshot.UnitEqual); + Assert.True(snapshot.PidLive); + Assert.False(snapshot.PidEqual); + Assert.Equal(2, snapshot.RestartCount); + Assert.DoesNotContain("home", JsonSerializer.Serialize(snapshot)); + } + + [Fact] + public void RejectsUnexpectedOrUnvalidatedOutputWithoutPersistingIt() + { + var snapshot = GatewayRestartFailureDiagnostic.Parse( + Result("probe=ok\nscope=user\nsensitive_path=/home/user/.openclaw/secret\n")); + Assert.Equal("invalid_output", snapshot.Probe); + Assert.DoesNotContain("sensitive_path", JsonSerializer.Serialize(snapshot)); + Assert.Equal("timeout", GatewayRestartFailureDiagnostic.Parse(Result("", timedOut: true)).Probe); + Assert.Equal("unavailable", GatewayRestartFailureDiagnostic.Parse(Result("", exitCode: 1)).Probe); + } + + [Fact] + public void ProbeRunsAsUserThroughStdinAndNeverPrintsRawStatus() + { + var script = GatewayRestartFailureDiagnostic.ProbeScript; + Assert.Contains("systemctl --user show", script); + Assert.Contains("read -r key value", script); + Assert.DoesNotContain("printf '%s\\n' \"$properties\"", script); + Assert.DoesNotContain("openclaw gateway restart", script); + Assert.DoesNotContain("ps ", script); + } + + private static CommandResult Result(string stdout, int exitCode = 0, bool timedOut = false) => + new(exitCode, stdout, "", TimeSpan.Zero, timedOut); +} diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index 71b4498d3..24b364804 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -77,6 +77,50 @@ public async Task RunAsync_CompatibilityFailure_PreservesTypedTerminalReason() Assert.Equal(GatewayCompatibilityFailureKind.ProtocolMismatch, result.CompatibilityFailure); } + [Fact] + public async Task RunAsync_FailureDiagnosticRunsBeforeRollback_WithoutChangingFailure() + { + var order = new List(); + var pipeline = new SetupPipeline( + [new MockStep( + "wizard", + (_, _) => Task.FromResult(StepResult.Fail("original failure")), + (_, _) => { order.Add("failed-step-rollback"); return Task.CompletedTask; })], + rollbackOnFailureOverride: true, + (_, stepId, result) => + { + Assert.Equal("wizard", stepId); + Assert.Equal("original failure", result.Message); + order.Add("diagnostic"); + return Task.CompletedTask; + }); + + var result = await pipeline.RunAsync(CreateContext()); + + Assert.Equal(["diagnostic", "failed-step-rollback"], order); + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal("original failure", result.Message); + } + + [Fact] + public async Task RunAsync_FailingDiagnosticStillRollsBackAndPreservesOriginalFailure() + { + var rolledBack = false; + var pipeline = new SetupPipeline( + [new MockStep( + "wizard", + (_, _) => Task.FromResult(StepResult.Fail("original failure")), + (_, _) => { rolledBack = true; return Task.CompletedTask; })], + rollbackOnFailureOverride: true, + (_, _, _) => throw new InvalidOperationException("diagnostic failure")); + + var result = await pipeline.RunAsync(CreateContext()); + + Assert.True(rolledBack); + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal("original failure", result.Message); + } + [Fact] public async Task RunAsync_RestartRequired_PreservesTypedTerminalReason() { From a2ce88e35244f9e48c3c23579eca3c507b518bd5 Mon Sep 17 00:00:00 2001 From: Scott Hanselman Date: Fri, 25 Sep 2026 08:03:03 +0200 Subject: [PATCH 2/5] test(setup): capture bounded service exit facts before rollback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/SETUP_ENGINE_REDESIGN.md | 3 +- .../Setup/GatewayRestartFailureDiagnostic.cs | 39 +++++++++++++++---- .../GatewayRestartFailureDiagnosticTests.cs | 38 ++++++++++++++++++ 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/docs/SETUP_ENGINE_REDESIGN.md b/docs/SETUP_ENGINE_REDESIGN.md index 81e4cf79b..03a1c0b8a 100644 --- a/docs/SETUP_ENGINE_REDESIGN.md +++ b/docs/SETUP_ENGINE_REDESIGN.md @@ -306,7 +306,8 @@ post-wizard Gateway restart, the observer awaits a fixed, read-only `systemctl --user show` probe **before** the normal owned-fixture rollback. The resulting `gateway-restart-diagnostic.json` contains only an allowlisted refusal category and coarse unit/state/PID-presence/start-identity-availability -facts. Failed or timed-out probes produce an explicit probe status, not raw +facts, plus allowlisted service result and bounded exit code/status. Failed or +timed-out probes produce an explicit probe status, not raw command output. The original setup failure and rollback are unchanged. The public Gateway CLI does not expose the rejected owner-lease predicate, diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs index ede4ffb50..c70193962 100644 --- a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs @@ -8,11 +8,11 @@ internal static class GatewayRestartFailureDiagnostic { internal const string ProbeScript = """ unit=openclaw-gateway.service - if ! properties=$(systemctl --user show "$unit" -p Id -p ActiveState -p SubState -p MainPID -p ExecMainPID -p ExecMainStartTimestampMonotonic -p NRestarts 2>/dev/null); then + if ! properties=$(systemctl --user show "$unit" -p Id -p ActiveState -p SubState -p MainPID -p ExecMainPID -p ExecMainStartTimestampMonotonic -p NRestarts -p Result -p ExecMainCode -p ExecMainStatus 2>/dev/null); then printf 'probe=unavailable\n' exit 0 fi - id= active= sub= pid= exec_pid= started= restarts= + id= active= sub= pid= exec_pid= started= restarts= service_result= exit_code= exit_status= while IFS='=' read -r key value; do case "$key" in Id) id=$value ;; @@ -22,6 +22,9 @@ exit 0 ExecMainPID) exec_pid=$value ;; ExecMainStartTimestampMonotonic) started=$value ;; NRestarts) restarts=$value ;; + Result) service_result=$value ;; + ExecMainCode) exit_code=$value ;; + ExecMainStatus) exit_status=$value ;; esac done <<< "$properties" printf 'probe=ok\nscope=user\n' @@ -44,6 +47,9 @@ exit 0 fi if [[ "$started" =~ ^[1-9][0-9]*$ ]]; then printf 'service_start_available=true\n'; else printf 'service_start_available=false\n'; fi if [[ "$restarts" =~ ^[0-9]{1,6}$ ]]; then printf 'restart_count=%s\n' "$restarts"; else printf 'restart_count=unknown\n'; fi + case "$service_result" in success|exit-code|signal|core-dump|timeout|watchdog|start-limit-hit|resources|protocol|oom-kill) printf 'service_result=%s\n' "$service_result" ;; *) printf 'service_result=unknown\n' ;; esac + if [[ "$exit_code" =~ ^[0-3]$ ]]; then printf 'exit_code=%s\n' "$exit_code"; else printf 'exit_code=unknown\n'; fi + if [[ "$exit_status" =~ ^(0|[1-9][0-9]{0,2})$ ]] && ((exit_status <= 255)); then printf 'exit_status=%s\n' "$exit_status"; else printf 'exit_status=unknown\n'; fi """; internal static string RefusalCategory(string? message) @@ -92,7 +98,10 @@ internal static async Task CaptureAsync( snapshot.PidEqual, snapshot.ProcessStartAvailable, snapshot.ServiceStartAvailable, - snapshot.RestartCount + snapshot.RestartCount, + snapshot.ServiceResult, + snapshot.ExecMainCode, + snapshot.ExecMainStatus }; await File.WriteAllTextAsync(artifactPath, JsonSerializer.Serialize(artifact)); ctx.Logger.Info($"Pre-rollback gateway restart diagnostic: {JsonSerializer.Serialize(artifact)}"); @@ -109,7 +118,10 @@ internal sealed record Snapshot( bool? PidEqual = null, bool? ProcessStartAvailable = null, bool? ServiceStartAvailable = null, - int? RestartCount = null); + int? RestartCount = null, + string? ServiceResult = null, + int? ExecMainCode = null, + int? ExecMainStatus = null); internal static Snapshot Parse(CommandResult result) { @@ -128,7 +140,7 @@ internal static Snapshot Parse(CommandResult result) if (fields.Count == 1 && fields.GetValueOrDefault("probe") == "unavailable") return new("unavailable"); - if (fields.Count != 11 || fields.GetValueOrDefault("probe") != "ok" || + if (fields.Count != 14 || fields.GetValueOrDefault("probe") != "ok" || fields.GetValueOrDefault("scope") != "user" || !TryBoolean(fields, "unit_equal", out var unitEqual) || !TryBoolean(fields, "pid_present", out var pidPresent) || @@ -143,14 +155,27 @@ active is not ("active" or "inactive" or "failed" or "activating" or "deactivati sub is not ("running" or "exited" or "dead" or "failed" or "start-pre" or "start-post" or "auto-restart" or "stop-sigterm" or "stop-post" or "unknown") || !fields.TryGetValue("restart_count", out var restartCountText) || !(restartCountText == "unknown" || - int.TryParse(restartCountText, out var count) && count is >= 0 and <= 999999)) + int.TryParse(restartCountText, out var count) && count is >= 0 and <= 999999) || + !fields.TryGetValue("service_result", out var serviceResult) || + serviceResult is not ("success" or "exit-code" or "signal" or "core-dump" or + "timeout" or "watchdog" or "start-limit-hit" or "resources" or + "protocol" or "oom-kill" or "unknown") || + !fields.TryGetValue("exit_code", out var exitCodeText) || + !(exitCodeText == "unknown" || + int.TryParse(exitCodeText, out var exitCode) && exitCode is >= 0 and <= 3) || + !fields.TryGetValue("exit_status", out var exitStatusText) || + !(exitStatusText == "unknown" || + int.TryParse(exitStatusText, out var exitStatus) && exitStatus is >= 0 and <= 255)) return new("invalid_output"); return new Snapshot( "ok", "user", unitEqual, active, sub, pidPresent, pidLive, pidEqualText == "unknown" ? null : pidEqualText == "true", processStart, serviceStart, - restartCountText == "unknown" ? null : int.Parse(restartCountText)); + restartCountText == "unknown" ? null : int.Parse(restartCountText), + serviceResult, + exitCodeText == "unknown" ? null : int.Parse(exitCodeText), + exitStatusText == "unknown" ? null : int.Parse(exitStatusText)); } private static bool TryBoolean(Dictionary fields, string name, out bool value) diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs index 08afc5f68..3f670a888 100644 --- a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs @@ -41,6 +41,9 @@ public void ParsesOnlyAllowlistedCoarseFacts() process_start_available=true service_start_available=true restart_count=2 + service_result=exit-code + exit_code=1 + exit_status=78 """; var snapshot = GatewayRestartFailureDiagnostic.Parse(Result(raw)); Assert.Equal("ok", snapshot.Probe); @@ -49,9 +52,43 @@ public void ParsesOnlyAllowlistedCoarseFacts() Assert.True(snapshot.PidLive); Assert.False(snapshot.PidEqual); Assert.Equal(2, snapshot.RestartCount); + Assert.Equal("exit-code", snapshot.ServiceResult); + Assert.Equal(1, snapshot.ExecMainCode); + Assert.Equal(78, snapshot.ExecMainStatus); Assert.DoesNotContain("home", JsonSerializer.Serialize(snapshot)); } + [Theory] + [InlineData("service_result=private-path")] + [InlineData("exit_code=4")] + [InlineData("exit_status=256")] + public void RejectsOutOfRangeServiceExitFacts(string replacement) + { + const string raw = """ + probe=ok + scope=user + unit_equal=true + active=activating + sub=auto-restart + pid_present=false + pid_live=false + pid_equal=unknown + process_start_available=false + service_start_available=true + restart_count=0 + service_result=unknown + exit_code=unknown + exit_status=unknown + """; + var fieldName = replacement.Split('=')[0]; + var output = raw.Replace($"{fieldName}=unknown", replacement, StringComparison.Ordinal); + + var snapshot = GatewayRestartFailureDiagnostic.Parse(Result(output)); + + Assert.Equal("invalid_output", snapshot.Probe); + Assert.DoesNotContain(replacement, JsonSerializer.Serialize(snapshot)); + } + [Fact] public void RejectsUnexpectedOrUnvalidatedOutputWithoutPersistingIt() { @@ -68,6 +105,7 @@ public void ProbeRunsAsUserThroughStdinAndNeverPrintsRawStatus() { var script = GatewayRestartFailureDiagnostic.ProbeScript; Assert.Contains("systemctl --user show", script); + Assert.Contains("-p ExecMainStatus", script); Assert.Contains("read -r key value", script); Assert.DoesNotContain("printf '%s\\n' \"$properties\"", script); Assert.DoesNotContain("openclaw gateway restart", script); From c99d2d0c29c35c26107eac775e05e80029c7b296 Mon Sep 17 00:00:00 2001 From: Karen Lai <7976322+karkarl@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:43:19 -0700 Subject: [PATCH 3/5] fix(setup): retry guarded restart intent contention Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 504f71d1-79b6-4d69-abd3-cbb098ca9e0b --- docs/SETUP_ENGINE_REDESIGN.md | 11 +-- .../GatewayWizardRestartRecoveryPolicy.cs | 14 +++ src/OpenClaw.SetupEngine/SetupWizardRunner.cs | 27 ++++-- ...GatewayWizardRestartRecoveryPolicyTests.cs | 21 +++++ .../SetupStepsTests.cs | 88 ++++++++++++++++++- 5 files changed, 150 insertions(+), 11 deletions(-) diff --git a/docs/SETUP_ENGINE_REDESIGN.md b/docs/SETUP_ENGINE_REDESIGN.md index 03a1c0b8a..bef5cf577 100644 --- a/docs/SETUP_ENGINE_REDESIGN.md +++ b/docs/SETUP_ENGINE_REDESIGN.md @@ -18,15 +18,16 @@ timing explanation, not an established cause of every refusal. Observed service states differ: local diagnostics captured `activating/auto-restart` with no MainPID, while hosted generic refusals captured an `active/running` unit and a live PID. Neither snapshot establishes the admission-time owner-lease predicate. -`SetupWizardRunner` recognizes only the exact -serving-owner refusal, waits for verified managed endpoint ownership using the -existing bounded provenance probe, and retries the normal CLI restart once. +`SetupWizardRunner` recognizes only the exact serving-owner refusal or the +combination of typed state-database coordinator contention and the exact +restart-intent-recording refusal. It waits for verified managed endpoint +ownership using the existing bounded provenance probe, then retries the normal +guarded CLI restart once. The probe allows up to 30 one-second retry delays, plus probe duration, for `NoListener` and `UnknownListener` tagged `ListenerSnapshotChanged`. Other unknown/conflicting listeners, other restart errors, and a repeated refusal still fail setup. Listener provenance does not prove owner-lease or coordinator -readiness; the retried CLI command retains those guards. Restart-intent recording -contention is a separate failure and is not retried here. There is no direct +readiness; the retried CLI command retains those guards. There is no direct systemd restart fallback or ownership bypass. > **Status note (2026-07-06):** Current default setup includes `WindowsNodeBootstrapContextStep`, which injects Windows-node context into the WSL workspace `AGENTS.md` after onboarding. diff --git a/src/OpenClaw.SetupEngine/GatewayWizardRestartRecoveryPolicy.cs b/src/OpenClaw.SetupEngine/GatewayWizardRestartRecoveryPolicy.cs index f42a2eaec..60d668ccf 100644 --- a/src/OpenClaw.SetupEngine/GatewayWizardRestartRecoveryPolicy.cs +++ b/src/OpenClaw.SetupEngine/GatewayWizardRestartRecoveryPolicy.cs @@ -15,6 +15,12 @@ internal static class GatewayWizardRestartRecoveryPolicy /// public const string HostedWizardTerminationError = "Error: TUI exited from signal SIGTERM"; + public const string RestartIntentCoordinatorContentionError = + "StateDatabaseCoordinatorContentionError: another OpenClaw process owns state-lifecycle"; + public const string RestartIntentRecordingRefusal = + "GATEWAY_RESTART_PREPARATION_REFUSED: Cannot record restart intent for the serving Gateway. Gateway was not signaled."; + public static readonly TimeSpan RestartIntentContentionRetryDelay = + TimeSpan.FromMilliseconds(500); public static bool IsTerminalRestartCandidate( string? gatewayVersion, @@ -93,6 +99,14 @@ public static bool IsRestartLikeDisconnect(Exception exception) public static bool IsRetryableGatewayStartupDisconnect(int? closeStatusCode) => closeStatusCode == 1013; + public static bool IsRestartIntentCoordinatorContention(string? message) => + message?.Contains( + RestartIntentCoordinatorContentionError, + StringComparison.Ordinal) == true && + message.Contains( + RestartIntentRecordingRefusal, + StringComparison.Ordinal); + public static async Task WaitForExpectedManagedGatewayAsync( Func> inspectAsync, int noListenerRetryCount, diff --git a/src/OpenClaw.SetupEngine/SetupWizardRunner.cs b/src/OpenClaw.SetupEngine/SetupWizardRunner.cs index 8fef4b403..ea1b90617 100644 --- a/src/OpenClaw.SetupEngine/SetupWizardRunner.cs +++ b/src/OpenClaw.SetupEngine/SetupWizardRunner.cs @@ -616,13 +616,30 @@ internal async Task RestoreReloadModeAsync() await StartGatewayStep.RestartAndWaitForHealthAsync( _ctx, CancellationToken.None); - if (!restartResult.IsSuccess && - restartResult.Message?.Contains(RestartServingOwnerDiagnostic, StringComparison.Ordinal) == true) + var servingOwnerUnavailable = + !restartResult.IsSuccess && + restartResult.Message?.Contains( + RestartServingOwnerDiagnostic, + StringComparison.Ordinal) == true; + var restartIntentContention = + !restartResult.IsSuccess && + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + restartResult.Message); + if (servingOwnerUnavailable || restartIntentContention) { - // Restoring hybrid reload can initiate a supervisor restart before the CLI - // records its intent. Never bypass that CLI ownership gate or adopt a listener. _ctx.Logger.Warn( - "Gateway restart owner was unavailable after restoring reload. Rechecking managed ownership before one restart retry."); + restartIntentContention + ? "Gateway restart intent was refused by coordinator contention after restoring reload. Rechecking managed ownership before one guarded restart retry." + : "Gateway restart owner was unavailable after restoring reload. Rechecking managed ownership before one guarded restart retry."); + if (restartIntentContention) + { + await _restorationDelayAsync( + GatewayWizardRestartRecoveryPolicy.RestartIntentContentionRetryDelay, + CancellationToken.None); + } + + // Endpoint provenance rejects a foreign listener before retry. The retried + // Gateway CLI command remains responsible for coordinator and owner admission. var retryOwnership = await VerifyExpectedManagedGatewayAsync( "before retrying gateway restart"); if (!retryOwnership.IsSuccess) diff --git a/tests/OpenClaw.SetupEngine.Tests/GatewayWizardRestartRecoveryPolicyTests.cs b/tests/OpenClaw.SetupEngine.Tests/GatewayWizardRestartRecoveryPolicyTests.cs index 3fe08efd4..bdb16568b 100644 --- a/tests/OpenClaw.SetupEngine.Tests/GatewayWizardRestartRecoveryPolicyTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/GatewayWizardRestartRecoveryPolicyTests.cs @@ -83,6 +83,27 @@ public void RetryableGatewayStartupDisconnect_IsNarrow( closeStatusCode)); } + [Fact] + public void RestartIntentCoordinatorContention_RequiresBothExactMarkers() + { + var contention = + $"{GatewayWizardRestartRecoveryPolicy.RestartIntentCoordinatorContentionError}. " + + GatewayWizardRestartRecoveryPolicy.RestartIntentRecordingRefusal; + + Assert.True( + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + contention)); + Assert.False( + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + GatewayWizardRestartRecoveryPolicy.RestartIntentCoordinatorContentionError)); + Assert.False( + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + GatewayWizardRestartRecoveryPolicy.RestartIntentRecordingRefusal)); + Assert.False( + GatewayWizardRestartRecoveryPolicy.IsRestartIntentCoordinatorContention( + "another process owns state-lifecycle")); + } + [Theory] [InlineData("done")] [InlineData("Done")] diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs index 9c4c88803..0d133e1cf 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs @@ -4309,6 +4309,92 @@ CommandResult Restart() } } + [Fact] + public async Task SetupWizard_RestartIntentContentionReverifiesOwnershipBeforeOneRetry() + { + var restarts = 0; + var inspections = 0; + var delays = new List<(TimeSpan Delay, CancellationToken CancellationToken)>(); + var commands = new FakeCommandRunner( + _ => Ok(), + (_, command, _) => command switch + { + var value when value.Contains("config set gateway.reload.mode") => Ok(), + var value when value.Contains("openclaw gateway restart") => Restart(), + var value when value.Contains("curl -s") => Ok("200"), + _ => Fail($"Unexpected command: {command}"), + }); + var ctx = CreateContext(commands: commands); + ctx.DistroName = "test-distro"; + ctx.EndpointProvenanceProbe = (_, _) => + { + inspections++; + return Task.FromResult(new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + ctx.Config.GatewayPort)); + }; + + var runner = new SetupWizardRunner( + ctx, + (delay, cancellationToken) => + { + delays.Add((delay, cancellationToken)); + return Task.CompletedTask; + }); + + var result = await runner.RestoreReloadModeAsync(); + + Assert.True(result.IsSuccess, result.Message); + Assert.Equal(2, restarts); + Assert.Equal(2, inspections); + var retryDelay = Assert.Single(delays); + Assert.Equal( + GatewayWizardRestartRecoveryPolicy.RestartIntentContentionRetryDelay, + retryDelay.Delay); + Assert.False(retryDelay.CancellationToken.CanBeCanceled); + Assert.DoesNotContain(commands.WslCalls, call => call.Command.Contains("systemctl")); + return; + + CommandResult Restart() + { + if (++restarts == 1) + { + return Fail( + $"{GatewayWizardRestartRecoveryPolicy.RestartIntentCoordinatorContentionError}. " + + GatewayWizardRestartRecoveryPolicy.RestartIntentRecordingRefusal); + } + + Assert.Equal(1, inspections); + return Ok(); + } + } + + [Theory] + [InlineData(GatewayEndpointProvenanceKind.UnknownListener)] + [InlineData(GatewayEndpointProvenanceKind.ConflictingOpenClawGateway)] + public async Task SetupWizard_RestartIntentContentionDoesNotRetryAnUntrustedListener( + GatewayEndpointProvenanceKind kind) + { + var restartFailure = + $"{GatewayWizardRestartRecoveryPolicy.RestartIntentCoordinatorContentionError}. " + + GatewayWizardRestartRecoveryPolicy.RestartIntentRecordingRefusal; + var commands = new FakeCommandRunner( + _ => Ok(), + (_, command, _) => command.Contains("config set gateway.reload.mode") + ? Ok() + : Fail(restartFailure)); + var ctx = CreateContext(commands: commands); + ctx.DistroName = "test-distro"; + ctx.EndpointProvenanceProbe = (_, _) => Task.FromResult( + new GatewayEndpointProvenance(kind, ctx.Config.GatewayPort)); + + var result = await new SetupWizardRunner(ctx).RestoreReloadModeAsync(); + + Assert.False(result.IsSuccess); + Assert.Contains("ownership verification failed", result.Message); + Assert.Single(commands.WslCalls, call => call.Command.Contains("openclaw gateway restart")); + } + [Theory] [InlineData(GatewayEndpointProvenanceKind.UnknownListener)] [InlineData(GatewayEndpointProvenanceKind.ConflictingOpenClawGateway)] @@ -4335,7 +4421,7 @@ public async Task SetupWizard_RestartOwnerGapDoesNotRetryAnUntrustedListener( [Theory] [InlineData(SetupWizardRunner.RestartServingOwnerDiagnostic, 2)] [InlineData("GATEWAY_RESTART_PREPARATION_REFUSED: Cannot verify the selected service command.", 1)] - [InlineData("StateDatabaseCoordinatorContentionError: another OpenClaw process owns state-lifecycle. GATEWAY_RESTART_PREPARATION_REFUSED: Cannot record restart intent for the serving Gateway. Gateway was not signaled.", 1)] + [InlineData("StateDatabaseCoordinatorContentionError: another OpenClaw process owns state-lifecycle. GATEWAY_RESTART_PREPARATION_REFUSED: Cannot record restart intent for the serving Gateway. Gateway was not signaled.", 2)] [InlineData("Unrelated restart failure", 1)] public async Task SetupWizard_RestartRetryRemainsBoundedAndSpecific(string error, int expectedRestarts) { From 8af2e4b18d4c88a1eb40c46fa69986cc5fef3f5a Mon Sep 17 00:00:00 2001 From: Scott Hanselman Date: Mon, 28 Sep 2026 10:54:42 -0700 Subject: [PATCH 4/5] fix(test): suppress proc diagnostic stderr Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1cae2b28-e7ec-42b3-96d8-cddf8565243e --- .../Setup/GatewayRestartFailureDiagnostic.cs | 3 +-- .../GatewayRestartFailureDiagnosticTests.cs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs index c70193962..462bf40e9 100644 --- a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs @@ -35,8 +35,7 @@ exit 0 printf 'pid_present=true\n' if kill -0 "$pid" 2>/dev/null; then printf 'pid_live=true\n'; else printf 'pid_live=false\n'; fi if [[ "$exec_pid" == "$pid" ]]; then printf 'pid_equal=true\n'; else printf 'pid_equal=false\n'; fi - if [[ -r "/proc/$pid/stat" ]]; then - read -r stat < "/proc/$pid/stat" + if [[ -r "/proc/$pid/stat" ]] && read -r stat < "/proc/$pid/stat" 2>/dev/null; then read -r -a fields <<< "${stat##*) }" if [[ "${fields[19]:-}" =~ ^[1-9][0-9]*$ ]]; then printf 'process_start_available=true\n'; else printf 'process_start_available=false\n'; fi else diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs index 3f670a888..9340660ae 100644 --- a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs @@ -112,6 +112,22 @@ public void ProbeRunsAsUserThroughStdinAndNeverPrintsRawStatus() Assert.DoesNotContain("ps ", script); } + [Fact] + public void ProbeSuppressesFailedProcReadBeforeCommandLogging() + { + var script = GatewayRestartFailureDiagnostic.ProbeScript; + + Assert.Contains( + """if [[ -r "/proc/$pid/stat" ]] && read -r stat < "/proc/$pid/stat" 2>/dev/null; then""", + script); + Assert.DoesNotContain( + """ + read -r stat < "/proc/$pid/stat" + read -r -a fields + """, + script); + } + private static CommandResult Result(string stdout, int exitCode = 0, bool timedOut = false) => new(exitCode, stdout, "", TimeSpan.Zero, timedOut); } From f6fab60a5cc7049981b7df1cbe99348fa162179b Mon Sep 17 00:00:00 2001 From: Scott Hanselman Date: Mon, 28 Sep 2026 11:17:51 -0700 Subject: [PATCH 5/5] fix(test): order proc diagnostic redirections Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1cae2b28-e7ec-42b3-96d8-cddf8565243e --- .../Setup/GatewayRestartFailureDiagnostic.cs | 3 ++- .../Setup/GatewayRestartFailureDiagnosticTests.cs | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs index 462bf40e9..04a4e4728 100644 --- a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnostic.cs @@ -35,7 +35,8 @@ exit 0 printf 'pid_present=true\n' if kill -0 "$pid" 2>/dev/null; then printf 'pid_live=true\n'; else printf 'pid_live=false\n'; fi if [[ "$exec_pid" == "$pid" ]]; then printf 'pid_equal=true\n'; else printf 'pid_equal=false\n'; fi - if [[ -r "/proc/$pid/stat" ]] && read -r stat < "/proc/$pid/stat" 2>/dev/null; then + # Redirection order is intentional: silence stderr before opening the racy proc path. + if [[ -r "/proc/$pid/stat" ]] && read -r stat 2>/dev/null < "/proc/$pid/stat"; then read -r -a fields <<< "${stat##*) }" if [[ "${fields[19]:-}" =~ ^[1-9][0-9]*$ ]]; then printf 'process_start_available=true\n'; else printf 'process_start_available=false\n'; fi else diff --git a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs index 9340660ae..293a0b07c 100644 --- a/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs +++ b/tests/OpenClaw.E2ETests/Setup/GatewayRestartFailureDiagnosticTests.cs @@ -118,13 +118,10 @@ public void ProbeSuppressesFailedProcReadBeforeCommandLogging() var script = GatewayRestartFailureDiagnostic.ProbeScript; Assert.Contains( - """if [[ -r "/proc/$pid/stat" ]] && read -r stat < "/proc/$pid/stat" 2>/dev/null; then""", + """if [[ -r "/proc/$pid/stat" ]] && read -r stat 2>/dev/null < "/proc/$pid/stat"; then""", script); Assert.DoesNotContain( - """ - read -r stat < "/proc/$pid/stat" - read -r -a fields - """, + """read -r stat < "/proc/$pid/stat" 2>/dev/null""", script); }