From f4a30b8c13bab055be3a301a5312241c7253822c Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Sat, 12 Sep 2026 00:34:51 -0700 Subject: [PATCH 01/22] fix: replace local ai model during recovery --- .../LocalAi/LlamaServerRuntimeService.cs | 15 +++ .../LocalAi/LocalAiManifest.cs | 69 +++++++++--- .../LocalAiGatewayConfiguration.cs | 104 ++++++++++++++++-- .../LocalAiInstallReconciler.cs | 84 +++++++++++++- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 72 +++++++++++- src/OpenClaw.SetupEngine/SetupContext.cs | 1 + src/OpenClaw.SetupEngine/SetupPipeline.cs | 1 + .../LocalAiGatewayUninstallTests.cs | 104 +++++++++++++++--- .../LocalAiInstallRecoveryTests.cs | 94 +++++++++++++++- .../SetupPipelineTests.cs | 5 +- .../LocalAiSetupUxContractTests.cs | 4 +- 11 files changed, 503 insertions(+), 50 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs index d3a4032a8..cdac2ddcc 100644 --- a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs +++ b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs @@ -12,6 +12,7 @@ // using OpenClaw.Shared; using OpenClaw.Shared.Inference.Catalog; +using System.Collections.Immutable; using System.Net; using System.Text; @@ -493,6 +494,7 @@ recoveryPort is not null { LocalAiInstallManifest verifiedManifest = install.Manifest with { + PreviousEndpoints = ReplacementEndpointHistory(install, ownership.Endpoint), Endpoint = ownership.Endpoint.AbsoluteUri, }; await _manifestStore.SaveAsync(verifiedManifest, cancellationToken).ConfigureAwait(false); @@ -866,6 +868,7 @@ private async Task BindVerifiedEndpointAsync( LocalAiInstallManifest verifiedManifest = install.Manifest with { + PreviousEndpoints = ReplacementEndpointHistory(install, endpoint), Endpoint = endpoint.AbsoluteUri, }; await _manifestStore.SaveAsync(verifiedManifest, cancellationToken).ConfigureAwait(false); @@ -873,6 +876,18 @@ private async Task BindVerifiedEndpointAsync( return _install; } + private static ImmutableArray ReplacementEndpointHistory( + LocalAiResolvedInstall install, + Uri endpoint) + { + ImmutableArray history = install.Manifest.PreviousEndpoints; + string? previous = install.Endpoint?.AbsoluteUri; + return install.Manifest.ReplacedManifest is not null && install.Endpoint != endpoint && + previous is not null && !history.Contains(previous, StringComparer.Ordinal) + ? history.Add(previous) + : history; + } + private async Task TryLoadInstallAsync(CancellationToken cancellationToken) { try diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index 895ab908c..ff11363af 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -252,6 +252,14 @@ public sealed record LocalAiInstallManifest /// managed llama.cpp model. Null means no prior primary model was configured. /// public string? GatewayFallbackModel { get; init; } + /// + /// The last committed receipt while a recovery flow replaces its model. + /// Cleared only after the Gateway has restarted on the replacement route. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LocalAiInstallManifest? ReplacedManifest { get; init; } + /// Earlier verified replacement endpoints that may still be published to the Gateway. + public ImmutableArray PreviousEndpoints { get; init; } = []; public required int ContextLength { get; init; } public KvCachePrecision KeyCachePrecision { get; init; } = KvCachePrecision.F16; public KvCachePrecision ValueCachePrecision { get; init; } = KvCachePrecision.F16; @@ -632,30 +640,59 @@ LocalAiInstallManifest.HubCacheReceiptSchemaVersion or LocalAiPortPolicy.Validate(manifest.RequestedPort); LocalAiGatewayModelPolicy.ValidateFallbackModel(manifest.GatewayFallbackModel); - Uri? endpoint = null; - if (manifest.Endpoint is not null) - { - if (!Uri.TryCreate(manifest.Endpoint, UriKind.Absolute, out endpoint) || - endpoint.Scheme != Uri.UriSchemeHttp || - !string.Equals(endpoint.Host, "127.0.0.1", StringComparison.Ordinal) || - endpoint.IsDefaultPort || - endpoint.Port is <= 0 or > 65535 || - endpoint.Port == 80 || - !string.IsNullOrEmpty(endpoint.UserInfo) || - !string.IsNullOrEmpty(endpoint.Query) || - !string.IsNullOrEmpty(endpoint.Fragment) || - !string.Equals(endpoint.AbsolutePath, "/v1", StringComparison.Ordinal)) + if (manifest.ReplacedManifest is { } replaced) + { + if (replaced.ReplacedManifest is not null || + string.Equals(replaced.ModelCatalogId, manifest.ModelCatalogId, StringComparison.Ordinal) || + !string.Equals(replaced.Engine, manifest.Engine, StringComparison.Ordinal) || + !string.Equals(replaced.EngineVersion, manifest.EngineVersion, StringComparison.Ordinal) || + !string.Equals(replaced.Architecture, manifest.Architecture, StringComparison.Ordinal) || + !string.Equals(replaced.RuntimeId, manifest.RuntimeId, StringComparison.Ordinal) || + !string.Equals(replaced.ExecutablePath, manifest.ExecutablePath, StringComparison.Ordinal) || + !replaced.RuntimeAssets.SequenceEqual(manifest.RuntimeAssets) || + replaced.RequestedPort != manifest.RequestedPort) { - throw new InvalidDataException("The local AI endpoint must be an HTTP IPv4 loopback /v1 address with an explicit non-reserved port."); + throw new InvalidDataException("The local AI model replacement receipt is invalid."); } + _ = ResolveAndValidate(replaced); + } + else if (!manifest.PreviousEndpoints.IsDefaultOrEmpty) + { + throw new InvalidDataException("Previous Local AI endpoints require a pending model replacement."); + } - if (manifest.RequestedPort != LocalAiPortPolicy.Automatic && endpoint.Port != manifest.RequestedPort) - throw new InvalidDataException("The verified Local AI endpoint does not match its requested fixed port."); + Uri? endpoint = ValidateEndpoint(manifest.Endpoint, manifest.RequestedPort); + HashSet previousEndpoints = manifest.PreviousEndpoints.ToHashSet(StringComparer.Ordinal); + if (previousEndpoints.Count != manifest.PreviousEndpoints.Length) + { + throw new InvalidDataException("Previous Local AI endpoints must be unique."); } + foreach (string previousEndpoint in previousEndpoints) + _ = ValidateEndpoint(previousEndpoint, manifest.RequestedPort); return new LocalAiResolvedInstall(manifest, executable, model, endpoint); } + private static Uri? ValidateEndpoint(string? value, int requestedPort) + { + if (value is null) + return null; + if (!Uri.TryCreate(value, UriKind.Absolute, out Uri? endpoint) || + endpoint.Scheme != Uri.UriSchemeHttp || + !string.Equals(endpoint.Host, "127.0.0.1", StringComparison.Ordinal) || + endpoint.IsDefaultPort || endpoint.Port is <= 0 or > 65535 || endpoint.Port == 80 || + !string.IsNullOrEmpty(endpoint.UserInfo) || + !string.IsNullOrEmpty(endpoint.Query) || + !string.IsNullOrEmpty(endpoint.Fragment) || + !string.Equals(endpoint.AbsolutePath, "/v1", StringComparison.Ordinal)) + { + throw new InvalidDataException("The local AI endpoint must be an HTTP IPv4 loopback /v1 address with an explicit non-reserved port."); + } + if (requestedPort != LocalAiPortPolicy.Automatic && endpoint.Port != requestedPort) + throw new InvalidDataException("The verified Local AI endpoint does not match its requested fixed port."); + return endpoint; + } + private static void ValidateModelPath( string path, LocalAiAssetReceipt asset, diff --git a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs index 75149de04..9073134e1 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs @@ -113,38 +113,67 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati bool retainedManagedPrimary = !prior.ProviderExisted && prior.PrimaryModelExisted && JsonEquals(prior.PrimaryModelJson!, expectedPrimary); + LocalAiResolvedInstall? recoveryInstall = ctx.LocalAiRecoveryOriginalInstall; + bool retainedRecoveryPrimary = !retainedManagedPrimary && + !prior.ProviderExisted && + prior.PrimaryModelExisted && + recoveryInstall is not null && + JsonEquals( + prior.PrimaryModelJson!, + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(recoveryInstall))); string? fallbackModel; bool recoveryProviderTransition = false; if (prior.ProviderExisted) { bool matchesCurrentInstall = install.Endpoint is not null && LocalAiGatewayProviderDefinition.MatchesProviderJson(prior.ProviderJson!, install); + LocalAiResolvedInstall? pendingRoute = MatchPendingRoute( + ctx.LocalAiRecoveryPendingInstall, + prior.ProviderJson!); + bool matchesPendingInstall = !matchesCurrentInstall && + pendingRoute is not null && prior.PrimaryModelExisted && + JsonEquals(prior.PrimaryModelJson!, expectedPrimary); bool matchesRecoveryInstall = false; - if (!matchesCurrentInstall && + if (!matchesCurrentInstall && !matchesPendingInstall && ctx.LocalAiRecoveryOriginalInstall is { Endpoint: not null } originalInstall) { string originalPrimary = JsonSerializer.Serialize( LocalAiGatewayProviderDefinition.BuildPrimaryModel(originalInstall)); matchesRecoveryInstall = + prior.PrimaryModelExisted && LocalAiGatewayProviderDefinition.MatchesProviderJson( prior.ProviderJson!, originalInstall) && - JsonEquals(originalPrimary, expectedPrimary); + JsonEquals(prior.PrimaryModelJson!, originalPrimary); } - if ((!matchesCurrentInstall && !matchesRecoveryInstall) || + if ((!matchesCurrentInstall && !matchesPendingInstall && !matchesRecoveryInstall) || !prior.PrimaryModelExisted || - !JsonEquals(prior.PrimaryModelJson!, expectedPrimary)) + (matchesCurrentInstall && !JsonEquals(prior.PrimaryModelJson!, expectedPrimary))) { return StepResult.Fail( "The existing llamacpp gateway route is not the exact companion-managed configuration; preserving it."); } - recoveryProviderTransition = matchesRecoveryInstall; + if (matchesCurrentInstall && install.Manifest.ReplacedManifest is not null) + { + // A previous process already published the replacement. Rollback must + // preserve that live route instead of reconstructing the older one. + ctx.LocalAiRecoveryOriginalInstall = null; + ctx.LocalAiRecoveryProviderTransition = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + } + else if (matchesPendingInstall) + { + ctx.LocalAiRecoveryOriginalInstall = pendingRoute; + } + recoveryProviderTransition = matchesRecoveryInstall || matchesPendingInstall; fallbackModel = install.Manifest.GatewayFallbackModel; } - else if (retainedManagedPrimary) + else if (retainedManagedPrimary || retainedRecoveryPrimary) { - fallbackModel = install.Manifest.GatewayFallbackModel; + fallbackModel = retainedRecoveryPrimary + ? recoveryInstall!.Manifest.GatewayFallbackModel + : install.Manifest.GatewayFallbackModel; } else if (prior.PrimaryModelExisted) { @@ -420,6 +449,47 @@ private static async Task RemoveManagedStateForUninstallAsync( LocalAiGatewayPriorState current = ParseSnapshot(currentResult.Stdout); if (!current.ProviderExisted && !current.PrimaryModelExisted) return; + + var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); + foreach (string previousEndpoint in install.Manifest.PreviousEndpoints) + { + LocalAiResolvedInstall previous = store.ResolveAndValidate(install.Manifest with + { + Endpoint = previousEndpoint, + }); + if (current.ProviderExisted && current.PrimaryModelExisted && + LocalAiGatewayProviderDefinition.MatchesProviderJson(current.ProviderJson!, previous) && + JsonEquals( + current.PrimaryModelJson!, + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(previous)))) + { + install = previous; + break; + } + } + + if (install.Manifest.ReplacedManifest is { } replacedManifest) + { + LocalAiResolvedInstall replaced = store.ResolveAndValidate(replacedManifest); + bool previousProvider = current.ProviderExisted && + replaced.Endpoint is not null && + LocalAiGatewayProviderDefinition.MatchesProviderJson(current.ProviderJson!, replaced); + bool previousPrimary = current.PrimaryModelExisted && + JsonEquals( + current.PrimaryModelJson!, + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(replaced))); + if (previousProvider || previousPrimary) + { + if ((current.ProviderExisted && !previousProvider) || + (current.PrimaryModelExisted && !previousPrimary)) + { + throw new InvalidDataException( + "Local AI gateway settings contain a mixed model replacement route; preserving them."); + } + install = replaced; + } + } + if (install.Endpoint is null) { throw new InvalidDataException( @@ -605,6 +675,26 @@ private static string ExtractOperationValue(string batchJson, int index) return document.RootElement[index].GetProperty("value").GetRawText(); } + private static LocalAiResolvedInstall? MatchPendingRoute( + LocalAiResolvedInstall? pending, + string providerJson) + { + if (pending?.Endpoint is not null && + LocalAiGatewayProviderDefinition.MatchesProviderJson(providerJson, pending)) + return pending; + foreach (string endpoint in pending?.Manifest.PreviousEndpoints ?? []) + { + LocalAiResolvedInstall previous = pending! with + { + Manifest = pending.Manifest with { Endpoint = endpoint }, + Endpoint = new Uri(endpoint), + }; + if (LocalAiGatewayProviderDefinition.MatchesProviderJson(providerJson, previous)) + return previous; + } + return null; + } + private static bool JsonEquals(string left, string right) { using JsonDocument leftDocument = JsonDocument.Parse(left); diff --git a/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs b/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs index 44bed86c9..46d89b7e9 100644 --- a/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs +++ b/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs @@ -10,7 +10,8 @@ internal sealed record LocalAiReconcileResult( LlamaRuntimeInstallResult? RuntimeInstall, HuggingFaceModelInstallResult? ModelInstall, LocalAiResolvedInstall? OriginalInstall = null, - ImmutableArray? AdditionalModelInstalls = null) + ImmutableArray? AdditionalModelInstalls = null, + LocalAiResolvedInstall? PendingReplacement = null) { public static LocalAiReconcileResult NotInstalled { get; } = new(false, null, null, null); @@ -157,8 +158,44 @@ public async Task ReconcileAsync( .ConfigureAwait(false); if (install is null) return LocalAiReconcileResult.NotInstalled; - LocalAiResolvedInstall originalInstall = install; - bool runtimeUpgradePending = ValidateRecipeMatch(install, plan, selectedGpuId, localDataDirectory); + LocalAiResolvedInstall originalInstall = install.Manifest.ReplacedManifest is { } replacedManifest + ? manifestStore.ResolveAndValidate(replacedManifest) + : install; + LocalAiResolvedInstall? pendingReplacement = install.Manifest.ReplacedManifest is null ? null : install; + bool replacingModel = !string.Equals( + install.Manifest.ModelCatalogId, + plan.Model.Id, + StringComparison.Ordinal); + if (install.Manifest.ReplacedManifest is not null && replacingModel) + { + throw new InvalidDataException( + "Complete the pending Local AI model replacement before selecting another model."); + } + if (replacingModel) + { + if (!allowIncompleteInstallation) + { + throw new InvalidDataException( + "The existing managed Local AI installation does not match the selected runtime, GPU, and model recipe."); + } + + ValidateReplacementSource(install, plan, selectedGpuId, localDataDirectory); + LlamaRuntimeInspection replacementRuntime = await _runtimeInspector + .InspectAsync(Path.GetDirectoryName(install.ExecutablePath)!, cancellationToken) + .ConfigureAwait(false); + return new LocalAiReconcileResult( + Reused: false, + ResolvedInstall: null, + RuntimeInstall: replacementRuntime.IsValid ? CreateRuntimeInstall(install) : null, + ModelInstall: null, + OriginalInstall: originalInstall, + PendingReplacement: pendingReplacement); + } + bool runtimeUpgradePending = ValidateRecipeMatch( + install, + plan, + selectedGpuId, + localDataDirectory); bool migrateLegacyGpuId = !string.Equals(install.Manifest.SelectedGpuId, selectedGpuId, StringComparison.Ordinal) && @@ -239,7 +276,8 @@ public async Task ReconcileAsync( RuntimeInstall: inspection.IsValid ? CreateRuntimeInstall(install) : null, ModelInstall: modelIsValid ? CreateModelInstall(install, localDataDirectory) : null, OriginalInstall: originalInstall, - AdditionalModelInstalls: modelIsValid ? CreateAdditionalModelInstalls(install) : null); + AdditionalModelInstalls: modelIsValid ? CreateAdditionalModelInstalls(install) : null, + PendingReplacement: pendingReplacement); } install = await MigrateLegacyModelAsync( @@ -268,7 +306,43 @@ public async Task ReconcileAsync( CreateRuntimeInstall(install), CreateModelInstall(install, localDataDirectory), OriginalInstall: allowIncompleteInstallation ? originalInstall : null, - AdditionalModelInstalls: CreateAdditionalModelInstalls(install)); + AdditionalModelInstalls: CreateAdditionalModelInstalls(install), + PendingReplacement: pendingReplacement); + } + + private static void ValidateReplacementSource( + LocalAiResolvedInstall install, + LocalInferencePlan plan, + string selectedGpuId, + string localDataDirectory) + { + // Validate the existing receipt against its own catalog model before using it + // as durable rollback provenance for the newly selected model. + _ = LlamaServerRouterConfiguration.Build(new LocalAiPaths(localDataDirectory), install); + + string expectedArchitecture = plan.Runtime.Architecture switch + { + System.Runtime.InteropServices.Architecture.X64 => "x64", + System.Runtime.InteropServices.Architecture.Arm64 => "arm64", + _ => throw new InvalidDataException("The selected Local AI runtime architecture is unsupported."), + }; + LocalAiComponentIdentity component = LlamaRuntimeInstaller.Component(plan.Runtime); + if (!string.Equals(install.Manifest.RuntimeId, plan.Runtime.Id, StringComparison.Ordinal) || + !string.Equals(install.Manifest.Architecture, expectedArchitecture, StringComparison.Ordinal) || + !GpuIdsMatch(install.Manifest.SelectedGpuId, selectedGpuId) || + !LocalAiPathPolicy.TryResolve( + localDataDirectory, + component, + out LocalAiSetupPaths setupPaths, + out _) || + !string.Equals( + Path.GetDirectoryName(install.ExecutablePath), + setupPaths.InstallDirectory, + StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidDataException( + "The existing managed Local AI installation cannot reuse the selected runtime and GPU."); + } } /// Read-only inspection for onboarding; unlike reconciliation, never migrates or saves receipts. diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 085e14040..72f60c206 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -283,7 +283,10 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati (result.OriginalInstall ?? result.ResolvedInstall) is { } originalInstall) { ctx.LocalAiRecoveryOriginalInstall = originalInstall; - ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryPendingInstall = result.PendingReplacement; + bool pendingReplacement = result.PendingReplacement is not null; + ctx.LocalAiRecoveryProviderTransition = pendingReplacement; + ctx.LocalAiRecoveryReceiptRollbackAllowed = !pendingReplacement; } if (!result.Reused) { @@ -550,6 +553,11 @@ or InvalidDataException public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) { ct.ThrowIfCancellationRequested(); + if (ctx.LocalAiRecoveryProviderTransition && + !ctx.LocalAiRecoveryReceiptRollbackAllowed) + { + return Task.CompletedTask; + } if (ctx.LocalAiModelInstall is { } install) { _acquirer.RemoveInstalledModel(ctx.LocalDataDir, install); @@ -716,6 +724,10 @@ originalInstall is not null && }; if (originalInstall is not null) { + bool replacingModel = !string.Equals( + originalInstall.Manifest.ModelCatalogId, + manifest.ModelCatalogId, + StringComparison.Ordinal); manifest = originalInstall.Manifest with { SchemaVersion = manifest.SchemaVersion, @@ -741,6 +753,15 @@ originalInstall is not null && ValueCachePrecision = manifest.ValueCachePrecision, DraftKeyCachePrecision = manifest.DraftKeyCachePrecision, DraftValueCachePrecision = manifest.DraftValueCachePrecision, + GatewayFallbackModel = ctx.LocalAiRecoveryPendingInstall is { } pendingInstall + ? pendingInstall.Manifest.GatewayFallbackModel + : originalInstall.Manifest.GatewayFallbackModel, + InstalledAtUtc = ctx.LocalAiRecoveryPendingInstall?.Manifest.InstalledAtUtc ?? + originalInstall.Manifest.InstalledAtUtc, + ReplacedManifest = replacingModel + ? originalInstall.Manifest.ReplacedManifest ?? originalInstall.Manifest + : null, + PreviousEndpoints = ReplacementEndpointHistory(ctx.LocalAiRecoveryPendingInstall), }; } @@ -750,6 +771,11 @@ originalInstall is not null && await store.SaveAsync(manifest, ct); ctx.LocalAiResolvedInstall = store.ResolveAndValidate(manifest); ctx.LocalAiManifestCreatedThisRun = !replacesExistingReceipt; + if (manifest.ReplacedManifest is not null) + { + ctx.LocalAiRecoveryProviderTransition = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + } return StepResult.Ok("Recorded the verified llama-server and Hugging Face installation."); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) @@ -847,6 +873,50 @@ private static ImmutableArray BuildRuntimeReceipts( return receipts.MoveToImmutable(); } + + private static ImmutableArray ReplacementEndpointHistory(LocalAiResolvedInstall? pending) + { + if (pending?.Endpoint is null || + pending.Manifest.PreviousEndpoints.Contains(pending.Endpoint.AbsoluteUri, StringComparer.Ordinal)) + { + return pending?.Manifest.PreviousEndpoints ?? []; + } + return pending.Manifest.PreviousEndpoints.Add(pending.Endpoint.AbsoluteUri); + } +} + +/// Commits a model replacement after its Gateway route has restarted successfully. +public sealed class FinalizeLocalAiModelReplacementStep : SetupStep +{ + public override string Id => "finalize-local-ai-model-replacement"; + public override string DisplayName => "Finalizing Local AI model replacement"; + public override bool CanRetry => false; + public override RetryPolicy Retry => RetryPolicy.None; + public override bool CanSkip(SetupContext ctx) => + ctx.LocalAiResolvedInstall?.Manifest.ReplacedManifest is null; + + public override async Task ExecuteAsync(SetupContext ctx, CancellationToken ct) + { + LocalAiResolvedInstall install = ctx.LocalAiResolvedInstall!; + LocalAiInstallManifest committed = install.Manifest with + { + ReplacedManifest = null, + PreviousEndpoints = [], + }; + try + { + var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); + await store.SaveAsync(committed, ct).ConfigureAwait(false); + ctx.LocalAiResolvedInstall = store.ResolveAndValidate(committed); + ctx.LocalAiRecoveryProviderTransition = false; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + return StepResult.Ok("Local AI model replacement is committed."); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + return StepResult.Fail("The Local AI model replacement could not be finalized.", ex); + } + } } /// Starts the companion-owned llama-server router without preloading a model. diff --git a/src/OpenClaw.SetupEngine/SetupContext.cs b/src/OpenClaw.SetupEngine/SetupContext.cs index ef06766fc..63975d6ed 100644 --- a/src/OpenClaw.SetupEngine/SetupContext.cs +++ b/src/OpenClaw.SetupEngine/SetupContext.cs @@ -583,6 +583,7 @@ public Func>? : null; internal LocalAiResolvedInstall? LocalAiRecoveryOriginalInstall { get; set; } internal LocalAiResolvedInstall? LocalAiUpgradeOriginalInstall { get; set; } + internal LocalAiResolvedInstall? LocalAiRecoveryPendingInstall { get; set; } internal bool LocalAiRecoveryProviderTransition { get; set; } internal bool LocalAiRecoveryReceiptRollbackAllowed { get; set; } internal bool LocalAiManifestCreatedThisRun { get; set; } diff --git a/src/OpenClaw.SetupEngine/SetupPipeline.cs b/src/OpenClaw.SetupEngine/SetupPipeline.cs index 4a267c8da..2959a5923 100644 --- a/src/OpenClaw.SetupEngine/SetupPipeline.cs +++ b/src/OpenClaw.SetupEngine/SetupPipeline.cs @@ -118,6 +118,7 @@ public static List BuildLocalAiRecoverySteps() => new VerifyLocalAiWslStep(), new ConfigureLocalAiGatewayStep(), new RestartGatewayStep(), + new FinalizeLocalAiModelReplacementStep(), ]; public static List BuildDefaultSteps() diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index b55506ada..57d93620d 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -58,14 +58,33 @@ public async Task Repair_RollbackUnsetsPrimaryAfterRetainedEndpointCycleWithoutF Assert.Null(commands.PrimaryJson); } - [Fact] - public async Task FreshProcessUninstall_RemovesExactManagedProviderAndPrimary() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task FreshProcessUninstall_RemovesExactManagedProviderAndPrimary(bool pendingReplacement) { using var temp = new TempDirectory("local-ai-gateway-uninstall-"); - LocalAiResolvedInstall install = await SaveManifestAsync(temp.Path); - string provider = LocalAiGatewayProviderDefinition.BuildProviderJson(install); + LocalAiResolvedInstall routed = await SaveManifestAsync(temp.Path); + if (pendingReplacement) + { + LocalAiInstallManifest published = routed.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39876/v1", + ReplacedManifest = routed.Manifest, + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + routed = store.ResolveAndValidate(published); + await store.SaveAsync(published with + { + Endpoint = "http://127.0.0.1:39877/v1", + PreviousEndpoints = [published.Endpoint!], + }); + } + string provider = LocalAiGatewayProviderDefinition.BuildProviderJson(routed); string primary = JsonSerializer.Serialize( - LocalAiGatewayProviderDefinition.BuildPrimaryModel(install)); + LocalAiGatewayProviderDefinition.BuildPrimaryModel(routed)); var commands = new GatewayStateCommandRunner(provider, primary); SetupContext context = CreateContext(temp.Path, commands); context.IsUninstalling = true; @@ -171,30 +190,89 @@ public async Task Recovery_ReplacesExactManagedProviderAfterAutomaticPortChanges { using var temp = new TempDirectory("local-ai-gateway-recovery-"); LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); - string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + LocalAiInstallManifest publishedManifest = original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39878/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = + [ + "http://127.0.0.1:39876/v1", + "http://127.0.0.1:39877/v1", + ], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(publishedManifest); + LocalAiResolvedInstall published = (await store.LoadAsync())!; + LocalAiResolvedInstall publishedRoute = published with + { + Manifest = published.Manifest with { Endpoint = published.Manifest.PreviousEndpoints[1] }, + Endpoint = new Uri(published.Manifest.PreviousEndpoints[1]), + }; + string publishedProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(publishedRoute); string primary = JsonSerializer.Serialize( - LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); - var commands = new GatewayStateCommandRunner(originalProvider, primary); + LocalAiGatewayProviderDefinition.BuildPrimaryModel(published)); + var commands = new GatewayStateCommandRunner(publishedProvider, primary); SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryPendingInstall = published; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with + LocalAiInstallManifest replacementManifest = publishedManifest with { - Endpoint = "http://127.0.0.1:39876/v1", + Endpoint = "http://127.0.0.1:39879/v1", + PreviousEndpoints = publishedManifest.PreviousEndpoints.Add(publishedManifest.Endpoint!), }; - var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); - context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); + context.LocalAiResolvedInstall = (await store.LoadAsync())!; var step = new ConfigureLocalAiGatewayStep(); StepResult result = await step.ExecuteAsync(context, CancellationToken.None); Assert.Equal(StepOutcome.Success, result.Outcome); Assert.True(context.LocalAiRecoveryProviderTransition); + Assert.Equal(replacementManifest.Endpoint, (await store.LoadAsync())!.Manifest.Endpoint); Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson( commands.ProviderJson!, context.LocalAiResolvedInstall)); - Assert.Equal(primary, commands.PrimaryJson); + Assert.Equal( + JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(context.LocalAiResolvedInstall)), + commands.PrimaryJson); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Recovery_ReplacementAcceptsProviderlessOriginalPrimary(bool retainedManagedPrimary) + { + const string fallback = "openai/gpt-5"; + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync( + temp.Path, + retainedManagedPrimary ? fallback : null); + string primary = JsonSerializer.Serialize(retainedManagedPrimary + ? LocalAiGatewayProviderDefinition.BuildPrimaryModel(original) + : fallback); + var commands = new GatewayStateCommandRunner(providerJson: null, primary); + SetupContext context = CreateRecoveryContext(temp.Path, commands); + context.LocalAiRecoveryOriginalInstall = original; + LocalAiInstallManifest replacement = original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39876/v1", + ReplacedManifest = original.Manifest, + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(replacement); + context.LocalAiResolvedInstall = store.ResolveAndValidate(replacement); + + StepResult result = await new ConfigureLocalAiGatewayStep() + .ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.Equal(fallback, (await store.LoadAsync())!.Manifest.GatewayFallbackModel); } [Fact] diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index 206db4311..664698448 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1490,7 +1490,43 @@ await Assert.ThrowsAsync(() => } [Fact] - public async Task Reconciler_RecoveryRetainsReceiptWhileMissingModelIsRepaired() + public async Task Reconciler_RecoveryReusesRuntimeWhenSelectedModelChanges() + { + using var temp = new TempDirectory(); + LocalInferencePlan installed = CatalogPlan(); + LocalModelInfo replacementModel = LocalModelCatalog.Models.First( + model => model.Id != installed.Model.Id); + var replacement = new LocalInferencePlan( + installed.Runtime, + replacementModel, + LocalModelCatalog.GetProfiles(replacementModel)[0], + LocalInferenceModelSelectionOrigin.Explicit); + const string gpuId = "GPU-0"; + var paths = new LocalAiPaths(temp.Path); + LocalAiInstallManifest manifest = CreateManifest(temp.Path, installed, gpuId); + await new LocalAiManifestStore(paths).SaveAsync(manifest); + + LocalAiReconcileResult result = await new LocalAiInstallReconciler( + new ValidRuntimeInspector(), + new AcceptingModelVerifier()) + .ReconcileAsync( + temp.Path, + replacement, + gpuId, + CancellationToken.None, + allowIncompleteInstallation: true); + + Assert.False(result.Reused); + Assert.False(result.RuntimeInstall!.CreatedThisRun); + Assert.Null(result.ModelInstall); + Assert.Equal(manifest.ModelCatalogId, result.OriginalInstall?.Manifest.ModelCatalogId); + Assert.Null((await new LocalAiManifestStore(paths).LoadAsync())!.Manifest.ReplacedManifest); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Reconciler_RecoveryRetainsReceiptWhileMissingModelIsRepaired(bool pendingReplacement) { using var temp = new TempDirectory(); LocalInferencePlan plan = CatalogPlan(); @@ -1498,6 +1534,16 @@ public async Task Reconciler_RecoveryRetainsReceiptWhileMissingModelIsRepaired() var paths = new LocalAiPaths(temp.Path); var store = new LocalAiManifestStore(paths); LocalAiInstallManifest manifest = CreateManifest(temp.Path, plan, gpuId); + if (pendingReplacement) + { + LocalModelInfo priorModel = LocalModelCatalog.Models.First(model => model.Id != plan.Model.Id); + var priorPlan = new LocalInferencePlan( + plan.Runtime, + priorModel, + LocalModelCatalog.GetProfiles(priorModel)[0], + LocalInferenceModelSelectionOrigin.Explicit); + manifest = manifest with { ReplacedManifest = CreateManifest(temp.Path, priorPlan, gpuId) }; + } await store.SaveAsync(manifest); var reconciler = new LocalAiInstallReconciler( new ValidRuntimeInspector(), @@ -1515,9 +1561,36 @@ public async Task Reconciler_RecoveryRetainsReceiptWhileMissingModelIsRepaired() Assert.Equal(manifest.Endpoint, result.OriginalInstall!.Manifest.Endpoint); Assert.NotNull(result.RuntimeInstall); Assert.Null(result.ModelInstall); + Assert.Equal(pendingReplacement, result.PendingReplacement is not null); Assert.True(File.Exists(paths.ManifestPath)); } + [Fact] + public async Task FinalizeReplacement_ClearsRollbackReceipt() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pending); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pending); + + StepResult result = await new FinalizeLocalAiModelReplacementStep() + .ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + LocalAiInstallManifest committed = (await store.LoadAsync())!.Manifest; + Assert.Null(committed.ReplacedManifest); + Assert.Empty(committed.PreviousEndpoints); + } + [Fact] public async Task ReconcileStep_RecoveryPinsIncompleteReceiptAsRollbackBaseline() { @@ -1552,8 +1625,10 @@ public async Task ReconcileStep_RecoveryPinsIncompleteReceiptAsRollbackBaseline( Assert.Null(context.LocalAiModelInstall); } - [Fact] - public async Task RecoveryPipeline_RewritesIncompleteReceiptAfterModelRepair() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RecoveryPipeline_RewritesIncompleteReceiptAfterModelRepair(bool pendingReplacement) { using var temp = new TempDirectory(); LocalInferencePlan plan = CatalogPlan(); @@ -1563,6 +1638,19 @@ public async Task RecoveryPipeline_RewritesIncompleteReceiptAfterModelRepair() RequestedPort = 18803, GatewayFallbackModel = "openai/gpt-5", }; + if (pendingReplacement) + { + LocalModelInfo priorModel = LocalModelCatalog.Models.First(model => model.Id != plan.Model.Id); + var priorPlan = new LocalInferencePlan( + plan.Runtime, + priorModel, + LocalModelCatalog.GetProfiles(priorModel)[0], + LocalInferenceModelSelectionOrigin.Explicit); + manifest = manifest with + { + ReplacedManifest = CreateManifest(temp.Path, priorPlan, gpuId) with { RequestedPort = 18803 }, + }; + } var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(manifest); LocalAiResolvedInstall original = (await store.LoadAsync())!; diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index c74b2d256..2d8755b1c 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -207,8 +207,9 @@ public void BuildLocalAiRecoverySteps_PreservesExistingWslGateway() Assert.Contains(steps, step => step is AcquireLocalAiRuntimeStep); Assert.Contains(steps, step => step is AcquireLocalAiModelStep); Assert.Contains(steps, step => step is VerifyLocalAiWslStep); - Assert.IsType(steps[^2]); - Assert.IsType(steps[^1]); + Assert.IsType(steps[^3]); + Assert.IsType(steps[^2]); + Assert.IsType(steps[^1]); Assert.True( steps.FindIndex(step => step is ValidateLocalAiRecoveryGatewayStep) < steps.FindIndex(step => step is AcquireLocalAiRuntimeStep)); diff --git a/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs b/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs index fc8892eea..cfcdbd67b 100644 --- a/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs @@ -297,9 +297,7 @@ public void CapabilitiesReview_GatesOnDeviceEligibilityAndReconcilesStaleSelecte "hardwareReason = DescribeLocalAiUnavailable(availability);", "LocalInferenceEligibilityResult selectedEligibility =", "LocalInferenceEligibility.Evaluate(_localAiHardware, selectedModelId);", - "if (_localAiRecoveryModelPinned)", - "eligibility = selectedEligibility;", - "else if (!selectedEligibility.CanInstall)", + "if (!selectedEligibility.CanInstall)", "_config.LocalAi.SelectedModelId = null;", "_config.LocalAi.SelectedModelId ??= _localAiRecommendedModelId ?? availability.Plan.Model.Id;", "eligibility ??= LocalInferenceEligibility.Evaluate(", From d7b3b608e8b2a08977735a6aba0b558636d1680b Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 28 Sep 2026 20:26:02 -0700 Subject: [PATCH 02/22] fix(local-ai): preserve receipt downgrade compatibility --- .../LocalAi/LlamaServerRuntimeService.cs | 5 +- .../LocalAi/LocalAiManifest.cs | 12 +++-- .../LocalAiGatewayConfiguration.cs | 2 +- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 9 ++-- .../LocalAiPortLifecycleTests.cs | 47 +++++++++++++++++++ .../LocalAiGatewayUninstallTests.cs | 6 +-- .../LocalAiInstallRecoveryTests.cs | 5 +- 7 files changed, 71 insertions(+), 15 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs index cdac2ddcc..6739fc6a6 100644 --- a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs +++ b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs @@ -880,10 +880,11 @@ private static ImmutableArray ReplacementEndpointHistory( LocalAiResolvedInstall install, Uri endpoint) { - ImmutableArray history = install.Manifest.PreviousEndpoints; + ImmutableArray history = install.Manifest.PreviousEndpoints ?? []; string? previous = install.Endpoint?.AbsoluteUri; return install.Manifest.ReplacedManifest is not null && install.Endpoint != endpoint && - previous is not null && !history.Contains(previous, StringComparer.Ordinal) + previous is not null && + !history.Contains(previous, StringComparer.Ordinal) ? history.Add(previous) : history; } diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index ff11363af..b78ecc095 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -259,7 +259,8 @@ public sealed record LocalAiInstallManifest [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public LocalAiInstallManifest? ReplacedManifest { get; init; } /// Earlier verified replacement endpoints that may still be published to the Gateway. - public ImmutableArray PreviousEndpoints { get; init; } = []; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ImmutableArray? PreviousEndpoints { get; init; } public required int ContextLength { get; init; } public KvCachePrecision KeyCachePrecision { get; init; } = KvCachePrecision.F16; public KvCachePrecision ValueCachePrecision { get; init; } = KvCachePrecision.F16; @@ -656,14 +657,17 @@ LocalAiInstallManifest.HubCacheReceiptSchemaVersion or } _ = ResolveAndValidate(replaced); } - else if (!manifest.PreviousEndpoints.IsDefaultOrEmpty) + else if (manifest.PreviousEndpoints is { IsDefaultOrEmpty: false }) { throw new InvalidDataException("Previous Local AI endpoints require a pending model replacement."); } Uri? endpoint = ValidateEndpoint(manifest.Endpoint, manifest.RequestedPort); - HashSet previousEndpoints = manifest.PreviousEndpoints.ToHashSet(StringComparer.Ordinal); - if (previousEndpoints.Count != manifest.PreviousEndpoints.Length) + ImmutableArray endpointHistory = manifest.PreviousEndpoints.GetValueOrDefault(); + if (endpointHistory.IsDefault) + endpointHistory = []; + HashSet previousEndpoints = endpointHistory.ToHashSet(StringComparer.Ordinal); + if (previousEndpoints.Count != endpointHistory.Length) { throw new InvalidDataException("Previous Local AI endpoints must be unique."); } diff --git a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs index 9073134e1..afdefc1d6 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs @@ -451,7 +451,7 @@ private static async Task RemoveManagedStateForUninstallAsync( return; var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); - foreach (string previousEndpoint in install.Manifest.PreviousEndpoints) + foreach (string previousEndpoint in install.Manifest.PreviousEndpoints ?? []) { LocalAiResolvedInstall previous = store.ResolveAndValidate(install.Manifest with { diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 72f60c206..e95af0c14 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -876,12 +876,13 @@ private static ImmutableArray BuildRuntimeReceipts( private static ImmutableArray ReplacementEndpointHistory(LocalAiResolvedInstall? pending) { + ImmutableArray history = pending?.Manifest.PreviousEndpoints ?? []; if (pending?.Endpoint is null || - pending.Manifest.PreviousEndpoints.Contains(pending.Endpoint.AbsoluteUri, StringComparer.Ordinal)) + history.Contains(pending.Endpoint.AbsoluteUri, StringComparer.Ordinal)) { - return pending?.Manifest.PreviousEndpoints ?? []; + return history; } - return pending.Manifest.PreviousEndpoints.Add(pending.Endpoint.AbsoluteUri); + return history.Add(pending.Endpoint.AbsoluteUri); } } @@ -901,7 +902,7 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati LocalAiInstallManifest committed = install.Manifest with { ReplacedManifest = null, - PreviousEndpoints = [], + PreviousEndpoints = null, }; try { diff --git a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs index 3eee568c0..7b315746f 100644 --- a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs +++ b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs @@ -112,6 +112,24 @@ public async Task Manifest_OmitsLegacyHardwareProfileIdFromNewWrites() Assert.Contains("\"draftValueCachePrecision\": \"q8_0\"", json, StringComparison.Ordinal); } + [Fact] + public async Task Manifest_OrdinaryReceiptRemainsReadableBySchemaFourReader() + { + using var temp = new TempDirectory("local-ai-manifest-"); + var paths = new LocalAiPaths(temp.Path); + await new LocalAiManifestStore(paths).SaveAsync(ValidManifest()); + + string json = await File.ReadAllTextAsync(paths.ManifestPath); + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + UnmappedMemberHandling = System.Text.Json.Serialization.JsonUnmappedMemberHandling.Disallow, + }; + + Assert.NotNull(JsonSerializer.Deserialize(json, options)); + Assert.DoesNotContain("previousEndpoints", json, StringComparison.Ordinal); + Assert.DoesNotContain("replacedManifest", json, StringComparison.Ordinal); + } + [Fact] public async Task Router_RejectsRuntimeArchitectureMismatchWithoutHardwareProfile() { @@ -2812,6 +2830,35 @@ private static LocalAiInstallManifest ValidManifest() }; } + private sealed record SchemaFourTransitionalManifest + { + public int SchemaVersion { get; init; } + public string? Engine { get; init; } + public string? EngineVersion { get; init; } + public string? Architecture { get; init; } + public string? HardwareProfileId { get; init; } + public string? RuntimeId { get; init; } + public string? ModelCatalogId { get; init; } + public string? SelectedGpuId { get; init; } + public string? ExecutablePath { get; init; } + public JsonElement RuntimeAssets { get; init; } + public string? ModelPath { get; init; } + public string? ModelCacheRoot { get; init; } + public string? CachedModelPath { get; init; } + public string? ModelId { get; init; } + public string? ModelAlias { get; init; } + public JsonElement ModelAsset { get; init; } + public int RequestedPort { get; init; } + public string? Endpoint { get; init; } + public string? GatewayFallbackModel { get; init; } + public int ContextLength { get; init; } + public string? KeyCachePrecision { get; init; } + public string? ValueCachePrecision { get; init; } + public string? DraftKeyCachePrecision { get; init; } + public string? DraftValueCachePrecision { get; init; } + public DateTimeOffset InstalledAtUtc { get; init; } + } + private sealed class SynchronizedEventLog : IReadOnlyCollection { private readonly Lock _gate = new(); diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index 57d93620d..1a8116c9f 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -207,8 +207,8 @@ public async Task Recovery_ReplacesExactManagedProviderAfterAutomaticPortChanges LocalAiResolvedInstall published = (await store.LoadAsync())!; LocalAiResolvedInstall publishedRoute = published with { - Manifest = published.Manifest with { Endpoint = published.Manifest.PreviousEndpoints[1] }, - Endpoint = new Uri(published.Manifest.PreviousEndpoints[1]), + Manifest = published.Manifest with { Endpoint = published.Manifest.PreviousEndpoints!.Value[1] }, + Endpoint = new Uri(published.Manifest.PreviousEndpoints!.Value[1]), }; string publishedProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(publishedRoute); string primary = JsonSerializer.Serialize( @@ -221,7 +221,7 @@ public async Task Recovery_ReplacesExactManagedProviderAfterAutomaticPortChanges LocalAiInstallManifest replacementManifest = publishedManifest with { Endpoint = "http://127.0.0.1:39879/v1", - PreviousEndpoints = publishedManifest.PreviousEndpoints.Add(publishedManifest.Endpoint!), + PreviousEndpoints = publishedManifest.PreviousEndpoints!.Value.Add(publishedManifest.Endpoint!), }; await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = (await store.LoadAsync())!; diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index 664698448..8630a1742 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1588,7 +1588,10 @@ public async Task FinalizeReplacement_ClearsRollbackReceipt() Assert.Equal(StepOutcome.Success, result.Outcome); LocalAiInstallManifest committed = (await store.LoadAsync())!.Manifest; Assert.Null(committed.ReplacedManifest); - Assert.Empty(committed.PreviousEndpoints); + Assert.Null(committed.PreviousEndpoints); + string json = await File.ReadAllTextAsync(new LocalAiPaths(temp.Path).ManifestPath); + Assert.DoesNotContain("previousEndpoints", json, StringComparison.Ordinal); + Assert.DoesNotContain("replacedManifest", json, StringComparison.Ordinal); } [Fact] From 94b4669a4bd9f5248559e8b240e6a3659b92f60c Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 28 Sep 2026 20:31:55 -0700 Subject: [PATCH 03/22] fix(local-ai): keep replacement recovery state scoped --- .../LocalAiGatewayConfiguration.cs | 7 ++-- .../Services/LocalAiGatewayDistroResolver.cs | 9 +++-- .../LocalAiGatewayUninstallTests.cs | 33 +++++++++++++++++++ .../LocalAiGatewayProviderCoordinatorTests.cs | 4 ++- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs index afdefc1d6..7c971e9de 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs @@ -457,11 +457,8 @@ private static async Task RemoveManagedStateForUninstallAsync( { Endpoint = previousEndpoint, }); - if (current.ProviderExisted && current.PrimaryModelExisted && - LocalAiGatewayProviderDefinition.MatchesProviderJson(current.ProviderJson!, previous) && - JsonEquals( - current.PrimaryModelJson!, - JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(previous)))) + if (current.ProviderExisted && + LocalAiGatewayProviderDefinition.MatchesProviderJson(current.ProviderJson!, previous)) { install = previous; break; diff --git a/src/OpenClaw.Tray.WinUI/Services/LocalAiGatewayDistroResolver.cs b/src/OpenClaw.Tray.WinUI/Services/LocalAiGatewayDistroResolver.cs index d192a7761..7756441dd 100644 --- a/src/OpenClaw.Tray.WinUI/Services/LocalAiGatewayDistroResolver.cs +++ b/src/OpenClaw.Tray.WinUI/Services/LocalAiGatewayDistroResolver.cs @@ -31,7 +31,8 @@ internal sealed record LocalAiRecoveryTarget( string DistroName, int GatewayPort, string? ModelCatalogId, - int? RequestedLocalAiPort); + int? RequestedLocalAiPort, + bool PinModelSelection); internal sealed record LocalAiSetupResolution( LocalAiSetupRoute Route, @@ -47,7 +48,8 @@ public static LocalAiSetupResolution Decide( bool hasDistroDataDirectory, bool distroIsAppOwned, string? installedModelCatalogId = null, - int? installedRequestedLocalAiPort = null) + int? installedRequestedLocalAiPort = null, + bool pinInstalledModelSelection = false) { if (owners.Count == 1) { @@ -69,7 +71,8 @@ public static LocalAiSetupResolution Decide( GatewayRecordEditing.ResolveManagedDistroName(owner)!.Trim(), uri.Port, installedModelCatalogId, - installedRequestedLocalAiPort)); + installedRequestedLocalAiPort, + pinInstalledModelSelection)); } return new(LocalAiSetupRoute.Blocked); diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index 1a8116c9f..2ea8b4923 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -97,6 +97,39 @@ await store.SaveAsync(published with command.Contains("LOCAL_AI_GATEWAY_UNSET", StringComparison.Ordinal)); } + [Fact] + public async Task FreshProcessUninstall_RemovesHistoricalManagedProviderWhenPrimaryIsMissing() + { + using var temp = new TempDirectory("local-ai-gateway-uninstall-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path); + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39876/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Manifest.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pendingManifest); + LocalAiResolvedInstall pending = (await store.LoadAsync())!; + LocalAiResolvedInstall historical = pending with + { + Manifest = pending.Manifest with { Endpoint = original.Manifest.Endpoint }, + Endpoint = original.Endpoint, + }; + var commands = new GatewayStateCommandRunner( + LocalAiGatewayProviderDefinition.BuildProviderJson(historical), + primaryJson: null); + SetupContext context = CreateContext(temp.Path, commands); + context.IsUninstalling = true; + + await new ConfigureLocalAiGatewayStep().RollbackAsync(context, CancellationToken.None); + + Assert.Null(commands.ProviderJson); + Assert.Null(commands.PrimaryJson); + } + [Fact] public async Task FreshProcessUninstall_AcceptsCliRedactedManagedApiKey() { diff --git a/tests/OpenClaw.Tray.Tests/LocalAiGatewayProviderCoordinatorTests.cs b/tests/OpenClaw.Tray.Tests/LocalAiGatewayProviderCoordinatorTests.cs index 31c1f5ba3..e9cf42657 100644 --- a/tests/OpenClaw.Tray.Tests/LocalAiGatewayProviderCoordinatorTests.cs +++ b/tests/OpenClaw.Tray.Tests/LocalAiGatewayProviderCoordinatorTests.cs @@ -695,7 +695,8 @@ public void LocalAiSetupRoute_UsesUniqueManagedOwnerEvenWhenItIsNotActive() hasDistroDataDirectory: true, distroIsAppOwned: true, installedModelCatalogId: LocalModelCatalog.Qwen38_27BModelId, - installedRequestedLocalAiPort: 28888); + installedRequestedLocalAiPort: 28888, + pinInstalledModelSelection: true); Assert.Equal(LocalAiSetupRoute.Recovery, resolution.Route); Assert.Equal("managed", resolution.RecoveryTarget?.GatewayId); @@ -703,6 +704,7 @@ public void LocalAiSetupRoute_UsesUniqueManagedOwnerEvenWhenItIsNotActive() Assert.Equal(29999, resolution.RecoveryTarget?.GatewayPort); Assert.Equal(LocalModelCatalog.Qwen38_27BModelId, resolution.RecoveryTarget?.ModelCatalogId); Assert.Equal(28888, resolution.RecoveryTarget?.RequestedLocalAiPort); + Assert.True(resolution.RecoveryTarget?.PinModelSelection); } [Fact] From e6d01c107b2df754f9d59b775080ac8532b62acd Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 28 Sep 2026 20:36:31 -0700 Subject: [PATCH 04/22] fix(local-ai): serialize replacement endpoint updates --- .../LocalAi/LlamaServerRuntimeService.cs | 38 +++-------- .../LocalAi/LocalAiManifest.cs | 54 +++++++++++++++- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 9 ++- .../LocalAiPortLifecycleTests.cs | 64 +++++++++++++++++++ .../LocalAiInstallRecoveryTests.cs | 6 ++ 5 files changed, 139 insertions(+), 32 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs index 6739fc6a6..960646447 100644 --- a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs +++ b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs @@ -12,7 +12,6 @@ // using OpenClaw.Shared; using OpenClaw.Shared.Inference.Catalog; -using System.Collections.Immutable; using System.Net; using System.Text; @@ -492,13 +491,11 @@ recoveryPort is not null .ConfigureAwait(false); if (probe.IsReadyForManagedModel(runtimeModelPath)) { - LocalAiInstallManifest verifiedManifest = install.Manifest with - { - PreviousEndpoints = ReplacementEndpointHistory(install, ownership.Endpoint), - Endpoint = ownership.Endpoint.AbsoluteUri, - }; - await _manifestStore.SaveAsync(verifiedManifest, cancellationToken).ConfigureAwait(false); - _install = _manifestStore.ResolveAndValidate(verifiedManifest); + _install = await _manifestStore.UpdateVerifiedEndpointAsync( + install.Manifest, + ownership.Endpoint, + cancellationToken) + .ConfigureAwait(false); LocalAiEndpointLifecycleResult published = await PublishRouteAsync( _install, @@ -866,29 +863,14 @@ private async Task BindVerifiedEndpointAsync( if (install.Endpoint == endpoint) return install; - LocalAiInstallManifest verifiedManifest = install.Manifest with - { - PreviousEndpoints = ReplacementEndpointHistory(install, endpoint), - Endpoint = endpoint.AbsoluteUri, - }; - await _manifestStore.SaveAsync(verifiedManifest, cancellationToken).ConfigureAwait(false); - _install = _manifestStore.ResolveAndValidate(verifiedManifest); + _install = await _manifestStore.UpdateVerifiedEndpointAsync( + install.Manifest, + endpoint, + cancellationToken) + .ConfigureAwait(false); return _install; } - private static ImmutableArray ReplacementEndpointHistory( - LocalAiResolvedInstall install, - Uri endpoint) - { - ImmutableArray history = install.Manifest.PreviousEndpoints ?? []; - string? previous = install.Endpoint?.AbsoluteUri; - return install.Manifest.ReplacedManifest is not null && install.Endpoint != endpoint && - previous is not null && - !history.Contains(previous, StringComparer.Ordinal) - ? history.Add(previous) - : history; - } - private async Task TryLoadInstallAsync(CancellationToken cancellationToken) { try diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index b78ecc095..74bff0ad4 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -493,6 +493,54 @@ public async Task SaveAsync(LocalAiInstallManifest manifest, CancellationToken c await SaveWithoutLockAsync(manifest, cancellationToken).ConfigureAwait(false); } + internal async Task UpdateVerifiedEndpointAsync( + LocalAiInstallManifest expectedManifest, + Uri endpoint, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(expectedManifest); + ArgumentNullException.ThrowIfNull(endpoint); + await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) + .ConfigureAwait(false); + LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); + if (!HasSameRuntimeAndModel(current, expectedManifest)) + { + throw new InvalidDataException( + "The Local AI installation changed before its verified endpoint could be recorded."); + } + + ImmutableArray? history = current.PreviousEndpoints; + if (current.ReplacedManifest is null) + { + history = null; + } + else if (current.Endpoint is { } previousEndpoint && + !string.Equals(previousEndpoint, endpoint.AbsoluteUri, StringComparison.Ordinal)) + { + ImmutableArray values = history ?? []; + if (!values.Contains(previousEndpoint, StringComparer.Ordinal)) + history = values.Add(previousEndpoint); + } + + LocalAiInstallManifest updated = current with + { + Endpoint = endpoint.AbsoluteUri, + PreviousEndpoints = history, + }; + LocalAiResolvedInstall resolved = ResolveAndValidate(updated); + await SaveWithoutLockAsync(updated, cancellationToken).ConfigureAwait(false); + return resolved; + } + + private static bool HasSameRuntimeAndModel( + LocalAiInstallManifest current, + LocalAiInstallManifest expected) => + string.Equals(current.RuntimeId, expected.RuntimeId, StringComparison.Ordinal) && + string.Equals(current.SelectedGpuId, expected.SelectedGpuId, StringComparison.Ordinal) && + string.Equals(current.ModelCatalogId, expected.ModelCatalogId, StringComparison.Ordinal) && + string.Equals(current.ModelPath, expected.ModelPath, StringComparison.OrdinalIgnoreCase) && + string.Equals(current.CachedModelPath, expected.CachedModelPath, StringComparison.OrdinalIgnoreCase); + private async Task SaveWithoutLockAsync( LocalAiInstallManifest manifest, CancellationToken cancellationToken) @@ -657,7 +705,7 @@ LocalAiInstallManifest.HubCacheReceiptSchemaVersion or } _ = ResolveAndValidate(replaced); } - else if (manifest.PreviousEndpoints is { IsDefaultOrEmpty: false }) + else if (manifest.PreviousEndpoints is not null) { throw new InvalidDataException("Previous Local AI endpoints require a pending model replacement."); } @@ -672,7 +720,11 @@ LocalAiInstallManifest.HubCacheReceiptSchemaVersion or throw new InvalidDataException("Previous Local AI endpoints must be unique."); } foreach (string previousEndpoint in previousEndpoints) + { + if (string.IsNullOrWhiteSpace(previousEndpoint)) + throw new InvalidDataException("Previous Local AI endpoints must be non-empty endpoint strings."); _ = ValidateEndpoint(previousEndpoint, manifest.RequestedPort); + } return new LocalAiResolvedInstall(manifest, executable, model, endpoint); } diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index e95af0c14..1a6881467 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -874,10 +874,13 @@ private static ImmutableArray BuildRuntimeReceipts( return receipts.MoveToImmutable(); } - private static ImmutableArray ReplacementEndpointHistory(LocalAiResolvedInstall? pending) + private static ImmutableArray? ReplacementEndpointHistory(LocalAiResolvedInstall? pending) { - ImmutableArray history = pending?.Manifest.PreviousEndpoints ?? []; - if (pending?.Endpoint is null || + if (pending is null) + return null; + + ImmutableArray history = pending.Manifest.PreviousEndpoints ?? []; + if (pending.Endpoint is null || history.Contains(pending.Endpoint.AbsoluteUri, StringComparer.Ordinal)) { return history; diff --git a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs index 7b315746f..fe95b5e7a 100644 --- a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs +++ b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs @@ -130,6 +130,70 @@ public async Task Manifest_OrdinaryReceiptRemainsReadableBySchemaFourReader() Assert.DoesNotContain("replacedManifest", json, StringComparison.Ordinal); } + [Fact] + public async Task Manifest_EndpointUpdateDoesNotRestoreFinalizedReplacementState() + { + using var temp = new TempDirectory("local-ai-manifest-"); + var paths = new LocalAiPaths(temp.Path); + var store = new LocalAiManifestStore(paths); + LocalAiInstallManifest original = ValidManifest() with + { + Endpoint = "http://127.0.0.1:28765/v1", + }; + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:28766/v1", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + await store.SaveAsync(pending); + await store.SaveAsync(pending with + { + ReplacedManifest = null, + PreviousEndpoints = null, + }); + + LocalAiResolvedInstall updated = await store.UpdateVerifiedEndpointAsync( + pending, + new Uri("http://127.0.0.1:28767/v1")); + + Assert.Equal("http://127.0.0.1:28767/v1", updated.Manifest.Endpoint); + Assert.Null(updated.Manifest.ReplacedManifest); + Assert.Null(updated.Manifest.PreviousEndpoints); + string json = await File.ReadAllTextAsync(paths.ManifestPath); + Assert.DoesNotContain("previousEndpoints", json, StringComparison.Ordinal); + Assert.DoesNotContain("replacedManifest", json, StringComparison.Ordinal); + } + + [Fact] + public async Task Manifest_RejectsNullPreviousEndpoint() + { + using var temp = new TempDirectory("local-ai-manifest-"); + var paths = new LocalAiPaths(temp.Path); + var store = new LocalAiManifestStore(paths); + LocalAiInstallManifest original = ValidManifest() with + { + Endpoint = "http://127.0.0.1:28765/v1", + }; + await store.SaveAsync(original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:28766/v1", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }); + JsonObject json = (JsonNode.Parse(await File.ReadAllTextAsync(paths.ManifestPath)) as JsonObject)!; + json["previousEndpoints"] = new JsonArray { null }; + await File.WriteAllTextAsync(paths.ManifestPath, json.ToJsonString()); + + InvalidDataException error = await Assert.ThrowsAsync(() => store.LoadAsync()); + + Assert.Contains("non-empty endpoint", error.Message, StringComparison.Ordinal); + } + [Fact] public async Task Router_RejectsRuntimeArchitectureMismatchWithoutHardwareProfile() { diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index 8630a1742..45a767c7b 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1692,6 +1692,12 @@ public async Task RecoveryPipeline_RewritesIncompleteReceiptAfterModelRepair(boo Assert.Equal(CacheRoot(temp.Path), repaired.Manifest.ModelCacheRoot); Assert.Equal(repaired.Manifest.CachedModelPath, repaired.ModelPath); Assert.False(context.LocalAiManifestCreatedThisRun); + if (!pendingReplacement) + { + Assert.Null(repaired.Manifest.PreviousEndpoints); + string json = await File.ReadAllTextAsync(new LocalAiPaths(temp.Path).ManifestPath); + Assert.DoesNotContain("previousEndpoints", json, StringComparison.Ordinal); + } } [Fact] From 9cf986f13ec7c95ed63c764b8a1335aef2dcfff3 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 28 Sep 2026 20:40:33 -0700 Subject: [PATCH 05/22] fix(local-ai): finalize the current replacement receipt --- .../LocalAi/LocalAiManifest.cs | 24 +++++++++++++++ src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 10 ++----- .../LocalAiPortLifecycleTests.cs | 30 +++++++++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index 74bff0ad4..7fc8271e0 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -532,6 +532,30 @@ internal async Task UpdateVerifiedEndpointAsync( return resolved; } + internal async Task FinalizeReplacementAsync( + LocalAiInstallManifest expectedManifest, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(expectedManifest); + await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) + .ConfigureAwait(false); + LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); + if (!HasSameRuntimeAndModel(current, expectedManifest)) + { + throw new InvalidDataException( + "The Local AI installation changed before its model replacement could be finalized."); + } + + LocalAiInstallManifest finalized = current with + { + ReplacedManifest = null, + PreviousEndpoints = null, + }; + LocalAiResolvedInstall resolved = ResolveAndValidate(finalized); + await SaveWithoutLockAsync(finalized, cancellationToken).ConfigureAwait(false); + return resolved; + } + private static bool HasSameRuntimeAndModel( LocalAiInstallManifest current, LocalAiInstallManifest expected) => diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 1a6881467..52a539c79 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -902,16 +902,12 @@ public override bool CanSkip(SetupContext ctx) => public override async Task ExecuteAsync(SetupContext ctx, CancellationToken ct) { LocalAiResolvedInstall install = ctx.LocalAiResolvedInstall!; - LocalAiInstallManifest committed = install.Manifest with - { - ReplacedManifest = null, - PreviousEndpoints = null, - }; try { var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); - await store.SaveAsync(committed, ct).ConfigureAwait(false); - ctx.LocalAiResolvedInstall = store.ResolveAndValidate(committed); + ctx.LocalAiResolvedInstall = await store + .FinalizeReplacementAsync(install.Manifest, ct) + .ConfigureAwait(false); ctx.LocalAiRecoveryProviderTransition = false; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; return StepResult.Ok("Local AI model replacement is committed."); diff --git a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs index fe95b5e7a..ec1590961 100644 --- a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs +++ b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs @@ -167,6 +167,36 @@ await store.SaveAsync(pending with Assert.DoesNotContain("replacedManifest", json, StringComparison.Ordinal); } + [Fact] + public async Task Manifest_FinalizationPreservesConcurrentlyUpdatedEndpoint() + { + using var temp = new TempDirectory("local-ai-manifest-"); + var paths = new LocalAiPaths(temp.Path); + var store = new LocalAiManifestStore(paths); + LocalAiInstallManifest original = ValidManifest() with + { + Endpoint = "http://127.0.0.1:28765/v1", + }; + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:28766/v1", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + await store.SaveAsync(pending); + await store.UpdateVerifiedEndpointAsync( + pending, + new Uri("http://127.0.0.1:28767/v1")); + + LocalAiResolvedInstall finalized = await store.FinalizeReplacementAsync(pending); + + Assert.Equal("http://127.0.0.1:28767/v1", finalized.Manifest.Endpoint); + Assert.Null(finalized.Manifest.ReplacedManifest); + Assert.Null(finalized.Manifest.PreviousEndpoints); + } + [Fact] public async Task Manifest_RejectsNullPreviousEndpoint() { From 4eb5e09306bb5f51317f9945cee014fdb2d10e75 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 28 Sep 2026 20:47:09 -0700 Subject: [PATCH 06/22] fix(local-ai): close replacement rollback races --- .../LocalAi/LocalAiManifest.cs | 31 +++++++++++++++++++ src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 12 +++++++ .../LocalAiPortLifecycleTests.cs | 26 ++++++++++------ .../LocalAiInstallRecoveryTests.cs | 29 +++++++++++++++++ 4 files changed, 88 insertions(+), 10 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index 7fc8271e0..5cd1a1cd0 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -508,6 +508,11 @@ internal async Task UpdateVerifiedEndpointAsync( throw new InvalidDataException( "The Local AI installation changed before its verified endpoint could be recorded."); } + if (expectedManifest.ReplacedManifest is not null && current.ReplacedManifest is null) + { + throw new InvalidDataException( + "The Local AI model replacement was finalized before its verified endpoint could be recorded."); + } ImmutableArray? history = current.PreviousEndpoints; if (current.ReplacedManifest is null) @@ -545,6 +550,12 @@ internal async Task FinalizeReplacementAsync( throw new InvalidDataException( "The Local AI installation changed before its model replacement could be finalized."); } + if (expectedManifest.ReplacedManifest is null || current.ReplacedManifest is null || + !string.Equals(current.Endpoint, expectedManifest.Endpoint, StringComparison.Ordinal)) + { + throw new InvalidDataException( + "The Local AI model replacement changed before it could be finalized."); + } LocalAiInstallManifest finalized = current with { @@ -556,6 +567,26 @@ internal async Task FinalizeReplacementAsync( return resolved; } + internal async Task RestoreReplacedManifestAsync( + LocalAiInstallManifest expectedManifest, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(expectedManifest); + await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) + .ConfigureAwait(false); + LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); + if (!HasSameRuntimeAndModel(current, expectedManifest) || + current.ReplacedManifest is not { } replaced) + { + throw new InvalidDataException( + "The Local AI model replacement changed before its original receipt could be restored."); + } + + LocalAiResolvedInstall resolved = ResolveAndValidate(replaced); + await SaveWithoutLockAsync(replaced, cancellationToken).ConfigureAwait(false); + return resolved; + } + private static bool HasSameRuntimeAndModel( LocalAiInstallManifest current, LocalAiInstallManifest expected) => diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 52a539c79..d536f1de2 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -816,7 +816,19 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) } if (!ctx.LocalAiManifestCreatedThisRun) + { + if (ctx.LocalAiRecoveryReceiptRollbackAllowed && + ctx.LocalAiResolvedInstall?.Manifest.ReplacedManifest is not null) + { + var recoveryStore = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); + ctx.LocalAiResolvedInstall = await recoveryStore + .RestoreReplacedManifestAsync(ctx.LocalAiResolvedInstall.Manifest, ct) + .ConfigureAwait(false); + ctx.LocalAiRecoveryProviderTransition = false; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + } return; + } var paths = new LocalAiPaths(ctx.LocalDataDir); await new LocalAiManifestStore(paths).DeleteAsync(ct); diff --git a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs index ec1590961..244b694ec 100644 --- a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs +++ b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs @@ -155,11 +155,14 @@ await store.SaveAsync(pending with PreviousEndpoints = null, }); - LocalAiResolvedInstall updated = await store.UpdateVerifiedEndpointAsync( - pending, - new Uri("http://127.0.0.1:28767/v1")); - - Assert.Equal("http://127.0.0.1:28767/v1", updated.Manifest.Endpoint); + InvalidDataException error = await Assert.ThrowsAsync(() => + store.UpdateVerifiedEndpointAsync( + pending, + new Uri("http://127.0.0.1:28767/v1"))); + + Assert.Contains("finalized", error.Message, StringComparison.Ordinal); + LocalAiResolvedInstall updated = (await store.LoadAsync())!; + Assert.Equal("http://127.0.0.1:28766/v1", updated.Manifest.Endpoint); Assert.Null(updated.Manifest.ReplacedManifest); Assert.Null(updated.Manifest.PreviousEndpoints); string json = await File.ReadAllTextAsync(paths.ManifestPath); @@ -168,7 +171,7 @@ await store.SaveAsync(pending with } [Fact] - public async Task Manifest_FinalizationPreservesConcurrentlyUpdatedEndpoint() + public async Task Manifest_FinalizationRejectsConcurrentlyUpdatedEndpoint() { using var temp = new TempDirectory("local-ai-manifest-"); var paths = new LocalAiPaths(temp.Path); @@ -190,11 +193,14 @@ await store.UpdateVerifiedEndpointAsync( pending, new Uri("http://127.0.0.1:28767/v1")); - LocalAiResolvedInstall finalized = await store.FinalizeReplacementAsync(pending); + InvalidDataException error = await Assert.ThrowsAsync(() => + store.FinalizeReplacementAsync(pending)); - Assert.Equal("http://127.0.0.1:28767/v1", finalized.Manifest.Endpoint); - Assert.Null(finalized.Manifest.ReplacedManifest); - Assert.Null(finalized.Manifest.PreviousEndpoints); + Assert.Contains("changed", error.Message, StringComparison.Ordinal); + LocalAiResolvedInstall retained = (await store.LoadAsync())!; + Assert.Equal("http://127.0.0.1:28767/v1", retained.Manifest.Endpoint); + Assert.NotNull(retained.Manifest.ReplacedManifest); + Assert.NotNull(retained.Manifest.PreviousEndpoints); } [Fact] diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index 45a767c7b..4aa1c6739 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1594,6 +1594,35 @@ public async Task FinalizeReplacement_ClearsRollbackReceipt() Assert.DoesNotContain("replacedManifest", json, StringComparison.Ordinal); } + [Fact] + public async Task PersistRollback_RestoresOriginalReceiptBeforeReplacementCleanup() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pending); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pending); + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryReceiptRollbackAllowed = true; + + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + + LocalAiInstallManifest restored = (await store.LoadAsync())!.Manifest; + Assert.Equal(original.ModelCatalogId, restored.ModelCatalogId); + Assert.Null(restored.ReplacedManifest); + Assert.Null(restored.PreviousEndpoints); + Assert.False(context.LocalAiRecoveryProviderTransition); + Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + } + [Fact] public async Task ReconcileStep_RecoveryPinsIncompleteReceiptAsRollbackBaseline() { From 527377ba39facd68fc75f41b6195c63ec03698ea Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 28 Sep 2026 20:58:37 -0700 Subject: [PATCH 07/22] test(local-ai): prove drift rejection before gateway writes --- .../LocalAiGatewayUninstallTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index 2ea8b4923..cfbc7eb7c 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -309,7 +309,7 @@ public async Task Recovery_ReplacementAcceptsProviderlessOriginalPrimary(bool re } [Fact] - public async Task Recovery_PreservesProviderThatMatchesNeitherEndpoint() + public async Task Recovery_DriftedProviderRejectsBeforeGatewayMutation() { using var temp = new TempDirectory("local-ai-gateway-recovery-"); LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path); @@ -340,6 +340,8 @@ public async Task Recovery_PreservesProviderThatMatchesNeitherEndpoint() Assert.Equal(StepOutcome.Failed, result.Outcome); Assert.Equal(driftedProvider, commands.ProviderJson); Assert.Equal(primary, commands.PrimaryJson); + Assert.DoesNotContain(commands.WslCalls, command => + command.Contains("LOCAL_AI_GATEWAY_CONFIGURED", StringComparison.Ordinal)); } [Fact] From 49a3d742d529db5f82c2dc8ad5959759a991995e Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 28 Sep 2026 21:15:31 -0700 Subject: [PATCH 08/22] fix(local-ai): preserve resumed replacement rollback receipt --- .../LocalAi/LocalAiManifest.cs | 35 +++++++++++++++---- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 8 +++-- .../LocalAiGatewayUninstallTests.cs | 14 +++++--- .../LocalAiInstallRecoveryTests.cs | 1 + 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index 5cd1a1cd0..5f5e71c4f 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -567,23 +567,46 @@ internal async Task FinalizeReplacementAsync( return resolved; } - internal async Task RestoreReplacedManifestAsync( + internal async Task RestoreRecoveryManifestAsync( LocalAiInstallManifest expectedManifest, + LocalAiInstallManifest recoveryManifest, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(expectedManifest); + ArgumentNullException.ThrowIfNull(recoveryManifest); await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) .ConfigureAwait(false); LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); - if (!HasSameRuntimeAndModel(current, expectedManifest) || - current.ReplacedManifest is not { } replaced) + if (!HasSameRuntimeAndModel(current, expectedManifest) || current.ReplacedManifest is null) { throw new InvalidDataException( - "The Local AI model replacement changed before its original receipt could be restored."); + "The Local AI model replacement changed before its recovery receipt could be restored."); } - LocalAiResolvedInstall resolved = ResolveAndValidate(replaced); - await SaveWithoutLockAsync(replaced, cancellationToken).ConfigureAwait(false); + bool restoresOriginal = JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(current.ReplacedManifest), + JsonSerializer.SerializeToElement(recoveryManifest)); + bool endpointWasPublished = recoveryManifest.Endpoint is { } recoveryEndpoint && + (string.Equals(current.Endpoint, recoveryEndpoint, StringComparison.Ordinal) || + (current.PreviousEndpoints?.Contains(recoveryEndpoint, StringComparer.Ordinal) ?? false)); + bool restoresPendingRoute = endpointWasPublished && JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(current), + JsonSerializer.SerializeToElement(recoveryManifest with + { + Endpoint = current.Endpoint, + PreviousEndpoints = current.PreviousEndpoints, + })); + if (!restoresOriginal && !restoresPendingRoute) + { + throw new InvalidDataException( + "The requested Local AI recovery receipt is not an authorized replacement route."); + } + + LocalAiInstallManifest restored = restoresPendingRoute + ? current with { Endpoint = recoveryManifest.Endpoint } + : recoveryManifest; + LocalAiResolvedInstall resolved = ResolveAndValidate(restored); + await SaveWithoutLockAsync(restored, cancellationToken).ConfigureAwait(false); return resolved; } diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index d536f1de2..03a3e5f60 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -818,11 +818,15 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) if (!ctx.LocalAiManifestCreatedThisRun) { if (ctx.LocalAiRecoveryReceiptRollbackAllowed && - ctx.LocalAiResolvedInstall?.Manifest.ReplacedManifest is not null) + ctx.LocalAiResolvedInstall?.Manifest.ReplacedManifest is not null && + ctx.LocalAiRecoveryOriginalInstall is { } recoveryInstall) { var recoveryStore = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); ctx.LocalAiResolvedInstall = await recoveryStore - .RestoreReplacedManifestAsync(ctx.LocalAiResolvedInstall.Manifest, ct) + .RestoreRecoveryManifestAsync( + ctx.LocalAiResolvedInstall.Manifest, + recoveryInstall.Manifest, + ct) .ConfigureAwait(false); ctx.LocalAiRecoveryProviderTransition = false; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index cfbc7eb7c..7ac6e0b7f 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -261,16 +261,22 @@ public async Task Recovery_ReplacesExactManagedProviderAfterAutomaticPortChanges var step = new ConfigureLocalAiGatewayStep(); StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + await step.RollbackAsync(context, CancellationToken.None); + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); Assert.Equal(StepOutcome.Success, result.Outcome); - Assert.True(context.LocalAiRecoveryProviderTransition); - Assert.Equal(replacementManifest.Endpoint, (await store.LoadAsync())!.Manifest.Endpoint); + Assert.False(context.LocalAiRecoveryProviderTransition); + LocalAiResolvedInstall restored = (await store.LoadAsync())!; + Assert.Equal(publishedRoute.Manifest.Endpoint, restored.Manifest.Endpoint); + Assert.Equal(published.Manifest.ModelCatalogId, restored.Manifest.ModelCatalogId); + Assert.NotNull(restored.Manifest.ReplacedManifest); + Assert.Equal(replacementManifest.PreviousEndpoints, restored.Manifest.PreviousEndpoints); Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson( commands.ProviderJson!, - context.LocalAiResolvedInstall)); + restored)); Assert.Equal( JsonSerializer.Serialize( - LocalAiGatewayProviderDefinition.BuildPrimaryModel(context.LocalAiResolvedInstall)), + LocalAiGatewayProviderDefinition.BuildPrimaryModel(restored)), commands.PrimaryJson); } diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index 4aa1c6739..3ae44ea50 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1610,6 +1610,7 @@ public async Task PersistRollback_RestoresOriginalReceiptBeforeReplacementCleanu await store.SaveAsync(pending); SetupContext context = CreateContext(temp.Path, confirmDestructive: false); context.LocalAiResolvedInstall = store.ResolveAndValidate(pending); + context.LocalAiRecoveryOriginalInstall = store.ResolveAndValidate(original); context.LocalAiRecoveryProviderTransition = true; context.LocalAiRecoveryReceiptRollbackAllowed = true; From fbf50a3ae6c8193803cef132699dea1a21a166e1 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Tue, 29 Sep 2026 22:56:13 -0700 Subject: [PATCH 09/22] fix(local-ai): restart gateway after rollback --- .../RestartGatewayStep.cs | 12 +++++++ .../SetupPipelineTests.cs | 36 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/OpenClaw.SetupEngine/RestartGatewayStep.cs b/src/OpenClaw.SetupEngine/RestartGatewayStep.cs index 6f4ae0b43..7eee4452a 100644 --- a/src/OpenClaw.SetupEngine/RestartGatewayStep.cs +++ b/src/OpenClaw.SetupEngine/RestartGatewayStep.cs @@ -27,4 +27,16 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati ctx.LocalAiRecoveryStoppedWsl = false; return result; } + + public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + if (!string.IsNullOrWhiteSpace(ctx.Config.LocalAiRecoveryGatewayId)) + { + // Configuration rollback runs after this step in reverse order. Arm the + // early recovery guard to restart the Gateway once the prior route is restored. + ctx.LocalAiRecoveryStoppedWsl = true; + } + return Task.CompletedTask; + } } diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index 2d8755b1c..852935270 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -333,6 +333,42 @@ public async Task RestartGatewayStep_FailureArmsRecoveryRollbackRestart() Assert.True(context.LocalAiRecoveryStoppedWsl); } + [Fact] + public async Task FinalizationFailure_RestartsGatewayAfterConfigurationRollback() + { + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config); + bool configurationRestored = false; + bool restartedAfterRestore = false; + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep((_, _) => + { + restartedAfterRestore = configurationRestored; + return Task.FromResult(StepResult.Ok("restarted")); + }), + new MockStep( + "configure-local-ai-gateway", + (_, _) => Task.FromResult(StepResult.Ok("configured")), + (_, _) => + { + configurationRestored = true; + return Task.CompletedTask; + }), + new RestartGatewayStep((_, _) => Task.FromResult(StepResult.Ok("restarted"))), + new MockStep( + "finalize-local-ai-model-replacement", + (_, _) => Task.FromResult(StepResult.Fail("finalization failed"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.True(configurationRestored); + Assert.True(restartedAfterRestore); + Assert.False(context.LocalAiRecoveryStoppedWsl); + } + /// /// Regression guard for a rollback race: if the Gateway could not be confirmed switched back /// to the original (A) endpoint, the replacement (B) runtime must be kept alive rather than From 29034e2db0946cae419cb9273ad875b2634a8338 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Tue, 29 Sep 2026 23:02:01 -0700 Subject: [PATCH 10/22] test(local-ai): cover fixed-port replacement routes --- .../LocalAiGatewayUninstallTests.cs | 77 ++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index 7ac6e0b7f..3f82ffb9b 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -130,6 +130,32 @@ public async Task FreshProcessUninstall_RemovesHistoricalManagedProviderWhenPrim Assert.Null(commands.PrimaryJson); } + [Fact] + public async Task FreshProcessUninstall_RemovesPendingReplacementOnFixedPort() + { + using var temp = new TempDirectory("local-ai-gateway-uninstall-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, requestedPort: 28765); + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + ReplacedManifest = original.Manifest, + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pendingManifest); + LocalAiResolvedInstall pending = (await store.LoadAsync())!; + var commands = new GatewayStateCommandRunner( + LocalAiGatewayProviderDefinition.BuildProviderJson(pending), + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(pending))); + SetupContext context = CreateContext(temp.Path, commands); + context.IsUninstalling = true; + + await new ConfigureLocalAiGatewayStep().RollbackAsync(context, CancellationToken.None); + + Assert.Null(commands.ProviderJson); + Assert.Null(commands.PrimaryJson); + } + [Fact] public async Task FreshProcessUninstall_AcceptsCliRedactedManagedApiKey() { @@ -280,6 +306,52 @@ public async Task Recovery_ReplacesExactManagedProviderAfterAutomaticPortChanges commands.PrimaryJson); } + [Fact] + public async Task Recovery_ReplacesAndRollsBackModelOnFixedPort() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync( + temp.Path, + fallbackModel: "openai/gpt-5", + requestedPort: 28765); + var commands = new GatewayStateCommandRunner( + LocalAiGatewayProviderDefinition.BuildProviderJson(original), + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(original))); + SetupContext context = CreateRecoveryContext(temp.Path, commands); + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryReceiptRollbackAllowed = true; + LocalAiInstallManifest replacementManifest = original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + ReplacedManifest = original.Manifest, + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(replacementManifest); + LocalAiResolvedInstall replacement = (await store.LoadAsync())!; + context.LocalAiResolvedInstall = replacement; + var step = new ConfigureLocalAiGatewayStep(); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson( + commands.ProviderJson!, + replacement)); + Assert.Equal( + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(replacement)), + commands.PrimaryJson); + + await step.RollbackAsync(context, CancellationToken.None); + + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson( + commands.ProviderJson!, + original)); + Assert.Equal( + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)), + commands.PrimaryJson); + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -766,7 +838,8 @@ private static SetupContext CreateRecoveryContext( private static async Task SaveManifestAsync( string localDataDirectory, - string? fallbackModel = null) + string? fallbackModel = null, + int requestedPort = 0) { var paths = new LocalAiPaths(localDataDirectory); const string revision = "5bc3e238d916f48a861bac2f8a1990a0e9b7e98d"; @@ -799,7 +872,7 @@ private static async Task SaveManifestAsync( SizeBytes = 1, Sha256 = new string('b', 64), }, - RequestedPort = 0, + RequestedPort = requestedPort, Endpoint = "http://127.0.0.1:28765/v1", GatewayFallbackModel = fallbackModel, ContextLength = LocalModelCatalog.NativeContextTokens, From be69b132f56ee17804e5b42bf790aabdc3e4e51a Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Tue, 29 Sep 2026 23:32:38 -0700 Subject: [PATCH 11/22] fix(local-ai): preserve replacement across runtime upgrades --- .../LocalAiInstallReconciler.cs | 3 +- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 15 +++++- .../LocalAiInstallRecoveryTests.cs | 49 +++++++++++++++---- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs b/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs index 46d89b7e9..81071d4b6 100644 --- a/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs +++ b/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs @@ -245,7 +245,8 @@ public async Task ReconcileAsync( RuntimeInstall: null, ModelInstall: modelIsValid ? CreateModelInstall(install, localDataDirectory) : null, OriginalInstall: originalInstall, - AdditionalModelInstalls: modelIsValid ? CreateAdditionalModelInstalls(install) : null); + AdditionalModelInstalls: modelIsValid ? CreateAdditionalModelInstalls(install) : null, + PendingReplacement: pendingReplacement); } if (!inspection.IsValid || !modelIsValid) diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 03a3e5f60..50cec5663 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -728,6 +728,19 @@ originalInstall is not null && originalInstall.Manifest.ModelCatalogId, manifest.ModelCatalogId, StringComparison.Ordinal); + LocalAiInstallManifest replacedManifest = originalInstall.Manifest.ReplacedManifest ?? + originalInstall.Manifest; + // A pending model replacement can outlive a catalog runtime bump. Both sides + // of the replacement receipt must describe the newly acquired shared runtime, + // while retaining the prior model as rollback provenance. + replacedManifest = replacedManifest with + { + EngineVersion = manifest.EngineVersion, + Architecture = manifest.Architecture, + RuntimeId = manifest.RuntimeId, + ExecutablePath = manifest.ExecutablePath, + RuntimeAssets = manifest.RuntimeAssets, + }; manifest = originalInstall.Manifest with { SchemaVersion = manifest.SchemaVersion, @@ -759,7 +772,7 @@ originalInstall is not null && InstalledAtUtc = ctx.LocalAiRecoveryPendingInstall?.Manifest.InstalledAtUtc ?? originalInstall.Manifest.InstalledAtUtc, ReplacedManifest = replacingModel - ? originalInstall.Manifest.ReplacedManifest ?? originalInstall.Manifest + ? replacedManifest : null, PreviousEndpoints = ReplacementEndpointHistory(ctx.LocalAiRecoveryPendingInstall), }; diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index 3ae44ea50..6851f8d9b 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1280,14 +1280,21 @@ public async Task Reconciler_UpgradesRetiredRuntimeReceiptInsteadOfFailingSetup( } [Theory] - [InlineData(false, null)] - [InlineData(true, null)] - [InlineData(false, "after-reconcile")] - [InlineData(true, "after-reconcile")] - [InlineData(false, "after-persist")] - [InlineData(true, "after-persist")] + [InlineData(false, false, null)] + [InlineData(true, false, null)] + [InlineData(false, true, null)] + [InlineData(true, true, null)] + [InlineData(false, false, "after-reconcile")] + [InlineData(true, false, "after-reconcile")] + [InlineData(false, true, "after-reconcile")] + [InlineData(true, true, "after-reconcile")] + [InlineData(false, false, "after-persist")] + [InlineData(true, false, "after-persist")] + [InlineData(false, true, "after-persist")] + [InlineData(true, true, "after-persist")] public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailure( bool usesHubCache, + bool pendingReplacement, string? failureStage) { using var temp = new TempDirectory(); @@ -1307,6 +1314,18 @@ public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailur { GatewayFallbackModel = "openai/gpt-5", }; + if (pendingReplacement) + { + manifest = manifest with + { + ReplacedManifest = manifest with + { + ModelCatalogId = "prior-model", + ModelAlias = "prior-model", + }, + PreviousEndpoints = [manifest.Endpoint!], + }; + } string oldExecutable = paths.ResolveContainedPath(manifest.ExecutablePath, "executable"); Directory.CreateDirectory(Path.GetDirectoryName(oldExecutable)!); await File.WriteAllTextAsync(oldExecutable, "old-server"); @@ -1355,9 +1374,14 @@ public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailur new ReconcileLocalAiInstallationStep(reconciler), new UpgradeCheckpointStep("after-reconcile", ctx => { - Assert.Null(ctx.LocalAiRecoveryOriginalInstall); - Assert.Equal(manifest.SchemaVersion, ctx.LocalAiUpgradeOriginalInstall?.Manifest.SchemaVersion); - Assert.Equal(oldExecutable, ctx.LocalAiUpgradeOriginalInstall?.ExecutablePath); + Assert.Equal(pendingReplacement, ctx.LocalAiRecoveryOriginalInstall is not null); + Assert.Equal(pendingReplacement, ctx.LocalAiRecoveryPendingInstall is not null); + Assert.Equal( + pendingReplacement ? null : manifest.SchemaVersion, + ctx.LocalAiUpgradeOriginalInstall?.Manifest.SchemaVersion); + Assert.Equal( + pendingReplacement ? null : oldExecutable, + ctx.LocalAiUpgradeOriginalInstall?.ExecutablePath); Assert.Equal(cacheRoot, ctx.LocalAiModelInstall?.CacheRoot); cachedModel = ctx.LocalAiModelInstall!.ModelPath; return failureStage == "after-reconcile"; @@ -1376,6 +1400,13 @@ public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailur Assert.Equal(manifest.InstalledAtUtc, upgraded.Manifest.InstalledAtUtc); Assert.Equal(manifest.GatewayFallbackModel, upgraded.Manifest.GatewayFallbackModel); Assert.Null(upgraded.Endpoint); + Assert.Equal(pendingReplacement, upgraded.Manifest.ReplacedManifest is not null); + if (pendingReplacement) + { + Assert.Equal("prior-model", upgraded.Manifest.ReplacedManifest!.ModelCatalogId); + Assert.Equal(upgraded.Manifest.RuntimeId, upgraded.Manifest.ReplacedManifest.RuntimeId); + Assert.Equal(upgraded.Manifest.RuntimeAssets, upgraded.Manifest.ReplacedManifest.RuntimeAssets); + } newExecutable = upgraded.ExecutablePath; Assert.True(File.Exists(newExecutable)); return failureStage == "after-persist"; From e5f6bca8cbe53c0ec4e9bd900cf3d8cd6b50ca1b Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Tue, 29 Sep 2026 23:46:07 -0700 Subject: [PATCH 12/22] fix(local-ai): restore pending receipt after upgrade failure --- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 3 +++ .../LocalAiInstallRecoveryTests.cs | 9 +++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 50cec5663..a7954635f 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -295,6 +295,9 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati if (ctx.LocalAiRecoveryOriginalInstall is null && result.OriginalInstall is { } retainedReceipt) ctx.LocalAiUpgradeOriginalInstall ??= retainedReceipt; + else if (result.PendingReplacement is { } pendingInstall && + result.RuntimeInstall is null) + ctx.LocalAiUpgradeOriginalInstall ??= pendingInstall; ctx.LocalAiRuntimeInstall = result.RuntimeInstall; ctx.LocalAiModelInstall = result.ModelInstall; ctx.LocalAiAdditionalModelInstalls = result.AdditionalModelInstalls diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index 6851f8d9b..2a8b65539 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1339,6 +1339,7 @@ public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailur var context = CreateContext(temp.Path, confirmDestructive: false); context.Config.LocalAi.Enabled = true; + context.Config.LocalAiRecoveryGatewayId = pendingReplacement ? "gateway-id" : null; context.Config.RollbackOnFailure = true; context.LocalAiPort = manifest.RequestedPort; context.LocalAiEligibility = new LocalInferenceEligibilityResult( @@ -1376,12 +1377,8 @@ public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailur { Assert.Equal(pendingReplacement, ctx.LocalAiRecoveryOriginalInstall is not null); Assert.Equal(pendingReplacement, ctx.LocalAiRecoveryPendingInstall is not null); - Assert.Equal( - pendingReplacement ? null : manifest.SchemaVersion, - ctx.LocalAiUpgradeOriginalInstall?.Manifest.SchemaVersion); - Assert.Equal( - pendingReplacement ? null : oldExecutable, - ctx.LocalAiUpgradeOriginalInstall?.ExecutablePath); + Assert.Equal(manifest.SchemaVersion, ctx.LocalAiUpgradeOriginalInstall?.Manifest.SchemaVersion); + Assert.Equal(oldExecutable, ctx.LocalAiUpgradeOriginalInstall?.ExecutablePath); Assert.Equal(cacheRoot, ctx.LocalAiModelInstall?.CacheRoot); cachedModel = ctx.LocalAiModelInstall!.ModelPath; return failureStage == "after-reconcile"; From a70f71610638448f52cc431b7702d402415fc202 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 30 Sep 2026 15:28:32 -0700 Subject: [PATCH 13/22] fix(local-ai): make replacement rollback transaction-safe --- .../LocalAi/LocalAiManifest.cs | 5 +- .../LocalAiGatewayConfiguration.cs | 10 ++ .../LocalAiRecoveryPolicy.cs | 1 + src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 37 ++++++-- src/OpenClaw.SetupEngine/SetupContext.cs | 3 + .../SetupInstallationProgress.cs | 3 +- .../LocalAiInstallRecoveryTests.cs | 91 ++++++++++++++++++- .../LocalAiOnboardingTests.cs | 3 +- .../SetupPipelineTests.cs | 2 + 9 files changed, 141 insertions(+), 14 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index 5f5e71c4f..f7790cad3 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -577,7 +577,10 @@ internal async Task RestoreRecoveryManifestAsync( await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) .ConfigureAwait(false); LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); - if (!HasSameRuntimeAndModel(current, expectedManifest) || current.ReplacedManifest is null) + if (!JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(current), + JsonSerializer.SerializeToElement(expectedManifest)) || + current.ReplacedManifest is null) { throw new InvalidDataException( "The Local AI model replacement changed before its recovery receipt could be restored."); diff --git a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs index 7c971e9de..e5251fc88 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs @@ -229,6 +229,11 @@ pendingRoute is not null && prior.PrimaryModelExisted && } string batchJson = LocalAiGatewayConfigBuilder.BuildBatchJson(ctx); + if (ctx.LocalAiRecoveryProviderTransition) + { + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + } CommandResult result = await ApplyBatchAsync(ctx, batchJson, "LOCAL_AI_GATEWAY_CONFIGURED", ct); if (result.ExitCode != 0 || result.TimedOut || !result.Stdout.Contains("LOCAL_AI_GATEWAY_CONFIGURED", StringComparison.Ordinal)) @@ -291,6 +296,7 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) JsonEquals(current.PrimaryModelJson!, prior.PrimaryModelJson!))) { ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; return; } @@ -362,7 +368,10 @@ await ReconcileFailedRecoveryRestoreAsync( } } if (recoveryOriginal is not null) + { ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; + } } private static async Task ReconcileFailedRecoveryRestoreAsync( @@ -405,6 +414,7 @@ private static async Task ReconcileFailedRecoveryRestoreAsync( if (originalRestored) { ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; return; } diff --git a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs index 50b903bdb..18a14f53f 100644 --- a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs +++ b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs @@ -226,6 +226,7 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) } ctx.LocalAiRecoveryProviderTransition = false; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + ctx.LocalAiRecoveryRollbackUncertain = false; ctx.LocalAiGatewayPriorState = null; } diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index a7954635f..4b7c8a4fe 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -287,6 +287,9 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati bool pendingReplacement = result.PendingReplacement is not null; ctx.LocalAiRecoveryProviderTransition = pendingReplacement; ctx.LocalAiRecoveryReceiptRollbackAllowed = !pendingReplacement; + // A recovered replacement may already be the active Gateway route. Only the + // process that created a fresh replacement knows it has not published it yet. + ctx.LocalAiRecoveryRollbackUncertain = pendingReplacement; } if (!result.Reused) { @@ -425,6 +428,8 @@ or UnauthorizedAccessException public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) { ct.ThrowIfCancellationRequested(); + if (!ctx.LocalAiRecoveryCleanupAllowed) + return Task.CompletedTask; if (ctx.LocalAiRuntimeInstall is { } install) { _acquirer.RemoveInstalledRuntime(ctx.LocalDataDir, install); @@ -556,8 +561,7 @@ or InvalidDataException public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) { ct.ThrowIfCancellationRequested(); - if (ctx.LocalAiRecoveryProviderTransition && - !ctx.LocalAiRecoveryReceiptRollbackAllowed) + if (!ctx.LocalAiRecoveryCleanupAllowed) { return Task.CompletedTask; } @@ -833,19 +837,29 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) if (!ctx.LocalAiManifestCreatedThisRun) { - if (ctx.LocalAiRecoveryReceiptRollbackAllowed && + if (ctx.LocalAiRecoveryCleanupAllowed && ctx.LocalAiResolvedInstall?.Manifest.ReplacedManifest is not null && ctx.LocalAiRecoveryOriginalInstall is { } recoveryInstall) { var recoveryStore = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); - ctx.LocalAiResolvedInstall = await recoveryStore - .RestoreRecoveryManifestAsync( - ctx.LocalAiResolvedInstall.Manifest, - recoveryInstall.Manifest, - ct) - .ConfigureAwait(false); + try + { + ctx.LocalAiResolvedInstall = await recoveryStore + .RestoreRecoveryManifestAsync( + ctx.LocalAiResolvedInstall.Manifest, + recoveryInstall.Manifest, + ct) + .ConfigureAwait(false); + } + catch (InvalidDataException) + { + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + throw; + } ctx.LocalAiRecoveryProviderTransition = false; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + ctx.LocalAiRecoveryRollbackUncertain = false; } return; } @@ -862,6 +876,8 @@ internal static async Task RestoreUpgradeReceiptAsync(SetupContext ctx, Cancella { if (ctx.LocalAiUpgradeOriginalInstall is not { } originalInstall) return; + if (!ctx.LocalAiRecoveryCleanupAllowed) + return; var paths = new LocalAiPaths(ctx.LocalDataDir); var store = new LocalAiManifestStore(paths); @@ -942,6 +958,7 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati .ConfigureAwait(false); ctx.LocalAiRecoveryProviderTransition = false; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + ctx.LocalAiRecoveryRollbackUncertain = false; return StepResult.Ok("Local AI model replacement is committed."); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) @@ -1026,7 +1043,7 @@ public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) // it confirmed the Gateway no longer routes to this runtime's endpoint. If that could not // be confirmed, the Gateway may still be pointed at this runtime; disposing it here would // orphan the active route instead of the intended, coordinated rollback. - if (ctx.LocalAiRecoveryProviderTransition && !ctx.LocalAiRecoveryReceiptRollbackAllowed) + if (!ctx.LocalAiRecoveryCleanupAllowed) { ctx.Logger.Warn( "Keeping the replacement llama-server router running because the Gateway configuration " + diff --git a/src/OpenClaw.SetupEngine/SetupContext.cs b/src/OpenClaw.SetupEngine/SetupContext.cs index 63975d6ed..5cff84d26 100644 --- a/src/OpenClaw.SetupEngine/SetupContext.cs +++ b/src/OpenClaw.SetupEngine/SetupContext.cs @@ -586,6 +586,9 @@ public Func>? internal LocalAiResolvedInstall? LocalAiRecoveryPendingInstall { get; set; } internal bool LocalAiRecoveryProviderTransition { get; set; } internal bool LocalAiRecoveryReceiptRollbackAllowed { get; set; } + internal bool LocalAiRecoveryRollbackUncertain { get; set; } + internal bool LocalAiRecoveryCleanupAllowed => + !LocalAiRecoveryRollbackUncertain || LocalAiRecoveryReceiptRollbackAllowed; internal bool LocalAiManifestCreatedThisRun { get; set; } internal ILocalAiRuntime? LocalAiRuntime { get; set; } internal HostHardwareInfo? LocalAiGpuBaseline { get; set; } diff --git a/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs b/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs index a6908b00c..1c4742337 100644 --- a/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs +++ b/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs @@ -86,7 +86,8 @@ private static SetupInstallationStatus Aggregate(IEnumerable entries) "configure-local-ai-gateway" or "install-service" => SetupInstallationPhase.Install, "start-gateway" or "restart-gateway" or "mint-token" or "finalize-tailscale-serve" or "pair-operator" or "pair-node" or "verify-e2e" or - "run-wizard" or "windows-node-context" or "start-keepalive" => SetupInstallationPhase.Connect, + "run-wizard" or "windows-node-context" or "start-keepalive" or + "finalize-local-ai-model-replacement" => SetupInstallationPhase.Connect, _ => throw new ArgumentOutOfRangeException(nameof(stepId), stepId, "Installation step needs an explicit presentation phase."), }; } diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index 2a8b65539..a7ae7a14f 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1640,7 +1640,7 @@ public async Task PersistRollback_RestoresOriginalReceiptBeforeReplacementCleanu context.LocalAiResolvedInstall = store.ResolveAndValidate(pending); context.LocalAiRecoveryOriginalInstall = store.ResolveAndValidate(original); context.LocalAiRecoveryProviderTransition = true; - context.LocalAiRecoveryReceiptRollbackAllowed = true; + context.LocalAiRecoveryReceiptRollbackAllowed = false; await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); @@ -1652,6 +1652,78 @@ public async Task PersistRollback_RestoresOriginalReceiptBeforeReplacementCleanu Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); } + [Fact] + public async Task Rollback_PreservesPublishedUpgradeWhenGatewayCompensationIsUncertain() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest replacement = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(replacement); + var acquirer = new TrackingRuntimeAcquirer(); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(replacement); + context.LocalAiUpgradeOriginalInstall = store.ResolveAndValidate(original); + context.LocalAiRuntimeInstall = new LlamaRuntimeInstallResult( + temp.Path, + Path.Combine(temp.Path, "llama-server.exe"), + LlamaRuntimeInstallDisposition.Installed, + CreatedThisRun: true, + VerifiedArchives: [], + Rollback: null); + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryRollbackUncertain = true; + context.LocalAiRecoveryReceiptRollbackAllowed = false; + + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + await new AcquireLocalAiRuntimeStep(acquirer).RollbackAsync(context, CancellationToken.None); + + Assert.Equal(replacement.ModelCatalogId, (await store.LoadAsync())!.Manifest.ModelCatalogId); + Assert.NotNull(context.LocalAiUpgradeOriginalInstall); + Assert.NotNull(context.LocalAiRuntimeInstall); + Assert.Equal(0, acquirer.RemoveCalls); + } + + [Fact] + public async Task PersistRollback_RejectsStaleReplacementHistoryAndBlocksCleanup() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:18802/v1", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + LocalAiInstallManifest newer = pending with + { + Endpoint = "http://127.0.0.1:18803/v1", + PreviousEndpoints = [original.Endpoint!, pending.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(newer); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pending); + context.LocalAiRecoveryOriginalInstall = store.ResolveAndValidate(original); + context.LocalAiRecoveryProviderTransition = true; + + await Assert.ThrowsAsync(() => + new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None)); + + LocalAiInstallManifest retained = (await store.LoadAsync())!.Manifest; + Assert.Equal(newer.Endpoint, retained.Endpoint); + Assert.Equal(newer.PreviousEndpoints, retained.PreviousEndpoints); + Assert.True(context.LocalAiRecoveryRollbackUncertain); + Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + Assert.False(context.LocalAiRecoveryCleanupAllowed); + } + [Fact] public async Task ReconcileStep_RecoveryPinsIncompleteReceiptAsRollbackBaseline() { @@ -1750,6 +1822,7 @@ public async Task RecoveryPipeline_RewritesIncompleteReceiptAfterModelRepair(boo Assert.Equal(CacheRoot(temp.Path), repaired.Manifest.ModelCacheRoot); Assert.Equal(repaired.Manifest.CachedModelPath, repaired.ModelPath); Assert.False(context.LocalAiManifestCreatedThisRun); + Assert.Equal(pendingReplacement, context.LocalAiRecoveryRollbackUncertain); if (!pendingReplacement) { Assert.Null(repaired.Manifest.PreviousEndpoints); @@ -2221,6 +2294,22 @@ public Task InspectAsync( Task.FromResult(new LlamaRuntimeInspection(false, "invalid", "simulated corrupted runtime")); } + private sealed class TrackingRuntimeAcquirer : ILlamaRuntimeAcquirer + { + public int RemoveCalls { get; private set; } + + public Task InstallAsync( + string localDataDirectory, + LlamaRuntimeVariant runtime, + IProgress? progress, + CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public void RemoveInstalledRuntime( + string localDataDirectory, + LlamaRuntimeInstallResult install) => RemoveCalls++; + } + private sealed class AcceptingModelVerifier : ILocalAiModelFileVerifier { public Task VerifyActiveAsync( diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiOnboardingTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiOnboardingTests.cs index cd268471f..b4fdea480 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiOnboardingTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiOnboardingTests.cs @@ -599,7 +599,8 @@ public async Task Host_ExplicitUseRequiresPublicationAdmissionAndReturnsExactIde private static SetupLocalAiHost Host(GatewayRegistry registry, FakeRuntime runtime, Func install, Func? provider = null) => new(() => Task.FromResult(new LocalAiSetupResolution(LocalAiSetupRoute.Recovery, - new("gateway", "Managed", 18789, install()?.Manifest.ModelCatalogId, install()?.Manifest.RequestedPort))), + new("gateway", "Managed", 18789, install()?.Manifest.ModelCatalogId, + install()?.Manifest.RequestedPort, PinModelSelection: false))), () => registry, () => runtime, _ => Task.FromResult(install()), (_, _) => Task.FromResult(true), _ => Task.FromResult(Hardware), provider ?? (() => throw new InvalidOperationException("No route mutation expected."))); diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index 852935270..5acac0a5a 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -379,6 +379,7 @@ public async Task StartLocalAiRuntimeStep_KeepsReplacementRuntimeWhenGatewayRoll { var context = CreateContext(LocalAiRecoveryConfig()); context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryRollbackUncertain = true; context.LocalAiRecoveryReceiptRollbackAllowed = false; var runtime = new DisposeTrackingRuntime(); context.LocalAiRuntime = runtime; @@ -395,6 +396,7 @@ public async Task StartLocalAiRuntimeStep_DisposesRuntimeWhenGatewayRollbackConf { var context = CreateContext(LocalAiRecoveryConfig()); context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryRollbackUncertain = true; context.LocalAiRecoveryReceiptRollbackAllowed = true; var runtime = new DisposeTrackingRuntime(); context.LocalAiRuntime = runtime; From efcb0bb848d0f08485c04abd9747626e1b00037b Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 30 Sep 2026 18:54:00 -0700 Subject: [PATCH 14/22] fix(local-ai): settle receipt rollback after endpoint health --- .../LocalAiRecoveryPolicy.cs | 20 +++-- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 7 +- src/OpenClaw.SetupEngine/SetupPipeline.cs | 4 +- .../LocalAiInstallRecoveryTests.cs | 12 ++- .../SetupPipelineTests.cs | 90 ++++++++++++++++++- 5 files changed, 118 insertions(+), 15 deletions(-) diff --git a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs index 18a14f53f..561bfc9eb 100644 --- a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs +++ b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs @@ -208,26 +208,34 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) ctx.Logger.Warn( "The previous Local AI endpoint could not be verified as healthy; preserving the replacement " + "receipt instead of restoring a receipt for an endpoint that is not confirmed reachable."); + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; } else { try { var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); - await store.SaveAsync(originalInstall.Manifest, ct).ConfigureAwait(false); - ctx.LocalAiResolvedInstall = store.ResolveAndValidate(originalInstall.Manifest); + ctx.LocalAiResolvedInstall = await store + .RestoreRecoveryManifestAsync( + ctx.LocalAiResolvedInstall!.Manifest, + originalInstall.Manifest, + ct) + .ConfigureAwait(false); + ctx.LocalAiRecoveryProviderTransition = false; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + ctx.LocalAiRecoveryRollbackUncertain = false; + ctx.LocalAiGatewayPriorState = null; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) { receiptError = ex; + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; ctx.Logger.Warn( $"Restoring the previous Local AI endpoint receipt failed ({ex.GetType().Name})."); } } - ctx.LocalAiRecoveryProviderTransition = false; - ctx.LocalAiRecoveryReceiptRollbackAllowed = false; - ctx.LocalAiRecoveryRollbackUncertain = false; - ctx.LocalAiGatewayPriorState = null; } if (ctx.LocalAiRecoveryStoppedWsl) diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 4b7c8a4fe..3cbeffa25 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -837,7 +837,11 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) if (!ctx.LocalAiManifestCreatedThisRun) { - if (ctx.LocalAiRecoveryCleanupAllowed && + // Before Gateway configuration is enrolled, this step still owns restoring a fresh + // replacement receipt. Once configuration starts, the recovery guard must settle the + // route and endpoint-health decision before any receipt or resource cleanup occurs. + if (ctx.LocalAiGatewayPriorState is null && + !ctx.LocalAiRecoveryRollbackUncertain && ctx.LocalAiResolvedInstall?.Manifest.ReplacedManifest is not null && ctx.LocalAiRecoveryOriginalInstall is { } recoveryInstall) { @@ -859,7 +863,6 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) } ctx.LocalAiRecoveryProviderTransition = false; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; - ctx.LocalAiRecoveryRollbackUncertain = false; } return; } diff --git a/src/OpenClaw.SetupEngine/SetupPipeline.cs b/src/OpenClaw.SetupEngine/SetupPipeline.cs index 2959a5923..6fa9f926d 100644 --- a/src/OpenClaw.SetupEngine/SetupPipeline.cs +++ b/src/OpenClaw.SetupEngine/SetupPipeline.cs @@ -101,7 +101,6 @@ public static List BuildLocalAiRecoverySteps() => [ new PreflightOsStep(), new ValidateLocalAiRecoveryGatewayStep(), - new PreserveLocalAiRecoveryGatewayStep(), new PreflightLocalAiHardwareStep(), new PreflightWslStep(), new EnsureWslPlatformStep(reusePreflightResult: true), @@ -116,6 +115,9 @@ public static List BuildLocalAiRecoverySteps() => new ValidateLocalAiRecoveryGatewayStep(finalCheck: true), new ConfigureLocalAiWslNetworkingStep(), new VerifyLocalAiWslStep(), + // Roll this guard back immediately after Gateway compensation so it can settle the + // receipt and endpoint-health decision before runtime and asset cleanup begins. + new PreserveLocalAiRecoveryGatewayStep(), new ConfigureLocalAiGatewayStep(), new RestartGatewayStep(), new FinalizeLocalAiModelReplacementStep(), diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index a7ae7a14f..bcefa9d24 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1623,7 +1623,7 @@ public async Task FinalizeReplacement_ClearsRollbackReceipt() } [Fact] - public async Task PersistRollback_RestoresOriginalReceiptBeforeReplacementCleanup() + public async Task PersistRollback_RestoresOriginalReceiptBeforeGatewayGuard() { using var temp = new TempDirectory(); LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); @@ -1689,7 +1689,7 @@ public async Task Rollback_PreservesPublishedUpgradeWhenGatewayCompensationIsUnc } [Fact] - public async Task PersistRollback_RejectsStaleReplacementHistoryAndBlocksCleanup() + public async Task RecoveryGuard_RejectsStaleReplacementHistoryAndBlocksCleanup() { using var temp = new TempDirectory(); LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); @@ -1713,8 +1713,12 @@ public async Task PersistRollback_RejectsStaleReplacementHistoryAndBlocksCleanup context.LocalAiRecoveryOriginalInstall = store.ResolveAndValidate(original); context.LocalAiRecoveryProviderTransition = true; - await Assert.ThrowsAsync(() => - new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None)); + context.LocalAiRecoveryReceiptRollbackAllowed = true; + await Assert.ThrowsAsync(() => + new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("restarted")), + (_, _) => Task.FromResult(true)) + .RollbackAsync(context, CancellationToken.None)); LocalAiInstallManifest retained = (await store.LoadAsync())!.Manifest; Assert.Equal(newer.Endpoint, retained.Endpoint); diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index 5acac0a5a..3fcf7ed56 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -215,7 +215,10 @@ public void BuildLocalAiRecoverySteps_PreservesExistingWslGateway() steps.FindIndex(step => step is AcquireLocalAiRuntimeStep)); Assert.True( steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep) < - steps.FindIndex(step => step is ConfigureLocalAiWslNetworkingStep)); + steps.FindIndex(step => step is ConfigureLocalAiGatewayStep)); + Assert.True( + steps.FindIndex(step => step is VerifyLocalAiWslStep) < + steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep)); Assert.IsType( steps[steps.FindIndex(step => step is ConfigureLocalAiWslNetworkingStep) - 1]); } @@ -461,9 +464,19 @@ public async Task PreserveLocalAiRecoveryGateway_RestoresReceiptWhenOriginalEndp using var temp = new TempDirectory("local-ai-recovery-rollback-"); var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); LocalAiResolvedInstall originalInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18801); + LocalAiResolvedInstall replacementInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + LocalAiInstallManifest pendingManifest = replacementInstall.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + ReplacedManifest = originalInstall.Manifest, + PreviousEndpoints = [originalInstall.Manifest.Endpoint!], + }; + replacementInstall = replacementInstall with { Manifest = pendingManifest }; + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(pendingManifest); context.LocalAiRecoveryOriginalInstall = originalInstall; context.LocalAiRecoveryReceiptRollbackAllowed = true; - context.LocalAiResolvedInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + context.LocalAiResolvedInstall = replacementInstall; var step = new PreserveLocalAiRecoveryGatewayStep( (_, _) => Task.FromResult(StepResult.Ok("restarted")), (_, _) => Task.FromResult(true)); @@ -475,6 +488,79 @@ public async Task PreserveLocalAiRecoveryGateway_RestoresReceiptWhenOriginalEndp Assert.Equal(originalInstall.Endpoint, context.LocalAiResolvedInstall!.Endpoint); } + [Fact] + public async Task RecoveryRollback_PreservesReplacementWhenCompensatedOriginalEndpointIsUnhealthy() + { + using var temp = new TempDirectory("local-ai-recovery-rollback-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall originalInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18801); + LocalAiResolvedInstall replacementInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + LocalAiInstallManifest pendingManifest = replacementInstall.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + ReplacedManifest = originalInstall.Manifest, + PreviousEndpoints = [originalInstall.Manifest.Endpoint!], + }; + replacementInstall = replacementInstall with { Manifest = pendingManifest }; + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(pendingManifest); + context.LocalAiRecoveryOriginalInstall = originalInstall; + context.LocalAiResolvedInstall = replacementInstall; + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiGatewayPriorState = new LocalAiGatewayPriorState( + ProviderExisted: true, + ProviderJson: "{}", + PrimaryModelExisted: true, + PrimaryModelJson: "\"test-model\""); + var runtime = new DisposeTrackingRuntime(); + context.LocalAiRuntime = runtime; + var probedEndpoints = new List(); + var persist = new PersistLocalAiManifestStep(); + var start = new StartLocalAiRuntimeStep(_ => runtime); + var pipeline = new SetupPipeline([ + new MockStep( + "persist-local-ai-manifest", + (_, _) => Task.FromResult(StepResult.Ok("persisted")), + persist.RollbackAsync), + new MockStep( + "start-local-ai-runtime", + (_, _) => Task.FromResult(StepResult.Ok("started")), + start.RollbackAsync), + new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("restarted")), + (install, _) => + { + probedEndpoints.Add(install.Endpoint); + return Task.FromResult(false); + }), + new MockStep( + "configure-local-ai-gateway", + (_, _) => Task.FromResult(StepResult.Ok("configured")), + (ctx, _) => + { + ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; + return Task.CompletedTask; + }), + new MockStep( + "finalize-local-ai-model-replacement", + (_, _) => Task.FromResult(StepResult.Fail("finalization failed"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal([originalInstall.Endpoint], probedEndpoints); + Assert.Equal(0, runtime.DisposeCalls); + Assert.False(context.LocalAiRecoveryCleanupAllowed); + LocalAiInstallManifest retained = (await new LocalAiManifestStore( + new LocalAiPaths(context.LocalDataDir)).LoadAsync())!.Manifest; + Assert.Equal(pendingManifest.ModelCatalogId, retained.ModelCatalogId); + Assert.NotNull(retained.ReplacedManifest); + } + private sealed class DisposeTrackingRuntime : ILocalAiRuntime { public int DisposeCalls { get; private set; } From 531ddf5b35a0213e18ada97183599a190cfbc515 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 30 Sep 2026 22:23:22 -0700 Subject: [PATCH 15/22] fix(local-ai): distinguish current gateway rollback --- .../LocalAiGatewayConfiguration.cs | 4 +- .../LocalAiRecoveryPolicy.cs | 1 + src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 6 +- src/OpenClaw.SetupEngine/SetupContext.cs | 1 + .../SetupInstallationProgress.cs | 5 +- .../LocalAiGatewayUninstallTests.cs | 74 ++++++++----------- .../LocalAiInstallRecoveryTests.cs | 1 + .../SetupInstallationProgressTests.cs | 4 +- 8 files changed, 47 insertions(+), 49 deletions(-) diff --git a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs index e5251fc88..083bdc7cc 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs @@ -154,7 +154,8 @@ pendingRoute is not null && prior.PrimaryModelExisted && return StepResult.Fail( "The existing llamacpp gateway route is not the exact companion-managed configuration; preserving it."); } - if (matchesCurrentInstall && install.Manifest.ReplacedManifest is not null) + if (matchesCurrentInstall && install.Manifest.ReplacedManifest is not null && + !ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun) { // A previous process already published the replacement. Rollback must // preserve that live route instead of reconstructing the older one. @@ -231,6 +232,7 @@ pendingRoute is not null && prior.PrimaryModelExisted && string batchJson = LocalAiGatewayConfigBuilder.BuildBatchJson(ctx); if (ctx.LocalAiRecoveryProviderTransition) { + ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = true; ctx.LocalAiRecoveryRollbackUncertain = true; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; } diff --git a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs index 561bfc9eb..34e5db2e1 100644 --- a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs +++ b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs @@ -226,6 +226,7 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) ctx.LocalAiRecoveryReceiptRollbackAllowed = false; ctx.LocalAiRecoveryRollbackUncertain = false; ctx.LocalAiGatewayPriorState = null; + ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = false; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) { diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 3cbeffa25..20c7adbd6 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -879,7 +879,10 @@ internal static async Task RestoreUpgradeReceiptAsync(SetupContext ctx, Cancella { if (ctx.LocalAiUpgradeOriginalInstall is not { } originalInstall) return; - if (!ctx.LocalAiRecoveryCleanupAllowed) + // A resumed pending replacement uses that pending receipt as the upgrade baseline, so + // restoring it before this process touches Gateway preserves any route published by an + // earlier process. After Gateway I/O begins, the recovery guard must settle authority. + if (!ctx.LocalAiRecoveryCleanupAllowed && ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun) return; var paths = new LocalAiPaths(ctx.LocalDataDir); @@ -962,6 +965,7 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati ctx.LocalAiRecoveryProviderTransition = false; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; ctx.LocalAiRecoveryRollbackUncertain = false; + ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = false; return StepResult.Ok("Local AI model replacement is committed."); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) diff --git a/src/OpenClaw.SetupEngine/SetupContext.cs b/src/OpenClaw.SetupEngine/SetupContext.cs index 5cff84d26..ed6465bdd 100644 --- a/src/OpenClaw.SetupEngine/SetupContext.cs +++ b/src/OpenClaw.SetupEngine/SetupContext.cs @@ -595,6 +595,7 @@ public Func>? internal LlamaServerInferenceVerification? LocalAiInferenceVerification { get; set; } internal LocalAiGpuLoadEvidence? LocalAiGpuLoadEvidence { get; set; } internal LocalAiGatewayPriorState? LocalAiGatewayPriorState { get; set; } + internal bool LocalAiRecoveryGatewayConfigurationStartedThisRun { get; set; } internal bool IsUninstalling { get; set; } internal bool LocalAiRecoveryStoppedWsl { get; set; } diff --git a/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs b/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs index 1c4742337..e51d72bd5 100644 --- a/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs +++ b/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs @@ -74,14 +74,15 @@ private static SetupInstallationStatus Aggregate(IEnumerable entries) { "validate-distro-path" or "preflight-os" or "preflight-local-ai-hardware" or "preflight-wsl" or "preflight-windows-tailscale" or "ensure-wsl-platform" or "validate-local-ai-recovery-gateway" or - "preserve-local-ai-recovery-gateway" or "reconcile-local-ai-installation" or + "reconcile-local-ai-installation" or "cleanup-distro" or "cleanup-gateway" or "preflight-port" or "wsl-create" or "wsl-configure" or "validate-wsl-lockdown" => SetupInstallationPhase.Prepare, "acquire-local-ai-runtime" or "acquire-local-ai-model" or "persist-local-ai-manifest" or "start-local-ai-runtime" or "configure-local-ai-wsl-networking" => localAiRecovery ? SetupInstallationPhase.Install : SetupInstallationPhase.Prepare, "capture-local-ai-gpu-baseline" or "verify-local-ai-inference" or "verify-local-ai-gpu-load" or - "revalidate-local-ai-recovery-gateway" or "install-cli" or "verify-local-ai-wsl" or + "revalidate-local-ai-recovery-gateway" or "preserve-local-ai-recovery-gateway" or + "install-cli" or "verify-local-ai-wsl" or "install-tailscale" or "authorize-tailscale" or "configure-gateway" or "configure-local-ai-gateway" or "install-service" => SetupInstallationPhase.Install, "start-gateway" or "restart-gateway" or "mint-token" or diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index 3f82ffb9b..a7a3c7aaf 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -288,6 +288,10 @@ public async Task Recovery_ReplacesExactManagedProviderAfterAutomaticPortChanges StepResult result = await step.ExecuteAsync(context, CancellationToken.None); await step.RollbackAsync(context, CancellationToken.None); + await new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("not needed")), + (_, _) => Task.FromResult(true)) + .RollbackAsync(context, CancellationToken.None); await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); Assert.Equal(StepOutcome.Success, result.Outcome); @@ -402,10 +406,7 @@ public async Task Recovery_DriftedProviderRejectsBeforeGatewayMutation() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); context.LocalAiResolvedInstall = new LocalAiResolvedInstall( replacementManifest, original.ExecutablePath, @@ -434,10 +435,7 @@ public async Task Recovery_RollbackRestoresOriginalProviderAndReceipt() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -470,10 +468,7 @@ public async Task Recovery_RollbackPreservesEndpointCycleManagedPrimary() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -508,10 +503,7 @@ public async Task Recovery_FailedProviderSwitchRestoresOriginalReceipt() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -544,10 +536,7 @@ public async Task Recovery_FailureBeforeProviderConfigurationRestoresOriginalRec SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -584,10 +573,7 @@ public async Task Recovery_RetryPreservesOriginalProviderRollbackBaseline() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -624,10 +610,7 @@ public async Task Recovery_FailedProviderCompensationKeepsReplacementReceipt() context.Config.RollbackOnFailure = true; context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -653,8 +636,10 @@ public async Task Recovery_FailedProviderCompensationKeepsReplacementReceipt() commands.ProviderJson!, context.LocalAiResolvedInstall)); Assert.Equal(new Uri(replacementManifest.Endpoint!), (await store.LoadAsync())!.Endpoint); - Assert.False(context.LocalAiRecoveryProviderTransition); + Assert.True(context.LocalAiRecoveryProviderTransition); Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + Assert.False(context.LocalAiRecoveryCleanupAllowed); + Assert.True(context.LocalAiRecoveryGatewayConfigurationStartedThisRun); } [Fact] @@ -670,10 +655,7 @@ public async Task Recovery_LostRollbackAcknowledgementRestoresOriginalReceipt() context.Config.RollbackOnFailure = true; context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -718,10 +700,7 @@ public async Task Recovery_RollbackCancellationKeepsReplacementReceipt() context.Config.RollbackOnFailure = true; context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -747,8 +726,9 @@ public async Task Recovery_RollbackCancellationKeepsReplacementReceipt() commands.ProviderJson!, context.LocalAiResolvedInstall)); Assert.Equal(new Uri(replacementManifest.Endpoint!), (await store.LoadAsync())!.Endpoint); - Assert.False(context.LocalAiRecoveryProviderTransition); + Assert.True(context.LocalAiRecoveryProviderTransition); Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + Assert.False(context.LocalAiRecoveryCleanupAllowed); } [Fact] @@ -762,10 +742,7 @@ public async Task Recovery_ProviderCreationRollbackCancellationKeepsReplacementR context.Config.RollbackOnFailure = true; context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -791,8 +768,9 @@ public async Task Recovery_ProviderCreationRollbackCancellationKeepsReplacementR commands.ProviderJson!, context.LocalAiResolvedInstall)); Assert.Equal(new Uri(replacementManifest.Endpoint!), (await store.LoadAsync())!.Endpoint); - Assert.False(context.LocalAiRecoveryProviderTransition); + Assert.True(context.LocalAiRecoveryProviderTransition); Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + Assert.False(context.LocalAiRecoveryCleanupAllowed); } private static SetupContext CreateContext(string localDataDirectory, ICommandRunner commands) @@ -808,6 +786,16 @@ private static SetupContext CreateContext(string localDataDirectory, ICommandRun localDataDir: localDataDirectory); } + private static LocalAiInstallManifest ReplacementManifest(LocalAiResolvedInstall original) => + original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39876/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Endpoint!.AbsoluteUri], + }; + private static SetupContext CreateRecoveryContext( string localDataDirectory, ICommandRunner commands) diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index bcefa9d24..c56ab49e3 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1678,6 +1678,7 @@ public async Task Rollback_PreservesPublishedUpgradeWhenGatewayCompensationIsUnc context.LocalAiRecoveryProviderTransition = true; context.LocalAiRecoveryRollbackUncertain = true; context.LocalAiRecoveryReceiptRollbackAllowed = false; + context.LocalAiRecoveryGatewayConfigurationStartedThisRun = true; await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); await new AcquireLocalAiRuntimeStep(acquirer).RollbackAsync(context, CancellationToken.None); diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupInstallationProgressTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupInstallationProgressTests.cs index 179ce904d..9c4f8aeb1 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupInstallationProgressTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupInstallationProgressTests.cs @@ -107,9 +107,9 @@ public void Cancellation_BetweenStepsDoesNotOverwriteFailureOrSuccess() } [Fact] - public void Recovery_PreparesExistingGatewayThenLocalAiWithoutGatewayInstallClaim() + public void Recovery_GroupsGatewayPreservationWithLocalAiInstallation() { - Assert.Equal(SetupInstallationPhase.Prepare, SetupInstallationProgress.PhaseFor("preserve-local-ai-recovery-gateway", true)); + Assert.Equal(SetupInstallationPhase.Install, SetupInstallationProgress.PhaseFor("preserve-local-ai-recovery-gateway", true)); Assert.Equal(SetupInstallationPhase.Install, SetupInstallationProgress.PhaseFor("acquire-local-ai-model", true)); Assert.Equal(SetupInstallationPhase.Connect, SetupInstallationProgress.PhaseFor("restart-gateway", true)); } From 1ec7448ec9cafe1ebc2d8a8490f78c9ff7caf1a1 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 30 Sep 2026 22:49:03 -0700 Subject: [PATCH 16/22] fix(local-ai): clean pre-gateway runtime upgrades --- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 2 + .../LocalAiInstallRecoveryTests.cs | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 20c7adbd6..479d836d3 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -892,6 +892,8 @@ internal static async Task RestoreUpgradeReceiptAsync(SetupContext ctx, Cancella ctx.LocalAiResolvedInstall = store.ResolveAndValidate(originalInstall.Manifest); ctx.LocalAiManifestCreatedThisRun = false; ctx.LocalAiUpgradeOriginalInstall = null; + if (!ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun) + ctx.LocalAiRecoveryRollbackUncertain = false; } private static ImmutableArray BuildRuntimeReceipts( diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index c56ab49e3..15700da34 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1652,6 +1652,43 @@ public async Task PersistRollback_RestoresOriginalReceiptBeforeGatewayGuard() Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); } + [Fact] + public async Task Rollback_RestoresUpgradeAndRemovesTaskOwnedRuntimeBeforeGatewayConfiguration() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest replacement = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(replacement); + var acquirer = new TrackingRuntimeAcquirer(); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(replacement); + context.LocalAiUpgradeOriginalInstall = store.ResolveAndValidate(original); + context.LocalAiRuntimeInstall = new LlamaRuntimeInstallResult( + temp.Path, + Path.Combine(temp.Path, "llama-server.exe"), + LlamaRuntimeInstallDisposition.Installed, + CreatedThisRun: true, + VerifiedArchives: [], + Rollback: null); + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryRollbackUncertain = true; + context.LocalAiRecoveryReceiptRollbackAllowed = false; + + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + await new AcquireLocalAiRuntimeStep(acquirer).RollbackAsync(context, CancellationToken.None); + + Assert.Equal(original.ModelCatalogId, (await store.LoadAsync())!.Manifest.ModelCatalogId); + Assert.Null(context.LocalAiUpgradeOriginalInstall); + Assert.Null(context.LocalAiRuntimeInstall); + Assert.False(context.LocalAiRecoveryRollbackUncertain); + Assert.Equal(1, acquirer.RemoveCalls); + } + [Fact] public async Task Rollback_PreservesPublishedUpgradeWhenGatewayCompensationIsUncertain() { From c64fddb3d7343fa79e10183276ffe4e4a0a87409 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 1 Oct 2026 00:56:39 -0700 Subject: [PATCH 17/22] fix(local-ai): hand off tray runtime during recovery --- .../LocalAi/LlamaServerRuntimeService.cs | 263 +++++++++--- .../LocalAi/LocalAiRuntimeModels.cs | 30 ++ .../Pages/ProgressPage.xaml.cs | 28 +- .../SetupWindow.xaml.cs | 3 + .../LocalAiGatewayConfiguration.cs | 178 ++++++++ .../LocalAiGpuVerification.cs | 6 +- src/OpenClaw.SetupEngine/LocalAiOnboarding.cs | 5 + .../LocalAiRecoveryPolicy.cs | 97 ++++- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 101 ++++- src/OpenClaw.SetupEngine/SetupContext.cs | 3 + src/OpenClaw.SetupEngine/SetupPipeline.cs | 14 + .../Services/SetupLocalAiHost.cs | 2 + .../LocalAiPortLifecycleTests.cs | 125 ++++++ .../LocalAiGatewayUninstallTests.cs | 259 ++++++++++++ .../SetupPipelineTests.cs | 396 +++++++++++++++++- 15 files changed, 1413 insertions(+), 97 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs index 960646447..7643e9d23 100644 --- a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs +++ b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs @@ -129,6 +129,7 @@ public sealed class LlamaServerRuntimeService : ILocalAiRuntime private bool _explicitStopRequested; private bool _automaticResumeSuppressed; private bool _gatewayRouteRequiresResolution; + private bool _setupOwnsEndpointLifecycle; private bool _disposed; private bool _acceptExitTasks = true; private int _disposeStarted; @@ -216,14 +217,143 @@ public async Task RefreshAsync(CancellationToken cancell } public async Task StopAsync(CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return await StopWithStateAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task StopForSetupAsync(CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + _setupOwnsEndpointLifecycle = true; + return await StopWithStateAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task RestartAsync(CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return await RestartCoreAsync( + cancellationToken, + enableAutomaticRecovery: !_setupOwnsEndpointLifecycle) + .ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task RestartForSetupAsync(CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + _setupOwnsEndpointLifecycle = true; + return await RestartCoreAsync(cancellationToken, enableAutomaticRecovery: false).ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task RestartForSetupRollbackAsync( + CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + ThrowIfDisposed(); + _setupOwnsEndpointLifecycle = false; + if (!await TryLoadInstallAsync(cancellationToken).ConfigureAwait(false)) + return Snapshot; + return await RestartCoreAsync(cancellationToken, enableAutomaticRecovery: true).ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + ThrowIfDisposed(); + LocalAiRuntimeSnapshot current = Snapshot; + if (current.State != LocalAiRuntimeState.Healthy || + current.Ownership != LocalAiOwnership.CompanionManaged) + { + throw new InvalidOperationException( + "The setup Gateway route cannot be acknowledged without a healthy managed Local AI runtime."); + } + await _options.EndpointLifecycle + .SetAutomaticRecoveryEnabledAsync(true, cancellationToken) + .ConfigureAwait(false); + _automaticResumeSuppressed = false; + _setupOwnsEndpointLifecycle = false; + _gatewayRouteRequiresResolution = false; + return SetSnapshot(current with + { + GatewayRouteRequiresResolution = false, + UpdatedAtUtc = _platform.UtcNow, + }); + } + finally + { + _operationGate.Release(); + } + } + + public async Task ReleaseSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) { await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); try { ThrowIfDisposed(); - _automaticResumeSuppressed = true; - await _options.EndpointLifecycle.SetAutomaticRecoveryEnabledAsync(false, cancellationToken).ConfigureAwait(false); - _explicitStopRequested = true; + await _options.EndpointLifecycle + .SetAutomaticRecoveryEnabledAsync(true, cancellationToken) + .ConfigureAwait(false); + _automaticResumeSuppressed = false; + _setupOwnsEndpointLifecycle = false; + LocalAiRuntimeSnapshot current = Snapshot; + return SetSnapshot(current with { UpdatedAtUtc = _platform.UtcNow }); + } + finally + { + _operationGate.Release(); + } + } + + private async Task StopWithStateAsync(CancellationToken cancellationToken) + { + ThrowIfDisposed(); + _automaticResumeSuppressed = true; + await _options.EndpointLifecycle + .SetAutomaticRecoveryEnabledAsync(false, cancellationToken) + .ConfigureAwait(false); + _explicitStopRequested = true; + try + { LocalAiRuntimeSnapshot stopped = await StopCoreAsync( LocalAiQuiesceReason.Teardown, cancellationToken) @@ -251,77 +381,69 @@ public async Task StopAsync(CancellationToken cancellati } throw; } - finally - { - _operationGate.Release(); - } } - public async Task RestartAsync(CancellationToken cancellationToken = default) + private async Task RestartCoreAsync( + CancellationToken cancellationToken, + bool enableAutomaticRecovery) { - await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + ThrowIfDisposed(); + await _options.EndpointLifecycle + .SetAutomaticRecoveryEnabledAsync(enableAutomaticRecovery, cancellationToken) + .ConfigureAwait(false); + _automaticResumeSuppressed = !enableAutomaticRecovery; + _explicitStopRequested = false; + LocalAiResolvedInstall? restartInstall = _install; try { - ThrowIfDisposed(); - await _options.EndpointLifecycle.SetAutomaticRecoveryEnabledAsync(true, cancellationToken).ConfigureAwait(false); - _automaticResumeSuppressed = false; - _explicitStopRequested = false; - LocalAiResolvedInstall? restartInstall = _install; - try - { - LocalAiRuntimeSnapshot stopped = await StopCoreAsync( - LocalAiQuiesceReason.EndpointCycle, - cancellationToken) - .ConfigureAwait(false); - restartInstall ??= _install; - if (_managedProcess is not null || stopped.State == LocalAiRuntimeState.Failed) - return stopped; + LocalAiRuntimeSnapshot stopped = await StopCoreAsync( + LocalAiQuiesceReason.EndpointCycle, + cancellationToken) + .ConfigureAwait(false); + restartInstall ??= _install; + if (_managedProcess is not null || stopped.State == LocalAiRuntimeState.Failed) + return stopped; - _restartAttempts = 0; - LocalAiRuntimeSnapshot restarted = await EnsureStartedCoreAsync(cancellationToken) - .ConfigureAwait(false); - if (restarted.State is LocalAiRuntimeState.Failed or LocalAiRuntimeState.NotInstalled && - restartInstall is not null) + _restartAttempts = 0; + LocalAiRuntimeSnapshot restarted = await EnsureStartedCoreAsync(cancellationToken) + .ConfigureAwait(false); + if (restarted.State is LocalAiRuntimeState.Failed or LocalAiRuntimeState.NotInstalled && + restartInstall is not null) + { + LocalAiResolvedInstall cleanupInstall = _install ?? restartInstall; + bool withdrawn = await WithdrawRouteAsync( + cleanupInstall, + "after restart startup did not complete").ConfigureAwait(false); + if (!withdrawn) { - LocalAiResolvedInstall cleanupInstall = _install ?? restartInstall; - bool withdrawn = await WithdrawRouteAsync( - cleanupInstall, - "after restart startup did not complete").ConfigureAwait(false); - if (!withdrawn) - { - return _managedProcess is { HasExited: false } - ? PublishManagedFailure( - "Local AI restart did not complete and gateway routing could not be safely disabled; the managed listener remains running.") - : PublishTerminalCleanupFailure( - "Local AI restart did not complete and gateway routing could not be safely disabled."); - } - if (_managedProcess is { HasExited: false }) - { - ++_generation; - await DisposeManagedProcessAsync(CancellationToken.None).ConfigureAwait(false); - return PublishTerminalCleanupFailure("Local AI restart did not complete."); - } + return _managedProcess is { HasExited: false } + ? PublishManagedFailure( + "Local AI restart did not complete and gateway routing could not be safely disabled; the managed listener remains running.") + : PublishTerminalCleanupFailure( + "Local AI restart did not complete and gateway routing could not be safely disabled."); + } + if (_managedProcess is { HasExited: false }) + { + ++_generation; + await DisposeManagedProcessAsync(CancellationToken.None).ConfigureAwait(false); + return PublishTerminalCleanupFailure("Local AI restart did not complete."); } - return restarted; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - LocalAiResolvedInstall? interruptedInstall = restartInstall ?? _install; - if (interruptedInstall is not null) - await CompleteInterruptedRestartAsync(interruptedInstall, "interrupted").ConfigureAwait(false); - throw; - } - catch (OperationCanceledException) - { - LocalAiResolvedInstall? canceledInstall = restartInstall ?? _install; - if (canceledInstall is not null) - await CompleteInterruptedRestartAsync(canceledInstall, "canceled").ConfigureAwait(false); - throw; } + return restarted; } - finally + catch (Exception ex) when (ex is not OperationCanceledException) { - _operationGate.Release(); + LocalAiResolvedInstall? interruptedInstall = restartInstall ?? _install; + if (interruptedInstall is not null) + await CompleteInterruptedRestartAsync(interruptedInstall, "interrupted").ConfigureAwait(false); + throw; + } + catch (OperationCanceledException) + { + LocalAiResolvedInstall? canceledInstall = restartInstall ?? _install; + if (canceledInstall is not null) + await CompleteInterruptedRestartAsync(canceledInstall, "canceled").ConfigureAwait(false); + throw; } } @@ -1189,6 +1311,12 @@ private async Task QuiesceRouteAsync( LocalAiQuiesceReason reason, CancellationToken cancellationToken) { + if (_setupOwnsEndpointLifecycle) + { + _gatewayRouteRequiresResolution = true; + return LocalAiEndpointLifecycleResult.Ok(); + } + _gatewayRouteRequiresResolution = true; LocalAiEndpointLifecycleResult result = await _options.EndpointLifecycle @@ -1203,6 +1331,9 @@ private async Task PublishRouteAsync( LocalAiResolvedInstall install, CancellationToken cancellationToken) { + if (_setupOwnsEndpointLifecycle) + return LocalAiEndpointLifecycleResult.Ok(); + LocalAiEndpointLifecycleResult result = await _options.EndpointLifecycle .PublishAsync(install, cancellationToken) .ConfigureAwait(false); @@ -1635,9 +1766,13 @@ private LocalAiRuntimeSnapshot Publish( { GatewayRouteRequiresResolution = _gatewayRouteRequiresResolution, }; + return SetSnapshot(value); + } + + private LocalAiRuntimeSnapshot SetSnapshot(LocalAiRuntimeSnapshot value) + { lock (_snapshotGate) _snapshot = value; - EventHandler? handler = StateChanged; if (handler is not null) { diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs b/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs index 4fc59a630..2cb288235 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs @@ -141,5 +141,35 @@ public interface ILocalAiRuntime : IAsyncDisposable Task ResumeAsync(CancellationToken cancellationToken = default); Task StopAsync(CancellationToken cancellationToken = default); Task RestartAsync(CancellationToken cancellationToken = default); + /// + /// Stops the managed process for a setup transaction without changing Gateway routing. + /// Implementations that publish endpoint lifecycle changes must suppress them here because + /// the setup pipeline coordinates the matching Gateway transaction separately. + /// + Task StopForSetupAsync(CancellationToken cancellationToken = default) => + StopAsync(cancellationToken); + /// + /// Restarts the managed process for a setup transaction without changing Gateway routing. + /// Implementations that publish endpoint lifecycle changes must suppress them here because + /// the setup pipeline coordinates the matching Gateway transaction separately. + /// + Task RestartForSetupAsync(CancellationToken cancellationToken = default) => + RestartAsync(cancellationToken); + /// + /// Adopts setup's restored receipt, then restarts through the ordinary Gateway lifecycle. + /// Use only before setup has begun its own Gateway configuration transaction. + /// + Task RestartForSetupRollbackAsync( + CancellationToken cancellationToken = default) => RestartAsync(cancellationToken); + /// + /// Acknowledges that setup committed or compensated the Gateway route for the current endpoint. + /// + Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) => Task.FromResult(Snapshot); + /// + /// Returns endpoint lifecycle ownership to the runtime without claiming Gateway reconciliation. + /// + Task ReleaseSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) => Task.FromResult(Snapshot); Task RefreshAsync(CancellationToken cancellationToken = default); } diff --git a/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs b/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs index 21e04abff..63d6b91eb 100644 --- a/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs +++ b/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs @@ -149,6 +149,13 @@ private async Task StartPipelineAsync() var steps = BuildSteps(config, _localAiRecoveryOnly); var setupOwner = _window; ctx.ExpectedGatewayRegistry = config.NativeLocalAiAcquisition ? null : setupOwner?.BeginGatewaySetup(); + ctx.LocalAiRuntime = setupOwner?.BorrowManagedLocalAiRuntime(); + ctx.LocalAiRuntimeBorrowed = ctx.LocalAiRuntime is not null; + if (ctx.LocalAiRuntimeBorrowed && !config.RollbackOnFailure) + { + throw new InvalidOperationException( + "Local AI recovery requires transactional rollback when borrowing the tray runtime."); + } ctx.PersistTraySettings = _window is { } settingsOwner ? settingsOwner.PersistPipelineSettings : null; _pipeline = new SetupPipeline(steps); _pipeline.StepProgress += OnStepProgress; @@ -156,9 +163,24 @@ private async Task StartPipelineAsync() var pipeline = _pipeline; var result = await SetupPipeline.RunWithSettlementAsync( () => Task.Run(() => pipeline.RunAsync(ctx), cts.Token), - outcome => config.NativeLocalAiAcquisition ? Task.CompletedTask : setupOwner?.SettleGatewaySetupAsync(ctx.ExpectedGatewayRegistry, - outcome?.Outcome == PipelineOutcome.Success ? config.LocalAiRecoveryGatewayId ?? ctx.GatewayRecordId : null) - ?? Task.CompletedTask); + async outcome => + { + try + { + await SetupPipeline.ReleaseBorrowedLocalAiRuntimeAfterFailureAsync(ctx, outcome); + } + finally + { + if (!config.NativeLocalAiAcquisition && setupOwner is not null) + { + await setupOwner.SettleGatewaySetupAsync( + ctx.ExpectedGatewayRegistry, + outcome?.Outcome == PipelineOutcome.Success + ? config.LocalAiRecoveryGatewayId ?? ctx.GatewayRecordId + : null); + } + } + }); sw.Stop(); _pipelineFinished = true; if (_closed || _window?.IsClosed == true) diff --git a/src/OpenClaw.SetupEngine.UI/SetupWindow.xaml.cs b/src/OpenClaw.SetupEngine.UI/SetupWindow.xaml.cs index 515a8cee7..fe375d3cb 100644 --- a/src/OpenClaw.SetupEngine.UI/SetupWindow.xaml.cs +++ b/src/OpenClaw.SetupEngine.UI/SetupWindow.xaml.cs @@ -490,6 +490,9 @@ public void SetWelcomeInstallSelected(bool installSelected) internal Task GetLocalAiHardwareAsync(bool forceRefresh = false) => _localAiHardwareProbe.GetAsync(forceRefresh); + internal ILocalAiRuntime? BorrowManagedLocalAiRuntime() => + _startAtLocalAiRecoveryReview ? _localAiHost?.BorrowManagedRuntime() : null; + internal Task GetWslViabilityAsync(bool refresh = false) => _wslViabilityProbe.GetAsync(refresh); diff --git a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs index 083bdc7cc..5fa83a378 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs @@ -69,6 +69,7 @@ public static string BuildRecoveryRestoreBatchJson( ]; return JsonSerializer.Serialize(operations); } + } public sealed class ConfigureLocalAiGatewayStep : SetupStep @@ -78,6 +79,9 @@ public sealed class ConfigureLocalAiGatewayStep : SetupStep private const string MissingValue = "MISSING"; private const string FailedValuePrefix = "FAILED:"; private const string BatchVariable = "OPENCLAW_LOCAL_AI_BATCH_B64"; + private const string ProviderVariable = "OPENCLAW_LOCAL_AI_PROVIDER_B64"; + private const string ExpectedProviderVariable = "OPENCLAW_LOCAL_AI_EXPECTED_PROVIDER_B64"; + private const string ConditionalSetUnsupportedMarker = "LOCAL_AI_CONDITIONAL_SET_UNSUPPORTED"; private const int MaximumSnapshotBytes = 1024 * 1024; public override string Id => "configure-local-ai-gateway"; @@ -245,6 +249,25 @@ pendingRoute is not null && prior.PrimaryModelExisted && : $"Local AI gateway configuration failed (exit {result.ExitCode})."); } + if (ctx.LocalAiRuntimeBorrowed && ctx.LocalAiRuntime is { } borrowedRuntime) + { + try + { + await borrowedRuntime.AcknowledgeSetupGatewayRouteAsync(ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + await RollbackAsync(ctx, CancellationToken.None).ConfigureAwait(false); + throw; + } + catch (Exception ex) + { + await RollbackAsync(ctx, CancellationToken.None).ConfigureAwait(false); + return StepResult.Fail( + "The Local AI runtime could not acknowledge the committed Gateway route.", ex); + } + } + return StepResult.Ok("Gateway configured to use the managed llama-server provider"); } @@ -376,6 +399,161 @@ await ReconcileFailedRecoveryRestoreAsync( } } + internal static async Task RestoreRecoveryRouteAsync( + SetupContext ctx, + LocalAiGatewayPriorState prior, + LocalAiResolvedInstall expectedInstall, + LocalAiResolvedInstall restoredInstall, + CancellationToken ct) + { + if (!prior.ProviderExisted) + return true; + + CommandResult currentResult = await CaptureStateAsync(ctx, ct).ConfigureAwait(false); + if (currentResult.ExitCode != 0 || currentResult.TimedOut) + return false; + LocalAiGatewayPriorState current; + try + { + current = ParseSnapshot(currentResult.Stdout); + } + catch (Exception ex) when (ex is FormatException or JsonException or InvalidDataException) + { + return false; + } + if (ProviderRoutesToEndpoint(current.ProviderJson!, restoredInstall.Endpoint!)) + return true; + if (!LocalAiGatewayProviderDefinition.MatchesProviderJson( + prior.ProviderJson!, + expectedInstall)) + { + return false; + } + // The runtime restart can move an automatic endpoint and take long enough for another + // actor to update Gateway. Only rewrite the restored route while we still own the exact + // rollback state that was captured before setup. + if (current.ProviderExisted != prior.ProviderExisted || + !JsonEquals(current.ProviderJson!, prior.ProviderJson!) || + current.PrimaryModelExisted != prior.PrimaryModelExisted || + (current.PrimaryModelExisted && + !JsonEquals(current.PrimaryModelJson!, prior.PrimaryModelJson!))) + { + return false; + } + + CommandResult restore = await ApplyConditionalProviderAsync( + ctx, + expectedInstall, + restoredInstall, + ct).ConfigureAwait(false); + if (restore.ExitCode == 42 && + restore.Stdout.Contains(ConditionalSetUnsupportedMarker, StringComparison.Ordinal)) + { + // Older protocol-v4 Gateways predate atomic conditional config writes. A separate + // read then write can overwrite a concurrent owner, so leave route resolution + // pending instead of attempting a lossy compatibility update. + return false; + } + if (restore.ExitCode != 0 || restore.TimedOut || + !restore.Stdout.Contains("LOCAL_AI_GATEWAY_RESTORED", StringComparison.Ordinal)) + { + return false; + } + + CommandResult verifiedResult = await CaptureStateAsync(ctx, ct).ConfigureAwait(false); + if (verifiedResult.ExitCode != 0 || verifiedResult.TimedOut) + return false; + try + { + LocalAiGatewayPriorState verified = ParseSnapshot(verifiedResult.Stdout); + bool matches = verified.ProviderExisted && + LocalAiGatewayProviderDefinition.MatchesProviderJson( + verified.ProviderJson!, + restoredInstall); + if (!matches) + return false; + // The conditional write changes only the owned provider. A primary-model update + // racing after the ownership snapshot is independent user state and is preserved. + return true; + } + catch (Exception ex) when (ex is FormatException or JsonException or InvalidDataException) + { + return false; + } + } + + private static bool ProviderRoutesToEndpoint(string providerJson, Uri endpoint) + { + try + { + using JsonDocument provider = JsonDocument.Parse(providerJson); + return provider.RootElement.TryGetProperty("baseUrl", out JsonElement baseUrl) && + baseUrl.ValueKind == JsonValueKind.String && + Uri.TryCreate(baseUrl.GetString(), UriKind.Absolute, out Uri? configured) && + string.Equals( + configured.AbsoluteUri.TrimEnd('/'), + endpoint.AbsoluteUri.TrimEnd('/'), + StringComparison.OrdinalIgnoreCase); + } + catch (JsonException) + { + return false; + } + } + + private static Task ApplyConditionalProviderAsync( + SetupContext ctx, + LocalAiResolvedInstall expectedInstall, + LocalAiResolvedInstall restoredInstall, + CancellationToken ct) + { + string script = $$""" + set -eu + {{ctx.WslPathPrefix}} + if ! openclaw config set --help | grep -Fq -- '--expect-current-json'; then + echo {{ConditionalSetUnsupportedMarker}} + exit 42 + fi + provider_json="$(printf '%s' "$OPENCLAW_LOCAL_AI_PROVIDER_B64" | base64 -d)" + expected_provider_json="$(printf '%s' "$OPENCLAW_LOCAL_AI_EXPECTED_PROVIDER_B64" | base64 -d)" + openclaw config set {{LocalAiGatewayConfigBuilder.ProviderPath}} "$provider_json" \ + --strict-json --replace --expect-current-json "$expected_provider_json" + echo LOCAL_AI_GATEWAY_RESTORED + """; + var environment = new Dictionary(StringComparer.Ordinal) + { + [ProviderVariable] = Convert.ToBase64String(Encoding.UTF8.GetBytes( + LocalAiGatewayProviderDefinition.BuildProviderJson(restoredInstall))), + [ExpectedProviderVariable] = Convert.ToBase64String(Encoding.UTF8.GetBytes( + LocalAiGatewayProviderDefinition.BuildProviderJson(expectedInstall))), + }; + return ctx.Commands.RunInWslAsync( + ctx.DistroName!, + script, + TimeSpan.FromMinutes(2), + environment, + ct, + ctx.Config.Wsl.User, + inputViaStdin: true); + } + + internal static async Task AcknowledgeBorrowedRuntimeRouteAsync( + SetupContext ctx, + CancellationToken ct) + { + if (!ctx.LocalAiRuntimeBorrowed || ctx.LocalAiRuntime is not { } borrowedRuntime) + return true; + try + { + await borrowedRuntime.AcknowledgeSetupGatewayRouteAsync(ct).ConfigureAwait(false); + return true; + } + catch (Exception ex) when (ex is InvalidOperationException or ObjectDisposedException) + { + return false; + } + } + private static async Task ReconcileFailedRecoveryRestoreAsync( SetupContext ctx, LocalAiGatewayPriorState prior, diff --git a/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs b/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs index 6316da3d4..5eb783148 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs @@ -310,7 +310,7 @@ ctx.LocalAiInferenceVerification is null || } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - await VerifyLocalAiInferenceStep.ResetRouterAsync(runtime); + await VerifyLocalAiInferenceStep.ResetRouterAsync(runtime, ctx.LocalAiRuntimeBorrowed); throw; } catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) @@ -318,7 +318,9 @@ ctx.LocalAiInferenceVerification is null || failure = ex; } - LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync(runtime); + LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync( + runtime, + ctx.LocalAiRuntimeBorrowed); if (failure is not null) return StepResult.Fail($"Local AI GPU verification failed: {failure.Message}", failure); if (reset.State != LocalAiRuntimeState.Healthy || diff --git a/src/OpenClaw.SetupEngine/LocalAiOnboarding.cs b/src/OpenClaw.SetupEngine/LocalAiOnboarding.cs index 73da67d25..b381798c1 100644 --- a/src/OpenClaw.SetupEngine/LocalAiOnboarding.cs +++ b/src/OpenClaw.SetupEngine/LocalAiOnboarding.cs @@ -213,6 +213,11 @@ public static Task InspectAsync(LocalAiResolvedInstall install, Cancellati /// Observation never calls a runtime refresh (which may publish or withdraw a route). public interface ISetupLocalAiHost { + /// + /// Borrows the tray-owned runtime for a recovery transaction. The setup pipeline may restart + /// this runtime, but its lifetime remains owned by the tray. + /// + ILocalAiRuntime? BorrowManagedRuntime() => null; OpenClaw.Connection.GatewayRegistrySnapshot BeginGatewaySetup(); Task ReconcileGatewaySetupAsync(OpenClaw.Connection.GatewayRegistrySnapshot expectedOutput, string? completedGatewayId); Task ObserveAsync(CancellationToken ct); diff --git a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs index 34e5db2e1..8af18902e 100644 --- a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs +++ b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs @@ -167,6 +167,13 @@ public sealed class PreserveLocalAiRecoveryGatewayStep : SetupStep { private readonly Func> _restart; private readonly Func> _probeOriginalEndpoint; + private readonly Func< + SetupContext, + LocalAiGatewayPriorState, + LocalAiResolvedInstall, + LocalAiResolvedInstall, + CancellationToken, + Task> _restoreRecoveryRoute; public PreserveLocalAiRecoveryGatewayStep() : this(StartGatewayStep.RestartAndWaitForHealthAsync, ProbeOriginalEndpointAsync) @@ -175,10 +182,18 @@ public PreserveLocalAiRecoveryGatewayStep() internal PreserveLocalAiRecoveryGatewayStep( Func> restart, - Func>? probeOriginalEndpoint = null) + Func>? probeOriginalEndpoint = null, + Func< + SetupContext, + LocalAiGatewayPriorState, + LocalAiResolvedInstall, + LocalAiResolvedInstall, + CancellationToken, + Task>? restoreRecoveryRoute = null) { _restart = restart ?? throw new ArgumentNullException(nameof(restart)); _probeOriginalEndpoint = probeOriginalEndpoint ?? ProbeOriginalEndpointAsync; + _restoreRecoveryRoute = restoreRecoveryRoute ?? ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync; } public override string Id => "preserve-local-ai-recovery-gateway"; @@ -203,6 +218,71 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) ctx.Logger.Warn( "The previous Local AI endpoint receipt was not restored because gateway provider rollback did not complete."); } + else if (ctx.LocalAiRuntimeBorrowed && ctx.LocalAiRuntime is { } borrowedRuntime) + { + try + { + var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); + ctx.LocalAiResolvedInstall = await store + .RestoreRecoveryManifestAsync( + ctx.LocalAiResolvedInstall!.Manifest, + originalInstall.Manifest, + ct) + .ConfigureAwait(false); + LocalAiRuntimeSnapshot restored = await borrowedRuntime + .RestartForSetupAsync(ct) + .ConfigureAwait(false); + LocalAiResolvedInstall restoredInstall = await store.LoadAsync(ct).ConfigureAwait(false) + ?? throw new InvalidDataException( + "The previous Local AI receipt was unavailable after restarting its runtime."); + if (restored.State != LocalAiRuntimeState.Healthy || + restored.Ownership != LocalAiOwnership.CompanionManaged || + restored.ModelId != restoredInstall.Manifest.ModelCatalogId || + restored.Endpoint != restoredInstall.Endpoint || + restored.ModelEvidence.State is not + (LocalAiModelAvailabilityState.Verified or LocalAiModelAvailabilityState.Loaded)) + { + throw new InvalidDataException( + restored.Detail ?? "The previous Local AI runtime could not be restored."); + } + ctx.LocalAiResolvedInstall = restoredInstall; + if (ctx.LocalAiGatewayPriorState is { } prior && + !await _restoreRecoveryRoute( + ctx, + prior, + originalInstall, + restoredInstall, + ct) + .ConfigureAwait(false)) + { + throw new InvalidDataException( + "The previous Local AI gateway route could not be updated to its restored endpoint."); + } + if (!await _probeOriginalEndpoint(restoredInstall, ct).ConfigureAwait(false)) + { + throw new InvalidDataException( + "The previous Local AI endpoint was not healthy after its runtime was restored."); + } + if (!await ConfigureLocalAiGatewayStep + .AcknowledgeBorrowedRuntimeRouteAsync(ctx, ct) + .ConfigureAwait(false)) + { + throw new InvalidDataException( + "The restored Local AI route could not be acknowledged by its runtime owner."); + } + ctx.LocalAiBorrowedRuntimeRestored = true; + CompleteReceiptRollback(ctx); + } + catch (Exception ex) when ( + ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + receiptError = ex; + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + ctx.Logger.Warn( + $"Restoring the previous Local AI runtime failed ({ex.GetType().Name})."); + } + } else if (!await _probeOriginalEndpoint(originalInstall, ct).ConfigureAwait(false)) { ctx.Logger.Warn( @@ -222,11 +302,7 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) originalInstall.Manifest, ct) .ConfigureAwait(false); - ctx.LocalAiRecoveryProviderTransition = false; - ctx.LocalAiRecoveryReceiptRollbackAllowed = false; - ctx.LocalAiRecoveryRollbackUncertain = false; - ctx.LocalAiGatewayPriorState = null; - ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = false; + CompleteReceiptRollback(ctx); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) { @@ -256,6 +332,15 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) } } + private static void CompleteReceiptRollback(SetupContext ctx) + { + ctx.LocalAiRecoveryProviderTransition = false; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + ctx.LocalAiRecoveryRollbackUncertain = false; + ctx.LocalAiGatewayPriorState = null; + ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = false; + } + /// /// Confirms the original (pre-recovery) llama-server endpoint is actually alive before the /// Gateway is pointed back at it. A stale manifest receipt alone cannot tell us whether the diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 479d836d3..9784781fd 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -832,11 +832,14 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) if (ctx.LocalAiUpgradeOriginalInstall is not null) { await RestoreUpgradeReceiptAsync(ctx, ct); + if (ctx.LocalAiUpgradeOriginalInstall is null) + await RestartBorrowedRuntimeAsync(ctx, ct); return; } if (!ctx.LocalAiManifestCreatedThisRun) { + bool restoredRecoveryReceipt = false; // Before Gateway configuration is enrolled, this step still owns restoring a fresh // replacement receipt. Once configuration starts, the recovery guard must settle the // route and endpoint-health decision before any receipt or resource cleanup occurs. @@ -863,7 +866,10 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) } ctx.LocalAiRecoveryProviderTransition = false; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + restoredRecoveryReceipt = true; } + if (restoredRecoveryReceipt) + await RestartBorrowedRuntimeAsync(ctx, ct); return; } @@ -875,6 +881,30 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) ctx.LocalAiManifestCreatedThisRun = false; } + private static async Task RestartBorrowedRuntimeAsync(SetupContext ctx, CancellationToken ct) + { + if (!ctx.LocalAiRuntimeBorrowed || ctx.LocalAiRuntime is null) + return; + // Before ConfigureLocalAiGatewayStep begins, the tray runtime still owns Gateway + // publication. Use its ordinary lifecycle so an automatic-port move is published. + LocalAiRuntimeSnapshot restored = await ctx.LocalAiRuntime.RestartForSetupRollbackAsync(ct); + LocalAiResolvedInstall expected = await new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)) + .LoadAsync(ct) + .ConfigureAwait(false) + ?? throw new InvalidOperationException("The restored Local AI receipt is unavailable."); + ctx.LocalAiResolvedInstall = expected; + if (restored.State != LocalAiRuntimeState.Healthy || + restored.Ownership != LocalAiOwnership.CompanionManaged || + restored.ModelId != expected.Manifest.ModelCatalogId || + restored.Endpoint != expected.Endpoint || + restored.ModelEvidence.State is not + (LocalAiModelAvailabilityState.Verified or LocalAiModelAvailabilityState.Loaded)) + { + throw new InvalidOperationException( + restored.Detail ?? "The previous Local AI runtime could not be restored."); + } + } + internal static async Task RestoreUpgradeReceiptAsync(SetupContext ctx, CancellationToken ct) { if (ctx.LocalAiUpgradeOriginalInstall is not { } originalInstall) @@ -1001,20 +1031,37 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati { if (ctx.LocalAiResolvedInstall is null) return StepResult.Terminal("llama-server startup requires a verified installation receipt."); - if (ctx.LocalAiRuntime is not null) + if (ctx.LocalAiRuntime is not null && !ctx.LocalAiRuntimeBorrowed) return StepResult.Terminal("A Local AI runtime is already attached to this setup transaction."); + if (ctx.LocalAiRuntimeBorrowed && + ctx.LocalAiUpgradeOriginalInstall is null && + ctx.LocalAiResolvedInstall.Manifest.ReplacedManifest is null) + { + return StepResult.Terminal( + "Borrowing the tray Local AI runtime requires a recorded model replacement or runtime upgrade."); + } - ILocalAiRuntime runtime = _runtimeFactory(ctx); + ILocalAiRuntime runtime = ctx.LocalAiRuntime ?? _runtimeFactory(ctx); ctx.LocalAiRuntime = runtime; try { - LocalAiRuntimeSnapshot snapshot = await runtime.EnsureStartedAsync(ct); + LocalAiRuntimeSnapshot snapshot; + if (ctx.LocalAiRuntimeBorrowed) + { + ctx.LocalAiBorrowedRuntimeRestartedThisRun = true; + snapshot = await runtime.RestartForSetupAsync(ct); + } + else + { + snapshot = await runtime.EnsureStartedAsync(ct); + } if (snapshot.State != LocalAiRuntimeState.Healthy || snapshot.Ownership != LocalAiOwnership.CompanionManaged || snapshot.ProcessId is null || + snapshot.ModelId != ctx.LocalAiResolvedInstall.Manifest.ModelCatalogId || snapshot.ModelEvidence.State != LocalAiModelAvailabilityState.Verified) { - await DisposeRuntimeAsync(ctx); + await CleanUpFailedRuntimeAsync(ctx); return StepResult.Fail( snapshot.Detail ?? "The managed llama-server router did not become healthy."); } @@ -1024,7 +1071,7 @@ snapshot.ProcessId is null || .LoadAsync(ct); if (verifiedInstall?.Endpoint is null || verifiedInstall.Endpoint != snapshot.Endpoint) { - await DisposeRuntimeAsync(ctx); + await CleanUpFailedRuntimeAsync(ctx); return StepResult.Fail( "llama-server became healthy without committing its verified endpoint receipt."); } @@ -1035,18 +1082,23 @@ snapshot.ProcessId is null || } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - await DisposeRuntimeAsync(ctx); + await CleanUpFailedRuntimeAsync(ctx); throw; } catch (Exception ex) { - await DisposeRuntimeAsync(ctx); + await CleanUpFailedRuntimeAsync(ctx); return StepResult.Fail($"llama-server startup failed: {ex.Message}", ex); } } public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) { + if (ctx.LocalAiRuntimeBorrowed && !ctx.LocalAiBorrowedRuntimeRestartedThisRun) + return Task.CompletedTask; + if (ctx.LocalAiBorrowedRuntimeRestored) + return Task.CompletedTask; + // During a recovery provider transition, ConfigureLocalAiGatewayStep's rollback (which // runs before this step's rollback) sets LocalAiRecoveryReceiptRollbackAllowed only when // it confirmed the Gateway no longer routes to this runtime's endpoint. If that could not @@ -1057,8 +1109,12 @@ public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) ctx.Logger.Warn( "Keeping the replacement llama-server router running because the Gateway configuration " + "rollback could not confirm it no longer routes to this endpoint."); - return Task.CompletedTask; + return ctx.LocalAiRuntimeBorrowed && ctx.LocalAiRuntime is { } borrowedRuntime + ? borrowedRuntime.ReleaseSetupGatewayRouteAsync(ct) + : Task.CompletedTask; } + if (ctx.LocalAiRuntimeBorrowed) + return StopBorrowedRuntimeAsync(ctx, ct); return DisposeRuntimeAsync(ctx).AsTask(); } @@ -1082,6 +1138,23 @@ private static async ValueTask DisposeRuntimeAsync(SetupContext ctx) ctx.LocalAiRuntime = null; await runtime.DisposeAsync(); } + + private static async Task CleanUpFailedRuntimeAsync(SetupContext ctx) + { + if (ctx.LocalAiRuntimeBorrowed) + { + if (ctx.LocalAiRuntime is not null) + await ctx.LocalAiRuntime.StopForSetupAsync(CancellationToken.None); + return; + } + await DisposeRuntimeAsync(ctx); + } + + private static async Task StopBorrowedRuntimeAsync(SetupContext ctx, CancellationToken ct) + { + if (ctx.LocalAiRuntime is not null) + await ctx.LocalAiRuntime.StopForSetupAsync(ct); + } } /// @@ -1141,7 +1214,7 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - await ResetRouterAsync(runtime); + await ResetRouterAsync(runtime, ctx.LocalAiRuntimeBorrowed); throw; } catch (OperationCanceledException ex) @@ -1185,7 +1258,7 @@ private static async Task CaptureFailureDetailAsync( var paths = new LocalAiPaths(ctx.LocalDataDir); IReadOnlyList diagnostics = await LocalAiLogTail.ReadDiagnosticLinesAsync(paths, CancellationToken.None); - await ResetRouterAsync(runtime); + await ResetRouterAsync(runtime, ctx.LocalAiRuntimeBorrowed); // Echo into the setup log the UI already links, so the root cause remains available if // the router restart rotates the managed llama-server logs. foreach (string line in diagnostics) @@ -1193,11 +1266,15 @@ private static async Task CaptureFailureDetailAsync( return new LocalAiFailureDetail(diagnostics, paths.LogsDirectory); } - internal static async Task ResetRouterAsync(ILocalAiRuntime runtime) + internal static async Task ResetRouterAsync( + ILocalAiRuntime runtime, + bool setupScoped = false) { try { - return await runtime.RestartAsync(CancellationToken.None); + return setupScoped + ? await runtime.RestartForSetupAsync(CancellationToken.None) + : await runtime.RestartAsync(CancellationToken.None); } catch { diff --git a/src/OpenClaw.SetupEngine/SetupContext.cs b/src/OpenClaw.SetupEngine/SetupContext.cs index ed6465bdd..e40b3bb14 100644 --- a/src/OpenClaw.SetupEngine/SetupContext.cs +++ b/src/OpenClaw.SetupEngine/SetupContext.cs @@ -591,6 +591,9 @@ public Func>? !LocalAiRecoveryRollbackUncertain || LocalAiRecoveryReceiptRollbackAllowed; internal bool LocalAiManifestCreatedThisRun { get; set; } internal ILocalAiRuntime? LocalAiRuntime { get; set; } + internal bool LocalAiRuntimeBorrowed { get; set; } + internal bool LocalAiBorrowedRuntimeRestartedThisRun { get; set; } + internal bool LocalAiBorrowedRuntimeRestored { get; set; } internal HostHardwareInfo? LocalAiGpuBaseline { get; set; } internal LlamaServerInferenceVerification? LocalAiInferenceVerification { get; set; } internal LocalAiGpuLoadEvidence? LocalAiGpuLoadEvidence { get; set; } diff --git a/src/OpenClaw.SetupEngine/SetupPipeline.cs b/src/OpenClaw.SetupEngine/SetupPipeline.cs index 6fa9f926d..cb33b490f 100644 --- a/src/OpenClaw.SetupEngine/SetupPipeline.cs +++ b/src/OpenClaw.SetupEngine/SetupPipeline.cs @@ -169,6 +169,20 @@ public static List BuildDefaultSteps() public sealed class SetupPipeline { + internal static Task ReleaseBorrowedLocalAiRuntimeAfterFailureAsync( + SetupContext ctx, + PipelineResult? result) + { + if (result?.Outcome == PipelineOutcome.Success || + !ctx.LocalAiRuntimeBorrowed || + !ctx.LocalAiBorrowedRuntimeRestartedThisRun || + ctx.LocalAiRuntime is not { } borrowedRuntime) + { + return Task.CompletedTask; + } + return borrowedRuntime.ReleaseSetupGatewayRouteAsync(CancellationToken.None); + } + public static async Task RunWithSettlementAsync( Func> run, Func settle) { diff --git a/src/OpenClaw.Tray.WinUI/Services/SetupLocalAiHost.cs b/src/OpenClaw.Tray.WinUI/Services/SetupLocalAiHost.cs index 4fa0ba772..0eac792f3 100644 --- a/src/OpenClaw.Tray.WinUI/Services/SetupLocalAiHost.cs +++ b/src/OpenClaw.Tray.WinUI/Services/SetupLocalAiHost.cs @@ -67,6 +67,8 @@ public void ReleaseNative(IGatewayAiSetupTransport transport) if (ReferenceEquals(_nativeTransport, transport)) _nativeTransport = null; } + public ILocalAiRuntime? BorrowManagedRuntime() => getRuntime(); + public GatewayRegistrySnapshot BeginGatewaySetup() => _setupRegistryBaseline = (getRegistry() ?? throw new InvalidOperationException("The Gateway registry is unavailable.")).CapturePersistedSnapshot(); diff --git a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs index 244b694ec..0af553c1a 100644 --- a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs +++ b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs @@ -2353,6 +2353,131 @@ public async Task RestartAsync_UsesEndpointCycleUntilReplacementIsPublished() Assert.DoesNotContain("quiesce:Teardown", events); } + [Fact] + public async Task RestartForSetupAsync_RestartsProcessWithoutChangingGatewayLifecycle() + { + using var temp = new TempDirectory("local-ai-port-"); + LocalAiPaths paths = await PrepareInstallAsync(temp); + var events = new SynchronizedEventLog(); + var platform = new FakePlatform(); + var host = new FakeProcessHost(platform, events, selectedPort: 28_769); + var lifecycle = new FakeLifecycle(events); + await using var runtime = CreateRuntime( + paths, + host, + platform, + new FakeClient(events), + lifecycle); + LocalAiRuntimeSnapshot started = await runtime.EnsureStartedAsync(); + Assert.Equal(LocalAiRuntimeState.Healthy, started.State); + events.Clear(); + + LocalAiRuntimeSnapshot restarted = await runtime.RestartForSetupAsync(); + + Assert.Equal(LocalAiRuntimeState.Healthy, restarted.State); + Assert.Equal(["stop", "start", "probe:28769"], events); + Assert.True(restarted.GatewayRouteRequiresResolution); + + events.Clear(); + LocalAiRuntimeSnapshot stillSetupOwned = await runtime.RestartAsync(); + + Assert.True(stillSetupOwned.GatewayRouteRequiresResolution); + Assert.Equal(["stop", "start", "probe:28769"], events); + await runtime.AcknowledgeSetupGatewayRouteAsync(); + Assert.Equal([true, false, false, true], lifecycle.RecoveryIntents); + } + + [Fact] + public async Task AcknowledgeSetupGatewayRouteAsync_ClearsResolutionWithoutLifecycleIo() + { + using var temp = new TempDirectory("local-ai-port-"); + LocalAiPaths paths = await PrepareInstallAsync(temp); + var events = new SynchronizedEventLog(); + var platform = new FakePlatform(); + var host = new FakeProcessHost(platform, events, selectedPort: 28_769); + var lifecycle = new FakeLifecycle(events); + await using var runtime = CreateRuntime( + paths, + host, + platform, + new FakeClient(events), + lifecycle); + LocalAiRuntimeSnapshot started = await runtime.RestartForSetupAsync(); + Assert.Equal(LocalAiRuntimeState.Healthy, started.State); + Assert.True(started.GatewayRouteRequiresResolution); + events.Clear(); + + LocalAiRuntimeSnapshot acknowledged = await runtime.AcknowledgeSetupGatewayRouteAsync(); + + Assert.False(acknowledged.GatewayRouteRequiresResolution); + Assert.Empty(events); + Assert.Equal([false, true], lifecycle.RecoveryIntents); + } + + [Fact] + public async Task ReleaseSetupGatewayRouteAsync_ReturnsLifecycleOwnershipWithoutAcknowledgingRoute() + { + using var temp = new TempDirectory("local-ai-port-"); + LocalAiPaths paths = await PrepareInstallAsync(temp); + var events = new SynchronizedEventLog(); + var platform = new FakePlatform(); + var host = new FakeProcessHost(platform, events, selectedPort: 28_769); + var lifecycle = new FakeLifecycle(events); + await using var runtime = CreateRuntime( + paths, + host, + platform, + new FakeClient(events), + lifecycle); + LocalAiRuntimeSnapshot started = await runtime.RestartForSetupAsync(); + Assert.True(started.GatewayRouteRequiresResolution); + + LocalAiRuntimeSnapshot released = await runtime.ReleaseSetupGatewayRouteAsync(); + events.Clear(); + LocalAiRuntimeSnapshot restarted = await runtime.RestartAsync(); + + Assert.True(released.GatewayRouteRequiresResolution); + Assert.False(restarted.GatewayRouteRequiresResolution); + Assert.Equal([false, true, true], lifecycle.RecoveryIntents); + Assert.Contains("quiesce:EndpointCycle", events); + Assert.Contains("publish:28769", events); + } + + [Fact] + public async Task RestartForSetupRollbackAsync_AdoptsRestoredReceiptBeforeGatewayLifecycle() + { + using var temp = new TempDirectory("local-ai-port-"); + LocalAiPaths paths = await PrepareInstallAsync(temp); + var events = new SynchronizedEventLog(); + var platform = new FakePlatform(); + var lifecycle = new FakeLifecycle(events); + var host = new FakeProcessHost(platform, events, selectedPort: 28_769); + await using var runtime = CreateRuntime( + paths, + host, + platform, + new FakeClient(events), + lifecycle); + LocalAiRuntimeSnapshot started = await runtime.EnsureStartedAsync(); + Assert.Equal(LocalAiRuntimeState.Healthy, started.State); + LocalAiResolvedInstall current = (await new LocalAiManifestStore(paths).LoadAsync())!; + LocalAiInstallManifest restoredManifest = current.Manifest with + { + Endpoint = "http://127.0.0.1:28768/v1", + }; + await new LocalAiManifestStore(paths).SaveAsync(restoredManifest); + events.Clear(); + lifecycle.QuiescedEndpoints.Clear(); + + LocalAiRuntimeSnapshot restarted = await runtime.RestartForSetupRollbackAsync(); + + Assert.Equal(LocalAiRuntimeState.Healthy, restarted.State); + Assert.Equal(new Uri("http://127.0.0.1:28768/v1"), lifecycle.QuiescedEndpoints[0]); + Assert.Equal(new Uri("http://127.0.0.1:28769/v1"), restarted.Endpoint); + Assert.False(restarted.GatewayRouteRequiresResolution); + Assert.Contains("publish:28769", events); + } + [Fact] public async Task RestartAsync_InitialEndpointCycleExceptionCompletesTeardownBeforeStopping() { diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index a7a3c7aaf..412496317 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -37,6 +37,27 @@ public async Task Repair_RollbackRestoresFallbackAfterRetainedEndpointCycle() Assert.Equal(JsonSerializer.Serialize("openai/gpt-5"), commands.PrimaryJson); } + [Fact] + public async Task Configure_AcknowledgementFailureCompensatesCommittedGatewayRoute() + { + using var temp = new TempDirectory("local-ai-gateway-ack-"); + LocalAiResolvedInstall install = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string fallback = JsonSerializer.Serialize("openai/gpt-5"); + var commands = new GatewayStateCommandRunner(providerJson: null, fallback); + SetupContext context = CreateContext(temp.Path, commands); + context.LocalAiResolvedInstall = install; + context.LocalAiEligibility = LocalInferenceEligibility.Evaluate(CreateSparkHardware()); + context.LocalAiRuntimeBorrowed = true; + context.LocalAiRuntime = new AcknowledgementFailingRuntime(); + + StepResult result = await new ConfigureLocalAiGatewayStep() + .ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Failed, result.Outcome); + Assert.Null(commands.ProviderJson); + Assert.Equal(fallback, commands.PrimaryJson); + } + [Fact] public async Task Repair_RollbackUnsetsPrimaryAfterRetainedEndpointCycleWithoutFallback() { @@ -597,6 +618,180 @@ public async Task Recovery_RetryPreservesOriginalProviderRollbackBaseline() Assert.False(context.LocalAiRecoveryProviderTransition); } + [Fact] + public async Task RestoreRecoveryRouteAsync_PreservesConcurrentGatewayChanges() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + var prior = new LocalAiGatewayPriorState( + ProviderExisted: true, + ProviderJson: originalProvider, + PrimaryModelExisted: true, + PrimaryModelJson: originalPrimary); + var commands = new GatewayStateCommandRunner( + LocalAiGatewayProviderDefinition.BuildProviderJson(original), + JsonSerializer.Serialize("openai/concurrent-model")); + SetupContext context = CreateRecoveryContext(temp.Path, commands); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:28766/v1" }, + Endpoint = new Uri("http://127.0.0.1:28766/v1"), + }; + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + moved, + CancellationToken.None); + + Assert.False(restored); + Assert.Equal(JsonSerializer.Serialize("openai/concurrent-model"), commands.PrimaryJson); + Assert.DoesNotContain( + commands.WslCalls, + command => command.Contains("LOCAL_AI_GATEWAY_RESTORED", StringComparison.Ordinal)); + } + + [Fact] + public async Task RestoreRecoveryRouteAsync_PreservesCustomizedProviderWhenEndpointIsUnchanged() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string customizedProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original) + .Replace("\"timeoutSeconds\":300", "\"timeoutSeconds\":301", StringComparison.Ordinal); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + var prior = new LocalAiGatewayPriorState(true, customizedProvider, true, originalPrimary); + var commands = new GatewayStateCommandRunner(customizedProvider, originalPrimary) + { + SupportsConditionalProviderSet = false, + }; + SetupContext context = CreateRecoveryContext(temp.Path, commands); + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + original, + CancellationToken.None); + + Assert.True(restored); + Assert.Equal(customizedProvider, commands.ProviderJson); + Assert.DoesNotContain( + commands.WslCalls, + command => command.Contains("LOCAL_AI_GATEWAY_RESTORED", StringComparison.Ordinal)); + } + + [Fact] + public async Task RestoreRecoveryRouteAsync_ConditionallyMovesOwnedProviderEndpoint() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + var prior = new LocalAiGatewayPriorState( + ProviderExisted: true, + ProviderJson: originalProvider, + PrimaryModelExisted: true, + PrimaryModelJson: originalPrimary); + var commands = new GatewayStateCommandRunner(originalProvider, originalPrimary); + SetupContext context = CreateRecoveryContext(temp.Path, commands); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:28766/v1" }, + Endpoint = new Uri("http://127.0.0.1:28766/v1"), + }; + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + moved, + CancellationToken.None); + + Assert.True(restored); + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson(commands.ProviderJson!, moved)); + Assert.Equal(originalPrimary, commands.PrimaryJson); + Assert.Contains( + commands.WslCalls, + command => command.Contains("--expect-current-json", StringComparison.Ordinal)); + } + + [Fact] + public async Task RestoreRecoveryRouteAsync_PreservesPrimaryChangeRacingProviderCas() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + string concurrentPrimary = JsonSerializer.Serialize("openai/concurrent-model"); + var prior = new LocalAiGatewayPriorState(true, originalProvider, true, originalPrimary); + var commands = new GatewayStateCommandRunner(originalProvider, originalPrimary) + { + PrimaryJsonAfterConditionalProviderSet = concurrentPrimary, + }; + SetupContext context = CreateRecoveryContext(temp.Path, commands); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:28766/v1" }, + Endpoint = new Uri("http://127.0.0.1:28766/v1"), + }; + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + moved, + CancellationToken.None); + + Assert.True(restored); + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson(commands.ProviderJson!, moved)); + Assert.Equal(concurrentPrimary, commands.PrimaryJson); + } + + [Fact] + public async Task RestoreRecoveryRouteAsync_RejectsLegacyGatewayCliWithoutConditionalWrites() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + var prior = new LocalAiGatewayPriorState(true, originalProvider, true, originalPrimary); + var commands = new GatewayStateCommandRunner(originalProvider, originalPrimary) + { + SupportsConditionalProviderSet = false, + }; + SetupContext context = CreateRecoveryContext(temp.Path, commands); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:28766/v1" }, + Endpoint = new Uri("http://127.0.0.1:28766/v1"), + }; + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + moved, + CancellationToken.None); + + Assert.False(restored); + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson(commands.ProviderJson!, original)); + Assert.Equal(originalPrimary, commands.PrimaryJson); + Assert.Contains( + commands.WslCalls, + command => command.Contains("LOCAL_AI_CONDITIONAL_SET_UNSUPPORTED", StringComparison.Ordinal)); + Assert.DoesNotContain( + commands.WslCalls, + command => command.Contains("OPENCLAW_LOCAL_AI_BATCH_B64", StringComparison.Ordinal)); + } + [Fact] public async Task Recovery_FailedProviderCompensationKeepsReplacementReceipt() { @@ -885,6 +1080,8 @@ private sealed class GatewayStateCommandRunner( public bool FailRestoreBatchOnce { get; set; } public bool LoseRestoreAcknowledgementOnce { get; set; } public bool ThrowOnNextCapture { get; set; } + public bool SupportsConditionalProviderSet { get; set; } = true; + public string? PrimaryJsonAfterConditionalProviderSet { get; set; } public List WslCalls { get; } = []; public Task RunAsync( @@ -908,6 +1105,42 @@ public Task RunInWslAsync( { ct.ThrowIfCancellationRequested(); WslCalls.Add(command); + if (environment is not null && environment.Count == 2 && + command.Contains("--expect-current-json", StringComparison.Ordinal)) + { + if (!SupportsConditionalProviderSet) + { + return Task.FromResult(new CommandResult( + 42, + "LOCAL_AI_CONDITIONAL_SET_UNSUPPORTED", + "", + TimeSpan.Zero, + TimedOut: false)); + } + string providerJson = Encoding.UTF8.GetString(Convert.FromBase64String( + environment["OPENCLAW_LOCAL_AI_PROVIDER_B64"])); + string expectedProviderJson = Encoding.UTF8.GetString(Convert.FromBase64String( + environment["OPENCLAW_LOCAL_AI_EXPECTED_PROVIDER_B64"])); + using JsonDocument currentProvider = JsonDocument.Parse(ProviderJson!); + using JsonDocument expectedProvider = JsonDocument.Parse(expectedProviderJson); + if (!JsonElement.DeepEquals(currentProvider.RootElement, expectedProvider.RootElement)) + { + return Task.FromResult(new CommandResult( + 1, + "", + "gateway provider changed", + TimeSpan.Zero, + TimedOut: false)); + } + ProviderJson = providerJson; + PrimaryJson = PrimaryJsonAfterConditionalProviderSet ?? PrimaryJson; + return Task.FromResult(new CommandResult( + 0, + "LOCAL_AI_GATEWAY_RESTORED", + "", + TimeSpan.Zero, + TimedOut: false)); + } if (environment is not null && environment.Count == 1) { if (FailRestoreBatchOnce && @@ -1019,6 +1252,32 @@ private static string EncodeOrMissing(string? value) => value is null : Convert.ToBase64String(Encoding.UTF8.GetBytes(value)); } + private sealed class AcknowledgementFailingRuntime : ILocalAiRuntime + { + public LocalAiRuntimeSnapshot Snapshot => LocalAiRuntimeSnapshot.Initial( + new Uri("http://127.0.0.1:18800/v1"), + DateTimeOffset.UtcNow); + public event EventHandler? StateChanged + { + add { } + remove { } + } + public Task EnsureStartedAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task ResumeAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task StopAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task RestartAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task RefreshAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) => + throw new IOException("acknowledgement failed"); + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + private sealed class DelegatingRollbackStep( string id, Func rollback, diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index 3fcf7ed56..b76892bc7 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -427,6 +427,286 @@ public async Task StartLocalAiRuntimeStep_DisposesRuntimeOutsideRecoveryTransiti Assert.Null(context.LocalAiRuntime); } + [Fact] + public async Task StartLocalAiRuntimeStep_RestartsBorrowedTrayRuntimeForReplacement() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + context.LocalAiUpgradeOriginalInstall = install; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + + StepResult result = await new StartLocalAiRuntimeStep().ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(0, runtime.RestartCalls); + Assert.Equal(0, runtime.EnsureStartedCalls); + Assert.Equal(0, runtime.DisposeCalls); + Assert.Same(runtime, context.LocalAiRuntime); + } + + [Fact] + public async Task StartLocalAiRuntimeStep_RejectsBorrowedRuntimeWithoutReplacementOrUpgrade() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-no-replacement-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + var step = new StartLocalAiRuntimeStep(); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + await step.RollbackAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.FailedTerminal, result.Outcome); + Assert.Equal(0, runtime.RestartForSetupCalls); + Assert.Equal(0, runtime.StopForSetupCalls); + Assert.False(context.LocalAiBorrowedRuntimeRestartedThisRun); + } + + [Fact] + public async Task ResetRouterAsync_UsesSetupScopedRestartForBorrowedRuntime() + { + var runtime = new DisposeTrackingRuntime(); + + await VerifyLocalAiInferenceStep.ResetRouterAsync(runtime, setupScoped: true); + + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(0, runtime.RestartCalls); + } + + [Fact] + public async Task ReleaseBorrowedLocalAiRuntimeAfterFailureAsync_ReleasesNoRollbackOwnership() + { + var context = CreateContext(LocalAiRecoveryConfig()); + var runtime = new DisposeTrackingRuntime(); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + context.LocalAiBorrowedRuntimeRestartedThisRun = true; + + await SetupPipeline.ReleaseBorrowedLocalAiRuntimeAfterFailureAsync( + context, + new PipelineResult(PipelineOutcome.Failed)); + + Assert.Equal(1, runtime.ReleaseSetupGatewayRouteCalls); + + context.LocalAiBorrowedRuntimeRestartedThisRun = false; + await SetupPipeline.ReleaseBorrowedLocalAiRuntimeAfterFailureAsync( + context, + new PipelineResult(PipelineOutcome.Failed)); + + Assert.Equal(1, runtime.ReleaseSetupGatewayRouteCalls); + + await SetupPipeline.ReleaseBorrowedLocalAiRuntimeAfterFailureAsync( + context, + new PipelineResult(PipelineOutcome.Success)); + + Assert.Equal(1, runtime.ReleaseSetupGatewayRouteCalls); + } + + [Fact] + public async Task BorrowedTrayRuntime_RollbackRestoresRuntimeBeforeReconcilingAutomaticPort() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-rollback-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18801); + original = original with + { + Manifest = original.Manifest with { RequestedPort = 0 }, + }; + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:18802/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Manifest.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)); + await store.SaveAsync(pendingManifest); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pendingManifest); + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryProviderTransition = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(context.LocalAiResolvedInstall)) + { + RestartForSetupHandler = async ct => + { + LocalAiResolvedInstall restored = await store.LoadAsync(ct) + ?? throw new InvalidDataException("restored receipt missing"); + LocalAiInstallManifest movedManifest = restored.Manifest with + { + Endpoint = "http://127.0.0.1:18803/v1", + }; + await store.SaveAsync(movedManifest, ct); + return HealthySnapshot(store.ResolveAndValidate(movedManifest)); + }, + }; + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + context.LocalAiBorrowedRuntimeRestartedThisRun = true; + var rollbackOrder = new List(); + Uri? restoredRoute = null; + Uri? probedEndpoint = null; + var persist = new PersistLocalAiManifestStep(); + var start = new StartLocalAiRuntimeStep(_ => runtime); + var preserve = new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("gateway restarted")), + (install, _) => + { + rollbackOrder.Add("probe"); + Assert.Equal(0, runtime.AcknowledgeSetupGatewayRouteCalls); + probedEndpoint = install.Endpoint; + return Task.FromResult(true); + }, + (_, _, _, install, _) => + { + rollbackOrder.Add("route"); + restoredRoute = install.Endpoint; + return Task.FromResult(true); + }); + var pipeline = new SetupPipeline([ + new MockStep( + "persist-local-ai-manifest", + (_, _) => Task.FromResult(StepResult.Ok("persisted")), + persist.RollbackAsync), + new MockStep( + "start-local-ai-runtime", + (_, _) => Task.FromResult(StepResult.Ok("started")), + start.RollbackAsync), + preserve, + new MockStep( + "configure-local-ai-gateway", + (ctx, _) => + { + ctx.LocalAiGatewayPriorState = new LocalAiGatewayPriorState( + ProviderExisted: true, + ProviderJson: "{}", + PrimaryModelExisted: true, + PrimaryModelJson: "\"prior-model\""); + ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = true; + ctx.LocalAiRecoveryRollbackUncertain = true; + return Task.FromResult(StepResult.Ok("configured")); + }, + (ctx, _) => + { + rollbackOrder.Add("gateway"); + ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; + return Task.CompletedTask; + }), + new MockStep( + "failure", + (_, _) => Task.FromResult(StepResult.Fail("failed after gateway configuration"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal("failure", result.FailedStepId); + LocalAiResolvedInstall restored = (await store.LoadAsync())!; + Assert.Equal(new Uri("http://127.0.0.1:18803/v1"), probedEndpoint); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(1, runtime.AcknowledgeSetupGatewayRouteCalls); + Assert.Equal(["gateway", "route", "probe"], rollbackOrder); + Assert.Equal(original.Manifest.ModelCatalogId, restored.Manifest.ModelCatalogId); + Assert.Equal(new Uri("http://127.0.0.1:18803/v1"), restored.Endpoint); + Assert.Equal(restored.Endpoint, restoredRoute); + Assert.Equal(restored.Endpoint, probedEndpoint); + Assert.Equal(0, runtime.RestartCalls); + Assert.Equal(0, runtime.StopForSetupCalls); + Assert.Equal(0, runtime.StopCalls); + Assert.Equal(0, runtime.DisposeCalls); + Assert.Same(runtime, context.LocalAiRuntime); + Assert.True(context.LocalAiBorrowedRuntimeRestored); + Assert.False(context.LocalAiRecoveryRollbackUncertain); + } + + [Fact] + public async Task BorrowedTrayRuntime_PreGatewayRollbackPublishesRestoredAutomaticPort() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-pre-gateway-rollback-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18801); + original = original with + { + Manifest = original.Manifest with { RequestedPort = 0 }, + }; + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:18802/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Manifest.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)); + await store.SaveAsync(pendingManifest); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pendingManifest); + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryProviderTransition = true; + Uri? publishedEndpoint = null; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(context.LocalAiResolvedInstall)) + { + RestartHandler = async ct => + { + LocalAiResolvedInstall restored = await store.LoadAsync(ct) + ?? throw new InvalidDataException("restored receipt missing"); + LocalAiInstallManifest movedManifest = restored.Manifest with + { + Endpoint = "http://127.0.0.1:18803/v1", + }; + await store.SaveAsync(movedManifest, ct); + LocalAiResolvedInstall moved = store.ResolveAndValidate(movedManifest); + publishedEndpoint = moved.Endpoint; + return HealthySnapshot(moved); + }, + }; + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + context.LocalAiBorrowedRuntimeRestartedThisRun = true; + var persist = new PersistLocalAiManifestStep(); + var start = new StartLocalAiRuntimeStep(_ => runtime); + var pipeline = new SetupPipeline([ + new MockStep( + "persist-local-ai-manifest", + (_, _) => Task.FromResult(StepResult.Ok("persisted")), + persist.RollbackAsync), + new MockStep( + "start-local-ai-runtime", + (_, _) => Task.FromResult(StepResult.Ok("started")), + start.RollbackAsync), + new MockStep( + "failure-before-gateway", + (_, _) => Task.FromResult(StepResult.Fail("failed before gateway configuration"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + LocalAiResolvedInstall restored = (await store.LoadAsync())!; + Assert.Equal(original.Manifest.ModelCatalogId, restored.Manifest.ModelCatalogId); + Assert.Equal(new Uri("http://127.0.0.1:18803/v1"), restored.Endpoint); + Assert.Equal(restored.Endpoint, publishedEndpoint); + Assert.Equal(1, runtime.StopForSetupCalls); + Assert.Equal(1, runtime.RestartForSetupRollbackCalls); + Assert.Equal(0, runtime.RestartCalls); + Assert.Equal(0, runtime.RestartForSetupCalls); + Assert.Equal(0, runtime.DisposeCalls); + Assert.False(context.LocalAiRecoveryRollbackUncertain); + } + /// /// Regression guard: a stale manifest receipt is not enough to prove the original (A) /// endpoint is still alive. Rollback must probe it before pointing the Gateway back at it. @@ -561,22 +841,99 @@ public async Task RecoveryRollback_PreservesReplacementWhenCompensatedOriginalEn Assert.NotNull(retained.ReplacedManifest); } - private sealed class DisposeTrackingRuntime : ILocalAiRuntime + private sealed class DisposeTrackingRuntime(LocalAiRuntimeSnapshot? snapshot = null) : ILocalAiRuntime { public int DisposeCalls { get; private set; } - - public LocalAiRuntimeSnapshot Snapshot => throw new NotSupportedException(); + public int EnsureStartedCalls { get; private set; } + public int StopCalls { get; private set; } + public int RestartCalls { get; private set; } + public int StopForSetupCalls { get; private set; } + public int RestartForSetupCalls { get; private set; } + public int RestartForSetupRollbackCalls { get; private set; } + public int AcknowledgeSetupGatewayRouteCalls { get; private set; } + public int ReleaseSetupGatewayRouteCalls { get; private set; } + public Func>? RestartHandler { get; init; } + public Func>? RestartForSetupHandler { get; init; } + + public LocalAiRuntimeSnapshot Snapshot { get; private set; } = snapshot ?? + LocalAiRuntimeSnapshot.Initial(new Uri("http://127.0.0.1:18800/v1"), DateTimeOffset.UtcNow); public Task ResumeAsync(CancellationToken cancellationToken = default) => - throw new NotSupportedException(); - public Task EnsureStartedAsync(CancellationToken cancellationToken = default) => - throw new NotSupportedException(); + Task.FromResult(Snapshot); - public Task StopAsync(CancellationToken cancellationToken = default) => - throw new NotSupportedException(); + public Task EnsureStartedAsync(CancellationToken cancellationToken = default) + { + EnsureStartedCalls++; + return Task.FromResult(Snapshot); + } - public Task RestartAsync(CancellationToken cancellationToken = default) => - throw new NotSupportedException(); + public Task StopAsync(CancellationToken cancellationToken = default) + { + StopCalls++; + Snapshot = Snapshot with + { + State = LocalAiRuntimeState.Stopped, + Ownership = LocalAiOwnership.None, + ProcessId = null, + ProcessStartedAtUtc = null, + }; + return Task.FromResult(Snapshot); + } + + public async Task RestartAsync(CancellationToken cancellationToken = default) + { + RestartCalls++; + Snapshot = RestartHandler is null + ? snapshot ?? Snapshot + : await RestartHandler(cancellationToken); + return Snapshot; + } + + public Task StopForSetupAsync(CancellationToken cancellationToken = default) + { + StopForSetupCalls++; + Snapshot = Snapshot with + { + State = LocalAiRuntimeState.Stopped, + Ownership = LocalAiOwnership.None, + ProcessId = null, + ProcessStartedAtUtc = null, + }; + return Task.FromResult(Snapshot); + } + + public async Task RestartForSetupAsync(CancellationToken cancellationToken = default) + { + RestartForSetupCalls++; + Snapshot = RestartForSetupHandler is null + ? snapshot ?? Snapshot + : await RestartForSetupHandler(cancellationToken); + return Snapshot; + } + + public async Task RestartForSetupRollbackAsync( + CancellationToken cancellationToken = default) + { + RestartForSetupRollbackCalls++; + Snapshot = RestartHandler is null + ? snapshot ?? Snapshot + : await RestartHandler(cancellationToken); + return Snapshot; + } + + public Task ReleaseSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) + { + ReleaseSetupGatewayRouteCalls++; + return Task.FromResult(Snapshot); + } + + public Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) + { + AcknowledgeSetupGatewayRouteCalls++; + return Task.FromResult(Snapshot); + } public Task RefreshAsync(CancellationToken cancellationToken = default) => throw new NotSupportedException(); @@ -594,6 +951,25 @@ public ValueTask DisposeAsync() } } + private static LocalAiRuntimeSnapshot HealthySnapshot(LocalAiResolvedInstall install) => new( + LocalAiRuntimeState.Healthy, + LocalAiOwnership.CompanionManaged, + install.Endpoint!, + install.Manifest.EngineVersion, + install.Manifest.ModelCatalogId, + new LocalAiModelEvidence( + LocalAiModelAvailabilityState.Verified, + DateTimeOffset.UtcNow, + install.Manifest.ModelAsset.Sha256, + install.Manifest.ModelAsset.SizeBytes), + 42, + DateTimeOffset.UtcNow, + null, + DateTimeOffset.UtcNow) + { + GatewayRouteRequiresResolution = false, + }; + private static LocalAiResolvedInstall CreateLocalAiResolvedInstall(string localDataDirectory, int port) { var paths = new LocalAiPaths(localDataDirectory); From d03e4ec69343eafe29319dd9755b440814e09da8 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 1 Oct 2026 04:20:36 -0700 Subject: [PATCH 18/22] test(setup): assert runtime release settlement order --- .../NativeLocalAiAcquisitionTests.cs | 3 ++- .../OnboardingPresentationContractTests.cs | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/OpenClaw.SetupEngine.Tests/NativeLocalAiAcquisitionTests.cs b/tests/OpenClaw.SetupEngine.Tests/NativeLocalAiAcquisitionTests.cs index 21b1d1f73..48b7e44e2 100644 --- a/tests/OpenClaw.SetupEngine.Tests/NativeLocalAiAcquisitionTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/NativeLocalAiAcquisitionTests.cs @@ -37,6 +37,7 @@ public void WslRecoveryRetainsItsExistingNetworkingAndGatewayVerification() Assert.Contains(steps, step => step is ConfigureLocalAiWslNetworkingStep); Assert.Contains(steps, step => step is VerifyLocalAiWslStep); Assert.Contains(steps, step => step is ConfigureLocalAiGatewayStep); - Assert.IsType(steps[^1]); + Assert.IsType(steps[^2]); + Assert.IsType(steps[^1]); } } diff --git a/tests/OpenClaw.Tray.Tests/OnboardingPresentationContractTests.cs b/tests/OpenClaw.Tray.Tests/OnboardingPresentationContractTests.cs index 9a488d03c..50b945e37 100644 --- a/tests/OpenClaw.Tray.Tests/OnboardingPresentationContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/OnboardingPresentationContractTests.cs @@ -19,9 +19,13 @@ public void PipelineRegistryWriters_ProduceTheExpectedAdoptionSnapshot() } var progress = Read(@"src\OpenClaw.SetupEngine.UI\Pages\ProgressPage.xaml.cs"); Assert.Contains("ctx.ExpectedGatewayRegistry = config.NativeLocalAiAcquisition ? null : setupOwner?.BeginGatewaySetup()", progress); - Assert.Contains("outcome => config.NativeLocalAiAcquisition ? Task.CompletedTask :", progress); + Assert.Contains("ReleaseBorrowedLocalAiRuntimeAfterFailureAsync(ctx, outcome)", progress); + Assert.Contains("if (!config.NativeLocalAiAcquisition && setupOwner is not null)", progress); Assert.Contains("SetupPipeline.RunWithSettlementAsync", progress); - Assert.True(progress.IndexOf("SettleGatewaySetupAsync(ctx.ExpectedGatewayRegistry", StringComparison.Ordinal) < + Assert.Contains("await setupOwner.SettleGatewaySetupAsync(", progress); + Assert.True(progress.IndexOf("ReleaseBorrowedLocalAiRuntimeAfterFailureAsync(ctx, outcome)", StringComparison.Ordinal) < + progress.IndexOf("await setupOwner.SettleGatewaySetupAsync(", StringComparison.Ordinal)); + Assert.True(progress.IndexOf("await setupOwner.SettleGatewaySetupAsync(", StringComparison.Ordinal) < progress.IndexOf("if (_closed || _window?.IsClosed == true)", StringComparison.Ordinal)); } From 616f1b14e76983da648e514dbb73d1794dd23bc1 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 1 Oct 2026 07:38:35 -0700 Subject: [PATCH 19/22] fix(setup): harden borrowed runtime recovery --- .../LocalAi/LocalAiManifest.cs | 23 + .../LocalAiGpuVerification.cs | 6 +- .../LocalAiRecoveryPolicy.cs | 109 ++++- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 44 +- src/OpenClaw.SetupEngine/SetupPipeline.cs | 7 +- .../SetupPipelineTests.cs | 410 +++++++++++++++++- 6 files changed, 572 insertions(+), 27 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index f7790cad3..6074fcafe 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -613,6 +613,29 @@ internal async Task RestoreRecoveryManifestAsync( return resolved; } + internal async Task RestoreManifestIfUnchangedAsync( + LocalAiInstallManifest expectedManifest, + LocalAiInstallManifest recoveryManifest, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(expectedManifest); + ArgumentNullException.ThrowIfNull(recoveryManifest); + await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) + .ConfigureAwait(false); + LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); + if (!JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(current), + JsonSerializer.SerializeToElement(expectedManifest))) + { + throw new InvalidDataException( + "The Local AI installation changed before its recovery receipt could be restored."); + } + + LocalAiResolvedInstall resolved = ResolveAndValidate(recoveryManifest); + await SaveWithoutLockAsync(recoveryManifest, cancellationToken).ConfigureAwait(false); + return resolved; + } + private static bool HasSameRuntimeAndModel( LocalAiInstallManifest current, LocalAiInstallManifest expected) => diff --git a/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs b/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs index 5eb783148..d4dc4bf5e 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs @@ -310,7 +310,7 @@ ctx.LocalAiInferenceVerification is null || } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - await VerifyLocalAiInferenceStep.ResetRouterAsync(runtime, ctx.LocalAiRuntimeBorrowed); + await VerifyLocalAiInferenceStep.ResetRouterAsync(ctx, runtime); throw; } catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) @@ -318,9 +318,7 @@ ctx.LocalAiInferenceVerification is null || failure = ex; } - LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync( - runtime, - ctx.LocalAiRuntimeBorrowed); + LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync(ctx, runtime); if (failure is not null) return StepResult.Fail($"Local AI GPU verification failed: {failure.Message}", failure); if (reset.State != LocalAiRuntimeState.Healthy || diff --git a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs index 8af18902e..5606293d0 100644 --- a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs +++ b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs @@ -163,6 +163,81 @@ private static IReadOnlyList LoadGatewayRecords(string dataDir) } } +public sealed class ValidateLocalAiRecoveryGatewayCompatibilityStep : SetupStep +{ + internal const string SupportedMarker = "LOCAL_AI_CONDITIONAL_SET_SUPPORTED"; + internal const string UnsupportedMarker = "LOCAL_AI_CONDITIONAL_SET_UNSUPPORTED"; + + private readonly Func> _probe; + + public ValidateLocalAiRecoveryGatewayCompatibilityStep() + : this(ProbeConditionalSetSupportAsync) + { + } + + internal ValidateLocalAiRecoveryGatewayCompatibilityStep( + Func> probe) => + _probe = probe ?? throw new ArgumentNullException(nameof(probe)); + + public override string Id => "validate-local-ai-recovery-gateway-compatibility"; + public override string DisplayName => "Check gateway recovery compatibility"; + public override bool CanRetry => false; + + public override async Task ExecuteAsync(SetupContext ctx, CancellationToken ct) + { + LocalAiResolvedInstall?[] recoveryInstalls = + [ + ctx.LocalAiRecoveryPendingInstall, + ctx.LocalAiRecoveryOriginalInstall, + ctx.LocalAiResolvedInstall, + ]; + if (recoveryInstalls.All(install => install is null)) + { + return StepResult.Skip( + "No prior Local AI route requires conditional recovery support."); + } + if (recoveryInstalls.All(install => install is null || install.Manifest.RequestedPort != 0)) + return StepResult.Skip("Fixed-port Local AI recovery does not require conditional route updates."); + + CommandResult result = await _probe(ctx, ct).ConfigureAwait(false); + if (result.ExitCode == 0 && + result.Stdout.Contains(SupportedMarker, StringComparison.Ordinal)) + { + return StepResult.Ok("Gateway supports safe automatic-port recovery."); + } + if (result.ExitCode == 42 && + result.Stdout.Contains(UnsupportedMarker, StringComparison.Ordinal)) + { + return StepResult.Terminal( + "This Gateway version cannot safely recover Local AI with an automatic port. Update the Gateway, then retry recovery."); + } + return StepResult.Fail("OpenClaw could not verify Gateway support for safe automatic-port recovery."); + } + + private static Task ProbeConditionalSetSupportAsync( + SetupContext ctx, + CancellationToken ct) + { + string script = $$""" + set -eu + {{ctx.WslPathPrefix}} + if openclaw config set --help | grep -Fq -- '--expect-current-json'; then + echo {{SupportedMarker}} + exit 0 + fi + echo {{UnsupportedMarker}} + exit 42 + """; + return ctx.Commands.RunInWslAsync( + ctx.DistroName!, + script, + TimeSpan.FromMinutes(1), + ct: ct, + user: ctx.Config.Wsl.User, + inputViaStdin: true); + } +} + public sealed class PreserveLocalAiRecoveryGatewayStep : SetupStep { private readonly Func> _restart; @@ -223,15 +298,28 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) try { var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); - ctx.LocalAiResolvedInstall = await store - .RestoreRecoveryManifestAsync( - ctx.LocalAiResolvedInstall!.Manifest, - originalInstall.Manifest, - ct) - .ConfigureAwait(false); - LocalAiRuntimeSnapshot restored = await borrowedRuntime - .RestartForSetupAsync(ct) - .ConfigureAwait(false); + if (ctx.LocalAiResolvedInstall!.Manifest.ReplacedManifest is not null) + { + ctx.LocalAiResolvedInstall = await store + .RestoreRecoveryManifestAsync( + ctx.LocalAiResolvedInstall.Manifest, + originalInstall.Manifest, + ct) + .ConfigureAwait(false); + } + else + { + ctx.LocalAiResolvedInstall = await store + .RestoreManifestIfUnchangedAsync( + ctx.LocalAiResolvedInstall.Manifest, + originalInstall.Manifest, + ct) + .ConfigureAwait(false); + } + bool runtimeOwnsGatewayRoute = ctx.LocalAiGatewayPriorState is null; + LocalAiRuntimeSnapshot restored = runtimeOwnsGatewayRoute + ? await borrowedRuntime.RestartForSetupRollbackAsync(ct).ConfigureAwait(false) + : await borrowedRuntime.RestartForSetupAsync(ct).ConfigureAwait(false); LocalAiResolvedInstall restoredInstall = await store.LoadAsync(ct).ConfigureAwait(false) ?? throw new InvalidDataException( "The previous Local AI receipt was unavailable after restarting its runtime."); @@ -263,7 +351,8 @@ restored.ModelEvidence.State is not throw new InvalidDataException( "The previous Local AI endpoint was not healthy after its runtime was restored."); } - if (!await ConfigureLocalAiGatewayStep + if (!runtimeOwnsGatewayRoute && + !await ConfigureLocalAiGatewayStep .AcknowledgeBorrowedRuntimeRouteAsync(ctx, ct) .ConfigureAwait(false)) { diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 9784781fd..25c48236d 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -1034,11 +1034,12 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati if (ctx.LocalAiRuntime is not null && !ctx.LocalAiRuntimeBorrowed) return StepResult.Terminal("A Local AI runtime is already attached to this setup transaction."); if (ctx.LocalAiRuntimeBorrowed && + ctx.LocalAiRecoveryOriginalInstall is null && ctx.LocalAiUpgradeOriginalInstall is null && ctx.LocalAiResolvedInstall.Manifest.ReplacedManifest is null) { return StepResult.Terminal( - "Borrowing the tray Local AI runtime requires a recorded model replacement or runtime upgrade."); + "Borrowing the tray Local AI runtime requires an armed recovery or recorded upgrade."); } ILocalAiRuntime runtime = ctx.LocalAiRuntime ?? _runtimeFactory(ctx); @@ -1214,7 +1215,7 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - await ResetRouterAsync(runtime, ctx.LocalAiRuntimeBorrowed); + await ResetRouterAsync(ctx, runtime); throw; } catch (OperationCanceledException ex) @@ -1258,7 +1259,7 @@ private static async Task CaptureFailureDetailAsync( var paths = new LocalAiPaths(ctx.LocalDataDir); IReadOnlyList diagnostics = await LocalAiLogTail.ReadDiagnosticLinesAsync(paths, CancellationToken.None); - await ResetRouterAsync(runtime, ctx.LocalAiRuntimeBorrowed); + await ResetRouterAsync(ctx, runtime); // Echo into the setup log the UI already links, so the root cause remains available if // the router restart rotates the managed llama-server logs. foreach (string line in diagnostics) @@ -1281,6 +1282,43 @@ internal static async Task ResetRouterAsync( return runtime.Snapshot; } } + + internal static async Task ResetRouterAsync( + SetupContext ctx, + ILocalAiRuntime runtime) + { + LocalAiRuntimeSnapshot reset = await ResetRouterAsync(runtime, ctx.LocalAiRuntimeBorrowed); + if (!ctx.LocalAiRuntimeBorrowed) + return reset; + + try + { + LocalAiResolvedInstall? restartedInstall = await new LocalAiManifestStore( + new LocalAiPaths(ctx.LocalDataDir)) + .LoadAsync(CancellationToken.None) + .ConfigureAwait(false); + LocalAiInstallManifest expected = ctx.LocalAiResolvedInstall!.Manifest; + bool onlyEndpointStateChanged = restartedInstall is not null && + JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(restartedInstall.Manifest), + JsonSerializer.SerializeToElement(expected with + { + Endpoint = restartedInstall.Manifest.Endpoint, + PreviousEndpoints = restartedInstall.Manifest.PreviousEndpoints, + })); + if (restartedInstall?.Endpoint == reset.Endpoint && onlyEndpointStateChanged) + { + // This refresh updates only the rollback compare-and-swap baseline. The caller + // still decides whether the runtime restart itself was healthy and successful. + ctx.LocalAiResolvedInstall = restartedInstall; + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + // Keep the prior compare-and-swap baseline so rollback fails closed. + } + return reset; + } } /// Proves the app-owned WSL distro can reach the native loopback router. diff --git a/src/OpenClaw.SetupEngine/SetupPipeline.cs b/src/OpenClaw.SetupEngine/SetupPipeline.cs index cb33b490f..230d1c918 100644 --- a/src/OpenClaw.SetupEngine/SetupPipeline.cs +++ b/src/OpenClaw.SetupEngine/SetupPipeline.cs @@ -105,9 +105,13 @@ public static List BuildLocalAiRecoverySteps() => new PreflightWslStep(), new EnsureWslPlatformStep(reusePreflightResult: true), new ReconcileLocalAiInstallationStep(), + new ValidateLocalAiRecoveryGatewayCompatibilityStep(), new AcquireLocalAiRuntimeStep(), new AcquireLocalAiModelStep(), new PersistLocalAiManifestStep(), + // Arm receipt, runtime, and final restart recovery before the borrowed runtime changes. + // Reverse rollback still restores WSL networking before this guard settles the Gateway. + new PreserveLocalAiRecoveryGatewayStep(), new StartLocalAiRuntimeStep(), new CaptureLocalAiGpuBaselineStep(), new VerifyLocalAiInferenceStep(), @@ -115,9 +119,6 @@ public static List BuildLocalAiRecoverySteps() => new ValidateLocalAiRecoveryGatewayStep(finalCheck: true), new ConfigureLocalAiWslNetworkingStep(), new VerifyLocalAiWslStep(), - // Roll this guard back immediately after Gateway compensation so it can settle the - // receipt and endpoint-health decision before runtime and asset cleanup begins. - new PreserveLocalAiRecoveryGatewayStep(), new ConfigureLocalAiGatewayStep(), new RestartGatewayStep(), new FinalizeLocalAiModelReplacementStep(), diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index b76892bc7..3ae1e9a9a 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -199,6 +199,7 @@ public void BuildLocalAiRecoverySteps_PreservesExistingWslGateway() Assert.DoesNotContain(steps, step => step is ValidateDistroInstallPathStep); Assert.Equal(2, steps.Count(step => step is ValidateLocalAiRecoveryGatewayStep)); Assert.Contains(steps, step => step is PreserveLocalAiRecoveryGatewayStep); + Assert.Contains(steps, step => step is ValidateLocalAiRecoveryGatewayCompatibilityStep); Assert.DoesNotContain(steps, step => step is CleanupStaleDistroStep); Assert.DoesNotContain(steps, step => step is CleanupStaleGatewayStep); Assert.DoesNotContain(steps, step => step is CreateWslInstanceStep); @@ -211,16 +212,22 @@ public void BuildLocalAiRecoverySteps_PreservesExistingWslGateway() Assert.IsType(steps[^2]); Assert.IsType(steps[^1]); Assert.True( - steps.FindIndex(step => step is ValidateLocalAiRecoveryGatewayStep) < + steps.FindIndex(step => step is ReconcileLocalAiInstallationStep) < + steps.FindIndex(step => step is ValidateLocalAiRecoveryGatewayCompatibilityStep)); + Assert.True( + steps.FindIndex(step => step is ValidateLocalAiRecoveryGatewayCompatibilityStep) < steps.FindIndex(step => step is AcquireLocalAiRuntimeStep)); + Assert.True( + steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep) < + steps.FindIndex(step => step is StartLocalAiRuntimeStep)); Assert.True( steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep) < steps.FindIndex(step => step is ConfigureLocalAiGatewayStep)); Assert.True( - steps.FindIndex(step => step is VerifyLocalAiWslStep) < - steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep)); - Assert.IsType( - steps[steps.FindIndex(step => step is ConfigureLocalAiWslNetworkingStep) - 1]); + steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep) < + steps.FindIndex(step => step is ConfigureLocalAiWslNetworkingStep)); + Assert.IsType( + steps[steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep) - 1]); } [Fact] @@ -237,6 +244,119 @@ public async Task ValidateLocalAiRecoveryGateway_MissingDistro_BlocksBeforeRecov Assert.Contains("run full setup", result.Message, StringComparison.OrdinalIgnoreCase); } + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_AutomaticPortRejectsLegacyGateway() + { + using var temp = new TempDirectory("local-ai-compatibility-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Port = 18801; + var context = CreateContext(config, localDataDir: temp.Path); + context.LocalAiRecoveryOriginalInstall = CreateLocalAiResolvedInstall(temp.Path, 18801); + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + Task.FromResult(new CommandResult( + 42, + ValidateLocalAiRecoveryGatewayCompatibilityStep.UnsupportedMarker, + string.Empty, + TimeSpan.Zero, + false))); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.FailedTerminal, result.Outcome); + Assert.Contains("Update the Gateway", result.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_NoPriorReceiptSkipsProbe() + { + var context = CreateContext(LocalAiRecoveryConfig()); + var probeCalls = 0; + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + { + probeCalls++; + return Task.FromResult(new CommandResult(1, string.Empty, "failed", TimeSpan.Zero, false)); + }); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Skipped, result.Outcome); + Assert.Equal(0, probeCalls); + } + + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_FixedPortSkipsProbe() + { + using var temp = new TempDirectory("local-ai-compatibility-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Port = 0; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(temp.Path, 18801); + context.LocalAiRecoveryOriginalInstall = install with + { + Manifest = install.Manifest with { RequestedPort = 18801 }, + }; + var probeCalls = 0; + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + { + probeCalls++; + return Task.FromResult(new CommandResult(1, string.Empty, "failed", TimeSpan.Zero, false)); + }); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Skipped, result.Outcome); + Assert.Equal(0, probeCalls); + } + + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_AutomaticPortAcceptsConditionalWrites() + { + using var temp = new TempDirectory("local-ai-compatibility-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + context.LocalAiRecoveryOriginalInstall = CreateLocalAiResolvedInstall(temp.Path, 18801); + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + Task.FromResult(new CommandResult( + 0, + ValidateLocalAiRecoveryGatewayCompatibilityStep.SupportedMarker, + string.Empty, + TimeSpan.Zero, + false))); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + } + + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_ChecksAutomaticRollbackReceipt() + { + using var temp = new TempDirectory("local-ai-compatibility-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall automatic = CreateLocalAiResolvedInstall(temp.Path, 18801); + LocalAiResolvedInstall fixedPort = automatic with + { + Manifest = automatic.Manifest with { RequestedPort = 18802 }, + }; + context.LocalAiRecoveryOriginalInstall = automatic; + context.LocalAiRecoveryPendingInstall = fixedPort; + var probeCalls = 0; + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + { + probeCalls++; + return Task.FromResult(new CommandResult( + 42, + ValidateLocalAiRecoveryGatewayCompatibilityStep.UnsupportedMarker, + string.Empty, + TimeSpan.Zero, + false)); + }); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.FailedTerminal, result.Outcome); + Assert.Equal(1, probeCalls); + } + [Fact] public async Task ValidateLocalAiRecoveryGateway_AppOwnedDistro_AllowsRecovery() { @@ -372,6 +492,157 @@ public async Task FinalizationFailure_RestartsGatewayAfterConfigurationRollback( Assert.False(context.LocalAiRecoveryStoppedWsl); } + [Fact] + public async Task LaterFailure_RestoresWslNetworkingBeforeRecoveryGuardRestartsGateway() + { + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config); + var rollbackOrder = new List(); + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep((_, _) => + { + rollbackOrder.Add("restart"); + return Task.FromResult(StepResult.Ok("restarted")); + }), + new MockStep( + "configure-local-ai-wsl-networking", + (ctx, _) => + { + ctx.LocalAiRecoveryStoppedWsl = true; + return Task.FromResult(StepResult.Ok("configured")); + }, + (_, _) => + { + rollbackOrder.Add("networking"); + return Task.CompletedTask; + }), + new MockStep("failure", (_, _) => Task.FromResult(StepResult.Fail("failed"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal(["networking", "restart"], rollbackOrder); + } + + [Fact] + public async Task WslNetworkingFailure_RollsBackArmedRecoveryGuard() + { + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config); + var restartCalls = 0; + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep((_, _) => + { + restartCalls++; + return Task.FromResult(StepResult.Ok("restarted")); + }), + new MockStep( + "configure-local-ai-wsl-networking", + (ctx, _) => + { + ctx.LocalAiRecoveryStoppedWsl = true; + return Task.FromResult(StepResult.Fail("failed after stopping WSL")); + }), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal(1, restartCalls); + Assert.False(context.LocalAiRecoveryStoppedWsl); + } + + [Fact] + public async Task BorrowedRuntimeFailure_RollsBackArmedRecoveryGuard() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-failure-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Enabled = true; + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(temp.Path, 18801); + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + context.LocalAiRecoveryOriginalInstall = install; + context.LocalAiRecoveryReceiptRollbackAllowed = true; + var restartCalls = 0; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)) + { + RestartForSetupHandler = _ => + { + restartCalls++; + LocalAiRuntimeSnapshot snapshot = HealthySnapshot(install); + return Task.FromResult(restartCalls == 1 + ? snapshot with + { + State = LocalAiRuntimeState.Failed, + Ownership = LocalAiOwnership.None, + ProcessId = null, + } + : snapshot); + }, + }; + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("not needed")), + (_, _) => Task.FromResult(true)), + new StartLocalAiRuntimeStep(_ => runtime), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(1, runtime.RestartForSetupRollbackCalls); + Assert.True(context.LocalAiBorrowedRuntimeRestored); + Assert.Equal(LocalAiRuntimeState.Healthy, runtime.Snapshot.State); + } + + [Fact] + public async Task OrdinaryRecoveryRollback_PreservesConcurrentlyUpdatedReceipt() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-concurrent-receipt-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Enabled = true; + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, 18801); + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(original.Manifest); + context.LocalAiResolvedInstall = original; + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryReceiptRollbackAllowed = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(original)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + LocalAiInstallManifest concurrent = original.Manifest with + { + Endpoint = "http://127.0.0.1:18803/v1", + }; + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("not needed")), + (_, _) => Task.FromResult(true)), + new MockStep( + "failure", + (_, _) => Task.FromResult(StepResult.Fail("failed")), + async (_, ct) => await store.SaveAsync(concurrent, ct)), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal(concurrent.Endpoint, (await store.LoadAsync())!.Manifest.Endpoint); + Assert.True(context.LocalAiRecoveryRollbackUncertain); + Assert.Equal(0, runtime.RestartForSetupRollbackCalls); + } + /// /// Regression guard for a rollback race: if the Gateway could not be confirmed switched back /// to the original (A) endpoint, the replacement (B) runtime must be kept alive rather than @@ -451,13 +722,14 @@ public async Task StartLocalAiRuntimeStep_RestartsBorrowedTrayRuntimeForReplacem } [Fact] - public async Task StartLocalAiRuntimeStep_RejectsBorrowedRuntimeWithoutReplacementOrUpgrade() + public async Task StartLocalAiRuntimeStep_RestartsBorrowedTrayRuntimeForOrdinaryRecovery() { using var temp = new TempDirectory("local-ai-borrowed-runtime-no-replacement-"); var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(install.Manifest); context.LocalAiResolvedInstall = install; + context.LocalAiRecoveryOriginalInstall = install; var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)); context.LocalAiRuntime = runtime; context.LocalAiRuntimeBorrowed = true; @@ -466,9 +738,30 @@ public async Task StartLocalAiRuntimeStep_RejectsBorrowedRuntimeWithoutReplaceme StepResult result = await step.ExecuteAsync(context, CancellationToken.None); await step.RollbackAsync(context, CancellationToken.None); + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(1, runtime.StopForSetupCalls); + Assert.True(context.LocalAiBorrowedRuntimeRestartedThisRun); + } + + [Fact] + public async Task StartLocalAiRuntimeStep_RejectsBorrowedRuntimeOutsideRecovery() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-unarmed-"); + var context = CreateContext(new SetupConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + + StepResult result = await new StartLocalAiRuntimeStep().ExecuteAsync( + context, + CancellationToken.None); + Assert.Equal(StepOutcome.FailedTerminal, result.Outcome); Assert.Equal(0, runtime.RestartForSetupCalls); - Assert.Equal(0, runtime.StopForSetupCalls); Assert.False(context.LocalAiBorrowedRuntimeRestartedThisRun); } @@ -483,6 +776,109 @@ public async Task ResetRouterAsync_UsesSetupScopedRestartForBorrowedRuntime() Assert.Equal(0, runtime.RestartCalls); } + [Fact] + public async Task ResetRouterAsync_RefreshesBorrowedRuntimeReceiptAfterAutomaticPortMove() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-reset-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, port: 18801); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:18803/v1" }, + Endpoint = new Uri("http://127.0.0.1:18803/v1"), + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(original.Manifest); + context.LocalAiResolvedInstall = original; + context.LocalAiRuntimeBorrowed = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(original)) + { + RestartForSetupHandler = async ct => + { + await store.SaveAsync(moved.Manifest, ct); + return HealthySnapshot(moved); + }, + }; + + LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync( + context, + runtime); + + Assert.Equal(moved.Endpoint, reset.Endpoint); + Assert.Equal(moved.Endpoint, context.LocalAiResolvedInstall.Endpoint); + } + + [Fact] + public async Task ResetRouterAsync_DoesNotAdoptConcurrentReceiptChanges() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-reset-concurrent-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, port: 18801); + LocalAiResolvedInstall concurrent = original with + { + Manifest = original.Manifest with + { + Endpoint = "http://127.0.0.1:18803/v1", + ContextLength = original.Manifest.ContextLength + 1, + }, + Endpoint = new Uri("http://127.0.0.1:18803/v1"), + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(original.Manifest); + context.LocalAiResolvedInstall = original; + context.LocalAiRuntimeBorrowed = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(original)) + { + RestartForSetupHandler = async ct => + { + await store.SaveAsync(concurrent.Manifest, ct); + return HealthySnapshot(concurrent); + }, + }; + + await VerifyLocalAiInferenceStep.ResetRouterAsync(context, runtime); + + Assert.Same(original, context.LocalAiResolvedInstall); + Assert.Equal(concurrent.Manifest.ContextLength, (await store.LoadAsync())!.Manifest.ContextLength); + } + + [Fact] + public async Task ResetRouterAsync_RefreshesEndpointBaselineAfterFailedReset() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-reset-failed-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, port: 18801); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:18803/v1" }, + Endpoint = new Uri("http://127.0.0.1:18803/v1"), + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(original.Manifest); + context.LocalAiResolvedInstall = original; + context.LocalAiRuntimeBorrowed = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(original)) + { + RestartForSetupHandler = async ct => + { + await store.SaveAsync(moved.Manifest, ct); + return HealthySnapshot(moved) with + { + State = LocalAiRuntimeState.Failed, + Ownership = LocalAiOwnership.None, + ProcessId = null, + }; + }, + }; + + LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync( + context, + runtime); + + Assert.Equal(LocalAiRuntimeState.Failed, reset.State); + Assert.Equal(moved.Endpoint, context.LocalAiResolvedInstall.Endpoint); + } + [Fact] public async Task ReleaseBorrowedLocalAiRuntimeAfterFailureAsync_ReleasesNoRollbackOwnership() { From 526d61c55244de4b987c0e4fcc55cae915bccb60 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 1 Oct 2026 07:57:40 -0700 Subject: [PATCH 20/22] fix(setup): classify recovery compatibility progress --- src/OpenClaw.SetupEngine/SetupInstallationProgress.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs b/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs index e51d72bd5..1dcffbee6 100644 --- a/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs +++ b/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs @@ -74,6 +74,7 @@ private static SetupInstallationStatus Aggregate(IEnumerable entries) { "validate-distro-path" or "preflight-os" or "preflight-local-ai-hardware" or "preflight-wsl" or "preflight-windows-tailscale" or "ensure-wsl-platform" or "validate-local-ai-recovery-gateway" or + "validate-local-ai-recovery-gateway-compatibility" or "reconcile-local-ai-installation" or "cleanup-distro" or "cleanup-gateway" or "preflight-port" or "wsl-create" or "wsl-configure" or "validate-wsl-lockdown" => SetupInstallationPhase.Prepare, From 320e4674c2174424f5bc2fabf4dd70f47c176e59 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 1 Oct 2026 08:32:13 -0700 Subject: [PATCH 21/22] fix(setup): admit first recovery install runtime --- src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 3 ++- .../SetupPipelineTests.cs | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index 25c48236d..2fc5288cb 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -1036,7 +1036,8 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati if (ctx.LocalAiRuntimeBorrowed && ctx.LocalAiRecoveryOriginalInstall is null && ctx.LocalAiUpgradeOriginalInstall is null && - ctx.LocalAiResolvedInstall.Manifest.ReplacedManifest is null) + ctx.LocalAiResolvedInstall.Manifest.ReplacedManifest is null && + string.IsNullOrWhiteSpace(ctx.Config.LocalAiRecoveryGatewayId)) { return StepResult.Terminal( "Borrowing the tray Local AI runtime requires an armed recovery or recorded upgrade."); diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index 3ae1e9a9a..53c8439df 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -744,6 +744,27 @@ public async Task StartLocalAiRuntimeStep_RestartsBorrowedTrayRuntimeForOrdinary Assert.True(context.LocalAiBorrowedRuntimeRestartedThisRun); } + [Fact] + public async Task StartLocalAiRuntimeStep_RestartsBorrowedTrayRuntimeForFirstRecoveryInstall() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-first-install-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + + StepResult result = await new StartLocalAiRuntimeStep().ExecuteAsync( + context, + CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.True(context.LocalAiBorrowedRuntimeRestartedThisRun); + } + [Fact] public async Task StartLocalAiRuntimeStep_RejectsBorrowedRuntimeOutsideRecovery() { From 6f232a9c664db31c4e3db17fea7360298d5033ab Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 1 Oct 2026 17:24:03 -0700 Subject: [PATCH 22/22] fix(setup): retire settled Local AI rollback baselines --- .../LocalAiRecoveryPolicy.cs | 4 + .../LocalAiGatewayUninstallTests.cs | 164 +++++++++++++++--- 2 files changed, 145 insertions(+), 23 deletions(-) diff --git a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs index d5eb62112..71b5b209d 100644 --- a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs +++ b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs @@ -426,6 +426,10 @@ restored.ModelEvidence.State is not private static void CompleteReceiptRollback(SetupContext ctx) { + // The recovery guard now owns the settled receipt. Retire the earlier upgrade and + // recovery baselines so later reverse rollback steps cannot restore them again. + ctx.LocalAiUpgradeOriginalInstall = null; + ctx.LocalAiRecoveryOriginalInstall = null; ctx.LocalAiRecoveryProviderTransition = false; ctx.LocalAiRecoveryReceiptRollbackAllowed = false; ctx.LocalAiRecoveryRollbackUncertain = false; diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index 7769a2f5c..863600daa 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -792,8 +792,11 @@ public async Task RestoreRecoveryRouteAsync_RejectsLegacyGatewayCliWithoutCondit command => command.Contains("OPENCLAW_LOCAL_AI_BATCH_B64", StringComparison.Ordinal)); } - [Fact] - public async Task Recovery_UpgradedPendingRouteRestoresCoherentRuntimeGeneration() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Recovery_UpgradedRouteSettlementSurvivesRemainingRollback( + bool priorRouteWasPending) { using var temp = new TempDirectory("local-ai-gateway-recovery-"); LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); @@ -823,47 +826,69 @@ public async Task Recovery_UpgradedPendingRouteRestoresCoherentRuntimeGeneration }; await store.SaveAsync(upgradedReplacement); - string priorProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(oldPending); + LocalAiResolvedInstall priorRoute = priorRouteWasPending ? oldPending : original; + string priorProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(priorRoute); string primary = JsonSerializer.Serialize( - LocalAiGatewayProviderDefinition.BuildPrimaryModel(oldPending)); + LocalAiGatewayProviderDefinition.BuildPrimaryModel(priorRoute)); var commands = new GatewayStateCommandRunner(priorProvider, primary); SetupContext context = CreateRecoveryContext(temp.Path, commands); context.Config.RollbackOnFailure = true; context.LocalAiRecoveryOriginalInstall = store.ResolveAndValidate(upgradedOriginal); context.LocalAiRecoveryPendingInstall = store.ResolveAndValidate(upgradedPendingRoute); + context.LocalAiUpgradeOriginalInstall = oldPending; context.LocalAiResolvedInstall = store.ResolveAndValidate(upgradedReplacement); + context.LocalAiRuntimeBorrowed = true; + var runtime = new SettlementTrackingRuntime(store); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeInstall = new LlamaRuntimeInstallResult( + Path.GetDirectoryName(context.LocalAiResolvedInstall.ExecutablePath)!, + context.LocalAiResolvedInstall.ExecutablePath, + LlamaRuntimeInstallDisposition.Installed, + CreatedThisRun: true, + VerifiedArchives: [], + Rollback: null); + var runtimeAcquirer = new TrackingRuntimeAcquirer(); var configure = new ConfigureLocalAiGatewayStep(); StepResult configured = await configure.ExecuteAsync(context, CancellationToken.None); - var pipeline = new SetupPipeline( - [ - new PreserveLocalAiRecoveryGatewayStep( + await configure.RollbackAsync(context, CancellationToken.None); + await new PreserveLocalAiRecoveryGatewayStep( (_, _) => Task.FromResult(StepResult.Ok("not needed")), - (_, _) => Task.FromResult(true)), - new DelegatingRollbackStep("configured", configure.RollbackAsync), - new DelegatingRollbackStep( - "fail", - (_, _) => Task.CompletedTask, - (_, _) => Task.FromResult(StepResult.Fail("forced failure"))), - ]); - - PipelineResult result = await pipeline.RunAsync(context); + (_, _) => Task.FromResult(true), + (_, _, _, _, _) => Task.FromResult(true)) + .RollbackAsync(context, CancellationToken.None); + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + await new AcquireLocalAiRuntimeStep(runtimeAcquirer) + .RollbackAsync(context, CancellationToken.None); Assert.Equal(StepOutcome.Success, configured.Outcome); - Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(0, runtime.RestartForSetupRollbackCalls); + Assert.Null(context.LocalAiUpgradeOriginalInstall); + Assert.Null(context.LocalAiRecoveryOriginalInstall); LocalAiResolvedInstall restored = Assert.IsType( await store.LoadAsync()); - Assert.Equal(oldPending.Endpoint, restored.Endpoint); + Assert.Equal(priorRoute.Endpoint, restored.Endpoint); + Assert.Equal(priorRoute.Manifest.ModelCatalogId, restored.Manifest.ModelCatalogId); Assert.Equal(upgradedReplacement.RuntimeId, restored.Manifest.RuntimeId); - LocalAiInstallManifest restoredOriginal = Assert.IsType( - restored.Manifest.ReplacedManifest); - Assert.Equal(upgradedOriginal.RuntimeId, restoredOriginal.RuntimeId); - Assert.Equal(upgradedOriginal.ModelCatalogId, restoredOriginal.ModelCatalogId); - Assert.Equal(upgradedOriginal.Endpoint, restoredOriginal.Endpoint); + if (priorRouteWasPending) + { + LocalAiInstallManifest restoredOriginal = Assert.IsType( + restored.Manifest.ReplacedManifest); + Assert.Equal(upgradedOriginal.RuntimeId, restoredOriginal.RuntimeId); + Assert.Equal(upgradedOriginal.ModelCatalogId, restoredOriginal.ModelCatalogId); + Assert.Equal(upgradedOriginal.Endpoint, restoredOriginal.Endpoint); + } + else + { + Assert.Null(restored.Manifest.ReplacedManifest); + } Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson( commands.ProviderJson!, restored)); Assert.Equal(primary, commands.PrimaryJson); + Assert.Null(context.LocalAiRuntimeInstall); + Assert.Equal(0, runtimeAcquirer.RemoveCalls); Assert.False(context.LocalAiRecoveryRollbackUncertain); Assert.False(context.LocalAiRecoveryProviderTransition); } @@ -1354,6 +1379,99 @@ public Task AcknowledgeSetupGatewayRouteAsync( public ValueTask DisposeAsync() => ValueTask.CompletedTask; } + private sealed class SettlementTrackingRuntime(LocalAiManifestStore store) : ILocalAiRuntime + { + public int RestartForSetupCalls { get; private set; } + public int RestartForSetupRollbackCalls { get; private set; } + + public LocalAiRuntimeSnapshot Snapshot { get; private set; } = LocalAiRuntimeSnapshot.Initial( + new Uri("http://127.0.0.1:18800/v1"), + DateTimeOffset.UtcNow); + + public event EventHandler? StateChanged + { + add { } + remove { } + } + + public Task EnsureStartedAsync( + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task ResumeAsync( + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task StopAsync( + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task RestartAsync( + CancellationToken cancellationToken = default) => LoadSnapshotAsync(cancellationToken); + + public async Task RestartForSetupAsync( + CancellationToken cancellationToken = default) + { + RestartForSetupCalls++; + return await LoadSnapshotAsync(cancellationToken); + } + + public Task RefreshAsync( + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) => Task.FromResult(Snapshot); + + public async Task RestartForSetupRollbackAsync( + CancellationToken cancellationToken = default) + { + RestartForSetupRollbackCalls++; + return await LoadSnapshotAsync(cancellationToken); + } + + private async Task LoadSnapshotAsync( + CancellationToken cancellationToken) + { + LocalAiResolvedInstall restored = await store.LoadAsync(cancellationToken) + ?? throw new InvalidDataException("The restored Local AI receipt is unavailable."); + Snapshot = HealthySnapshot(restored); + return Snapshot; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class TrackingRuntimeAcquirer : ILlamaRuntimeAcquirer + { + public int RemoveCalls { get; private set; } + + public Task InstallAsync( + string localDataDirectory, + LlamaRuntimeVariant runtime, + IProgress? progress, + CancellationToken cancellationToken) => throw new NotSupportedException(); + + public void RemoveInstalledRuntime( + string localDataDirectory, + LlamaRuntimeInstallResult install) => RemoveCalls++; + } + + private static LocalAiRuntimeSnapshot HealthySnapshot(LocalAiResolvedInstall install) => new( + LocalAiRuntimeState.Healthy, + LocalAiOwnership.CompanionManaged, + install.Endpoint!, + install.Manifest.EngineVersion, + install.Manifest.ModelCatalogId, + new LocalAiModelEvidence( + LocalAiModelAvailabilityState.Verified, + DateTimeOffset.UtcNow, + install.Manifest.ModelAsset.Sha256, + install.Manifest.ModelAsset.SizeBytes), + 42, + DateTimeOffset.UtcNow, + null, + DateTimeOffset.UtcNow) + { + GatewayRouteRequiresResolution = false, + }; + private sealed class DelegatingRollbackStep( string id, Func rollback,