diff --git a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs index ec0398621..b3acd1752 100644 --- a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs +++ b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs @@ -129,6 +129,7 @@ public sealed class LlamaServerRuntimeService : ILocalAiRuntime private bool _explicitStopRequested; private bool _automaticResumeSuppressed; private bool _gatewayRouteRequiresResolution; + private bool _setupOwnsEndpointLifecycle; private bool _disposed; private bool _acceptExitTasks = true; private int _disposeStarted; @@ -284,14 +285,143 @@ public async Task RefreshAsync(CancellationToken cancell } public async Task StopAsync(CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return await StopWithStateAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task StopForSetupAsync(CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + _setupOwnsEndpointLifecycle = true; + return await StopWithStateAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task RestartAsync(CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return await RestartCoreAsync( + cancellationToken, + enableAutomaticRecovery: !_setupOwnsEndpointLifecycle) + .ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task RestartForSetupAsync(CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + _setupOwnsEndpointLifecycle = true; + return await RestartCoreAsync(cancellationToken, enableAutomaticRecovery: false).ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task RestartForSetupRollbackAsync( + CancellationToken cancellationToken = default) { await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); try { ThrowIfDisposed(); - _automaticResumeSuppressed = true; - await _options.EndpointLifecycle.SetAutomaticRecoveryEnabledAsync(false, cancellationToken).ConfigureAwait(false); - _explicitStopRequested = true; + _setupOwnsEndpointLifecycle = false; + if (!await TryLoadInstallAsync(cancellationToken).ConfigureAwait(false)) + return Snapshot; + return await RestartCoreAsync(cancellationToken, enableAutomaticRecovery: true).ConfigureAwait(false); + } + finally + { + _operationGate.Release(); + } + } + + public async Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + ThrowIfDisposed(); + LocalAiRuntimeSnapshot current = Snapshot; + if (current.State != LocalAiRuntimeState.Healthy || + current.Ownership != LocalAiOwnership.CompanionManaged) + { + throw new InvalidOperationException( + "The setup Gateway route cannot be acknowledged without a healthy managed Local AI runtime."); + } + await _options.EndpointLifecycle + .SetAutomaticRecoveryEnabledAsync(true, cancellationToken) + .ConfigureAwait(false); + _automaticResumeSuppressed = false; + _setupOwnsEndpointLifecycle = false; + _gatewayRouteRequiresResolution = false; + return SetSnapshot(current with + { + GatewayRouteRequiresResolution = false, + UpdatedAtUtc = _platform.UtcNow, + }); + } + finally + { + _operationGate.Release(); + } + } + + public async Task ReleaseSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) + { + await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + ThrowIfDisposed(); + await _options.EndpointLifecycle + .SetAutomaticRecoveryEnabledAsync(true, cancellationToken) + .ConfigureAwait(false); + _automaticResumeSuppressed = false; + _setupOwnsEndpointLifecycle = false; + LocalAiRuntimeSnapshot current = Snapshot; + return SetSnapshot(current with { UpdatedAtUtc = _platform.UtcNow }); + } + finally + { + _operationGate.Release(); + } + } + + private async Task StopWithStateAsync(CancellationToken cancellationToken) + { + ThrowIfDisposed(); + _automaticResumeSuppressed = true; + await _options.EndpointLifecycle + .SetAutomaticRecoveryEnabledAsync(false, cancellationToken) + .ConfigureAwait(false); + _explicitStopRequested = true; + try + { LocalAiRuntimeSnapshot stopped = await StopCoreAsync( LocalAiQuiesceReason.Teardown, cancellationToken) @@ -319,95 +449,104 @@ public async Task StopAsync(CancellationToken cancellati } throw; } - finally - { - _operationGate.Release(); - } } - public async Task RestartAsync(CancellationToken cancellationToken = default) + private async Task RestartCoreAsync( + CancellationToken cancellationToken, + bool enableAutomaticRecovery) { - await _operationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + ThrowIfDisposed(); + bool publishEndpointLifecycle = enableAutomaticRecovery && !_setupOwnsEndpointLifecycle; bool admitted = false; bool completedSuccessfully = false; + if (publishEndpointLifecycle && + (_install is not null || await TryLoadInstallAsync(cancellationToken).ConfigureAwait(false))) + { + await _options.EndpointLifecycle + .PrepareStartAsync(_install!, cancellationToken) + .ConfigureAwait(false); + } + await _options.EndpointLifecycle + .SetAutomaticRecoveryEnabledAsync( + publishEndpointLifecycle ? false : enableAutomaticRecovery, + cancellationToken) + .ConfigureAwait(false); + admitted = true; + _automaticResumeSuppressed = publishEndpointLifecycle || !enableAutomaticRecovery; + _explicitStopRequested = false; + LocalAiResolvedInstall? restartInstall = _install; try { - ThrowIfDisposed(); - if (_install is not null || await TryLoadInstallAsync(cancellationToken).ConfigureAwait(false)) - await _options.EndpointLifecycle.PrepareStartAsync(_install!, cancellationToken).ConfigureAwait(false); - await _options.EndpointLifecycle.SetAutomaticRecoveryEnabledAsync(false, cancellationToken).ConfigureAwait(false); - admitted = true; - _automaticResumeSuppressed = false; - _explicitStopRequested = false; - LocalAiResolvedInstall? restartInstall = _install; - try - { - LocalAiRuntimeSnapshot stopped = await StopCoreAsync( - LocalAiQuiesceReason.EndpointCycle, - cancellationToken) - .ConfigureAwait(false); - restartInstall ??= _install; - if (_managedProcess is not null || stopped.State == LocalAiRuntimeState.Failed) - return SuppressIncompleteStart(); + LocalAiRuntimeSnapshot stopped = await StopCoreAsync( + LocalAiQuiesceReason.EndpointCycle, + cancellationToken) + .ConfigureAwait(false); + restartInstall ??= _install; + if (_managedProcess is not null || stopped.State == LocalAiRuntimeState.Failed) + return publishEndpointLifecycle ? SuppressIncompleteStart() : stopped; - _restartAttempts = 0; - LocalAiRuntimeSnapshot restarted = await EnsureStartedCoreAsync(cancellationToken) - .ConfigureAwait(false); - if (restarted.State is LocalAiRuntimeState.Failed or LocalAiRuntimeState.NotInstalled && - restartInstall is not null) + _restartAttempts = 0; + LocalAiRuntimeSnapshot restarted = await EnsureStartedCoreAsync(cancellationToken) + .ConfigureAwait(false); + if (restarted.State is LocalAiRuntimeState.Failed or LocalAiRuntimeState.NotInstalled && + restartInstall is not null) + { + LocalAiResolvedInstall cleanupInstall = _install ?? restartInstall; + bool withdrawn = await WithdrawRouteAsync( + cleanupInstall, + "after restart startup did not complete").ConfigureAwait(false); + if (!withdrawn) { - LocalAiResolvedInstall cleanupInstall = _install ?? restartInstall; - bool withdrawn = await WithdrawRouteAsync( - cleanupInstall, - "after restart startup did not complete").ConfigureAwait(false); - if (!withdrawn) - { - return _managedProcess is { HasExited: false } - ? PublishManagedFailure( - "Local AI restart did not complete and gateway routing could not be safely disabled; the managed listener remains running.") - : PublishTerminalCleanupFailure( - "Local AI restart did not complete and gateway routing could not be safely disabled."); - } - if (_managedProcess is { HasExited: false }) - { - ++_generation; - await DisposeManagedProcessAsync(CancellationToken.None).ConfigureAwait(false); - return PublishTerminalCleanupFailure("Local AI restart did not complete."); - } + return _managedProcess is { HasExited: false } + ? PublishManagedFailure( + "Local AI restart did not complete and gateway routing could not be safely disabled; the managed listener remains running.") + : PublishTerminalCleanupFailure( + "Local AI restart did not complete and gateway routing could not be safely disabled."); } - if (restarted.State == LocalAiRuntimeState.Healthy) + if (_managedProcess is { HasExited: false }) { - var completed = await _options.EndpointLifecycle.CompleteStartAsync(_install!, cancellationToken).ConfigureAwait(false); - if (!completed.Success) - { - _gatewayRouteRequiresResolution = true; - return PublishManagedFailure(completed.Detail ?? "Local AI publication requires reconciliation."); - } - await _options.EndpointLifecycle.SetAutomaticRecoveryEnabledAsync(true, CancellationToken.None).ConfigureAwait(false); - completedSuccessfully = true; + ++_generation; + await DisposeManagedProcessAsync(CancellationToken.None).ConfigureAwait(false); + return PublishTerminalCleanupFailure("Local AI restart did not complete."); } - return completedSuccessfully ? restarted : SuppressIncompleteStart(); } - catch (Exception ex) when (ex is not OperationCanceledException) + if (publishEndpointLifecycle && restarted.State == LocalAiRuntimeState.Healthy) { - LocalAiResolvedInstall? interruptedInstall = restartInstall ?? _install; - if (interruptedInstall is not null) - await CompleteInterruptedRestartAsync(interruptedInstall, "interrupted").ConfigureAwait(false); - throw; - } - catch (OperationCanceledException) - { - LocalAiResolvedInstall? canceledInstall = restartInstall ?? _install; - if (canceledInstall is not null) - await CompleteInterruptedRestartAsync(canceledInstall, "canceled").ConfigureAwait(false); - throw; + LocalAiEndpointLifecycleResult completed = await _options.EndpointLifecycle + .CompleteStartAsync(_install!, cancellationToken) + .ConfigureAwait(false); + if (!completed.Success) + { + _gatewayRouteRequiresResolution = true; + return PublishManagedFailure( + completed.Detail ?? "Local AI publication requires reconciliation."); + } + await _options.EndpointLifecycle + .SetAutomaticRecoveryEnabledAsync(true, CancellationToken.None) + .ConfigureAwait(false); + _automaticResumeSuppressed = false; + completedSuccessfully = true; } + return restarted; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LocalAiResolvedInstall? interruptedInstall = restartInstall ?? _install; + if (interruptedInstall is not null) + await CompleteInterruptedRestartAsync(interruptedInstall, "interrupted").ConfigureAwait(false); + throw; + } + catch (OperationCanceledException) + { + LocalAiResolvedInstall? canceledInstall = restartInstall ?? _install; + if (canceledInstall is not null) + await CompleteInterruptedRestartAsync(canceledInstall, "canceled").ConfigureAwait(false); + throw; } finally { - if (admitted && !completedSuccessfully) + if (publishEndpointLifecycle && admitted && !completedSuccessfully) SuppressIncompleteStart(); - _operationGate.Release(); } } @@ -577,12 +716,11 @@ recoveryPort is not null .ConfigureAwait(false); if (probe.IsReadyForManagedModel(runtimeModelPath)) { - LocalAiInstallManifest verifiedManifest = install.Manifest with - { - Endpoint = ownership.Endpoint.AbsoluteUri, - }; - await _manifestStore.SaveAsync(verifiedManifest, cancellationToken).ConfigureAwait(false); - _install = _manifestStore.ResolveAndValidate(verifiedManifest); + _install = await _manifestStore.UpdateVerifiedEndpointAsync( + install.Manifest, + ownership.Endpoint, + cancellationToken) + .ConfigureAwait(false); LocalAiEndpointLifecycleResult published = await PublishRouteAsync( _install, @@ -950,12 +1088,11 @@ private async Task BindVerifiedEndpointAsync( if (install.Endpoint == endpoint) return install; - LocalAiInstallManifest verifiedManifest = install.Manifest with - { - Endpoint = endpoint.AbsoluteUri, - }; - await _manifestStore.SaveAsync(verifiedManifest, cancellationToken).ConfigureAwait(false); - _install = _manifestStore.ResolveAndValidate(verifiedManifest); + _install = await _manifestStore.UpdateVerifiedEndpointAsync( + install.Manifest, + endpoint, + cancellationToken) + .ConfigureAwait(false); return _install; } @@ -1277,6 +1414,12 @@ private async Task QuiesceRouteAsync( LocalAiQuiesceReason reason, CancellationToken cancellationToken) { + if (_setupOwnsEndpointLifecycle) + { + _gatewayRouteRequiresResolution = true; + return LocalAiEndpointLifecycleResult.Ok(); + } + _gatewayRouteRequiresResolution = true; LocalAiEndpointLifecycleResult result = await _options.EndpointLifecycle @@ -1291,6 +1434,9 @@ private async Task PublishRouteAsync( LocalAiResolvedInstall install, CancellationToken cancellationToken) { + if (_setupOwnsEndpointLifecycle) + return LocalAiEndpointLifecycleResult.Ok(); + LocalAiEndpointLifecycleResult result = await _options.EndpointLifecycle .PublishAsync(install, cancellationToken) .ConfigureAwait(false); @@ -1724,9 +1870,13 @@ private LocalAiRuntimeSnapshot Publish( { GatewayRouteRequiresResolution = _gatewayRouteRequiresResolution, }; + return SetSnapshot(value); + } + + private LocalAiRuntimeSnapshot SetSnapshot(LocalAiRuntimeSnapshot value) + { lock (_snapshotGate) _snapshot = value; - EventHandler? handler = StateChanged; if (handler is not null) { diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index 895ab908c..6074fcafe 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -252,6 +252,15 @@ public sealed record LocalAiInstallManifest /// managed llama.cpp model. Null means no prior primary model was configured. /// public string? GatewayFallbackModel { get; init; } + /// + /// The last committed receipt while a recovery flow replaces its model. + /// Cleared only after the Gateway has restarted on the replacement route. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LocalAiInstallManifest? ReplacedManifest { get; init; } + /// Earlier verified replacement endpoints that may still be published to the Gateway. + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ImmutableArray? PreviousEndpoints { get; init; } public required int ContextLength { get; init; } public KvCachePrecision KeyCachePrecision { get; init; } = KvCachePrecision.F16; public KvCachePrecision ValueCachePrecision { get; init; } = KvCachePrecision.F16; @@ -484,6 +493,158 @@ public async Task SaveAsync(LocalAiInstallManifest manifest, CancellationToken c await SaveWithoutLockAsync(manifest, cancellationToken).ConfigureAwait(false); } + internal async Task UpdateVerifiedEndpointAsync( + LocalAiInstallManifest expectedManifest, + Uri endpoint, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(expectedManifest); + ArgumentNullException.ThrowIfNull(endpoint); + await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) + .ConfigureAwait(false); + LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); + if (!HasSameRuntimeAndModel(current, expectedManifest)) + { + throw new InvalidDataException( + "The Local AI installation changed before its verified endpoint could be recorded."); + } + if (expectedManifest.ReplacedManifest is not null && current.ReplacedManifest is null) + { + throw new InvalidDataException( + "The Local AI model replacement was finalized before its verified endpoint could be recorded."); + } + + ImmutableArray? history = current.PreviousEndpoints; + if (current.ReplacedManifest is null) + { + history = null; + } + else if (current.Endpoint is { } previousEndpoint && + !string.Equals(previousEndpoint, endpoint.AbsoluteUri, StringComparison.Ordinal)) + { + ImmutableArray values = history ?? []; + if (!values.Contains(previousEndpoint, StringComparer.Ordinal)) + history = values.Add(previousEndpoint); + } + + LocalAiInstallManifest updated = current with + { + Endpoint = endpoint.AbsoluteUri, + PreviousEndpoints = history, + }; + LocalAiResolvedInstall resolved = ResolveAndValidate(updated); + await SaveWithoutLockAsync(updated, cancellationToken).ConfigureAwait(false); + return resolved; + } + + internal async Task FinalizeReplacementAsync( + LocalAiInstallManifest expectedManifest, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(expectedManifest); + await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) + .ConfigureAwait(false); + LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); + if (!HasSameRuntimeAndModel(current, expectedManifest)) + { + throw new InvalidDataException( + "The Local AI installation changed before its model replacement could be finalized."); + } + if (expectedManifest.ReplacedManifest is null || current.ReplacedManifest is null || + !string.Equals(current.Endpoint, expectedManifest.Endpoint, StringComparison.Ordinal)) + { + throw new InvalidDataException( + "The Local AI model replacement changed before it could be finalized."); + } + + LocalAiInstallManifest finalized = current with + { + ReplacedManifest = null, + PreviousEndpoints = null, + }; + LocalAiResolvedInstall resolved = ResolveAndValidate(finalized); + await SaveWithoutLockAsync(finalized, cancellationToken).ConfigureAwait(false); + return resolved; + } + + internal async Task RestoreRecoveryManifestAsync( + LocalAiInstallManifest expectedManifest, + LocalAiInstallManifest recoveryManifest, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(expectedManifest); + ArgumentNullException.ThrowIfNull(recoveryManifest); + await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) + .ConfigureAwait(false); + LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); + if (!JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(current), + JsonSerializer.SerializeToElement(expectedManifest)) || + current.ReplacedManifest is null) + { + throw new InvalidDataException( + "The Local AI model replacement changed before its recovery receipt could be restored."); + } + + bool restoresOriginal = JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(current.ReplacedManifest), + JsonSerializer.SerializeToElement(recoveryManifest)); + bool endpointWasPublished = recoveryManifest.Endpoint is { } recoveryEndpoint && + (string.Equals(current.Endpoint, recoveryEndpoint, StringComparison.Ordinal) || + (current.PreviousEndpoints?.Contains(recoveryEndpoint, StringComparer.Ordinal) ?? false)); + bool restoresPendingRoute = endpointWasPublished && JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(current), + JsonSerializer.SerializeToElement(recoveryManifest with + { + Endpoint = current.Endpoint, + PreviousEndpoints = current.PreviousEndpoints, + })); + if (!restoresOriginal && !restoresPendingRoute) + { + throw new InvalidDataException( + "The requested Local AI recovery receipt is not an authorized replacement route."); + } + + LocalAiInstallManifest restored = restoresPendingRoute + ? current with { Endpoint = recoveryManifest.Endpoint } + : recoveryManifest; + LocalAiResolvedInstall resolved = ResolveAndValidate(restored); + await SaveWithoutLockAsync(restored, cancellationToken).ConfigureAwait(false); + return resolved; + } + + internal async Task RestoreManifestIfUnchangedAsync( + LocalAiInstallManifest expectedManifest, + LocalAiInstallManifest recoveryManifest, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(expectedManifest); + ArgumentNullException.ThrowIfNull(recoveryManifest); + await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken) + .ConfigureAwait(false); + LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false); + if (!JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(current), + JsonSerializer.SerializeToElement(expectedManifest))) + { + throw new InvalidDataException( + "The Local AI installation changed before its recovery receipt could be restored."); + } + + LocalAiResolvedInstall resolved = ResolveAndValidate(recoveryManifest); + await SaveWithoutLockAsync(recoveryManifest, cancellationToken).ConfigureAwait(false); + return resolved; + } + + private static bool HasSameRuntimeAndModel( + LocalAiInstallManifest current, + LocalAiInstallManifest expected) => + string.Equals(current.RuntimeId, expected.RuntimeId, StringComparison.Ordinal) && + string.Equals(current.SelectedGpuId, expected.SelectedGpuId, StringComparison.Ordinal) && + string.Equals(current.ModelCatalogId, expected.ModelCatalogId, StringComparison.Ordinal) && + string.Equals(current.ModelPath, expected.ModelPath, StringComparison.OrdinalIgnoreCase) && + string.Equals(current.CachedModelPath, expected.CachedModelPath, StringComparison.OrdinalIgnoreCase); + private async Task SaveWithoutLockAsync( LocalAiInstallManifest manifest, CancellationToken cancellationToken) @@ -632,30 +793,66 @@ LocalAiInstallManifest.HubCacheReceiptSchemaVersion or LocalAiPortPolicy.Validate(manifest.RequestedPort); LocalAiGatewayModelPolicy.ValidateFallbackModel(manifest.GatewayFallbackModel); - Uri? endpoint = null; - if (manifest.Endpoint is not null) - { - if (!Uri.TryCreate(manifest.Endpoint, UriKind.Absolute, out endpoint) || - endpoint.Scheme != Uri.UriSchemeHttp || - !string.Equals(endpoint.Host, "127.0.0.1", StringComparison.Ordinal) || - endpoint.IsDefaultPort || - endpoint.Port is <= 0 or > 65535 || - endpoint.Port == 80 || - !string.IsNullOrEmpty(endpoint.UserInfo) || - !string.IsNullOrEmpty(endpoint.Query) || - !string.IsNullOrEmpty(endpoint.Fragment) || - !string.Equals(endpoint.AbsolutePath, "/v1", StringComparison.Ordinal)) + if (manifest.ReplacedManifest is { } replaced) + { + if (replaced.ReplacedManifest is not null || + string.Equals(replaced.ModelCatalogId, manifest.ModelCatalogId, StringComparison.Ordinal) || + !string.Equals(replaced.Engine, manifest.Engine, StringComparison.Ordinal) || + !string.Equals(replaced.EngineVersion, manifest.EngineVersion, StringComparison.Ordinal) || + !string.Equals(replaced.Architecture, manifest.Architecture, StringComparison.Ordinal) || + !string.Equals(replaced.RuntimeId, manifest.RuntimeId, StringComparison.Ordinal) || + !string.Equals(replaced.ExecutablePath, manifest.ExecutablePath, StringComparison.Ordinal) || + !replaced.RuntimeAssets.SequenceEqual(manifest.RuntimeAssets) || + replaced.RequestedPort != manifest.RequestedPort) { - throw new InvalidDataException("The local AI endpoint must be an HTTP IPv4 loopback /v1 address with an explicit non-reserved port."); + throw new InvalidDataException("The local AI model replacement receipt is invalid."); } + _ = ResolveAndValidate(replaced); + } + else if (manifest.PreviousEndpoints is not null) + { + throw new InvalidDataException("Previous Local AI endpoints require a pending model replacement."); + } - if (manifest.RequestedPort != LocalAiPortPolicy.Automatic && endpoint.Port != manifest.RequestedPort) - throw new InvalidDataException("The verified Local AI endpoint does not match its requested fixed port."); + Uri? endpoint = ValidateEndpoint(manifest.Endpoint, manifest.RequestedPort); + ImmutableArray endpointHistory = manifest.PreviousEndpoints.GetValueOrDefault(); + if (endpointHistory.IsDefault) + endpointHistory = []; + HashSet previousEndpoints = endpointHistory.ToHashSet(StringComparer.Ordinal); + if (previousEndpoints.Count != endpointHistory.Length) + { + throw new InvalidDataException("Previous Local AI endpoints must be unique."); + } + foreach (string previousEndpoint in previousEndpoints) + { + if (string.IsNullOrWhiteSpace(previousEndpoint)) + throw new InvalidDataException("Previous Local AI endpoints must be non-empty endpoint strings."); + _ = ValidateEndpoint(previousEndpoint, manifest.RequestedPort); } return new LocalAiResolvedInstall(manifest, executable, model, endpoint); } + private static Uri? ValidateEndpoint(string? value, int requestedPort) + { + if (value is null) + return null; + if (!Uri.TryCreate(value, UriKind.Absolute, out Uri? endpoint) || + endpoint.Scheme != Uri.UriSchemeHttp || + !string.Equals(endpoint.Host, "127.0.0.1", StringComparison.Ordinal) || + endpoint.IsDefaultPort || endpoint.Port is <= 0 or > 65535 || endpoint.Port == 80 || + !string.IsNullOrEmpty(endpoint.UserInfo) || + !string.IsNullOrEmpty(endpoint.Query) || + !string.IsNullOrEmpty(endpoint.Fragment) || + !string.Equals(endpoint.AbsolutePath, "/v1", StringComparison.Ordinal)) + { + throw new InvalidDataException("The local AI endpoint must be an HTTP IPv4 loopback /v1 address with an explicit non-reserved port."); + } + if (requestedPort != LocalAiPortPolicy.Automatic && endpoint.Port != requestedPort) + throw new InvalidDataException("The verified Local AI endpoint does not match its requested fixed port."); + return endpoint; + } + private static void ValidateModelPath( string path, LocalAiAssetReceipt asset, diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs b/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs index 726c86f33..747d3af95 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs @@ -146,5 +146,35 @@ Task ReconcileStoppedAsync(CancellationToken cancellatio => Task.FromResult(Snapshot); Task StopAsync(CancellationToken cancellationToken = default); Task RestartAsync(CancellationToken cancellationToken = default); + /// + /// Stops the managed process for a setup transaction without changing Gateway routing. + /// Implementations that publish endpoint lifecycle changes must suppress them here because + /// the setup pipeline coordinates the matching Gateway transaction separately. + /// + Task StopForSetupAsync(CancellationToken cancellationToken = default) => + StopAsync(cancellationToken); + /// + /// Restarts the managed process for a setup transaction without changing Gateway routing. + /// Implementations that publish endpoint lifecycle changes must suppress them here because + /// the setup pipeline coordinates the matching Gateway transaction separately. + /// + Task RestartForSetupAsync(CancellationToken cancellationToken = default) => + RestartAsync(cancellationToken); + /// + /// Adopts setup's restored receipt, then restarts through the ordinary Gateway lifecycle. + /// Use only before setup has begun its own Gateway configuration transaction. + /// + Task RestartForSetupRollbackAsync( + CancellationToken cancellationToken = default) => RestartAsync(cancellationToken); + /// + /// Acknowledges that setup committed or compensated the Gateway route for the current endpoint. + /// + Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) => Task.FromResult(Snapshot); + /// + /// Returns endpoint lifecycle ownership to the runtime without claiming Gateway reconciliation. + /// + Task ReleaseSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) => Task.FromResult(Snapshot); Task RefreshAsync(CancellationToken cancellationToken = default); } diff --git a/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs b/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs index 21e04abff..63d6b91eb 100644 --- a/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs +++ b/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs @@ -149,6 +149,13 @@ private async Task StartPipelineAsync() var steps = BuildSteps(config, _localAiRecoveryOnly); var setupOwner = _window; ctx.ExpectedGatewayRegistry = config.NativeLocalAiAcquisition ? null : setupOwner?.BeginGatewaySetup(); + ctx.LocalAiRuntime = setupOwner?.BorrowManagedLocalAiRuntime(); + ctx.LocalAiRuntimeBorrowed = ctx.LocalAiRuntime is not null; + if (ctx.LocalAiRuntimeBorrowed && !config.RollbackOnFailure) + { + throw new InvalidOperationException( + "Local AI recovery requires transactional rollback when borrowing the tray runtime."); + } ctx.PersistTraySettings = _window is { } settingsOwner ? settingsOwner.PersistPipelineSettings : null; _pipeline = new SetupPipeline(steps); _pipeline.StepProgress += OnStepProgress; @@ -156,9 +163,24 @@ private async Task StartPipelineAsync() var pipeline = _pipeline; var result = await SetupPipeline.RunWithSettlementAsync( () => Task.Run(() => pipeline.RunAsync(ctx), cts.Token), - outcome => config.NativeLocalAiAcquisition ? Task.CompletedTask : setupOwner?.SettleGatewaySetupAsync(ctx.ExpectedGatewayRegistry, - outcome?.Outcome == PipelineOutcome.Success ? config.LocalAiRecoveryGatewayId ?? ctx.GatewayRecordId : null) - ?? Task.CompletedTask); + async outcome => + { + try + { + await SetupPipeline.ReleaseBorrowedLocalAiRuntimeAfterFailureAsync(ctx, outcome); + } + finally + { + if (!config.NativeLocalAiAcquisition && setupOwner is not null) + { + await setupOwner.SettleGatewaySetupAsync( + ctx.ExpectedGatewayRegistry, + outcome?.Outcome == PipelineOutcome.Success + ? config.LocalAiRecoveryGatewayId ?? ctx.GatewayRecordId + : null); + } + } + }); sw.Stop(); _pipelineFinished = true; if (_closed || _window?.IsClosed == true) diff --git a/src/OpenClaw.SetupEngine.UI/SetupWindow.xaml.cs b/src/OpenClaw.SetupEngine.UI/SetupWindow.xaml.cs index 515a8cee7..fe375d3cb 100644 --- a/src/OpenClaw.SetupEngine.UI/SetupWindow.xaml.cs +++ b/src/OpenClaw.SetupEngine.UI/SetupWindow.xaml.cs @@ -490,6 +490,9 @@ public void SetWelcomeInstallSelected(bool installSelected) internal Task GetLocalAiHardwareAsync(bool forceRefresh = false) => _localAiHardwareProbe.GetAsync(forceRefresh); + internal ILocalAiRuntime? BorrowManagedLocalAiRuntime() => + _startAtLocalAiRecoveryReview ? _localAiHost?.BorrowManagedRuntime() : null; + internal Task GetWslViabilityAsync(bool refresh = false) => _wslViabilityProbe.GetAsync(refresh); diff --git a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs index 75149de04..5fa83a378 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGatewayConfiguration.cs @@ -69,6 +69,7 @@ public static string BuildRecoveryRestoreBatchJson( ]; return JsonSerializer.Serialize(operations); } + } public sealed class ConfigureLocalAiGatewayStep : SetupStep @@ -78,6 +79,9 @@ public sealed class ConfigureLocalAiGatewayStep : SetupStep private const string MissingValue = "MISSING"; private const string FailedValuePrefix = "FAILED:"; private const string BatchVariable = "OPENCLAW_LOCAL_AI_BATCH_B64"; + private const string ProviderVariable = "OPENCLAW_LOCAL_AI_PROVIDER_B64"; + private const string ExpectedProviderVariable = "OPENCLAW_LOCAL_AI_EXPECTED_PROVIDER_B64"; + private const string ConditionalSetUnsupportedMarker = "LOCAL_AI_CONDITIONAL_SET_UNSUPPORTED"; private const int MaximumSnapshotBytes = 1024 * 1024; public override string Id => "configure-local-ai-gateway"; @@ -113,38 +117,68 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati bool retainedManagedPrimary = !prior.ProviderExisted && prior.PrimaryModelExisted && JsonEquals(prior.PrimaryModelJson!, expectedPrimary); + LocalAiResolvedInstall? recoveryInstall = ctx.LocalAiRecoveryOriginalInstall; + bool retainedRecoveryPrimary = !retainedManagedPrimary && + !prior.ProviderExisted && + prior.PrimaryModelExisted && + recoveryInstall is not null && + JsonEquals( + prior.PrimaryModelJson!, + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(recoveryInstall))); string? fallbackModel; bool recoveryProviderTransition = false; if (prior.ProviderExisted) { bool matchesCurrentInstall = install.Endpoint is not null && LocalAiGatewayProviderDefinition.MatchesProviderJson(prior.ProviderJson!, install); + LocalAiResolvedInstall? pendingRoute = MatchPendingRoute( + ctx.LocalAiRecoveryPendingInstall, + prior.ProviderJson!); + bool matchesPendingInstall = !matchesCurrentInstall && + pendingRoute is not null && prior.PrimaryModelExisted && + JsonEquals(prior.PrimaryModelJson!, expectedPrimary); bool matchesRecoveryInstall = false; - if (!matchesCurrentInstall && + if (!matchesCurrentInstall && !matchesPendingInstall && ctx.LocalAiRecoveryOriginalInstall is { Endpoint: not null } originalInstall) { string originalPrimary = JsonSerializer.Serialize( LocalAiGatewayProviderDefinition.BuildPrimaryModel(originalInstall)); matchesRecoveryInstall = + prior.PrimaryModelExisted && LocalAiGatewayProviderDefinition.MatchesProviderJson( prior.ProviderJson!, originalInstall) && - JsonEquals(originalPrimary, expectedPrimary); + JsonEquals(prior.PrimaryModelJson!, originalPrimary); } - if ((!matchesCurrentInstall && !matchesRecoveryInstall) || + if ((!matchesCurrentInstall && !matchesPendingInstall && !matchesRecoveryInstall) || !prior.PrimaryModelExisted || - !JsonEquals(prior.PrimaryModelJson!, expectedPrimary)) + (matchesCurrentInstall && !JsonEquals(prior.PrimaryModelJson!, expectedPrimary))) { return StepResult.Fail( "The existing llamacpp gateway route is not the exact companion-managed configuration; preserving it."); } - recoveryProviderTransition = matchesRecoveryInstall; + if (matchesCurrentInstall && install.Manifest.ReplacedManifest is not null && + !ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun) + { + // A previous process already published the replacement. Rollback must + // preserve that live route instead of reconstructing the older one. + ctx.LocalAiRecoveryOriginalInstall = null; + ctx.LocalAiRecoveryProviderTransition = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + } + else if (matchesPendingInstall) + { + ctx.LocalAiRecoveryOriginalInstall = pendingRoute; + } + recoveryProviderTransition = matchesRecoveryInstall || matchesPendingInstall; fallbackModel = install.Manifest.GatewayFallbackModel; } - else if (retainedManagedPrimary) + else if (retainedManagedPrimary || retainedRecoveryPrimary) { - fallbackModel = install.Manifest.GatewayFallbackModel; + fallbackModel = retainedRecoveryPrimary + ? recoveryInstall!.Manifest.GatewayFallbackModel + : install.Manifest.GatewayFallbackModel; } else if (prior.PrimaryModelExisted) { @@ -200,6 +234,12 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati } string batchJson = LocalAiGatewayConfigBuilder.BuildBatchJson(ctx); + if (ctx.LocalAiRecoveryProviderTransition) + { + ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = true; + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + } CommandResult result = await ApplyBatchAsync(ctx, batchJson, "LOCAL_AI_GATEWAY_CONFIGURED", ct); if (result.ExitCode != 0 || result.TimedOut || !result.Stdout.Contains("LOCAL_AI_GATEWAY_CONFIGURED", StringComparison.Ordinal)) @@ -209,6 +249,25 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati : $"Local AI gateway configuration failed (exit {result.ExitCode})."); } + if (ctx.LocalAiRuntimeBorrowed && ctx.LocalAiRuntime is { } borrowedRuntime) + { + try + { + await borrowedRuntime.AcknowledgeSetupGatewayRouteAsync(ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + await RollbackAsync(ctx, CancellationToken.None).ConfigureAwait(false); + throw; + } + catch (Exception ex) + { + await RollbackAsync(ctx, CancellationToken.None).ConfigureAwait(false); + return StepResult.Fail( + "The Local AI runtime could not acknowledge the committed Gateway route.", ex); + } + } + return StepResult.Ok("Gateway configured to use the managed llama-server provider"); } @@ -262,6 +321,7 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) JsonEquals(current.PrimaryModelJson!, prior.PrimaryModelJson!))) { ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; return; } @@ -333,7 +393,165 @@ await ReconcileFailedRecoveryRestoreAsync( } } if (recoveryOriginal is not null) + { ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; + } + } + + internal static async Task RestoreRecoveryRouteAsync( + SetupContext ctx, + LocalAiGatewayPriorState prior, + LocalAiResolvedInstall expectedInstall, + LocalAiResolvedInstall restoredInstall, + CancellationToken ct) + { + if (!prior.ProviderExisted) + return true; + + CommandResult currentResult = await CaptureStateAsync(ctx, ct).ConfigureAwait(false); + if (currentResult.ExitCode != 0 || currentResult.TimedOut) + return false; + LocalAiGatewayPriorState current; + try + { + current = ParseSnapshot(currentResult.Stdout); + } + catch (Exception ex) when (ex is FormatException or JsonException or InvalidDataException) + { + return false; + } + if (ProviderRoutesToEndpoint(current.ProviderJson!, restoredInstall.Endpoint!)) + return true; + if (!LocalAiGatewayProviderDefinition.MatchesProviderJson( + prior.ProviderJson!, + expectedInstall)) + { + return false; + } + // The runtime restart can move an automatic endpoint and take long enough for another + // actor to update Gateway. Only rewrite the restored route while we still own the exact + // rollback state that was captured before setup. + if (current.ProviderExisted != prior.ProviderExisted || + !JsonEquals(current.ProviderJson!, prior.ProviderJson!) || + current.PrimaryModelExisted != prior.PrimaryModelExisted || + (current.PrimaryModelExisted && + !JsonEquals(current.PrimaryModelJson!, prior.PrimaryModelJson!))) + { + return false; + } + + CommandResult restore = await ApplyConditionalProviderAsync( + ctx, + expectedInstall, + restoredInstall, + ct).ConfigureAwait(false); + if (restore.ExitCode == 42 && + restore.Stdout.Contains(ConditionalSetUnsupportedMarker, StringComparison.Ordinal)) + { + // Older protocol-v4 Gateways predate atomic conditional config writes. A separate + // read then write can overwrite a concurrent owner, so leave route resolution + // pending instead of attempting a lossy compatibility update. + return false; + } + if (restore.ExitCode != 0 || restore.TimedOut || + !restore.Stdout.Contains("LOCAL_AI_GATEWAY_RESTORED", StringComparison.Ordinal)) + { + return false; + } + + CommandResult verifiedResult = await CaptureStateAsync(ctx, ct).ConfigureAwait(false); + if (verifiedResult.ExitCode != 0 || verifiedResult.TimedOut) + return false; + try + { + LocalAiGatewayPriorState verified = ParseSnapshot(verifiedResult.Stdout); + bool matches = verified.ProviderExisted && + LocalAiGatewayProviderDefinition.MatchesProviderJson( + verified.ProviderJson!, + restoredInstall); + if (!matches) + return false; + // The conditional write changes only the owned provider. A primary-model update + // racing after the ownership snapshot is independent user state and is preserved. + return true; + } + catch (Exception ex) when (ex is FormatException or JsonException or InvalidDataException) + { + return false; + } + } + + private static bool ProviderRoutesToEndpoint(string providerJson, Uri endpoint) + { + try + { + using JsonDocument provider = JsonDocument.Parse(providerJson); + return provider.RootElement.TryGetProperty("baseUrl", out JsonElement baseUrl) && + baseUrl.ValueKind == JsonValueKind.String && + Uri.TryCreate(baseUrl.GetString(), UriKind.Absolute, out Uri? configured) && + string.Equals( + configured.AbsoluteUri.TrimEnd('/'), + endpoint.AbsoluteUri.TrimEnd('/'), + StringComparison.OrdinalIgnoreCase); + } + catch (JsonException) + { + return false; + } + } + + private static Task ApplyConditionalProviderAsync( + SetupContext ctx, + LocalAiResolvedInstall expectedInstall, + LocalAiResolvedInstall restoredInstall, + CancellationToken ct) + { + string script = $$""" + set -eu + {{ctx.WslPathPrefix}} + if ! openclaw config set --help | grep -Fq -- '--expect-current-json'; then + echo {{ConditionalSetUnsupportedMarker}} + exit 42 + fi + provider_json="$(printf '%s' "$OPENCLAW_LOCAL_AI_PROVIDER_B64" | base64 -d)" + expected_provider_json="$(printf '%s' "$OPENCLAW_LOCAL_AI_EXPECTED_PROVIDER_B64" | base64 -d)" + openclaw config set {{LocalAiGatewayConfigBuilder.ProviderPath}} "$provider_json" \ + --strict-json --replace --expect-current-json "$expected_provider_json" + echo LOCAL_AI_GATEWAY_RESTORED + """; + var environment = new Dictionary(StringComparer.Ordinal) + { + [ProviderVariable] = Convert.ToBase64String(Encoding.UTF8.GetBytes( + LocalAiGatewayProviderDefinition.BuildProviderJson(restoredInstall))), + [ExpectedProviderVariable] = Convert.ToBase64String(Encoding.UTF8.GetBytes( + LocalAiGatewayProviderDefinition.BuildProviderJson(expectedInstall))), + }; + return ctx.Commands.RunInWslAsync( + ctx.DistroName!, + script, + TimeSpan.FromMinutes(2), + environment, + ct, + ctx.Config.Wsl.User, + inputViaStdin: true); + } + + internal static async Task AcknowledgeBorrowedRuntimeRouteAsync( + SetupContext ctx, + CancellationToken ct) + { + if (!ctx.LocalAiRuntimeBorrowed || ctx.LocalAiRuntime is not { } borrowedRuntime) + return true; + try + { + await borrowedRuntime.AcknowledgeSetupGatewayRouteAsync(ct).ConfigureAwait(false); + return true; + } + catch (Exception ex) when (ex is InvalidOperationException or ObjectDisposedException) + { + return false; + } } private static async Task ReconcileFailedRecoveryRestoreAsync( @@ -376,6 +594,7 @@ private static async Task ReconcileFailedRecoveryRestoreAsync( if (originalRestored) { ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; return; } @@ -420,6 +639,44 @@ private static async Task RemoveManagedStateForUninstallAsync( LocalAiGatewayPriorState current = ParseSnapshot(currentResult.Stdout); if (!current.ProviderExisted && !current.PrimaryModelExisted) return; + + var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); + foreach (string previousEndpoint in install.Manifest.PreviousEndpoints ?? []) + { + LocalAiResolvedInstall previous = store.ResolveAndValidate(install.Manifest with + { + Endpoint = previousEndpoint, + }); + if (current.ProviderExisted && + LocalAiGatewayProviderDefinition.MatchesProviderJson(current.ProviderJson!, previous)) + { + install = previous; + break; + } + } + + if (install.Manifest.ReplacedManifest is { } replacedManifest) + { + LocalAiResolvedInstall replaced = store.ResolveAndValidate(replacedManifest); + bool previousProvider = current.ProviderExisted && + replaced.Endpoint is not null && + LocalAiGatewayProviderDefinition.MatchesProviderJson(current.ProviderJson!, replaced); + bool previousPrimary = current.PrimaryModelExisted && + JsonEquals( + current.PrimaryModelJson!, + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(replaced))); + if (previousProvider || previousPrimary) + { + if ((current.ProviderExisted && !previousProvider) || + (current.PrimaryModelExisted && !previousPrimary)) + { + throw new InvalidDataException( + "Local AI gateway settings contain a mixed model replacement route; preserving them."); + } + install = replaced; + } + } + if (install.Endpoint is null) { throw new InvalidDataException( @@ -605,6 +862,26 @@ private static string ExtractOperationValue(string batchJson, int index) return document.RootElement[index].GetProperty("value").GetRawText(); } + private static LocalAiResolvedInstall? MatchPendingRoute( + LocalAiResolvedInstall? pending, + string providerJson) + { + if (pending?.Endpoint is not null && + LocalAiGatewayProviderDefinition.MatchesProviderJson(providerJson, pending)) + return pending; + foreach (string endpoint in pending?.Manifest.PreviousEndpoints ?? []) + { + LocalAiResolvedInstall previous = pending! with + { + Manifest = pending.Manifest with { Endpoint = endpoint }, + Endpoint = new Uri(endpoint), + }; + if (LocalAiGatewayProviderDefinition.MatchesProviderJson(providerJson, previous)) + return previous; + } + return null; + } + private static bool JsonEquals(string left, string right) { using JsonDocument leftDocument = JsonDocument.Parse(left); diff --git a/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs b/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs index 6316da3d4..d4dc4bf5e 100644 --- a/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs +++ b/src/OpenClaw.SetupEngine/LocalAiGpuVerification.cs @@ -310,7 +310,7 @@ ctx.LocalAiInferenceVerification is null || } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - await VerifyLocalAiInferenceStep.ResetRouterAsync(runtime); + await VerifyLocalAiInferenceStep.ResetRouterAsync(ctx, runtime); throw; } catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) @@ -318,7 +318,7 @@ ctx.LocalAiInferenceVerification is null || failure = ex; } - LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync(runtime); + LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync(ctx, runtime); if (failure is not null) return StepResult.Fail($"Local AI GPU verification failed: {failure.Message}", failure); if (reset.State != LocalAiRuntimeState.Healthy || diff --git a/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs b/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs index 44bed86c9..81071d4b6 100644 --- a/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs +++ b/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs @@ -10,7 +10,8 @@ internal sealed record LocalAiReconcileResult( LlamaRuntimeInstallResult? RuntimeInstall, HuggingFaceModelInstallResult? ModelInstall, LocalAiResolvedInstall? OriginalInstall = null, - ImmutableArray? AdditionalModelInstalls = null) + ImmutableArray? AdditionalModelInstalls = null, + LocalAiResolvedInstall? PendingReplacement = null) { public static LocalAiReconcileResult NotInstalled { get; } = new(false, null, null, null); @@ -157,8 +158,44 @@ public async Task ReconcileAsync( .ConfigureAwait(false); if (install is null) return LocalAiReconcileResult.NotInstalled; - LocalAiResolvedInstall originalInstall = install; - bool runtimeUpgradePending = ValidateRecipeMatch(install, plan, selectedGpuId, localDataDirectory); + LocalAiResolvedInstall originalInstall = install.Manifest.ReplacedManifest is { } replacedManifest + ? manifestStore.ResolveAndValidate(replacedManifest) + : install; + LocalAiResolvedInstall? pendingReplacement = install.Manifest.ReplacedManifest is null ? null : install; + bool replacingModel = !string.Equals( + install.Manifest.ModelCatalogId, + plan.Model.Id, + StringComparison.Ordinal); + if (install.Manifest.ReplacedManifest is not null && replacingModel) + { + throw new InvalidDataException( + "Complete the pending Local AI model replacement before selecting another model."); + } + if (replacingModel) + { + if (!allowIncompleteInstallation) + { + throw new InvalidDataException( + "The existing managed Local AI installation does not match the selected runtime, GPU, and model recipe."); + } + + ValidateReplacementSource(install, plan, selectedGpuId, localDataDirectory); + LlamaRuntimeInspection replacementRuntime = await _runtimeInspector + .InspectAsync(Path.GetDirectoryName(install.ExecutablePath)!, cancellationToken) + .ConfigureAwait(false); + return new LocalAiReconcileResult( + Reused: false, + ResolvedInstall: null, + RuntimeInstall: replacementRuntime.IsValid ? CreateRuntimeInstall(install) : null, + ModelInstall: null, + OriginalInstall: originalInstall, + PendingReplacement: pendingReplacement); + } + bool runtimeUpgradePending = ValidateRecipeMatch( + install, + plan, + selectedGpuId, + localDataDirectory); bool migrateLegacyGpuId = !string.Equals(install.Manifest.SelectedGpuId, selectedGpuId, StringComparison.Ordinal) && @@ -208,7 +245,8 @@ public async Task ReconcileAsync( RuntimeInstall: null, ModelInstall: modelIsValid ? CreateModelInstall(install, localDataDirectory) : null, OriginalInstall: originalInstall, - AdditionalModelInstalls: modelIsValid ? CreateAdditionalModelInstalls(install) : null); + AdditionalModelInstalls: modelIsValid ? CreateAdditionalModelInstalls(install) : null, + PendingReplacement: pendingReplacement); } if (!inspection.IsValid || !modelIsValid) @@ -239,7 +277,8 @@ public async Task ReconcileAsync( RuntimeInstall: inspection.IsValid ? CreateRuntimeInstall(install) : null, ModelInstall: modelIsValid ? CreateModelInstall(install, localDataDirectory) : null, OriginalInstall: originalInstall, - AdditionalModelInstalls: modelIsValid ? CreateAdditionalModelInstalls(install) : null); + AdditionalModelInstalls: modelIsValid ? CreateAdditionalModelInstalls(install) : null, + PendingReplacement: pendingReplacement); } install = await MigrateLegacyModelAsync( @@ -268,7 +307,43 @@ public async Task ReconcileAsync( CreateRuntimeInstall(install), CreateModelInstall(install, localDataDirectory), OriginalInstall: allowIncompleteInstallation ? originalInstall : null, - AdditionalModelInstalls: CreateAdditionalModelInstalls(install)); + AdditionalModelInstalls: CreateAdditionalModelInstalls(install), + PendingReplacement: pendingReplacement); + } + + private static void ValidateReplacementSource( + LocalAiResolvedInstall install, + LocalInferencePlan plan, + string selectedGpuId, + string localDataDirectory) + { + // Validate the existing receipt against its own catalog model before using it + // as durable rollback provenance for the newly selected model. + _ = LlamaServerRouterConfiguration.Build(new LocalAiPaths(localDataDirectory), install); + + string expectedArchitecture = plan.Runtime.Architecture switch + { + System.Runtime.InteropServices.Architecture.X64 => "x64", + System.Runtime.InteropServices.Architecture.Arm64 => "arm64", + _ => throw new InvalidDataException("The selected Local AI runtime architecture is unsupported."), + }; + LocalAiComponentIdentity component = LlamaRuntimeInstaller.Component(plan.Runtime); + if (!string.Equals(install.Manifest.RuntimeId, plan.Runtime.Id, StringComparison.Ordinal) || + !string.Equals(install.Manifest.Architecture, expectedArchitecture, StringComparison.Ordinal) || + !GpuIdsMatch(install.Manifest.SelectedGpuId, selectedGpuId) || + !LocalAiPathPolicy.TryResolve( + localDataDirectory, + component, + out LocalAiSetupPaths setupPaths, + out _) || + !string.Equals( + Path.GetDirectoryName(install.ExecutablePath), + setupPaths.InstallDirectory, + StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidDataException( + "The existing managed Local AI installation cannot reuse the selected runtime and GPU."); + } } /// Read-only inspection for onboarding; unlike reconciliation, never migrates or saves receipts. diff --git a/src/OpenClaw.SetupEngine/LocalAiOnboarding.cs b/src/OpenClaw.SetupEngine/LocalAiOnboarding.cs index fec5cffd3..1449ae244 100644 --- a/src/OpenClaw.SetupEngine/LocalAiOnboarding.cs +++ b/src/OpenClaw.SetupEngine/LocalAiOnboarding.cs @@ -217,6 +217,11 @@ public static Task InspectAsync(LocalAiResolvedInstall install, Cancellati /// Observation never calls a runtime refresh (which may publish or withdraw a route). public interface ISetupLocalAiHost { + /// + /// Borrows the tray-owned runtime for a recovery transaction. The setup pipeline may restart + /// this runtime, but its lifetime remains owned by the tray. + /// + ILocalAiRuntime? BorrowManagedRuntime() => null; OpenClaw.Connection.GatewayRegistrySnapshot BeginGatewaySetup(); Task ReconcileGatewaySetupAsync(OpenClaw.Connection.GatewayRegistrySnapshot expectedOutput, string? completedGatewayId); Task ObserveAsync(CancellationToken ct); diff --git a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs index 50b903bdb..71b5b209d 100644 --- a/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs +++ b/src/OpenClaw.SetupEngine/LocalAiRecoveryPolicy.cs @@ -163,10 +163,92 @@ private static IReadOnlyList LoadGatewayRecords(string dataDir) } } +public sealed class ValidateLocalAiRecoveryGatewayCompatibilityStep : SetupStep +{ + internal const string SupportedMarker = "LOCAL_AI_CONDITIONAL_SET_SUPPORTED"; + internal const string UnsupportedMarker = "LOCAL_AI_CONDITIONAL_SET_UNSUPPORTED"; + + private readonly Func> _probe; + + public ValidateLocalAiRecoveryGatewayCompatibilityStep() + : this(ProbeConditionalSetSupportAsync) + { + } + + internal ValidateLocalAiRecoveryGatewayCompatibilityStep( + Func> probe) => + _probe = probe ?? throw new ArgumentNullException(nameof(probe)); + + public override string Id => "validate-local-ai-recovery-gateway-compatibility"; + public override string DisplayName => "Check gateway recovery compatibility"; + public override bool CanRetry => false; + + public override async Task ExecuteAsync(SetupContext ctx, CancellationToken ct) + { + LocalAiResolvedInstall?[] recoveryInstalls = + [ + ctx.LocalAiRecoveryPendingInstall, + ctx.LocalAiRecoveryOriginalInstall, + ctx.LocalAiResolvedInstall, + ]; + if (recoveryInstalls.All(install => install is null)) + { + return StepResult.Skip( + "No prior Local AI route requires conditional recovery support."); + } + if (recoveryInstalls.All(install => install is null || install.Manifest.RequestedPort != 0)) + return StepResult.Skip("Fixed-port Local AI recovery does not require conditional route updates."); + + CommandResult result = await _probe(ctx, ct).ConfigureAwait(false); + if (result.ExitCode == 0 && + result.Stdout.Contains(SupportedMarker, StringComparison.Ordinal)) + { + return StepResult.Ok("Gateway supports safe automatic-port recovery."); + } + if (result.ExitCode == 42 && + result.Stdout.Contains(UnsupportedMarker, StringComparison.Ordinal)) + { + return StepResult.Terminal( + "This Gateway version cannot safely recover Local AI with an automatic port. Update the Gateway, then retry recovery."); + } + return StepResult.Fail("OpenClaw could not verify Gateway support for safe automatic-port recovery."); + } + + private static Task ProbeConditionalSetSupportAsync( + SetupContext ctx, + CancellationToken ct) + { + string script = $$""" + set -eu + {{ctx.WslPathPrefix}} + if openclaw config set --help | grep -Fq -- '--expect-current-json'; then + echo {{SupportedMarker}} + exit 0 + fi + echo {{UnsupportedMarker}} + exit 42 + """; + return ctx.Commands.RunInWslAsync( + ctx.DistroName!, + script, + TimeSpan.FromMinutes(1), + ct: ct, + user: ctx.Config.Wsl.User, + inputViaStdin: true); + } +} + public sealed class PreserveLocalAiRecoveryGatewayStep : SetupStep { private readonly Func> _restart; private readonly Func> _probeOriginalEndpoint; + private readonly Func< + SetupContext, + LocalAiGatewayPriorState, + LocalAiResolvedInstall, + LocalAiResolvedInstall, + CancellationToken, + Task> _restoreRecoveryRoute; public PreserveLocalAiRecoveryGatewayStep() : this(StartGatewayStep.RestartAndWaitForHealthAsync, ProbeOriginalEndpointAsync) @@ -175,10 +257,18 @@ public PreserveLocalAiRecoveryGatewayStep() internal PreserveLocalAiRecoveryGatewayStep( Func> restart, - Func>? probeOriginalEndpoint = null) + Func>? probeOriginalEndpoint = null, + Func< + SetupContext, + LocalAiGatewayPriorState, + LocalAiResolvedInstall, + LocalAiResolvedInstall, + CancellationToken, + Task>? restoreRecoveryRoute = null) { _restart = restart ?? throw new ArgumentNullException(nameof(restart)); _probeOriginalEndpoint = probeOriginalEndpoint ?? ProbeOriginalEndpointAsync; + _restoreRecoveryRoute = restoreRecoveryRoute ?? ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync; } public override string Id => "preserve-local-ai-recovery-gateway"; @@ -203,30 +293,118 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) ctx.Logger.Warn( "The previous Local AI endpoint receipt was not restored because gateway provider rollback did not complete."); } + else if (ctx.LocalAiRuntimeBorrowed && ctx.LocalAiRuntime is { } borrowedRuntime) + { + try + { + var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); + if (ctx.LocalAiResolvedInstall!.Manifest.ReplacedManifest is not null) + { + ctx.LocalAiResolvedInstall = await store + .RestoreRecoveryManifestAsync( + ctx.LocalAiResolvedInstall.Manifest, + originalInstall.Manifest, + ct) + .ConfigureAwait(false); + } + else + { + ctx.LocalAiResolvedInstall = await store + .RestoreManifestIfUnchangedAsync( + ctx.LocalAiResolvedInstall.Manifest, + originalInstall.Manifest, + ct) + .ConfigureAwait(false); + } + bool runtimeOwnsGatewayRoute = ctx.LocalAiGatewayPriorState is null; + LocalAiRuntimeSnapshot restored = runtimeOwnsGatewayRoute + ? await borrowedRuntime.RestartForSetupRollbackAsync(ct).ConfigureAwait(false) + : await borrowedRuntime.RestartForSetupAsync(ct).ConfigureAwait(false); + LocalAiResolvedInstall restoredInstall = await store.LoadAsync(ct).ConfigureAwait(false) + ?? throw new InvalidDataException( + "The previous Local AI receipt was unavailable after restarting its runtime."); + if (restored.State != LocalAiRuntimeState.Healthy || + restored.Ownership != LocalAiOwnership.CompanionManaged || + restored.ModelId != restoredInstall.Manifest.ModelCatalogId || + restored.Endpoint != restoredInstall.Endpoint || + restored.ModelEvidence.State is not + (LocalAiModelAvailabilityState.Verified or LocalAiModelAvailabilityState.Loaded)) + { + throw new InvalidDataException( + restored.Detail ?? "The previous Local AI runtime could not be restored."); + } + ctx.LocalAiResolvedInstall = restoredInstall; + AcquireLocalAiRuntimeStep.TransferCleanupOwnershipToRestoredRuntime( + ctx, + restoredInstall); + if (ctx.LocalAiGatewayPriorState is { } prior && + !await _restoreRecoveryRoute( + ctx, + prior, + originalInstall, + restoredInstall, + ct) + .ConfigureAwait(false)) + { + throw new InvalidDataException( + "The previous Local AI gateway route could not be updated to its restored endpoint."); + } + if (!await _probeOriginalEndpoint(restoredInstall, ct).ConfigureAwait(false)) + { + throw new InvalidDataException( + "The previous Local AI endpoint was not healthy after its runtime was restored."); + } + if (!runtimeOwnsGatewayRoute && + !await ConfigureLocalAiGatewayStep + .AcknowledgeBorrowedRuntimeRouteAsync(ctx, ct) + .ConfigureAwait(false)) + { + throw new InvalidDataException( + "The restored Local AI route could not be acknowledged by its runtime owner."); + } + ctx.LocalAiBorrowedRuntimeRestored = true; + CompleteReceiptRollback(ctx); + } + catch (Exception ex) when ( + ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + receiptError = ex; + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + ctx.Logger.Warn( + $"Restoring the previous Local AI runtime failed ({ex.GetType().Name})."); + } + } else if (!await _probeOriginalEndpoint(originalInstall, ct).ConfigureAwait(false)) { ctx.Logger.Warn( "The previous Local AI endpoint could not be verified as healthy; preserving the replacement " + "receipt instead of restoring a receipt for an endpoint that is not confirmed reachable."); + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; } else { try { var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); - await store.SaveAsync(originalInstall.Manifest, ct).ConfigureAwait(false); - ctx.LocalAiResolvedInstall = store.ResolveAndValidate(originalInstall.Manifest); + ctx.LocalAiResolvedInstall = await store + .RestoreRecoveryManifestAsync( + ctx.LocalAiResolvedInstall!.Manifest, + originalInstall.Manifest, + ct) + .ConfigureAwait(false); + CompleteReceiptRollback(ctx); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) { receiptError = ex; + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; ctx.Logger.Warn( $"Restoring the previous Local AI endpoint receipt failed ({ex.GetType().Name})."); } } - ctx.LocalAiRecoveryProviderTransition = false; - ctx.LocalAiRecoveryReceiptRollbackAllowed = false; - ctx.LocalAiGatewayPriorState = null; } if (ctx.LocalAiRecoveryStoppedWsl) @@ -246,6 +424,19 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) } } + private static void CompleteReceiptRollback(SetupContext ctx) + { + // The recovery guard now owns the settled receipt. Retire the earlier upgrade and + // recovery baselines so later reverse rollback steps cannot restore them again. + ctx.LocalAiUpgradeOriginalInstall = null; + ctx.LocalAiRecoveryOriginalInstall = null; + ctx.LocalAiRecoveryProviderTransition = false; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + ctx.LocalAiRecoveryRollbackUncertain = false; + ctx.LocalAiGatewayPriorState = null; + ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = false; + } + /// /// Confirms the original (pre-recovery) llama-server endpoint is actually alive before the /// Gateway is pointed back at it. A stale manifest receipt alone cannot tell us whether the diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index e83cb8262..5aff8c53b 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -3,6 +3,7 @@ using System.Text; using System.Text.Json; using OpenClaw.Connection.LocalAi; +using OpenClaw.Shared.IO; using OpenClaw.Shared.Inference; using OpenClaw.Shared.Inference.Catalog; @@ -283,7 +284,13 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati (result.OriginalInstall ?? result.ResolvedInstall) is { } originalInstall) { ctx.LocalAiRecoveryOriginalInstall = originalInstall; - ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryPendingInstall = result.PendingReplacement; + bool pendingReplacement = result.PendingReplacement is not null; + ctx.LocalAiRecoveryProviderTransition = pendingReplacement; + ctx.LocalAiRecoveryReceiptRollbackAllowed = !pendingReplacement; + // A recovered replacement may already be the active Gateway route. Only the + // process that created a fresh replacement knows it has not published it yet. + ctx.LocalAiRecoveryRollbackUncertain = pendingReplacement; } if (!result.Reused) { @@ -292,6 +299,9 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati if (ctx.LocalAiRecoveryOriginalInstall is null && result.OriginalInstall is { } retainedReceipt) ctx.LocalAiUpgradeOriginalInstall ??= retainedReceipt; + else if (result.PendingReplacement is { } pendingInstall && + result.RuntimeInstall is null) + ctx.LocalAiUpgradeOriginalInstall ??= pendingInstall; ctx.LocalAiRuntimeInstall = result.RuntimeInstall; ctx.LocalAiModelInstall = result.ModelInstall; ctx.LocalAiAdditionalModelInstalls = result.AdditionalModelInstalls @@ -424,6 +434,8 @@ or UnauthorizedAccessException public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) { ct.ThrowIfCancellationRequested(); + if (!ctx.LocalAiRecoveryCleanupAllowed) + return Task.CompletedTask; if (ctx.LocalAiRuntimeInstall is { } install) { _acquirer.RemoveInstalledRuntime(ctx.LocalDataDir, install); @@ -432,6 +444,21 @@ public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) return Task.CompletedTask; } + + internal static void TransferCleanupOwnershipToRestoredRuntime( + SetupContext ctx, + LocalAiResolvedInstall restoredInstall) + { + if (ctx.LocalAiRuntimeInstall is not { CreatedThisRun: true } runtimeInstall) + return; + + string restoredDirectory = WindowsPathSafety.NormalizePath( + Path.GetDirectoryName(restoredInstall.ExecutablePath) + ?? throw new InvalidDataException("The restored Local AI executable path has no directory.")); + string acquiredDirectory = WindowsPathSafety.NormalizePath(runtimeInstall.InstallDirectory); + if (WindowsPathSafety.PathEquals(restoredDirectory, acquiredDirectory)) + ctx.LocalAiRuntimeInstall = null; + } } /// Downloads one immutable, recipe-selected GGUF directly from Hugging Face. @@ -555,6 +582,10 @@ or InvalidDataException public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) { ct.ThrowIfCancellationRequested(); + if (!ctx.LocalAiRecoveryCleanupAllowed) + { + return Task.CompletedTask; + } if (ctx.LocalAiModelInstall is { } install) { _acquirer.RemoveInstalledModel(ctx.LocalDataDir, install); @@ -721,6 +752,23 @@ originalInstall is not null && }; if (originalInstall is not null) { + bool replacingModel = !string.Equals( + originalInstall.Manifest.ModelCatalogId, + manifest.ModelCatalogId, + StringComparison.Ordinal); + LocalAiInstallManifest replacedManifest = originalInstall.Manifest.ReplacedManifest ?? + originalInstall.Manifest; + // A pending model replacement can outlive a catalog runtime bump. Both sides + // of the replacement receipt must describe the newly acquired shared runtime, + // while retaining the prior model as rollback provenance. + replacedManifest = replacedManifest with + { + EngineVersion = manifest.EngineVersion, + Architecture = manifest.Architecture, + RuntimeId = manifest.RuntimeId, + ExecutablePath = manifest.ExecutablePath, + RuntimeAssets = manifest.RuntimeAssets, + }; manifest = originalInstall.Manifest with { SchemaVersion = manifest.SchemaVersion, @@ -746,6 +794,15 @@ originalInstall is not null && ValueCachePrecision = manifest.ValueCachePrecision, DraftKeyCachePrecision = manifest.DraftKeyCachePrecision, DraftValueCachePrecision = manifest.DraftValueCachePrecision, + GatewayFallbackModel = ctx.LocalAiRecoveryPendingInstall is { } pendingInstall + ? pendingInstall.Manifest.GatewayFallbackModel + : originalInstall.Manifest.GatewayFallbackModel, + InstalledAtUtc = ctx.LocalAiRecoveryPendingInstall?.Manifest.InstalledAtUtc ?? + originalInstall.Manifest.InstalledAtUtc, + ReplacedManifest = replacingModel + ? replacedManifest + : null, + PreviousEndpoints = ReplacementEndpointHistory(ctx.LocalAiRecoveryPendingInstall), }; } @@ -755,6 +812,28 @@ originalInstall is not null && await store.SaveAsync(manifest, ct); ctx.LocalAiResolvedInstall = store.ResolveAndValidate(manifest); ctx.LocalAiManifestCreatedThisRun = !replacesExistingReceipt; + if (manifest.ReplacedManifest is not null) + { + if (ctx.LocalAiRecoveryOriginalInstall is not null) + { + ctx.LocalAiRecoveryOriginalInstall = store.ResolveAndValidate( + manifest.ReplacedManifest); + } + if (ctx.LocalAiRecoveryPendingInstall is { } pendingRouteBaseline) + { + // Keep the published pending route as rollback provenance, but move + // it onto the upgraded runtime generation. Otherwise Gateway capture + // can replace the coherent upgraded original with the retired receipt. + LocalAiInstallManifest upgradedPendingRoute = manifest with + { + Endpoint = pendingRouteBaseline.Manifest.Endpoint, + }; + ctx.LocalAiRecoveryPendingInstall = store.ResolveAndValidate( + upgradedPendingRoute); + } + ctx.LocalAiRecoveryProviderTransition = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + } return StepResult.Ok("Recorded the verified llama-server and Hugging Face installation."); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) @@ -804,11 +883,46 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) if (ctx.LocalAiUpgradeOriginalInstall is not null) { await RestoreUpgradeReceiptAsync(ctx, ct); + if (ctx.LocalAiUpgradeOriginalInstall is null) + await RestartBorrowedRuntimeAsync(ctx, ct); return; } if (!ctx.LocalAiManifestCreatedThisRun) + { + bool restoredRecoveryReceipt = false; + // Before Gateway configuration is enrolled, this step still owns restoring a fresh + // replacement receipt. Once configuration starts, the recovery guard must settle the + // route and endpoint-health decision before any receipt or resource cleanup occurs. + if (ctx.LocalAiGatewayPriorState is null && + !ctx.LocalAiRecoveryRollbackUncertain && + ctx.LocalAiResolvedInstall?.Manifest.ReplacedManifest is not null && + ctx.LocalAiRecoveryOriginalInstall is { } recoveryInstall) + { + var recoveryStore = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); + try + { + ctx.LocalAiResolvedInstall = await recoveryStore + .RestoreRecoveryManifestAsync( + ctx.LocalAiResolvedInstall.Manifest, + recoveryInstall.Manifest, + ct) + .ConfigureAwait(false); + } + catch (InvalidDataException) + { + ctx.LocalAiRecoveryRollbackUncertain = true; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + throw; + } + ctx.LocalAiRecoveryProviderTransition = false; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + restoredRecoveryReceipt = true; + } + if (restoredRecoveryReceipt) + await RestartBorrowedRuntimeAsync(ctx, ct); return; + } var paths = new LocalAiPaths(ctx.LocalDataDir); await new LocalAiManifestStore(paths).DeleteAsync(ct); @@ -818,10 +932,40 @@ public override async Task RollbackAsync(SetupContext ctx, CancellationToken ct) ctx.LocalAiManifestCreatedThisRun = false; } + private static async Task RestartBorrowedRuntimeAsync(SetupContext ctx, CancellationToken ct) + { + if (!ctx.LocalAiRuntimeBorrowed || ctx.LocalAiRuntime is null) + return; + // Before ConfigureLocalAiGatewayStep begins, the tray runtime still owns Gateway + // publication. Use its ordinary lifecycle so an automatic-port move is published. + LocalAiRuntimeSnapshot restored = await ctx.LocalAiRuntime.RestartForSetupRollbackAsync(ct); + LocalAiResolvedInstall expected = await new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)) + .LoadAsync(ct) + .ConfigureAwait(false) + ?? throw new InvalidOperationException("The restored Local AI receipt is unavailable."); + ctx.LocalAiResolvedInstall = expected; + if (restored.State != LocalAiRuntimeState.Healthy || + restored.Ownership != LocalAiOwnership.CompanionManaged || + restored.ModelId != expected.Manifest.ModelCatalogId || + restored.Endpoint != expected.Endpoint || + restored.ModelEvidence.State is not + (LocalAiModelAvailabilityState.Verified or LocalAiModelAvailabilityState.Loaded)) + { + throw new InvalidOperationException( + restored.Detail ?? "The previous Local AI runtime could not be restored."); + } + AcquireLocalAiRuntimeStep.TransferCleanupOwnershipToRestoredRuntime(ctx, expected); + } + internal static async Task RestoreUpgradeReceiptAsync(SetupContext ctx, CancellationToken ct) { if (ctx.LocalAiUpgradeOriginalInstall is not { } originalInstall) return; + // A resumed pending replacement uses that pending receipt as the upgrade baseline, so + // restoring it before this process touches Gateway preserves any route published by an + // earlier process. After Gateway I/O begins, the recovery guard must settle authority. + if (!ctx.LocalAiRecoveryCleanupAllowed && ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun) + return; var paths = new LocalAiPaths(ctx.LocalDataDir); var store = new LocalAiManifestStore(paths); @@ -830,6 +974,8 @@ internal static async Task RestoreUpgradeReceiptAsync(SetupContext ctx, Cancella ctx.LocalAiResolvedInstall = store.ResolveAndValidate(originalInstall.Manifest); ctx.LocalAiManifestCreatedThisRun = false; ctx.LocalAiUpgradeOriginalInstall = null; + if (!ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun) + ctx.LocalAiRecoveryRollbackUncertain = false; } private static ImmutableArray BuildRuntimeReceipts( @@ -865,6 +1011,52 @@ private static ImmutableArray BuildRuntimeReceipts( return receipts.MoveToImmutable(); } + + private static ImmutableArray? ReplacementEndpointHistory(LocalAiResolvedInstall? pending) + { + if (pending is null) + return null; + + ImmutableArray history = pending.Manifest.PreviousEndpoints ?? []; + if (pending.Endpoint is null || + history.Contains(pending.Endpoint.AbsoluteUri, StringComparer.Ordinal)) + { + return history; + } + return history.Add(pending.Endpoint.AbsoluteUri); + } +} + +/// Commits a model replacement after its Gateway route has restarted successfully. +public sealed class FinalizeLocalAiModelReplacementStep : SetupStep +{ + public override string Id => "finalize-local-ai-model-replacement"; + public override string DisplayName => "Finalizing Local AI model replacement"; + public override bool CanRetry => false; + public override RetryPolicy Retry => RetryPolicy.None; + public override bool CanSkip(SetupContext ctx) => + ctx.LocalAiResolvedInstall?.Manifest.ReplacedManifest is null; + + public override async Task ExecuteAsync(SetupContext ctx, CancellationToken ct) + { + LocalAiResolvedInstall install = ctx.LocalAiResolvedInstall!; + try + { + var store = new LocalAiManifestStore(new LocalAiPaths(ctx.LocalDataDir)); + ctx.LocalAiResolvedInstall = await store + .FinalizeReplacementAsync(install.Manifest, ct) + .ConfigureAwait(false); + ctx.LocalAiRecoveryProviderTransition = false; + ctx.LocalAiRecoveryReceiptRollbackAllowed = false; + ctx.LocalAiRecoveryRollbackUncertain = false; + ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = false; + return StepResult.Ok("Local AI model replacement is committed."); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + return StepResult.Fail("The Local AI model replacement could not be finalized.", ex); + } + } } /// Starts the companion-owned llama-server router without preloading a model. @@ -891,64 +1083,107 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati { if (ctx.LocalAiResolvedInstall is null) return StepResult.Terminal("llama-server startup requires a verified installation receipt."); - if (ctx.LocalAiRuntime is not null) + if (ctx.LocalAiRuntime is not null && !ctx.LocalAiRuntimeBorrowed) return StepResult.Terminal("A Local AI runtime is already attached to this setup transaction."); + if (ctx.LocalAiRuntimeBorrowed && + ctx.LocalAiRecoveryOriginalInstall is null && + ctx.LocalAiUpgradeOriginalInstall is null && + ctx.LocalAiResolvedInstall.Manifest.ReplacedManifest is null && + string.IsNullOrWhiteSpace(ctx.Config.LocalAiRecoveryGatewayId)) + { + return StepResult.Terminal( + "Borrowing the tray Local AI runtime requires an armed recovery or recorded upgrade."); + } - ILocalAiRuntime runtime = _runtimeFactory(ctx); + ILocalAiRuntime runtime = ctx.LocalAiRuntime ?? _runtimeFactory(ctx); ctx.LocalAiRuntime = runtime; + LocalAiInstallManifest expectedManifest = ctx.LocalAiResolvedInstall.Manifest; try { - LocalAiRuntimeSnapshot snapshot = await runtime.EnsureStartedAsync(ct); + LocalAiRuntimeSnapshot snapshot; + if (ctx.LocalAiRuntimeBorrowed) + { + ctx.LocalAiBorrowedRuntimeRestartedThisRun = true; + snapshot = await runtime.RestartForSetupAsync(ct); + } + else + { + snapshot = await runtime.EnsureStartedAsync(ct); + } if (snapshot.State != LocalAiRuntimeState.Healthy || snapshot.Ownership != LocalAiOwnership.CompanionManaged || snapshot.ProcessId is null || + snapshot.ModelId != ctx.LocalAiResolvedInstall.Manifest.ModelCatalogId || snapshot.ModelEvidence.State != LocalAiModelAvailabilityState.Verified) { - await DisposeRuntimeAsync(ctx); + await CleanUpFailedRuntimeAsync(ctx); return StepResult.Fail( snapshot.Detail ?? "The managed llama-server router did not become healthy."); } LocalAiResolvedInstall? verifiedInstall = await new LocalAiManifestStore( new LocalAiPaths(ctx.LocalDataDir)) - .LoadAsync(ct); - if (verifiedInstall?.Endpoint is null || verifiedInstall.Endpoint != snapshot.Endpoint) + .LoadAsync(CancellationToken.None); + if (verifiedInstall?.Endpoint is null || + verifiedInstall.Endpoint != snapshot.Endpoint || + !IsEndpointOnlyTransition(expectedManifest, verifiedInstall.Manifest)) { - await DisposeRuntimeAsync(ctx); + await CleanUpFailedRuntimeAsync(ctx); return StepResult.Fail( "llama-server became healthy without committing its verified endpoint receipt."); } ctx.LocalAiResolvedInstall = verifiedInstall; + ct.ThrowIfCancellationRequested(); return StepResult.Ok( "The companion-owned llama-server router is healthy. The model remains unloaded until the first request."); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - await DisposeRuntimeAsync(ctx); + await CleanUpFailedRuntimeAsync(ctx); throw; } catch (Exception ex) { - await DisposeRuntimeAsync(ctx); + await CleanUpFailedRuntimeAsync(ctx); return StepResult.Fail($"llama-server startup failed: {ex.Message}", ex); } } + internal static bool IsEndpointOnlyTransition( + LocalAiInstallManifest expected, + LocalAiInstallManifest current) => + JsonElement.DeepEquals( + JsonSerializer.SerializeToElement(current), + JsonSerializer.SerializeToElement(expected with + { + Endpoint = current.Endpoint, + PreviousEndpoints = current.PreviousEndpoints, + })); + public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) { + if (ctx.LocalAiRuntimeBorrowed && !ctx.LocalAiBorrowedRuntimeRestartedThisRun) + return Task.CompletedTask; + if (ctx.LocalAiBorrowedRuntimeRestored) + return Task.CompletedTask; + // During a recovery provider transition, ConfigureLocalAiGatewayStep's rollback (which // runs before this step's rollback) sets LocalAiRecoveryReceiptRollbackAllowed only when // it confirmed the Gateway no longer routes to this runtime's endpoint. If that could not // be confirmed, the Gateway may still be pointed at this runtime; disposing it here would // orphan the active route instead of the intended, coordinated rollback. - if (ctx.LocalAiRecoveryProviderTransition && !ctx.LocalAiRecoveryReceiptRollbackAllowed) + if (!ctx.LocalAiRecoveryCleanupAllowed) { ctx.Logger.Warn( "Keeping the replacement llama-server router running because the Gateway configuration " + "rollback could not confirm it no longer routes to this endpoint."); - return Task.CompletedTask; + return ctx.LocalAiRuntimeBorrowed && ctx.LocalAiRuntime is { } borrowedRuntime + ? borrowedRuntime.ReleaseSetupGatewayRouteAsync(ct) + : Task.CompletedTask; } + if (ctx.LocalAiRuntimeBorrowed) + return StopBorrowedRuntimeAsync(ctx, ct); return DisposeRuntimeAsync(ctx).AsTask(); } @@ -972,6 +1207,23 @@ private static async ValueTask DisposeRuntimeAsync(SetupContext ctx) ctx.LocalAiRuntime = null; await runtime.DisposeAsync(); } + + private static async Task CleanUpFailedRuntimeAsync(SetupContext ctx) + { + if (ctx.LocalAiRuntimeBorrowed) + { + if (ctx.LocalAiRuntime is not null) + await ctx.LocalAiRuntime.StopForSetupAsync(CancellationToken.None); + return; + } + await DisposeRuntimeAsync(ctx); + } + + private static async Task StopBorrowedRuntimeAsync(SetupContext ctx, CancellationToken ct) + { + if (ctx.LocalAiRuntime is not null) + await ctx.LocalAiRuntime.StopForSetupAsync(ct); + } } /// @@ -1031,7 +1283,7 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - await ResetRouterAsync(runtime); + await ResetRouterAsync(ctx, runtime); throw; } catch (OperationCanceledException ex) @@ -1075,7 +1327,7 @@ private static async Task CaptureFailureDetailAsync( var paths = new LocalAiPaths(ctx.LocalDataDir); IReadOnlyList diagnostics = await LocalAiLogTail.ReadDiagnosticLinesAsync(paths, CancellationToken.None); - await ResetRouterAsync(runtime); + await ResetRouterAsync(ctx, runtime); // Echo into the setup log the UI already links, so the root cause remains available if // the router restart rotates the managed llama-server logs. foreach (string line in diagnostics) @@ -1083,17 +1335,54 @@ private static async Task CaptureFailureDetailAsync( return new LocalAiFailureDetail(diagnostics, paths.LogsDirectory); } - internal static async Task ResetRouterAsync(ILocalAiRuntime runtime) + internal static async Task ResetRouterAsync( + ILocalAiRuntime runtime, + bool setupScoped = false) { try { - return await runtime.RestartAsync(CancellationToken.None); + return setupScoped + ? await runtime.RestartForSetupAsync(CancellationToken.None) + : await runtime.RestartAsync(CancellationToken.None); } catch { return runtime.Snapshot; } } + + internal static async Task ResetRouterAsync( + SetupContext ctx, + ILocalAiRuntime runtime) + { + LocalAiRuntimeSnapshot reset = await ResetRouterAsync(runtime, ctx.LocalAiRuntimeBorrowed); + if (!ctx.LocalAiRuntimeBorrowed) + return reset; + + try + { + LocalAiResolvedInstall? restartedInstall = await new LocalAiManifestStore( + new LocalAiPaths(ctx.LocalDataDir)) + .LoadAsync(CancellationToken.None) + .ConfigureAwait(false); + LocalAiInstallManifest expected = ctx.LocalAiResolvedInstall!.Manifest; + bool onlyEndpointStateChanged = restartedInstall is not null && + StartLocalAiRuntimeStep.IsEndpointOnlyTransition( + expected, + restartedInstall.Manifest); + if (restartedInstall?.Endpoint == reset.Endpoint && onlyEndpointStateChanged) + { + // This refresh updates only the rollback compare-and-swap baseline. The caller + // still decides whether the runtime restart itself was healthy and successful. + ctx.LocalAiResolvedInstall = restartedInstall; + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + // Keep the prior compare-and-swap baseline so rollback fails closed. + } + return reset; + } } /// Proves the app-owned WSL distro can reach the native loopback router. diff --git a/src/OpenClaw.SetupEngine/RestartGatewayStep.cs b/src/OpenClaw.SetupEngine/RestartGatewayStep.cs index 6f4ae0b43..7eee4452a 100644 --- a/src/OpenClaw.SetupEngine/RestartGatewayStep.cs +++ b/src/OpenClaw.SetupEngine/RestartGatewayStep.cs @@ -27,4 +27,16 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati ctx.LocalAiRecoveryStoppedWsl = false; return result; } + + public override Task RollbackAsync(SetupContext ctx, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + if (!string.IsNullOrWhiteSpace(ctx.Config.LocalAiRecoveryGatewayId)) + { + // Configuration rollback runs after this step in reverse order. Arm the + // early recovery guard to restart the Gateway once the prior route is restored. + ctx.LocalAiRecoveryStoppedWsl = true; + } + return Task.CompletedTask; + } } diff --git a/src/OpenClaw.SetupEngine/SetupContext.cs b/src/OpenClaw.SetupEngine/SetupContext.cs index ef06766fc..e40b3bb14 100644 --- a/src/OpenClaw.SetupEngine/SetupContext.cs +++ b/src/OpenClaw.SetupEngine/SetupContext.cs @@ -583,14 +583,22 @@ public Func>? : null; internal LocalAiResolvedInstall? LocalAiRecoveryOriginalInstall { get; set; } internal LocalAiResolvedInstall? LocalAiUpgradeOriginalInstall { get; set; } + internal LocalAiResolvedInstall? LocalAiRecoveryPendingInstall { get; set; } internal bool LocalAiRecoveryProviderTransition { get; set; } internal bool LocalAiRecoveryReceiptRollbackAllowed { get; set; } + internal bool LocalAiRecoveryRollbackUncertain { get; set; } + internal bool LocalAiRecoveryCleanupAllowed => + !LocalAiRecoveryRollbackUncertain || LocalAiRecoveryReceiptRollbackAllowed; internal bool LocalAiManifestCreatedThisRun { get; set; } internal ILocalAiRuntime? LocalAiRuntime { get; set; } + internal bool LocalAiRuntimeBorrowed { get; set; } + internal bool LocalAiBorrowedRuntimeRestartedThisRun { get; set; } + internal bool LocalAiBorrowedRuntimeRestored { get; set; } internal HostHardwareInfo? LocalAiGpuBaseline { get; set; } internal LlamaServerInferenceVerification? LocalAiInferenceVerification { get; set; } internal LocalAiGpuLoadEvidence? LocalAiGpuLoadEvidence { get; set; } internal LocalAiGatewayPriorState? LocalAiGatewayPriorState { get; set; } + internal bool LocalAiRecoveryGatewayConfigurationStartedThisRun { get; set; } internal bool IsUninstalling { get; set; } internal bool LocalAiRecoveryStoppedWsl { get; set; } diff --git a/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs b/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs index a6908b00c..1dcffbee6 100644 --- a/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs +++ b/src/OpenClaw.SetupEngine/SetupInstallationProgress.cs @@ -74,19 +74,22 @@ private static SetupInstallationStatus Aggregate(IEnumerable entries) { "validate-distro-path" or "preflight-os" or "preflight-local-ai-hardware" or "preflight-wsl" or "preflight-windows-tailscale" or "ensure-wsl-platform" or "validate-local-ai-recovery-gateway" or - "preserve-local-ai-recovery-gateway" or "reconcile-local-ai-installation" or + "validate-local-ai-recovery-gateway-compatibility" or + "reconcile-local-ai-installation" or "cleanup-distro" or "cleanup-gateway" or "preflight-port" or "wsl-create" or "wsl-configure" or "validate-wsl-lockdown" => SetupInstallationPhase.Prepare, "acquire-local-ai-runtime" or "acquire-local-ai-model" or "persist-local-ai-manifest" or "start-local-ai-runtime" or "configure-local-ai-wsl-networking" => localAiRecovery ? SetupInstallationPhase.Install : SetupInstallationPhase.Prepare, "capture-local-ai-gpu-baseline" or "verify-local-ai-inference" or "verify-local-ai-gpu-load" or - "revalidate-local-ai-recovery-gateway" or "install-cli" or "verify-local-ai-wsl" or + "revalidate-local-ai-recovery-gateway" or "preserve-local-ai-recovery-gateway" or + "install-cli" or "verify-local-ai-wsl" or "install-tailscale" or "authorize-tailscale" or "configure-gateway" or "configure-local-ai-gateway" or "install-service" => SetupInstallationPhase.Install, "start-gateway" or "restart-gateway" or "mint-token" or "finalize-tailscale-serve" or "pair-operator" or "pair-node" or "verify-e2e" or - "run-wizard" or "windows-node-context" or "start-keepalive" => SetupInstallationPhase.Connect, + "run-wizard" or "windows-node-context" or "start-keepalive" or + "finalize-local-ai-model-replacement" => SetupInstallationPhase.Connect, _ => throw new ArgumentOutOfRangeException(nameof(stepId), stepId, "Installation step needs an explicit presentation phase."), }; } diff --git a/src/OpenClaw.SetupEngine/SetupPipeline.cs b/src/OpenClaw.SetupEngine/SetupPipeline.cs index 4a267c8da..230d1c918 100644 --- a/src/OpenClaw.SetupEngine/SetupPipeline.cs +++ b/src/OpenClaw.SetupEngine/SetupPipeline.cs @@ -101,14 +101,17 @@ public static List BuildLocalAiRecoverySteps() => [ new PreflightOsStep(), new ValidateLocalAiRecoveryGatewayStep(), - new PreserveLocalAiRecoveryGatewayStep(), new PreflightLocalAiHardwareStep(), new PreflightWslStep(), new EnsureWslPlatformStep(reusePreflightResult: true), new ReconcileLocalAiInstallationStep(), + new ValidateLocalAiRecoveryGatewayCompatibilityStep(), new AcquireLocalAiRuntimeStep(), new AcquireLocalAiModelStep(), new PersistLocalAiManifestStep(), + // Arm receipt, runtime, and final restart recovery before the borrowed runtime changes. + // Reverse rollback still restores WSL networking before this guard settles the Gateway. + new PreserveLocalAiRecoveryGatewayStep(), new StartLocalAiRuntimeStep(), new CaptureLocalAiGpuBaselineStep(), new VerifyLocalAiInferenceStep(), @@ -118,6 +121,7 @@ public static List BuildLocalAiRecoverySteps() => new VerifyLocalAiWslStep(), new ConfigureLocalAiGatewayStep(), new RestartGatewayStep(), + new FinalizeLocalAiModelReplacementStep(), ]; public static List BuildDefaultSteps() @@ -166,6 +170,20 @@ public static List BuildDefaultSteps() public sealed class SetupPipeline { + internal static Task ReleaseBorrowedLocalAiRuntimeAfterFailureAsync( + SetupContext ctx, + PipelineResult? result) + { + if (result?.Outcome == PipelineOutcome.Success || + !ctx.LocalAiRuntimeBorrowed || + !ctx.LocalAiBorrowedRuntimeRestartedThisRun || + ctx.LocalAiRuntime is not { } borrowedRuntime) + { + return Task.CompletedTask; + } + return borrowedRuntime.ReleaseSetupGatewayRouteAsync(CancellationToken.None); + } + public static async Task RunWithSettlementAsync( Func> run, Func settle) { diff --git a/src/OpenClaw.Tray.WinUI/Services/LocalAiGatewayDistroResolver.cs b/src/OpenClaw.Tray.WinUI/Services/LocalAiGatewayDistroResolver.cs index d192a7761..7756441dd 100644 --- a/src/OpenClaw.Tray.WinUI/Services/LocalAiGatewayDistroResolver.cs +++ b/src/OpenClaw.Tray.WinUI/Services/LocalAiGatewayDistroResolver.cs @@ -31,7 +31,8 @@ internal sealed record LocalAiRecoveryTarget( string DistroName, int GatewayPort, string? ModelCatalogId, - int? RequestedLocalAiPort); + int? RequestedLocalAiPort, + bool PinModelSelection); internal sealed record LocalAiSetupResolution( LocalAiSetupRoute Route, @@ -47,7 +48,8 @@ public static LocalAiSetupResolution Decide( bool hasDistroDataDirectory, bool distroIsAppOwned, string? installedModelCatalogId = null, - int? installedRequestedLocalAiPort = null) + int? installedRequestedLocalAiPort = null, + bool pinInstalledModelSelection = false) { if (owners.Count == 1) { @@ -69,7 +71,8 @@ public static LocalAiSetupResolution Decide( GatewayRecordEditing.ResolveManagedDistroName(owner)!.Trim(), uri.Port, installedModelCatalogId, - installedRequestedLocalAiPort)); + installedRequestedLocalAiPort, + pinInstalledModelSelection)); } return new(LocalAiSetupRoute.Blocked); diff --git a/src/OpenClaw.Tray.WinUI/Services/SetupLocalAiHost.cs b/src/OpenClaw.Tray.WinUI/Services/SetupLocalAiHost.cs index 49dae3d53..6618a0956 100644 --- a/src/OpenClaw.Tray.WinUI/Services/SetupLocalAiHost.cs +++ b/src/OpenClaw.Tray.WinUI/Services/SetupLocalAiHost.cs @@ -67,6 +67,8 @@ public void ReleaseNative(IGatewayAiSetupTransport transport) if (ReferenceEquals(_nativeTransport, transport)) _nativeTransport = null; } + public ILocalAiRuntime? BorrowManagedRuntime() => getRuntime(); + public GatewayRegistrySnapshot BeginGatewaySetup() => _setupRegistryBaseline = (getRegistry() ?? throw new InvalidOperationException("The Gateway registry is unavailable.")).CapturePersistedSnapshot(); diff --git a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs index d619ef608..498bf782f 100644 --- a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs +++ b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs @@ -112,6 +112,124 @@ public async Task Manifest_OmitsLegacyHardwareProfileIdFromNewWrites() Assert.Contains("\"draftValueCachePrecision\": \"q8_0\"", json, StringComparison.Ordinal); } + [Fact] + public async Task Manifest_OrdinaryReceiptRemainsReadableBySchemaFourReader() + { + using var temp = new TempDirectory("local-ai-manifest-"); + var paths = new LocalAiPaths(temp.Path); + await new LocalAiManifestStore(paths).SaveAsync(ValidManifest()); + + string json = await File.ReadAllTextAsync(paths.ManifestPath); + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + UnmappedMemberHandling = System.Text.Json.Serialization.JsonUnmappedMemberHandling.Disallow, + }; + + Assert.NotNull(JsonSerializer.Deserialize(json, options)); + Assert.DoesNotContain("previousEndpoints", json, StringComparison.Ordinal); + Assert.DoesNotContain("replacedManifest", json, StringComparison.Ordinal); + } + + [Fact] + public async Task Manifest_EndpointUpdateDoesNotRestoreFinalizedReplacementState() + { + using var temp = new TempDirectory("local-ai-manifest-"); + var paths = new LocalAiPaths(temp.Path); + var store = new LocalAiManifestStore(paths); + LocalAiInstallManifest original = ValidManifest() with + { + Endpoint = "http://127.0.0.1:28765/v1", + }; + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:28766/v1", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + await store.SaveAsync(pending); + await store.SaveAsync(pending with + { + ReplacedManifest = null, + PreviousEndpoints = null, + }); + + InvalidDataException error = await Assert.ThrowsAsync(() => + store.UpdateVerifiedEndpointAsync( + pending, + new Uri("http://127.0.0.1:28767/v1"))); + + Assert.Contains("finalized", error.Message, StringComparison.Ordinal); + LocalAiResolvedInstall updated = (await store.LoadAsync())!; + Assert.Equal("http://127.0.0.1:28766/v1", updated.Manifest.Endpoint); + Assert.Null(updated.Manifest.ReplacedManifest); + Assert.Null(updated.Manifest.PreviousEndpoints); + string json = await File.ReadAllTextAsync(paths.ManifestPath); + Assert.DoesNotContain("previousEndpoints", json, StringComparison.Ordinal); + Assert.DoesNotContain("replacedManifest", json, StringComparison.Ordinal); + } + + [Fact] + public async Task Manifest_FinalizationRejectsConcurrentlyUpdatedEndpoint() + { + using var temp = new TempDirectory("local-ai-manifest-"); + var paths = new LocalAiPaths(temp.Path); + var store = new LocalAiManifestStore(paths); + LocalAiInstallManifest original = ValidManifest() with + { + Endpoint = "http://127.0.0.1:28765/v1", + }; + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:28766/v1", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + await store.SaveAsync(pending); + await store.UpdateVerifiedEndpointAsync( + pending, + new Uri("http://127.0.0.1:28767/v1")); + + InvalidDataException error = await Assert.ThrowsAsync(() => + store.FinalizeReplacementAsync(pending)); + + Assert.Contains("changed", error.Message, StringComparison.Ordinal); + LocalAiResolvedInstall retained = (await store.LoadAsync())!; + Assert.Equal("http://127.0.0.1:28767/v1", retained.Manifest.Endpoint); + Assert.NotNull(retained.Manifest.ReplacedManifest); + Assert.NotNull(retained.Manifest.PreviousEndpoints); + } + + [Fact] + public async Task Manifest_RejectsNullPreviousEndpoint() + { + using var temp = new TempDirectory("local-ai-manifest-"); + var paths = new LocalAiPaths(temp.Path); + var store = new LocalAiManifestStore(paths); + LocalAiInstallManifest original = ValidManifest() with + { + Endpoint = "http://127.0.0.1:28765/v1", + }; + await store.SaveAsync(original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:28766/v1", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }); + JsonObject json = (JsonNode.Parse(await File.ReadAllTextAsync(paths.ManifestPath)) as JsonObject)!; + json["previousEndpoints"] = new JsonArray { null }; + await File.WriteAllTextAsync(paths.ManifestPath, json.ToJsonString()); + + InvalidDataException error = await Assert.ThrowsAsync(() => store.LoadAsync()); + + Assert.Contains("non-empty endpoint", error.Message, StringComparison.Ordinal); + } + [Fact] public async Task Router_RejectsRuntimeArchitectureMismatchWithoutHardwareProfile() { @@ -2235,6 +2353,131 @@ public async Task RestartAsync_UsesEndpointCycleUntilReplacementIsPublished() Assert.DoesNotContain("quiesce:Teardown", events); } + [Fact] + public async Task RestartForSetupAsync_RestartsProcessWithoutChangingGatewayLifecycle() + { + using var temp = new TempDirectory("local-ai-port-"); + LocalAiPaths paths = await PrepareInstallAsync(temp); + var events = new SynchronizedEventLog(); + var platform = new FakePlatform(); + var host = new FakeProcessHost(platform, events, selectedPort: 28_769); + var lifecycle = new FakeLifecycle(events); + await using var runtime = CreateRuntime( + paths, + host, + platform, + new FakeClient(events), + lifecycle); + LocalAiRuntimeSnapshot started = await runtime.EnsureStartedAsync(); + Assert.Equal(LocalAiRuntimeState.Healthy, started.State); + events.Clear(); + + LocalAiRuntimeSnapshot restarted = await runtime.RestartForSetupAsync(); + + Assert.Equal(LocalAiRuntimeState.Healthy, restarted.State); + Assert.Equal(["stop", "start", "probe:28769"], events); + Assert.True(restarted.GatewayRouteRequiresResolution); + + events.Clear(); + LocalAiRuntimeSnapshot stillSetupOwned = await runtime.RestartAsync(); + + Assert.True(stillSetupOwned.GatewayRouteRequiresResolution); + Assert.Equal(["stop", "start", "probe:28769"], events); + await runtime.AcknowledgeSetupGatewayRouteAsync(); + Assert.Equal([false, true, false, false, true], lifecycle.RecoveryIntents); + } + + [Fact] + public async Task AcknowledgeSetupGatewayRouteAsync_ClearsResolutionWithoutLifecycleIo() + { + using var temp = new TempDirectory("local-ai-port-"); + LocalAiPaths paths = await PrepareInstallAsync(temp); + var events = new SynchronizedEventLog(); + var platform = new FakePlatform(); + var host = new FakeProcessHost(platform, events, selectedPort: 28_769); + var lifecycle = new FakeLifecycle(events); + await using var runtime = CreateRuntime( + paths, + host, + platform, + new FakeClient(events), + lifecycle); + LocalAiRuntimeSnapshot started = await runtime.RestartForSetupAsync(); + Assert.Equal(LocalAiRuntimeState.Healthy, started.State); + Assert.True(started.GatewayRouteRequiresResolution); + events.Clear(); + + LocalAiRuntimeSnapshot acknowledged = await runtime.AcknowledgeSetupGatewayRouteAsync(); + + Assert.False(acknowledged.GatewayRouteRequiresResolution); + Assert.Empty(events); + Assert.Equal([false, true], lifecycle.RecoveryIntents); + } + + [Fact] + public async Task ReleaseSetupGatewayRouteAsync_ReturnsLifecycleOwnershipWithoutAcknowledgingRoute() + { + using var temp = new TempDirectory("local-ai-port-"); + LocalAiPaths paths = await PrepareInstallAsync(temp); + var events = new SynchronizedEventLog(); + var platform = new FakePlatform(); + var host = new FakeProcessHost(platform, events, selectedPort: 28_769); + var lifecycle = new FakeLifecycle(events); + await using var runtime = CreateRuntime( + paths, + host, + platform, + new FakeClient(events), + lifecycle); + LocalAiRuntimeSnapshot started = await runtime.RestartForSetupAsync(); + Assert.True(started.GatewayRouteRequiresResolution); + + LocalAiRuntimeSnapshot released = await runtime.ReleaseSetupGatewayRouteAsync(); + events.Clear(); + LocalAiRuntimeSnapshot restarted = await runtime.RestartAsync(); + + Assert.True(released.GatewayRouteRequiresResolution); + Assert.False(restarted.GatewayRouteRequiresResolution); + Assert.Equal([false, true, false, true], lifecycle.RecoveryIntents); + Assert.Contains("quiesce:EndpointCycle", events); + Assert.Contains("publish:28769", events); + } + + [Fact] + public async Task RestartForSetupRollbackAsync_AdoptsRestoredReceiptBeforeGatewayLifecycle() + { + using var temp = new TempDirectory("local-ai-port-"); + LocalAiPaths paths = await PrepareInstallAsync(temp); + var events = new SynchronizedEventLog(); + var platform = new FakePlatform(); + var lifecycle = new FakeLifecycle(events); + var host = new FakeProcessHost(platform, events, selectedPort: 28_769); + await using var runtime = CreateRuntime( + paths, + host, + platform, + new FakeClient(events), + lifecycle); + LocalAiRuntimeSnapshot started = await runtime.EnsureStartedAsync(); + Assert.Equal(LocalAiRuntimeState.Healthy, started.State); + LocalAiResolvedInstall current = (await new LocalAiManifestStore(paths).LoadAsync())!; + LocalAiInstallManifest restoredManifest = current.Manifest with + { + Endpoint = "http://127.0.0.1:28768/v1", + }; + await new LocalAiManifestStore(paths).SaveAsync(restoredManifest); + events.Clear(); + lifecycle.QuiescedEndpoints.Clear(); + + LocalAiRuntimeSnapshot restarted = await runtime.RestartForSetupRollbackAsync(); + + Assert.Equal(LocalAiRuntimeState.Healthy, restarted.State); + Assert.Equal(new Uri("http://127.0.0.1:28768/v1"), lifecycle.QuiescedEndpoints[0]); + Assert.Equal(new Uri("http://127.0.0.1:28769/v1"), restarted.Endpoint); + Assert.False(restarted.GatewayRouteRequiresResolution); + Assert.Contains("publish:28769", events); + } + [Fact] public async Task RestartAsync_InitialEndpointCycleExceptionCompletesTeardownBeforeStopping() { @@ -3155,6 +3398,35 @@ private static LocalAiInstallManifest ValidManifest() }; } + private sealed record SchemaFourTransitionalManifest + { + public int SchemaVersion { get; init; } + public string? Engine { get; init; } + public string? EngineVersion { get; init; } + public string? Architecture { get; init; } + public string? HardwareProfileId { get; init; } + public string? RuntimeId { get; init; } + public string? ModelCatalogId { get; init; } + public string? SelectedGpuId { get; init; } + public string? ExecutablePath { get; init; } + public JsonElement RuntimeAssets { get; init; } + public string? ModelPath { get; init; } + public string? ModelCacheRoot { get; init; } + public string? CachedModelPath { get; init; } + public string? ModelId { get; init; } + public string? ModelAlias { get; init; } + public JsonElement ModelAsset { get; init; } + public int RequestedPort { get; init; } + public string? Endpoint { get; init; } + public string? GatewayFallbackModel { get; init; } + public int ContextLength { get; init; } + public string? KeyCachePrecision { get; init; } + public string? ValueCachePrecision { get; init; } + public string? DraftKeyCachePrecision { get; init; } + public string? DraftValueCachePrecision { get; init; } + public DateTimeOffset InstalledAtUtc { get; init; } + } + private sealed class SynchronizedEventLog : IReadOnlyCollection { private readonly Lock _gate = new(); diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs index b55506ada..863600daa 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGatewayUninstallTests.cs @@ -37,6 +37,27 @@ public async Task Repair_RollbackRestoresFallbackAfterRetainedEndpointCycle() Assert.Equal(JsonSerializer.Serialize("openai/gpt-5"), commands.PrimaryJson); } + [Fact] + public async Task Configure_AcknowledgementFailureCompensatesCommittedGatewayRoute() + { + using var temp = new TempDirectory("local-ai-gateway-ack-"); + LocalAiResolvedInstall install = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string fallback = JsonSerializer.Serialize("openai/gpt-5"); + var commands = new GatewayStateCommandRunner(providerJson: null, fallback); + SetupContext context = CreateContext(temp.Path, commands); + context.LocalAiResolvedInstall = install; + context.LocalAiEligibility = LocalInferenceEligibility.Evaluate(CreateSparkHardware()); + context.LocalAiRuntimeBorrowed = true; + context.LocalAiRuntime = new AcknowledgementFailingRuntime(); + + StepResult result = await new ConfigureLocalAiGatewayStep() + .ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Failed, result.Outcome); + Assert.Null(commands.ProviderJson); + Assert.Equal(fallback, commands.PrimaryJson); + } + [Fact] public async Task Repair_RollbackUnsetsPrimaryAfterRetainedEndpointCycleWithoutFallback() { @@ -58,14 +79,33 @@ public async Task Repair_RollbackUnsetsPrimaryAfterRetainedEndpointCycleWithoutF Assert.Null(commands.PrimaryJson); } - [Fact] - public async Task FreshProcessUninstall_RemovesExactManagedProviderAndPrimary() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task FreshProcessUninstall_RemovesExactManagedProviderAndPrimary(bool pendingReplacement) { using var temp = new TempDirectory("local-ai-gateway-uninstall-"); - LocalAiResolvedInstall install = await SaveManifestAsync(temp.Path); - string provider = LocalAiGatewayProviderDefinition.BuildProviderJson(install); + LocalAiResolvedInstall routed = await SaveManifestAsync(temp.Path); + if (pendingReplacement) + { + LocalAiInstallManifest published = routed.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39876/v1", + ReplacedManifest = routed.Manifest, + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + routed = store.ResolveAndValidate(published); + await store.SaveAsync(published with + { + Endpoint = "http://127.0.0.1:39877/v1", + PreviousEndpoints = [published.Endpoint!], + }); + } + string provider = LocalAiGatewayProviderDefinition.BuildProviderJson(routed); string primary = JsonSerializer.Serialize( - LocalAiGatewayProviderDefinition.BuildPrimaryModel(install)); + LocalAiGatewayProviderDefinition.BuildPrimaryModel(routed)); var commands = new GatewayStateCommandRunner(provider, primary); SetupContext context = CreateContext(temp.Path, commands); context.IsUninstalling = true; @@ -78,6 +118,65 @@ public async Task FreshProcessUninstall_RemovesExactManagedProviderAndPrimary() command.Contains("LOCAL_AI_GATEWAY_UNSET", StringComparison.Ordinal)); } + [Fact] + public async Task FreshProcessUninstall_RemovesHistoricalManagedProviderWhenPrimaryIsMissing() + { + using var temp = new TempDirectory("local-ai-gateway-uninstall-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path); + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39876/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Manifest.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pendingManifest); + LocalAiResolvedInstall pending = (await store.LoadAsync())!; + LocalAiResolvedInstall historical = pending with + { + Manifest = pending.Manifest with { Endpoint = original.Manifest.Endpoint }, + Endpoint = original.Endpoint, + }; + var commands = new GatewayStateCommandRunner( + LocalAiGatewayProviderDefinition.BuildProviderJson(historical), + primaryJson: null); + SetupContext context = CreateContext(temp.Path, commands); + context.IsUninstalling = true; + + await new ConfigureLocalAiGatewayStep().RollbackAsync(context, CancellationToken.None); + + Assert.Null(commands.ProviderJson); + Assert.Null(commands.PrimaryJson); + } + + [Fact] + public async Task FreshProcessUninstall_RemovesPendingReplacementOnFixedPort() + { + using var temp = new TempDirectory("local-ai-gateway-uninstall-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, requestedPort: 28765); + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + ReplacedManifest = original.Manifest, + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pendingManifest); + LocalAiResolvedInstall pending = (await store.LoadAsync())!; + var commands = new GatewayStateCommandRunner( + LocalAiGatewayProviderDefinition.BuildProviderJson(pending), + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(pending))); + SetupContext context = CreateContext(temp.Path, commands); + context.IsUninstalling = true; + + await new ConfigureLocalAiGatewayStep().RollbackAsync(context, CancellationToken.None); + + Assert.Null(commands.ProviderJson); + Assert.Null(commands.PrimaryJson); + } + [Fact] public async Task FreshProcessUninstall_AcceptsCliRedactedManagedApiKey() { @@ -171,34 +270,149 @@ public async Task Recovery_ReplacesExactManagedProviderAfterAutomaticPortChanges { using var temp = new TempDirectory("local-ai-gateway-recovery-"); LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); - string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + LocalAiInstallManifest publishedManifest = original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39878/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = + [ + "http://127.0.0.1:39876/v1", + "http://127.0.0.1:39877/v1", + ], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(publishedManifest); + LocalAiResolvedInstall published = (await store.LoadAsync())!; + LocalAiResolvedInstall publishedRoute = published with + { + Manifest = published.Manifest with { Endpoint = published.Manifest.PreviousEndpoints!.Value[1] }, + Endpoint = new Uri(published.Manifest.PreviousEndpoints!.Value[1]), + }; + string publishedProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(publishedRoute); string primary = JsonSerializer.Serialize( - LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); - var commands = new GatewayStateCommandRunner(originalProvider, primary); + LocalAiGatewayProviderDefinition.BuildPrimaryModel(published)); + var commands = new GatewayStateCommandRunner(publishedProvider, primary); + SetupContext context = CreateRecoveryContext(temp.Path, commands); + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryPendingInstall = published; + context.LocalAiRecoveryReceiptRollbackAllowed = true; + LocalAiInstallManifest replacementManifest = publishedManifest with + { + Endpoint = "http://127.0.0.1:39879/v1", + PreviousEndpoints = publishedManifest.PreviousEndpoints!.Value.Add(publishedManifest.Endpoint!), + }; + await store.SaveAsync(replacementManifest); + context.LocalAiResolvedInstall = (await store.LoadAsync())!; + var step = new ConfigureLocalAiGatewayStep(); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + await step.RollbackAsync(context, CancellationToken.None); + await new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("not needed")), + (_, _) => Task.FromResult(true)) + .RollbackAsync(context, CancellationToken.None); + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.False(context.LocalAiRecoveryProviderTransition); + LocalAiResolvedInstall restored = (await store.LoadAsync())!; + Assert.Equal(publishedRoute.Manifest.Endpoint, restored.Manifest.Endpoint); + Assert.Equal(published.Manifest.ModelCatalogId, restored.Manifest.ModelCatalogId); + Assert.NotNull(restored.Manifest.ReplacedManifest); + Assert.Equal(replacementManifest.PreviousEndpoints, restored.Manifest.PreviousEndpoints); + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson( + commands.ProviderJson!, + restored)); + Assert.Equal( + JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(restored)), + commands.PrimaryJson); + } + + [Fact] + public async Task Recovery_ReplacesAndRollsBackModelOnFixedPort() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync( + temp.Path, + fallbackModel: "openai/gpt-5", + requestedPort: 28765); + var commands = new GatewayStateCommandRunner( + LocalAiGatewayProviderDefinition.BuildProviderJson(original), + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(original))); SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; LocalAiInstallManifest replacementManifest = original.Manifest with { - Endpoint = "http://127.0.0.1:39876/v1", + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + ReplacedManifest = original.Manifest, }; var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); - context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); + LocalAiResolvedInstall replacement = (await store.LoadAsync())!; + context.LocalAiResolvedInstall = replacement; var step = new ConfigureLocalAiGatewayStep(); StepResult result = await step.ExecuteAsync(context, CancellationToken.None); Assert.Equal(StepOutcome.Success, result.Outcome); - Assert.True(context.LocalAiRecoveryProviderTransition); Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson( commands.ProviderJson!, - context.LocalAiResolvedInstall)); - Assert.Equal(primary, commands.PrimaryJson); + replacement)); + Assert.Equal( + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(replacement)), + commands.PrimaryJson); + + await step.RollbackAsync(context, CancellationToken.None); + + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson( + commands.ProviderJson!, + original)); + Assert.Equal( + JsonSerializer.Serialize(LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)), + commands.PrimaryJson); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Recovery_ReplacementAcceptsProviderlessOriginalPrimary(bool retainedManagedPrimary) + { + const string fallback = "openai/gpt-5"; + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync( + temp.Path, + retainedManagedPrimary ? fallback : null); + string primary = JsonSerializer.Serialize(retainedManagedPrimary + ? LocalAiGatewayProviderDefinition.BuildPrimaryModel(original) + : fallback); + var commands = new GatewayStateCommandRunner(providerJson: null, primary); + SetupContext context = CreateRecoveryContext(temp.Path, commands); + context.LocalAiRecoveryOriginalInstall = original; + LocalAiInstallManifest replacement = original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39876/v1", + ReplacedManifest = original.Manifest, + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(replacement); + context.LocalAiResolvedInstall = store.ResolveAndValidate(replacement); + + StepResult result = await new ConfigureLocalAiGatewayStep() + .ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.Equal(fallback, (await store.LoadAsync())!.Manifest.GatewayFallbackModel); } [Fact] - public async Task Recovery_PreservesProviderThatMatchesNeitherEndpoint() + public async Task Recovery_DriftedProviderRejectsBeforeGatewayMutation() { using var temp = new TempDirectory("local-ai-gateway-recovery-"); LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path); @@ -213,10 +427,7 @@ public async Task Recovery_PreservesProviderThatMatchesNeitherEndpoint() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); context.LocalAiResolvedInstall = new LocalAiResolvedInstall( replacementManifest, original.ExecutablePath, @@ -229,6 +440,8 @@ public async Task Recovery_PreservesProviderThatMatchesNeitherEndpoint() Assert.Equal(StepOutcome.Failed, result.Outcome); Assert.Equal(driftedProvider, commands.ProviderJson); Assert.Equal(primary, commands.PrimaryJson); + Assert.DoesNotContain(commands.WslCalls, command => + command.Contains("LOCAL_AI_GATEWAY_CONFIGURED", StringComparison.Ordinal)); } [Fact] @@ -243,10 +456,7 @@ public async Task Recovery_RollbackRestoresOriginalProviderAndReceipt() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -279,10 +489,7 @@ public async Task Recovery_RollbackPreservesEndpointCycleManagedPrimary() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -317,10 +524,7 @@ public async Task Recovery_FailedProviderSwitchRestoresOriginalReceipt() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -353,10 +557,7 @@ public async Task Recovery_FailureBeforeProviderConfigurationRestoresOriginalRec SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -393,10 +594,7 @@ public async Task Recovery_RetryPreservesOriginalProviderRollbackBaseline() SetupContext context = CreateRecoveryContext(temp.Path, commands); context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -420,6 +618,281 @@ public async Task Recovery_RetryPreservesOriginalProviderRollbackBaseline() Assert.False(context.LocalAiRecoveryProviderTransition); } + [Fact] + public async Task RestoreRecoveryRouteAsync_PreservesConcurrentGatewayChanges() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + var prior = new LocalAiGatewayPriorState( + ProviderExisted: true, + ProviderJson: originalProvider, + PrimaryModelExisted: true, + PrimaryModelJson: originalPrimary); + var commands = new GatewayStateCommandRunner( + LocalAiGatewayProviderDefinition.BuildProviderJson(original), + JsonSerializer.Serialize("openai/concurrent-model")); + SetupContext context = CreateRecoveryContext(temp.Path, commands); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:28766/v1" }, + Endpoint = new Uri("http://127.0.0.1:28766/v1"), + }; + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + moved, + CancellationToken.None); + + Assert.False(restored); + Assert.Equal(JsonSerializer.Serialize("openai/concurrent-model"), commands.PrimaryJson); + Assert.DoesNotContain( + commands.WslCalls, + command => command.Contains("LOCAL_AI_GATEWAY_RESTORED", StringComparison.Ordinal)); + } + + [Fact] + public async Task RestoreRecoveryRouteAsync_PreservesCustomizedProviderWhenEndpointIsUnchanged() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string customizedProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original) + .Replace("\"timeoutSeconds\":300", "\"timeoutSeconds\":301", StringComparison.Ordinal); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + var prior = new LocalAiGatewayPriorState(true, customizedProvider, true, originalPrimary); + var commands = new GatewayStateCommandRunner(customizedProvider, originalPrimary) + { + SupportsConditionalProviderSet = false, + }; + SetupContext context = CreateRecoveryContext(temp.Path, commands); + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + original, + CancellationToken.None); + + Assert.True(restored); + Assert.Equal(customizedProvider, commands.ProviderJson); + Assert.DoesNotContain( + commands.WslCalls, + command => command.Contains("LOCAL_AI_GATEWAY_RESTORED", StringComparison.Ordinal)); + } + + [Fact] + public async Task RestoreRecoveryRouteAsync_ConditionallyMovesOwnedProviderEndpoint() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + var prior = new LocalAiGatewayPriorState( + ProviderExisted: true, + ProviderJson: originalProvider, + PrimaryModelExisted: true, + PrimaryModelJson: originalPrimary); + var commands = new GatewayStateCommandRunner(originalProvider, originalPrimary); + SetupContext context = CreateRecoveryContext(temp.Path, commands); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:28766/v1" }, + Endpoint = new Uri("http://127.0.0.1:28766/v1"), + }; + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + moved, + CancellationToken.None); + + Assert.True(restored); + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson(commands.ProviderJson!, moved)); + Assert.Equal(originalPrimary, commands.PrimaryJson); + Assert.Contains( + commands.WslCalls, + command => command.Contains("--expect-current-json", StringComparison.Ordinal)); + } + + [Fact] + public async Task RestoreRecoveryRouteAsync_PreservesPrimaryChangeRacingProviderCas() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + string concurrentPrimary = JsonSerializer.Serialize("openai/concurrent-model"); + var prior = new LocalAiGatewayPriorState(true, originalProvider, true, originalPrimary); + var commands = new GatewayStateCommandRunner(originalProvider, originalPrimary) + { + PrimaryJsonAfterConditionalProviderSet = concurrentPrimary, + }; + SetupContext context = CreateRecoveryContext(temp.Path, commands); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:28766/v1" }, + Endpoint = new Uri("http://127.0.0.1:28766/v1"), + }; + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + moved, + CancellationToken.None); + + Assert.True(restored); + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson(commands.ProviderJson!, moved)); + Assert.Equal(concurrentPrimary, commands.PrimaryJson); + } + + [Fact] + public async Task RestoreRecoveryRouteAsync_RejectsLegacyGatewayCliWithoutConditionalWrites() + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + string originalProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(original); + string originalPrimary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(original)); + var prior = new LocalAiGatewayPriorState(true, originalProvider, true, originalPrimary); + var commands = new GatewayStateCommandRunner(originalProvider, originalPrimary) + { + SupportsConditionalProviderSet = false, + }; + SetupContext context = CreateRecoveryContext(temp.Path, commands); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:28766/v1" }, + Endpoint = new Uri("http://127.0.0.1:28766/v1"), + }; + + bool restored = await ConfigureLocalAiGatewayStep.RestoreRecoveryRouteAsync( + context, + prior, + original, + moved, + CancellationToken.None); + + Assert.False(restored); + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson(commands.ProviderJson!, original)); + Assert.Equal(originalPrimary, commands.PrimaryJson); + Assert.Contains( + commands.WslCalls, + command => command.Contains("LOCAL_AI_CONDITIONAL_SET_UNSUPPORTED", StringComparison.Ordinal)); + Assert.DoesNotContain( + commands.WslCalls, + command => command.Contains("OPENCLAW_LOCAL_AI_BATCH_B64", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Recovery_UpgradedRouteSettlementSurvivesRemainingRollback( + bool priorRouteWasPending) + { + using var temp = new TempDirectory("local-ai-gateway-recovery-"); + LocalAiResolvedInstall original = await SaveManifestAsync(temp.Path, "openai/gpt-5"); + LocalAiInstallManifest oldPendingManifest = ReplacementManifest(original); + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + LocalAiResolvedInstall oldPending = store.ResolveAndValidate(oldPendingManifest); + LocalAiInstallManifest upgradedOriginal = original.Manifest with + { + EngineVersion = "b11026", + RuntimeId = "b11026-cuda13-arm64", + }; + LocalAiInstallManifest upgradedReplacement = oldPendingManifest with + { + EngineVersion = upgradedOriginal.EngineVersion, + RuntimeId = upgradedOriginal.RuntimeId, + Endpoint = "http://127.0.0.1:39877/v1", + ReplacedManifest = upgradedOriginal, + PreviousEndpoints = + [ + original.Endpoint!.AbsoluteUri, + oldPending.Endpoint!.AbsoluteUri, + ], + }; + LocalAiInstallManifest upgradedPendingRoute = upgradedReplacement with + { + Endpoint = oldPending.Manifest.Endpoint, + }; + await store.SaveAsync(upgradedReplacement); + + LocalAiResolvedInstall priorRoute = priorRouteWasPending ? oldPending : original; + string priorProvider = LocalAiGatewayProviderDefinition.BuildProviderJson(priorRoute); + string primary = JsonSerializer.Serialize( + LocalAiGatewayProviderDefinition.BuildPrimaryModel(priorRoute)); + var commands = new GatewayStateCommandRunner(priorProvider, primary); + SetupContext context = CreateRecoveryContext(temp.Path, commands); + context.Config.RollbackOnFailure = true; + context.LocalAiRecoveryOriginalInstall = store.ResolveAndValidate(upgradedOriginal); + context.LocalAiRecoveryPendingInstall = store.ResolveAndValidate(upgradedPendingRoute); + context.LocalAiUpgradeOriginalInstall = oldPending; + context.LocalAiResolvedInstall = store.ResolveAndValidate(upgradedReplacement); + context.LocalAiRuntimeBorrowed = true; + var runtime = new SettlementTrackingRuntime(store); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeInstall = new LlamaRuntimeInstallResult( + Path.GetDirectoryName(context.LocalAiResolvedInstall.ExecutablePath)!, + context.LocalAiResolvedInstall.ExecutablePath, + LlamaRuntimeInstallDisposition.Installed, + CreatedThisRun: true, + VerifiedArchives: [], + Rollback: null); + var runtimeAcquirer = new TrackingRuntimeAcquirer(); + + var configure = new ConfigureLocalAiGatewayStep(); + StepResult configured = await configure.ExecuteAsync(context, CancellationToken.None); + await configure.RollbackAsync(context, CancellationToken.None); + await new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("not needed")), + (_, _) => Task.FromResult(true), + (_, _, _, _, _) => Task.FromResult(true)) + .RollbackAsync(context, CancellationToken.None); + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + await new AcquireLocalAiRuntimeStep(runtimeAcquirer) + .RollbackAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, configured.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(0, runtime.RestartForSetupRollbackCalls); + Assert.Null(context.LocalAiUpgradeOriginalInstall); + Assert.Null(context.LocalAiRecoveryOriginalInstall); + LocalAiResolvedInstall restored = Assert.IsType( + await store.LoadAsync()); + Assert.Equal(priorRoute.Endpoint, restored.Endpoint); + Assert.Equal(priorRoute.Manifest.ModelCatalogId, restored.Manifest.ModelCatalogId); + Assert.Equal(upgradedReplacement.RuntimeId, restored.Manifest.RuntimeId); + if (priorRouteWasPending) + { + LocalAiInstallManifest restoredOriginal = Assert.IsType( + restored.Manifest.ReplacedManifest); + Assert.Equal(upgradedOriginal.RuntimeId, restoredOriginal.RuntimeId); + Assert.Equal(upgradedOriginal.ModelCatalogId, restoredOriginal.ModelCatalogId); + Assert.Equal(upgradedOriginal.Endpoint, restoredOriginal.Endpoint); + } + else + { + Assert.Null(restored.Manifest.ReplacedManifest); + } + Assert.True(LocalAiGatewayProviderDefinition.MatchesProviderJson( + commands.ProviderJson!, + restored)); + Assert.Equal(primary, commands.PrimaryJson); + Assert.Null(context.LocalAiRuntimeInstall); + Assert.Equal(0, runtimeAcquirer.RemoveCalls); + Assert.False(context.LocalAiRecoveryRollbackUncertain); + Assert.False(context.LocalAiRecoveryProviderTransition); + } + [Fact] public async Task Recovery_FailedProviderCompensationKeepsReplacementReceipt() { @@ -433,10 +906,7 @@ public async Task Recovery_FailedProviderCompensationKeepsReplacementReceipt() context.Config.RollbackOnFailure = true; context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -462,8 +932,10 @@ public async Task Recovery_FailedProviderCompensationKeepsReplacementReceipt() commands.ProviderJson!, context.LocalAiResolvedInstall)); Assert.Equal(new Uri(replacementManifest.Endpoint!), (await store.LoadAsync())!.Endpoint); - Assert.False(context.LocalAiRecoveryProviderTransition); + Assert.True(context.LocalAiRecoveryProviderTransition); Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + Assert.False(context.LocalAiRecoveryCleanupAllowed); + Assert.True(context.LocalAiRecoveryGatewayConfigurationStartedThisRun); } [Fact] @@ -479,10 +951,7 @@ public async Task Recovery_LostRollbackAcknowledgementRestoresOriginalReceipt() context.Config.RollbackOnFailure = true; context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -527,10 +996,7 @@ public async Task Recovery_RollbackCancellationKeepsReplacementReceipt() context.Config.RollbackOnFailure = true; context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -556,8 +1022,9 @@ public async Task Recovery_RollbackCancellationKeepsReplacementReceipt() commands.ProviderJson!, context.LocalAiResolvedInstall)); Assert.Equal(new Uri(replacementManifest.Endpoint!), (await store.LoadAsync())!.Endpoint); - Assert.False(context.LocalAiRecoveryProviderTransition); + Assert.True(context.LocalAiRecoveryProviderTransition); Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + Assert.False(context.LocalAiRecoveryCleanupAllowed); } [Fact] @@ -571,10 +1038,7 @@ public async Task Recovery_ProviderCreationRollbackCancellationKeepsReplacementR context.Config.RollbackOnFailure = true; context.LocalAiRecoveryOriginalInstall = original; context.LocalAiRecoveryReceiptRollbackAllowed = true; - LocalAiInstallManifest replacementManifest = original.Manifest with - { - Endpoint = "http://127.0.0.1:39876/v1", - }; + LocalAiInstallManifest replacementManifest = ReplacementManifest(original); var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(replacementManifest); context.LocalAiResolvedInstall = store.ResolveAndValidate(replacementManifest); @@ -600,8 +1064,9 @@ public async Task Recovery_ProviderCreationRollbackCancellationKeepsReplacementR commands.ProviderJson!, context.LocalAiResolvedInstall)); Assert.Equal(new Uri(replacementManifest.Endpoint!), (await store.LoadAsync())!.Endpoint); - Assert.False(context.LocalAiRecoveryProviderTransition); + Assert.True(context.LocalAiRecoveryProviderTransition); Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + Assert.False(context.LocalAiRecoveryCleanupAllowed); } private static SetupContext CreateContext(string localDataDirectory, ICommandRunner commands) @@ -617,6 +1082,16 @@ private static SetupContext CreateContext(string localDataDirectory, ICommandRun localDataDir: localDataDirectory); } + private static LocalAiInstallManifest ReplacementManifest(LocalAiResolvedInstall original) => + original.Manifest with + { + ModelCatalogId = LocalModelCatalog.Qwen27BModelId, + ModelAlias = LocalModelCatalog.Qwen27BModelId, + Endpoint = "http://127.0.0.1:39876/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Endpoint!.AbsoluteUri], + }; + private static SetupContext CreateRecoveryContext( string localDataDirectory, ICommandRunner commands) @@ -647,7 +1122,8 @@ private static SetupContext CreateRecoveryContext( private static async Task SaveManifestAsync( string localDataDirectory, - string? fallbackModel = null) + string? fallbackModel = null, + int requestedPort = 0) { var paths = new LocalAiPaths(localDataDirectory); const string revision = "5bc3e238d916f48a861bac2f8a1990a0e9b7e98d"; @@ -680,7 +1156,7 @@ private static async Task SaveManifestAsync( SizeBytes = 1, Sha256 = new string('b', 64), }, - RequestedPort = 0, + RequestedPort = requestedPort, Endpoint = "http://127.0.0.1:28765/v1", GatewayFallbackModel = fallbackModel, ContextLength = LocalModelCatalog.NativeContextTokens, @@ -705,6 +1181,8 @@ private sealed class GatewayStateCommandRunner( public bool FailRestoreBatchOnce { get; set; } public bool LoseRestoreAcknowledgementOnce { get; set; } public bool ThrowOnNextCapture { get; set; } + public bool SupportsConditionalProviderSet { get; set; } = true; + public string? PrimaryJsonAfterConditionalProviderSet { get; set; } public List WslCalls { get; } = []; public Task RunAsync( @@ -728,6 +1206,42 @@ public Task RunInWslAsync( { ct.ThrowIfCancellationRequested(); WslCalls.Add(command); + if (environment is not null && environment.Count == 2 && + command.Contains("--expect-current-json", StringComparison.Ordinal)) + { + if (!SupportsConditionalProviderSet) + { + return Task.FromResult(new CommandResult( + 42, + "LOCAL_AI_CONDITIONAL_SET_UNSUPPORTED", + "", + TimeSpan.Zero, + TimedOut: false)); + } + string providerJson = Encoding.UTF8.GetString(Convert.FromBase64String( + environment["OPENCLAW_LOCAL_AI_PROVIDER_B64"])); + string expectedProviderJson = Encoding.UTF8.GetString(Convert.FromBase64String( + environment["OPENCLAW_LOCAL_AI_EXPECTED_PROVIDER_B64"])); + using JsonDocument currentProvider = JsonDocument.Parse(ProviderJson!); + using JsonDocument expectedProvider = JsonDocument.Parse(expectedProviderJson); + if (!JsonElement.DeepEquals(currentProvider.RootElement, expectedProvider.RootElement)) + { + return Task.FromResult(new CommandResult( + 1, + "", + "gateway provider changed", + TimeSpan.Zero, + TimedOut: false)); + } + ProviderJson = providerJson; + PrimaryJson = PrimaryJsonAfterConditionalProviderSet ?? PrimaryJson; + return Task.FromResult(new CommandResult( + 0, + "LOCAL_AI_GATEWAY_RESTORED", + "", + TimeSpan.Zero, + TimedOut: false)); + } if (environment is not null && environment.Count == 1) { if (FailRestoreBatchOnce && @@ -839,6 +1353,125 @@ private static string EncodeOrMissing(string? value) => value is null : Convert.ToBase64String(Encoding.UTF8.GetBytes(value)); } + private sealed class AcknowledgementFailingRuntime : ILocalAiRuntime + { + public LocalAiRuntimeSnapshot Snapshot => LocalAiRuntimeSnapshot.Initial( + new Uri("http://127.0.0.1:18800/v1"), + DateTimeOffset.UtcNow); + public event EventHandler? StateChanged + { + add { } + remove { } + } + public Task EnsureStartedAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task ResumeAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task StopAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task RestartAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task RefreshAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) => + throw new IOException("acknowledgement failed"); + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class SettlementTrackingRuntime(LocalAiManifestStore store) : ILocalAiRuntime + { + public int RestartForSetupCalls { get; private set; } + public int RestartForSetupRollbackCalls { get; private set; } + + public LocalAiRuntimeSnapshot Snapshot { get; private set; } = LocalAiRuntimeSnapshot.Initial( + new Uri("http://127.0.0.1:18800/v1"), + DateTimeOffset.UtcNow); + + public event EventHandler? StateChanged + { + add { } + remove { } + } + + public Task EnsureStartedAsync( + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task ResumeAsync( + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task StopAsync( + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task RestartAsync( + CancellationToken cancellationToken = default) => LoadSnapshotAsync(cancellationToken); + + public async Task RestartForSetupAsync( + CancellationToken cancellationToken = default) + { + RestartForSetupCalls++; + return await LoadSnapshotAsync(cancellationToken); + } + + public Task RefreshAsync( + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) => Task.FromResult(Snapshot); + + public async Task RestartForSetupRollbackAsync( + CancellationToken cancellationToken = default) + { + RestartForSetupRollbackCalls++; + return await LoadSnapshotAsync(cancellationToken); + } + + private async Task LoadSnapshotAsync( + CancellationToken cancellationToken) + { + LocalAiResolvedInstall restored = await store.LoadAsync(cancellationToken) + ?? throw new InvalidDataException("The restored Local AI receipt is unavailable."); + Snapshot = HealthySnapshot(restored); + return Snapshot; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class TrackingRuntimeAcquirer : ILlamaRuntimeAcquirer + { + public int RemoveCalls { get; private set; } + + public Task InstallAsync( + string localDataDirectory, + LlamaRuntimeVariant runtime, + IProgress? progress, + CancellationToken cancellationToken) => throw new NotSupportedException(); + + public void RemoveInstalledRuntime( + string localDataDirectory, + LlamaRuntimeInstallResult install) => RemoveCalls++; + } + + private static LocalAiRuntimeSnapshot HealthySnapshot(LocalAiResolvedInstall install) => new( + LocalAiRuntimeState.Healthy, + LocalAiOwnership.CompanionManaged, + install.Endpoint!, + install.Manifest.EngineVersion, + install.Manifest.ModelCatalogId, + new LocalAiModelEvidence( + LocalAiModelAvailabilityState.Verified, + DateTimeOffset.UtcNow, + install.Manifest.ModelAsset.Sha256, + install.Manifest.ModelAsset.SizeBytes), + 42, + DateTimeOffset.UtcNow, + null, + DateTimeOffset.UtcNow) + { + GatewayRouteRequiresResolution = false, + }; + private sealed class DelegatingRollbackStep( string id, Func rollback, diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index 338ed74a2..26c36ee1c 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -1280,14 +1280,21 @@ public async Task Reconciler_UpgradesRetiredRuntimeReceiptInsteadOfFailingSetup( } [Theory] - [InlineData(false, null)] - [InlineData(true, null)] - [InlineData(false, "after-reconcile")] - [InlineData(true, "after-reconcile")] - [InlineData(false, "after-persist")] - [InlineData(true, "after-persist")] + [InlineData(false, false, null)] + [InlineData(true, false, null)] + [InlineData(false, true, null)] + [InlineData(true, true, null)] + [InlineData(false, false, "after-reconcile")] + [InlineData(true, false, "after-reconcile")] + [InlineData(false, true, "after-reconcile")] + [InlineData(true, true, "after-reconcile")] + [InlineData(false, false, "after-persist")] + [InlineData(true, false, "after-persist")] + [InlineData(false, true, "after-persist")] + [InlineData(true, true, "after-persist")] public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailure( bool usesHubCache, + bool pendingReplacement, string? failureStage) { using var temp = new TempDirectory(); @@ -1307,6 +1314,18 @@ public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailur { GatewayFallbackModel = "openai/gpt-5", }; + if (pendingReplacement) + { + manifest = manifest with + { + ReplacedManifest = manifest with + { + ModelCatalogId = "prior-model", + ModelAlias = "prior-model", + }, + PreviousEndpoints = [manifest.Endpoint!], + }; + } string oldExecutable = paths.ResolveContainedPath(manifest.ExecutablePath, "executable"); Directory.CreateDirectory(Path.GetDirectoryName(oldExecutable)!); await File.WriteAllTextAsync(oldExecutable, "old-server"); @@ -1320,6 +1339,7 @@ public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailur var context = CreateContext(temp.Path, confirmDestructive: false); context.Config.LocalAi.Enabled = true; + context.Config.LocalAiRecoveryGatewayId = pendingReplacement ? "gateway-id" : null; context.Config.RollbackOnFailure = true; context.LocalAiPort = manifest.RequestedPort; context.LocalAiEligibility = new LocalInferenceEligibilityResult( @@ -1355,7 +1375,8 @@ public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailur new ReconcileLocalAiInstallationStep(reconciler), new UpgradeCheckpointStep("after-reconcile", ctx => { - Assert.Null(ctx.LocalAiRecoveryOriginalInstall); + Assert.Equal(pendingReplacement, ctx.LocalAiRecoveryOriginalInstall is not null); + Assert.Equal(pendingReplacement, ctx.LocalAiRecoveryPendingInstall is not null); Assert.Equal(manifest.SchemaVersion, ctx.LocalAiUpgradeOriginalInstall?.Manifest.SchemaVersion); Assert.Equal(oldExecutable, ctx.LocalAiUpgradeOriginalInstall?.ExecutablePath); Assert.Equal(cacheRoot, ctx.LocalAiModelInstall?.CacheRoot); @@ -1376,6 +1397,22 @@ public async Task RuntimeUpgrade_MigratesModelAndRestoresOriginalReceiptOnFailur Assert.Equal(manifest.InstalledAtUtc, upgraded.Manifest.InstalledAtUtc); Assert.Equal(manifest.GatewayFallbackModel, upgraded.Manifest.GatewayFallbackModel); Assert.Null(upgraded.Endpoint); + Assert.Equal(pendingReplacement, upgraded.Manifest.ReplacedManifest is not null); + if (pendingReplacement) + { + Assert.Equal("prior-model", upgraded.Manifest.ReplacedManifest!.ModelCatalogId); + Assert.Equal(upgraded.Manifest.RuntimeId, upgraded.Manifest.ReplacedManifest.RuntimeId); + Assert.Equal(upgraded.Manifest.RuntimeAssets, upgraded.Manifest.ReplacedManifest.RuntimeAssets); + Assert.Equal( + upgraded.Manifest.ReplacedManifest, + ctx.LocalAiRecoveryOriginalInstall?.Manifest); + LocalAiResolvedInstall pendingRoute = Assert.IsType( + ctx.LocalAiRecoveryPendingInstall); + Assert.Equal(manifest.Endpoint, pendingRoute.Manifest.Endpoint); + Assert.Equal(upgraded.Manifest.RuntimeId, pendingRoute.Manifest.RuntimeId); + Assert.Equal(upgraded.Manifest.RuntimeAssets, pendingRoute.Manifest.RuntimeAssets); + Assert.Equal(upgraded.Manifest.ReplacedManifest, pendingRoute.Manifest.ReplacedManifest); + } newExecutable = upgraded.ExecutablePath; Assert.True(File.Exists(newExecutable)); return failureStage == "after-persist"; @@ -1490,7 +1527,43 @@ await Assert.ThrowsAsync(() => } [Fact] - public async Task Reconciler_RecoveryRetainsReceiptWhileMissingModelIsRepaired() + public async Task Reconciler_RecoveryReusesRuntimeWhenSelectedModelChanges() + { + using var temp = new TempDirectory(); + LocalInferencePlan installed = CatalogPlan(); + LocalModelInfo replacementModel = LocalModelCatalog.Models.First( + model => model.Id != installed.Model.Id); + var replacement = new LocalInferencePlan( + installed.Runtime, + replacementModel, + LocalModelCatalog.GetProfiles(replacementModel)[0], + LocalInferenceModelSelectionOrigin.Explicit); + const string gpuId = "GPU-0"; + var paths = new LocalAiPaths(temp.Path); + LocalAiInstallManifest manifest = CreateManifest(temp.Path, installed, gpuId); + await new LocalAiManifestStore(paths).SaveAsync(manifest); + + LocalAiReconcileResult result = await new LocalAiInstallReconciler( + new ValidRuntimeInspector(), + new AcceptingModelVerifier()) + .ReconcileAsync( + temp.Path, + replacement, + gpuId, + CancellationToken.None, + allowIncompleteInstallation: true); + + Assert.False(result.Reused); + Assert.False(result.RuntimeInstall!.CreatedThisRun); + Assert.Null(result.ModelInstall); + Assert.Equal(manifest.ModelCatalogId, result.OriginalInstall?.Manifest.ModelCatalogId); + Assert.Null((await new LocalAiManifestStore(paths).LoadAsync())!.Manifest.ReplacedManifest); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Reconciler_RecoveryRetainsReceiptWhileMissingModelIsRepaired(bool pendingReplacement) { using var temp = new TempDirectory(); LocalInferencePlan plan = CatalogPlan(); @@ -1498,6 +1571,16 @@ public async Task Reconciler_RecoveryRetainsReceiptWhileMissingModelIsRepaired() var paths = new LocalAiPaths(temp.Path); var store = new LocalAiManifestStore(paths); LocalAiInstallManifest manifest = CreateManifest(temp.Path, plan, gpuId); + if (pendingReplacement) + { + LocalModelInfo priorModel = LocalModelCatalog.Models.First(model => model.Id != plan.Model.Id); + var priorPlan = new LocalInferencePlan( + plan.Runtime, + priorModel, + LocalModelCatalog.GetProfiles(priorModel)[0], + LocalInferenceModelSelectionOrigin.Explicit); + manifest = manifest with { ReplacedManifest = CreateManifest(temp.Path, priorPlan, gpuId) }; + } await store.SaveAsync(manifest); var reconciler = new LocalAiInstallReconciler( new ValidRuntimeInspector(), @@ -1515,9 +1598,183 @@ public async Task Reconciler_RecoveryRetainsReceiptWhileMissingModelIsRepaired() Assert.Equal(manifest.Endpoint, result.OriginalInstall!.Manifest.Endpoint); Assert.NotNull(result.RuntimeInstall); Assert.Null(result.ModelInstall); + Assert.Equal(pendingReplacement, result.PendingReplacement is not null); Assert.True(File.Exists(paths.ManifestPath)); } + [Fact] + public async Task FinalizeReplacement_ClearsRollbackReceipt() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pending); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pending); + + StepResult result = await new FinalizeLocalAiModelReplacementStep() + .ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + LocalAiInstallManifest committed = (await store.LoadAsync())!.Manifest; + Assert.Null(committed.ReplacedManifest); + Assert.Null(committed.PreviousEndpoints); + string json = await File.ReadAllTextAsync(new LocalAiPaths(temp.Path).ManifestPath); + Assert.DoesNotContain("previousEndpoints", json, StringComparison.Ordinal); + Assert.DoesNotContain("replacedManifest", json, StringComparison.Ordinal); + } + + [Fact] + public async Task PersistRollback_RestoresOriginalReceiptBeforeGatewayGuard() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pending); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pending); + context.LocalAiRecoveryOriginalInstall = store.ResolveAndValidate(original); + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryReceiptRollbackAllowed = false; + + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + + LocalAiInstallManifest restored = (await store.LoadAsync())!.Manifest; + Assert.Equal(original.ModelCatalogId, restored.ModelCatalogId); + Assert.Null(restored.ReplacedManifest); + Assert.Null(restored.PreviousEndpoints); + Assert.False(context.LocalAiRecoveryProviderTransition); + Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + } + + [Fact] + public async Task Rollback_RestoresUpgradeAndRemovesTaskOwnedRuntimeBeforeGatewayConfiguration() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest replacement = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(replacement); + var acquirer = new TrackingRuntimeAcquirer(); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(replacement); + context.LocalAiUpgradeOriginalInstall = store.ResolveAndValidate(original); + context.LocalAiRuntimeInstall = new LlamaRuntimeInstallResult( + temp.Path, + Path.Combine(temp.Path, "llama-server.exe"), + LlamaRuntimeInstallDisposition.Installed, + CreatedThisRun: true, + VerifiedArchives: [], + Rollback: null); + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryRollbackUncertain = true; + context.LocalAiRecoveryReceiptRollbackAllowed = false; + + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + await new AcquireLocalAiRuntimeStep(acquirer).RollbackAsync(context, CancellationToken.None); + + Assert.Equal(original.ModelCatalogId, (await store.LoadAsync())!.Manifest.ModelCatalogId); + Assert.Null(context.LocalAiUpgradeOriginalInstall); + Assert.Null(context.LocalAiRuntimeInstall); + Assert.False(context.LocalAiRecoveryRollbackUncertain); + Assert.Equal(1, acquirer.RemoveCalls); + } + + [Fact] + public async Task Rollback_PreservesPublishedUpgradeWhenGatewayCompensationIsUncertain() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest replacement = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(replacement); + var acquirer = new TrackingRuntimeAcquirer(); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(replacement); + context.LocalAiUpgradeOriginalInstall = store.ResolveAndValidate(original); + context.LocalAiRuntimeInstall = new LlamaRuntimeInstallResult( + temp.Path, + Path.Combine(temp.Path, "llama-server.exe"), + LlamaRuntimeInstallDisposition.Installed, + CreatedThisRun: true, + VerifiedArchives: [], + Rollback: null); + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryRollbackUncertain = true; + context.LocalAiRecoveryReceiptRollbackAllowed = false; + context.LocalAiRecoveryGatewayConfigurationStartedThisRun = true; + + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + await new AcquireLocalAiRuntimeStep(acquirer).RollbackAsync(context, CancellationToken.None); + + Assert.Equal(replacement.ModelCatalogId, (await store.LoadAsync())!.Manifest.ModelCatalogId); + Assert.NotNull(context.LocalAiUpgradeOriginalInstall); + Assert.NotNull(context.LocalAiRuntimeInstall); + Assert.Equal(0, acquirer.RemoveCalls); + } + + [Fact] + public async Task RecoveryGuard_RejectsStaleReplacementHistoryAndBlocksCleanup() + { + using var temp = new TempDirectory(); + LocalAiInstallManifest original = CreateManifest(temp.Path, CatalogPlan(), "GPU-0"); + LocalAiInstallManifest pending = original with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:18802/v1", + ReplacedManifest = original, + PreviousEndpoints = [original.Endpoint!], + }; + LocalAiInstallManifest newer = pending with + { + Endpoint = "http://127.0.0.1:18803/v1", + PreviousEndpoints = [original.Endpoint!, pending.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(newer); + SetupContext context = CreateContext(temp.Path, confirmDestructive: false); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pending); + context.LocalAiRecoveryOriginalInstall = store.ResolveAndValidate(original); + context.LocalAiRecoveryProviderTransition = true; + + context.LocalAiRecoveryReceiptRollbackAllowed = true; + await Assert.ThrowsAsync(() => + new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("restarted")), + (_, _) => Task.FromResult(true)) + .RollbackAsync(context, CancellationToken.None)); + + LocalAiInstallManifest retained = (await store.LoadAsync())!.Manifest; + Assert.Equal(newer.Endpoint, retained.Endpoint); + Assert.Equal(newer.PreviousEndpoints, retained.PreviousEndpoints); + Assert.True(context.LocalAiRecoveryRollbackUncertain); + Assert.False(context.LocalAiRecoveryReceiptRollbackAllowed); + Assert.False(context.LocalAiRecoveryCleanupAllowed); + } + [Fact] public async Task ReconcileStep_RecoveryPinsIncompleteReceiptAsRollbackBaseline() { @@ -1552,8 +1809,10 @@ public async Task ReconcileStep_RecoveryPinsIncompleteReceiptAsRollbackBaseline( Assert.Null(context.LocalAiModelInstall); } - [Fact] - public async Task RecoveryPipeline_RewritesIncompleteReceiptAfterModelRepair() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RecoveryPipeline_RewritesIncompleteReceiptAfterModelRepair(bool pendingReplacement) { using var temp = new TempDirectory(); LocalInferencePlan plan = CatalogPlan(); @@ -1563,6 +1822,19 @@ public async Task RecoveryPipeline_RewritesIncompleteReceiptAfterModelRepair() RequestedPort = 18803, GatewayFallbackModel = "openai/gpt-5", }; + if (pendingReplacement) + { + LocalModelInfo priorModel = LocalModelCatalog.Models.First(model => model.Id != plan.Model.Id); + var priorPlan = new LocalInferencePlan( + plan.Runtime, + priorModel, + LocalModelCatalog.GetProfiles(priorModel)[0], + LocalInferenceModelSelectionOrigin.Explicit); + manifest = manifest with + { + ReplacedManifest = CreateManifest(temp.Path, priorPlan, gpuId) with { RequestedPort = 18803 }, + }; + } var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); await store.SaveAsync(manifest); LocalAiResolvedInstall original = (await store.LoadAsync())!; @@ -1601,6 +1873,13 @@ public async Task RecoveryPipeline_RewritesIncompleteReceiptAfterModelRepair() Assert.Equal(CacheRoot(temp.Path), repaired.Manifest.ModelCacheRoot); Assert.Equal(repaired.Manifest.CachedModelPath, repaired.ModelPath); Assert.False(context.LocalAiManifestCreatedThisRun); + Assert.Equal(pendingReplacement, context.LocalAiRecoveryRollbackUncertain); + if (!pendingReplacement) + { + Assert.Null(repaired.Manifest.PreviousEndpoints); + string json = await File.ReadAllTextAsync(new LocalAiPaths(temp.Path).ManifestPath); + Assert.DoesNotContain("previousEndpoints", json, StringComparison.Ordinal); + } } [Fact] @@ -2464,6 +2743,22 @@ public Task InspectAsync( Task.FromResult(new LlamaRuntimeInspection(false, "invalid", "simulated corrupted runtime")); } + private sealed class TrackingRuntimeAcquirer : ILlamaRuntimeAcquirer + { + public int RemoveCalls { get; private set; } + + public Task InstallAsync( + string localDataDirectory, + LlamaRuntimeVariant runtime, + IProgress? progress, + CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public void RemoveInstalledRuntime( + string localDataDirectory, + LlamaRuntimeInstallResult install) => RemoveCalls++; + } + /// Accepts the runtime but cancels the install as inspection returns. private sealed class CancellingRuntimeInspector(CancellationTokenSource cancellation) : ILlamaRuntimeInspector { diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiOnboardingTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiOnboardingTests.cs index 2e89081d2..31fcf30ed 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiOnboardingTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiOnboardingTests.cs @@ -599,7 +599,8 @@ public async Task Host_ExplicitUseRequiresPublicationAdmissionAndReturnsExactIde private static SetupLocalAiHost Host(GatewayRegistry registry, FakeRuntime runtime, Func install, Func? provider = null) => new(() => Task.FromResult(new LocalAiSetupResolution(LocalAiSetupRoute.Recovery, - new("gateway", "Managed", 18789, install()?.Manifest.ModelCatalogId, install()?.Manifest.RequestedPort))), + new("gateway", "Managed", 18789, install()?.Manifest.ModelCatalogId, + install()?.Manifest.RequestedPort, PinModelSelection: false))), () => registry, () => runtime, _ => Task.FromResult(install()), (_, _) => Task.FromResult(true), _ => Task.FromResult(Hardware), provider ?? (() => throw new InvalidOperationException("No route mutation expected."))); @@ -637,7 +638,13 @@ public async Task WslUseAndRepairRejectNativeOwnershipBeforeAdmission(bool repai new LocalAiGatewayDistroResolver(registry), NullLogger.Instance), NullLogger.Instance); var host = new SetupLocalAiHost( () => Task.FromResult(new LocalAiSetupResolution(LocalAiSetupRoute.Recovery, - new("gateway", "Managed", 18789, install.Manifest.ModelCatalogId, install.Manifest.RequestedPort))), + new( + "gateway", + "Managed", + 18789, + install.Manifest.ModelCatalogId, + install.Manifest.RequestedPort, + PinModelSelection: false))), () => registry, () => runtime, _ => Task.FromResult(install), (_, _) => Task.FromResult(true), _ => Task.FromResult(Hardware), () => throw new InvalidOperationException("WSL admission must not run."), diff --git a/tests/OpenClaw.SetupEngine.Tests/NativeLocalAiAcquisitionTests.cs b/tests/OpenClaw.SetupEngine.Tests/NativeLocalAiAcquisitionTests.cs index 21b1d1f73..48b7e44e2 100644 --- a/tests/OpenClaw.SetupEngine.Tests/NativeLocalAiAcquisitionTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/NativeLocalAiAcquisitionTests.cs @@ -37,6 +37,7 @@ public void WslRecoveryRetainsItsExistingNetworkingAndGatewayVerification() Assert.Contains(steps, step => step is ConfigureLocalAiWslNetworkingStep); Assert.Contains(steps, step => step is VerifyLocalAiWslStep); Assert.Contains(steps, step => step is ConfigureLocalAiGatewayStep); - Assert.IsType(steps[^1]); + Assert.IsType(steps[^2]); + Assert.IsType(steps[^1]); } } diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupInstallationProgressTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupInstallationProgressTests.cs index 179ce904d..9c4f8aeb1 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupInstallationProgressTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupInstallationProgressTests.cs @@ -107,9 +107,9 @@ public void Cancellation_BetweenStepsDoesNotOverwriteFailureOrSuccess() } [Fact] - public void Recovery_PreparesExistingGatewayThenLocalAiWithoutGatewayInstallClaim() + public void Recovery_GroupsGatewayPreservationWithLocalAiInstallation() { - Assert.Equal(SetupInstallationPhase.Prepare, SetupInstallationProgress.PhaseFor("preserve-local-ai-recovery-gateway", true)); + Assert.Equal(SetupInstallationPhase.Install, SetupInstallationProgress.PhaseFor("preserve-local-ai-recovery-gateway", true)); Assert.Equal(SetupInstallationPhase.Install, SetupInstallationProgress.PhaseFor("acquire-local-ai-model", true)); Assert.Equal(SetupInstallationPhase.Connect, SetupInstallationProgress.PhaseFor("restart-gateway", true)); } diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs index c74b2d256..a4641040e 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupPipelineTests.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; using OpenClaw.Connection.LocalAi; +using OpenClaw.Shared.Inference.Catalog; using OpenClaw.TestSupport; namespace OpenClaw.SetupEngine.Tests; @@ -199,6 +200,7 @@ public void BuildLocalAiRecoverySteps_PreservesExistingWslGateway() Assert.DoesNotContain(steps, step => step is ValidateDistroInstallPathStep); Assert.Equal(2, steps.Count(step => step is ValidateLocalAiRecoveryGatewayStep)); Assert.Contains(steps, step => step is PreserveLocalAiRecoveryGatewayStep); + Assert.Contains(steps, step => step is ValidateLocalAiRecoveryGatewayCompatibilityStep); Assert.DoesNotContain(steps, step => step is CleanupStaleDistroStep); Assert.DoesNotContain(steps, step => step is CleanupStaleGatewayStep); Assert.DoesNotContain(steps, step => step is CreateWslInstanceStep); @@ -207,16 +209,26 @@ public void BuildLocalAiRecoverySteps_PreservesExistingWslGateway() Assert.Contains(steps, step => step is AcquireLocalAiRuntimeStep); Assert.Contains(steps, step => step is AcquireLocalAiModelStep); Assert.Contains(steps, step => step is VerifyLocalAiWslStep); - Assert.IsType(steps[^2]); - Assert.IsType(steps[^1]); + Assert.IsType(steps[^3]); + Assert.IsType(steps[^2]); + Assert.IsType(steps[^1]); Assert.True( - steps.FindIndex(step => step is ValidateLocalAiRecoveryGatewayStep) < + steps.FindIndex(step => step is ReconcileLocalAiInstallationStep) < + steps.FindIndex(step => step is ValidateLocalAiRecoveryGatewayCompatibilityStep)); + Assert.True( + steps.FindIndex(step => step is ValidateLocalAiRecoveryGatewayCompatibilityStep) < steps.FindIndex(step => step is AcquireLocalAiRuntimeStep)); + Assert.True( + steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep) < + steps.FindIndex(step => step is StartLocalAiRuntimeStep)); + Assert.True( + steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep) < + steps.FindIndex(step => step is ConfigureLocalAiGatewayStep)); Assert.True( steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep) < steps.FindIndex(step => step is ConfigureLocalAiWslNetworkingStep)); - Assert.IsType( - steps[steps.FindIndex(step => step is ConfigureLocalAiWslNetworkingStep) - 1]); + Assert.IsType( + steps[steps.FindIndex(step => step is PreserveLocalAiRecoveryGatewayStep) - 1]); } [Fact] @@ -233,6 +245,119 @@ public async Task ValidateLocalAiRecoveryGateway_MissingDistro_BlocksBeforeRecov Assert.Contains("run full setup", result.Message, StringComparison.OrdinalIgnoreCase); } + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_AutomaticPortRejectsLegacyGateway() + { + using var temp = new TempDirectory("local-ai-compatibility-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Port = 18801; + var context = CreateContext(config, localDataDir: temp.Path); + context.LocalAiRecoveryOriginalInstall = CreateLocalAiResolvedInstall(temp.Path, 18801); + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + Task.FromResult(new CommandResult( + 42, + ValidateLocalAiRecoveryGatewayCompatibilityStep.UnsupportedMarker, + string.Empty, + TimeSpan.Zero, + false))); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.FailedTerminal, result.Outcome); + Assert.Contains("Update the Gateway", result.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_NoPriorReceiptSkipsProbe() + { + var context = CreateContext(LocalAiRecoveryConfig()); + var probeCalls = 0; + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + { + probeCalls++; + return Task.FromResult(new CommandResult(1, string.Empty, "failed", TimeSpan.Zero, false)); + }); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Skipped, result.Outcome); + Assert.Equal(0, probeCalls); + } + + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_FixedPortSkipsProbe() + { + using var temp = new TempDirectory("local-ai-compatibility-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Port = 0; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(temp.Path, 18801); + context.LocalAiRecoveryOriginalInstall = install with + { + Manifest = install.Manifest with { RequestedPort = 18801 }, + }; + var probeCalls = 0; + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + { + probeCalls++; + return Task.FromResult(new CommandResult(1, string.Empty, "failed", TimeSpan.Zero, false)); + }); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Skipped, result.Outcome); + Assert.Equal(0, probeCalls); + } + + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_AutomaticPortAcceptsConditionalWrites() + { + using var temp = new TempDirectory("local-ai-compatibility-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + context.LocalAiRecoveryOriginalInstall = CreateLocalAiResolvedInstall(temp.Path, 18801); + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + Task.FromResult(new CommandResult( + 0, + ValidateLocalAiRecoveryGatewayCompatibilityStep.SupportedMarker, + string.Empty, + TimeSpan.Zero, + false))); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + } + + [Fact] + public async Task ValidateRecoveryGatewayCompatibility_ChecksAutomaticRollbackReceipt() + { + using var temp = new TempDirectory("local-ai-compatibility-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall automatic = CreateLocalAiResolvedInstall(temp.Path, 18801); + LocalAiResolvedInstall fixedPort = automatic with + { + Manifest = automatic.Manifest with { RequestedPort = 18802 }, + }; + context.LocalAiRecoveryOriginalInstall = automatic; + context.LocalAiRecoveryPendingInstall = fixedPort; + var probeCalls = 0; + var step = new ValidateLocalAiRecoveryGatewayCompatibilityStep((_, _) => + { + probeCalls++; + return Task.FromResult(new CommandResult( + 42, + ValidateLocalAiRecoveryGatewayCompatibilityStep.UnsupportedMarker, + string.Empty, + TimeSpan.Zero, + false)); + }); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.FailedTerminal, result.Outcome); + Assert.Equal(1, probeCalls); + } + [Fact] public async Task ValidateLocalAiRecoveryGateway_AppOwnedDistro_AllowsRecovery() { @@ -332,6 +457,193 @@ public async Task RestartGatewayStep_FailureArmsRecoveryRollbackRestart() Assert.True(context.LocalAiRecoveryStoppedWsl); } + [Fact] + public async Task FinalizationFailure_RestartsGatewayAfterConfigurationRollback() + { + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config); + bool configurationRestored = false; + bool restartedAfterRestore = false; + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep((_, _) => + { + restartedAfterRestore = configurationRestored; + return Task.FromResult(StepResult.Ok("restarted")); + }), + new MockStep( + "configure-local-ai-gateway", + (_, _) => Task.FromResult(StepResult.Ok("configured")), + (_, _) => + { + configurationRestored = true; + return Task.CompletedTask; + }), + new RestartGatewayStep((_, _) => Task.FromResult(StepResult.Ok("restarted"))), + new MockStep( + "finalize-local-ai-model-replacement", + (_, _) => Task.FromResult(StepResult.Fail("finalization failed"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.True(configurationRestored); + Assert.True(restartedAfterRestore); + Assert.False(context.LocalAiRecoveryStoppedWsl); + } + + [Fact] + public async Task LaterFailure_RestoresWslNetworkingBeforeRecoveryGuardRestartsGateway() + { + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config); + var rollbackOrder = new List(); + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep((_, _) => + { + rollbackOrder.Add("restart"); + return Task.FromResult(StepResult.Ok("restarted")); + }), + new MockStep( + "configure-local-ai-wsl-networking", + (ctx, _) => + { + ctx.LocalAiRecoveryStoppedWsl = true; + return Task.FromResult(StepResult.Ok("configured")); + }, + (_, _) => + { + rollbackOrder.Add("networking"); + return Task.CompletedTask; + }), + new MockStep("failure", (_, _) => Task.FromResult(StepResult.Fail("failed"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal(["networking", "restart"], rollbackOrder); + } + + [Fact] + public async Task WslNetworkingFailure_RollsBackArmedRecoveryGuard() + { + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config); + var restartCalls = 0; + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep((_, _) => + { + restartCalls++; + return Task.FromResult(StepResult.Ok("restarted")); + }), + new MockStep( + "configure-local-ai-wsl-networking", + (ctx, _) => + { + ctx.LocalAiRecoveryStoppedWsl = true; + return Task.FromResult(StepResult.Fail("failed after stopping WSL")); + }), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal(1, restartCalls); + Assert.False(context.LocalAiRecoveryStoppedWsl); + } + + [Fact] + public async Task BorrowedRuntimeFailure_RollsBackArmedRecoveryGuard() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-failure-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Enabled = true; + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(temp.Path, 18801); + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + context.LocalAiRecoveryOriginalInstall = install; + context.LocalAiRecoveryReceiptRollbackAllowed = true; + var restartCalls = 0; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)) + { + RestartForSetupHandler = _ => + { + restartCalls++; + LocalAiRuntimeSnapshot snapshot = HealthySnapshot(install); + return Task.FromResult(restartCalls == 1 + ? snapshot with + { + State = LocalAiRuntimeState.Failed, + Ownership = LocalAiOwnership.None, + ProcessId = null, + } + : snapshot); + }, + }; + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("not needed")), + (_, _) => Task.FromResult(true)), + new StartLocalAiRuntimeStep(_ => runtime), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(1, runtime.RestartForSetupRollbackCalls); + Assert.True(context.LocalAiBorrowedRuntimeRestored); + Assert.Equal(LocalAiRuntimeState.Healthy, runtime.Snapshot.State); + } + + [Fact] + public async Task OrdinaryRecoveryRollback_PreservesConcurrentlyUpdatedReceipt() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-concurrent-receipt-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Enabled = true; + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, 18801); + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(original.Manifest); + context.LocalAiResolvedInstall = original; + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryReceiptRollbackAllowed = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(original)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + LocalAiInstallManifest concurrent = original.Manifest with + { + Endpoint = "http://127.0.0.1:18803/v1", + }; + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("not needed")), + (_, _) => Task.FromResult(true)), + new MockStep( + "failure", + (_, _) => Task.FromResult(StepResult.Fail("failed")), + async (_, ct) => await store.SaveAsync(concurrent, ct)), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal(concurrent.Endpoint, (await store.LoadAsync())!.Manifest.Endpoint); + Assert.True(context.LocalAiRecoveryRollbackUncertain); + Assert.Equal(0, runtime.RestartForSetupRollbackCalls); + } + /// /// Regression guard for a rollback race: if the Gateway could not be confirmed switched back /// to the original (A) endpoint, the replacement (B) runtime must be kept alive rather than @@ -342,6 +654,7 @@ public async Task StartLocalAiRuntimeStep_KeepsReplacementRuntimeWhenGatewayRoll { var context = CreateContext(LocalAiRecoveryConfig()); context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryRollbackUncertain = true; context.LocalAiRecoveryReceiptRollbackAllowed = false; var runtime = new DisposeTrackingRuntime(); context.LocalAiRuntime = runtime; @@ -358,6 +671,7 @@ public async Task StartLocalAiRuntimeStep_DisposesRuntimeWhenGatewayRollbackConf { var context = CreateContext(LocalAiRecoveryConfig()); context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryRollbackUncertain = true; context.LocalAiRecoveryReceiptRollbackAllowed = true; var runtime = new DisposeTrackingRuntime(); context.LocalAiRuntime = runtime; @@ -385,6 +699,547 @@ public async Task StartLocalAiRuntimeStep_DisposesRuntimeOutsideRecoveryTransiti Assert.Null(context.LocalAiRuntime); } + [Fact] + public async Task StartLocalAiRuntimeStep_RestartsBorrowedTrayRuntimeForReplacement() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + context.LocalAiUpgradeOriginalInstall = install; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + + StepResult result = await new StartLocalAiRuntimeStep().ExecuteAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(0, runtime.RestartCalls); + Assert.Equal(0, runtime.EnsureStartedCalls); + Assert.Equal(0, runtime.DisposeCalls); + Assert.Same(runtime, context.LocalAiRuntime); + } + + [Fact] + public async Task StartLocalAiRuntimeStep_RestartsBorrowedTrayRuntimeForOrdinaryRecovery() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-no-replacement-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + context.LocalAiRecoveryOriginalInstall = install; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + var step = new StartLocalAiRuntimeStep(); + + StepResult result = await step.ExecuteAsync(context, CancellationToken.None); + await step.RollbackAsync(context, CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(1, runtime.StopForSetupCalls); + Assert.True(context.LocalAiBorrowedRuntimeRestartedThisRun); + } + + [Fact] + public async Task StartLocalAiRuntimeStep_RestartsBorrowedTrayRuntimeForFirstRecoveryInstall() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-first-install-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + + StepResult result = await new StartLocalAiRuntimeStep().ExecuteAsync( + context, + CancellationToken.None); + + Assert.Equal(StepOutcome.Success, result.Outcome); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.True(context.LocalAiBorrowedRuntimeRestartedThisRun); + } + + [Fact] + public async Task StartLocalAiRuntimeStep_CancellationAfterEndpointCommitAdoptsRollbackBaseline() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-cancel-after-endpoint-"); + using var cancellation = new CancellationTokenSource(); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Enabled = true; + config.RollbackOnFailure = true; + var context = CreateContext(config, cancellation.Token, localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, port: 18801); + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:18802/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Manifest.Endpoint!], + }; + LocalAiInstallManifest movedManifest = pendingManifest with + { + Endpoint = "http://127.0.0.1:18803/v1", + PreviousEndpoints = [original.Manifest.Endpoint!, pendingManifest.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pendingManifest); + LocalAiResolvedInstall pending = store.ResolveAndValidate(pendingManifest); + LocalAiResolvedInstall moved = store.ResolveAndValidate(movedManifest); + context.LocalAiResolvedInstall = pending; + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRecoveryReceiptRollbackAllowed = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(pending)) + { + RestartForSetupHandler = async _ => + { + await store.SaveAsync(movedManifest, CancellationToken.None); + cancellation.Cancel(); + return HealthySnapshot(moved); + }, + RestartHandler = async ct => + { + LocalAiResolvedInstall restored = await store.LoadAsync(ct) + ?? throw new InvalidDataException("restored receipt missing"); + return HealthySnapshot(restored); + }, + }; + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + var pipeline = new SetupPipeline([ + new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("not needed")), + (_, _) => Task.FromResult(true)), + new StartLocalAiRuntimeStep(_ => runtime), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Cancelled, result.Outcome); + LocalAiResolvedInstall restored = (await store.LoadAsync())!; + Assert.Equal(original.Manifest.ModelCatalogId, restored.Manifest.ModelCatalogId); + Assert.Equal(1, runtime.RestartForSetupRollbackCalls); + Assert.True(context.LocalAiBorrowedRuntimeRestored); + Assert.False(context.LocalAiRecoveryRollbackUncertain); + } + + [Fact] + public async Task StartLocalAiRuntimeStep_RejectsBorrowedRuntimeOutsideRecovery() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-unarmed-"); + var context = CreateContext(new SetupConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall install = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(install.Manifest); + context.LocalAiResolvedInstall = install; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(install)); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + + StepResult result = await new StartLocalAiRuntimeStep().ExecuteAsync( + context, + CancellationToken.None); + + Assert.Equal(StepOutcome.FailedTerminal, result.Outcome); + Assert.Equal(0, runtime.RestartForSetupCalls); + Assert.False(context.LocalAiBorrowedRuntimeRestartedThisRun); + } + + [Fact] + public async Task ResetRouterAsync_UsesSetupScopedRestartForBorrowedRuntime() + { + var runtime = new DisposeTrackingRuntime(); + + await VerifyLocalAiInferenceStep.ResetRouterAsync(runtime, setupScoped: true); + + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(0, runtime.RestartCalls); + } + + [Fact] + public async Task ResetRouterAsync_RefreshesBorrowedRuntimeReceiptAfterAutomaticPortMove() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-reset-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, port: 18801); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:18803/v1" }, + Endpoint = new Uri("http://127.0.0.1:18803/v1"), + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(original.Manifest); + context.LocalAiResolvedInstall = original; + context.LocalAiRuntimeBorrowed = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(original)) + { + RestartForSetupHandler = async ct => + { + await store.SaveAsync(moved.Manifest, ct); + return HealthySnapshot(moved); + }, + }; + + LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync( + context, + runtime); + + Assert.Equal(moved.Endpoint, reset.Endpoint); + Assert.Equal(moved.Endpoint, context.LocalAiResolvedInstall.Endpoint); + } + + [Fact] + public async Task ResetRouterAsync_DoesNotAdoptConcurrentReceiptChanges() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-reset-concurrent-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, port: 18801); + LocalAiResolvedInstall concurrent = original with + { + Manifest = original.Manifest with + { + Endpoint = "http://127.0.0.1:18803/v1", + ContextLength = original.Manifest.ContextLength + 1, + }, + Endpoint = new Uri("http://127.0.0.1:18803/v1"), + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(original.Manifest); + context.LocalAiResolvedInstall = original; + context.LocalAiRuntimeBorrowed = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(original)) + { + RestartForSetupHandler = async ct => + { + await store.SaveAsync(concurrent.Manifest, ct); + return HealthySnapshot(concurrent); + }, + }; + + await VerifyLocalAiInferenceStep.ResetRouterAsync(context, runtime); + + Assert.Same(original, context.LocalAiResolvedInstall); + Assert.Equal(concurrent.Manifest.ContextLength, (await store.LoadAsync())!.Manifest.ContextLength); + } + + [Fact] + public async Task ResetRouterAsync_RefreshesEndpointBaselineAfterFailedReset() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-reset-failed-"); + var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, port: 18801); + LocalAiResolvedInstall moved = original with + { + Manifest = original.Manifest with { Endpoint = "http://127.0.0.1:18803/v1" }, + Endpoint = new Uri("http://127.0.0.1:18803/v1"), + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(original.Manifest); + context.LocalAiResolvedInstall = original; + context.LocalAiRuntimeBorrowed = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(original)) + { + RestartForSetupHandler = async ct => + { + await store.SaveAsync(moved.Manifest, ct); + return HealthySnapshot(moved) with + { + State = LocalAiRuntimeState.Failed, + Ownership = LocalAiOwnership.None, + ProcessId = null, + }; + }, + }; + + LocalAiRuntimeSnapshot reset = await VerifyLocalAiInferenceStep.ResetRouterAsync( + context, + runtime); + + Assert.Equal(LocalAiRuntimeState.Failed, reset.State); + Assert.Equal(moved.Endpoint, context.LocalAiResolvedInstall.Endpoint); + } + + [Fact] + public async Task ReleaseBorrowedLocalAiRuntimeAfterFailureAsync_ReleasesNoRollbackOwnership() + { + var context = CreateContext(LocalAiRecoveryConfig()); + var runtime = new DisposeTrackingRuntime(); + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + context.LocalAiBorrowedRuntimeRestartedThisRun = true; + + await SetupPipeline.ReleaseBorrowedLocalAiRuntimeAfterFailureAsync( + context, + new PipelineResult(PipelineOutcome.Failed)); + + Assert.Equal(1, runtime.ReleaseSetupGatewayRouteCalls); + + context.LocalAiBorrowedRuntimeRestartedThisRun = false; + await SetupPipeline.ReleaseBorrowedLocalAiRuntimeAfterFailureAsync( + context, + new PipelineResult(PipelineOutcome.Failed)); + + Assert.Equal(1, runtime.ReleaseSetupGatewayRouteCalls); + + await SetupPipeline.ReleaseBorrowedLocalAiRuntimeAfterFailureAsync( + context, + new PipelineResult(PipelineOutcome.Success)); + + Assert.Equal(1, runtime.ReleaseSetupGatewayRouteCalls); + } + + [Fact] + public async Task BorrowedTrayRuntime_RollbackRestoresRuntimeBeforeReconcilingAutomaticPort() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-rollback-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18801); + original = original with + { + Manifest = original.Manifest with { RequestedPort = 0 }, + }; + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:18802/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Manifest.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)); + await store.SaveAsync(pendingManifest); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pendingManifest); + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryProviderTransition = true; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(context.LocalAiResolvedInstall)) + { + RestartForSetupHandler = async ct => + { + LocalAiResolvedInstall restored = await store.LoadAsync(ct) + ?? throw new InvalidDataException("restored receipt missing"); + LocalAiInstallManifest movedManifest = restored.Manifest with + { + Endpoint = "http://127.0.0.1:18803/v1", + }; + await store.SaveAsync(movedManifest, ct); + return HealthySnapshot(store.ResolveAndValidate(movedManifest)); + }, + }; + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + context.LocalAiBorrowedRuntimeRestartedThisRun = true; + var rollbackOrder = new List(); + Uri? restoredRoute = null; + Uri? probedEndpoint = null; + var persist = new PersistLocalAiManifestStep(); + var start = new StartLocalAiRuntimeStep(_ => runtime); + var preserve = new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("gateway restarted")), + (install, _) => + { + rollbackOrder.Add("probe"); + Assert.Equal(0, runtime.AcknowledgeSetupGatewayRouteCalls); + probedEndpoint = install.Endpoint; + return Task.FromResult(true); + }, + (_, _, _, install, _) => + { + rollbackOrder.Add("route"); + restoredRoute = install.Endpoint; + return Task.FromResult(true); + }); + var pipeline = new SetupPipeline([ + new MockStep( + "persist-local-ai-manifest", + (_, _) => Task.FromResult(StepResult.Ok("persisted")), + persist.RollbackAsync), + new MockStep( + "start-local-ai-runtime", + (_, _) => Task.FromResult(StepResult.Ok("started")), + start.RollbackAsync), + preserve, + new MockStep( + "configure-local-ai-gateway", + (ctx, _) => + { + ctx.LocalAiGatewayPriorState = new LocalAiGatewayPriorState( + ProviderExisted: true, + ProviderJson: "{}", + PrimaryModelExisted: true, + PrimaryModelJson: "\"prior-model\""); + ctx.LocalAiRecoveryGatewayConfigurationStartedThisRun = true; + ctx.LocalAiRecoveryRollbackUncertain = true; + return Task.FromResult(StepResult.Ok("configured")); + }, + (ctx, _) => + { + rollbackOrder.Add("gateway"); + ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; + return Task.CompletedTask; + }), + new MockStep( + "failure", + (_, _) => Task.FromResult(StepResult.Fail("failed after gateway configuration"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal("failure", result.FailedStepId); + LocalAiResolvedInstall restored = (await store.LoadAsync())!; + Assert.Equal(new Uri("http://127.0.0.1:18803/v1"), probedEndpoint); + Assert.Equal(1, runtime.RestartForSetupCalls); + Assert.Equal(1, runtime.AcknowledgeSetupGatewayRouteCalls); + Assert.Equal(["gateway", "route", "probe"], rollbackOrder); + Assert.Equal(original.Manifest.ModelCatalogId, restored.Manifest.ModelCatalogId); + Assert.Equal(new Uri("http://127.0.0.1:18803/v1"), restored.Endpoint); + Assert.Equal(restored.Endpoint, restoredRoute); + Assert.Equal(restored.Endpoint, probedEndpoint); + Assert.Equal(0, runtime.RestartCalls); + Assert.Equal(0, runtime.StopForSetupCalls); + Assert.Equal(0, runtime.StopCalls); + Assert.Equal(0, runtime.DisposeCalls); + Assert.Same(runtime, context.LocalAiRuntime); + Assert.True(context.LocalAiBorrowedRuntimeRestored); + Assert.False(context.LocalAiRecoveryRollbackUncertain); + } + + [Fact] + public async Task BorrowedTrayRuntime_PreGatewayRollbackPublishesRestoredAutomaticPort() + { + using var temp = new TempDirectory("local-ai-borrowed-runtime-pre-gateway-rollback-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18801); + original = original with + { + Manifest = original.Manifest with { RequestedPort = 0 }, + }; + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:18802/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Manifest.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)); + await store.SaveAsync(pendingManifest); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pendingManifest); + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryProviderTransition = true; + Uri? publishedEndpoint = null; + var runtime = new DisposeTrackingRuntime(HealthySnapshot(context.LocalAiResolvedInstall)) + { + RestartHandler = async ct => + { + LocalAiResolvedInstall restored = await store.LoadAsync(ct) + ?? throw new InvalidDataException("restored receipt missing"); + LocalAiInstallManifest movedManifest = restored.Manifest with + { + Endpoint = "http://127.0.0.1:18803/v1", + }; + await store.SaveAsync(movedManifest, ct); + LocalAiResolvedInstall moved = store.ResolveAndValidate(movedManifest); + publishedEndpoint = moved.Endpoint; + return HealthySnapshot(moved); + }, + }; + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + context.LocalAiBorrowedRuntimeRestartedThisRun = true; + var persist = new PersistLocalAiManifestStep(); + var start = new StartLocalAiRuntimeStep(_ => runtime); + var pipeline = new SetupPipeline([ + new MockStep( + "persist-local-ai-manifest", + (_, _) => Task.FromResult(StepResult.Ok("persisted")), + persist.RollbackAsync), + new MockStep( + "start-local-ai-runtime", + (_, _) => Task.FromResult(StepResult.Ok("started")), + start.RollbackAsync), + new MockStep( + "failure-before-gateway", + (_, _) => Task.FromResult(StepResult.Fail("failed before gateway configuration"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + LocalAiResolvedInstall restored = (await store.LoadAsync())!; + Assert.Equal(original.Manifest.ModelCatalogId, restored.Manifest.ModelCatalogId); + Assert.Equal(new Uri("http://127.0.0.1:18803/v1"), restored.Endpoint); + Assert.Equal(restored.Endpoint, publishedEndpoint); + Assert.Equal(1, runtime.StopForSetupCalls); + Assert.Equal(1, runtime.RestartForSetupRollbackCalls); + Assert.Equal(0, runtime.RestartCalls); + Assert.Equal(0, runtime.RestartForSetupCalls); + Assert.Equal(0, runtime.DisposeCalls); + Assert.False(context.LocalAiRecoveryRollbackUncertain); + } + + [Fact] + public async Task RestoredBorrowedRuntimeTransfersExactAcquisitionCleanupOwnership() + { + using var temp = new TempDirectory("local-ai-restored-runtime-cleanup-ownership-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.LocalAi.Enabled = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall original = CreateLocalAiResolvedInstall(temp.Path, port: 18801); + LocalAiInstallManifest pendingManifest = original.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + Endpoint = "http://127.0.0.1:18802/v1", + ReplacedManifest = original.Manifest, + PreviousEndpoints = [original.Manifest.Endpoint!], + }; + var store = new LocalAiManifestStore(new LocalAiPaths(temp.Path)); + await store.SaveAsync(pendingManifest); + context.LocalAiResolvedInstall = store.ResolveAndValidate(pendingManifest); + context.LocalAiRecoveryOriginalInstall = original; + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiRuntimeInstall = new LlamaRuntimeInstallResult( + Path.GetDirectoryName(original.ExecutablePath)!, + original.ExecutablePath, + LlamaRuntimeInstallDisposition.Installed, + CreatedThisRun: true, + VerifiedArchives: [], + Rollback: null); + var runtime = new DisposeTrackingRuntime(HealthySnapshot(context.LocalAiResolvedInstall)) + { + RestartHandler = async ct => + { + LocalAiResolvedInstall restored = await store.LoadAsync(ct) + ?? throw new InvalidDataException("restored receipt missing"); + return HealthySnapshot(restored); + }, + }; + context.LocalAiRuntime = runtime; + context.LocalAiRuntimeBorrowed = true; + var acquirer = new TrackingRuntimeAcquirer(); + + await new PersistLocalAiManifestStep().RollbackAsync(context, CancellationToken.None); + await new AcquireLocalAiRuntimeStep(acquirer).RollbackAsync(context, CancellationToken.None); + + Assert.Equal(original.Manifest.ModelCatalogId, (await store.LoadAsync())!.Manifest.ModelCatalogId); + Assert.Null(context.LocalAiRuntimeInstall); + Assert.Equal(0, acquirer.RemoveCalls); + Assert.Equal(1, runtime.RestartForSetupRollbackCalls); + } + /// /// Regression guard: a stale manifest receipt is not enough to prove the original (A) /// endpoint is still alive. Rollback must probe it before pointing the Gateway back at it. @@ -422,9 +1277,19 @@ public async Task PreserveLocalAiRecoveryGateway_RestoresReceiptWhenOriginalEndp using var temp = new TempDirectory("local-ai-recovery-rollback-"); var context = CreateContext(LocalAiRecoveryConfig(), localDataDir: temp.Path); LocalAiResolvedInstall originalInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18801); + LocalAiResolvedInstall replacementInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + LocalAiInstallManifest pendingManifest = replacementInstall.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + ReplacedManifest = originalInstall.Manifest, + PreviousEndpoints = [originalInstall.Manifest.Endpoint!], + }; + replacementInstall = replacementInstall with { Manifest = pendingManifest }; + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(pendingManifest); context.LocalAiRecoveryOriginalInstall = originalInstall; context.LocalAiRecoveryReceiptRollbackAllowed = true; - context.LocalAiResolvedInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + context.LocalAiResolvedInstall = replacementInstall; var step = new PreserveLocalAiRecoveryGatewayStep( (_, _) => Task.FromResult(StepResult.Ok("restarted")), (_, _) => Task.FromResult(true)); @@ -436,22 +1301,172 @@ public async Task PreserveLocalAiRecoveryGateway_RestoresReceiptWhenOriginalEndp Assert.Equal(originalInstall.Endpoint, context.LocalAiResolvedInstall!.Endpoint); } - private sealed class DisposeTrackingRuntime : ILocalAiRuntime + [Fact] + public async Task RecoveryRollback_PreservesReplacementWhenCompensatedOriginalEndpointIsUnhealthy() { - public int DisposeCalls { get; private set; } + using var temp = new TempDirectory("local-ai-recovery-rollback-"); + SetupConfig config = LocalAiRecoveryConfig(); + config.RollbackOnFailure = true; + var context = CreateContext(config, localDataDir: temp.Path); + LocalAiResolvedInstall originalInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18801); + LocalAiResolvedInstall replacementInstall = CreateLocalAiResolvedInstall(context.LocalDataDir, port: 18802); + LocalAiInstallManifest pendingManifest = replacementInstall.Manifest with + { + ModelCatalogId = "replacement-model", + ModelAlias = "replacement-model", + ReplacedManifest = originalInstall.Manifest, + PreviousEndpoints = [originalInstall.Manifest.Endpoint!], + }; + replacementInstall = replacementInstall with { Manifest = pendingManifest }; + await new LocalAiManifestStore(new LocalAiPaths(context.LocalDataDir)).SaveAsync(pendingManifest); + context.LocalAiRecoveryOriginalInstall = originalInstall; + context.LocalAiResolvedInstall = replacementInstall; + context.LocalAiRecoveryProviderTransition = true; + context.LocalAiGatewayPriorState = new LocalAiGatewayPriorState( + ProviderExisted: true, + ProviderJson: "{}", + PrimaryModelExisted: true, + PrimaryModelJson: "\"test-model\""); + var runtime = new DisposeTrackingRuntime(); + context.LocalAiRuntime = runtime; + var probedEndpoints = new List(); + var persist = new PersistLocalAiManifestStep(); + var start = new StartLocalAiRuntimeStep(_ => runtime); + var pipeline = new SetupPipeline([ + new MockStep( + "persist-local-ai-manifest", + (_, _) => Task.FromResult(StepResult.Ok("persisted")), + persist.RollbackAsync), + new MockStep( + "start-local-ai-runtime", + (_, _) => Task.FromResult(StepResult.Ok("started")), + start.RollbackAsync), + new PreserveLocalAiRecoveryGatewayStep( + (_, _) => Task.FromResult(StepResult.Ok("restarted")), + (install, _) => + { + probedEndpoints.Add(install.Endpoint); + return Task.FromResult(false); + }), + new MockStep( + "configure-local-ai-gateway", + (_, _) => Task.FromResult(StepResult.Ok("configured")), + (ctx, _) => + { + ctx.LocalAiRecoveryReceiptRollbackAllowed = true; + ctx.LocalAiRecoveryRollbackUncertain = false; + return Task.CompletedTask; + }), + new MockStep( + "finalize-local-ai-model-replacement", + (_, _) => Task.FromResult(StepResult.Fail("finalization failed"))), + ]); + + PipelineResult result = await pipeline.RunAsync(context); + + Assert.Equal(PipelineOutcome.Failed, result.Outcome); + Assert.Equal([originalInstall.Endpoint], probedEndpoints); + Assert.Equal(0, runtime.DisposeCalls); + Assert.False(context.LocalAiRecoveryCleanupAllowed); + LocalAiInstallManifest retained = (await new LocalAiManifestStore( + new LocalAiPaths(context.LocalDataDir)).LoadAsync())!.Manifest; + Assert.Equal(pendingManifest.ModelCatalogId, retained.ModelCatalogId); + Assert.NotNull(retained.ReplacedManifest); + } - public LocalAiRuntimeSnapshot Snapshot => throw new NotSupportedException(); + private sealed class DisposeTrackingRuntime(LocalAiRuntimeSnapshot? snapshot = null) : ILocalAiRuntime + { + public int DisposeCalls { get; private set; } + public int EnsureStartedCalls { get; private set; } + public int StopCalls { get; private set; } + public int RestartCalls { get; private set; } + public int StopForSetupCalls { get; private set; } + public int RestartForSetupCalls { get; private set; } + public int RestartForSetupRollbackCalls { get; private set; } + public int AcknowledgeSetupGatewayRouteCalls { get; private set; } + public int ReleaseSetupGatewayRouteCalls { get; private set; } + public Func>? RestartHandler { get; init; } + public Func>? RestartForSetupHandler { get; init; } + + public LocalAiRuntimeSnapshot Snapshot { get; private set; } = snapshot ?? + LocalAiRuntimeSnapshot.Initial(new Uri("http://127.0.0.1:18800/v1"), DateTimeOffset.UtcNow); public Task ResumeAsync(CancellationToken cancellationToken = default) => - throw new NotSupportedException(); - public Task EnsureStartedAsync(CancellationToken cancellationToken = default) => - throw new NotSupportedException(); + Task.FromResult(Snapshot); - public Task StopAsync(CancellationToken cancellationToken = default) => - throw new NotSupportedException(); + public Task EnsureStartedAsync(CancellationToken cancellationToken = default) + { + EnsureStartedCalls++; + return Task.FromResult(Snapshot); + } - public Task RestartAsync(CancellationToken cancellationToken = default) => - throw new NotSupportedException(); + public Task StopAsync(CancellationToken cancellationToken = default) + { + StopCalls++; + Snapshot = Snapshot with + { + State = LocalAiRuntimeState.Stopped, + Ownership = LocalAiOwnership.None, + ProcessId = null, + ProcessStartedAtUtc = null, + }; + return Task.FromResult(Snapshot); + } + + public async Task RestartAsync(CancellationToken cancellationToken = default) + { + RestartCalls++; + Snapshot = RestartHandler is null + ? snapshot ?? Snapshot + : await RestartHandler(cancellationToken); + return Snapshot; + } + + public Task StopForSetupAsync(CancellationToken cancellationToken = default) + { + StopForSetupCalls++; + Snapshot = Snapshot with + { + State = LocalAiRuntimeState.Stopped, + Ownership = LocalAiOwnership.None, + ProcessId = null, + ProcessStartedAtUtc = null, + }; + return Task.FromResult(Snapshot); + } + + public async Task RestartForSetupAsync(CancellationToken cancellationToken = default) + { + RestartForSetupCalls++; + Snapshot = RestartForSetupHandler is null + ? snapshot ?? Snapshot + : await RestartForSetupHandler(cancellationToken); + return Snapshot; + } + + public async Task RestartForSetupRollbackAsync( + CancellationToken cancellationToken = default) + { + RestartForSetupRollbackCalls++; + Snapshot = RestartHandler is null + ? snapshot ?? Snapshot + : await RestartHandler(cancellationToken); + return Snapshot; + } + + public Task ReleaseSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) + { + ReleaseSetupGatewayRouteCalls++; + return Task.FromResult(Snapshot); + } + + public Task AcknowledgeSetupGatewayRouteAsync( + CancellationToken cancellationToken = default) + { + AcknowledgeSetupGatewayRouteCalls++; + return Task.FromResult(Snapshot); + } public Task RefreshAsync(CancellationToken cancellationToken = default) => throw new NotSupportedException(); @@ -469,6 +1484,41 @@ public ValueTask DisposeAsync() } } + private sealed class TrackingRuntimeAcquirer : ILlamaRuntimeAcquirer + { + public int RemoveCalls { get; private set; } + + public Task InstallAsync( + string localDataDirectory, + LlamaRuntimeVariant runtime, + IProgress? progress, + CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public void RemoveInstalledRuntime( + string localDataDirectory, + LlamaRuntimeInstallResult install) => RemoveCalls++; + } + + private static LocalAiRuntimeSnapshot HealthySnapshot(LocalAiResolvedInstall install) => new( + LocalAiRuntimeState.Healthy, + LocalAiOwnership.CompanionManaged, + install.Endpoint!, + install.Manifest.EngineVersion, + install.Manifest.ModelCatalogId, + new LocalAiModelEvidence( + LocalAiModelAvailabilityState.Verified, + DateTimeOffset.UtcNow, + install.Manifest.ModelAsset.Sha256, + install.Manifest.ModelAsset.SizeBytes), + 42, + DateTimeOffset.UtcNow, + null, + DateTimeOffset.UtcNow) + { + GatewayRouteRequiresResolution = false, + }; + private static LocalAiResolvedInstall CreateLocalAiResolvedInstall(string localDataDirectory, int port) { var paths = new LocalAiPaths(localDataDirectory); diff --git a/tests/OpenClaw.Tray.Tests/LocalAiGatewayProviderCoordinatorTests.cs b/tests/OpenClaw.Tray.Tests/LocalAiGatewayProviderCoordinatorTests.cs index 31c1f5ba3..e9cf42657 100644 --- a/tests/OpenClaw.Tray.Tests/LocalAiGatewayProviderCoordinatorTests.cs +++ b/tests/OpenClaw.Tray.Tests/LocalAiGatewayProviderCoordinatorTests.cs @@ -695,7 +695,8 @@ public void LocalAiSetupRoute_UsesUniqueManagedOwnerEvenWhenItIsNotActive() hasDistroDataDirectory: true, distroIsAppOwned: true, installedModelCatalogId: LocalModelCatalog.Qwen38_27BModelId, - installedRequestedLocalAiPort: 28888); + installedRequestedLocalAiPort: 28888, + pinInstalledModelSelection: true); Assert.Equal(LocalAiSetupRoute.Recovery, resolution.Route); Assert.Equal("managed", resolution.RecoveryTarget?.GatewayId); @@ -703,6 +704,7 @@ public void LocalAiSetupRoute_UsesUniqueManagedOwnerEvenWhenItIsNotActive() Assert.Equal(29999, resolution.RecoveryTarget?.GatewayPort); Assert.Equal(LocalModelCatalog.Qwen38_27BModelId, resolution.RecoveryTarget?.ModelCatalogId); Assert.Equal(28888, resolution.RecoveryTarget?.RequestedLocalAiPort); + Assert.True(resolution.RecoveryTarget?.PinModelSelection); } [Fact] diff --git a/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs b/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs index 22e33cf61..294ab0685 100644 --- a/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs @@ -308,9 +308,7 @@ public void CapabilitiesReview_GatesOnDeviceEligibilityAndReconcilesStaleSelecte "hardwareReason = DescribeLocalAiUnavailable(availability);", "LocalInferenceEligibilityResult selectedEligibility =", "LocalInferenceEligibility.Evaluate(_localAiHardware, selectedModelId);", - "if (_localAiRecoveryModelPinned)", - "eligibility = selectedEligibility;", - "else if (!selectedEligibility.CanInstall)", + "if (!selectedEligibility.CanInstall)", "_config.LocalAi.SelectedModelId = null;", "_config.LocalAi.SelectedModelId ??= _localAiRecommendedModelId ?? availability.Plan.Model.Id;", "eligibility ??= LocalInferenceEligibility.Evaluate(", diff --git a/tests/OpenClaw.Tray.Tests/OnboardingPresentationContractTests.cs b/tests/OpenClaw.Tray.Tests/OnboardingPresentationContractTests.cs index 9a488d03c..50b945e37 100644 --- a/tests/OpenClaw.Tray.Tests/OnboardingPresentationContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/OnboardingPresentationContractTests.cs @@ -19,9 +19,13 @@ public void PipelineRegistryWriters_ProduceTheExpectedAdoptionSnapshot() } var progress = Read(@"src\OpenClaw.SetupEngine.UI\Pages\ProgressPage.xaml.cs"); Assert.Contains("ctx.ExpectedGatewayRegistry = config.NativeLocalAiAcquisition ? null : setupOwner?.BeginGatewaySetup()", progress); - Assert.Contains("outcome => config.NativeLocalAiAcquisition ? Task.CompletedTask :", progress); + Assert.Contains("ReleaseBorrowedLocalAiRuntimeAfterFailureAsync(ctx, outcome)", progress); + Assert.Contains("if (!config.NativeLocalAiAcquisition && setupOwner is not null)", progress); Assert.Contains("SetupPipeline.RunWithSettlementAsync", progress); - Assert.True(progress.IndexOf("SettleGatewaySetupAsync(ctx.ExpectedGatewayRegistry", StringComparison.Ordinal) < + Assert.Contains("await setupOwner.SettleGatewaySetupAsync(", progress); + Assert.True(progress.IndexOf("ReleaseBorrowedLocalAiRuntimeAfterFailureAsync(ctx, outcome)", StringComparison.Ordinal) < + progress.IndexOf("await setupOwner.SettleGatewaySetupAsync(", StringComparison.Ordinal)); + Assert.True(progress.IndexOf("await setupOwner.SettleGatewaySetupAsync(", StringComparison.Ordinal) < progress.IndexOf("if (_closed || _window?.IsClosed == true)", StringComparison.Ordinal)); }