Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
f4a30b8
fix: replace local ai model during recovery
RomneyDa Sep 12, 2026
d7b3b60
fix(local-ai): preserve receipt downgrade compatibility
RomneyDa Sep 29, 2026
94b4669
fix(local-ai): keep replacement recovery state scoped
RomneyDa Sep 29, 2026
e6d01c1
fix(local-ai): serialize replacement endpoint updates
RomneyDa Sep 29, 2026
9cf986f
fix(local-ai): finalize the current replacement receipt
RomneyDa Sep 29, 2026
4eb5e09
fix(local-ai): close replacement rollback races
RomneyDa Sep 29, 2026
527377b
test(local-ai): prove drift rejection before gateway writes
RomneyDa Sep 29, 2026
49a3d74
fix(local-ai): preserve resumed replacement rollback receipt
RomneyDa Sep 29, 2026
fbf50a3
fix(local-ai): restart gateway after rollback
RomneyDa Sep 30, 2026
29034e2
test(local-ai): cover fixed-port replacement routes
RomneyDa Sep 30, 2026
be69b13
fix(local-ai): preserve replacement across runtime upgrades
RomneyDa Sep 30, 2026
e5f6bca
fix(local-ai): restore pending receipt after upgrade failure
RomneyDa Sep 30, 2026
a70f716
fix(local-ai): make replacement rollback transaction-safe
RomneyDa Sep 30, 2026
efcb0bb
fix(local-ai): settle receipt rollback after endpoint health
RomneyDa Oct 1, 2026
531ddf5
fix(local-ai): distinguish current gateway rollback
RomneyDa Oct 1, 2026
1ec7448
fix(local-ai): clean pre-gateway runtime upgrades
RomneyDa Oct 1, 2026
c64fddb
fix(local-ai): hand off tray runtime during recovery
RomneyDa Oct 1, 2026
d03e4ec
test(setup): assert runtime release settlement order
RomneyDa Oct 1, 2026
616f1b1
fix(setup): harden borrowed runtime recovery
RomneyDa Oct 1, 2026
526d61c
fix(setup): classify recovery compatibility progress
RomneyDa Oct 1, 2026
320e467
fix(setup): admit first recovery install runtime
RomneyDa Oct 1, 2026
ab1c16b
fix(setup): settle Local AI rollback ownership
RomneyDa Oct 1, 2026
6f232a9
fix(setup): retire settled Local AI rollback baselines
RomneyDa Oct 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
322 changes: 236 additions & 86 deletions src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs

Large diffs are not rendered by default.

229 changes: 213 additions & 16 deletions src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,15 @@ public sealed record LocalAiInstallManifest
/// managed llama.cpp model. Null means no prior primary model was configured.
/// </summary>
public string? GatewayFallbackModel { get; init; }
/// <summary>
/// The last committed receipt while a recovery flow replaces its model.
/// Cleared only after the Gateway has restarted on the replacement route.
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public LocalAiInstallManifest? ReplacedManifest { get; init; }
/// <summary>Earlier verified replacement endpoints that may still be published to the Gateway.</summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public ImmutableArray<string>? PreviousEndpoints { get; init; }
public required int ContextLength { get; init; }
public KvCachePrecision KeyCachePrecision { get; init; } = KvCachePrecision.F16;
public KvCachePrecision ValueCachePrecision { get; init; } = KvCachePrecision.F16;
Expand Down Expand Up @@ -484,6 +493,158 @@ public async Task SaveAsync(LocalAiInstallManifest manifest, CancellationToken c
await SaveWithoutLockAsync(manifest, cancellationToken).ConfigureAwait(false);
}

internal async Task<LocalAiResolvedInstall> UpdateVerifiedEndpointAsync(
LocalAiInstallManifest expectedManifest,
Uri endpoint,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(expectedManifest);
ArgumentNullException.ThrowIfNull(endpoint);
await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken)
.ConfigureAwait(false);
LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false);
if (!HasSameRuntimeAndModel(current, expectedManifest))
{
throw new InvalidDataException(
"The Local AI installation changed before its verified endpoint could be recorded.");
}
if (expectedManifest.ReplacedManifest is not null && current.ReplacedManifest is null)
{
throw new InvalidDataException(
"The Local AI model replacement was finalized before its verified endpoint could be recorded.");
}

ImmutableArray<string>? history = current.PreviousEndpoints;
if (current.ReplacedManifest is null)
{
history = null;
}
else if (current.Endpoint is { } previousEndpoint &&
!string.Equals(previousEndpoint, endpoint.AbsoluteUri, StringComparison.Ordinal))
{
ImmutableArray<string> values = history ?? [];
if (!values.Contains(previousEndpoint, StringComparer.Ordinal))
history = values.Add(previousEndpoint);
}

LocalAiInstallManifest updated = current with
{
Endpoint = endpoint.AbsoluteUri,
PreviousEndpoints = history,
};
LocalAiResolvedInstall resolved = ResolveAndValidate(updated);
await SaveWithoutLockAsync(updated, cancellationToken).ConfigureAwait(false);
return resolved;
}

internal async Task<LocalAiResolvedInstall> FinalizeReplacementAsync(
LocalAiInstallManifest expectedManifest,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(expectedManifest);
await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken)
.ConfigureAwait(false);
LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false);
if (!HasSameRuntimeAndModel(current, expectedManifest))
{
throw new InvalidDataException(
"The Local AI installation changed before its model replacement could be finalized.");
}
if (expectedManifest.ReplacedManifest is null || current.ReplacedManifest is null ||
!string.Equals(current.Endpoint, expectedManifest.Endpoint, StringComparison.Ordinal))
{
throw new InvalidDataException(
"The Local AI model replacement changed before it could be finalized.");
}

LocalAiInstallManifest finalized = current with
{
ReplacedManifest = null,
PreviousEndpoints = null,
};
LocalAiResolvedInstall resolved = ResolveAndValidate(finalized);
await SaveWithoutLockAsync(finalized, cancellationToken).ConfigureAwait(false);
return resolved;
}

internal async Task<LocalAiResolvedInstall> RestoreRecoveryManifestAsync(
LocalAiInstallManifest expectedManifest,
LocalAiInstallManifest recoveryManifest,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(expectedManifest);
ArgumentNullException.ThrowIfNull(recoveryManifest);
await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken)
.ConfigureAwait(false);
LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false);
if (!JsonElement.DeepEquals(
JsonSerializer.SerializeToElement(current),
JsonSerializer.SerializeToElement(expectedManifest)) ||
current.ReplacedManifest is null)
{
throw new InvalidDataException(
"The Local AI model replacement changed before its recovery receipt could be restored.");
}

bool restoresOriginal = JsonElement.DeepEquals(
JsonSerializer.SerializeToElement(current.ReplacedManifest),
JsonSerializer.SerializeToElement(recoveryManifest));
bool endpointWasPublished = recoveryManifest.Endpoint is { } recoveryEndpoint &&
(string.Equals(current.Endpoint, recoveryEndpoint, StringComparison.Ordinal) ||
(current.PreviousEndpoints?.Contains(recoveryEndpoint, StringComparer.Ordinal) ?? false));
bool restoresPendingRoute = endpointWasPublished && JsonElement.DeepEquals(
JsonSerializer.SerializeToElement(current),
JsonSerializer.SerializeToElement(recoveryManifest with
{
Endpoint = current.Endpoint,
PreviousEndpoints = current.PreviousEndpoints,
}));
if (!restoresOriginal && !restoresPendingRoute)
{
throw new InvalidDataException(
"The requested Local AI recovery receipt is not an authorized replacement route.");
}

LocalAiInstallManifest restored = restoresPendingRoute
? current with { Endpoint = recoveryManifest.Endpoint }
: recoveryManifest;
LocalAiResolvedInstall resolved = ResolveAndValidate(restored);
await SaveWithoutLockAsync(restored, cancellationToken).ConfigureAwait(false);
return resolved;
}

internal async Task<LocalAiResolvedInstall> RestoreManifestIfUnchangedAsync(
LocalAiInstallManifest expectedManifest,
LocalAiInstallManifest recoveryManifest,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(expectedManifest);
ArgumentNullException.ThrowIfNull(recoveryManifest);
await using FileStream writeLock = await AcquireManifestWriteLockAsync(cancellationToken)
.ConfigureAwait(false);
LocalAiInstallManifest current = await ReadManifestAsync(cancellationToken).ConfigureAwait(false);
if (!JsonElement.DeepEquals(
JsonSerializer.SerializeToElement(current),
JsonSerializer.SerializeToElement(expectedManifest)))
{
throw new InvalidDataException(
"The Local AI installation changed before its recovery receipt could be restored.");
}

LocalAiResolvedInstall resolved = ResolveAndValidate(recoveryManifest);
await SaveWithoutLockAsync(recoveryManifest, cancellationToken).ConfigureAwait(false);
return resolved;
}

private static bool HasSameRuntimeAndModel(
LocalAiInstallManifest current,
LocalAiInstallManifest expected) =>
string.Equals(current.RuntimeId, expected.RuntimeId, StringComparison.Ordinal) &&
string.Equals(current.SelectedGpuId, expected.SelectedGpuId, StringComparison.Ordinal) &&
string.Equals(current.ModelCatalogId, expected.ModelCatalogId, StringComparison.Ordinal) &&
string.Equals(current.ModelPath, expected.ModelPath, StringComparison.OrdinalIgnoreCase) &&
string.Equals(current.CachedModelPath, expected.CachedModelPath, StringComparison.OrdinalIgnoreCase);

private async Task SaveWithoutLockAsync(
LocalAiInstallManifest manifest,
CancellationToken cancellationToken)
Expand Down Expand Up @@ -632,30 +793,66 @@ LocalAiInstallManifest.HubCacheReceiptSchemaVersion or
LocalAiPortPolicy.Validate(manifest.RequestedPort);
LocalAiGatewayModelPolicy.ValidateFallbackModel(manifest.GatewayFallbackModel);

Uri? endpoint = null;
if (manifest.Endpoint is not null)
{
if (!Uri.TryCreate(manifest.Endpoint, UriKind.Absolute, out endpoint) ||
endpoint.Scheme != Uri.UriSchemeHttp ||
!string.Equals(endpoint.Host, "127.0.0.1", StringComparison.Ordinal) ||
endpoint.IsDefaultPort ||
endpoint.Port is <= 0 or > 65535 ||
endpoint.Port == 80 ||
!string.IsNullOrEmpty(endpoint.UserInfo) ||
!string.IsNullOrEmpty(endpoint.Query) ||
!string.IsNullOrEmpty(endpoint.Fragment) ||
!string.Equals(endpoint.AbsolutePath, "/v1", StringComparison.Ordinal))
if (manifest.ReplacedManifest is { } replaced)
{
if (replaced.ReplacedManifest is not null ||
string.Equals(replaced.ModelCatalogId, manifest.ModelCatalogId, StringComparison.Ordinal) ||
!string.Equals(replaced.Engine, manifest.Engine, StringComparison.Ordinal) ||
!string.Equals(replaced.EngineVersion, manifest.EngineVersion, StringComparison.Ordinal) ||
!string.Equals(replaced.Architecture, manifest.Architecture, StringComparison.Ordinal) ||
!string.Equals(replaced.RuntimeId, manifest.RuntimeId, StringComparison.Ordinal) ||
!string.Equals(replaced.ExecutablePath, manifest.ExecutablePath, StringComparison.Ordinal) ||
!replaced.RuntimeAssets.SequenceEqual(manifest.RuntimeAssets) ||
replaced.RequestedPort != manifest.RequestedPort)
{
throw new InvalidDataException("The local AI endpoint must be an HTTP IPv4 loopback /v1 address with an explicit non-reserved port.");
throw new InvalidDataException("The local AI model replacement receipt is invalid.");
}
_ = ResolveAndValidate(replaced);
}
else if (manifest.PreviousEndpoints is not null)
{
throw new InvalidDataException("Previous Local AI endpoints require a pending model replacement.");
}

if (manifest.RequestedPort != LocalAiPortPolicy.Automatic && endpoint.Port != manifest.RequestedPort)
throw new InvalidDataException("The verified Local AI endpoint does not match its requested fixed port.");
Uri? endpoint = ValidateEndpoint(manifest.Endpoint, manifest.RequestedPort);
ImmutableArray<string> endpointHistory = manifest.PreviousEndpoints.GetValueOrDefault();
if (endpointHistory.IsDefault)
endpointHistory = [];
HashSet<string> previousEndpoints = endpointHistory.ToHashSet(StringComparer.Ordinal);
if (previousEndpoints.Count != endpointHistory.Length)
{
throw new InvalidDataException("Previous Local AI endpoints must be unique.");
}
foreach (string previousEndpoint in previousEndpoints)
{
if (string.IsNullOrWhiteSpace(previousEndpoint))
throw new InvalidDataException("Previous Local AI endpoints must be non-empty endpoint strings.");
_ = ValidateEndpoint(previousEndpoint, manifest.RequestedPort);
}

return new LocalAiResolvedInstall(manifest, executable, model, endpoint);
}

private static Uri? ValidateEndpoint(string? value, int requestedPort)
{
if (value is null)
return null;
if (!Uri.TryCreate(value, UriKind.Absolute, out Uri? endpoint) ||
endpoint.Scheme != Uri.UriSchemeHttp ||
!string.Equals(endpoint.Host, "127.0.0.1", StringComparison.Ordinal) ||
endpoint.IsDefaultPort || endpoint.Port is <= 0 or > 65535 || endpoint.Port == 80 ||
!string.IsNullOrEmpty(endpoint.UserInfo) ||
!string.IsNullOrEmpty(endpoint.Query) ||
!string.IsNullOrEmpty(endpoint.Fragment) ||
!string.Equals(endpoint.AbsolutePath, "/v1", StringComparison.Ordinal))
{
throw new InvalidDataException("The local AI endpoint must be an HTTP IPv4 loopback /v1 address with an explicit non-reserved port.");
}
if (requestedPort != LocalAiPortPolicy.Automatic && endpoint.Port != requestedPort)
throw new InvalidDataException("The verified Local AI endpoint does not match its requested fixed port.");
return endpoint;
}

private static void ValidateModelPath(
string path,
LocalAiAssetReceipt asset,
Expand Down
30 changes: 30 additions & 0 deletions src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs
Original file line number Diff line number Diff line change
Expand Up @@ -146,5 +146,35 @@ Task<LocalAiRuntimeSnapshot> ReconcileStoppedAsync(CancellationToken cancellatio
=> Task.FromResult(Snapshot);
Task<LocalAiRuntimeSnapshot> StopAsync(CancellationToken cancellationToken = default);
Task<LocalAiRuntimeSnapshot> RestartAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Stops the managed process for a setup transaction without changing Gateway routing.
/// Implementations that publish endpoint lifecycle changes must suppress them here because
/// the setup pipeline coordinates the matching Gateway transaction separately.
/// </summary>
Task<LocalAiRuntimeSnapshot> StopForSetupAsync(CancellationToken cancellationToken = default) =>
StopAsync(cancellationToken);
/// <summary>
/// Restarts the managed process for a setup transaction without changing Gateway routing.
/// Implementations that publish endpoint lifecycle changes must suppress them here because
/// the setup pipeline coordinates the matching Gateway transaction separately.
/// </summary>
Task<LocalAiRuntimeSnapshot> RestartForSetupAsync(CancellationToken cancellationToken = default) =>
RestartAsync(cancellationToken);
/// <summary>
/// Adopts setup's restored receipt, then restarts through the ordinary Gateway lifecycle.
/// Use only before setup has begun its own Gateway configuration transaction.
/// </summary>
Task<LocalAiRuntimeSnapshot> RestartForSetupRollbackAsync(
CancellationToken cancellationToken = default) => RestartAsync(cancellationToken);
/// <summary>
/// Acknowledges that setup committed or compensated the Gateway route for the current endpoint.
/// </summary>
Task<LocalAiRuntimeSnapshot> AcknowledgeSetupGatewayRouteAsync(
CancellationToken cancellationToken = default) => Task.FromResult(Snapshot);
/// <summary>
/// Returns endpoint lifecycle ownership to the runtime without claiming Gateway reconciliation.
/// </summary>
Task<LocalAiRuntimeSnapshot> ReleaseSetupGatewayRouteAsync(
CancellationToken cancellationToken = default) => Task.FromResult(Snapshot);
Task<LocalAiRuntimeSnapshot> RefreshAsync(CancellationToken cancellationToken = default);
}
28 changes: 25 additions & 3 deletions src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -149,16 +149,38 @@ private async Task StartPipelineAsync()
var steps = BuildSteps(config, _localAiRecoveryOnly);
var setupOwner = _window;
ctx.ExpectedGatewayRegistry = config.NativeLocalAiAcquisition ? null : setupOwner?.BeginGatewaySetup();
ctx.LocalAiRuntime = setupOwner?.BorrowManagedLocalAiRuntime();
ctx.LocalAiRuntimeBorrowed = ctx.LocalAiRuntime is not null;
if (ctx.LocalAiRuntimeBorrowed && !config.RollbackOnFailure)
{
throw new InvalidOperationException(
"Local AI recovery requires transactional rollback when borrowing the tray runtime.");
}
ctx.PersistTraySettings = _window is { } settingsOwner ? settingsOwner.PersistPipelineSettings : null;
_pipeline = new SetupPipeline(steps);
_pipeline.StepProgress += OnStepProgress;

var pipeline = _pipeline;
var result = await SetupPipeline.RunWithSettlementAsync(
() => Task.Run(() => pipeline.RunAsync(ctx), cts.Token),
outcome => config.NativeLocalAiAcquisition ? Task.CompletedTask : setupOwner?.SettleGatewaySetupAsync(ctx.ExpectedGatewayRegistry,
outcome?.Outcome == PipelineOutcome.Success ? config.LocalAiRecoveryGatewayId ?? ctx.GatewayRecordId : null)
?? Task.CompletedTask);
async outcome =>
{
try
{
await SetupPipeline.ReleaseBorrowedLocalAiRuntimeAfterFailureAsync(ctx, outcome);
}
finally
{
if (!config.NativeLocalAiAcquisition && setupOwner is not null)
{
await setupOwner.SettleGatewaySetupAsync(
ctx.ExpectedGatewayRegistry,
outcome?.Outcome == PipelineOutcome.Success
? config.LocalAiRecoveryGatewayId ?? ctx.GatewayRecordId
: null);
}
}
});
sw.Stop();
_pipelineFinished = true;
if (_closed || _window?.IsClosed == true)
Expand Down
3 changes: 3 additions & 0 deletions src/OpenClaw.SetupEngine.UI/SetupWindow.xaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,9 @@ public void SetWelcomeInstallSelected(bool installSelected)
internal Task<HostHardwareInfo> GetLocalAiHardwareAsync(bool forceRefresh = false) =>
_localAiHardwareProbe.GetAsync(forceRefresh);

internal ILocalAiRuntime? BorrowManagedLocalAiRuntime() =>
_startAtLocalAiRecoveryReview ? _localAiHost?.BorrowManagedRuntime() : null;

internal Task<WslViabilityResult> GetWslViabilityAsync(bool refresh = false) =>
_wslViabilityProbe.GetAsync(refresh);

Expand Down
Loading
Loading