From 9b38ea951f19b3736950d5c9b8e3134def61a5d9 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Tue, 1 Sep 2026 15:10:57 -0400 Subject: [PATCH 1/3] fix(local-ai): recommend Qwen3.8 with measured context profiles --- .../LocalAi/LlamaServerRouterConfiguration.cs | 34 +++- .../LocalAi/LlamaServerRuntimeService.cs | 7 +- .../LocalAi/LocalAiManifest.cs | 27 ++- .../LocalAi/LocalAiRuntimeModels.cs | 9 +- .../Pages/CapabilitiesPage.xaml | 18 +- .../Pages/CapabilitiesPage.xaml.cs | 96 ++++----- .../Pages/ProgressPage.xaml.cs | 2 +- .../LocalAiInstallReconciler.cs | 5 + src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs | 7 +- src/OpenClaw.SetupEngine/SetupContext.cs | 3 + .../SetupReviewSummary.cs | 26 ++- .../Catalog/LocalInferenceEligibility.cs | 8 +- .../LocalInferenceEligibilityDiagnostics.cs | 2 +- .../Catalog/LocalInferenceSelector.cs | 99 +++++++-- .../Inference/Catalog/LocalModelCatalog.cs | 192 +++++++++++++++--- .../Pages/LocalAiPage.xaml.cs | 4 +- .../Presentation/LocalAiPageViewModel.cs | 26 ++- .../Strings/en-us/Resources.resw | 2 +- .../Strings/fr-fr/Resources.resw | 2 +- .../Strings/nl-nl/Resources.resw | 2 +- .../Strings/zh-cn/Resources.resw | 2 +- .../Strings/zh-tw/Resources.resw | 2 +- .../LocalAiPortLifecycleTests.cs | 20 ++ .../LocalAiGpuRestartEndpointTests.cs | 2 +- .../LocalAiInstallRecoveryTests.cs | 11 +- .../LocalAiPortHandoffTests.cs | 2 +- .../SetupConfigTests.cs | 17 +- .../LocalInferenceQualificationTests.cs | 132 +++++++++--- .../AppRefactorContractTests.cs | 4 +- .../LocalAiSetupUxContractTests.cs | 18 +- .../Presentation/LocalAiPageViewModelTests.cs | 9 +- 31 files changed, 596 insertions(+), 194 deletions(-) diff --git a/src/OpenClaw.Connection/LocalAi/LlamaServerRouterConfiguration.cs b/src/OpenClaw.Connection/LocalAi/LlamaServerRouterConfiguration.cs index dff74bfcc..a95a43727 100644 --- a/src/OpenClaw.Connection/LocalAi/LlamaServerRouterConfiguration.cs +++ b/src/OpenClaw.Connection/LocalAi/LlamaServerRouterConfiguration.cs @@ -33,7 +33,7 @@ public static LlamaServerRouterLaunchPlan Build( LocalModelInfo model = LocalModelCatalog.Find(manifest.ModelCatalogId) ?? throw new InvalidDataException("The managed local AI model is no longer qualified."); - ValidateQualifiedReceipt(manifest, runtime, model); + LocalInferenceRunProfile profile = ValidateQualifiedReceipt(manifest, runtime, model); string presetPath = paths.ResolveContainedPath( Path.GetRelativePath(paths.RootDirectory, paths.RouterPresetPath), @@ -58,11 +58,11 @@ public static LlamaServerRouterLaunchPlan Build( .WithComparers(StringComparer.OrdinalIgnoreCase) .Add("CUDA_VISIBLE_DEVICES", manifest.SelectedGpuId), presetPath, - BuildPreset(model, install.ModelPath), + BuildPreset(model, profile, install.ModelPath), model.Id); } - private static void ValidateQualifiedReceipt( + private static LocalInferenceRunProfile ValidateQualifiedReceipt( LocalAiInstallManifest manifest, LlamaRuntimeVariant runtime, LocalModelInfo model) @@ -78,12 +78,21 @@ private static void ValidateQualifiedReceipt( throw new InvalidDataException("The managed local AI architecture and runtime receipt do not match."); } if (!string.Equals(manifest.EngineVersion, LlamaRuntimeCatalog.ReleaseTag, StringComparison.Ordinal) || - !string.Equals(manifest.ModelAlias, model.Id, StringComparison.Ordinal) || - manifest.ContextLength != model.Recipe.ContextTokens) + !string.Equals(manifest.ModelAlias, model.Id, StringComparison.Ordinal)) { throw new InvalidDataException("The managed local AI model recipe receipt does not match the qualified catalog."); } + LocalInferenceRunProfile profile = LocalModelCatalog.FindProfile( + model, + manifest.ContextLength, + manifest.KeyCachePrecision, + manifest.ValueCachePrecision, + manifest.DraftKeyCachePrecision, + manifest.DraftValueCachePrecision) + ?? throw new InvalidDataException( + "The managed local AI context and KV cache receipt do not match a qualified catalog profile."); + if (manifest.RuntimeAssets.Length != runtime.Artifacts.Count || runtime.Artifacts.Any(artifact => !manifest.RuntimeAssets.Any(receipt => string.Equals(receipt.FileName, Path.GetFileName(artifact.RelativePath), StringComparison.Ordinal) && @@ -103,9 +112,14 @@ private static void ValidateQualifiedReceipt( { throw new InvalidDataException("The managed model artifact receipt does not match the qualified catalog."); } + + return profile; } - private static string BuildPreset(LocalModelInfo model, string modelPath) + private static string BuildPreset( + LocalModelInfo model, + LocalInferenceRunProfile profile, + string modelPath) { if (modelPath.IndexOfAny(['\r', '\n']) >= 0) throw new InvalidDataException("The managed model path cannot be represented safely in a llama-server preset."); @@ -118,11 +132,13 @@ private static string BuildPreset(LocalModelInfo model, string modelPath) preset.Append('[').Append(model.Id).AppendLine("]"); preset.Append("model = ").AppendLine(modelPath); preset.AppendLine("load-on-startup = false"); - preset.Append("ctx-size = ").AppendLine(Invariant(recipe.ContextTokens)); + preset.Append("ctx-size = ").AppendLine(Invariant(profile.ContextTokens)); preset.Append("n-predict = ").AppendLine(Invariant(LocalAiGatewayProviderDefinition.MaximumOutputTokens)); preset.Append("parallel = ").AppendLine(Invariant(recipe.ParallelRequests)); - preset.AppendLine("cache-type-k = f16"); - preset.AppendLine("cache-type-v = f16"); + preset.Append("cache-type-k = ").AppendLine(LocalModelCatalog.ToLlamaServerCacheType(profile.KeyCachePrecision)); + preset.Append("cache-type-v = ").AppendLine(LocalModelCatalog.ToLlamaServerCacheType(profile.ValueCachePrecision)); + preset.Append("cache-type-k-draft = ").AppendLine(LocalModelCatalog.ToLlamaServerCacheType(profile.DraftKeyCachePrecision)); + preset.Append("cache-type-v-draft = ").AppendLine(LocalModelCatalog.ToLlamaServerCacheType(profile.DraftValueCachePrecision)); preset.Append("batch-size = ").AppendLine(Invariant(recipe.BatchTokens)); preset.Append("ubatch-size = ").AppendLine(Invariant(recipe.MicroBatchTokens)); preset.AppendLine("flash-attn = on"); diff --git a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs index 9d2a31ad3..b350bdb91 100644 --- a/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs +++ b/src/OpenClaw.Connection/LocalAi/LlamaServerRuntimeService.cs @@ -802,7 +802,12 @@ private LocalAiRuntimeSnapshot Publish( processId, processStartedAtUtc, detail, - now); + now, + _install?.Manifest.ContextLength, + _install?.Manifest.KeyCachePrecision, + _install?.Manifest.ValueCachePrecision, + _install?.Manifest.DraftKeyCachePrecision, + _install?.Manifest.DraftValueCachePrecision); lock (_snapshotGate) _snapshot = value; diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs index edb42ee91..effb99a1a 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiManifest.cs @@ -1,6 +1,7 @@ using System.Collections.Immutable; using System.Text.Json; using System.Text.Json.Serialization; +using OpenClaw.Shared.Inference.Catalog; namespace OpenClaw.Connection.LocalAi; @@ -158,6 +159,10 @@ public sealed record LocalAiInstallManifest /// public string? GatewayFallbackModel { get; init; } public required int ContextLength { get; init; } + public KvCachePrecision KeyCachePrecision { get; init; } = KvCachePrecision.F16; + public KvCachePrecision ValueCachePrecision { get; init; } = KvCachePrecision.F16; + public KvCachePrecision DraftKeyCachePrecision { get; init; } = KvCachePrecision.F16; + public KvCachePrecision DraftValueCachePrecision { get; init; } = KvCachePrecision.F16; public DateTimeOffset InstalledAtUtc { get; init; } = DateTimeOffset.UtcNow; } @@ -213,11 +218,18 @@ public static void ValidateFallbackModel(string? model) /// Persists the installation manifest with same-directory atomic replacement. public sealed class LocalAiManifestStore { - private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + private static readonly JsonSerializerOptions JsonOptions = CreateJsonOptions(); + + private static JsonSerializerOptions CreateJsonOptions() { - WriteIndented = true, - UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow, - }; + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = true, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow, + }; + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower, allowIntegerValues: false)); + return options; + } private readonly LocalAiPaths _paths; @@ -330,6 +342,13 @@ public LocalAiResolvedInstall ResolveAndValidate(LocalAiInstallManifest manifest } if (manifest.ContextLength <= 0) throw new InvalidDataException("The local AI manifest context length must be positive."); + if (!Enum.IsDefined(manifest.KeyCachePrecision) || + !Enum.IsDefined(manifest.ValueCachePrecision) || + !Enum.IsDefined(manifest.DraftKeyCachePrecision) || + !Enum.IsDefined(manifest.DraftValueCachePrecision)) + { + throw new InvalidDataException("The local AI manifest KV cache precision is unsupported."); + } if (manifest.RuntimeAssets.IsDefaultOrEmpty) throw new InvalidDataException("The local AI manifest must record at least one runtime asset receipt."); diff --git a/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs b/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs index cb86d844f..2146869d4 100644 --- a/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs +++ b/src/OpenClaw.Connection/LocalAi/LocalAiRuntimeModels.cs @@ -1,3 +1,5 @@ +using OpenClaw.Shared.Inference.Catalog; + namespace OpenClaw.Connection.LocalAi; public enum LocalAiRuntimeState @@ -85,7 +87,12 @@ public sealed record LocalAiRuntimeSnapshot( int? ProcessId, DateTimeOffset? ProcessStartedAtUtc, string? Detail, - DateTimeOffset UpdatedAtUtc) + DateTimeOffset UpdatedAtUtc, + int? ContextLength = null, + KvCachePrecision? KeyCachePrecision = null, + KvCachePrecision? ValueCachePrecision = null, + KvCachePrecision? DraftKeyCachePrecision = null, + KvCachePrecision? DraftValueCachePrecision = null) { public static LocalAiRuntimeSnapshot Initial(Uri endpoint, DateTimeOffset now) => new( diff --git a/src/OpenClaw.SetupEngine.UI/Pages/CapabilitiesPage.xaml b/src/OpenClaw.SetupEngine.UI/Pages/CapabilitiesPage.xaml index c40d557b6..2c9c0d020 100644 --- a/src/OpenClaw.SetupEngine.UI/Pages/CapabilitiesPage.xaml +++ b/src/OpenClaw.SetupEngine.UI/Pages/CapabilitiesPage.xaml @@ -188,25 +188,13 @@ - - - - - - + Title="WSL networking change required" + Message="Setup will enable mirrored WSL networking and stop all running WSL distributions once; save any WSL work first." + Visibility="Collapsed" /> model.Weights.SizeBytes) - .FirstOrDefault(model => - _localAiSelectedGpuCapacityBytes is { } capacityBytes && - LocalInferenceEligibility.GetRequiredMemoryBytes(model) <= capacityBytes)?.Id; + _localAiRecommendedModelId = deviceEligibility.CanInstall + ? deviceEligibility.Plan?.Model.Id + : null; if (!deviceEligibility.CanInstall || deviceEligibility.Plan is null || deviceEligibility.SelectedGpu is null) { @@ -401,6 +393,7 @@ _localAiSelectedGpuCapacityBytes is { } capacityBytes && SetLocalAiOptionAvailability(isAvailable: true); _localAiSelectionEligible = eligibility.Status == LocalInferenceEligibilityStatus.Eligible; _config!.LocalAi.SelectedModelId ??= eligibility.Plan!.Model.Id; + _config.LocalAi.SelectedProfileId = eligibility.Plan!.Profile.Id; PopulateLocalAiModels(); _suppressLocalAiToggle = true; LocalAiToggle.IsOn = _config!.LocalAi.Enabled; @@ -571,7 +564,6 @@ private void SetLocalAiOptionAvailability(bool isAvailable, string? helpText = n LocalAiOptionContent.Opacity = isAvailable ? 1 : 0.55; LocalAiToggle.IsEnabled = isAvailable; LocalAiModelSelector.IsEnabled = isAvailable; - LocalAiNetworkingConsentCheckBox.IsEnabled = isAvailable; AutomationProperties.SetHelpText( LocalAiOptionContent, isAvailable @@ -630,21 +622,23 @@ private void PopulateLocalAiModels() _suppressLocalAiSelection = true; LocalAiModelSelector.Items.Clear(); int selectedIndex = 0; - LocalModelInfo[] fittingModels = LocalModelCatalog.Models - .Where(model => - _localAiSelectedGpuCapacityBytes is { } capacityBytes && - LocalInferenceEligibility.GetRequiredMemoryBytes(model) <= capacityBytes) + (LocalModelInfo Model, LocalInferencePlan Plan)[] fittingModels = LocalModelCatalog.Models + .Select(model => (Model: model, Eligibility: LocalInferenceEligibility.Evaluate(_localAiHardware!, model.Id))) + .Where(candidate => candidate.Eligibility.CanInstall && candidate.Eligibility.Plan is not null) + .Select(candidate => (candidate.Model, candidate.Eligibility.Plan!)) .ToArray(); for (int index = 0; index < fittingModels.Length; index++) { - LocalModelInfo model = fittingModels[index]; + (LocalModelInfo model, LocalInferencePlan plan) = fittingModels[index]; bool isRecommended = string.Equals( _localAiRecommendedModelId, model.Id, StringComparison.OrdinalIgnoreCase); LocalAiModelSelector.Items.Add(new ComboBoxItem { - Content = $"{model.DisplayName} ({FormatSize(model.Weights.SizeBytes)})" + + Content = $"{model.DisplayName} ({FormatSize(model.Weights.SizeBytes)}, " + + $"{FormatContext(plan.Profile.ContextTokens)}, " + + $"{LocalModelCatalog.ToDisplayCacheType(plan.Profile.KeyCachePrecision)} KV)" + (isRecommended ? " - Recommended" : string.Empty), Tag = model.Id, }); @@ -676,17 +670,6 @@ private void LocalAiModelSelector_SelectionChanged(object sender, SelectionChang ApplySetupReviewSummary(_config); } - private void LocalAiNetworkingConsent_Changed(object sender, RoutedEventArgs e) - { - if (_suppressLocalAiConsent || _config is null) - return; - _config.LocalAi.WslMirroredNetworkingConsent = - LocalAiToggle.IsOn == true && - _localAiNetworkingConsentRequired && - LocalAiNetworkingConsentCheckBox.IsChecked == true; - UpdatePrimaryButtonState(); - } - private void UpdateLocalAiOptions(bool forceNetworkingConsent = false) { var config = _config!; @@ -695,12 +678,11 @@ private void UpdateLocalAiOptions(bool forceNetworkingConsent = false) config.SkipWizard = enabled || _skipWizardWithoutLocalAi; LocalAiDetailsPanel.Visibility = enabled ? Visibility.Visible : Visibility.Collapsed; LocalAiNetworkingInspectionError.Visibility = Visibility.Collapsed; - _localAiNetworkingConsentRequired = false; + _localAiNetworkingChangeRequired = false; if (!enabled) { LocalAiNetworkingConsentPanel.Visibility = Visibility.Collapsed; - SetLocalAiNetworkingConsent(false); config.LocalAi.WslMirroredNetworkingConsent = false; UpdatePrimaryButtonState(); return; @@ -710,12 +692,11 @@ private void UpdateLocalAiOptions(bool forceNetworkingConsent = false) WslGlobalConfigStatus status = forceNetworkingConsent ? new(false, false) : _localAiNetworkingStatus ?? new(false, false); - _localAiNetworkingConsentRequired = !status.IsMirrored; - LocalAiNetworkingConsentPanel.Visibility = _localAiNetworkingConsentRequired + _localAiNetworkingChangeRequired = !status.IsMirrored; + LocalAiNetworkingConsentPanel.Visibility = _localAiNetworkingChangeRequired ? Visibility.Visible : Visibility.Collapsed; - SetLocalAiNetworkingConsent(false); - config.LocalAi.WslMirroredNetworkingConsent = false; + config.LocalAi.WslMirroredNetworkingConsent = true; UpdatePrimaryButtonState(); } @@ -731,40 +712,34 @@ private void UpdateLocalAiModelDetails() if (eligibility.Plan is not { } plan || eligibility.SelectedGpu is not { } gpu) { _localAiSelectionEligible = false; + _config!.LocalAi.SelectedProfileId = null; LocalAiHardwareStatusText.Text = "This model is not qualified for the detected hardware."; UpdatePrimaryButtonState(); return; } _localAiSelectionEligible = eligibility.Status == LocalInferenceEligibilityStatus.Eligible; + _config!.LocalAi.SelectedProfileId = plan.Profile.Id; LocalAiHardwareStatusText.Text = eligibility.Status switch { LocalInferenceEligibilityStatus.Eligible => - $"Detected {gpu.Name} with {FormatOptionalSize(eligibility.DetectedTotalMemoryBytes)}. " + - $"The selected model requires {FormatSize(eligibility.RequiredTotalMemoryBytes)}.", + $"{FormatMemorySize(eligibility.RequiredTotalMemoryBytes)} required \u00b7 " + + $"{FormatOptionalMemorySize(eligibility.DetectedTotalMemoryBytes)} CUDA-visible on {gpu.Name}", LocalInferenceEligibilityStatus.EligibleButBusy => - $"Detected {gpu.Name}, but only {FormatOptionalSize(eligibility.AvailableFreeMemoryBytes)} of " + - $"{FormatSize(eligibility.RequiredFreeMemoryBytes)} required GPU memory is currently free. " + + $"Detected {gpu.Name}, but only {FormatOptionalMemorySize(eligibility.AvailableFreeMemoryBytes)} of " + + $"{FormatMemorySize(eligibility.RequiredFreeMemoryBytes)} required GPU memory is currently free. " + "Close GPU applications and retry setup.", _ => DescribeLocalAiUnavailable(eligibility), }; LocalAiEngineDetailText.Text = "llama-server for Windows; " + - $"{FormatSize(plan.Runtime.Artifacts.Sum(artifact => artifact.SizeBytes))} verified download"; + $"{FormatSize(plan.Runtime.Artifacts.Sum(artifact => artifact.SizeBytes))} verified download; " + + "loads on first request"; LocalAiModelDetailText.Text = $"{plan.Model.DisplayName}, {FormatSize(plan.Model.Weights.SizeBytes)} from Hugging Face"; - LocalAiSettingsDetailText.Text = - $"{plan.Model.Recipe.ContextTokens / 1024}K context, FP16 KV cache, full CUDA offload, loads on first request"; UpdatePrimaryButtonState(); } - private void SetLocalAiNetworkingConsent(bool value) - { - _suppressLocalAiConsent = true; - LocalAiNetworkingConsentCheckBox.IsChecked = value; - _suppressLocalAiConsent = false; - } - private void UpdatePrimaryButtonState() { // Local AI availability being merely pending (Checking/ProbeUnknown) must never let @@ -776,15 +751,24 @@ private void UpdatePrimaryButtonState() PrimaryButton.IsEnabled = _step != 3 || LocalAiToggle.IsOn != true || - (_localAiSelectionEligible && - (!_localAiNetworkingConsentRequired || LocalAiNetworkingConsentCheckBox.IsChecked == true)); + _localAiSelectionEligible; } private static string FormatSize(long bytes) => $"{bytes / 1_000_000_000d:0.#} GB"; - private static string FormatOptionalSize(long? bytes) => - bytes is { } value ? FormatSize(value) : "an unknown amount"; + private static string FormatMemorySize(long bytes) => + $"{bytes / (1024d * 1024d * 1024d):0.#} GiB"; + + private static string FormatOptionalMemorySize(long? bytes) => + bytes is { } value ? FormatMemorySize(value) : "an unknown amount"; + + private static string FormatContext(int tokens) => + tokens % 1024 == 0 + ? $"{tokens / 1024}K" + : tokens % 1000 == 0 + ? $"{tokens / 1000}K" + : $"{tokens:N0} tokens"; private void TailscaleToggle_Toggled(object sender, RoutedEventArgs e) { diff --git a/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs b/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs index ccda8c2de..e805c8ef3 100644 --- a/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs +++ b/src/OpenClaw.SetupEngine.UI/Pages/ProgressPage.xaml.cs @@ -121,7 +121,7 @@ private void RenderProgressPreview() row.SetStatus(status); } if (localAiPreview && _rows.TryGetValue("local-ai-model", out var modelRow)) - modelRow.SetDetail("Downloading Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", 8_701_231_104, 22_663_387_424, SetupDetailProgressUnit.Bytes); + modelRow.SetDetail("Downloading Qwen3.8-27B-UD-Q4_K_M.gguf", 6_322_405_376, 16_464_440_224, SetupDetailProgressUnit.Bytes); LogText.Text = "[12:04:01] [info] Windows 11 26100 · WSL 2 present\n" + "[12:04:03] [info] port 127.0.0.1:18789 available\n" + diff --git a/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs b/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs index 100a6130d..9b3b9b55f 100644 --- a/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs +++ b/src/OpenClaw.SetupEngine/LocalAiInstallReconciler.cs @@ -119,6 +119,11 @@ private static void ValidateRecipeMatch( !string.Equals(manifest.Architecture, expectedArchitecture, StringComparison.Ordinal) || !string.Equals(manifest.RuntimeId, plan.Runtime.Id, StringComparison.Ordinal) || !string.Equals(manifest.ModelCatalogId, plan.Model.Id, StringComparison.Ordinal) || + manifest.ContextLength != plan.Profile.ContextTokens || + manifest.KeyCachePrecision != plan.Profile.KeyCachePrecision || + manifest.ValueCachePrecision != plan.Profile.ValueCachePrecision || + manifest.DraftKeyCachePrecision != plan.Profile.DraftKeyCachePrecision || + manifest.DraftValueCachePrecision != plan.Profile.DraftValueCachePrecision || !string.Equals(manifest.SelectedGpuId, selectedGpuId, StringComparison.Ordinal)) { throw new InvalidDataException( diff --git a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs index a8b2511e3..dd731f583 100644 --- a/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs +++ b/src/OpenClaw.SetupEngine/LocalAiSetupSteps.cs @@ -52,6 +52,7 @@ public override Task ExecuteAsync(SetupContext ctx, CancellationToke ctx.Config.LocalAi.SelectedModelId); ctx.LocalAiHardware = hardware; ctx.LocalAiEligibility = eligibility; + ctx.Config.LocalAi.SelectedProfileId = eligibility.Plan?.Profile.Id; if (eligibility.Status == LocalInferenceEligibilityStatus.Unsupported) { @@ -557,7 +558,11 @@ ctx.LocalAiRuntimeInstall is not { } runtimeInstall || }, RequestedPort = requestedPort, Endpoint = null, - ContextLength = plan.Model.Recipe.ContextTokens, + ContextLength = plan.Profile.ContextTokens, + KeyCachePrecision = plan.Profile.KeyCachePrecision, + ValueCachePrecision = plan.Profile.ValueCachePrecision, + DraftKeyCachePrecision = plan.Profile.DraftKeyCachePrecision, + DraftValueCachePrecision = plan.Profile.DraftValueCachePrecision, }; var store = new LocalAiManifestStore(paths); diff --git a/src/OpenClaw.SetupEngine/SetupContext.cs b/src/OpenClaw.SetupEngine/SetupContext.cs index 144aa6b63..2f416bd94 100644 --- a/src/OpenClaw.SetupEngine/SetupContext.cs +++ b/src/OpenClaw.SetupEngine/SetupContext.cs @@ -142,6 +142,9 @@ public sealed class LocalAiConfig { public bool Enabled { get; set; } public string? SelectedModelId { get; set; } + /// Runtime-only effective profile selected from detected GPU capacity. + [JsonIgnore] + public string? SelectedProfileId { get; set; } /// Managed llama-server port. Zero selects a free loopback port during setup. public int Port { get; set; } public bool WslMirroredNetworkingConsent { get; set; } diff --git a/src/OpenClaw.SetupEngine/SetupReviewSummary.cs b/src/OpenClaw.SetupEngine/SetupReviewSummary.cs index c1f53770b..aac62245a 100644 --- a/src/OpenClaw.SetupEngine/SetupReviewSummary.cs +++ b/src/OpenClaw.SetupEngine/SetupReviewSummary.cs @@ -62,6 +62,8 @@ public static SetupReviewSummary Build(SetupConfig config, string? dataDir = nul $"curl -fsSL --proto '=https' --tlsv1.2 | bash -s -- --version {release.Version}{runtimeArgument}"; LocalModelInfo localAiModel = LocalModelCatalog.Find(config.LocalAi.SelectedModelId) ?? LocalModelCatalog.Default; + LocalInferenceRunProfile? localAiProfile = + LocalModelCatalog.FindProfile(localAiModel, config.LocalAi.SelectedProfileId); string[] localAiCommands = config.LocalAi.Enabled ? [ @@ -106,12 +108,32 @@ public static SetupReviewSummary Build(SetupConfig config, string? dataDir = nul ? $"Local AI verified with {localAiModel.DisplayName}" : null, LocalAiDescription = config.LocalAi.Enabled - ? "llama-server · " + - $"{localAiModel.Recipe.ContextTokens / 1024}K context · FP16 KV · full CUDA offload · loads on first request" + ? localAiProfile is null + ? "llama-server for Windows · loads on first request · " + + "context and KV profile selected from detected GPU" + : "llama-server for Windows · loads on first request · " + + $"{FormatContext(localAiProfile.ContextTokens)} context · " + + $"{FormatKvCache(localAiProfile)}" : null, }; } + private static string FormatContext(int tokens) => + tokens % 1024 == 0 + ? $"{tokens / 1024}K" + : tokens % 1000 == 0 + ? $"{tokens / 1000}K" + : $"{tokens:N0} tokens"; + + private static string FormatKvCache(LocalInferenceRunProfile profile) + { + string target = LocalModelCatalog.ToDisplayCacheType(profile.KeyCachePrecision); + string draft = LocalModelCatalog.ToDisplayCacheType(profile.DraftKeyCachePrecision); + return target == draft + ? $"{target} target and MTP draft KV" + : $"{target} target KV and {draft} MTP draft KV"; + } + private static string Display(string? value, string fallback) => string.IsNullOrWhiteSpace(value) ? fallback : value.Trim(); diff --git a/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceEligibility.cs b/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceEligibility.cs index 4892dde2a..1e1437ebf 100644 --- a/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceEligibility.cs +++ b/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceEligibility.cs @@ -43,8 +43,10 @@ public static class LocalInferenceEligibility public const long RuntimeWorkspaceReserveBytes = LocalModelCatalog.RuntimeWorkspaceReserveBytes; public static Version MinimumNvidiaDriverVersion { get; } = new(615, 0); - public static long GetRequiredMemoryBytes(LocalModelInfo model) => - LocalInferenceQualificationPolicy.GetRequiredMemoryBytes(model); + public static long GetRequiredMemoryBytes( + LocalModelInfo model, + LocalInferenceRunProfile profile) => + LocalInferenceQualificationPolicy.GetRequiredMemoryBytes(model, profile); public static LocalInferenceEligibilityResult Evaluate( HostHardwareInfo hardware, @@ -61,7 +63,7 @@ public static LocalInferenceEligibilityResult Evaluate( } LocalInferencePlan plan = selection.Plan; - long requiredMemoryBytes = GetRequiredMemoryBytes(plan.Model); + long requiredMemoryBytes = GetRequiredMemoryBytes(plan.Model, plan.Profile); CandidateAssessment? selected = hardware.NvidiaGpus .Select(gpu => Assess(gpu, plan.Runtime, requiredMemoryBytes)) .OrderBy(candidate => StatusRank(candidate.Status)) diff --git a/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceEligibilityDiagnostics.cs b/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceEligibilityDiagnostics.cs index e92bf8803..f314a2b40 100644 --- a/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceEligibilityDiagnostics.cs +++ b/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceEligibilityDiagnostics.cs @@ -65,5 +65,5 @@ public static LocalInferenceUnavailableReason GetUnavailableReason(LocalInferenc LocalInferenceEligibility.MinimumNvidiaDriverVersion.ToString()); } - private static double ToGigabytes(long bytes) => bytes / 1_000_000_000d; + private static double ToGigabytes(long bytes) => bytes / (1024d * 1024d * 1024d); } diff --git a/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceSelector.cs b/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceSelector.cs index 1cc8e1462..1d5e0f0a9 100644 --- a/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceSelector.cs +++ b/src/OpenClaw.Shared/Inference/Catalog/LocalInferenceSelector.cs @@ -29,6 +29,7 @@ public enum LocalInferenceModelSelectionOrigin public sealed record LocalInferencePlan( LlamaRuntimeVariant Runtime, LocalModelInfo Model, + LocalInferenceRunProfile Profile, LocalInferenceModelSelectionOrigin ModelSelectionOrigin); /// The deterministic result of selecting from the pinned local inference catalog. @@ -78,16 +79,11 @@ public static LocalInferenceSelectionResult Select( return LocalInferenceSelectionResult.Unsupported(LocalInferenceSelectionFailureCode.NoNvidiaGpu); LocalModelInfo? model; + LocalInferenceRunProfile profile; LocalInferenceModelSelectionOrigin modelSelectionOrigin; if (string.IsNullOrWhiteSpace(requestedModelId)) { - model = LocalModelCatalog.Models - .OrderByDescending(candidate => candidate.Weights.SizeBytes) - .FirstOrDefault(candidate => hardware.NvidiaGpus.Any(gpu => - LocalInferenceQualificationPolicy.HasRuntimePrerequisites(gpu, runtime) && - LocalInferenceQualificationPolicy.GetEffectiveTotalMemoryBytes(gpu) >= - LocalInferenceQualificationPolicy.GetRequiredMemoryBytes(candidate))) - ?? LocalModelCatalog.Models.OrderBy(candidate => candidate.Weights.SizeBytes).First(); + (model, profile) = SelectDefaultModelAndProfile(hardware, runtime); modelSelectionOrigin = LocalInferenceModelSelectionOrigin.Default; } else @@ -95,12 +91,41 @@ public static LocalInferenceSelectionResult Select( model = LocalModelCatalog.Find(requestedModelId); if (model is null) return LocalInferenceSelectionResult.Unsupported(LocalInferenceSelectionFailureCode.UnknownModel); + profile = SelectBestFittingProfile(hardware, runtime, model) ?? + LocalModelCatalog.GetProfiles(model)[^1]; modelSelectionOrigin = LocalInferenceModelSelectionOrigin.Explicit; } return LocalInferenceSelectionResult.Selected( - new LocalInferencePlan(runtime, model, modelSelectionOrigin)); + new LocalInferencePlan(runtime, model, profile, modelSelectionOrigin)); } + + private static (LocalModelInfo Model, LocalInferenceRunProfile Profile) SelectDefaultModelAndProfile( + HostHardwareInfo hardware, + LlamaRuntimeVariant runtime) + { + foreach (LocalModelInfo candidate in LocalModelCatalog.Models + .OrderByDescending(model => model.RecommendationPriority) + .ThenByDescending(model => model.Weights.SizeBytes)) + { + LocalInferenceRunProfile? profile = SelectBestFittingProfile(hardware, runtime, candidate); + if (profile is not null) + return (candidate, profile); + } + + LocalModelInfo fallback = LocalModelCatalog.Models.OrderBy(model => model.Weights.SizeBytes).First(); + return (fallback, LocalModelCatalog.GetProfiles(fallback)[^1]); + } + + private static LocalInferenceRunProfile? SelectBestFittingProfile( + HostHardwareInfo hardware, + LlamaRuntimeVariant runtime, + LocalModelInfo model) => + LocalModelCatalog.GetProfiles(model).FirstOrDefault(profile => + hardware.NvidiaGpus.Any(gpu => + LocalInferenceQualificationPolicy.HasRuntimePrerequisites(gpu, runtime) && + LocalInferenceQualificationPolicy.GetEffectiveTotalMemoryBytes(gpu) >= + LocalInferenceQualificationPolicy.GetRequiredMemoryBytes(model, profile))); } internal static class LocalInferenceQualificationPolicy @@ -121,24 +146,55 @@ public static bool HasRuntimePrerequisites(GpuInfo gpu, LlamaRuntimeVariant runt gpu.CudaMajorVersion >= runtime.CudaVersion.Major; } - public static long GetRequiredMemoryBytes(LocalModelInfo model) + public static long GetRequiredMemoryBytes( + LocalModelInfo model, + LocalInferenceRunProfile profile) { ArgumentNullException.ThrowIfNull(model); + ArgumentNullException.ThrowIfNull(profile); return SaturatingAdd( - SaturatingAdd(model.Weights.SizeBytes, GetKvCacheMemoryBytes(model.Recipe)), - model.Recipe.RuntimeWorkspaceBytes); + SaturatingAdd( + SaturatingAdd(model.Weights.SizeBytes, GetKvCacheMemoryBytes(model.Recipe, profile)), + GetDraftKvCacheMemoryBytes(model.Recipe, profile)), + profile.RuntimeWorkspaceBytes); + } + + internal static long GetKvCacheMemoryBytes( + LocalModelRunRecipe recipe, + LocalInferenceRunProfile profile) + { + ArgumentNullException.ThrowIfNull(recipe); + ArgumentNullException.ThrowIfNull(profile); + long vectorsPerTypePerToken = SaturatingMultiply( + recipe.FullAttentionLayerCount, + recipe.KeyValueHeadCount); + long bytesPerToken = SaturatingAdd( + SaturatingMultiply( + vectorsPerTypePerToken, + EncodedBytes(recipe.KeyValueHeadDimension, profile.KeyCachePrecision)), + SaturatingMultiply( + vectorsPerTypePerToken, + EncodedBytes(recipe.KeyValueHeadDimension, profile.ValueCachePrecision))); + return SaturatingMultiply(bytesPerToken, profile.ContextTokens); } - internal static long GetKvCacheMemoryBytes(LocalModelRunRecipe recipe) + internal static long GetDraftKvCacheMemoryBytes( + LocalModelRunRecipe recipe, + LocalInferenceRunProfile profile) { ArgumentNullException.ThrowIfNull(recipe); - long keyBytes = BytesPerElement(recipe.KeyCachePrecision); - long valueBytes = BytesPerElement(recipe.ValueCachePrecision); - long bytesPerToken = SaturatingMultiply( - SaturatingMultiply(recipe.FullAttentionLayerCount, recipe.KeyValueHeadCount), - recipe.KeyValueHeadDimension); - bytesPerToken = SaturatingMultiply(bytesPerToken, SaturatingAdd(keyBytes, valueBytes)); - return SaturatingMultiply(bytesPerToken, recipe.ContextTokens); + ArgumentNullException.ThrowIfNull(profile); + + // The pinned Qwen MTP artifacts contain one draft attention layer with + // the same KV head count and head dimension as the target model. + long bytesPerToken = SaturatingAdd( + SaturatingMultiply( + recipe.KeyValueHeadCount, + EncodedBytes(recipe.KeyValueHeadDimension, profile.DraftKeyCachePrecision)), + SaturatingMultiply( + recipe.KeyValueHeadCount, + EncodedBytes(recipe.KeyValueHeadDimension, profile.DraftValueCachePrecision))); + return SaturatingMultiply(bytesPerToken, profile.ContextTokens); } public static long GetEffectiveTotalMemoryBytes(GpuInfo gpu) => @@ -167,9 +223,10 @@ private static bool IsStableGpuId(string? value) => !string.IsNullOrWhiteSpace(value) && !value.Any(character => char.IsControl(character) || char.IsWhiteSpace(character)); - private static long BytesPerElement(KvCachePrecision precision) => precision switch + private static long EncodedBytes(long elementCount, KvCachePrecision precision) => precision switch { - KvCachePrecision.F16 => 2, + KvCachePrecision.F16 => SaturatingMultiply(elementCount, 2), + KvCachePrecision.Q8_0 => SaturatingMultiply((SaturatingAdd(elementCount, 31)) / 32, 34), _ => throw new ArgumentOutOfRangeException(nameof(precision)), }; diff --git a/src/OpenClaw.Shared/Inference/Catalog/LocalModelCatalog.cs b/src/OpenClaw.Shared/Inference/Catalog/LocalModelCatalog.cs index c80cec169..37d251d9b 100644 --- a/src/OpenClaw.Shared/Inference/Catalog/LocalModelCatalog.cs +++ b/src/OpenClaw.Shared/Inference/Catalog/LocalModelCatalog.cs @@ -6,6 +6,7 @@ namespace OpenClaw.Shared.Inference.Catalog; public enum KvCachePrecision { F16 = 0, + Q8_0 = 1, } /// Speculative decoding implementation used by a model recipe. @@ -27,24 +28,18 @@ public sealed record ModelSamplingPreset( public sealed record LocalModelRunRecipe { public LocalModelRunRecipe( - int contextTokens, - KvCachePrecision keyCachePrecision, - KvCachePrecision valueCachePrecision, int batchTokens, int microBatchTokens, int parallelRequests, int fullAttentionLayerCount, int keyValueHeadCount, int keyValueHeadDimension, - long runtimeWorkspaceBytes, bool flashAttention, bool offloadAllLayers, SpeculativeDecodingMode speculativeDecoding, int speculativeDraftMaxTokens, ModelSamplingPreset sampling) { - if (contextTokens <= 0) - throw new ArgumentOutOfRangeException(nameof(contextTokens)); if (batchTokens <= 0) throw new ArgumentOutOfRangeException(nameof(batchTokens)); if (microBatchTokens <= 0 || microBatchTokens > batchTokens) @@ -57,22 +52,16 @@ public LocalModelRunRecipe( throw new ArgumentOutOfRangeException(nameof(keyValueHeadCount)); if (keyValueHeadDimension <= 0) throw new ArgumentOutOfRangeException(nameof(keyValueHeadDimension)); - if (runtimeWorkspaceBytes <= 0) - throw new ArgumentOutOfRangeException(nameof(runtimeWorkspaceBytes)); if (speculativeDraftMaxTokens <= 0) throw new ArgumentOutOfRangeException(nameof(speculativeDraftMaxTokens)); ArgumentNullException.ThrowIfNull(sampling); - ContextTokens = contextTokens; - KeyCachePrecision = keyCachePrecision; - ValueCachePrecision = valueCachePrecision; BatchTokens = batchTokens; MicroBatchTokens = microBatchTokens; ParallelRequests = parallelRequests; FullAttentionLayerCount = fullAttentionLayerCount; KeyValueHeadCount = keyValueHeadCount; KeyValueHeadDimension = keyValueHeadDimension; - RuntimeWorkspaceBytes = runtimeWorkspaceBytes; FlashAttention = flashAttention; OffloadAllLayers = offloadAllLayers; SpeculativeDecoding = speculativeDecoding; @@ -80,16 +69,12 @@ public LocalModelRunRecipe( Sampling = sampling; } - public int ContextTokens { get; } - public KvCachePrecision KeyCachePrecision { get; } - public KvCachePrecision ValueCachePrecision { get; } public int BatchTokens { get; } public int MicroBatchTokens { get; } public int ParallelRequests { get; } public int FullAttentionLayerCount { get; } public int KeyValueHeadCount { get; } public int KeyValueHeadDimension { get; } - public long RuntimeWorkspaceBytes { get; } public bool FlashAttention { get; } public bool OffloadAllLayers { get; } public SpeculativeDecodingMode SpeculativeDecoding { get; } @@ -107,19 +92,69 @@ public sealed record LocalModelInfo( LocalModelRunRecipe Recipe, bool IsDefault, bool IsExplicitAlternative, - bool SupportsVision); + bool SupportsVision, + int RecommendationPriority = 0); + +/// The capacity-sensitive settings selected for one model launch. +public sealed record LocalInferenceRunProfile +{ + public LocalInferenceRunProfile( + string id, + int contextTokens, + KvCachePrecision keyCachePrecision, + KvCachePrecision valueCachePrecision, + KvCachePrecision draftKeyCachePrecision, + KvCachePrecision draftValueCachePrecision, + long runtimeWorkspaceBytes) + { + ArgumentException.ThrowIfNullOrWhiteSpace(id); + if (contextTokens <= 0) + throw new ArgumentOutOfRangeException(nameof(contextTokens)); + if (runtimeWorkspaceBytes <= 0) + throw new ArgumentOutOfRangeException(nameof(runtimeWorkspaceBytes)); + + Id = id; + ContextTokens = contextTokens; + KeyCachePrecision = keyCachePrecision; + ValueCachePrecision = valueCachePrecision; + DraftKeyCachePrecision = draftKeyCachePrecision; + DraftValueCachePrecision = draftValueCachePrecision; + RuntimeWorkspaceBytes = runtimeWorkspaceBytes; + } + + public string Id { get; } + public int ContextTokens { get; } + public KvCachePrecision KeyCachePrecision { get; } + public KvCachePrecision ValueCachePrecision { get; } + public KvCachePrecision DraftKeyCachePrecision { get; } + public KvCachePrecision DraftValueCachePrecision { get; } + public long RuntimeWorkspaceBytes { get; } +} /// Immutable Hugging Face model pins offered by the Windows local inference flow. public static class LocalModelCatalog { + public const string Qwen38_27BModelId = "qwen3.8-27b-mtp-ud-q4-k-m"; public const string Qwen35BModelId = "qwen3.6-35b-a3b-mtp-q4-k-m"; public const string Qwen27BModelId = "qwen3.6-27b-mtp-q4-k-m"; public const string Qwen9BModelId = "qwen3.5-9b-mtp-q4-k-m"; public const int NativeContextTokens = 262_144; + public const int IntermediateContextTokens = 196_608; + public const int ReducedContextTokens = 131_072; + public const int MinimumContextTokens = 65_536; - // The pinned 262K MTP recipe also allocates draft KV, compute buffers, - // recurrent state, and backend workspace beyond weights and primary KV. + // Measured-conservative allowances for compute buffers, recurrent state, + // CUDA graphs, allocator alignment, and miscellaneous backend allocations. + // These buffers shrink with context size; the tiers retain at least about + // 0.5 GiB of guard over the corresponding RTX 5090 peak measurements. public const long RuntimeWorkspaceReserveBytes = 8L * 1024 * 1024 * 1024; + public const long IntermediateContextWorkspaceReserveBytes = 7L * 1024 * 1024 * 1024; + public const long ReducedContextWorkspaceReserveBytes = 5L * 1024 * 1024 * 1024; + public const long MinimumContextWorkspaceReserveBytes = 4L * 1024 * 1024 * 1024; + + private static readonly HuggingFaceRevisionSource s_qwen38_27BSource = new( + "unsloth/Qwen3.8-27B-GGUF", + "313447f257f7ebde0b968e4778feef774546ed81"); private static readonly HuggingFaceRevisionSource s_qwen35BSource = new( "unsloth/Qwen3.6-35B-A3B-MTP-GGUF", @@ -136,6 +171,25 @@ public static class LocalModelCatalog private static readonly ReadOnlyCollection s_models = Array.AsReadOnly( new[] { + new LocalModelInfo( + Qwen38_27BModelId, + "Qwen3.8 27B (UD-Q4_K_M)", + "Qwen3.8", + "UD-Q4_K_M", + ModelArtifact( + Qwen38_27BModelId, + s_qwen38_27BSource, + "Qwen3.8-27B-UD-Q4_K_M.gguf", + 16_464_440_224, + "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482"), + Recipe( + fullAttentionLayerCount: 16, + keyValueHeadCount: 4, + temperature: 1.0), + IsDefault: true, + IsExplicitAlternative: false, + SupportsVision: false, + RecommendationPriority: 400), new LocalModelInfo( Qwen35BModelId, "Qwen3.6 35B-A3B (UD-Q4_K_M)", @@ -151,9 +205,10 @@ public static class LocalModelCatalog fullAttentionLayerCount: 10, keyValueHeadCount: 2, temperature: 0.6), - IsDefault: true, - IsExplicitAlternative: false, - SupportsVision: false), + IsDefault: false, + IsExplicitAlternative: true, + SupportsVision: false, + RecommendationPriority: 300), new LocalModelInfo( Qwen27BModelId, "Qwen3.6 27B (Q4_K_M)", @@ -171,7 +226,8 @@ public static class LocalModelCatalog temperature: 1.0), IsDefault: false, IsExplicitAlternative: true, - SupportsVision: false), + SupportsVision: false, + RecommendationPriority: 200), new LocalModelInfo( Qwen9BModelId, "Qwen3.5 9B (Q4_K_M)", @@ -189,9 +245,16 @@ public static class LocalModelCatalog temperature: 1.0), IsDefault: false, IsExplicitAlternative: true, - SupportsVision: false), + SupportsVision: false, + RecommendationPriority: 100), }); + private static readonly IReadOnlyDictionary> + s_profilesByModel = s_models.ToDictionary( + model => model.Id, + model => Array.AsReadOnly(CreateProfiles(model)), + StringComparer.OrdinalIgnoreCase); + private static readonly ReadOnlyCollection s_explicitAlternatives = Array.AsReadOnly(s_models.Where(model => model.IsExplicitAlternative).ToArray()); @@ -201,11 +264,88 @@ public static class LocalModelCatalog public static IReadOnlyList ExplicitAlternatives => s_explicitAlternatives; + public static IReadOnlyList GetProfiles(LocalModelInfo model) + { + ArgumentNullException.ThrowIfNull(model); + return s_profilesByModel.TryGetValue(model.Id, out ReadOnlyCollection? profiles) + ? profiles + : throw new ArgumentException("The model is not part of the local inference catalog.", nameof(model)); + } + + public static LocalInferenceRunProfile? FindProfile(LocalModelInfo model, string? profileId) => + string.IsNullOrWhiteSpace(profileId) + ? null + : GetProfiles(model).SingleOrDefault(profile => + string.Equals(profile.Id, profileId, StringComparison.OrdinalIgnoreCase)); + + public static LocalInferenceRunProfile? FindProfile( + LocalModelInfo model, + int contextTokens, + KvCachePrecision keyCachePrecision, + KvCachePrecision valueCachePrecision, + KvCachePrecision draftKeyCachePrecision, + KvCachePrecision draftValueCachePrecision) => + GetProfiles(model).SingleOrDefault(profile => + profile.ContextTokens == contextTokens && + profile.KeyCachePrecision == keyCachePrecision && + profile.ValueCachePrecision == valueCachePrecision && + profile.DraftKeyCachePrecision == draftKeyCachePrecision && + profile.DraftValueCachePrecision == draftValueCachePrecision); + + public static string ToLlamaServerCacheType(KvCachePrecision precision) => precision switch + { + KvCachePrecision.F16 => "f16", + KvCachePrecision.Q8_0 => "q8_0", + _ => throw new ArgumentOutOfRangeException(nameof(precision)), + }; + + public static string ToDisplayCacheType(KvCachePrecision precision) => precision switch + { + KvCachePrecision.F16 => "F16", + KvCachePrecision.Q8_0 => "Q8_0", + _ => throw new ArgumentOutOfRangeException(nameof(precision)), + }; + public static LocalModelInfo? Find(string? id) => string.IsNullOrWhiteSpace(id) ? null : s_models.SingleOrDefault(model => string.Equals(model.Id, id, StringComparison.OrdinalIgnoreCase)); + private static LocalInferenceRunProfile[] CreateProfiles(LocalModelInfo model) => + [ + Profile(model, NativeContextTokens, KvCachePrecision.F16), + Profile(model, NativeContextTokens, KvCachePrecision.Q8_0), + Profile(model, IntermediateContextTokens, KvCachePrecision.F16), + Profile(model, IntermediateContextTokens, KvCachePrecision.Q8_0), + Profile(model, ReducedContextTokens, KvCachePrecision.F16), + Profile(model, ReducedContextTokens, KvCachePrecision.Q8_0), + Profile(model, MinimumContextTokens, KvCachePrecision.F16), + Profile(model, MinimumContextTokens, KvCachePrecision.Q8_0), + ]; + + private static LocalInferenceRunProfile Profile( + LocalModelInfo model, + int contextTokens, + KvCachePrecision precision) + { + long workspaceBytes = contextTokens switch + { + NativeContextTokens => RuntimeWorkspaceReserveBytes, + IntermediateContextTokens => IntermediateContextWorkspaceReserveBytes, + ReducedContextTokens => ReducedContextWorkspaceReserveBytes, + MinimumContextTokens => MinimumContextWorkspaceReserveBytes, + _ => throw new ArgumentOutOfRangeException(nameof(contextTokens)), + }; + return new LocalInferenceRunProfile( + $"ctx-{contextTokens}-{ToLlamaServerCacheType(precision)}", + contextTokens, + precision, + precision, + precision, + precision, + workspaceBytes); + } + private static PinnedArtifact ModelArtifact( string id, HuggingFaceRevisionSource source, @@ -226,16 +366,12 @@ private static LocalModelRunRecipe Recipe( int keyValueHeadCount, double temperature) => new( - contextTokens: NativeContextTokens, - keyCachePrecision: KvCachePrecision.F16, - valueCachePrecision: KvCachePrecision.F16, batchTokens: 4_096, microBatchTokens: 4_096, parallelRequests: 1, fullAttentionLayerCount: fullAttentionLayerCount, keyValueHeadCount: keyValueHeadCount, keyValueHeadDimension: 256, - runtimeWorkspaceBytes: RuntimeWorkspaceReserveBytes, flashAttention: true, offloadAllLayers: true, speculativeDecoding: SpeculativeDecodingMode.DraftMtp, diff --git a/src/OpenClaw.Tray.WinUI/Pages/LocalAiPage.xaml.cs b/src/OpenClaw.Tray.WinUI/Pages/LocalAiPage.xaml.cs index ecc2743b6..a188191a8 100644 --- a/src/OpenClaw.Tray.WinUI/Pages/LocalAiPage.xaml.cs +++ b/src/OpenClaw.Tray.WinUI/Pages/LocalAiPage.xaml.cs @@ -63,8 +63,8 @@ private void RefreshFromViewModel() ModelRecipeText.Text = string.Format( CultureInfo.CurrentCulture, LocalizationHelper.GetString("LocalAiPage_ModelRecipeFormat"), - LocalAiPageViewModel.ContextLengthText, - LocalAiPageViewModel.KvCacheText); + _viewModel.ContextLengthText, + _viewModel.KvCacheText); ModelStatusDot.Fill = ModelStatusText.Foreground = ResolveBrush(_viewModel.ModelState switch { LocalAiModelPresentationState.Loaded => "SystemFillColorSuccessBrush", diff --git a/src/OpenClaw.Tray.WinUI/Presentation/LocalAiPageViewModel.cs b/src/OpenClaw.Tray.WinUI/Presentation/LocalAiPageViewModel.cs index e1a4cae77..2d50bdf3e 100644 --- a/src/OpenClaw.Tray.WinUI/Presentation/LocalAiPageViewModel.cs +++ b/src/OpenClaw.Tray.WinUI/Presentation/LocalAiPageViewModel.cs @@ -78,8 +78,10 @@ public LocalAiPageViewModel( public string? EngineDetail => _runtimeSnapshot.Detail; public string? ModelName => LocalModelCatalog.Find(_runtimeSnapshot.ModelId)?.DisplayName ?? _runtimeSnapshot.ModelId; - public const string ContextLengthText = "256K"; - public const string KvCacheText = "FP16"; + public string ContextLengthText => _runtimeSnapshot.ContextLength is { } tokens + ? FormatContext(tokens) + : "Unknown"; + public string KvCacheText => FormatKvCache(_runtimeSnapshot); public LocalAiModelPresentationState ModelState => _runtimeSnapshot.ModelEvidence.State switch { @@ -522,6 +524,26 @@ private void ApplyGatewaySnapshot(GatewayConnectionSnapshot snapshot) OnPropertyChanged(null); } + private static string FormatContext(int tokens) => + tokens % 1024 == 0 + ? $"{tokens / 1024}K" + : tokens % 1000 == 0 + ? $"{tokens / 1000}K" + : $"{tokens:N0} tokens"; + + private static string FormatKvCache(LocalAiRuntimeSnapshot snapshot) + { + if (snapshot.KeyCachePrecision is not { } targetPrecision || + snapshot.DraftKeyCachePrecision is not { } draftPrecision) + { + return "Unknown"; + } + + string target = LocalModelCatalog.ToDisplayCacheType(targetPrecision); + string draft = LocalModelCatalog.ToDisplayCacheType(draftPrecision); + return target == draft ? $"{target} target + MTP draft" : $"{target} target + {draft} MTP draft"; + } + public void Dispose() { if (_disposed) diff --git a/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw index 8c427b265..adf820c14 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw @@ -7137,7 +7137,7 @@ Make sure the gateway is running. The selected model unknown an unknown amount - {0:0.#} GB + {0:0.#} GiB Local AI available OpenClaw cannot safely read the global .wslconfig file. Check that the file is valid and readable, then try setup again. Local AI availability could not be verified diff --git a/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw index fb45940d9..34579dd49 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw @@ -7073,7 +7073,7 @@ Le binaire wxc-exec est introuvable. {1} S'il s'agit d'une build développeur, c Le modèle sélectionné inconnu une quantité inconnue - {0:0.#} GB + {0:0.#} GiB IA locale disponible OpenClaw ne peut pas lire le fichier .wslconfig global en toute sécurité. Vérifiez que le fichier est valide et lisible, puis réessayez la configuration. La disponibilité de l'IA locale n'a pas pu être vérifiée diff --git a/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw index fcda37507..05a298c12 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw @@ -7074,7 +7074,7 @@ Het binaire bestand wxc-exec is niet gevonden. {1} Als dit een ontwikkelaarsbuil Het geselecteerde model onbekend een onbekende hoeveelheid - {0:0.#} GB + {0:0.#} GiB Lokale AI beschikbaar OpenClaw kan het globale .wslconfig-bestand niet veilig lezen. Controleer of het bestand geldig en leesbaar is en probeer de installatie opnieuw. Beschikbaarheid van lokale AI kon niet worden geverifieerd diff --git a/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw index 73ef37a89..947bc4d61 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw @@ -7073,7 +7073,7 @@ 所选模型 未知 未知数量 - {0:0.#} GB + {0:0.#} GiB 本地 AI 可用 OpenClaw 无法安全读取全局 .wslconfig 文件。请检查该文件是否有效且可读,然后重试安装。 无法验证本地 AI 可用性 diff --git a/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw index 616645227..4f1f60562 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw @@ -7073,7 +7073,7 @@ 所選模型 未知 未知數量 - {0:0.#} GB + {0:0.#} GiB 本機 AI 可用 OpenClaw 無法安全讀取全域 .wslconfig 檔案。請確認檔案有效且可讀取,然後再試一次設定。 無法驗證本機 AI 可用性 diff --git a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs index 899eedd5f..97fad1e9f 100644 --- a/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs +++ b/tests/OpenClaw.Connection.Tests/LocalAiPortLifecycleTests.cs @@ -5,6 +5,7 @@ using System.Collections.Immutable; using System.Net; using System.Text.Json; +using System.Text.Json.Nodes; namespace OpenClaw.Connection.Tests; @@ -52,11 +53,19 @@ public async Task Manifest_AcceptsAndIgnoresLegacyHardwareProfileId() var paths = new LocalAiPaths(temp.Path); var store = new LocalAiManifestStore(paths); await store.SaveAsync(ValidManifest() with { HardwareProfileId = "retired-profile-id" }); + JsonObject legacyJson = (JsonNode.Parse(await File.ReadAllTextAsync(paths.ManifestPath)) as JsonObject)!; + legacyJson.Remove("keyCachePrecision"); + legacyJson.Remove("valueCachePrecision"); + legacyJson.Remove("draftKeyCachePrecision"); + legacyJson.Remove("draftValueCachePrecision"); + await File.WriteAllTextAsync(paths.ManifestPath, legacyJson.ToJsonString()); LocalAiResolvedInstall saved = (await store.LoadAsync())!; LlamaServerRouterLaunchPlan launch = LlamaServerRouterConfiguration.Build(paths, saved); Assert.Equal("retired-profile-id", saved.Manifest.HardwareProfileId); + Assert.Equal(KvCachePrecision.F16, saved.Manifest.KeyCachePrecision); + Assert.Contains("cache-type-k = f16", launch.PresetContent); Assert.Equal(LocalModelCatalog.Qwen35BModelId, launch.ModelAlias); } @@ -81,6 +90,11 @@ public async Task RouterPreset_BoundsOmittedGenerationAtGatewayMaximum() Assert.Contains( $"n-predict = {gatewayMaximum}", launch.PresetContent.Split(Environment.NewLine)); + Assert.Contains("ctx-size = 262144", launch.PresetContent.Split(Environment.NewLine)); + Assert.Contains("cache-type-k = q8_0", launch.PresetContent.Split(Environment.NewLine)); + Assert.Contains("cache-type-v = q8_0", launch.PresetContent.Split(Environment.NewLine)); + Assert.Contains("cache-type-k-draft = q8_0", launch.PresetContent.Split(Environment.NewLine)); + Assert.Contains("cache-type-v-draft = q8_0", launch.PresetContent.Split(Environment.NewLine)); } [Fact] @@ -93,6 +107,8 @@ public async Task Manifest_OmitsLegacyHardwareProfileIdFromNewWrites() string json = await File.ReadAllTextAsync(paths.ManifestPath); Assert.DoesNotContain("hardwareProfileId", json, StringComparison.OrdinalIgnoreCase); + Assert.Contains("\"keyCachePrecision\": \"q8_0\"", json, StringComparison.Ordinal); + Assert.Contains("\"draftValueCachePrecision\": \"q8_0\"", json, StringComparison.Ordinal); } [Fact] @@ -658,6 +674,10 @@ private static LocalAiInstallManifest ValidManifest() RequestedPort = 0, Endpoint = null, ContextLength = 262_144, + KeyCachePrecision = KvCachePrecision.Q8_0, + ValueCachePrecision = KvCachePrecision.Q8_0, + DraftKeyCachePrecision = KvCachePrecision.Q8_0, + DraftValueCachePrecision = KvCachePrecision.Q8_0, InstalledAtUtc = DateTimeOffset.Parse("2026-08-18T12:00:00Z"), }; } diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiGpuRestartEndpointTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiGpuRestartEndpointTests.cs index 43a9463cd..7aa007506 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiGpuRestartEndpointTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiGpuRestartEndpointTests.cs @@ -132,7 +132,7 @@ private static LocalAiResolvedInstall CreateInstall( ModelAsset = receipt with { FileName = Path.GetFileName(modelPath) }, RequestedPort = 0, Endpoint = endpoint.AbsoluteUri, - ContextLength = model.Recipe.ContextTokens, + ContextLength = LocalModelCatalog.GetProfiles(model)[0].ContextTokens, }; return new LocalAiResolvedInstall(manifest, executable, modelPath, endpoint); } diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs index f2e3549c9..b0afbcf98 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiInstallRecoveryTests.cs @@ -587,6 +587,7 @@ private static LocalInferencePlan CatalogPlan() return new LocalInferencePlan( runtime, LocalModelCatalog.Default, + LocalModelCatalog.GetProfiles(LocalModelCatalog.Default)[0], LocalInferenceModelSelectionOrigin.Default); } @@ -638,7 +639,11 @@ private static LocalAiInstallManifest CreateManifest( Sha256 = plan.Model.Weights.Sha256.Value, }, Endpoint = "http://127.0.0.1:18803/v1", - ContextLength = plan.Model.Recipe.ContextTokens, + ContextLength = plan.Profile.ContextTokens, + KeyCachePrecision = plan.Profile.KeyCachePrecision, + ValueCachePrecision = plan.Profile.ValueCachePrecision, + DraftKeyCachePrecision = plan.Profile.DraftKeyCachePrecision, + DraftValueCachePrecision = plan.Profile.DraftValueCachePrecision, }; } @@ -659,16 +664,12 @@ private static LocalModelInfo CreateModel(byte[] bytes) "Q4", artifact, new LocalModelRunRecipe( - 1024, - KvCachePrecision.F16, - KvCachePrecision.F16, 128, 128, 1, 1, 1, 128, - 8L * 1024 * 1024 * 1024, true, true, SpeculativeDecodingMode.DraftMtp, diff --git a/tests/OpenClaw.SetupEngine.Tests/LocalAiPortHandoffTests.cs b/tests/OpenClaw.SetupEngine.Tests/LocalAiPortHandoffTests.cs index 077c7cc81..2741715d8 100644 --- a/tests/OpenClaw.SetupEngine.Tests/LocalAiPortHandoffTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/LocalAiPortHandoffTests.cs @@ -111,7 +111,7 @@ private static SetupContext CreateContext(LocalAiConfig localAi, string? localDa GpuVendor.Nvidia, "NVIDIA RTX Spark N1X (6144-core Blackwell RTX GPU)", GpuVisibleMemoryBytes: 25_702_694_912, - FreeGpuVisibleMemoryBytes: 25_000_000_000, + FreeGpuVisibleMemoryBytes: 25_702_694_912, DriverVersion: "616.00", CudaMajorVersion: 13, StableId: "GPU-SPARK"), diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupConfigTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupConfigTests.cs index 45ccdc767..cf1b10f32 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupConfigTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupConfigTests.cs @@ -1,5 +1,6 @@ using System.Text.Json; using System.Runtime.Versioning; +using OpenClaw.Shared.Inference.Catalog; namespace OpenClaw.SetupEngine.Tests; @@ -400,6 +401,7 @@ public void SetupReviewSummary_UsesActiveSetupConfig() { Environment.SetEnvironmentVariable("OPENCLAW_TRAY_DATA_DIR", Path.Combine(_tempDir, "roaming")); Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", Path.Combine(_tempDir, "local")); + LocalModelInfo qwen35B = LocalModelCatalog.Find(LocalModelCatalog.Qwen35BModelId)!; var config = new SetupConfig { DistroName = "CustomClaw", @@ -411,7 +413,12 @@ public void SetupReviewSummary_UsesActiveSetupConfig() InstallUrl = "https://example.test/install.sh", Version = GatewayReleasePolicy.SecurityFloor }, - LocalAi = { Enabled = true } + LocalAi = + { + Enabled = true, + SelectedModelId = qwen35B.Id, + SelectedProfileId = LocalModelCatalog.GetProfiles(qwen35B)[1].Id, + } }; var summary = SetupReviewSummaryBuilder.Build(config); @@ -426,7 +433,13 @@ public void SetupReviewSummary_UsesActiveSetupConfig() Assert.Contains("CustomClaw", summary.ExactCommands); Assert.Contains("19999", summary.ExactCommands); Assert.Equal("CustomClaw · LAN:19999", summary.CompletionGatewaySummary); - Assert.StartsWith("llama-server · ", summary.LocalAiDescription, StringComparison.Ordinal); + Assert.StartsWith( + "llama-server for Windows · loads on first request · ", + summary.LocalAiDescription, + StringComparison.Ordinal); + Assert.Contains("256K context", summary.LocalAiDescription, StringComparison.Ordinal); + Assert.Contains("Q8_0 target and MTP draft KV", summary.LocalAiDescription, StringComparison.Ordinal); + Assert.DoesNotContain("full CUDA offload", summary.LocalAiDescription, StringComparison.Ordinal); Assert.DoesNotContain("immutable revision", summary.LocalAiDescription, StringComparison.Ordinal); Assert.DoesNotContain("llama-server b", summary.LocalAiDescription, StringComparison.Ordinal); } diff --git a/tests/OpenClaw.Shared.Tests/LocalInferenceQualificationTests.cs b/tests/OpenClaw.Shared.Tests/LocalInferenceQualificationTests.cs index ad7e0b5d5..705674d41 100644 --- a/tests/OpenClaw.Shared.Tests/LocalInferenceQualificationTests.cs +++ b/tests/OpenClaw.Shared.Tests/LocalInferenceQualificationTests.cs @@ -21,24 +21,57 @@ public void Evaluate_RoutesRuntimeByArchitectureWithoutGpuSkuPairing( Assert.Equal(LocalInferenceEligibilityStatus.Eligible, result.Status); Assert.Equal(expectedRuntimeId, result.Plan?.Runtime.Id); + Assert.Equal(LocalModelCatalog.Qwen38_27BModelId, result.Plan?.Model.Id); + Assert.Equal(LocalModelCatalog.IntermediateContextTokens, result.Plan?.Profile.ContextTokens); + Assert.Equal(KvCachePrecision.Q8_0, result.Plan?.Profile.KeyCachePrecision); } [Fact] - public void Evaluate_UnsetModelChoosesLargestModelThatFitsTotalCapacity() + public void Evaluate_UnsetModelChoosesHighestPriorityModelThatFitsTotalCapacity() { - LocalInferenceEligibilityResult result = LocalInferenceEligibility.Evaluate( - Hardware(RuntimeArchitecture.X64, Gpu("NVIDIA arbitrary adapter", "GPU-24", 24, 24))); - - Assert.Equal(LocalInferenceEligibilityStatus.Eligible, result.Status); - Assert.Equal(LocalModelCatalog.Qwen9BModelId, result.Plan?.Model.Id); - Assert.Equal(LocalInferenceModelSelectionOrigin.Default, result.Plan?.ModelSelectionOrigin); + var cases = new[] + { + (TotalBytes: 34_190_458_880L, FreeBytes: 32_432_455_680L, + ModelId: LocalModelCatalog.Qwen38_27BModelId, + ContextTokens: LocalModelCatalog.IntermediateContextTokens, + Precision: KvCachePrecision.Q8_0, + RequiredBytes: 31_253_556_128L), + (TotalBytes: 24 * GiB, FreeBytes: 24 * GiB, + ModelId: LocalModelCatalog.Qwen38_27BModelId, + ContextTokens: LocalModelCatalog.MinimumContextTokens, + Precision: KvCachePrecision.F16, + RequiredBytes: 25_322_810_272L), + (TotalBytes: 16 * GiB, FreeBytes: 16 * GiB, + ModelId: LocalModelCatalog.Qwen9BModelId, + ContextTokens: LocalModelCatalog.ReducedContextTokens, + Precision: KvCachePrecision.F16, + RequiredBytes: 16_069_374_304L), + }; + foreach (var testCase in cases) + { + GpuInfo gpu = Gpu("NVIDIA arbitrary adapter", "GPU-capacity", 1, 1) with + { + GpuVisibleMemoryBytes = testCase.TotalBytes, + FreeGpuVisibleMemoryBytes = testCase.FreeBytes, + }; + LocalInferenceEligibilityResult result = LocalInferenceEligibility.Evaluate( + Hardware(RuntimeArchitecture.X64, gpu)); + + Assert.Equal(LocalInferenceEligibilityStatus.Eligible, result.Status); + Assert.Equal(testCase.ModelId, result.Plan?.Model.Id); + Assert.Equal(testCase.ContextTokens, result.Plan?.Profile.ContextTokens); + Assert.Equal(testCase.Precision, result.Plan?.Profile.KeyCachePrecision); + Assert.Equal(testCase.RequiredBytes, result.RequiredTotalMemoryBytes); + Assert.True(result.Plan?.Profile.ContextTokens >= LocalModelCatalog.MinimumContextTokens); + Assert.Equal(LocalInferenceModelSelectionOrigin.Default, result.Plan?.ModelSelectionOrigin); + } } [Fact] public void Evaluate_UnsetModelRejectsCapacityBelowSmallestCompleteRecipe() { LocalInferenceEligibilityResult result = LocalInferenceEligibility.Evaluate( - Hardware(RuntimeArchitecture.X64, Gpu("NVIDIA arbitrary adapter", "GPU-16", 16, 16))); + Hardware(RuntimeArchitecture.X64, Gpu("NVIDIA arbitrary adapter", "GPU-10", 10, 10))); Assert.Equal(LocalInferenceEligibilityStatus.Unsupported, result.Status); Assert.Equal(LocalInferenceEligibilityFailureCode.InsufficientGpuMemory, result.FailureCode); @@ -48,30 +81,77 @@ public void Evaluate_UnsetModelRejectsCapacityBelowSmallestCompleteRecipe() [Fact] public void Evaluate_ExplicitModelNeverDowngradesAndReportsExactCapacity() { - LocalInferenceEligibilityResult result = LocalInferenceEligibility.Evaluate( - Hardware(RuntimeArchitecture.X64, Gpu("NVIDIA arbitrary adapter", "GPU-16", 16, 16)), - LocalModelCatalog.Qwen35BModelId); - - Assert.Equal(LocalInferenceEligibilityStatus.Unsupported, result.Status); - Assert.Equal(LocalInferenceEligibilityFailureCode.InsufficientGpuMemory, result.FailureCode); - Assert.Equal(LocalModelCatalog.Qwen35BModelId, result.Plan?.Model.Id); - Assert.Equal(LocalModelCatalog.Default.Weights.SizeBytes + 13 * GiB, result.RequiredTotalMemoryBytes); - Assert.Equal(16 * GiB, result.DetectedTotalMemoryBytes); + var cases = new[] + { + (ModelId: LocalModelCatalog.Qwen38_27BModelId, TotalGiB: 32, + Status: LocalInferenceEligibilityStatus.Eligible, + ContextTokens: LocalModelCatalog.IntermediateContextTokens, + Precision: KvCachePrecision.Q8_0, RequiredBytes: 31_253_556_128L), + (ModelId: LocalModelCatalog.Qwen35BModelId, TotalGiB: 32, + Status: LocalInferenceEligibilityStatus.Eligible, + ContextTokens: LocalModelCatalog.IntermediateContextTokens, + Precision: KvCachePrecision.Q8_0, RequiredBytes: 32_532_584_736L), + (ModelId: LocalModelCatalog.Qwen27BModelId, TotalGiB: 32, + Status: LocalInferenceEligibilityStatus.Eligible, + ContextTokens: LocalModelCatalog.IntermediateContextTokens, + Precision: KvCachePrecision.Q8_0, RequiredBytes: 31_895_889_024L), + (ModelId: LocalModelCatalog.Qwen9BModelId, TotalGiB: 32, + Status: LocalInferenceEligibilityStatus.Eligible, + ContextTokens: LocalModelCatalog.NativeContextTokens, + Precision: KvCachePrecision.F16, RequiredBytes: 24_122_437_984L), + (ModelId: LocalModelCatalog.Qwen35BModelId, TotalGiB: 16, + Status: LocalInferenceEligibilityStatus.Unsupported, + ContextTokens: LocalModelCatalog.MinimumContextTokens, + Precision: KvCachePrecision.Q8_0, RequiredBytes: 27_742_689_568L), + }; + foreach (var testCase in cases) + { + LocalInferenceEligibilityResult result = LocalInferenceEligibility.Evaluate( + Hardware(RuntimeArchitecture.X64, Gpu( + "NVIDIA arbitrary adapter", "GPU-explicit", testCase.TotalGiB, testCase.TotalGiB)), + testCase.ModelId); + + Assert.Equal(testCase.Status, result.Status); + Assert.Equal(testCase.ModelId, result.Plan?.Model.Id); + Assert.Equal(testCase.ContextTokens, result.Plan?.Profile.ContextTokens); + Assert.Equal(testCase.Precision, result.Plan?.Profile.KeyCachePrecision); + Assert.Equal(testCase.RequiredBytes, result.RequiredTotalMemoryBytes); + Assert.Equal(testCase.TotalGiB * GiB, result.DetectedTotalMemoryBytes); + if (testCase.Status == LocalInferenceEligibilityStatus.Unsupported) + Assert.Equal(LocalInferenceEligibilityFailureCode.InsufficientGpuMemory, result.FailureCode); + } } [Theory] - [InlineData(LocalModelCatalog.Qwen35BModelId, 5)] - [InlineData(LocalModelCatalog.Qwen27BModelId, 16)] - [InlineData(LocalModelCatalog.Qwen9BModelId, 8)] + [InlineData(LocalModelCatalog.Qwen35BModelId, 5_120, 512, 2_720, 272, 8)] + [InlineData(LocalModelCatalog.Qwen38_27BModelId, 16_384, 1_024, 8_704, 544, 8)] + [InlineData(LocalModelCatalog.Qwen27BModelId, 16_384, 1_024, 8_704, 544, 8)] + [InlineData(LocalModelCatalog.Qwen9BModelId, 8_192, 1_024, 4_352, 544, 8)] public void GetRequiredMemoryBytes_IncludesRecipeKvCacheAndWorkspace( string modelId, - long expectedCacheGiB) + long expectedF16CacheMiB, + long expectedF16DraftCacheMiB, + long expectedQ8CacheMiB, + long expectedQ8DraftCacheMiB, + long expectedQ8WorkspaceGiB) { LocalModelInfo model = LocalModelCatalog.Find(modelId)!; - - long required = LocalInferenceEligibility.GetRequiredMemoryBytes(model); - - Assert.Equal(model.Weights.SizeBytes + (expectedCacheGiB + 8) * GiB, required); + LocalInferenceRunProfile f16Profile = LocalModelCatalog.GetProfiles(model)[0]; + LocalInferenceRunProfile q8Profile = LocalModelCatalog.GetProfiles(model)[1]; + + long f16Required = LocalInferenceEligibility.GetRequiredMemoryBytes(model, f16Profile); + long q8Required = LocalInferenceEligibility.GetRequiredMemoryBytes(model, q8Profile); + + Assert.Equal( + model.Weights.SizeBytes + + (expectedF16CacheMiB + expectedF16DraftCacheMiB) * 1024 * 1024 + + LocalModelCatalog.RuntimeWorkspaceReserveBytes, + f16Required); + Assert.Equal( + model.Weights.SizeBytes + + (expectedQ8CacheMiB + expectedQ8DraftCacheMiB) * 1024 * 1024 + + expectedQ8WorkspaceGiB * GiB, + q8Required); } [Fact] @@ -102,7 +182,7 @@ public void Evaluate_CountsSharedMemoryForAnyNvidiaGpuAndUnknownSharedFreeIsNotB Hardware(RuntimeArchitecture.Arm64, gpu)); Assert.Equal(LocalInferenceEligibilityStatus.Eligible, result.Status); - Assert.Equal(LocalModelCatalog.Qwen9BModelId, result.Plan?.Model.Id); + Assert.Equal(LocalModelCatalog.Qwen38_27BModelId, result.Plan?.Model.Id); Assert.Equal(24 * GiB, result.DetectedTotalMemoryBytes); Assert.Null(result.AvailableFreeMemoryBytes); } diff --git a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs index 63ef3ce84..321adc1c0 100644 --- a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs @@ -1443,10 +1443,12 @@ public void CapabilitiesPage_FiltersModelsBySelectedGpuCapacityAndShowsMemoryEvi var diagnostics = File.ReadAllText(Path.Combine(root, "src", "OpenClaw.Shared", "Inference", "Catalog", "LocalInferenceEligibilityDiagnostics.cs")); var resources = File.ReadAllText(Path.Combine(root, "src", "OpenClaw.Tray.WinUI", "Strings", "en-us", "Resources.resw")); - Assert.Contains("LocalInferenceEligibility.GetRequiredMemoryBytes(model) <= capacityBytes", source); + Assert.Contains("LocalInferenceEligibility.Evaluate(_localAiHardware!, model.Id)", source); Assert.Contains("eligibility.RequiredTotalMemoryBytes", diagnostics); Assert.Contains("eligibility.DetectedTotalMemoryBytes", diagnostics); Assert.Contains("model weights, KV cache, and runtime workspace", resources); + Assert.Contains("bytes / (1024d * 1024d * 1024d)", diagnostics); + Assert.Contains("GiB", resources); Assert.DoesNotContain("2 GiB runtime margin", source); Assert.DoesNotContain("HardwareProfile", source); Assert.DoesNotContain("RTX PRO 6000", source); diff --git a/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs b/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs index eaab47ba4..ea1c0397c 100644 --- a/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/LocalAiSetupUxContractTests.cs @@ -95,6 +95,7 @@ public void CapabilitiesReview_SeparatesReasonActionFromDisabledOptions() "Pages", "CapabilitiesPage.xaml.cs")); string infoBar = ExtractElement(xaml, "LocalAiUnavailablePanel", ""); + string networkingInfoBar = ExtractElement(xaml, "LocalAiNetworkingConsentPanel", "/>"); Assert.Contains("Title=\"Local AI is not available\"", xaml); Assert.Contains("Severity=\"Informational\"", xaml); @@ -142,7 +143,16 @@ public void CapabilitiesReview_SeparatesReasonActionFromDisabledOptions() Assert.Contains("LocalAiOptionContent.Opacity = isAvailable ? 1 : 0.55", source); Assert.Contains("LocalAiToggle.IsEnabled = isAvailable", source); Assert.Contains("LocalAiModelSelector.IsEnabled = isAvailable", source); - Assert.Contains("LocalAiNetworkingConsentCheckBox.IsEnabled = isAvailable", source); + Assert.Contains("Title=\"WSL networking change required\"", networkingInfoBar); + Assert.Contains("Message=\"Setup will enable mirrored WSL networking", networkingInfoBar); + Assert.DoesNotContain(" diff --git a/tests/OpenClaw.Tray.Tests/Presentation/LocalAiPageViewModelTests.cs b/tests/OpenClaw.Tray.Tests/Presentation/LocalAiPageViewModelTests.cs index f3b3b3b33..6ccdd4a57 100644 --- a/tests/OpenClaw.Tray.Tests/Presentation/LocalAiPageViewModelTests.cs +++ b/tests/OpenClaw.Tray.Tests/Presentation/LocalAiPageViewModelTests.cs @@ -388,6 +388,8 @@ public async Task QualifiedHardware_EnablesApplicableOptionsAndRoutesActions() Assert.True(viewModel.IsLocalAiAvailable); Assert.True(viewModel.IsSetupAvailable); Assert.Null(viewModel.LocalAiUnavailableReason); + Assert.Equal("256K", viewModel.ContextLengthText); + Assert.Equal("Q8_0 target + MTP draft", viewModel.KvCacheText); Assert.True(viewModel.CanStop); Assert.True(viewModel.CanRestart); Assert.True(viewModel.CanOpenLogs); @@ -689,7 +691,12 @@ private static LocalAiRuntimeSnapshot CreateInstalledSnapshot( ProcessId: 1234, ProcessStartedAtUtc: now, Detail: null, - UpdatedAtUtc: now); + UpdatedAtUtc: now, + ContextLength: LocalModelCatalog.NativeContextTokens, + KeyCachePrecision: KvCachePrecision.Q8_0, + ValueCachePrecision: KvCachePrecision.Q8_0, + DraftKeyCachePrecision: KvCachePrecision.Q8_0, + DraftValueCachePrecision: KvCachePrecision.Q8_0); } private sealed class FixedHardwareProbe(HostHardwareInfo hardware) : IHostHardwareProbe From c784fddd5538b5abfaf8f77dccf6281a1a23eb71 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 1 Sep 2026 23:21:19 -0700 Subject: [PATCH 2/3] docs: add GUI end-to-end testing plan for Azure Windows via Crabbox Proposes tests/OpenClaw.GuiE2ETests built on FlaUI with a fake gateway tier and a real WSL gateway tier, a scenario catalog with parity tests, Crabbox orchestration into the interactive desktop session, CI wiring, proof-pool integration and a phased rollout. Co-Authored-By: Claude Fable 5.1 --- tests/OpenClaw.GuiE2ETests/PLAN.md | 139 +++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 tests/OpenClaw.GuiE2ETests/PLAN.md diff --git a/tests/OpenClaw.GuiE2ETests/PLAN.md b/tests/OpenClaw.GuiE2ETests/PLAN.md new file mode 100644 index 000000000..12cd1d657 --- /dev/null +++ b/tests/OpenClaw.GuiE2ETests/PLAN.md @@ -0,0 +1,139 @@ +# Plan: Automated GUI end-to-end testing of OpenClaw Windows Companion on Azure Windows + +## Context + +The repo already has strong non-GUI E2E coverage (`tests\OpenClaw.E2ETests`, real WSL gateway, driven over local MCP) and a real-process accessibility lane (`tests\OpenClaw.Tray.UITests\AccessibilityAppFixture.cs`) that launches `OpenClaw.Tray.WinUI.exe`, navigates via `openclaw://hub/` deep links and inspects the UI through UI Automation. What is missing is user-level GUI testing: nobody clicks buttons, types into fields, or asserts what a user sees. The proof pool `windows-winui-interactive` in `.github\proof-pools.json` still relies on a **manual** "exercise-changed-ui" step, and there is no automated Windows host provisioning; `.agents\skills\crabbox\SKILL.md` documents leasing Azure Windows hosts via the external Crabbox CLI but nothing in CI uses it. + +Goal: a new GUI E2E test project driven by **FlaUI (UIA3)**, scenarios authored as **C# xUnit + page objects** with a **scenario catalog** so tests can be added incrementally, a **two-tier gateway** strategy (fake in-process WebSocket gateway for every run; real WSL gateway for nightly/on-demand), and an orchestration layer that runs the suite on **Azure Windows VMs leased through Crabbox** with an interactive desktop, wired into GitHub Actions. + +Decisions already made by the user: Crabbox leases, FlaUI, two-tier gateway, C# xUnit + page objects + catalog. + +## Verified ground truth (drives the design) + +- Onboarding shows when `RequiresSetup(_settings)` or `OPENCLAW_FORCE_ONBOARDING=1` (`src\OpenClaw.Tray.WinUI\App.xaml.cs:839`). Seeding `EnableMcpServer=true` skips the wizard (what `AccessibilityAppFixture` does); an empty data dir triggers it. +- Isolated instances do not register the URI scheme; navigation is done by spawning a second `OpenClaw.Tray.WinUI.exe ` with the same `OPENCLAW_TRAY_DATA_DIR` (`AccessibilityAppFixture.NavigateAsync`). Page readiness comes from `AccessibilityNavigationSignal.WritePageReady` (env `OPENCLAW_ACCESSIBILITY_NAVIGATION_SIGNAL`) plus `PageMarker` AutomationIds on 18 hub pages (inventory in `AccessibilityScanTests.PageTestData`). +- `OPENCLAW_UI_AUTOMATION=1` keeps the tray flyout from auto-dismissing (`TrayMenuWindow.xaml.cs:219`). +- Dialogs already have ids: `ExecApprovalDenyAction/AllowAlwaysAction/AllowOnceAction` (`Dialogs\ExecApprovalDialog.cs:182-203`), `PairingRejectAction/LaterAction/ApproveAction` (`Dialogs\PairingApprovalDialog.cs:153-169`). ConnectionPage has 44 ids (`AddGatewayHeaderAction`, `LobbyDirectTile`, `LobbySetupCodeTile`, `AddDirectUrl/Token/Name`, `AddSetupCodeInput/Decode`, `AddSave`, `ConnectionToggle`, `NodeModeToggle`, `StatusStrip`, `NodeReconnect`, `RecoveryApplyRepair`, ...). +- Gateway protocol the fake must speak (`src\OpenClaw.Shared\OpenClawGatewayClient.cs`, `ConnectEnvelopeBuilder.cs`, `WindowsNodeClient.cs`): optional `connect.challenge` event; `{"type":"req","method":"connect",...}` answered by `{"type":"res","ok":true,"payload":{"type":"hello-ok","protocol":4,...}}`; pairing-required is `ok:false` with `details.code = "PAIRING_REQUIRED"` then `device.pair.resolved`/`node.pair.resolved` events; node commands arrive as `node.invoke.request` events answered with `node.invoke.result`; exec approvals are `exec.approval.requested`/`exec.approval.resolve`. After hello-ok the client fires `health`, `sessions.list`, `node.list`, `usage.status`, etc. Reusable basis: `LoopbackGatewayServer` in `tests\OpenClaw.Shared.Tests\GatewayProtocolLiveRoundTripTests.cs` (`OnMethod`, `WaitFrameAsync`). +- `scripts\run-proof-tests.ps1` rejects zero-test and zero-pass runs. `scripts\validate-proof-pools.ps1:736` only allows the `$env:OPENCLAW_RUN_E2E = '1'; ` prefix for proof-test commands; it must be extended for `OPENCLAW_RUN_GUI_E2E`. +- No `Directory.Packages.props`; package versions go in the csproj. `tests\Directory.Build.props` sets net10.0 + TreatWarningsAsErrors. +- Crabbox: `warmup --desktop` is the only way to get an interactive desktop, and it must be requested at lease creation. SSH-run scripts execute in a non-interactive session, so UI tests must be launched into the desktop session. + +## Deliverable 1: new test project `tests\OpenClaw.GuiE2ETests` + +**csproj** mirrors `tests\OpenClaw.Tray.UITests\OpenClaw.Tray.UITests.csproj`: TFM `net10.0-windows10.0.22621.0`, `Platforms x64;ARM64`, `RuntimeIdentifiers win-x64;win-arm64`, no `UseWinUI`, `FrameworkReference Microsoft.WindowsDesktop.App`, `WarningsNotAsErrors NU1904`. Packages: `FlaUI.Core`, `FlaUI.UIA3`, `Microsoft.WindowsAppSDK $(MicrosoftWindowsAppSDKVersion)`, `Microsoft.Windows.SDK.BuildTools`. ProjectReferences: `src\OpenClaw.Tray.WinUI` (puts the exe beside the test assembly), `src\OpenClaw.Shared`, `src\OpenClaw.Connection`, `tests\OpenClaw.TestSupport`, `tests\OpenClaw.E2ETests` (add `InternalsVisibleTo("OpenClaw.GuiE2ETests")` in `tests\OpenClaw.E2ETests\AssemblyInfo.cs` to reuse `E2ESetupFixture`, `FakeOllamaServer`, `McpClient`, `E2ETestGate`). `xunit.runner.json` with `parallelizeTestCollections=false`. Add to `openclaw-windows-node.slnx`. + +**Folders** + +- `Harness\` + - `GuiE2EGate.cs`: `[GuiE2EFact]` (needs `OPENCLAW_RUN_GUI_E2E=1`), `[GatewayGuiE2EFact]` (also `OPENCLAW_RUN_E2E=1`), same "1|true" pattern as `E2EFactAttribute.cs`. + - `Traits.cs`: `Tier` (`Fake`|`Gateway`), `Workflow` (`Smoke`,`Onboarding`,`Connection`,`Pairing`,`NodeMode`,`Sandbox`,`Settings`,`LocalAi`,`Chat`,`Tray`,`ExecApproval`,`Recovery`,`Updates`), `Pool`, `Quarantine`. + - `GuiApp.cs`: the fixture. Port of `AccessibilityAppFixture` + `IsolatedTrayInstance`: isolated data/localappdata dirs, seeded `settings.json` and optional `gateways.json`, env (`OPENCLAW_TRAY_DATA_DIR`, `..._APPDATA_DIR`, `..._LOCALAPPDATA_DIR`, `OPENCLAW_MCP_PORT`, `OPENCLAW_SUPPRESS_EXTERNAL_BROWSER=1`, `OPENCLAW_SKIP_UPDATE_CHECK=1`, `OPENCLAW_LANGUAGE=en-US`, `OPENCLAW_UI_AUTOMATION=1`, `OPENCLAW_ACCESSIBILITY_NAVIGATION_SIGNAL`, optional `OPENCLAW_FORCE_ONBOARDING=1`), stdout/stderr capture, `FlaUI.Core.Application.Attach(pid)` with `UIA3Automation`, window finders (`HubWindow()`, `SetupWindow()`, `TrayMenuWindow()`, `ChatWindow()`, `FindDialog(id)`) by process id + title/marker, `NavigateAsync(tag, pageName, markerId)`, `SendDeepLinkAsync(uri)`, `RestartAsync()` for persistence scenarios, `Mcp` (`McpClient`) for `app.status` polling as a second oracle, `EnsureAlive()` reading `crash.log`. + - `Wait.cs`: `Until(cond, timeout, interval, description)` with last-observed-state in failure message. No sleeps in tests. + - `ArtifactSink.cs`: per-test folder `TestResults\GuiE2E\\.\`; `CaptureWindow(window, name)` via FlaUI `Capture` with the blank-image guard from `AccessibilityAppFixture`; UIA tree dump + redacted logs on failure; copies only `*.log/*.jsonl`, never `settings.json`, `gateways.json`, `device-key*`, `gateways\` (rules from `E2ESetupFixture.ShouldCopyArtifactFile` / `SanitizeForLog`). + - `GuiScenarioBase.cs`: `IAsyncLifetime` base owning `GuiApp` + optional `FakeGatewayServer`; `RunAsync(body)` captures evidence on failure and rethrows. + - `SettingsFile.cs`, `GatewaysFile.cs`: typed on-disk readers with `WaitForValueAsync` (saves are async); port `ReadActiveGatewayRecord`. +- `FakeGateway\` + - `FakeGatewayServer.cs`: HttpListener WebSocket server on loopback, multi-connection (operator + node), records frames (`ReceivedFrames`, `WaitForMethodAsync`), `OnMethod`, `Broadcast(event, payload)`, `DropAllConnections()`, `Pause()/Resume()`, `RequirePairing` + `ApprovePending()`, `AcceptedTokens`, `SetupCode` (base64 `{url,bootstrapToken}` per `SetupCodeDecoder`), strict mode that fails on unknown methods. + - `FakeGatewayProtocol.cs`: builders for `connect.challenge`, `hello-ok` (protocol 4, deviceToken, scopes, `snapshot.sessionDefaults.mainSessionKey`), `PAIRING_REQUIRED`, `health`, `chat.send` ack + canned `chat` event, `node.invoke.request` for `system.run`, `exec.approval.requested`. + - `FakeGatewayDefaults.cs`: empty-but-valid responders for the post-hello request storm (`health`, `sessions.list`, `node.list`, `usage.status`, `usage.cost`, `commands.list`, `agents.list`, `channels.status`, `config.get`, `subscribe`). + - `FakeGatewayServerTests.cs`: always-on `[Fact]`s exercising the fake with the real `OpenClawGatewayClient` (mirror of `GatewayProtocolLiveRoundTripTests`). +- `Pages\`: page objects (`HubShell`, `ConnectionPage`, `SettingsPage`, `PermissionsPage`, `SandboxPage`, `LocalAiPage`, `ChatPage`, `DebugPage`, `TrayMenu`, `Setup\SecurityNoticePage|WelcomePage|AdvancedSetupPage|CapabilitiesPage|ProgressPage|CompletePage`, `Dialogs\PairingApprovalDialog|ExecApprovalDialog|CommandPalette`). Each finds elements by `cf.ByAutomationId`, exposes intent methods (`AddDirectGateway(url, token)`), returns only after readiness signal or UIA condition. `AutomationIds.cs` holds all id constants. +- `Scenarios\`: one class per workflow (`OnboardingScenarios.cs`, `GatewayConnectionScenarios.cs`, `PairingScenarios.cs`, `NodeModeScenarios.cs`, `SandboxScenarios.cs`, `SettingsScenarios.cs`, `LocalAiScenarios.cs`, `ChatScenarios.cs`, `TrayScenarios.cs`, `ExecApprovalScenarios.cs`, `RecoveryScenarios.cs`, `UpdateScenarios.cs`, `Gateway\RealGatewayOnboardingScenarios.cs` in a `[Collection("RealGateway")]` sharing `E2ESetupFixture`). +- `Catalog\`: `gui-e2e-catalog.json`, `gui-e2e-catalog.schema.json`, `CatalogTests.cs` (always-on): every `[GuiE2EFact]`/`[GatewayGuiE2EFact]` has exactly one entry with matching `testId`, traits equal `tier/workflow/pool`, every listed AutomationId exists as a literal under `src\**` (grep from `OPENCLAW_REPO_ROOT`), quarantined entries carry `quarantineIssue`. + +## Deliverable 2: scenario catalog format + +```json +{ "$schema": "./gui-e2e-catalog.schema.json", "schemaVersion": 1, + "scenarios": [{ + "id": "GUI-CONN-001", + "testId": "OpenClaw.GuiE2ETests.Scenarios.GatewayConnectionScenarios.AddDirectGateway_ConnectsAndPersists", + "title": "Add gateway by direct URL and token, connect", + "workflow": "Connection", "tier": "Fake", "pool": "windows-winui-interactive", "shard": "connection", + "automationIds": ["AddGatewayHeaderAction","LobbyDirectTile","AddDirectUrl","AddDirectToken","AddSave","StatusStrip"], + "preconditions": ["isolated data dir, EnableMcpServer=true, no gateways.json", "FakeGatewayServer with AcceptedTokens=[test-token]"], + "steps": ["navigate hub/connection", "click AddGatewayHeaderAction", "..."], + "expected": ["StatusStrip = Connected", "gateways.json active url = fake url", "fake received connect with auth.token"], + "evidence": ["screenshot:connection-connected", "trx", "fake-frames.jsonl"], + "owner": "windows-node-maintainers", "addedIn": "0.1.0", + "quarantine": false, "quarantineIssue": null }] } +``` + +Consumers: `CatalogTests.cs` (parity), `scripts\gui-e2e\Get-GuiE2EShards.ps1` (emits a JSON matrix of `shard` → xUnit `--filter` for CI), `scripts\gui-e2e\Export-GuiE2ECatalog.ps1` (regenerates the scenario table in `docs\GUI_E2E_TESTING.md` between `` markers; `scripts\validate-docs.ps1` fails when stale), PR template hint under `## Required proof pools`. + +## Deliverable 3: initial scenario set + +| ID | Scenario | Tier | Key ids | Assertions (UIA + disk + fake frames + screenshot) | +|---|---|---|---|---| +| GUI-ONB-001 | First run, connect existing gateway via setup code | Fake, `OPENCLAW_FORCE_ONBOARDING=1` | new `SecurityNoticeContinue`, `WelcomeConnectExistingGatewayChoice`, `WelcomeNextButton`, new AdvancedSetup ids, `CompleteLaunchButton` | Setup window closes, Hub opens, gateways.json has fake URL, fake saw `connect` with `bootstrapToken`, `app.status` Connected | +| GUI-ONB-002 | First run, install local gateway | Gateway | `WelcomeInstallLocalGatewayChoice`, new `ProgressPageMarker`, `CompletePageMarker` | Progress steps succeed, real gateway Connected via `WaitForConnectionReady` | +| GUI-CONN-001 | Add gateway by direct URL+token | Fake | `AddGatewayHeaderAction`, `LobbyDirectTile`, `AddDirectUrl/Token/Name`, `AddSave`, `StatusStrip` | connect frame has `auth.token`; active record on disk; Connected | +| GUI-CONN-002 | Add gateway by setup code | Fake | `LobbySetupCodeTile`, `AddSetupCodeInput`, `AddSetupCodeDecode`, `AddSave` | decode shows URL; connect uses bootstrapToken; device key file exists (not copied) | +| GUI-CONN-003 | Connect/disconnect toggle | Fake | `ConnectionToggle`, `StatusStrip` | socket closes / new connect frame; `app.status` flips | +| GUI-PAIR-001 | Pairing approval dialog approve / reject | Fake `RequirePairing` | `PairingApproveAction`, `PairingRejectAction` | dialog appears; approve → `device.pair.resolved` → reconnect with deviceToken; reject → not paired | +| GUI-NODE-001 | Node mode + capability toggles persist | Fake | `NodeModeToggle`, PermissionsPage toggles | settings.json flags; node connect `caps`; survives `RestartAsync` | +| GUI-SBX-001 | Sandbox policy change | Fake | new `SandboxEnabledToggle`, `SandboxScopeExpander` | settings key changes; status title; persists | +| GUI-SET-001 | Settings toggles persist across restart | Fake | SettingsPage ids | `AutoStart`, theme, `GlobalHotkeyEnabled` on disk = UIA state after restart | +| GUI-LAI-001 | Local AI page with `FakeOllamaServer` | Fake (may become Gateway) | `LocalAiStart/Stop/Restart/ChangeModel` | status transitions; fake Ollama request count > 0 | +| GUI-CHAT-001 | Chat send + canned reply | Fake | `ChatComposerInput`, new `ChatComposerSend`, new transcript ids | fake sees `chat.send`; reply text visible | +| GUI-CHAT-002 | Quick send via `openclaw://agent?message=` | Fake | same | fake receives `chat.send` with message | +| GUI-TRAY-001 | Tray menu open + navigate | Fake | `TrayMenuPanel`, new `TrayMenu*` ids | `openclaw://tray` opens menu; click Settings → `SettingsPageMarker` | +| GUI-EXEC-001 | Exec approval allow once / deny | Fake | `ExecApprovalAllowOnceAction`, `ExecApprovalDenyAction` | fake sends `node.invoke.request system.run`; result frame ok with stdout / denied | +| GUI-REC-001 | Network recovery | Fake | `StatusStrip`, `NodeReconnect` | `DropAllConnections`+`Pause` → reconnecting; `Resume` → new connect frames, Connected within 60 s | +| GUI-UPD-001 | Check-for-updates semantics | Fake | `SettingsPageCheckUpdates` | no update dialog at startup in isolated mode; clicking shows graceful InfoBar offline | + +Smoke subset (`Workflow=Smoke`, per PR, ≤5 min): launch+navigate, CONN-001, SET-001, CHAT-001. + +## Deliverable 4: product-side changes in `src` + +- Add AutomationIds: `SecurityNoticePage.xaml` (`SecurityNoticePageMarker`, `SecurityNoticeContinue`), `AdvancedSetupPage.xaml` (marker, inputs, `AdvancedSetupNext/Back`), `ProgressPage.xaml` (marker, steps panel), `CompletePage.xaml` (marker, `CompleteLaunchButton`, `CompleteStartupToggle`), `SetupWindow.xaml` (`SetupWindowRoot`), `SandboxPage.xaml` (marker if missing, `SandboxEnabledToggle`, `SandboxScopeExpander`), `TrayMenuWindow.xaml` (`TrayMenu` per button), chat composer/transcript (`ChatComposerSend`, `ChatTranscriptList`, message items with `Name` = text), `CommandPaletteDialog.xaml`, dialog roots (`ExecApprovalDialogRoot`, `PairingApprovalDialogRoot`), `ChatWebViewHost` on WebView2 host borders (asserted present, never traversed). +- Extend `AccessibilityNavigationSignal.WritePageReady` calls to setup wizard page transitions and dialog show (`Setup:WelcomePage`, `Dialog:ExecApproval`, `Dialog:Pairing`, `Tray:Menu`). Same env var, no new hook. +- Verify isolated-mode guards: AutoStart registry write and hotkey registration must no-op when `DataDirOverride != null`; add guard if missing. +- `scripts\validate-proof-pools.ps1:736`: allow prefix `(?:\$env:OPENCLAW_RUN_(?:E2E|GUI_E2E) = '1'; )*`; add a case to `scripts\test-proof-pool-validator.ps1`. +- `tests\OpenClaw.E2ETests\AssemblyInfo.cs`: `InternalsVisibleTo`. + +## Deliverable 5: Crabbox orchestration (`scripts\gui-e2e\`) + +Interactive-session problem: `crabbox run` executes over SSH in a non-interactive session; WinUI windows are not visible to UIA there and screenshots are black. Chosen mechanism: **scheduled task in the desktop session, created from SSH, polled via a completion marker** (`schtasks /Create /SC ONCE /RU /IT /RL HIGHEST` then `/Run`; poll `TestResults\GuiE2E\done.json`). Fallback if `/IT` is refused on the image: `crabbox desktop launch ... -- powershell -File scripts\gui-e2e\Invoke-GuiE2E-Remote.ps1` and poll the same marker. + +- `Test-GuiE2EHost.ps1`: hard prerequisite checks (WebView2 runtime registry key, .NET 10 SDK, an Active interactive session via `query user` and `explorer.exe`, current `SessionId` equals it unless `-SkipSessionCheck`, resolution ≥1600x900, DPI 96, `LogonUI` not running, `OPENCLAW_REPO_ROOT`). Writes `host-report.json`. Never skips. +- `Invoke-GuiE2E.ps1` (local + remote runner): `-Tier Fake|Gateway|All -Filter -RuntimeIdentifier -NoBuild`; builds tray + test project; sets gates; calls `scripts\run-proof-tests.ps1 -Project 'tests\OpenClaw.GuiE2ETests\OpenClaw.GuiE2ETests.csproj' -Filter 'Trait=Tier=Fake' -ResultName 'gui-e2e-fake' -RuntimeIdentifier win-x64`. +- `Invoke-GuiE2E-Remote.ps1`: runs inside the desktop session, tees to `remote.log`, writes `done.json {exitCode,started,finished}` in `finally`. +- `Invoke-GuiE2E-Crabbox.ps1` (controller): resolve `$Crabbox` per SKILL.md; `crabbox warmup --provider azure --target windows --windows-mode normal --desktop --keep --idle-timeout 90m --ttl 240m --timing-json` (or `-LeaseId`); `crabbox run --id --preflight --timing-json --script-stdin` with remote script: host check (`-SkipSessionCheck`), build once, register + run scheduled task, poll `done.json` up to `-TimeoutMinutes`, print `remote.log`, delete task; second `crabbox run` tars `TestResults\GuiE2E` + `TestResults\ProofPools\gui-e2e-*` back (or `crabbox results` if supported); `finally { crabbox stop }` unless `-KeepLease`. Never print `crabbox config path` contents or `az login` output. +- `Get-GuiE2EShards.ps1`, `Export-GuiE2ECatalog.ps1` (see Deliverable 2). + +## Deliverable 6: CI + +- New `.github\workflows\gui-e2e.yml`: triggers `workflow_dispatch` (`tier`, `filter`, `keep_lease`), nightly `schedule` `0 3 * * *`, `pull_request` labeled `gui-e2e`. Job on `windows-latest`, `permissions: id-token: write`, `concurrency: gui-e2e-azure`, `timeout-minutes: 150`. Steps: checkout, setup-dotnet 10, `azure/login@v3` (OIDC), obtain crabbox, `crabbox azure login --location ${{ vars.CRABBOX_AZURE_LOCATION }}`, `crabbox doctor`, `Invoke-GuiE2E-Crabbox.ps1`, TRX step summary (per-outcome counts, failed scenario ids), `upload-artifact gui-e2e-results` (`if: always()`), final `if: always()` `crabbox stop` by lease id from step output. +- `ci.yml`: add `gui-e2e-smoke` job on `windows-latest` (same restore/build as `e2etests`), `OPENCLAW_RUN_GUI_E2E=1`, filter `Trait=Workflow=Smoke|FullyQualifiedName~GuiE2ETests.Catalog|FullyQualifiedName~FakeGatewayServerTests`, 25 min, TRX zero-test guard, upload `TestResults\GuiE2E`. Runs per PR (the Accessibility lane already proves UIA on the real exe works on `windows-latest`). Full fake tier stays nightly/label-triggered on Crabbox until two weeks of green nightlies. +- `.github\proof-pools.json`: add `run-gui-e2e` proof-test command to `windows-winui-interactive` (`$env:OPENCLAW_RUN_GUI_E2E = '1'; .\scripts\run-proof-tests.ps1 -Project 'tests\OpenClaw.GuiE2ETests\OpenClaw.GuiE2ETests.csproj' -Filter 'Trait=Tier=Fake' -ResultName 'gui-e2e-fake' -RuntimeIdentifier win-x64`), keep `exercise-changed-ui` manual for uncovered paths; add `run-gui-e2e-gateway` (`Trait=Tier=Gateway`, both env prefixes) to `windows-wsl-gateway-e2e`. Update `docs\PROOF_POOLS.md`. + +## Deliverable 7: docs + +New `docs\GUI_E2E_TESTING.md` (architecture, how to run locally, Crabbox lane, generated scenario table, extension guide, flakiness policy, evidence rules). Update `docs\TEST_COVERAGE.md`, `docs\PROOF_POOLS.md`, `DEVELOPMENT.md`, `AGENTS.md` (targeted validation rule: UI changes run `Invoke-GuiE2E.ps1 -Tier Fake` or declare `windows-winui-interactive`), `.github\pull_request_template.md` hint. + +Extension guide (goes in the doc): 1) add/verify AutomationIds in `src` (PascalCase `[Action|Toggle|Input|Marker]`, stable across locales, never user content) and `Pages\AutomationIds.cs`; 2) add/extend page object with intent methods; 3) write scenario with gate attribute + `Tier/Workflow/Pool` traits inside `RunAsync`, one named screenshot minimum; 4) add catalog entry, run `CatalogTests` and `Export-GuiE2ECatalog.ps1`; 5) optionally tag `Workflow=Smoke` (cap 5 min). Flakiness: no in-test retries; explicit timeouts; ≥2 failures in last 10 nightlies without a product bug → `Quarantine=true` + issue, excluded from proof filters (`&Trait!=Quarantine=true` equivalent), fix or delete within 2 weeks. Evidence follows `windows-winui-interactive.evidencePolicy`: app-window capture only, never raw settings/gateways/device files, `SanitizeForLog` on logs and fake frames. + +## Phased rollout + +- **Phase 0, spike (2-3 days):** project skeleton, `GuiApp` attaches via FlaUI, one scenario (navigate to `hub/connection`, marker visible, non-blank screenshot). Run locally and in a scratch workflow on `windows-latest`. Confirm UIA tree exposes AutomationIds under the WinUI `DesktopChildSiteBridge`. +- **Phase 1 (~2 weeks):** `FakeGatewayServer` + its tests; AutomationIds in `src`; page objects for Hub/Connection/Settings/Chat/dialogs; CONN-001/002/003, SET-001, CHAT-001; catalog + `CatalogTests`; `gui-e2e-smoke` job. Verify 3 consecutive green local runs of `Invoke-GuiE2E.ps1 -Tier Fake`, smoke green in CI, and that `run-proof-tests.ps1` fails when the gate is unset (negative check). +- **Phase 2 (~2 weeks):** Crabbox scripts + scheduled-task mechanism, `gui-e2e.yml`, tier 2 ONB-002 on `E2ESetupFixture`. Verify nightly produces TRX + screenshots from the VM, `host-report.json` shows session/DPI checks passed, `crabbox list` shows no leaked leases. +- **Phase 3 (~2 weeks):** remaining scenarios, docs, proof-pool integration. Verify `scripts\validate-proof-pools.ps1`, `scripts\test-proof-pool-validator.ps1`, `scripts\validate-docs.ps1` pass. + +## Verification (end to end) + +1. `dotnet test tests\OpenClaw.GuiE2ETests` with no gate → only `CatalogTests` + `FakeGatewayServerTests` run; GUI scenarios skipped with reason. +2. `.\scripts\gui-e2e\Invoke-GuiE2E.ps1 -Tier Fake` locally → all fake-tier scenarios pass, `TestResults\ProofPools\gui-e2e-fake\gui-e2e-fake.trx` non-empty, per-scenario screenshots present and non-blank. +3. `.\scripts\gui-e2e\Invoke-GuiE2E-Crabbox.ps1 -Tier Fake` from a workstation with `az login` → lease warmed with desktop, host report passes, TRX + screenshots downloaded, lease stopped. +4. `gui-e2e.yml` `workflow_dispatch` run → artifact `gui-e2e-results` contains TRX, PNGs, redacted logs; step summary lists scenario ids. +5. PR with an intentionally removed AutomationId → `gui-e2e-smoke` and `CatalogTests` fail. + +## Open items (need owner input, not invented) + +- Crabbox binary distribution for CI (`vars.CRABBOX_DOWNLOAD_URL` + token, or private release asset) and which Azure federated identity may create VMs (likely new `AZURE_GUI_E2E_*` secrets, distinct from the release signing identity); `vars.CRABBOX_AZURE_LOCATION`. +- Whether the `--desktop` image allows `schtasks /IT` for the autologon user, its lock-screen policy and default DPI/resolution, and whether WSL2 is enabled inside the normal-mode desktop image (needed for tier 2 with a desktop; otherwise tier 2 GUI onboarding is limited to connect-existing against a gateway on a second WSL2 lease). +- Local AI page may need a WSL-backed runtime; LAI-001 may move to tier 2 or need an endpoint override. +- Update-check scenario depends on an update-feed override existing in `UpdateCoordinator`; otherwise only the offline negative is testable. +- Fake gateway protocol drift: keep responders minimal, run strict mode nightly, cross-reference `docs\gateway-protocol-drift-guard.md`. From fd0c59ecf2f252718c98278442f88c9dbd1f696b Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 1 Sep 2026 23:42:04 -0700 Subject: [PATCH 3/3] docs: add GUI E2E implementation notes, Phase 0 kickoff prompt, and skill stub Verified reuse points, app env hooks, fake gateway wire contract, repo build conventions and AutomationId inventory for implementers; a scoped Phase 0 spike prompt; and a gui-e2e skill stub to be completed in Phase 3. Co-Authored-By: Claude Fable 5.1 --- .agents/skills/gui-e2e/SKILL.md | 57 ++++ .../IMPLEMENTATION_NOTES.md | 243 ++++++++++++++++++ tests/OpenClaw.GuiE2ETests/KICKOFF_PROMPT.md | 84 ++++++ 3 files changed, 384 insertions(+) create mode 100644 .agents/skills/gui-e2e/SKILL.md create mode 100644 tests/OpenClaw.GuiE2ETests/IMPLEMENTATION_NOTES.md create mode 100644 tests/OpenClaw.GuiE2ETests/KICKOFF_PROMPT.md diff --git a/.agents/skills/gui-e2e/SKILL.md b/.agents/skills/gui-e2e/SKILL.md new file mode 100644 index 000000000..4e329725a --- /dev/null +++ b/.agents/skills/gui-e2e/SKILL.md @@ -0,0 +1,57 @@ +--- +name: gui-e2e +description: "Run and extend the automated GUI end-to-end lane (FlaUI over the real OpenClaw tray, fake or real gateway, Azure Windows via Crabbox). Use when a change touches tray, Settings, onboarding, chat, permissions, sandbox, connection UI, or when the user asks for GUI E2E coverage." +--- + +# GUI end-to-end lane + +Status: **planned, not yet implemented.** The design lives in +`tests\OpenClaw.GuiE2ETests\PLAN.md`; verified facts for implementers are in +`tests\OpenClaw.GuiE2ETests\IMPLEMENTATION_NOTES.md`; the Phase 0 starting +prompt is `tests\OpenClaw.GuiE2ETests\KICKOFF_PROMPT.md`. Fill in the sections +below as each phase lands and remove this status line in Phase 3. + +## When to use + +- A change touches any hub page, the setup wizard, tray menu, chat surface, + dialogs, or connection/pairing UI. +- A PR declares the `windows-winui-interactive` or `windows-wsl-gateway-e2e` + proof pool and the automated `run-gui-e2e` command exists. +- The user asks to add or run a GUI scenario. + +## Run locally (Phase 1+) + +```powershell +$env:OPENCLAW_REPO_ROOT = (Get-Location).Path +.\scripts\gui-e2e\Invoke-GuiE2E.ps1 -Tier Fake +``` + +Results: `TestResults\ProofPools\gui-e2e-fake\gui-e2e-fake.trx`, screenshots and +redacted logs under `TestResults\GuiE2E\\`. The script refuses zero-test runs. + +## Run on Azure via Crabbox (Phase 2+) + +Requires the Crabbox CLI and Azure auth as described in +`.agents\skills\crabbox\SKILL.md`. UI tests need a desktop lease +(`warmup --desktop`); the script launches the suite inside the interactive +session through a scheduled task and polls for completion. + +```powershell +.\scripts\gui-e2e\Invoke-GuiE2E-Crabbox.ps1 -Tier Fake +``` + +Always report the provider and lease id, and confirm the lease was stopped. + +## Add a scenario (Phase 1+) + +1. Add or verify AutomationIds in `src` (PascalCase `[Action|Toggle|Input|Marker]`). +2. Add or extend a page object in `tests\OpenClaw.GuiE2ETests\Pages\`. +3. Write the test in `Scenarios\Scenarios.cs` with `[GuiE2EFact]` and `Tier`, `Workflow`, `Pool` traits. +4. Add the entry to `Catalog\gui-e2e-catalog.json`; run `dotnet test --filter FullyQualifiedName~Catalog`. +5. Regenerate the docs table with `.\scripts\gui-e2e\Export-GuiE2ECatalog.ps1`. + +## Rules + +- Never run against real `%APPDATA%\OpenClawTray`; the harness always isolates state. +- Capture app windows only; never copy `settings.json`, `gateways.json`, or device key files into artifacts. +- A skipped or quarantined scenario is not proof. Report blockers explicitly. diff --git a/tests/OpenClaw.GuiE2ETests/IMPLEMENTATION_NOTES.md b/tests/OpenClaw.GuiE2ETests/IMPLEMENTATION_NOTES.md new file mode 100644 index 000000000..e2c7e7d4e --- /dev/null +++ b/tests/OpenClaw.GuiE2ETests/IMPLEMENTATION_NOTES.md @@ -0,0 +1,243 @@ +# Implementation notes for the GUI E2E lane + +Companion to [PLAN.md](PLAN.md). Everything here was verified against the +repository at the time of writing (branch `dev/plarroy/gui-e2e-plan`). Re-verify +line numbers with `grep` before relying on them; symbol names are the stable part. + +## 1. Code to port, with exact sources + +| Need | Copy from | What to take | +|---|---|---| +| Launch isolated tray, seed settings, deep-link navigation, nav-signal file parsing, page-marker wait, foreground + `CopyFromScreen` screenshot with blank-image guard, crash-log detection, P/Invokes | `tests\OpenClaw.Tray.UITests\AccessibilityAppFixture.cs` | `StartProcess` (env set, lines ~338-368), `NavigateAsync` + `WaitForNavigationSignalAsync` + `ReadNavigationSignals` (tab-separated `guid\tPageName` lines), `WaitForPageMarkerAsync`, `CaptureHubScreenshotIfRequested` (sampled-colour guard), `EnsureTargetIsAlive`, `WaitForHubWindow` | +| Isolated dirs with separate LOCALAPPDATA, MCP port allocation, MCP-ready loop, tool-availability wait, stdout/stderr capture, log copy, on-disk gateway record reader | `tests\OpenClaw.E2ETests\IsolatedTrayInstance.cs` | `WriteSettings` dictionary, `SpawnTray`, `WaitForMcpReadyAsync` (reads `mcp-token.txt`, GETs `http://127.0.0.1:{port}/`), `ReadActiveGatewayRecord`, `ReadCredentialState`, `WaitForConnectionReady` (polls MCP `app.status` for `connectionStatus`, `nodeConnected`, `nodePaired`), `LocateTrayExe`, `FindFreePort` | +| Loopback WebSocket gateway stub with method responders and frame capture | `tests\OpenClaw.Shared.Tests\GatewayProtocolLiveRoundTripTests.cs` (`LoopbackGatewayServer`, ~line 355) | HttpListener bind-with-retry, `OnMethod(string, Func)`, `_frames` queue, `WaitFrameAsync(method, occurrence, timeoutMs)` | +| Artifact redaction and secret-file exclusion | `tests\OpenClaw.E2ETests\Setup\E2ESetupFixture.cs` | `SanitizeForLog` (~line 900: redacts `token|authorization|secret|password` values, `Bearer` tokens, any 48+ char base64-ish run), `ShouldCopyArtifactFile` (~line 966: never `gateways.json`, `settings.json`, `device-key*`, anything under a `gateways\` segment), `CopyLogsFrom`, `AllocateFreePort`, `OPENCLAW_E2E_TRAY_EXE` override handling | +| MCP JSON-RPC client | `tests\OpenClaw.E2ETests\McpClient.cs` | reuse as-is via `InternalsVisibleTo` | +| Fake Ollama HTTP server | `tests\OpenClaw.E2ETests\FakeOllamaServer.cs` | reuse as-is | +| Gate attribute pattern | `tests\OpenClaw.E2ETests\E2EFactAttribute.cs` | `E2ETestGate.IsEnabled` "1|true" check; `MxcE2EFactAttribute` shows GITHUB_ACTIONS-aware skipping | +| Page inventory (tag, page name, marker id) for all 18 hub pages | `tests\OpenClaw.Tray.UITests\AccessibilityScanTests.cs` `PageTestData()` | copy the list into `Pages\AutomationIds.cs` | + +Existing internals in `tests\OpenClaw.E2ETests` are `internal`; add +`[assembly: InternalsVisibleTo("OpenClaw.GuiE2ETests")]` to +`tests\OpenClaw.E2ETests\AssemblyInfo.cs`. + +## 2. App hooks the harness relies on + +| Env var | Effect | Where | +|---|---|---| +| `OPENCLAW_TRAY_DATA_DIR`, `OPENCLAW_TRAY_APPDATA_DIR`, `OPENCLAW_TRAY_LOCALAPPDATA_DIR` | isolate all state; `DataDirOverride != null` also disables URI-scheme registration and the startup update check | `src\OpenClaw.Tray.WinUI\App.xaml.cs` (~684, ~699) | +| `OPENCLAW_FORCE_ONBOARDING=1` | always show the setup wizard at startup | `App.xaml.cs:839-840` | +| `OPENCLAW_ACCESSIBILITY_NAVIGATION_SIGNAL=` | app appends `guid\tPageName` when a hub page is ready | `src\OpenClaw.Tray.WinUI\Services\AccessibilityNavigationSignal.cs`, called from `Windows\HubWindow.xaml.cs` | +| `OPENCLAW_UI_AUTOMATION=1` | tray flyout does not auto-dismiss on deactivate | `Windows\TrayMenuWindow.xaml.cs:219` | +| `OPENCLAW_MCP_PORT` | fixed loopback MCP port; token written to `\mcp-token.txt` | `IsolatedTrayInstance.WaitForMcpReadyAsync` | +| `OPENCLAW_SUPPRESS_EXTERNAL_BROWSER=1` | no browser launches during tests | used by `IsolatedTrayInstance.SpawnTray` | +| `OPENCLAW_SKIP_UPDATE_CHECK=1`, `OPENCLAW_LANGUAGE=en-US` | belt-and-braces; language pins UIA `Name` strings | `AccessibilityAppFixture.StartProcess` | +| `OPENCLAW_E2E_TRAY_EXE` | override exe path (must be an existing `.exe`) | `E2ESetupFixture.ResolveTrayExecutable` | + +Wizard skip rule: `StartupSetupState.RequiresSetup` returns false when +`EnableMcpServer=true` is seeded, so the minimal seed used by +`AccessibilityAppFixture` (`EnableMcpServer=true, GlobalHotkeyEnabled=false, AutoStart=false`) +lands directly on the Hub. An empty data dir triggers onboarding naturally. + +Deep-link routes handled by `src\OpenClaw.Tray.WinUI\Services\DeepLinkHandler.cs`: +`hub/`, `tray|tray-menu|menu`, `settings`, `chat`, `check-updates`, +`agent?message=` (send). Scheme constant: `OpenClawTray.AppIdentity.ProtocolScheme` +(`openclaw`, or `openclaw-dev` for DevBuild). Because isolated instances do +not register the scheme, send a link by starting a second +`OpenClaw.Tray.WinUI.exe ` with the same data-dir env; it forwards over +IPC and exits. + +## 3. Fake gateway wire contract + +Sources: `src\OpenClaw.Shared\ConnectEnvelopeBuilder.cs` (~292-330), +`src\OpenClaw.Shared\OpenClawGatewayClient.cs` (hello-ok ~2309-2378, errors +~2603-2690, events ~3255-3320, challenge ~3520-3540), +`src\OpenClaw.Shared\WindowsNodeClient.cs` (invoke ~488-540, result ~645-665), +`src\OpenClaw.Connection\SetupCodeDecoder.cs`. Protocol constants live in +`GatewayProtocolContract` (`MinimumSupportedVersion`, `MaximumSupportedVersion`). + +Client to gateway, first frame after socket open (or after challenge): + +```json +{ "type": "req", "id": "", "method": "connect", + "params": { + "minProtocol": 3, "maxProtocol": 4, + "client": { "id": "openclaw-windows-tray|node-host", "version": "...", "platform": "windows", + "deviceFamily": "...", "mode": "operator|node", "displayName": "..." }, + "role": "operator|node", "scopes": ["..."], "caps": ["..."], "commands": ["..."], "permissions": {}, + "auth": { "token": "..." } | { "bootstrapToken": "..." } | { "deviceToken": "..." }, + "locale": "en-US", "userAgent": "openclaw-windows-tray/|openclaw-windows-node/", + "device": { "id": "...", "publicKey": "...", "signature": "...", "nonce": "...", "signedAt": ... } } } +``` + +Gateway to client, optional challenge (client answers with the connect above): + +```json +{ "type": "event", "event": "connect.challenge", "payload": { "nonce": "", "ts": 1700000000000 } } +``` + +Successful handshake. `id` must equal the connect request id. + +```json +{ "type": "res", "id": "", "ok": true, + "payload": { "type": "hello-ok", "protocol": 4, + "auth": { "deviceToken": "", "scopes": ["operator.admin"] }, + "device": { "id": "" }, + "snapshot": { "sessionDefaults": { "mainSessionKey": "agent:main:main" } } } } +``` + +Fields read by the client: `protocol`, `auth.deviceToken` or `auth.deviceTokens[]` +(entries `{role, deviceToken, scopes}`), `device.id`, `snapshot.sessionDefaults.mainSessionKey`. +Omitting `auth.deviceToken` is valid for direct-token auth. + +Pairing required (client then waits for the resolved event and reconnects): + +```json +{ "type": "res", "id": "", "ok": false, + "error": { "message": "pairing required", "details": { "code": "PAIRING_REQUIRED", "requestId": "" } } } +``` + +```json +{ "type": "event", "event": "device.pair.resolved", "payload": { "requestId": "", "decision": "approved" } } +{ "type": "event", "event": "node.pair.resolved", "payload": { "requestId": "", "decision": "approved" } } +``` + +Node invocation (gateway to node) and its answer (node to gateway): + +```json +{ "type": "event", "event": "node.invoke.request", + "payload": { "id": "", "nodeId": "", "command": "system.run", "paramsJSON": "{\"command\":\"cmd /c echo gui-e2e\"}" } } +``` + +```json +{ "type": "req", "id": "", "method": "node.invoke.result", + "params": { "id": "", "nodeId": "", "ok": true, "payload": { ... }, "error": null } } +``` + +Exec approvals: gateway emits events `exec.approval.requested` / +`exec.approval.resolved` (payload has `id`); the operator client resolves with +`{"method":"exec.approval.resolve","params":{"id":"","decision":"allow-once|allow-always|deny"}}`. + +Setup code: `Convert.ToBase64String(UTF8({"url":"ws://127.0.0.1:","bootstrapToken":""}))`. +Limits enforced by the decoder: total ≤ 2048 chars, token ≤ 512 chars. + +Post-handshake request storm from the operator client that the fake should +answer with empty-but-valid payloads (`{"type":"res","id":...,"ok":true,"payload":{...}}`): +`health`, `sessions.list`, `node.list`, `usage.status`, `usage.cost`, +`agents.list`, `config.get`, `skills.status`, `node.pair.list`, `cron.list`, +`chat.history`, `sessions.subscribe`. Chat: request `chat.send` (params include +`sessionKey`, `message`); reply with `{"runId":"","status":"started"}` +then push a `chat` event carrying the assistant message. Return +`{"ok":false,"error":{"message":"unsupported"}}` for anything else so the UI +fails fast instead of hanging. `docs\gateway-protocol-drift-guard.md` describes +how protocol drift is caught; keep fake responders minimal. + +## 4. Repo conventions that will bite + +- `tests\Directory.Build.props`: `TargetFramework net10.0`, `Nullable`, + `TreatWarningsAsErrors=true`, `NuGetAuditMode=all`. Any new warning fails the + build. `System.Drawing.Common` advisory NU1904 arrives transitively through the + Tray.WinUI reference; add `$(WarningsNotAsErrors);NU1904` + exactly as `tests\OpenClaw.Tray.UITests\OpenClaw.Tray.UITests.csproj` does. +- Do **not** set `true` in a test project. It turns the + project into a WinExe and breaks the test host (explained in the UITests csproj comment). +- TFM for anything referencing Tray.WinUI: `net10.0-windows10.0.22621.0`, with + `x64;ARM64` and `win-x64;win-arm64`. + CI builds with `-r win-x64`; without declared RIDs a `--no-restore` build fails with NETSDK1047. +- First-run gotcha (AGENTS.md): `dotnet test --no-restore` silently no-ops when + the test `bin\` does not exist yet. Build the test project first or omit `--no-restore`. +- Polling loops and best-effort cleanup need a `// slopwatch-ignore: SW00x ` + comment (SW003 for best-effort teardown, SW004 for bounded delays). See + `tests\OpenClaw.Connection.Tests\GatewayConnectionManagerTests.cs:31` and `:1484`. +- Test naming: `*Tests.cs`; real-behaviour proofs use `*ProofTests.cs`. +- xUnit traits are used for lane selection (`[Trait("Category", "Accessibility")]` + in CI filters). Filter syntax on the CLI: `--filter "Trait=Tier=Fake"`; xUnit + 2.x actually matches `Tier=Fake` as a trait name/value pair, so verify the + exact filter string against `dotnet test --list-tests` before wiring CI. +- `scripts\run-proof-tests.ps1` is the only permitted way to run proof tests + from a proof pool. It writes `TestResults\ProofPools\\.trx` and + fails on zero tests or zero passes. `scripts\validate-proof-pools.ps1:736` + allows only this command shape: + `^(?:$env:OPENCLAW_RUN_E2E = '1'; )?.\scripts\run-proof-tests.ps1 -Project '

' -Filter '' -ResultName ''(?: -RuntimeIdentifier (?:win-x64|win-arm64))?$` + Extend the env prefix group for `OPENCLAW_RUN_GUI_E2E` and add a case in + `scripts\test-proof-pool-validator.ps1`. +- No `Directory.Packages.props`; pin package versions directly in the csproj. + Windows App SDK version comes from the `$(MicrosoftWindowsAppSDKVersion)` property. +- Add the new project to `openclaw-windows-node.slnx`. E2E tests are + intentionally excluded from the default `dotnet test` at repo root; check how + `tests\OpenClaw.E2ETests` is handled in the slnx and CI before deciding + whether the GUI project follows the same pattern (recommended: include it, since + its always-on `CatalogTests` and `FakeGatewayServerTests` are cheap and the + GUI scenarios self-skip without the gate). + +## 5. Existing AutomationId inventory (what you can use today) + +- Hub pages: every page in `AccessibilityScanTests.PageTestData()` has a + `PageMarker` except ChatPage, which uses `ChatComposerInput` as marker. +- `Pages\ConnectionPage.xaml` (44): `PendingApprovalsBanner`, `StatusStrip`, + `ConnectionPageMarker`, `StripPrimaryAction`, `StripTerminalAction`, + `StripDashboardAction`, `ConnectionToggle`, `GatewayHostOpenTerminalAction`, + `GatewayHostStartAction`, `GatewayHostStopAction`, `GatewayHostRestartAction`, + `NodeCard`, `NodeModeToggle`, `NodeTrustApproveCopyAction`, `NodeApproveCopyAction`, + `NodeReconnect`, `RecoveryCard`, `RecoveryRestartTunnel`, `RecoveryEditTunnel`, + `RecoveryApplyRepair`, `RecoveryRepairResult`, `RecoveryCopyApprove`, + `RecoveryConnect`, `RecoveryDisconnect`, `SavedGatewaysCard`, `GatewaysScanAction`, + `AddGatewayHeaderAction`, `WelcomeAddTilesCard`, `WelcomeInstallLocalGateway`, + `LobbyDirectTile`, `LobbySetupCodeTile`, `WelcomeScanAction`, `AddGatewayPanel`, + `AddGatewayBack`, `AddMethodSelector`, `AddDirectUrl`, `AddDirectToken`, + `AddDirectName`, `AddRemoteHelpLink`, `AddSetupCodeInput`, `AddSetupCodeDecode`, + `AddInstallLocalGateway`, `AddSecurityAdvice`, `AddSave`. +- `Pages\SettingsPage.xaml` (26, includes `SettingsPageCheckUpdates`), + `Pages\LocalAiPage.xaml` (21: `LocalAiStart`, `LocalAiStop`, `LocalAiRestart`, + `LocalAiOpenLogs`, `LocalAiChangeModel`, ...), `Pages\DebugPage.xaml` (14), + `Pages\PermissionsPage.xaml` (8 static plus `{Binding RemoveRuleAutomationId}`). + Run `grep -o 'AutomationId="[A-Za-z]*"' ` for the full lists. +- Setup wizard: `WelcomePage.xaml` (`WelcomeInstallLocalGatewayChoice`, + `WelcomeInstallCheckProgress`, `WelcomeLocalAiAvailable`, + `WelcomeConnectExistingGatewayChoice`, `WelcomeBackButton`, `WelcomeNextButton`), + `CapabilitiesPage.xaml` (9 `LocalAi*` ids), `CompletePage.xaml` + (`LocalAiCompletionSummary` only). `SecurityNoticePage`, `AdvancedSetupPage`, + `ProgressPage` have none. +- Dialogs: `Dialogs\ExecApprovalDialog.cs` sets `ExecApprovalDenyAction`, + `ExecApprovalAllowAlwaysAction`, `ExecApprovalAllowOnceAction`; + `Dialogs\PairingApprovalDialog.cs` sets `PairingRejectAction`, + `PairingLaterAction`, `PairingApproveAction`. +- Tray: `TrayMenuWindow.xaml` has only `TrayMenuPanel`. +- Chat composer (code-built in `Chat\ReactorChatComposer.cs`): `ChatComposerInput`, + `ChatComposerAttach`, `ChatComposerSpeakerToggle`. + +## 6. Validation the repo requires before a PR + +From `AGENTS.md` and `.agents\skills\openclaw-proof-validation\SKILL.md`: + +```powershell +$env:OPENCLAW_REPO_ROOT = (Get-Location).Path +.\build.ps1 +dotnet test .\tests\OpenClaw.Shared.Tests\OpenClaw.Shared.Tests.csproj --no-restore +dotnet test .\tests\OpenClaw.Tray.Tests\OpenClaw.Tray.Tests.csproj --no-restore +``` + +Plus, for this work: build and run the new project, and run +`.\scripts\validate-proof-pools.ps1`, `.\scripts\test-proof-pool-validator.ps1` +and `.\scripts\validate-docs.ps1` whenever `.github\proof-pools.json` or docs change. + +PR body must contain `## Required proof pools`, `## Validation` and +`## Real Behavior Proof` (template: `.github\pull_request_template.md`). Select +pool IDs from `docs\PROOF_POOLS.md` or declare `none` with a reason. Never +present a skipped test as validation. Use isolated tray data for any proof run +(`.\run-app-local.ps1 -Isolated -AllowNonMain`). + +Architecture rules: read `docs\ARCHITECTURE.md` before touching any listed god +object; `HubWindow.xaml.cs` and `App.xaml.cs` are large, so keep product-side +edits to AutomationIds and `WritePageReady` calls. + +## 7. Known unknowns to resolve early + +- FlaUI attach to a WinUI 3 window: content lives under a + `Microsoft.UI.Content.DesktopChildSiteBridge` child; scope searches to the page + marker's parent to keep `FindFirstDescendant` fast. +- Whether `AutoStart` and hotkey registration are guarded in isolated mode; + grep `DataDirOverride` in the settings/autostart services before running + GUI-SET-001 on a shared machine. +- The exact xUnit trait filter string that works with `dotnet test --filter`. +- Crabbox desktop image capabilities (see PLAN.md open items). diff --git a/tests/OpenClaw.GuiE2ETests/KICKOFF_PROMPT.md b/tests/OpenClaw.GuiE2ETests/KICKOFF_PROMPT.md new file mode 100644 index 000000000..6e6d85d6c --- /dev/null +++ b/tests/OpenClaw.GuiE2ETests/KICKOFF_PROMPT.md @@ -0,0 +1,84 @@ +# Kickoff prompt: Phase 0 spike for the GUI E2E lane + +Copy everything below the line into a fresh session (any capable coding model) +started at the repository root on the branch `dev/plarroy/gui-e2e-plan`. + +--- + +You are implementing **Phase 0 only** of the plan in +`tests\OpenClaw.GuiE2ETests\PLAN.md`. Read that file and +`tests\OpenClaw.GuiE2ETests\IMPLEMENTATION_NOTES.md` first, then `AGENTS.md`. +Do not start Phase 1 work. When the acceptance criteria below are met, commit, +push to the `larroy-fork` remote on the current branch, and stop with a report. + +## Goal + +Prove that FlaUI (UIA3) can attach to the real `OpenClaw.Tray.WinUI.exe` +running with isolated state, navigate to the Connection page through the +existing deep-link path, find an element by AutomationId, and capture a +non-blank screenshot of the Hub window. This must work locally and on a +GitHub-hosted `windows-latest` runner. + +## Scope: files you may create or modify + +- `tests\OpenClaw.GuiE2ETests\OpenClaw.GuiE2ETests.csproj` (new; mirror + `tests\OpenClaw.Tray.UITests\OpenClaw.Tray.UITests.csproj`, add `FlaUI.Core` + and `FlaUI.UIA3`, no `UseWinUI`, `NU1904` exemption, RIDs `win-x64;win-arm64`). +- `tests\OpenClaw.GuiE2ETests\xunit.runner.json` (`parallelizeTestCollections: false`). +- `tests\OpenClaw.GuiE2ETests\Harness\GuiE2EGate.cs` (`GuiE2EFactAttribute`, gate `OPENCLAW_RUN_GUI_E2E`). +- `tests\OpenClaw.GuiE2ETests\Harness\GuiApp.cs` (port of + `AccessibilityAppFixture` launch/navigate logic plus `FlaUI.Core.Application.Attach`). +- `tests\OpenClaw.GuiE2ETests\Harness\Wait.cs`, `Harness\ArtifactSink.cs` (minimal). +- `tests\OpenClaw.GuiE2ETests\Pages\AutomationIds.cs` (page inventory from + `AccessibilityScanTests.PageTestData()` plus the ConnectionPage ids listed in the notes). +- `tests\OpenClaw.GuiE2ETests\Scenarios\SmokeScenarios.cs` with one test. +- `openclaw-windows-node.slnx` (add the project). +- `.github\workflows\gui-e2e-spike.yml` (temporary, `workflow_dispatch` only; + delete it in Phase 1 when the real `gui-e2e-smoke` job lands in `ci.yml`). + +Do **not** modify anything under `src\`, `.github\proof-pools.json`, +`scripts\`, or existing test projects, except adding +`[assembly: InternalsVisibleTo("OpenClaw.GuiE2ETests")]` to +`tests\OpenClaw.E2ETests\AssemblyInfo.cs` if you reuse its internals. + +## The one scenario + +`SmokeScenarios.HubLaunches_ConnectionPageIsReachable_AndScreenshotIsNotBlank` +tagged `[GuiE2EFact]`, `[Trait("Tier","Fake")]`, `[Trait("Workflow","Smoke")]`: + +1. Start the tray with isolated dirs and the seed + `{"SettingsSchemaVersion":1,"EnableMcpServer":true,"GlobalHotkeyEnabled":false,"AutoStart":false}` + and env `OPENCLAW_TRAY_DATA_DIR`, `OPENCLAW_TRAY_APPDATA_DIR`, + `OPENCLAW_TRAY_LOCALAPPDATA_DIR`, `OPENCLAW_SKIP_UPDATE_CHECK=1`, + `OPENCLAW_LANGUAGE=en-US`, `OPENCLAW_UI_AUTOMATION=1`, + `OPENCLAW_SUPPRESS_EXTERNAL_BROWSER=1`, + `OPENCLAW_ACCESSIBILITY_NAVIGATION_SIGNAL=\nav.ready`, passing the + argument `://hub/connection` where scheme is `OpenClawTray.AppIdentity.ProtocolScheme`. +2. Wait for the main window handle, attach FlaUI, get the Hub `Window`. +3. Navigate to `hub/settings` via a second exe process, wait for the nav signal + line `SettingsPage`, then find `SettingsPageMarker` by AutomationId with FlaUI. + Navigate back to `hub/connection` and find `ConnectionPageMarker` and `AddGatewayHeaderAction`. +4. Capture the Hub window to `TestResults\GuiE2E\\smoke\hub-connection.png` + and assert it is non-blank (sampled colours ≥ 3, reuse the guard from `AccessibilityAppFixture`). +5. Dump the UIA tree under the Hub window to `uia-tree.txt` (helps Phase 1 write page objects). +6. Kill the process tree and delete temp dirs in `Dispose`. + +## Acceptance criteria + +- `dotnet build tests\OpenClaw.GuiE2ETests -c Debug -r win-x64` succeeds with zero warnings. +- Without the gate: `dotnet test tests\OpenClaw.GuiE2ETests -c Debug -r win-x64 --no-build` + reports the scenario as skipped with the reason text, not failed. +- With `OPENCLAW_RUN_GUI_E2E=1`: the scenario passes locally three times in a row. +- The spike workflow passes on `windows-latest` and uploads the PNG, the UIA tree + dump and the TRX as an artifact. Use the same restore/build steps as the + `e2etests` job in `.github\workflows\ci.yml` (restore, build Shared, + SetupEngine, Tray.WinUI with `-r win-x64`, then the test project). +- Required repo validation still passes (`.\build.ps1`, the two `dotnet test` + commands in `AGENTS.md`). + +## Report back + +State: exact commit SHA pushed, the workflow run URL, whether AutomationIds +were visible under the WinUI `DesktopChildSiteBridge` node in the UIA dump, +attach/navigation timings, and anything in `IMPLEMENTATION_NOTES.md` that +turned out to be wrong. List open questions for Phase 1. Do not proceed to Phase 1.