diff --git a/.github/workflows/sandbox-e2e.yml b/.github/workflows/sandbox-e2e.yml new file mode 100644 index 000000000..482d18a85 --- /dev/null +++ b/.github/workflows/sandbox-e2e.yml @@ -0,0 +1,132 @@ +name: Windows Sandbox E2E Tests + +# Windows Sandbox requires full Hyper-V container infrastructure which is NOT +# available on GitHub-hosted runners. The feature reports "Enabled" but +# WindowsSandbox.exe cannot boot VMs (no nested Hyper-V container support). +# +# This workflow: +# - Builds and verifies sandbox binaries on every PR (always works) +# - Attempts to run E2E tests but expects them to fail on hosted runners +# - Succeeds fully on self-hosted runners with Windows Sandbox enabled +# - Can be triggered manually via workflow_dispatch on capable machines +# +# Unlike NanVix/MicroVM which uses WHP (a lightweight hypervisor API that +# works on Azure VMs), Windows Sandbox needs the full container stack. + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +jobs: + sandbox-e2e: + name: Sandbox E2E + runs-on: windows-latest + timeout-minutes: 45 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust toolchain + run: | + rustup update stable + rustup target add x86_64-pc-windows-msvc + + - name: Build + working-directory: src + run: cargo build --target x86_64-pc-windows-msvc + + - name: Exclude build output from Windows Defender + shell: pwsh + run: | + $binDir = Join-Path $env:GITHUB_WORKSPACE "src\target\x86_64-pc-windows-msvc\debug" + Add-MpPreference -ExclusionPath $binDir + Write-Host "Added Defender exclusion for $binDir" + + - name: Verify Sandbox binaries + shell: pwsh + run: | + $binDir = Join-Path $env:GITHUB_WORKSPACE "src\target\x86_64-pc-windows-msvc\debug" + $required = @("wxc-exec.exe", "wxc-sandbox-daemon.exe", "wxc-sandbox-guest.exe") + $missing = $required | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } + if ($missing) { + Write-Host "::error::Missing binaries: $($missing -join ', ')" + exit 1 + } + Get-ChildItem $binDir -Include $required -Recurse | Format-Table Name, Length + + - name: Check Windows Sandbox availability + id: sandbox-check + shell: pwsh + run: | + # Check if the feature is installed (may be "enabled" but non-functional + # on GitHub-hosted runners due to lack of nested Hyper-V container support). + $info = dism /online /get-featureinfo /featurename:Containers-DisposableClientVM 2>&1 | + Select-String "State" + if ($info -notmatch "Enabled") { + Write-Host "::warning::Windows Sandbox feature is not enabled - skipping E2E tests." + "sandbox_available=false" | Out-File -Append -FilePath $env:GITHUB_OUTPUT + } else { + Write-Host "Windows Sandbox feature is enabled. Will attempt E2E tests." + Write-Host "Note: On GitHub-hosted runners, the feature may be enabled but" + Write-Host "non-functional (WindowsSandbox.exe cannot boot VMs). Tests will" + Write-Host "report a warning if they fail for this reason." + "sandbox_available=true" | Out-File -Append -FilePath $env:GITHUB_OUTPUT + } + exit 0 + + - name: Run Sandbox E2E Tests + if: steps.sandbox-check.outputs.sandbox_available == 'true' + id: sandbox-tests + continue-on-error: true + shell: pwsh + working-directory: test_scripts + run: | + $wxcExe = Join-Path $env:GITHUB_WORKSPACE "src\target\x86_64-pc-windows-msvc\debug\wxc-exec.exe" + $configDir = Join-Path $env:GITHUB_WORKSPACE "test_configs" + .\run_sandbox_tests.ps1 -WxcExePath $wxcExe -ConfigDir $configDir + + - name: Report sandbox test outcome + if: steps.sandbox-check.outputs.sandbox_available == 'true' && !cancelled() + shell: pwsh + run: | + $outcome = "${{ steps.sandbox-tests.outcome }}" + if ($outcome -eq "success") { + Write-Host "All sandbox E2E tests passed!" -ForegroundColor Green + } elseif ($outcome -eq "failure") { + Write-Host "::warning::Sandbox E2E tests failed." + Write-Host "::warning::This is expected on GitHub-hosted runners where Windows Sandbox" + Write-Host "::warning::is enabled but non-functional (no nested Hyper-V container support)." + Write-Host "::warning::Tests pass on self-hosted runners with full Sandbox support." + } + + - name: Print performance summary + if: steps.sandbox-check.outputs.sandbox_available == 'true' && !cancelled() + shell: pwsh + run: | + $jsonPath = Join-Path $env:GITHUB_WORKSPACE "sandbox-perf-results.json" + if (Test-Path $jsonPath) { + $data = Get-Content $jsonPath -Raw | ConvertFrom-Json + Write-Host "`n=== Sandbox Performance Summary ===" + Write-Host "Commit: $($data.commit)" + Write-Host "Timestamp: $($data.timestamp)" + Write-Host "" + Write-Host ("{0,-40} {1,10} {2,8}" -f "Test", "Time (ms)", "Status") + Write-Host ("{0,-40} {1,10} {2,8}" -f "----", "---------", "------") + foreach ($r in $data.results) { + Write-Host ("{0,-40} {1,10} {2,8}" -f $r.description, $r.wall_time_ms, $r.status) + } + } else { + Write-Host "No performance results found (tests may not have run)." + } + + - name: Upload performance results + if: steps.sandbox-check.outputs.sandbox_available == 'true' && !cancelled() + uses: actions/upload-artifact@v4 + with: + name: sandbox-perf-results-${{ github.event.pull_request.number || github.run_number }} + retention-days: 30 + path: sandbox-perf-results.json + if-no-files-found: ignore diff --git a/test_scripts/run_sandbox_tests.ps1 b/test_scripts/run_sandbox_tests.ps1 index f3b636d7e..9c3fcef02 100644 --- a/test_scripts/run_sandbox_tests.ps1 +++ b/test_scripts/run_sandbox_tests.ps1 @@ -1,35 +1,61 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -# Windows Sandbox E2E test runner. -# Requires: Windows 11 Pro/Enterprise, Windows Sandbox enabled, Python on host. -# Cannot run in GitHub Actions CI (needs Hyper-V + Sandbox feature). -# -# Usage: -# .\run_sandbox_tests.ps1 # debug build -# .\run_sandbox_tests.ps1 -Release # release build - +<# +.SYNOPSIS + Runs all Windows Sandbox E2E tests with performance measurement. + +.DESCRIPTION + - Checks if Windows Sandbox feature is enabled + - Locates wxc-exec.exe and wxc-sandbox-daemon.exe + - Starts the sandbox daemon as a background process + - Runs each test config, validates exit codes and output + - Captures per-test wall-clock timing + - Writes sandbox-perf-results.json for CI artifact consumption + - Reports pass/fail summary with performance table + +.PARAMETER WxcExePath + Path to wxc-exec.exe. Defaults to ..\src\target\debug\wxc-exec.exe + +.PARAMETER ConfigDir + Path to test configs directory. Defaults to ..\test_configs + +.PARAMETER Release + Use release build binaries instead of debug. + +.EXAMPLE + .\run_sandbox_tests.ps1 + .\run_sandbox_tests.ps1 -Release + .\run_sandbox_tests.ps1 -WxcExePath C:\build\wxc-exec.exe -ConfigDir C:\configs +#> param( + [string]$WxcExePath = "", + [string]$ConfigDir = "", [switch]$Release ) $ErrorActionPreference = "Stop" $RepoRoot = Split-Path -Parent $PSScriptRoot -$TestConfigs = Join-Path $RepoRoot "test_configs" -# Find binaries -if ($Release) { - $BinDir = Join-Path $RepoRoot "src\target\release" -} else { - $BinDir = Join-Path $RepoRoot "src\target\debug" +# Resolve paths +if (-not $ConfigDir) { + $ConfigDir = Join-Path $RepoRoot "test_configs" +} +if (-not $WxcExePath) { + $profile = if ($Release) { "release" } else { "debug" } + $WxcExePath = Join-Path $RepoRoot "src\target\$profile\wxc-exec.exe" } -$WxcExec = Join-Path $BinDir "wxc-exec.exe" +$BinDir = Split-Path $WxcExePath $Daemon = Join-Path $BinDir "wxc-sandbox-daemon.exe" -if (-not (Test-Path $WxcExec)) { - Write-Host "ERROR: wxc-exec.exe not found at $WxcExec" -ForegroundColor Red - Write-Host "Run 'cargo build$(if ($Release) { ' --release' })' first." -ForegroundColor Yellow +# -- Preflight ---------------------------------------------------------------- + +Write-Host "`n=== Windows Sandbox E2E Tests ===" -ForegroundColor Cyan + +if (-not (Test-Path $WxcExePath)) { + Write-Host "ERROR: wxc-exec.exe not found at $WxcExePath" -ForegroundColor Red + Write-Host " Build with: cd src && cargo build" -ForegroundColor Yellow exit 1 } if (-not (Test-Path $Daemon)) { @@ -37,38 +63,74 @@ if (-not (Test-Path $Daemon)) { exit 1 } -# Preflight: check Windows Sandbox is available $sandboxFeature = dism /online /get-featureinfo /featurename:Containers-DisposableClientVM 2>&1 | Select-String "State" if ($sandboxFeature -notmatch "Enabled") { - Write-Host "ERROR: Windows Sandbox feature is not enabled." -ForegroundColor Red - Write-Host "Run: dism /online /enable-feature /featurename:Containers-DisposableClientVM /all" -ForegroundColor Yellow + Write-Host "SKIP: Windows Sandbox feature is not enabled." -ForegroundColor Yellow + Write-Host " Enable with: dism /online /enable-feature /featurename:Containers-DisposableClientVM /all" + exit 0 +} + +Write-Host "wxc-exec: $WxcExePath" +Write-Host "daemon: $Daemon" +Write-Host "configs: $ConfigDir" + +# -- Test definitions --------------------------------------------------------- + +$tests = @( + @{ Config = "sandbox_echo.json"; ExpectedExit = 0; Description = "Echo (cmd.exe)"; OutputContains = "Hello from sandbox!" }, + @{ Config = "basic_sandbox.json"; ExpectedExit = 0; Description = "Python hello world"; OutputContains = "executed successfully" }, + @{ Config = "sandbox_powershell.json"; ExpectedExit = 0; Description = "PowerShell"; OutputContains = "PowerShell works" }, + @{ Config = "sandbox_powershell_env.json"; ExpectedExit = 0; Description = "PowerShell environment"; OutputContains = "ComputerName=" }, + @{ Config = "sandbox_stderr.json"; ExpectedExit = 0; Description = "stderr relay"; OutputContains = "stdout-message" }, + @{ Config = "sandbox_exit_code.json"; ExpectedExit = 42; Description = "Exit code propagation"; OutputContains = "" }, + @{ Config = "sandbox_timeout.json"; ExpectedExit = -1; Description = "Timeout kills process"; OutputContains = "" }, + @{ Config = "sandbox_echo.json"; ExpectedExit = 0; Description = "Multi-exec #1 (echo)"; OutputContains = "Hello from sandbox!" }, + @{ Config = "sandbox_echo.json"; ExpectedExit = 0; Description = "Multi-exec #2 (echo)"; OutputContains = "Hello from sandbox!" }, + @{ Config = "sandbox_echo.json"; ExpectedExit = 0; Description = "Multi-exec #3 (echo)"; OutputContains = "Hello from sandbox!" } +) + +# -- Cleanup & start daemon --------------------------------------------------- + +Write-Host "`nCleaning up stale sandbox processes..." -ForegroundColor Yellow +Get-Process -Name "wxc-sandbox-daemon","WindowsSandbox*" -ErrorAction SilentlyContinue | + ForEach-Object { Stop-Process -Id $_.Id -Force -ErrorAction SilentlyContinue } +Remove-Item "$env:TEMP\wxc-sandbox-rendezvous\*" -ErrorAction SilentlyContinue +Start-Sleep 5 + +Write-Host "Starting sandbox daemon..." -ForegroundColor Yellow +$daemonProc = Start-Process -FilePath $Daemon -ArgumentList "wxc-sandbox","300000" ` + -PassThru -NoNewWindow -RedirectStandardError "$env:TEMP\wxc-sandbox-daemon.log" +Start-Sleep 2 + +if ($daemonProc.HasExited) { + Write-Host "ERROR: Daemon exited immediately. Check $env:TEMP\wxc-sandbox-daemon.log" -ForegroundColor Red exit 1 } +Write-Host "Daemon started (PID $($daemonProc.Id))`n" -ForegroundColor Green + +# -- Run tests ---------------------------------------------------------------- -# Helpers -function Run-SandboxTest { - param( - [string]$ConfigFile, - [int]$ExpectedExit = 0, - [string]$OutputContains = "", - [switch]$ExpectNonZero - ) +$passed = 0 +$failed = 0 +$results = @() - $configPath = Join-Path $TestConfigs $ConfigFile +foreach ($test in $tests) { + $configPath = Join-Path $ConfigDir $test.Config if (-not (Test-Path $configPath)) { - return @{ Name = $ConfigFile; Pass = $false; Reason = "Config file not found" } + Write-Host " SKIP $($test.Config) (file not found)" -ForegroundColor Yellow + continue } - Write-Host " Running $ConfigFile... " -NoNewline + Write-Host " $($test.Description) ($($test.Config))... " -NoNewline - # wxc-exec outputs base64-encoded stdout/stderr when not attached to a - # terminal (e.g. when daemon was started via Start-Process). We capture - # everything and try to decode any base64 lines we find. - $output = & $WxcExec --debug $configPath 2>&1 | Out-String - $exitCode = $LASTEXITCODE + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $output = & $WxcExePath --debug --experimental $configPath 2>&1 | Out-String + $actualExit = $LASTEXITCODE + $sw.Stop() + $elapsedMs = $sw.ElapsedMilliseconds - # Build a combined string: raw output + any decoded base64 lines. + # Decode any base64 lines in the output $decoded = $output $lines = $output -split "`n" | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" } foreach ($line in $lines) { @@ -82,98 +144,68 @@ function Run-SandboxTest { } # Validate - $pass = $true - $reason = "" + $ok = $true + $expectedExit = $test.ExpectedExit - if ($ExpectNonZero) { - if ($exitCode -eq 0) { - $pass = $false - $reason = "Expected non-zero exit, got 0" - } + # For timeout test, accept any non-zero exit + if ($test.Description -match "Timeout") { + if ($actualExit -eq 0) { $ok = $false } } else { - if ($exitCode -ne $ExpectedExit) { - $pass = $false - $reason = "Expected exit $ExpectedExit, got $exitCode" - } + if ($actualExit -ne $expectedExit) { $ok = $false } } - if ($pass -and $OutputContains -and $decoded -notmatch [regex]::Escape($OutputContains)) { - $pass = $false - $reason = "Output missing '$OutputContains'" + if ($ok -and $test.OutputContains -and $decoded -notmatch [regex]::Escape($test.OutputContains)) { + $ok = $false } - if ($pass) { - Write-Host "PASS" -ForegroundColor Green + $status = if ($ok) { "PASS" } else { "FAIL" } + if ($ok) { + Write-Host "PASS (${elapsedMs}ms)" -ForegroundColor Green + $passed++ } else { - Write-Host "FAIL" -ForegroundColor Red - Write-Host " Reason: $reason" -ForegroundColor Red - $meaningful = $output -split "`n" | Where-Object { $_.Trim() -ne "" } | Select-Object -Last 5 - foreach ($line in $meaningful) { - Write-Host " > $($line.TrimEnd())" -ForegroundColor Gray - } + Write-Host "FAIL (exit=$actualExit, ${elapsedMs}ms)" -ForegroundColor Red + $failed++ } - return @{ Name = $ConfigFile; Pass = $pass; Reason = $reason } + $results += @{ + Test = $test.Config + Description = $test.Description + WallTimeMs = $elapsedMs + Exit = $actualExit + Status = $status + } } -# Clean up stale processes -Write-Host "`nSandbox E2E Tests" -ForegroundColor Cyan -Write-Host "=================" -ForegroundColor Cyan -Write-Host "`nCleaning up stale sandbox processes..." -ForegroundColor Yellow -Get-Process -Name "wxc-sandbox-daemon","WindowsSandbox*" -ErrorAction SilentlyContinue | - ForEach-Object { Stop-Process -Id $_.Id -Force -ErrorAction SilentlyContinue } -Remove-Item "$env:TEMP\wxc-sandbox-rendezvous\*" -ErrorAction SilentlyContinue -Start-Sleep 5 - -# Start daemon -Write-Host "Starting sandbox daemon..." -ForegroundColor Yellow -$daemonProc = Start-Process -FilePath $Daemon -ArgumentList "wxc-sandbox","300000" ` - -PassThru -NoNewWindow -RedirectStandardError "$env:TEMP\wxc-sandbox-daemon.log" -Start-Sleep 2 - -if ($daemonProc.HasExited) { - Write-Host "ERROR: Daemon exited immediately. Check $env:TEMP\wxc-sandbox-daemon.log" -ForegroundColor Red - exit 1 -} -Write-Host "Daemon started (PID $($daemonProc.Id))`n" -ForegroundColor Green +# -- Performance summary ------------------------------------------------------ -# Run tests -[System.Collections.ArrayList]$results = @() - -Write-Host "--- Basic Tests ---" -ForegroundColor Cyan -$null = $results.Add((Run-SandboxTest "sandbox_echo.json" -OutputContains "Hello from sandbox!")) -$null = $results.Add((Run-SandboxTest "basic_sandbox.json" -OutputContains "executed successfully")) -$null = $results.Add((Run-SandboxTest "sandbox_powershell.json" -OutputContains "PowerShell works")) -$null = $results.Add((Run-SandboxTest "sandbox_powershell_env.json" -OutputContains "ComputerName=")) -$null = $results.Add((Run-SandboxTest "sandbox_stderr.json" -OutputContains "stdout-message")) -$null = $results.Add((Run-SandboxTest "sandbox_exit_code.json" -ExpectedExit 42)) - -Write-Host "`n--- Timeout Test ---" -ForegroundColor Cyan -$null = $results.Add((Run-SandboxTest "sandbox_timeout.json" -ExpectNonZero)) - -Write-Host "`n--- Multi-Exec Test (3x echo on same VM) ---" -ForegroundColor Cyan -for ($iter = 1; $iter -le 3; $iter++) { - $result = Run-SandboxTest "sandbox_echo.json" -OutputContains "Hello from sandbox!" - $result.Name = "multi-exec #$iter (sandbox_echo.json)" - $null = $results.Add($result) +Write-Host "`n=== Performance ===" -ForegroundColor Cyan +Write-Host (" {0,-40} {1,10} {2,8}" -f "Test", "Time (ms)", "Status") +Write-Host (" {0,-40} {1,10} {2,8}" -f "----", "---------", "------") +foreach ($r in $results) { + $color = if ($r.Status -eq "PASS") { "Green" } else { "Red" } + Write-Host (" {0,-40} {1,10} {2,8}" -f $r.Description, $r.WallTimeMs, $r.Status) -ForegroundColor $color } -# Summary -$passed = ($results | Where-Object { $_.Pass }).Count -$failed = ($results | Where-Object { -not $_.Pass }).Count -$total = $results.Count - -Write-Host "`n===================" -ForegroundColor Cyan -if ($failed -eq 0) { - Write-Host "ALL $total TESTS PASSED" -ForegroundColor Green -} else { - Write-Host "$passed/$total passed, $failed FAILED:" -ForegroundColor Red - $results | Where-Object { -not $_.Pass } | ForEach-Object { - Write-Host " FAIL: $($_.Name) - $($_.Reason)" -ForegroundColor Red +# Write JSON results for CI artifact +$perfOutput = @{ + commit = if ($env:GITHUB_SHA) { $env:GITHUB_SHA } else { "local" } + timestamp = (Get-Date -Format "o") + results = $results | ForEach-Object { + @{ + test = $_.Test + description = $_.Description + wall_time_ms = $_.WallTimeMs + exit_code = $_.Exit + status = $_.Status + } } } +$perfJsonPath = Join-Path $RepoRoot "sandbox-perf-results.json" +$perfOutput | ConvertTo-Json -Depth 3 | Set-Content $perfJsonPath -Encoding UTF8 +Write-Host "`n Performance results written to: $perfJsonPath" + +# -- Cleanup ------------------------------------------------------------------ -# Cleanup Write-Host "`nStopping daemon..." -ForegroundColor Yellow if (-not $daemonProc.HasExited) { Stop-Process -Id $daemonProc.Id -Force -ErrorAction SilentlyContinue @@ -183,4 +215,22 @@ Get-Process -Name "WindowsSandbox*" -ErrorAction SilentlyContinue | Write-Host "Daemon log: $env:TEMP\wxc-sandbox-daemon.log" -ForegroundColor Gray -exit $(if ($failed -gt 0) { 1 } else { 0 }) +# -- Summary ------------------------------------------------------------------ + +$total = $passed + $failed +Write-Host "`n=== Results ===" -ForegroundColor Cyan +if ($total -eq 0) { + Write-Host " ERROR: No tests were executed." -ForegroundColor Red + exit 1 +} +Write-Host " Passed: $passed / $total" +if ($failed -gt 0) { + Write-Host " Failed: $failed / $total" -ForegroundColor Red + $results | Where-Object { $_.Status -eq "FAIL" } | ForEach-Object { + Write-Host " - $($_.Description) ($($_.Test), exit=$($_.Exit))" -ForegroundColor Red + } + exit 1 +} else { + Write-Host " All Sandbox E2E tests passed!" -ForegroundColor Green + exit 0 +}