Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 49 additions & 6 deletions reference/cindy-project.md

Large diffs are not rendered by default.

55 changes: 49 additions & 6 deletions reference/cindy-project.zh-CN.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion reference/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ flakes were loaded local sandboxes — gated by `ACP_TEST_CLAUDE_NATIVE`, not CI
The script self-checks on every run: a new top-level `tests/*.test.ts` that
matches a process-spawn pattern but is NOT registered in its `LIVE_FILES` list
fails the job. When you add a suite that spawns processes, register it in
`scripts/run-ci-tests.mjs` and review its concurrency assumptions (#1248:
`scripts/live-files.mjs` (append-only) and review its concurrency assumptions (#1248:
per-file XDG homes, `listen(0)` ports, no machine-global state) before it
rides c=4 unreviewed.

Expand Down
3 changes: 2 additions & 1 deletion scripts/ci-shard-weights.json
Original file line number Diff line number Diff line change
Expand Up @@ -847,6 +847,7 @@
"tests/incremental-ack-pull.test.ts": 10547,
"tests/issue4323-drift-escalation.test.ts": 2194,
"tests/pi-host-stub.test.ts": 136,
"tests/server-close-all-connections.test.ts": 3212
"tests/server-close-all-connections.test.ts": 3212,
"tests/live-deploy-scripts.test.ts": 221000
}
}
389 changes: 389 additions & 0 deletions scripts/live-deploy/Deploy-LiveProxy.ps1

Large diffs are not rendered by default.

620 changes: 620 additions & 0 deletions scripts/live-deploy/LiveProxy.Common.ps1

Large diffs are not rendered by default.

154 changes: 154 additions & 0 deletions scripts/live-deploy/Rollback-LiveProxy.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
#requires -Version 7.2
<#
.SYNOPSIS
Rolls one live proxy deployment back to the release it replaced, from that deployment's backup folder.

.DESCRIPTION
Undoes a deployment made by Deploy-LiveProxy.ps1 (step 8 of "Deploy only with an explicit request" in
reference/cindy-project.md): holds supervision, stops the deployed release, restores the launcher and
the receipts from the backup folder, lets the supervisor start the previous release and checks it.
Everything it needs except the two paths comes from deploy-record.json in the backup folder.

If the previous release does not come up healthy, the hold is put back and nothing further is started.

Exit codes: 0 restored (or preflight passed), 2 refused (nothing changed), 4 rollback failed (hold in
place). Any other exit code means the state is unknown: run again with -PreflightOnly.

.EXAMPLE
pwsh -NoProfile -NonInteractive -File Rollback-LiveProxy.ps1 -RuntimeRoot "<runtime root>" -BackupRoot "<runtime root>\backups\<backup folder>" -PreflightOnly
#>
[CmdletBinding()]
param(
[Parameter(Mandatory)] [string]$RuntimeRoot,
[Parameter(Mandatory)] [string]$BackupRoot,
[int]$HealthWaitSeconds = 90,
[int]$StopWaitSeconds = 20,
[string]$TestTaskDriver,
[switch]$PreflightOnly
)
. (Join-Path $PSScriptRoot 'LiveProxy.Common.ps1')

function Stop-Refused([string]$Why) { throw [LpStop]::new(2, 'refused', "refused: $Why; nothing was changed", '', 'see the reason above') }

# Strict shape check of the deployment record: exact keys, expected formats.
function Assert-RecordShape($Rec) {
$keys = @('schema', 'deploymentId', 'createdAt', 'runtimeRoot', 'port', 'taskName', 'previous', 'candidate', 'nodeModules', 'configSha256', 'launcherSha256Before', 'launcherSha256After', 'deployProcess', 'created', 'stage', 'reachedStage')
$sub = @{
previous = @('proxyPath', 'version', 'commit'); candidate = @('releaseName', 'proxyPath', 'version', 'commit', 'sourceCommit', 'sha256')
nodeModules = @('linkType', 'target'); deployProcess = @('pid', 'created'); created = @('releaseDir', 'backupDir')
}
$same = { param($have, $want) (@($have | Sort-Object) -join ',') -ceq (@($want | Sort-Object) -join ',') }
if (-not ($Rec -is [System.Collections.IDictionary]) -or -not (& $same $Rec.Keys $keys)) { throw 'unexpected or missing fields' }
foreach ($k in $sub.Keys) { if (-not ($Rec[$k] -is [System.Collections.IDictionary]) -or -not (& $same $Rec[$k].Keys $sub[$k])) { throw "unexpected or missing fields in $k" } }
if ($Rec.schema -ne 1) { throw 'unknown schema version' }
if ($script:LpStages -notcontains [string]$Rec.stage -or $script:LpForwardStages -notcontains [string]$Rec.reachedStage) { throw 'unknown stage' }
foreach ($side in 'previous', 'candidate') {
if ([string]$Rec[$side].version -notmatch $script:LpPatterns.Version -or [string]$Rec[$side].commit -cnotmatch $script:LpPatterns.Commit) { throw "malformed $side identity" }
if (-not (Test-LpSafeText ([string]$Rec[$side].proxyPath))) { throw "malformed $side path" }
}
if ([string]$Rec.candidate.releaseName -notmatch $script:LpPatterns.ReleaseName) { throw 'malformed release name' }
foreach ($h in 'launcherSha256Before', 'launcherSha256After', 'configSha256') { if ([string]$Rec[$h] -notmatch $script:LpPatterns.Sha256) { throw "malformed $h" } }
if (-not ($Rec.port -is [int] -or $Rec.port -is [long]) -or $Rec.port -lt 1 -or $Rec.port -gt 65535) { throw 'malformed port' }
if (-not (Test-LpSafeText ([string]$Rec.taskName)) -or [string]$Rec.taskName -match '[*?\[\]]') { throw 'malformed task name' }
}

$ctx = $null
try {
$root = Resolve-LpPath $RuntimeRoot
if (Test-LpReparseInPath $root) { Stop-Refused 'a folder in the runtime root path is a link' }
$script:LpRoot = $root
# A full run takes the operation lock before it looks at anything; -PreflightOnly creates no file at all.
if (-not $PreflightOnly) { try { Enter-LpOperation $root } catch { Stop-Refused $_.Exception.Message } }

$driver = $null
try { $driver = Resolve-LpTaskDriver $RuntimeRoot $root $TestTaskDriver } catch { Stop-Refused $_.Exception.Message }
if ($driver) { . $driver; Write-LpEvent 'STEP' 'TEST SEAM ACTIVE: scheduled-task operations go to the test driver' }

$backup = $null
try { $backup = Resolve-LpPath $BackupRoot } catch { Stop-Refused 'the backup folder does not exist' }
if ([System.IO.Path]::GetDirectoryName($backup) -ne (Join-Path $root 'backups')) { Stop-Refused 'the backup folder is not directly under <runtime root>\backups' }
$recordPath = Join-Path $backup 'deploy-record.json'
$restoreNames = @('Start-BillionContextProxy.ps1', 'tuned-deployment.json', 'supervision-receipt.json')
foreach ($p in @($backup, $recordPath) + @($restoreNames | ForEach-Object { Join-Path $backup $_ })) {
if (-not (Test-Path -LiteralPath $p)) { Stop-Refused "$(Get-LpRelative $p) is missing" }
if (Test-LpReparsePoint $p) { Stop-Refused "$(Get-LpRelative $p) is a link" }
}
$rec = $null
try { $rec = [System.IO.File]::ReadAllText($recordPath) | ConvertFrom-Json -AsHashtable; Assert-RecordShape $rec } catch { Stop-Refused "the deployment record is not valid ($($_.Exception.Message))" }
if (-not [string]::Equals([string]$rec.runtimeRoot, $root, [System.StringComparison]::OrdinalIgnoreCase)) { Stop-Refused 'the deployment record belongs to another runtime root' }
if ([string]$rec.deploymentId -cne [System.IO.Path]::GetFileName($backup)) { Stop-Refused 'the deployment record does not belong to this backup folder' }
$releasesDir = Join-Path $root 'releases'
foreach ($side in 'previous', 'candidate') {
$pp = [string]$rec[$side].proxyPath
$relDir = [System.IO.Path]::GetDirectoryName([System.IO.Path]::GetDirectoryName($pp))
if ([System.IO.Path]::GetFullPath($pp) -ne $pp -or [System.IO.Path]::GetDirectoryName($relDir) -ne $releasesDir -or [System.IO.Path]::GetFileName($pp) -ne 'index.mjs') { Stop-Refused "the $side release path in the record is not <runtime root>\releases\<name>\dist\index.mjs" }
}
if (-not (Test-Path -LiteralPath ([string]$rec.previous.proxyPath) -PathType Leaf)) { Stop-Refused 'the previous release bundle is missing' }

$ctx = @{
Root = $root; Port = [int]$rec.port; TaskName = [string]$rec.taskName; HealthWait = $HealthWaitSeconds; StopWait = $StopWaitSeconds
LauncherPath = Join-Path $root 'Start-BillionContextProxy.ps1'; InhibitPath = Join-Path $root 'maintenance.inhibit'
LockPath = Join-Path $root 'supervisor.lock'; StatusPath = Join-Path $root 'supervisor-status.json'
ConfigPath = Join-Path $root 'config\billion-context\billion-context.json'; ConfigSha = $null
Previous = [ordered]@{ proxyPath = [string]$rec.previous.proxyPath; version = [string]$rec.previous.version; commit = [string]$rec.previous.commit }
Candidate = [ordered]@{ releaseName = [string]$rec.candidate.releaseName; proxyPath = [string]$rec.candidate.proxyPath; version = [string]$rec.candidate.version; commit = [string]$rec.candidate.commit }
BackupRoot = $backup; DeploymentId = [string]$rec.deploymentId; Record = $rec; RecordPath = $recordPath; LiftAt = [datetime]::MinValue; TaskBaseline = $null
}

$backupLauncher = [System.IO.File]::ReadAllBytes((Join-Path $backup 'Start-BillionContextProxy.ps1'))
if (-not (Test-LpSameHash (Get-LpSha256OfBytes $backupLauncher) ([string]$rec.launcherSha256Before))) { Stop-Refused 'the launcher in the backup folder is not the one this deployment saved' }
$id = $null
try { $id = Read-LpLauncherIdentity ([System.Text.Encoding]::ASCII.GetString($backupLauncher)) } catch { Stop-Refused "the launcher in the backup folder is not readable ($($_.Exception.Message))" }
if ($id.ProxyPath.Value -cne $ctx.Previous.proxyPath -or $id.ExpectedVersion.Value -cne $ctx.Previous.version -or $id.ExpectedCommit.Value -cne $ctx.Previous.commit -or [int]$id.Port.Value -ne $ctx.Port) { Stop-Refused 'the launcher in the backup folder does not name the previous release recorded for this deployment' }

$ownHold = Test-LpOwnInhibit $ctx
if ((Test-Path -LiteralPath $ctx.InhibitPath) -and -not $ownHold) { Stop-Refused 'maintenance.inhibit is present and belongs to other maintenance' }
$task = Get-LiveProxyTaskInfo $ctx.TaskName
if ($task.Count -ne 1) { Stop-Refused "expected exactly one scheduled task named in the record, found $($task.Count)" }
$actionGap = Get-LpTaskActionGap $task $ctx.LauncherPath
if ($actionGap) { Stop-Refused $actionGap }
$ctx.TaskBaseline = $task

$dp = Get-CimInstance Win32_Process -Filter "ProcessId=$([int]$rec.deployProcess.pid)" -OperationTimeoutSec 15 -ErrorAction Stop
if ($null -ne $dp -and [Math]::Abs(($dp.CreationDate.ToUniversalTime() - ([datetime]$rec.deployProcess.created).ToUniversalTime()).TotalSeconds) -lt 1) { Stop-Refused 'the deploy script for this deployment is still running' }

$liveSha = Get-LpSha256 $ctx.LauncherPath
$isBefore = Test-LpSameHash $liveSha ([string]$rec.launcherSha256Before)
$isAfter = Test-LpSameHash $liveSha ([string]$rec.launcherSha256After)
$stage = [string]$rec.stage
if (-not $isBefore -and -not $isAfter) { Stop-Refused 'the launcher was changed after this deployment; this is not the current deployment''s backup' }
if ($stage -eq 'rolled-back') { Stop-Refused 'this deployment was already rolled back' }
if (@('prepared', 'aborted') -contains $stage) { Stop-Refused "this deployment stopped at stage '$stage' and changed nothing" }
# A disabled task is this deployment's doing only if its own hold is in place or it stopped inside the hold window.
if (-not $task.Enabled -and -not $ownHold -and @('holding', 'held', 'cut', 'rolling-back', 'rollback-failed') -notcontains $stage) { Stop-Refused "the scheduled task is disabled and neither this deployment's hold nor its stage '$stage' explains it; other maintenance may be in progress" }

$ctx.ConfigSha = Get-LpSha256 $ctx.ConfigPath
$receipts = @{}
foreach ($name in 'tuned-deployment.json', 'supervision-receipt.json') { $receipts[(Join-Path $root $name)] = [System.IO.File]::ReadAllBytes((Join-Path $backup $name)) }
$clients = Get-LpClientCount $ctx
Write-LpEvent 'STEP' "deployment stage '$stage'; launcher currently names the $(if ($isAfter -and -not $isBefore) { 'deployed' } else { 'previous' }) release; open client connections: $clients"
$plan = "hold supervision, stop any process of release $($ctx.Candidate.releaseName), restore the launcher and receipts where they differ from the backup, let the supervisor start $($ctx.Previous.version), $($ctx.Previous.commit) and check it"
if ($PreflightOnly) {
Write-LpEvent 'STEP' "would do: $plan"
throw [LpStop]::new(0, 'preflight-ok', 'rollback preflight passed; nothing was changed', '', 'run again without -PreflightOnly to roll back')
}
# Journal the rollback before anything is changed, so that an interrupted run can be recognised and retried.
try { Set-LpStage $ctx 'rolling-back' } catch { Stop-Refused "the deployment record cannot be written ($($_.Exception.Message))" }
} catch [LpStop] {
Complete-Lp $null $_.Exception $null
} catch {
Complete-Lp $null ([LpStop]::new(2, 'refused', "refused: $($_.Exception.Message); nothing was changed", '', 'fix the problem above and run again')) $null
}

try {
Invoke-LpRollback $ctx $backupLauncher $receipts $false

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(ai: kilo/openai/gpt-6-astra)

WARNING: An interrupted standalone rollback cannot resume

Starting from healthy or receipts-written, terminate this rollback after Remove-LpHold removes the inhibit but before it enables the task (LiveProxy.Common.ps1:351-353). The candidate has been stopped, the task remains disabled, and the record still has the original completed-deploy stage because this script journals nothing until line 139. A retry is then refused by line 119 as unexplained foreign maintenance, leaving the service down. Persist a rollback-in-progress stage before mutation and recognize it during recovery; add the standalone equivalent of the existing deploy-killed-during-enable test.

# The previous release is live and checked from here on: a bookkeeping error must not read as a failed rollback.
$note = ''
try { Set-LpStage $ctx 'rolled-back' } catch { $note = ' The deployment record could not be updated; running this rollback again is harmless.' }
Complete-Lp $ctx ([LpStop]::new(0, 'restored', "the previous release $($ctx.Previous.version), $($ctx.Previous.commit) is live under supervision; receipts describe it.$note", '', 'report the failure, the release left live and the backup location (step 10). A later deployment needs a new -ReleaseName.')) 'rollback-result.json'
} catch [LpStop] {
Complete-Lp $ctx $_.Exception 'rollback-result.json'
} catch {
# Exit 4 always means the hold was re-applied and evidence was collected.
try { Stop-LpRollbackFailed $ctx "unexpected error ($($_.Exception.Message))" 'rollback-error' } catch [LpStop] { Complete-Lp $ctx $_.Exception 'rollback-result.json' }
}
5 changes: 5 additions & 0 deletions scripts/live-files.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,11 @@ export const LIVE_FILES = [
"tests/ci-stall-watch.test.ts",
// #69: runs copies of run-ci-tests.mjs as subprocesses; no ports.
"tests/ci-runner-stall-gate.test.ts",
// #80: runs the live-proxy deploy and rollback scripts against throw-away
// sandboxes (Windows only; skipped elsewhere). Every sandbox has its own
// temp folder, a listen(0)-probed port and a file-backed stand-in for the
// scheduled task, so no zone band and no machine-global state is used.
"tests/live-deploy-scripts.test.ts",
];

/** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry;
Expand Down
110 changes: 110 additions & 0 deletions tests/fixtures/live-deploy/Start-BillionContextProxy.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
param(
[string]$RuntimeRoot = (Split-Path -Parent $PSCommandPath),
[string]$NodePath = '__NODE__',
[string]$ProxyPath = '__PROXY__',
[int]$Port = 0,
[string]$ExpectedVersion = '__VERSION__',
[string]$ExpectedCommit = '__COMMIT__',
[double]$PollSeconds = 0.3,
[double]$HeartbeatSeconds = 0.4,
[switch]$ReportOnly
)
# TEST STAND-IN for the live proxy supervisor. Written for tests/live-deploy-scripts.test.ts only; it is
# not the real launcher and is never installed. It honours the same contract the deploy scripts rely on:
# one supervisor per runtime root (exclusive lock), maintenance.inhibit suppresses adopt/start/restart,
# adopt a verified listener, start a proxy only when none matches and the port is free, otherwise do
# nothing, and report through supervisor-status.json. Runs under Windows PowerShell.
$ErrorActionPreference = 'Stop'
$inhibitPath = Join-Path $RuntimeRoot 'maintenance.inhibit'
$lockPath = Join-Path $RuntimeRoot 'supervisor.lock'
$statusPath = Join-Path $RuntimeRoot 'supervisor-status.json'
$configPath = Join-Path $RuntimeRoot 'config\billion-context\billion-context.json'
$proxyArgs = '"' + $ProxyPath + '" start --port ' + $Port + ' --host 127.0.0.1 --config "' + $configPath + '"'
$expectedCommandLine = '"' + $NodePath + '" ' + $proxyArgs
$started = (Get-Date).ToString('o')

function Get-Owners {
return @(Get-CimInstance -Namespace ROOT/StandardCimv2 -ClassName MSFT_NetTCPConnection -Filter "LocalPort=$Port AND State=2" -ErrorAction Stop | Select-Object -ExpandProperty OwningProcess -Unique)
}
function Get-Matching {
return @(Get-CimInstance Win32_Process -Filter "Name='node.exe'" -ErrorAction Stop | Where-Object { $null -ne $_.CommandLine -and $_.CommandLine.Trim() -ieq $expectedCommandLine })
}
function Test-Health([int]$expectPid) {
try {
$req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$Port/__bili/health")
$req.Proxy = $null; $req.KeepAlive = $false; $req.Timeout = 2000
$resp = $req.GetResponse()
try { $h = (New-Object System.IO.StreamReader($resp.GetResponseStream())).ReadToEnd() | ConvertFrom-Json } finally { $resp.Close() }
return (($h.ok -eq $true) -and ([string]$h.version -eq $ExpectedVersion) -and ([string]$h.commit -eq $ExpectedCommit) -and ([string]$h.pid -eq [string]$expectPid))
} catch { return $false }
}
function Resolve-Decision {
if (Test-Path -LiteralPath $inhibitPath) { return @{ action = 'inhibited' } }
if (!(Test-Path -LiteralPath $NodePath) -or !(Test-Path -LiteralPath $ProxyPath)) { return @{ action = 'blocked' } }
$owners = @(Get-Owners)
$candidates = @(Get-Matching)
if ($owners.Count -eq 0) {
if ($candidates.Count -eq 0) { return @{ action = 'spawn' } }
return @{ action = 'waiting' }
}
if ($owners.Count -ne 1) { return @{ action = 'conflict' } }
$owner = [int]$owners[0]
$match = @($candidates | Where-Object { $_.ProcessId -eq $owner })
if ($match.Count -ne 1 -or $candidates.Count -ne 1) { return @{ action = 'conflict' } }
if (!(Test-Health $owner)) { return @{ action = 'conflict' } }
return @{ action = 'adopt'; pid = $owner }
}
function Write-Status([hashtable]$s) {
try {
$s.supervisorPid = $PID; $s.supervisorStarted = $started; $s.heartbeatAt = (Get-Date).ToString('o')
$tmp = "$statusPath.tmp"
[System.IO.File]::WriteAllText($tmp, ($s | ConvertTo-Json -Compress))
if (Test-Path -LiteralPath $statusPath) { [System.IO.File]::Replace($tmp, $statusPath, [NullString]::Value) } else { [System.IO.File]::Move($tmp, $statusPath) }
} catch { }
}
# Test control: <runtime root>\status-control.json = { "<commit>": { "healthMatch": false, "childPid": 1 } }
# overrides what this supervisor reports for that release; the proxy and its health are not affected.
function Get-StatusOverride {
try {
$c = [System.IO.File]::ReadAllText((Join-Path $RuntimeRoot 'status-control.json')) | ConvertFrom-Json
return $c.$ExpectedCommit
} catch { return $null }
}
function Watch-Child($proc, [string]$mode) {
$created = $proc.StartTime.ToString('o')
while (!$proc.HasExited) {
$s = @{ state = $mode; childPid = $proc.Id; childCreated = $created; healthMatch = (Test-Health $proc.Id); inhibit = (Test-Path -LiteralPath $inhibitPath) }
$o = Get-StatusOverride
if ($null -ne $o) { foreach ($p in $o.PSObject.Properties) { $s[$p.Name] = $p.Value } }
Write-Status $s
Start-Sleep -Milliseconds ([int]($HeartbeatSeconds * 1000))
}
}

if ($ReportOnly) {
try { Resolve-Decision | ConvertTo-Json -Compress; exit 0 } catch { @{ action = 'error' } | ConvertTo-Json -Compress; exit 2 }
}

try { $lockStream = [System.IO.File]::Open($lockPath, 'OpenOrCreate', 'ReadWrite', 'None') } catch { exit 0 }
Add-Content -LiteralPath (Join-Path $RuntimeRoot 'sandbox-pids.log') -Value "$PID"

while ($true) {
try {
$d = Resolve-Decision
if ($d.action -eq 'adopt') {
$proc = [System.Diagnostics.Process]::GetProcessById([int]$d.pid)
if (!(Test-Path -LiteralPath $inhibitPath)) { Watch-Child $proc 'adopted'; continue }
} elseif ($d.action -eq 'spawn') {
if (!(Test-Path -LiteralPath $inhibitPath) -and @(Get-Owners).Count -eq 0 -and @(Get-Matching).Count -eq 0) {
$proc = Start-Process -FilePath $NodePath -ArgumentList $proxyArgs -WindowStyle Hidden -PassThru -WorkingDirectory $RuntimeRoot
$null = $proc.Handle
Watch-Child $proc 'spawned'
continue
}
}
Write-Status @{ state = $d.action }
} catch {
Write-Status @{ state = 'error' }
}
Start-Sleep -Milliseconds ([int]($PollSeconds * 1000))
}
Loading
Loading