From b8cbdadd1296099c2cd529f41c7f6ab5e5054296 Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Tue, 17 Nov 2026 23:39:36 -0500 Subject: [PATCH 1/5] feat(tooling): maintained live-proxy deploy and rollback scripts (#80) Add scripts/live-deploy with Deploy-LiveProxy.ps1 and Rollback-LiveProxy.ps1, which follow the documented cutover order: preflight, hold supervision, edit the launcher, let the supervisor start the new release, verify, write receipts. A failed deployment rolls back; a failed rollback puts the hold back, leaves the proxy as it is and exits 4. Nothing about a release is hardcoded: identity, port, task name and the expected config hash are parameters, and the rollback reads the deployment record written by the deploy. The scripts are exercised only against throw-away sandboxes (stand-in launcher, stand-in proxy, file-backed task driver behind a marker-gated seam that is refused inside the live installation area). The test is registered as a process-spawning suite and weighted for sharding. Docs: reference/cindy-project (en + zh-CN) describe the scripts, their step coverage and exit codes; reference/testing.md names the right registration file. Assisted-by: Cindy (https://cindy.app) --- reference/cindy-project.md | 55 +- reference/cindy-project.zh-CN.md | 55 +- reference/testing.md | 2 +- scripts/ci-shard-weights.json | 3 +- scripts/live-deploy/Deploy-LiveProxy.ps1 | 381 +++++++ scripts/live-deploy/LiveProxy.Common.ps1 | 528 ++++++++++ scripts/live-deploy/Rollback-LiveProxy.ps1 | 143 +++ scripts/live-files.mjs | 5 + .../live-deploy/Start-BillionContextProxy.ps1 | 99 ++ tests/fixtures/live-deploy/TestTaskDriver.ps1 | 67 ++ tests/fixtures/live-deploy/stand-in-proxy.mjs | 50 + tests/live-deploy-scripts.test.ts | 970 ++++++++++++++++++ 12 files changed, 2344 insertions(+), 14 deletions(-) create mode 100644 scripts/live-deploy/Deploy-LiveProxy.ps1 create mode 100644 scripts/live-deploy/LiveProxy.Common.ps1 create mode 100644 scripts/live-deploy/Rollback-LiveProxy.ps1 create mode 100644 tests/fixtures/live-deploy/Start-BillionContextProxy.ps1 create mode 100644 tests/fixtures/live-deploy/TestTaskDriver.ps1 create mode 100644 tests/fixtures/live-deploy/stand-in-proxy.mjs create mode 100644 tests/live-deploy-scripts.test.ts diff --git a/reference/cindy-project.md b/reference/cindy-project.md index 5cb9c7c8a8..ccd81f3e18 100644 --- a/reference/cindy-project.md +++ b/reference/cindy-project.md @@ -88,13 +88,56 @@ The receipt identifies the active release, prior release, bundle hash and backup 2. Verify the existing supervisor, configuration hash, listener PID/process identity and idle state. Prepare a new release directory and back up the supervisor and deployment receipt. Preserve authentication, settings, originals and the scheduled task's sign-in/recovery ownership. 3. Build the live bundle from the small local start-up wrapper `live-entry.ts` with the kept esbuild script `build.mjs`, not from the standard CLI entry; building the standard CLI alone does not capture this deployment's runtime policy. Neither file is in the repository: both are in the kept tooling directory named below. Copy the wrapper to `src/live-entry.ts` in a checkout of the pinned source SHA and run `build.mjs` from that checkout's root with the full source SHA as its argument; do not commit the copied files. The wrapper requires an explicit `--port` (it rejects a non-integer value and ports below 1024 or above 65535), then calls `startServer({ ...loadOptions(), host: "127.0.0.1", port, passthrough: false, autoUpdate: false, autoRestartOnUpdate: false }, { boundedMaintenance: { timeoutMs: 15000, maxTokens: 1024 } })` and logs the enabled bounded maintenance. `build.mjs` bundles `src/live-entry.ts` → `out/dist/index.mjs` (bundle/esm/node20, alias `@earendil-works/pi-coding-agent` → `src/agent/pi-host-stub.ts`, external `jimp, mysql2, mysql2/promise, bufferutil, utf-8-validate`, createRequire banner) and writes `out/dist/build-info.json` with `sourceCommit`, `bundleSha256`, `commit` (the first 8 chars plus `-live`) and `localPatchSha256`; record those hashes in the receipt. The candidate folder handed to the cutover holds `dist\index.mjs`, `dist\build-info.json`, `package.json` and `LICENSE`. 4. Hold supervision before editing the launcher, in this order: create `maintenance.inhibit` in the runtime root, which suppresses supervisor adoption, start and restart; disable the scheduled task, whose repeating trigger would otherwise relaunch the supervisor; then stop the supervisor. A monitored child is left running while inhibited, so the old proxy keeps serving: confirm it is still healthy. Editing launcher defaults under a live supervisor is unsafe: adoption is by PID plus process creation time, rechecked after a handle is held, and one supervisor per installation is held by an OS-exclusive lock on `supervisor.lock`, so a second launch exits 0 immediately. The hold is temporary: step 6 lifts it at cutover, and step 8 lifts it again after a rollback. -5. Change only the launcher defaults that identify the release, and nothing else in the launcher: `ProxyPath` and `ExpectedCommit`, plus `ExpectedVersion` when the version changes. They must match the new release, otherwise the supervisor fails closed. Take the new values from the candidate, not from this page: `ProxyPath` is `dist\index.mjs` in the new release directory, `ExpectedCommit` is `commit` in the candidate's `dist\build-info.json`, and `ExpectedVersion` is the version its health endpoint reported in the step 1 isolated test. The values of the release being replaced are the launcher's current defaults; the receipts record the same release in `releaseRoot` and `build` of `tuned-deployment.json` and in `masterDeploy.proxyRelease` of `supervision-receipt.json`, which also carries the version. The kept `Deploy-Master.ps1` rewrites exactly the `ProxyPath` and `ExpectedCommit` lines and stops if its edit would touch any other line, and its health check has one fixed version written into it, so it does not cover a version change. The other defaults stay as they are: `Port` = `8787`, `NodePath` = `C:\Program Files\nodejs\node.exe`, `RestartDelaySeconds` 5, `PollSeconds` 5, `HeartbeatSeconds` 15 and a `-ReportOnly` switch. -6. Cut over only when idle: if any client connection to the live port is open, stand down instead of waiting for requests to drain. Stop only the old proxy, wait for the port to be released, then remove `maintenance.inhibit`, enable the scheduled task and start it. The supervisor then starts the new release itself, because no matching proxy exists and nothing listens on the port; do not start a proxy by hand. With supervision running, expect the supervisor to fail closed on a foreign or unverified listener, unreadable/booting/mismatched health, a matching process that is not listening, missing executables, and inspection errors or timeouts: nothing is killed and nothing is started, and state is rechecked every `PollSeconds` and logged only when it changes. The cutover has succeeded only when health reports the expected version and commit from the single listener's PID, with exactly one proxy process of the new release and exactly one supervisor; the kept script waits up to 90 seconds for the health result and then checks the process counts once. Otherwise take the rollback branch in step 8. -7. With supervision running, check the manifest's source identity, single-listener ownership and unchanged configuration hash. Exercise request forwarding, actual folding, exact original text/image retrieval and session-scoped checker logs. Use one small authenticated provider request when authorized. The kept script checks only the health identity and process counts of step 6, then rewrites the receipts and writes `Rollback-MasterDeploy.ps1`, so these checks run after it has finished, and a failure here is rolled back with the receipts already rewritten. -8. If the cutover or these checks fail, roll back under a new hold: create `maintenance.inhibit`, disable the scheduled task, stop the supervisor and then the candidate proxy if one was started. Restore the saved launcher, and restore both receipts if they were already rewritten: a failure inside the cutover leaves the receipts untouched, a failure in step 7 after the kept script has finished does not, and `Rollback-MasterDeploy.ps1` in that cutover's backup restores all three files. Editing or restoring the launcher under a live supervisor is unsafe for the same reason as in step 4. Then lift the hold as in step 6, so that the supervisor starts the previous release, and repeat the step 6 success test and the step 7 checks against it. If the restored release fails them too, the rollback has failed: put the hold back (inhibit set, task disabled, supervisor stopped), make no further start, restore or edit attempt, and report to the owner with the evidence: the failed checks, `supervisor.log`, `bili.log` and what, if anything, holds the port. The live proxy then stays unsupervised or down until the owner decides; that is the intended end state, so that nothing restarts a failing proxy in a loop. +5. Change only the launcher defaults that identify the release, and nothing else in the launcher: `ProxyPath` and `ExpectedCommit`, plus `ExpectedVersion` when the version changes. They must match the new release, otherwise the supervisor fails closed. Take the new values from the candidate, not from this page: `ProxyPath` is `dist\index.mjs` in the new release directory, `ExpectedCommit` is `commit` in the candidate's `dist\build-info.json`, and `ExpectedVersion` is the version its health endpoint reported in the step 1 isolated test. The values of the release being replaced are the launcher's current defaults; the receipts record the same release in `releaseRoot` and `build` of `tuned-deployment.json` and in `masterDeploy.proxyRelease` of `supervision-receipt.json`, which also carries the version. `Deploy-LiveProxy.ps1` takes the candidate's version, commit, source commit and bundle hash as parameters and refuses a candidate that does not match them; it rewrites only those of the three lines whose value changes and stops before the hold if its edit would touch any other line. The other defaults stay as they are: `Port` = `8787`, `NodePath` = `C:\Program Files\nodejs\node.exe`, `RestartDelaySeconds` 5, `PollSeconds` 5, `HeartbeatSeconds` 15 and a `-ReportOnly` switch. +6. Cut over only when idle: if any client connection to the live port is open, stand down instead of waiting for requests to drain. The script counts a loopback connection as a client unless its owning process is the script itself or the supervisor, whose health probes are not clients. Stop only the old proxy, wait for the port to be released, then remove `maintenance.inhibit`, enable the scheduled task and start it. The supervisor then starts the new release itself, because no matching proxy exists and nothing listens on the port; do not start a proxy by hand. With supervision running, expect the supervisor to fail closed on a foreign or unverified listener, unreadable/booting/mismatched health, a matching process that is not listening, missing executables, and inspection errors or timeouts: nothing is killed and nothing is started, and state is rechecked every `PollSeconds` and logged only when it changes. The cutover has succeeded only when health reports the expected version and commit from the single listener's PID, with exactly one proxy process of the new release and exactly one supervisor; `Deploy-LiveProxy.ps1` waits up to `-HealthWaitSeconds` (90 by default) for all of this together with the supervisor's own report described in step 9. Otherwise take the rollback branch in step 8. +7. With supervision running, check the manifest's source identity, single-listener ownership and unchanged configuration hash. Exercise request forwarding, actual folding, exact original text/image retrieval and session-scoped checker logs. Use one small authenticated provider request when authorized. `Deploy-LiveProxy.ps1` checks only the success test of step 6 and the supervision state of step 9, then rewrites the receipts and reports `deployed`, so these checks run after it has finished, and a failure here is rolled back with `Rollback-LiveProxy.ps1`, with the receipts already rewritten. +8. If the cutover or these checks fail, roll back under a new hold: create `maintenance.inhibit`, disable the scheduled task, stop the supervisor and then the candidate proxy if one was started. Restore the saved launcher, and restore both receipts if they were already rewritten: a failure inside the cutover leaves the receipts untouched, a failure in step 7 after the deploy script has finished does not, and `Rollback-LiveProxy.ps1`, given that deployment's backup folder, restores all three files. Editing or restoring the launcher under a live supervisor is unsafe for the same reason as in step 4. Then lift the hold as in step 6, so that the supervisor starts the previous release, and repeat the step 6 success test and the step 7 checks against it. If the restored release fails them too, the rollback has failed: put the hold back (inhibit set, task disabled, supervisor stopped), make no further start, restore or edit attempt, and report to the owner with the evidence: the failed checks, `supervisor.log`, `bili.log` and what, if anything, holds the port. The live proxy then stays unsupervised or down until the owner decides; that is the intended end state, so that nothing restarts a failing proxy in a loop. Both scripts apply it themselves, leave any proxy process as it is, save the evidence in the backup folder and exit with code 4. 9. Confirm that maintenance has ended, on the success path and the rollback path alike: `maintenance.inhibit` is absent, the scheduled task is enabled and exactly one supervisor is running. While that file exists the supervisor refuses adoption, start and restart, so a deployment that leaves it behind has no automatic adoption or restart even when every check passed. Verify the supervisor is monitoring the release now live, candidate or restored: `supervisor-status.json` shows a monitored child with `healthMatch` true and no inhibit, its `childPid` is the single listener's PID, and the health endpoint reports the expected version and commit from that PID. The only exception is the failed rollback in step 8, which deliberately ends with the hold in place. 10. On the success path, finalize the deployment receipt and report the deployed SHA, results and rollback location. On the rollback path nothing was deployed: confirm the receipts describe the restored release, and report the failure, the release left live and the backup location. After a failed rollback, report that no release is confirmed live and that the hold is in place. A local proxy deployment does not publish npm packages or update the hosting Cindy application. -Each master cutover keeps its tooling in its own backup: the `tooling\` directory under the backup root that the deployment receipt `tuned-deployment.json` names in `masterBackupRoot`, currently `%APPDATA%\CindyGlobal\billion-context-proxy\backups\before-master-20261114-214204`. It contains exactly `Deploy-Master.ps1`, `build.mjs`, `fold-test.mjs`, `iso-test.mjs` and `live-entry.ts`; the earlier `before-master-20261114-191918` backup holds a byte-identical copy. By step: `iso-test.mjs` and `fold-test.mjs` serve step 1, `live-entry.ts` and `build.mjs` step 3, and `Deploy-Master.ps1` automates the preflight, release directory and backup of step 2, the hold and launcher edit of steps 4–5, the cutover of step 6, the launcher-only rollback of step 8 when the cutover fails, and, straight after the cutover, the receipt rewrite that step 7 describes. Beside `tooling\`, the backup root holds the pre-cutover `Start-BillionContextProxy.ps1`, `proxy.identity.json`, `supervision-receipt.json`, `supervisor-status.json` and `tuned-deployment.json`, and `Rollback-MasterDeploy.ps1`, which undoes that one completed cutover: it restores the launcher and both receipts from that backup and restarts the previous release through the scheduled task. Each backup's rollback script targets its own cutover, so use the one in the backup the receipt names. The live launcher is `Start-BillionContextProxy.ps1` in the runtime root. `Rollback-TunedLiveProxy.ps1` in the same root is a different, deeper rollback: it restores the configuration and the launcher from the receipt's `backupRoot`, the backup taken before the first tuned deployment on 2026-11-07, not the previous release. +The deploy and rollback scripts are in the repository, in `scripts/live-deploy/`, and are described below. The build and isolated-test tooling is not: each master cutover kept it in its own backup, in the `tooling\` directory under the backup root that the deployment receipt `tuned-deployment.json` names in `masterBackupRoot`, currently `%APPDATA%\CindyGlobal\billion-context-proxy\backups\before-master-20261114-214204`. It contains exactly `Deploy-Master.ps1`, `build.mjs`, `fold-test.mjs`, `iso-test.mjs` and `live-entry.ts`; the earlier `before-master-20261114-191918` backup holds a byte-identical copy. By step: `iso-test.mjs` and `fold-test.mjs` serve step 1, `live-entry.ts` and `build.mjs` step 3, and `Deploy-Master.ps1` is the one-off script that performed the 2026-11-14 cutovers, kept as history and superseded by `Deploy-LiveProxy.ps1`. Pass the tooling directory to the deploy script as `-ToolingDir`, so that the next backup keeps a copy. Beside `tooling\`, the backup root holds the pre-cutover `Start-BillionContextProxy.ps1`, `proxy.identity.json`, `supervision-receipt.json`, `supervisor-status.json` and `tuned-deployment.json`, and `Rollback-MasterDeploy.ps1`, the rollback text that `Deploy-Master.ps1` generated for that one completed cutover: it restores the launcher and both receipts from that backup and restarts the previous release through the scheduled task. A backup made by `Deploy-LiveProxy.ps1` holds `deploy-record.json` instead, which is what `Rollback-LiveProxy.ps1` reads; that script cannot roll back a cutover made by the old one. The live launcher is `Start-BillionContextProxy.ps1` in the runtime root. `Rollback-TunedLiveProxy.ps1` in the same root is a different, deeper rollback: it restores the configuration and the launcher from the receipt's `backupRoot`, the backup taken before the first tuned deployment on 2026-11-07, not the previous release. -Steps 4–9 follow the order the kept `Deploy-Master.ps1` implements: the supervisor, not the operator, starts the new release, and health is checked with supervision running. That order is the sanctioned one by owner decision of 2026-11-16 ([#78](https://github.com/anuzzolilli-oss/billion-context/issues/78)); the earlier wording, which started and checked the new proxy by hand under the inhibit, is superseded, and with it the question of how a hand-started proxy comes under supervision. The kept scripts still depart from the steps when a rollback fails. The failure branch of `Deploy-Master.ps1` and `Rollback-MasterDeploy.ps1` both remove the inhibit and start the scheduled task without requiring the previous release to be healthy (the first only reports the result, the second does not check), so after either one, do the step 8 checks yourself and put the hold back by hand if they fail. The script rewrites the receipts and writes `Rollback-MasterDeploy.ps1` only after a successful cutover. Inspect the kept tooling before reuse: it contains pinned build hashes and previous-build assumptions. Do not execute an old installer against a different current release without rebuilding its identity and rollback checks. +Steps 4–9 follow the order the kept `Deploy-Master.ps1` implements: the supervisor, not the operator, starts the new release, and health is checked with supervision running. That order is the sanctioned one by owner decision of 2026-11-16 ([#78](https://github.com/anuzzolilli-oss/billion-context/issues/78)); the earlier wording, which started and checked the new proxy by hand under the inhibit, is superseded, and with it the question of how a hand-started proxy comes under supervision. The repository scripts implement that order, including the failed-rollback end state of step 8 ([#80](https://github.com/anuzzolilli-oss/billion-context/issues/80)). The kept `Deploy-Master.ps1` and `Rollback-MasterDeploy.ps1` do not: both remove the inhibit and start the scheduled task without requiring the previous release to be healthy (the first only reports the result, the second does not check), and they contain pinned build hashes and previous-build assumptions. Do not run `Deploy-Master.ps1` for a new deployment. If `Rollback-MasterDeploy.ps1` has to be used to undo the 2026-11-14 cutover, do the step 8 checks yourself and put the hold back by hand if they fail. + +### Deploy and rollback scripts + +`scripts/live-deploy/Deploy-LiveProxy.ps1` and `scripts/live-deploy/Rollback-LiveProxy.ps1` need PowerShell 7 and take every installation-specific value as a parameter; nothing about a release, the port or the task is written into them. Run them non-interactively, and run the deploy script with `-PreflightOnly` first: it performs every check, prints the plan and changes nothing. + +```powershell +pwsh -NoProfile -NonInteractive -File scripts\live-deploy\Deploy-LiveProxy.ps1 ` + -RuntimeRoot "" -Port -TaskName "" ` + -CandidateRoot "" -ReleaseName "" ` + -CandidateVersion "" -CandidateCommit "" ` + -CandidateSourceCommit "" -CandidateBundleSha256 "" ` + -ExpectedConfigSha256 "" -ToolingDir "" -PreflightOnly + +pwsh -NoProfile -NonInteractive -File scripts\live-deploy\Rollback-LiveProxy.ps1 ` + -RuntimeRoot "" -BackupRoot "\backups\" -PreflightOnly +``` + +Take the four candidate values from the record of step 1, not from the candidate folder, so that the script proves the folder holds the artifact that was tested. The expected configuration hash is the operator's too: `deployedConfigSha256` in `tuned-deployment.json` is not consulted and not rewritten. + +| Step | Covered by | +|---|---| +| 1, 3 | By hand, with the kept build and test tooling | +| 2, 4, 5, 6 | `Deploy-LiveProxy.ps1` | +| 7 | Receipt rewrite: `Deploy-LiveProxy.ps1`. Functional checks: by hand, after it reports `deployed` | +| 8 | During the cutover: `Deploy-LiveProxy.ps1`. After a failed step 7: `Rollback-LiveProxy.ps1` | +| 9 | Both scripts, as part of their success test | +| 10 | Receipts and the printed result: the scripts. The report to the owner: the operator | + +Each script prints one line per event and ends with a `NEXT` line and a `RESULT` line; the deploy script prints the backup folder on a `BACKUP` line and keeps `deploy-record.json` and `result.json` there. + +| Exit code | Result | State left behind | What to do | +|---|---|---|---| +| 0 | `deployed` | New release healthy under supervision | Run the step 7 checks; the rollback command is on the `NEXT` line | +| 0 | `restored` (rollback script) | Previous release healthy under supervision | Report as in step 10; a later deployment needs a new release name | +| 0 | `preflight-ok` | Nothing changed | Run again without `-PreflightOnly` | +| 2 | `refused` | Nothing changed | Fix the failing checks | +| 3 | `rolled-back` | Previous release healthy under supervision; the new release directory is removed | `reason=not-idle`: retry later. `hold-failed`, `cutover-failed`: investigate first | +| 4 | `rollback-failed` | Hold in place; proxy processes as they were; no release confirmed live | Report to the owner with the evidence file and the two logs; retry with `Rollback-LiveProxy.ps1` once the cause is fixed | +| 5 | `deployed-receipts-incomplete` | New release healthy under supervision; a receipt was not replaced | Copy the pending receipt files named on the `NEXT` line, then run the step 7 checks | +| 6 | `error-before-hold` | Nothing running was changed; the new release directory is removed | Fix the problem and run again | +| other | none printed | Unknown, for example a killed script | Read `deploy-record.json`, then run `Rollback-LiveProxy.ps1 -PreflightOnly` | + +The scripts are tested against a sandbox, never against the live proxy: `tests/live-deploy-scripts.test.ts` builds a temporary runtime root with a stand-in launcher and proxy and a test driver in place of the scheduled task. The scripts accept that driver only for a runtime root that carries `sandbox.marker`, refuse the real scheduled task for such a root, and refuse the driver for any root in the live installation area. diff --git a/reference/cindy-project.zh-CN.md b/reference/cindy-project.zh-CN.md index 8191093bf4..cf41d4ad6c 100644 --- a/reference/cindy-project.zh-CN.md +++ b/reference/cindy-project.zh-CN.md @@ -88,13 +88,56 @@ Get-Content (Join-Path $proxyRoot 'tuned-deployment.json') 2. 验证现有 supervisor、配置哈希、监听 PID/进程身份及空闲状态。准备新的 release 目录,备份 supervisor 和部署回执;保留认证、设置、原文及计划任务的登录启动/恢复职责。 3. 运行版本的 bundle 由小型本地启动 wrapper `live-entry.ts` 配合保留的 esbuild 脚本 `build.mjs` 构建,而非标准 CLI 入口;只构建标准 CLI 无法体现该部署的运行时策略。这两个文件都不在仓库中:它们位于下文所述的保留工具目录。把 wrapper 复制到已固定源码 SHA 的检出目录中的 `src/live-entry.ts`,在该检出目录的根目录运行 `build.mjs`,并以完整源码 SHA 作为参数;不得提交复制进来的文件。该 wrapper 要求显式 `--port`(拒绝非整数以及小于 1024 或大于 65535 的端口),随后调用 `startServer({ ...loadOptions(), host: "127.0.0.1", port, passthrough: false, autoUpdate: false, autoRestartOnUpdate: false }, { boundedMaintenance: { timeoutMs: 15000, maxTokens: 1024 } })`,并记录已启用的 bounded maintenance。`build.mjs` 打包 `src/live-entry.ts` → `out/dist/index.mjs`(bundle/esm/node20,别名 `@earendil-works/pi-coding-agent` → `src/agent/pi-host-stub.ts`,external 为 `jimp, mysql2, mysql2/promise, bufferutil, utf-8-validate`,带 createRequire banner),并写出 `out/dist/build-info.json`,内含 `sourceCommit`、`bundleSha256`、`commit`(前 8 位加 `-live`)与 `localPatchSha256`;把这些哈希记入回执。交给切换步骤的候选目录包含 `dist\index.mjs`、`dist\build-info.json`、`package.json` 和 `LICENSE`。 4. 编辑 launcher 前必须先暂停监管,顺序如下:在运行根目录创建 `maintenance.inhibit`,该文件会抑制 supervisor 的接管、启动与重启;禁用计划任务,否则其重复触发器会重新拉起 supervisor;然后停止 supervisor。被抑制期间已监控的子进程继续运行,因此旧代理仍在服务:确认它仍然健康。在 supervisor 存活时修改 launcher 默认值不安全:接管按 PID 加进程创建时间进行,并在持有句柄后复核;每个安装只允许一个 supervisor,由 `supervisor.lock` 上的操作系统排他锁保持,因此第二次启动会立即以 0 退出。这一暂停是临时的:步骤 6 在切换时解除,回滚后由步骤 8 再次解除。 -5. 仅更改 launcher 中标识 release 的默认值,launcher 的其他内容一律不动:`ProxyPath` 和 `ExpectedCommit`,版本号变化时再加上 `ExpectedVersion`。它们必须与新 release 一致,否则 supervisor 会失败关闭(fail closed)。新值取自候选产物,而非本页:`ProxyPath` 是新 release 目录中的 `dist\index.mjs`,`ExpectedCommit` 是候选产物 `dist\build-info.json` 中的 `commit`,`ExpectedVersion` 是步骤 1 隔离测试中其健康端点报告的版本号。被替换 release 的值即 launcher 当前的默认值;回执也记录同一 release:`tuned-deployment.json` 的 `releaseRoot` 与 `build`,以及 `supervision-receipt.json` 的 `masterDeploy.proxyRelease`(其中还含版本号)。保留的 `Deploy-Master.ps1` 只改写 `ProxyPath` 与 `ExpectedCommit` 两行,若改动会触及其他行则停止,且其健康检查中写死了一个固定版本号,因此不覆盖版本号变化的情况。其余默认值保持不变:`Port` = `8787`、`NodePath` = `C:\Program Files\nodejs\node.exe`、`RestartDelaySeconds` 5、`PollSeconds` 5、`HeartbeatSeconds` 15 及 `-ReportOnly` 开关。 -6. 仅在空闲时切换:只要运行端口上还有客户端连接,就中止,而不是等待请求结束。只停止旧代理,等待端口释放,然后删除 `maintenance.inhibit`、启用计划任务并启动它。此时没有匹配的代理且端口无人监听,因此由 supervisor 自行启动新 release;不得手动启动代理。监管运行期间,遇到外来或未经验证的监听者、不可读/正在启动/不匹配的健康结果、匹配但未监听的进程、缺失的可执行文件,以及检查错误或超时,supervisor 会失败关闭:既不终止也不启动任何进程,状态每 `PollSeconds` 复核一次,且仅在变化时记录日志。只有当健康结果由唯一监听者的 PID 报告预期版本号与 commit,且新 release 的代理进程恰好一个、supervisor 恰好一个时,切换才算成功;保留的脚本最多等待 90 秒取得该健康结果,随后检查一次进程数量。否则走步骤 8 的回滚分支。 -7. 在监管运行的状态下,检查 manifest 源码身份、唯一监听者及配置哈希不变。验证转发、实际折叠、原文/图片精确检索及该测试 session 的检查日志。获授权后可发送一个小型认证提供商请求。保留的脚本只检查步骤 6 的健康身份与进程数量,随后即改写回执并写出 `Rollback-MasterDeploy.ps1`,因此这些检查在脚本结束后进行,此处失败时须在回执已被改写的情况下回滚。 -8. 切换或上述检查失败时,在重新暂停监管后回滚:创建 `maintenance.inhibit`、禁用计划任务、停止 supervisor,再停止已启动的候选代理(如有)。恢复保存的 launcher,若回执已被改写则一并恢复两份回执:切换过程中失败时回执尚未改动,保留的脚本结束后在步骤 7 失败时则已改写,该次切换备份中的 `Rollback-MasterDeploy.ps1` 会恢复这三个文件。在 supervisor 存活时修改或恢复 launcher,与步骤 4 同理并不安全。随后按步骤 6 的方式解除暂停,由 supervisor 启动旧 release,并对它重做步骤 6 的成功判定与步骤 7 的检查。若恢复后的 release 同样未通过,即回滚失败:重新暂停监管(inhibit 已设、计划任务已禁用、supervisor 已停止),不再尝试任何启动、恢复或编辑,并携带证据向所有者报告:未通过的检查、`supervisor.log`、`bili.log`,以及端口当前由谁占用(如有)。此后运行中的代理处于无人监管或停止状态,直至所有者决定;这是有意设定的终态,以免失败的代理被反复重启。 +5. 仅更改 launcher 中标识 release 的默认值,launcher 的其他内容一律不动:`ProxyPath` 和 `ExpectedCommit`,版本号变化时再加上 `ExpectedVersion`。它们必须与新 release 一致,否则 supervisor 会失败关闭(fail closed)。新值取自候选产物,而非本页:`ProxyPath` 是新 release 目录中的 `dist\index.mjs`,`ExpectedCommit` 是候选产物 `dist\build-info.json` 中的 `commit`,`ExpectedVersion` 是步骤 1 隔离测试中其健康端点报告的版本号。被替换 release 的值即 launcher 当前的默认值;回执也记录同一 release:`tuned-deployment.json` 的 `releaseRoot` 与 `build`,以及 `supervision-receipt.json` 的 `masterDeploy.proxyRelease`(其中还含版本号)。`Deploy-LiveProxy.ps1` 以参数接收候选产物的版本号、commit、源码 commit 与 bundle 哈希,候选产物与之不符即拒绝;它只改写这三行中取值发生变化的行,若改动会触及其他行则在暂停监管之前停止。其余默认值保持不变:`Port` = `8787`、`NodePath` = `C:\Program Files\nodejs\node.exe`、`RestartDelaySeconds` 5、`PollSeconds` 5、`HeartbeatSeconds` 15 及 `-ReportOnly` 开关。 +6. 仅在空闲时切换:只要运行端口上还有客户端连接,就中止,而不是等待请求结束。脚本把回环连接计为客户端,除非其所属进程是脚本自身或 supervisor,后者的健康探测不算客户端。只停止旧代理,等待端口释放,然后删除 `maintenance.inhibit`、启用计划任务并启动它。此时没有匹配的代理且端口无人监听,因此由 supervisor 自行启动新 release;不得手动启动代理。监管运行期间,遇到外来或未经验证的监听者、不可读/正在启动/不匹配的健康结果、匹配但未监听的进程、缺失的可执行文件,以及检查错误或超时,supervisor 会失败关闭:既不终止也不启动任何进程,状态每 `PollSeconds` 复核一次,且仅在变化时记录日志。只有当健康结果由唯一监听者的 PID 报告预期版本号与 commit,且新 release 的代理进程恰好一个、supervisor 恰好一个时,切换才算成功;`Deploy-LiveProxy.ps1` 最多等待 `-HealthWaitSeconds`(默认 90)秒,直到上述条件连同步骤 9 所述的 supervisor 自身报告全部成立。否则走步骤 8 的回滚分支。 +7. 在监管运行的状态下,检查 manifest 源码身份、唯一监听者及配置哈希不变。验证转发、实际折叠、原文/图片精确检索及该测试 session 的检查日志。获授权后可发送一个小型认证提供商请求。`Deploy-LiveProxy.ps1` 只检查步骤 6 的成功判定与步骤 9 的监管状态,随后即改写回执并报告 `deployed`,因此这些检查在脚本结束后进行,此处失败时须在回执已被改写的情况下用 `Rollback-LiveProxy.ps1` 回滚。 +8. 切换或上述检查失败时,在重新暂停监管后回滚:创建 `maintenance.inhibit`、禁用计划任务、停止 supervisor,再停止已启动的候选代理(如有)。恢复保存的 launcher,若回执已被改写则一并恢复两份回执:切换过程中失败时回执尚未改动,部署脚本结束后在步骤 7 失败时则已改写,`Rollback-LiveProxy.ps1` 在给定该次部署的备份目录后会恢复这三个文件。在 supervisor 存活时修改或恢复 launcher,与步骤 4 同理并不安全。随后按步骤 6 的方式解除暂停,由 supervisor 启动旧 release,并对它重做步骤 6 的成功判定与步骤 7 的检查。若恢复后的 release 同样未通过,即回滚失败:重新暂停监管(inhibit 已设、计划任务已禁用、supervisor 已停止),不再尝试任何启动、恢复或编辑,并携带证据向所有者报告:未通过的检查、`supervisor.log`、`bili.log`,以及端口当前由谁占用(如有)。此后运行中的代理处于无人监管或停止状态,直至所有者决定;这是有意设定的终态,以免失败的代理被反复重启。两个脚本都会自行落到这一终态,不触碰任何代理进程,把证据保存到备份目录,并以退出码 4 结束。 9. 确认维护状态已结束,成功路径与回滚路径同样适用:`maintenance.inhibit` 不存在、计划任务已启用,且恰好有一个 supervisor 在运行。该文件存在期间 supervisor 拒绝接管、启动与重启,因此遗留该文件的部署即使通过全部检查,也没有自动接管与重启。确认 supervisor 正在监控当前运行的 release(候选版本或已恢复版本):`supervisor-status.json` 显示一个受监控的子进程,`healthMatch` 为 true 且无 inhibit,其 `childPid` 即唯一监听者的 PID,且健康端点由该 PID 报告预期版本号与 commit。唯一的例外是步骤 8 的回滚失败,它有意以暂停监管的状态结束。 10. 成功路径:完成部署回执,报告已部署的 SHA、结果及回滚位置。回滚路径:没有完成任何部署,确认回执描述的是已恢复的 release,并报告失败、保持运行的 release 及备份位置。回滚失败时:报告没有任何 release 被确认在运行,且监管处于暂停状态。本地代理部署不包含 npm 发包或更新承载任务的 Cindy 应用。 -每次 master 切换都把工具保留在各自的备份中:即部署回执 `tuned-deployment.json` 的 `masterBackupRoot` 所指备份根目录下的 `tooling\` 目录,当前为 `%APPDATA%\CindyGlobal\billion-context-proxy\backups\before-master-20261114-214204`。其中恰好包含 `Deploy-Master.ps1`、`build.mjs`、`fold-test.mjs`、`iso-test.mjs` 和 `live-entry.ts`;更早的 `before-master-20261114-191918` 备份中有一份逐字节相同的副本。按步骤对应:`iso-test.mjs` 与 `fold-test.mjs` 用于步骤 1,`live-entry.ts` 与 `build.mjs` 用于步骤 3,`Deploy-Master.ps1` 自动完成步骤 2 的预检、release 目录与备份,步骤 4–5 的暂停监管与 launcher 编辑,步骤 6 的切换,切换失败时步骤 8 中仅恢复 launcher 的回滚,以及切换后随即进行的、步骤 7 所述的回执改写。在 `tooling\` 旁边,该备份根目录保存切换前的 `Start-BillionContextProxy.ps1`、`proxy.identity.json`、`supervision-receipt.json`、`supervisor-status.json` 和 `tuned-deployment.json`,以及 `Rollback-MasterDeploy.ps1`;后者撤销这一次已完成的切换:从该备份恢复 launcher 与两份回执,并通过计划任务重启旧 release。每个备份中的回滚脚本只针对它自己的那次切换,因此应使用回执所指备份中的那一个。运行中的 launcher 是运行根目录中的 `Start-BillionContextProxy.ps1`。同一根目录下的 `Rollback-TunedLiveProxy.ps1` 是另一种更深的回滚:它从回执的 `backupRoot`(2026-11-07 首次 tuned 部署之前的备份)恢复配置与 launcher,而不是回到上一个 release。 +部署与回滚脚本位于仓库的 `scripts/live-deploy/`,说明见下文。构建与隔离测试工具则不在仓库中:每次 master 切换都把它们保留在各自的备份中,即部署回执 `tuned-deployment.json` 的 `masterBackupRoot` 所指备份根目录下的 `tooling\` 目录,当前为 `%APPDATA%\CindyGlobal\billion-context-proxy\backups\before-master-20261114-214204`。其中恰好包含 `Deploy-Master.ps1`、`build.mjs`、`fold-test.mjs`、`iso-test.mjs` 和 `live-entry.ts`;更早的 `before-master-20261114-191918` 备份中有一份逐字节相同的副本。按步骤对应:`iso-test.mjs` 与 `fold-test.mjs` 用于步骤 1,`live-entry.ts` 与 `build.mjs` 用于步骤 3,`Deploy-Master.ps1` 是执行 2026-11-14 各次切换的一次性脚本,作为历史保留,已被 `Deploy-LiveProxy.ps1` 取代。请把该工具目录以 `-ToolingDir` 传给部署脚本,使下一个备份也保留一份副本。在 `tooling\` 旁边,该备份根目录保存切换前的 `Start-BillionContextProxy.ps1`、`proxy.identity.json`、`supervision-receipt.json`、`supervisor-status.json` 和 `tuned-deployment.json`,以及 `Rollback-MasterDeploy.ps1`;后者是 `Deploy-Master.ps1` 为这一次已完成的切换生成的回滚文本:从该备份恢复 launcher 与两份回执,并通过计划任务重启旧 release。由 `Deploy-LiveProxy.ps1` 生成的备份则保存 `deploy-record.json`,`Rollback-LiveProxy.ps1` 读取的正是它;该脚本无法回滚由旧脚本完成的切换。运行中的 launcher 是运行根目录中的 `Start-BillionContextProxy.ps1`。同一根目录下的 `Rollback-TunedLiveProxy.ps1` 是另一种更深的回滚:它从回执的 `backupRoot`(2026-11-07 首次 tuned 部署之前的备份)恢复配置与 launcher,而不是回到上一个 release。 -步骤 4–9 采用保留的 `Deploy-Master.ps1` 所实现的顺序:由 supervisor 而非操作者启动新 release,健康检查在监管运行的状态下进行。依据所有者 2026-11-16 的决定([#78](https://github.com/anuzzolilli-oss/billion-context/issues/78)),这是认可的顺序;此前在 inhibit 下手动启动并检查新代理的写法已被取代,手动启动的代理如何纳入监管的问题也随之不复存在。回滚失败时,保留的脚本仍与上述步骤不同。`Deploy-Master.ps1` 的失败分支与 `Rollback-MasterDeploy.ps1` 都会删除 inhibit 并启动计划任务,而不要求旧 release 健康(前者只报告结果,后者不做检查),因此在其中任何一个运行之后,须自行完成步骤 8 的检查,未通过时手动重新暂停监管。只有切换成功后脚本才改写回执并写出 `Rollback-MasterDeploy.ps1`。复用前检查这些保留的工具:它们含固定的构建哈希与旧版本假设。不得在当前版本已变化时直接执行旧安装脚本,应重新建立身份及回滚检查。 +步骤 4–9 采用保留的 `Deploy-Master.ps1` 所实现的顺序:由 supervisor 而非操作者启动新 release,健康检查在监管运行的状态下进行。依据所有者 2026-11-16 的决定([#78](https://github.com/anuzzolilli-oss/billion-context/issues/78)),这是认可的顺序;此前在 inhibit 下手动启动并检查新代理的写法已被取代,手动启动的代理如何纳入监管的问题也随之不复存在。仓库中的脚本实现了这一顺序,包括步骤 8 的回滚失败终态([#80](https://github.com/anuzzolilli-oss/billion-context/issues/80))。保留的 `Deploy-Master.ps1` 与 `Rollback-MasterDeploy.ps1` 则不然:两者都会删除 inhibit 并启动计划任务,而不要求旧 release 健康(前者只报告结果,后者不做检查),并且含固定的构建哈希与旧版本假设。不得用 `Deploy-Master.ps1` 进行新的部署。如果必须用 `Rollback-MasterDeploy.ps1` 撤销 2026-11-14 的切换,须自行完成步骤 8 的检查,未通过时手动重新暂停监管。 + +### 部署与回滚脚本 + +`scripts/live-deploy/Deploy-LiveProxy.ps1` 与 `scripts/live-deploy/Rollback-LiveProxy.ps1` 需要 PowerShell 7,所有与具体安装相关的值都以参数传入;脚本中不写入任何 release、端口或计划任务的信息。请以非交互方式运行,并先用 `-PreflightOnly` 运行部署脚本:它执行全部检查、打印计划,且不做任何更改。 + +```powershell +pwsh -NoProfile -NonInteractive -File scripts\live-deploy\Deploy-LiveProxy.ps1 ` + -RuntimeRoot "<运行根目录>" -Port <端口> -TaskName "<计划任务名称>" ` + -CandidateRoot "<候选目录>" -ReleaseName "<新 release 目录名>" ` + -CandidateVersion "<步骤 1 测试报告的版本号>" -CandidateCommit "<步骤 1 测试报告的 commit>" ` + -CandidateSourceCommit "<已固定的源码 SHA>" -CandidateBundleSha256 "<构建时记录的 bundle 哈希>" ` + -ExpectedConfigSha256 "<步骤 2 核实的配置哈希>" -ToolingDir "<保留的工具目录>" -PreflightOnly + +pwsh -NoProfile -NonInteractive -File scripts\live-deploy\Rollback-LiveProxy.ps1 ` + -RuntimeRoot "<运行根目录>" -BackupRoot "<运行根目录>\backups\<备份目录>" -PreflightOnly +``` + +四个候选值取自步骤 1 的记录,而非候选目录,这样脚本才能证明该目录中的产物就是经过测试的那一个。预期配置哈希同样由操作者提供:`tuned-deployment.json` 中的 `deployedConfigSha256` 既不被读取,也不被改写。 + +| 步骤 | 由谁完成 | +|---|---| +| 1、3 | 手动,使用保留的构建与测试工具 | +| 2、4、5、6 | `Deploy-LiveProxy.ps1` | +| 7 | 回执改写:`Deploy-LiveProxy.ps1`。功能检查:在其报告 `deployed` 之后手动进行 | +| 8 | 切换过程中:`Deploy-LiveProxy.ps1`。步骤 7 失败之后:`Rollback-LiveProxy.ps1` | +| 9 | 两个脚本,作为其成功判定的一部分 | +| 10 | 回执与打印的结果:脚本。向所有者的报告:操作者 | + +每个脚本对每个事件打印一行,并以一行 `NEXT` 和一行 `RESULT` 结束;部署脚本在 `BACKUP` 行打印备份目录,并在其中保存 `deploy-record.json` 与 `result.json`。 + +| 退出码 | 结果 | 留下的状态 | 接下来做什么 | +|---|---|---|---| +| 0 | `deployed` | 新 release 在监管下健康运行 | 进行步骤 7 的检查;回滚命令见 `NEXT` 行 | +| 0 | `restored`(回滚脚本) | 旧 release 在监管下健康运行 | 按步骤 10 报告;之后的部署需要新的 release 名称 | +| 0 | `preflight-ok` | 未做任何更改 | 去掉 `-PreflightOnly` 再次运行 | +| 2 | `refused` | 未做任何更改 | 修正未通过的检查 | +| 3 | `rolled-back` | 旧 release 在监管下健康运行;新 release 目录已删除 | `reason=not-idle`:稍后重试。`hold-failed`、`cutover-failed`:先查明原因 | +| 4 | `rollback-failed` | 监管处于暂停状态;代理进程保持原状;没有任何 release 被确认在运行 | 携带证据文件与两份日志向所有者报告;原因修复后用 `Rollback-LiveProxy.ps1` 重试 | +| 5 | `deployed-receipts-incomplete` | 新 release 在监管下健康运行;有回执未被替换 | 复制 `NEXT` 行列出的待写回执文件,然后进行步骤 7 的检查 | +| 6 | `error-before-hold` | 运行中的一切均未更改;新 release 目录已删除 | 修正问题后再次运行 | +| 其他 | 未打印 | 未知,例如脚本被终止 | 阅读 `deploy-record.json`,然后运行 `Rollback-LiveProxy.ps1 -PreflightOnly` | + +这些脚本只在沙箱中测试,从不针对运行中的代理:`tests/live-deploy-scripts.test.ts` 建立一个临时运行根目录,其中是替身 launcher 与替身代理,并以测试驱动代替计划任务。脚本只对带有 `sandbox.marker` 的运行根目录接受该驱动,对这样的根目录拒绝使用真实的计划任务,并对位于正式安装区域内的任何根目录拒绝该驱动。 diff --git a/reference/testing.md b/reference/testing.md index 8bd619552b..8f7d3a1009 100644 --- a/reference/testing.md +++ b/reference/testing.md @@ -63,7 +63,7 @@ flakes were loaded local sandboxes — gated by `ACP_TEST_CLAUDE_NATIVE`, not CI The script self-checks on every run: a new top-level `tests/*.test.ts` that matches a process-spawn pattern but is NOT registered in its `LIVE_FILES` list fails the job. When you add a suite that spawns processes, register it in -`scripts/run-ci-tests.mjs` and review its concurrency assumptions (#1248: +`scripts/live-files.mjs` (append-only) and review its concurrency assumptions (#1248: per-file XDG homes, `listen(0)` ports, no machine-global state) before it rides c=4 unreviewed. diff --git a/scripts/ci-shard-weights.json b/scripts/ci-shard-weights.json index 71a305ff15..0dcd681671 100644 --- a/scripts/ci-shard-weights.json +++ b/scripts/ci-shard-weights.json @@ -847,6 +847,7 @@ "tests/incremental-ack-pull.test.ts": 10547, "tests/issue4323-drift-escalation.test.ts": 2194, "tests/pi-host-stub.test.ts": 136, - "tests/server-close-all-connections.test.ts": 3212 + "tests/server-close-all-connections.test.ts": 3212, + "tests/live-deploy-scripts.test.ts": 221000 } } diff --git a/scripts/live-deploy/Deploy-LiveProxy.ps1 b/scripts/live-deploy/Deploy-LiveProxy.ps1 new file mode 100644 index 0000000000..a56f9731ff --- /dev/null +++ b/scripts/live-deploy/Deploy-LiveProxy.ps1 @@ -0,0 +1,381 @@ +#requires -Version 7.0 +<# +.SYNOPSIS +Deploys a tested candidate build as the live proxy release, following the documented procedure. + +.DESCRIPTION +Automates steps 2, 4-6, the receipt rewrite of step 7, the deploy-time rollback of step 8 and the +supervision check of step 9 in reference/cindy-project.md ("Deploy only with an explicit request"). +The functional checks of step 7 stay manual and run after this script reports "deployed". + +The supervisor, not this script, starts the new release. Every failure after the hold begins ends +either with the previous release healthy under supervision, or with the hold back in place. + +Exit codes: 0 deployed, 2 refused (nothing changed), 3 rolled back, 4 rollback failed (hold in +place), 5 deployed but a receipt was not updated, 6 error before the hold (nothing changed). +Any other exit code means the state is unknown: read deploy-record.json in the backup folder. + +.EXAMPLE +pwsh -NoProfile -NonInteractive -File Deploy-LiveProxy.ps1 -RuntimeRoot "" -Port -TaskName "" -CandidateRoot "" -ReleaseName "" -CandidateVersion "" -CandidateCommit "" -CandidateSourceCommit "" -CandidateBundleSha256 "" -ExpectedConfigSha256 "" -PreflightOnly +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string]$RuntimeRoot, + [Parameter(Mandatory)] [int]$Port, + [Parameter(Mandatory)] [string]$TaskName, + [Parameter(Mandatory)] [string]$CandidateRoot, + [Parameter(Mandatory)] [string]$ReleaseName, + [Parameter(Mandatory)] [string]$CandidateVersion, + [Parameter(Mandatory)] [string]$CandidateCommit, + [Parameter(Mandatory)] [string]$CandidateSourceCommit, + [Parameter(Mandatory)] [string]$CandidateBundleSha256, + [Parameter(Mandatory)] [string]$ExpectedConfigSha256, + [string]$ToolingDir, + [int]$HealthWaitSeconds = 90, + [int]$StopWaitSeconds = 20, + [string]$TestTaskDriver, + [switch]$PreflightOnly +) +. (Join-Path $PSScriptRoot 'LiveProxy.Common.ps1') + +$ctx = $null +$script:checkFailures = 0 +function Add-Check([string]$Label, [scriptblock]$Test) { + # The test returns $null (or nothing) on success, otherwise a short "expected ..., observed ..." text. + $problem = $null + try { $problem = & $Test } catch { $problem = "could not be checked ($($_.Exception.Message))" } + if ($null -eq $problem -or $problem -eq '') { Write-LpEvent 'CHECK' "PASS $Label" } + else { Write-LpEvent 'CHECK' "FAIL ${Label}: $problem"; $script:checkFailures++ } +} +function Stop-IfChecksFailed([string]$What) { + if ($script:checkFailures -gt 0) { throw [LpStop]::new(2, 'refused', "$What refused: $script:checkFailures check(s) failed; nothing was changed", '', 'fix the failing checks listed above and run again') } +} +function Remove-CreatedRelease([string]$ReleaseDir) { + if (-not (Test-Path -LiteralPath $ReleaseDir)) { return } + $link = Join-Path $ReleaseDir 'node_modules' + if (Test-Path -LiteralPath $link) { + if (-not (Test-LpReparsePoint $link)) { throw 'node_modules in the new release is not a link; the release directory was left in place' } + [System.IO.Directory]::Delete($link) # removes the link itself, never its target + } + Remove-Item -LiteralPath $ReleaseDir -Recurse -Force +} +# Receipts are edited as JSON trees so that fields this script does not own keep their exact text. +function Set-JsonProp($Object, [string]$Key, $Value) { + $Object.set_Item($Key, [System.Text.Json.Nodes.JsonNode]::Parse((ConvertTo-Json -InputObject $Value -Compress -Depth 6))) +} +function Copy-JsonProp($Object, [string]$To, [string]$From) { + $source = $Object.get_Item($From) + if ($null -eq $source) { $Object.set_Item($To, [System.Text.Json.Nodes.JsonNode]$null) } + else { $Object.set_Item($To, [System.Text.Json.Nodes.JsonNode]::Parse($source.ToJsonString())) } +} +function ConvertTo-JsonBytes($Node) { + $opt = [System.Text.Json.JsonSerializerOptions]::new() + $opt.WriteIndented = $true + $opt.Encoder = [System.Text.Encodings.Web.JavaScriptEncoder]::UnsafeRelaxedJsonEscaping + return $script:LpUtf8.GetBytes($Node.ToJsonString($opt)) +} + +try { + # ---- input validation: nothing is read from the runtime root before this passes ---------------- + Add-Check 'candidate version format' { if ($CandidateVersion -notmatch $script:LpPatterns.Version) { 'expected ..' } } + Add-Check 'candidate commit format' { if ($CandidateCommit -cnotmatch $script:LpPatterns.Commit) { 'expected 7-12 lower-case hex characters with an optional -suffix' } } + Add-Check 'candidate source commit format' { if ($CandidateSourceCommit -cnotmatch $script:LpPatterns.SourceCommit) { 'expected 40 lower-case hex characters' } } + Add-Check 'candidate bundle hash format' { if ($CandidateBundleSha256 -notmatch $script:LpPatterns.Sha256) { 'expected 64 hex characters' } } + Add-Check 'expected config hash format' { if ($ExpectedConfigSha256 -notmatch $script:LpPatterns.Sha256) { 'expected 64 hex characters' } } + Add-Check 'release name' { + if ($ReleaseName -notmatch $script:LpPatterns.ReleaseName -or $ReleaseName.Length -gt 64) { return 'expected letters, digits, dot, dash and underscore, at most 64 characters' } + if ($ReleaseName -match '^(?i)(con|prn|aux|nul|com[1-9]|lpt[1-9])(\.|$)') { return 'reserved device name' } + } + Add-Check 'task name' { if ($TaskName -match '[*?\[\]]') { 'wildcard characters are not allowed' } } + Add-Check 'port' { if ($Port -lt 1 -or $Port -gt 65535) { 'expected 1-65535' } } + Add-Check 'no quote, backtick, dollar sign, control or non-ASCII character in any value' { + foreach ($v in @($RuntimeRoot, $CandidateRoot, $ReleaseName, $TaskName, $CandidateVersion, $CandidateCommit, $ToolingDir)) { if ($v -and -not (Test-LpSafeText $v)) { return 'a value contains a forbidden character' } } + } + $root = $null; $candidateRootPath = $null + Add-Check 'runtime root' { + $script:root = Resolve-LpPath $RuntimeRoot + if (-not (Test-LpSafeText $script:root)) { return 'the resolved path contains a forbidden character' } + if (Test-LpReparseInPath $script:root) { return 'a folder in the path is a link' } + } + Add-Check 'candidate folder' { + $script:candidateRootPath = Resolve-LpPath $CandidateRoot + if (Test-LpReparseInPath $script:candidateRootPath) { return 'a folder in the path is a link' } + if ($script:root -and (Test-LpPathWithin $script:candidateRootPath $script:root)) { return 'the candidate folder is inside the runtime root' } + } + Stop-IfChecksFailed 'input' + $root = $script:root; $candidateRootPath = $script:candidateRootPath + $script:LpRoot = $root + + $driver = $null + Add-Check 'task driver allowed for this runtime root' { $script:driverPath = Resolve-LpTaskDriver $RuntimeRoot $root $TestTaskDriver; $null } + Stop-IfChecksFailed 'input' + if ($script:driverPath) { . $script:driverPath; Write-LpEvent 'STEP' 'TEST SEAM ACTIVE: scheduled-task operations go to the test driver' } + + $releasesDir = Join-Path $root 'releases' + $newRoot = Join-Path $releasesDir $ReleaseName + $newProxyPath = Join-Path $newRoot 'dist\index.mjs' + Add-Check 'release directory location' { if ([System.IO.Path]::GetDirectoryName([System.IO.Path]::GetFullPath($newRoot)) -ne $releasesDir) { 'the release name does not resolve to a folder directly under releases' } } + Stop-IfChecksFailed 'input' + + $ctx = @{ + Root = $root; Port = $Port; TaskName = $TaskName; HealthWait = $HealthWaitSeconds; StopWait = $StopWaitSeconds + LauncherPath = Join-Path $root 'Start-BillionContextProxy.ps1'; InhibitPath = Join-Path $root 'maintenance.inhibit' + LockPath = Join-Path $root 'supervisor.lock'; StatusPath = Join-Path $root 'supervisor-status.json' + ConfigPath = Join-Path $root 'config\billion-context\billion-context.json'; ConfigSha = $ExpectedConfigSha256 + Previous = $null; Candidate = [ordered]@{ releaseName = $ReleaseName; proxyPath = $newProxyPath; version = $CandidateVersion; commit = $CandidateCommit; sourceCommit = $CandidateSourceCommit } + BackupRoot = $null; DeploymentId = $null; Record = @{ stage = 'none' }; RecordPath = $null; LiftAt = [datetime]::MinValue; TaskBaseline = $null + } + $tunedPath = Join-Path $root 'tuned-deployment.json' + $supervisionPath = Join-Path $root 'supervision-receipt.json' + + # ---- preflight: read-only, every check runs ---------------------------------------------------- + $candidateFiles = [ordered]@{ 'index.mjs' = 'dist\index.mjs'; 'build-info.json' = 'dist\build-info.json'; 'package.json' = 'package.json'; 'LICENSE' = 'LICENSE' } + $candidateBytes = @{}; $candidateSha = [ordered]@{} + Add-Check 'candidate files and identity' { + foreach ($name in $candidateFiles.Keys) { + $p = Join-Path $candidateRootPath $candidateFiles[$name] + if (-not (Test-Path -LiteralPath $p -PathType Leaf)) { return "$($candidateFiles[$name]) is missing" } + if (Test-LpReparsePoint $p) { return "$($candidateFiles[$name]) is a link" } + $candidateBytes[$name] = [System.IO.File]::ReadAllBytes($p) + $candidateSha[$name] = Get-LpSha256OfBytes $candidateBytes[$name] + } + $info = $script:LpUtf8.GetString($candidateBytes['build-info.json']) | ConvertFrom-Json -AsHashtable + if ([string]$info['commit'] -cne $CandidateCommit) { return "expected commit $CandidateCommit, the candidate reports $($info['commit'])" } + if ([string]$info['sourceCommit'] -cne $CandidateSourceCommit) { return 'the candidate reports a different source commit' } + if ($info.ContainsKey('bundleSha256') -and -not (Test-LpSameHash ([string]$info['bundleSha256']) $CandidateBundleSha256)) { return 'the bundle hash in build-info.json differs from the expected one' } + $pkg = $script:LpUtf8.GetString($candidateBytes['package.json']) | ConvertFrom-Json -AsHashtable + if ([string]$pkg['version'] -cne $CandidateVersion) { return "expected version $CandidateVersion, the candidate package says $($pkg['version'])" } + if (-not (Test-LpSameHash $candidateSha['index.mjs'] $CandidateBundleSha256)) { return 'the bundle hash differs from the expected one' } + } + $launcherBytes = $null; $launcherText = $null; $edit = $null; $editedBytes = $null; $prevReleaseDir = $null + Add-Check 'launcher identity lines' { + $script:launcherBytes = [System.IO.File]::ReadAllBytes($ctx.LauncherPath) + if (-not (Test-LpPlainAscii $script:launcherBytes)) { return 'the launcher is not plain ASCII without a byte-order mark' } + $script:launcherText = [System.Text.Encoding]::ASCII.GetString($script:launcherBytes) + $id = Read-LpLauncherIdentity $script:launcherText + $ctx.Previous = [ordered]@{ proxyPath = $id.ProxyPath.Value; version = $id.ExpectedVersion.Value; commit = $id.ExpectedCommit.Value } + $script:launcherPort = [int]$id.Port.Value + $script:prevReleaseDir = [System.IO.Path]::GetDirectoryName([System.IO.Path]::GetDirectoryName($ctx.Previous.proxyPath)) + if ([System.IO.Path]::GetDirectoryName($script:prevReleaseDir) -ne $releasesDir -or [System.IO.Path]::GetFileName($ctx.Previous.proxyPath) -ne 'index.mjs') { return 'the current release path is not \releases\\dist\index.mjs' } + if (-not (Test-Path -LiteralPath $ctx.Previous.proxyPath -PathType Leaf)) { return 'the current release bundle is missing' } + } + $launcherBytes = $script:launcherBytes; $launcherText = $script:launcherText; $prevReleaseDir = $script:prevReleaseDir + if ($null -eq $ctx.Previous) { Stop-IfChecksFailed 'preflight' } + Add-Check 'port matches the launcher' { if ($script:launcherPort -ne $Port) { "the launcher uses another port than -Port $Port" } } + Add-Check 'scheduled task runs this launcher' { + $t = Get-LiveProxyTaskInfo $TaskName + if ($t.Count -ne 1) { return "expected exactly one task with this name, found $($t.Count)" } + if (-not $t.Enabled) { return 'the task is disabled' } + if ($t.ActionArguments.IndexOf($ctx.LauncherPath, [System.StringComparison]::OrdinalIgnoreCase) -lt 0) { return 'the task action does not run the launcher in this runtime root' } + $ctx.TaskBaseline = $t + } + Add-Check 'configuration hash' { if (-not (Test-LpSameHash (Get-LpSha256 $ctx.ConfigPath) $ExpectedConfigSha256)) { 'the configuration file differs from -ExpectedConfigSha256' } } + $listenerProcess = $null + Add-Check 'current release healthy from the single listener' { + $l = @(Get-LpListenerPids $Port) + if ($l.Count -ne 1) { return "expected one listener, found $($l.Count)" } + $h = Get-LpHealth $Port + if (-not $h.ok -or $h.version -cne $ctx.Previous.version -or $h.commit -cne $ctx.Previous.commit -or $h.pid -ne [string]$l[0]) { return "expected version=$($ctx.Previous.version) commit=$($ctx.Previous.commit) from pid $($l[0]), observed $(Format-LpHealth $h)" } + $p = @(Get-LpProxies $ctx.Previous.proxyPath) + if ($p.Count -ne 1 -or [string]$p[0].ProcessId -ne [string]$l[0]) { return 'the listener is not the single process running the current release' } + $script:listenerProcess = $p[0] + } + $supervisorProcess = $null + Add-Check 'exactly one supervisor holding the lock' { + $s = @(Get-LpSupervisors $ctx) + if ($s.Count -ne 1) { return "expected one supervisor, found $($s.Count)" } + if (-not (Test-LpLockHeld $ctx.LockPath)) { return 'the installation lock is not held' } + $script:supervisorProcess = $s[0] + } + Add-Check 'no maintenance in progress' { if (Test-Path -LiteralPath $ctx.InhibitPath) { 'maintenance.inhibit is present' } } + Add-Check 'new release directory is absent' { if (Test-Path -LiteralPath $newRoot) { "releases\$ReleaseName already exists; choose another -ReleaseName" } } + $nodeModules = $null + Add-Check 'current release node_modules link' { + $nm = Get-Item -LiteralPath (Join-Path $prevReleaseDir 'node_modules') -Force + if (@('Junction', 'SymbolicLink') -notcontains [string]$nm.LinkType) { return 'node_modules of the current release is not a junction or symbolic link' } + $target = [string]@($nm.Target)[0] + if (-not [System.IO.Path]::IsPathRooted($target)) { $target = Join-Path $prevReleaseDir $target } + if (-not (Test-Path -LiteralPath $target -PathType Container)) { return 'the node_modules link target is not an existing folder' } + $script:nodeModules = [ordered]@{ linkType = [string]$nm.LinkType; target = [System.IO.Path]::GetFullPath($target) } + } + Add-Check 'launcher edit touches only the release identity lines' { + if (-not (Test-LpSafeText $newProxyPath)) { return 'the new release path contains a forbidden character' } + $script:edit = New-LpEditedLauncher $launcherText $newProxyPath $CandidateCommit $CandidateVersion + if ($script:edit.Changed -notcontains 'ProxyPath') { return 'the release path would not change' } + $script:editedBytes = [System.Text.Encoding]::ASCII.GetBytes($script:edit.Text) + } + $edit = $script:edit; $editedBytes = $script:editedBytes + Add-Check 'edited launcher parses and reports a decision under Windows PowerShell' { + if ($null -eq $editedBytes) { return 'the edited launcher could not be built' } + $r = Invoke-LpLauncherDryRun $editedBytes $root + if ($r.ExitCode -ne 0 -or -not $r.Action) { return "exit code $($r.ExitCode), decision '$($r.Action)'" } + } + $toolingPath = $null + if ($ToolingDir) { + Add-Check 'tooling folder' { + $script:toolingPath = Resolve-LpPath $ToolingDir + if (Test-LpReparseInPath $script:toolingPath) { return 'a folder in the path is a link' } + if (Test-LpPathWithin $script:toolingPath $releasesDir) { return 'the tooling folder is inside releases' } + foreach ($f in (Get-ChildItem -LiteralPath $script:toolingPath -File -Force)) { if (Test-LpReparsePoint $f.FullName) { return "$($f.Name) is a link" } } + } + $toolingPath = $script:toolingPath + } + $clients = -1 + Add-Check 'idle (no client connection)' { + $script:clients = Get-LpClientCount $ctx + if ($script:clients -gt 0 -and -not $PreflightOnly) { return "$script:clients client connection(s) open; not idle" } + } + $clients = $script:clients + Stop-IfChecksFailed 'preflight' + $listenerProcess = $script:listenerProcess; $supervisorProcess = $script:supervisorProcess; $nodeModules = $script:nodeModules + + $stamp = Get-Date -Format 'yyyyMMdd-HHmmss' + $deploymentId = "before-deploy-$stamp-$ReleaseName" + $backupRoot = Join-Path (Join-Path $root 'backups') $deploymentId + Write-LpEvent 'STEP' "plan: replace $($ctx.Previous.version), $($ctx.Previous.commit) ($(Get-LpRelative $prevReleaseDir)) with $CandidateVersion, $CandidateCommit (releases\$ReleaseName)" + Write-LpEvent 'STEP' "plan: launcher defaults to change: $($edit.Changed -join ', '); backup folder backups\$deploymentId; open client connections now: $clients" + if ($PreflightOnly) { throw [LpStop]::new(0, 'preflight-ok', 'preflight passed; nothing was changed', '', 'run again without -PreflightOnly when the proxy is idle') } +} catch [LpStop] { + Complete-Lp $null $_.Exception $null +} catch { + Complete-Lp $null ([LpStop]::new(2, 'refused', "refused: $($_.Exception.Message); nothing was changed", '', 'fix the problem above and run again')) $null +} + +# ---- preparation: release directory, backup, record (additive; nothing running is touched) -------- +$createdRelease = $false +try { + New-Item -ItemType Directory -Path $newRoot | Out-Null + $createdRelease = $true + New-Item -ItemType Directory -Path (Join-Path $newRoot 'dist') | Out-Null + foreach ($name in $candidateFiles.Keys) { + $dest = Join-Path $newRoot $candidateFiles[$name] + [System.IO.File]::WriteAllBytes($dest, $candidateBytes[$name]) + if (-not (Test-LpSameHash (Get-LpSha256 $dest) $candidateSha[$name])) { throw "the copied $name does not match the candidate" } + } + New-Item -ItemType $nodeModules.linkType -Path (Join-Path $newRoot 'node_modules') -Target $nodeModules.target | Out-Null + New-Item -ItemType Directory -Path $backupRoot | Out-Null + $ctx.BackupRoot = $backupRoot; $ctx.DeploymentId = $deploymentId; $ctx.RecordPath = Join-Path $backupRoot 'deploy-record.json' + foreach ($f in 'Start-BillionContextProxy.ps1', 'tuned-deployment.json', 'supervision-receipt.json') { Copy-Item -LiteralPath (Join-Path $root $f) -Destination $backupRoot } + foreach ($f in 'proxy.identity.json', 'supervisor-status.json') { if (Test-Path -LiteralPath (Join-Path $root $f)) { Copy-Item -LiteralPath (Join-Path $root $f) -Destination $backupRoot } } + if ($toolingPath) { + $toolDest = Join-Path $backupRoot 'tooling' + New-Item -ItemType Directory -Path $toolDest | Out-Null + Get-ChildItem -LiteralPath $toolingPath -File -Force | ForEach-Object { Copy-Item -LiteralPath $_.FullName -Destination $toolDest } + } else { Write-LpEvent 'STEP' 'no -ToolingDir given: this backup holds no tooling folder' } + if (-not (Test-LpSameHash (Get-LpSha256 (Join-Path $backupRoot 'Start-BillionContextProxy.ps1')) (Get-LpSha256OfBytes $launcherBytes))) { throw 'the launcher changed while it was being backed up' } + $self = Get-CimInstance Win32_Process -Filter "ProcessId=$PID" + $ctx.Record = [ordered]@{ + schema = 1; deploymentId = $deploymentId; createdAt = (Get-Date).ToString('o'); runtimeRoot = $root; port = $Port; taskName = $TaskName + previous = $ctx.Previous + candidate = [ordered]@{ releaseName = $ReleaseName; proxyPath = $newProxyPath; version = $CandidateVersion; commit = $CandidateCommit; sourceCommit = $CandidateSourceCommit; sha256 = $candidateSha } + nodeModules = $nodeModules; configSha256 = $ExpectedConfigSha256.ToUpperInvariant() + launcherSha256Before = Get-LpSha256OfBytes $launcherBytes; launcherSha256After = Get-LpSha256OfBytes $editedBytes + deployProcess = [ordered]@{ pid = $PID; created = $self.CreationDate.ToString('o') } + created = [ordered]@{ releaseDir = $true; backupDir = $true }; stage = 'prepared'; reachedStage = 'prepared' + } + Save-LpRecord $ctx + Write-LpEvent 'BACKUP' $backupRoot + $r = Invoke-LpLauncherDryRun $editedBytes $root + if ($r.ExitCode -ne 0 -or $r.Action -ne 'conflict') { throw "with the new release in place the edited launcher decided '$($r.Action)' (exit $($r.ExitCode)); expected 'conflict' while the current release is still serving" } + $ctx.TaskBaseline = Get-LiveProxyTaskInfo $TaskName + Set-LpStage $ctx 'holding' + try { New-LpInhibit $ctx 'deploy' } catch { throw "maintenance.inhibit could not be created ($($_.Exception.Message)); other maintenance may have started" } +} catch { + $note = '' + try { if ($ctx.RecordPath -and (Test-Path -LiteralPath $ctx.RecordPath)) { Set-LpStage $ctx 'aborted' } } catch { $note = ' The deployment record could not be marked aborted.' } + try { if ($createdRelease) { Remove-CreatedRelease $newRoot } } catch { $note += " $($_.Exception.Message)." } + Complete-Lp $ctx ([LpStop]::new(6, 'error-before-hold', "stopped before the hold: $($_.Exception.Message). Nothing running was changed.$note", '', 'fix the problem and run again; the backup folder was kept for the record')) 'result.json' +} + +# ---- hold window: from here until the candidate is healthy, every error is rolled back ------------ +$reason = 'cutover-failed' +try { + Disable-LiveProxyTask $TaskName + Stop-LpProcessVerified $supervisorProcess + Assert-LpNoSupervisor $ctx 'after the hold' + $h = Get-LpHealth $Port + if (-not $h.ok -or $h.version -cne $ctx.Previous.version -or $h.commit -cne $ctx.Previous.commit) { throw 'the current release stopped answering after the supervisor was stopped' } + Write-LpEvent 'STEP' 'hold applied: inhibit set, task disabled, supervisor stopped; current release still healthy' + + if (-not (Test-LpSameHash (Get-LpSha256 $ctx.LauncherPath) $ctx.Record.launcherSha256Before)) { + Stop-LpRollbackFailed $ctx 'launcher edit: the launcher was changed by someone else after preflight; it was left as found' 'foreign-launcher-edit' + } + Set-LpStage $ctx 'held' + Write-LpBytesAtomic $ctx.LauncherPath $editedBytes + Write-LpEvent 'STEP' "launcher updated: $($edit.Changed -join ', ')" + + if ((Get-LpClientCount $ctx) -gt 0) { $reason = 'not-idle'; throw 'a client connected during preparation: not idle, standing down' } + + Set-LpStage $ctx 'cut' + Stop-LpProcessVerified $listenerProcess + if (-not (Wait-LpUntil { @(Get-LpListenerPids $Port).Count -eq 0 } $StopWaitSeconds 'waiting for the port')) { throw 'the port was not released' } + Write-LpEvent 'STEP' 'current release stopped, port released' + Remove-LpHold $ctx + $gap = Test-LpSuccess $ctx $ctx.Candidate + if ($null -ne $gap) { throw "the new release is not healthy under supervision: $gap" } + Write-LpEvent 'STEP' "new release healthy under supervision: $CandidateVersion, $CandidateCommit" +} catch [LpStop] { + Complete-Lp $ctx $_.Exception 'result.json' +} catch { + $holdOnly = -not (Test-LpReached $ctx 'held') + if ($holdOnly) { $reason = 'hold-failed' } + Write-LpEvent 'STEP' "FAILED: $($_.Exception.Message); rolling back" + try { + Invoke-LpRollback $ctx $launcherBytes @{} $holdOnly + Set-LpStage $ctx 'rolled-back' + $note = '' + try { Remove-CreatedRelease $newRoot } catch { $note = " The release directory could not be removed: $($_.Exception.Message)." } + $next = if ($reason -eq 'not-idle') { 'retry later with the same parameters, when no client is connected' } else { 'investigate the failure above before retrying' } + Complete-Lp $ctx ([LpStop]::new(3, 'rolled-back', "nothing was deployed; the previous release is live under supervision.$note", $reason, $next)) 'result.json' + } catch [LpStop] { + Complete-Lp $ctx $_.Exception 'result.json' + } catch { + Complete-Lp $ctx ([LpStop]::new(4, 'rollback-failed', "rollback stopped unexpectedly: $($_.Exception.Message). Check by hand that maintenance.inhibit is present, the task is disabled and no supervisor runs.", 'rollback-error', 'report to the owner; no release is confirmed live')) 'result.json' + } +} + +# ---- receipts: the candidate is live; nothing from here on is rolled back -------------------------- +$rollbackCommand = "pwsh -NoProfile -NonInteractive -File `"$(Join-Path $PSScriptRoot 'Rollback-LiveProxy.ps1')`" -RuntimeRoot `"$root`" -BackupRoot `"$backupRoot`"" +$pending = [ordered]@{ $tunedPath = Join-Path $backupRoot 'tuned-deployment.pending.json'; $supervisionPath = Join-Path $backupRoot 'supervision-receipt.pending.json' } +$notReplaced = @($tunedPath, $supervisionPath) +try { + Set-LpStage $ctx 'healthy' + $live = Get-LpHealth $Port + $launcherSha = Get-LpSha256 $ctx.LauncherPath + $tuned = [System.Text.Json.Nodes.JsonNode]::Parse([System.IO.File]::ReadAllText($tunedPath)).AsObject() + Set-JsonProp $tuned 'masterDeployedAt' ((Get-Date).ToString('o')) + Set-JsonProp $tuned 'masterBackupRoot' $backupRoot + Copy-JsonProp $tuned 'masterPreviousReleaseRoot' 'releaseRoot' + Copy-JsonProp $tuned 'masterPreviousBuild' 'build' + Copy-JsonProp $tuned 'masterPreviousBundleSha256' 'bundleSha256' + Copy-JsonProp $tuned 'masterPreviousSourceCommit' 'sourceCommit' + Set-JsonProp $tuned 'releaseRoot' $newRoot + Set-JsonProp $tuned 'bundleSha256' $candidateSha['index.mjs'] + Set-JsonProp $tuned 'build' $CandidateCommit + Set-JsonProp $tuned 'previousBuild' $ctx.Previous.commit + Set-JsonProp $tuned 'sourceCommit' $CandidateSourceCommit + Set-JsonProp $tuned 'livePid' ([int]$live.pid) + Set-JsonProp $tuned 'verifiedAt' ((Get-Date).ToUniversalTime().ToString('o')) + Set-JsonProp $tuned 'deployedSupervisorSha256' $launcherSha + $changedText = if ($edit.Changed.Count -gt 1) { (($edit.Changed | Select-Object -SkipLast 1) -join ', ') + ' and ' + $edit.Changed[-1] } else { $edit.Changed[0] } + $supervision = [System.Text.Json.Nodes.JsonNode]::Parse([System.IO.File]::ReadAllText($supervisionPath)).AsObject() + $masterDeploy = [ordered]@{ + at = (Get-Date).ToString('o'); launcherSha256 = $launcherSha.ToLowerInvariant(); change = "$changedText defaults only" + proxyRelease = "releases/$ReleaseName ($CandidateVersion, $CandidateCommit)"; backup = $backupRoot + } + Set-JsonProp $supervision 'masterDeploy' $masterDeploy + $content = @{ $tunedPath = ConvertTo-JsonBytes $tuned; $supervisionPath = ConvertTo-JsonBytes $supervision } + foreach ($path in $pending.Keys) { [System.IO.File]::WriteAllBytes($pending[$path], $content[$path]) } + foreach ($path in @($tunedPath, $supervisionPath)) { + Write-LpBytesAtomic $path $content[$path] + $notReplaced = @($notReplaced | Where-Object { $_ -ne $path }) + } + Set-LpStage $ctx 'receipts-written' + Write-LpEvent 'STEP' 'receipts updated' +} catch { + $names = ($notReplaced | ForEach-Object { [System.IO.Path]::GetFileName($_) }) -join ', ' + $copies = ($notReplaced | Where-Object { Test-Path -LiteralPath $pending[$_] } | ForEach-Object { "Copy-Item -LiteralPath `"$($pending[$_])`" -Destination `"$_`" -Force" }) -join '; ' + if (-not $copies) { $copies = 'the pending receipt files were not written; update the receipts by hand' } + Complete-Lp $ctx ([LpStop]::new(5, 'deployed-receipts-incomplete', "the new release is live under supervision, but not every receipt was updated ($names): $($_.Exception.Message)", '', "finish the receipts: $copies. Then run the step 7 checks. To roll back instead: $rollbackCommand")) 'result.json' +} +Complete-Lp $ctx ([LpStop]::new(0, 'deployed', "deployed $CandidateVersion, $CandidateCommit (source $CandidateSourceCommit) as releases\$ReleaseName", '', "run the step 7 checks now. If they fail, roll back with: $rollbackCommand")) 'result.json' diff --git a/scripts/live-deploy/LiveProxy.Common.ps1 b/scripts/live-deploy/LiveProxy.Common.ps1 new file mode 100644 index 0000000000..0c3eb6368b --- /dev/null +++ b/scripts/live-deploy/LiveProxy.Common.ps1 @@ -0,0 +1,528 @@ +# Shared functions for Deploy-LiveProxy.ps1 and Rollback-LiveProxy.ps1 (dot-sourced; PowerShell 7). +# Procedure: reference/cindy-project.md, "Deploy only with an explicit request". +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$script:LpUtf8 = [System.Text.UTF8Encoding]::new($false) +$script:LpPollMs = 500 +$script:LpSeam = $false +$script:LpRoot = $null +$script:LpStages = @('prepared', 'aborted', 'holding', 'held', 'cut', 'healthy', 'receipts-written', 'rolled-back', 'rollback-failed') +$script:LpForwardStages = @('prepared', 'holding', 'held', 'cut', 'healthy', 'receipts-written') +$script:LpPatterns = @{ + Version = '^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$' + Commit = '^[0-9a-f]{7,12}(-[a-z0-9]+)?$' + SourceCommit = '^[0-9a-f]{40}$' + Sha256 = '^[0-9A-Fa-f]{64}$' + ReleaseName = '^[A-Za-z0-9][A-Za-z0-9_-]*(\.[A-Za-z0-9_-]+)*$' +} + +# A planned stop: carries the exit code, result name, optional reason and the NEXT line. +class LpStop : System.Exception { + [int]$Code + [string]$Name + [string]$Reason + [string]$Next + LpStop([int]$code, [string]$name, [string]$message, [string]$reason, [string]$next) : base($message) { + $this.Code = $code; $this.Name = $name; $this.Reason = $reason; $this.Next = $next + } +} + +function Write-LpEvent([string]$Kind, [string]$Text) { + Write-Host ("{0} {1} {2}" -f (Get-Date).ToString('o'), $Kind, $Text) +} + +# ---- paths ------------------------------------------------------------------------------------- + +if (-not ('LpNative' -as [type])) { + Add-Type -Namespace '' -Name 'LpNative' -MemberDefinition @' +[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] +public static extern uint GetLongPathName(string shortPath, System.Text.StringBuilder longPath, uint bufferLength); +'@ +} + +# Absolute, long-form spelling of an existing path. One string, used everywhere afterwards. +function Resolve-LpPath([string]$Path) { + if (-not [System.IO.Path]::IsPathRooted($Path)) { throw "path is not absolute: $Path" } + $full = [System.IO.Path]::GetFullPath($Path).TrimEnd('\') + if (-not (Test-Path -LiteralPath $full)) { throw "path does not exist: $Path" } + $sb = [System.Text.StringBuilder]::new(1024) + $n = [LpNative]::GetLongPathName($full, $sb, [uint32]$sb.Capacity) + if ($n -gt 0 -and $n -lt $sb.Capacity) { $full = $sb.ToString().TrimEnd('\') } + return $full +} + +# True when the path itself or any parent folder is a junction, symbolic link or other reparse point. +function Test-LpReparseInPath([string]$Path) { + $p = $Path + while ($p) { + if (Test-Path -LiteralPath $p) { + $item = Get-Item -LiteralPath $p -Force + if ($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { return $true } + } + $parent = [System.IO.Path]::GetDirectoryName($p) + if (-not $parent -or $parent -eq $p) { break } + $p = $parent + } + return $false +} + +function Test-LpReparsePoint([string]$Path) { + return [bool]((Get-Item -LiteralPath $Path -Force).Attributes -band [System.IO.FileAttributes]::ReparsePoint) +} + +function ConvertTo-LpComparablePath([string]$Path) { + $p = $Path -replace '/', '\' + if ($p.StartsWith('\\?\')) { $p = $p.Substring(4) } + return $p.TrimEnd('\') +} + +# True when $Child is $Parent or lies below it. Pure string comparison on comparable forms. +function Test-LpPathWithin([string]$Child, [string]$Parent) { + $c = ConvertTo-LpComparablePath $Child + $p = ConvertTo-LpComparablePath $Parent + if ($c.Equals($p, [System.StringComparison]::OrdinalIgnoreCase)) { return $true } + return $c.StartsWith($p + '\', [System.StringComparison]::OrdinalIgnoreCase) +} + +# Pure guard for the test seam: is this runtime root inside the live installation area? +# The caller supplies both forms of the root (as given and long form) and the roaming base. +function Test-LpLiveArea([string[]]$RootForms, [string]$RoamingBase) { + $area = (ConvertTo-LpComparablePath $RoamingBase) + '\CindyGlobal' + foreach ($form in $RootForms) { if (Test-LpPathWithin $form $area) { return $true } } + return $false +} + +function Get-LpRelative([string]$Path) { + if ($script:LpRoot -and (Test-LpPathWithin $Path $script:LpRoot)) { + $rest = (ConvertTo-LpComparablePath $Path).Substring((ConvertTo-LpComparablePath $script:LpRoot).Length).TrimStart('\') + if ($rest) { return $rest } else { return '.' } + } + return $Path +} + +function Test-LpSafeText([string]$Value) { + if ($null -eq $Value) { return $false } + foreach ($ch in $Value.ToCharArray()) { + if ([int]$ch -gt 126 -or [int]$ch -lt 32) { return $false } + if ($ch -eq "'" -or $ch -eq '`' -or $ch -eq '$') { return $false } + } + return $true +} + +# ---- hashing and atomic writes ----------------------------------------------------------------- + +function Get-LpSha256([string]$Path) { return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToUpperInvariant() } + +function Get-LpSha256OfBytes([byte[]]$Bytes) { + $sha = [System.Security.Cryptography.SHA256]::Create() + try { return ([System.BitConverter]::ToString($sha.ComputeHash($Bytes)) -replace '-', '').ToUpperInvariant() } finally { $sha.Dispose() } +} + +function Test-LpSameHash([string]$A, [string]$B) { return [string]::Equals($A, $B, [System.StringComparison]::OrdinalIgnoreCase) } + +# Write to a temporary file beside the target, verify, then rename over the target. +function Write-LpBytesAtomic([string]$Path, [byte[]]$Bytes) { + $tmp = "$Path.lp-tmp" + [System.IO.File]::WriteAllBytes($tmp, $Bytes) + try { + if (-not (Test-LpSameHash (Get-LpSha256 $tmp) (Get-LpSha256OfBytes $Bytes))) { throw "temporary file for $(Get-LpRelative $Path) does not match what was written" } + [System.IO.File]::Move($tmp, $Path, $true) + } finally { + if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } + } + if (-not (Test-LpSameHash (Get-LpSha256 $Path) (Get-LpSha256OfBytes $Bytes))) { throw "$(Get-LpRelative $Path) does not match what was written" } +} + +function Write-LpJsonAtomic([string]$Path, $Object) { + Write-LpBytesAtomic $Path ($script:LpUtf8.GetBytes(($Object | ConvertTo-Json -Depth 6))) +} + +# ---- launcher text ----------------------------------------------------------------------------- + +# Reads the release identity defaults out of the launcher text. Throws when a line is missing or repeated. +function Read-LpLauncherIdentity([string]$Text) { + $lines = $Text -split "`n" + $shapes = [ordered]@{ + ProxyPath = '^(\s*)\[string\]\$ProxyPath = ''([^'']*)'',(\r?)$' + ExpectedCommit = '^(\s*)\[string\]\$ExpectedCommit = ''([^'']*)'',(\r?)$' + ExpectedVersion = '^(\s*)\[string\]\$ExpectedVersion = ''([^'']*)'',(\r?)$' + Port = '^(\s*)\[int\]\$Port = (\d+),(\r?)$' + } + $found = @{} + foreach ($name in $shapes.Keys) { + $hits = @(for ($i = 0; $i -lt $lines.Count; $i++) { if ($lines[$i] -match $shapes[$name]) { [pscustomobject]@{ Index = $i; Indent = $Matches[1]; Value = $Matches[2]; Cr = $Matches[3] } } }) + if ($hits.Count -ne 1) { throw "launcher has $($hits.Count) lines defining $name, expected exactly one" } + $found[$name] = $hits[0] + } + return $found +} + +# Returns the edited text and the names of the defaults that changed. +function New-LpEditedLauncher([string]$Text, [string]$ProxyPath, [string]$Commit, [string]$Version) { + $id = Read-LpLauncherIdentity $Text + $lines = $Text -split "`n" + $changed = @() + $want = [ordered]@{ ProxyPath = $ProxyPath; ExpectedCommit = $Commit; ExpectedVersion = $Version } + foreach ($name in $want.Keys) { + $line = $id[$name] + if ($line.Value -ceq $want[$name]) { continue } + $lines[$line.Index] = "$($line.Indent)[string]`$$name = '$($want[$name])',$($line.Cr)" + $changed += $name + } + $new = $lines -join "`n" + # Positional comparison: nothing but the intended lines may differ. + $old = $Text -split "`n" + $diff = @(for ($i = 0; $i -lt $old.Count; $i++) { if ($old[$i] -cne $lines[$i]) { $i } }) + if ($old.Count -ne $lines.Count -or $diff.Count -ne $changed.Count) { throw "launcher edit would change $($diff.Count) lines, expected $($changed.Count)" } + return @{ Text = $new; Changed = $changed } +} + +function Test-LpPlainAscii([byte[]]$Bytes) { + foreach ($b in $Bytes) { if ($b -gt 127) { return $false } } + return $true +} + +# Runs launcher text under Windows PowerShell with -ReportOnly, from a private temporary folder. +function Invoke-LpLauncherDryRun([byte[]]$LauncherBytes, [string]$RuntimeRoot) { + $dir = Join-Path ([System.IO.Path]::GetTempPath()) ("lp-dryrun-" + [System.Guid]::NewGuid().ToString('N')) + [void][System.IO.Directory]::CreateDirectory($dir) + try { + $file = Join-Path $dir 'launcher-candidate.ps1' + [System.IO.File]::WriteAllBytes($file, $LauncherBytes) + $want = Get-LpSha256OfBytes $LauncherBytes + if (-not (Test-LpSameHash (Get-LpSha256 $file) $want)) { throw 'dry-run copy of the launcher does not match' } + $ps = Join-Path $env:WINDIR 'System32\WindowsPowerShell\v1.0\powershell.exe' + if (-not (Test-Path -LiteralPath $ps)) { throw 'Windows PowerShell was not found' } + if (-not (Test-LpSameHash (Get-LpSha256 $file) $want)) { throw 'dry-run copy of the launcher changed before it ran' } + $out = (& $ps -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $file -RuntimeRoot $RuntimeRoot -ReportOnly 2>&1 | Out-String) + $code = $LASTEXITCODE + $action = $null + foreach ($line in ($out -split "`r?`n")) { + $t = $line.Trim() + if ($t.StartsWith('{') -and $t.EndsWith('}')) { + try { $j = $t | ConvertFrom-Json; if ($j.PSObject.Properties['action']) { $action = [string]$j.action } } catch { } + } + } + return @{ ExitCode = $code; Action = $action } + } finally { + Remove-Item -LiteralPath $dir -Recurse -Force -ErrorAction SilentlyContinue + } +} + +# ---- probes (inspection errors propagate; they never read as "free" or "idle") ------------------- + +function Get-LpConnections([int]$Port, [int]$State) { + return @(Get-CimInstance -Namespace ROOT/StandardCimv2 -ClassName MSFT_NetTCPConnection -Filter "LocalPort=$Port AND State=$State" -OperationTimeoutSec 15 -ErrorAction Stop) +} +# PowerShell unrolls a one-item list on return: callers wrap these calls in @( ) before counting. +function Get-LpListenerPids([int]$Port) { return @(Get-LpConnections $Port 2 | Select-Object -ExpandProperty OwningProcess -Unique) } +# Open client connections to the port. A client is a loopback connection whose owning process is neither +# this script nor the supervisor (both only probe health); the client side of each connection names its owner. +function Get-LpClientCount($Ctx) { + $own = @([int]$PID) + @(Get-LpSupervisors $Ctx | ForEach-Object { [int]$_.ProcessId }) + $rows = @(Get-CimInstance -Namespace ROOT/StandardCimv2 -ClassName MSFT_NetTCPConnection -Filter "RemotePort=$($Ctx.Port) AND State=5" -OperationTimeoutSec 15 -ErrorAction Stop) + return @($rows | Where-Object { @('127.0.0.1', '::1') -contains [string]$_.RemoteAddress -and $own -notcontains [int]$_.OwningProcess }).Count +} + +# Processes of one image whose command line contains the given path as a quoted token. +function Get-LpProcesses([string]$ImageName, [string]$QuotedPath) { + $token = '"' + $QuotedPath + '"' + return @(Get-CimInstance Win32_Process -Filter "Name='$ImageName'" -OperationTimeoutSec 15 -ErrorAction Stop | + Where-Object { $null -ne $_.CommandLine -and $_.CommandLine.IndexOf($token, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 }) +} +function Get-LpSupervisors($Ctx) { return @(Get-LpProcesses 'powershell.exe' $Ctx.LauncherPath) } +function Get-LpProxies([string]$ProxyPath) { return @(Get-LpProcesses 'node.exe' $ProxyPath) } + +# Stops one process after re-checking that the PID still belongs to the process that was matched. +function Stop-LpProcessVerified($Process) { + $now = Get-CimInstance Win32_Process -Filter "ProcessId=$($Process.ProcessId)" -OperationTimeoutSec 15 -ErrorAction Stop + if ($null -eq $now) { return } + if ($now.CreationDate -ne $Process.CreationDate) { throw "pid $($Process.ProcessId) now belongs to a different process; not stopping it" } + Stop-Process -Id $Process.ProcessId -Force -ErrorAction Stop +} + +function Wait-LpUntil([scriptblock]$Condition, [int]$Seconds, [string]$What) { + $watch = [System.Diagnostics.Stopwatch]::StartNew() + $nextNote = 10 + while ($true) { + if (& $Condition) { return $true } + if ($watch.Elapsed.TotalSeconds -ge $Seconds) { return $false } + if ($watch.Elapsed.TotalSeconds -ge $nextNote) { Write-LpEvent 'WAIT' "$What ($([int]$watch.Elapsed.TotalSeconds) of $Seconds s)"; $nextNote += 10 } + Start-Sleep -Milliseconds $script:LpPollMs + } +} + +function Get-LpHealth([int]$Port) { + try { + $h = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/__bili/health" -TimeoutSec 5 -NoProxy -MaximumRedirection 0 + return [pscustomobject]@{ ok = ($h.ok -eq $true); version = [string]$h.version; commit = [string]$h.commit; pid = [string]$h.pid; error = $null } + } catch { + return [pscustomobject]@{ ok = $false; version = ''; commit = ''; pid = ''; error = $_.Exception.GetType().Name } + } +} +function Format-LpHealth($Health) { + if ($Health.error) { return "unreadable ($($Health.error))" } + return "ok=$($Health.ok) version=$($Health.version) commit=$($Health.commit) pid=$($Health.pid)" +} + +# One probe of the installation lock. Opens the existing file only; never creates it. +function Test-LpLockHeld([string]$LockPath) { + if (-not (Test-Path -LiteralPath $LockPath)) { return $false } + try { + $s = [System.IO.File]::Open($LockPath, [System.IO.FileMode]::Open, [System.IO.FileAccess]::ReadWrite, [System.IO.FileShare]::None) + $s.Dispose() + return $false + } catch [System.IO.IOException] { + return $true + } +} + +# ---- scheduled task (default driver; a test driver may replace these four functions) ------------- + +function Get-LpTaskObjects([string]$TaskName) { return @(Get-ScheduledTask -ErrorAction Stop | Where-Object { $_.TaskName -eq $TaskName }) } +function Get-LiveProxyTaskInfo([string]$TaskName) { + $t = @(Get-LpTaskObjects $TaskName) + if ($t.Count -ne 1) { return [pscustomobject]@{ Count = $t.Count; Enabled = $false; Execute = ''; ActionArguments = ''; TriggerCount = 0 } } + $a = @($t[0].Actions) + return [pscustomobject]@{ + Count = 1; Enabled = [bool]$t[0].Settings.Enabled + Execute = [string]($a | Select-Object -First 1).Execute; ActionArguments = (($a | ForEach-Object { [string]$_.Arguments }) -join ' | ') + TriggerCount = @($t[0].Triggers).Count + } +} +function Disable-LiveProxyTask([string]$TaskName) { Get-LpTaskObjects $TaskName | Disable-ScheduledTask -ErrorAction Stop | Out-Null } +function Enable-LiveProxyTask([string]$TaskName) { Get-LpTaskObjects $TaskName | Enable-ScheduledTask -ErrorAction Stop | Out-Null } +function Start-LiveProxyTask([string]$TaskName) { Get-LpTaskObjects $TaskName | Start-ScheduledTask -ErrorAction Stop } + +# Decides which task driver may be used for this runtime root. Returns the driver path to dot-source, or $null. +function Resolve-LpTaskDriver([string]$RootAsGiven, [string]$RootResolved, [string]$TestTaskDriver) { + $marked = Test-Path -LiteralPath (Join-Path $RootResolved 'sandbox.marker') + if (-not $TestTaskDriver) { + if ($marked) { throw 'this runtime root is a test sandbox (sandbox.marker is present): the real scheduled task may not be used with it' } + return $null + } + if (-not $marked) { throw '-TestTaskDriver is accepted only for a test sandbox (sandbox.marker is absent)' } + $roaming = [System.Environment]::GetFolderPath([System.Environment+SpecialFolder]::ApplicationData) + if (Test-LpLiveArea @($RootAsGiven, $RootResolved) $roaming) { throw '-TestTaskDriver is refused for a runtime root inside the live installation area' } + $driver = Resolve-LpPath $TestTaskDriver + if ((Get-Item -LiteralPath $driver -Force).PSIsContainer -or (Test-LpReparseInPath $driver)) { throw '-TestTaskDriver must be a regular file with no link in its path' } + $script:LpSeam = $true + return $driver +} + +# ---- hold and lift ----------------------------------------------------------------------------- + +function Get-LpInhibitText($Ctx) { + if (-not (Test-Path -LiteralPath $Ctx.InhibitPath)) { return $null } + return ([System.IO.File]::ReadAllText($Ctx.InhibitPath)).Trim() +} +function Test-LpOwnInhibit($Ctx) { + $t = Get-LpInhibitText $Ctx + return ($null -ne $t -and ($t -ceq "deploy $($Ctx.DeploymentId)" -or $t -ceq "rollback $($Ctx.DeploymentId)")) +} + +# Creates the inhibit file; fails when one exists (create-new). +function New-LpInhibit($Ctx, [string]$Verb) { + $bytes = $script:LpUtf8.GetBytes("$Verb $($Ctx.DeploymentId)") + $s = [System.IO.File]::Open($Ctx.InhibitPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) + try { $s.Write($bytes, 0, $bytes.Length) } finally { $s.Dispose() } +} + +function Assert-LpNoSupervisor($Ctx, [string]$When) { + $free = Wait-LpUntil { @(Get-LpSupervisors $Ctx).Count -eq 0 } $Ctx.StopWait 'waiting for the supervisor to stop' + if (-not $free) { throw "a supervisor is still running $When" } + if (Test-LpLockHeld $Ctx.LockPath) { throw "the installation lock is still held $When" } +} + +# Hold used by rollback: tolerates zero supervisors, does not check health. +function Set-LpRollbackHold($Ctx) { + $text = Get-LpInhibitText $Ctx + if ($null -eq $text) { New-LpInhibit $Ctx 'rollback' } + elseif (-not (Test-LpOwnInhibit $Ctx)) { throw 'maintenance.inhibit belongs to other maintenance' } + Disable-LiveProxyTask $Ctx.TaskName + foreach ($s in (Get-LpSupervisors $Ctx)) { Stop-LpProcessVerified $s } + Assert-LpNoSupervisor $Ctx 'under the rollback hold' + Write-LpEvent 'STEP' 'hold applied: inhibit set, task disabled, no supervisor' +} + +function Remove-LpHold($Ctx) { + Assert-LpNoSupervisor $Ctx 'before the hold is lifted' + Remove-Item -LiteralPath $Ctx.InhibitPath -Force + $Ctx.LiftAt = (Get-Date).ToUniversalTime() + Enable-LiveProxyTask $Ctx.TaskName + Start-LiveProxyTask $Ctx.TaskName + Write-LpEvent 'STEP' 'hold lifted: inhibit removed, task enabled and started' +} + +# ---- success test (step 6 and step 9) ----------------------------------------------------------- + +# One evaluation. Returns $null when every condition holds, otherwise the first reason it does not. +function Get-LpSuccessGap($Ctx, $Identity) { + $listeners = @(Get-LpListenerPids $Ctx.Port) + if ($listeners.Count -ne 1) { return "listeners on the port: $($listeners.Count)" } + $listener = [string]$listeners[0] + $h = Get-LpHealth $Ctx.Port + if (-not $h.ok -or $h.version -cne $Identity.version -or $h.commit -cne $Identity.commit -or $h.pid -ne $listener) { return "health $(Format-LpHealth $h), expected version=$($Identity.version) commit=$($Identity.commit) from pid $listener" } + $proxies = @(Get-LpProxies $Identity.proxyPath) + if ($proxies.Count -ne 1 -or [string]$proxies[0].ProcessId -ne $listener) { return "processes running the expected release: $($proxies.Count), listener pid $listener" } + $sups = @(Get-LpSupervisors $Ctx) + if ($sups.Count -ne 1) { return "supervisors: $($sups.Count)" } + if (-not (Test-Path -LiteralPath $Ctx.StatusPath)) { return 'supervisor-status.json is absent' } + try { $st = [System.IO.File]::ReadAllText($Ctx.StatusPath) | ConvertFrom-Json -AsHashtable } catch { return 'supervisor-status.json is unreadable' } + $beat = $null + try { $beat = ([datetime]$st['heartbeatAt']).ToUniversalTime() } catch { return 'supervisor-status.json has no heartbeat time' } + if ($beat -le $Ctx.LiftAt) { return 'no supervisor heartbeat since the hold was lifted' } + if ([string]$st['supervisorPid'] -ne [string]$sups[0].ProcessId) { return 'status file was written by another supervisor' } + if (@('adopted', 'spawned') -notcontains [string]$st['state']) { return "supervisor state is $($st['state'])" } + if ($st['healthMatch'] -ne $true) { return 'supervisor reports no health match' } + if ($st['inhibit'] -eq $true) { return 'supervisor reports the inhibit' } + if ([string]$st['childPid'] -ne $listener) { return 'supervisor monitors a process that is not the listener' } + if (Test-Path -LiteralPath $Ctx.InhibitPath) { return 'maintenance.inhibit is present' } + if (-not (Get-LiveProxyTaskInfo $Ctx.TaskName).Enabled) { return 'the task is not enabled' } + if (-not (Test-LpSameHash (Get-LpSha256 $Ctx.ConfigPath) $Ctx.ConfigSha)) { return 'the configuration file changed' } + return $null +} + +# Polls the success test, then probes the lock once and compares the task with its baseline. +function Test-LpSuccess($Ctx, $Identity) { + $gap = [ref]'not evaluated' + $ok = Wait-LpUntil { $gap.Value = Get-LpSuccessGap $Ctx $Identity; $null -eq $gap.Value } $Ctx.HealthWait "waiting for $($Identity.commit) under supervision" + if (-not $ok) { return $gap.Value } + if (-not (Test-LpLockHeld $Ctx.LockPath)) { return 'the installation lock is not held' } + $t = Get-LiveProxyTaskInfo $Ctx.TaskName + $b = $Ctx.TaskBaseline + if ($t.Count -ne 1 -or $t.Execute -ne $b.Execute -or $t.ActionArguments -ne $b.ActionArguments -or $t.TriggerCount -ne $b.TriggerCount) { return 'the scheduled task definition changed' } + return $null +} + +# ---- record, result, evidence ------------------------------------------------------------------- + +function Save-LpRecord($Ctx) { Write-LpJsonAtomic $Ctx.RecordPath $Ctx.Record } + +function Set-LpStage($Ctx, [string]$Stage) { + if ($script:LpForwardStages -contains $Stage) { $Ctx.Record.reachedStage = $Stage } + $Ctx.Record.stage = $Stage + Save-LpRecord $Ctx +} + +function Test-LpReached($Ctx, [string]$Stage) { + return ([array]::IndexOf($script:LpForwardStages, [string]$Ctx.Record.reachedStage) -ge [array]::IndexOf($script:LpForwardStages, $Stage)) +} + +function Write-LpEvidence($Ctx, [string]$FailingStep, [string[]]$HoldGaps) { + $lines = [System.Collections.Generic.List[string]]::new() + $add = { param($label, [scriptblock]$get) try { $lines.Add("${label}: $(& $get)") } catch { $lines.Add("${label}: could not be read ($($_.Exception.GetType().Name))") } } + $lines.Add("failing step: $FailingStep") + if ($HoldGaps.Count -gt 0) { $lines.Add("hold NOT fully in place: $($HoldGaps -join '; ')") } else { $lines.Add('hold in place: inhibit set, task disabled, no supervisor') } + & $add 'health' { Format-LpHealth (Get-LpHealth $Ctx.Port) } + & $add 'listener pids' { (Get-LpListenerPids $Ctx.Port) -join ',' } + & $add 'supervisors' { ((Get-LpSupervisors $Ctx) | ForEach-Object { "pid $($_.ProcessId) created $($_.CreationDate.ToString('o'))" }) -join '; ' } + foreach ($which in 'Previous', 'Candidate') { + $path = $Ctx[$which].proxyPath + & $add "$($which.ToLower()) release processes" { ((Get-LpProxies $path) | ForEach-Object { "pid $($_.ProcessId) created $($_.CreationDate.ToString('o'))" }) -join '; ' } + } + & $add 'inhibit' { $t = Get-LpInhibitText $Ctx; if ($null -eq $t) { 'absent' } else { "present ($t)" } } + & $add 'task enabled' { (Get-LiveProxyTaskInfo $Ctx.TaskName).Enabled } + & $add 'installation lock held' { Test-LpLockHeld $Ctx.LockPath } + & $add 'launcher matches the pre-deployment launcher' { Test-LpSameHash (Get-LpSha256 $Ctx.LauncherPath) $Ctx.Record.launcherSha256Before } + $lines.Add('logs to attach: supervisor.log, state\billion-context\bili.log') + foreach ($l in $lines) { Write-LpEvent 'STEP' "evidence: $l" } + try { + $file = Join-Path $Ctx.BackupRoot ("evidence-{0}.txt" -f (Get-Date -Format 'yyyyMMdd-HHmmss')) + [System.IO.File]::WriteAllLines($file, $lines, $script:LpUtf8) + return $file + } catch { return $null } +} + +# Re-applies the hold after a failed rollback and stops. Touches no proxy process. +function Stop-LpRollbackFailed($Ctx, [string]$FailingStep, [string]$Reason) { + $gaps = @() + try { + $text = Get-LpInhibitText $Ctx + if ($null -eq $text) { New-LpInhibit $Ctx 'rollback' } elseif (-not (Test-LpOwnInhibit $Ctx)) { $gaps += 'inhibit present but owned by other maintenance' } + } catch { $gaps += 'inhibit could not be set' } + try { Disable-LiveProxyTask $Ctx.TaskName } catch { $gaps += 'task could not be disabled' } + try { + foreach ($s in (Get-LpSupervisors $Ctx)) { Stop-LpProcessVerified $s } + if (-not (Wait-LpUntil { @(Get-LpSupervisors $Ctx).Count -eq 0 } $Ctx.StopWait 'waiting for the supervisor to stop')) { $gaps += 'a supervisor is still running' } + } catch { $gaps += 'supervisor could not be stopped' } + try { Set-LpStage $Ctx 'rollback-failed' } catch { Write-LpEvent 'STEP' 'the deployment record could not be updated' } + $file = Write-LpEvidence $Ctx $FailingStep $gaps + $next = "report to the owner with $(if ($file) { $file } else { 'the evidence lines above' }); no release is confirmed live and no further start was attempted. To retry: Rollback-LiveProxy.ps1 -RuntimeRoot -BackupRoot `"$($Ctx.BackupRoot)`". To lift the hold by hand if the owner judges the proxy fine: remove maintenance.inhibit, enable the task, start the task. A later deployment needs a new -ReleaseName." + throw [LpStop]::new(4, 'rollback-failed', "rollback failed at: $FailingStep", $Reason, $next) +} + +# Prints NEXT and RESULT, writes the result file, and ends the process. +function Complete-Lp($Ctx, [LpStop]$Stop, [string]$ResultFileName) { + $seam = if ($script:LpSeam) { ' TEST SEAM ACTIVE' } else { '' } + $reason = if ($Stop.Reason) { " reason=$($Stop.Reason)" } else { '' } + if ($Stop.Message) { Write-LpEvent 'STEP' $Stop.Message } + if ($Stop.Next) { Write-LpEvent 'NEXT' $Stop.Next } + if ($Ctx -and $Ctx.BackupRoot -and $ResultFileName -and (Test-Path -LiteralPath $Ctx.BackupRoot)) { + try { + Write-LpJsonAtomic (Join-Path $Ctx.BackupRoot $ResultFileName) ([ordered]@{ + result = $Stop.Name; exit = $Stop.Code; reason = $Stop.Reason; stage = $Ctx.Record.stage; backup = $Ctx.BackupRoot + previous = $Ctx.Previous; candidate = $Ctx.Candidate; at = (Get-Date).ToString('o') + }) + } catch { Write-LpEvent 'STEP' 'the result file could not be written' } + } + Write-LpEvent 'RESULT' "$($Stop.Name) (exit $($Stop.Code))$reason$seam" + exit $Stop.Code +} + +# ---- rollback core (deploy-time and standalone) -------------------------------------------------- + +# $RestoreLauncherBytes: the pre-deployment launcher. $Receipts: @{ path = bytes } to restore, or empty. +# Returns normally when the previous release is healthy under supervision; otherwise throws LpStop 4. +function Invoke-LpRollback($Ctx, [byte[]]$RestoreLauncherBytes, [hashtable]$Receipts, [bool]$SkipHoldAndRestore) { + Write-LpEvent 'ROLLBACK' "from $($Ctx.Candidate.releaseName) ($($Ctx.Candidate.version), $($Ctx.Candidate.commit)) to $($Ctx.Previous.version), $($Ctx.Previous.commit)" + $step = 'rollback hold' + try { + if ($SkipHoldAndRestore) { + $step = 'launcher check' + if (-not (Test-LpSameHash (Get-LpSha256 $Ctx.LauncherPath) $Ctx.Record.launcherSha256Before)) { throw 'the launcher is not the pre-deployment launcher' } + } else { + Set-LpRollbackHold $Ctx + $step = 'stop candidate release' + $stopped = $false + foreach ($p in (Get-LpProxies $Ctx.Candidate.proxyPath)) { Stop-LpProcessVerified $p; $stopped = $true } + if ($stopped) { + if (-not (Wait-LpUntil { @(Get-LpProxies $Ctx.Candidate.proxyPath).Count -eq 0 } $Ctx.StopWait 'waiting for the candidate release to stop')) { throw 'the candidate release did not stop' } + if (@(Get-LpProxies $Ctx.Previous.proxyPath).Count -eq 0 -and -not (Wait-LpUntil { @(Get-LpListenerPids $Ctx.Port).Count -eq 0 } $Ctx.StopWait 'waiting for the port')) { throw 'the port was not released' } + Write-LpEvent 'STEP' 'candidate release stopped' + } + $step = 'launcher restore' + if (Test-LpSameHash (Get-LpSha256 $Ctx.LauncherPath) $Ctx.Record.launcherSha256Before) { + Write-LpEvent 'STEP' 'launcher already is the pre-deployment launcher' + } else { + if (-not (Test-LpSameHash (Get-LpSha256OfBytes $RestoreLauncherBytes) $Ctx.Record.launcherSha256Before)) { throw 'the saved launcher does not match the deployment record' } + Write-LpBytesAtomic $Ctx.LauncherPath $RestoreLauncherBytes + Write-LpEvent 'STEP' 'launcher restored and verified' + } + $step = 'receipt restore' + foreach ($path in $Receipts.Keys) { + if (Test-LpSameHash (Get-LpSha256 $path) (Get-LpSha256OfBytes $Receipts[$path])) { continue } + Write-LpBytesAtomic $path $Receipts[$path] + Write-LpEvent 'STEP' "$(Get-LpRelative $path) restored and verified" + } + } + $step = 'lift after rollback' + if ($SkipHoldAndRestore) { + # The hold was never completed: remove this deployment's inhibit and re-enable supervision as it was. + if (Test-LpOwnInhibit $Ctx) { Remove-Item -LiteralPath $Ctx.InhibitPath -Force } + $Ctx.LiftAt = (Get-Date).ToUniversalTime() + Enable-LiveProxyTask $Ctx.TaskName + Start-LiveProxyTask $Ctx.TaskName + Write-LpEvent 'STEP' 'inhibit removed, task enabled and started' + } else { + Remove-LpHold $Ctx + } + } catch [LpStop] { throw } catch { + Stop-LpRollbackFailed $Ctx "$step ($($_.Exception.Message))" 'rollback-error' + } + $gap = $null + try { $gap = Test-LpSuccess $Ctx $Ctx.Previous } catch { $gap = "check failed ($($_.Exception.Message))" } + if ($null -ne $gap) { Stop-LpRollbackFailed $Ctx "previous release check: $gap" 'previous-unhealthy' } + Write-LpEvent 'STEP' "previous release healthy under supervision: $($Ctx.Previous.version), $($Ctx.Previous.commit)" +} diff --git a/scripts/live-deploy/Rollback-LiveProxy.ps1 b/scripts/live-deploy/Rollback-LiveProxy.ps1 new file mode 100644 index 0000000000..a4cdd3a23c --- /dev/null +++ b/scripts/live-deploy/Rollback-LiveProxy.ps1 @@ -0,0 +1,143 @@ +#requires -Version 7.0 +<# +.SYNOPSIS +Rolls one live proxy deployment back to the release it replaced, from that deployment's backup folder. + +.DESCRIPTION +Undoes a deployment made by Deploy-LiveProxy.ps1 (step 8 of "Deploy only with an explicit request" in +reference/cindy-project.md): holds supervision, stops the deployed release, restores the launcher and +the receipts from the backup folder, lets the supervisor start the previous release and checks it. +Everything it needs except the two paths comes from deploy-record.json in the backup folder. + +If the previous release does not come up healthy, the hold is put back and nothing further is started. + +Exit codes: 0 restored (or preflight passed), 2 refused (nothing changed), 4 rollback failed (hold in +place). Any other exit code means the state is unknown: run again with -PreflightOnly. + +.EXAMPLE +pwsh -NoProfile -NonInteractive -File Rollback-LiveProxy.ps1 -RuntimeRoot "" -BackupRoot "\backups\" -PreflightOnly +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string]$RuntimeRoot, + [Parameter(Mandatory)] [string]$BackupRoot, + [int]$HealthWaitSeconds = 90, + [int]$StopWaitSeconds = 20, + [string]$TestTaskDriver, + [switch]$PreflightOnly +) +. (Join-Path $PSScriptRoot 'LiveProxy.Common.ps1') + +function Stop-Refused([string]$Why) { throw [LpStop]::new(2, 'refused', "refused: $Why; nothing was changed", '', 'see the reason above') } + +# Strict shape check of the deployment record: exact keys, expected formats. +function Assert-RecordShape($Rec) { + $keys = @('schema', 'deploymentId', 'createdAt', 'runtimeRoot', 'port', 'taskName', 'previous', 'candidate', 'nodeModules', 'configSha256', 'launcherSha256Before', 'launcherSha256After', 'deployProcess', 'created', 'stage', 'reachedStage') + $sub = @{ + previous = @('proxyPath', 'version', 'commit'); candidate = @('releaseName', 'proxyPath', 'version', 'commit', 'sourceCommit', 'sha256') + nodeModules = @('linkType', 'target'); deployProcess = @('pid', 'created'); created = @('releaseDir', 'backupDir') + } + $same = { param($have, $want) (@($have | Sort-Object) -join ',') -ceq (@($want | Sort-Object) -join ',') } + if (-not ($Rec -is [System.Collections.IDictionary]) -or -not (& $same $Rec.Keys $keys)) { throw 'unexpected or missing fields' } + foreach ($k in $sub.Keys) { if (-not ($Rec[$k] -is [System.Collections.IDictionary]) -or -not (& $same $Rec[$k].Keys $sub[$k])) { throw "unexpected or missing fields in $k" } } + if ($Rec.schema -ne 1) { throw 'unknown schema version' } + if ($script:LpStages -notcontains [string]$Rec.stage -or $script:LpForwardStages -notcontains [string]$Rec.reachedStage) { throw 'unknown stage' } + foreach ($side in 'previous', 'candidate') { + if ([string]$Rec[$side].version -notmatch $script:LpPatterns.Version -or [string]$Rec[$side].commit -cnotmatch $script:LpPatterns.Commit) { throw "malformed $side identity" } + if (-not (Test-LpSafeText ([string]$Rec[$side].proxyPath))) { throw "malformed $side path" } + } + if ([string]$Rec.candidate.releaseName -notmatch $script:LpPatterns.ReleaseName) { throw 'malformed release name' } + foreach ($h in 'launcherSha256Before', 'launcherSha256After', 'configSha256') { if ([string]$Rec[$h] -notmatch $script:LpPatterns.Sha256) { throw "malformed $h" } } + if (-not ($Rec.port -is [int] -or $Rec.port -is [long]) -or $Rec.port -lt 1 -or $Rec.port -gt 65535) { throw 'malformed port' } + if (-not (Test-LpSafeText ([string]$Rec.taskName)) -or [string]$Rec.taskName -match '[*?\[\]]') { throw 'malformed task name' } +} + +$ctx = $null +try { + $root = Resolve-LpPath $RuntimeRoot + if (Test-LpReparseInPath $root) { Stop-Refused 'a folder in the runtime root path is a link' } + $script:LpRoot = $root + $driver = $null + try { $driver = Resolve-LpTaskDriver $RuntimeRoot $root $TestTaskDriver } catch { Stop-Refused $_.Exception.Message } + if ($driver) { . $driver; Write-LpEvent 'STEP' 'TEST SEAM ACTIVE: scheduled-task operations go to the test driver' } + + $backup = $null + try { $backup = Resolve-LpPath $BackupRoot } catch { Stop-Refused 'the backup folder does not exist' } + if ([System.IO.Path]::GetDirectoryName($backup) -ne (Join-Path $root 'backups')) { Stop-Refused 'the backup folder is not directly under \backups' } + $recordPath = Join-Path $backup 'deploy-record.json' + $restoreNames = @('Start-BillionContextProxy.ps1', 'tuned-deployment.json', 'supervision-receipt.json') + foreach ($p in @($backup, $recordPath) + @($restoreNames | ForEach-Object { Join-Path $backup $_ })) { + if (-not (Test-Path -LiteralPath $p)) { Stop-Refused "$(Get-LpRelative $p) is missing" } + if (Test-LpReparsePoint $p) { Stop-Refused "$(Get-LpRelative $p) is a link" } + } + $rec = $null + try { $rec = [System.IO.File]::ReadAllText($recordPath) | ConvertFrom-Json -AsHashtable; Assert-RecordShape $rec } catch { Stop-Refused "the deployment record is not valid ($($_.Exception.Message))" } + if (-not [string]::Equals([string]$rec.runtimeRoot, $root, [System.StringComparison]::OrdinalIgnoreCase)) { Stop-Refused 'the deployment record belongs to another runtime root' } + if ([string]$rec.deploymentId -cne [System.IO.Path]::GetFileName($backup)) { Stop-Refused 'the deployment record does not belong to this backup folder' } + $releasesDir = Join-Path $root 'releases' + foreach ($side in 'previous', 'candidate') { + $pp = [string]$rec[$side].proxyPath + $relDir = [System.IO.Path]::GetDirectoryName([System.IO.Path]::GetDirectoryName($pp)) + if ([System.IO.Path]::GetFullPath($pp) -ne $pp -or [System.IO.Path]::GetDirectoryName($relDir) -ne $releasesDir -or [System.IO.Path]::GetFileName($pp) -ne 'index.mjs') { Stop-Refused "the $side release path in the record is not \releases\\dist\index.mjs" } + } + if (-not (Test-Path -LiteralPath ([string]$rec.previous.proxyPath) -PathType Leaf)) { Stop-Refused 'the previous release bundle is missing' } + + $ctx = @{ + Root = $root; Port = [int]$rec.port; TaskName = [string]$rec.taskName; HealthWait = $HealthWaitSeconds; StopWait = $StopWaitSeconds + LauncherPath = Join-Path $root 'Start-BillionContextProxy.ps1'; InhibitPath = Join-Path $root 'maintenance.inhibit' + LockPath = Join-Path $root 'supervisor.lock'; StatusPath = Join-Path $root 'supervisor-status.json' + ConfigPath = Join-Path $root 'config\billion-context\billion-context.json'; ConfigSha = $null + Previous = [ordered]@{ proxyPath = [string]$rec.previous.proxyPath; version = [string]$rec.previous.version; commit = [string]$rec.previous.commit } + Candidate = [ordered]@{ releaseName = [string]$rec.candidate.releaseName; proxyPath = [string]$rec.candidate.proxyPath; version = [string]$rec.candidate.version; commit = [string]$rec.candidate.commit } + BackupRoot = $backup; DeploymentId = [string]$rec.deploymentId; Record = $rec; RecordPath = $recordPath; LiftAt = [datetime]::MinValue; TaskBaseline = $null + } + + $backupLauncher = [System.IO.File]::ReadAllBytes((Join-Path $backup 'Start-BillionContextProxy.ps1')) + if (-not (Test-LpSameHash (Get-LpSha256OfBytes $backupLauncher) ([string]$rec.launcherSha256Before))) { Stop-Refused 'the launcher in the backup folder is not the one this deployment saved' } + $id = $null + try { $id = Read-LpLauncherIdentity ([System.Text.Encoding]::ASCII.GetString($backupLauncher)) } catch { Stop-Refused "the launcher in the backup folder is not readable ($($_.Exception.Message))" } + if ($id.ProxyPath.Value -cne $ctx.Previous.proxyPath -or $id.ExpectedVersion.Value -cne $ctx.Previous.version -or $id.ExpectedCommit.Value -cne $ctx.Previous.commit -or [int]$id.Port.Value -ne $ctx.Port) { Stop-Refused 'the launcher in the backup folder does not name the previous release recorded for this deployment' } + + $ownHold = Test-LpOwnInhibit $ctx + if ((Test-Path -LiteralPath $ctx.InhibitPath) -and -not $ownHold) { Stop-Refused 'maintenance.inhibit is present and belongs to other maintenance' } + $task = Get-LiveProxyTaskInfo $ctx.TaskName + if ($task.Count -ne 1) { Stop-Refused "expected exactly one scheduled task named in the record, found $($task.Count)" } + if ($task.ActionArguments.IndexOf($ctx.LauncherPath, [System.StringComparison]::OrdinalIgnoreCase) -lt 0) { Stop-Refused 'the task action does not run the launcher in this runtime root' } + $ctx.TaskBaseline = $task + + $dp = Get-CimInstance Win32_Process -Filter "ProcessId=$([int]$rec.deployProcess.pid)" -OperationTimeoutSec 15 -ErrorAction Stop + if ($null -ne $dp -and [Math]::Abs(($dp.CreationDate.ToUniversalTime() - ([datetime]$rec.deployProcess.created).ToUniversalTime()).TotalSeconds) -lt 1) { Stop-Refused 'the deploy script for this deployment is still running' } + + $liveSha = Get-LpSha256 $ctx.LauncherPath + $isBefore = Test-LpSameHash $liveSha ([string]$rec.launcherSha256Before) + $isAfter = Test-LpSameHash $liveSha ([string]$rec.launcherSha256After) + $stage = [string]$rec.stage + if (-not $isBefore -and -not $isAfter) { Stop-Refused 'the launcher was changed after this deployment; this is not the current deployment''s backup' } + if ($stage -eq 'rolled-back') { Stop-Refused 'this deployment was already rolled back' } + if (@('prepared', 'aborted') -contains $stage) { Stop-Refused "this deployment stopped at stage '$stage' and changed nothing" } + + $ctx.ConfigSha = Get-LpSha256 $ctx.ConfigPath + $receipts = @{} + foreach ($name in 'tuned-deployment.json', 'supervision-receipt.json') { $receipts[(Join-Path $root $name)] = [System.IO.File]::ReadAllBytes((Join-Path $backup $name)) } + $clients = Get-LpClientCount $ctx + Write-LpEvent 'STEP' "deployment stage '$stage'; launcher currently names the $(if ($isAfter -and -not $isBefore) { 'deployed' } else { 'previous' }) release; open client connections: $clients" + $plan = "hold supervision, stop any process of release $($ctx.Candidate.releaseName), restore the launcher and receipts where they differ from the backup, let the supervisor start $($ctx.Previous.version), $($ctx.Previous.commit) and check it" + if ($PreflightOnly) { + Write-LpEvent 'STEP' "would do: $plan" + throw [LpStop]::new(0, 'preflight-ok', 'rollback preflight passed; nothing was changed', '', 'run again without -PreflightOnly to roll back') + } +} catch [LpStop] { + Complete-Lp $null $_.Exception $null +} catch { + Complete-Lp $null ([LpStop]::new(2, 'refused', "refused: $($_.Exception.Message); nothing was changed", '', 'fix the problem above and run again')) $null +} + +try { + Invoke-LpRollback $ctx $backupLauncher $receipts $false + Set-LpStage $ctx 'rolled-back' + Complete-Lp $ctx ([LpStop]::new(0, 'restored', "the previous release $($ctx.Previous.version), $($ctx.Previous.commit) is live under supervision; receipts describe it", '', 'report the failure, the release left live and the backup location (step 10). A later deployment needs a new -ReleaseName.')) 'rollback-result.json' +} catch [LpStop] { + Complete-Lp $ctx $_.Exception 'rollback-result.json' +} catch { + Complete-Lp $ctx ([LpStop]::new(4, 'rollback-failed', "rollback stopped unexpectedly: $($_.Exception.Message). Check by hand that maintenance.inhibit is present, the task is disabled and no supervisor runs.", 'rollback-error', 'report to the owner; no release is confirmed live')) 'rollback-result.json' +} diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index 764f687f16..7c7946a73a 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -122,6 +122,11 @@ export const LIVE_FILES = [ "tests/ci-stall-watch.test.ts", // #69: runs copies of run-ci-tests.mjs as subprocesses; no ports. "tests/ci-runner-stall-gate.test.ts", + // #80: runs the live-proxy deploy and rollback scripts against throw-away + // sandboxes (Windows only; skipped elsewhere). Every sandbox has its own + // temp folder, a listen(0)-probed port and a file-backed stand-in for the + // scheduled task, so no zone band and no machine-global state is used. + "tests/live-deploy-scripts.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; diff --git a/tests/fixtures/live-deploy/Start-BillionContextProxy.ps1 b/tests/fixtures/live-deploy/Start-BillionContextProxy.ps1 new file mode 100644 index 0000000000..519e5fed9d --- /dev/null +++ b/tests/fixtures/live-deploy/Start-BillionContextProxy.ps1 @@ -0,0 +1,99 @@ +param( + [string]$RuntimeRoot = (Split-Path -Parent $PSCommandPath), + [string]$NodePath = '__NODE__', + [string]$ProxyPath = '__PROXY__', + [int]$Port = 0, + [string]$ExpectedVersion = '__VERSION__', + [string]$ExpectedCommit = '__COMMIT__', + [double]$PollSeconds = 0.3, + [double]$HeartbeatSeconds = 0.4, + [switch]$ReportOnly +) +# TEST STAND-IN for the live proxy supervisor. Written for tests/live-deploy-scripts.test.ts only; it is +# not the real launcher and is never installed. It honours the same contract the deploy scripts rely on: +# one supervisor per runtime root (exclusive lock), maintenance.inhibit suppresses adopt/start/restart, +# adopt a verified listener, start a proxy only when none matches and the port is free, otherwise do +# nothing, and report through supervisor-status.json. Runs under Windows PowerShell. +$ErrorActionPreference = 'Stop' +$inhibitPath = Join-Path $RuntimeRoot 'maintenance.inhibit' +$lockPath = Join-Path $RuntimeRoot 'supervisor.lock' +$statusPath = Join-Path $RuntimeRoot 'supervisor-status.json' +$configPath = Join-Path $RuntimeRoot 'config\billion-context\billion-context.json' +$proxyArgs = '"' + $ProxyPath + '" start --port ' + $Port + ' --host 127.0.0.1 --config "' + $configPath + '"' +$expectedCommandLine = '"' + $NodePath + '" ' + $proxyArgs +$started = (Get-Date).ToString('o') + +function Get-Owners { + return @(Get-CimInstance -Namespace ROOT/StandardCimv2 -ClassName MSFT_NetTCPConnection -Filter "LocalPort=$Port AND State=2" -ErrorAction Stop | Select-Object -ExpandProperty OwningProcess -Unique) +} +function Get-Matching { + return @(Get-CimInstance Win32_Process -Filter "Name='node.exe'" -ErrorAction Stop | Where-Object { $null -ne $_.CommandLine -and $_.CommandLine.Trim() -ieq $expectedCommandLine }) +} +function Test-Health([int]$expectPid) { + try { + $req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$Port/__bili/health") + $req.Proxy = $null; $req.KeepAlive = $false; $req.Timeout = 2000 + $resp = $req.GetResponse() + try { $h = (New-Object System.IO.StreamReader($resp.GetResponseStream())).ReadToEnd() | ConvertFrom-Json } finally { $resp.Close() } + return (($h.ok -eq $true) -and ([string]$h.version -eq $ExpectedVersion) -and ([string]$h.commit -eq $ExpectedCommit) -and ([string]$h.pid -eq [string]$expectPid)) + } catch { return $false } +} +function Resolve-Decision { + if (Test-Path -LiteralPath $inhibitPath) { return @{ action = 'inhibited' } } + if (!(Test-Path -LiteralPath $NodePath) -or !(Test-Path -LiteralPath $ProxyPath)) { return @{ action = 'blocked' } } + $owners = @(Get-Owners) + $candidates = @(Get-Matching) + if ($owners.Count -eq 0) { + if ($candidates.Count -eq 0) { return @{ action = 'spawn' } } + return @{ action = 'waiting' } + } + if ($owners.Count -ne 1) { return @{ action = 'conflict' } } + $owner = [int]$owners[0] + $match = @($candidates | Where-Object { $_.ProcessId -eq $owner }) + if ($match.Count -ne 1 -or $candidates.Count -ne 1) { return @{ action = 'conflict' } } + if (!(Test-Health $owner)) { return @{ action = 'conflict' } } + return @{ action = 'adopt'; pid = $owner } +} +function Write-Status([hashtable]$s) { + try { + $s.supervisorPid = $PID; $s.supervisorStarted = $started; $s.heartbeatAt = (Get-Date).ToString('o') + $tmp = "$statusPath.tmp" + [System.IO.File]::WriteAllText($tmp, ($s | ConvertTo-Json -Compress)) + if (Test-Path -LiteralPath $statusPath) { [System.IO.File]::Replace($tmp, $statusPath, [NullString]::Value) } else { [System.IO.File]::Move($tmp, $statusPath) } + } catch { } +} +function Watch-Child($proc, [string]$mode) { + $created = $proc.StartTime.ToString('o') + while (!$proc.HasExited) { + Write-Status @{ state = $mode; childPid = $proc.Id; childCreated = $created; healthMatch = (Test-Health $proc.Id); inhibit = (Test-Path -LiteralPath $inhibitPath) } + Start-Sleep -Milliseconds ([int]($HeartbeatSeconds * 1000)) + } +} + +if ($ReportOnly) { + try { Resolve-Decision | ConvertTo-Json -Compress; exit 0 } catch { @{ action = 'error' } | ConvertTo-Json -Compress; exit 2 } +} + +try { $lockStream = [System.IO.File]::Open($lockPath, 'OpenOrCreate', 'ReadWrite', 'None') } catch { exit 0 } +Add-Content -LiteralPath (Join-Path $RuntimeRoot 'sandbox-pids.log') -Value "$PID" + +while ($true) { + try { + $d = Resolve-Decision + if ($d.action -eq 'adopt') { + $proc = [System.Diagnostics.Process]::GetProcessById([int]$d.pid) + if (!(Test-Path -LiteralPath $inhibitPath)) { Watch-Child $proc 'adopted'; continue } + } elseif ($d.action -eq 'spawn') { + if (!(Test-Path -LiteralPath $inhibitPath) -and @(Get-Owners).Count -eq 0 -and @(Get-Matching).Count -eq 0) { + $proc = Start-Process -FilePath $NodePath -ArgumentList $proxyArgs -WindowStyle Hidden -PassThru -WorkingDirectory $RuntimeRoot + $null = $proc.Handle + Watch-Child $proc 'spawned' + continue + } + } + Write-Status @{ state = $d.action } + } catch { + Write-Status @{ state = 'error' } + } + Start-Sleep -Milliseconds ([int]($PollSeconds * 1000)) +} diff --git a/tests/fixtures/live-deploy/TestTaskDriver.ps1 b/tests/fixtures/live-deploy/TestTaskDriver.ps1 new file mode 100644 index 0000000000..42b2723bcf --- /dev/null +++ b/tests/fixtures/live-deploy/TestTaskDriver.ps1 @@ -0,0 +1,67 @@ +# TEST DRIVER standing in for the Windows scheduled task, used only by tests/live-deploy-scripts.test.ts. +# The deploy scripts dot-source it when -TestTaskDriver is given and the runtime root is a marked sandbox. +# State lives in \sandbox-task.json: +# { taskName, enabled, port, calls: { info, disable, enable, start }, +# failOn: [ { op, call } ], the n-th call of that operation throws +# onCall: [ { op, call, action } ] } the n-th call first performs a fault action +# Actions: openClient, makeLauncherReadOnly, writeForeignInhibit, editLauncher, killSelf. + +function Get-LpTestTaskStatePath { return (Join-Path $script:LpRoot 'sandbox-task.json') } +function Get-LpTestTaskState { return ([System.IO.File]::ReadAllText((Get-LpTestTaskStatePath)) | ConvertFrom-Json -AsHashtable) } +function Save-LpTestTaskState($State) { [System.IO.File]::WriteAllText((Get-LpTestTaskStatePath), ($State | ConvertTo-Json -Depth 6)) } +function Get-LpTestLauncher { return (Join-Path $script:LpRoot 'Start-BillionContextProxy.ps1') } +function Get-LpTestWindowsPowerShell { return (Join-Path $env:WINDIR 'System32\WindowsPowerShell\v1.0\powershell.exe') } + +function Invoke-LpTestFault([string]$Op) { + $s = Get-LpTestTaskState + $s.calls[$Op] = [int]$s.calls[$Op] + 1 + Save-LpTestTaskState $s + $n = $s.calls[$Op] + foreach ($a in @($s.onCall)) { + if ($null -eq $a -or $a.op -ne $Op -or [int]$a.call -ne $n) { continue } + switch ([string]$a.action) { + 'openClient' { + # A separate process holds a connection to the proxy port open, like a real client would. + $cmd = "`$c = New-Object System.Net.Sockets.TcpClient('127.0.0.1', $([int]$s.port)); Start-Sleep -Seconds 120 # sandbox `"$script:LpRoot`"" + $p = Start-Process -FilePath (Get-LpTestWindowsPowerShell) -ArgumentList @('-NoProfile', '-NonInteractive', '-Command', $cmd) -WindowStyle Hidden -PassThru + Add-Content -LiteralPath (Join-Path $script:LpRoot 'sandbox-pids.log') -Value "$($p.Id)" + $ctxLike = @{ Port = [int]$s.port; LauncherPath = (Get-LpTestLauncher) } + $null = Wait-LpUntil { (Get-LpClientCount $ctxLike) -gt 0 } 20 'waiting for the test client to connect' + } + 'makeLauncherReadOnly' { Set-ItemProperty -LiteralPath (Get-LpTestLauncher) -Name IsReadOnly -Value $true } + 'writeForeignInhibit' { [System.IO.File]::WriteAllText((Join-Path $script:LpRoot 'maintenance.inhibit'), 'someone else') } + 'editLauncher' { Add-Content -LiteralPath (Get-LpTestLauncher) -Value '# foreign edit' } + 'killSelf' { Stop-Process -Id $PID -Force } + } + } + foreach ($f in @($s.failOn)) { + if ($null -ne $f -and $f.op -eq $Op -and [int]$f.call -eq $n) { throw "test driver: '$Op' refused (injected fault)" } + } +} + +function Get-LiveProxyTaskInfo([string]$TaskName) { + Invoke-LpTestFault 'info' + $s = Get-LpTestTaskState + if ($TaskName -ne [string]$s.taskName) { return [pscustomobject]@{ Count = 0; Enabled = $false; Execute = ''; ActionArguments = ''; TriggerCount = 0 } } + return [pscustomobject]@{ + Count = 1; Enabled = [bool]$s.enabled; Execute = 'powershell.exe' + ActionArguments = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$(Get-LpTestLauncher)`""; TriggerCount = 2 + } +} +function Disable-LiveProxyTask([string]$TaskName) { + Invoke-LpTestFault 'disable' + $s = Get-LpTestTaskState; $s.enabled = $false; Save-LpTestTaskState $s +} +function Enable-LiveProxyTask([string]$TaskName) { + Invoke-LpTestFault 'enable' + $s = Get-LpTestTaskState; $s.enabled = $true; Save-LpTestTaskState $s +} +function Start-LiveProxyTask([string]$TaskName) { + Invoke-LpTestFault 'start' + $s = Get-LpTestTaskState + if (-not $s.enabled) { return } + # The real task ignores a new start while an instance is running. + if (Test-LpLockHeld (Join-Path $script:LpRoot 'supervisor.lock')) { return } + $arguments = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$(Get-LpTestLauncher)`"" + Start-Process -FilePath (Get-LpTestWindowsPowerShell) -ArgumentList $arguments -WindowStyle Hidden | Out-Null +} diff --git a/tests/fixtures/live-deploy/stand-in-proxy.mjs b/tests/fixtures/live-deploy/stand-in-proxy.mjs new file mode 100644 index 0000000000..5444fe3f46 --- /dev/null +++ b/tests/fixtures/live-deploy/stand-in-proxy.mjs @@ -0,0 +1,50 @@ +// TEST STAND-IN for the live proxy bundle, used only by tests/live-deploy-scripts.test.ts. +// Installed in a sandbox as releases//dist/index.mjs. Answers /__bili/health with the version of +// the package.json beside dist/ and the commit of dist/build-info.json, like the real build does. +// /proxy-control.json can make one release misbehave: +// { "": { "unhealthy": true } } health reports ok:false +// { "": { "unhealthyFromStart": 2 } } healthy on the first start, unhealthy from the second +// { "": { "exitAtStart": true } } the process exits before listening +import fs from "node:fs"; +import http from "node:http"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const dist = path.dirname(fileURLToPath(import.meta.url)); +const releaseDir = path.dirname(dist); +const releaseName = path.basename(releaseDir); +const runtimeRoot = path.dirname(path.dirname(releaseDir)); +const port = Number(process.argv[process.argv.indexOf("--port") + 1]); +const version = JSON.parse(fs.readFileSync(path.join(releaseDir, "package.json"), "utf8")).version; +const commit = JSON.parse(fs.readFileSync(path.join(dist, "build-info.json"), "utf8")).commit; + +const counterFile = path.join(runtimeRoot, `proxy-starts-${releaseName}.txt`); +let starts = 1; +try { + starts = Number(fs.readFileSync(counterFile, "utf8")) + 1; +} catch { + // first start +} +fs.writeFileSync(counterFile, String(starts)); +fs.appendFileSync(path.join(runtimeRoot, "sandbox-pids.log"), `${process.pid}\n`); + +function control() { + try { + return JSON.parse(fs.readFileSync(path.join(runtimeRoot, "proxy-control.json"), "utf8"))[releaseName] ?? {}; + } catch { + return {}; + } +} + +if (control().exitAtStart) process.exit(3); + +http.createServer((req, res) => { + if (req.url !== "/__bili/health") { + res.writeHead(404).end(); + return; + } + const c = control(); + const unhealthy = c.unhealthy === true || (typeof c.unhealthyFromStart === "number" && starts >= c.unhealthyFromStart); + res.writeHead(200, { "content-type": "application/json", connection: "close" }); + res.end(JSON.stringify({ ok: !unhealthy, version, commit, pid: process.pid })); +}).listen(port, "127.0.0.1"); diff --git a/tests/live-deploy-scripts.test.ts b/tests/live-deploy-scripts.test.ts new file mode 100644 index 0000000000..c0207ef760 --- /dev/null +++ b/tests/live-deploy-scripts.test.ts @@ -0,0 +1,970 @@ +// Tests for scripts/live-deploy/ (issue #80): the live-proxy deploy and rollback scripts, driven against +// throw-away sandboxes. A sandbox is a temporary runtime root with a stand-in launcher, a stand-in proxy +// and a test driver in place of the Windows scheduled task (tests/fixtures/live-deploy/). +// +// Nothing here may reach a real installation: every sandbox lives under the temp folder with a random +// name, uses a port handed out by the OS, and is torn down by PIDs its own processes reported. +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import http from "node:http"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { describe, test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const WIN = process.platform === "win32"; +const SKIP = WIN ? false : "live-proxy deploy scripts are Windows-only (PowerShell, scheduled task, CIM)"; +const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const SCRIPTS = path.join(REPO, "scripts", "live-deploy"); +const FIXTURES = path.join(REPO, "tests", "fixtures", "live-deploy"); +const DEPLOY = path.join(SCRIPTS, "Deploy-LiveProxy.ps1"); +const ROLLBACK = path.join(SCRIPTS, "Rollback-LiveProxy.ps1"); +const COMMON = path.join(SCRIPTS, "LiveProxy.Common.ps1"); +const DRIVER = path.join(FIXTURES, "TestTaskDriver.ps1"); +const WINPS = path.join(process.env.WINDIR ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); +const LAUNCHER = "Start-BillionContextProxy.ps1"; +const FORBIDDEN_PORT = 8787; + +const PREV = { name: "prev-1", version: "0.9.1", commit: "aaaaaaa1-live", source: "a".repeat(40) }; +const CAND = { version: "0.9.2", commit: "bbbbbbb2-live", source: "b".repeat(40) }; + +// ---- harness ---------------------------------------------------------------------------------------- + +const sha256 = (buf: Buffer | string) => crypto.createHash("sha256").update(buf).digest("hex").toUpperCase(); +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +const tempBase = () => fs.realpathSync.native(os.tmpdir()); + +/** Refuses anything that is not a throw-away sandbox. Exported shape is tested below. */ +function assertSandbox(root: string, port: number): void { + const base = tempBase().toLowerCase(); + const r = path.resolve(root).toLowerCase(); + if (path.dirname(r) !== base || !/^bili-live-deploy-[0-9a-f]{10}[a-z0-9]*$/.test(path.basename(r))) { + throw new Error("sandbox root must be \\bili-live-deploy-"); + } + const roaming = (process.env.APPDATA ?? "").toLowerCase(); + if (roaming && (r === roaming || r.startsWith(roaming + path.sep))) throw new Error("sandbox root must not be under the roaming application-data folder"); + if (port === FORBIDDEN_PORT) throw new Error("the live port must never be used by a sandbox"); +} + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const s = net.createServer(); + s.once("error", reject); + s.listen(0, "127.0.0.1", () => { + const port = (s.address() as net.AddressInfo).port; + s.close(() => resolve(port)); + }); + }); +} + +interface Candidate { + dir: string; + version: string; + commit: string; + source: string; + sha: string; +} +interface Sandbox { + root: string; + port: number; + taskName: string; + configSha: string; + canary: string; + cand: Candidate; + extraDirs: string[]; +} + +function writeReleaseFiles(dir: string, version: string, commit: string, source: string, tag: string): string { + fs.mkdirSync(path.join(dir, "dist"), { recursive: true }); + const bundle = fs.readFileSync(path.join(FIXTURES, "stand-in-proxy.mjs"), "utf8").replace(/\r\n/g, "\n") + `// ${tag}\n`; + fs.writeFileSync(path.join(dir, "dist", "index.mjs"), bundle); + fs.writeFileSync(path.join(dir, "dist", "build-info.json"), JSON.stringify({ commit, sourceCommit: source })); + fs.writeFileSync(path.join(dir, "package.json"), JSON.stringify({ name: "stand-in", version, type: "module" })); + fs.writeFileSync(path.join(dir, "LICENSE"), "test fixture\n"); + return sha256(bundle); +} + +function launcherText(sb: { root: string; port: number }, proxyPath: string, version: string, commit: string): string { + const values: Record = { __NODE__: process.execPath, __PROXY__: proxyPath, __VERSION__: version, __COMMIT__: commit }; + const portLine = `[int]$Port = ${sb.port},`; + const text = fs + .readFileSync(path.join(FIXTURES, LAUNCHER), "utf8") + .replace(/\r\n/g, "\n") + .replace(/__(NODE|PROXY|VERSION|COMMIT)__/g, (m) => values[m]) + .replace("[int]$Port = 0,", () => portLine); + assert.equal(text.includes(portLine), true, "the launcher template must carry the port line"); + return text; +} + +function candidateDir(root: string): string { + return `${root}-cand`; +} + +async function makeSandbox(suffix = "", cand = CAND): Promise { + const port = await freePort(); + const root = path.join(tempBase(), `bili-live-deploy-${crypto.randomBytes(5).toString("hex")}${suffix}`); + assertSandbox(root, port); + const canary = `CANARY-${crypto.randomBytes(6).toString("hex")}`; + fs.mkdirSync(path.join(root, "config", "billion-context"), { recursive: true }); + fs.mkdirSync(path.join(root, "releases")); + fs.mkdirSync(path.join(root, "backups")); + fs.mkdirSync(path.join(root, "shared-node-modules")); + fs.writeFileSync(path.join(root, "sandbox.marker"), "test sandbox\n"); + const config = JSON.stringify({ note: canary }); + fs.writeFileSync(path.join(root, "config", "billion-context", "billion-context.json"), config); + const prevDir = path.join(root, "releases", PREV.name); + const prevSha = writeReleaseFiles(prevDir, PREV.version, PREV.commit, PREV.source, "previous"); + fs.symlinkSync(path.join(root, "shared-node-modules"), path.join(prevDir, "node_modules"), "junction"); + const sb: Sandbox = { + root, + port, + taskName: `Sandbox task ${path.basename(root)}`, + configSha: sha256(config), + canary, + cand: { dir: candidateDir(root), ...cand, sha: "" }, + extraDirs: [], + }; + fs.writeFileSync(path.join(root, LAUNCHER), launcherText(sb, path.join(prevDir, "dist", "index.mjs"), PREV.version, PREV.commit), "ascii"); + fs.writeFileSync( + path.join(root, "tuned-deployment.json"), + JSON.stringify({ deployedAt: "2026-01-01T00:00:00.0000000-05:00", releaseRoot: prevDir, bundleSha256: prevSha, build: PREV.commit, sourceCommit: PREV.source, deployedConfigSha256: "stale-on-purpose", livePid: 0 }, null, 2), + ); + fs.writeFileSync(path.join(root, "supervision-receipt.json"), JSON.stringify({ task: "sandbox", proxyRelease: "stale-on-purpose" }, null, 2)); + setTask(sb, { taskName: sb.taskName, enabled: true, port, calls: { info: 0, disable: 0, enable: 0, start: 0 }, failOn: [], onCall: [] }); + sb.cand.sha = writeReleaseFiles(sb.cand.dir, cand.version, cand.commit, cand.source, "candidate"); + try { + startSupervisor(sb); + await waitSupervised(sb, PREV.version, PREV.commit); + } catch (e) { + await teardown(sb); + throw e; + } + return sb; +} + +const taskPath = (sb: Sandbox) => path.join(sb.root, "sandbox-task.json"); +const getTask = (sb: Sandbox) => JSON.parse(fs.readFileSync(taskPath(sb), "utf8")); +const setTask = (sb: Sandbox, state: unknown) => fs.writeFileSync(taskPath(sb), JSON.stringify(state)); +function patchTask(sb: Sandbox, patch: Record): void { + setTask(sb, { ...getTask(sb), ...patch }); +} + +function startSupervisor(sb: Sandbox): void { + const child = spawn(WINPS, [`-NoProfile -NonInteractive -ExecutionPolicy Bypass -File "${path.join(sb.root, LAUNCHER)}"`], { + // Not detached: Windows PowerShell exits at once without a console. + stdio: "ignore", + windowsHide: true, + windowsVerbatimArguments: true, + }); + child.unref(); +} + +function health(port: number): Promise<{ ok: boolean; version: string; commit: string; pid: number } | null> { + return new Promise((resolve) => { + const req = http.get({ host: "127.0.0.1", port, path: "/__bili/health", agent: false, timeout: 2000 }, (res) => { + let body = ""; + res.on("data", (c) => (body += c)); + res.on("end", () => { + try { + resolve(JSON.parse(body)); + } catch { + resolve(null); + } + }); + }); + req.on("timeout", () => req.destroy()); + req.on("error", () => resolve(null)); + }); +} + +function status(sb: Sandbox): Record | null { + try { + return JSON.parse(fs.readFileSync(path.join(sb.root, "supervisor-status.json"), "utf8")); + } catch { + return null; + } +} + +async function waitFor(what: string, cond: () => Promise | boolean, ms = 30000): Promise { + const end = Date.now() + ms; + for (;;) { + if (await cond()) return; + if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); + await sleep(150); + } +} + +/** Waits until the given release answers health and the sandbox supervisor monitors it. */ +async function waitSupervised(sb: Sandbox, version: string, commit: string): Promise { + let pid = 0; + await waitFor(`${commit} under supervision`, async () => { + const h = await health(sb.port); + const st = status(sb); + if (!h || !h.ok || h.version !== version || h.commit !== commit || !st) return false; + pid = h.pid; + return (st.state === "adopted" || st.state === "spawned") && st.childPid === h.pid && st.healthMatch === true; + }); + return pid; +} + +const alive = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +/** Stops the processes the sandbox itself reported, after checking each still belongs to this sandbox. */ +function stopSandboxProcesses(sb: Sandbox): void { + let pids: number[] = []; + try { + pids = [...new Set(fs.readFileSync(path.join(sb.root, "sandbox-pids.log"), "utf8").split(/\r?\n/).map(Number).filter((n) => n > 0))]; + } catch { + return; + } + if (pids.length === 0) return; + const script = + "$ids = $env:LP_TD_PIDS -split ','; $token = $env:LP_TD_ROOT; " + + "Get-CimInstance Win32_Process | Where-Object { $ids -contains [string]$_.ProcessId -and $null -ne $_.CommandLine -and " + + "($_.CommandLine.IndexOf($token + '\\', [System.StringComparison]::OrdinalIgnoreCase) -ge 0 -or $_.CommandLine.IndexOf($token + '\"', [System.StringComparison]::OrdinalIgnoreCase) -ge 0) } | " + + "ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }"; + spawnSync("pwsh", ["-NoProfile", "-NonInteractive", "-Command", script], { env: { ...process.env, LP_TD_PIDS: pids.join(","), LP_TD_ROOT: sb.root }, encoding: "utf8", timeout: 60000 }); +} + +function clearReadOnly(dir: string): void { + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + if (e.isFile()) fs.chmodSync(path.join(dir, e.name), 0o666); + } +} + +async function teardown(sb: Sandbox): Promise { + stopSandboxProcesses(sb); + await sleep(300); + stopSandboxProcesses(sb); // a supervisor may have started one more child while it was being stopped + for (const dir of [...sb.extraDirs, sb.cand.dir, sb.root]) { + const st = fs.lstatSync(dir, { throwIfNoEntry: false }); + if (!st) continue; + if (st.isSymbolicLink()) { + fs.rmSync(dir, { force: true, maxRetries: 20, retryDelay: 200 }); + continue; + } + try { + clearReadOnly(dir); + } catch { + // best effort + } + fs.rmSync(dir, { recursive: true, force: true, maxRetries: 20, retryDelay: 200 }); + } +} + +async function withSandbox(fn: (sb: Sandbox) => Promise, suffix = "", cand = CAND): Promise { + const sb = await makeSandbox(suffix, cand); + try { + await fn(sb); + } finally { + await teardown(sb); + } +} + +interface Run { + code: number | null; + out: string; +} +function runScript(script: string, args: Record): Promise { + const argv: string[] = []; + for (const [k, v] of Object.entries(args)) { + if (v === null) continue; + argv.push(`-${k}`); + if (v !== true) argv.push(String(v)); + } + // Asynchronous, so that scenarios in separate sandboxes can run side by side. + return new Promise((resolve, reject) => { + const child = spawn("pwsh", ["-NoProfile", "-NonInteractive", "-File", script, ...argv], { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8").on("data", (d) => (stdout += d)); + child.stderr.setEncoding("utf8").on("data", (d) => (stderr += d)); + const timer = setTimeout(() => child.kill(), 240000); + child.once("error", (e) => { + clearTimeout(timer); + reject(e); + }); + child.once("close", (code) => { + clearTimeout(timer); + resolve({ code, out: `${stdout}\n${stderr}` }); + }); + }); +} + +function deploy(sb: Sandbox, overrides: Record = {}): Promise { + return runScript(DEPLOY, { + RuntimeRoot: sb.root, + Port: sb.port, + TaskName: sb.taskName, + CandidateRoot: sb.cand.dir, + ReleaseName: "cand-1", + CandidateVersion: sb.cand.version, + CandidateCommit: sb.cand.commit, + CandidateSourceCommit: sb.cand.source, + CandidateBundleSha256: sb.cand.sha, + ExpectedConfigSha256: sb.configSha, + HealthWaitSeconds: 30, + StopWaitSeconds: 30, + TestTaskDriver: DRIVER, + ...overrides, + }); +} + +function rollback(sb: Sandbox, backup: string, overrides: Record = {}): Promise { + return runScript(ROLLBACK, { RuntimeRoot: sb.root, BackupRoot: backup, HealthWaitSeconds: 30, StopWaitSeconds: 30, TestTaskDriver: DRIVER, ...overrides }); +} + +function expectResult(r: Run, code: number, name: string, reason?: string): void { + assert.equal(r.code, code, `expected exit ${code}\n${r.out}`); + assert.match(r.out, new RegExp(` RESULT ${name} \\(exit ${code}\\)${reason ? ` reason=${reason}` : ""}`), r.out); + assert.match(r.out, /TEST SEAM ACTIVE/, "a sandbox run must announce the test seam"); +} + +const backups = (sb: Sandbox) => fs.readdirSync(path.join(sb.root, "backups")).map((n) => path.join(sb.root, "backups", n)); +const onlyBackup = (sb: Sandbox) => { + const b = backups(sb); + assert.equal(b.length, 1, "expected exactly one backup folder"); + return b[0]; +}; +const record = (backup: string) => JSON.parse(fs.readFileSync(path.join(backup, "deploy-record.json"), "utf8")); +const inhibit = (sb: Sandbox) => { + try { + return fs.readFileSync(path.join(sb.root, "maintenance.inhibit"), "utf8").trim(); + } catch { + return null; + } +}; +const launcher = (sb: Sandbox) => fs.readFileSync(path.join(sb.root, LAUNCHER), "utf8"); +const releaseDir = (sb: Sandbox, name = "cand-1") => path.join(sb.root, "releases", name); +const supervisorPid = (sb: Sandbox) => Number(status(sb)?.supervisorPid ?? 0); + +function assertUntouched(sb: Sandbox, launcherBefore: string): void { + assert.equal(inhibit(sb), null, "no inhibit may be left behind"); + assert.equal(fs.existsSync(releaseDir(sb)), false, "no release directory may be created"); + assert.equal(backups(sb).length, 0, "no backup folder may be created"); + assert.equal(launcher(sb), launcherBefore, "the launcher must be unchanged"); + assert.equal(getTask(sb).enabled, true, "the task must stay enabled"); +} + +/** Hold in place: inhibit owned by this deployment, task disabled, no supervisor process. */ +function assertHold(sb: Sandbox, backup: string, supervisorBefore: number): void { + assert.match(inhibit(sb) ?? "", new RegExp(`^(deploy|rollback) ${path.basename(backup)}$`)); + assert.equal(getTask(sb).enabled, false, "the task must be disabled"); + assert.equal(alive(supervisorBefore), false, "the supervisor must be stopped"); +} + +// ---- static checks ---------------------------------------------------------------------------------- + +describe("live-deploy scripts: static rules", () => { + const sources = [DEPLOY, ROLLBACK, COMMON].map((f) => ({ file: path.basename(f), text: fs.readFileSync(f, "utf8") })); + + test("no release identity, port, task name or personal path is written into the scripts", () => { + const banned: [string, RegExp][] = [ + ["40-hex string", /\b[0-9a-fA-F]{40}\b/], + ["version literal", /(? s.text).join("\n"); + assert.equal((all.match(/CindyGlobal/g) ?? []).length, 1, "the live installation folder may be named once, in the seam guard"); + // Negative control: the scan does find these shapes when they are present. + assert.equal(banned[1][1].test("version -eq '0.1.213'"), true); + assert.equal(banned[1][1].test("http://127.0.0.1:1/x"), false); + }); + + test("the scripts take exactly the documented parameters and no test hook", () => { + const params = (text: string) => { + const block = text.slice(text.indexOf("param("), text.indexOf("\n)", text.indexOf("param("))); + return [...block.matchAll(/\$([A-Za-z0-9]+)(?=\s*(?:=|,|$))/gm)].map((m) => m[1]).sort(); + }; + const deployParams = "CandidateBundleSha256 CandidateCommit CandidateRoot CandidateSourceCommit CandidateVersion ExpectedConfigSha256 HealthWaitSeconds Port PreflightOnly ReleaseName RuntimeRoot StopWaitSeconds TaskName TestTaskDriver ToolingDir"; + assert.deepEqual(params(sources[0].text), deployParams.split(" ")); + assert.deepEqual(params(sources[1].text), "BackupRoot HealthWaitSeconds PreflightOnly RuntimeRoot StopWaitSeconds TestTaskDriver".split(" ")); + for (const { file, text } of sources) { + const env = [...text.matchAll(/\$env:([A-Za-z_]+)/g)].map((m) => m[1].toUpperCase()); + assert.deepEqual([...new Set(env)].filter((n) => n !== "WINDIR"), [], `${file} reads an environment variable for behaviour`); + } + }); + + test("committed files of this feature contain no user-profile path", () => { + const files = [DEPLOY, ROLLBACK, COMMON, ...fs.readdirSync(FIXTURES).map((f) => path.join(FIXTURES, f)), fileURLToPath(import.meta.url)]; + const profile = new RegExp(["[A-Za-z]:", "Users", "[^<\\\\]"].join("\\\\")); + for (const f of files) assert.equal(profile.test(fs.readFileSync(f, "utf8")), false, `${path.basename(f)} contains a user-profile path`); + assert.equal(profile.test(["C:", "Users", "someone", "x"].join("\\")), true, "negative control"); + }); + + test("the harness refuses anything that is not a throw-away sandbox", () => { + const good = path.join(tempBase(), "bili-live-deploy-0123456789"); + assert.doesNotThrow(() => assertSandbox(good, 40000)); + assert.throws(() => assertSandbox(good, FORBIDDEN_PORT), /live port/); + assert.throws(() => assertSandbox(path.join(tempBase(), "something-else"), 40000), /bili-live-deploy/); + assert.throws(() => assertSandbox(path.join(tempBase(), "nested", "bili-live-deploy-0123456789"), 40000), /bili-live-deploy/); + const roaming = process.env.APPDATA; + if (roaming) assert.throws(() => assertSandbox(path.join(roaming, "bili-live-deploy-0123456789"), 40000)); + }); + + test("the seam guard recognises the live installation area in every spelling", { skip: SKIP }, () => { + // The guard is a pure function, so no folder is created anywhere near a real installation. + const base = "X:\\Profiles\\someone\\AppData\\Roaming"; + const cases: [string, boolean][] = [ + [`${base}\\CindyGlobal\\billion-context-proxy`, true], + [`${base.toUpperCase()}\\CINDYGLOBAL\\x\\`, true], + [`\\\\?\\${base}\\CindyGlobal\\x`, true], + [`${base}\\CindyGlobal`, true], + [`${base}\\CindyGlobalOther\\x`, false], + [`${base}\\Other\\CindyGlobal\\x`, false], + ["X:\\Temp\\bili-live-deploy-0123456789", false], + ]; + const lines = cases.map(([p]) => `Test-LpLiveArea @('${p}') '${base}\\'`).join("; "); + const r = spawnSync("pwsh", ["-NoProfile", "-NonInteractive", "-Command", `. '${COMMON}'; ${lines}`], { encoding: "utf8", timeout: 60000 }); + assert.equal(r.status, 0, r.stderr); + assert.deepEqual(r.stdout.trim().split(/\r?\n/), cases.map(([, want]) => (want ? "True" : "False"))); + }); +}); + +// ---- refusals: nothing may change --------------------------------------------------------------------- + +describe("live-deploy scripts: refusals leave everything untouched", { skip: SKIP }, () => { + test("input validation and preflight refuse, each for its own reason", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + const refuse = async (label: string, args: Record, reasons: RegExp[]) => { + const r = await deploy(sb, { PreflightOnly: true, ...args }); + assert.equal(r.code, 2, `${label}: expected exit 2\n${r.out}`); + assert.match(r.out, / RESULT refused \(exit 2\)/, label); + for (const re of reasons) assert.match(r.out, re, `${label}\n${r.out}`); + assertUntouched(sb, before); + }; + + // Positive control first: the same sandbox passes when nothing is wrong. + const ok = await deploy(sb, { PreflightOnly: true }); + expectResult(ok, 0, "preflight-ok"); + assert.equal(/ CHECK FAIL /.test(ok.out), false, ok.out); + assert.match(ok.out, /plan: replace 0\.9\.1, aaaaaaa1-live .* with 0\.9\.2, bbbbbbb2-live/); + assertUntouched(sb, before); + + // Input validation. + await refuse("version", { CandidateVersion: "1.2" }, [/FAIL candidate version format/]); + await refuse("commit", { CandidateCommit: "XYZ" }, [/FAIL candidate commit format/]); + await refuse("source commit", { CandidateSourceCommit: "abc123" }, [/FAIL candidate source commit format/]); + await refuse("bundle hash", { CandidateBundleSha256: "zz" }, [/FAIL candidate bundle hash format/]); + await refuse("config hash", { ExpectedConfigSha256: "zz" }, [/FAIL expected config hash format/]); + await refuse("release name", { ReleaseName: ".." }, [/FAIL release name/]); + await refuse("reserved name", { ReleaseName: "nul" }, [/reserved device name/]); + await refuse("task wildcard", { TaskName: "Sandbox*" }, [/wildcard characters are not allowed/]); + await refuse("relative root", { RuntimeRoot: "relative\\root" }, [/FAIL runtime root: .*not absolute/]); + await refuse("candidate inside root", { CandidateRoot: path.join(sb.root, "releases", PREV.name) }, [/inside the runtime root/]); + const quoted = `${sb.root}-q'x`; + fs.mkdirSync(quoted); + sb.extraDirs.push(quoted); + await refuse("quote in a path", { CandidateRoot: quoted }, [/forbidden character/]); + const link = `${sb.root}-link`; + fs.symlinkSync(sb.root, link, "junction"); + sb.extraDirs.push(link); + await refuse("root reached through a link", { RuntimeRoot: link }, [/FAIL runtime root: a folder in the path is a link/]); + + // Sandbox seam. + await refuse("real task driver on a marked sandbox", { TestTaskDriver: null }, [/real scheduled task may not be used/]); + fs.renameSync(path.join(sb.root, "sandbox.marker"), path.join(sb.root, "sandbox.marker.off")); + await refuse("test driver without the marker", {}, [/sandbox\.marker is absent/]); + fs.renameSync(path.join(sb.root, "sandbox.marker.off"), path.join(sb.root, "sandbox.marker")); + await refuse("test driver that is a folder", { TestTaskDriver: FIXTURES }, [/must be a regular file/]); + + // Preflight: candidate identity (one check, so one reason per run). + await refuse("candidate commit", { CandidateCommit: "ccccccc3-live" }, [/expected commit ccccccc3-live/]); + await refuse("candidate source", { CandidateSourceCommit: "c".repeat(40) }, [/different source commit/]); + await refuse("candidate version", { CandidateVersion: "9.9.9" }, [/expected version 9\.9\.9/]); + await refuse("candidate bundle", { CandidateBundleSha256: "0".repeat(64) }, [/bundle hash differs/]); + fs.renameSync(path.join(sb.cand.dir, "LICENSE"), path.join(sb.cand.dir, "LICENSE.off")); + await refuse("candidate file missing", {}, [/LICENSE is missing/]); + fs.renameSync(path.join(sb.cand.dir, "LICENSE.off"), path.join(sb.cand.dir, "LICENSE")); + + // Preflight: independent checks all run and all report in one pass. + fs.writeFileSync(path.join(sb.root, "maintenance.inhibit"), "someone else"); + patchTask(sb, { enabled: false }); + const otherPort = await freePort(); + const r = await deploy(sb, { PreflightOnly: true, ExpectedConfigSha256: "1".repeat(64), ReleaseName: PREV.name, ToolingDir: path.join(sb.root, "no-such-folder"), Port: otherPort }); + assert.equal(r.code, 2, r.out); + for (const re of [ + /FAIL port matches the launcher/, + /FAIL scheduled task runs this launcher: the task is disabled/, + /FAIL configuration hash/, + /FAIL current release healthy from the single listener/, + /FAIL no maintenance in progress/, + /FAIL new release directory is absent/, + /FAIL tooling folder/, + /PASS candidate files and identity/, + /PASS current release node_modules link/, + ]) { + assert.match(r.out, re, r.out); + } + fs.rmSync(path.join(sb.root, "maintenance.inhibit")); + patchTask(sb, { enabled: true }); + assertUntouched(sb, before); + await refuse("unknown task", { TaskName: "No such task" }, [/expected exactly one task with this name, found 0/]); + + // Preflight: launcher shape. + fs.appendFileSync(path.join(sb.root, LAUNCHER), " [string]$ExpectedCommit = 'again',\n"); + { + const run = await deploy(sb, { PreflightOnly: true }); + assert.equal(run.code, 2, run.out); + assert.match(run.out, /launcher has 2 lines defining ExpectedCommit/); + } + fs.writeFileSync(path.join(sb.root, LAUNCHER), before + "# caf\u00e9\n", "utf8"); + { + const run = await deploy(sb, { PreflightOnly: true }); + assert.equal(run.code, 2, run.out); + assert.match(run.out, /not plain ASCII/); + } + fs.writeFileSync(path.join(sb.root, LAUNCHER), before, "ascii"); + + // Preflight: node_modules must be a link. + const nm = path.join(sb.root, "releases", PREV.name, "node_modules"); + fs.rmdirSync(nm); + fs.mkdirSync(nm); + await refuse("node_modules is a folder", {}, [/not a junction or symbolic link/]); + fs.rmdirSync(nm); + fs.symlinkSync(path.join(sb.root, "shared-node-modules"), nm, "junction"); + + // Preflight: an unhealthy current release. + fs.writeFileSync(path.join(sb.root, "proxy-control.json"), JSON.stringify({ [PREV.name]: { unhealthy: true } })); + await refuse("unhealthy current release", {}, [/FAIL current release healthy from the single listener: .*observed ok=False/]); + fs.rmSync(path.join(sb.root, "proxy-control.json")); + await waitSupervised(sb, PREV.version, PREV.commit); + + // Idle: a connected client is reported by -PreflightOnly and refused by a full run. + const client = net.connect({ host: "127.0.0.1", port: sb.port }); + await new Promise((res, rej) => client.once("connect", res).once("error", rej)); + try { + const dry = await deploy(sb, { PreflightOnly: true }); + expectResult(dry, 0, "preflight-ok"); + assert.match(dry.out, /open client connections now: [1-9]/); + const full = await deploy(sb); + assert.equal(full.code, 2, full.out); + assert.match(full.out, /client connection\(s\) open; not idle/); + assertUntouched(sb, before); + } finally { + client.destroy(); + } + + // No supervisor (last: it changes the sandbox). + const sup = supervisorPid(sb); + process.kill(sup); + await waitFor("supervisor to exit", () => !alive(sup)); + await refuse("no supervisor", {}, [/expected one supervisor, found 0/]); + }); + }); +}); + +// ---- deployments ------------------------------------------------------------------------------------ + +// Every scenario owns its sandbox (folder, port, task state), so they run side by side. +describe("live-deploy scripts: deployment and rollback", { skip: SKIP, concurrency: 4 }, () => { + test("deploys, proves supervision, rewrites receipts; the standalone rollback restores everything", async () => { + // A second sandbox whose name is a prefix-sibling of the first proves the scripts never touch another installation. + const other = await makeSandbox("a10"); + try { + await withSandbox(async (sb) => { + const otherPids = { sup: supervisorPid(other), proxy: (await health(other.port))!.pid }; + const before = launcher(sb); + const tunedBefore = fs.readFileSync(path.join(sb.root, "tuned-deployment.json")); + const supBefore = fs.readFileSync(path.join(sb.root, "supervision-receipt.json")); + const tooling = `${sb.root}-tools`; + fs.mkdirSync(tooling); + fs.writeFileSync(path.join(tooling, "build.mjs"), "// kept tooling\n"); + sb.extraDirs.push(tooling); + + const r = await deploy(sb, { ToolingDir: tooling }); + expectResult(r, 0, "deployed"); + const backup = onlyBackup(sb); + assert.match(r.out, new RegExp(` BACKUP ${backup.replace(/[\\.]/g, "\\$&")}`)); + assert.match(r.out, / NEXT run the step 7 checks now\. If they fail, roll back with: pwsh .*Rollback-LiveProxy\.ps1/); + + // Supervision: the supervisor, not the script, runs the new release. + const livePid = await waitSupervised(sb, CAND.version, CAND.commit); + assert.equal(inhibit(sb), null); + assert.equal(getTask(sb).enabled, true); + assert.equal(status(sb)!.state, "spawned", "the supervisor must have started the new release itself"); + + // Launcher: only the identity lines changed. + const after = launcher(sb).split("\n"); + const changed = before.split("\n").flatMap((l, i) => (l === after[i] ? [] : [after[i].trim()])); + assert.deepEqual(changed, [ + `[string]$ProxyPath = '${path.join(releaseDir(sb), "dist", "index.mjs")}',`, + `[string]$ExpectedVersion = '${CAND.version}',`, + `[string]$ExpectedCommit = '${CAND.commit}',`, + ]); + + // Release directory and backup. + assert.equal(fs.lstatSync(path.join(releaseDir(sb), "node_modules")).isSymbolicLink(), true); + assert.equal(fs.realpathSync(path.join(releaseDir(sb), "node_modules")), fs.realpathSync(path.join(sb.root, "shared-node-modules"))); + for (const f of [LAUNCHER, "tuned-deployment.json", "supervision-receipt.json", "deploy-record.json", "result.json", path.join("tooling", "build.mjs")]) { + assert.equal(fs.existsSync(path.join(backup, f)), true, `${f} missing from the backup`); + } + assert.equal(fs.readFileSync(path.join(backup, LAUNCHER), "utf8"), before); + const rec = record(backup); + assert.equal(rec.stage, "receipts-written"); + assert.equal(rec.reachedStage, "receipts-written"); + assert.equal(rec.deploymentId, path.basename(backup)); + + // Receipts: the documented fields, computed values, nothing else lost. + const tuned = JSON.parse(fs.readFileSync(path.join(sb.root, "tuned-deployment.json"), "utf8")); + const old = JSON.parse(tunedBefore.toString()); + assert.equal(tuned.releaseRoot, releaseDir(sb)); + assert.equal(tuned.build, CAND.commit); + assert.equal(tuned.previousBuild, PREV.commit); + assert.equal(tuned.sourceCommit, CAND.source); + assert.equal(tuned.bundleSha256, sb.cand.sha); + assert.equal(tuned.livePid, livePid); + assert.equal(tuned.masterBackupRoot, backup); + assert.equal(tuned.masterPreviousReleaseRoot, old.releaseRoot); + assert.equal(tuned.masterPreviousBuild, old.build); + assert.equal(tuned.masterPreviousBundleSha256, old.bundleSha256); + assert.equal(tuned.masterPreviousSourceCommit, old.sourceCommit); + assert.equal(tuned.deployedAt, old.deployedAt, "an untouched date field must keep its exact text"); + assert.equal(tuned.deployedConfigSha256, "stale-on-purpose", "the stale config hash is neither trusted nor rewritten"); + assert.equal(tuned.deployedSupervisorSha256, sha256(launcher(sb))); + const sup = JSON.parse(fs.readFileSync(path.join(sb.root, "supervision-receipt.json"), "utf8")); + assert.equal(sup.proxyRelease, "stale-on-purpose"); + assert.equal(sup.masterDeploy.proxyRelease, `releases/cand-1 (${CAND.version}, ${CAND.commit})`); + assert.equal(sup.masterDeploy.change, "ProxyPath, ExpectedCommit and ExpectedVersion defaults only"); + assert.equal(sup.masterDeploy.launcherSha256, sha256(launcher(sb)).toLowerCase()); + + // Nothing was written where the scripts have no business. + assert.deepEqual(fs.readdirSync(path.join(sb.root, "config", "billion-context")), ["billion-context.json"]); + assert.equal(fs.existsSync(path.join(sb.root, "state")), false); + assert.equal(r.out.includes(sb.canary), false, "config content must never be printed"); + // The sandbox root, the temp folder and the scripts' own folder (in the ready-to-paste rollback + // command) are under a profile on every machine; nothing else may be. + const redacted = [sb.root, path.dirname(sb.root), SCRIPTS].reduce((text, known) => text.split(known).join(""), r.out); + assert.equal(/Users\\/i.test(redacted), false, "no personal path other than the sandbox and the scripts' folder may be printed"); + + // Standalone rollback: preflight changes nothing, then the real thing. + const dry = await rollback(sb, backup, { PreflightOnly: true }); + expectResult(dry, 0, "preflight-ok"); + assert.equal(fs.existsSync(path.join(backup, "rollback-result.json")), false); + assert.equal((await health(sb.port))!.commit, CAND.commit); + + const back = await rollback(sb, backup); + expectResult(back, 0, "restored"); + assert.match(back.out, / ROLLBACK from cand-1 \(0\.9\.2, bbbbbbb2-live\) to 0\.9\.1, aaaaaaa1-live/); + await waitSupervised(sb, PREV.version, PREV.commit); + assert.equal(launcher(sb), before); + assert.deepEqual(fs.readFileSync(path.join(sb.root, "tuned-deployment.json")), tunedBefore); + assert.deepEqual(fs.readFileSync(path.join(sb.root, "supervision-receipt.json")), supBefore); + assert.equal(inhibit(sb), null); + assert.equal(getTask(sb).enabled, true); + assert.equal(alive(livePid), false, "the deployed release must be stopped"); + assert.equal(fs.existsSync(releaseDir(sb)), true, "the standalone rollback keeps the release directory"); + assert.equal(record(backup).stage, "rolled-back"); + assert.equal(JSON.parse(fs.readFileSync(path.join(backup, "result.json"), "utf8")).result, "deployed", "the deploy result must not be overwritten"); + assert.equal(JSON.parse(fs.readFileSync(path.join(backup, "rollback-result.json"), "utf8")).result, "restored"); + + const again = await rollback(sb, backup); + assert.equal(again.code, 2, again.out); + assert.match(again.out, /already rolled back/); + + // The other installation was never touched. + assert.equal(supervisorPid(other), otherPids.sup); + assert.equal(alive(otherPids.sup), true); + assert.equal((await health(other.port))!.pid, otherPids.proxy); + assert.equal(inhibit(other), null); + assert.equal(getTask(other).calls.disable, 0); + }, "a1"); + } finally { + await teardown(other); + } + }); + + test("a rebuild of the same commit changes only the release path", async () => { + await withSandbox( + async (sb) => { + const r = await deploy(sb); + expectResult(r, 0, "deployed"); + assert.match(r.out, /launcher updated: ProxyPath$/m); + const sup = JSON.parse(fs.readFileSync(path.join(sb.root, "supervision-receipt.json"), "utf8")); + assert.equal(sup.masterDeploy.change, "ProxyPath defaults only"); + const h = await health(sb.port); + assert.equal(h!.commit, PREV.commit); + assert.equal(String(status(sb)!.childPid), String(h!.pid)); + }, + "", + { version: PREV.version, commit: PREV.commit, source: PREV.source }, + ); + }); + + test("a candidate that never becomes healthy is rolled back to the previous release", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + fs.writeFileSync(path.join(sb.root, "proxy-control.json"), JSON.stringify({ "cand-1": { unhealthy: true } })); + const r = await deploy(sb, { HealthWaitSeconds: 6 }); + expectResult(r, 3, "rolled-back", "cutover-failed"); + await waitSupervised(sb, PREV.version, PREV.commit); + assert.equal(launcher(sb), before); + assert.equal(inhibit(sb), null); + assert.equal(getTask(sb).enabled, true); + assert.equal(fs.existsSync(releaseDir(sb)), false, "the release directory this run created must be removed"); + assert.equal(fs.existsSync(path.join(sb.root, "shared-node-modules")), true, "removing the release must not follow the node_modules link"); + const backup = onlyBackup(sb); + assert.equal(record(backup).stage, "rolled-back"); + assert.equal(record(backup).reachedStage, "cut"); + assert.equal(JSON.parse(fs.readFileSync(path.join(sb.root, "tuned-deployment.json"), "utf8")).build, PREV.commit, "receipts must be untouched"); + const after = await rollback(sb, backup); + assert.equal(after.code, 2, after.out); + assert.match(after.out, /already rolled back/); + }); + }); + + test("a client that connects after the hold makes the script stand down without stopping the proxy", async () => { + await withSandbox(async (sb) => { + const pid = (await health(sb.port))!.pid; + patchTask(sb, { onCall: [{ op: "disable", call: 1, action: "openClient" }] }); + const r = await deploy(sb); + expectResult(r, 3, "rolled-back", "not-idle"); + assert.match(r.out, / NEXT retry later with the same parameters/); + await waitSupervised(sb, PREV.version, PREV.commit); + assert.equal((await health(sb.port))!.pid, pid, "the serving proxy must never have been stopped"); + assert.equal(status(sb)!.state, "adopted"); + assert.equal(record(onlyBackup(sb)).reachedStage, "held"); + assert.equal(fs.existsSync(releaseDir(sb)), false); + }); + }); + + test("a failed rollback ends with the hold in place and the proxy left as it is; a retry restores", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + const sup = supervisorPid(sb); + fs.writeFileSync(path.join(sb.root, "proxy-control.json"), JSON.stringify({ "cand-1": { unhealthy: true }, [PREV.name]: { unhealthyFromStart: 2 } })); + const r = await deploy(sb, { HealthWaitSeconds: 6 }); + expectResult(r, 4, "rollback-failed", "previous-unhealthy"); + const backup = onlyBackup(sb); + assertHold(sb, backup, sup); + assert.equal(launcher(sb), before, "the launcher was restored before the previous release was tried"); + const left = await health(sb.port); + assert.equal(left!.ok, false); + assert.equal(left!.commit, PREV.commit); + assert.equal(alive(left!.pid), true, "the unhealthy proxy must be left running"); + assert.equal(record(backup).stage, "rollback-failed"); + assert.equal(record(backup).reachedStage, "cut"); + assert.match(r.out, / NEXT report to the owner with .*evidence-.*no release is confirmed live/); + const evidence = fs.readdirSync(backup).filter((f) => f.startsWith("evidence-")); + assert.equal(evidence.length, 1); + const text = fs.readFileSync(path.join(backup, evidence[0]), "utf8"); + assert.match(text, /hold in place: inhibit set, task disabled, no supervisor/); + assert.match(text, /health: ok=False/); + assert.equal(text.includes(sb.canary), false, "evidence must not contain config content"); + assert.equal(r.out.includes(sb.canary), false); + // No supervisor comes back by itself while the hold is in place. + await sleep(1500); + assert.equal(status(sb)!.supervisorPid === sup || !alive(Number(status(sb)!.supervisorPid)), true); + + // The owner fixes the release; the standalone rollback accepts its own hold and finishes the job. + fs.rmSync(path.join(sb.root, "proxy-control.json")); + const retry = await rollback(sb, backup); + expectResult(retry, 0, "restored"); + await waitSupervised(sb, PREV.version, PREV.commit); + assert.equal((await health(sb.port))!.pid, left!.pid, "the running proxy is adopted, not restarted"); + assert.equal(inhibit(sb), null); + assert.equal(getTask(sb).enabled, true); + }); + }); + + test("an error while restoring the launcher ends with the hold in place", async () => { + await withSandbox(async (sb) => { + const sup = supervisorPid(sb); + fs.writeFileSync(path.join(sb.root, "proxy-control.json"), JSON.stringify({ "cand-1": { unhealthy: true } })); + patchTask(sb, { onCall: [{ op: "disable", call: 2, action: "makeLauncherReadOnly" }] }); + const r = await deploy(sb, { HealthWaitSeconds: 6 }); + expectResult(r, 4, "rollback-failed", "rollback-error"); + assert.match(r.out, /rollback failed at: launcher restore/); + assertHold(sb, onlyBackup(sb), sup); + assert.equal((await health(sb.port)) === null, true, "nothing was started after the failure"); + }); + }); + + test("when the hold cannot be re-applied the script still stops and says which part is missing", async () => { + await withSandbox(async (sb) => { + fs.writeFileSync(path.join(sb.root, "proxy-control.json"), JSON.stringify({ "cand-1": { unhealthy: true }, [PREV.name]: { unhealthyFromStart: 2 } })); + patchTask(sb, { failOn: [{ op: "disable", call: 3 }] }); + const r = await deploy(sb, { HealthWaitSeconds: 6 }); + expectResult(r, 4, "rollback-failed", "previous-unhealthy"); + assert.match(r.out, /hold NOT fully in place: task could not be disabled/); + assert.match(inhibit(sb) ?? "", /^rollback /); + assert.equal(getTask(sb).enabled, true); + }); + }); + + test("a hold that cannot be applied is undone with the proxy untouched", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + const pid = (await health(sb.port))!.pid; + const sup = supervisorPid(sb); + patchTask(sb, { failOn: [{ op: "disable", call: 1 }] }); + const r = await deploy(sb); + expectResult(r, 3, "rolled-back", "hold-failed"); + assert.equal(inhibit(sb), null); + assert.equal(getTask(sb).enabled, true); + assert.equal(alive(sup), true, "the supervisor was never stopped"); + assert.equal((await health(sb.port))!.pid, pid); + assert.equal(launcher(sb), before); + assert.equal(record(onlyBackup(sb)).reachedStage, "holding"); + assert.equal(fs.existsSync(releaseDir(sb)), false); + }); + }); + + test("maintenance started by someone else after preflight is left alone", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + const sup = supervisorPid(sb); + patchTask(sb, { onCall: [{ op: "info", call: 2, action: "writeForeignInhibit" }] }); + const r = await deploy(sb); + expectResult(r, 6, "error-before-hold"); + assert.equal(inhibit(sb), "someone else", "the other maintenance's inhibit must be untouched"); + assert.equal(getTask(sb).enabled, true); + assert.equal(getTask(sb).calls.disable, 0); + assert.equal(alive(sup), true); + assert.equal(launcher(sb), before); + assert.equal(fs.existsSync(releaseDir(sb)), false); + const backup = onlyBackup(sb); + assert.equal(record(backup).stage, "aborted"); + const refused = await rollback(sb, backup); + assert.equal(refused.code, 2, refused.out); + assert.match(refused.out, /belongs to other maintenance/); + fs.rmSync(path.join(sb.root, "maintenance.inhibit")); + const nothing = await rollback(sb, backup); + assert.equal(nothing.code, 2, nothing.out); + assert.match(nothing.out, /stopped at stage 'aborted' and changed nothing/); + }); + }); + + test("a launcher edited by someone else during the hold is never overwritten", async () => { + await withSandbox(async (sb) => { + const sup = supervisorPid(sb); + const pid = (await health(sb.port))!.pid; + patchTask(sb, { onCall: [{ op: "disable", call: 1, action: "editLauncher" }] }); + const r = await deploy(sb); + expectResult(r, 4, "rollback-failed", "foreign-launcher-edit"); + assert.match(launcher(sb), /# foreign edit/); + assertHold(sb, onlyBackup(sb), sup); + assert.equal(alive(pid), true, "the serving proxy is left as it is"); + const refused = await rollback(sb, onlyBackup(sb)); + assert.equal(refused.code, 2, refused.out); + assert.match(refused.out, /launcher was changed after this deployment/); + }); + }); + + test("a receipt that cannot be replaced does not roll back a healthy release", async () => { + await withSandbox(async (sb) => { + fs.chmodSync(path.join(sb.root, "tuned-deployment.json"), 0o444); + const r = await deploy(sb); + expectResult(r, 5, "deployed-receipts-incomplete"); + assert.match(r.out, /not every receipt was updated \(tuned-deployment\.json, supervision-receipt\.json\)/); + assert.match(r.out, / NEXT finish the receipts: Copy-Item -LiteralPath .*tuned-deployment\.pending\.json/); + await waitSupervised(sb, CAND.version, CAND.commit); + const backup = onlyBackup(sb); + assert.equal(record(backup).stage, "healthy"); + assert.equal(JSON.parse(fs.readFileSync(path.join(backup, "tuned-deployment.pending.json"), "utf8")).build, CAND.commit); + assert.equal(JSON.parse(fs.readFileSync(path.join(sb.root, "tuned-deployment.json"), "utf8")).build, PREV.commit); + // Rolling back from this stage works and leaves the receipts as they were. + const back = await rollback(sb, backup); + expectResult(back, 0, "restored"); + await waitSupervised(sb, PREV.version, PREV.commit); + }); + }); + + test("a deploy script that died mid-run can be rolled back from its record", async () => { + // Killed after its own inhibit exists but before the task was disabled. + await withSandbox(async (sb) => { + const pid = (await health(sb.port))!.pid; + patchTask(sb, { onCall: [{ op: "disable", call: 1, action: "killSelf" }] }); + const r = await deploy(sb); + assert.notEqual(r.code, 0); + assert.equal(/ RESULT /.test(r.out), false, "a killed script prints no result"); + const backup = onlyBackup(sb); + assert.equal(record(backup).stage, "holding"); + assert.match(inhibit(sb) ?? "", /^deploy /); + patchTask(sb, { onCall: [] }); + const back = await rollback(sb, backup); + expectResult(back, 0, "restored"); + await waitSupervised(sb, PREV.version, PREV.commit); + assert.equal((await health(sb.port))!.pid, pid); + assert.equal(inhibit(sb), null); + }); + // Killed in the middle of the lift: previous proxy stopped, launcher edited, task still disabled. + await withSandbox(async (sb) => { + const before = launcher(sb); + patchTask(sb, { onCall: [{ op: "enable", call: 1, action: "killSelf" }] }); + const r = await deploy(sb); + assert.notEqual(r.code, 0); + const backup = onlyBackup(sb); + assert.equal(record(backup).stage, "cut"); + assert.equal(inhibit(sb), null); + assert.equal(getTask(sb).enabled, false); + assert.equal(await health(sb.port), null); + patchTask(sb, { onCall: [] }); + const back = await rollback(sb, backup); + expectResult(back, 0, "restored"); + await waitSupervised(sb, PREV.version, PREV.commit); + assert.equal(launcher(sb), before); + assert.equal(getTask(sb).enabled, true); + }); + }); + + test("the standalone rollback refuses a backup that is not the current deployment's", async () => { + await withSandbox(async (sb) => { + expectResult(await deploy(sb), 0, "deployed"); + const backup = onlyBackup(sb); + const recPath = path.join(backup, "deploy-record.json"); + const good = fs.readFileSync(recPath, "utf8"); + const live = launcher(sb); + const refuse = async (label: string, reason: RegExp, target = backup) => { + const r = await rollback(sb, target); + assert.equal(r.code, 2, `${label}: expected exit 2\n${r.out}`); + assert.match(r.out, reason, `${label}\n${r.out}`); + assert.equal(inhibit(sb), null, label); + assert.equal(launcher(sb), live, label); + assert.equal(getTask(sb).enabled, true, label); + }; + const withRecord = async (label: string, change: (rec: Record) => void, reason: RegExp) => { + const rec = JSON.parse(good); + change(rec); + fs.writeFileSync(recPath, JSON.stringify(rec)); + await refuse(label, reason); + fs.writeFileSync(recPath, good); + }; + await withRecord("unknown field", (r) => (r.extra = 1), /deployment record is not valid \(unexpected or missing fields\)/); + await withRecord("other root", (r) => (r.runtimeRoot = `${sb.root}-elsewhere`), /belongs to another runtime root/); + await withRecord("forbidden port", (r) => (r.port = 70000), /malformed port/); + await withRecord("path outside releases", (r) => (r.candidate.proxyPath = path.join(sb.root, "dist", "index.mjs")), /candidate release path in the record is not/); + await withRecord("nothing changed", (r) => (r.stage = "prepared"), /stopped at stage 'prepared' and changed nothing/); + await withRecord("other previous release", (r) => (r.previous.commit = "ddddddd4-live"), /does not name the previous release/); + fs.renameSync(recPath, `${recPath}.off`); + await refuse("record missing", /deploy-record\.json is missing/); + fs.renameSync(`${recPath}.off`, recPath); + await refuse("outside backups", /not directly under \\backups/, path.join(sb.root, "releases", PREV.name)); + fs.writeFileSync(path.join(sb.root, "maintenance.inhibit"), "someone else"); + const foreign = await rollback(sb, backup); + assert.equal(foreign.code, 2, foreign.out); + assert.match(foreign.out, /belongs to other maintenance/); + assert.equal(inhibit(sb), "someone else"); + fs.rmSync(path.join(sb.root, "maintenance.inhibit")); + // Still the current deployment: the same backup is accepted once nothing is wrong. + expectResult(await rollback(sb, backup, { PreflightOnly: true }), 0, "preflight-ok"); + }); + }); +}); From b04db286b81267308af4b9fb86856504694a743a Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Tue, 17 Nov 2026 23:48:32 -0500 Subject: [PATCH 2/5] fix(tooling): rollback accepts a disabled task only when the deployment explains it (#80) The standalone rollback refused nothing about the task's enabled state. It now accepts a disabled task only when this deployment's own hold is in place or its record stopped inside the hold window; otherwise it refuses, since someone else may be doing maintenance. Covered by a new refusal in the sandbox test. Also give the "candidate never healthy" scenario a wait that is not tight: the same wait bounds the previous release coming back. Assisted-by: Cindy (https://cindy.app) --- scripts/live-deploy/Rollback-LiveProxy.ps1 | 2 ++ tests/live-deploy-scripts.test.ts | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/live-deploy/Rollback-LiveProxy.ps1 b/scripts/live-deploy/Rollback-LiveProxy.ps1 index a4cdd3a23c..dd85bb68fd 100644 --- a/scripts/live-deploy/Rollback-LiveProxy.ps1 +++ b/scripts/live-deploy/Rollback-LiveProxy.ps1 @@ -115,6 +115,8 @@ try { if (-not $isBefore -and -not $isAfter) { Stop-Refused 'the launcher was changed after this deployment; this is not the current deployment''s backup' } if ($stage -eq 'rolled-back') { Stop-Refused 'this deployment was already rolled back' } if (@('prepared', 'aborted') -contains $stage) { Stop-Refused "this deployment stopped at stage '$stage' and changed nothing" } + # A disabled task is this deployment's doing only if its own hold is in place or it stopped inside the hold window. + if (-not $task.Enabled -and -not $ownHold -and @('holding', 'held', 'cut', 'rollback-failed') -notcontains $stage) { Stop-Refused "the scheduled task is disabled and neither this deployment's hold nor its stage '$stage' explains it; other maintenance may be in progress" } $ctx.ConfigSha = Get-LpSha256 $ctx.ConfigPath $receipts = @{} diff --git a/tests/live-deploy-scripts.test.ts b/tests/live-deploy-scripts.test.ts index c0207ef760..f89d814ce0 100644 --- a/tests/live-deploy-scripts.test.ts +++ b/tests/live-deploy-scripts.test.ts @@ -710,9 +710,10 @@ describe("live-deploy scripts: deployment and rollback", { skip: SKIP, concurren test("a candidate that never becomes healthy is rolled back to the previous release", async () => { await withSandbox(async (sb) => { + // The same wait also bounds the previous release coming back, so it must not be tight. const before = launcher(sb); fs.writeFileSync(path.join(sb.root, "proxy-control.json"), JSON.stringify({ "cand-1": { unhealthy: true } })); - const r = await deploy(sb, { HealthWaitSeconds: 6 }); + const r = await deploy(sb, { HealthWaitSeconds: 20 }); expectResult(r, 3, "rolled-back", "cutover-failed"); await waitSupervised(sb, PREV.version, PREV.commit); assert.equal(launcher(sb), before); @@ -963,6 +964,14 @@ describe("live-deploy scripts: deployment and rollback", { skip: SKIP, concurren assert.match(foreign.out, /belongs to other maintenance/); assert.equal(inhibit(sb), "someone else"); fs.rmSync(path.join(sb.root, "maintenance.inhibit")); + // A task disabled by someone else: a finished deployment does not explain it. + patchTask(sb, { enabled: false }); + const disabled = await rollback(sb, backup); + assert.equal(disabled.code, 2, disabled.out); + assert.match(disabled.out, /scheduled task is disabled and neither this deployment's hold nor its stage/); + assert.equal(getTask(sb).enabled, false, "a task disabled by someone else stays disabled"); + assert.equal(launcher(sb), live); + patchTask(sb, { enabled: true }); // Still the current deployment: the same backup is accepted once nothing is wrong. expectResult(await rollback(sb, backup, { PreflightOnly: true }), 0, "preflight-ok"); }); From 209793c6a80dfbaa0f4dbcfc37a90ab55559cd38 Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Wed, 18 Nov 2026 00:23:12 -0500 Subject: [PATCH 3/5] fix(tooling): address the automated review of the live-deploy scripts (#80) Scripts - Rollback never overwrites a launcher that is neither the saved nor the deployed one: it stops with the hold in place (foreign-launcher-edit). - One deploy or rollback per runtime root: an exclusive operation lock is held from before the checks until the process ends. - The standalone rollback journals stage rolling-back before it changes anything, so an interrupted run can be retried. - Processes are matched by how they were started (node entry point, PowerShell -File script), not by a path appearing anywhere in the command line. - A process is stopped through one handle whose start time was verified. - The task action must be powershell.exe with this launcher as -File; the task is compared with the validated one before the hold instead of re-read. - Lifting the hold removes only this run's inhibit; a half-written inhibit is removed again; a missing live receipt is restored from the backup. - A record that cannot be updated after a successful rollback no longer reads as exit 4; an unexpected error after rollback re-applies the hold first. - Minimum PowerShell is 7.2 (JsonNode). Tests: foreign edit during rollback, healthy candidate with supervision that does not match, second run refused, interrupted standalone rollback retried, task action look-alikes, refusals now assert no task operation and unchanged processes, and the prefix-sibling sandbox really shares a name prefix. Assisted-by: Cindy (https://cindy.app) --- reference/cindy-project.md | 2 +- reference/cindy-project.zh-CN.md | 2 +- scripts/live-deploy/Deploy-LiveProxy.ps1 | 18 ++- scripts/live-deploy/LiveProxy.Common.ps1 | 88 ++++++++++++--- scripts/live-deploy/Rollback-LiveProxy.ps1 | 21 +++- .../live-deploy/Start-BillionContextProxy.ps1 | 13 ++- tests/fixtures/live-deploy/TestTaskDriver.ps1 | 5 +- tests/live-deploy-scripts.test.ts | 104 ++++++++++++++++-- 8 files changed, 213 insertions(+), 40 deletions(-) diff --git a/reference/cindy-project.md b/reference/cindy-project.md index ccd81f3e18..bc57cb509a 100644 --- a/reference/cindy-project.md +++ b/reference/cindy-project.md @@ -101,7 +101,7 @@ Steps 4–9 follow the order the kept `Deploy-Master.ps1` implements: the superv ### Deploy and rollback scripts -`scripts/live-deploy/Deploy-LiveProxy.ps1` and `scripts/live-deploy/Rollback-LiveProxy.ps1` need PowerShell 7 and take every installation-specific value as a parameter; nothing about a release, the port or the task is written into them. Run them non-interactively, and run the deploy script with `-PreflightOnly` first: it performs every check, prints the plan and changes nothing. +`scripts/live-deploy/Deploy-LiveProxy.ps1` and `scripts/live-deploy/Rollback-LiveProxy.ps1` need PowerShell 7.2 or later and take every installation-specific value as a parameter; nothing about a release, the port or the task is written into them. Only one deploy or rollback can run for a runtime root at a time; a second one is refused. Run them non-interactively, and run the deploy script with `-PreflightOnly` first: it performs every check, prints the plan and changes nothing. ```powershell pwsh -NoProfile -NonInteractive -File scripts\live-deploy\Deploy-LiveProxy.ps1 ` diff --git a/reference/cindy-project.zh-CN.md b/reference/cindy-project.zh-CN.md index cf41d4ad6c..9aa0e78c39 100644 --- a/reference/cindy-project.zh-CN.md +++ b/reference/cindy-project.zh-CN.md @@ -101,7 +101,7 @@ Get-Content (Join-Path $proxyRoot 'tuned-deployment.json') ### 部署与回滚脚本 -`scripts/live-deploy/Deploy-LiveProxy.ps1` 与 `scripts/live-deploy/Rollback-LiveProxy.ps1` 需要 PowerShell 7,所有与具体安装相关的值都以参数传入;脚本中不写入任何 release、端口或计划任务的信息。请以非交互方式运行,并先用 `-PreflightOnly` 运行部署脚本:它执行全部检查、打印计划,且不做任何更改。 +`scripts/live-deploy/Deploy-LiveProxy.ps1` 与 `scripts/live-deploy/Rollback-LiveProxy.ps1` 需要 PowerShell 7.2 或更高版本,所有与具体安装相关的值都以参数传入;脚本中不写入任何 release、端口或计划任务的信息。同一运行时根目录同一时间只能运行一个部署或回滚,第二个会被拒绝。请以非交互方式运行,并先用 `-PreflightOnly` 运行部署脚本:它执行全部检查、打印计划,且不做任何更改。 ```powershell pwsh -NoProfile -NonInteractive -File scripts\live-deploy\Deploy-LiveProxy.ps1 ` diff --git a/scripts/live-deploy/Deploy-LiveProxy.ps1 b/scripts/live-deploy/Deploy-LiveProxy.ps1 index a56f9731ff..7453303438 100644 --- a/scripts/live-deploy/Deploy-LiveProxy.ps1 +++ b/scripts/live-deploy/Deploy-LiveProxy.ps1 @@ -1,4 +1,4 @@ -#requires -Version 7.0 +#requires -Version 7.2 <# .SYNOPSIS Deploys a tested candidate build as the live proxy release, following the documented procedure. @@ -128,6 +128,9 @@ try { $tunedPath = Join-Path $root 'tuned-deployment.json' $supervisionPath = Join-Path $root 'supervision-receipt.json' + # A full run takes the operation lock before it looks at anything; -PreflightOnly creates no file at all. + if (-not $PreflightOnly) { Enter-LpOperation $root } + # ---- preflight: read-only, every check runs ---------------------------------------------------- $candidateFiles = [ordered]@{ 'index.mjs' = 'dist\index.mjs'; 'build-info.json' = 'dist\build-info.json'; 'package.json' = 'package.json'; 'LICENSE' = 'LICENSE' } $candidateBytes = @{}; $candidateSha = [ordered]@{} @@ -166,7 +169,8 @@ try { $t = Get-LiveProxyTaskInfo $TaskName if ($t.Count -ne 1) { return "expected exactly one task with this name, found $($t.Count)" } if (-not $t.Enabled) { return 'the task is disabled' } - if ($t.ActionArguments.IndexOf($ctx.LauncherPath, [System.StringComparison]::OrdinalIgnoreCase) -lt 0) { return 'the task action does not run the launcher in this runtime root' } + $actionGap = Get-LpTaskActionGap $t $ctx.LauncherPath + if ($actionGap) { return $actionGap } $ctx.TaskBaseline = $t } Add-Check 'configuration hash' { if (-not (Test-LpSameHash (Get-LpSha256 $ctx.ConfigPath) $ExpectedConfigSha256)) { 'the configuration file differs from -ExpectedConfigSha256' } } @@ -277,7 +281,7 @@ try { Write-LpEvent 'BACKUP' $backupRoot $r = Invoke-LpLauncherDryRun $editedBytes $root if ($r.ExitCode -ne 0 -or $r.Action -ne 'conflict') { throw "with the new release in place the edited launcher decided '$($r.Action)' (exit $($r.ExitCode)); expected 'conflict' while the current release is still serving" } - $ctx.TaskBaseline = Get-LiveProxyTaskInfo $TaskName + if (-not (Test-LpSameTask (Get-LiveProxyTaskInfo $TaskName) $ctx.TaskBaseline)) { throw 'the scheduled task was changed after preflight' } Set-LpStage $ctx 'holding' try { New-LpInhibit $ctx 'deploy' } catch { throw "maintenance.inhibit could not be created ($($_.Exception.Message)); other maintenance may have started" } } catch { @@ -322,15 +326,17 @@ try { Write-LpEvent 'STEP' "FAILED: $($_.Exception.Message); rolling back" try { Invoke-LpRollback $ctx $launcherBytes @{} $holdOnly - Set-LpStage $ctx 'rolled-back' + # The previous release is live and checked from here on: a bookkeeping error must not read as a failed rollback. $note = '' - try { Remove-CreatedRelease $newRoot } catch { $note = " The release directory could not be removed: $($_.Exception.Message)." } + try { Set-LpStage $ctx 'rolled-back' } catch { $note = ' The deployment record could not be updated.' } + try { Remove-CreatedRelease $newRoot } catch { $note += " The release directory could not be removed: $($_.Exception.Message)." } $next = if ($reason -eq 'not-idle') { 'retry later with the same parameters, when no client is connected' } else { 'investigate the failure above before retrying' } Complete-Lp $ctx ([LpStop]::new(3, 'rolled-back', "nothing was deployed; the previous release is live under supervision.$note", $reason, $next)) 'result.json' } catch [LpStop] { Complete-Lp $ctx $_.Exception 'result.json' } catch { - Complete-Lp $ctx ([LpStop]::new(4, 'rollback-failed', "rollback stopped unexpectedly: $($_.Exception.Message). Check by hand that maintenance.inhibit is present, the task is disabled and no supervisor runs.", 'rollback-error', 'report to the owner; no release is confirmed live')) 'result.json' + # Exit 4 always means the hold was re-applied and evidence was collected. + try { Stop-LpRollbackFailed $ctx "unexpected error ($($_.Exception.Message))" 'rollback-error' } catch [LpStop] { Complete-Lp $ctx $_.Exception 'result.json' } } } diff --git a/scripts/live-deploy/LiveProxy.Common.ps1 b/scripts/live-deploy/LiveProxy.Common.ps1 index 0c3eb6368b..006c791eaf 100644 --- a/scripts/live-deploy/LiveProxy.Common.ps1 +++ b/scripts/live-deploy/LiveProxy.Common.ps1 @@ -7,7 +7,7 @@ $script:LpUtf8 = [System.Text.UTF8Encoding]::new($false) $script:LpPollMs = 500 $script:LpSeam = $false $script:LpRoot = $null -$script:LpStages = @('prepared', 'aborted', 'holding', 'held', 'cut', 'healthy', 'receipts-written', 'rolled-back', 'rollback-failed') +$script:LpStages = @('prepared', 'aborted', 'holding', 'held', 'cut', 'healthy', 'receipts-written', 'rolling-back', 'rolled-back', 'rollback-failed') $script:LpForwardStages = @('prepared', 'holding', 'held', 'cut', 'healthy', 'receipts-written') $script:LpPatterns = @{ Version = '^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$' @@ -225,21 +225,38 @@ function Get-LpClientCount($Ctx) { return @($rows | Where-Object { @('127.0.0.1', '::1') -contains [string]$_.RemoteAddress -and $own -notcontains [int]$_.OwningProcess }).Count } -# Processes of one image whose command line contains the given path as a quoted token. -function Get-LpProcesses([string]$ImageName, [string]$QuotedPath) { - $token = '"' + $QuotedPath + '"' +# Processes of one image whose command line contains the given text. The text names how the process was +# started, not just a path: a process that merely receives the path as an argument does not match. +function Get-LpProcesses([string]$ImageName, [string]$Needle) { return @(Get-CimInstance Win32_Process -Filter "Name='$ImageName'" -OperationTimeoutSec 15 -ErrorAction Stop | - Where-Object { $null -ne $_.CommandLine -and $_.CommandLine.IndexOf($token, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 }) + Where-Object { $null -ne $_.CommandLine -and $_.CommandLine.IndexOf($Needle, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 }) +} +# The supervisor is Windows PowerShell running the launcher as its -File script. +function Get-LpSupervisors($Ctx) { return @(Get-LpProcesses 'powershell.exe' ('-File "' + $Ctx.LauncherPath + '"')) } +# A proxy is node running the bundle as its entry point: "" "" start ... +function Get-LpProxies([string]$ProxyPath) { + $needle = '" "' + $ProxyPath + '" start ' + return @(Get-LpProcesses 'node.exe' $needle | Where-Object { $_.CommandLine.IndexOf($needle, [System.StringComparison]::OrdinalIgnoreCase) -eq $_.CommandLine.IndexOf('" "', [System.StringComparison]::Ordinal) }) } -function Get-LpSupervisors($Ctx) { return @(Get-LpProcesses 'powershell.exe' $Ctx.LauncherPath) } -function Get-LpProxies([string]$ProxyPath) { return @(Get-LpProcesses 'node.exe' $ProxyPath) } -# Stops one process after re-checking that the PID still belongs to the process that was matched. +# Stops the process that was matched. One handle is opened, its start time is compared with the matched +# process, and the same handle is used to stop it, so a PID reused in between cannot be hit. function Stop-LpProcessVerified($Process) { - $now = Get-CimInstance Win32_Process -Filter "ProcessId=$($Process.ProcessId)" -OperationTimeoutSec 15 -ErrorAction Stop - if ($null -eq $now) { return } - if ($now.CreationDate -ne $Process.CreationDate) { throw "pid $($Process.ProcessId) now belongs to a different process; not stopping it" } - Stop-Process -Id $Process.ProcessId -Force -ErrorAction Stop + $p = $null + try { $p = [System.Diagnostics.Process]::GetProcessById([int]$Process.ProcessId) } catch [System.ArgumentException] { return } + try { + try { + $null = $p.SafeHandle + $started = $p.StartTime.ToUniversalTime() + } catch { + if ($p.HasExited) { return } + throw + } + if ([Math]::Abs(($started - $Process.CreationDate.ToUniversalTime()).TotalSeconds) -ge 1) { throw "pid $($Process.ProcessId) now belongs to a different process; not stopping it" } + try { $p.Kill() } catch { if (-not $p.HasExited) { throw } } + } finally { + $p.Dispose() + } } function Wait-LpUntil([scriptblock]$Condition, [int]$Seconds, [string]$What) { @@ -291,6 +308,16 @@ function Get-LiveProxyTaskInfo([string]$TaskName) { TriggerCount = @($t[0].Triggers).Count } } +# Reason the task cannot be the one that runs this launcher, or $null. The action must be Windows PowerShell +# with the launcher as its -File script; a path that merely starts with the launcher path does not count. +function Get-LpTaskActionGap($Task, [string]$LauncherPath) { + if ([System.IO.Path]::GetFileName([string]$Task.Execute) -ine 'powershell.exe') { return 'the task action is not Windows PowerShell (powershell.exe)' } + if (([string]$Task.ActionArguments).IndexOf('-File "' + $LauncherPath + '"', [System.StringComparison]::OrdinalIgnoreCase) -lt 0) { return 'the task action does not run the launcher in this runtime root' } + return $null +} +function Test-LpSameTask($A, $B) { + return ($A.Count -eq $B.Count -and $A.Enabled -eq $B.Enabled -and $A.Execute -eq $B.Execute -and $A.ActionArguments -eq $B.ActionArguments -and $A.TriggerCount -eq $B.TriggerCount) +} function Disable-LiveProxyTask([string]$TaskName) { Get-LpTaskObjects $TaskName | Disable-ScheduledTask -ErrorAction Stop | Out-Null } function Enable-LiveProxyTask([string]$TaskName) { Get-LpTaskObjects $TaskName | Enable-ScheduledTask -ErrorAction Stop | Out-Null } function Start-LiveProxyTask([string]$TaskName) { Get-LpTaskObjects $TaskName | Start-ScheduledTask -ErrorAction Stop } @@ -311,6 +338,19 @@ function Resolve-LpTaskDriver([string]$RootAsGiven, [string]$RootResolved, [stri return $driver } +# ---- one operation at a time --------------------------------------------------------------------- + +# Deploy and rollback hold this lock from before their checks until the process ends, so two runs for the +# same runtime root cannot interleave. The file is removed when the handle closes, also if the process dies. +function Enter-LpOperation([string]$Root) { + $path = Join-Path $Root 'live-deploy.lock' + try { + $script:LpOperationLock = [System.IO.FileStream]::new($path, [System.IO.FileMode]::OpenOrCreate, [System.IO.FileAccess]::ReadWrite, [System.IO.FileShare]::None, 1, [System.IO.FileOptions]::DeleteOnClose) + } catch [System.IO.IOException] { + throw 'another deploy or rollback is running for this runtime root' + } +} + # ---- hold and lift ----------------------------------------------------------------------------- function Get-LpInhibitText($Ctx) { @@ -326,7 +366,16 @@ function Test-LpOwnInhibit($Ctx) { function New-LpInhibit($Ctx, [string]$Verb) { $bytes = $script:LpUtf8.GetBytes("$Verb $($Ctx.DeploymentId)") $s = [System.IO.File]::Open($Ctx.InhibitPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) - try { $s.Write($bytes, 0, $bytes.Length) } finally { $s.Dispose() } + $written = $false + try { + $s.Write($bytes, 0, $bytes.Length) + $s.Flush($true) + $written = $true + } finally { + $s.Dispose() + # The file exists only because this call created it: a marker without its owner text must not stay. + if (-not $written) { Remove-Item -LiteralPath $Ctx.InhibitPath -Force -ErrorAction SilentlyContinue } + } } function Assert-LpNoSupervisor($Ctx, [string]$When) { @@ -348,6 +397,7 @@ function Set-LpRollbackHold($Ctx) { function Remove-LpHold($Ctx) { Assert-LpNoSupervisor $Ctx 'before the hold is lifted' + if (-not (Test-LpOwnInhibit $Ctx)) { throw 'maintenance.inhibit is no longer the one this run created; it was left as found' } Remove-Item -LiteralPath $Ctx.InhibitPath -Force $Ctx.LiftAt = (Get-Date).ToUniversalTime() Enable-LiveProxyTask $Ctx.TaskName @@ -369,7 +419,12 @@ function Get-LpSuccessGap($Ctx, $Identity) { $sups = @(Get-LpSupervisors $Ctx) if ($sups.Count -ne 1) { return "supervisors: $($sups.Count)" } if (-not (Test-Path -LiteralPath $Ctx.StatusPath)) { return 'supervisor-status.json is absent' } - try { $st = [System.IO.File]::ReadAllText($Ctx.StatusPath) | ConvertFrom-Json -AsHashtable } catch { return 'supervisor-status.json is unreadable' } + # The supervisor rewrites this file every heartbeat; a read that lands on the rewrite is tried again. + $st = $null + foreach ($try in 1..5) { + try { $st = [System.IO.File]::ReadAllText($Ctx.StatusPath) | ConvertFrom-Json -AsHashtable; break } catch { Start-Sleep -Milliseconds 100 } + } + if ($null -eq $st) { return 'supervisor-status.json is unreadable' } $beat = $null try { $beat = ([datetime]$st['heartbeatAt']).ToUniversalTime() } catch { return 'supervisor-status.json has no heartbeat time' } if ($beat -le $Ctx.LiftAt) { return 'no supervisor heartbeat since the hold was lifted' } @@ -496,13 +551,16 @@ function Invoke-LpRollback($Ctx, [byte[]]$RestoreLauncherBytes, [hashtable]$Rece if (Test-LpSameHash (Get-LpSha256 $Ctx.LauncherPath) $Ctx.Record.launcherSha256Before) { Write-LpEvent 'STEP' 'launcher already is the pre-deployment launcher' } else { + if (-not (Test-LpSameHash (Get-LpSha256 $Ctx.LauncherPath) $Ctx.Record.launcherSha256After)) { + Stop-LpRollbackFailed $Ctx 'launcher restore: the launcher is neither the saved nor the deployed one; it was changed by someone else and left as found' 'foreign-launcher-edit' + } if (-not (Test-LpSameHash (Get-LpSha256OfBytes $RestoreLauncherBytes) $Ctx.Record.launcherSha256Before)) { throw 'the saved launcher does not match the deployment record' } Write-LpBytesAtomic $Ctx.LauncherPath $RestoreLauncherBytes Write-LpEvent 'STEP' 'launcher restored and verified' } $step = 'receipt restore' foreach ($path in $Receipts.Keys) { - if (Test-LpSameHash (Get-LpSha256 $path) (Get-LpSha256OfBytes $Receipts[$path])) { continue } + if ((Test-Path -LiteralPath $path -PathType Leaf) -and (Test-LpSameHash (Get-LpSha256 $path) (Get-LpSha256OfBytes $Receipts[$path]))) { continue } Write-LpBytesAtomic $path $Receipts[$path] Write-LpEvent 'STEP' "$(Get-LpRelative $path) restored and verified" } diff --git a/scripts/live-deploy/Rollback-LiveProxy.ps1 b/scripts/live-deploy/Rollback-LiveProxy.ps1 index dd85bb68fd..d33074b743 100644 --- a/scripts/live-deploy/Rollback-LiveProxy.ps1 +++ b/scripts/live-deploy/Rollback-LiveProxy.ps1 @@ -1,4 +1,4 @@ -#requires -Version 7.0 +#requires -Version 7.2 <# .SYNOPSIS Rolls one live proxy deployment back to the release it replaced, from that deployment's backup folder. @@ -57,6 +57,9 @@ try { $root = Resolve-LpPath $RuntimeRoot if (Test-LpReparseInPath $root) { Stop-Refused 'a folder in the runtime root path is a link' } $script:LpRoot = $root + # A full run takes the operation lock before it looks at anything; -PreflightOnly creates no file at all. + if (-not $PreflightOnly) { try { Enter-LpOperation $root } catch { Stop-Refused $_.Exception.Message } } + $driver = $null try { $driver = Resolve-LpTaskDriver $RuntimeRoot $root $TestTaskDriver } catch { Stop-Refused $_.Exception.Message } if ($driver) { . $driver; Write-LpEvent 'STEP' 'TEST SEAM ACTIVE: scheduled-task operations go to the test driver' } @@ -102,7 +105,8 @@ try { if ((Test-Path -LiteralPath $ctx.InhibitPath) -and -not $ownHold) { Stop-Refused 'maintenance.inhibit is present and belongs to other maintenance' } $task = Get-LiveProxyTaskInfo $ctx.TaskName if ($task.Count -ne 1) { Stop-Refused "expected exactly one scheduled task named in the record, found $($task.Count)" } - if ($task.ActionArguments.IndexOf($ctx.LauncherPath, [System.StringComparison]::OrdinalIgnoreCase) -lt 0) { Stop-Refused 'the task action does not run the launcher in this runtime root' } + $actionGap = Get-LpTaskActionGap $task $ctx.LauncherPath + if ($actionGap) { Stop-Refused $actionGap } $ctx.TaskBaseline = $task $dp = Get-CimInstance Win32_Process -Filter "ProcessId=$([int]$rec.deployProcess.pid)" -OperationTimeoutSec 15 -ErrorAction Stop @@ -116,7 +120,7 @@ try { if ($stage -eq 'rolled-back') { Stop-Refused 'this deployment was already rolled back' } if (@('prepared', 'aborted') -contains $stage) { Stop-Refused "this deployment stopped at stage '$stage' and changed nothing" } # A disabled task is this deployment's doing only if its own hold is in place or it stopped inside the hold window. - if (-not $task.Enabled -and -not $ownHold -and @('holding', 'held', 'cut', 'rollback-failed') -notcontains $stage) { Stop-Refused "the scheduled task is disabled and neither this deployment's hold nor its stage '$stage' explains it; other maintenance may be in progress" } + if (-not $task.Enabled -and -not $ownHold -and @('holding', 'held', 'cut', 'rolling-back', 'rollback-failed') -notcontains $stage) { Stop-Refused "the scheduled task is disabled and neither this deployment's hold nor its stage '$stage' explains it; other maintenance may be in progress" } $ctx.ConfigSha = Get-LpSha256 $ctx.ConfigPath $receipts = @{} @@ -128,6 +132,8 @@ try { Write-LpEvent 'STEP' "would do: $plan" throw [LpStop]::new(0, 'preflight-ok', 'rollback preflight passed; nothing was changed', '', 'run again without -PreflightOnly to roll back') } + # Journal the rollback before anything is changed, so that an interrupted run can be recognised and retried. + try { Set-LpStage $ctx 'rolling-back' } catch { Stop-Refused "the deployment record cannot be written ($($_.Exception.Message))" } } catch [LpStop] { Complete-Lp $null $_.Exception $null } catch { @@ -136,10 +142,13 @@ try { try { Invoke-LpRollback $ctx $backupLauncher $receipts $false - Set-LpStage $ctx 'rolled-back' - Complete-Lp $ctx ([LpStop]::new(0, 'restored', "the previous release $($ctx.Previous.version), $($ctx.Previous.commit) is live under supervision; receipts describe it", '', 'report the failure, the release left live and the backup location (step 10). A later deployment needs a new -ReleaseName.')) 'rollback-result.json' + # The previous release is live and checked from here on: a bookkeeping error must not read as a failed rollback. + $note = '' + try { Set-LpStage $ctx 'rolled-back' } catch { $note = ' The deployment record could not be updated; running this rollback again is harmless.' } + Complete-Lp $ctx ([LpStop]::new(0, 'restored', "the previous release $($ctx.Previous.version), $($ctx.Previous.commit) is live under supervision; receipts describe it.$note", '', 'report the failure, the release left live and the backup location (step 10). A later deployment needs a new -ReleaseName.')) 'rollback-result.json' } catch [LpStop] { Complete-Lp $ctx $_.Exception 'rollback-result.json' } catch { - Complete-Lp $ctx ([LpStop]::new(4, 'rollback-failed', "rollback stopped unexpectedly: $($_.Exception.Message). Check by hand that maintenance.inhibit is present, the task is disabled and no supervisor runs.", 'rollback-error', 'report to the owner; no release is confirmed live')) 'rollback-result.json' + # Exit 4 always means the hold was re-applied and evidence was collected. + try { Stop-LpRollbackFailed $ctx "unexpected error ($($_.Exception.Message))" 'rollback-error' } catch [LpStop] { Complete-Lp $ctx $_.Exception 'rollback-result.json' } } diff --git a/tests/fixtures/live-deploy/Start-BillionContextProxy.ps1 b/tests/fixtures/live-deploy/Start-BillionContextProxy.ps1 index 519e5fed9d..4f36ca18ac 100644 --- a/tests/fixtures/live-deploy/Start-BillionContextProxy.ps1 +++ b/tests/fixtures/live-deploy/Start-BillionContextProxy.ps1 @@ -62,10 +62,21 @@ function Write-Status([hashtable]$s) { if (Test-Path -LiteralPath $statusPath) { [System.IO.File]::Replace($tmp, $statusPath, [NullString]::Value) } else { [System.IO.File]::Move($tmp, $statusPath) } } catch { } } +# Test control: \status-control.json = { "": { "healthMatch": false, "childPid": 1 } } +# overrides what this supervisor reports for that release; the proxy and its health are not affected. +function Get-StatusOverride { + try { + $c = [System.IO.File]::ReadAllText((Join-Path $RuntimeRoot 'status-control.json')) | ConvertFrom-Json + return $c.$ExpectedCommit + } catch { return $null } +} function Watch-Child($proc, [string]$mode) { $created = $proc.StartTime.ToString('o') while (!$proc.HasExited) { - Write-Status @{ state = $mode; childPid = $proc.Id; childCreated = $created; healthMatch = (Test-Health $proc.Id); inhibit = (Test-Path -LiteralPath $inhibitPath) } + $s = @{ state = $mode; childPid = $proc.Id; childCreated = $created; healthMatch = (Test-Health $proc.Id); inhibit = (Test-Path -LiteralPath $inhibitPath) } + $o = Get-StatusOverride + if ($null -ne $o) { foreach ($p in $o.PSObject.Properties) { $s[$p.Name] = $p.Value } } + Write-Status $s Start-Sleep -Milliseconds ([int]($HeartbeatSeconds * 1000)) } } diff --git a/tests/fixtures/live-deploy/TestTaskDriver.ps1 b/tests/fixtures/live-deploy/TestTaskDriver.ps1 index 42b2723bcf..dda79188bc 100644 --- a/tests/fixtures/live-deploy/TestTaskDriver.ps1 +++ b/tests/fixtures/live-deploy/TestTaskDriver.ps1 @@ -2,6 +2,7 @@ # The deploy scripts dot-source it when -TestTaskDriver is given and the runtime root is a marked sandbox. # State lives in \sandbox-task.json: # { taskName, enabled, port, calls: { info, disable, enable, start }, +# execute, launcherSuffix, optional: what the task action claims to run # failOn: [ { op, call } ], the n-th call of that operation throws # onCall: [ { op, call, action } ] } the n-th call first performs a fault action # Actions: openClient, makeLauncherReadOnly, writeForeignInhibit, editLauncher, killSelf. @@ -44,8 +45,8 @@ function Get-LiveProxyTaskInfo([string]$TaskName) { $s = Get-LpTestTaskState if ($TaskName -ne [string]$s.taskName) { return [pscustomobject]@{ Count = 0; Enabled = $false; Execute = ''; ActionArguments = ''; TriggerCount = 0 } } return [pscustomobject]@{ - Count = 1; Enabled = [bool]$s.enabled; Execute = 'powershell.exe' - ActionArguments = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$(Get-LpTestLauncher)`""; TriggerCount = 2 + Count = 1; Enabled = [bool]$s.enabled; Execute = $(if ($s['execute']) { [string]$s['execute'] } else { 'powershell.exe' }) + ActionArguments = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$(Get-LpTestLauncher)$([string]$s['launcherSuffix'])`""; TriggerCount = 2 } } function Disable-LiveProxyTask([string]$TaskName) { diff --git a/tests/live-deploy-scripts.test.ts b/tests/live-deploy-scripts.test.ts index f89d814ce0..5fb3d9073c 100644 --- a/tests/live-deploy-scripts.test.ts +++ b/tests/live-deploy-scripts.test.ts @@ -103,9 +103,9 @@ function candidateDir(root: string): string { return `${root}-cand`; } -async function makeSandbox(suffix = "", cand = CAND): Promise { +async function makeSandbox(suffix = "", cand = CAND, stem = crypto.randomBytes(5).toString("hex")): Promise { const port = await freePort(); - const root = path.join(tempBase(), `bili-live-deploy-${crypto.randomBytes(5).toString("hex")}${suffix}`); + const root = path.join(tempBase(), `bili-live-deploy-${stem}${suffix}`); assertSandbox(root, port); const canary = `CANARY-${crypto.randomBytes(6).toString("hex")}`; fs.mkdirSync(path.join(root, "config", "billion-context"), { recursive: true }); @@ -262,8 +262,8 @@ async function teardown(sb: Sandbox): Promise { } } -async function withSandbox(fn: (sb: Sandbox) => Promise, suffix = "", cand = CAND): Promise { - const sb = await makeSandbox(suffix, cand); +async function withSandbox(fn: (sb: Sandbox) => Promise, suffix = "", cand = CAND, stem?: string): Promise { + const sb = await makeSandbox(suffix, cand, stem); try { await fn(sb); } finally { @@ -348,7 +348,12 @@ const launcher = (sb: Sandbox) => fs.readFileSync(path.join(sb.root, LAUNCHER), const releaseDir = (sb: Sandbox, name = "cand-1") => path.join(sb.root, "releases", name); const supervisorPid = (sb: Sandbox) => Number(status(sb)?.supervisorPid ?? 0); -function assertUntouched(sb: Sandbox, launcherBefore: string): void { +function assertUntouched(sb: Sandbox, launcherBefore: string, ids: { sup?: number; proxy?: number } = {}): void { + const calls = getTask(sb).calls; + assert.deepEqual({ disable: calls.disable, enable: calls.enable, start: calls.start }, { disable: 0, enable: 0, start: 0 }, "the task must not be disabled, enabled or started"); + if (ids.sup) assert.equal(alive(ids.sup) && supervisorPid(sb) === ids.sup, true, "the same supervisor must still be running"); + if (ids.proxy) assert.equal(alive(ids.proxy), true, "the same proxy process must still be running"); + assert.equal(fs.existsSync(path.join(sb.root, "live-deploy.lock")), false, "no operation lock may be left behind"); assert.equal(inhibit(sb), null, "no inhibit may be left behind"); assert.equal(fs.existsSync(releaseDir(sb)), false, "no release directory may be created"); assert.equal(backups(sb).length, 0, "no backup folder may be created"); @@ -443,12 +448,14 @@ describe("live-deploy scripts: refusals leave everything untouched", { skip: SKI test("input validation and preflight refuse, each for its own reason", async () => { await withSandbox(async (sb) => { const before = launcher(sb); + // Identities of the running supervisor and proxy: no refusal may replace either. + let ids: { sup?: number; proxy?: number } = { sup: supervisorPid(sb), proxy: (await health(sb.port))!.pid }; const refuse = async (label: string, args: Record, reasons: RegExp[]) => { const r = await deploy(sb, { PreflightOnly: true, ...args }); assert.equal(r.code, 2, `${label}: expected exit 2\n${r.out}`); assert.match(r.out, / RESULT refused \(exit 2\)/, label); for (const re of reasons) assert.match(r.out, re, `${label}\n${r.out}`); - assertUntouched(sb, before); + assertUntouched(sb, before, ids); }; // Positive control first: the same sandbox passes when nothing is wrong. @@ -517,6 +524,12 @@ describe("live-deploy scripts: refusals leave everything untouched", { skip: SKI patchTask(sb, { enabled: true }); assertUntouched(sb, before); await refuse("unknown task", { TaskName: "No such task" }, [/expected exactly one task with this name, found 0/]); + // A task whose action only resembles this launcher, or is not run by Windows PowerShell. + patchTask(sb, { launcherSuffix: ".disabled" }); + await refuse("task runs another file", {}, [/FAIL scheduled task runs this launcher: the task action does not run the launcher/]); + patchTask(sb, { launcherSuffix: "", execute: "pwsh.exe" }); + await refuse("task runs another program", {}, [/FAIL scheduled task runs this launcher: the task action is not Windows PowerShell/]); + patchTask(sb, { execute: "" }); // Preflight: launcher shape. fs.appendFileSync(path.join(sb.root, LAUNCHER), " [string]$ExpectedCommit = 'again',\n"); @@ -566,6 +579,7 @@ describe("live-deploy scripts: refusals leave everything untouched", { skip: SKI const sup = supervisorPid(sb); process.kill(sup); await waitFor("supervisor to exit", () => !alive(sup)); + ids = { proxy: ids.proxy }; await refuse("no supervisor", {}, [/expected one supervisor, found 0/]); }); }); @@ -577,9 +591,11 @@ describe("live-deploy scripts: refusals leave everything untouched", { skip: SKI describe("live-deploy scripts: deployment and rollback", { skip: SKIP, concurrency: 4 }, () => { test("deploys, proves supervision, rewrites receipts; the standalone rollback restores everything", async () => { // A second sandbox whose name is a prefix-sibling of the first proves the scripts never touch another installation. - const other = await makeSandbox("a10"); + const stem = crypto.randomBytes(5).toString("hex"); + const other = await makeSandbox("0", CAND, stem); try { await withSandbox(async (sb) => { + assert.equal(other.root.startsWith(sb.root) && other.root !== sb.root, true, "the other root must extend this root's name"); const otherPids = { sup: supervisorPid(other), proxy: (await health(other.port))!.pid }; const before = launcher(sb); const tunedBefore = fs.readFileSync(path.join(sb.root, "tuned-deployment.json")); @@ -685,7 +701,7 @@ describe("live-deploy scripts: deployment and rollback", { skip: SKIP, concurren assert.equal((await health(other.port))!.pid, otherPids.proxy); assert.equal(inhibit(other), null); assert.equal(getTask(other).calls.disable, 0); - }, "a1"); + }, "", CAND, stem); } finally { await teardown(other); } @@ -869,6 +885,78 @@ describe("live-deploy scripts: deployment and rollback", { skip: SKIP, concurren }); }); + test("a launcher edited by someone else while the rollback runs is never overwritten", async () => { + await withSandbox(async (sb) => { + const sup = supervisorPid(sb); + fs.writeFileSync(path.join(sb.root, "proxy-control.json"), JSON.stringify({ "cand-1": { unhealthy: true } })); + // The second "disable" is the rollback's own hold: the edit lands after the rollback has started. + patchTask(sb, { onCall: [{ op: "disable", call: 2, action: "editLauncher" }] }); + const r = await deploy(sb, { HealthWaitSeconds: 6 }); + expectResult(r, 4, "rollback-failed", "foreign-launcher-edit"); + assertHold(sb, onlyBackup(sb), sup); + assert.match(launcher(sb), /# foreign edit/, "the foreign edit must survive"); + assert.match(launcher(sb), /cand-1/, "the launcher must be left as found, not restored"); + }); + }); + + test("a healthy candidate that the supervisor does not vouch for is rolled back", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + // The candidate itself answers health correctly in both runs; only the supervisor's report is off. + for (const [override, why] of [ + [{ healthMatch: false }, /not healthy under supervision: supervisor reports no health match/], + [{ childPid: 1 }, /not healthy under supervision: supervisor monitors a process that is not the listener/], + ] as const) { + fs.writeFileSync(path.join(sb.root, "status-control.json"), JSON.stringify({ [CAND.commit]: override })); + const r = await deploy(sb, { HealthWaitSeconds: 20 }); + expectResult(r, 3, "rolled-back", "cutover-failed"); + assert.match(r.out, why); + await waitSupervised(sb, PREV.version, PREV.commit); + assert.equal(launcher(sb), before); + assert.equal(fs.existsSync(releaseDir(sb)), false); + } + }); + }); + + test("a second deploy is refused while one is running", async () => { + await withSandbox(async (sb) => { + const first = deploy(sb); + await waitFor("the first run to write its backup", () => backups(sb).length === 1); + const second = await deploy(sb, { ReleaseName: "cand-2" }); + assert.equal(second.code, 2, second.out); + assert.match(second.out, /another deploy or rollback is running for this runtime root/); + assert.equal(fs.existsSync(releaseDir(sb, "cand-2")), false); + expectResult(await first, 0, "deployed"); + assert.equal(backups(sb).length, 1); + assert.equal(fs.existsSync(path.join(sb.root, "live-deploy.lock")), false, "the operation lock goes away with the process"); + }); + }); + + test("a standalone rollback that died mid-run can be run again", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + expectResult(await deploy(sb), 0, "deployed"); + const backup = onlyBackup(sb); + // Killed in the middle of its lift: candidate stopped, inhibit removed, task still disabled. + patchTask(sb, { onCall: [{ op: "enable", call: 2, action: "killSelf" }] }); + const killed = await rollback(sb, backup); + assert.notEqual(killed.code, 0); + assert.equal(record(backup).stage, "rolling-back"); + assert.equal(inhibit(sb), null); + assert.equal(getTask(sb).enabled, false); + patchTask(sb, { onCall: [] }); + // A live receipt that went missing is restored from the backup, not a reason to stop. + fs.rmSync(path.join(sb.root, "tuned-deployment.json")); + const back = await rollback(sb, backup); + expectResult(back, 0, "restored"); + await waitSupervised(sb, PREV.version, PREV.commit); + assert.equal(launcher(sb), before); + assert.equal(getTask(sb).enabled, true); + assert.deepEqual(fs.readFileSync(path.join(sb.root, "tuned-deployment.json")), fs.readFileSync(path.join(backup, "tuned-deployment.json"))); + assert.equal(record(backup).stage, "rolled-back"); + }); + }); + test("a receipt that cannot be replaced does not roll back a healthy release", async () => { await withSandbox(async (sb) => { fs.chmodSync(path.join(sb.root, "tuned-deployment.json"), 0o444); From 3fb4a72e6f97bafa98c5c905a4c13a7edb318f3e Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Wed, 18 Nov 2026 00:40:40 -0500 Subject: [PATCH 4/5] fix(tooling): second review round for the live-deploy scripts (#80) - Supervisor, proxy and task action are recognised from parsed arguments: the host's own -File operand and node's entry point, not a path that appears later as data for another script. - Process identity is compared at millisecond precision before the same handle is used to stop it. - A half-written inhibit is removed also when closing the file is what failed. - The deploy re-checks that the inhibit is still its own before it changes anything; a foreign inhibit is kept and supervision is not restarted. Tests: task changed after preflight, inhibit replaced during the hold, a launcher passed as data to another script, and the lock scenario now pauses the first run while it owns the lock (deploy and rollback are both refused). Assisted-by: Cindy (https://cindy.app) --- scripts/live-deploy/Deploy-LiveProxy.ps1 | 2 + scripts/live-deploy/LiveProxy.Common.ps1 | 61 +++++++++++++----- tests/fixtures/live-deploy/TestTaskDriver.ps1 | 8 ++- tests/live-deploy-scripts.test.ts | 64 +++++++++++++++++-- 4 files changed, 113 insertions(+), 22 deletions(-) diff --git a/scripts/live-deploy/Deploy-LiveProxy.ps1 b/scripts/live-deploy/Deploy-LiveProxy.ps1 index 7453303438..e990574472 100644 --- a/scripts/live-deploy/Deploy-LiveProxy.ps1 +++ b/scripts/live-deploy/Deploy-LiveProxy.ps1 @@ -304,6 +304,8 @@ try { if (-not (Test-LpSameHash (Get-LpSha256 $ctx.LauncherPath) $ctx.Record.launcherSha256Before)) { Stop-LpRollbackFailed $ctx 'launcher edit: the launcher was changed by someone else after preflight; it was left as found' 'foreign-launcher-edit' } + # Nothing has been changed yet: if the inhibit is no longer this run's, someone else's maintenance has begun. + if (-not (Test-LpOwnInhibit $ctx)) { throw 'maintenance.inhibit is no longer the one this run created' } Set-LpStage $ctx 'held' Write-LpBytesAtomic $ctx.LauncherPath $editedBytes Write-LpEvent 'STEP' "launcher updated: $($edit.Changed -join ', ')" diff --git a/scripts/live-deploy/LiveProxy.Common.ps1 b/scripts/live-deploy/LiveProxy.Common.ps1 index 006c791eaf..a48600429d 100644 --- a/scripts/live-deploy/LiveProxy.Common.ps1 +++ b/scripts/live-deploy/LiveProxy.Common.ps1 @@ -225,18 +225,40 @@ function Get-LpClientCount($Ctx) { return @($rows | Where-Object { @('127.0.0.1', '::1') -contains [string]$_.RemoteAddress -and $own -notcontains [int]$_.OwningProcess }).Count } -# Processes of one image whose command line contains the given text. The text names how the process was -# started, not just a path: a process that merely receives the path as an argument does not match. -function Get-LpProcesses([string]$ImageName, [string]$Needle) { - return @(Get-CimInstance Win32_Process -Filter "Name='$ImageName'" -OperationTimeoutSec 15 -ErrorAction Stop | - Where-Object { $null -ne $_.CommandLine -and $_.CommandLine.IndexOf($Needle, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 }) +# Splits a command line into its arguments: blanks separate, double quotes group. Enough for the two +# command lines this installation produces; anything more exotic simply does not match below. +function Split-LpCommandLine([string]$Line) { + $out = [System.Collections.Generic.List[string]]::new() + $cur = [System.Text.StringBuilder]::new(); $inQuote = $false; $has = $false + foreach ($ch in $Line.ToCharArray()) { + if ($ch -eq '"') { $inQuote = -not $inQuote; $has = $true; continue } + if (-not $inQuote -and [char]::IsWhiteSpace($ch)) { if ($has) { $out.Add($cur.ToString()); $null = $cur.Clear(); $has = $false }; continue } + $null = $cur.Append($ch); $has = $true + } + if ($has) { $out.Add($cur.ToString()) } + return , $out.ToArray() +} +# True when these PowerShell host arguments run the launcher itself: the first -File is the host's own +# (nothing before it hands control to a command), and its operand is the launcher. A launcher path that +# only appears later, as data for another script, does not count. +function Test-LpRunsLauncher([string[]]$HostArgs, [string]$LauncherPath) { + for ($i = 0; $i -lt $HostArgs.Count; $i++) { + $a = $HostArgs[$i] + if ($a -match '^-(c|command|e|ec|encodedcommand)$') { return $false } + if ($a -ieq '-File') { return ($i + 1 -lt $HostArgs.Count -and $HostArgs[$i + 1] -ieq $LauncherPath) } + } + return $false +} +function Get-LpProcessesOf([string]$ImageName) { + return @(Get-CimInstance Win32_Process -Filter "Name='$ImageName'" -OperationTimeoutSec 15 -ErrorAction Stop | Where-Object { $null -ne $_.CommandLine }) } # The supervisor is Windows PowerShell running the launcher as its -File script. -function Get-LpSupervisors($Ctx) { return @(Get-LpProcesses 'powershell.exe' ('-File "' + $Ctx.LauncherPath + '"')) } -# A proxy is node running the bundle as its entry point: "" "" start ... +function Get-LpSupervisors($Ctx) { + return @(Get-LpProcessesOf 'powershell.exe' | Where-Object { $t = Split-LpCommandLine $_.CommandLine; $t.Count -gt 1 -and (Test-LpRunsLauncher @($t | Select-Object -Skip 1) $Ctx.LauncherPath) }) +} +# A proxy is node running the bundle as its entry point: start ... function Get-LpProxies([string]$ProxyPath) { - $needle = '" "' + $ProxyPath + '" start ' - return @(Get-LpProcesses 'node.exe' $needle | Where-Object { $_.CommandLine.IndexOf($needle, [System.StringComparison]::OrdinalIgnoreCase) -eq $_.CommandLine.IndexOf('" "', [System.StringComparison]::Ordinal) }) + return @(Get-LpProcessesOf 'node.exe' | Where-Object { $t = Split-LpCommandLine $_.CommandLine; $t.Count -ge 3 -and $t[1] -ieq $ProxyPath -and $t[2] -ceq 'start' }) } # Stops the process that was matched. One handle is opened, its start time is compared with the matched @@ -252,7 +274,8 @@ function Stop-LpProcessVerified($Process) { if ($p.HasExited) { return } throw } - if ([Math]::Abs(($started - $Process.CreationDate.ToUniversalTime()).TotalSeconds) -ge 1) { throw "pid $($Process.ProcessId) now belongs to a different process; not stopping it" } + # Both values come from the same kernel timestamp; they differ only by rounding below a millisecond. + if ([Math]::Abs(($started - $Process.CreationDate.ToUniversalTime()).TotalMilliseconds) -ge 1) { throw "pid $($Process.ProcessId) now belongs to a different process; not stopping it" } try { $p.Kill() } catch { if (-not $p.HasExited) { throw } } } finally { $p.Dispose() @@ -312,7 +335,7 @@ function Get-LiveProxyTaskInfo([string]$TaskName) { # with the launcher as its -File script; a path that merely starts with the launcher path does not count. function Get-LpTaskActionGap($Task, [string]$LauncherPath) { if ([System.IO.Path]::GetFileName([string]$Task.Execute) -ine 'powershell.exe') { return 'the task action is not Windows PowerShell (powershell.exe)' } - if (([string]$Task.ActionArguments).IndexOf('-File "' + $LauncherPath + '"', [System.StringComparison]::OrdinalIgnoreCase) -lt 0) { return 'the task action does not run the launcher in this runtime root' } + if (-not (Test-LpRunsLauncher (Split-LpCommandLine ([string]$Task.ActionArguments)) $LauncherPath)) { return 'the task action does not run the launcher in this runtime root' } return $null } function Test-LpSameTask($A, $B) { @@ -368,12 +391,16 @@ function New-LpInhibit($Ctx, [string]$Verb) { $s = [System.IO.File]::Open($Ctx.InhibitPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) $written = $false try { - $s.Write($bytes, 0, $bytes.Length) - $s.Flush($true) - $written = $true + try { + $s.Write($bytes, 0, $bytes.Length) + $s.Flush($true) + $written = $true + } finally { + $s.Dispose() + } } finally { - $s.Dispose() - # The file exists only because this call created it: a marker without its owner text must not stay. + # The file exists only because this call created it: a marker without its owner text must not stay, + # also when closing the file is what failed. if (-not $written) { Remove-Item -LiteralPath $Ctx.InhibitPath -Force -ErrorAction SilentlyContinue } } } @@ -568,6 +595,8 @@ function Invoke-LpRollback($Ctx, [byte[]]$RestoreLauncherBytes, [hashtable]$Rece $step = 'lift after rollback' if ($SkipHoldAndRestore) { # The hold was never completed: remove this deployment's inhibit and re-enable supervision as it was. + # An inhibit that is no longer this deployment's belongs to someone else's maintenance: nothing is started. + if ((Test-Path -LiteralPath $Ctx.InhibitPath) -and -not (Test-LpOwnInhibit $Ctx)) { throw 'maintenance.inhibit now belongs to other maintenance; it was left as found and supervision was not restarted' } if (Test-LpOwnInhibit $Ctx) { Remove-Item -LiteralPath $Ctx.InhibitPath -Force } $Ctx.LiftAt = (Get-Date).ToUniversalTime() Enable-LiveProxyTask $Ctx.TaskName diff --git a/tests/fixtures/live-deploy/TestTaskDriver.ps1 b/tests/fixtures/live-deploy/TestTaskDriver.ps1 index dda79188bc..c8535ec651 100644 --- a/tests/fixtures/live-deploy/TestTaskDriver.ps1 +++ b/tests/fixtures/live-deploy/TestTaskDriver.ps1 @@ -5,7 +5,8 @@ # execute, launcherSuffix, optional: what the task action claims to run # failOn: [ { op, call } ], the n-th call of that operation throws # onCall: [ { op, call, action } ] } the n-th call first performs a fault action -# Actions: openClient, makeLauncherReadOnly, writeForeignInhibit, editLauncher, killSelf. +# Actions: openClient, makeLauncherReadOnly, writeForeignInhibit, editLauncher, killSelf, changeTaskAction, +# pause (writes sandbox-paused, then waits for sandbox-release so a test can act while this run is in progress). function Get-LpTestTaskStatePath { return (Join-Path $script:LpRoot 'sandbox-task.json') } function Get-LpTestTaskState { return ([System.IO.File]::ReadAllText((Get-LpTestTaskStatePath)) | ConvertFrom-Json -AsHashtable) } @@ -33,6 +34,11 @@ function Invoke-LpTestFault([string]$Op) { 'writeForeignInhibit' { [System.IO.File]::WriteAllText((Join-Path $script:LpRoot 'maintenance.inhibit'), 'someone else') } 'editLauncher' { Add-Content -LiteralPath (Get-LpTestLauncher) -Value '# foreign edit' } 'killSelf' { Stop-Process -Id $PID -Force } + 'changeTaskAction' { $s['launcherSuffix'] = '.moved'; Save-LpTestTaskState $s } + 'pause' { + [System.IO.File]::WriteAllText((Join-Path $script:LpRoot 'sandbox-paused'), 'paused') + $null = Wait-LpUntil { Test-Path -LiteralPath (Join-Path $script:LpRoot 'sandbox-release') } 120 'paused by the test' + } } } foreach ($f in @($s.failOn)) { diff --git a/tests/live-deploy-scripts.test.ts b/tests/live-deploy-scripts.test.ts index 5fb3d9073c..b40c3779bc 100644 --- a/tests/live-deploy-scripts.test.ts +++ b/tests/live-deploy-scripts.test.ts @@ -527,6 +527,8 @@ describe("live-deploy scripts: refusals leave everything untouched", { skip: SKI // A task whose action only resembles this launcher, or is not run by Windows PowerShell. patchTask(sb, { launcherSuffix: ".disabled" }); await refuse("task runs another file", {}, [/FAIL scheduled task runs this launcher: the task action does not run the launcher/]); + patchTask(sb, { launcherSuffix: `.other.ps1" -File "${path.join(sb.root, LAUNCHER)}` }); + await refuse("task passes the launcher as data to another script", {}, [/FAIL scheduled task runs this launcher: the task action does not run the launcher/]); patchTask(sb, { launcherSuffix: "", execute: "pwsh.exe" }); await refuse("task runs another program", {}, [/FAIL scheduled task runs this launcher: the task action is not Windows PowerShell/]); patchTask(sb, { execute: "" }); @@ -918,14 +920,66 @@ describe("live-deploy scripts: deployment and rollback", { skip: SKIP, concurren }); }); + test("a task changed after preflight stops the run before anything is touched", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + const ids = { sup: supervisorPid(sb), proxy: (await health(sb.port))!.pid }; + // The second task read is the comparison with the task that preflight validated. + patchTask(sb, { onCall: [{ op: "info", call: 2, action: "changeTaskAction" }] }); + const r = await deploy(sb); + expectResult(r, 6, "error-before-hold"); + assert.match(r.out, /the scheduled task was changed after preflight/); + assert.equal(record(onlyBackup(sb)).stage, "aborted"); + const calls = getTask(sb).calls; + assert.deepEqual([calls.disable, calls.enable, calls.start], [0, 0, 0]); + assert.equal(inhibit(sb), null); + assert.equal(launcher(sb), before); + assert.equal(fs.existsSync(releaseDir(sb)), false); + assert.equal(alive(ids.sup) && supervisorPid(sb) === ids.sup, true); + assert.equal((await health(sb.port))!.pid, ids.proxy); + }); + }); + + test("an inhibit replaced by someone else during the hold is kept and nothing is restarted", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + const proxy = (await health(sb.port))!.pid; + const sup = supervisorPid(sb); + // This run's own inhibit exists by the first "disable"; the fault overwrites it with a foreign one. + patchTask(sb, { onCall: [{ op: "disable", call: 1, action: "writeForeignInhibit" }] }); + const r = await deploy(sb); + expectResult(r, 4, "rollback-failed", "rollback-error"); + assert.match(r.out, /maintenance\.inhibit is no longer the one this run created/); + assert.match(r.out, /hold NOT fully in place: inhibit present but owned by other maintenance/); + assert.equal(inhibit(sb), "someone else", "the foreign inhibit must be kept exactly"); + const task = getTask(sb); + assert.equal(task.enabled, false); + assert.deepEqual([task.calls.enable, task.calls.start], [0, 0], "supervision must not be restarted under someone else's maintenance"); + assert.equal(alive(sup), false); + assert.equal(launcher(sb), before, "the launcher must not have been edited"); + const h = await health(sb.port); + assert.equal(h!.pid, proxy, "the running proxy must not have been stopped"); + assert.equal(h!.commit, PREV.commit); + }); + }); + test("a second deploy is refused while one is running", async () => { await withSandbox(async (sb) => { + // The first run is paused inside its hold, so it provably still owns the lock when the second starts. + patchTask(sb, { onCall: [{ op: "disable", call: 1, action: "pause" }] }); const first = deploy(sb); - await waitFor("the first run to write its backup", () => backups(sb).length === 1); - const second = await deploy(sb, { ReleaseName: "cand-2" }); - assert.equal(second.code, 2, second.out); - assert.match(second.out, /another deploy or rollback is running for this runtime root/); - assert.equal(fs.existsSync(releaseDir(sb, "cand-2")), false); + try { + await waitFor("the first run to pause", () => fs.existsSync(path.join(sb.root, "sandbox-paused"))); + const second = await deploy(sb, { ReleaseName: "cand-2" }); + assert.equal(second.code, 2, second.out); + assert.match(second.out, /another deploy or rollback is running for this runtime root/); + assert.equal(fs.existsSync(releaseDir(sb, "cand-2")), false); + const back = await rollback(sb, onlyBackup(sb)); + assert.equal(back.code, 2, back.out); + assert.match(back.out, /another deploy or rollback is running for this runtime root/); + } finally { + fs.writeFileSync(path.join(sb.root, "sandbox-release"), "go"); + } expectResult(await first, 0, "deployed"); assert.equal(backups(sb).length, 1); assert.equal(fs.existsSync(path.join(sb.root, "live-deploy.lock")), false, "the operation lock goes away with the process"); From 6ac461eeaae17309af7cb6d16b69f7351d2c784f Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Wed, 18 Nov 2026 00:56:59 -0500 Subject: [PATCH 5/5] fix(tooling): third review round for the live-deploy scripts (#80) - The supervisor and task-action check accepts only the host options this installation uses, spelled in full, before -File; abbreviated or unknown options do not match. - Tests: host-argument cases (including -f and the launcher passed as data), a stale process snapshot is refused and the process stays alive, and an inhibit replaced during the rollback is kept with no further lift. Assisted-by: Cindy (https://cindy.app) --- scripts/live-deploy/LiveProxy.Common.ps1 | 13 +++-- tests/live-deploy-scripts.test.ts | 64 ++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/scripts/live-deploy/LiveProxy.Common.ps1 b/scripts/live-deploy/LiveProxy.Common.ps1 index a48600429d..eeaf733826 100644 --- a/scripts/live-deploy/LiveProxy.Common.ps1 +++ b/scripts/live-deploy/LiveProxy.Common.ps1 @@ -238,14 +238,19 @@ function Split-LpCommandLine([string]$Line) { if ($has) { $out.Add($cur.ToString()) } return , $out.ToArray() } -# True when these PowerShell host arguments run the launcher itself: the first -File is the host's own -# (nothing before it hands control to a command), and its operand is the launcher. A launcher path that -# only appears later, as data for another script, does not count. +# True when these PowerShell host arguments run the launcher itself: only the options this installation +# uses may come before -File, spelled in full, and the operand of -File is the launcher. Anything else, +# including abbreviated options, does not match: a launcher path that only appears later, as data for +# another script, never counts. function Test-LpRunsLauncher([string[]]$HostArgs, [string]$LauncherPath) { + $switches = @('-NoProfile', '-NonInteractive', '-NoLogo') + $valued = @('-ExecutionPolicy', '-WindowStyle') for ($i = 0; $i -lt $HostArgs.Count; $i++) { $a = $HostArgs[$i] - if ($a -match '^-(c|command|e|ec|encodedcommand)$') { return $false } if ($a -ieq '-File') { return ($i + 1 -lt $HostArgs.Count -and $HostArgs[$i + 1] -ieq $LauncherPath) } + if ($switches -icontains $a) { continue } + if ($valued -icontains $a) { $i++; continue } + return $false } return $false } diff --git a/tests/live-deploy-scripts.test.ts b/tests/live-deploy-scripts.test.ts index b40c3779bc..f2d2d8eb4e 100644 --- a/tests/live-deploy-scripts.test.ts +++ b/tests/live-deploy-scripts.test.ts @@ -442,6 +442,50 @@ describe("live-deploy scripts: static rules", () => { }); }); +// Pure functions and one disposable process: no sandbox, nothing near a real installation. +describe("live-deploy scripts: process identity", { skip: SKIP }, () => { + const runCommon = (body: string) => { + const r = spawnSync("pwsh", ["-NoProfile", "-NonInteractive", "-Command", `. '${COMMON}'; ${body}`], { encoding: "utf8", timeout: 120000 }); + assert.equal(r.status, 0, `${r.stdout}\n${r.stderr}`); + return r.stdout.trim().split(/\r?\n/); + }; + + test("only the host's own -File operand makes a process the supervisor", () => { + const L = "X:\\rt\\Start-BillionContextProxy.ps1"; + const cases: [string, boolean][] = [ + [`-NoProfile -NonInteractive -WindowStyle Hidden -ExecutionPolicy Bypass -File "${L}"`, true], + [`-noprofile -FILE "${L.toUpperCase()}"`, true], + [`-File "${L}" -Port 1`, true], + [`-NoProfile -File "X:\\tools\\Watch.ps1" -File "${L}"`, false], + [`-NoProfile -f "X:\\tools\\Watch.ps1" -File "${L}"`, false], + [`-NoProfile -Command "x" -File "${L}"`, false], + [`-NoProfile -File "${L}.disabled"`, false], + [`-NoProfile -ExecutionPolicy -File "${L}"`, false], + ["-NoProfile -File", false], + ]; + const lines = cases.map(([args]) => `Test-LpRunsLauncher (Split-LpCommandLine '${args}') '${L}'`).join("; "); + assert.deepEqual(runCommon(lines), cases.map(([, want]) => (want ? "True" : "False"))); + }); + + test("a process is stopped only when it is the one that was matched", () => { + const out = runCommon( + [ + "$p = Start-Process -FilePath (Join-Path $env:WINDIR 'System32\\WindowsPowerShell\\v1.0\\powershell.exe') -ArgumentList '-NoProfile -NonInteractive -Command Start-Sleep -Seconds 120' -WindowStyle Hidden -PassThru", + "try {", + ' $snap = Get-CimInstance Win32_Process -Filter "ProcessId=$($p.Id)"', + // Same PID, creation time half a second earlier: what a reused PID would look like. + " $stale = [pscustomobject]@{ ProcessId = $snap.ProcessId; CreationDate = $snap.CreationDate.AddMilliseconds(-500) }", + " try { Stop-LpProcessVerified $stale; 'no error' } catch { $_.Exception.Message -replace '\\d+', 'N' }", + " Start-Sleep -Milliseconds 300; \"alive=$(-not $p.HasExited)\"", + " Stop-LpProcessVerified $snap", + " \"stopped=$($p.WaitForExit(10000))\"", + "} finally { if (-not $p.HasExited) { $p.Kill() } }", + ].join("\n"), + ); + assert.deepEqual(out, ["pid N now belongs to a different process; not stopping it", "alive=True", "stopped=True"]); + }); +}); + // ---- refusals: nothing may change --------------------------------------------------------------------- describe("live-deploy scripts: refusals leave everything untouched", { skip: SKIP }, () => { @@ -963,6 +1007,26 @@ describe("live-deploy scripts: deployment and rollback", { skip: SKIP, concurren }); }); + test("an inhibit replaced by someone else during the rollback is kept and nothing is restarted", async () => { + await withSandbox(async (sb) => { + const before = launcher(sb); + const sup = supervisorPid(sb); + fs.writeFileSync(path.join(sb.root, "proxy-control.json"), JSON.stringify({ "cand-1": { unhealthy: true } })); + // The second "disable" is the rollback's hold, taken right after the rollback created its own inhibit. + patchTask(sb, { onCall: [{ op: "disable", call: 2, action: "writeForeignInhibit" }] }); + const r = await deploy(sb, { HealthWaitSeconds: 6 }); + expectResult(r, 4, "rollback-failed", "rollback-error"); + assert.match(r.out, /lift after rollback \(maintenance\.inhibit is no longer the one this run created; it was left as found\)/); + assert.equal(inhibit(sb), "someone else", "the foreign inhibit must be kept exactly"); + const task = getTask(sb); + assert.equal(task.enabled, false); + // One enable and one start: the deployment's own lift. The rollback must not have lifted anything. + assert.deepEqual([task.calls.enable, task.calls.start], [1, 1]); + assert.equal(alive(sup), false); + assert.equal(launcher(sb), before, "the launcher was restored before the lift was refused"); + }); + }); + test("a second deploy is refused while one is running", async () => { await withSandbox(async (sb) => { // The first run is paused inside its hold, so it provably still owns the lock when the second starts.