diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86d644b..b9a2be7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,9 @@ jobs: # One platform failing says nothing about the others, so all of them report. fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + # macOS because people deploy from Macs, and because its temporary directory + # is reached through a symlink, which once failed five tests nobody else saw. + os: [ubuntu-latest, windows-latest, macos-latest] # 22 is the engines floor; 24 is the current LTS line. node: ['22', '24'] defaults: diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 0b7831e..caa63ac 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -11,6 +11,10 @@ # the suites it targets in minutes rather than an hour. The summary job scores # either. # +# The scheduled run is the whole suite against the newest canary, every night, and its +# score is published to the `evidence` branch, which the site's evidence page reads. A +# run started by hand is never published: it may be a subset, or an older ref. +# # Contract: nextjs.org/docs/app/api-reference/adapters/testing-adapters # Author: Gowtham name: conformance @@ -27,8 +31,9 @@ on: default: '' type: string schedule: - # Weekly, so a Next.js regression is found before a release rather than after. - - cron: '0 3 * * 1' + # Nightly, so a Next.js change that breaks self-hosting is found and shown the + # day it lands on canary, rather than when it reaches a release. + - cron: '0 3 * * *' jobs: build: @@ -37,6 +42,7 @@ jobs: timeout-minutes: 45 outputs: nextjs-sha: ${{ steps.ref.outputs.sha }} + nextjs-version: ${{ steps.ref.outputs.version }} groups: ${{ steps.groups.outputs.list }} steps: # The test matrix is computed here rather than from the input inside the @@ -63,7 +69,9 @@ jobs: fetch-depth: 25 - id: ref - run: echo "sha=$(git -C nextjs rev-parse HEAD)" >> "$GITHUB_OUTPUT" + run: | + echo "sha=$(git -C nextjs rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "version=$(node -p "require('./nextjs/packages/next/package.json').version")" >> "$GITHUB_OUTPUT" # Building Next.js from source is most of this job's time, and it depends on # nothing but the Next.js commit. Keyed on that commit, every run against the @@ -217,3 +225,51 @@ jobs: name: summary path: summary.json retention-days: 90 + + publish: + name: Publish the nightly result + needs: [build, summary] + if: ${{ github.event_name == 'schedule' && needs.summary.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + # The one job that writes to the repository, and only to the evidence branch. + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: conformance/publish-evidence.mjs + sparse-checkout-cone-mode: false + path: adapter + + - uses: actions/setup-node@v4 + with: { node-version: '22' } + + - uses: actions/download-artifact@v4 + with: { name: summary } + + # The branch holds nothing but the published files, so it is created empty the + # first time rather than branched from main. + - name: Check out the evidence branch + run: | + if git ls-remote --exit-code --heads "https://github.com/${GITHUB_REPOSITORY}.git" evidence > /dev/null; then + git clone --depth 1 --branch evidence "https://x-access-token:${{ github.token }}@github.com/${GITHUB_REPOSITORY}.git" evidence + else + git init -q -b evidence evidence + git -C evidence remote add origin "https://x-access-token:${{ github.token }}@github.com/${GITHUB_REPOSITORY}.git" + fi + + - name: Write the result + env: + NEXTJS_VERSION: ${{ needs.build.outputs.nextjs-version }} + NEXTJS_SHA: ${{ needs.build.outputs.nextjs-sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: node adapter/conformance/publish-evidence.mjs summary.json evidence + + - name: Push it + working-directory: evidence + run: | + git add latest.json history.json + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -q -m "Nightly suite on Next.js ${{ needs.build.outputs.nextjs-version }}" + git push -q origin evidence diff --git a/CHANGELOG.md b/CHANGELOG.md index dd24d7a..995482f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,98 @@ All notable changes to this repository. Attribution rules: `AGENTS.md` §1.1. ## [Unreleased] +### Added + +- **A server keeps what each app writes at runtime under a limit.** `server add` installs + a cache guard, run every ten minutes, that keeps each app's optimized images, fetch + cache and runtime-rendered pages under 10% of the disk, never under 1 GiB, deleting + the least recently used first, by the later of a file's last access and last write so + that a disk keeping no access times is evicted oldest first, and logging it under + `nextship-cache-guard`. Only what + Next.js wrote at runtime is ever deleted, never a file the build produced. A site + that moved off Vercel reported its ISR cache growing about 145 GB a day under + crawlers until the disk filled and the site went down; nothing in Next.js bounds it. + Setup version 3, so `server status` asks servers set up earlier to run `server add` + again. (Ragul D) +- **A tab opened before a deployment keeps working after it, and a rollback within the + hour is instant.** On a server, the deployment a new build replaces keeps running for an + hour, and Caddy sends it every request that names its build, as Next.js does on each + client navigation, Server Action and script an open tab loads; a page load gets the new + build. Without it such a tab asks the new build for files and Server Actions it lacks, + the failure Next.js issue #99165 still reports. `rollback` to that deployment switches + Caddy back to its running container, 208 ms in the end-to-end test, keeping its own env, + and a timer stops the kept container when its hour is over. `env push` and domain changes + keep nothing extra. A server set up by 1.1.x has no such timer, so a deployment to one + keeps nothing and says to run `server add` again. The timer never stops the container + the Caddy site names as live, `previous` is emptied before every switch and written with + the history in one command, and an instant rollback reads what it decides on only once + it holds the app's lock: three ways the live app could have been stopped or Caddy + pointed at a stopped container, found in review before release. (Ragul D) +- **Folders an app writes to.** `"writable": ["data"]` in `nextship.json` gives an app on a + server a volume per folder that it can write to and that outlives a deployment, which is + what a SQLite file or uploads beside the code need now that the rest of the container is + read-only. `doctor` warns when the source looks like it writes files and none is + declared, `destroy` says it deletes that data, and `server move` says it does not copy + it. (Ragul D) +- **`doctor` hands over what `vercel.json` did.** A cron job becomes a crontab line that + calls its route with `CRON_SECRET` as a bearer token, as Vercel does. Redirects, rewrites + and headers that use only fields `next.config` accepts are printed as a `next.config` + snippet; any other, or a redirect that does not say whether it is permanent, is named + rather than guessed. The part of a migration that needs Vercel's API, domains and env var + names, is not built. (Ragul D) +- **A certificate for the server's own address, built and not yet claimed.** On a server + added by a public IPv4 address, the first app's site also asks Let's Encrypt for its + six-day IP address certificate, and `deploy` reports the `https://` address only once it + verifies. Plain HTTP on the address keeps serving whether or not a certificate comes: + with only `https://` named, Caddy redirected HTTP there before any certificate existed, + found in review and measured with a CA that could not be reached. Verified against + Pebble, Let's Encrypt's test server, with issuance failing and then succeeding. The + README and the site do not describe it until it has run against Let's Encrypt on a real + server; `docs/roadmap.md` holds it until then. (Ragul D) +- **Previews.** `nextship deploy --preview ` deploys the current source as a second + app on the same server, with its own env file and domains, never taking the server's + address and asking crawlers not to index it. The same flag scopes `rollback`, `logs`, + `env`, `domain`, `images` and `destroy`. A preview is found again from the app it + previews, recorded on the server, so a CI run holding only the committed `nextship.json` + can update or remove it; `docs/vm.md` §6 has a workflow for one per pull request, not yet + run in GitHub Actions. Destroying an app destroys its previews with it and names each in + the plan, since nothing could find them afterwards. (Ragul D) +- **The compatibility suite runs every night against the newest Next.js canary, and the + evidence page shows the result.** The scheduled run publishes its score to the + `evidence` branch, and the page reads it when it opens, saying so plainly when there is + nothing to read rather than showing a number. A run started by hand is never + published. (Ragul D) + +### Changed + +- **The default memory limit on a server counts two containers per app.** For the hour + after a deployment the replaced container runs beside the live one, and each was allowed + the app's whole share: 1600m twice for one app on a 2 GB server. The share is now 80% of + RAM across two containers per app, 800m in that case. `--memory` still overrides it, and + the plan shows the figure. (Ragul D) +- **A deployment to a server switches about three seconds sooner.** A container that was + ready at once waited 4.1 s to be seen as healthy, a quarter of a 16.6 s warm deployment, + because Docker's first check and the deploy's polling were both two seconds apart. Both + are half a second now; three warm deployments took 13.3 to 15.0 s. (Ragul D) +- **An app on a server runs in a hardened container.** Its root filesystem is read-only + apart from the two `.next` volumes and a 64 MB `/tmp` that is emptied on restart and + cannot run a binary; every Linux capability is dropped, privileges cannot be gained, + and it is limited to 512 processes. The December 2025 break-ins through CVE-2025-55182 + on self-hosted Next.js rewrote `next.config.js` and lockfiles and ran miners from + `/tmp`; none of that can persist now. Measured with the streaming fixture: streaming, + on-demand ISR, the Edge route and image optimization all work, and nothing is written + outside the volumes, and `conformance/vm/e2e.sh` checks it on every run. An app that + writes files outside `.next` now fails with `EROFS`, which the README and `docs/vm.md` + §4 state. (Ragul D) + +### Fixed + +- **The unit tests pass on macOS.** Five prune tests created their fixtures under the + temporary directory as named, which on macOS is a link to `/private/var`, so every path + prune resolved looked outside the build root. CI ran only Linux and Windows and never saw + it; it runs macOS too now. The image was never affected: prune runs under `/src` there. + (Ragul D) + ## [1.1.3] - 2026-10-10 Three fixes to deploying to your own server, all found or proven on a live Droplet. A diff --git a/README.md b/README.md index dc8f556..2ed9722 100644 --- a/README.md +++ b/README.md @@ -287,8 +287,8 @@ is reported as a blocker. | Checked | Why it matters | |---|---| | `@vercel/analytics` and similar packages | They stop reporting and nothing errors | -| Cron jobs in `vercel.json` | They will simply never run | -| Routing rules in `vercel.json` | They stop applying | +| Cron jobs in `vercel.json` | They will simply never run. `doctor` prints a crontab line for each that calls its route as Vercel does, with `CRON_SECRET` as a bearer token | +| Routing rules in `vercel.json` | They stop applying. `doctor` prints the redirects, rewrites and headers that carry over unchanged as a `next.config` snippet, and names each one that needs a decision, such as a redirect that does not say whether it is permanent | | `VERCEL_URL` and `VERCEL_ENV` read in source | They become undefined | | Packages in `node_modules` that nothing declares | They exist on your machine and not in the image, so the build fails on an import that resolves locally | | A missing lockfile | Installs are no longer reproducible | @@ -354,7 +354,7 @@ deployed, and a page that did not compile failed once, with no retry. | Option | Default | Meaning | |---|---|---| | `--build ` | `remote` | `remote` builds on the server's own Docker, reached over SSH through a private socket, or a named pipe on Windows, so nothing but the build context leaves your machine, no port is opened, and the image is built natively for the server. `local` builds here for the server's architecture and streams the image with `docker save \| ssh docker load`. Recorded in `nextship.json`. A server under 2 GB of RAM is refused for remote builds | -| `--memory ` | an even share of 80% of RAM across the server's apps | The container's memory limit, such as `512m` | +| `--memory ` | an even share of 80% of RAM across the containers the server's apps can run at once: two per app, the live one and the one kept for an hour after a deployment | The container's memory limit, such as `512m`. One app on a 2 GB server gets 800m by default | ``` > Plan @@ -371,7 +371,7 @@ deployed, and a page that did not compile failed once, with no retry. ``` A deployment starts a new container with no published port, waits for Docker to report it -healthy, then points Caddy at it and stops the previous one. Nothing a visitor sees +healthy, then points Caddy at it. Nothing a visitor sees changes until the new container is healthy, so a deployment that fails leaves the previous one serving, and a deployment that succeeds drops no request. Its last log lines are printed when it fails. The first app deployed on a server answers `http://`; @@ -382,8 +382,44 @@ so every machine that deploys, including CI, builds with the same key. A key alr your local file is copied to the server on the first deploy. If the server and your local file hold different keys, `deploy` refuses rather than choosing one. -`rollback` on a server starts the earlier deployment's image again the same way. It uses -the current env file, not the one that deployment first ran with, and the plan says so. +**A tab opened before a deployment keeps working after it.** The deployment a new build +replaces keeps running for an hour, and Caddy sends it every request that names its build, +which Next.js does on each client navigation, Server Action and script an open tab loads. +Without this, such a tab asks the new build for files and Server Actions it does not have. +A page load names no build and gets the new one. After the hour, the old container stops +and a tab still open from before loads the new build on its next navigation. The cost is a +second container's memory for that hour, which the default memory limit already counts. A +change that starts the same build again, such as `env push`, keeps nothing extra. A server +set up by nextship 1.1.x has no timer to stop the kept container, so a deployment to one +keeps nothing and says so: run `nextship server add` again with `--yes` to update it. + +`rollback` on a server to the deployment replaced within the last hour is instant: that +container is still running, so Caddy is pointed back at it and nothing is built or +started, and it keeps the env it ran with. A rollback further back starts that +deployment's image again with the current env file, and the plan says which is which. + +**Folders your app writes to.** Everything in an app's container is read-only apart from +`.next` and `/tmp`. An app that keeps a SQLite file or uploads beside its code lists the +folders in `nextship.json`: + +```json +{ "writable": ["data", "public/uploads"] } +``` + +Each is a volume the app can write to, and what it writes there outlives a deployment, which +it never did inside a container. A folder is relative to the app's directory, and +`.next`, `node_modules` and anything starting with a dot are refused. `nextship doctor` +warns when the source looks like it writes files and no folder is declared. `destroy` +deletes that data with the app, and `server move` does not copy it: both say so before +they act. + +**Previews.** `nextship deploy --preview pr-42` deploys the current source as a second app on +the same server, `-pr-42`, with its own env file and domains, and never writes +`nextship.json`. The same flag scopes `rollback`, `logs`, `env`, `domain`, `images` and +`destroy` to it, and the preview is found again from the app it previews, so a CI run with +only the committed `nextship.json` can update or remove it. A preview answers only on the +domains attached to it and asks crawlers not to index it. A workflow that deploys one per +pull request is in [`docs/vm.md`](./docs/vm.md) §6. ### `nextship rollback` @@ -635,6 +671,10 @@ Removes the app this project created, and nothing else. | `--yes` | off | Execute the plan. Without it, `destroy` only prints the plan | | `--images` | off | Also remove this project's images, then start garbage collection | +On a server, an app's previews are destroyed with it, images included, and the plan names +each one. A preview is found through the app it previews, so one left behind could never be +reached again. + **The app name is required**, and this is the only command that asks for one. Every other command acts on whatever directory you are in, which is fine when nothing can be destroyed. Here it is not: `--yes` typed in the wrong project would remove the wrong @@ -698,7 +738,9 @@ server presenting any other key is refused. With `--yes` it installs Docker CE if the server has none, creates a `nextship` user and proves a login as that user from your machine, runs Caddy on ports 80 and 443, limits the journal, enables security updates and ufw, installs a watchdog for containers that stay -unhealthy, and turns off SSH password logins last. Every later command connects as +unhealthy, a cache guard that keeps what each app writes at runtime under a limit and a +timer that stops a replaced deployment after its hour, and turns off SSH password logins +last. Every later command connects as `nextship`. Running it again against a server that is set up changes nothing; running it from a second project records the same server for that project. Each step is described in [`docs/vm.md`](./docs/vm.md). @@ -908,6 +950,10 @@ On a server, `server add` creates: caddy/sites/ one Caddy site per app /usr/local/lib/nextship/watchdog.sh restarts containers unhealthy for 3 checks in a row /etc/systemd/system/nextship-watchdog.* the timer that runs it every minute +/usr/local/lib/nextship/cache-guard.sh keeps each app's runtime cache under 10% of the disk +/etc/systemd/system/nextship-cache-guard.* the timer that runs it every ten minutes +/usr/local/lib/nextship/retire.sh stops a replaced deployment once its hour for open tabs is over +/etc/systemd/system/nextship-retire.* the timer that runs it every minute /etc/systemd/journald.conf.d/nextship.conf SystemMaxUse=500M, only if no limit was set /etc/ssh/sshd_config.d/10-nextship.conf unless --no-ssh-hardening /etc/apt/apt.conf.d/20auto-upgrades unless --no-auto-updates @@ -997,10 +1043,22 @@ deploy: is no CDN in front unless you add one, the operating system is yours to keep patched and rebooted, and backups are yours. The `nextship` user is root-equivalent. See [`docs/vm.md`](./docs/vm.md). -- **On a server, rollback uses the current env file.** Rolling back code does not roll - back a variable changed since, which App Platform does. +- **On a server, a rollback further back than the last hour uses the current env file.** + Rolling back code does not roll back a variable changed since, which App Platform does. + A rollback to the deployment replaced within the hour keeps its own env. - **On a server, only the first app deployed answers `http://`.** Every other app answers only on the domains attached to it. +- **On a server, your app cannot write its own files.** The container is read-only apart + from `.next`, a 64 MB `/tmp` that cannot run a binary, and the folders you list as + `writable` in `nextship.json`, so an attacker who gets code running through a flaw in + your app can neither rewrite it nor leave a program behind. An app that saves uploads to + a folder it has not listed deploys, passes its health check, and fails with `EROFS` on + its first write; `nextship doctor` warns about source that looks like it does. A package + that unpacks an executable into `/tmp`, such as a headless Chromium, cannot run it. +- **On a server, writable folders live on that one server.** They are not backed up, and + `server move` does not copy them. For the hour after a deployment the previous + deployment's container has the same folders mounted, so a SQLite file there can have + two processes writing to it. - **A server's env file cannot hold a value with a line break.** Docker reads it one line per variable, so `env push` refuses such a value rather than truncating it. Encode it, for example as base64. @@ -1083,7 +1141,7 @@ conformance/ scripts for the official Next.js adapter compatibility su packages/ adapter/ Next.js Adapter API implementation, injected via NEXT_ADAPTER_PATH cli/ every command; runtime/ holds the files copied into a build, - and runtime/vm/ the setup and watchdog scripts sent to a server + and runtime/vm/ the setup and timer scripts sent to a server site/ the marketing site and documentation, exported as static files ``` diff --git a/conformance/publish-evidence.mjs b/conformance/publish-evidence.mjs new file mode 100644 index 0000000..00e33d9 --- /dev/null +++ b/conformance/publish-evidence.mjs @@ -0,0 +1,86 @@ +#!/usr/bin/env node +/** + * Turns a scored compatibility suite run into the files the site reads. + * + * The nightly run against the newest Next.js canary is only evidence if someone can + * see it without opening GitHub Actions, so its score is written to the `evidence` + * branch, which the site's evidence page reads in the browser. `latest.json` is the + * last run; `history.json` keeps the last ninety, so a regression shows as a drop on + * a date rather than as a number that quietly changed. A run that scored no suite at + * all is refused rather than published, because "0 of 0" would read as a result. + * + * Usage: node publish-evidence.mjs + * Reads NEXTJS_VERSION, NEXTJS_SHA and RUN_URL from the environment, and writes + * latest.json and history.json into the directory, keeping the history already there. + * + * Author: Ragul D + */ + +import { readFile, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +export const HISTORY_LENGTH = 90 + +/** The published record of one run. Only the names of failing suites, so the file stays small. */ +export function nightlyRecord(summary, meta) { + if (!summary?.suites?.total) { + throw new Error('The run scored no suites, so there is nothing to publish. Check the test jobs of this run for why none reported.') + } + for (const [name, value] of Object.entries(meta)) { + if (!value) throw new Error(`${name} is not set. The workflow passes it from the build job; check that step's output.`) + } + return { + date: meta.date, + nextjs: { version: meta.version, sha: meta.sha }, + run: meta.runUrl, + suites: { passed: summary.suites.passed, total: summary.suites.total }, + assertions: { passed: summary.assertions.passed, total: summary.assertions.passed + summary.assertions.failed }, + failing: summary.failing.map((suite) => suite.name), + } +} + +/** The history with this run appended, one entry per date, the newest last, at most HISTORY_LENGTH long. */ +export function appendHistory(history, record) { + const entry = { + date: record.date, + version: record.nextjs.version, + suites: record.suites, + assertions: record.assertions, + } + // A run started again on the same day replaces that day's entry rather than + // counting twice. + const day = (date) => date.slice(0, 10) + return [...history.filter((previous) => day(previous.date) !== day(entry.date)), entry].slice(-HISTORY_LENGTH) +} + +async function main() { + const [summaryPath, directory] = process.argv.slice(2) + if (!summaryPath || !directory) { + console.error('usage: node publish-evidence.mjs ') + process.exit(2) + } + const summary = JSON.parse(await readFile(summaryPath, 'utf8')) + const record = nightlyRecord(summary, { + date: new Date().toISOString(), + version: process.env.NEXTJS_VERSION, + sha: process.env.NEXTJS_SHA, + runUrl: process.env.RUN_URL, + }) + + const historyPath = path.join(directory, 'history.json') + let history = [] + try { + history = JSON.parse(await readFile(historyPath, 'utf8')) + } catch (error) { + // Only a first publish has no history. A file that exists but cannot be read is + // stopped on, since writing over it would erase every earlier night. + if (error.code !== 'ENOENT') throw error + } + + await writeFile(path.join(directory, 'latest.json'), JSON.stringify(record, null, 2) + '\n') + await writeFile(historyPath, JSON.stringify(appendHistory(history, record), null, 2) + '\n') + console.log(`Published ${record.suites.passed} of ${record.suites.total} suites on Next.js ${record.nextjs.version}`) +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) await main() diff --git a/conformance/publish-evidence.test.mjs b/conformance/publish-evidence.test.mjs new file mode 100644 index 0000000..49c95f0 --- /dev/null +++ b/conformance/publish-evidence.test.mjs @@ -0,0 +1,58 @@ +/** + * Tests for what the nightly run publishes to the site. + * + * Each case is a way the published evidence could say something the run did not + * show: a run that scored nothing reading as a result, a missing version, a rerun + * counted twice, or a history that grows without end. + * + * Run with: node --test conformance/*.test.mjs + * Author: Ragul D + */ + +import { test } from 'node:test' +import assert from 'node:assert/strict' + +import { HISTORY_LENGTH, appendHistory, nightlyRecord } from './publish-evidence.mjs' + +const summary = { + suites: { total: 1123, passed: 1121, failed: 2 }, + assertions: { passed: 3590, failed: 9 }, + failing: [ + { name: 'test/e2e/a.test.ts', passed: 1, failed: 5, failures: ['x'] }, + { name: 'test/e2e/b.test.ts', passed: 0, failed: 4, failures: ['y'] }, + ], +} +const meta = { date: '2026-09-29T03:40:00.000Z', version: '16.5.0-canary.3', sha: 'abc123', runUrl: 'https://github.com/gowtham472/nextship/actions/runs/1' } + +test('a run is published with its totals, its Next.js version and the names of what failed', () => { + assert.deepEqual(nightlyRecord(summary, meta), { + date: meta.date, + nextjs: { version: '16.5.0-canary.3', sha: 'abc123' }, + run: meta.runUrl, + suites: { passed: 1121, total: 1123 }, + assertions: { passed: 3590, total: 3599 }, + failing: ['test/e2e/a.test.ts', 'test/e2e/b.test.ts'], + }) +}) + +test('a run that scored no suites is refused, so "0 of 0" is never shown as a result', () => { + assert.throws(() => nightlyRecord({ ...summary, suites: { total: 0, passed: 0, failed: 0 } }, meta), /scored no suites/) +}) + +test('a run without its Next.js version is refused rather than published unlabelled', () => { + assert.throws(() => nightlyRecord(summary, { ...meta, version: undefined }), /version is not set/) +}) + +test('a second run on the same day replaces that day, and history keeps only the newest', () => { + const record = nightlyRecord(summary, meta) + const rerun = { ...record, date: '2026-09-29T09:00:00.000Z', suites: { passed: 1123, total: 1123 } } + const history = appendHistory(appendHistory([], record), rerun) + assert.equal(history.length, 1) + assert.equal(history[0].suites.passed, 1123) + + const long = Array.from({ length: HISTORY_LENGTH }, (_, i) => ({ ...history[0], date: `2026-01-${String(i + 1).padStart(2, '0')}T03:00:00Z` })) + const next = appendHistory(long, record) + assert.equal(next.length, HISTORY_LENGTH) + assert.equal(next.at(-1).date, meta.date) + assert.equal(next[0].date, long[1].date) +}) diff --git a/conformance/vm/e2e.sh b/conformance/vm/e2e.sh index d888589..d534d09 100755 --- a/conformance/vm/e2e.sh +++ b/conformance/vm/e2e.sh @@ -6,7 +6,12 @@ # previous one serving, rollback returns to the previous deployment, env push takes effect # without downtime, a replaced deployment's logs are still readable, pruning keeps the # live image, a regenerated ISR page and an optimized image survive a container restart, -# a domain gets its own site, and destroying one app leaves another serving. +# the app's container is read-only and cannot run what it writes, a tab from before a +# deployment reaches its own build, a rollback within the hour is instant, a preview +# serves beside the app and leaves it alone when destroyed, a domain gets its own site, +# a declared writable folder outlives a deployment, the retire timer never stops the live +# container, a server with the older setup keeps nothing, destroying one app leaves +# another serving, and destroying an app takes its previews with it. # # It ends by destroying both apps it created, so a server it ran against is left with only # what `server add` set up. @@ -106,11 +111,49 @@ check "idempotence" "a second server add changes nothing" # ---------------------------------------------------------------------- deploy +# A route that answers with a stamp written into its source, so a response says +# which build served it. Each build below rewrites the stamp. +stamp() { + mkdir -p app/api/build + printf 'export const dynamic = "force-dynamic"\nexport function GET() { return Response.json({ build: "%s" }) }\n' "$1" > app/api/build/route.js +} +# The deployment id a build names on every request an open tab makes, read from +# the ?dpl= Next.js puts on the page's scripts. +page_dpl() { curl -s "${URL}/" | grep -o 'dpl=dpl-[A-Za-z0-9_.-]*' | head -1 | cut -d= -f2; } +build_of() { curl -s "$@" | sed -n 's/.*"build":"\([a-z]*\)".*/\1/p'; } + +stamp first +# A folder the app may write to, declared before the first deployment, and a route +# that writes a note there and reads it back. +node -e "const fs = require('fs'); const c = JSON.parse(fs.readFileSync('nextship.json', 'utf8')); c.writable = ['data']; fs.writeFileSync('nextship.json', JSON.stringify(c, null, 2) + '\n')" +mkdir -p app/api/data +cat > app/api/data/route.js <<'JS' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +export const dynamic = 'force-dynamic' +const file = process.cwd() + '/data/note.txt' +export async function POST(request) { + await mkdir(process.cwd() + '/data', { recursive: true }) + await writeFile(file, await request.text()) + return Response.json({ written: true }) +} +export async function GET() { + try { + return new Response(await readFile(file, 'utf8')) + } catch (error) { + return new Response(String(error.code), { status: 404 }) + } +} +JS commit "record the server" nextship deploy --yes curl -sf -o /dev/null "${URL}/" || fail "deploy" "${URL}/ does not answer after deploying" check "deploy" "the fixture serves at ${URL}" +# The container's root is read-only, so this write works only because `data` was +# declared writable. It is read back after the next deployment, further down. +[ "$(curl -s -X POST --data 'written by the first build' "${URL}/api/data")" = '{"written":true}' ] || + fail "writable" "the app could not write to the folder it declared: $(curl -s -X POST --data x "${URL}/api/data")" + node "${HERE}/../streaming/measure.mjs" "${URL}/stream" EDGE="$(curl -s "${URL}/edge")" [ "${EDGE}" = '{"runtime":"edge-runtime"}' ] || fail "edge" "/edge answered: ${EDGE:-nothing}" @@ -118,7 +161,11 @@ check "edge" "/edge ran in the Edge runtime, through Caddy" # ------------------------------------------------------------- zero downtime +FIRST_DPL="$(page_dpl)" +FIRST_SCRIPT="$(curl -s "${URL}/" | grep -o '/_next/static/[^"]*?dpl=dpl-[A-Za-z0-9_.-]*' | head -1)" +[ -n "${FIRST_DPL}" ] && [ -n "${FIRST_SCRIPT}" ] || fail "skew" "the first build's page named no deployment id on its scripts" echo "// second build $(date +%s)" >> app/layout.jsx +stamp second commit "second build" poll / "${WORK}/codes" & LOOP_PIDS+=($!) poll /stream "${WORK}/streams" & LOOP_PIDS+=($!) @@ -130,6 +177,26 @@ wait "${LOOP_PIDS[@]}" LOOP_PIDS=() all_ok "${WORK}/codes" "zero downtime" all_ok "${WORK}/streams" "streams across the switch" + +# ------------------------------------------------------------ skew protection + +# A tab that loaded the first build keeps naming it: x-deployment-id on navigations +# and Server Actions, ?dpl= on scripts. Those requests must reach the first build's +# container, kept running, while a request naming nothing gets the new build. +SECOND_DPL="$(page_dpl)" +[ -n "${SECOND_DPL}" ] && [ "${SECOND_DPL}" != "${FIRST_DPL}" ] || fail "skew" "the second build names the same deployment id as the first" +[ "$(build_of "${URL}/api/build")" = second ] || fail "skew" "a request naming no build did not get the new one" +[ "$(build_of -H "X-Deployment-Id: ${FIRST_DPL}" "${URL}/api/build")" = first ] || + fail "skew" "a request with the first build's x-deployment-id did not reach it" +[ "$(build_of "${URL}/api/build?dpl=${FIRST_DPL}")" = first ] || fail "skew" "a request with ?dpl= for the first build did not reach it" +[ "$(curl -s -o /dev/null -w '%{http_code}' "${URL}${FIRST_SCRIPT}")" = 200 ] || fail "skew" "the first build's script ${FIRST_SCRIPT} is gone" +check "skew" "requests naming the first build reach it after the second went live, and the rest get the second" + +# The second build is a new image and a new container, and the note is still there. +[ "$(build_of "${URL}/api/build")" = second ] || fail "writable" "the second build is not serving" +[ "$(curl -s "${URL}/api/data")" = 'written by the first build' ] || + fail "writable" "what the first build wrote did not survive the deployment: $(curl -s "${URL}/api/data")" +check "writable" "the app wrote to the folder it declared, and a later deployment read it back" SECOND="$(nextship rollback | sed -n 's/^ current *\([^ ]*\).*/\1/p')" # ------------------------------------------------------------ failed startup @@ -155,10 +222,62 @@ commit "remove the failing startup" TARGET="$(nextship rollback | sed -n 's/^ roll back *\([^ ]*\).*/\1/p')" [ -n "${TARGET}" ] || fail "rollback" "the plan named nothing to roll back to" -nextship rollback --yes +# The first build was kept running when the second replaced it, so going back to it +# is a switch of the proxy, not a new container. +ROLLED="$(nextship rollback --yes)" +echo "${ROLLED}" [ "$(nextship rollback | sed -n 's/^ current *\([^ ]*\).*/\1/p')" = "${TARGET}" ] || fail "rollback" "the live deployment is not ${TARGET}" curl -sf -o /dev/null "${URL}/" || fail "rollback" "${URL}/ does not answer after rolling back" check "rollback" "${TARGET} serves again" +SWITCH_MS="$(sed -n 's/.*which was still running, in \([0-9]*\) ms.*/\1/p' <<< "${ROLLED}")" +[ -n "${SWITCH_MS}" ] || fail "instant rollback" "the rollback started a new container instead of switching to the one still running" +[ "$(build_of "${URL}/api/build")" = first ] || fail "instant rollback" "the first build does not serve after rolling back to it" +[ "$(build_of -H "X-Deployment-Id: ${SECOND_DPL}" "${URL}/api/build")" = second ] || + fail "instant rollback" "the build rolled back from was not kept for the tabs that loaded it" +check "instant rollback" "switched back to the first build's running container in ${SWITCH_MS} ms, keeping the second for its tabs" + +# ------------------------------------------------------------------- retire + +# Not waited out: the kept container's stop time is moved to the past and the timer's +# service run, as the timer would an hour after the rollback. +SUDO="" +on_server "sudo -n true" 2> /dev/null && SUDO="sudo -n" +KEPT="$(on_server "${SUDO} cut -d' ' -f1 /etc/nextship/apps/nextship-streaming-fixture/previous")" +[ -n "${KEPT}" ] || fail "retire" "no kept container is recorded" +on_server "printf '%s 1\n' ${KEPT} | ${SUDO} tee /etc/nextship/apps/nextship-streaming-fixture/previous > /dev/null && ${SUDO} systemctl start nextship-retire.service" +[ "$(on_server "${SUDO} docker inspect -f '{{.State.Running}}' ${KEPT}")" = false ] || fail "retire" "${KEPT} still runs after its time ran out" +[ "$(curl -s -o /dev/null -w '%{http_code}' -H "X-Deployment-Id: ${SECOND_DPL}" "${URL}/api/build")" = 200 ] || + fail "retire" "a request naming the stopped build was not served by the live one" +[ "$(build_of -H "X-Deployment-Id: ${SECOND_DPL}" "${URL}/api/build")" = first ] || fail "retire" "a request naming the stopped build got something other than the live build" +check "retire" "the kept container stopped when its time ran out, and requests naming it fell back to the live build" + +# The worst `previous` can say: the live container, with its time already up. The +# timer must refuse, because the Caddy site names that container as the one serving. +LIVE_NOW="$(on_server "${SUDO} sed -n 's/^# live //p' /etc/nextship/caddy/sites/nextship-streaming-fixture.caddy")" +[ -n "${LIVE_NOW}" ] || fail "retire" "the Caddy site names no live container" +on_server "printf '%s 1\n' ${LIVE_NOW} | ${SUDO} tee /etc/nextship/apps/nextship-streaming-fixture/previous > /dev/null && ${SUDO} systemctl start nextship-retire.service" +[ "$(on_server "${SUDO} docker inspect -f '{{.State.Running}}' ${LIVE_NOW}")" = true ] || fail "retire" "the timer stopped the live container ${LIVE_NOW}" +[ "$(build_of "${URL}/api/build")" = first ] || fail "retire" "the app stopped serving after the timer ran" +on_server "${SUDO} journalctl -t nextship-retire --no-pager -n 5" | grep -q "not stopping ${LIVE_NOW}" || fail "retire" "the refusal was not logged" +check "retire" "told to stop the live container, the timer refused and said so" + +# ------------------------------------------- a server set up by an older nextship + +# A server set up before the retire timer existed has nothing to stop a kept +# container. Marked as one, a deployment must keep nothing and say what to run. +on_server "printf '{\"setupVersion\":2}\n' | ${SUDO} tee /etc/nextship/server.json > /dev/null" +stamp oldsetup +commit "a build for a server with the older setup" +OLD_SETUP="$(nextship deploy --yes)" +echo "${OLD_SETUP}" +on_server "printf '{\"setupVersion\":3}\n' | ${SUDO} tee /etc/nextship/server.json > /dev/null" +grep -q 'nextship server add' <<< "${OLD_SETUP}" || fail "older setup" "the deployment did not say to run server add again" +grep -q 'kept .* running until' <<< "${OLD_SETUP}" && fail "older setup" "a container was kept on a server with no timer to stop it" +[ "$(build_of "${URL}/api/build")" = oldsetup ] || fail "older setup" "the deployment is not serving" +RUNNING="$(on_server "${SUDO} docker ps --filter label=sh.nextship.app=nextship-streaming-fixture --filter status=running --format '{{.Names}}'" | wc -l | tr -d ' ')" +[ "${RUNNING}" = 1 ] || fail "older setup" "${RUNNING} containers of the app are running, where only the live one should be" +on_server "${SUDO} grep -q '@previous' /etc/nextship/caddy/sites/nextship-streaming-fixture.caddy" && fail "older setup" "the site routes to a container that was stopped" +check "older setup" "a server with no retire timer kept nothing, stopped the replaced container, and said to run server add" # ---------------------------------------------------------------------- env @@ -196,11 +315,14 @@ nextship images prune --keep 1 --yes # Output is captured before it is searched: grep -q exits at its first match, and the # closed pipe would fail the command under pipefail. IMAGES="$(nextship images)" -KEPT="$(grep -c '^ dpl-' <<< "${IMAGES}" || true)" -[ "${KEPT}" -eq 1 ] || fail "images prune" "${KEPT} images remain after keeping 1" +REMAINING="$(grep -c '^ dpl-' <<< "${IMAGES}" || true)" +# The build the env route deployment replaced is still kept running for its tabs, +# and Docker cannot remove an image a running container uses, so it stays too. +[ "${REMAINING}" -eq 2 ] || fail "images prune" "${REMAINING} images remain after keeping 1, where the live and the kept one were expected" grep -q 'deployed now' <<< "${IMAGES}" || fail "images prune" "the live image was removed" +grep -q 'kept running for tabs' <<< "${IMAGES}" || fail "images prune" "the kept deployment's image was removed" curl -sf -o /dev/null "${URL}/" || fail "images prune" "the app stopped serving" -check "images prune" "one image kept, the live one, and the app still serves" +check "images prune" "the live image and the one still running for older tabs kept, nothing else, and the app still serves" # --------------------------------------------------------------- durability @@ -257,8 +379,12 @@ check "durability" "an ISR page was regenerated and an image was optimized and c # A restart, not a redeploy: a redeploy builds a new image and so a new build # volume, which would prove nothing about what is kept. -CONTAINER="$(on_server "${DOCKER} ps --filter label=sh.nextship.app=nextship-streaming-fixture --filter status=running --format '{{.Names}}'" | head -1)" -[ -n "${CONTAINER}" ] || fail "durability" "no running container found for the app" +# The live one by its build: the deployment it replaced, a different build, is kept +# running beside it, and an older container of the live build is stopped. +LIVE_TAG="$(nextship rollback | sed -n 's/^ current *\([^ ]*\).*/\1/p')" +CONTAINER="$(on_server "${DOCKER} ps --filter label=sh.nextship.app=nextship-streaming-fixture --filter label=sh.nextship.image=${LIVE_TAG} --filter status=running --format '{{.Names}}'")" +[ "$(wc -l <<< "${CONTAINER}" | tr -d ' ')" = 1 ] && [ -n "${CONTAINER}" ] || + fail "durability" "expected one running container of the live build ${LIVE_TAG}, found: ${CONTAINER:-none}" # The restart and the wait are one connection: sixty round trips to poll a # health status is slower than the restart itself. @@ -278,6 +404,36 @@ AFTER_CACHED="$(cache_state "${IMAGE_URL}")" fail "durability" "the optimized image was re-encoded after the restart rather than read from the cache: ${AFTER_CACHED:-no header}" check "durability" "the regenerated page and the optimized image both survived a container restart" +# ---------------------------------------------------------------- hardening + +# The same container has just regenerated a page and optimized an image, so what +# follows proves the protections are enforced, not only that the app tolerates them. +# Each is tried from inside, as code running through a flaw in the app would try it. +HARDENING="$(on_server "${DOCKER} inspect -f '{{.HostConfig.ReadonlyRootfs}} {{.HostConfig.CapDrop}} {{.HostConfig.SecurityOpt}} {{.HostConfig.PidsLimit}}' ${CONTAINER}")" +[ "${HARDENING}" = "true [ALL] [no-new-privileges] 512" ] || + fail "hardening" "the container is not run read-only with no capabilities: ${HARDENING}" +on_server "${DOCKER} exec ${CONTAINER} sh -c 'touch next.config.js' 2> /dev/null" && + fail "hardening" "code inside the container could rewrite the app" +on_server "${DOCKER} exec ${CONTAINER} sh -c 'cp /bin/true /tmp/dropped && /tmp/dropped' 2> /dev/null" && + fail "hardening" "code inside the container could run a program it wrote to /tmp" +# The image has no `data` folder, so Docker makes the mount point for the declared +# writable folder in the container's own layer when it creates the container. That +# is the one change expected there, and its parent directory with it; anything +# else is a write that got past the read-only root. +WRITTEN="$(on_server "${DOCKER} diff ${CONTAINER}" | grep -v -x -e 'C /src' -e 'A /src/data' || true)" +[ -z "${WRITTEN}" ] || fail "hardening" "the container wrote outside its volumes: ${WRITTEN}" +check "hardening" "the app can rewrite none of its files, cannot run a program from /tmp, and wrote nothing outside its volumes" + +# ------------------------------------------------------------- cache guard + +# Run now rather than waited for, as the timer runs it. Under its limit it must +# delete nothing, so the regenerated page is still the one that survived the restart. +on_server "systemctl is-active --quiet nextship-cache-guard.timer" || fail "cache guard" "its timer is not active" +on_server "sudo -n systemctl start nextship-cache-guard.service 2> /dev/null || systemctl start nextship-cache-guard.service" || + fail "cache guard" "a run failed: $(on_server 'journalctl -u nextship-cache-guard.service -n 5 --no-pager' 2>&1)" +[ "$(curl -s "${URL}/isr")" = "${REGENERATED}" ] || fail "cache guard" "a run under the limit changed what the app serves" +check "cache guard" "its timer is active, and a run under the limit left the app's cache as it was" + # ------------------------------------------------------------------ domains nextship domain add e2e.nextship.test --yes @@ -285,6 +441,33 @@ nextship domain add e2e.nextship.test --yes fail "domain add" "the domain did not get its own HTTPS site" check "domain add" "the domain redirects to HTTPS through its own site" +# ------------------------------------------------------------------ preview + +# A preview of this app on the same server, from a build the app never ran. It must +# serve that build, leave the app's own address and build alone, keep out of +# nextship.json, ask crawlers not to index it, and go away without touching the app. +MAIN_BUILD="$(build_of "${URL}/api/build")" +RECORDED="$(cat nextship.json)" +stamp preview +commit "a preview build" +nextship deploy --preview e2e --yes +[ "$(cat nextship.json)" = "${RECORDED}" ] || fail "preview" "deploying a preview changed nextship.json" +[ "$(build_of "${URL}/api/build")" = "${MAIN_BUILD}" ] || fail "preview" "the app's own address stopped serving its build" +PREVIEW_CONTAINER="$(on_server "${DOCKER} ps --filter label=sh.nextship.app=nextship-streaming-fixture-e2e --filter status=running --format '{{.Names}}'")" +[ -n "${PREVIEW_CONTAINER}" ] || fail "preview" "no running container for the preview" +PREVIEW_BUILD="$(on_server "${DOCKER} exec ${PREVIEW_CONTAINER} node -e \"fetch('http://127.0.0.1:3000/api/build').then(r => r.json()).then(b => console.log(b.build))\"")" +[ "${PREVIEW_BUILD}" = preview ] || fail "preview" "the preview serves ${PREVIEW_BUILD:-nothing}, not its own build" +nextship domain add e2e-preview.nextship.test --preview e2e --yes +[ "$(curl -s -o /dev/null -w '%{http_code}' -H 'Host: e2e-preview.nextship.test' "${URL}/")" = 308 ] || + fail "preview" "the preview's domain did not get its own HTTPS site" +on_server "${SUDO} grep -q 'X-Robots-Tag \"noindex, nofollow\"' /etc/nextship/caddy/sites/nextship-streaming-fixture-e2e.caddy" || + fail "preview" "the preview's site does not ask crawlers not to index it" +if nextship logs --preview not-a-preview > /dev/null 2>&1; then fail "preview" "a preview that does not exist was not refused"; fi +nextship destroy nextship-streaming-fixture-e2e --preview e2e --images --yes +[ "$(cat nextship.json)" = "${RECORDED}" ] || fail "preview" "destroying the preview changed nextship.json" +[ "$(build_of "${URL}/api/build")" = "${MAIN_BUILD}" ] || fail "preview" "the app stopped serving when its preview was destroyed" +check "preview" "served its own build beside the app, kept out of nextship.json, asked crawlers not to index it, and was destroyed leaving the app serving" + # -------------------------------------------------- a second app, then destroy SECOND_APP="${WORK}/second" @@ -308,5 +491,12 @@ curl -sf -o /dev/null "${URL}/" || fail "destroy" "the first app stopped serving [ "$(curl -s -o /dev/null -w '%{http_code}' -H 'Host: e2e.nextship.test' "${URL}/")" = 308 ] || fail "destroy" "the first app lost its domain" check "destroy" "destroying one app left the other and Caddy serving" -nextship destroy nextship-streaming-fixture --images --yes -check "cleanup" "both apps are destroyed" +# A preview left when its app is destroyed could never be reached again: the app's id +# is what finds it, and destroying the app removes the id from nextship.json. +nextship deploy --preview orphan --yes +DESTROYED="$(nextship destroy nextship-streaming-fixture --images --yes)" +echo "${DESTROYED}" +grep -q 'preview DESTROY "nextship-streaming-fixture-orphan"' <<< "${DESTROYED}" || fail "destroy" "the plan did not name the preview it removes" +LEFT="$(on_server "${SUDO} docker ps -a --filter label=sh.nextship.managed=true --format '{{.Names}}' | grep -v '^nextship-caddy$' || true; ${SUDO} ls /etc/nextship/apps; ${SUDO} docker volume ls -q --filter label=sh.nextship.managed=true")" +[ -z "${LEFT}" ] || fail "destroy" "containers, app records or volumes are left on the server: ${LEFT}" +check "cleanup" "both apps are destroyed, the preview with its app, and nothing of them is left on the server" diff --git a/docs/design.md b/docs/design.md index de6ba0d..b00aea4 100644 --- a/docs/design.md +++ b/docs/design.md @@ -592,7 +592,7 @@ Everything nextship keeps on the server lives in a few places: | `/etc/nextship/apps//deployments.json` | Deployment history, newest first: `{ id, imageTag, createdAt, cause, served, live }` | | `/etc/nextship/apps//env`, `secrets` | Runtime variables and the Server Actions key, mode 0600, written from SSH stdin | | `/etc/nextship/apps//lock` | A directory created with `mkdir`, so two deployments cannot interleave, holding an owner file | -| `/etc/nextship/default-app` | The app that answers `http://`: the first one to deploy | +| `/etc/nextship/default-app` | The app that answers `http://`: the first one to deploy. On a server added by a public IPv4 address it also answers its site asks for a certificate for the address, not yet run against Let's Encrypt (Release sequence, below) | | Volumes `nextship--build-`, `nextship--cache` | What Next.js writes at runtime, kept across restarts (below) | | `/etc/nextship/caddy/sites/.caddy` | The Caddy site for the app | | Docker network `nextship` | Caddy and every app container; no app publishes a port | @@ -707,11 +707,111 @@ both sides are refused; with none anywhere, one is generated and stored only on record the deployment as not served, and leave the previous container serving. 5. Render the Caddy site pointing at the new container, validate it, move it into place and reload Caddy. A failed validation or reload restores the previous file. -6. Stop the previous container (kept, not removed, until images are pruned) and update - `deployments.json`. - -Rollback runs steps 3 to 6 with an earlier deployment's image. **It uses the current env -file**, which differs from App Platform, where a rollback restores the old spec. `env push`, +6. Update `deployments.json`, and stop the previous containers (stopped, not removed, until + images are pruned), except the one this deployment replaced when it is a different + build: that keeps running for an hour, and `retire.sh` stops it then. + +**Skew protection.** A tab loaded before a deployment keeps naming its build on every +request: `x-deployment-id` on client navigations and Server Actions and `?dpl=` on scripts +and styles, measured in a browser against Next.js 16.3.4 with `deploymentId` set. A page +load names none. So the Caddy site routes a request naming the replaced build to its +container, with the live container second under `lb_policy first` and `fail_duration 30s`, +and everything else to the live one. Once the replaced container stops, a request naming it +fails over to the live one, which is what every request got before. Next.js issue #99165 +reports stale tabs failing even with `deploymentId` set because a Server Action response +does not carry the deployment id; routing the request to the build that owns the action +means the response is that build's own. The replaced build is kept only when it is a +different image: `env push` and a domain change start the same image, and routing by a +deployment id both containers share would send every tab, new ones included, to the one +being replaced. `deployments.json` records the kept one with `keptUntil`, and beside it +`previous` holds ` ` for `retire.sh`, run every minute by a timer +`server add` installs, since the server has no JSON parser. `images prune` and the prune +after a deployment keep the kept deployment's image, which Docker would refuse to remove. + +Three things keep the kept container from doing harm, each found in review of the first +version. **A server with no retire timer keeps nothing.** The timer came with setup +version 3, so a deployment reads the server's version, and on an older one stops the +replaced container as 1.1.x did, writes no `@previous` route, and says to run `server add` +again; kept there, the container would have run with its memory limit until the next +deployment. **The timer cannot stop the live container.** `previous` is emptied before +Caddy is pointed anywhere and written again, with the history, in one remote command after +it, each to a temporary file moved into place; and `retire.sh` stops nothing the app's +Caddy site names on its `# live ` line, or when that line is missing. Before +this, a rollback to the kept container, or a CLI that died between the switch and the +second write, left `previous` naming the container Caddy served, and Docker's restart +policy does not bring back a container that was stopped. **The memory limit counts both +containers.** Where the server keeps a replaced deployment, each app counts twice in the +share of 80% of RAM, so one app on a 2 GB server is limited to 800m in each of its two +containers, not 1600m in each. + +**HTTPS on the server's address (Built, not yet run against Let's Encrypt).** When the +server was added by a literal public IPv4 address (`publicIpv4`: not private, shared, +loopback, link-local, documentation or multicast), the default app's site also has a block +for the address whose ACME issuer asks for Let's Encrypt's `shortlived` profile, the only one +under which Let's Encrypt issues IP address certificates since they became generally +available on 2026-01-15. The block names both `http://
` and `https://
`. +Named for HTTPS alone, Caddy's automatic HTTPS redirected plain HTTP for that host from the +moment it started, and that host-specific redirect won over the catch-all `http://:80` +site: measured with a CA that could not be reached, `http://
` answered 308 to an +address with no certificate, which is the one address a first deployment prints. Caddy +answers the HTTP-01 challenge before any site route. IPv4 only: IPv6 issuance was fixed in +Caddy after 2.10.2, the release `caddy:2.10` installs (caddyserver/caddy#7399). `address` +reports `https://` only once a TLS connection to the address verifies, and the certificate +check sends no server name for an address, which TLS forbids. + +Verified against Pebble, Let's Encrypt's test server, with a `shortlived` profile of six +days, on a private network with Caddy 2.10.2 running the site file `renderSite` produced for +`10.77.0.10`. With Pebble down, Caddy logged the failed issuance and `http://10.77.0.10` +answered 200 from the app with no redirect. With Pebble then started, Caddy's retry obtained +a certificate within 60 s through HTTP-01 on port 80, naming `IP Address:10.77.0.10` and +valid from 2026-10-10 18:30 to 2026-10-16 18:30; HTTP still answered 200, and a client +trusting only Pebble's root fetched the app over `https://10.77.0.10`. Pebble issues to +anything that answers, so this does not show Let's Encrypt's own policy or rate limits: the +README and the site say nothing of this until it has run on a server with a public address. + +**Writable folders.** A read-only root leaves an app that keeps a SQLite file or uploads +beside its code nowhere to write, and nothing at deploy time can see that it will fail. So +`nextship.json` may list `writable` folders, relative to the app's directory, refused when +they start with a dot (which covers `..` and `.next`), sit under `node_modules`, repeat, or +nest. Each is a named volume per app, `nextship--data-` with `/` as `__`, +labelled as the app's so `destroy` removes it, and mounted into every container of the +app, so what is written there outlives a deployment. Docker creates a volume's directory +owned by root and fills it from the image only when the image has that folder, so before +the app starts each volume is handed to the image's `app` user by `chown` in a container of +the same image, run as root with no network. For a folder the image does not have, Docker +makes the mount point in the container's own layer when it creates the container, so +`docker diff` lists that directory and its parent, and nothing else. + +**Previews.** `--preview ` (`preview.ts`) scopes a command to the app +`-` on the project's server. Its `app.json` records `previewOf`, the id of the +app it previews, and `ownedApp` accepts a preview only when that id is the one in +`nextship.json`, so a preview is found again by a CI run holding only the committed file and +is never adopted by name alone. `deploy --preview` creates the record and never writes +`nextship.json`; `destroy --preview` leaves it alone. Destroying an app destroys its +previews first, images included, and the plan names each: with the app's id gone from +`nextship.json`, nothing could match a preview's `previewOf` again, and it would keep its +containers, volumes and domains. A preview name too long for the app is refused with the +length it may be, before the app name check that would have said to rename the app. A preview's Caddy site never claims the +server's address and sends `X-Robots-Tag: noindex, nofollow` on every name. Its env file +starts empty. + +Verified by `conformance/vm/e2e.sh` on a local Ubuntu 24.04 stand-in, 25 checks of 25: +after a second build went live, a request naming the first build by `x-deployment-id` or by +`?dpl=` was served by it, and the first build's script still loaded, while a request naming +nothing got the second; rolling back to the first switched to its running container in 208 ms +and kept the second for its tabs; with the kept container's stop time moved to the past, the +retire service stopped it and a request naming it was served by the live build; a preview +deployed from a build the app never ran served that build beside the app, left `nextship.json` +and the app's address unchanged, sent `X-Robots-Tag` on its domain's site, and was destroyed +leaving the app serving. Every earlier check passed unchanged beside them. + +Rollback to the kept deployment is a Caddy switch to its container, which is still running, +and the deployment being left is kept in its place. The history and the container's health +are read only once the app's lock is held: read before it, a deployment that went live in +between had already stopped that container, and the rollback pointed Caddy at it. The container keeps the env it started +with. A rollback to anything older runs steps 3 to 6 with that deployment's image, and **it +uses the current env file**, which differs from App Platform, where a rollback restores the +old spec. `env push`, `env rm` and a domain change run the same sequence with the live image. After a deployment goes live, images beyond the newest five served deployments are removed with their stopped containers, which is safe on a server where removing an image frees its space at @@ -734,6 +834,15 @@ failed its deployment while 123 of 123 requests to the previous one returned 200 `rollback --yes` moved Caddy to a new container of the previous image. A warm remote build and deployment took 16.5 s; `--build local` from an M3 Max took 60 s. +A warm deployment of a one-line change, timed stage by stage on the stand-in (Docker Desktop +on an M3 Max): planning 0.6 s, `next build` in the builder 7.7 to 9.1 s, packaging the runtime +image 3.4 s, of which starting the second `docker build` is most and copying the app 0.2 s, +starting the container 0.4 s, waiting for it to be healthy, and switching Caddy 0.5 s. The wait +was 4.1 s of a 16.6 s deployment for a container ready at once, because Docker's first check +came two seconds after start and the deploy polled every two seconds; both are half a second +now, and three deployments took 15.0, 13.5 and 13.3 s with a 1.1 s wait. What remains is +mostly `next build`, which nextship does not own, so no deployment time is promised. + #### Runtime writes (Implemented) Measured with `docker diff` in a deployed container after exercising each feature: @@ -753,6 +862,32 @@ would be owned by root. After a restart with both volumes, a page regenerated by `revalidatePath` kept its content, the optimized image stayed cached, and `docker diff` reported no writes to the container layer. +#### Cache guard (Implemented) + +Nothing in Next.js bounds what it writes while an app serves, and one site that moved to its +own server reported its ISR cache growing about 145 GB a day under crawlers until the disk +filled. `runtime/vm/cache-guard.sh`, installed by `server add` and run every ten minutes by a +systemd timer, keeps each app's runtime writes under 10% of the disk Docker stores volumes +on, and never under 1 GiB. It counts everything in the app's cache volume, and in each build +volume only the files modified after that volume was created, so a file the build produced is +never deleted and a page with no revalidate time keeps what it was prerendered into. Over the +limit, it deletes by last use until 80% of the limit remains, then any directory left +empty, and logs the count under `nextship-cache-guard`. Last use is the later of a file's +access and write times. Access times alone would not do: ext4's default `relatime` updates +one at most daily, and under `noatime` never, where every file ties and the busiest would +go with the idle. With the write time taken too, a volume that keeps no access times is +evicted oldest written first. The guard reads only the cache and build volumes, by name, +and never a writable folder's volume. + +Verified on a stand-in whose Docker storage was a 12 GB ext4 disk, limit 1199 MiB, with the +streaming fixture deployed: the files the guard counted were exactly the optimized image and +the six files of the regenerated `/isr` page, none of the build's own. With 1.4 GB of filler +last read in 2020, a run removed the five oldest files, 500 MiB, down to 901 MiB, kept the +image and the page, and logged it. With the real entries made the oldest, a run removed +them with the filler; `/isr` and `/` answered 200, the image was encoded again and then +cached, `index.html` from the build was still there, and after a restart emptied Next.js's +memory `/isr` rendered anew and wrote its files back. + #### Day-two commands (Implemented) `env push` and `env rm` rewrite the env file from SSH stdin and start the live image again @@ -810,6 +945,22 @@ Server Actions key, and left the old server serving. - Only containers labelled `sh.nextship.managed=true` and `sh.nextship.app=` are ever stopped or removed. Other workloads and the global Docker daemon configuration are never modified. `destroy` never removes the server, Caddy, other apps or DNS. +- An app's container is hardened so that code an attacker runs through a flaw in the app + cannot stay. The root filesystem is read-only, so only the two volumes of + [Runtime writes](#runtime-writes-implemented) and a 64 MB `/tmp` are writable, and `/tmp` + is mounted `noexec`, emptied on restart. Every capability is dropped, `no-new-privileges` + stops a setuid binary from raising them, and a limit of 512 processes stops a fork bomb. + The trigger was the December 2025 break-ins through CVE-2025-55182 on self-hosted + Next.js, which edited `next.config.js` and lockfiles and ran miners dropped in `/tmp`. + Measured in a nextship image of the streaming fixture run with these flags and both + volumes: healthy in 4 s; `/stream` first byte 83 ms against a 2.04 s total; on-demand ISR + regenerated; the Edge route and the image optimizer answered; `docker diff` listed no + write outside the volumes; writing to the app directory failed with `Read-only file + system`; a binary copied to `/tmp` was refused with `Permission denied`; `CapEff` was + zero and `NoNewPrivs` 1; and the app ran 12 processes and threads under concurrent + streaming and image optimization. `conformance/vm/e2e.sh` now checks the same from inside + a deployed container, after it has regenerated a page and optimized an image, and passed + in full on a local Ubuntu 24.04 stand-in with every other check unchanged. #### What is verified, and what is not @@ -944,7 +1095,10 @@ Not verified yet, and the live checklist in `docs/roadmap.md`: | **One server.** | No failover: if the server is down, the app is down. Backups and recovery are the owner's | | **No CDN.** | Static assets and media are served by the container through Caddy | | **The owner owns the OS.** | `server add` enables security updates, but kernel reboots, disk and provider incidents are the owner's to watch | -| **Rollback uses the current env file.** | Rolling back code does not roll back a variable changed since | +| **A rollback older than an hour uses the current env file.** | Rolling back code does not roll back a variable changed since. A rollback to the deployment replaced within the hour keeps its own | +| **The replaced deployment runs for an hour.** | Its container holds memory beside the live one for that hour, within the same memory limit per container. A tab open longer than that loads the new build on its next navigation | +| **The app's files are read-only.** | An app that writes outside `.next` and the folders it lists as `writable` fails with `EROFS` on that write, after deploying and passing its health check; `doctor` warns about source that looks like it does. `/tmp` holds 64 MB and cannot run a binary, which breaks packages that unpack an executable there, such as a headless Chromium | +| **Writable folders live on one server.** | They are not backed up, `server move` does not copy them, and `destroy` deletes them, each said before it acts. For the hour a replaced deployment is kept, two containers have the same folder mounted | ## 10. Deploy lifecycle diff --git a/docs/roadmap.md b/docs/roadmap.md index 46028e1..2759ff6 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -267,6 +267,28 @@ Known from the Droplet runs, to fix or document here: `conformance/vm/e2e.sh` ne with no other app on it, and after adding swap `server add` prints `done swap already on`, the post-apply check's wording rather than what it did. +## The server target's next phase (Planned, not tied to a version) + +Chosen from what people self-hosting Next.js report as unsolved: open Next.js issues, +competitors' issue trackers and security advisories, and published break-ins, read in +September 2026. Built in this order, each shipped and verified on its own, because the ones +first are the ones a new user meets in their first ten minutes: + +| Order | Item | State and what decides it | +|---|---|---| +| 1 | **Hardened containers.** Read-only root filesystem, a `noexec` `/tmp`, no capabilities, `no-new-privileges`, a process limit | **Done.** See `design.md` §9.3, Security | +| 2 | **HTTPS on a bare IP.** A first deployment answers `https://` with a valid certificate, before any domain exists | **Built, verified against Pebble, and not described in the README or the site yet.** See `design.md` §9.3, Release sequence. It becomes a documented feature after one deployment to a server on a public IPv4 address, against Let's Encrypt itself, with plain HTTP checked while the certificate is still being issued | +| 3 | **A cap on the cache volumes, and disk watched while apps run.** Least recently used eviction when an app's runtime writes pass their limit | **Done.** See `design.md` §9.3, Cache guard | +| 4 | **Skew protection and instant rollback.** The previous deployment's container keeps running for an hour, Caddy sends each request to the build the page was loaded from, and a rollback within that hour only reloads Caddy | **Done.** See `design.md` §9.3, Release sequence | +| 5 | **The compatibility suite nightly against the newest canary, published on the site** | **Built, not yet run.** `conformance.yml` runs nightly and publishes to the `evidence` branch, which the evidence page reads. Verified once the first scheduled run publishes, which needs this merged to `main` | +| 6 | **A preview deployment per pull request on the same server**, removed when the pull request closes | **Done**, `--preview `, verified end to end. The pull request workflow in `vm.md` §6 has not run in GitHub Actions | +| 7 | **Deploy time measured stage by stage**, then shortened where nextship owns the time | **Done.** The health wait was 4.1 s of a 16.6 s warm deployment and is 1.1 s now; three deployments took 13.3 to 15.0 s, and 7.7 s or more of that is `next build`. See `design.md` §9.3, Release sequence. Under ten seconds is not reachable without a faster `next build`, so it is not promised | +| 8 | **Build once, promote to another environment** with its own `NEXT_PUBLIC_*` values | **Concluded no.** See "Deliberately not built" | +| 9 | **A migration plan from Vercel**: domains, cron jobs, routing rules and the names of env vars | **Local part done:** `doctor` prints a crontab line per cron job and a `next.config` snippet of the routes that carry over unchanged. **Not built:** domains and env var names, which need Vercel's API and a Vercel account to verify against. Vercel never returns a sensitive env var's value, so that part would ask for them, for the reason `env pull` is not built | +| 10 | **Peak build memory in the build result**, and a warning near the limit | `server add` already adds swap below 4 GB, and remote builds are refused below 2 GB | + +Correctness across several servers stays under v2 below, with its trigger. + ## AWS: on demand, after the Lightsail streaming experiment Previously v1.1. Moved behind the VM target: an EC2 instance is a Linux server over SSH, which @@ -528,6 +550,8 @@ Recorded so the decision stays visible rather than looking like an oversight. | Lambda, or any serverless AWS compute | It would be the cheapest option by far, and that is not the deciding factor. Lambda cannot run `next start`, so the app has to be split into functions and the routing pipeline re-implemented: middleware matching, dynamic segments, the `rsc` and `_rsc` cache-key discipline, PPR resume, ISR through object storage and a queue, image optimization as its own function. That is the work `design.md` §2.1 exists to avoid, and the work that cost other projects years. It also cannot port: DigitalOcean has no Lambda, so a serverless-first design would make the driver interface a fiction | | `env pull` | App Platform never returns a stored secret, so a pull could only ever return `NEXT_PUBLIC_*` values and the names of the rest. `nextship env` already lists the names. A command that returns blanks for everything that matters is worse than no command | | Log forwarding to an external sink | Needs a destination and credentials that are the user's to choose. There is nothing to verify against, and shipping unverified infrastructure code is how the defects in this project's own history got made | +| Promoting one build to another environment by replacing `NEXT_PUBLIC_*` placeholders | Measured on Next.js 16.3.4 with Turbopack, building a client component with placeholder values: a value used as a string or inside a template literal survived and could be replaced, but `FLAG === 'on' ? a : b` compiled to the losing branch alone, with no trace of the flag, and `URL.length > 30 ? a : b` compiled to the branch the placeholder's own length chose, while the placeholder itself still appeared elsewhere in the bundle. The second is undetectable: a replacement would look complete and ship an app that decided on the placeholder. The placeholders also land in the prerendered HTML and the server chunks. Promoting safely means building again with the target's values, which `deploy` already does | +| Stripping `x-middleware-subrequest` at the proxy | The header drove CVE-2025-29927, which was fixed in Next.js 15.2.3. nextship refuses anything older than 16.2, so no version it deploys is affected, and a rule for a header no supported version honours would be decoration | | A dedicated health endpoint | Would mean replacing Next.js's `startServer` and owning keep-alive, upgrades and error handling on the most critical path in the system, to save one render every few seconds for apps that prerender nothing. The health path is chosen from the build's prerendered routes instead | --- diff --git a/docs/vm.md b/docs/vm.md index acad1a6..88250d6 100644 --- a/docs/vm.md +++ b/docs/vm.md @@ -54,6 +54,8 @@ only what is not already done, which is why a second run changes nothing. | `updates` | Installs security updates automatically with unattended-upgrades, using the distribution's default security origins | `--no-auto-updates` | | `firewall` | If ufw is installed, allows the SSH port, 80 and 443, then enables it. SSH is allowed before anything is enabled | `--no-firewall` | | `watchdog` | A systemd timer that restarts a nextship container unhealthy for three checks in a row, logging why to the journal under `nextship-watchdog` | | +| `cache-guard` | A systemd timer that, every ten minutes, keeps what each app has written at runtime under 10% of the disk, and never under 1 GiB, by deleting what was used least recently. Logged to the journal under `nextship-cache-guard` | | +| `retire` | A systemd timer that, every minute, stops the deployment a newer one replaced once its hour for open tabs is over, logged under `nextship-retire` | | | `ssh-hardening` | Turns off password and keyboard-interactive logins and allows root by key only, in `/etc/ssh/sshd_config.d/10-nextship.conf`. Runs last, only after the `nextship` login worked, and removes its file again if `sshd -t` rejects it | `--no-ssh-hardening` | Docker publishes Caddy's ports itself, which bypasses ufw for those two ports. That is @@ -80,6 +82,16 @@ does not protect any port you publish from another container yourself. privileged container, so anyone who can log in as `nextship` controls the server. Treat its keys as root's. It is a separate user so nextship's files and containers have one owner, not to limit what that owner can do. +- **An app cannot change its own files.** Its container's filesystem is read-only apart + from the two volumes Next.js writes to and a 64 MB `/tmp` that is emptied on restart and + cannot run a binary. It keeps no Linux capability, cannot gain privileges, and is limited + to 512 processes. Code an attacker runs through a flaw in your app can neither rewrite + the app nor leave a program behind, and a restart removes whatever it wrote to `/tmp`. + An app that keeps files beside its code, a SQLite database or uploads, lists those + folders as `writable` in `nextship.json`, such as `"writable": ["data"]`: each becomes a + volume it can write to, kept across deployments, deleted by `destroy` and not copied by + `server move`. A write anywhere else fails with `EROFS` when it happens, and a package + that unpacks an executable into `/tmp`, such as a headless Chromium, cannot run it. - **What is never touched:** containers without nextship's labels, the Docker daemon's configuration, and DNS. @@ -96,6 +108,7 @@ keeps on the server: |---|---| | `/etc/nextship/apps//app.json` | The app's id and its domains. The id is what `nextship.json` records; a directory with another id is refused | | `/etc/nextship/apps//deployments.json` | The deployment history rollback chooses from | +| `/etc/nextship/apps//previous` | The replaced deployment kept running for open tabs, and when `retire.sh` stops it | | `/etc/nextship/apps//env` | Runtime variables, mode 0600, in Docker's env file format | | `/etc/nextship/apps//secrets` | The Server Actions key, mode 0600 | | `/etc/nextship/caddy/sites/.caddy` | The app's Caddy site, regenerated on every change | @@ -107,13 +120,20 @@ keeps on the server: attention. **Memory.** Each container is limited to an even share of 80% of the server's RAM across -the apps on it at the time it starts, at least 256 MiB. `--memory` overrides it. Adding an +the containers its apps can run at once when it starts, at least 256 MiB. That is two per +app, the live one and the one kept for an hour after a deployment, so one app on a 2 GB +server gets 800m, and a preview counts as an app. `--memory` overrides it. Adding an app does not shrink the limits of containers already running until they are deployed again. **Recovering.** A container that exits is restarted by Docker. A container that stays up but stops answering its health check is restarted by the watchdog after three failed -minutes, and the journal says so under `nextship-watchdog`. A deployment or rollback +minutes, and the journal says so under `nextship-watchdog`. What Next.js writes while an +app runs, optimized images and pages it regenerates or renders for a new dynamic segment, +is kept under 10% of the disk per app by the cache guard, which deletes the least recently +used first and logs each run that deleted anything under `nextship-cache-guard`. Only what +was written at runtime is deleted, never a file the build produced, and Next.js renders a +deleted page or encodes a deleted image again on its next request. A deployment or rollback stopped with Ctrl+C, by closing its terminal, or by `kill` gives its lock back before it exits. One that was ended some other way, a lost power supply or `kill -9`, can still leave its lock: that is reported with its owner after 30 minutes and never removed @@ -185,6 +205,71 @@ refuse the second anyway, but the refusal would fail that workflow run. Not yet run in GitHub Actions: this workflow is written from what nextship does locally, and running it is on the v1.1 checklist. +### A preview per pull request + +`--preview ` scopes a command to a preview: a second app on the same server named +`-`, with its own containers, env file and domains. `deploy --preview ` +creates it the first time and updates it after that. Its record on the server names the app +it previews, which is how a later run with only the committed `nextship.json` finds it again, +and `nextship.json` is never written for it. A preview never takes the server's own address, +answers only on the domains attached to it, and every response asks crawlers not to index +it. It starts with an empty env file: push values meant for a preview with +`nextship env push --preview `, and never production's database. + +Point a wildcard record, `*.preview.example.com`, at the server once. Each preview's domain +then gets its own certificate from Caddy when it is added. Then add a second workflow: + +```yaml +name: preview +on: + pull_request: + types: [opened, synchronize, reopened, closed] +concurrency: preview-${{ github.event.number }} +permissions: + contents: read + pull-requests: write +jobs: + preview: + runs-on: ubuntu-latest + env: + PREVIEW: pr-${{ github.event.number }} + HOST: pr-${{ github.event.number }}.preview.example.com + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - run: npm ci + - name: Load the deploy key + run: | + eval "$(ssh-agent -s)" + echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" + echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" + ssh-add - <<< "${{ secrets.NEXTSHIP_SSH_KEY }}" + - name: Deploy the preview + if: github.event.action != 'closed' + run: | + npx --yes nextship-cli deploy --preview "$PREVIEW" --yes --build local + npx --yes nextship-cli domain --preview "$PREVIEW" | grep -q "$HOST" || + npx --yes nextship-cli domain add "$HOST" --preview "$PREVIEW" --yes + gh pr comment ${{ github.event.number }} --body "Preview: https://$HOST" + - name: Remove the preview + if: github.event.action == 'closed' + run: npx --yes nextship-cli destroy "$(node -p "require('./nextship.json').name")-$PREVIEW" --preview "$PREVIEW" --images --yes +``` + +Each preview is a whole container, with the same memory limit as any app on the server, so +the server's memory decides how many pull requests can have one at once. + +**A preview runs the pull request's code with the deploy key.** GitHub gives no secrets to a +pull request from a fork, so only branches in the repository get a preview, but anyone who +can push a branch can run code on the runner that holds a key to the server. Use this where +everyone with push access would be trusted with the server anyway. + +Not yet run in GitHub Actions. What it runs is verified: `conformance/vm/e2e.sh` deploys a +preview, attaches a domain, and destroys it (`design.md` §9.3). + ## 7. Backups and recovery A server is one machine. Know what is on it that exists nowhere else: diff --git a/packages/cli/runtime/vm/cache-guard.sh b/packages/cli/runtime/vm/cache-guard.sh new file mode 100644 index 0000000..1cb405f --- /dev/null +++ b/packages/cli/runtime/vm/cache-guard.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# @nextship/cli: cache guard +# +# Run every ten minutes by a systemd timer that `nextship server add` installs. +# +# Next.js writes to disk for as long as an app serves: optimized images and the +# fetch cache in `.next/cache`, and every page it regenerates or renders for a new +# dynamic segment beside the build's own files in `.next`. Nothing in Next.js bounds +# either, and a crawler walking an app with dynamic segments can write gigabytes a +# day until the disk is full and every app on the server stops. This keeps what each +# app has written at runtime under a limit by deleting what was used least recently. +# +# Only what Next.js wrote at runtime is ever deleted: everything in the app's cache +# volume, and the files in a build volume modified after that volume was created. +# What the build produced is never touched, so a page with no revalidate time keeps +# the file it was prerendered into. Deleting an entry is safe while the app runs: +# Next.js reads a missing file as a cache miss and renders the page or encodes the +# image again. Every eviction is logged to the journal under nextship-cache-guard. +# +# "Used" is the later of a file's last access and its last write. Access times alone +# are not enough: under `relatime`, the default, the kernel updates one at most once +# a day, and under `noatime` never, where sorting by access would order files by +# nothing and evict the busiest with the idle. With the write time taken too, a +# volume that keeps no access times is evicted oldest written first, and one that +# does is evicted least recently used first, to the day. +# +# Author: Ragul D +# Design: ../../../../docs/design.md §9.3 + +set -euo pipefail + +# Evicting down to below the limit, rather than to it, means one eviction is followed +# by some minutes of room instead of another eviction on the next run. +TARGET_PERCENT=80 +APPS=/etc/nextship/apps + +# An app may keep 10% of the disk Docker stores volumes on, and never less than 1 GiB, +# which is room for thousands of pages and optimized images. +limit_kib() { + local disk + disk="$(df -Pk "$1" | awk 'NR == 2 { print $2 }')" + local share=$(( disk / 10 )) + [ "$share" -lt 1048576 ] && share=1048576 + echo "$share" +} + +# Prints " " for each file Next.js wrote at runtime under the +# given volume directories, where last used is the later of its access and write +# times. A build volume is given as "@": only its files modified after +# that moment count. +runtime_files() { + local entry dir since + for entry in "$@"; do + dir="${entry%@*}" + [ -d "$dir" ] || continue + if [ "$entry" = "$dir" ]; then + find "$dir" -type f -printf '%A@ %T@ %k %p\n' + else + since="${entry##*@}" + find "$dir" -type f -newermt "@$since" -printf '%A@ %T@ %k %p\n' + fi + done | awk '{ used = ($1 > $2) ? $1 : $2; sub(/^[^ ]+ [^ ]+ /, ""); print used, $0 }' +} + +# Deletes the least recently used runtime files of one app until what remains is at +# most TARGET_PERCENT of the limit. Prints the number of files and KiB it removed. +evict() { + local limit="$1" + shift + local listing total target removed=0 freed=0 atime kib path entry + listing="$(runtime_files "$@" | sort -n)" + total="$(awk '{ sum += $2 } END { print sum + 0 }' <<< "$listing")" + if [ "$total" -le "$limit" ]; then + echo "0 0" + return + fi + target=$(( limit * TARGET_PERCENT / 100 )) + while read -r atime kib path; do + [ -n "$path" ] || continue + [ "$total" -le "$target" ] && break + rm -f -- "$path" + total=$(( total - kib )) + removed=$(( removed + 1 )) + freed=$(( freed + kib )) + done <<< "$listing" + # An optimized image is a directory holding one file, so its directory goes too. + for entry in "$@"; do + [ -d "${entry%@*}" ] && find "${entry%@*}" -mindepth 1 -type d -empty -delete + done + echo "$removed $freed" +} + +main() { + [ -d "$APPS" ] || exit 0 + local app name volumes limit mount created result + for app in $(ls "$APPS"); do + volumes=() + mount="$(docker volume inspect -f '{{.Mountpoint}}' "nextship-$app-cache" 2> /dev/null || true)" + [ -n "$mount" ] && volumes+=("$mount") + for name in $(docker volume ls -q --filter "name=^nextship-$app-build-dpl-"); do + mount="$(docker volume inspect -f '{{.Mountpoint}}' "$name")" + created="$(date -d "$(docker volume inspect -f '{{.CreatedAt}}' "$name")" +%s)" + volumes+=("$mount@$created") + done + [ "${#volumes[@]}" -gt 0 ] || continue + limit="$(limit_kib "${volumes[0]%@*}")" + result="$(evict "$limit" "${volumes[@]}")" + if [ "${result%% *}" != 0 ]; then + logger -t nextship-cache-guard "$app: removed ${result%% *} least recently used cache files, $(( ${result##* } / 1024 )) MiB, to stay under its $(( limit / 1024 )) MiB limit" + fi + done +} + +# Sourced by its tests, which call evict on a directory; run by the timer otherwise. +if [ "${BASH_SOURCE[0]}" = "$0" ]; then main; fi diff --git a/packages/cli/runtime/vm/retire.sh b/packages/cli/runtime/vm/retire.sh new file mode 100644 index 0000000..e47c189 --- /dev/null +++ b/packages/cli/runtime/vm/retire.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +# @nextship/cli: retire the previous deployment +# +# Run every minute by a systemd timer that `nextship server add` installs. +# +# After a deployment the one it replaced keeps running for an hour, so a tab that +# loaded it can still fetch its files and call its Server Actions (caddy.ts). Each +# app's `previous` file, written by nextship with its deployment history, names that +# container and when it stops, as ` `. This stops it once +# that moment has passed and empties the file. Caddy's route to it then fails over +# to the live container, which is what such a request got before it was kept. +# +# Only a container labelled as this app's nextship container is ever stopped, so a +# damaged file cannot stop anything else. Stopped, not removed: its logs stay +# readable and the next deployment's cleanup removes it like any other. +# +# The container Caddy serves is never stopped, whatever `previous` says. The app's +# Caddy site names it on a `# live ` line, and that file is what Caddy +# reads, so it cannot disagree with what is serving. nextship empties `previous` +# before it points Caddy anywhere, so the two should never name the same container; +# this is for the case where they do, because stopping the live app is not undone by +# Docker's restart policy. With no such line to read, nothing is stopped either. +# +# Author: Ragul D +# Design: ../../../../docs/design.md §9.3 + +set -euo pipefail + +APPS=/etc/nextship/apps +SITES=/etc/nextship/caddy/sites +[ -d "$APPS" ] || exit 0 +now="$(date +%s)" + +for file in "$APPS"/*/previous; do + [ -s "$file" ] || continue + app="$(basename "$(dirname "$file")")" + read -r container until < "$file" || true + [ -n "$container" ] || continue + case "$until" in '' | *[!0-9]*) logger -t nextship-retire "$app: unreadable $file, left alone"; continue ;; esac + [ "$now" -ge "$until" ] || continue + live="$(sed -n 's/^# live //p' "$SITES/$app.caddy" 2> /dev/null | head -n 1)" + if [ -z "$live" ] || [ "$live" = "$container" ]; then + logger -t nextship-retire "$app: not stopping $container: ${live:+it is the container Caddy serves}${live:-the site file names no live container}" + : > "$file" + continue + fi + label="$(docker inspect -f '{{index .Config.Labels "sh.nextship.app"}}' "$container" 2> /dev/null || true)" + if [ "$label" = "$app" ]; then + # 30 s, the time a deployment gives a replaced container to finish its requests. + docker stop -t 30 "$container" > /dev/null && logger -t nextship-retire "$app: stopped $container, kept for the tabs that loaded it until now" + fi + : > "$file" +done diff --git a/packages/cli/runtime/vm/setup.sh b/packages/cli/runtime/vm/setup.sh index 55c80ec..865c32c 100755 --- a/packages/cli/runtime/vm/setup.sh +++ b/packages/cli/runtime/vm/setup.sh @@ -26,6 +26,9 @@ # NEXTSHIP_SSH_PORT the port that connection used, which the firewall keeps open # NEXTSHIP_MIN_DOCKER the oldest Docker major nextship builds with (docker.ts) # NEXTSHIP_WATCHDOG_B64 the watchdog script, base64, installed by the watchdog step +# NEXTSHIP_CACHE_GUARD_B64 the cache guard script, base64, installed by the +# cache-guard step +# NEXTSHIP_RETIRE_B64 the retire script, base64, installed by the retire step # NEXTSHIP_OS_RELEASE tests only: an os-release file to read instead of # /etc/os-release, so the OS check runs against fixtures # @@ -36,12 +39,12 @@ set -euo pipefail # Raised whenever a step changes, so `server status` can tell a server set up by an # older nextship to run `server add --yes` again. -SETUP_VERSION=2 +SETUP_VERSION=3 # Pinned to a minor so a server gets the same Caddy until nextship moves it on purpose. CADDY_IMAGE="caddy:2.10" -STEPS="os arch resources ports docker user dirs network caddy journald swap updates firewall watchdog ssh-hardening" +STEPS="os arch resources ports docker user dirs network caddy journald swap updates firewall watchdog cache-guard retire ssh-hardening" OS_RELEASE="${NEXTSHIP_OS_RELEASE:-/etc/os-release}" MIN_DOCKER="${NEXTSHIP_MIN_DOCKER:-23}" @@ -382,6 +385,98 @@ apply_watchdog() { systemctl restart nextship-watchdog.timer } +# ---------------------------------------------------------------- cache-guard + +CACHE_GUARD=/usr/local/lib/nextship/cache-guard.sh +CACHE_GUARD_SERVICE="[Unit] +Description=Keep what nextship apps write at runtime under their disk limit + +[Service] +Type=oneshot +ExecStart=$CACHE_GUARD +" +# Ten minutes because each run walks every file an app has cached, which is too much +# work for every minute, and a disk does not fill in ten minutes at the rates seen. +CACHE_GUARD_TIMER="[Unit] +Description=Run the nextship cache guard every ten minutes + +[Timer] +OnBootSec=5min +OnUnitActiveSec=10min +AccuracySec=1min + +[Install] +WantedBy=timers.target +" + +check_cache_guard() { + local expected="" + [ -n "${NEXTSHIP_CACHE_GUARD_B64:-}" ] && expected="$(printf '%s' "$NEXTSHIP_CACHE_GUARD_B64" | base64 -d)" + if [ ! -f "$CACHE_GUARD" ] || [ "$(cat "$CACHE_GUARD")" != "$expected" ] || + [ "$(cat /etc/systemd/system/nextship-cache-guard.timer 2> /dev/null)" != "$(printf '%s' "$CACHE_GUARD_TIMER")" ] || + [ "$(cat /etc/systemd/system/nextship-cache-guard.service 2> /dev/null)" != "$(printf '%s' "$CACHE_GUARD_SERVICE")" ]; then + result change "install a cache guard that keeps each app's runtime cache under 10% of the disk" + return + fi + if ! systemctl is-active --quiet nextship-cache-guard.timer; then result change "start the cache guard timer"; return; fi + result ok "cache guard runs every ten minutes" +} +apply_cache_guard() { + install -d -m 0755 /usr/local/lib/nextship + printf '%s' "$NEXTSHIP_CACHE_GUARD_B64" | base64 -d > "$CACHE_GUARD" + chmod 0755 "$CACHE_GUARD" + printf '%s' "$CACHE_GUARD_SERVICE" > /etc/systemd/system/nextship-cache-guard.service + printf '%s' "$CACHE_GUARD_TIMER" > /etc/systemd/system/nextship-cache-guard.timer + systemctl daemon-reload + systemctl enable nextship-cache-guard.timer > /dev/null 2>&1 + systemctl restart nextship-cache-guard.timer +} + +# --------------------------------------------------------------------- retire + +RETIRE=/usr/local/lib/nextship/retire.sh +RETIRE_SERVICE="[Unit] +Description=Stop the previous nextship deployment once its tabs have had their hour + +[Service] +Type=oneshot +ExecStart=$RETIRE +" +RETIRE_TIMER="[Unit] +Description=Run the nextship retire check every minute + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +AccuracySec=10s + +[Install] +WantedBy=timers.target +" + +check_retire() { + local expected="" + [ -n "${NEXTSHIP_RETIRE_B64:-}" ] && expected="$(printf '%s' "$NEXTSHIP_RETIRE_B64" | base64 -d)" + if [ ! -f "$RETIRE" ] || [ "$(cat "$RETIRE")" != "$expected" ] || + [ "$(cat /etc/systemd/system/nextship-retire.timer 2> /dev/null)" != "$(printf '%s' "$RETIRE_TIMER")" ] || + [ "$(cat /etc/systemd/system/nextship-retire.service 2> /dev/null)" != "$(printf '%s' "$RETIRE_SERVICE")" ]; then + result change "install a timer that stops the previous deployment an hour after it was replaced" + return + fi + if ! systemctl is-active --quiet nextship-retire.timer; then result change "start the retire timer"; return; fi + result ok "previous deployments stop an hour after they are replaced" +} +apply_retire() { + install -d -m 0755 /usr/local/lib/nextship + printf '%s' "$NEXTSHIP_RETIRE_B64" | base64 -d > "$RETIRE" + chmod 0755 "$RETIRE" + printf '%s' "$RETIRE_SERVICE" > /etc/systemd/system/nextship-retire.service + printf '%s' "$RETIRE_TIMER" > /etc/systemd/system/nextship-retire.timer + systemctl daemon-reload + systemctl enable nextship-retire.timer > /dev/null 2>&1 + systemctl restart nextship-retire.timer +} + # -------------------------------------------------------------- ssh-hardening # Named 10- because sshd keeps the first value it reads and reads drop-ins in diff --git a/packages/cli/scripts/verify-pack.mjs b/packages/cli/scripts/verify-pack.mjs index 20a949c..a9129a3 100644 --- a/packages/cli/scripts/verify-pack.mjs +++ b/packages/cli/scripts/verify-pack.mjs @@ -27,7 +27,7 @@ const manifest = JSON.parse(readFileSync(path.join(packageRoot, 'package.json'), * The adapter is copied into every build, and `server add` sends the two VM * scripts to the server; the rest are the README and what Apache-2.0 requires. */ -const REQUIRED = ['runtime/adapter.mjs', 'runtime/vm/setup.sh', 'runtime/vm/watchdog.sh', 'README.md', 'LICENSE', 'NOTICE'] +const REQUIRED = ['runtime/adapter.mjs', 'runtime/vm/setup.sh', 'runtime/vm/watchdog.sh', 'runtime/vm/cache-guard.sh', 'runtime/vm/retire.sh', 'README.md', 'LICENSE', 'NOTICE'] const fail = (message) => { throw new Error(message) diff --git a/packages/cli/src/config.test.ts b/packages/cli/src/config.test.ts index ce31689..9789f45 100644 --- a/packages/cli/src/config.test.ts +++ b/packages/cli/src/config.test.ts @@ -14,7 +14,7 @@ import assert from 'node:assert/strict' import { mkdtemp, readFile, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' -import { readConfig, writeConfig, type ServerRecord } from './config.js' +import { readConfig, writableProblem, writeConfig, type ServerRecord } from './config.js' import { NextshipError } from './errors.js' async function withConfig(contents: string): Promise { @@ -156,3 +156,33 @@ test('a digitalocean config is written as version 1, a vm config as version 2', await writeConfig(root, { version: 1, target: 'vm', name: 'demo', server: server as ServerRecord }) assert.equal(JSON.parse(await readFile(path.join(root, 'nextship.json'), 'utf8')).version, 2) }) + +const vmWith = (writable: unknown): string => + JSON.stringify({ + version: 2, + target: 'vm', + name: 'shop', + server: { host: '46.101.1.2', port: 22, user: 'nextship', hostKey: 'ssh-ed25519 AAAA', arch: 'amd64' }, + writable, + }) + +// A writable folder becomes a mount inside the app's container, so a path that +// climbs out of the app, or lands on what Next.js or the image owns, is refused. +test('a writable folder is a plain relative folder, never one that escapes the app or shadows what it runs from', () => { + for (const good of ['data', 'uploads', 'public/uploads', 'var/db.d']) assert.equal(writableProblem(good), null, good) + for (const bad of ['', '/data', '../data', 'data/..', '.next', '.next/cache', 'data/', 'a//b', 'a b', 'node_modules/x', 'a__b', 7, null]) { + assert.notEqual(writableProblem(bad), null, String(bad)) + } +}) + +test('writable folders are read from nextship.json, and a list that would hide or repeat one is refused', async () => { + assert.deepEqual((await readConfig(await withConfig(vmWith(['data', 'public/uploads']))))?.writable, ['data', 'public/uploads']) + await assert.rejects(readConfig(await withConfig(vmWith(['data', 'data']))), /same writable folder twice/) + await assert.rejects(readConfig(await withConfig(vmWith(['data', 'data/cache']))), /inside another/) + await assert.rejects(readConfig(await withConfig(vmWith(['../etc']))), /writable folder "\.\.\/etc"/) + await assert.rejects(readConfig(await withConfig(vmWith('data'))), /not a list of folders/) + await assert.rejects( + readConfig(await withConfig(JSON.stringify({ version: 1, target: 'digitalocean', name: 'shop', region: 'blr', registry: 'r', writable: ['data'] }))), + (error: unknown) => error instanceof NextshipError && /only the vm target uses/.test(error.message) + ) +}) diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index 1feba01..82fa73a 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -58,6 +58,32 @@ export interface ProjectConfig { server?: ServerRecord /** VM only. Absent means `remote`. */ build?: BuildMode + /** + * VM only. Folders under the app's directory the app may write to, such as `data` + * or `uploads`. Everything else in an app's container is read-only, and each folder + * here is a volume, so what the app writes there outlives a deployment. + */ + writable?: string[] +} + +/** How many writable folders one app may declare: each is a volume and a mount. */ +const WRITABLE_LIMIT = 8 + +/** + * Why a writable folder is refused, or null when it is acceptable: a relative + * path of plain names. A leading dot is refused, which covers `..` and `.next`, + * where Next.js's own volumes are mounted; `__` is refused because the volume is + * named with it in place of `/`. + */ +export function writableProblem(folder: unknown): string | null { + if (typeof folder !== 'string') return 'is not a string' + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*(\/[A-Za-z0-9][A-Za-z0-9._-]*)*$/.test(folder)) { + return 'is not a relative folder of letters, digits, dots, hyphens and underscores, with no leading dot and no trailing slash' + } + if (folder.includes('__')) return 'contains a double underscore' + if (folder.split('/')[0] === 'node_modules') return 'is inside node_modules, which the image owns' + if (folder.length > 100) return 'is over 100 characters' + return null } export async function readConfig(root: string): Promise { @@ -130,8 +156,8 @@ function validateConfig(config: ProjectConfig): void { if (typeof config.registry !== 'string' || config.registry.length === 0) { refuse('has no registry for DigitalOcean') } - if (config.server !== undefined || config.build !== undefined) { - refuse('records a server or build mode, which only the vm target uses') + if (config.server !== undefined || config.build !== undefined || config.writable !== undefined) { + refuse('records a server, build mode or writable folders, which only the vm target uses') } return } @@ -154,6 +180,20 @@ function validateConfig(config: ProjectConfig): void { if (config.region !== undefined || config.registry !== undefined) { refuse('records a region or registry, which only the digitalocean target uses') } + if (config.writable !== undefined) { + if (!Array.isArray(config.writable)) refuse('has a `writable` that is not a list of folders') + if (config.writable.length > WRITABLE_LIMIT) refuse(`lists ${config.writable.length} writable folders, more than the ${WRITABLE_LIMIT} an app may have`) + for (const folder of config.writable) { + const problem = writableProblem(folder) + if (problem) refuse(`has a writable folder ${JSON.stringify(folder)} that ${problem}`) + } + if (new Set(config.writable).size !== config.writable.length) refuse('lists the same writable folder twice') + // One volume mounted inside another hides whatever the outer one holds there. + for (const folder of config.writable) { + const outer = config.writable.find((other) => other !== folder && folder.startsWith(`${other}/`)) + if (outer) refuse(`lists the writable folder "${folder}" inside another, "${outer}"; list the outer one only`) + } + } } /** The lowest version that can hold everything in this config. */ diff --git a/packages/cli/src/deploy.ts b/packages/cli/src/deploy.ts index be763e7..ca5a9da 100644 --- a/packages/cli/src/deploy.ts +++ b/packages/cli/src/deploy.ts @@ -20,7 +20,8 @@ import { buildProject, storedEncryptionKey } from './build.js' import { packageImage } from './packaging.js' import { readConfig, writeConfig, TARGET_IDS, type BuildMode, type ProjectConfig, type TargetId } from './config.js' import { CONTAINER_PORT } from './image/dockerfile.js' -import { client as apiClient } from './owned-app.js' +import { client as apiClient, previewConfig } from './owned-app.js' +import { selectedPreview } from './preview.js' import { DEFAULT_REGION } from './targets/digitalocean-target.js' import type { EnvRecord, Target } from './targets/target.js' import { untilStopped } from './interrupt.js' @@ -105,15 +106,28 @@ export function settingsFor( export async function deploy(project: ProjectInfo, options: DeployOptions): Promise { const existing = await readConfig(project.root) - const settings = settingsFor(project.name, existing, options) + const preview = selectedPreview() + // A preview is of the deployed app, and is authorised by that app's id. + const previewOf = preview === null ? null : existing?.appId + if (preview !== null && !previewOf) { + throw new NextshipError('A preview is a preview of the deployed app, and this project has none yet.', 'Run `nextship deploy` first.') + } + const recorded = settingsFor(project.name, existing, options) + const settings = preview === null ? recorded : previewConfig(recorded, preview) const client = apiClient(settings, existing ? options.target : undefined) const name = settings.name // Read-only reconnaissance first, so the plan describes reality. const delivery = await client.planDelivery(name) const apps = await client.listApps() - const owned = existing?.appId ? apps.find((app) => app.id === existing.appId) : undefined - const nameClash = apps.find((app) => app.name === name && app.id !== existing?.appId) + const owned = previewOf + ? apps.find((app) => app.name === name && app.previewOf === previewOf) + : existing?.appId + ? apps.find((app) => app.id === existing.appId) + : undefined + const nameClash = previewOf + ? apps.find((app) => app.name === name && app.previewOf !== previewOf) + : apps.find((app) => app.name === name && app.id !== existing?.appId) if (nameClash) { throw new NextshipError( @@ -121,7 +135,7 @@ export async function deploy(project: ProjectInfo, options: DeployOptions): Prom 'nextship will not modify an app it does not own. Rename this project, or set a different name in nextship.json.' ) } - if (existing?.appId && !owned) { + if (!previewOf && existing?.appId && !owned) { throw new NextshipError( `nextship.json records app ${existing.appId}, which no longer exists ${client.appScope}.`, 'Remove the appId from nextship.json to create a new app, after confirming the old one is really gone.' @@ -144,11 +158,15 @@ export async function deploy(project: ProjectInfo, options: DeployOptions): Prom delivery, }) for (const line of lines) detail(line) + if (preview !== null) { + detail(`preview of "${recorded.name}", with its own env file and domains, and asks crawlers not to index it`) + if (!owned) detail(` answers nothing until \`nextship domain add --preview ${preview}\``) + } if (preserved.length > 0) detail(`preserved ${preserved.join(', ')}, kept as they are`) detail(`untouched ${apps.length} existing app(s) ${client.appScope}`) detail(client.deployRemoves ?? 'nothing is ever deleted by this command') - warnAboutMissingRuntimeEnv(project, existingEnv, owned !== undefined) + warnAboutMissingRuntimeEnv(project, existingEnv, owned !== undefined, preview) if (!options.confirmed) { ok('This was a plan only. Nothing was created or changed.') @@ -211,6 +229,7 @@ export async function deploy(project: ProjectInfo, options: DeployOptions): Prom instanceSize: options.instanceSize ?? null, memory: options.memory ?? null, healthPath: build.manifest.healthPath, + ...(previewOf ? { previewOf } : {}), }) // From here to the end of the release, every path out has to reach either @@ -220,14 +239,19 @@ export async function deploy(project: ProjectInfo, options: DeployOptions): Prom let handedOff = false try { // Written before waiting, so a timeout still leaves the app recorded as ours - // rather than orphaned and unadoptable on the next run. - const config: ProjectConfig = { - ...settings, - ...(delivered.store ? { registry: delivered.store } : {}), - appId: released.appId, + // rather than orphaned and unadoptable on the next run. A preview is recorded + // on the server instead, where the run that removes it will look. + if (previewOf) { + detail(`recorded on the server as a preview of "${recorded.name}"`) + } else { + const config: ProjectConfig = { + ...settings, + ...(delivered.store ? { registry: delivered.store } : {}), + appId: released.appId, + } + await writeConfig(project.root, config) + detail('recorded in nextship.json') } - await writeConfig(project.root, config) - detail('recorded in nextship.json') if (!released.deploymentId) { warn(`${client.displayName} reported no deployment to follow. Check its control panel.`) @@ -259,14 +283,16 @@ export async function deploy(project: ProjectInfo, options: DeployOptions): Prom * It is a warning rather than a prompt because uploading development values is * often exactly the wrong thing to do, which is why `env push` is explicit. */ -function warnAboutMissingRuntimeEnv(project: ProjectInfo, existing: EnvRecord[], appExists: boolean): void { +function warnAboutMissingRuntimeEnv(project: ProjectInfo, existing: EnvRecord[], appExists: boolean, preview: string | null): void { if (project.envFiles.length === 0) return if (appExists && existing.length > 0) return warn( - `This project has ${project.envFiles.join(', ')}, but the app will have no runtime environment. ` + + `This project has ${project.envFiles.join(', ')}, but the ${preview === null ? 'app' : 'preview'} will have no runtime environment. ` + 'Values Next.js inlines at build time still work; anything read at request time will be undefined. ' + - 'Run `nextship env push` if those values belong in production.' + (preview === null + ? 'Run `nextship env push` if those values belong in production.' + : `Run \`nextship env push --preview ${preview}\` with values meant for a preview, never production's database.`) ) } diff --git a/packages/cli/src/destroy.test.ts b/packages/cli/src/destroy.test.ts index 7a7ea4a..c0060e9 100644 --- a/packages/cli/src/destroy.test.ts +++ b/packages/cli/src/destroy.test.ts @@ -10,7 +10,7 @@ import { test } from 'node:test' import assert from 'node:assert/strict' -import { nameMatches } from './destroy.js' +import { nameMatches, previewsOf } from './destroy.js' import { NextshipError } from './errors.js' function rejects(run: () => unknown, fragment: string): void { @@ -47,3 +47,19 @@ test('the error names the app that would actually be destroyed', () => { } ) }) + +// The defect: destroying an app left its previews running, and with the app's id +// gone from nextship.json nothing could ever match them again. +test('the previews destroyed with an app are exactly the ones that name it', () => { + const apps = [ + { id: 'main', name: 'shop' }, + { id: 'p1', name: 'shop-pr-1', previewOf: 'main' }, + { id: 'p2', name: 'shop-pr-2', previewOf: 'main' }, + { id: 'other', name: 'blog' }, + { id: 'p3', name: 'blog-pr-1', previewOf: 'other' }, + // Named like a preview of shop, but not one: never matched by name. + { id: 'x', name: 'shop-pr-9' }, + ] + assert.deepEqual(previewsOf(apps, 'main').map((entry) => entry.id), ['p1', 'p2']) + assert.deepEqual(previewsOf(apps, 'none'), []) +}) diff --git a/packages/cli/src/destroy.ts b/packages/cli/src/destroy.ts index 80f7b08..0d9be21 100644 --- a/packages/cli/src/destroy.ts +++ b/packages/cli/src/destroy.ts @@ -23,7 +23,9 @@ import { NextshipError } from './errors.js' import type { ProjectInfo } from './detect.js' import { readConfig, writeConfig } from './config.js' -import { ownedApp } from './owned-app.js' +import { client, ownedApp, previewConfig } from './owned-app.js' +import type { AppRef } from './targets/target.js' +import { selectedPreview } from './preview.js' import { orderForDeletion } from './images.js' import { detail, ok, step, warn } from './util/log.js' @@ -51,6 +53,16 @@ export function nameMatches(given: string, recorded: string): void { ) } +/** + * The previews of an app, which go when it goes. A preview is found by the id of + * the app it previews, and destroying that app removes the id from `nextship.json`, + * so a preview left behind could never be reached by any command again while its + * containers, volumes and domains stayed on the server. + */ +export function previewsOf(apps: AppRef[], appId: string): AppRef[] { + return apps.filter((entry) => entry.previewOf === appId) +} + export async function destroy(project: ProjectInfo, options: DestroyOptions): Promise { const app = await ownedApp(project) const config = await readConfig(project.root) @@ -71,7 +83,13 @@ export async function destroy(project: ProjectInfo, options: DestroyOptions): Pr detail(`app DESTROY "${app.name}" (${app.appId})`) for (const line of plan.lines) detail(line) - const others = (await app.target.listApps()).filter((entry) => entry.id !== app.appId) + const apps = await app.target.listApps() + // A preview being destroyed has none of its own. + const previews = selectedPreview() === null ? previewsOf(apps, app.appId) : [] + for (const preview of previews) { + detail(`preview DESTROY "${preview.name}" with it, images included: a preview cannot be reached once the app it previews is gone`) + } + const others = apps.filter((entry) => entry.id !== app.appId && !previews.includes(entry)) detail(`untouched ${others.length} other app(s) ${app.target.appScope}`) warn('This cannot be undone. The app, its deployments and its history are removed.') @@ -79,18 +97,29 @@ export async function destroy(project: ProjectInfo, options: DestroyOptions): Pr if (!options.confirmed) { ok('This was a plan only. Nothing was destroyed.') - detail(`Run \`nextship destroy ${app.name} --yes\` to execute it.`) + const preview = selectedPreview() + detail(`Run \`nextship destroy ${app.name}${preview === null ? '' : ` --preview ${preview}`} --yes\` to execute it.`) return } step('Destroying') + // Previews first: if one cannot be removed, the app and its id are still there, + // and the same command can be run again. + for (const preview of previews) { + const scoped = client(previewConfig(app.config, preview.name.slice(app.name.length + 1))) + const images = await scoped.images(preview.name) + await scoped.destroyApp(preview.id) + await scoped.removeImages(preview.name, orderForDeletion(images).map((manifest) => manifest.id)) + detail(`preview "${preview.name}" destroyed`) + } await app.target.destroyApp(app.appId) ok(`App "${app.name}" destroyed.`) // Written before anything else can fail, so a later error cannot leave the // project pointing at an app that is already gone, which every command would - // then refuse to act on. - if (config) { + // then refuse to act on. A preview is not in nextship.json, and the app it + // previews still exists. + if (config && selectedPreview() === null) { await writeConfig(project.root, { ...config, appId: undefined }) detail('nextship.json no longer records an app, so `nextship deploy` will create a new one') } diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index 5ab8c15..c7b474c 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -12,7 +12,7 @@ import assert from 'node:assert/strict' import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' -import { diagnose, type Finding } from './doctor.js' +import { cronLines, diagnose, translateRoutes, type Finding } from './doctor.js' import type { ProjectInfo } from './detect.js' const projectFor = (root: string, overrides: Partial = {}): ProjectInfo => ({ @@ -236,7 +236,7 @@ test('on a vm target, revalidation is not warned about and cron advice names the try { const findings = await diagnose(projectFor(root), 'vm') assert.equal(titled(findings, 'On-demand revalidation'), undefined) - assert.match(titled(findings, 'cron job')?.action ?? '', /systemd timer/) + assert.match(titled(findings, 'cron job')?.action ?? '', /on your server, for example with these crontab lines/) assert.doesNotMatch(titled(findings, 'cron job')?.action ?? '', /DigitalOcean/) assert.equal(titled(findings, 'does not survive a restart'), undefined) assert.ok(titled(findings, 'survive a restart')) @@ -263,3 +263,66 @@ test('a vm project is warned that it runs on one server, and about request.url w await rm(root, { recursive: true, force: true }) } }) + +// A cron job left behind on Vercel never runs and nothing reports it, so doctor +// hands over the line that runs it, calling the route the way Vercel does. +test('each Vercel cron job becomes a crontab line that calls its route with the secret Vercel sends', () => { + assert.deepEqual(cronLines([{ path: '/api/digest', schedule: '0 5 * * *' }]), [ + '0 5 * * * curl -fsS -H "Authorization: Bearer $CRON_SECRET" "$APP_URL/api/digest"', + ]) + assert.match(cronLines([{ path: 'api/x', schedule: '* * * * *' }])[0], /^# not translated/) + assert.match(cronLines([{ path: '/api/x' }])[0], /^# not translated/) +}) + +// next.config takes the same shape as vercel.json for these, so an entry that uses +// nothing else is handed over as it is; anything else is named, never guessed. +test('routes carry over into next.config only when nothing about them has to be guessed', () => { + const redirects = translateRoutes('redirects', [ + { source: '/old', destination: '/new', permanent: true }, + { source: '/moved', destination: '/there' }, + { source: '/legacy', destination: '/x', permanent: false, regex: true }, + ]) + assert.deepEqual(redirects.translated, [{ source: '/old', destination: '/new', permanent: true }]) + assert.deepEqual(redirects.left, ['/moved does not say whether it is permanent', '/legacy uses regex']) + + const headers = translateRoutes('headers', [{ source: '/(.*)', headers: [{ key: 'X-Frame-Options', value: 'DENY' }] }]) + assert.equal(headers.translated.length, 1) + assert.deepEqual(translateRoutes('rewrites', [{ destination: '/x' }]).left, ['{"destination":"/x"} has no source']) +}) + +test('doctor prints the translated routes as a next.config snippet, and names what it left', async () => { + const root = await fixture({ + 'package.json': JSON.stringify({ name: 'demo', dependencies: { next: '16.3.4' } }), + 'vercel.json': JSON.stringify({ redirects: [{ source: '/a', destination: '/b', permanent: true }, { source: '/c', destination: '/d' }] }), + }) + try { + const finding = titled(await diagnose(projectFor(root), 'vm'), 'defines redirects') + assert.deepEqual(finding?.lines, ['async redirects() {', ' return [{"source":"/a","destination":"/b","permanent":true}]', '},']) + assert.match(finding?.action ?? '', /1 of 2 carries over as it is.*\/c does not say whether it is permanent/) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +// The defect: an app that writes beside its code deploys, passes its health check +// on a prerendered page, and fails with EROFS on its first write, with nothing at +// deploy time to say so. +test('on a server, source that looks like it writes local files is told about writable folders, unless it declared some', async () => { + const files = { + 'package.json': JSON.stringify({ name: 'demo', dependencies: { next: '16.3.4' } }), + 'app/api/upload/route.ts': "import { writeFile } from 'node:fs/promises'\nexport async function POST() { await writeFile('uploads/a', 'x') }", + 'lib/db.ts': "import Database from 'better-sqlite3'\nexport const db = new Database('data/app.db')", + } + const root = await fixture(files) + const declared = await fixture({ ...files, 'nextship.json': JSON.stringify({ version: 2, target: 'vm', name: 'demo', writable: ['data', 'uploads'] }) }) + try { + const finding = titled(await diagnose(projectFor(root), 'vm'), 'may write files') + assert.match(finding?.title ?? '', /better-sqlite3, writeFile/) + assert.match(finding?.action ?? '', /"writable": \["data"\]/) + assert.equal(titled(await diagnose(projectFor(root), 'digitalocean'), 'may write files'), undefined, 'App Platform containers are writable') + assert.equal(titled(await diagnose(projectFor(declared), 'vm'), 'may write files'), undefined, 'declared folders answer it') + } finally { + await rm(root, { recursive: true, force: true }) + await rm(declared, { recursive: true, force: true }) + } +}) diff --git a/packages/cli/src/doctor.ts b/packages/cli/src/doctor.ts index 016f1e3..2abb3dd 100644 --- a/packages/cli/src/doctor.ts +++ b/packages/cli/src/doctor.ts @@ -29,6 +29,8 @@ export interface Finding { consequence: string /** What to do about it. */ action: string + /** Text to use as it stands, such as crontab lines or a next.config snippet, when the action needs it. */ + lines?: string[] } /** Vercel packages that degrade silently rather than failing when they are off Vercel. */ @@ -158,19 +160,27 @@ async function vercelConfigFindings(root: string, target: TargetId): Promise 0) { + const { translated, left } = translateRoutes(key, config[key]) findings.push({ level: 'warning', title: `vercel.json defines ${key}`, consequence: `These ${key} stop applying. Only the equivalents in next.config are used.`, - action: `Move them into next.config, which works on any host.`, + action: + translated.length > 0 + ? `Move them into next.config, which works on any host. ${translated.length} of ${config[key].length} ${translated.length === 1 ? 'carries' : 'carry'} over as ${translated.length === 1 ? 'it is' : 'they are'}, in the lines below.` + + (left.length > 0 ? ` Translate the other ${left.length} by hand: ${left.join('; ')}.` : '') + : `Move them into next.config, which works on any host. None carries over as it is: ${left.join('; ')}.`, + ...(translated.length > 0 ? { lines: [`async ${key}() {`, ` return ${JSON.stringify(translated)}`, '},'] } : {}), }) } } @@ -178,6 +188,48 @@ async function vercelConfigFindings(root: string, target: TargetId): Promise { + const { path: route, schedule } = (cron ?? {}) as { path?: unknown; schedule?: unknown } + if (typeof route !== 'string' || !route.startsWith('/') || typeof schedule !== 'string') { + return `# not translated, it has no path starting with / or no schedule: ${JSON.stringify(cron)}` + } + return `${schedule} curl -fsS -H "Authorization: Bearer $CRON_SECRET" "$APP_URL${route}"` + }) +} + +/** + * The fields next.config accepts for each kind of route. vercel.json uses the same + * shape for these, so an entry that uses nothing else carries over unchanged. An + * entry with any other field, or a redirect that does not say whether it is + * permanent, which next.config requires, is left for a person to decide. + */ +const NEXT_ROUTE_FIELDS = { + redirects: ['source', 'destination', 'permanent', 'statusCode', 'basePath', 'locale', 'has', 'missing'], + rewrites: ['source', 'destination', 'basePath', 'locale', 'has', 'missing'], + headers: ['source', 'headers', 'basePath', 'locale', 'has', 'missing'], +} as const + +export function translateRoutes( + key: keyof typeof NEXT_ROUTE_FIELDS, + entries: unknown[] +): { translated: Record[]; left: string[] } { + const translated: Record[] = [] + const left: string[] = [] + for (const entry of entries) { + const route = (entry ?? {}) as Record + const source = typeof route.source === 'string' ? route.source : JSON.stringify(entry) + const unknown = Object.keys(route).filter((field) => !(NEXT_ROUTE_FIELDS[key] as readonly string[]).includes(field)) + if (typeof route.source !== 'string') left.push(`${source} has no source`) + else if (unknown.length > 0) left.push(`${source} uses ${unknown.join(', ')}`) + else if (key === 'redirects' && route.permanent === undefined && route.statusCode === undefined) { + left.push(`${source} does not say whether it is permanent`) + } else translated.push(route) + } + return { translated, left } +} + /** * The calls that revalidate on demand. App Platform's CDN keeps fully static pages * for as long as Next.js allows, which is a year, so these only reach visitors of a @@ -271,6 +323,23 @@ function runtimeFindings(project: ProjectInfo, target: TargetId): Finding[] { * What changes when the app runs on one server instead of a platform. Warned on * every vm project, because nothing in the source can show these are handled. */ +/** + * What in a source file suggests the app writes to its own disk: the file system + * calls that create or change a file, and the SQLite drivers, which keep their + * database in one. A sign, not proof: the path may be /tmp, which is writable. + */ +const LOCAL_WRITE_SIGNS = [ + 'writeFile(', + 'writeFileSync(', + 'appendFile(', + 'appendFileSync(', + 'createWriteStream(', + "'better-sqlite3'", + '"better-sqlite3"', + "'node:sqlite'", + '"node:sqlite"', +] + async function serverFindings(root: string): Promise { const findings: Finding[] = [ { @@ -282,6 +351,27 @@ async function serverFindings(root: string): Promise { }, ] + // Read as plain JSON, not through readConfig: doctor still reports what it can + // when nextship.json is damaged, and says so separately. + const declared = (await readJson(path.join(root, 'nextship.json')))?.writable + const writes = new Set() + if (!Array.isArray(declared) || declared.length === 0) { + for await (const file of sourceFiles(root)) { + const contents = await readFile(file, 'utf8').catch(() => '') + for (const sign of LOCAL_WRITE_SIGNS) if (contents.includes(sign)) writes.add(sign.replace(/[('"]/g, '')) + } + } + if (writes.size > 0) { + findings.push({ + level: 'warning', + title: `The source may write files beside the app: ${[...writes].sort().join(', ')}`, + consequence: + "On a server the app's container is read-only apart from .next and /tmp. A write anywhere else fails with EROFS at the moment it happens, not at deploy: the app deploys and passes its health check first.", + action: + 'If the app writes under its own directory, list the folders in nextship.json, for example "writable": ["data"]. Each becomes a volume the app can write to, kept across deployments. Writes to /tmp, object storage or a database need nothing.', + }) + } + for await (const file of sourceFiles(root)) { const contents = await readFile(file, 'utf8').catch(() => '') if (contents.includes('request.url') || contents.includes('req.url')) { diff --git a/packages/cli/src/images.test.ts b/packages/cli/src/images.test.ts index ca8d000..5563457 100644 --- a/packages/cli/src/images.test.ts +++ b/packages/cli/src/images.test.ts @@ -33,7 +33,7 @@ const registry = (...tags: Array<[string, number]>): ImageRecord[] => test('the image a live tag points to is never removed, though the API reports it untagged', () => { const manifests = registry(['new', 9], ['old', 8]) - const plan = planPrune(manifests, 'new', 1) + const plan = planPrune(manifests, ['new'], 1) const removed = new Set(plan.remove.map((m) => m.id)) assert.ok(!removed.has('sha256:image-new'), 'this is the image the app is running') @@ -42,7 +42,7 @@ test('the image a live tag points to is never removed, though the API reports it }) test('an image no retained tag can reach is removed, along with its index', () => { - const plan = planPrune(registry(['new', 9], ['old', 8]), 'new', 1) + const plan = planPrune(registry(['new', 9], ['old', 8]), ['new'], 1) assert.deepEqual( plan.remove.map((m) => m.id).sort(), @@ -54,7 +54,7 @@ test('an image no retained tag can reach is removed, along with its index', () = test('the deployed image is kept even when it falls outside the limit', () => { // 'old' is the oldest and would be dropped on age alone, but it is running. - const plan = planPrune(registry(['new', 9], ['mid', 8], ['old', 7]), 'old', 1) + const plan = planPrune(registry(['new', 9], ['mid', 8], ['old', 7]), ['old'], 1) const removed = new Set(plan.remove.map((m) => m.id)) assert.ok(!removed.has('sha256:image-old'), 'pruning the running image breaks the next restart') @@ -62,14 +62,14 @@ test('the deployed image is kept even when it falls outside the limit', () => { }) test('reclaimable size counts the images removed, not the tiny index alone', () => { - const plan = planPrune(registry(['new', 9], ['old', 8]), 'new', 1) + const plan = planPrune(registry(['new', 9], ['old', 8]), ['new'], 1) assert.ok(plan.reclaimableBytes > 181 * MIB, 'a figure near zero would be the index only') assert.ok(plan.reclaimableBytes < 183 * MIB) }) test('nothing is removed when everything is still reachable', () => { - const plan = planPrune(registry(['new', 9], ['old', 8]), 'new', 5) + const plan = planPrune(registry(['new', 9], ['old', 8]), ['new'], 5) assert.deepEqual(plan.remove, []) assert.equal(plan.reclaimableBytes, 0) @@ -82,7 +82,7 @@ test('a manifest shared by two retained indexes survives', () => { { id: 'sha256:shared', tags: [], children: [], sizeBytes: 182 * MIB, updatedAt: '2026-09-08T00:00:00Z' }, ] - const plan = planPrune(shared, 'a', 1) + const plan = planPrune(shared, ['a'], 1) const removed = new Set(plan.remove.map((m) => m.id)) assert.ok(removed.has('sha256:index-b'), 'the unreachable index goes') @@ -97,7 +97,7 @@ test('an orphan left by an earlier tag-only deletion is collected', () => { { id: 'sha256:image-orphan', tags: [], children: [], sizeBytes: 182 * MIB, updatedAt: '2026-09-07T00:00:00Z' }, ] - const plan = planPrune(manifests, 'live', 5) + const plan = planPrune(manifests, ['live'], 5) assert.deepEqual( plan.remove.map((m) => m.id).sort(), @@ -107,20 +107,20 @@ test('an orphan left by an earlier tag-only deletion is collected', () => { }) test('an unknown deployed tag still prunes by age rather than refusing', () => { - const plan = planPrune(registry(['c', 9], ['b', 8], ['a', 7]), null, 1) + const plan = planPrune(registry(['c', 9], ['b', 8], ['a', 7]), [], 1) assert.deepEqual(plan.keepTags, ['c']) assert.deepEqual(plan.removeTags.sort(), ['a', 'b']) }) test('an empty repository plans nothing', () => { - assert.deepEqual(planPrune([], null, 5), { keepTags: [], removeTags: [], remove: [], reclaimableBytes: 0 }) + assert.deepEqual(planPrune([], [], 5), { keepTags: [], removeTags: [], remove: [], reclaimableBytes: 0 }) }) test('keeping zero images is refused, because rollback would have no target', () => { for (const bad of [0, -1, 1.5, Number.NaN]) { assert.throws( - () => planPrune(registry(['a', 9]), 'a', bad), + () => planPrune(registry(['a', 9]), ['a'], bad), (error: unknown) => { assert.ok(error instanceof NextshipError) assert.match(error.message, /at least 1/) @@ -172,3 +172,12 @@ test('ordering emits everything even for a malformed graph', () => { assert.equal(orderForDeletion(cyclic).length, 2, 'a bad graph must not hang the command') }) + +// On a server the deployment a newer one replaced keeps running for an hour, and +// Docker refuses to remove an image a running container uses, so a prune that +// planned to would fail half way. +test('every tag in use is kept, the live one and the one still running for older tabs', () => { + const plan = planPrune(registry(['c', 9], ['b', 8], ['a', 7]), ['c', 'a'], 1) + assert.deepEqual(plan.keepTags.sort(), ['a', 'c']) + assert.deepEqual(plan.removeTags, ['b']) +}) diff --git a/packages/cli/src/images.ts b/packages/cli/src/images.ts index a76a691..d04bf14 100644 --- a/packages/cli/src/images.ts +++ b/packages/cli/src/images.ts @@ -31,7 +31,7 @@ import { NextshipError } from './errors.js' import type { ProjectInfo } from './detect.js' import { readConfig } from './config.js' import { ownedApp } from './owned-app.js' -import type { ImageRecord } from './targets/target.js' +import type { DeploymentRecord, ImageRecord } from './targets/target.js' import { detail, ok, step, warn } from './util/log.js' /** @@ -107,12 +107,13 @@ export function orderForDeletion(remove: ImageRecord[]): ImageRecord[] { /** * Decides what goes. * - * The live tag is retained whatever `keep` says. Pruning the image a running - * deployment was created from leaves an app that runs until something + * Tags in use are retained whatever `keep` says: the live one, and on a server the + * one still running for tabs opened before the last deployment. Pruning the image a + * running deployment was created from leaves an app that runs until something * reschedules it and then cannot start, a failure that appears hours later and - * looks nothing like its cause. + * looks nothing like its cause, and on a server Docker refuses to remove it at all. */ -export function planPrune(manifests: ImageRecord[], live: string | null, keep: number): PrunePlan { +export function planPrune(manifests: ImageRecord[], inUse: string[], keep: number): PrunePlan { if (!Number.isInteger(keep) || keep < 1) { throw new NextshipError( `--keep must be a whole number of at least 1, not "${keep}".`, @@ -125,8 +126,8 @@ export function planPrune(manifests: ImageRecord[], live: string | null, keep: n const dropped: ImageRecord[] = [] for (const manifest of tagged) { - const isLive = live !== null && manifest.tags.includes(live) - if (isLive || roots.length < keep) roots.push(manifest) + const used = manifest.tags.some((tag) => inUse.includes(tag)) + if (used || roots.length < keep) roots.push(manifest) else dropped.push(manifest) } @@ -178,16 +179,24 @@ async function repository(project: ProjectInfo): Promise { * The tag the app is deployed from, read from the deployment the target reports * as live rather than from any one platform's spec. */ -async function liveTag(app: { target: { deployments: (id: string) => Promise> }; appId: string }): Promise { +/** The live deployment's tag, and the tag still running for older tabs, if any. */ +async function tagsInUse(app: { target: { deployments: (id: string) => Promise }; appId: string }): Promise<{ live: string | null; kept: string | null }> { const deployments = await app.target.deployments(app.appId) - return deployments.find((entry) => entry.live)?.imageTag ?? null + const now = Date.now() + return { + live: deployments.find((entry) => entry.live)?.imageTag ?? null, + kept: deployments.find((entry) => !entry.live && entry.keptUntil && Date.parse(entry.keptUntil) > now)?.imageTag ?? null, + } } +const inUseList = (tags: { live: string | null; kept: string | null }): string[] => + [tags.live, tags.kept].filter((tag): tag is string => tag !== null) + /** Lists the images in this project's repository and what each one is for. */ export async function listImages(project: ProjectInfo): Promise { const app = await ownedApp(project) const repo = await repository(project) - const live = await liveTag(app) + const inUse = await tagsInUse(app) const manifests = await app.target.images(repo) const tagged = manifests.filter((manifest) => manifest.tags.length > 0) @@ -195,14 +204,19 @@ export async function listImages(project: ProjectInfo): Promise { if (tagged.length === 0) detail('No tagged images. Run `nextship deploy` to push one.') for (const manifest of tagged) { - const role = live !== null && manifest.tags.includes(live) ? 'deployed now' : 'kept for rollback' + const role = + inUse.live !== null && manifest.tags.includes(inUse.live) + ? 'deployed now' + : inUse.kept !== null && manifest.tags.includes(inUse.kept) + ? 'kept running for tabs opened before the last deployment' + : 'kept for rollback' detail(`${manifest.tags.join(', ')} ${manifest.updatedAt} ${role}`) } // Untagged manifests are normal: they are the platform images a tagged index // points at. Only those no tag can reach are waste, so they are counted rather // than listed, and identifying them is what `prune` does. - const orphans = planPrune(manifests, live, Number.MAX_SAFE_INTEGER).remove + const orphans = planPrune(manifests, inUseList(inUse), Number.MAX_SAFE_INTEGER).remove if (orphans.length > 0) { detail(`orphaned ${orphans.length} image(s) no tag points to, up to ${megabytes(totalBytes(orphans))}`) } @@ -216,9 +230,9 @@ export async function listImages(project: ProjectInfo): Promise { export async function pruneImages(project: ProjectInfo, options: PruneOptions): Promise { const app = await ownedApp(project) const repo = await repository(project) - const live = await liveTag(app) + const inUse = await tagsInUse(app) const manifests = await app.target.images(repo) - const plan = planPrune(manifests, live, options.keep) + const plan = planPrune(manifests, inUseList(inUse), options.keep) step('Plan') detail(`repository ${repo}`) @@ -235,7 +249,8 @@ export async function pruneImages(project: ProjectInfo, options: PruneOptions): } else { if (plan.removeTags.length > 0) detail(`remove ${plan.removeTags.join(', ')}`) detail(`remove ${plan.remove.length} image(s), up to ${megabytes(plan.reclaimableBytes)}`) - detail(`deployed ${live ?? 'unknown'}, never removed`) + detail(`deployed ${inUse.live ?? 'unknown'}, never removed`) + if (inUse.kept) detail(`kept ${inUse.kept}, running for tabs opened before the last deployment, never removed`) // Only when a tagged deployment is going. Removing orphans costs nothing // that could be rolled back to, and warning about it either way trains // people to ignore the warning that matters. diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 15094f8..a83074b 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -31,6 +31,7 @@ import { DEFAULT_REGION } from './targets/digitalocean-target.js' import { NextshipError } from './errors.js' import { DEFAULT_PLATFORM } from './image/dockerfile.js' import type { BuildPlacement } from './docker.js' +import { takePreview } from './preview.js' import { VERSION } from './version.js' import { detail, fail, ok, step, warn } from './util/log.js' @@ -88,7 +89,8 @@ Deploy options, vm target --build remote builds on the server (default), local builds here and streams the image over SSH. Recorded in nextship.json --memory Container memory limit, such as 512m (default: an even share - of 80% of the server's RAM across its apps) + of 80% of the server's RAM across two containers per app, + the live one and the one kept an hour after a deployment) Rollback options --yes Execute the plan. Without it, rollback only prints the plan @@ -119,6 +121,13 @@ Server add and server move options --no-swap Do not add a swap file on a server under 4 GB of RAM --no-ssh-hardening Leave password logins and root login as they are +Preview options, vm target + --preview Act on the preview instead of the app: a separate app on + the same server named -, with its own env and domains. + Taken by deploy, rollback, logs, env, domain, images and destroy. + \`deploy --preview \` creates it; \`destroy - + --preview \` removes it + Domain options --yes Execute the plan. Without it, domain only prints the plan --primary Make the domain the app's main address rather than an alias @@ -161,6 +170,7 @@ async function main(argv: string[]): Promise { return } + const rest = takePreview(command, argv.slice(1)) switch (command) { case 'detect': return runDetect() @@ -173,21 +183,21 @@ async function main(argv: string[]): Promise { case 'doctor': return runDoctor() case 'deploy': - return runDeploy(argv.slice(1)) + return runDeploy(rest) case 'rollback': - return runRollback(argv.slice(1)) + return runRollback(rest) case 'logs': - return runLogs(argv.slice(1)) + return runLogs(rest) case 'env': - return runEnv(argv.slice(1)) + return runEnv(rest) case 'domain': - return runDomain(argv.slice(1)) + return runDomain(rest) case 'images': - return runImages(argv.slice(1)) + return runImages(rest) case 'destroy': - return runDestroy(argv.slice(1)) + return runDestroy(rest) case 'server': - return runServer(argv.slice(1)) + return runServer(rest) default: throw new NextshipError(`Unknown command \`${command}\`.`, 'Run `nextship --help` to see the available commands.') } @@ -292,6 +302,7 @@ function report(finding: Finding): void { if (finding.level === 'note') detail(line) else warn(line) detail(` ${finding.action}`) + for (const line of finding.lines ?? []) detail(` ${line}`) } /** Reads `--flag value` pairs, so an unknown flag is an error rather than silently ignored. */ diff --git a/packages/cli/src/owned-app.ts b/packages/cli/src/owned-app.ts index c7e51fb..2cb0dba 100644 --- a/packages/cli/src/owned-app.ts +++ b/packages/cli/src/owned-app.ts @@ -23,6 +23,7 @@ import { DigitalOceanTarget } from './targets/digitalocean-target.js' import type { Target } from './targets/target.js' import { VmTarget } from './targets/vm/vm-target.js' import { docsUrl } from './links.js' +import { previewAppName, selectedPreview } from './preview.js' export interface OwnedApp { target: Target @@ -100,7 +101,12 @@ export async function closeTargets(): Promise { await Promise.all(opened.splice(0).map((driver) => driver.close())) } -/** The app recorded for this project, refusing to guess when there is none. */ +/** + * The app recorded for this project, refusing to guess when there is none. With + * `--preview`, the preview of it: found by name on the project's server and accepted + * only when its record names this project's app, so a preview is never adopted by + * name alone either. + */ export async function ownedApp(project: ProjectInfo): Promise { const config = await readConfig(project.root) if (!config?.appId) { @@ -109,5 +115,33 @@ export async function ownedApp(project: ProjectInfo): Promise { 'Run `nextship deploy` first. nextship only acts on apps listed in nextship.json.' ) } - return { target: client(config), appId: config.appId, name: config.name, config } + const preview = selectedPreview() + if (preview === null) return { target: client(config), appId: config.appId, name: config.name, config } + + const scoped = previewConfig(config, preview) + const target = client(scoped) + const app = (await target.listApps()).find((entry) => entry.name === scoped.name) + if (!app || app.previewOf !== config.appId) { + throw new NextshipError( + app ? `An app named "${scoped.name}" exists on the server, and it is not a preview of this project.` : `This project has no preview named "${preview}".`, + app ? 'nextship will not act on an app it does not own. Choose another preview name.' : `Create it with \`nextship deploy --preview ${preview}\`.` + ) + } + return { target, appId: app.id, name: scoped.name, config: scoped } +} + +/** + * The project's settings as they apply to one of its previews: the same server and + * build mode under the preview's own app name, with no app id, since the preview's + * id lives on the server and never in `nextship.json`. + */ +export function previewConfig(config: ProjectConfig, preview: string): ProjectConfig { + if (config.target !== 'vm') { + throw new NextshipError( + 'Previews run on a server, and this project deploys to DigitalOcean App Platform.', + 'Drop --preview. A preview is a second app on the same server, which App Platform has no equivalent of here.' + ) + } + const { appId: _appId, ...rest } = config + return { ...rest, name: previewAppName(config.name, preview) } } diff --git a/packages/cli/src/preview.test.ts b/packages/cli/src/preview.test.ts new file mode 100644 index 0000000..10c82e5 --- /dev/null +++ b/packages/cli/src/preview.test.ts @@ -0,0 +1,67 @@ +/** + * @nextship/cli: tests for preview deployments + * + * A preview name ends up in an app name, a container name and a hostname, and the + * flag scopes every step of a command to another app, so both are pinned: what a + * name may be, and which commands may be pointed at a preview at all. + * + * Author: Ragul D + * Design: ../../../docs/design.md §9.3 + */ + +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { NextshipError } from './errors.js' +import { previewConfig } from './owned-app.js' +import { assertPreviewName, previewAppName, selectedPreview, takePreview } from './preview.js' +import type { ProjectConfig } from './config.js' + +test('a preview is named after the app it previews', () => { + assert.equal(previewAppName('acme-web', 'pr-42'), 'acme-web-pr-42') +}) + +test('a preview name is refused unless an app name, a container name and a hostname all accept it', () => { + for (const good of ['pr-42', 'a', 'feature-login', '0']) assert.equal(assertPreviewName(good), good) + for (const bad of ['', 'PR-42', 'pr_42', '-pr', 'pr-', 'pr.42', 'a'.repeat(33), 'x;rm']) { + assert.throws(() => assertPreviewName(bad), NextshipError, bad) + } +}) + +test('--preview is taken off the arguments and scopes the command', () => { + assert.deepEqual(takePreview('env', ['push', '--preview', 'pr-7', '--yes']), ['push', '--yes']) + assert.equal(selectedPreview(), 'pr-7') + assert.deepEqual(takePreview('deploy', ['--yes']), ['--yes'], 'arguments without it are left as they are') +}) + +// A command that does not act on one app would otherwise ignore the flag and act +// on the server, or on the project's own app, while the user believes it is scoped. +test('--preview is refused by commands that do not act on one app, and without a name', () => { + assert.throws(() => takePreview('server', ['status', '--preview', 'pr-7']), /takes no --preview/) + assert.throws(() => takePreview('detect', ['--preview', 'pr-7']), /takes no --preview/) + assert.throws(() => takePreview('deploy', ['--preview']), /needs a name/) + assert.throws(() => takePreview('deploy', ['--preview', '--yes']), /needs a name/) +}) + +const server = { host: '203.0.113.10', port: 22, user: 'nextship', hostKey: 'k', arch: 'amd64' as const } + +test('a preview runs on the project\'s server under its own name, with no app id of its own in nextship.json', () => { + const config: ProjectConfig = { version: 2, target: 'vm', name: 'acme-web', appId: 'main-id', server, build: 'local' } + assert.deepEqual(previewConfig(config, 'pr-42'), { version: 2, target: 'vm', name: 'acme-web-pr-42', server, build: 'local' }) +}) + +test('a preview is refused on App Platform, which has no second app on the same server', () => { + const config: ProjectConfig = { version: 2, target: 'digitalocean', name: 'acme-web', appId: 'main-id', region: 'blr', registry: 'r' } + assert.throws(() => previewConfig(config, 'pr-42'), /Previews run on a server/) +}) + +// The defect: the name went on to the app name check, whose refusal says to +// rename the app, which is not what the user typed or can change here. +test('a preview name too long for the app says how long it may be, not to rename the app', () => { + const app = 'a'.repeat(55) + assert.equal(previewAppName(app, 'pr-1234').length, 63) + assert.throws( + () => previewAppName(app, 'pr-12345'), + (error: unknown) => error instanceof NextshipError && /64 characters, over the 63/.test(error.message) && /at most 7 characters/.test(error.action) + ) + assert.throws(() => previewAppName('a'.repeat(63), 'x'), (error: unknown) => error instanceof NextshipError && /leaves no room/.test(error.action)) +}) diff --git a/packages/cli/src/preview.ts b/packages/cli/src/preview.ts new file mode 100644 index 0000000..f4dc54b --- /dev/null +++ b/packages/cli/src/preview.ts @@ -0,0 +1,83 @@ +/** + * @nextship/cli: preview deployments + * + * A preview is another app on the project's server, named `-`, with its + * own containers, env file and domains, and it answers only on the domains attached to + * it. What makes it this project's is recorded on the server rather than in + * `nextship.json`: its app record names the app it previews by id. A CI runner that + * checks out the repository has the committed `nextship.json` and nothing else, so a + * preview it deployed for one pull request has to be found again by the run that + * closes that pull request, from the id both runs share. + * + * `--preview ` is taken off the command line before any command parses it, and + * scopes the whole command to that preview. Only commands that act on one app take it. + * + * Author: Ragul D + * Design: ../../../docs/design.md §9.3 + */ + +import { NextshipError } from './errors.js' + +/** The commands that act on one app, and so can act on a preview instead. */ +const PREVIEW_COMMANDS = ['deploy', 'rollback', 'logs', 'env', 'domain', 'images', 'destroy'] + +let selected: string | null = null + +/** + * Removes `--preview ` from a command's arguments and records it for the + * command, refusing it where it has no meaning. + */ +export function takePreview(command: string, argv: string[]): string[] { + const at = argv.indexOf('--preview') + if (at === -1) return argv + const name = argv[at + 1] + if (!name || name.startsWith('--')) { + throw new NextshipError('--preview needs a name.', 'For example `nextship deploy --preview pr-42`.') + } + if (!PREVIEW_COMMANDS.includes(command)) { + throw new NextshipError(`\`nextship ${command}\` does not act on one app, so it takes no --preview.`, 'Drop --preview.') + } + selected = assertPreviewName(name) + return [...argv.slice(0, at), ...argv.slice(at + 2)] +} + +/** The preview this command acts on, or null for the project's own app. */ +export function selectedPreview(): string | null { + return selected +} + +/** + * A preview name becomes part of an app name, a container name and a hostname a user + * may give it, so it is held to what all three accept. + */ +export function assertPreviewName(name: string): string { + if (!/^[a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?$/.test(name)) { + throw new NextshipError( + `"${name}" is not a preview name.`, + 'Use lowercase letters, digits and hyphens, up to 32 characters, starting and ending with a letter or digit, such as pr-42.' + ) + } + return name +} + +/** The longest app name a server accepts (`assertAppName`), which a preview's name has to fit in too. */ +const APP_NAME_LIMIT = 63 + +/** + * The preview's app name. Its length is checked here, because the refusal an app + * name over the limit gets on its own tells the user to rename the app, when what + * they can shorten is the preview name they just typed. + */ +export function previewAppName(appName: string, preview: string): string { + const name = `${appName}-${assertPreviewName(preview)}` + if (name.length > APP_NAME_LIMIT) { + const room = APP_NAME_LIMIT - appName.length - 1 + throw new NextshipError( + `The preview "${preview}" would be the app "${name}", ${name.length} characters, over the ${APP_NAME_LIMIT} an app name may have.`, + room >= 1 + ? `Use a preview name of at most ${room} character${room === 1 ? '' : 's'} for this app.` + : `The app name "${appName}" leaves no room for a preview name. Previews need an app name under ${APP_NAME_LIMIT - 1} characters.` + ) + } + return name +} diff --git a/packages/cli/src/prune.test.ts b/packages/cli/src/prune.test.ts index 24aef0b..276a0f8 100644 --- a/packages/cli/src/prune.test.ts +++ b/packages/cli/src/prune.test.ts @@ -8,7 +8,7 @@ import { test } from 'node:test' import assert from 'node:assert/strict' import { spawnSync } from 'node:child_process' -import { access, lstat, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { access, lstat, mkdir, mkdtemp, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' @@ -16,7 +16,11 @@ import { fileURLToPath } from 'node:url' const PRUNE = fileURLToPath(new URL('../runtime/prune.cjs', import.meta.url)) async function fixture(layout: Record): Promise { - const root = await mkdtemp(path.join(tmpdir(), 'nextship-prune-')) + // The real path, because prune resolves the paths it follows through links and + // compares them with the root it was given. On macOS the temporary directory is + // under /var, a link to /private/var, so a root named through the link made every + // resolved path look outside it. In the image the root is /src, which is not a link. + const root = await realpath(await mkdtemp(path.join(tmpdir(), 'nextship-prune-'))) for (const [relative, content] of Object.entries(layout)) { const file = path.join(root, relative) await mkdir(path.dirname(file), { recursive: true }) diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 0f50204..188b78f 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -32,7 +32,7 @@ import { fingerprint, parseKeyLine, scanHostKey, type HostKey } from './targets/ import { Ssh, assertHost, assertUser, shellQuote } from './targets/vm/ssh.js' import { client } from './owned-app.js' import { CONTAINER_PORT } from './image/dockerfile.js' -import { MOVE_STEPS, rebootProgress, statusWarnings, type VmTarget } from './targets/vm/vm-target.js' +import { MOVE_STEPS, rebootProgress, statusWarnings, writableVolumes, type VmTarget } from './targets/vm/vm-target.js' import { detail, ok, step, warn } from './util/log.js' /** The user every later command connects as. Created by the `user` step. */ @@ -54,6 +54,8 @@ export const SETUP_STEPS = [ 'updates', 'firewall', 'watchdog', + 'cache-guard', + 'retire', 'ssh-hardening', ] as const @@ -198,15 +200,18 @@ function runtimeFile(name: string): string { ) } -/** The setup script with the watchdog it installs, ready to send on stdin. */ +/** The setup script with the timer scripts it installs, ready to send on stdin. */ async function setupInput(): Promise { - const [script, watchdog] = await Promise.all([ + const [script, watchdog, cacheGuard, retire] = await Promise.all([ readFile(runtimeFile('setup.sh'), 'utf8'), readFile(runtimeFile('watchdog.sh'), 'utf8'), + readFile(runtimeFile('cache-guard.sh'), 'utf8'), + readFile(runtimeFile('retire.sh'), 'utf8'), ]) - // An assignment ahead of the script rather than an argument, so the command - // line stays short. Base64 has no characters a shell treats specially. - return `export NEXTSHIP_WATCHDOG_B64=${Buffer.from(watchdog).toString('base64')}\n${script}` + // Assignments ahead of the script rather than arguments, so the command line + // stays short. Base64 has no characters a shell treats specially. + const encode = (text: string) => Buffer.from(text).toString('base64') + return `export NEXTSHIP_WATCHDOG_B64=${encode(watchdog)}\nexport NEXTSHIP_CACHE_GUARD_B64=${encode(cacheGuard)}\nexport NEXTSHIP_RETIRE_B64=${encode(retire)}\n${script}` } /** The version setup.sh records on the server, read from the script this CLI ships. */ @@ -377,6 +382,13 @@ export async function moveServer(project: ProjectInfo, options: AddServerOptions detail(`to ${options.address}`) detail(`copies ${variables} env variable(s), the Server Actions key, ${exported.record.domains.length} domain(s), and image ${exported.live.imageTag}`) detail('not copied deployment history and older images, so rollback starts fresh there; cached pages and optimized images start cold') + if (config.writable && config.writable.length > 0) { + warn( + `What the app wrote in ${config.writable.join(', ')} is not copied. It stays in the volumes ` + + `${writableVolumes(config.name, config.writable).map((entry) => entry.volume).join(', ')} on ${config.server.host}, and the app starts with empty folders on the new server. ` + + 'Copy that data across yourself before sending visitors to the new server.' + ) + } for (const [index, entry] of MOVE_STEPS.entries()) detail(`step ${index + 1} ${entry}`) detail('DNS untouched; the records to change are printed once the app serves on the new server') diff --git a/packages/cli/src/targets/target.ts b/packages/cli/src/targets/target.ts index f3bdfc4..d6c1183 100644 --- a/packages/cli/src/targets/target.ts +++ b/packages/cli/src/targets/target.ts @@ -29,6 +29,8 @@ export interface AppRef { id: string name: string + /** On a server, for a preview: the id of the app it previews, which is what makes it that project's. */ + previewOf?: string } /** @@ -51,6 +53,11 @@ export interface DeploymentRecord { createdAt: string /** The image it runs, which is the nextship deployment id. */ imageTag: string | null + /** + * On a server, while the deployment a newer one replaced keeps running for the + * tabs that loaded it: until when. Absent on targets that keep nothing running. + */ + keptUntil?: string | null } /** How far along a domain is, normalised because every platform names these differently. */ @@ -115,6 +122,8 @@ export interface ReleaseRequest { memory: string | null /** A prerendered route the health check can poll without rendering. */ healthPath?: string | null + /** A server only: set when this release creates a preview, to the id of the app it previews. */ + previewOf?: string } /** Progress reporting, so a driver can narrate a wait without importing the logger. */ diff --git a/packages/cli/src/targets/vm/cache-guard.test.ts b/packages/cli/src/targets/vm/cache-guard.test.ts new file mode 100644 index 0000000..850a4c5 --- /dev/null +++ b/packages/cli/src/targets/vm/cache-guard.test.ts @@ -0,0 +1,110 @@ +import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' + +// The guard runs on the server, which is Linux with GNU find; its `-printf` and +// `-newermt @` have no BSD equivalent, so these run where the server's tools are. +const linuxOnly = { skip: process.platform !== 'linux' } + +const GUARD = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../runtime/vm/cache-guard.sh') + +/** A 64 KiB file whose last access is `accessed` seconds after the epoch and last change `modified`. */ +function entry(file: string, accessed: number, modified: number): void { + mkdirSync(path.dirname(file), { recursive: true }) + writeFileSync(file, Buffer.alloc(64 * 1024, 1)) + execFileSync('touch', ['-m', '-d', `@${modified}`, file]) + execFileSync('touch', ['-a', '-d', `@${accessed}`, file]) +} + +/** Runs `evict ` from the real script; returns " " it removed. */ +function evict(limitKib: number, ...volumes: string[]): string { + // The script is named by $1, not $0: sourced with $0 set to its own path, it would + // take itself to be run by the timer and start guarding this machine. + const script = 'guard="$1"; shift; source "$guard" && evict "$@"' + return execFileSync('bash', ['-c', script, 'cache-guard-test', GUARD, String(limitKib), ...volumes], { encoding: 'utf8' }).trim() +} + +function volume(): string { + return mkdtempSync(path.join(tmpdir(), 'nextship-cache-guard-')) +} + +test('nothing is removed while an app is under its limit', linuxOnly, () => { + const cache = volume() + entry(path.join(cache, 'images/a/1.webp'), 1000, 1000) + entry(path.join(cache, 'images/b/2.webp'), 2000, 2000) + assert.equal(evict(1024, cache), '0 0') + assert.ok(existsSync(path.join(cache, 'images/a/1.webp'))) + rmSync(cache, { recursive: true }) +}) + +test('over the limit, the least recently used go first, down to 80% of it, with their directories', linuxOnly, () => { + const cache = volume() + for (let i = 0; i < 10; i++) entry(path.join(cache, `images/${i}/img.webp`), 1000 + i, 1000) + // Ten 64 KiB entries against a 512 KiB limit: 640 KiB must come down to 409 KiB or + // less, which is four removed, and the four are the oldest by access. + const [files] = evict(512, cache).split(' ') + assert.equal(files, '4') + for (let i = 0; i < 4; i++) assert.ok(!existsSync(path.join(cache, `images/${i}`)), `entry ${i} should be evicted`) + for (let i = 4; i < 10; i++) assert.ok(existsSync(path.join(cache, `images/${i}/img.webp`)), `entry ${i} should be kept`) + rmSync(cache, { recursive: true }) +}) + +// The defect this prevents: a page with no revalidate time losing the file it was +// prerendered into, which Next.js would then have to render at runtime, where the +// data it was built from may not exist. +test('in a build volume, only files changed after it was created can go, however far over the limit', linuxOnly, () => { + const build = volume() + const created = 5000 + entry(path.join(build, 'server/app/about.html'), 100, created - 100) + entry(path.join(build, 'server/app/blog/one.html'), 200, created + 100) + entry(path.join(build, 'server/app/blog/two.html'), 300, created + 200) + const [files] = evict(64, `${build}@${created}`).split(' ') + assert.equal(files, '2') + assert.ok(existsSync(path.join(build, 'server/app/about.html'))) + assert.ok(!existsSync(path.join(build, 'server/app/blog'))) + rmSync(build, { recursive: true }) +}) + +test('an app\'s limit covers its cache and build volumes together', linuxOnly, () => { + const cache = volume() + const build = volume() + entry(path.join(cache, 'fetch-cache/old'), 1000, 6000) + entry(path.join(build, 'server/app/new.html'), 9000, 6000) + entry(path.join(build, 'server/app/newer.html'), 9500, 6000) + // 192 KiB against 128 must come down to 102 KiB or less: the two oldest by access + // go, one from each volume, and the newest stays. + assert.equal(evict(128, cache, `${build}@5000`).split(' ')[0], '2') + assert.ok(!existsSync(path.join(cache, 'fetch-cache/old'))) + assert.ok(existsSync(path.join(build, 'server/app/newer.html'))) + rmSync(cache, { recursive: true }) + rmSync(build, { recursive: true }) +}) + +// The defect: sorted by access time alone, a volume mounted `noatime` never updates +// one, so every file tied at its creation and the guard evicted in no order at +// all, the busiest entries included. +test('where access times are not kept, the oldest written goes first', linuxOnly, () => { + const cache = volume() + // Every access time is the same, as it is when the kernel never updates them. + entry(path.join(cache, 'images/old/img.webp'), 100, 5000) + entry(path.join(cache, 'images/mid/img.webp'), 100, 7000) + entry(path.join(cache, 'images/new/img.webp'), 100, 9000) + assert.equal(evict(128, cache).split(' ')[0], '2') + assert.ok(existsSync(path.join(cache, 'images/new/img.webp')), 'the newest written is kept') + assert.ok(!existsSync(path.join(cache, 'images/old')) && !existsSync(path.join(cache, 'images/mid'))) + rmSync(cache, { recursive: true }) +}) + +test('a file read recently is kept over one written more recently but not read since', linuxOnly, () => { + const cache = volume() + entry(path.join(cache, 'images/read/img.webp'), 9000, 1000) + entry(path.join(cache, 'images/written/img.webp'), 100, 5000) + entry(path.join(cache, 'images/idle/img.webp'), 100, 2000) + assert.equal(evict(128, cache).split(' ')[0], '2') + assert.ok(existsSync(path.join(cache, 'images/read/img.webp'))) + rmSync(cache, { recursive: true }) +}) diff --git a/packages/cli/src/targets/vm/caddy.test.ts b/packages/cli/src/targets/vm/caddy.test.ts index ee7e925..8aab9b4 100644 --- a/packages/cli/src/targets/vm/caddy.test.ts +++ b/packages/cli/src/targets/vm/caddy.test.ts @@ -12,13 +12,14 @@ import { test } from 'node:test' import assert from 'node:assert/strict' import { renderSite } from './caddy.js' -const base = { name: 'demo', container: 'demo-r20260915-100000-abcdef', domains: [], isDefault: false } +const base = { name: 'demo', container: 'demo-r20260915-100000-abcdef', domains: [], isDefault: false, previous: null, preview: false, ipCertificate: null } test('the default app with no domain answers plain HTTP on port 80, streaming unbuffered', () => { assert.equal( renderSite({ ...base, isDefault: true }), [ '# Written by nextship for demo. Regenerated on every deployment; edits are overwritten.', + '# live demo-r20260915-100000-abcdef', 'http://:80 {', ' reverse_proxy demo-r20260915-100000-abcdef:3000 {', ' flush_interval -1', @@ -60,3 +61,68 @@ test('a name that could break out of the site file is refused', () => { assert.throws(() => renderSite({ ...base, domains: [{ domain: 'x.com {\n}', primary: true, minimumTls: '1.2' }] })) assert.throws(() => renderSite({ ...base, name: 'demo {' })) }) + +// Next.js names the build on every request an open tab makes: the header on +// navigations and Server Actions, the query on JavaScript and CSS. A page load +// names none and must get the live build. +test('while a previous deployment runs, requests naming its build go to it first, and the rest to the live one', () => { + const site = renderSite({ + ...base, + isDefault: true, + domains: [{ domain: 'app.example.com', primary: true, minimumTls: '1.2' }], + previous: { container: 'demo-r20260915-090000-123456', deploymentId: 'dpl-aaa-111' }, + }) + const route = ' @previous expression `{header.X-Deployment-Id} == "dpl-aaa-111" || {query.dpl} == "dpl-aaa-111"`' + assert.equal(site.split('\n').filter((line) => line === route).length, 2, 'both the address and the domain route by build') + assert.match( + site, + /handle @previous \{\n reverse_proxy demo-r20260915-090000-123456:3000 demo-r20260915-100000-abcdef:3000 \{\n lb_policy first\n fail_duration 30s\n flush_interval -1/ + ) + assert.match(site, / handle \{\n reverse_proxy demo-r20260915-100000-abcdef:3000 \{\n flush_interval -1/) +}) + +test('a deployment id that could break out of the expression is refused', () => { + assert.throws(() => renderSite({ ...base, isDefault: true, previous: { container: 'demo-r1', deploymentId: 'dpl" || true || "' } })) +}) + +// A preview is the site under another name, so a crawler that finds it would index +// a copy. Crawler traffic to preview URLs is also what has run up hosting bills. +test('a preview tells crawlers not to index it, on every name it answers', () => { + const site = renderSite({ + ...base, + preview: true, + domains: [ + { domain: 'pr-42.preview.example.com', primary: true, minimumTls: '1.2' }, + { domain: 'secure-pr-42.preview.example.com', primary: false, minimumTls: '1.3' }, + ], + }) + assert.equal(site.match(/ header X-Robots-Tag "noindex, nofollow"/g)?.length, 2) + assert.doesNotMatch(renderSite({ ...base, domains: [{ domain: 'app.example.com', primary: true, minimumTls: '1.2' }] }), /X-Robots-Tag/) +}) + +// Let's Encrypt certifies an IP address only with its six-day profile, and a first +// deployment should be reachable over HTTPS before any domain exists. +test('a default app on a public IPv4 address also answers HTTPS on it, with a short-lived certificate', () => { + const site = renderSite({ ...base, isDefault: true, ipCertificate: '46.101.1.2' }) + assert.match(site, /^http:\/\/:80 \{/m, 'plain HTTP keeps working') + assert.match(site, /^http:\/\/46\.101\.1\.2, https:\/\/46\.101\.1\.2 \{\n tls \{\n issuer acme \{\n profile shortlived\n \}\n \}\n reverse_proxy demo-r20260915-100000-abcdef:3000 \{/m) + assert.doesNotMatch(renderSite({ ...base, isDefault: false, ipCertificate: '46.101.1.2' }), /46\.101/, 'only the default app answers on the address') + assert.throws(() => renderSite({ ...base, isDefault: true, ipCertificate: '46.101.1.2 {' }), /not an IPv4 address/) +}) + +// The defect: with only `https://
`, Caddy redirected plain HTTP on the +// address to HTTPS before a certificate existed, so the one address a first +// deployment printed led to a certificate error for as long as issuance took, or +// for good where Let's Encrypt could not reach the server. +test('the address is named for plain HTTP too, so Caddy serves it rather than redirecting to a certificate that may not exist', () => { + const site = renderSite({ ...base, isDefault: true, ipCertificate: '46.101.1.2' }) + assert.doesNotMatch(site, /^https:\/\/46\.101\.1\.2 \{/m) + assert.match(site, /^http:\/\/46\.101\.1\.2, https:\/\/46\.101\.1\.2 \{/m) +}) + +// retire.sh refuses to stop the container this line names, whatever `previous` says. +test('every site names its live container on a line of its own, with or without an address', () => { + assert.match(renderSite(base), /^# live demo-r20260915-100000-abcdef$/m) + const kept = renderSite({ ...base, isDefault: true, previous: { container: 'demo-r20260915-090000-123456', deploymentId: 'dpl-aaa-111' } }) + assert.deepEqual(kept.match(/^# live .*$/gm), ['# live demo-r20260915-100000-abcdef'], 'only the live one, never the kept one') +}) diff --git a/packages/cli/src/targets/vm/caddy.ts b/packages/cli/src/targets/vm/caddy.ts index da8d7ac..49b139d 100644 --- a/packages/cli/src/targets/vm/caddy.ts +++ b/packages/cli/src/targets/vm/caddy.ts @@ -14,12 +14,32 @@ * refuses it, which covers the instant between Caddy moving to a new container * and that container accepting a connection it has not seen before. * + * For an hour after a deployment the previous one keeps running, and a request + * that names its build goes to it. Next.js names the build on every request an open + * tab makes: `x-deployment-id` on client navigations and Server Actions, and + * `?dpl=` on every JavaScript and CSS file. Without this, a tab opened before the + * deployment asks the new container for files and Server Actions that exist only + * in the build it loaded, and fails. A page load names no build, so it gets the + * new one. Once the previous container has stopped, its route fails over to the + * live container, which is what every request got before this existed. + * + * The server's default app also answers `https://
` when the server was added + * by a public IPv4 address, with a certificate Let's Encrypt issues for the address + * itself, so a first deployment is reachable over HTTPS before any domain exists. Let's + * Encrypt issues IP address certificates only with its `shortlived` profile, valid for + * about six days, which Caddy renews on its own. Caddy answers the ACME challenge before + * any site route, so the plain HTTP site in front of the app does not intercept it. + * Plain HTTP on the address keeps serving the app whether or not a certificate comes. + * IPv4 only: IPv6 issuance was fixed after the Caddy release setup installs. + * * Author: Ragul D * Design: ../../../../../docs/design.md §9.3 */ +import { isIPv4 } from 'node:net' +import { NextshipError } from '../../errors.js' import { CONTAINER_PORT } from '../../image/dockerfile.js' -import { assertAppName, assertDomain } from './ssh.js' +import { assertAppName, assertDeploymentId, assertDomain } from './ssh.js' export interface SiteDomain { domain: string @@ -34,21 +54,77 @@ interface SiteOptions { domains: SiteDomain[] /** The first app deployed on a server answers plain HTTP on its address. */ isDefault: boolean + /** The deployment before the live one, while it still runs for the tabs that loaded it. */ + previous: { container: string; deploymentId: string } | null + /** A preview asks crawlers not to index it: it is a copy of the site under another name. */ + preview: boolean + /** The public IPv4 address the default app also answers over HTTPS, or null. */ + ipCertificate: string | null } -const proxy = (container: string): string[] => [ - ` reverse_proxy ${container}:${CONTAINER_PORT} {`, +const proxy = (upstreams: string[], extra: string[] = []): string[] => [ + ` reverse_proxy ${upstreams.map((container) => `${container}:${CONTAINER_PORT}`).join(' ')} {`, + ...extra, ' flush_interval -1', ' lb_try_duration 5s', ' }', ] +const indent = (lines: string[]): string[] => lines.map((line) => ` ${line}`) + +function routes(options: SiteOptions): string[] { + const noindex = options.preview ? [' header X-Robots-Tag "noindex, nofollow"'] : [] + return [...noindex, ...upstreams(options)] +} + +function upstreams(options: SiteOptions): string[] { + if (options.previous === null) return proxy([options.container]) + const id = assertDeploymentId(options.previous.deploymentId) + return [ + ` @previous expression \`{header.X-Deployment-Id} == "${id}" || {query.dpl} == "${id}"\``, + ' handle @previous {', + // `first` sends to the previous container while it answers. A failed dial marks + // it down for 30 s and the retry goes to the live one, so a request that names a + // build no longer running is served as it would be with no route at all. + ...indent(proxy([options.previous.container, options.container], [' lb_policy first', ' fail_duration 30s'])), + ' }', + ' handle {', + ...indent(proxy([options.container])), + ' }', + ] +} + export function renderSite(options: SiteOptions): string { assertAppName(options.name) - const lines = [`# Written by nextship for ${options.name}. Regenerated on every deployment; edits are overwritten.`] + const lines = [ + `# Written by nextship for ${options.name}. Regenerated on every deployment; edits are overwritten.`, + // Read by retire.sh, which stops nothing this line names: the site file is what + // Caddy serves from, so it is the one record of the live container that cannot + // disagree with what is serving. + `# live ${options.container}`, + ] if (options.isDefault) { - lines.push('http://:80 {', ...proxy(options.container), '}') + lines.push('http://:80 {', ...routes(options), '}') + if (options.ipCertificate !== null) { + if (!isIPv4(options.ipCertificate)) throw new NextshipError(`"${options.ipCertificate}" is not an IPv4 address.`, 'This is a nextship defect. Please report it.') + lines.push( + // Both schemes, named. With `https://` alone, Caddy redirects plain HTTP for + // this host to HTTPS from the moment it starts, before any certificate + // exists and for good if Let's Encrypt never issues one, and that + // host-specific redirect wins over the catch-all site above. Measured with a + // CA that could not be reached: `http://
` answered 308 to an address + // with no certificate. Naming the HTTP site makes Caddy serve it instead. + `http://${options.ipCertificate}, https://${options.ipCertificate} {`, + ' tls {', + ' issuer acme {', + ' profile shortlived', + ' }', + ' }', + ...routes(options), + '}' + ) + } } // Caddy applies a block's tls settings to every name in it, so domains are @@ -59,7 +135,7 @@ export function renderSite(options: SiteOptions): string { .sort((a, b) => Number(b.primary) - Number(a.primary)) .map((entry) => assertDomain(entry.domain)) if (names.length === 0) continue - lines.push(`${names.join(', ')} {`, ' tls {', ` protocols tls${minimum}`, ' }', ...proxy(options.container), '}') + lines.push(`${names.join(', ')} {`, ' tls {', ` protocols tls${minimum}`, ' }', ...routes(options), '}') } return `${lines.join('\n')}\n` diff --git a/packages/cli/src/targets/vm/deployments.test.ts b/packages/cli/src/targets/vm/deployments.test.ts index 1dda956..b624315 100644 --- a/packages/cli/src/targets/vm/deployments.test.ts +++ b/packages/cli/src/targets/vm/deployments.test.ts @@ -16,6 +16,8 @@ import { imagesToKeep, newReleaseId, parseDeployments, + PREVIOUS_KEPT_MS, + previousServing, recordFailed, recordLive, recordStarted, @@ -23,18 +25,19 @@ import { } from './deployments.js' const at = '2026-09-15T10:00:00.000Z' +const now = new Date(at) test('a first deployment goes from started to live, and becomes a rollback target', () => { let history = recordStarted([], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null }) - assert.deepEqual(history[0], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null, served: false, live: false }) - history = recordLive(history, 'r1') + assert.deepEqual(history[0], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null, served: false, live: false, keptUntil: null }) + history = recordLive(history, 'r1', now, true) assert.equal(history[0].live, true) assert.equal(history[0].served, true) }) test('an update moves live to the new deployment and keeps the old one served', () => { - let history = recordLive(recordStarted([], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null }), 'r1') - history = recordLive(recordStarted(history, { id: 'r2', imageTag: 'dpl-b', cause: 'deploy', createdAt: at, healthPath: null }), 'r2') + let history = recordLive(recordStarted([], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null }), 'r1', now, true) + history = recordLive(recordStarted(history, { id: 'r2', imageTag: 'dpl-b', cause: 'deploy', createdAt: at, healthPath: null }), 'r2', now, true) assert.deepEqual(history.map((entry) => [entry.id, entry.live, entry.served]), [['r2', true, true], ['r1', false, true]]) assert.deepEqual(rollbackCandidates(history, 'r2').map((entry) => entry.id), ['r1']) }) @@ -42,7 +45,7 @@ test('an update moves live to the new deployment and keeps the old one served', // A failed deployment must not take live away from the one still serving, or // rollback would report nothing live while traffic is flowing. test('a failed health check leaves the previous deployment live and is never a rollback target', () => { - let history = recordLive(recordStarted([], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null }), 'r1') + let history = recordLive(recordStarted([], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null }), 'r1', now, true) history = recordFailed(recordStarted(history, { id: 'r2', imageTag: 'dpl-b', cause: 'deploy', createdAt: at, healthPath: null }), 'r2', 'failed health check') assert.equal(history.find((entry) => entry.live)?.id, 'r1') assert.equal(history[0].cause, 'deploy, failed health check') @@ -50,31 +53,31 @@ test('a failed health check leaves the previous deployment live and is never a r }) test('a rollback is a new deployment of an old image, recorded with its cause', () => { - let history = recordLive(recordStarted([], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null }), 'r1') - history = recordLive(recordStarted(history, { id: 'r2', imageTag: 'dpl-b', cause: 'deploy', createdAt: at, healthPath: null }), 'r2') - history = recordLive(recordStarted(history, { id: 'r3', imageTag: 'dpl-a', cause: 'rollback to r1', createdAt: at, healthPath: null }), 'r3') + let history = recordLive(recordStarted([], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null }), 'r1', now, true) + history = recordLive(recordStarted(history, { id: 'r2', imageTag: 'dpl-b', cause: 'deploy', createdAt: at, healthPath: null }), 'r2', now, true) + history = recordLive(recordStarted(history, { id: 'r3', imageTag: 'dpl-a', cause: 'rollback to r1', createdAt: at, healthPath: null }), 'r3', now, true) assert.deepEqual(history.map((entry) => entry.live), [true, false, false]) assert.equal(history[0].imageTag, 'dpl-a') }) test('marking an unknown deployment live is a defect, not a silent no-op', () => { - assert.throws(() => recordLive([], 'r9'), /not in the history/) + assert.throws(() => recordLive([], 'r9', now, true), /not in the history/) }) test('the history round-trips, and a damaged file is refused rather than replaced', () => { - const history = recordLive(recordStarted([], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null }), 'r1') + const history = recordLive(recordStarted([], { id: 'r1', imageTag: 'dpl-a', cause: 'deploy', createdAt: at, healthPath: null }), 'r1', now, true) assert.deepEqual(parseDeployments(serializeDeployments(history)), history) assert.deepEqual(parseDeployments(''), []) assert.throws(() => parseDeployments('{'), /not valid JSON/) assert.throws(() => parseDeployments('{}'), /not a list/) assert.throws(() => parseDeployments('[{"id":"r1"}]'), /unreadable entry at position 1/) - // A history written before healthPath was recorded still reads. - const { healthPath: _omitted, ...older } = history[0] + // A history written before healthPath and keptUntil were recorded still reads. + const { healthPath: _health, keptUntil: _kept, ...older } = history[0] assert.equal(parseDeployments(JSON.stringify([older]))[0].healthPath, null) + assert.equal(parseDeployments(JSON.stringify([older]))[0].keptUntil, null) }) test('release ids are unique per release even for the same image, and valid container name suffixes', () => { - const now = new Date(at) const a = newReleaseId(now) assert.notEqual(a, newReleaseId(now)) assert.match(a, /^r20260915-100000-[0-9a-f]{6}$/) @@ -83,13 +86,61 @@ test('release ids are unique per release even for the same image, and valid cont test('images kept are the live one and the newest served ones, never a failed one', () => { let history: ReturnType = [] for (const [id, tag] of [['r1', 'a'], ['r2', 'b'], ['r3', 'c'], ['r4', 'd']]) { - history = recordLive(recordStarted(history, { id, imageTag: tag, cause: 'deploy', createdAt: at, healthPath: null }), id) + history = recordLive(recordStarted(history, { id, imageTag: tag, cause: 'deploy', createdAt: at, healthPath: null }), id, now, true) } history = recordFailed(recordStarted(history, { id: 'r5', imageTag: 'broken', cause: 'deploy', createdAt: at, healthPath: null }), 'r5', 'failed') assert.deepEqual([...imagesToKeep(history, 2)].sort(), ['c', 'd']) - // Rolled back to the oldest: it is live, so it is kept even beyond the count. - history = recordLive(recordStarted(history, { id: 'r6', imageTag: 'a', cause: 'rollback to r1', createdAt: at, healthPath: null }), 'r6') - assert.deepEqual([...imagesToKeep(history, 1)], ['a']) + // Rolled back to the oldest: it is live, so it is kept even beyond the count, and + // so is the one it replaced, whose container keeps running for its tabs. + history = recordLive(recordStarted(history, { id: 'r6', imageTag: 'a', cause: 'rollback to r1', createdAt: at, healthPath: null }), 'r6', now, true) + assert.deepEqual([...imagesToKeep(history, 1)].sort(), ['a', 'd']) assert.ok(!imagesToKeep(history, 10).has('broken')) }) + +const start = (history: ReturnType, id: string, imageTag: string) => + recordStarted(history, { id, imageTag, cause: 'deploy', createdAt: at, healthPath: null }) + +// The defect this prevents: a tab loaded before a deployment asking the new +// container for its build's files and Server Actions, which only the old one has. +test('a new build keeps the one it replaced for an hour, and no other', () => { + let history = recordLive(start([], 'r1', 'dpl-a'), 'r1', now, true) + history = recordLive(start(history, 'r2', 'dpl-b'), 'r2', now, true) + assert.equal(previousServing(history, now)?.id, 'r1') + assert.equal(history.find((entry) => entry.id === 'r1')?.keptUntil, new Date(now.getTime() + PREVIOUS_KEPT_MS).toISOString()) + history = recordLive(start(history, 'r3', 'dpl-c'), 'r3', now, true) + assert.deepEqual(history.filter((entry) => entry.keptUntil !== null).map((entry) => entry.id), ['r2']) + assert.equal(previousServing(history, new Date(now.getTime() + PREVIOUS_KEPT_MS + 1)), null, 'nothing is kept once its hour is over') +}) + +// Both containers would answer to one deployment id, so routing by it would send +// every tab, new ones too, to the container being replaced. +test('the same build started again is not kept, and does not end a keep already running', () => { + let history = recordLive(start([], 'r1', 'dpl-a'), 'r1', now, true) + history = recordLive(start(history, 'r2', 'dpl-b'), 'r2', now, true) + const later = new Date(now.getTime() + 60_000) + history = recordLive(start(history, 'r3', 'dpl-b'), 'r3', later, true) + assert.equal(previousServing(history, later)?.id, 'r1') + assert.equal(history.find((entry) => entry.id === 'r1')?.keptUntil, new Date(now.getTime() + PREVIOUS_KEPT_MS).toISOString()) +}) + +// An instant rollback: the kept deployment becomes live again without a new +// container, and the one it replaces is kept in its place. +test('going back to the kept deployment makes it live and keeps the one it replaces', () => { + let history = recordLive(start([], 'r1', 'dpl-a'), 'r1', now, true) + history = recordLive(start(history, 'r2', 'dpl-b'), 'r2', now, true) + history = recordLive(history, 'r1', now, true) + assert.equal(history.find((entry) => entry.live)?.id, 'r1') + assert.equal(previousServing(history, now)?.id, 'r2') + assert.equal(history.filter((entry) => entry.keptUntil !== null).length, 1) +}) + +// A server set up before the retire timer existed has nothing to stop a kept +// container, which would run with its whole memory limit until the next deployment. +test('on a server with no retire timer, a new build keeps nothing', () => { + let history = recordLive(start([], 'r1', 'dpl-a'), 'r1', now, false) + history = recordLive(start(history, 'r2', 'dpl-b'), 'r2', now, false) + assert.equal(previousServing(history, now), null) + assert.ok(history.every((entry) => entry.keptUntil === null)) + assert.equal(history.find((entry) => entry.live)?.id, 'r2') +}) diff --git a/packages/cli/src/targets/vm/deployments.ts b/packages/cli/src/targets/vm/deployments.ts index 8ba6e87..40338e3 100644 --- a/packages/cli/src/targets/vm/deployments.ts +++ b/packages/cli/src/targets/vm/deployments.ts @@ -36,8 +36,21 @@ export function newReleaseId(now: Date): string { */ export interface VmDeployment extends DeploymentRecord { healthPath: string | null + /** + * Set on the deployment a new one replaced: its container keeps running until + * then, for the tabs that loaded it (`caddy.ts`). At most one entry has it. + */ + keptUntil: string | null } +/** + * How long the deployment before the live one keeps running. An hour covers a tab + * left open through a deployment for the length of an ordinary working session, at + * the cost of a second container's memory for that hour. A tab older than that + * reloads the page on its next navigation, as it would with nothing kept. + */ +export const PREVIOUS_KEPT_MS = 60 * 60 * 1000 + /** Reads the file, refusing one that is not a history rather than starting a new one over it. */ export function parseDeployments(contents: string): VmDeployment[] { if (contents.trim() === '') return [] @@ -57,7 +70,8 @@ export function parseDeployments(contents: string): VmDeployment[] { typeof record.cause !== 'string' || typeof record.createdAt !== 'string' || !(typeof record.imageTag === 'string' || record.imageTag === null) || - !(record.healthPath === undefined || record.healthPath === null || typeof record.healthPath === 'string') + !(record.healthPath === undefined || record.healthPath === null || typeof record.healthPath === 'string') || + !(record.keptUntil === undefined || record.keptUntil === null || typeof record.keptUntil === 'string') ) { throw corrupt(`has an unreadable entry at position ${index + 1}`) } @@ -69,6 +83,7 @@ export function parseDeployments(contents: string): VmDeployment[] { createdAt: record.createdAt, imageTag: record.imageTag, healthPath: record.healthPath ?? null, + keptUntil: record.keptUntil ?? null, } }) } @@ -88,19 +103,49 @@ export function recordStarted( deployments: VmDeployment[], entry: { id: string; imageTag: string; cause: string; createdAt: string; healthPath: string | null } ): VmDeployment[] { - return [{ ...entry, served: false, live: false }, ...deployments] + return [{ ...entry, served: false, live: false, keptUntil: null }, ...deployments] } /** * The deployment took traffic. It becomes the only live one, and stays a valid * rollback target from now on. + * + * The deployment it replaced is kept for PREVIOUS_KEPT_MS when it is a different + * build. The same build started again, by `env push` or a domain change, is not: + * both would answer to one deployment id, and routing by it would send every tab, + * new ones included, to the container being replaced. A deployment already kept + * stays kept until its time runs out, unless it is the build going live. + * + * `canKeep` is false on a server with no retire timer, which keeps nothing: a kept + * container there would never be stopped. */ -export function recordLive(deployments: VmDeployment[], id: string): VmDeployment[] { - if (!deployments.some((entry) => entry.id === id)) { +export function recordLive(deployments: VmDeployment[], id: string, now: Date, canKeep: boolean): VmDeployment[] { + const next = deployments.find((entry) => entry.id === id) + if (!next) { throw new NextshipError(`Deployment ${id} is not in the history.`, 'This is a nextship defect. Please report it.') } - return deployments.map((entry) => - entry.id === id ? { ...entry, served: true, live: true } : { ...entry, live: false } + const replaced = deployments.find((entry) => entry.live && entry.id !== id) + const existing = previousServing(deployments, now) + let kept: { id: string; until: string } | null = null + if (!canKeep) { + kept = null + } else if (replaced && replaced.imageTag !== next.imageTag) { + kept = { id: replaced.id, until: new Date(now.getTime() + PREVIOUS_KEPT_MS).toISOString() } + } else if (existing && existing.id !== id && existing.imageTag !== next.imageTag && existing.keptUntil) { + kept = { id: existing.id, until: existing.keptUntil } + } + return deployments.map((entry) => { + if (entry.id === id) return { ...entry, served: true, live: true, keptUntil: null } + return { ...entry, live: false, keptUntil: entry.id === kept?.id ? kept.until : null } + }) +} + +/** The deployment kept running for the tabs that loaded it, if its time has not run out. */ +export function previousServing(deployments: VmDeployment[], now: Date): VmDeployment | null { + return ( + deployments.find( + (entry) => !entry.live && entry.served && entry.keptUntil !== null && Date.parse(entry.keptUntil) > now.getTime() + ) ?? null ) } @@ -121,6 +166,8 @@ export function imagesToKeep(deployments: DeploymentRecord[], keep: number): Set const kept = new Set() const live = deployments.find((entry) => entry.live)?.imageTag if (live) kept.add(live) + // Its container still runs, and Docker will not remove an image a container uses. + for (const entry of deployments) if (entry.keptUntil && entry.imageTag) kept.add(entry.imageTag) for (const entry of deployments) { if (kept.size >= keep) break if (entry.served && entry.imageTag) kept.add(entry.imageTag) diff --git a/packages/cli/src/targets/vm/vm-release.test.ts b/packages/cli/src/targets/vm/vm-release.test.ts index 047b276..2c39b21 100644 --- a/packages/cli/src/targets/vm/vm-release.test.ts +++ b/packages/cli/src/targets/vm/vm-release.test.ts @@ -139,11 +139,11 @@ const APP_RECORD = JSON.stringify({ id: 'app-uuid', name: 'acme-web', createdAt: */ function server( shell: FakeShell, - options: { deployments?: unknown[]; defaultApp?: string } = {} + options: { deployments?: unknown[]; defaultApp?: string; setupVersion?: number } = {} ): FakeShell { - const { deployments = [], defaultApp = '' } = options + const { deployments = [], defaultApp = '', setupVersion = 3 } = options return shell - .answer(/MemTotal/, `4096\n20480\n27.0.0\n${defaultApp}\nno\n`) + .answer(/MemTotal/, `4096\n20480\n27.0.0\n${defaultApp}\nno\n${setupVersion}\n`) .answer(/cat .*app\.json/, APP_RECORD) .answer(/cat .*deployments\.json/, JSON.stringify(deployments)) .answer(/cat .*\/env/, '') @@ -438,3 +438,88 @@ test('only a clock reading reaches the remote command', async () => { await assert.rejects(new TestVmTarget(CONFIG, shell).buildSessionLost('1790229580; rm -rf /'), NextshipError) assert.equal(shell.commands.length, 0, 'nothing was sent') }) + +// ------------------------------------------------- the kept deployment + +const LIVE_OLD = { + id: 'r20260101-000000-aaaaaa', + imageTag: 'old111', + cause: 'deploy', + createdAt: '2026-01-01T00:00:00Z', + healthPath: '/', + served: true, + live: true, + keptUntil: null, +} +const OLD_CONTAINER = `acme-web-${LIVE_OLD.id}` + +/** A server serving one deployment of another build, whose container is running. */ +const serving = (options: { setupVersion?: number } = {}): FakeShell => + server(new FakeShell(), { deployments: [LIVE_OLD], ...options }) + .answer(/State\.Status/, 'running healthy') + .answer(/docker ps .*status=running/, `${OLD_CONTAINER}\n`) + +// The defect: `previous` still named a container while Caddy was being pointed at +// it, or was left naming it by a CLI that died before the write after the switch. +// The retire timer then stopped the live app when the hour ran out, and Docker's +// restart policy does not bring back a container that was stopped. +test('the retire timer is given nothing to stop before Caddy is switched, and its container again only after', async () => { + const shell = serving() + await deploy(shell) + + shell.before(/: > .*\/previous/, /caddy reload/, 'previous must be empty before the switch') + const reload = shell.at(/caddy reload/) + const written = shell.at(/previous\.next/, reload) + assert.notEqual(written, -1, 'previous is written again after the switch') + assert.match(shell.commands[written], new RegExp(`'${OLD_CONTAINER} \\d+\\n'`), 'it names the replaced container and when it stops') + // One command, so the history and previous cannot be left disagreeing. + assert.match(shell.commands[written], /deployments\.json\.next/) + assert.ok(!shell.commands.some((command) => new RegExp(`docker stop .*${OLD_CONTAINER}`).test(command)), 'the replaced build keeps running') +}) + +// The defect: a server set up by 1.1.x has no retire timer, so the replaced +// container ran, with its whole memory limit, until the next deployment. +test('a server set up before the retire timer keeps nothing, stops the replaced container, and says why', async () => { + const shell = serving({ setupVersion: 2 }) + const { target, releaseId } = await start(shell) + const phases: string[] = [] + await target.awaitRelease('app-uuid', releaseId, (phase) => phases.push(phase), true) + + assert.ok(shell.deployments().every((entry) => !(entry as { keptUntil?: string | null }).keptUntil), 'nothing is recorded as kept') + assert.ok(shell.commands.some((command) => command.includes('docker stop -t 30') && command.includes(OLD_CONTAINER)), 'the replaced container is stopped') + assert.ok(phases.some((phase) => /nextship server add/.test(phase)), `the reason and the remedy are reported: ${phases.join(' | ')}`) + const site = shell.stdins[shell.at(/caddy validate/)] ?? '' + assert.doesNotMatch(site, /@previous/, 'no route to a container that is about to stop') +}) + +const KEPT_OLD = { ...LIVE_OLD, live: false, keptUntil: '2999-01-01T00:00:00.000Z' } +const LIVE_NEW = { ...LIVE_OLD, id: 'r20260102-000000-bbbbbb', imageTag: 'new222', live: true } + +// The defect: the history and the container were read, then the lock taken, and +// neither read again. A deployment that went live in between had stopped the +// container, and the rollback pointed Caddy at it. +test('an instant rollback reads the history and the container only once it holds the lock', async () => { + const shell = server(new FakeShell(), { deployments: [LIVE_NEW, KEPT_OLD] }).answer(/State\.Status/, 'running healthy') + await new TestVmTarget(CONFIG, shell).rollback('app-uuid', KEPT_OLD.id, () => {}) + + // rollback reads the history once to find its target. What decides the switch is + // read again once the lock is held. + const locked = shell.at(/mkdir .*\/lock/) + const reload = shell.at(/caddy reload/) + const reread = shell.at(/cat .*deployments\.json/, locked) + assert.ok(locked !== -1 && reread > locked && reread < reload, `the history is read again under the lock, before the switch\n${shell.commands.join('\n')}`) + shell.before(/mkdir .*\/lock/, /State\.Status/, 'the lock comes before the container is inspected') + shell.before(/: > .*\/previous/, /caddy reload/, 'previous is emptied before Caddy points at the kept container') + assert.equal(shell.at(/'docker' 'run'/), -1, 'no new container is started') + assert.equal(shell.deployments().find((entry) => entry.live)?.id, KEPT_OLD.id) +}) + +test('a kept container that is no longer healthy is not switched to: its image is started again instead', async () => { + const shell = server(new FakeShell(), { deployments: [LIVE_NEW, KEPT_OLD] }) + .answer(/State\.Status/, 'running healthy') + .answer(new RegExp(`State\\.Status.*${KEPT_OLD.id}`), 'exited ') + await new TestVmTarget(CONFIG, shell).rollback('app-uuid', KEPT_OLD.id, () => {}) + + assert.notEqual(shell.at(/'docker' 'run'/), -1, `a new container of that image is started\n${shell.commands.join('\n')}`) + assert.notEqual(shell.deployments().find((entry) => entry.live)?.id, KEPT_OLD.id, 'the stopped container is not what goes live') +}) diff --git a/packages/cli/src/targets/vm/vm-target.test.ts b/packages/cli/src/targets/vm/vm-target.test.ts index d3a6339..f0b5da3 100644 --- a/packages/cli/src/targets/vm/vm-target.test.ts +++ b/packages/cli/src/targets/vm/vm-target.test.ts @@ -27,13 +27,17 @@ import { runArguments, withDomain, withoutDomain, + publicIpv4, + writableVolumes, } from './vm-target.js' -const facts = (overrides = {}) => ({ memMib: 3900, diskFreeMib: 20000, dockerVersion: '27.3.1', defaultApp: null, ipv6: false, ...overrides }) +const facts = (overrides = {}) => ({ memMib: 3900, diskFreeMib: 20000, dockerVersion: '27.3.1', defaultApp: null, ipv6: false, setupVersion: 3, ...overrides }) const value = (args: string[], flag: string): string[] => args.flatMap((arg, index) => (args[index - 1] === flag ? [arg] : [])) test('the facts a server reports in one round trip are read, and a partial answer is refused', () => { - assert.deepEqual(parseFacts('3900\n20480\n27.3.1\nshop\nyes\n'), { memMib: 3900, diskFreeMib: 20480, dockerVersion: '27.3.1', defaultApp: 'shop', ipv6: true }) + assert.deepEqual(parseFacts('3900\n20480\n27.3.1\nshop\nyes\n3\n'), { memMib: 3900, diskFreeMib: 20480, dockerVersion: '27.3.1', defaultApp: 'shop', ipv6: true, setupVersion: 3 }) + // A server with no server.json prints nothing for the setup version. + assert.equal(parseFacts('3900\n20480\n27.3.1\nshop\nyes\n\n').setupVersion, null) assert.equal(parseFacts('3900\n20480\n27.3.1\n\nno\n').defaultApp, null) assert.throws(() => parseFacts('3900\n'), /did not report/) }) @@ -52,16 +56,27 @@ test('a remote build on a server under 2 GB of RAM is refused, pointing at --bui }) test('memory is an even share of 80% of RAM across apps, floored, and --memory overrides it', () => { - assert.equal(memoryLimit(4000, 1, null), '3200m') - assert.equal(memoryLimit(4000, 4, null), '800m') - assert.equal(memoryLimit(4000, 0, null), '3200m') - assert.equal(memoryLimit(961, 10, null), '256m') - assert.equal(memoryLimit(4000, 2, '1G'), '1g') - assert.throws(() => memoryLimit(4000, 1, '1.5gb'), /is not a size/) + assert.equal(memoryLimit(4000, 1, null, false), '3200m') + assert.equal(memoryLimit(4000, 4, null, false), '800m') + assert.equal(memoryLimit(4000, 0, null, false), '3200m') + assert.equal(memoryLimit(961, 10, null, false), '256m') + assert.equal(memoryLimit(4000, 2, '1G', true), '1g') + assert.throws(() => memoryLimit(4000, 1, '1.5gb', false), /is not a size/) +}) + +// The defect: one app on a 2 GB server was allowed 1600m, and for the hour after a +// deployment two of its containers were each allowed that. +test('where the replaced deployment is kept running, each app counts as two containers', () => { + assert.equal(memoryLimit(2000, 1, null, true), '800m') + assert.equal(memoryLimit(4000, 2, null, true), '800m') + // The live and the kept container together never pass the 80% the server gives away. + for (const [mem, apps] of [[2000, 1], [4000, 3], [8000, 5]]) { + assert.ok(Number.parseInt(memoryLimit(mem, apps, null, true), 10) * apps * 2 <= mem * 0.8) + } }) const run = (overrides = {}) => - runArguments({ name: 'shop', releaseId: 'r20260915-100000-abcdef', imageTag: 'dpl-1a2b-3c4d', memory: '800m', healthPath: '/about', dockerMajor: 27, workdir: '/src', ...overrides }) + runArguments({ name: 'shop', releaseId: 'r20260915-100000-abcdef', imageTag: 'dpl-1a2b-3c4d', memory: '800m', healthPath: '/about', dockerMajor: 27, workdir: '/src', writable: [], ...overrides }) // The one property every other safety claim leans on: nothing reaches a // container except through Caddy. @@ -82,11 +97,27 @@ test('a deployment publishes no port and carries the labels removal is scoped by assert.equal(args.at(-1), 'shop:dpl-1a2b-3c4d') }) +// The defect this prevents: an attacker's code, run through a flaw in the app, +// rewriting the app's files or leaving a binary that survives a restart. +test('a deployment runs read-only with no capabilities, and only the volumes and a non-executable /tmp are writable', () => { + const args = run() + assert.ok(args.includes('--read-only')) + assert.deepEqual(value(args, '--tmpfs'), ['/tmp:rw,noexec,nosuid,nodev,size=64m']) + assert.deepEqual(value(args, '--cap-drop'), ['ALL']) + assert.deepEqual(value(args, '--security-opt'), ['no-new-privileges']) + assert.deepEqual(value(args, '--pids-limit'), ['512']) + assert.deepEqual(value(args, '--volume'), [ + 'nextship-shop-build-dpl-1a2b-3c4d:/src/.next', + 'nextship-shop-cache:/src/.next/cache', + ]) + assert.ok(!args.includes('--cap-add') && !args.includes('--privileged')) +}) + test('the health check matches the image, with a fast start interval only where Docker has one', () => { const args = run() assert.deepEqual(value(args, '--health-interval'), ['30s']) assert.deepEqual(value(args, '--health-retries'), ['3']) - assert.deepEqual(value(args, '--health-start-interval'), ['2s']) + assert.deepEqual(value(args, '--health-start-interval'), ['500ms']) assert.deepEqual(value(run({ dockerMajor: 24 }), '--health-start-interval'), []) }) @@ -194,3 +225,32 @@ test('a build that failed on the app itself is not taken for a lost session', () assert.equal(lostBuildSession(JOURNAL_APP_FAILED), false) assert.equal(lostBuildSession(''), false) }) + +// No public CA may certify these, so asking Let's Encrypt for one would only fail, +// forever and every few minutes, in Caddy's log. +test('only a literal public IPv4 address gets a certificate of its own', () => { + for (const address of ['46.101.1.2', '159.89.10.20', '8.8.8.8']) assert.equal(publicIpv4(address), address) + for (const host of [ + 'example.com', '2a03:b0c0::1', '10.0.0.5', '172.16.4.1', '172.31.255.255', '192.168.1.10', '127.0.0.1', + '100.64.0.1', '169.254.1.1', '203.0.113.10', '198.51.100.7', '192.0.2.1', '224.0.0.1', '0.0.0.0', + ]) { + assert.equal(publicIpv4(host), null, host) + } + assert.equal(publicIpv4('172.32.0.1'), '172.32.0.1', 'just outside 172.16.0.0/12 is public') +}) + +// The container's root is read-only, so an app that keeps a database file or +// uploads beside its code needs a place it has named, and that place has to +// outlive the container. +test('each writable folder is a volume of its own, per app, mounted where the app expects it', () => { + const writable = writableVolumes('shop', ['data', 'public/uploads']) + assert.deepEqual(writable, [ + { folder: 'data', volume: 'nextship-shop-data-data' }, + { folder: 'public/uploads', volume: 'nextship-shop-data-public__uploads' }, + ]) + const args = run({ writable }) + assert.deepEqual(value(args, '--volume').slice(2), ['nextship-shop-data-data:/src/data', 'nextship-shop-data-public__uploads:/src/public/uploads']) + assert.ok(args.includes('--read-only'), 'the rest stays read-only') + // Named by the app, not the image: a new build mounts the same data. + assert.deepEqual(value(run({ writable, imageTag: 'dpl-9z9z-8y8y' }), '--volume').slice(2), value(args, '--volume').slice(2)) +}) diff --git a/packages/cli/src/targets/vm/vm-target.ts b/packages/cli/src/targets/vm/vm-target.ts index db3f5ad..0f2b768 100644 --- a/packages/cli/src/targets/vm/vm-target.ts +++ b/packages/cli/src/targets/vm/vm-target.ts @@ -36,6 +36,7 @@ import { hostname } from 'node:os' import path from 'node:path' import { connect as tlsConnect } from 'node:tls' import { lookup, resolve4 } from 'node:dns/promises' +import { isIP, isIPv4 } from 'node:net' import { NextshipError } from '../../errors.js' import { docsUrl } from '../../links.js' import type { ProjectConfig, ServerRecord } from '../../config.js' @@ -66,6 +67,7 @@ import { imagesToKeep, newReleaseId, parseDeployments, + previousServing, recordFailed, recordLive, recordStarted, @@ -82,7 +84,8 @@ const DEFAULT_APP_FILE = `${ETC}/default-app` /** How long a new container has to report healthy before the deployment fails. */ const HEALTH_TIMEOUT_MS = 5 * 60 * 1000 -const HEALTH_POLL_MS = 2000 +/** Matches the start interval in `runArguments`, so the switch follows the first passing check. */ +const HEALTH_POLL_MS = 500 /** Remote builds need room for the Next.js compiler beside what the server already runs. */ const REMOTE_BUILD_MIN_MIB = 1800 @@ -99,6 +102,8 @@ interface AppRecord { name: string createdAt: string domains: SiteDomain[] + /** Set on a preview: the id of the app it previews (`preview.ts`). */ + previewOf?: string } /** What a server reports in one round trip, for plans and guards. */ @@ -108,13 +113,21 @@ interface ServerFacts { dockerVersion: string defaultApp: string | null ipv6: boolean + /** The setup version `server add` last recorded, or null on a server that has none. */ + setupVersion: number | null } +/** + * The setup version that installs the retire timer. A server set up before it has + * nothing to stop a kept container, so a deployment to one keeps none. + */ +const RETIRE_SETUP_VERSION = 3 + // ------------------------------------------------------------------- pure /** Reads the one-shot facts script's output. */ export function parseFacts(output: string): ServerFacts { - const [mem, disk, docker, defaultApp, ipv6] = output.split('\n') + const [mem, disk, docker, defaultApp, ipv6, setup] = output.split('\n') const memMib = Number.parseInt(mem, 10) const diskFreeMib = Number.parseInt(disk, 10) if (!Number.isFinite(memMib) || !Number.isFinite(diskFreeMib) || !docker) { @@ -123,7 +136,15 @@ export function parseFacts(output: string): ServerFacts { 'Run `nextship server add` again to check the server is set up.' ) } - return { memMib, diskFreeMib, dockerVersion: docker.trim(), defaultApp: defaultApp?.trim() || null, ipv6: ipv6?.trim() === 'yes' } + const setupVersion = Number.parseInt(setup ?? '', 10) + return { + memMib, + diskFreeMib, + dockerVersion: docker.trim(), + defaultApp: defaultApp?.trim() || null, + ipv6: ipv6?.trim() === 'yes', + setupVersion: Number.isFinite(setupVersion) ? setupVersion : null, + } } /** Refuses a deployment the server has no room for, before anything is built. */ @@ -144,16 +165,23 @@ export function assertRoom(facts: ServerFacts, build: 'remote' | 'local'): void /** * The memory limit for a container: an even share of most of the server's RAM - * across its apps, so one app cannot starve the others or the server itself. + * across the containers its apps can run at once, so one app cannot starve the + * others or the server itself. A preview is an app and is counted as one. + * + * Where the server keeps a replaced deployment running, each app has two containers + * for the hour after a deployment, the live one and the kept one, so each app counts + * twice. Counted once, one app on a 2 GB server was allowed 1600m in each of two + * containers, twice what the server was meant to give away. */ -export function memoryLimit(memMib: number, appCount: number, requested: string | null): string { +export function memoryLimit(memMib: number, appCount: number, requested: string | null, keepsPrevious: boolean): string { if (requested !== null) { if (!/^\d+[mg]$/i.test(requested)) { throw new NextshipError(`--memory "${requested}" is not a size.`, 'Use megabytes or gigabytes, for example 512m or 2g.') } return requested.toLowerCase() } - return `${Math.max(256, Math.floor((memMib * APP_MEMORY_SHARE) / Math.max(appCount, 1)))}m` + const containers = Math.max(appCount, 1) * (keepsPrevious ? 2 : 1) + return `${Math.max(256, Math.floor((memMib * APP_MEMORY_SHARE) / containers))}m` } /** The health check a container runs: the adapter's cheap path, polled the way the image's own check polls `/`. */ @@ -174,9 +202,24 @@ export function healthCommand(healthPath: string | null): string { * `docker run` for one deployment. No port is published: Caddy reaches the * container on the nextship network, so nothing on the server's public address * bypasses the proxy. The interval, timeout and retries match the image's own - * HEALTHCHECK; the start interval makes Docker check every two seconds while the + * HEALTHCHECK; the start interval makes Docker check every half second while the * container starts, where Docker supports it, so a healthy deployment switches in - * seconds rather than after the first 30 second interval. + * about a second rather than after the first 30 second interval. Measured on a warm + * deployment: at two seconds, with the deploy polling every two seconds too, a + * container that was ready at once was seen as healthy four seconds after it started, + * a quarter of the whole deployment. + * + * The container is hardened so that code an attacker runs inside it cannot stay. + * The self-hosted break-ins through CVE-2025-55182 in December 2025 edited + * `next.config.js` and lockfiles and ran miners dropped in `/tmp`. Here the root + * filesystem is read-only, so the only writable places are the two volumes + * Next.js writes to and a `/tmp` that is emptied on restart and cannot hold an + * executable. No capability is kept, because the image runs as its `app` user on + * a port above 1024 and needs none, and nothing can gain privileges through a + * setuid binary. Measured in a nextship image: under concurrent streaming and + * image optimization the app ran 12 processes and threads, so the process limit + * stops a fork bomb without coming near a real app, and `docker diff` showed no + * write outside the volumes. */ export function runArguments(options: { name: string @@ -187,6 +230,8 @@ export function runArguments(options: { dockerMajor: number /** The image's working directory, where its `.next` is. */ workdir: string + /** The app's declared writable folders, each with the volume that holds it. */ + writable: WritableVolume[] }): string[] { const { name, releaseId, imageTag } = options const volumes = cacheVolumes(name, imageTag) @@ -197,10 +242,16 @@ export function runArguments(options: { '--restart', 'unless-stopped', '--env-file', `${ETC}/apps/${name}/env`, '--memory', options.memory, + '--read-only', + '--tmpfs', '/tmp:rw,noexec,nosuid,nodev,size=64m', + '--cap-drop', 'ALL', + '--security-opt', 'no-new-privileges', + '--pids-limit', '512', '--log-driver', 'journald', '--log-opt', `tag=${name}`, '--volume', `${volumes.build}:${options.workdir}/.next`, '--volume', `${volumes.cache}:${options.workdir}/.next/cache`, + ...options.writable.flatMap((entry) => ['--volume', `${entry.volume}:${options.workdir}/${entry.folder}`]), '--label', 'sh.nextship.managed=true', '--label', `sh.nextship.app=${name}`, '--label', `sh.nextship.deployment=${releaseId}`, @@ -211,7 +262,7 @@ export function runArguments(options: { '--health-start-period', '10s', '--health-retries', '3', // API 1.44, Docker 25. Setup accepts Docker 23, which would refuse the flag. - ...(options.dockerMajor >= 25 ? ['--health-start-interval', '2s'] : []), + ...(options.dockerMajor >= 25 ? ['--health-start-interval', '500ms'] : []), `${name}:${imageTag}`, ] } @@ -236,6 +287,25 @@ export function runArguments(options: { * `.next/cache/images` is not mounted directly: it does not exist in the image, so * its volume would be created owned by root and the app could not write to it. */ +export interface WritableVolume { + folder: string + volume: string +} + +/** + * The volume behind each folder an app declares writable in `nextship.json`. + * + * The container's root is read-only, so an app that keeps a SQLite file or uploads + * beside its code has nowhere to write unless it says where. Each folder is a + * named volume per app, not per image, so what is written there outlives a + * deployment, which it never did when it sat in a container's own layer. `/` in a + * folder becomes `__` in the name, which Docker allows and the config refuses in a + * folder, so two folders cannot share a volume. + */ +export function writableVolumes(name: string, folders: string[]): WritableVolume[] { + return folders.map((folder) => ({ folder, volume: `nextship-${name}-data-${folder.replace(/\//g, '__')}` })) +} + function cacheVolumes(name: string, imageTag: string): { build: string; cache: string } { return { build: `nextship-${name}-build-${imageTag}`, cache: `nextship-${name}-cache` } } @@ -406,6 +476,7 @@ export class VmTarget implements Target { private readonly server: ServerRecord private readonly name: string private readonly build: 'remote' | 'local' + private readonly writable: WritableVolume[] private connection: Promise | null = null constructor(config: ProjectConfig) { @@ -415,6 +486,7 @@ export class VmTarget implements Target { this.server = config.server this.name = assertAppName(config.name) this.build = config.build ?? 'remote' + this.writable = writableVolumes(this.name, config.writable ?? []) this.displayName = `your server ${config.server.host}` } @@ -447,7 +519,8 @@ export class VmTarget implements Target { ] readonly rollbackNotes = [ - 'the current env file is used, not the one that deployment ran with: variables changed since then keep their new values', + 'the deployment replaced within the last hour is still running: going back to it is one Caddy switch, and it keeps the env it ran with', + 'an older deployment is started again with the current env file, not the one it ran with: variables changed since then keep their new values', ] // ------------------------------------------------------------ connection @@ -492,6 +565,7 @@ export class VmTarget implements Target { "docker version --format '{{.Server.Version}}'", `cat ${DEFAULT_APP_FILE} 2> /dev/null || echo`, "if ip -6 addr show scope global 2> /dev/null | grep -q inet6; then echo yes; else echo no; fi", + `sed -n 's/.*"setupVersion":\\([0-9]*\\).*/\\1/p' ${ETC}/server.json 2> /dev/null || true`, ].join('; ') return parseFacts(await this.run(script, 'read the server\'s memory and disk')) } @@ -508,7 +582,7 @@ export class VmTarget implements Target { .filter((line) => line.trim() !== '') .map((line) => { const record = JSON.parse(line) as AppRecord - return { id: record.id, name: record.name } + return { id: record.id, name: record.name, ...(record.previewOf ? { previewOf: record.previewOf } : {}) } }) } @@ -551,8 +625,10 @@ export class VmTarget implements Target { const isDefault = facts.defaultApp === this.name || (facts.defaultApp === null && others.length === 0) const domains = context.appId ? (await this.appRecord(context.appId)).domains : [] + const keepsPrevious = (facts.setupVersion ?? 0) >= RETIRE_SETUP_VERSION + const ip = publicIpv4(this.server.host) const address = isDefault - ? `address http://${this.server.host}${domains.length > 0 ? `, and ${domains.map((entry) => entry.domain).join(', ')}` : ''}` + ? `address http://${this.server.host}${ip ? `, and https://${ip} once Let's Encrypt issues its six-day certificate for the address` : ''}${domains.length > 0 ? `, and ${domains.map((entry) => entry.domain).join(', ')}` : ''}` : domains.length > 0 ? `address ${domains.map((entry) => entry.domain).join(', ')}` : `address none until \`nextship domain add\`: ${facts.defaultApp ?? 'another app'} answers http://${this.server.host}` @@ -563,8 +639,11 @@ export class VmTarget implements Target { context.appId ? `app UPDATE "${context.name}" (${context.appId}), which nextship created` : `app CREATE "${context.name}" in ${ETC}/apps/${context.name}`, - `memory up to ${memoryLimit(facts.memMib, appCount, context.memory)} of ${facts.memMib} MiB`, + `memory up to ${memoryLimit(facts.memMib, appCount, context.memory, keepsPrevious)} of ${facts.memMib} MiB${keepsPrevious ? ', for each of the live container and the one kept for an hour after a deployment' : ''}`, address, + ...(this.writable.length > 0 + ? [`writable ${this.writable.map((entry) => entry.folder).join(', ')}: kept in a volume each, across deployments; everything else in the container is read-only`] + : []), 'switch Caddy moves traffic only once the new container is healthy; until then the current one serves', 'caddy the site is regenerated from the domains recorded for this app', ] @@ -766,6 +845,7 @@ export class VmTarget implements Target { cause: 'deploy', memory: request.memory, healthPath: request.healthPath ?? null, + previewOf: request.previewOf, }) return { appId: started.appId, deploymentId: started.releaseId } } @@ -835,12 +915,48 @@ export class VmTarget implements Target { return result.code === 0 ? parseDeployments(result.stdout) : [] } + /** + * Writes the history, and beside it `previous`: the kept container and when it + * stops, as ` `, empty when none is kept. The retire + * timer on the server reads that line, since the server has no JSON parser. + * + * One remote command writes both, each to a temporary file moved into place, so a + * connection lost part way leaves the pair as it was rather than a history that + * says one thing and a `previous` that says another. `previous` moves first: if + * only it lands, the timer knows of a container the history does not, which costs + * nothing, where the reverse would leave a kept container nothing ever stops. + */ private async writeDeployments(deployments: VmDeployment[]): Promise { - await this.run( - `install -m 600 /dev/stdin ${q(`${this.appDir()}/deployments.json`)}`, - 'write the deployment history', - serializeDeployments(deployments) - ) + const kept = previousServing(deployments, new Date()) + // Safe to place in the command: an app name and a release id are both validated, + // and the rest is digits. + const previous = kept?.keptUntil ? `${this.name}-${kept.id} ${Math.floor(Date.parse(kept.keptUntil) / 1000)}\n` : '' + const script = [ + 'set -eu', + `d=${q(this.appDir())}`, + 'install -m 600 /dev/stdin "$d/deployments.json.next"', + `printf '%s' ${q(previous)} > "$d/previous.next" && chmod 600 "$d/previous.next"`, + 'mv "$d/previous.next" "$d/previous"', + 'mv "$d/deployments.json.next" "$d/deployments.json"', + ].join('\n') + await this.run(script, 'write the deployment history', serializeDeployments(deployments)) + } + + /** + * Empties `previous`, so the retire timer stops nothing, before Caddy is pointed + * anywhere new. Without this, a rollback to the kept container, or a CLI that died + * between the switch and the write that follows it, left `previous` naming the + * container Caddy now serves, and the timer stopped the live app when its hour + * ran out. Docker's restart policy does not bring back a container that was stopped. + */ + private async clearPrevious(): Promise { + await this.run(`: > ${q(`${this.appDir()}/previous`)}`, 'stop the retire timer acting during the switch') + } + + /** The Caddy route for the kept deployment, from a history. */ + private previousRoute(deployments: VmDeployment[], now: Date): { container: string; deploymentId: string } | null { + const kept = previousServing(deployments, now) + return kept?.imageTag ? { container: `${this.name}-${kept.id}`, deploymentId: kept.imageTag } : null } /** @@ -850,7 +966,14 @@ export class VmTarget implements Target { */ private async startRelease( appId: string | null, - options: { imageTag: string; cause: string; memory: string | null; healthPath: string | null; before?: () => Promise } + options: { + imageTag: string + cause: string + memory: string | null + healthPath: string | null + before?: () => Promise + previewOf?: string + } ): Promise<{ appId: string; releaseId: string }> { await this.lock() try { @@ -865,7 +988,13 @@ export class VmTarget implements Target { 'nextship will not modify an app it does not own. Rename this project, or set a different name in nextship.json.' ) } - record = { id: randomUUID(), name: this.name, createdAt: new Date().toISOString(), domains: [] } + record = { + id: randomUUID(), + name: this.name, + createdAt: new Date().toISOString(), + domains: [], + ...(options.previewOf ? { previewOf: options.previewOf } : {}), + } await this.writeAppRecord(record) } await this.run(`touch ${q(`${this.appDir()}/env`)} && chmod 600 ${q(`${this.appDir()}/env`)}`, 'prepare the env file') @@ -885,14 +1014,32 @@ export class VmTarget implements Target { ].join(' > /dev/null && ') + ' > /dev/null', 'create the cache volumes' ) + if (this.writable.length > 0) { + // Docker creates a volume's directory owned by root, and fills it from the + // image, with the image's ownership, only when the image has that folder. + // A folder the build did not produce would be one the app's own user cannot + // write to, so each is handed to the image's `app` user before the app + // starts. Run as root in a container of the same image, which is where the + // name `app` means the right uid. + const mounts = this.writable.flatMap((entry, index) => ['--volume', `${entry.volume}:/nextship-writable/${index}`]) + const targets = this.writable.map((_entry, index) => `/nextship-writable/${index}`) + await this.run( + [ + ...this.writable.map((entry) => ['docker', 'volume', 'create', ...labels, entry.volume].map(q).join(' ') + ' > /dev/null'), + ['docker', 'run', '--rm', '--user', '0:0', '--network', 'none', '--entrypoint', 'chown', ...mounts, reference, 'app:app', ...targets].map(q).join(' '), + ].join(' && '), + 'prepare the writable folders' + ) + } const args = runArguments({ workdir, name: this.name, releaseId, imageTag: assertDeploymentId(options.imageTag), - memory: memoryLimit(facts.memMib, apps.length, options.memory), + memory: memoryLimit(facts.memMib, apps.length, options.memory, (facts.setupVersion ?? 0) >= RETIRE_SETUP_VERSION), healthPath: options.healthPath, dockerMajor: Number.parseInt(facts.dockerVersion, 10), + writable: this.writable, }) const history = recordStarted(await this.readDeployments(), { @@ -937,8 +1084,14 @@ export class VmTarget implements Target { } const record = await this.appRecord(appId) + const now = new Date() + // A server set up before the retire timer existed has nothing to stop a kept + // container, which would then run, with its memory, until the next deployment. + const canKeep = ((await this.facts()).setupVersion ?? 0) >= RETIRE_SETUP_VERSION + const history = recordLive(await this.readDeployments(), releaseId, now, canKeep) + await this.clearPrevious() try { - await this.switchSite(record, container) + await this.switchSite(record, container, this.previousRoute(history, now)) } catch (error) { await this.discard(container, releaseId, 'was refused by Caddy', onPhase) throw new NextshipError( @@ -953,7 +1106,6 @@ export class VmTarget implements Target { // time: a CLI that dies in between would otherwise leave deployments.json // naming a stopped container as live, which `domain add` would then rebuild // the site from and point the proxy back at something that is not running. - const history = recordLive(await this.readDeployments(), releaseId) await this.writeDeployments(history) // A previous container that is restarting or paused is not serving, but it @@ -963,7 +1115,15 @@ export class VmTarget implements Target { `docker ps --filter label=sh.nextship.managed=true --filter ${q(`label=sh.nextship.app=${this.name}`)} --filter status=running --filter status=restarting --filter status=paused --format '{{.Names}}'`, 'list the running containers' ) - const previous = previousContainers(running, this.name, container) + // The deployment this one replaced keeps running for the tabs that loaded it, if + // it is a different build; `retire.sh` stops it when its time runs out. + const kept = previousServing(history, now) + const previous = previousContainers(running, this.name, container).filter((name) => name !== `${this.name}-${kept?.id}`) + if (kept) onPhase(`kept ${this.name}-${kept.id} running until ${kept.keptUntil} for tabs that loaded it`) + if (!canKeep && replacing) { + onPhase('the previous deployment was not kept running for open tabs: this server was set up by an older nextship, which has no timer to stop it') + onPhase('Run `nextship server add` with --yes to update the server. Until then a tab opened before a deployment reloads on its next navigation.') + } if (previous.length > 0) { // 30 s matches the time a request is allowed to finish: Next.js exits on // SIGTERM once in-flight responses and after() callbacks complete. @@ -1041,12 +1201,21 @@ export class VmTarget implements Target { * the previous file is put back and Caddy reloaded again, so a bad site never * takes the proxy down for every app on the server. */ - private async switchSite(record: AppRecord, container: string): Promise { + private async switchSite( + record: AppRecord, + container: string, + previous: { container: string; deploymentId: string } | null + ): Promise { + // A preview never takes the server's own address, which belongs to a real app. + const isDefault = record.previewOf === undefined && (await this.claimDefaultApp()) === this.name const site = renderSite({ name: this.name, container, domains: record.domains, - isDefault: (await this.claimDefaultApp()) === this.name, + isDefault, + previous, + preview: record.previewOf !== undefined, + ipCertificate: isDefault ? publicIpv4(this.server.host) : null, }) await this.applySite(site) } @@ -1163,9 +1332,13 @@ export class VmTarget implements Target { } }) ) + // HTTPS only once it really answers with a valid certificate, which can take a + // minute after the first deployment, or never on a server Let's Encrypt cannot reach. + const ip = isDefault ? publicIpv4(this.server.host) : null + const secure = ip !== null && (await certificateState(ip, ip)).live return { platformHost: isDefault ? this.server.host : null, - platformUrl: isDefault ? `http://${this.server.host}` : null, + platformUrl: isDefault ? `${secure ? 'https' : 'http'}://${this.server.host}` : null, domains, } } @@ -1192,6 +1365,7 @@ export class VmTarget implements Target { 'It was pruned. Check out that commit and run `nextship deploy` to build it again. Nothing was changed.' ) } + if (await this.rollbackInstantly(appId, target, onPhase)) return const live = (await this.readDeployments()).some((entry) => entry.live) const started = await this.startRelease(appId, { imageTag: target.imageTag, @@ -1206,6 +1380,44 @@ export class VmTarget implements Target { } } + /** + * Returns to the kept deployment by pointing Caddy back at its container, which + * never stopped, so nothing is built or started and the switch is one reload. + * The deployment being left is kept in its place, so a mistaken rollback can be + * undone the same way. Returns false, changing nothing, when the target is not + * the kept deployment or its container is not running and healthy; the caller + * then starts the target's image again as it always did. + * + * The container runs with the env it started with, which is the env of that + * deployment, where a rollback that starts a new container reads the current one. + */ + private async rollbackInstantly(appId: string, target: VmDeployment, onPhase: PhaseReporter): Promise { + const began = Date.now() + const container = `${this.name}-${target.id}` + + // Everything is read under the lock. Read before it, a deployment that went live + // in between had already stopped this container, as no longer the kept one, and + // the rollback then pointed Caddy at a container that was not running. + await this.lock() + try { + const now = new Date() + const before = await this.readDeployments() + if (previousServing(before, now)?.id !== target.id) return false + const state = await this.exec(`docker inspect -f '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' ${q(container)}`) + if (state.code !== 0 || state.stdout.trim() !== 'running healthy') return false + + const history = recordLive(before, target.id, now, true) + await this.clearPrevious() + await this.switchSite(await this.appRecord(appId), container, this.previousRoute(history, now)) + await this.writeDeployments(history) + onPhase(`Caddy now sends traffic to ${container}, which was still running, in ${Date.now() - began} ms`) + onPhase('it runs with the env it was deployed with, not the current env file') + return true + } finally { + await this.unlock() + } + } + /** Starts the live image again, for a change that needs a new container but no new build. */ private async redeploy(appId: string, cause: string, before: () => Promise): Promise { const history = await this.readDeployments() @@ -1317,7 +1529,8 @@ export class VmTarget implements Target { try { const record = await this.appRecord(appId) const updated = { ...record, domains: change(record.domains) } - const live = (await this.readDeployments()).find((entry) => entry.live) + const history = await this.readDeployments() + const live = history.find((entry) => entry.live) if (live) { const facts = await this.facts() const site = renderSite({ @@ -1325,6 +1538,9 @@ export class VmTarget implements Target { container: `${this.name}-${live.id}`, domains: updated.domains, isDefault: facts.defaultApp === this.name, + previous: this.previousRoute(history, new Date()), + preview: record.previewOf !== undefined, + ipCertificate: facts.defaultApp === this.name ? publicIpv4(this.server.host) : null, }) // The site is validated and live before the record says so, so a domain // Caddy refused is never recorded as attached. @@ -1663,6 +1879,9 @@ export class VmTarget implements Target { : 'address none, this app answers only on its domains', ...domains.map((domain) => `domain ${domain.domain} stops serving this app`), `containers remove every container of this app, its cache volumes and ${this.appDir()}`, + ...(this.writable.length > 0 + ? [`data remove what the app wrote in ${this.writable.map((entry) => entry.folder).join(', ')}: volumes ${this.writable.map((entry) => entry.volume).join(', ')}`] + : []), options.images && options.imageCount > 0 ? `images remove all ${options.imageCount} image(s) of this app from the server` : options.images @@ -1671,13 +1890,17 @@ export class VmTarget implements Target { `server kept, with Caddy and every other app on ${this.server.host}`, 'DNS untouched, nextship did not create your records', ], - warnings: - domains.length > 0 + warnings: [ + ...(this.writable.length > 0 + ? [`Everything the app wrote in ${this.writable.map((entry) => entry.folder).join(', ')} is deleted with it. Copy it off the server first if you need it.`] + : []), + ...(domains.length > 0 ? [ `The DNS records for ${domains.map((domain) => domain.domain).join(', ')} keep pointing at ${this.server.host}, ` + "which stops serving this app on them: the server's default app, if any, answers them over plain HTTP. Remove or repoint them.", ] - : [], + : []), + ], } } @@ -1748,7 +1971,9 @@ async function serverAddress(host: string): Promise { */ async function certificateState(host: string, domain: string): Promise<{ live: boolean; detail: string }> { return new Promise((resolve) => { - const socket = tlsConnect({ host, port: 443, servername: domain, rejectUnauthorized: true, timeout: 5000 }) + // No server name for an address: TLS forbids one, and Node then checks the + // certificate against the address itself. + const socket = tlsConnect({ host, port: 443, ...(isIP(domain) ? {} : { servername: domain }), rejectUnauthorized: true, timeout: 5000 }) socket.once('secureConnect', () => { socket.destroy() resolve({ live: true, detail: 'certificate issued' }) @@ -1762,3 +1987,25 @@ async function certificateState(host: string, domain: string): Promise<{ live: b }) }) } + +/** + * The server's address when Let's Encrypt can issue a certificate for it: a literal + * IPv4 address outside the private, shared, loopback, link-local, documentation and + * multicast ranges, which no public CA may certify. A server added by hostname has a + * name to get a certificate for already, through `domain add`. + */ +export function publicIpv4(host: string): string | null { + if (!isIPv4(host)) return null + const [a, b, c] = host.split('.').map(Number) + const reserved = + a === 0 || a === 10 || a === 127 || a >= 224 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) || + (a === 192 && b === 0 && (c === 0 || c === 2)) || + (a === 198 && (b === 18 || b === 19)) || + (a === 198 && b === 51 && c === 100) || + (a === 203 && b === 0 && c === 113) + return reserved ? null : host +} diff --git a/site/components/docs/nightly-result.tsx b/site/components/docs/nightly-result.tsx new file mode 100644 index 0000000..4af98d9 --- /dev/null +++ b/site/components/docs/nightly-result.tsx @@ -0,0 +1,85 @@ +'use client' + +import { useEffect, useState } from 'react' + +import { NIGHTLY_RESULT_URL, NIGHTLY_RUNS_URL, type NightlyResult as Result } from '@/lib/evidence' + +/** + * Last night's compatibility suite result, read from the `evidence` branch when the + * page opens. + * + * When the file cannot be read, the page says so and links to the runs instead of + * showing a number: before the first published night there is no file, and a figure + * rendered from nothing would be exactly the fabricated claim the evidence page exists + * to avoid. + * + * Author: Ragul D + */ +export function NightlyResult() { + const [state, setState] = useState<{ result: Result } | 'loading' | 'unavailable'>('loading') + + useEffect(() => { + const controller = new AbortController() + const load = async () => { + try { + const response = await fetch(NIGHTLY_RESULT_URL, { signal: controller.signal, cache: 'no-store' }) + if (!response.ok) throw new Error(`HTTP ${response.status}`) + setState({ result: (await response.json()) as Result }) + } catch { + // Unmounting aborts the fetch; nothing is left to update then. + if (!controller.signal.aborted) setState('unavailable') + } + } + void load() + return () => controller.abort() + }, []) + + const link = 'font-medium text-accent-text underline decoration-accent-text/30 underline-offset-4 hover:decoration-accent-text' + + if (state === 'loading') { + return

Reading last night's result…

+ } + if (state === 'unavailable') { + return ( +

+ Last night's result could not be read here. Every run is listed on{' '} + + GitHub Actions + + . +

+ ) + } + + const { result } = state + const rows = [ + ['Run', new Date(result.date).toUTCString().replace(' GMT', ' UTC')], + ['Next.js', `${result.nextjs.version} (${result.nextjs.sha.slice(0, 9)})`], + ['Suites passing', `${result.suites.passed} of ${result.suites.total}`], + ['Assertions passing', `${result.assertions.passed} of ${result.assertions.total}`], + ['Failing suites', result.failing.length === 0 ? 'None' : result.failing.join(', ')], + ] + return ( +
+ + + {rows.map(([label, value]) => ( + + + + + ))} + + + + +
+ {label} + {value}
+ + The full run on GitHub Actions + +
+
+ ) +} diff --git a/site/components/mdx.tsx b/site/components/mdx.tsx index 9c1cb61..70e5db2 100644 --- a/site/components/mdx.tsx +++ b/site/components/mdx.tsx @@ -3,6 +3,7 @@ import type { MDXComponents } from 'mdx/types' import { Callout } from '@/components/docs/callout' import { CodeBlock } from '@/components/docs/code-block' +import { NightlyResult } from '@/components/docs/nightly-result' /** * How MDX elements render. @@ -94,4 +95,5 @@ export const mdxComponents: MDXComponents = { pre: (props) => 
,
 
   Callout,
+  NightlyResult,
 }
diff --git a/site/content/docs/commands.mdx b/site/content/docs/commands.mdx
index ae2ea2d..b6010de 100644
--- a/site/content/docs/commands.mdx
+++ b/site/content/docs/commands.mdx
@@ -91,8 +91,8 @@ in `node_modules` that nothing declares: that one fails the build, and is a bloc
 | Checked | Why it matters |
 |---|---|
 | Analytics and similar packages | They stop reporting and nothing errors |
-| Cron jobs in platform config | They will simply never run |
-| Routing rules in platform config | They stop applying |
+| Cron jobs in platform config | They will simply never run. `doctor` prints a crontab line for each that calls its route |
+| Routing rules in platform config | They stop applying. `doctor` prints those that carry over unchanged as a `next.config` snippet, and names the rest |
 | Platform URL and environment variables read in source | They become undefined |
 | Packages in `node_modules` that nothing declares | They exist on your machine and not in the image, so the build fails on an import that resolves locally |
 | A missing lockfile | Installs are no longer reproducible |
diff --git a/site/content/docs/evidence.mdx b/site/content/docs/evidence.mdx
index bf30df1..9cb01da 100644
--- a/site/content/docs/evidence.mdx
+++ b/site/content/docs/evidence.mdx
@@ -35,8 +35,12 @@ Vercel, are in the
 
 A full run now takes 319 runner-minutes against 788, in the same 32 groups.
 
-The suite runs from GitHub Actions in this repository, weekly and on demand, so a
-regression shows up without anyone remembering to look.
+The suite runs from GitHub Actions in this repository every night against the newest
+Next.js canary, and on demand, so a change in Next.js that breaks self-hosting shows up the
+day it lands rather than when it reaches a release. The table above is the result recorded
+for a release. The nightly one is below, read from the repository when you open this page:
+
+
 
 ## Streaming
 
diff --git a/site/content/docs/vm.mdx b/site/content/docs/vm.mdx
index 18e3ee9..7762067 100644
--- a/site/content/docs/vm.mdx
+++ b/site/content/docs/vm.mdx
@@ -33,8 +33,9 @@ The first command changes nothing. It shows the server's host key fingerprint, w
 compare with the one your provider shows, and every setup step with what it would change.
 The second installs Docker if needed, creates a `nextship` user and proves a login as it,
 runs Caddy on ports 80 and 443, limits the journal, enables security updates and ufw,
-installs a watchdog for containers that stay unhealthy, and turns off SSH password logins
-last. From then on the host key is pinned in `nextship.json` and a server presenting any
+installs a watchdog for containers that stay unhealthy, a cache guard that keeps what each
+app writes at runtime under a limit and a timer that stops a replaced deployment after its
+hour, and turns off SSH password logins last. From then on the host key is pinned in `nextship.json` and a server presenting any
 other key is refused.
 
 `--no-firewall`, `--no-auto-updates`, `--no-swap` and `--no-ssh-hardening` opt out of those
@@ -71,7 +72,8 @@ nextship domain add app.example.com --yes
 | Cache after a restart | Starts cold | Regenerated pages and optimized images kept in volumes |
 | Logs of a replaced deployment | Gone | `nextship logs --deployment ` reads the journal |
 | CDN | In front, keeps static pages | None unless you add one |
-| Rollback | Restores the old spec, env included | Starts the old image with the current env file |
+| Rollback | Restores the old spec, env included | Within an hour, switches back to the replaced deployment, still running with its env. Older, starts the old image with the current env file |
+| A tab open across a deployment | Loads the new build on its next navigation | Keeps working for an hour: requests naming its build reach it |
 | Server Actions key | In a local file | On the server, shared by every machine that deploys |
 | Apps | One per App Platform app | Many per server, each with its own domains |
 | Failover | Managed | None: one server |
@@ -92,5 +94,20 @@ Two things on the server exist nowhere else: `/etc/nextship/apps//env` and
 `/etc/nextship/apps//secrets`. Keep a copy of both. The `nextship` user is in the docker
 group, which is root-equivalent, so treat its keys as root's.
 
+A tab opened before a deployment keeps working after it: the deployment a new build replaces
+keeps running for an hour and gets every request that names its build. A rollback within that
+hour switches back to it at once.
+
+`nextship deploy --preview pr-42` deploys a preview: a second app on the same server with its
+own env file and domains, which asks crawlers not to index it. A workflow that deploys one per
+pull request is in the
+[server guide](https://github.com/gowtham472/nextship/blob/main/docs/vm.md#a-preview-per-pull-request).
+
+Each app's container is read-only apart from `.next` and a 64 MB `/tmp` that cannot run a
+binary, with no Linux capabilities and a limit of 512 processes. Code an attacker runs through
+a flaw in your app can neither rewrite it nor leave a program behind. An app that keeps a
+SQLite file or uploads beside its code lists the folders in `nextship.json`, as
+`"writable": ["data"]`: each becomes a volume it can write to, kept across deployments.
+
 The full guide, including GitHub Actions and recovery, is
 [docs/vm.md](https://github.com/gowtham472/nextship/blob/main/docs/vm.md).
diff --git a/site/lib/evidence.ts b/site/lib/evidence.ts
new file mode 100644
index 0000000..023944c
--- /dev/null
+++ b/site/lib/evidence.ts
@@ -0,0 +1,24 @@
+/**
+ * Where the nightly compatibility suite publishes its result.
+ *
+ * `conformance/publish-evidence.mjs`, run by the scheduled workflow, writes
+ * `latest.json` to the repository's `evidence` branch. The page reads it in the
+ * browser rather than at build time, so the evidence page shows last night's run
+ * without the site being rebuilt every night.
+ *
+ * Author: Ragul D
+ */
+export const NIGHTLY_RESULT_URL = 'https://raw.githubusercontent.com/gowtham472/nextship/evidence/latest.json'
+
+/** Where every run, published or not, can be read in full. */
+export const NIGHTLY_RUNS_URL = 'https://github.com/gowtham472/nextship/actions/workflows/conformance.yml'
+
+/** What `publish-evidence.mjs` writes, as far as the page reads it. */
+export interface NightlyResult {
+  date: string
+  nextjs: { version: string; sha: string }
+  run: string
+  suites: { passed: number; total: number }
+  assertions: { passed: number; total: number }
+  failing: string[]
+}
diff --git a/site/lib/markdown.ts b/site/lib/markdown.ts
index be6063e..861d637 100644
--- a/site/lib/markdown.ts
+++ b/site/lib/markdown.ts
@@ -1,4 +1,5 @@
 import type { Doc } from './docs'
+import { NIGHTLY_RESULT_URL } from './evidence'
 import { SITE_URL } from './site'
 
 /**
@@ -6,7 +7,8 @@ import { SITE_URL } from './site'
  * than a rendered page.
  *
  * The MDX is already markdown apart from one component, so conversion is small and
- * strict: a callout becomes a blockquote, and links become absolute, pointing a docs
+ * strict: a callout becomes a blockquote, the nightly result becomes the address of its
+ * file, and links become absolute, pointing a docs
  * link at that page's markdown so an agent can keep reading in the same format. Any
  * other component fails the build, because an agent handed raw JSX would read it as
  * content, and a page that quietly lost a component is worse than one that refuses to
@@ -29,6 +31,8 @@ export function toMarkdown(doc: Doc): string {
         .map((line) => (line ? `> ${line}` : '>'))
         .join('\n')
     })
+    // Read in the browser on the page; an agent is pointed at the same file.
+    .replace(//g, `Last night's result is published as JSON at ${NIGHTLY_RESULT_URL}.`)
     .replace(/\]\((\/[^)\s]*)\)/g, (_, target: string) => `](${absolute(target)})`)
 
   const leftover = /<[A-Z][A-Za-z]*/.exec(body.replace(/```[\s\S]*?```/g, ''))