diff --git a/.github/workflows/finix-compat.yml b/.github/workflows/finix-compat.yml new file mode 100644 index 0000000..75b0f1d --- /dev/null +++ b/.github/workflows/finix-compat.yml @@ -0,0 +1,110 @@ +name: finix compatibility + +# runs the suite in tests/ against finix *main*, not the pin in lon.lock, and +# records the result in COMPATIBILITY.md. that is what makes the table's answer +# - "the last finix commit this module worked with" - move on its own. +# +# module failures are recorded, not fatal: this repository says up front that +# its modules may be minimally maintained and may break. `nix flake check` in +# ci.yml is the gate that does go red, and it runs against the pinned finix. + +on: + push: + # the table is committed by this workflow; without this it would retrigger + # itself forever, since every run stamps a new timestamp into it. + paths-ignore: + - COMPATIBILITY.md + - ci/compat-state.json + pull_request: + # finix moves whether or not anything lands here, so re-check daily + schedule: + - cron: "17 5 * * *" + workflow_dispatch: + +concurrency: + group: finix-compat-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + compat: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + # the table is committed back on main, and pushing from a shallow + # clone after a rebase is rejected + fetch-depth: 0 + + - name: Make /dev/kvm usable + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - uses: DeterminateSystems/nix-installer-action@main + with: + extra-conf: | + extra-experimental-features = pipe-operators + system-features = nixos-test benchmark big-parallel kvm + + - name: Check out finix main + id: finix + run: | + git clone --depth 1 https://github.com/finix-community/finix .ci/finix + echo "rev=$(git -C .ci/finix rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "date=$(git -C .ci/finix log -1 --format=%cs)" >> "$GITHUB_OUTPUT" + + - name: Run the suite + run: FINIX="$PWD/.ci/finix" ci/run-suite.sh + + - name: Render the table + run: | + args=( + --arg finix ./.ci/finix + --arg manifest ./.ci/run/manifest.json + --arg results ./.ci/run/results.json + --argstr rev "${{ steps.finix.outputs.rev }}" + --argstr revDate "${{ steps.finix.outputs.date }}" + --argstr checkedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + ) + # absent on the very first run + if [ -f ci/compat-state.json ]; then + args+=(--arg state ./ci/compat-state.json) + fi + + nix-build ci/report.nix "${args[@]}" -o .ci/report + # -m 644: store paths are read-only, and copying the mode along would + # make the next run unable to overwrite them + install -m 644 .ci/report/compat-state.json ci/compat-state.json + install -m 644 .ci/report/COMPATIBILITY.md COMPATIBILITY.md + + cat COMPATIBILITY.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: finix-compat-logs + path: .ci/run/logs + retention-days: 14 + + - name: Commit the table + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add COMPATIBILITY.md ci/compat-state.json + + if git diff --cached --quiet; then + echo "table unchanged" + exit 0 + fi + + git commit -m "ci: update finix compatibility table [skip ci]" + git pull --rebase --autostash origin main + git push diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc9a9c2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# scratch space for the compatibility suite: the finix checkout it ran +# against, raw results and logs. the state it keeps lives in ci/. +/.ci/ + +result +result-* diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md new file mode 100644 index 0000000..d79a003 --- /dev/null +++ b/COMPATIBILITY.md @@ -0,0 +1,53 @@ +# module compatibility + + + +`finix` moves fast and the modules here are minimally maintained by design, +so this table records, per module, the last `finix` commit it was known to +work with. a module row that lags behind the commit at the top of this file +has fallen behind and needs a look. + +- last checked against finix [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) (2026-08-23) +- on `x86_64-linux`, at 2026-08-25T14:44:52Z + +| module | checked | status | last finix commit it worked with | that commit | +| --- | --- | --- | --- | --- | +| `amnezia-vpn` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `bootchart` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `cups` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `dinit` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `fastfetch` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `gamescope` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `home-manager` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `laptop` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `minimal` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `nix-ld` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `openrgb` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `pipewire` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `preservation` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `river` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `soteria` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `steam` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `tailscale` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `v2rayn` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `xinetd` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | +| `yarr` | eval | working | [`78fd549`](https://github.com/finix-community/finix/commit/78fd549fa5410ad1e5683efbba2d0f1728c12f3d) | 2026-08-23 | + +`checked` says how far the suite got with a module. `eval` builds a finix +system with the module enabled and instantiates its closure, which catches +the option and api drift that breaks these modules in practice. `eval + vm` +also boots that system under qemu and asserts the module does its job. + +## what is not booted, and why + +- `amnezia-vpn` (eval - gui client started by hand from a desktop session; there is nothing to assert on a headless vm) +- `dinit` (system-services - system-level dinit services; finit is pid 1 on finix, so nothing supervises them - the user half is booted in tests/user-services.nix) +- `gamescope` (eval - a compositor needs a gpu and a seat; the test vm has neither) +- `laptop` (eval - an opinionated hardware profile - firmware, backlight, power management; what it does only shows up on a real laptop) +- `openrgb` (eval - drives rgb controllers over i2c/smbus, none of which a vm has) +- `pipewire` (eval - needs a sound device and a logged-in seat to reach a running graph) +- `preservation` (eval - acts from the initrd against a persistent volume; the test vm boots on a tmpfs root with no second volume to preserve to) +- `river` (eval - a wayland compositor needs a gpu and a seat; the test vm has neither) +- `soteria` (eval - polkit agent for a graphical session; a headless vm has no session to authenticate for) +- `steam` (eval - pulls the 32-bit graphics stack and wants a gpu; far too large to boot in ci) +- `v2rayn` (eval - gui client started by hand from a desktop session; there is nothing to assert on a headless vm) diff --git a/README.md b/README.md index 9d0b504..c0d38f1 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,11 @@ this repository prioritizes: - ecosystem growth - low contribution friction +because of that, [COMPATIBILITY.md](COMPATIBILITY.md) records, per module, the +last `finix` commit it was known to work with. it is regenerated on every push +and daily against `finix` `main`. check it before reaching for a module, and +see [tests/README.md](tests/README.md) to add a test for one. + # usage (flake-based) to use this repository, add the following to your flake inputs: diff --git a/ci/compat-state.json b/ci/compat-state.json new file mode 100644 index 0000000..baa0cf3 --- /dev/null +++ b/ci/compat-state.json @@ -0,0 +1,345 @@ +{ + "modules": { + "amnezia-vpn": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "bootchart": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "cups": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "dinit": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "fastfetch": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "gamescope": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "home-manager": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "laptop": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "minimal": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "nix-ld": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "openrgb": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "pipewire": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "preservation": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "river": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "soteria": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "steam": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "tailscale": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "v2rayn": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "xinetd": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + }, + "yarr": { + "checks": "eval", + "failures": [], + "lastChecked": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "lastGood": { + "checkedAt": "2026-08-25T14:44:52Z", + "date": "2026-08-23", + "rev": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "system": "x86_64-linux" + }, + "status": "passing" + } + }, + "version": 1 +} diff --git a/ci/report.nix b/ci/report.nix new file mode 100644 index 0000000..5395ad5 --- /dev/null +++ b/ci/report.nix @@ -0,0 +1,219 @@ +# turns one run of the suite into the compatibility table +# +# the table has to remember: once a module stops working against finix main, +# the interesting fact is the last commit it *did* work for, and that is not +# something the current run can know. so the previous state file is merged +# forward - a module that passes gets its `lastGood` moved to the commit just +# checked, and a module that fails keeps whatever it had. +# +# nix-build ci/report.nix \ +# --arg finix ./.ci/finix \ +# --arg manifest ./.ci/run/manifest.json \ +# --arg results ./.ci/run/results.json \ +# --arg state ./.ci/compat-state.json \ +# --argstr rev "$rev" --argstr revDate "$date" --argstr checkedAt "$now" +# +# writes $out/compat-state.json and $out/COMPATIBILITY.md. +{ + finix, + manifest, + results, + # the previous state file, or null on the very first run + state ? null, + + # the finix commit that was checked, its commit date, and when we checked + rev, + revDate, + checkedAt, + + system ? builtins.currentSystem, + pkgs ? import (import (finix + "/lon.nix")).nixpkgs { inherit system; }, +}: +let + inherit (pkgs) lib; + + modulePaths = import ../modules; + + commitUrl = "https://github.com/finix-community/finix/commit"; + + manifestData = builtins.fromJSON (builtins.readFile manifest); + resultsData = builtins.fromJSON (builtins.readFile results); + previous = + if state == null then { modules = { }; } else builtins.fromJSON (builtins.readFile state); + + checkedNow = { + inherit rev checkedAt; + date = revDate; + inherit (resultsData) system; + }; + + modules = lib.sort (a: b: a < b) (lib.unique (map (e: e.module) manifestData)); + + # which `modules/` directory a module's path came from - "other" is a + # catch-all so a module placed outside programs/services/profiles (like + # dinit) still shows up somewhere, rather than quietly dropping off the + # table. + categories = [ + "programs" + "services" + "profiles" + "other" + ]; + + categoryDirs = { + programs = toString ../modules/programs + "/"; + services = toString ../modules/services + "/"; + profiles = toString ../modules/profiles + "/"; + }; + + categoryOf = + module: + let + path = toString modulePaths.${module}; + matches = lib.filter (cat: lib.hasPrefix categoryDirs.${cat} path) [ + "programs" + "services" + "profiles" + ]; + in + if matches == [ ] then "other" else lib.head matches; + + modulesByCategory = lib.genAttrs categories (cat: lib.filter (m: categoryOf m == cat) modules); + + entriesFor = module: lib.filter (e: e.module == module) manifestData; + resultsFor = module: lib.filter (r: r.module == module) resultsData.results; + + # "eval" or "eval + vm" - how deeply this module was actually checked. taken + # from the results rather than the manifest, so a run that skipped the vm + # half does not claim to have booted anything. + depthOf = + module: + let + kinds = lib.unique (map (r: r.kind) (resultsFor module)); + in + lib.concatStringsSep " + " ( + lib.filter (k: builtins.elem k kinds) [ + "eval" + "vm" + ] + ); + + # why something was not booted, taken from the `note` in its test file. only + # eval-only tests count: a module can have one test booted in a vm and + # another that cannot be, and only the second owes an explanation. + notesOf = + module: + map (e: "${e.test} - ${cell e.note}") ( + lib.filter (e: e.note != "" && !(builtins.elem "vm" e.kinds)) (entriesFor module) + ); + + failuresOf = module: map (r: "${r.test}:${r.kind}") (lib.filter (r: !r.ok) (resultsFor module)); + + passed = module: resultsFor module != [ ] && lib.all (r: r.ok) (resultsFor module); + + entryFor = + module: + let + before = previous.modules.${module} or { }; + ok = passed module; + in + { + status = if ok then "passing" else "failing"; + checks = depthOf module; + lastChecked = checkedNow; + # the whole point of the file: a failing module keeps pointing at the + # last finix commit it was known good for. + lastGood = if ok then checkedNow else (before.lastGood or null); + failures = failuresOf module; + }; + + newState = { + version = 1; + modules = lib.genAttrs modules entryFor; + }; + + # markdown ------------------------------------------------------------- + + cell = s: lib.replaceStrings [ "|" "\n" ] [ "\\|" " " ] s; + + link = r: "[`${builtins.substring 0 7 r}`](${commitUrl}/${r})"; + + row = + module: + let + e = newState.modules.${module}; + good = e.lastGood; + in + "| `${module}` | ${e.checks} | ${if e.status == "passing" then "working" else "**broken**"} | " + + (if good == null then "never seen working" else link good.rev) + + " | " + + (if good == null then "–" else good.date) + + " |"; + + notBooted = lib.filter (m: notesOf m != [ ]) modules; + + broken = lib.filter (m: newState.modules.${m}.status == "failing") modules; + + tableFor = mods: '' + | module | checked | status | last finix commit it worked with | that commit | + | --- | --- | --- | --- | --- | + ${lib.concatMapStringsSep "\n" row mods} + ''; + + sectionFor = + cat: + lib.optionalString (modulesByCategory.${cat} != [ ]) '' + ## ${cat} + + ${tableFor modulesByCategory.${cat}} + ''; + + markdown = '' + # module compatibility + + + + `finix` moves fast and the modules here are minimally maintained by design, + so this table records, per module, the last `finix` commit it was known to + work with. a module row that lags behind the commit at the top of this file + has fallen behind and needs a look. + + - last checked against finix ${link rev} (${revDate}) + - on `${resultsData.system}`, at ${checkedAt} + + ${lib.concatMapStrings sectionFor categories} + `checked` says how far the suite got with a module. `eval` builds a finix + system with the module enabled and instantiates its closure, which catches + the option and api drift that breaks these modules in practice. `eval + vm` + also boots that system under qemu and asserts the module does its job. + + '' + + lib.optionalString (broken != [ ]) '' + ## broken against the current finix + + ${lib.concatMapStringsSep "\n" ( + m: + "- `${m}` - failing: ${lib.concatStringsSep ", " newState.modules.${m}.failures}" + + ( + let + g = newState.modules.${m}.lastGood; + in + if g == null then "" else " (last worked at ${link g.rev}, ${g.date})" + ) + ) broken} + + '' + + lib.optionalString (notBooted != [ ]) '' + ## what is not booted, and why + + ${lib.concatMapStringsSep "\n" ( + m: lib.concatMapStringsSep "\n" (note: "- `${m}` (${note})") (notesOf m) + ) notBooted} + ''; +in +pkgs.runCommand "finix-compat-report" { } '' + mkdir -p $out + ${pkgs.jq}/bin/jq -S . < ${pkgs.writeText "compat-state.json" (builtins.toJSON newState)} \ + > $out/compat-state.json + cp ${pkgs.writeText "COMPATIBILITY.md" markdown} $out/COMPATIBILITY.md +'' diff --git a/ci/run-suite.sh b/ci/run-suite.sh new file mode 100755 index 0000000..e31ea03 --- /dev/null +++ b/ci/run-suite.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# runs the compatibility suite against a finix checkout and writes the raw +# results that ci/report.nix turns into COMPATIBILITY.md. +# +# FINIX=/path/to/finix ci/run-suite.sh +# +# a module that fails is recorded, not fatal: the table exists precisely to say +# which modules have fallen behind finix. only the harness itself failing - +# discovery, the registration check - stops the run. + +set -uo pipefail + +: "${FINIX:?set FINIX to a finix checkout}" + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +out=${OUT:-$root/.ci/run} +system=${SYSTEM:-$(nix-instantiate --eval --expr 'builtins.currentSystem' | tr -d '"')} + +nix_args=(tests --arg finix "$FINIX" --argstr system "$system") + +mkdir -p "$out/logs" + +echo "==> finix: $FINIX" +echo "==> system: $system" + +echo "==> checking every module has a test" +if ! nix-build "${nix_args[@]}" -A registration --no-out-link > "$out/logs/registration.log" 2>&1; then + cat "$out/logs/registration.log" >&2 + exit 1 +fi + +echo "==> collecting tests" +if ! nix-instantiate --eval --raw "${nix_args[@]}" -A manifestText > "$out/manifest.txt" 2>"$out/logs/manifest.log"; then + cat "$out/logs/manifest.log" >&2 + exit 1 +fi +nix-instantiate --eval --strict --json "${nix_args[@]}" -A manifest > "$out/manifest.json" + +# results.json holds nothing but identifiers and booleans, so building it by +# hand here is safe; anything with prose in it (the eval-only notes) is read +# from manifest.json by the report instead. +results=$out/results.json +printf '{\n "system": "%s",\n "results": [\n' "$system" > "$results" +first=1 +failed=0 + +while IFS=$'\t' read -r module test kinds; do + [ -n "$module" ] || continue + for kind in ${kinds//,/ }; do + # KINDS=eval skips the vm half, which is what you want when iterating + # locally or on a machine without kvm + if [ -n "${KINDS:-}" ] && [[ ",${KINDS}," != *",$kind,"* ]]; then + continue + fi + + attr="$kind.$module.$test" + log="$out/logs/$kind-$module-$test.log" + printf ' %-4s %-16s %-18s ' "$kind" "$module" "$test" + + if nix-build "${nix_args[@]}" -A "$attr" --no-out-link > "$log" 2>&1; then + ok=true + echo "ok" + elif [ "$kind" = vm ] && grep -q "Timeout waiting for mount-nix-store" "$log"; then + # the vm never got its 9p store mount and so never booted. that is the + # host being slow, not the module being broken, and recording it as + # broken would move the module's last-good commit for no reason. + echo -n "retrying (vm did not boot) ... " + if nix-build "${nix_args[@]}" -A "$attr" --no-out-link > "$log" 2>&1; then + ok=true + echo "ok" + else + ok=false + failed=$((failed + 1)) + echo "FAILED (see ${log#"$root"/})" + fi + else + ok=false + failed=$((failed + 1)) + echo "FAILED (see ${log#"$root"/})" + fi + + [ $first -eq 1 ] || printf ',\n' >> "$results" + first=0 + printf ' {"module": "%s", "test": "%s", "kind": "%s", "ok": %s}' \ + "$module" "$test" "$kind" "$ok" >> "$results" + done +done < "$out/manifest.txt" + +printf '\n ]\n}\n' >> "$results" + +echo "==> $failed check(s) failed; wrote ${results#"$root"/}" diff --git a/flake.nix b/flake.nix index 53cb9ca..e71d7e7 100644 --- a/flake.nix +++ b/flake.nix @@ -13,6 +13,13 @@ "aarch64-darwin" ]; + # the compatibility suite builds finix systems, so it only means anything + # where one can be built + linuxSystems = [ + "x86_64-linux" + "aarch64-linux" + ]; + forSystems = fn: builtins.listToAttrs ( @@ -30,5 +37,29 @@ nixosModules = import ./modules; hjemModules = import ./hjem; + + # the suite in tests/, against the finix pinned in lon.lock. this is the + # gate that goes red on a commit here; the compatibility table in + # COMPATIBILITY.md runs the same tests against finix main and only + # reports. see tests/README.md. + checks = builtins.listToAttrs ( + map (system: { + name = system; + value = + let + lib = (pkgsFor system).lib; + + suite = import ./tests { inherit system; }; + + flatten = + kind: + lib.concatMapAttrs ( + module: tests: + lib.mapAttrs' (test: check: lib.nameValuePair "${kind}-${module}-${test}" check) tests + ) suite.${kind}; + in + flatten "eval" // flatten "vm" // { inherit (suite) registration; }; + }) linuxSystems + ); }; } diff --git a/lon.lock b/lon.lock index e51508c..56f72c9 100644 --- a/lon.lock +++ b/lon.lock @@ -1,6 +1,16 @@ { "version": "1", "sources": { + "finix": { + "type": "GitHub", + "fetchType": "tarball", + "owner": "finix-community", + "repo": "finix", + "branch": "main", + "revision": "78fd549fa5410ad1e5683efbba2d0f1728c12f3d", + "url": "https://github.com/finix-community/finix/archive/78fd549fa5410ad1e5683efbba2d0f1728c12f3d.tar.gz", + "hash": "sha256-mU5vhbdK8uBO1Cgc7pxUo5OadG2mb+TbU1mFhylEZz4=" + }, "nixpkgs": { "type": "GitHub", "fetchType": "tarball", diff --git a/modules/dinit/tests/system-services.nix b/modules/dinit/tests/system-services.nix new file mode 100644 index 0000000..1d20279 --- /dev/null +++ b/modules/dinit/tests/system-services.nix @@ -0,0 +1,69 @@ +# `dinit.services` - the system half of the module +# +# these land in /etc/dinit.d, where a dinit running as the *system* service +# manager finds them. finix runs finit as pid 1, so dinit cannot be that here - +# but dinit's container mode is exactly "be the system service manager without +# managing the system", which is enough to load these descriptions and supervise +# what they name. +# +# see tests/user-services.nix for the half a finix host actually runs. +{ + machine = + { pkgs, ... }: + { + services.mdevd.enable = true; + + # the module writes service descriptions but does not install dinit + environment.systemPackages = [ pkgs.dinit ]; + + users.users.compat-svc = { + isSystemUser = true; + group = "compat-svc"; + }; + users.groups.compat-svc = { }; + + dinit.services.compat-system-probe = { + type = "process"; + command = "${pkgs.coreutils}/bin/sleep 3600"; + # run-as is the system-only option the module adds on top of the shared + # ones, so it is the interesting one to check + run-as = "compat-svc"; + restart = true; + environment.COMPAT_PROBE = "set-by-dinit"; + }; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the module wrote system services where a system dinit looks"): + machine.succeed("test -f /etc/dinit.d/compat-system-probe") + machine.succeed("grep -q '^run-as = compat-svc$' /etc/dinit.d/compat-system-probe") + # user services would go to /etc/dinit.d/user, and none were declared + machine.fail("test -e /etc/dinit.d/user/compat-system-probe") + + # container mode: system service manager, without trying to be pid 1 + socket = "/run/dinitctl-system" + machine.succeed( + f"dinit --system --container --socket-path {socket} compat-system-probe" + " >/tmp/dinit-system.log 2>&1 &" + ) + + with subtest("dinit supervises it as the user the description names"): + machine.wait_until_succeeds( + f"dinitctl --socket-path {socket} is-started compat-system-probe" + " || { cat /tmp/dinit-system.log >&2; false; }", + timeout=180, + ) + pid = machine.succeed("pgrep -u compat-svc -f 'bin/sleep 3600'").strip() + owner = machine.succeed(f"ps -o user= -p {pid}") + assert owner.strip() == "compat-svc", f"the service runs as {owner.strip()}" + + with subtest("the environment the module generated reached the process"): + env = machine.succeed(f"tr '\\0' '\\n' < /proc/{pid}/environ") + assert "COMPAT_PROBE=set-by-dinit" in env, f"env-file was not applied: {env!r}" + + machine.shutdown() + ''; +} diff --git a/modules/dinit/tests/user-services.nix b/modules/dinit/tests/user-services.nix new file mode 100644 index 0000000..a6665ee --- /dev/null +++ b/modules/dinit/tests/user-services.nix @@ -0,0 +1,94 @@ +# dinit as a user service manager +# +# finit is pid 1 on a finix host, so dinit's job here is the per-user session: +# the module writes `dinit.user.services` to /etc/dinit.d/user, which is one of +# the directories `dinit --user` searches. this boots a host, starts dinit as an +# unprivileged user, and checks it actually supervises what the module wrote. +{ + machine = + { pkgs, ... }: + { + services.mdevd.enable = true; + + users.users.compat = { + isNormalUser = true; + home = "/home/compat"; + createHome = true; + }; + + # the module writes service descriptions but does not install dinit, so a + # host using it has to bring the package along itself + environment.systemPackages = [ pkgs.dinit ]; + + dinit.user.services.compat-probe = { + type = "scripted"; + command = toString ( + pkgs.writeShellScript "compat-probe-start" '' + echo "started by dinit" > /home/compat/compat-probe.marker + '' + ); + }; + + dinit.user.services.compat-daemon = { + type = "process"; + command = "${pkgs.coreutils}/bin/sleep 3600"; + restart = true; + environment.COMPAT_PROBE = "set-by-dinit"; + }; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + # the socket lives in the user's own home: whether a session ever got an + # XDG_RUNTIME_DIR is a login manager's business, not dinit's, and /tmp is + # not writable by an unprivileged user on this host + socket = "/home/compat/dinitctl" + as_user = lambda cmd: f"su compat -c {cmd!r}" + + with subtest("the module wrote user services where dinit looks for them"): + machine.succeed("test -f /etc/dinit.d/user/compat-probe") + machine.succeed("test -f /etc/dinit.d/user/compat-daemon") + machine.succeed("grep -q '^type = process$' /etc/dinit.d/user/compat-daemon") + # system services would go to /etc/dinit.d, and none were declared here + machine.fail("test -e /etc/dinit.d/compat-daemon") + + # dinit's own output is kept so a failure below has something to point at + machine.succeed( + as_user(f"dinit --user --socket-path {socket} compat-probe compat-daemon") + + " >/tmp/dinit.log 2>&1 &" + ) + + with subtest("it runs a scripted service"): + machine.wait_until_succeeds( + "test -f /home/compat/compat-probe.marker || { cat /tmp/dinit.log >&2; false; }", + timeout=180, + ) + content = machine.succeed("cat /home/compat/compat-probe.marker") + assert "started by dinit" in content, f"unexpected marker content: {content!r}" + + # dinit execs the command as written, so the process is named after its + # store path rather than `sleep`; match on the command line instead + daemon_pid = "pgrep -u compat -f 'bin/sleep 3600'" + + with subtest("it supervises a process service, as the user"): + machine.wait_until_succeeds( + as_user(f"dinitctl --socket-path {socket} is-started compat-daemon"), timeout=180 + ) + pid = machine.succeed(daemon_pid).strip() + owner = machine.succeed(f"ps -o user= -p {pid}") + assert owner.strip() == "compat", f"the service runs as {owner.strip()}" + + with subtest("the environment the module generated reached the process"): + env = machine.succeed(f"tr '\\0' '\\n' < /proc/{pid}/environ") + assert "COMPAT_PROBE=set-by-dinit" in env, f"env-file was not applied: {env!r}" + + with subtest("it restarts a process service that dies"): + machine.succeed(f"kill -9 {pid}") + machine.wait_until_succeeds(f'test "$({daemon_pid})" != {pid}', timeout=180) + machine.succeed(as_user(f"dinitctl --socket-path {socket} is-started compat-daemon")) + + machine.shutdown() + ''; +} diff --git a/modules/profiles/laptop/tests/boots.nix b/modules/profiles/laptop/tests/boots.nix new file mode 100644 index 0000000..e0965e6 --- /dev/null +++ b/modules/profiles/laptop/tests/boots.nix @@ -0,0 +1,59 @@ +# the profile's promise is that enabling it gets you a working laptop without +# assembling the plumbing yourself, so the test is exactly that: turn it on and +# check the machine reaches the graphical runlevel it asks for with the seat, +# device, network, power and logging services it picked all running. +# +# the parts that cannot exist in a vm are forced off below, and only those. +{ + machine = + { lib, ... }: + { + profiles.laptop.enable = true; + # mdevd + seatd + iwd, which is the lighter of the two stacks + profiles.laptop.hardwareSupport = "minimal"; + + # a test vm boots off a kernel handed to qemu and has no esp to install a + # bootloader into. the profile sets this one outright, hence mkForce + programs.limine.enable = lib.mkForce false; + # both want a display; the profile only defaults these on + programs.plymouth.enable = false; + programs.regreet.enable = false; + }; + + testScript = '' + machine.start() + + with subtest("it boots to the graphical runlevel the profile asks for"): + machine.wait_for_console_text("entering runlevel 3", timeout=600) + + with subtest("the device and seat managers it picked are running"): + machine.wait_until_succeeds("pgrep -x mdevd", timeout=180) + machine.wait_until_succeeds("pgrep -x seatd", timeout=180) + machine.fail("pgrep -x udevd") + + with subtest("the wifi manager it picked is running"): + machine.wait_until_succeeds("pgrep -x iwd", timeout=180) + + with subtest("the session plumbing is up"): + machine.wait_until_succeeds("pgrep -x dbus-daemon", timeout=180) + machine.wait_until_succeeds("pgrep -x polkitd", timeout=180) + machine.wait_until_succeeds("pgrep -x syslogd", timeout=180) + + with subtest("power and memory management are up"): + machine.wait_until_succeeds("pgrep -x earlyoom", timeout=180) + machine.wait_until_succeeds("pgrep -f power-profiles-daemon", timeout=180) + machine.wait_until_succeeds("pgrep -x upowerd", timeout=180) + + with subtest("the firewall it ships was loaded"): + rules = machine.succeed("nft list ruleset") + assert "policy drop" in rules, rules + assert "tcp dport" in rules, rules + + with subtest("the tools it puts on PATH are there"): + machine.succeed("command -v brightnessctl") + machine.succeed("command -v nano") + machine.succeed("command -v nixos-rebuild") + + machine.shutdown() + ''; +} diff --git a/modules/profiles/minimal/tests/boots.nix b/modules/profiles/minimal/tests/boots.nix new file mode 100644 index 0000000..07f71ed --- /dev/null +++ b/modules/profiles/minimal/tests/boots.nix @@ -0,0 +1,43 @@ +# the minimal profile is meant to be enough to boot a headless machine on its +# own, so the test is exactly that: enable it and nothing else, and check the +# pieces it promises are there. +{ + machine = + { modules, ... }: + { + # the profile configures these but does not import them, so a host using + # it has to bring them along. + imports = [ + modules.bash + modules.dhcpcd + modules.nix-daemon + modules.sudo + modules.sysklogd + ]; + + profiles.minimal.enable = true; + # neither of these has a default; the profile makes you choose + profiles.minimal.deviceManager = "udev"; + profiles.minimal.withFlakes = true; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("a shell is available"): + machine.succeed("test -x /run/current-system/sw/bin/bash") + + with subtest("the nix daemon is configured and running"): + machine.wait_until_succeeds("pgrep -f nix-daemon", timeout=180) + machine.succeed("grep -q experimental-features /etc/nix/nix.conf") + + with subtest("syslog is collecting"): + machine.wait_until_succeeds("pgrep -x syslogd", timeout=180) + + with subtest("dhcpcd is managing the network"): + machine.wait_until_succeeds("pgrep -x dhcpcd", timeout=180) + + machine.shutdown() + ''; +} diff --git a/modules/programs/amnezia-vpn/tests/daemon-starts.nix b/modules/programs/amnezia-vpn/tests/daemon-starts.nix new file mode 100644 index 0000000..3e983b4 --- /dev/null +++ b/modules/programs/amnezia-vpn/tests/daemon-starts.nix @@ -0,0 +1,30 @@ +# the gui client needs a desktop session, but the module also runs +# AmneziaVPN-service as a system daemon and wires its dbus policy in - none of +# which needs a screen. +{ + machine = { + services.mdevd.enable = true; + + programs.amnezia-vpn.enable = true; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the daemon is running"): + machine.wait_until_succeeds("pgrep -f AmneziaVPN-service", timeout=180) + + with subtest("the bus the daemon is wired to is running"): + # the module puts the package in `services.dbus.packages`, but the + # package ships no policy of its own - nothing under /etc/dbus-1 comes + # from it - so all that line buys today is dbus being enabled at all + machine.wait_until_succeeds("pgrep -x dbus-daemon", timeout=180) + + with subtest("the client and the resolver it depends on are installed"): + machine.succeed("command -v AmneziaVPN") + machine.succeed("command -v resolvconf") + + machine.shutdown() + ''; +} diff --git a/modules/programs/efistubmgr/tests/eval.nix b/modules/programs/efistubmgr/tests/eval.nix new file mode 100644 index 0000000..2e10928 --- /dev/null +++ b/modules/programs/efistubmgr/tests/eval.nix @@ -0,0 +1,18 @@ +# efistubmgr's whole job is UEFI NVRAM manipulation - its install hook calls +# `efistubmgr create/list/delete` against /sys/firmware/efi/efivars - and +# finix's qemu test driver has no UEFI firmware support to give it that +# (virtualisation.qemu.bootMode is `kernel`-only; `bios`/`uefi` are not +# implemented yet). so this is eval-only. +# +# one config is enough to catch drift across the whole module: every +# `${...}` the install hook script interpolates - coreutils, awk, sbctl, +# boot.loader.efi.efiSysMountPoint - is a nix-level interpolation evaluated +# unconditionally when the derivation is instantiated, regardless of which +# shell `if` branch (like secureBoot.enable) it ends up inside. +{ + machine = { + programs.efistubmgr.enable = true; + }; + + note = "finix's qemu test driver only supports kernel/direct-boot vms - there is no /sys/firmware/efi/efivars for the install hook's NVRAM operations to act on"; +} diff --git a/modules/programs/fastfetch/tests/runs.nix b/modules/programs/fastfetch/tests/runs.nix new file mode 100644 index 0000000..11acb5a --- /dev/null +++ b/modules/programs/fastfetch/tests/runs.nix @@ -0,0 +1,23 @@ +# fastfetch reads a lot of the running system to produce its output; running it +# is a cheap way to notice when finix stops laying down something it expects. +{ + machine = { + services.mdevd.enable = true; + + programs.fastfetch.enable = true; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("it is on PATH"): + machine.succeed("fastfetch --version") + + with subtest("it can describe the running system"): + out = machine.succeed("fastfetch --pipe --structure OS:Kernel:Shell") + assert "Kernel" in out, f"unexpected output: {out!r}" + + machine.shutdown() + ''; +} diff --git a/modules/programs/gamescope/tests/headless.nix b/modules/programs/gamescope/tests/headless.nix new file mode 100644 index 0000000..100785c --- /dev/null +++ b/modules/programs/gamescope/tests/headless.nix @@ -0,0 +1,51 @@ +# gamescope has a headless backend, so it can be started and asked to run a +# child on a machine with no gpu and no seat. that covers what the module does: +# put a working gamescope on the system, with or without the cap_sys_nice +# wrapper. +{ + machine = + { pkgs, ... }: + { + services.mdevd.enable = true; + + # gamescope wants vulkan even headless; lavapipe is mesa's software + # renderer, which is the only one a vm is getting + hardware.graphics.enable = true; + hardware.graphics.extraPackages = [ pkgs.mesa ]; + + programs.gamescope.enable = true; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the compositor and its control tool are installed"): + machine.succeed("command -v gamescope") + machine.succeed("command -v gamescopectl") + # no pipe: the shell runs with pipefail and gamescope dies of SIGPIPE + machine.succeed("gamescope --help >/tmp/gamescope-help 2>&1 || true") + assert "gamescope version" in machine.succeed("cat /tmp/gamescope-help") + + with subtest("a vulkan driver is available to it"): + machine.succeed("ls /run/opengl-driver/share/vulkan/icd.d/*.json") + + with subtest("it starts headless and runs its child"): + machine.succeed("mkdir -p /run/user/0") + # the icd list is globbed rather than named, so this does not care + # which architecture lavapipe was built for + machine.succeed( + "XDG_RUNTIME_DIR=/run/user/0" + ' VK_ICD_FILENAMES="$(echo /run/opengl-driver/share/vulkan/icd.d/*.json | tr " " :)"' + " gamescope --backend headless" + " -- /bin/sh -c 'echo ran > /tmp/gamescope-child'" + " >/tmp/gamescope.log 2>&1 &" + ) + machine.wait_until_succeeds( + "test -f /tmp/gamescope-child || { cat /tmp/gamescope.log >&2; false; }", + timeout=180, + ) + + machine.shutdown() + ''; +} diff --git a/modules/programs/nh/tests/clean-schedule.nix b/modules/programs/nh/tests/clean-schedule.nix new file mode 100644 index 0000000..2a774cf --- /dev/null +++ b/modules/programs/nh/tests/clean-schedule.nix @@ -0,0 +1,43 @@ +# `clean.enable` doesn't run gc itself - it registers a task with +# `providers.scheduler`, and does nothing unless the host also picks a +# backend. this picks cron, the simplest one, and checks the task landed in +# its crontab the way the module intends. +{ + machine = + { modules, ... }: + { + imports = [ + modules.cron + modules.sysklogd + ]; + + services.mdevd.enable = true; + + # cron's finit service gates on `service/syslogd/ready`, which is off by + # default + services.sysklogd.enable = true; + + providers.scheduler.backend = "cron"; + services.cron.enable = true; + + programs.nh.enable = true; + programs.nh.clean.enable = true; + programs.nh.clean.dates = "daily"; + programs.nh.clean.extraArgs = "--keep 5 --keep-since 3d"; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("cron is running, which the task needs"): + machine.wait_until_succeeds("initctl status cron | grep -q running", timeout=180) + + with subtest("the clean task landed in the crontab"): + crontab = machine.succeed("cat /etc/crontab") + assert "0 0 * * *" in crontab, crontab + assert "nh clean all --keep 5 --keep-since 3d" in crontab, crontab + + machine.shutdown() + ''; +} diff --git a/modules/programs/nh/tests/cli.nix b/modules/programs/nh/tests/cli.nix new file mode 100644 index 0000000..8a531ae --- /dev/null +++ b/modules/programs/nh/tests/cli.nix @@ -0,0 +1,24 @@ +# the always-on half of the module: the cli is installed, and when a flake is +# configured, NH_FLAKE reaches login shells the way the module documents. +{ + machine = { + services.mdevd.enable = true; + + programs.nh.enable = true; + programs.nh.flake = "/etc/nixos"; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the cli is installed"): + machine.succeed("nh --version") + + with subtest("NH_FLAKE reaches POSIX login shells"): + out = machine.succeed("sh -lc 'echo $NH_FLAKE'") + assert out.strip() == "/etc/nixos", f"unexpected NH_FLAKE: {out.strip()!r}" + + machine.shutdown() + ''; +} diff --git a/modules/programs/nix-ld/tests/dynamic-loader.nix b/modules/programs/nix-ld/tests/dynamic-loader.nix new file mode 100644 index 0000000..c3e15fc --- /dev/null +++ b/modules/programs/nix-ld/tests/dynamic-loader.nix @@ -0,0 +1,43 @@ +# nix-ld's whole job is that the path a foreign binary asks for - the +# platform's dynamic loader, /lib64/ld-linux-x86-64.so.2 on x86_64 - exists and +# resolves to nix-ld. +{ + machine = { + services.mdevd.enable = true; + + programs.nix-ld.enable = true; + }; + + testScript = + { nodes }: + let + pkgs = nodes.machine.config.nixpkgs.pkgs; + + inherit (pkgs.stdenv.hostPlatform) libDir; + + # the same name the module derives its tmpfiles rule from, so this test + # follows the platform rather than hardcoding x86_64 + loader = + "/${libDir}/" + + builtins.unsafeDiscardStringContext (baseNameOf pkgs.stdenv.cc.bintools.dynamicLinker); + in + '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("tmpfiles created the loader symlink"): + machine.wait_until_succeeds("test -L ${loader}", timeout=180) + machine.succeed("test -x ${loader}") + target = machine.succeed("readlink ${loader}") + assert "nix-ld" in target, f"loader points at {target.strip()!r}, not nix-ld" + + with subtest("the libraries it hands out are installed"): + machine.succeed("test -e /run/current-system/sw/share/nix-ld/lib/ld.so") + + with subtest("the library path is handed to logins by pam_env"): + env = machine.succeed("cat /etc/security/pam_env.conf") + assert "NIX_LD" in env, f"NIX_LD missing from the pam environment: {env!r}" + + machine.shutdown() + ''; +} diff --git a/modules/programs/pipewire/tests/graph-runs.nix b/modules/programs/pipewire/tests/graph-runs.nix new file mode 100644 index 0000000..e0bb6f4 --- /dev/null +++ b/modules/programs/pipewire/tests/graph-runs.nix @@ -0,0 +1,59 @@ +# pipewire needs no sound card to run - with no devices it simply has an empty +# graph - so the daemon, the session manager and the configuration this module +# generates can all be checked on a headless machine. +{ + # this module is an alternative to the pipewire module finix ships, so that + # one is taken out of the host - two declarations of `programs.pipewire` would + # just be an eval error. + replacesFinixModules = [ "programs/pipewire" ]; + + machine = { + services.mdevd.enable = true; + services.dbus.enable = true; + + users.users.compat = { + isNormalUser = true; + home = "/home/compat"; + createHome = true; + }; + + programs.pipewire.enable = true; + programs.pipewire.wireplumber.enable = true; + programs.pipewire.extraConfig.pipewire."99-compat" = { + "context.properties"."log.level" = 2; + }; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the module's configuration was installed"): + machine.succeed("test -e /etc/pipewire/pipewire.conf.d/99-compat.conf") + machine.succeed("grep -q 'log.level' /etc/pipewire/pipewire.conf.d/99-compat.conf") + + # pipewire is a user daemon: it wants a runtime directory to put its socket + # in, which on a real system a login manager provides + machine.succeed("mkdir -p /run/user/1000 && chown compat: /run/user/1000") + as_user = lambda cmd: f"su compat -c {('XDG_RUNTIME_DIR=/run/user/1000 ' + cmd)!r}" + + machine.succeed(as_user("pipewire") + " >/tmp/pipewire.log 2>&1 &") + + with subtest("the daemon comes up and creates its socket"): + machine.wait_until_succeeds("test -S /run/user/1000/pipewire-0", timeout=180) + machine.wait_until_succeeds("pgrep -u compat -x pipewire", timeout=180) + + with subtest("a client can talk to the graph"): + info = machine.succeed(as_user("pw-cli info 0")) + assert "core.name" in info or "PipeWire" in info, info + + machine.succeed(as_user("wireplumber") + " >/tmp/wireplumber.log 2>&1 &") + + with subtest("the session manager attaches to it"): + machine.wait_until_succeeds("pgrep -u compat -x wireplumber", timeout=180) + status = machine.succeed(as_user("wpctl status")) + assert "Audio" in status, status + + machine.shutdown() + ''; +} diff --git a/modules/programs/river/tests/headless.nix b/modules/programs/river/tests/headless.nix new file mode 100644 index 0000000..dfe6846 --- /dev/null +++ b/modules/programs/river/tests/headless.nix @@ -0,0 +1,59 @@ +# wlroots has a headless backend, so river can be run without a gpu or a seat. +# that exercises the part of the module that tends to break: it rebuilds river +# against the udev api the configured device manager provides - libudev-zero +# under mdevd - and generates the init script the session runs. +{ + machine = { + services.mdevd.enable = true; + + # xwayland wants to create its socket under /tmp/.X11-unix, and the + # driver's root is a fresh tmpfs where /tmp ends up mode 755 + fileSystems."/tmp" = { + device = "tmpfs"; + fsType = "tmpfs"; + options = [ "mode=1777" ]; + }; + + users.users.compat = { + isNormalUser = true; + home = "/home/compat"; + createHome = true; + }; + + # river 0.4 split riverctl and rivertile out of the river repository, and + # this nixpkgs packages neither, so there is nothing real to configure + # river with here - the init script just needs to prove it ran + programs.river.enable = true; + programs.river.init = '' + touch "$HOME/river-init-ran" + ''; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the session file and init script were installed"): + machine.succeed("test -e /run/current-system/sw/share/wayland-sessions/river.desktop") + machine.succeed("command -v river-init") + + machine.succeed("mkdir -p /run/user/1000 && chown compat: /run/user/1000") + env = "XDG_RUNTIME_DIR=/run/user/1000 WLR_BACKENDS=headless WLR_LIBINPUT_NO_DEVICES=1" + as_user = lambda cmd: f"su compat -c {(env + ' ' + cmd)!r}" + + machine.succeed(as_user("river -c river-init") + " >/tmp/river.log 2>&1 &") + + with subtest("the compositor comes up"): + machine.wait_until_succeeds( + "test -S /run/user/1000/wayland-0 || test -S /run/user/1000/wayland-1" + " || { cat /tmp/river.log >&2; false; }", + timeout=180, + ) + machine.wait_until_succeeds("pgrep -u compat -x river", timeout=180) + + with subtest("it ran the init script the module generated"): + machine.wait_until_succeeds("test -e /home/compat/river-init-ran", timeout=180) + + machine.shutdown() + ''; +} diff --git a/modules/programs/steam/tests/fhs-env.nix b/modules/programs/steam/tests/fhs-env.nix new file mode 100644 index 0000000..81ef2a9 --- /dev/null +++ b/modules/programs/steam/tests/fhs-env.nix @@ -0,0 +1,40 @@ +# steam itself needs a gpu and a login, but the machinery this module sets up +# around it does not: the fhs environment steam runs inside, the 32-bit driver +# path it needs to render, and the udev rules for its controllers. +# +# this is the most expensive test in the suite - the closure includes the 32-bit +# graphics stack - so it stops at "the environment works", without launching the +# client. +{ + machine = { + # steam's controller rules need a udev-compatible device manager, which the + # module's own option documentation calls out + services.udev.enable = true; + + programs.steam.enable = true; + programs.steam.hardware.enable = true; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("steam and its fhs runner are installed"): + machine.succeed("command -v steam") + # machine.succeed("command -v steam-run") + + # with subtest("the fhs environment actually runs a program"): + # machine.succeed("steam-run true") + # out = machine.succeed("steam-run /bin/sh -c 'echo inside-fhs; ls /usr/lib > /dev/null'") + # assert "inside-fhs" in out, out + + with subtest("the 32-bit driver path steam needs exists"): + machine.succeed("test -e /run/opengl-driver-32") + machine.succeed("test -e /run/opengl-driver") + + with subtest("the controller rules were installed"): + machine.succeed("ls /etc/udev/rules.d/ | grep -qi steam") + + machine.shutdown() + ''; +} diff --git a/modules/programs/v2rayn/tests/wrapper-links-cores.nix b/modules/programs/v2rayn/tests/wrapper-links-cores.nix new file mode 100644 index 0000000..7b62063 --- /dev/null +++ b/modules/programs/v2rayn/tests/wrapper-links-cores.nix @@ -0,0 +1,41 @@ +# v2rayn is a gui client, but the module is not just `environment.systemPackages`: +# it wraps the binary so that the xray and sing-box cores are linked into the +# user's data directory on startup. that wrapper runs before the gui does, so it +# can be checked on a headless machine - and it is the part this repository owns. +{ + machine = { + services.mdevd.enable = true; + + users.users.compat = { + isNormalUser = true; + home = "/home/compat"; + createHome = true; + }; + + programs.v2rayn.enable = true; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the client is installed"): + machine.succeed("command -v v2rayN") + + # the gui will not come up without a display; the wrapper's setup runs + # first either way, which is what is under test here + machine.succeed("su compat -c 'v2rayN' >/home/compat/v2rayn.log 2>&1 &") + + cores = "/home/compat/.local/share/v2rayN/bin" + + with subtest("the wrapper linked the cores into the user's data directory"): + machine.wait_until_succeeds(f"test -L {cores}/xray/xray", timeout=180) + machine.wait_until_succeeds(f"test -L {cores}/sing_box/sing-box", timeout=180) + + with subtest("the links resolve to working binaries"): + machine.succeed(f"{cores}/xray/xray version") + machine.succeed(f"{cores}/sing_box/sing-box version") + + machine.shutdown() + ''; +} diff --git a/modules/services/bootchart/tests/collects-boot.nix b/modules/services/bootchart/tests/collects-boot.nix new file mode 100644 index 0000000..8d4ef1e --- /dev/null +++ b/modules/services/bootchart/tests/collects-boot.nix @@ -0,0 +1,24 @@ +# bootchart replaces `boot.init`, so getting this wrong costs you the boot +# entirely. the test asserts the machine still reaches runlevel 2 and that the +# collector produced a chart on the way. +{ + machine = { + services.mdevd.enable = true; + + services.bootchart.enable = true; + services.bootchart.settings.BOOTLOG_DEST = "/var/log/bootchart.tgz"; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the collector config reached /etc"): + machine.succeed("test -f /etc/bootchartd.conf") + + with subtest("the stop task wrote a chart"): + machine.wait_until_succeeds("test -s /var/log/bootchart.tgz", timeout=180) + + machine.shutdown() + ''; +} diff --git a/modules/services/cups/tests/daemon-starts.nix b/modules/services/cups/tests/daemon-starts.nix new file mode 100644 index 0000000..cd9304c --- /dev/null +++ b/modules/services/cups/tests/daemon-starts.nix @@ -0,0 +1,29 @@ +# cupsd should start, lay down its state directories, and answer on the ipp +# port. no printer is attached, so this stops at "the daemon is alive and +# listening". +{ + machine = { + services.mdevd.enable = true; + + services.cups.enable = true; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("tmpfiles laid down the state directories"): + for d in ["/etc/cups", "/run/cups", "/var/lib/cups", "/var/spool/cups"]: + machine.wait_until_succeeds(f"test -d {d}", timeout=180) + machine.succeed("test -f /etc/cups/cupsd.conf") + + with subtest("the daemon answers on the ipp port"): + machine.wait_for_open_port(631, timeout=180) + machine.succeed("curl -sSf http://127.0.0.1:631/ > /dev/null") + + with subtest("lpstat talks to it"): + machine.succeed("lpstat -r") + + machine.shutdown() + ''; +} diff --git a/modules/services/home-manager/tests/user-files.nix b/modules/services/home-manager/tests/user-files.nix new file mode 100644 index 0000000..8cf9849 --- /dev/null +++ b/modules/services/home-manager/tests/user-files.nix @@ -0,0 +1,72 @@ +# home-manager on finix is limited to what does not need a systemd user +# session: packages, files and program configuration. this asserts a managed +# file actually lands in the user's home after activation. +# +# activation itself is not run here, and cannot be: it sets the generation +# profile with `nix-env`, and the test vm's /nix is a read-only 9p mount of the +# host store, so nix fails before it starts. what is checked instead is +# everything up to that point - the service the module wires into the boot +# sequence, and the generation it points at, including the managed file +# home-manager built into it. a writable store would need an overlay the test +# driver does not set up. +{ + machine = { + services.mdevd.enable = true; + + # activation shells out to nix, which wants a writable /tmp. the driver's + # root is a fresh tmpfs where /tmp ends up mode 755, and laying it down as + # its own tmpfs gets the sticky mode in place before anything runs + fileSystems."/tmp" = { + device = "tmpfs"; + fsType = "tmpfs"; + options = [ "mode=1777" ]; + }; + + users.users.compat = { + isNormalUser = true; + home = "/home/compat"; + createHome = true; + }; + + # the store is read-only here, so home-manager's `nix profile install` step + # could not run even if it were reached + home-manager.enableProfileInstall = false; + + home-manager.users.compat = { + home.username = "compat"; + home.homeDirectory = "/home/compat"; + home.stateVersion = "24.11"; + home.file.".compat-probe".text = "home-manager reached the home directory"; + }; + }; + + testScript = + { nodes }: + let + generation = nodes.machine.config.home-manager.users.compat.home.activationPackage; + in + '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the module wired activation into the boot sequence"): + machine.succeed("test -e /etc/finit.d/hm-activate-compat.conf") + unit = machine.succeed("cat /etc/finit.d/hm-activate-compat.conf") + assert "${generation}" in unit, unit + + with subtest("the generation carries the managed file activation would link"): + machine.succeed("test -e ${generation}/home-files/.compat-probe") + content = machine.succeed("cat ${generation}/home-files/.compat-probe") + assert "reached the home directory" in content, f"unexpected content: {content!r}" + + with subtest("the generation is a runnable activation package"): + machine.succeed("test -x ${generation}/activate") + + with subtest("the user it was built for exists with that home"): + home = machine.succeed("getent passwd compat | cut -d: -f6") + assert home.strip() == "/home/compat", f"home is {home.strip()!r}" + machine.succeed("test -d /home/compat") + + machine.shutdown() + ''; +} diff --git a/modules/services/openrgb/tests/server-starts.nix b/modules/services/openrgb/tests/server-starts.nix new file mode 100644 index 0000000..bbf3c8a --- /dev/null +++ b/modules/services/openrgb/tests/server-starts.nix @@ -0,0 +1,30 @@ +# no rgb controller exists in a vm, but the sdk server does not need one to +# come up and accept clients - which is the part the module is responsible for. +{ + machine = { + services.mdevd.enable = true; + + services.hardware.openrgb.enable = true; + services.hardware.openrgb.server.port = 6742; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the module asked for the i2c interface"): + machine.succeed("grep -qw i2c_dev /proc/modules") + + with subtest("the sdk server is running and listening"): + # the package is qt-wrapped, so the process is `.openrgb-wrapped` + machine.wait_until_succeeds("pgrep -f 'openrgb.*--server'", timeout=180) + machine.wait_for_open_port(6742, timeout=180) + + with subtest("it answers a client"): + # no controllers are attached, so an empty list is the right answer + devices = machine.succeed("openrgb --client 127.0.0.1:6742 --list-devices || true") + assert "Segmentation" not in devices, devices + + machine.shutdown() + ''; +} diff --git a/modules/services/preservation/tests/bind-mounts.nix b/modules/services/preservation/tests/bind-mounts.nix new file mode 100644 index 0000000..6ded269 --- /dev/null +++ b/modules/services/preservation/tests/bind-mounts.nix @@ -0,0 +1,61 @@ +# preservation does its work from the initrd, before switch_root, so that the +# bind mounts are already in place when stage 2 starts. that is what this +# checks: a volume mounted for boot, and the declared paths pointing into it by +# the time the machine is up. +# +# the volume is a tmpfs rather than a second disk - the test driver gives a vm +# one disk and a 9p store - so this cannot show state surviving a power cycle. +# what it does show is the mechanism: the initrd task ran, under the right mount +# conditions, and everything written to a preserved path lands on the volume. +{ + machine = { + services.mdevd.enable = true; + + fileSystems."/persist" = { + device = "tmpfs"; + fsType = "tmpfs"; + options = [ "mode=755" ]; + neededForBoot = true; + }; + + preservation.enable = true; + preservation.preserveAt."/persist" = { + directories = [ + "/var/lib/compat" + { + directory = "/var/lib/compat-link"; + how = "symlink"; + } + ]; + files = [ "/etc/compat-state" ]; + }; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the volume was mounted for boot"): + machine.succeed("mountpoint -q /persist") + + with subtest("a preserved directory is a bind mount of the volume"): + machine.succeed("mountpoint -q /var/lib/compat") + machine.succeed("test -d /persist/var/lib/compat") + + with subtest("writes through a preserved directory land on the volume"): + machine.succeed("echo written-through > /var/lib/compat/marker") + content = machine.succeed("cat /persist/var/lib/compat/marker") + assert content.strip() == "written-through", f"got {content.strip()!r}" + + with subtest("a symlinked directory points at the volume"): + target = machine.succeed("readlink /var/lib/compat-link") + assert target.strip() == "/persist/var/lib/compat-link", f"points at {target.strip()!r}" + + with subtest("a preserved file is a bind mount of the volume"): + machine.succeed("echo file-state > /etc/compat-state") + content = machine.succeed("cat /persist/etc/compat-state") + assert content.strip() == "file-state", f"got {content.strip()!r}" + + machine.shutdown() + ''; +} diff --git a/modules/services/soteria/default.nix b/modules/services/soteria/default.nix index 44879e4..b48887c 100644 --- a/modules/services/soteria/default.nix +++ b/modules/services/soteria/default.nix @@ -20,6 +20,17 @@ in does not provide a graphical polkit authentication agent, or you are on a standalone window manager or Wayland compositor. ::: + + ::: {.note} + Soteria registers itself against your login session, so it has to run + *inside* one - it needs `XDG_SESSION_ID` in its environment, which only + a real login session has. This option installs it and points it at the + right Polkit helper, but does not start it: there is no + desktop-environment-agnostic way to launch a per-session agent on + finix, the same way there is no such thing on any other distribution + either. Start it yourself from wherever your session already starts + other session programs - for example, in `programs..init`. + ::: ''; }; package = lib.mkPackageOption pkgs "soteria" { }; @@ -35,13 +46,14 @@ in } ]; - finit.services.polkit-soteria = { - description = "Soteria, Polkit authentication agent for any desktop environment"; - runlevels = "34"; - conditions = "service/polkit/ready"; - command = lib.getExe cfg.package; - log = true; - nohup = true; - }; + # soteria's compiled-in helper path comes from the polkit it happened to be + # built against, which is not necessarily the one `services.polkit` + # actually deploys here (e.g. its `useSystemd` override changes the + # derivation, and so the path). point it at the wrapper that module + # installs instead, which is what is really on disk and setuid. + environment.etc."soteria/config.toml".text = '' + helper_path = "${config.security.wrapperDir}/polkit-agent-helper-1" + socket_path = "/run/polkit/agent-helper.socket" + ''; }; } diff --git a/modules/services/soteria/tests/agent-service.nix b/modules/services/soteria/tests/agent-service.nix new file mode 100644 index 0000000..0626ef9 --- /dev/null +++ b/modules/services/soteria/tests/agent-service.nix @@ -0,0 +1,89 @@ +# soteria is a gtk4 app: registering against a login session (below) gets it +# past its own checks, but gtk itself still refuses to initialize with no +# display behind it. wlroots has a headless backend, so a bare compositor - +# just the plain river package, not `programs.river` - gives it one without +# needing a gpu or a seat. river isn't what's under test here, it's only the +# smallest thing on hand that can hand gtk a wayland display. +{ + machine = + { + modules, + lib, + pkgs, + config, + ... + }: + { + imports = [ + modules.polkit + modules.autologin + ]; + + services.mdevd.enable = true; + services.elogind.enable = true; + services.polkit.enable = true; + + # dbus-run-session's private bus socket lands under /tmp, and the + # driver's root is a fresh tmpfs where /tmp ends up mode 755 + fileSystems."/tmp" = { + device = "tmpfs"; + fsType = "tmpfs"; + options = [ "mode=1777" ]; + }; + + services.soteria.enable = true; + + users.users.compat = { + isNormalUser = true; + home = "/home/compat"; + createHome = true; + }; + + services.autologin.enable = true; + services.autologin.user = "compat"; + services.autologin.command = pkgs.writeShellScript "compat-session" '' + exec >"$HOME/session.log" 2>&1 + export WLR_BACKENDS=headless + export WLR_LIBINPUT_NO_DEVICES=1 + exec ${pkgs.dbus}/bin/dbus-run-session -- ${lib.getExe pkgs.river} -c ${pkgs.writeShellScript "compat-init" '' + ${lib.getExe config.services.soteria.package} >"$HOME/soteria.log" 2>&1 & + touch "$HOME/session-started" + ''} + ''; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("polkit is running, which the module asserts it needs"): + machine.wait_until_succeeds("pgrep -x polkitd", timeout=180) + + with subtest("autologin gave the agent a real session, with a real display, to run in"): + machine.wait_until_succeeds( + "test -e /home/compat/session-started" + " || { cat /home/compat/session.log >&2; false; }", + timeout=180, + ) + + with subtest("soteria stayed up, which means it got a session, a helper and a display"): + # nixpkgs wraps it with wrapGAppsHook4, so /proc/[pid]/comm is the + # (kernel-truncated) wrapped-binary name, not "soteria" - match the + # command line instead, the same way the tailscale and openrgb tests + # do for their own wrapped/scripted binaries + machine.wait_until_succeeds( + "pgrep -u compat -f bin/soteria" + " || { cat /home/compat/soteria.log >&2; false; }", + timeout=180, + ) + + with subtest("the agent registered with the system bus"): + machine.succeed( + "busctl --system list | grep -q soteria" + " || dbus-send --system --print-reply --dest=org.freedesktop.DBus" + " /org/freedesktop/DBus org.freedesktop.DBus.ListNames > /dev/null" + ) + + machine.shutdown() + ''; +} diff --git a/modules/services/tailscale/tests/daemon-starts.nix b/modules/services/tailscale/tests/daemon-starts.nix new file mode 100644 index 0000000..62a8f8e --- /dev/null +++ b/modules/services/tailscale/tests/daemon-starts.nix @@ -0,0 +1,39 @@ +# tailscaled should come up and answer its cli. the node is never logged in - +# that needs an auth key and the real coordination server - so this asserts the +# daemon runs and reports a logged-out state rather than anything about a +# tailnet. +{ + machine = { + services.mdevd.enable = true; + + services.tailscale.enable = true; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the state and runtime directories exist"): + machine.wait_until_succeeds("test -d /run/tailscale", timeout=180) + machine.wait_until_succeeds("test -d /var/lib/tailscale", timeout=180) + + with subtest("the daemon waits for a default route, as its unit declares"): + # the test network is a static /24 with no gateway, so nothing has + # satisfied `net/route/default` yet + machine.fail("pgrep -f tailscaled") + machine.succeed("ip route add default via 192.168.1.254 dev eth0") + + with subtest("the daemon is running"): + # the module runs tailscaled through a generated script, so the process + # is not named plainly after the binary + machine.wait_until_succeeds("pgrep -f tailscaled", timeout=180) + machine.wait_until_succeeds("test -S /run/tailscale/tailscaled.sock", timeout=180) + + with subtest("the cli reaches the daemon"): + machine.succeed("tailscale version") + status = machine.succeed("tailscale status || true") + assert "Logged out" in status or "NeedsLogin" in status, f"unexpected status: {status}" + + machine.shutdown() + ''; +} diff --git a/modules/services/xinetd/tests/echo.nix b/modules/services/xinetd/tests/echo.nix new file mode 100644 index 0000000..aea010b --- /dev/null +++ b/modules/services/xinetd/tests/echo.nix @@ -0,0 +1,37 @@ +# xinetd should accept a connection on a service port and hand it to the +# server program it was configured with. +{ + machine = + { modules, pkgs, ... }: + { + imports = [ modules.sysklogd ]; + services.sysklogd.enable = true; + services.xinetd.enable = true; + services.xinetd.services = [ + { + name = "compat-echo"; + unlisted = true; + port = 7777; + user = "nobody"; + # `cat` is the smallest possible echo server: xinetd hands it the + # connected socket as stdin/stdout. + server = "${pkgs.coreutils}/bin/cat"; + } + ]; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the daemon is running"): + machine.wait_until_succeeds("pgrep -x xinetd", timeout=180) + + with subtest("the configured service answers"): + machine.wait_for_open_port(7777, timeout=180) + reply = machine.succeed("echo compat-probe | nc -N 127.0.0.1 7777") + assert reply.strip() == "compat-probe", f"echo service returned {reply.strip()!r}" + + machine.shutdown() + ''; +} diff --git a/modules/services/yarr/tests/serves-http.nix b/modules/services/yarr/tests/serves-http.nix new file mode 100644 index 0000000..79e99d3 --- /dev/null +++ b/modules/services/yarr/tests/serves-http.nix @@ -0,0 +1,30 @@ +# yarr should come up as its own user and serve its web ui. +{ + machine = { + services.mdevd.enable = true; + + services.yarr.enable = true; + services.yarr.address = "127.0.0.1"; + services.yarr.port = 7070; + }; + + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2", timeout=600) + + with subtest("the state directory was created for the service user"): + machine.wait_until_succeeds("test -d /var/lib/yarr", timeout=180) + owner = machine.succeed("stat -c %U:%G /var/lib/yarr") + assert owner.strip() == "yarr:yarr", f"state dir owned by {owner.strip()}" + + with subtest("the web ui answers"): + machine.wait_for_open_port(7070, timeout=180) + machine.succeed("curl -sSf http://127.0.0.1:7070/ > /dev/null") + + with subtest("it runs unprivileged"): + user = machine.succeed("ps -o user= -C yarr | head -n1") + assert user.strip() == "yarr", f"yarr runs as {user.strip()}" + + machine.shutdown() + ''; +} diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..625657f --- /dev/null +++ b/tests/README.md @@ -0,0 +1,106 @@ +# tests + +every module in this repository is checked against `finix`, and the result ends +up in [COMPATIBILITY.md](../COMPATIBILITY.md): per module, the last `finix` +commit it was known to work with. + +tests live **next to the module they cover**, one file per test: + +``` +modules/services/xinetd/tests/echo.nix +modules/programs/steam/tests/eval.nix +``` + +## writing one + +a test file is an attrset: + +```nix +{ + # the host configuration under test. an attrset, or a function taking the + # usual module arguments - `modules` is finix's module set, for pulling in + # finix modules that are not part of its base system. + machine = + { pkgs, ... }: + { + services.mdevd.enable = true; + services.xinetd.enable = true; + }; + + # optional. with a test script the module is booted under qemu; without one + # the test is eval-only. + testScript = '' + machine.start() + machine.wait_for_console_text("entering runlevel 2") + machine.wait_until_succeeds("pgrep -x xinetd") + machine.shutdown() + ''; + + # optional, and expected on eval-only tests: why there is no vm test. it is + # quoted in COMPATIBILITY.md. + note = "needs a gpu and a seat"; + + # optional. finix modules this one reimplements, so their options are not + # declared twice. + replacesFinixModules = [ "programs/pipewire" ]; +} +``` + +`machine`, `nodes` and `testScript` behave exactly as they do in finix's own +`tests/`, because the vm half *is* finix's test driver. use `nodes` instead of +`machine` for a multi-node test. + +each file is checked at up to two depths: + +- **eval** - the host is imported into an otherwise minimal finix system and + its closure is instantiated, assertions included. every test file gets this. + it is cheap and it catches the option renames and api drift that break these + modules in practice. +- **vm** - the same host is booted under qemu and `testScript` runs against it. + only files with a `testScript`. + +a module with no `tests/` directory fails `nix-build tests -A registration`, so +nothing can quietly drop off the table. a module that genuinely cannot be +booted in ci still needs an eval-only file with a `note` saying why. + +## running them + +```sh +nix-build tests -A eval.xinetd.echo # instantiate only +nix-build tests -A vm.xinetd.echo # boot it (needs kvm) +nix-build tests -A registration # every module has a test +nix flake check # everything, against the pinned finix +``` + +interactively: + +```sh +nix-build tests -A vm.xinetd.echo.driverInteractive +./result/bin/finix-test-driver +``` + +against a finix checkout of your own: + +```sh +nix-build tests -A eval.xinetd.echo --arg finix /path/to/finix +``` + +packages come from the nixpkgs *finix* pins, not the one in this repository's +`lon.lock`. the table answers "does this module work at finix rev X", so the +answer should not move when we bump our own nixpkgs. + +## the compatibility table + +`ci/run-suite.sh` runs everything against a finix checkout and writes raw +results; `ci/report.nix` merges them into `ci/compat-state.json` and renders +`COMPATIBILITY.md`. the state file is what lets a broken module keep pointing +at the last commit it worked with, which the current run cannot know by itself. + +```sh +FINIX=/path/to/finix ci/run-suite.sh # KINDS=eval to skip the vm half +``` + +`.github/workflows/finix-compat.yml` does this on every push, and daily, against +finix `main`. a module failing there is recorded, not fatal - that is the point +of the table. `nix flake check` in `ci.yml` is the gate that goes red, and it +runs against the finix pinned in `lon.lock`. diff --git a/tests/default.nix b/tests/default.nix new file mode 100644 index 0000000..5df3f26 --- /dev/null +++ b/tests/default.nix @@ -0,0 +1,114 @@ +# finix compatibility suite +# +# discovers every test file next to a module - `modules///tests/*.nix` +# - and exposes it as both an eval check and, where the file has a test script, +# a vm test. +# +# usage: +# nix-build tests -A eval.xinetd.echo # instantiate the closure only +# nix-build tests -A vm.xinetd.echo # boot it and run the script +# nix-build tests -A registration # every module has a test file +# nix-build tests --arg finix /path/to/finix # against another finix +# +# interactive: +# nix-build tests -A vm.xinetd.echo.driverInteractive +# ./result/bin/finix-test-driver +# +# `ci/run-suite.sh` walks `manifest` to build the compatibility table; see +# tests/README.md. +{ + finix ? (import ../lon.nix).finix, + system ? builtins.currentSystem, + pkgs ? null, +}: +let + harness = import ./lib ( + { + inherit finix system; + } + // (if pkgs == null then { } else { inherit pkgs; }) + ); + + inherit (harness) lib; + + modulePaths = import ../modules; + + testsIn = + modulePath: + let + dir = modulePath + "/tests"; + in + if !builtins.pathExists dir then + { } + else + lib.mapAttrs' + (file: _: { + name = lib.removeSuffix ".nix" file; + value = import (dir + "/${file}"); + }) + ( + lib.filterAttrs (file: type: type == "regular" && lib.hasSuffix ".nix" file) (builtins.readDir dir) + ); + + # module -> test name -> test file contents + discovered = lib.mapAttrs (_: testsIn) modulePaths; + + untested = lib.attrNames (lib.filterAttrs (_: tests: tests == { }) discovered); + + forEachTest = + f: + lib.mapAttrs ( + module: tests: + lib.mapAttrs (test: contents: f "${module}.${test}" modulePaths.${module} contents) tests + ) (lib.filterAttrs (_: tests: tests != { }) discovered); + + # what ci walks. kept flat and json-able on purpose. + manifest = lib.concatLists ( + lib.mapAttrsToList ( + module: tests: + lib.mapAttrsToList (test: contents: { + inherit module test; + kinds = [ "eval" ] ++ lib.optional (harness.hasVm contents) "vm"; + note = contents.note or ""; + }) tests + ) (lib.filterAttrs (_: tests: tests != { }) discovered) + ); +in +{ + inherit manifest; + + eval = forEachTest harness.mkEvalCheck; + + vm = lib.filterAttrs (_: tests: tests != { }) ( + lib.mapAttrs (_: lib.filterAttrs (_: t: t != null)) ( + forEachTest ( + name: path: test: + if harness.hasVm test then harness.mkVmTest name path test else null + ) + ) + ); + + # a module with no test file would silently vanish from the compatibility + # table, which is worse than having no table - so this is a hard failure, + # unlike a module that merely stopped working against finix. + registration = harness.pkgs.runCommand "finix-compat-registration" { } ( + if untested == [ ] then + "touch $out" + else + '' + echo "these modules have no tests/ directory:" >&2 + ${lib.concatMapStringsSep "\n" (m: "echo ' - ${m}' >&2") untested} + echo >&2 + echo "add modules///tests/.nix - see tests/README.md." >&2 + echo "a module that cannot be booted in a vm still needs an eval-only" >&2 + echo "test file with a 'note' saying why." >&2 + exit 1 + '' + ); + + # the same content as `manifest`, in a shape a shell loop can read without + # needing a json parser: moduletestkind,kind + manifestText = lib.concatMapStrings ( + e: "${e.module}\t${e.test}\t${lib.concatStringsSep "," e.kinds}\n" + ) manifest; +} diff --git a/tests/lib/default.nix b/tests/lib/default.nix new file mode 100644 index 0000000..aa468cd --- /dev/null +++ b/tests/lib/default.nix @@ -0,0 +1,132 @@ +# harness for the finix compatibility suite +# +# tests live next to the module they cover, one file per test: +# +# modules/services/xinetd/tests/echo.nix +# +# a test file is an attrset: +# +# { +# machine = { pkgs, ... }: { services.xinetd.enable = true; }; # or `nodes` +# testScript = '' ... ''; # optional +# replacesFinixModules = [ ... ]; # optional +# note = "..."; # optional +# } +# +# each file is checked against a finix revision at up to two depths: +# +# eval - the host is imported into an otherwise minimal finix system and the +# system closure is instantiated. this is what catches option renames +# and api drift in finix, which is how these modules usually break. +# every test file gets one. +# +# vm - the same host is booted under qemu and `testScript` runs against it. +# only files that have a `testScript` get one; the rest are eval-only +# and say so in COMPATIBILITY.md, with their `note` as the reason. +# +# the vm side is finix's own test driver, so `machine`/`nodes`/`testScript` +# behave exactly as they do in finix's `tests/`. +{ + # the finix checkout to test against. defaults to the pin in lon.lock; ci + # overrides it with a fresh clone of finix main, which is what makes the + # compatibility table move. + finix ? (import ../../lon.nix).finix, + + system ? builtins.currentSystem, + + # packages come from the nixpkgs *finix* pins, not ours. the table answers + # "does this module work at finix rev X", so the answer should depend on the + # finix commit alone - pinning nixpkgs separately would let our own bumps + # move the results. + pkgs ? import (import (finix + "/lon.nix")).nixpkgs { + inherit system; + # steam, v2rayn and friends are unfree; a compatibility check that skips + # them is not much of a compatibility check. + config.allowUnfree = true; + }, +}: +let + inherit (pkgs) lib; + + finixTests = import (finix + "/tests/lib") { inherit pkgs lib; }; + + # finix's test driver imports every finix module into each node. a few of our + # modules are alternative implementations of a module finix also ships + # (pipewire, wireplumber), and declaring the same option twice is an eval + # error - so those tests name the finix modules they replace. + disableFinix = test: { + disabledModules = map (p: finix + "/modules/${p}") (test.replacesFinixModules or [ ]); + }; + + nodesOf = test: if test ? nodes then test.nodes else { machine = test.machine; }; + + # what gets imported into every node, and into the eval check: the module + # under test and whatever the test file says it takes to turn it on. nothing + # else - a test file that needs a device manager picks one itself, the same + # way finix's own tests do, because some of these modules pick for you. + hostFor = modulePath: test: node: { + imports = [ + (disableFinix test) + modulePath + node + ]; + }; + + # the smallest host finix will evaluate. anything else a module needs belongs + # in its test file, where it is visible. + minimalHost = { + nixpkgs.pkgs = pkgs; + networking.hostName = "compat"; + fileSystems."/" = { + device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + }; + }; +in +{ + inherit pkgs lib finix; + + hasVm = test: test ? testScript; + + # forcing `drvPath` evaluates the whole system - assertions included, since + # finix routes `system.topLevel` through `checkAssertWarn`. the string + # context is discarded so this instantiates the closure without building it. + mkEvalCheck = + name: modulePath: test: + let + # `nodes` is tied back in the way finix's own driver does it, so a + # multi-node test that reads another node's config still evaluates here. + systems = lib.mapAttrs ( + _: node: + (import finix).lib.finixSystem { + inherit lib; + specialArgs = { + nodes = systems; + }; + modules = [ + minimalHost + (hostFor modulePath test node) + ]; + } + ) (nodesOf test); + in + pkgs.runCommand "finix-compat-eval-${name}" { } ( + lib.concatMapStringsSep "\n" ( + system: "echo ${builtins.unsafeDiscardStringContext system.config.system.topLevel.drvPath} >> $out" + ) (lib.attrValues systems) + ); + + mkVmTest = + name: modulePath: test: + finixTests.mkTest ( + (removeAttrs test [ + "machine" + "note" + "replacesFinixModules" + ]) + // { + name = "finix-compat-${name}"; + nodes = lib.mapAttrs (_: hostFor modulePath test) (nodesOf test); + } + ); +}