From 909b63e724d4b5e1df176a337c10cfaaf5cffe86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kyle=20=F0=9F=90=86?= Date: Wed, 12 Aug 2026 10:43:31 -0400 Subject: [PATCH 1/2] Fix the Nix package build and guard it in CI --- .github/workflows/ci.yml | 45 ++++++++++++++++++++++++++++++++++++++++ nix/deps.nix | 33 +++++++++++++++++------------ nix/package.nix | 2 +- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99220a9..8ce24a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,51 @@ jobs: - name: Verify vendor/httpz is upstream plus the recorded patch run: ./scripts/verify-vendored-httpz.sh + nix: + name: nix package build + permissions: + contents: read + runs-on: ubuntu-latest + if: github.event.pull_request.draft != true + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31.11.0 + + # nix/deps.nix is the offline package cache for the Nix build, mirroring + # build.zig.zon by hand. Nothing else checks it, so it silently went stale + # when httpz became a path dependency and when the websocket pin moved: the + # cache no longer contained the revision zig asks for, and `nix build` + # failed with "package not found". This job is what catches that drift. + - name: Build the Nix package + run: nix build .# --print-build-logs + + # Proves the packaged binary actually runs, not merely that it linked: a + # stale or wrong dependency can still produce something that fails on + # startup. NIP-11 is served before any storage write, so this needs no + # fixtures. + - name: Check the packaged binary serves NIP-11 + run: | + out="$(nix build .# --no-link --print-out-paths)" + env WISP_HOST=127.0.0.1 WISP_PORT=7802 \ + WISP_STORAGE_PATH="$RUNNER_TEMP/wisp-nix" \ + "$out/bin/wisp" relay > relay-nix.log 2>&1 & + pid=$! + for i in $(seq 1 40); do nc -z 127.0.0.1 7802 && break; sleep 0.25; done + nc -z 127.0.0.1 7802 || { echo "packaged relay never listened"; cat relay-nix.log; kill "$pid" 2>/dev/null; exit 1; } + body="$(curl -sf -m 5 -H 'Accept: application/nostr+json' http://127.0.0.1:7802/)" + kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null || true + echo "$body" | grep -q '"software"' || { echo "NIP-11 response missing software field: $body"; exit 1; } + echo "packaged binary served NIP-11" + + # The flake also exposes checks..wisp-module, a NixOS VM test that + # boots the module and drives a real WebSocket client. It is NOT run here: + # it needs /dev/kvm, which the standard GitHub runners do not provide. + # Run it locally with `nix flake check` when touching nix/module.nix. + integration: permissions: contents: read diff --git a/nix/deps.nix b/nix/deps.nix index 4902192..4789f3d 100644 --- a/nix/deps.nix +++ b/nix/deps.nix @@ -1,9 +1,24 @@ -# generated by zon2nix (https://github.com/nix-community/zon2nix) +# Offline package cache for the Nix build: one entry per Zig package that +# `zig build` resolves, keyed by the exact hash from a build.zig.zon. +# +# MAINTAINED BY HAND. Originally generated by zon2nix +# (https://github.com/nix-community/zon2nix), but zon2nix now fails on our +# manifest with `error: parseError` because it cannot express the vendored path +# dependency on vendor/httpz. Do not expect to regenerate this file; edit it. +# +# The entries must cover the union of build.zig.zon AND vendor/httpz/build.zig.zon +# (the vendored copy pulls in metrics and websocket of its own). A `name` here has +# to match the dependency's `.hash` character for character, or zig reports +# "package not found" against this store path. httpz itself has no entry: it is a +# path dependency, so it is part of the source tree rather than the cache. # # `extension = "tar.gz"` is added to each fetchzip: the codeload `/tar.gz/` URLs have a bare # rev basename with no archive suffix, so without it fetchzip's unpackFile fails with "do not know # how to unpack source archive" on any cache miss. The extension names the intermediate download -# `download.tar.gz` so unpackFile recognises it. Re-add this after regenerating with zon2nix. +# `download.tar.gz` so unpackFile recognises it. +# +# The `nix package build` job in .github/workflows/ci.yml is what catches drift +# here; before it existed this file was broken for two releases without a signal. { linkFarm, @@ -20,14 +35,6 @@ linkFarm "zig-packages" [ extension = "tar.gz"; }; } - { - name = "httpz-0.0.0-PNVzrEnoCADpf8B42EMPO2WmIlvsQJ0dJ8oCWjnl6CIX"; - path = fetchzip { - url = "https://codeload.github.com/karlseguin/http.zig/tar.gz/86a44b63bda353338f2dbddc35eeb260f0f5c299"; - hash = "sha256-QjQa+J+4W/Z7Bu5YcZP5T2iZ7n38rEH5mjH7fGL6Tkc="; - extension = "tar.gz"; - }; - } { name = "metrics-0.0.0-W7G4eIegAQD4XxA9Co7Atbw59u_2zvxYf406AZuoAHPM"; path = fetchzip { @@ -53,10 +60,10 @@ linkFarm "zig-packages" [ }; } { - name = "websocket-0.1.0-ZPISdangBAAgWPap_MVU6Nk4rj3GT3xuJuF0IIv8HN6G"; + name = "websocket-0.1.0-ZPISdUoQBQC7zw1uSwbOEYdxZiTxNN4nf9RlWBsN0_Nd"; path = fetchzip { - url = "https://codeload.github.com/karlseguin/websocket.zig/tar.gz/99df0d3533a41cbcd5ff59b9af68bbfe6169cc62"; - hash = "sha256-Ol+yLbGAfgb+tWzfK5LTC6Ti7m36U3y8J//JYoULAvw="; + url = "https://codeload.github.com/karlseguin/websocket.zig/tar.gz/b70e733bc0d0ba0a98ff5fe5ef64d3017c85f369"; + hash = "sha256-R+N4XZfObDDTKD4b+cBgEyQ8yA1IetWxD3g9+wNrFlE="; extension = "tar.gz"; }; } diff --git a/nix/package.nix b/nix/package.nix index de53442..8065205 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -18,7 +18,7 @@ let in stdenv.mkDerivation (finalAttrs: { pname = "wisp"; - version = "0.5.10"; # keep in sync with build.zig.zon .version + version = "0.6.0"; # keep in sync with build.zig.zon .version inherit src; nativeBuildInputs = [ zig_0_16 ]; From aebf6c1ddddd0e4a9086cd603a6dc9ea681c1f7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kyle=20=F0=9F=90=86?= Date: Wed, 12 Aug 2026 11:06:17 -0400 Subject: [PATCH 2/2] Address review: correct the --system integrity claim, cross-check dep revisions --- .github/workflows/ci.yml | 55 +++++++++++---- nix/deps.nix | 9 ++- nix/package.nix | 21 ++++-- scripts/verify-nix-deps.sh | 141 +++++++++++++++++++++++++++++++++++++ 4 files changed, 206 insertions(+), 20 deletions(-) create mode 100755 scripts/verify-nix-deps.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ce24a5..d344493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,11 @@ jobs: permissions: contents: read runs-on: ubuntu-latest + # Bounds two things this job could otherwise hang on: a relay that will not + # exit, and a flake.lock bump landing on a nixpkgs rev where zig_0_16 is not + # substitutable, which would start a source build of zig and LLVM. No job + # here sets a timeout otherwise, so the default is 6 hours. + timeout-minutes: 20 if: github.event.pull_request.draft != true steps: @@ -75,13 +80,19 @@ jobs: with: persist-credentials: false + # Cheap and offline, so it runs before nix is even installed: it names the + # offending file and hash in about a second, where nix build takes ~2 + # minutes to report the same class of problem as a store path. + - name: Check nix/deps.nix and the package version against the manifests + run: ./scripts/verify-nix-deps.sh + - uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31.11.0 # nix/deps.nix is the offline package cache for the Nix build, mirroring - # build.zig.zon by hand. Nothing else checks it, so it silently went stale - # when httpz became a path dependency and when the websocket pin moved: the - # cache no longer contained the revision zig asks for, and `nix build` - # failed with "package not found". This job is what catches that drift. + # the Zig manifests by hand. Nothing built it in CI, so it silently went + # stale when httpz became a path dependency and when the websocket pin + # moved: the cache no longer contained the revision zig asks for, and + # `nix build` failed with "package not found" for two releases. - name: Build the Nix package run: nix build .# --print-build-logs @@ -91,22 +102,42 @@ jobs: # fixtures. - name: Check the packaged binary serves NIP-11 run: | + stop() { + kill "$1" 2>/dev/null || return 0 + for i in $(seq 1 20); do kill -0 "$1" 2>/dev/null || return 0; sleep 0.25; done + kill -9 "$1" 2>/dev/null || true + } + # Re-evaluates rather than threading the path between steps; the + # derivation is already built, so this is a cached ~2s no-op. out="$(nix build .# --no-link --print-out-paths)" env WISP_HOST=127.0.0.1 WISP_PORT=7802 \ WISP_STORAGE_PATH="$RUNNER_TEMP/wisp-nix" \ "$out/bin/wisp" relay > relay-nix.log 2>&1 & pid=$! for i in $(seq 1 40); do nc -z 127.0.0.1 7802 && break; sleep 0.25; done - nc -z 127.0.0.1 7802 || { echo "packaged relay never listened"; cat relay-nix.log; kill "$pid" 2>/dev/null; exit 1; } - body="$(curl -sf -m 5 -H 'Accept: application/nostr+json' http://127.0.0.1:7802/)" - kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null || true - echo "$body" | grep -q '"software"' || { echo "NIP-11 response missing software field: $body"; exit 1; } - echo "packaged binary served NIP-11" + nc -z 127.0.0.1 7802 || { echo "packaged relay never listened"; cat relay-nix.log; stop "$pid"; exit 1; } + # Capture the status explicitly: under `bash -e` a bare assignment from + # a failing curl aborts the step here, losing both the relay log and + # the chance to stop the process. + body="$(curl -sf -m 5 -H 'Accept: application/nostr+json' http://127.0.0.1:7802/)" || rc=$? + rc="${rc:-0}" + stop "$pid"; wait "$pid" 2>/dev/null || true + if [ "$rc" -ne 0 ]; then + echo "NIP-11 request failed (curl exit $rc); relay log:" + cat relay-nix.log + exit 1 + fi + case "$body" in + *'"software"'*) echo "packaged binary served NIP-11" ;; + *) echo "NIP-11 response missing software field: $body"; cat relay-nix.log; exit 1 ;; + esac # The flake also exposes checks..wisp-module, a NixOS VM test that - # boots the module and drives a real WebSocket client. It is NOT run here: - # it needs /dev/kvm, which the standard GitHub runners do not provide. - # Run it locally with `nix flake check` when touching nix/module.nix. + # boots the module and drives a real WebSocket client. It is not run here + # only because of runtime: it builds a guest kernel and userland on top of + # this job's work. KVM *is* available on these runners (the installer + # enables it), so adding it is a cost decision, not a capability one. + # Run it with `nix flake check` when touching nix/module.nix. integration: permissions: diff --git a/nix/deps.nix b/nix/deps.nix index 4789f3d..2519ba6 100644 --- a/nix/deps.nix +++ b/nix/deps.nix @@ -17,8 +17,13 @@ # how to unpack source archive" on any cache miss. The extension names the intermediate download # `download.tar.gz` so unpackFile recognises it. # -# The `nix package build` job in .github/workflows/ci.yml is what catches drift -# here; before it existed this file was broken for two releases without a signal. +# Drift guards, and their limits: the `nix package build` job in +# .github/workflows/ci.yml builds this for real, which catches an entry that is +# MISSING, and scripts/verify-nix-deps.sh catches the same case cheaply plus a +# package.nix version that has drifted from build.zig.zon. Neither catches an +# entry that is EXTRA -- zig ignores what it does not resolve -- which is how the +# obsolete httpz entry survived here for two releases. That needs the transitive +# closure; see the note in scripts/verify-nix-deps.sh for the upgrade path. { linkFarm, diff --git a/nix/package.nix b/nix/package.nix index 8065205..d3da5d5 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -1,7 +1,9 @@ -# Hermetic Zig 0.16 build of the wisp relay. The three .zon dependencies (and their transitive -# closure) are pinned by URL+hash in build.zig.zon; deps.nix (generated by `zon2nix build.zig.zon`) -# fetches them as fixed-output derivations and links them into Zig's global package cache, so the -# build needs no network. System libs: lmdb (linked directly by build.zig), plus secp256k1 + openssl +# Hermetic Zig 0.16 build of the wisp relay. build.zig.zon declares two fetched dependencies +# (websocket, nostr) plus a path dependency on vendor/httpz, and deps.nix pins that whole transitive +# closure by URL+hash, fetches each as a fixed-output derivation, and links them into Zig's global +# package cache, so the build needs no network. deps.nix is maintained BY HAND and cannot be +# regenerated with zon2nix (it cannot express the path dependency); see the header there. +# System libs: lmdb (linked directly by build.zig), plus secp256k1 + openssl # (pulled in via the noscrypt crypto dependency). { lib, @@ -40,8 +42,15 @@ stdenv.mkDerivation (finalAttrs: { mkdir -p "$ZIG_GLOBAL_CACHE_DIR" # `--system ` resolves every build.zig.zon dependency from the pre-fetched FOD package set - # (keyed by hash) instead of the network, so the build is hermetic. Zig verifies each against the - # hash pinned in build.zig.zon. + # instead of the network, so the build is hermetic. + # + # The hash is a LOOKUP KEY here, not an attestation. In --system mode zig only checks that a + # directory of that name exists and adopts it; the "hash mismatch" comparison lives on the + # network fetch path and is never reached. Verified by experiment, not by reading: appending a + # line to a package's source inside the system dir and rebuilding still succeeds. So the + # `hash =` values in deps.nix are the ONLY integrity gate on this build, and an entry whose url + # and hash are changed together would be adopted with no complaint. scripts/verify-nix-deps.sh + # is what ties each entry's url back to the revision the manifest asks for. # Target the BASELINE microarchitecture, not the build machine's native CPU: Zig otherwise bakes in # host CPU features (AVX2, etc.) and the binary SIGILLs on older/emulated CPUs. Baseline is the # portable, reproducible choice for a distributed package; SIMD-heavy deps do runtime dispatch. diff --git a/scripts/verify-nix-deps.sh b/scripts/verify-nix-deps.sh new file mode 100755 index 0000000..aaba014 --- /dev/null +++ b/scripts/verify-nix-deps.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# Checks nix/deps.nix and nix/package.nix against the Zig manifests, offline. +# +# Why this exists at all: in `--system` mode (which nix/package.nix uses) the Zig +# package hash is a LOOKUP KEY, not an attestation. Zig checks that a directory +# of that name exists and adopts it; the "hash mismatch" comparison lives on the +# network fetch path and is never reached. Verified by experiment: appending a +# line to a package's source inside the system dir and rebuilding still succeeds. +# So the `hash =` values in deps.nix are the only integrity gate on the Nix +# build, and an entry whose url and hash were changed together would be fetched +# and adopted without anything disagreeing. +# +# What this closes: it ties each cache entry's url back to the revision the +# manifest actually asks for. Combined with nix's fixed-output hash (url -> +# content) and the network `zig build` in the other CI jobs (manifest hash -> +# content), the chain from manifest to compiled bytes is complete. +# +# It also catches two drift classes cheaply: +# 1. A direct dependency missing from the cache. `nix build` catches this too, +# but takes ~2 minutes and reports a store path; this names the hash in a +# second. +# 2. nix/package.nix's version drifting from build.zig.zon. Nothing compared +# these, and it silently lagged from 0.5.10 across two releases. NIP-11 +# reports its own literal, so the packaged version is invisible at runtime. +# +# What it deliberately does NOT catch, and why: an entry that no manifest +# requires (the obsolete httpz entry was one), and the url of a TRANSITIVE entry +# such as noscrypt or StringZilla. Both need the full closure, and the transitive +# manifests live inside fetched packages rather than the repo -- zig-pkg/ is +# gitignored. Resolving them means network fetches on every run. Upgrade path if +# that becomes worth it: walk the closure from a populated ZIG_GLOBAL_CACHE_DIR +# after `zig build --fetch`, then assert set equality and cross-check every url. +# +# Usage: scripts/verify-nix-deps.sh (from anywhere in the repo) +# Requires: bash, grep, awk, comm. No nix, no network. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +fail=0 + +# " /@" per direct dependency, from both in-repo +# manifests. Path dependencies have no .url/.hash and so contribute nothing: +# httpz is part of the source tree rather than the cache, which is why it has no +# entry here or in deps.nix. +# +# Manifest urls look like https://github.com///archive/.tar.gz or +# .../archive/refs/tags/.tar.gz. +manifest_pairs() { + awk ' + /\.url = "/ { + line = $0 + sub(/.*\.url = "/, "", line); sub(/".*/, "", line) + url = line + next + } + /\.hash = "/ && url != "" { + line = $0 + sub(/.*\.hash = "/, "", line); sub(/".*/, "", line) + slug = url + sub(/^https:\/\/github\.com\//, "", slug) + repo = slug; sub(/\/archive\/.*/, "", repo) + rev = slug; sub(/^.*\/archive\//, "", rev); sub(/\.tar\.gz$/, "", rev) + print line, repo "@" rev + url = "" + } + ' "$@" | sort -u +} + +# " /@" per cache entry. +# deps.nix urls look like https://codeload.github.com///tar.gz/ or +# .../tar.gz/refs/tags/. +deps_pairs() { + awk ' + /^ name = "/ { + line = $0 + sub(/.*name = "/, "", line); sub(/".*/, "", line) + name = line + next + } + /^ url = "/ && name != "" { + line = $0 + sub(/.*url = "/, "", line); sub(/".*/, "", line) + slug = line + sub(/^https:\/\/codeload\.github\.com\//, "", slug) + repo = slug; sub(/\/tar\.gz\/.*/, "", repo) + rev = slug; sub(/^.*\/tar\.gz\//, "", rev) + print name, repo "@" rev + name = "" + } + ' nix/deps.nix | sort -u +} + +required_pairs="$(manifest_pairs build.zig.zon vendor/httpz/build.zig.zon)" +present_pairs="$(deps_pairs)" + +required_hashes="$(printf '%s\n' "$required_pairs" | awk '{print $1}' | sort -u)" +present_hashes="$(printf '%s\n' "$present_pairs" | awk '{print $1}' | sort -u)" + +# Subset, not equality: deps.nix also legitimately carries transitive entries +# (noscrypt and StringZilla arrive via libnostr-z) that no in-repo manifest names. +missing="$(comm -23 <(printf '%s\n' "$required_hashes") <(printf '%s\n' "$present_hashes") || true)" +if [ -n "$missing" ]; then + echo "FAIL - nix/deps.nix is missing entries the manifests require:" + printf ' %s\n' $missing + echo " nix build would fail with \"package not found\" for these." + fail=1 +fi + +# The check that matters: same hash on both sides must mean the same revision. +checked=0 +while read -r hash want; do + [ -n "$hash" ] || continue + got="$(printf '%s\n' "$present_pairs" | awk -v h="$hash" '$1 == h {print $2}')" + [ -n "$got" ] || continue # already reported as missing above + if [ "$got" != "$want" ]; then + echo "FAIL - $hash points at different revisions:" + echo " manifest asks for $want" + echo " nix/deps.nix fetches $got" + echo " In --system mode zig does not verify content against the hash, so" + echo " this would be adopted silently." + fail=1 + else + checked=$((checked + 1)) + fi +done <<< "$required_pairs" + +zon_version="$(grep -oP '(?<=\.version = ")[^"]+' build.zig.zon | head -1)" +nix_version="$(grep -oP '(?<=^ version = ")[^"]+' nix/package.nix | head -1)" +if [ -z "$zon_version" ] || [ -z "$nix_version" ]; then + echo "FAIL - could not read a version from build.zig.zon ('$zon_version') or nix/package.nix ('$nix_version')" + fail=1 +elif [ "$zon_version" != "$nix_version" ]; then + echo "FAIL - version mismatch: build.zig.zon says '$zon_version', nix/package.nix says '$nix_version'" + fail=1 +fi + +[ "$fail" -eq 0 ] || exit 1 + +echo "ok - $checked direct entries match the manifests by hash and revision; package version $zon_version"