Fix the Nix package build and guard it in CI

.github/workflows/ci.yml

@@ -63,6 +63,82 @@ jobs:
       - name: Verify vendor/httpz is upstream plus the recorded patch
         run: ./scripts/verify-vendored-httpz.sh
 
+  nix:
+    name: nix package build
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    # Bounds two things this job could otherwise hang on: a relay that will not
+    # exit, and a flake.lock bump landing on a nixpkgs rev where zig_0_16 is not
+    # substitutable, which would start a source build of zig and LLVM. No job
+    # here sets a timeout otherwise, so the default is 6 hours.
+    timeout-minutes: 20
+    if: github.event.pull_request.draft != true
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      # Cheap and offline, so it runs before nix is even installed: it names the
+      # offending file and hash in about a second, where nix build takes ~2
+      # minutes to report the same class of problem as a store path.
+      - name: Check nix/deps.nix and the package version against the manifests
+        run: ./scripts/verify-nix-deps.sh
+
+      - uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31.11.0
+
+      # nix/deps.nix is the offline package cache for the Nix build, mirroring
+      # the Zig manifests by hand. Nothing built it in CI, so it silently went
+      # stale when httpz became a path dependency and when the websocket pin
+      # moved: the cache no longer contained the revision zig asks for, and
+      # `nix build` failed with "package not found" for two releases.
+      - name: Build the Nix package
+        run: nix build .# --print-build-logs
+
+      # Proves the packaged binary actually runs, not merely that it linked: a
+      # stale or wrong dependency can still produce something that fails on
+      # startup. NIP-11 is served before any storage write, so this needs no
+      # fixtures.
+      - name: Check the packaged binary serves NIP-11
+        run: |
+          stop() {
+            kill "$1" 2>/dev/null || return 0
+            for i in $(seq 1 20); do kill -0 "$1" 2>/dev/null || return 0; sleep 0.25; done
+            kill -9 "$1" 2>/dev/null || true
+          }
+          # Re-evaluates rather than threading the path between steps; the
+          # derivation is already built, so this is a cached ~2s no-op.
+          out="$(nix build .# --no-link --print-out-paths)"
+          env WISP_HOST=127.0.0.1 WISP_PORT=7802 \
+            WISP_STORAGE_PATH="$RUNNER_TEMP/wisp-nix" \
+            "$out/bin/wisp" relay > relay-nix.log 2>&1 &
+          pid=$!
+          for i in $(seq 1 40); do nc -z 127.0.0.1 7802 && break; sleep 0.25; done
+          nc -z 127.0.0.1 7802 || { echo "packaged relay never listened"; cat relay-nix.log; stop "$pid"; exit 1; }
+          # Capture the status explicitly: under `bash -e` a bare assignment from
+          # a failing curl aborts the step here, losing both the relay log and
+          # the chance to stop the process.
+          body="$(curl -sf -m 5 -H 'Accept: application/nostr+json' http://127.0.0.1:7802/)" || rc=$?
+          rc="${rc:-0}"
+          stop "$pid"; wait "$pid" 2>/dev/null || true
+          if [ "$rc" -ne 0 ]; then
+            echo "NIP-11 request failed (curl exit $rc); relay log:"
+            cat relay-nix.log
+            exit 1
+          fi
+          case "$body" in
+            *'"software"'*) echo "packaged binary served NIP-11" ;;
+            *) echo "NIP-11 response missing software field: $body"; cat relay-nix.log; exit 1 ;;
+          esac
+
+      # The flake also exposes checks.<system>.wisp-module, a NixOS VM test that
+      # boots the module and drives a real WebSocket client. It is not run here
+      # only because of runtime: it builds a guest kernel and userland on top of
+      # this job's work. KVM *is* available on these runners (the installer
+      # enables it), so adding it is a cost decision, not a capability one.
+      # Run it with `nix flake check` when touching nix/module.nix.
+
   integration:
     permissions:
       contents: read

nix/deps.nix

@@ -1,9 +1,29 @@
-# generated by zon2nix (https://github.com/nix-community/zon2nix)
+# Offline package cache for the Nix build: one entry per Zig package that
+# `zig build` resolves, keyed by the exact hash from a build.zig.zon.
+#
+# MAINTAINED BY HAND. Originally generated by zon2nix
+# (https://github.com/nix-community/zon2nix), but zon2nix now fails on our
+# manifest with `error: parseError` because it cannot express the vendored path
+# dependency on vendor/httpz. Do not expect to regenerate this file; edit it.
+#
+# The entries must cover the union of build.zig.zon AND vendor/httpz/build.zig.zon
+# (the vendored copy pulls in metrics and websocket of its own). A `name` here has
+# to match the dependency's `.hash` character for character, or zig reports
+# "package not found" against this store path. httpz itself has no entry: it is a
+# path dependency, so it is part of the source tree rather than the cache.
 #
 # `extension = "tar.gz"` is added to each fetchzip: the codeload `/tar.gz/<rev>` URLs have a bare
 # rev basename with no archive suffix, so without it fetchzip's unpackFile fails with "do not know
 # how to unpack source archive" on any cache miss. The extension names the intermediate download
-# `download.tar.gz` so unpackFile recognises it. Re-add this after regenerating with zon2nix.
+# `download.tar.gz` so unpackFile recognises it.
+#
+# Drift guards, and their limits: the `nix package build` job in
+# .github/workflows/ci.yml builds this for real, which catches an entry that is
+# MISSING, and scripts/verify-nix-deps.sh catches the same case cheaply plus a
+# package.nix version that has drifted from build.zig.zon. Neither catches an
+# entry that is EXTRA -- zig ignores what it does not resolve -- which is how the
+# obsolete httpz entry survived here for two releases. That needs the transitive
+# closure; see the note in scripts/verify-nix-deps.sh for the upgrade path.
 
 {
   linkFarm,
@@ -20,14 +40,6 @@ linkFarm "zig-packages" [
       extension = "tar.gz";
     };
   }
-  {
-    name = "httpz-0.0.0-PNVzrEnoCADpf8B42EMPO2WmIlvsQJ0dJ8oCWjnl6CIX";
-    path = fetchzip {
-      url = "https://codeload.github.com/karlseguin/http.zig/tar.gz/86a44b63bda353338f2dbddc35eeb260f0f5c299";
-      hash = "sha256-QjQa+J+4W/Z7Bu5YcZP5T2iZ7n38rEH5mjH7fGL6Tkc=";
-      extension = "tar.gz";
-    };
-  }
   {
     name = "metrics-0.0.0-W7G4eIegAQD4XxA9Co7Atbw59u_2zvxYf406AZuoAHPM";
     path = fetchzip {
@@ -53,10 +65,10 @@ linkFarm "zig-packages" [
     };
   }
   {
-    name = "websocket-0.1.0-ZPISdangBAAgWPap_MVU6Nk4rj3GT3xuJuF0IIv8HN6G";
+    name = "websocket-0.1.0-ZPISdUoQBQC7zw1uSwbOEYdxZiTxNN4nf9RlWBsN0_Nd";
     path = fetchzip {
-      url = "https://codeload.github.com/karlseguin/websocket.zig/tar.gz/99df0d3533a41cbcd5ff59b9af68bbfe6169cc62";
-      hash = "sha256-Ol+yLbGAfgb+tWzfK5LTC6Ti7m36U3y8J//JYoULAvw=";
+      url = "https://codeload.github.com/karlseguin/websocket.zig/tar.gz/b70e733bc0d0ba0a98ff5fe5ef64d3017c85f369";
+      hash = "sha256-R+N4XZfObDDTKD4b+cBgEyQ8yA1IetWxD3g9+wNrFlE=";
       extension = "tar.gz";
     };
   }

nix/package.nix

@@ -1,7 +1,9 @@
-# Hermetic Zig 0.16 build of the wisp relay. The three .zon dependencies (and their transitive
-# closure) are pinned by URL+hash in build.zig.zon; deps.nix (generated by `zon2nix build.zig.zon`)
-# fetches them as fixed-output derivations and links them into Zig's global package cache, so the
-# build needs no network. System libs: lmdb (linked directly by build.zig), plus secp256k1 + openssl
+# Hermetic Zig 0.16 build of the wisp relay. build.zig.zon declares two fetched dependencies
+# (websocket, nostr) plus a path dependency on vendor/httpz, and deps.nix pins that whole transitive
+# closure by URL+hash, fetches each as a fixed-output derivation, and links them into Zig's global
+# package cache, so the build needs no network. deps.nix is maintained BY HAND and cannot be
+# regenerated with zon2nix (it cannot express the path dependency); see the header there.
+# System libs: lmdb (linked directly by build.zig), plus secp256k1 + openssl
 # (pulled in via the noscrypt crypto dependency).
 {
   lib,
@@ -18,7 +20,7 @@ let
 in
 stdenv.mkDerivation (finalAttrs: {
   pname = "wisp";
-  version = "0.5.10"; # keep in sync with build.zig.zon .version
+  version = "0.6.0"; # keep in sync with build.zig.zon .version
   inherit src;
 
   nativeBuildInputs = [ zig_0_16 ];
@@ -40,8 +42,15 @@ stdenv.mkDerivation (finalAttrs: {
     mkdir -p "$ZIG_GLOBAL_CACHE_DIR"
 
     # `--system <dir>` resolves every build.zig.zon dependency from the pre-fetched FOD package set
-    # (keyed by hash) instead of the network, so the build is hermetic. Zig verifies each against the
-    # hash pinned in build.zig.zon.
+    # instead of the network, so the build is hermetic.
+    #
+    # The hash is a LOOKUP KEY here, not an attestation. In --system mode zig only checks that a
+    # directory of that name exists and adopts it; the "hash mismatch" comparison lives on the
+    # network fetch path and is never reached. Verified by experiment, not by reading: appending a
+    # line to a package's source inside the system dir and rebuilding still succeeds. So the
+    # `hash =` values in deps.nix are the ONLY integrity gate on this build, and an entry whose url
+    # and hash are changed together would be adopted with no complaint. scripts/verify-nix-deps.sh
+    # is what ties each entry's url back to the revision the manifest asks for.
     # Target the BASELINE microarchitecture, not the build machine's native CPU: Zig otherwise bakes in
     # host CPU features (AVX2, etc.) and the binary SIGILLs on older/emulated CPUs. Baseline is the
     # portable, reproducible choice for a distributed package; SIMD-heavy deps do runtime dispatch.

scripts/verify-nix-deps.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+# Checks nix/deps.nix and nix/package.nix against the Zig manifests, offline.
+#
+# Why this exists at all: in `--system` mode (which nix/package.nix uses) the Zig
+# package hash is a LOOKUP KEY, not an attestation. Zig checks that a directory
+# of that name exists and adopts it; the "hash mismatch" comparison lives on the
+# network fetch path and is never reached. Verified by experiment: appending a
+# line to a package's source inside the system dir and rebuilding still succeeds.
+# So the `hash =` values in deps.nix are the only integrity gate on the Nix
+# build, and an entry whose url and hash were changed together would be fetched
+# and adopted without anything disagreeing.
+#
+# What this closes: it ties each cache entry's url back to the revision the
+# manifest actually asks for. Combined with nix's fixed-output hash (url ->
+# content) and the network `zig build` in the other CI jobs (manifest hash ->
+# content), the chain from manifest to compiled bytes is complete.
+#
+# It also catches two drift classes cheaply:
+#   1. A direct dependency missing from the cache. `nix build` catches this too,
+#      but takes ~2 minutes and reports a store path; this names the hash in a
+#      second.
+#   2. nix/package.nix's version drifting from build.zig.zon. Nothing compared
+#      these, and it silently lagged from 0.5.10 across two releases. NIP-11
+#      reports its own literal, so the packaged version is invisible at runtime.
+#
+# What it deliberately does NOT catch, and why: an entry that no manifest
+# requires (the obsolete httpz entry was one), and the url of a TRANSITIVE entry
+# such as noscrypt or StringZilla. Both need the full closure, and the transitive
+# manifests live inside fetched packages rather than the repo -- zig-pkg/ is
+# gitignored. Resolving them means network fetches on every run. Upgrade path if
+# that becomes worth it: walk the closure from a populated ZIG_GLOBAL_CACHE_DIR
+# after `zig build --fetch`, then assert set equality and cross-check every url.
+#
+# Usage: scripts/verify-nix-deps.sh   (from anywhere in the repo)
+# Requires: bash, grep, awk, comm. No nix, no network.
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$repo_root"
+
+fail=0
+
+# "<hash> <owner>/<repo>@<rev>" per direct dependency, from both in-repo
+# manifests. Path dependencies have no .url/.hash and so contribute nothing:
+# httpz is part of the source tree rather than the cache, which is why it has no
+# entry here or in deps.nix.
+#
+# Manifest urls look like https://github.com/<o>/<r>/archive/<rev>.tar.gz or
+# .../archive/refs/tags/<tag>.tar.gz.
+manifest_pairs() {
+  awk '
+    /\.url = "/ {
+      line = $0
+      sub(/.*\.url = "/, "", line); sub(/".*/, "", line)
+      url = line
+      next
+    }
+    /\.hash = "/ && url != "" {
+      line = $0
+      sub(/.*\.hash = "/, "", line); sub(/".*/, "", line)
+      slug = url
+      sub(/^https:\/\/github\.com\//, "", slug)
+      repo = slug; sub(/\/archive\/.*/, "", repo)
+      rev = slug; sub(/^.*\/archive\//, "", rev); sub(/\.tar\.gz$/, "", rev)
+      print line, repo "@" rev
+      url = ""
+    }
+  ' "$@" | sort -u
+}
+
+# "<name> <owner>/<repo>@<rev>" per cache entry.
+# deps.nix urls look like https://codeload.github.com/<o>/<r>/tar.gz/<rev> or
+# .../tar.gz/refs/tags/<tag>.
+deps_pairs() {
+  awk '
+    /^    name = "/ {
+      line = $0
+      sub(/.*name = "/, "", line); sub(/".*/, "", line)
+      name = line
+      next
+    }
+    /^      url = "/ && name != "" {
+      line = $0
+      sub(/.*url = "/, "", line); sub(/".*/, "", line)
+      slug = line
+      sub(/^https:\/\/codeload\.github\.com\//, "", slug)
+      repo = slug; sub(/\/tar\.gz\/.*/, "", repo)
+      rev = slug; sub(/^.*\/tar\.gz\//, "", rev)
+      print name, repo "@" rev
+      name = ""
+    }
+  ' nix/deps.nix | sort -u
+}
+
+required_pairs="$(manifest_pairs build.zig.zon vendor/httpz/build.zig.zon)"
+present_pairs="$(deps_pairs)"
+
+required_hashes="$(printf '%s\n' "$required_pairs" | awk '{print $1}' | sort -u)"
+present_hashes="$(printf '%s\n' "$present_pairs" | awk '{print $1}' | sort -u)"
+
+# Subset, not equality: deps.nix also legitimately carries transitive entries
+# (noscrypt and StringZilla arrive via libnostr-z) that no in-repo manifest names.
+missing="$(comm -23 <(printf '%s\n' "$required_hashes") <(printf '%s\n' "$present_hashes") || true)"
+if [ -n "$missing" ]; then
+  echo "FAIL - nix/deps.nix is missing entries the manifests require:"
+  printf '  %s\n' $missing
+  echo "       nix build would fail with \"package not found\" for these."
+  fail=1
+fi
+
+# The check that matters: same hash on both sides must mean the same revision.
+checked=0
+while read -r hash want; do
+  [ -n "$hash" ] || continue
+  got="$(printf '%s\n' "$present_pairs" | awk -v h="$hash" '$1 == h {print $2}')"
+  [ -n "$got" ] || continue   # already reported as missing above
+  if [ "$got" != "$want" ]; then
+    echo "FAIL - $hash points at different revisions:"
+    echo "         manifest asks for $want"
+    echo "         nix/deps.nix fetches $got"
+    echo "       In --system mode zig does not verify content against the hash, so"
+    echo "       this would be adopted silently."
+    fail=1
+  else
+    checked=$((checked + 1))
+  fi
+done <<< "$required_pairs"
+
+zon_version="$(grep -oP '(?<=\.version = ")[^"]+' build.zig.zon | head -1)"
+nix_version="$(grep -oP '(?<=^  version = ")[^"]+' nix/package.nix | head -1)"
+if [ -z "$zon_version" ] || [ -z "$nix_version" ]; then
+  echo "FAIL - could not read a version from build.zig.zon ('$zon_version') or nix/package.nix ('$nix_version')"
+  fail=1
+elif [ "$zon_version" != "$nix_version" ]; then
+  echo "FAIL - version mismatch: build.zig.zon says '$zon_version', nix/package.nix says '$nix_version'"
+  fail=1
+fi
+
+[ "$fail" -eq 0 ] || exit 1
+
+echo "ok - $checked direct entries match the manifests by hash and revision; package version $zon_version"