fix: visible update failures, resumable downloads, systemd-aware Linux restart

.github/workflows/security.yml

@@ -29,7 +29,7 @@ jobs:
 
   # SAST. Accepted rule exclusions (documented, minimal — never blanket):
   #   G104 — best-effort os.Remove/Close on error-cleanup paths.
-  #   G204 — exec of gh / launchctl with internally-built args (no user input).
+  #   G204 — exec of launchctl with internally-built args (no user input).
   #   G301 — staging/install dirs are 0755 (operator-readable install tree).
   #   G302 — extracted artifacts are executables and must be 0755.
   #   G304 — file paths derive from InstallDir / temp dir, inherent to an updater.
@@ -57,7 +57,10 @@ jobs:
         # golang/govulncheck-action does its own git fetch and fails with
         # "fatal: unable to access ... error: 400" on org repos. Run the tool
         # directly after checkout + setup-go (rendezvous/common pattern).
-        run: go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+        # Pinned: govulncheck v1.8.0+ requires Go 1.26, and setup-go runs
+        # with GOTOOLCHAIN=local on the go.mod version (1.25.x), so @latest
+        # fails before scanning anything. Bump together with the Go 1.26 move.
+        run: go run golang.org/x/vuln/cmd/govulncheck@v1.7.0 ./...
 
   # Secret scan over the full history on push, the PR range on PR.
   gitleaks:

CHANGELOG.md

@@ -5,6 +5,61 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- `update-state.json` status file written after every check (`Config.StatusPath`,
+  default: next to `StatePath`). It records the last check time, result and
+  error, the run of consecutive failures, the installed and latest versions,
+  the last update, any daemon restart failure, and a heartbeat for the loop
+  (`loop_pid`, `next_check_at`). Read it with `ReadStatus`. `LastStatus()`
+  returns the same record in-process. Writers in different processes
+  serialise on an advisory lock (`update-state.json.lock`), so none of them
+  lose each other's updates. A `RunOnce` caller writes the file only when it
+  sets `StatusPath` or `StatePath`. `pilotctl update` does not do that yet.
+- `GITHUB_TOKEN` / `GH_TOKEN` is now used for the releases API as well as for
+  attestations. It is sent only to `api.github.com`, and the call is retried
+  without it if GitHub rejects it with a 401. Rate-limit errors say when the
+  limit resets and how to raise it.
+- Downloads resume with HTTP `Range` after a dropped or stalled transfer (up to
+  4 attempts, 5xx responses retried) and are aborted by `Stop()`.
+
+### Changed
+
+- **`RunOnce()` now returns `error`.** Callers such as `pilotctl update` can
+  report a failed update and exit non-zero instead of printing "ok". Existing
+  call sites that ignore the result still compile.
+- `RunOnce()` no longer exits the calling process when a release replaces
+  `pilot-updater`. It restarts the daemon and returns. The loop (`Start`)
+  still exits for its service manager, after recording the update.
+- Asset downloads have no fixed 30 s total timeout. They fail after 60 s
+  without data, or after 30 minutes in total. The TLS handshake timeout is
+  now 20 s.
+- `go` directive raised to 1.25.13 (standard-library security fixes).
+
+### Fixed
+
+- On Linux the daemon is now restarted after an update. The running daemon's
+  `/proc/<pid>/exe` reads `<path> (deleted)` once its binary is renamed over,
+  so the old exact-path match never found it and daemons kept running the old
+  version.
+- The updater sends that SIGTERM only when systemd will start the daemon
+  again: the daemon runs as `ExecStart=` of a system service with
+  `Restart=always` or `on-success`, read from `/proc/<pid>/cgroup` and the unit
+  files. Otherwise the daemon keeps running and `restart_error` says how to
+  restart it. This covers a daemon started with `pilotctl daemon start`
+  (containers, WSL, CI) and units from install.sh up to v1.9.0, which have
+  `Restart=on-failure`. Stopping those daemons would take the node offline
+  while the status file reported `updated`. After a SIGTERM the updater waits
+  up to 30 s for a new daemon on the new binary, and reports a failure if none
+  appears.
+- On Linux, `restart_error` is cleared once a check finds the daemon running
+  the installed binary, for example after a manual restart. Before, it stayed
+  until the next release.
+- The test suite no longer runs the real `launchctl`. On a developer Mac with
+  Pilot installed, it used to restart the running daemon.
+
 ## [v0.1.0]
 
 Initial release.

README.md

@@ -19,14 +19,74 @@ import "github.com/pilot-protocol/updater"
 
 ```go
 u := updater.New(updater.Config{
-    Repo:          "TeoSlayer/pilotprotocol",
+    Repo:          "pilot-protocol/pilotprotocol",
     InstallDir:    "/home/user/.pilot/bin",
     Version:       "v1.10.5",
     CheckInterval: 1 * time.Hour,
+    StatePath:     "/home/user/.pilot/auto-update.json", // {"enabled": bool}
 })
 u.Start()
 ```
 
+One-shot check, for `pilotctl update` and similar commands:
+
+```go
+u := updater.New(updater.Config{
+    Repo:       "pilot-protocol/pilotprotocol",
+    InstallDir: "/home/user/.pilot/bin",
+    StatusPath: "/home/user/.pilot/update-state.json", // record the run
+})
+if err := u.RunOnce(); err != nil {
+    // The check failed: report it and exit non-zero.
+}
+st := u.LastStatus() // st.LastResult is "up_to_date" or "updated"
+if st.RestartError != "" {
+    // Installed, but the daemon still runs the old version. The message
+    // says how to restart it.
+}
+```
+
+`RunOnce` returns the check's error. It never exits the calling process. If
+the release replaced `pilot-updater`, the daemon is still restarted onto the
+new binaries. A running updater service switches to its new binary the next
+time its service manager restarts it.
+
+A `RunOnce` caller writes the status file only when it sets `StatusPath` (or
+`StatePath`); otherwise only `LastStatus()` has the result. **`pilotctl
+update` does not do this yet.** As of web4 `main` on 2026-09-24,
+`cmd/pilotctl/updates.go` sets neither path and ignores the error `RunOnce`
+returns. So manual updates are not recorded in `update-state.json`, and a
+failed `pilotctl update` still prints success and exits 0. That stays true
+until pilotctl passes `StatusPath: ~/.pilot/update-state.json` and checks the
+error, which needs a web4 change after this version is released.
+
+### Restarting the daemon
+
+After installing new binaries the updater restarts the daemon onto them:
+
+- **macOS:** `launchctl kickstart -k gui/<uid>/network.pilotprotocol.pilot-daemon`.
+- **Linux:** SIGTERM, but only when systemd will start the daemon again. The
+  daemon must run as the main program (`ExecStart=`) of a system service whose
+  unit restarts it after a clean exit. pilot-daemon exits 0 on SIGTERM, so
+  that means `Restart=always` or `on-success`. The updater finds the service
+  from `/proc/<pid>/cgroup` and reads the unit file and its drop-ins. It does
+  not run `systemctl`. After the SIGTERM it waits up to 30 s for a new daemon
+  process on the new binary.
+
+On Linux the daemon is left running the old version, with `restart_error`
+saying how to restart it, when:
+
+- it was started with `pilotctl daemon start`, which is how containers, WSL
+  and CI run it because install.sh sets up no service there;
+- its unit has `Restart=on-failure`, as written by install.sh up to v1.9.0.
+  Updates never rewrite the unit, and systemd does not restart a clean exit
+  under that policy. Re-running install.sh rewrites the unit with
+  `Restart=always`;
+- it runs under a systemd user service, or inside another service, such as a
+  CI runner.
+
+Stopping the daemon in any of those cases would take the node offline.
+
 ### Pinning a version
 
 Set `PinnedVersion` to lock the updater to a specific release tag. When
@@ -37,7 +97,7 @@ install, then idles — it will **not** chase the latest release. Clear
 
 ```go
 u := updater.New(updater.Config{
-    Repo:          "TeoSlayer/pilotprotocol",
+    Repo:          "pilot-protocol/pilotprotocol",
     InstallDir:    "/home/user/.pilot/bin",
     PinnedVersion: "v1.10.5",  // stay on this version
 })
@@ -48,11 +108,155 @@ The in-process `Service` adapter is used when embedding into the
 daemon; a standalone sidecar binary built from this package is also
 supported.
 
+## Update status file
+
+After every check the updater writes `update-state.json` so that other tools
+can tell whether updates actually work. Before this file existed,
+`pilotctl update status` could only show whether auto-update was switched on,
+and a node could fail every hourly check for weeks without anyone noticing.
+
+The file goes to `Config.StatusPath`. When that is empty and `StatePath` is
+set, it goes next to the control file. For a standard install that is
+`~/.pilot/auto-update.json` → `~/.pilot/update-state.json`. With neither
+set, nothing is written.
+
+The long-running updater and any one-shot `RunOnce` caller configured with
+the same path share the file. `pilotctl update` does not pass one yet (see
+above). Each writer takes an advisory lock on `update-state.json.lock` and
+merges its fields into what is already there, so concurrent writers from
+different processes do not lose each other's changes. Writes are atomic
+(temp file + rename). A writer that cannot get the lock within 5 s writes
+anyway and logs a warning.
+
+```json
+{
+  "updater_version": "v1.13.10",
+  "repo": "pilot-protocol/pilotprotocol",
+  "last_check_at": "2026-09-24T10:00:12Z",
+  "last_check_trigger": "auto",
+  "last_result": "failed",
+  "last_error": "fetch latest release: GitHub API returned 403 (rate limit exceeded, resets at 2026-09-24T10:41:03Z; ...)",
+  "consecutive_failures": 3,
+  "last_success_at": "2026-09-24T07:00:09Z",
+  "current_version": "v1.13.9",
+  "latest_version": "v1.13.10",
+  "last_update_at": "2026-09-01T12:35:40Z",
+  "last_update_version": "v1.13.9",
+  "loop_pid": 812,
+  "loop_started_at": "2026-09-20T08:11:02Z",
+  "next_check_at": "2026-09-24T11:00:12Z"
+}
+```
+
+| Field | Meaning |
+|---|---|
+| `last_result` | `up_to_date`, `updated` or `failed`. |
+| `last_error` | Why the last check failed. Empty after a success. |
+| `consecutive_failures` | Failed checks since the last success. |
+| `last_success_at` | The last check that completed without error. |
+| `current_version` / `latest_version` | The installed release, and the release it was compared against (the pin, when pinned). |
+| `restart_error` | New binaries were installed but the daemon is not running them. Either it was left on the old version because stopping it would have left it down (see [Restarting the daemon](#restarting-the-daemon)) or the restart failed, or it was stopped and did not come back. Says how to restart it. Cleared by the next successful restart, and on Linux by the next check that finds the daemon running the installed binary. |
+| `loop_pid`, `loop_started_at` | The long-running updater process. |
+| `next_check_at` | When the loop wakes next. It is updated even while auto-update is disabled, so it doubles as a heartbeat. |
+
+Suggested readings for `pilotctl update status` and similar tools:
+
+- **Updates failing:** `consecutive_failures > 0`. Show `last_error`.
+- **Updates stalled:** `last_success_at` is more than 48 hours old while auto-update is enabled.
+- **No updater running:** auto-update is enabled, but the file is missing, `loop_pid` is not alive, or `next_check_at` is more than one interval in the past. This happens on Linux without systemd (containers, WSL, CI), where the installer cannot start the updater.
+- **Daemon on the old version:** `restart_error` is set. Show it: it names the command that restarts the daemon.
+
+Use `updater.ReadStatus(path)` to read the file. A missing file returns an
+error matching `errors.Is(err, fs.ErrNotExist)`.
+
+## GitHub API rate limits and `GITHUB_TOKEN`
+
+The updater calls the GitHub REST API for the release list and for the build
+attestation. Without a token GitHub allows 60 requests per hour per IP, and a
+shared NAT, VPN or CI egress can use that up. When that happens the
+check fails with an error that says so, and it is recorded in the status file.
+
+If `GITHUB_TOKEN` (or `GH_TOKEN`) is set in the updater's environment, it is
+sent to `api.github.com`, and only there, which raises the limit to 5,000 per
+hour. The token needs no scopes because the repository is public: a classic
+token with no scopes, or a fine-grained token with public read-only access,
+is enough. If GitHub rejects the token (401), the updater retries without it,
+so an expired token never stops updates.
+
+To set it for the service:
+
+- **systemd:** `sudo systemctl edit pilot-updater`, add
+  `[Service]` / `Environment=GITHUB_TOKEN=<token>`, then
+  `sudo systemctl restart pilot-updater`.
+- **launchd:** add an `EnvironmentVariables` dict with `GITHUB_TOKEN` to
+  `~/Library/LaunchAgents/network.pilotprotocol.pilot-updater.plist`, then
+  reload it with `launchctl bootout` / `launchctl bootstrap`.
+
+## Downloads
+
+Release archives are 17–20 MB. Downloads no longer have a fixed total
+timeout (the old 30 s limit needed a sustained ~5 Mbit/s). Instead:
+
+- An attempt fails only when no data arrives for 60 s.
+- A dropped or stalled transfer resumes with an HTTP `Range` request from the
+  bytes already on disk, or starts over if the server ignores `Range`. Up to
+  4 attempts are made. 5xx responses are retried and 4xx responses are not.
+- One file download is capped at 30 minutes in total.
+- `Stop()` aborts an in-flight download.
+
+The SHA-256 match against the attested `checksums.txt` still gates every install.
+
+## No external tools (no `gh`)
+
+The updater never runs `gh` or any other external tool. It checks the SLSA
+provenance of `checksums.txt` in-process with sigstore-go
+(`attestation.go`). The only command it runs is `launchctl kickstart`, which
+restarts the daemon under launchd on macOS. On Linux it reads `/proc` and the
+systemd unit files directly instead of running `systemctl`.
+`TestNoExternalToolDependency` fails the build if another exec is added.
+
+### Nodes stuck on v1.12.2–v1.13.4 (updater v0.2.3)
+
+pilotprotocol releases **v1.12.2 through v1.13.4** shipped updater v0.2.3,
+which ran `gh attestation verify` and refused every update when `gh` was not
+on its `PATH`. Service managers never put `gh` on the `PATH`: launchd's default is
+`/usr/bin:/bin:/usr/sbin:/sbin`, and most Linux hosts have no `gh` at all. So
+those nodes cannot update themselves. The updater log shows an hourly
+`gh CLI required for SLSA attestation verification` error. On macOS the log
+is `~/.pilot/updater.log`. On Linux, read it with `journalctl -u pilot-updater`.
+
+The broken updater cannot fix itself. On an affected node, do one of these:
+
+1. **Re-run the installer** (recommended). It installs the current binaries,
+   including a gh-free updater, and restarts the services:
+
+   ```sh
+   curl -fsSL https://pilotprotocol.network/install.sh | sh
+   ```
+
+2. **Or** run `pilotctl update` once from an interactive shell where `gh` is
+   installed and logged in (`gh auth status`). That one-shot update uses your
+   shell's `PATH`, installs the current release and replaces `pilot-updater`
+   with a gh-free version. The old `pilotctl update` exits silently after
+   doing so. Then restart both services so they run the new binaries:
+   `sudo systemctl restart pilot-updater pilot-daemon` on Linux, or on macOS
+   `launchctl kickstart -k gui/$(id -u)/network.pilotprotocol.pilot-updater`
+   and the same command for `network.pilotprotocol.pilot-daemon`.
+
+Check the result with `pilotctl version` and `~/.pilot/bin/pilot-updater --version`.
+Do **not** work around the problem with `PILOT_UPDATER_SKIP_ATTESTATION`,
+because that switches off provenance checks.
+
 ## Layout
 
 | File | What it does |
 |---|---|
-| `updater.go` | `Updater` — GitHub release polling, download, SHA verify, atomic rename. |
+| `updater.go` | `Updater`: check loop, `RunOnce`, release polling, install, daemon restart. |
+| `supervisor.go` | Linux restart safety: finds the daemon's systemd service and checks its restart policy before any SIGTERM. |
+| `download.go` | Resumable asset downloads with idle and total-duration limits. |
+| `github.go` | GitHub API calls: optional `GITHUB_TOKEN`, rate-limit errors. |
+| `attestation.go` | In-process SLSA provenance verification of `checksums.txt` (sigstore-go). |
+| `status.go` | `Status` / `update-state.json` persistence and `ReadStatus`. |
 | `version.go` | SemVer parsing and comparison. |
 | `service.go` | `*Service` — `coreapi.Service` adapter. Build tag `!no_updater`. |
 | `service_disabled.go` | Stub when build tag `no_updater` is set. |

attestation.go

@@ -14,7 +14,6 @@ import (
 	"regexp"
 	"strings"
 	"sync"
-	"time"
 
 	sgbundle "github.com/sigstore/sigstore-go/pkg/bundle"
 	"github.com/sigstore/sigstore-go/pkg/root"
@@ -47,8 +46,9 @@ var (
 )
 
 // githubToken returns a GitHub API token from the environment, if any. Optional:
-// used only to raise the API rate limit (see fetchAttestations). GITHUB_TOKEN is
-// the conventional name; GH_TOKEN is gh's alias.
+// used only to raise the API rate limit (see doGitHubAPI) for the releases and
+// attestations endpoints. GITHUB_TOKEN is the conventional name; GH_TOKEN is
+// gh's alias. The updater never runs the gh CLI itself.
 func githubToken() string {
 	for _, k := range []string{"GITHUB_TOKEN", "GH_TOKEN"} {
 		if v := strings.TrimSpace(os.Getenv(k)); v != "" {
@@ -89,8 +89,8 @@ func sha256Hex(path string) (string, error) {
 }
 
 // fetchAttestations retrieves the SLSA attestation bundles GitHub holds for the
-// given repository + artifact digest. Unauthenticated: the endpoint is public
-// for public repositories and requires no gh CLI or token.
+// given repository + artifact digest. The endpoint is public for public
+// repositories and requires no gh CLI or token.
 func fetchAttestations(client *http.Client, repo, digestHex string) ([]json.RawMessage, error) {
 	url := fmt.Sprintf("%s/repos/%s/attestations/sha256:%s", attestationAPIBase, repo, digestHex)
 	req, err := http.NewRequest(http.MethodGet, url, nil)
@@ -99,23 +99,19 @@ func fetchAttestations(client *http.Client, repo, digestHex string) ([]json.RawM
 	}
 	req.Header.Set("Accept", "application/vnd.github+json")
 	req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
-	// No token is required — the endpoint is public for public repositories.
-	// If one is present in the environment we send it purely to lift the
-	// 60-req/hr unauthenticated rate limit (5000/hr authenticated), so a busy
-	// fleet does not intermittently fail closed on a 403 rate-limit. Absence of
-	// a token never changes correctness, only the rate ceiling.
-	if tok := githubToken(); tok != "" {
-		req.Header.Set("Authorization", "Bearer "+tok)
-	}
 
-	resp, err := client.Do(req)
+	// No token is required. If GITHUB_TOKEN/GH_TOKEN is set, doGitHubAPI
+	// sends it purely to lift the 60-req/hr unauthenticated rate limit
+	// (5000/hr authenticated), so a busy fleet does not intermittently fail
+	// closed on a 403 rate-limit. A token never changes correctness, only the
+	// rate ceiling.
+	resp, err := doGitHubAPI(client, req)
 	if err != nil {
 		return nil, fmt.Errorf("fetch attestations: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
-		return nil, fmt.Errorf("GitHub attestations API returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+		return nil, githubAPIError("GitHub attestations API", resp)
 	}
 
 	var parsed githubAttestationsResponse
@@ -206,7 +202,7 @@ func realVerifyChecksumsAttestationFn(repo, tag, checksumsPath string) error {
 		verify.WithCertificateIdentity(certID),
 	)
 
-	client := &http.Client{Timeout: 30 * time.Second}
+	client := &http.Client{Timeout: apiTimeout, Transport: newTransport()}
 	rawBundles, err := fetchAttestations(client, repo, digestHex)
 	if err != nil {
 		return err