From 3f3c787f8c78fad87f7eba327ad916cfaa582e4d Mon Sep 17 00:00:00 2001 From: James Reilly Date: Sun, 25 Oct 2026 19:49:56 +0000 Subject: [PATCH 01/62] feat(omp): configure the omp auth broker client on every host The omp role installed omp but never pointed it at the fleet's auth broker (`omp auth-broker`, running on punjab), so a rebuilt machine had an empty ~/.omp/agent/config.yml and no provider credentials until someone ran /login by hand. The role now sets auth.broker.url from group_vars and, when Bitwarden is unlocked, writes auth.broker.token from the item omp-auth-broker-token. A locked vault leaves an existing token alone. The config file is set to 0600 and the role reports `omp auth-broker status`. just seed-omp-broker (run on punjab) creates the Bitwarden item from `omp auth-broker token`, which prints the token without rotating it. The broker's own systemd unit on punjab is still hand-made and unmanaged. --- Justfile | 4 ++ docs/src/roles/omp.md | 20 +++++++++- group_vars/all.yml | 5 +++ roles/omp/tasks/main.yml | 69 +++++++++++++++++++++++++++++++++++ scripts/bw-seed-omp-broker.sh | 38 +++++++++++++++++++ 5 files changed, 135 insertions(+), 1 deletion(-) create mode 100755 scripts/bw-seed-omp-broker.sh diff --git a/Justfile b/Justfile index 4461bdb9..6ad8d38f 100644 --- a/Justfile +++ b/Justfile @@ -516,6 +516,10 @@ docs-build: seed-kube: @{{ dotfiles_dir }}/scripts/bw-seed-kube.sh +# Seed Bitwarden with the omp auth-broker bearer token (run on the broker host, punjab) +seed-omp-broker: + @{{ dotfiles_dir }}/scripts/bw-seed-omp-broker.sh + # Show all online fleet machines' doctor status in parallel doctor-fleet: #!/usr/bin/env bash diff --git a/docs/src/roles/omp.md b/docs/src/roles/omp.md index 1900f85f..d391b403 100644 --- a/docs/src/roles/omp.md +++ b/docs/src/roles/omp.md @@ -1,7 +1,7 @@ # omp **Tags:** `dotfiles`, `omp` -**Secrets needed:** No for OMP/Kiro installation; `shell_ai` optionally provides the DeepSeek API key +**Secrets needed:** No for OMP/Kiro installation; the auth broker token and `shell_ai`'s optional DeepSeek key come from Bitwarden **Runs on:** All supported hosts except raw Termux Installs Oh My Pi and the Kiro provider on each supported computer. Homebrew hosts install OMP and Bun from the `can1357/tap` tap. musl hosts use OMP's checksum-verified binary installer and Bun's installer. The Kiro provider is installed from `github:elkibalion/omp-kiro-provider`. @@ -13,6 +13,24 @@ Installs Oh My Pi and the Kiro provider on each supported computer. Homebrew hos 3. Deploys personal skills to `~/.omp/agent/skills/james/` and symlinks them into `~/.claude/skills/`. 4. Exposes the Bitwarden DeepSeek key as `DEEPSEEK_API_KEY` through the shell secret files, not an agent-specific credential file. 5. Removes the legacy Pi CLI from raw Termux. OMP has no native Android/Bionic build; the Debian VM (`mumbai`) is supported normally. +6. Points OMP at the fleet auth broker (`auth.broker.url`) and, when Bitwarden is unlocked, writes the bearer token (`auth.broker.token`), so the machine needs no local provider `/login`. + +## Auth broker + +`omp auth-broker` is the fleet's credential vault: provider logins live on one broker, and every other host is a client that holds only a URL and a bearer token. The broker runs on `punjab` as a user unit (`omp-auth-broker.service`, `omp auth-broker serve --bind=:8765`). That unit is created by hand and is **not** managed by this repo. + +Clients get two keys in `~/.omp/agent/config.yml`: + +| Key | Source | +|---|---| +| `auth.broker.url` | `omp_auth_broker_url` in `group_vars/all.yml` (punjab's tailnet address, port 8765) | +| `auth.broker.token` | Bitwarden password item `omp-auth-broker-token` (`omp_auth_broker_bw_item`) | + +The token is written only when Bitwarden is unlocked; a locked vault leaves the existing token as it is. The config file is set to mode `0600`. + +To seed the token, run `just seed-omp-broker` **on punjab** with the vault unlocked. It reads `omp auth-broker token` (which does not rotate it) and creates or updates the Bitwarden item. Then run `just apply-tags omp` on each client. Verify with `omp auth-broker status`. + +If the token is ever rotated (`omp auth-broker token --regenerate` on punjab), seed again and re-apply on the clients. ## Skills and the package manifest diff --git a/group_vars/all.yml b/group_vars/all.yml index 82fa39dc..5b987e6f 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -4,6 +4,11 @@ user_name: James Reilly user_email: jreilly1821@gmail.com user_login: james brew_prefix: /home/linuxbrew/.linuxbrew +# OMP auth broker: punjab runs `omp auth-broker serve` on its tailnet address +# (a hand-made user unit, not managed here). Clients need this URL plus the +# bearer token from the Bitwarden item below; see docs/src/roles/omp.md. +omp_auth_broker_url: "http://{{ hostvars['punjab']['ansible_host'] }}:8765" +omp_auth_broker_bw_item: omp-auth-broker-token # Every inventory host reachable over SSH, keyed by inventory name, with the # address ssh should use (MagicDNS name unless noted). Port and login user # come from inventory.yml (ansible_port / ansible_user) — ssh_config.j2 and diff --git a/roles/omp/tasks/main.yml b/roles/omp/tasks/main.yml index a8d74999..8fdeed24 100644 --- a/roles/omp/tasks/main.yml +++ b/roles/omp/tasks/main.yml @@ -140,3 +140,72 @@ PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_env_path }}" register: omp_routing_merge changed_when: "'changed' in omp_routing_merge.stdout" + +# ── Auth broker (`omp auth-broker`): the fleet's credential vault ─────────── +# One canonical broker holds the provider logins (punjab, over the tailnet); +# every other host is a client that needs only its URL and bearer token, so no +# per-machine `/login` is required. The URL is not secret. The token comes from +# Bitwarden and is written only when the vault is unlocked, so a locked vault +# leaves any existing token untouched. +- name: Point OMP at the auth broker + ansible.builtin.shell: | + set -euo pipefail + cfg="$HOME/.omp/agent/config.yml" + mkdir -p "$(dirname "$cfg")" + [ -f "$cfg" ] || : > "$cfg" + before=$(sha256sum "$cfg") + URL='{{ omp_auth_broker_url }}' yq -i '.auth.broker.url = strenv(URL)' "$cfg" + chmod 600 "$cfg" + [ "$before" = "$(sha256sum "$cfg")" ] || echo changed + environment: + PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_env_path }}" + register: omp_broker_url_set + changed_when: "'changed' in omp_broker_url_set.stdout" + when: omp_auth_broker_url | length > 0 + +- name: Fetch and write the auth broker token + when: (omp_auth_broker_url | length > 0) and (bw_unlocked | default(false)) + environment: + BW_SESSION: "{{ bw_session }}" + PATH: "{{ ansible_env_path }}" + block: + - name: Fetch the broker bearer token from Bitwarden + ansible.builtin.command: bw get password {{ omp_auth_broker_bw_item }} + register: omp_broker_token + no_log: true + changed_when: false + failed_when: false + + - name: Write the broker token into the OMP config + ansible.builtin.shell: | + set -euo pipefail + [ -n "$TOKEN" ] || exit 0 + cfg="$HOME/.omp/agent/config.yml" + before=$(sha256sum "$cfg") + yq -i '.auth.broker.token = strenv(TOKEN)' "$cfg" + chmod 600 "$cfg" + [ "$before" = "$(sha256sum "$cfg")" ] || echo changed + environment: + TOKEN: "{{ omp_broker_token.stdout | default('') | trim }}" + PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_env_path }}" + register: omp_broker_token_set + no_log: true + changed_when: "'changed' in omp_broker_token_set.stdout" + when: omp_broker_token.rc == 0 + +- name: Check the auth broker answers + ansible.builtin.command: omp auth-broker status + environment: + PATH: "{{ ansible_facts['env']['HOME'] }}/.bun/bin:{{ ansible_facts['env']['HOME'] }}/.local/bin:{{ ansible_env_path }}" + register: omp_broker_status + changed_when: false + failed_when: false + when: omp_auth_broker_url | length > 0 + +- name: Report auth broker status + ansible.builtin.debug: + msg: >- + omp auth-broker: {{ (omp_broker_status.stdout | default('') ~ omp_broker_status.stderr | default('')) | trim }} + {{ '' if omp_broker_status.rc | default(1) == 0 else + '(not healthy. With the vault unlocked, run `just seed-omp-broker` on punjab once, then `just apply-tags omp` here.)' }} + when: omp_auth_broker_url | length > 0 diff --git a/scripts/bw-seed-omp-broker.sh b/scripts/bw-seed-omp-broker.sh new file mode 100755 index 00000000..845a2974 --- /dev/null +++ b/scripts/bw-seed-omp-broker.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Seed Bitwarden with the omp auth-broker bearer token. Run on the broker host +# (punjab), where `omp auth-broker token` reads the broker's own token. +# Usage: ./scripts/bw-seed-omp-broker.sh +# Re-run safely — updates the existing item in place. `omp auth-broker token` +# prints the current token and does not rotate it. Clients fetch the item with +# `just apply-tags omp`. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/bw-item.sh" + +if [ -z "${BW_SESSION:-}" ]; then + # Assign then export: `export X=$(cmd)` would mask a failed unlock from set -e. + BW_SESSION=$("$SCRIPT_DIR/bw-unlock.sh") + export BW_SESSION +fi + +NAME="${OMP_AUTH_BROKER_BW_ITEM:-omp-auth-broker-token}" + +token=$(omp auth-broker token) +if [ -z "$token" ]; then + echo "no token from 'omp auth-broker token' — is this the broker host?" >&2 + exit 1 +fi + +# type 1 = login; the role reads it with `bw get password`. +payload=$(bw get template item \ + | jq --arg name "$NAME" --arg tok "$token" ' + . + { + name: $name, + type: 1, + notes: "Bearer token for the omp auth-broker on punjab. Managed by scripts/bw-seed-omp-broker.sh.", + login: {username: "omp-auth-broker", password: $tok} + }') + +bw_upsert_item "$NAME" "$payload" 1 +bw sync >/dev/null +echo "Done. Run \`just apply-tags omp\` on other machines to pick up the token." From 7abe249fdac72f2667857ffe74affd59df98cf43 Mon Sep 17 00:00:00 2001 From: James Reilly Date: Sun, 25 Oct 2026 20:19:24 +0000 Subject: [PATCH 02/62] refactor(omp): move omp to a bash unit run by mise --- Justfile | 21 +- docs/src/SUMMARY.md | 1 + docs/src/roles/omp.md | 59 +++--- docs/src/units.md | 37 ++++ group_vars/all.yml | 6 +- mise.toml | 7 + roles/omp/tasks/main.yml | 211 --------------------- roles/shell_dotfiles/vars/main.yml | 2 +- scripts/run-units.sh | 56 ++++++ site.yml | 3 - tests/test_omp_unit.py | 74 ++++++++ tests/test_units_registry.py | 36 ++++ units/lib.sh | 25 +++ units/omp.sh | 113 +++++++++++ units/omp/auth-broker.conf | 5 + {roles/omp/files => units/omp}/kiro.json | 0 {roles/omp/files => units/omp}/omp-route | 0 {roles/omp/files => units/omp}/routing.yml | 2 +- 18 files changed, 403 insertions(+), 255 deletions(-) create mode 100644 docs/src/units.md create mode 100644 mise.toml delete mode 100644 roles/omp/tasks/main.yml create mode 100755 scripts/run-units.sh create mode 100644 tests/test_omp_unit.py create mode 100644 tests/test_units_registry.py create mode 100644 units/lib.sh create mode 100755 units/omp.sh create mode 100644 units/omp/auth-broker.conf rename {roles/omp/files => units/omp}/kiro.json (100%) rename {roles/omp/files => units/omp}/omp-route (100%) rename {roles/omp/files => units/omp}/routing.yml (95%) diff --git a/Justfile b/Justfile index 6ad8d38f..180f1f67 100644 --- a/Justfile +++ b/Justfile @@ -43,6 +43,8 @@ apply *args: else {{ _play }} -e "bw_session=${BW_SESSION:-}" site.yml $BECOME_ARGS {{ args }} || rc=$? fi + urc=0; scripts/run-units.sh || urc=$? + [ "$rc" -ne 0 ] || rc=$urc scripts/record-apply.py "$rc" apply "$SKIP_TAGS" exit $rc @@ -53,6 +55,8 @@ apply-tags tags: cd {{ dotfiles_dir }} rc=0 {{ _play }} -e "bw_session=${BW_SESSION:-}" site.yml --tags {{ tags }} || rc=$? + urc=0; scripts/run-units.sh --tags "{{ tags }}" || urc=$? + [ "$rc" -ne 0 ] || rc=$urc scripts/record-apply.py "$rc" apply-tags "{{ tags }}" exit $rc @@ -64,6 +68,8 @@ apply-nosecrets *args: git pull --ff-only || true rc=0 {{ _play }} site.yml --skip-tags secrets {{ args }} || rc=$? + urc=0; scripts/run-units.sh --no-secrets || urc=$? + [ "$rc" -ne 0 ] || rc=$urc scripts/record-apply.py "$rc" apply-nosecrets secrets exit $rc @@ -86,7 +92,18 @@ mesh-check: # Apply only dotfile configs (shell, git, tmux, etc.) dotfiles: - cd {{ dotfiles_dir }} && {{ _play }} site.yml --tags dotfiles + #!/usr/bin/env bash + set -uo pipefail + cd {{ dotfiles_dir }} + rc=0 + {{ _play }} site.yml --tags dotfiles || rc=$? + urc=0; scripts/run-units.sh --tags dotfiles || urc=$? + [ "$rc" -ne 0 ] || rc=$urc + exit $rc + +# Run the bash units (units/*.sh) in parallel via mise, e.g. `just units --tags omp` +units *args: + cd {{ dotfiles_dir }} && scripts/run-units.sh {{ args }} # Apply only packages (Homebrew + Flatpak) packages: @@ -443,7 +460,7 @@ doctor: echo "→ Machine identity" if [ -f /etc/dotfiles-machine ]; then pass "machine: $(cat /etc/dotfiles-machine)"; else warn "/etc/dotfiles-machine missing — run bootstrap"; fi echo "→ Core binaries" - for bin in brew bw gh tailscale ansible-playbook just; do + for bin in brew bw gh tailscale ansible-playbook just mise; do if command -v "$bin" >/dev/null 2>&1; then pass "$bin"; else warn "$bin missing"; fi done echo "→ Tailscale" diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 8adf5ef5..9f95290f 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -5,6 +5,7 @@ # Core Concepts - [Architecture](architecture.md) +- [Units](units.md) - [Inventory & Groups](inventory.md) - [Secrets with Bitwarden](bitwarden.md) - [Onboarding a New Machine](onboarding.md) diff --git a/docs/src/roles/omp.md b/docs/src/roles/omp.md index d391b403..37c3ecac 100644 --- a/docs/src/roles/omp.md +++ b/docs/src/roles/omp.md @@ -1,47 +1,42 @@ # omp -**Tags:** `dotfiles`, `omp` -**Secrets needed:** No for OMP/Kiro installation; the auth broker token and `shell_ai`'s optional DeepSeek key come from Bitwarden -**Runs on:** All supported hosts except raw Termux +**Unit:** `units/omp.sh` (mise task `unit:omp`; see [Units](../units.md)) +**Tags:** `dotfiles`, `omp` (`just apply-tags omp` runs it after the Ansible pass) +**Secrets needed:** No for the install and routing; the auth-broker token comes from Bitwarden when the vault is unlocked +**Runs on:** All supported hosts except raw Termux (the unit prints `SKIPPED: omp has no Android build`) -Installs Oh My Pi and the Kiro provider on each supported computer. Homebrew hosts install OMP and Bun from the `can1357/tap` tap. musl hosts use OMP's checksum-verified binary installer and Bun's installer. The Kiro provider is installed from `github:elkibalion/omp-kiro-provider`. +Installs Oh My Pi's Kiro provider, skills, routing and the fleet auth-broker client config. Homebrew hosts get `omp` and Bun from the `can1357/tap` tap through `just packages`; musl hosts use OMP's checksum-verified binary installer and Bun's installer, run by the unit itself. This is no longer an Ansible role. -## What It Does +## Steps -1. Installs and verifies `omp`. -2. Installs the Kiro provider plugin for the current user; it is available as `kiro/*` models. -3. Deploys personal skills to `~/.omp/agent/skills/james/` and symlinks them into `~/.claude/skills/`. -4. Exposes the Bitwarden DeepSeek key as `DEEPSEEK_API_KEY` through the shell secret files, not an agent-specific credential file. -5. Removes the legacy Pi CLI from raw Termux. OMP has no native Android/Bionic build; the Debian VM (`mumbai`) is supported normally. -6. Points OMP at the fleet auth broker (`auth.broker.url`) and, when Bitwarden is unlocked, writes the bearer token (`auth.broker.token`), so the machine needs no local provider `/login`. +Each step is independent: a failure prints `FAIL `, fails the unit, and the later steps still run. + +1. `omp installed`: on musl, install `omp` and Bun; elsewhere fail with `omp not installed; run: just packages` if missing. Steps that call `omp` print `skip ... (omp missing)` instead. +2. `skills synced`: copy `roles/pi/files/skills/` to `~/.omp/agent/skills/james/`; make `*.sh`/`*.mjs` executable. (`roles/pi` is skill source data, not an Ansible role.) +3. `claude skill links`: symlink each skill directory into `~/.claude/skills/`. +4. `kiro plugin`: `omp plugin install github:elkibalion/omp-kiro-provider` unless installed. +5. `ccleft` (soft): `go install` the quota probe. +6. `kiro.json`: `~/.config/omp/kiro.json` (kiro-cli ACP fallback on 401/403), creating `~/.config/omp`. +7. `omp-route`: the quota-aware launcher at `~/.local/bin/omp-route`. +8. `routing merged`: deep-merge `units/omp/routing.yml` into `~/.omp/agent/config.yml` with `yq` (lists replace). +9. `broker url`: set `auth.broker.url`; config mode `0600`. +10. `broker token`: set `auth.broker.token` from Bitwarden when the vault is unlocked. If no token ends up in the config the unit prints `PARTIAL: broker token not written ...`. +11. `broker status` (soft): print `omp auth-broker status`. ## Auth broker `omp auth-broker` is the fleet's credential vault: provider logins live on one broker, and every other host is a client that holds only a URL and a bearer token. The broker runs on `punjab` as a user unit (`omp-auth-broker.service`, `omp auth-broker serve --bind=:8765`). That unit is created by hand and is **not** managed by this repo. -Clients get two keys in `~/.omp/agent/config.yml`: - -| Key | Source | +| Key in `~/.omp/agent/config.yml` | Source | |---|---| -| `auth.broker.url` | `omp_auth_broker_url` in `group_vars/all.yml` (punjab's tailnet address, port 8765) | -| `auth.broker.token` | Bitwarden password item `omp-auth-broker-token` (`omp_auth_broker_bw_item`) | - -The token is written only when Bitwarden is unlocked; a locked vault leaves the existing token as it is. The config file is set to mode `0600`. - -To seed the token, run `just seed-omp-broker` **on punjab** with the vault unlocked. It reads `omp auth-broker token` (which does not rotate it) and creates or updates the Bitwarden item. Then run `just apply-tags omp` on each client. Verify with `omp auth-broker status`. - -If the token is ever rotated (`omp auth-broker token --regenerate` on punjab), seed again and re-apply on the clients. +| `auth.broker.url` | `units/omp/auth-broker.conf` (`OMP_AUTH_BROKER_URL`, punjab's tailnet address, port 8765; keep in step with `inventory.yml`) | +| `auth.broker.token` | Bitwarden password item `omp-auth-broker-token` (`OMP_AUTH_BROKER_BW_ITEM`) | -## Skills and the package manifest +The token is written only when Bitwarden is unlocked; a locked vault or `--no-secrets` leaves the existing token as it is. The config file is `0600`. -The playbook copies `roles/pi/files/skills/` to -`~/.omp/agent/skills/james/` and links those skill directories into -`~/.claude/skills/`. The `roles/pi` path is skill source data, not a separate -Ansible role. +To seed the token, run `just seed-omp-broker` **on punjab** with the vault unlocked. It reads `omp auth-broker token` (which does not rotate it) and creates or updates the Bitwarden item. Then run `just apply-tags omp` on each client. Verify with `omp auth-broker status`. If the token is rotated (`omp auth-broker token --regenerate` on punjab), seed again and re-apply on the clients. -`roles/pi/files/package.json` is tracked by Renovate and protected by -`deps-guard`; it is not installed by this role. OMP plugin installation is -handled separately by the role's explicit plugin tasks. +`DEEPSEEK_API_KEY` and the other provider keys are exposed to the shell by the `shell_ai` role's secret files, not by this unit. ## Kiro authentication @@ -52,8 +47,8 @@ omp /login kiro ``` -Complete the browser sign-in. Then select `kiro/auto` with `/model kiro/auto`. The playbook does not set Kiro as the default model because the OAuth session is not present until a user signs in. +Complete the browser sign-in, then select `kiro/auto` with `/model kiro/auto`. Kiro is not the default model because the OAuth session does not exist until a user signs in. ## Package source -The OMP and Bun Homebrew formulas are in `can1357/tap`; both are core packages for Homebrew-supported hosts. On musl, the official OMP binary installer verifies its SHA-256 and requires `libgcc` and `libstdc++`. Raw Termux receives neither OMP nor the obsolete Pi npm package. +The OMP and Bun Homebrew formulas are in `can1357/tap`; both are core packages for Homebrew-supported hosts. On musl, the official OMP binary installer verifies its SHA-256 and requires `libgcc` and `libstdc++`. `roles/pi/files/package.json` is tracked by Renovate and protected by `deps-guard`; the unit does not install it. diff --git a/docs/src/units.md b/docs/src/units.md new file mode 100644 index 00000000..42f5c721 --- /dev/null +++ b/docs/src/units.md @@ -0,0 +1,37 @@ +# Units + +A **unit** is an idempotent bash script in `units/` that configures one independent piece of a machine. Units exist because Ansible aborts a whole role when one task fails: on himachal the `omp` role died at `Deploy omp-kiro-provider config` (`~/.config/omp` did not exist), so the routing merge, `omp-route` and the auth-broker config never ran. In a unit, a failing step is recorded and the next independent step still runs; independent units run in parallel. + +[mise](https://mise.jdx.dev) is the runner: each unit is a `mise.toml` task, and `mise run` supplies the dependency graph, parallel jobs and `--continue-on-error`. + +Everything not listed under `units/` is still an Ansible role. + +## Running units + +| Command | What it does | +|---|---| +| `just units` | Run every unit in parallel (`just units --tags omp`, `--no-secrets` also accepted) | +| `mise run units` | Same, straight through mise | +| `mise run unit:omp` | One unit | +| `mise run --dry-run units` | Show what would run | +| `mise tasks deps units` | Show the task graph | +| `just apply`, `apply-nosecrets`, `apply-tags`, `dotfiles` | Run Ansible first, then `scripts/run-units.sh` (never skipped because Ansible failed; exit codes are combined) | + +`scripts/run-units.sh [--no-secrets] [--tags a,b]` is the one wrapper the recipes call. It maps Ansible tags to units (`tasks_for_tag`, mirroring the tags of the roles the units replaced), runs the selection with `mise run --continue-on-error --jobs 8 --output prefix --timeout 15m`, and exits with mise's status. Without `mise` on `PATH` it prints `mise not installed; running units sequentially` and runs the selected units one after another, continuing after failures, so hosts without mise (Termux, musl) keep updating. + +## Rules for a unit + +- Idempotent bash. `set -uo pipefail`, never `set -e`. +- Source `units/lib.sh`: `step "desc" cmd...` (a failure marks the unit failed and continues), `soft "desc" cmd...` (a failure is only a warning), `note "..."`, `finish` (exit 1 only if a `step` failed), `install_if_changed src dest mode`, `bw_session` (a usable Bitwarden session or nothing; never prompts; honours `UNITS_NO_SECRETS=1`). +- mise treats any non-zero exit as failure, so "skipped" and "partial" are printed as `SKIPPED: ...` / `PARTIAL: ...` lines with exit 0. +- Never echo a secret or put one on a command line. + +## Adding a unit + +1. Create `units/.sh` (executable) with a `# shellcheck source=units/lib.sh` line above its `. lib.sh` line. +2. Add `[tasks."unit:"]` with `file = "units/.sh"` to `mise.toml`. +3. Add `` to `ALL_UNITS` and to the right `tasks_for_tag` branches in `scripts/run-units.sh`. + +`tests/test_units_registry.py` fails if the three places disagree. + +`units/`, `mise.toml` and `scripts/run-units.sh` are executed by the fleet timer, so they are intentionally not carved out of CODEOWNERS. diff --git a/group_vars/all.yml b/group_vars/all.yml index 5b987e6f..bf9f9714 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -4,11 +4,6 @@ user_name: James Reilly user_email: jreilly1821@gmail.com user_login: james brew_prefix: /home/linuxbrew/.linuxbrew -# OMP auth broker: punjab runs `omp auth-broker serve` on its tailnet address -# (a hand-made user unit, not managed here). Clients need this URL plus the -# bearer token from the Bitwarden item below; see docs/src/roles/omp.md. -omp_auth_broker_url: "http://{{ hostvars['punjab']['ansible_host'] }}:8765" -omp_auth_broker_bw_item: omp-auth-broker-token # Every inventory host reachable over SSH, keyed by inventory name, with the # address ssh should use (MagicDNS name unless noted). Port and login user # come from inventory.yml (ansible_port / ansible_user) — ssh_config.j2 and @@ -86,6 +81,7 @@ core_brews: - tmux - direnv - just + - mise - make - shellcheck - shfmt diff --git a/mise.toml b/mise.toml new file mode 100644 index 00000000..cb4032f2 --- /dev/null +++ b/mise.toml @@ -0,0 +1,7 @@ +[tasks.units] +description = "Run every unit in parallel; one failing unit does not stop the others" +depends = ["unit:*"] + +[tasks."unit:omp"] +description = "omp: skills, Kiro plugin, routing, and the auth-broker client" +file = "units/omp.sh" diff --git a/roles/omp/tasks/main.yml b/roles/omp/tasks/main.yml deleted file mode 100644 index 8fdeed24..00000000 --- a/roles/omp/tasks/main.yml +++ /dev/null @@ -1,211 +0,0 @@ ---- -- name: Ensure installer cache exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.cache" - state: directory - mode: "0700" - when: is_musl | bool - -- name: Download OMP installer on musl systems - ansible.builtin.get_url: - url: https://omp.sh/install - dest: "{{ ansible_facts['env']['HOME'] }}/.cache/omp-install.sh" - mode: "0700" - when: is_musl | bool - -- name: Install OMP on musl systems - ansible.builtin.command: sh {{ ansible_facts['env']['HOME'] }}/.cache/omp-install.sh --binary - args: - creates: "{{ ansible_facts['env']['HOME'] }}/.local/bin/omp" - environment: - PI_INSTALL_DIR: "{{ ansible_facts['env']['HOME'] }}/.local/bin" - when: is_musl | bool - -- name: Download Bun installer on musl systems - ansible.builtin.get_url: - url: https://bun.sh/install - dest: "{{ ansible_facts['env']['HOME'] }}/.cache/bun-install.sh" - mode: "0700" - when: is_musl | bool - -- name: Install Bun on musl systems for OMP plugins - ansible.builtin.command: bash {{ ansible_facts['env']['HOME'] }}/.cache/bun-install.sh - args: - creates: "{{ ansible_facts['env']['HOME'] }}/.bun/bin/bun" - when: is_musl | bool - - -- name: Verify OMP is installed - ansible.builtin.command: omp --version - environment: - PATH: "{{ ansible_facts['env']['HOME'] }}/.bun/bin:{{ ansible_facts['env']['HOME'] }}/.local/bin:{{ ansible_env_path }}" - changed_when: false - -- name: Ensure OMP personal skills directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.omp/agent/skills/james" - state: directory - mode: "0755" - -- name: Deploy personal skills to OMP - ansible.builtin.copy: - src: "{{ playbook_dir }}/roles/pi/files/skills/" - dest: "{{ ansible_facts['env']['HOME'] }}/.omp/agent/skills/james/" - mode: "0644" - directory_mode: "0755" - -- name: Find skill scripts that need to be executable - ansible.builtin.find: - paths: "{{ ansible_facts['env']['HOME'] }}/.omp/agent/skills/james" - patterns: "*.sh,*.mjs" - recurse: true - register: omp_skill_scripts - -- name: Make skill scripts executable - ansible.builtin.file: - path: "{{ item.path }}" - mode: "0755" - loop: "{{ omp_skill_scripts.files }}" - loop_control: - label: "{{ item.path | basename }}" - -- name: Ensure Claude skills directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.claude/skills" - state: directory - mode: "0755" - -- name: Enumerate deployed OMP skills - ansible.builtin.find: - paths: "{{ ansible_facts['env']['HOME'] }}/.omp/agent/skills/james" - file_type: directory - recurse: false - register: omp_skill_dirs - -- name: Link OMP skills into Claude's skill directory - ansible.builtin.file: - src: "{{ item.path }}" - dest: "{{ ansible_facts['env']['HOME'] }}/.claude/skills/{{ item.path | basename }}" - state: link - force: true - loop: "{{ omp_skill_dirs.files }}" - loop_control: - label: "{{ item.path | basename }}" - -- name: Check installed OMP plugins - ansible.builtin.command: omp plugin list --json - environment: - PATH: "{{ ansible_facts['env']['HOME'] }}/.bun/bin:{{ ansible_facts['env']['HOME'] }}/.local/bin:{{ ansible_env_path }}" - register: omp_plugin_list - changed_when: false - failed_when: false - -- name: Install Kiro provider plugin from GitHub - ansible.builtin.command: - cmd: omp plugin install github:elkibalion/omp-kiro-provider - environment: - PATH: "{{ ansible_facts['env']['HOME'] }}/.bun/bin:{{ ansible_facts['env']['HOME'] }}/.local/bin:{{ ansible_env_path }}" - when: omp_plugin_list.rc != 0 or 'omp-kiro-provider' not in (omp_plugin_list.stdout | default('')) - -- name: Install ccleft (provider quota-remaining probe) - ansible.builtin.command: go install github.com/tuna-os/ccleft/cmd/ccleft@latest - environment: - PATH: "{{ ansible_facts['env']['HOME'] }}/.local/bin:/home/linuxbrew/.linuxbrew/bin:{{ ansible_env_path }}" - args: - creates: "{{ ansible_facts['env']['HOME'] }}/go/bin/ccleft" - failed_when: false - -- name: Deploy omp-kiro-provider config (kiro-cli ACP fallback on 401/403) - ansible.builtin.copy: - src: kiro.json - dest: "{{ ansible_facts['env']['HOME'] }}/.config/omp/kiro.json" - mode: "0644" - -- name: Install omp-route (quota-aware omp launcher via ccleft) - ansible.builtin.copy: - src: omp-route - dest: "{{ ansible_facts['env']['HOME'] }}/.local/bin/omp-route" - mode: "0755" - -- name: Merge provider routing into OMP config - ansible.builtin.shell: | - set -euo pipefail - cfg="$HOME/.omp/agent/config.yml" - mkdir -p "$(dirname "$cfg")" - [ -f "$cfg" ] || : > "$cfg" - before=$(sha256sum "$cfg") - yq -i '. *= (load("{{ role_path }}/files/routing.yml") | explode(.) | ... comments="")' "$cfg" - [ "$before" = "$(sha256sum "$cfg")" ] || echo changed - environment: - PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_env_path }}" - register: omp_routing_merge - changed_when: "'changed' in omp_routing_merge.stdout" - -# ── Auth broker (`omp auth-broker`): the fleet's credential vault ─────────── -# One canonical broker holds the provider logins (punjab, over the tailnet); -# every other host is a client that needs only its URL and bearer token, so no -# per-machine `/login` is required. The URL is not secret. The token comes from -# Bitwarden and is written only when the vault is unlocked, so a locked vault -# leaves any existing token untouched. -- name: Point OMP at the auth broker - ansible.builtin.shell: | - set -euo pipefail - cfg="$HOME/.omp/agent/config.yml" - mkdir -p "$(dirname "$cfg")" - [ -f "$cfg" ] || : > "$cfg" - before=$(sha256sum "$cfg") - URL='{{ omp_auth_broker_url }}' yq -i '.auth.broker.url = strenv(URL)' "$cfg" - chmod 600 "$cfg" - [ "$before" = "$(sha256sum "$cfg")" ] || echo changed - environment: - PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_env_path }}" - register: omp_broker_url_set - changed_when: "'changed' in omp_broker_url_set.stdout" - when: omp_auth_broker_url | length > 0 - -- name: Fetch and write the auth broker token - when: (omp_auth_broker_url | length > 0) and (bw_unlocked | default(false)) - environment: - BW_SESSION: "{{ bw_session }}" - PATH: "{{ ansible_env_path }}" - block: - - name: Fetch the broker bearer token from Bitwarden - ansible.builtin.command: bw get password {{ omp_auth_broker_bw_item }} - register: omp_broker_token - no_log: true - changed_when: false - failed_when: false - - - name: Write the broker token into the OMP config - ansible.builtin.shell: | - set -euo pipefail - [ -n "$TOKEN" ] || exit 0 - cfg="$HOME/.omp/agent/config.yml" - before=$(sha256sum "$cfg") - yq -i '.auth.broker.token = strenv(TOKEN)' "$cfg" - chmod 600 "$cfg" - [ "$before" = "$(sha256sum "$cfg")" ] || echo changed - environment: - TOKEN: "{{ omp_broker_token.stdout | default('') | trim }}" - PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_env_path }}" - register: omp_broker_token_set - no_log: true - changed_when: "'changed' in omp_broker_token_set.stdout" - when: omp_broker_token.rc == 0 - -- name: Check the auth broker answers - ansible.builtin.command: omp auth-broker status - environment: - PATH: "{{ ansible_facts['env']['HOME'] }}/.bun/bin:{{ ansible_facts['env']['HOME'] }}/.local/bin:{{ ansible_env_path }}" - register: omp_broker_status - changed_when: false - failed_when: false - when: omp_auth_broker_url | length > 0 - -- name: Report auth broker status - ansible.builtin.debug: - msg: >- - omp auth-broker: {{ (omp_broker_status.stdout | default('') ~ omp_broker_status.stderr | default('')) | trim }} - {{ '' if omp_broker_status.rc | default(1) == 0 else - '(not healthy. With the vault unlocked, run `just seed-omp-broker` on punjab once, then `just apply-tags omp` here.)' }} - when: omp_auth_broker_url | length > 0 diff --git a/roles/shell_dotfiles/vars/main.yml b/roles/shell_dotfiles/vars/main.yml index f6221585..9d89d4f5 100644 --- a/roles/shell_dotfiles/vars/main.yml +++ b/roles/shell_dotfiles/vars/main.yml @@ -94,7 +94,7 @@ alias_groups: - { name: md, cmd: "mkdir -p" } - { name: h, cmd: history } - # Quota-aware launcher (roles/omp): starts omp on the first provider with + # Quota-aware launcher (units/omp): starts omp on the first provider with # ccleft-reported quota left. `command omp` bypasses it. - guard: omp-route fish_kind: alias diff --git a/scripts/run-units.sh b/scripts/run-units.sh new file mode 100755 index 00000000..ac93b734 --- /dev/null +++ b/scripts/run-units.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Run the bash units (units/*.sh) through mise, in parallel, failure-isolated. +# Usage: scripts/run-units.sh [--no-secrets] [--tags a,b] +set -uo pipefail + +ALL_UNITS="omp" # run order for the sequential fallback; keep in step with mise.toml +tasks_for_tag() { case "$1" in # Ansible tag -> units (mirrors the old role tags) + dotfiles|omp) echo omp ;; +esac; } + +repo="$(cd "$(dirname "$0")/.." && pwd)" +export DOTFILES_DIR="$repo" +PATH=/home/linuxbrew/.linuxbrew/bin:$PATH + +tags="" +while [ $# -gt 0 ]; do + case "$1" in + --no-secrets) export UNITS_NO_SECRETS=1 ;; + --tags) tags="${2:-}"; shift ;; + *) echo "usage: $0 [--no-secrets] [--tags a,b]" >&2; exit 2 ;; + esac + shift +done + +wanted=" " +if [ -z "$tags" ]; then + wanted=" $ALL_UNITS " +else + IFS=',' read -ra tag_list <<< "$tags" + for t in "${tag_list[@]}"; do + for u in $(tasks_for_tag "$t"); do wanted="$wanted$u "; done + done +fi +selected=() +for u in $ALL_UNITS; do + case "$wanted" in *" $u "*) selected+=("$u") ;; esac +done +[ "${#selected[@]}" -gt 0 ] || exit 0 + +cd "$repo" || exit 1 +if command -v mise >/dev/null 2>&1; then + mise trust -q "$repo/mise.toml" >/dev/null 2>&1 || true + args=() + for u in "${selected[@]}"; do + [ "${#args[@]}" -gt 0 ] && args+=(":::") + args+=("unit:$u") + done + exec mise run --continue-on-error --jobs "${UNITS_JOBS:-8}" --output prefix --timeout 15m "${args[@]}" +fi + +echo "mise not installed; running units sequentially (brew install mise for parallel runs)" +rc=0 +for u in "${selected[@]}"; do + bash "$repo/units/$u.sh" || rc=1 +done +exit $rc diff --git a/site.yml b/site.yml index bc0581a1..46b332d3 100644 --- a/site.yml +++ b/site.yml @@ -75,9 +75,6 @@ tags: [dotfiles, shell, fonts] - role: shell_dotfiles tags: [dotfiles, shell, dotfiles] - - role: omp - tags: [dotfiles, omp] - when: not is_termux | bool - role: hive_ops tags: [hive, hive_ops] - role: git diff --git a/tests/test_omp_unit.py b/tests/test_omp_unit.py new file mode 100644 index 00000000..f4fbb3c6 --- /dev/null +++ b/tests/test_omp_unit.py @@ -0,0 +1,74 @@ +import shutil +import subprocess + +import pytest +import yaml + +from conftest import REPO, write_stub + +pytestmark = pytest.mark.skipif(shutil.which("yq") is None, reason="yq not installed") + +BROKER_URL = "http://100.78.73.8:8765" + +OMP_STUB = """\ +case "$1 $2" in + "--version "*) echo omp/0 ;; + "plugin list") echo '["omp-kiro-provider"]' ;; + "auth-broker status") echo "OK stub" ;; +esac +exit 0 +""" + + +def run_unit(env): + env = {**env, "DOTFILES_DIR": str(REPO), "UNITS_NO_SECRETS": "1"} + return subprocess.run( + ["bash", str(REPO / "units/omp.sh")], env=env, capture_output=True, text=True + ) + + +def broker_url(home): + cfg = yaml.safe_load((home / ".omp/agent/config.yml").read_text()) + return cfg["auth"]["broker"]["url"] + + +@pytest.fixture +def env(stub_env): + bindir, env = stub_env + write_stub(bindir, "omp", OMP_STUB) + return env + + +def snapshot(home): + paths = [".omp/agent/config.yml", ".config/omp/kiro.json", ".local/bin/omp-route"] + return {p: (home / p).read_bytes() for p in paths} + + +def test_fresh_home_is_configured(env, tmp_path): + home = tmp_path / "home" + r = run_unit(env) + assert r.returncode == 0, r.stdout + r.stderr + # himachal regression: ~/.config did not exist and the role aborted. + assert (home / ".config/omp/kiro.json").is_file() + assert (home / ".local/bin/omp-route").stat().st_mode & 0o111 + assert broker_url(home) == BROKER_URL + assert (home / ".omp/agent/config.yml").stat().st_mode & 0o777 == 0o600 + assert "PARTIAL:" in r.stdout + + +def test_second_run_changes_nothing(env, tmp_path): + home = tmp_path / "home" + assert run_unit(env).returncode == 0 + before = snapshot(home) + assert run_unit(env).returncode == 0 + assert snapshot(home) == before + + +def test_failing_step_does_not_stop_later_steps(env, tmp_path): + home = tmp_path / "home" + (home / ".claude").write_text("not a directory") + r = run_unit(env) + assert r.returncode == 1 + assert "FAIL claude skill links" in r.stdout + assert (home / ".config/omp/kiro.json").is_file() + assert broker_url(home) == BROKER_URL diff --git a/tests/test_units_registry.py b/tests/test_units_registry.py new file mode 100644 index 00000000..c1522032 --- /dev/null +++ b/tests/test_units_registry.py @@ -0,0 +1,36 @@ +import re +import tomllib + +from conftest import REPO + + +def mise_units(): + tasks = tomllib.loads((REPO / "mise.toml").read_text())["tasks"] + return {k.removeprefix("unit:"): v for k, v in tasks.items() if k.startswith("unit:")} + + +def wrapper(): + return (REPO / "scripts/run-units.sh").read_text() + + +def all_units(): + return set(re.search(r'^ALL_UNITS="([^"]*)"', wrapper(), re.M).group(1).split()) + + +def test_every_unit_is_registered_everywhere(): + on_disk = {p.stem for p in (REPO / "units").glob("*.sh")} - {"lib"} + assert set(mise_units()) == on_disk == all_units() + + +def test_mise_task_files_exist(): + for name, task in mise_units().items(): + assert task["file"] == f"units/{name}.sh" + assert (REPO / task["file"]).is_file() + + +def test_tag_map_only_names_known_units(): + case = re.search(r"tasks_for_tag\(\) \{ case .*?esac; \}", wrapper(), re.S).group(0) + named = set() + for echoed in re.findall(r'echo "?([a-z0-9 -]+?)"? ?;;', case): + named.update(echoed.split()) + assert named and named <= all_units() diff --git a/units/lib.sh b/units/lib.sh new file mode 100644 index 00000000..d90541e0 --- /dev/null +++ b/units/lib.sh @@ -0,0 +1,25 @@ +# shellcheck shell=bash +# Shared helpers for units/*.sh. Source this; do not execute it. +# Units use `set -uo pipefail` and never `set -e`: a failing step is recorded +# and the later independent steps still run. +UNIT_FAILED=0 +step() { local d="$1"; shift; if "$@"; then echo "ok $d"; else echo "FAIL $d (exit $?)"; UNIT_FAILED=1; fi; } +soft() { local d="$1"; shift; if "$@"; then echo "ok $d"; else echo "warn $d (exit $?, ignored)"; fi; } +finish() { [ "$UNIT_FAILED" = 0 ] || exit 1; exit 0; } # failed steps fail the unit; everything else is exit 0 +note() { echo "$*"; } # e.g. note "PARTIAL: vault locked, broker token not written" +install_if_changed() { # src dest mode + if [ -f "$2" ] && cmp -s "$1" "$2"; then chmod "$3" "$2"; return 0; fi + mkdir -p "$(dirname "$2")" && install -m "$3" "$1" "$2" +} +DOTFILES_DIR="${DOTFILES_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +IS_TERMUX=0; [ -d /data/data/com.termux/files/usr ] && IS_TERMUX=1 +IS_MUSL=0; { [ -e /lib/ld-musl-x86_64.so.1 ] || [ -e /lib/ld-musl-aarch64.so.1 ]; } && IS_MUSL=1 +# bw_session: print a usable Bitwarden session or nothing. Never prompts. +bw_session() { + [ "${UNITS_NO_SECRETS:-0}" = 1 ] && return 0 + command -v bw >/dev/null 2>&1 || return 0 + local s="${BW_SESSION:-}" + [ -n "$s" ] || { [ -s /tmp/bw_session ] && s="$(cat /tmp/bw_session)"; } + [ -n "$s" ] && bw unlock --check --session "$s" >/dev/null 2>&1 && printf '%s' "$s" + return 0 +} diff --git a/units/omp.sh b/units/omp.sh new file mode 100755 index 00000000..1d6d7b56 --- /dev/null +++ b/units/omp.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# omp unit: install, skills, Kiro plugin, routing, auth-broker client config. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +PATH="$HOME/.bun/bin:$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:$PATH" +cfg="$HOME/.omp/agent/config.yml" +skills_dest="$HOME/.omp/agent/skills/james" + +if [ "$IS_TERMUX" = 1 ]; then + note "SKIPPED: omp has no Android build" + exit 0 +fi + +# shellcheck source=units/omp/auth-broker.conf +. "$DOTFILES_DIR/units/omp/auth-broker.conf" + +omp_install() { + if ! command -v omp >/dev/null 2>&1 && [ "$IS_MUSL" = 1 ]; then + mkdir -p "$HOME/.cache" || return 1 + curl -fsSL https://omp.sh/install -o "$HOME/.cache/omp-install.sh" || return 1 + PI_INSTALL_DIR="$HOME/.local/bin" sh "$HOME/.cache/omp-install.sh" --binary || return 1 + if [ ! -x "$HOME/.bun/bin/bun" ]; then + curl -fsSL https://bun.sh/install -o "$HOME/.cache/bun-install.sh" || return 1 + bash "$HOME/.cache/bun-install.sh" || return 1 + fi + fi + if ! command -v omp >/dev/null 2>&1; then + echo "omp not installed; run: just packages" >&2 + return 1 + fi +} + +skills_sync() { + local src="$DOTFILES_DIR/roles/pi/files/skills" + mkdir -p "$skills_dest" || return 1 + if command -v rsync >/dev/null 2>&1; then + rsync -a "$src/" "$skills_dest/" || return 1 + else + cp -a "$src/." "$skills_dest/" || return 1 + fi + find "$skills_dest" -type f \( -name '*.sh' -o -name '*.mjs' \) -exec chmod 755 {} + +} + +claude_links() { + local d rc=0 + mkdir -p "$HOME/.claude/skills" || return 1 + for d in "$skills_dest"/*/; do + [ -d "$d" ] || continue + d="${d%/}" + ln -sfn "$d" "$HOME/.claude/skills/$(basename "$d")" || rc=1 + done + return $rc +} + +kiro_plugin() { + if omp plugin list --json 2>/dev/null | grep -q omp-kiro-provider; then + return 0 + fi + omp plugin install github:elkibalion/omp-kiro-provider +} + +ccleft_install() { + [ -x "$HOME/go/bin/ccleft" ] && return 0 + command -v go >/dev/null 2>&1 || return 0 + go install github.com/tuna-os/ccleft/cmd/ccleft@latest +} + +routing_merge() { + mkdir -p "$(dirname "$cfg")" || return 1 + [ -f "$cfg" ] || : > "$cfg" || return 1 + yq -i ". *= (load(\"$DOTFILES_DIR/units/omp/routing.yml\") | explode(.) | ... comments=\"\")" "$cfg" +} + +broker_url() { + mkdir -p "$(dirname "$cfg")" || return 1 + [ -f "$cfg" ] || : > "$cfg" || return 1 + URL="$OMP_AUTH_BROKER_URL" yq -i '.auth.broker.url = strenv(URL)' "$cfg" || return 1 + chmod 600 "$cfg" +} + +broker_token() { + local s tok + s="$(bw_session)" + [ -n "$s" ] || return 0 + tok="$(BW_SESSION="$s" bw get password "$OMP_AUTH_BROKER_BW_ITEM" 2>/dev/null)" + [ -n "$tok" ] || return 0 + TOKEN="$tok" yq -i '.auth.broker.token = strenv(TOKEN)' "$cfg" || return 1 + chmod 600 "$cfg" +} + +broker_status() { + omp auth-broker status +} + +have_omp() { command -v omp >/dev/null 2>&1; } + +step "omp installed" omp_install +step "skills synced" skills_sync +step "claude skill links" claude_links +if have_omp; then step "kiro plugin" kiro_plugin; else note "skip kiro plugin (omp missing)"; fi +soft "ccleft" ccleft_install +step "kiro.json" install_if_changed "$DOTFILES_DIR/units/omp/kiro.json" "$HOME/.config/omp/kiro.json" 0644 +step "omp-route" install_if_changed "$DOTFILES_DIR/units/omp/omp-route" "$HOME/.local/bin/omp-route" 0755 +step "routing merged" routing_merge +step "broker url" broker_url +step "broker token" broker_token +if [ "$(yq '.auth.broker | has("token")' "$cfg" 2>/dev/null)" != true ]; then + note "PARTIAL: broker token not written (vault locked or item missing); run: just seed-omp-broker on punjab, then just apply-tags omp" +fi +if have_omp; then soft "broker status" broker_status; else note "skip broker status (omp missing)"; fi +finish diff --git a/units/omp/auth-broker.conf b/units/omp/auth-broker.conf new file mode 100644 index 00000000..025c8f15 --- /dev/null +++ b/units/omp/auth-broker.conf @@ -0,0 +1,5 @@ +# Sourced by units/omp.sh. punjab runs `omp auth-broker serve` on its tailnet +# address (a hand-made user unit, not managed here); keep in step with +# inventory.yml's punjab ansible_host. +OMP_AUTH_BROKER_URL="${OMP_AUTH_BROKER_URL:-http://100.78.73.8:8765}" +OMP_AUTH_BROKER_BW_ITEM="${OMP_AUTH_BROKER_BW_ITEM:-omp-auth-broker-token}" diff --git a/roles/omp/files/kiro.json b/units/omp/kiro.json similarity index 100% rename from roles/omp/files/kiro.json rename to units/omp/kiro.json diff --git a/roles/omp/files/omp-route b/units/omp/omp-route similarity index 100% rename from roles/omp/files/omp-route rename to units/omp/omp-route diff --git a/roles/omp/files/routing.yml b/units/omp/routing.yml similarity index 95% rename from roles/omp/files/routing.yml rename to units/omp/routing.yml index b1a4e408..4338163a 100644 --- a/roles/omp/files/routing.yml +++ b/units/omp/routing.yml @@ -1,4 +1,4 @@ -# Merged into ~/.omp/agent/config.yml by roles/omp (yq deep-merge; lists replace). +# Merged into ~/.omp/agent/config.yml by units/omp (yq deep-merge; lists replace). # Order everywhere: Claude Max → Codex → Antigravity → Kiro → Muse contributor. # omp-route (aliased to `omp`) picks the starting model from ccleft quota; # these chains handle exhaustion mid-session. From f77b9634548638e97ce2ec0d74c5eb93a61a40b9 Mon Sep 17 00:00:00 2001 From: James Reilly Date: Sun, 25 Oct 2026 20:25:26 +0000 Subject: [PATCH 03/62] refactor: move file roles to chezmoi, secret roles to units --- .chezmoiroot | 1 + CLAUDE.md | 2 + CONTEXT.md | 3 +- Justfile | 2 +- docs/src/SUMMARY.md | 12 +- docs/src/architecture.md | 10 +- docs/src/bitwarden.md | 8 +- docs/src/index.md | 6 +- docs/src/roles/git.md | 18 +-- docs/src/roles/neovim.md | 11 +- docs/src/roles/omp.md | 2 +- docs/src/roles/shell_ai.md | 29 ++-- docs/src/roles/shell_atuin.md | 22 +-- docs/src/roles/shell_dotfiles.md | 37 +++-- docs/src/units.md | 9 ++ group_vars/all.yml | 2 +- home/.chezmoi.toml.tmpl | 4 + .../vars/main.yml => home/.chezmoidata.yaml | 5 +- .../run_after_10-ghostty-terminfo.sh | 6 + .../.chezmoiscripts/run_after_20-nvim-sync.sh | 7 + .../templates/bashrc.j2 => home/dot_bashrc | 0 home/dot_config/atuin/config.toml | 18 +++ .../dot_config/fish/conf.d/aliases.fish.tmpl | 33 ++-- .../dot_config/fish}/config.fish | 0 .../dot_config/gh/config.yml | 0 .../dot_config/ghostty/config | 2 +- .../files => home/dot_config}/nvim/init.lua | 0 .../dot_config}/nvim/lua/config/autocmds.lua | 0 .../dot_config}/nvim/lua/config/keymaps.lua | 0 .../dot_config}/nvim/lua/config/options.lua | 0 .../dot_config}/nvim/lua/plugins/coding.lua | 0 .../nvim/lua/plugins/colorscheme.lua | 0 .../dot_config}/nvim/lua/plugins/editor.lua | 0 .../dot_config}/nvim/lua/plugins/lsp.lua | 0 .../dot_config}/nvim/lua/plugins/ui.lua | 0 .../dot_config/shell/aliases.sh.tmpl | 27 ++-- .../dot_config/shell/ghostty.terminfo | 0 .../files => home/dot_config}/starship.toml | 0 .../gitconfig.j2 => home/dot_gitconfig.tmpl | 6 +- .../dot_gitignore_global | 0 .../files/gitmessage => home/dot_gitmessage | 0 .../files/inputrc => home/dot_inputrc | 0 .../files/tmux.conf => home/dot_tmux.conf | 0 .../zprofile.j2 => home/dot_zprofile.tmpl | 4 +- .../templates/zshrc.j2 => home/dot_zshrc | 0 mise.toml | 12 ++ roles/git/tasks/main.yml | 30 ---- roles/neovim/tasks/main.yml | 66 -------- roles/pi/files/skills/tavily-search/SKILL.md | 2 +- roles/shell_ai/tasks/main.yml | 141 ---------------- roles/shell_atuin/tasks/main.yml | 153 ------------------ roles/shell_dotfiles/tasks/main.yml | 136 ---------------- scripts/omp-login.sh | 4 +- scripts/run-units.sh | 12 +- site.yml | 10 -- tests/test_atuin_unit.py | 73 +++++++++ tests/test_brew_blocked.py | 2 +- tests/test_chezmoi_data.py | 19 +++ tests/test_shell_ai_unit.py | 74 +++++++++ units/atuin.sh | 51 ++++++ units/chezmoi.sh | 24 +++ units/lib.sh | 3 + units/omp.sh | 2 +- units/omp/omp-route | 2 +- units/shell-ai.sh | 69 ++++++++ 65 files changed, 517 insertions(+), 654 deletions(-) create mode 100644 .chezmoiroot create mode 100644 home/.chezmoi.toml.tmpl rename roles/shell_dotfiles/vars/main.yml => home/.chezmoidata.yaml (96%) create mode 100755 home/.chezmoiscripts/run_after_10-ghostty-terminfo.sh create mode 100755 home/.chezmoiscripts/run_after_20-nvim-sync.sh rename roles/shell_dotfiles/templates/bashrc.j2 => home/dot_bashrc (100%) create mode 100644 home/dot_config/atuin/config.toml rename roles/shell_dotfiles/templates/aliases.fish.j2 => home/dot_config/fish/conf.d/aliases.fish.tmpl (60%) rename {roles/shell_dotfiles/files => home/dot_config/fish}/config.fish (100%) rename roles/git/files/gh_config.yml => home/dot_config/gh/config.yml (100%) rename roles/shell_dotfiles/files/ghostty.config => home/dot_config/ghostty/config (76%) rename {roles/neovim/files => home/dot_config}/nvim/init.lua (100%) rename {roles/neovim/files => home/dot_config}/nvim/lua/config/autocmds.lua (100%) rename {roles/neovim/files => home/dot_config}/nvim/lua/config/keymaps.lua (100%) rename {roles/neovim/files => home/dot_config}/nvim/lua/config/options.lua (100%) rename {roles/neovim/files => home/dot_config}/nvim/lua/plugins/coding.lua (100%) rename {roles/neovim/files => home/dot_config}/nvim/lua/plugins/colorscheme.lua (100%) rename {roles/neovim/files => home/dot_config}/nvim/lua/plugins/editor.lua (100%) rename {roles/neovim/files => home/dot_config}/nvim/lua/plugins/lsp.lua (100%) rename {roles/neovim/files => home/dot_config}/nvim/lua/plugins/ui.lua (100%) rename roles/shell_dotfiles/templates/aliases.sh.j2 => home/dot_config/shell/aliases.sh.tmpl (72%) rename roles/shell_dotfiles/files/xterm-ghostty.terminfo => home/dot_config/shell/ghostty.terminfo (100%) rename {roles/shell_dotfiles/files => home/dot_config}/starship.toml (100%) rename roles/git/templates/gitconfig.j2 => home/dot_gitconfig.tmpl (93%) rename roles/git/files/gitignore_global => home/dot_gitignore_global (100%) rename roles/git/files/gitmessage => home/dot_gitmessage (100%) rename roles/shell_dotfiles/files/inputrc => home/dot_inputrc (100%) rename roles/shell_dotfiles/files/tmux.conf => home/dot_tmux.conf (100%) rename roles/shell_dotfiles/templates/zprofile.j2 => home/dot_zprofile.tmpl (84%) rename roles/shell_dotfiles/templates/zshrc.j2 => home/dot_zshrc (100%) delete mode 100644 roles/git/tasks/main.yml delete mode 100644 roles/neovim/tasks/main.yml delete mode 100644 roles/shell_ai/tasks/main.yml delete mode 100644 roles/shell_atuin/tasks/main.yml delete mode 100644 roles/shell_dotfiles/tasks/main.yml create mode 100644 tests/test_atuin_unit.py create mode 100644 tests/test_chezmoi_data.py create mode 100644 tests/test_shell_ai_unit.py create mode 100755 units/atuin.sh create mode 100755 units/chezmoi.sh mode change 100644 => 100755 units/lib.sh create mode 100755 units/shell-ai.sh diff --git a/.chezmoiroot b/.chezmoiroot new file mode 100644 index 00000000..5e72f75c --- /dev/null +++ b/.chezmoiroot @@ -0,0 +1 @@ +home diff --git a/CLAUDE.md b/CLAUDE.md index f8eabcc8..c13831c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,6 +53,8 @@ just add-machine NAME # Onboard + bootstrap an already-reachable machine via S - `kube` fetches `~/.kube/config` + `~/.talos/config` from Bitwarden — desktops only. - `ssh_keys` round-trips per-machine ed25519 keys through Bitwarden, building cross-machine `authorized_keys`. +After the playbook, `just apply*` runs bash **units** (`units/`, via mise through `scripts/run-units.sh`; see [`docs/src/units.md`](docs/src/units.md)): `omp`, `chezmoi`, `atuin`, `shell-ai`. `home/` is the chezmoi source for shell, git, neovim and atuin config; `ssh_keys` remains an Ansible role. + Roles are conditional on inventory group (`desktop`, `server`, `vps`, `test_fleet`, `termux_hosts`; `llm` is dormant/commented out) and on `skip_*` flags / `machine_profile` in `host_vars/`. See [`docs/src/roles/`](docs/src/roles/) for the full reference. ## Kubeconfig + Talos secrets diff --git a/CONTEXT.md b/CONTEXT.md index 872da52b..3ff8f72c 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -20,7 +20,8 @@ ## Playbook -- **Role** — an Ansible role that configures one concern (e.g. `shell_dotfiles`, `tailscale`, `proxy`). The interface is `tasks/main.yml`; the implementation is templates, files, and handlers. +- **Role** — an Ansible role that configures one concern (e.g. `tailscale`, `proxy`, `ssh_keys`). The interface is `tasks/main.yml`; the implementation is templates, files, and handlers. +- **Unit** — an idempotent bash script in `units/` run in parallel by mise after the playbook (`omp`, `chezmoi`, `atuin`, `shell-ai`). A failing step does not stop the rest of the unit. See `docs/src/units.md`. - **Phase** — a logical grouping of roles in `site.yml` (Phase 1: system + packages, Phase 2: secrets + auth, Phase 3: desktop, Phase 4: services). Phases are documented in comments, not enforced by the playbook. - **Tag** — an Ansible tag applied to roles so `just apply-tags ` can target a subset. diff --git a/Justfile b/Justfile index 180f1f67..08c48444 100644 --- a/Justfile +++ b/Justfile @@ -460,7 +460,7 @@ doctor: echo "→ Machine identity" if [ -f /etc/dotfiles-machine ]; then pass "machine: $(cat /etc/dotfiles-machine)"; else warn "/etc/dotfiles-machine missing — run bootstrap"; fi echo "→ Core binaries" - for bin in brew bw gh tailscale ansible-playbook just mise; do + for bin in brew bw gh tailscale ansible-playbook just mise chezmoi; do if command -v "$bin" >/dev/null 2>&1; then pass "$bin"; else warn "$bin missing"; fi done echo "→ Tailscale" diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 9f95290f..a7ea7562 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -61,13 +61,13 @@ ## Dotfiles - [shell_fonts](roles/shell_fonts.md) -- [shell_dotfiles](roles/shell_dotfiles.md) -- [shell_atuin](roles/shell_atuin.md) -- [shell_ai](roles/shell_ai.md) -- [omp](roles/omp.md) +- [shell_dotfiles (chezmoi)](roles/shell_dotfiles.md) +- [shell_atuin (chezmoi + unit)](roles/shell_atuin.md) +- [shell_ai (unit)](roles/shell_ai.md) +- [omp (unit)](roles/omp.md) - [hive_ops](roles/hive_ops.md) -- [git](roles/git.md) -- [neovim](roles/neovim.md) +- [git (chezmoi)](roles/git.md) +- [neovim (chezmoi)](roles/neovim.md) ## Secrets & Auth diff --git a/docs/src/architecture.md b/docs/src/architecture.md index 940ef1ee..a58fd2c5 100644 --- a/docs/src/architecture.md +++ b/docs/src/architecture.md @@ -63,13 +63,7 @@ Every role in `site.yml` carries one or more tags. Tags fall into two kinds: **p | System + packages | `termux_packages` | `system`, `packages`, `termux` | | System + packages | `bitwarden` | `secrets`, `bitwarden` | | System + packages | `shell_fonts` | `dotfiles`, `shell`, `fonts` | -| System + packages | `shell_dotfiles` | `dotfiles`, `shell`, `dotfiles` | -| System + packages | `omp` | `dotfiles`, `omp` | | System + packages | `hive_ops` | `hive`, `hive_ops` | -| System + packages | `git` | `dotfiles`, `git` | -| System + packages | `neovim` | `dotfiles`, `neovim`, `editor` | -| Secrets + auth | `shell_atuin` | `dotfiles`, `shell`, `atuin`, `secrets` | -| Secrets + auth | `shell_ai` | `dotfiles`, `shell`, `ai`, `secrets` | | Secrets + auth | `ssh_keys` | `secrets`, `ssh`, `ssh_keys` | | Secrets + auth | `ssh_mesh` | `ssh`, `mesh_check` | | Secrets + auth | `github` | `secrets`, `github` | @@ -91,9 +85,11 @@ Every role in `site.yml` carries one or more tags. Tags fall into two kinds: **p | Services | `server_hardening` | `system`, `hardening`, `security` | | Services | `fleet_facts` | `facts`, `fleet_facts` | +Shell, git, neovim and atuin config, omp, and the shell/atuin secrets are not roles: they are [units](units.md) (`chezmoi`, `omp`, `atuin`, `shell-ai`) that `just apply*` runs after the playbook. The old tags `dotfiles`, `shell`, `git`, `neovim`, `editor`, `atuin`, `ai`, `secrets`, `omp` still select them through `scripts/run-units.sh`. + Common phase-wide selections: -- `--tags secrets` — every BW-backed role only (`bitwarden`, `shell_atuin`, `shell_ai`, `ssh_keys`, `github`, `tailscale`, `kube`, `forgejo_registry`) +- `--tags secrets` — every BW-backed role only (`bitwarden`, `ssh_keys`, `github`, `tailscale`, `kube`, `forgejo_registry`) - `--tags desktop` — every desktop-app role (`flatpak`, `bluefin_common`, `gnome`, `zen_browser`, `pipewire_audio`) - `--tags services` — every long-running service role (`syncthing`, `systemd`, `bst_dashboard`, `kirocrew`, `proxy`, `tailscale_cert`) - `--skip-tags secrets` — everything except BW-backed roles. This is `just apply-nosecrets`, which the daily `dotfiles-update.timer` and the `dots` alias run; `bootstrap.sh` and `phone-apply.service` pass `--skip-tags secrets` directly diff --git a/docs/src/bitwarden.md b/docs/src/bitwarden.md index 68712e4e..e447cb9c 100644 --- a/docs/src/bitwarden.md +++ b/docs/src/bitwarden.md @@ -44,10 +44,10 @@ Every item a role or script reads (grep `bw get` / `bw list` under `roles/` and | `talosconfig-aws-migration` | Secure Note | `kube` role — AWS cluster Talos config | | `accounts.firefox.com` | Login + TOTP | `browser_fxa` role — Firefox Account credentials | | `github-token` | Login (password) | `github` role — PAT for `gh auth login` | -| `atuin.sh` | Login + custom field `key` | `shell_atuin` role — atuin sync login + mnemonic | -| `deepseek-api-key` | Login (password) | `shell_ai` role — pi `auth.json` | -| `forgejo` | Secure Note (`API Token:` line) | `shell_ai` role — `FORGEJO_TOKEN` | -| `tavily-api-key` | Login (password) | `shell_ai` role — `TAVILY_API_KEY` | +| `atuin.sh` | Login + custom field `key` | `atuin` unit — atuin sync login + mnemonic | +| `deepseek-api-key` | Login (password) | `shell-ai` unit — pi `auth.json` | +| `forgejo` | Secure Note (`API Token:` line) | `shell-ai` unit — `FORGEJO_TOKEN` | +| `tavily-api-key` | Login (password) | `shell-ai` unit — `TAVILY_API_KEY` | | `forgejo.manatee-basking.ts.net` | Login | `forgejo_registry` role — podman registry login | | `bw-api-key` | Login | `scripts/bw-resolve.sh` — `bw login --apikey` on a remote | | `James Bitwarden` | Login | `scripts/bw-resolve.sh` — non-interactive unlock on a remote | diff --git a/docs/src/index.md b/docs/src/index.md index 20969ce2..174e2d12 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -65,11 +65,13 @@ The `llm` group is dormant (commented out in `inventory.yml`). | Phase | Roles | Main tags | |-------|-------|-----------| -| 1 — System + packages + dotfiles | sudo, sshd, apk_packages, homebrew, termux_packages, bitwarden, shell_fonts, shell_dotfiles, omp, hive_ops, git, neovim | `system`, `packages`, `dotfiles` | -| 2 — Secrets + auth | shell_atuin, shell_ai, ssh_keys, ssh_mesh, github, tailscale, kube, forgejo_registry | `secrets` | +| 1 — System + packages + dotfiles | sudo, sshd, apk_packages, homebrew, termux_packages, bitwarden, shell_fonts, hive_ops | `system`, `packages`, `dotfiles` | +| 2 — Secrets + auth | ssh_keys, ssh_mesh, github, tailscale, kube, forgejo_registry | `secrets` | | 3 — Desktop apps | flatpak, bluefin_common, gnome, zen_browser (+ browser_fxa), pipewire_audio | `desktop` | | 4 — Services | syncthing, systemd, bst_dashboard, kirocrew, proxy, tailscale_cert, server_hardening | `services` | +After the playbook, `just apply*` runs the [bash units](units.md) (`chezmoi`, `omp`, `atuin`, `shell-ai`) in parallel through mise. + Only the `secrets`-tagged work needs Bitwarden. With the vault locked (or `--skip-tags secrets`), phases 1, 3 and 4 run normally and BW-touching tasks skip cleanly. diff --git a/docs/src/roles/git.md b/docs/src/roles/git.md index e49dd7ab..6b070e0f 100644 --- a/docs/src/roles/git.md +++ b/docs/src/roles/git.md @@ -1,20 +1,20 @@ # git +**Unit:** `chezmoi` (`units/chezmoi.sh`; see [Units](../units.md)) **Tags:** `dotfiles`, `git` **Secrets needed:** No -**Runs on:** All machines +**Runs on:** All machines with `chezmoi` installed -Deploys Git configuration with SSH commit signing. +Git configuration with SSH commit signing, as chezmoi-managed files under `home/`: -## What It Does - -1. Deploys `~/.gitconfig` with name, email, and signing preferences -2. Configures SSH commit signing using `~/.ssh/id_ed25519` -3. Deploys `~/.ssh/allowed_signers` for signature verification -4. Deploys GitHub CLI config at `~/.config/gh/config.yml` +| Target | Source | +|---|---| +| `~/.gitconfig` | `dot_gitconfig.tmpl` (data: `user_name`, `user_email`, `.chezmoi.homeDir`) | +| `~/.gitignore_global`, `~/.gitmessage` | `dot_gitignore_global`, `dot_gitmessage` | +| `~/.config/gh/config.yml` | `dot_config/gh/config.yml` | ## Key Settings - **Signing:** SSH (`ssh-ed25519` key) for all commits -- **Allowed signers:** All fleet machine keys from Bitwarden +- **Allowed signers:** `~/.ssh/allowed_signers` is written by the `ssh_keys` role from the fleet machine keys in Bitwarden - **Delta:** Git diff viewer (`git-delta`) for side-by-side diffs diff --git a/docs/src/roles/neovim.md b/docs/src/roles/neovim.md index 6c732d02..556eec1d 100644 --- a/docs/src/roles/neovim.md +++ b/docs/src/roles/neovim.md @@ -1,16 +1,11 @@ # neovim +**Unit:** `chezmoi` (`units/chezmoi.sh`; see [Units](../units.md)) **Tags:** `dotfiles`, `neovim`, `editor` **Secrets needed:** No -**Runs on:** All machines +**Runs on:** All machines with `chezmoi` installed -Deploys [Neovim](https://neovim.io/) configuration. - -## What It Does - -1. Creates `~/.config/nvim/` directory structure -2. Deploys init.lua and plugin configuration -3. Installs Lazy.nvim plugin manager and configured plugins +[Neovim](https://neovim.io/) configuration, deployed from `home/dot_config/nvim/` (`init.lua`, `lua/config/`, `lua/plugins/`) by chezmoi. After each apply, `.chezmoiscripts/run_after_20-nvim-sync.sh` bootstraps lazy.nvim and runs `nvim --headless "+Lazy! sync" +qa` (best effort, 300 s timeout, never fails the apply). ## Key Plugins diff --git a/docs/src/roles/omp.md b/docs/src/roles/omp.md index 37c3ecac..86570bd7 100644 --- a/docs/src/roles/omp.md +++ b/docs/src/roles/omp.md @@ -36,7 +36,7 @@ The token is written only when Bitwarden is unlocked; a locked vault or `--no-se To seed the token, run `just seed-omp-broker` **on punjab** with the vault unlocked. It reads `omp auth-broker token` (which does not rotate it) and creates or updates the Bitwarden item. Then run `just apply-tags omp` on each client. Verify with `omp auth-broker status`. If the token is rotated (`omp auth-broker token --regenerate` on punjab), seed again and re-apply on the clients. -`DEEPSEEK_API_KEY` and the other provider keys are exposed to the shell by the `shell_ai` role's secret files, not by this unit. +`DEEPSEEK_API_KEY` and the other provider keys are exposed to the shell by the [`shell-ai` unit](shell_ai.md)'s secret files, not by this unit. ## Kiro authentication diff --git a/docs/src/roles/shell_ai.md b/docs/src/roles/shell_ai.md index d771cc3a..08bd2988 100644 --- a/docs/src/roles/shell_ai.md +++ b/docs/src/roles/shell_ai.md @@ -1,22 +1,31 @@ # shell_ai +**Unit:** `shell-ai` (`units/shell-ai.sh`; see [Units](../units.md)) **Tags:** `dotfiles`, `shell`, `ai`, `secrets` -**Secrets needed:** Yes (`deepseek-api-key`, `forgejo`, `tavily-api-key`) +**Secrets needed:** Yes (`deepseek-api-key`, `forgejo`, `tavily-api-key`, `meta-api-key`, `kiro-api-key`) **Runs on:** All machines -Deploys API credentials for the AI coding tools and shell. +Writes API credentials for the AI coding tools and shell. ## What It Does -1. When `bw_unlocked`: - - `bw get password deepseek-api-key` → `DEEPSEEK_API_KEY` in shell secret files - - `bw get notes forgejo` (the `API Token:` line) → `FORGEJO_TOKEN` - - `bw get password tavily-api-key` → `TAVILY_API_KEY` - - Writes these exports to `~/.config/shell/secrets.sh` (bash/zsh) and - `~/.config/fish/conf.d/secrets.fish` (fish), both 0600 -2. Warns if `deepseek-api-key` is missing; Kiro OAuth remains a per-user OMP sign-in +With an unlocked vault (`bw_session`; never prompts): + +- `bw get password deepseek-api-key` → `DEEPSEEK_API_KEY` +- `bw get notes forgejo` (the `API Token:` line) → `FORGEJO_TOKEN` +- `bw get password tavily-api-key` → `TAVILY_API_KEY` +- `bw get notes meta-api-key` → `META_API_KEY` +- `bw get notes kiro-api-key` → `KIRO_API_KEY` + +The values present are written (single-quoted, 0600, via temp file + `mv`) to `~/.config/shell/secrets.sh` (bash/zsh; also exports `OMP_KIRO_CONFIG`) and `~/.config/fish/conf.d/secrets.fish` (fish; ends with a runtime Kiro key refresh from `/tmp/bw_session`). + +## Output lines + +- `SKIPPED: vault locked or --no-secrets` — files left untouched +- `warn: no secrets found in Bitwarden` — files left untouched ## Notes +- Not a chezmoi template on purpose: `bitwarden` template functions abort the whole template when one item is missing, while the unit writes whatever secrets exist. - A locked vault leaves any previously written secret files untouched — nothing is overwritten with empty values -- `TAVILY_API_KEY` is read by the `tavily-search` skill deployed by the [`omp`](omp.md) role +- `TAVILY_API_KEY` is read by the `tavily-search` skill deployed by the [`omp`](omp.md) unit diff --git a/docs/src/roles/shell_atuin.md b/docs/src/roles/shell_atuin.md index a6fef4d9..68bf168b 100644 --- a/docs/src/roles/shell_atuin.md +++ b/docs/src/roles/shell_atuin.md @@ -1,5 +1,6 @@ # shell_atuin +**Units:** `chezmoi` (config file), `atuin` (`units/atuin.sh`; see [Units](../units.md)) **Tags:** `dotfiles`, `shell`, `atuin`, `secrets` **Secrets needed:** Yes, for login (`atuin.sh` from Bitwarden) **Runs on:** All machines @@ -8,18 +9,19 @@ Configures [Atuin](https://atuin.sh/) shell history and logs it in to sync. ## What It Does -1. Writes `~/.config/atuin/config.toml` — sync to `api.atuin.sh` every 5m, - fuzzy search, `filter_mode = "global"` (including the Up-arrow binding) -2. Checks `atuin status`; if already logged in, skips the Bitwarden round-trip -3. Otherwise, when `bw_unlocked`, reads the `atuin.sh` item (username, - password, and a custom field `key` holding the mnemonic) and runs - `atuin login -u … -p … -k …` -4. Runs a best-effort `atuin sync` -5. Imports existing shell history once (`atuin import auto`, sentinel - `~/.local/share/atuin/.history_imported`) +1. chezmoi writes `~/.config/atuin/config.toml` from `home/dot_config/atuin/config.toml` — sync to `api.atuin.sh` every 5m, fuzzy search, `filter_mode = "global"` (including the Up-arrow binding) +2. The `atuin` unit checks `atuin status`; if it shows `[Sync]` the Bitwarden round-trip is skipped +3. Otherwise, with an unlocked vault, it reads the `atuin.sh` item (username, password, custom field `key` holding the mnemonic) and runs `atuin login -u … -p … -k …` (resetting `~/.local/share/atuin` first when no key file exists), then a best-effort `atuin sync` +4. Imports existing shell history once (`atuin import auto`, sentinel `~/.local/share/atuin/.history_imported`) + +## Output lines + +- `SKIPPED: atuin not installed` — install it (`core_brews`) +- `PARTIAL: atuin not logged in; vault locked` — `bw unlock`, then `just apply-tags atuin` +- `PARTIAL: Bitwarden item atuin.sh missing or incomplete` — fix the item (needs username, password, field `key`) ## Notes -- Skips login with a warning if the `atuin` binary is missing (it's in `core_brews`) - The mnemonic is passed to `atuin login -k`, never written to `~/.local/share/atuin/key` directly — atuin expects that file to be a binary key +- `atuin login` receives the password on its command line, visible to `ps` for the duration of the call - Don't set the Up-arrow filter to `session`: it looks broken in fresh terminals diff --git a/docs/src/roles/shell_dotfiles.md b/docs/src/roles/shell_dotfiles.md index 12c9eb6f..9261f77b 100644 --- a/docs/src/roles/shell_dotfiles.md +++ b/docs/src/roles/shell_dotfiles.md @@ -1,20 +1,31 @@ # shell_dotfiles -**Tags:** `dotfiles`, `shell` +**Unit:** `chezmoi` (`units/chezmoi.sh`; see [Units](../units.md)) +**Tags:** `dotfiles`, `shell` (`just apply-tags shell` runs `unit:chezmoi`) **Secrets needed:** No -**Runs on:** All machines +**Runs on:** All machines with `chezmoi` installed (`core_brews`) -Deploys the shell, prompt, multiplexer and terminal configs. +No longer an Ansible role: the shell, prompt, multiplexer and terminal configs are chezmoi-managed files under `home/` (`.chezmoiroot` points chezmoi there). -## What It Does +## Files (source in `home/`) -1. `~/.config/shell/aliases.sh` (bash/zsh) and `~/.config/fish/conf.d/aliases.fish` — shared aliases, templated -2. `~/.bashrc`, `~/.zshrc`, `~/.zprofile` — templated -3. `~/.config/fish/config.fish`, `~/.tmux.conf`, `~/.inputrc`, `~/.config/starship.toml` — copied as-is -4. Ghostty terminfo: ships `xterm-ghostty.terminfo` and compiles it into - `~/.terminfo` with `tic`, so SSH sessions from Ghostty don't break - ncurses apps. Warns and skips if `tic` isn't installed. -5. `~/.config/ghostty/config` — auto light/dark following GNOME (inert on headless hosts) +| Target | Source | +|---|---| +| `~/.bashrc`, `~/.zshrc`, `~/.inputrc`, `~/.tmux.conf` | `dot_bashrc`, `dot_zshrc`, `dot_inputrc`, `dot_tmux.conf` | +| `~/.zprofile` | `dot_zprofile.tmpl` (flatpak path on desktops) | +| `~/.config/shell/aliases.sh` | `dot_config/shell/aliases.sh.tmpl` | +| `~/.config/fish/conf.d/aliases.fish` | `dot_config/fish/conf.d/aliases.fish.tmpl` | +| `~/.config/fish/config.fish`, `~/.config/starship.toml` | `dot_config/fish/config.fish`, `dot_config/starship.toml` | +| `~/.config/ghostty/config` | `dot_config/ghostty/config` (inert on headless hosts) | +| `~/.config/shell/ghostty.terminfo` | `dot_config/shell/ghostty.terminfo`, compiled into `~/.terminfo` by `.chezmoiscripts/run_after_10-ghostty-terminfo.sh` (skips if `tic` is missing) so SSH from Ghostty keeps ncurses apps working | + +## Template data + +`home/.chezmoi.toml.tmpl` (regenerated by `apply --init`) provides `machine` and `is_desktop` (from `DOTFILES_MACHINE` / `DOTFILES_IS_DESKTOP`, exported by the unit; `is_desktop` is true when the host is in the `desktop` group of `inventory.yml`). `home/.chezmoidata.yaml` provides `user_name`, `user_email` and `alias_groups`, the single source for the bash/zsh and fish aliases. Add a plain alias there, not in the templates. + +## Adding a file + +Put it under `home/` using chezmoi names: `dot_` for a leading dot, `.tmpl` suffix to template it, `private_`/`executable_` for modes. chezmoi never deletes files it does not manage. ## Fleet aliases @@ -28,5 +39,5 @@ In fish these are abbreviations (`abbr`), not aliases. ## Notes -- Shell secrets (`~/.config/shell/secrets.sh`, `~/.config/fish/conf.d/secrets.fish`) are written by [`shell_ai`](shell_ai.md), not here -- Source files: `roles/shell_dotfiles/files/` and `roles/shell_dotfiles/templates/` +- Shell secrets (`~/.config/shell/secrets.sh`, `~/.config/fish/conf.d/secrets.fish`) are written by the [`shell-ai` unit](shell_ai.md), not here. +- One failing file does not stop the rest: the unit runs `chezmoi apply --init --keep-going --force --no-tty`. diff --git a/docs/src/units.md b/docs/src/units.md index 42f5c721..b4e907ac 100644 --- a/docs/src/units.md +++ b/docs/src/units.md @@ -35,3 +35,12 @@ Everything not listed under `units/` is still an Ansible role. `tests/test_units_registry.py` fails if the three places disagree. `units/`, `mise.toml` and `scripts/run-units.sh` are executed by the fleet timer, so they are intentionally not carved out of CODEOWNERS. + +## chezmoi + +File-based config (shell, git, neovim, atuin config) is managed by [chezmoi](https://www.chezmoi.io/) from `home/` (`.chezmoiroot` contains `home`, so `.chezmoi.toml.tmpl`, `.chezmoidata.yaml` and `.chezmoiscripts/` live there). `unit:chezmoi` runs `chezmoi --source --keep-going --force --no-tty apply --init`: `--keep-going` continues past a failing file or script, `--force` overwrites like the old roles did, `--no-tty` keeps the timer from hanging on a prompt, `--init` regenerates the config from `home/.chezmoi.toml.tmpl`. + +- Data: `machine`, `is_desktop` (true when the host is in the `desktop` group of `inventory.yml`), `user_name`, `user_email`, `alias_groups`. +- Secrets are **not** chezmoi templates: the `bitwarden` template function aborts a whole template on a missing item. The `shell-ai` and `atuin` units handle secrets. +- chezmoi never deletes files it does not manage (no `exact_` directories). +- `ssh_keys` stays an Ansible role (key lifecycle, `authorized_keys` merge). diff --git a/group_vars/all.yml b/group_vars/all.yml index bf9f9714..8af51101 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -82,6 +82,7 @@ core_brews: - direnv - just - mise + - chezmoi - make - shellcheck - shfmt @@ -126,7 +127,6 @@ desktop_brews: - ansible - block-goose-cli - charmbracelet/tap/crush - - chezmoi - cmake - cosign - curl diff --git a/home/.chezmoi.toml.tmpl b/home/.chezmoi.toml.tmpl new file mode 100644 index 00000000..02c862b3 --- /dev/null +++ b/home/.chezmoi.toml.tmpl @@ -0,0 +1,4 @@ +{{- $machine := env "DOTFILES_MACHINE" | default .chezmoi.hostname -}} +[data] + machine = {{ $machine | quote }} + is_desktop = {{ eq (env "DOTFILES_IS_DESKTOP") "1" }} diff --git a/roles/shell_dotfiles/vars/main.yml b/home/.chezmoidata.yaml similarity index 96% rename from roles/shell_dotfiles/vars/main.yml rename to home/.chezmoidata.yaml index 9d89d4f5..c4552e53 100644 --- a/roles/shell_dotfiles/vars/main.yml +++ b/home/.chezmoidata.yaml @@ -1,12 +1,15 @@ --- # Single source of truth for aliases shared across bash/zsh and fish. -# Rendered into aliases.sh.j2 (POSIX `alias`) and aliases.fish.j2 (fish +# Rendered into aliases.sh.tmpl (POSIX `alias`) and aliases.fish.tmpl (fish # `abbr`, or `alias` where fish_kind: alias is set — e.g. vim→nvim, where # an abbreviation's visual text-expansion would be surprising). # # Entries requiring shell-specific syntax (&&/`; and`, ${VAR:-default}, # heredocs, etc.) are NOT here — they're hand-written once per shell # template instead, clearly marked, to avoid subtle translation bugs. +user_name: James Reilly +user_email: jreilly1821@gmail.com + alias_groups: - guard: nvim fish_kind: alias diff --git a/home/.chezmoiscripts/run_after_10-ghostty-terminfo.sh b/home/.chezmoiscripts/run_after_10-ghostty-terminfo.sh new file mode 100755 index 00000000..a36fdba3 --- /dev/null +++ b/home/.chezmoiscripts/run_after_10-ghostty-terminfo.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Compile the xterm-ghostty terminfo so SSH sessions from Ghostty keep working ncurses apps. +src="$HOME/.config/shell/ghostty.terminfo"; out="$HOME/.terminfo/x/xterm-ghostty" +command -v tic >/dev/null 2>&1 || { echo "tic not installed; skipping xterm-ghostty terminfo"; exit 0; } +[ -f "$out" ] && [ "$out" -nt "$src" ] && exit 0 +tic -x -o "$HOME/.terminfo" "$src" diff --git a/home/.chezmoiscripts/run_after_20-nvim-sync.sh b/home/.chezmoiscripts/run_after_20-nvim-sync.sh new file mode 100755 index 00000000..1c43046a --- /dev/null +++ b/home/.chezmoiscripts/run_after_20-nvim-sync.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Best effort: bootstrap lazy.nvim and sync plugins; never fails the apply. +command -v nvim >/dev/null 2>&1 || exit 0 +lazy="$HOME/.local/share/nvim/lazy/lazy.nvim" +[ -d "$lazy" ] || git clone --filter=blob:none https://github.com/folke/lazy.nvim.git --branch=stable "$lazy" || true +timeout 300 nvim --headless "+Lazy! sync" +qa 2>&1 | tail -5 +exit 0 diff --git a/roles/shell_dotfiles/templates/bashrc.j2 b/home/dot_bashrc similarity index 100% rename from roles/shell_dotfiles/templates/bashrc.j2 rename to home/dot_bashrc diff --git a/home/dot_config/atuin/config.toml b/home/dot_config/atuin/config.toml new file mode 100644 index 00000000..62e781a3 --- /dev/null +++ b/home/dot_config/atuin/config.toml @@ -0,0 +1,18 @@ +sync_address = "https://api.atuin.sh" +sync_frequency = "5m" +auto_sync = true +search_mode = "fuzzy" +# ── filter_mode ────────────────────────────────────────────── +# "global" = search all history across every machine & session. +# "host" = only this machine. +# "session" = only this shell session (empty in fresh terminals!). +# ────────────────────────────────────────────────────────────── +# DO NOT set filter_mode_shell_up_key_binding to "session" — +# it makes the Up arrow appear broken in fresh terminals because +# a new shell session has no commands yet. +# ────────────────────────────────────────────────────────────── +filter_mode = "global" +filter_mode_shell_up_key_binding = "global" +style = "compact" +show_preview = true +max_preview_height = 4 diff --git a/roles/shell_dotfiles/templates/aliases.fish.j2 b/home/dot_config/fish/conf.d/aliases.fish.tmpl similarity index 60% rename from roles/shell_dotfiles/templates/aliases.fish.j2 rename to home/dot_config/fish/conf.d/aliases.fish.tmpl index 4bd92743..57a281c2 100644 --- a/roles/shell_dotfiles/templates/aliases.fish.j2 +++ b/home/dot_config/fish/conf.d/aliases.fish.tmpl @@ -1,27 +1,28 @@ -# Fish abbreviations + aliases — managed by Ansible +# Fish abbreviations + aliases — managed by chezmoi # Abbreviations expand visually on space so you see (and learn) the real command. # NOTE: bluefin-cli init handles: eza/ll/ls, bat/cat, ugrep/grep, # zoxide/cd, starship, atuin, fzf — do NOT duplicate those here # -# Most of this file is generated from roles/shell_dotfiles/vars/main.yml, +# Most of this file is generated from home/.chezmoidata.yaml, # the single source shared with the bash/zsh aliases template. To add/change # a plain alias, edit that file, not this one — it'll show up in both # shells. Only shell-specific-syntax aliases are hand-written below. -{% for group in alias_groups %} -{% if group.guard is defined %} -if command -q {{ group.guard }} -{% endif %} -{% for a in group.aliases %} -{% if group.fish_kind is defined and group.fish_kind == 'alias' %} -{{ ' ' if group.guard is defined else '' }}alias {{ a.name }} {{ a.cmd }} -{% else %} -{{ ' ' if group.guard is defined else '' }}abbr --add {{ a.name }} '{{ a.cmd }}' -{% endif %} -{% endfor %} -{% if group.guard is defined %} +{{- range $group := .alias_groups }} +{{- $guarded := hasKey $group "guard" }} +{{- if $guarded }} +if command -q {{ $group.guard }} +{{- end }} +{{- range $a := $group.aliases }} +{{- if and (hasKey $group "fish_kind") (eq $group.fish_kind "alias") }} +{{ if $guarded }} {{ end }}alias {{ $a.name }} {{ $a.cmd }} +{{- else }} +{{ if $guarded }} {{ end }}abbr --add {{ $a.name }} '{{ $a.cmd }}' +{{- end }} +{{- end }} +{{- if $guarded }} end -{% endif %} -{% endfor %} +{{- end }} +{{- end }} # ── Shell-specific aliases (syntax differs from bash/zsh; not in the shared source) ── abbr --add dots 'cd ~/.local/share/dotfiles; and just apply-nosecrets' abbr --add dots-apply 'cd ~/.local/share/dotfiles; and git pull --ff-only; and just apply' diff --git a/roles/shell_dotfiles/files/config.fish b/home/dot_config/fish/config.fish similarity index 100% rename from roles/shell_dotfiles/files/config.fish rename to home/dot_config/fish/config.fish diff --git a/roles/git/files/gh_config.yml b/home/dot_config/gh/config.yml similarity index 100% rename from roles/git/files/gh_config.yml rename to home/dot_config/gh/config.yml diff --git a/roles/shell_dotfiles/files/ghostty.config b/home/dot_config/ghostty/config similarity index 76% rename from roles/shell_dotfiles/files/ghostty.config rename to home/dot_config/ghostty/config index d51bef8d..e8ca9c01 100644 --- a/roles/shell_dotfiles/files/ghostty.config +++ b/home/dot_config/ghostty/config @@ -1,4 +1,4 @@ -# Managed by Ansible (roles/shell_dotfiles). Edits here are overwritten on apply. +# Managed by chezmoi (home/dot_config/ghostty/config). Edits here are overwritten on apply. # # Follow the GNOME light/dark preference automatically. Ghostty watches the # desktop color-scheme and swaps between the two themes live — no restart. diff --git a/roles/neovim/files/nvim/init.lua b/home/dot_config/nvim/init.lua similarity index 100% rename from roles/neovim/files/nvim/init.lua rename to home/dot_config/nvim/init.lua diff --git a/roles/neovim/files/nvim/lua/config/autocmds.lua b/home/dot_config/nvim/lua/config/autocmds.lua similarity index 100% rename from roles/neovim/files/nvim/lua/config/autocmds.lua rename to home/dot_config/nvim/lua/config/autocmds.lua diff --git a/roles/neovim/files/nvim/lua/config/keymaps.lua b/home/dot_config/nvim/lua/config/keymaps.lua similarity index 100% rename from roles/neovim/files/nvim/lua/config/keymaps.lua rename to home/dot_config/nvim/lua/config/keymaps.lua diff --git a/roles/neovim/files/nvim/lua/config/options.lua b/home/dot_config/nvim/lua/config/options.lua similarity index 100% rename from roles/neovim/files/nvim/lua/config/options.lua rename to home/dot_config/nvim/lua/config/options.lua diff --git a/roles/neovim/files/nvim/lua/plugins/coding.lua b/home/dot_config/nvim/lua/plugins/coding.lua similarity index 100% rename from roles/neovim/files/nvim/lua/plugins/coding.lua rename to home/dot_config/nvim/lua/plugins/coding.lua diff --git a/roles/neovim/files/nvim/lua/plugins/colorscheme.lua b/home/dot_config/nvim/lua/plugins/colorscheme.lua similarity index 100% rename from roles/neovim/files/nvim/lua/plugins/colorscheme.lua rename to home/dot_config/nvim/lua/plugins/colorscheme.lua diff --git a/roles/neovim/files/nvim/lua/plugins/editor.lua b/home/dot_config/nvim/lua/plugins/editor.lua similarity index 100% rename from roles/neovim/files/nvim/lua/plugins/editor.lua rename to home/dot_config/nvim/lua/plugins/editor.lua diff --git a/roles/neovim/files/nvim/lua/plugins/lsp.lua b/home/dot_config/nvim/lua/plugins/lsp.lua similarity index 100% rename from roles/neovim/files/nvim/lua/plugins/lsp.lua rename to home/dot_config/nvim/lua/plugins/lsp.lua diff --git a/roles/neovim/files/nvim/lua/plugins/ui.lua b/home/dot_config/nvim/lua/plugins/ui.lua similarity index 100% rename from roles/neovim/files/nvim/lua/plugins/ui.lua rename to home/dot_config/nvim/lua/plugins/ui.lua diff --git a/roles/shell_dotfiles/templates/aliases.sh.j2 b/home/dot_config/shell/aliases.sh.tmpl similarity index 72% rename from roles/shell_dotfiles/templates/aliases.sh.j2 rename to home/dot_config/shell/aliases.sh.tmpl index 03cf2028..34e2b919 100644 --- a/roles/shell_dotfiles/templates/aliases.sh.j2 +++ b/home/dot_config/shell/aliases.sh.tmpl @@ -2,21 +2,22 @@ # NOTE: bluefin-cli init handles: eza/ll/ls, bat/cat, ugrep/grep, # zoxide/cd, starship, atuin, fzf — do NOT duplicate those here # -# Most of this file is generated from roles/shell_dotfiles/vars/main.yml, +# Most of this file is generated from home/.chezmoidata.yaml, # the single source shared with the fish aliases template. To add/change # a plain alias, edit that file, not this one — it'll show up in both # shells. Only shell-specific-syntax aliases are hand-written below. -{% for group in alias_groups %} -{% if group.guard is defined %} -if command -v {{ group.guard }} &>/dev/null; then -{% endif %} -{% for a in group.aliases %} -{{ ' ' if group.guard is defined else '' }}alias {{ a.name }}='{{ a.cmd }}' -{% endfor %} -{% if group.guard is defined %} +{{- range $group := .alias_groups }} +{{- $guarded := hasKey $group "guard" }} +{{- if $guarded }} +if command -v {{ $group.guard }} &>/dev/null; then +{{- end }} +{{- range $a := $group.aliases }} +{{ if $guarded }} {{ end }}alias {{ $a.name }}='{{ $a.cmd }}' +{{- end }} +{{- if $guarded }} fi -{% endif %} -{% endfor %} +{{- end }} +{{- end }} # ── Shell-specific aliases (syntax differs from fish; not in the shared source) ── alias -- -='cd -' alias dots='cd ~/.local/share/dotfiles && just apply-nosecrets' @@ -26,9 +27,9 @@ alias reload='exec $SHELL -l' alias path='tr ":" "\n" <<< "$PATH"' alias myip='curl -s ifconfig.me && echo' alias please='sudo $(fc -ln -1)' -{% if is_desktop | bool %} +{{- if .is_desktop }} alias open='xdg-open' -{% endif %} +{{- end }} # ── Local secrets (not committed to git) ────────────────────────── [ -f ~/.config/shell/secrets.sh ] && source ~/.config/shell/secrets.sh diff --git a/roles/shell_dotfiles/files/xterm-ghostty.terminfo b/home/dot_config/shell/ghostty.terminfo similarity index 100% rename from roles/shell_dotfiles/files/xterm-ghostty.terminfo rename to home/dot_config/shell/ghostty.terminfo diff --git a/roles/shell_dotfiles/files/starship.toml b/home/dot_config/starship.toml similarity index 100% rename from roles/shell_dotfiles/files/starship.toml rename to home/dot_config/starship.toml diff --git a/roles/git/templates/gitconfig.j2 b/home/dot_gitconfig.tmpl similarity index 93% rename from roles/git/templates/gitconfig.j2 rename to home/dot_gitconfig.tmpl index 9ae3ce64..a28480d7 100644 --- a/roles/git/templates/gitconfig.j2 +++ b/home/dot_gitconfig.tmpl @@ -1,6 +1,6 @@ [user] - name = {{ user_name }} - email = {{ user_email }} + name = {{ .user_name }} + email = {{ .user_email }} signingkey = ~/.ssh/id_ed25519.pub [commit] @@ -11,7 +11,7 @@ format = ssh [gpg "ssh"] - allowedSignersFile = {{ ansible_facts['env']['HOME'] }}/.ssh/allowed_signers + allowedSignersFile = {{ .chezmoi.homeDir }}/.ssh/allowed_signers [init] defaultBranch = main diff --git a/roles/git/files/gitignore_global b/home/dot_gitignore_global similarity index 100% rename from roles/git/files/gitignore_global rename to home/dot_gitignore_global diff --git a/roles/git/files/gitmessage b/home/dot_gitmessage similarity index 100% rename from roles/git/files/gitmessage rename to home/dot_gitmessage diff --git a/roles/shell_dotfiles/files/inputrc b/home/dot_inputrc similarity index 100% rename from roles/shell_dotfiles/files/inputrc rename to home/dot_inputrc diff --git a/roles/shell_dotfiles/files/tmux.conf b/home/dot_tmux.conf similarity index 100% rename from roles/shell_dotfiles/files/tmux.conf rename to home/dot_tmux.conf diff --git a/roles/shell_dotfiles/templates/zprofile.j2 b/home/dot_zprofile.tmpl similarity index 84% rename from roles/shell_dotfiles/templates/zprofile.j2 rename to home/dot_zprofile.tmpl index a569c479..04395a0e 100644 --- a/roles/shell_dotfiles/templates/zprofile.j2 +++ b/home/dot_zprofile.tmpl @@ -1,5 +1,5 @@ [ -f /home/linuxbrew/.linuxbrew/bin/brew ] && eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" -{% if is_desktop | bool %} +{{ if .is_desktop -}} export PATH="$HOME/.local/share/flatpak/exports/bin:$PATH" -{% endif %} +{{ end -}} [ -f "$HOME/.cargo/env" ] && . "$HOME/.cargo/env" diff --git a/roles/shell_dotfiles/templates/zshrc.j2 b/home/dot_zshrc similarity index 100% rename from roles/shell_dotfiles/templates/zshrc.j2 rename to home/dot_zshrc diff --git a/mise.toml b/mise.toml index cb4032f2..267f10f8 100644 --- a/mise.toml +++ b/mise.toml @@ -5,3 +5,15 @@ depends = ["unit:*"] [tasks."unit:omp"] description = "omp: skills, Kiro plugin, routing, and the auth-broker client" file = "units/omp.sh" + +[tasks."unit:chezmoi"] +description = "chezmoi: shell, git, neovim and atuin config files from home/" +file = "units/chezmoi.sh" + +[tasks."unit:atuin"] +description = "atuin: sync login from Bitwarden and history import" +file = "units/atuin.sh" + +[tasks."unit:shell-ai"] +description = "shell-ai: API secrets from Bitwarden into the shell secret files" +file = "units/shell-ai.sh" diff --git a/roles/git/tasks/main.yml b/roles/git/tasks/main.yml deleted file mode 100644 index 993b7fb3..00000000 --- a/roles/git/tasks/main.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -- name: Deploy ~/.gitconfig - ansible.builtin.template: - src: gitconfig.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.gitconfig" - mode: "0644" - -- name: Deploy ~/.gitignore_global - ansible.builtin.copy: - src: gitignore_global - dest: "{{ ansible_facts['env']['HOME'] }}/.gitignore_global" - mode: "0644" - -- name: Deploy ~/.gitmessage commit template - ansible.builtin.copy: - src: gitmessage - dest: "{{ ansible_facts['env']['HOME'] }}/.gitmessage" - mode: "0644" - -- name: Create ~/.config/gh/ directory - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/gh" - state: directory - mode: "0755" - -- name: Deploy ~/.config/gh/config.yml - ansible.builtin.copy: - src: gh_config.yml - dest: "{{ ansible_facts['env']['HOME'] }}/.config/gh/config.yml" - mode: "0644" diff --git a/roles/neovim/tasks/main.yml b/roles/neovim/tasks/main.yml deleted file mode 100644 index d4143602..00000000 --- a/roles/neovim/tasks/main.yml +++ /dev/null @@ -1,66 +0,0 @@ ---- -- name: Ensure nvim config directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/nvim" - state: directory - mode: "0755" - -- name: Ensure nvim lua config directories exist - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/nvim/{{ item }}" - state: directory - mode: "0755" - loop: - - lua/config - - lua/plugins - -- name: Deploy nvim init.lua - ansible.builtin.copy: - src: nvim/init.lua - dest: "{{ ansible_facts['env']['HOME'] }}/.config/nvim/init.lua" - mode: "0644" - -- name: Deploy nvim config files - ansible.builtin.copy: - src: "nvim/lua/config/{{ item }}" - dest: "{{ ansible_facts['env']['HOME'] }}/.config/nvim/lua/config/{{ item }}" - mode: "0644" - loop: - - options.lua - - keymaps.lua - - autocmds.lua - -- name: Deploy nvim plugin specs - ansible.builtin.copy: - src: "nvim/lua/plugins/{{ item }}" - dest: "{{ ansible_facts['env']['HOME'] }}/.config/nvim/lua/plugins/{{ item }}" - mode: "0644" - loop: - - colorscheme.lua - - editor.lua - - lsp.lua - - ui.lua - - coding.lua - -- name: Bootstrap lazy.nvim (headless install) - ansible.builtin.shell: | - LAZY_DIR="${HOME}/.local/share/nvim/lazy/lazy.nvim" - if [ ! -d "$LAZY_DIR" ]; then - git clone --filter=blob:none https://github.com/folke/lazy.nvim.git \ - --branch=stable "$LAZY_DIR" - fi - changed_when: false - failed_when: false - environment: - PATH: "{{ ansible_env_path }}" - -- name: Install nvim plugins (headless) - ansible.builtin.shell: | - set -o pipefail - nvim --headless "+Lazy! sync" +qa 2>&1 | tail -5 - environment: - PATH: "{{ ansible_env_path }}" - HOME: "{{ ansible_facts['env']['HOME'] }}" - changed_when: false - failed_when: false - timeout: 300 diff --git a/roles/pi/files/skills/tavily-search/SKILL.md b/roles/pi/files/skills/tavily-search/SKILL.md index ad1ba9f1..43fb0ed1 100644 --- a/roles/pi/files/skills/tavily-search/SKILL.md +++ b/roles/pi/files/skills/tavily-search/SKILL.md @@ -20,6 +20,6 @@ Returns: title, url, content snippet for each result. ## API Key -API key: read from `$TAVILY_API_KEY` (provisioned by the `shell_ai` role from Bitwarden item `tavily-api-key`). +API key: read from `$TAVILY_API_KEY` (provisioned by the `shell-ai` unit from Bitwarden item `tavily-api-key`). Endpoint: `https://api.tavily.com/search` diff --git a/roles/shell_ai/tasks/main.yml b/roles/shell_ai/tasks/main.yml deleted file mode 100644 index 9898a173..00000000 --- a/roles/shell_ai/tasks/main.yml +++ /dev/null @@ -1,141 +0,0 @@ ---- -# ── AI Coding Tools configuration ──────────────────────────────── - -# ── Per-host secrets from Bitwarden (DeepSeek + Forgejo) ───────── -# Gated on `bw_unlocked` (set by the bitwarden role) so a locked vault -# leaves any previously-written secret files untouched instead of writing -# empty values. -- name: Fetch and deploy shell secrets from Bitwarden - when: bw_unlocked | default(false) - environment: - BW_SESSION: "{{ bw_session }}" - PATH: "{{ ansible_env_path }}" - block: - - name: Fetch DeepSeek API key from Bitwarden - ansible.builtin.command: bw get password deepseek-api-key - register: deepseek_key - no_log: true - changed_when: false - failed_when: false - - - - name: Fetch Forgejo API token from Bitwarden - ansible.builtin.shell: | - set -euo pipefail - bw get notes forgejo | grep "API Token:" | sed 's/API Token: //' - register: forgejo_token - no_log: true - changed_when: false - failed_when: false - - # The OMP DeepSeek provider consumes this env var; the API key never lands - # in the agent config or this public repository. - - name: Fetch Tavily API key from Bitwarden - ansible.builtin.command: bw get password tavily-api-key - register: tavily_key - no_log: true - changed_when: false - failed_when: false - - # Meta Model API key (Muse Spark) lives in a Bitwarden secure note. - - name: Fetch Meta API key from Bitwarden - ansible.builtin.command: bw get notes meta-api-key - register: meta_key - no_log: true - changed_when: false - failed_when: false - - # OMP's Kiro provider sends this API key directly to Kiro's runtime API. - # It is separate from the OAuth bearer stored by OMP/kiro-cli. - - name: Fetch Kiro API key from Bitwarden - ansible.builtin.command: bw get notes kiro-api-key - register: kiro_key - no_log: true - changed_when: false - failed_when: false - - - name: Write shell secrets file - ansible.builtin.copy: - content: | - # Managed by dotfiles playbook — DO NOT EDIT - export OMP_KIRO_CONFIG="$HOME/.config/omp/kiro.json" - {% if deepseek_key.rc == 0 and deepseek_key.stdout | trim | length > 0 %} - export DEEPSEEK_API_KEY={{ deepseek_key.stdout | trim | quote }} - {% endif %} - {% if forgejo_token.rc == 0 and forgejo_token.stdout | trim | length > 0 %} - export FORGEJO_TOKEN={{ forgejo_token.stdout | trim | quote }} - {% endif %} - {% if tavily_key.rc == 0 and tavily_key.stdout | trim | length > 0 %} - export TAVILY_API_KEY={{ tavily_key.stdout | trim | quote }} - {% endif %} - {% if meta_key.rc == 0 and meta_key.stdout | trim | length > 0 %} - export META_API_KEY={{ meta_key.stdout | trim | quote }} - {% endif %} - {% if kiro_key.rc == 0 and kiro_key.stdout | trim | length > 0 %} - export KIRO_API_KEY={{ kiro_key.stdout | trim | quote }} - {% endif %} - dest: "{{ ansible_facts['env']['HOME'] }}/.config/shell/secrets.sh" - mode: "0600" - no_log: true - when: >- - (deepseek_key.rc == 0 and deepseek_key.stdout | trim | length > 0) - or (forgejo_token.rc == 0 and forgejo_token.stdout | trim | length > 0) - or (tavily_key.rc == 0 and tavily_key.stdout | trim | length > 0) - or (meta_key.rc == 0 and meta_key.stdout | trim | length > 0) - or (kiro_key.rc == 0 and kiro_key.stdout | trim | length > 0) - - # Fish doesn't source secrets.sh; give it the same values via conf.d. - - name: Ensure fish conf.d directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/fish/conf.d" - state: directory - mode: "0755" - when: >- - (deepseek_key.rc == 0 and deepseek_key.stdout | trim | length > 0) - or (forgejo_token.rc == 0 and forgejo_token.stdout | trim | length > 0) - or (tavily_key.rc == 0 and tavily_key.stdout | trim | length > 0) - or (meta_key.rc == 0 and meta_key.stdout | trim | length > 0) - or (kiro_key.rc == 0 and kiro_key.stdout | trim | length > 0) - - - name: Write fish secrets file - ansible.builtin.copy: - content: | - # Managed by dotfiles playbook — DO NOT EDIT - {% if deepseek_key.rc == 0 and deepseek_key.stdout | trim | length > 0 %} - set -gx DEEPSEEK_API_KEY {{ deepseek_key.stdout | trim | quote }} - {% endif %} - {% if forgejo_token.rc == 0 and forgejo_token.stdout | trim | length > 0 %} - set -gx FORGEJO_TOKEN {{ forgejo_token.stdout | trim | quote }} - {% endif %} - {% if tavily_key.rc == 0 and tavily_key.stdout | trim | length > 0 %} - set -gx TAVILY_API_KEY {{ tavily_key.stdout | trim | quote }} - {% endif %} - {% if meta_key.rc == 0 and meta_key.stdout | trim | length > 0 %} - set -gx META_API_KEY {{ meta_key.stdout | trim | quote }} - {% endif %} - {% if kiro_key.rc == 0 and kiro_key.stdout | trim | length > 0 %} - set -gx KIRO_API_KEY {{ kiro_key.stdout | trim | quote }} - {% endif %} - if command -v bw >/dev/null 2>&1; and test -r /tmp/bw_session - set -gx KIRO_API_KEY (env BW_SESSION=(cat /tmp/bw_session) bw get item 2706eda5-e8ad-43df-9838-b4e00167e95d 2>/dev/null | jq -r .notes) - test "$KIRO_API_KEY" = null; and set -e KIRO_API_KEY - end - dest: "{{ ansible_facts['env']['HOME'] }}/.config/fish/conf.d/secrets.fish" - mode: "0600" - no_log: true - when: >- - (deepseek_key.rc == 0 and deepseek_key.stdout | trim | length > 0) - or (forgejo_token.rc == 0 and forgejo_token.stdout | trim | length > 0) - or (tavily_key.rc == 0 and tavily_key.stdout | trim | length > 0) - or (meta_key.rc == 0 and meta_key.stdout | trim | length > 0) - or (kiro_key.rc == 0 and kiro_key.stdout | trim | length > 0) - -- name: Warn if deepseek-api-key not found in Bitwarden - ansible.builtin.debug: - msg: >- - Bitwarden item 'deepseek-api-key' not found — skipping DEEPSEEK_API_KEY. - OMP/Kiro OAuth can still be configured with `/login kiro`. - when: - - bw_unlocked | default(false) - - (deepseek_key | default({})).rc | default(1) != 0 - or (deepseek_key | default({})).stdout | default('') | trim | length == 0 diff --git a/roles/shell_atuin/tasks/main.yml b/roles/shell_atuin/tasks/main.yml deleted file mode 100644 index d00b611a..00000000 --- a/roles/shell_atuin/tasks/main.yml +++ /dev/null @@ -1,153 +0,0 @@ ---- -- name: Ensure atuin config dir exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/atuin" - state: directory - mode: "0755" - -- name: Deploy atuin config - ansible.builtin.copy: - dest: "{{ ansible_facts['env']['HOME'] }}/.config/atuin/config.toml" - mode: "0644" - content: | - sync_address = "https://api.atuin.sh" - sync_frequency = "5m" - auto_sync = true - search_mode = "fuzzy" - # ── filter_mode ────────────────────────────────────────────── - # "global" = search all history across every machine & session. - # "host" = only this machine. - # "session" = only this shell session (empty in fresh terminals!). - # ────────────────────────────────────────────────────────────── - # DO NOT set filter_mode_shell_up_key_binding to "session" — - # it makes the Up arrow appear broken in fresh terminals because - # a new shell session has no commands yet. - # ────────────────────────────────────────────────────────────── - filter_mode = "global" - filter_mode_shell_up_key_binding = "global" - style = "compact" - show_preview = true - max_preview_height = 4 - -# ── Atuin setup from Bitwarden ──────────────────────────────────── -- name: Ensure atuin data dir exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/share/atuin" - state: directory - mode: "0700" - -# bw sync is handled by the bitwarden role earlier in the play — skip here. - -# Skip BW round-trip when already logged in — atuin sync status is the source of truth. -- name: Check Atuin sync status - ansible.builtin.command: atuin status - register: atuin_status_cmd - changed_when: false - failed_when: false - environment: - PATH: "{{ ansible_env_path }}" - -- name: Set Atuin logged in fact - ansible.builtin.set_fact: - atuin_logged_in: "{{ atuin_status_cmd.rc == 0 and '[Sync]' in atuin_status_cmd.stdout }}" - -- name: Check if atuin binary is installed - ansible.builtin.shell: command -v atuin - register: atuin_binary - changed_when: false - failed_when: false - -- name: Warn if atuin binary is missing - ansible.builtin.debug: - msg: >- - atuin binary not found — skipping atuin login/sync. - Install atuin (brew install atuin) and re-run. - when: atuin_binary.rc != 0 - -- name: Check if atuin key exists - ansible.builtin.stat: - path: "{{ ansible_facts['env']['HOME'] }}/.local/share/atuin/key" - register: atuin_existing_key - -- name: Fetch atuin credentials from Bitwarden - ansible.builtin.command: bw get item atuin.sh - register: atuin_bw_item - no_log: true - changed_when: false - failed_when: false - when: - - not atuin_logged_in - - bw_unlocked | default(false) - environment: - BW_SESSION: "{{ bw_session | default('') }}" - PATH: "{{ ansible_env_path }}" - -- name: Skip atuin login (already logged in or vault locked) - ansible.builtin.set_fact: - atuin_bw_item: {rc: 1, stdout: ""} - when: atuin_logged_in or not (bw_unlocked | default(false)) - -- name: Warn if atuin.sh not found in Bitwarden - ansible.builtin.debug: - msg: >- - Bitwarden item 'atuin.sh' not found or vault locked — skipping atuin login. - Run 'bw unlock' on this machine and re-run, or log in to atuin manually. - when: - - not atuin_logged_in - - atuin_bw_item.rc != 0 or (atuin_bw_item.stdout | default('')) | length == 0 - -- name: Parse atuin credentials - ansible.builtin.set_fact: - atuin_username: "{{ (atuin_bw_item.stdout | from_json).login.username }}" - atuin_password: "{{ (atuin_bw_item.stdout | from_json).login.password }}" - atuin_key: "{{ (atuin_bw_item.stdout | from_json).fields | selectattr('name', 'equalto', 'key') | map(attribute='value') | first }}" - no_log: true - when: atuin_bw_item.rc == 0 and (atuin_bw_item.stdout | default('')) | length > 0 - -# NOTE: We do NOT write atuin_key (mnemonic) to ~/.local/share/atuin/key directly. -# Atuin expects that file to be a base64-encoded binary key. -# Running 'atuin login -k' with the mnemonic correctly generates the binary key file. -- name: Remove atuin data directory before fresh login - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/share/atuin" - state: absent - when: > - atuin_bw_item.rc == 0 and (atuin_bw_item.stdout | default("")) | length > 0 - and not atuin_existing_key.stat.exists - -- name: Login to atuin sync - ansible.builtin.command: > - atuin login -u "{{ atuin_username }}" -p "{{ atuin_password }}" -k "{{ atuin_key }}" - no_log: true - when: > - atuin_bw_item.rc == 0 and (atuin_bw_item.stdout | default("")) | length > 0 - and not atuin_logged_in - and atuin_binary.rc == 0 - changed_when: true - environment: - PATH: "{{ ansible_env_path }}" - -- name: Sync atuin history (best-effort — key mismatch is non-fatal) - ansible.builtin.command: atuin sync - changed_when: false - failed_when: false - environment: - PATH: "{{ ansible_env_path }}" - when: atuin_bw_item.rc == 0 and (atuin_bw_item.stdout | default("")) | length > 0 - -- name: Check if shell history has already been imported into atuin - ansible.builtin.stat: - path: "{{ ansible_facts['env']['HOME'] }}/.local/share/atuin/.history_imported" - register: atuin_import_sentinel - -- name: Import existing shell history into atuin - ansible.builtin.shell: | - atuin import auto 2>&1 && - touch "{{ ansible_facts['env']['HOME'] }}/.local/share/atuin/.history_imported" - environment: - PATH: "{{ ansible_env_path }}" - when: - - atuin_binary.rc == 0 - - not atuin_import_sentinel.stat.exists - changed_when: false - failed_when: false diff --git a/roles/shell_dotfiles/tasks/main.yml b/roles/shell_dotfiles/tasks/main.yml deleted file mode 100644 index 4efdfec9..00000000 --- a/roles/shell_dotfiles/tasks/main.yml +++ /dev/null @@ -1,136 +0,0 @@ ---- -- name: Create ~/.config/shell/ directory - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/shell" - state: directory - mode: "0755" - -- name: Deploy shared aliases (bash/zsh) - ansible.builtin.template: - src: aliases.sh.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/shell/aliases.sh" - mode: "0644" - -- name: Deploy ~/.bashrc - ansible.builtin.template: - src: bashrc.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.bashrc" - mode: "0644" - -- name: Deploy ~/.zshrc - ansible.builtin.template: - src: zshrc.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.zshrc" - mode: "0644" - -- name: Deploy ~/.zprofile - ansible.builtin.template: - src: zprofile.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.zprofile" - mode: "0644" - -- name: Deploy ~/.tmux.conf - ansible.builtin.copy: - src: tmux.conf - dest: "{{ ansible_facts['env']['HOME'] }}/.tmux.conf" - mode: "0644" - -- name: Deploy ~/.inputrc (readline defaults for bash + readline apps) - ansible.builtin.copy: - src: inputrc - dest: "{{ ansible_facts['env']['HOME'] }}/.inputrc" - mode: "0644" - -- name: Ensure ~/.config/starship directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config" - state: directory - mode: "0755" - -- name: Deploy starship.toml - ansible.builtin.copy: - src: starship.toml - dest: "{{ ansible_facts['env']['HOME'] }}/.config/starship.toml" - mode: "0644" - -- name: Create ~/.config/fish/ directory - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/fish" - state: directory - mode: "0755" - -- name: Create ~/.config/fish/conf.d/ directory - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/fish/conf.d" - state: directory - mode: "0755" - -- name: Deploy ~/.config/fish/config.fish - ansible.builtin.copy: - src: config.fish - dest: "{{ ansible_facts['env']['HOME'] }}/.config/fish/config.fish" - mode: "0644" - -- name: Deploy fish aliases - ansible.builtin.template: - src: aliases.fish.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/fish/conf.d/aliases.fish" - mode: "0644" - -# ── Ghostty terminfo ────────────────────────────────────────────── -# SSH-ing in from a Ghostty terminal exports TERM=xterm-ghostty. Hosts -# without that terminfo entry break ncurses apps ("missing or unsuitable -# terminal: xterm-ghostty" from tmux/vim/etc). Ship the entry and compile -# it into the user's ~/.terminfo so every host understands the term. -- name: Check whether tic (ncurses) is available - ansible.builtin.command: which tic - register: _tic_bin - changed_when: false - failed_when: false - -- name: Deploy ghostty terminfo source - ansible.builtin.copy: - src: xterm-ghostty.terminfo - dest: "{{ ansible_facts['env']['HOME'] }}/.config/shell/ghostty.terminfo" - mode: "0644" - register: _ghostty_ti_src - when: _tic_bin.rc == 0 - -- name: Check whether xterm-ghostty terminfo is already installed - ansible.builtin.command: infocmp -x xterm-ghostty - register: _ghostty_ti_installed - changed_when: false - failed_when: false - when: _tic_bin.rc == 0 - -- name: Compile ghostty terminfo into ~/.terminfo - ansible.builtin.command: > - tic -x -o {{ ansible_facts['env']['HOME'] }}/.terminfo - {{ ansible_facts['env']['HOME'] }}/.config/shell/ghostty.terminfo - when: - - _tic_bin.rc == 0 - - _ghostty_ti_installed.rc != 0 or _ghostty_ti_src.changed - changed_when: true - -- name: Warn if tic is unavailable (ghostty terminfo not installable) - ansible.builtin.debug: - msg: >- - tic (ncurses) is not installed on {{ inventory_hostname }}; the - xterm-ghostty terminfo cannot be compiled. SSH sessions from Ghostty - will need `TERM=xterm-256color` until ncurses is present. - when: _tic_bin.rc != 0 - -# ── Ghostty config (auto light/dark, follows GNOME) ─────────────── -# Inert on headless hosts (Ghostty never launched there); active wherever -# Ghostty runs. Kept alongside the rest of the managed shell config. -- name: Create ~/.config/ghostty/ directory - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/ghostty" - state: directory - mode: "0755" - -- name: Deploy ghostty config - ansible.builtin.copy: - src: ghostty.config - dest: "{{ ansible_facts['env']['HOME'] }}/.config/ghostty/config" - mode: "0644" diff --git a/scripts/omp-login.sh b/scripts/omp-login.sh index 48305b43..6b6ca43d 100755 --- a/scripts/omp-login.sh +++ b/scripts/omp-login.sh @@ -4,7 +4,7 @@ # scripts/omp-login.sh # status + log into anything missing # # - API-key providers (DeepSeek, Tavily, ...) need no omp login: omp reads them -# from env, and roles/shell_ai exports them from Bitwarden into the shell. +# from env, and units/shell-ai.sh exports them from Bitwarden into the shell. # This script only reports whether they resolve. # - OAuth providers (Claude Max, Codex Pro, Antigravity) need a one-time browser # approval. They are stored in ~/.omp/agent/agent.db and refresh on their own, @@ -32,7 +32,7 @@ done if session=$("$SCRIPT_DIR/bw-unlock.sh" 2>/dev/null); then for item in deepseek-api-key tavily-api-key meta-api-key; do if BW_SESSION="$session" bw get item "$item" >/dev/null 2>&1; then - printf ' ✓ %s (in Bitwarden; exported by shell_ai role)\n' "$item" + printf ' ✓ %s (in Bitwarden; exported by shell-ai unit)\n' "$item" else printf ' ⚠ %s not found in Bitwarden\n' "$item" fi diff --git a/scripts/run-units.sh b/scripts/run-units.sh index ac93b734..ad101ec6 100755 --- a/scripts/run-units.sh +++ b/scripts/run-units.sh @@ -3,14 +3,20 @@ # Usage: scripts/run-units.sh [--no-secrets] [--tags a,b] set -uo pipefail -ALL_UNITS="omp" # run order for the sequential fallback; keep in step with mise.toml +ALL_UNITS="chezmoi omp atuin shell-ai" # run order for the sequential fallback; keep in step with mise.toml tasks_for_tag() { case "$1" in # Ansible tag -> units (mirrors the old role tags) - dotfiles|omp) echo omp ;; + dotfiles) echo "chezmoi omp atuin shell-ai" ;; + shell) echo "chezmoi atuin shell-ai" ;; + git|neovim|editor) echo chezmoi ;; + atuin) echo atuin ;; + ai) echo shell-ai ;; + secrets) echo "atuin shell-ai" ;; + omp) echo omp ;; esac; } repo="$(cd "$(dirname "$0")/.." && pwd)" export DOTFILES_DIR="$repo" -PATH=/home/linuxbrew/.linuxbrew/bin:$PATH +PATH="$PATH:/home/linuxbrew/.linuxbrew/bin" tags="" while [ $# -gt 0 ]; do diff --git a/site.yml b/site.yml index 46b332d3..aff4351a 100644 --- a/site.yml +++ b/site.yml @@ -73,20 +73,10 @@ tags: [secrets, bitwarden] - role: shell_fonts tags: [dotfiles, shell, fonts] - - role: shell_dotfiles - tags: [dotfiles, shell, dotfiles] - role: hive_ops tags: [hive, hive_ops] - - role: git - tags: [dotfiles, git] - - role: neovim - tags: [dotfiles, neovim, editor] # Phase 2: Secrets + auth - - role: shell_atuin - tags: [dotfiles, shell, atuin, secrets] - - role: shell_ai - tags: [dotfiles, shell, ai, secrets] - role: ssh_keys tags: [secrets, ssh, ssh_keys] - role: ssh_mesh diff --git a/tests/test_atuin_unit.py b/tests/test_atuin_unit.py new file mode 100644 index 00000000..fb6a76f6 --- /dev/null +++ b/tests/test_atuin_unit.py @@ -0,0 +1,73 @@ +import shutil +import subprocess +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub + +pytestmark = pytest.mark.skipif(shutil.which("jq") is None, reason="jq not installed") + +ITEM = '{"login":{"username":"U","password":"P"},"fields":[{"name":"key","value":"K"}]}' + + +def stubs(bindir, status_out, status_rc=1, unlock_rc=0): + write_stub( + bindir, + "atuin", + f"""echo "atuin $*" >> "$HOME/calls" +case "$1" in + status) echo '{status_out}'; exit {status_rc} ;; +esac +exit 0 +""", + ) + write_stub( + bindir, + "bw", + f"""echo "bw $*" >> "$HOME/calls" +[ "$1 $2" = "unlock --check" ] && exit {unlock_rc} +[ "$1 $2 $3" = "get item atuin.sh" ] && echo '{ITEM}' +exit 0 +""", + ) + + +def run(env): + env = {**env, "BW_SESSION": "x", "DOTFILES_DIR": str(REPO), "BREW_PREFIX": "/nonexistent"} + r = subprocess.run(["bash", str(REPO / "units/atuin.sh")], env=env, capture_output=True, text=True) + calls = Path(env["HOME"], "calls") + return r, calls.read_text() if calls.exists() else "" + + +def test_logs_in_with_bitwarden_credentials(stub_env): + bindir, env = stub_env + stubs(bindir, "not logged in") + r, calls = run(env) + assert r.returncode == 0, r.stdout + r.stderr + assert "atuin login -u U -p P -k K" in calls + + +def test_already_logged_in_never_touches_bitwarden(stub_env): + bindir, env = stub_env + stubs(bindir, "[Sync] ok", status_rc=0) + r, calls = run(env) + assert r.returncode == 0 + assert "login" not in calls and "bw " not in calls + + +def test_locked_vault_is_partial_and_does_not_login(stub_env): + bindir, env = stub_env + stubs(bindir, "not logged in", unlock_rc=1) + r, calls = run(env) + assert r.returncode == 0 + assert "PARTIAL:" in r.stdout + assert "login" not in calls + + +def test_missing_atuin_is_skipped(stub_env): + bindir, env = stub_env + env["PATH"] = f"{bindir}:/usr/bin:/bin" + r, _ = run(env) + assert r.returncode == 0 + assert "SKIPPED:" in r.stdout diff --git a/tests/test_brew_blocked.py b/tests/test_brew_blocked.py index e7e31555..127b5e45 100644 --- a/tests/test_brew_blocked.py +++ b/tests/test_brew_blocked.py @@ -39,7 +39,7 @@ def test_no_install_list_names_a_blocked_package(): def test_no_alias_routes_through_trash(): - groups = yaml.safe_load((REPO / "roles" / "shell_dotfiles" / "vars" / "main.yml").read_text())["alias_groups"] + groups = yaml.safe_load((REPO / "home" / ".chezmoidata.yaml").read_text())["alias_groups"] assert all(g.get("guard") != "trash" for g in groups) cmds = [a["cmd"] for g in groups for a in g.get("aliases", [])] assert not [c for c in cmds if c.split()[0] in {"trash", "trash-put"}] diff --git a/tests/test_chezmoi_data.py b/tests/test_chezmoi_data.py new file mode 100644 index 00000000..a79c4c06 --- /dev/null +++ b/tests/test_chezmoi_data.py @@ -0,0 +1,19 @@ +import yaml + +from conftest import REPO + + +def load(rel): + return yaml.safe_load((REPO / rel).read_text()) + + +def test_identity_matches_ansible_vars(): + # group_vars/all.yml feeds ssh_keys; home/.chezmoidata.yaml feeds ~/.gitconfig. + ans, chez = load("group_vars/all.yml"), load("home/.chezmoidata.yaml") + assert (chez["user_name"], chez["user_email"]) == (ans["user_name"], ans["user_email"]) + + +def test_every_alias_has_name_and_cmd(): + for group in load("home/.chezmoidata.yaml")["alias_groups"]: + for a in group["aliases"]: + assert a.get("name") and a.get("cmd"), a diff --git a/tests/test_shell_ai_unit.py b/tests/test_shell_ai_unit.py new file mode 100644 index 00000000..77464762 --- /dev/null +++ b/tests/test_shell_ai_unit.py @@ -0,0 +1,74 @@ +import subprocess + +import pytest + +from conftest import REPO, write_stub + +NASTY = "pa'ss\"w$ord" + +BW_STUB = """\ +if [ "$1 $2" = "unlock --check" ]; then exit "${STUB_UNLOCK_RC:-0}"; fi +[ "$1" = get ] || exit 1 +case "$2 $3" in + "password deepseek-api-key") printf '%s\\n' "$STUB_DEEPSEEK" ;; + "notes forgejo") printf 'Name: x\\nAPI Token: forge123\\n' ;; + "password tavily-api-key") printf 'tav456\\n' ;; + *) exit 1 ;; +esac +""" + + +@pytest.fixture +def env(stub_env): + bindir, env = stub_env + write_stub(bindir, "bw", BW_STUB) + env.update(BW_SESSION="x", DOTFILES_DIR=str(REPO), STUB_DEEPSEEK=NASTY, BREW_PREFIX="/nonexistent") + return env + + +def run(env, **extra): + return subprocess.run( + ["bash", str(REPO / "units/shell-ai.sh")], env={**env, **extra}, capture_output=True, text=True + ) + + +def home(env): + from pathlib import Path + + return Path(env["HOME"]) + + +def test_secrets_round_trip_and_modes(env): + r = run(env) + assert r.returncode == 0, r.stdout + r.stderr + sh = home(env) / ".config/shell/secrets.sh" + fish = home(env) / ".config/fish/conf.d/secrets.fish" + assert sh.stat().st_mode & 0o777 == 0o600 and fish.stat().st_mode & 0o777 == 0o600 + out = subprocess.run( + ["bash", "-c", f'. {sh}; printf "%s\\n%s\\n%s" "$DEEPSEEK_API_KEY" "$FORGEJO_TOKEN" "$TAVILY_API_KEY"'], + capture_output=True, + text=True, + ).stdout + assert out == f"{NASTY}\nforge123\ntav456" + assert "set -gx FORGEJO_TOKEN 'forge123'" in fish.read_text() + assert "2706eda5-e8ad-43df-9838-b4e00167e95d" in fish.read_text() + assert NASTY not in r.stdout + r.stderr + + +def test_missing_item_still_writes_the_others(env): + r = run(env, STUB_DEEPSEEK="") + assert r.returncode == 0 + text = (home(env) / ".config/shell/secrets.sh").read_text() + assert "DEEPSEEK_API_KEY" not in text + assert "FORGEJO_TOKEN='forge123'" in text + + +@pytest.mark.parametrize("extra", [{"STUB_UNLOCK_RC": "1"}, {"UNITS_NO_SECRETS": "1"}]) +def test_locked_or_nosecrets_leaves_files_untouched(env, extra): + sh = home(env) / ".config/shell/secrets.sh" + sh.parent.mkdir(parents=True) + sh.write_text("previous\n") + r = run(env, **extra) + assert r.returncode == 0 + assert "SKIPPED:" in r.stdout + assert sh.read_text() == "previous\n" diff --git a/units/atuin.sh b/units/atuin.sh new file mode 100755 index 00000000..72c20be7 --- /dev/null +++ b/units/atuin.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# atuin unit: log in to atuin sync from Bitwarden and import shell history. +# The config file itself (~/.config/atuin/config.toml) is deployed by chezmoi. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +data="$HOME/.local/share/atuin" + +if ! command -v atuin >/dev/null 2>&1; then + note "SKIPPED: atuin not installed (brew install atuin)" + exit 0 +fi + +step "atuin data dir" install -d -m 0700 "$data" + +logged_in() { + local out + out="$(atuin status 2>&1)" || return 1 + case "$out" in *"[Sync]"*) return 0 ;; *) return 1 ;; esac +} + +atuin_login() { + local s item u p k + s="$(bw_session)" + if [ -z "$s" ]; then + note "PARTIAL: atuin not logged in; vault locked (bw unlock, then just apply-tags atuin)" + return 0 + fi + item="$(BW_SESSION="$s" bw get item atuin.sh 2>/dev/null)" + u="$(printf '%s' "$item" | jq -r '.login.username // empty' 2>/dev/null)" + p="$(printf '%s' "$item" | jq -r '.login.password // empty' 2>/dev/null)" + k="$(printf '%s' "$item" | jq -r '.fields[]? | select(.name=="key") | .value' 2>/dev/null | head -n1)" + if [ -z "$u" ] || [ -z "$p" ] || [ -z "$k" ]; then + note "PARTIAL: Bitwarden item atuin.sh missing or incomplete; atuin login skipped" + return 0 + fi + # atuin expects a binary key file; `login -k` regenerates it from the mnemonic, so reset first. + [ -e "$data/key" ] || rm -rf "$data" + atuin login -u "$u" -p "$p" -k "$k" || return 1 + soft "atuin sync" atuin sync +} + +if ! logged_in; then + step "atuin login" atuin_login +fi + +if [ ! -e "$data/.history_imported" ]; then + soft "atuin import" sh -c 'atuin import auto 2>&1 && mkdir -p ~/.local/share/atuin && touch ~/.local/share/atuin/.history_imported' +fi +finish diff --git a/units/chezmoi.sh b/units/chezmoi.sh new file mode 100755 index 00000000..2765a44a --- /dev/null +++ b/units/chezmoi.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# chezmoi unit: apply the file-based dotfiles from home/ (shell, git, neovim, atuin config). +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if ! command -v chezmoi >/dev/null 2>&1 && [ "$IS_TERMUX" = 1 ]; then + soft "pkg install chezmoi" pkg install -y chezmoi +fi +if ! command -v chezmoi >/dev/null 2>&1; then + note "SKIPPED: chezmoi not installed (run: just packages)" + exit 0 +fi + +is_desktop=0 +if command -v yq >/dev/null 2>&1; then + [ "$(yq ".all.children.desktop.hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml" 2>/dev/null)" = true ] && is_desktop=1 +else + note "warn: yq missing, assuming non-desktop" +fi +export DOTFILES_MACHINE="$MACHINE" DOTFILES_IS_DESKTOP="$is_desktop" + +step "chezmoi apply" chezmoi --source "$DOTFILES_DIR" --keep-going --force --no-tty apply --init +finish diff --git a/units/lib.sh b/units/lib.sh old mode 100644 new mode 100755 index d90541e0..4f391827 --- a/units/lib.sh +++ b/units/lib.sh @@ -2,6 +2,8 @@ # Shared helpers for units/*.sh. Source this; do not execute it. # Units use `set -uo pipefail` and never `set -e`: a failing step is recorded # and the later independent steps still run. +# Homebrew tools come last so a stub or an earlier install wins; BREW_PREFIX is overridable for tests. +PATH="$PATH:${BREW_PREFIX:-/home/linuxbrew/.linuxbrew}/bin" UNIT_FAILED=0 step() { local d="$1"; shift; if "$@"; then echo "ok $d"; else echo "FAIL $d (exit $?)"; UNIT_FAILED=1; fi; } soft() { local d="$1"; shift; if "$@"; then echo "ok $d"; else echo "warn $d (exit $?, ignored)"; fi; } @@ -23,3 +25,4 @@ bw_session() { [ -n "$s" ] && bw unlock --check --session "$s" >/dev/null 2>&1 && printf '%s' "$s" return 0 } +MACHINE="$(cat /etc/dotfiles-machine 2>/dev/null || hostname)" diff --git a/units/omp.sh b/units/omp.sh index 1d6d7b56..dfb8ceb1 100755 --- a/units/omp.sh +++ b/units/omp.sh @@ -4,7 +4,7 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -PATH="$HOME/.bun/bin:$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:$PATH" +PATH="$HOME/.bun/bin:$HOME/.local/bin:$PATH" cfg="$HOME/.omp/agent/config.yml" skills_dest="$HOME/.omp/agent/skills/james" diff --git a/units/omp/omp-route b/units/omp/omp-route index e147f9db..a4c79098 100755 --- a/units/omp/omp-route +++ b/units/omp/omp-route @@ -7,7 +7,7 @@ # own usage-aware fallback and retry.fallbackChains.default (same order) take # over mid-session. # -# omp-route [omp args...] # aliased to `omp` by shell_dotfiles +# omp-route [omp args...] # aliased to `omp` by the chezmoi shell aliases # omp-route --pick # print the chosen model and why, then exit set -uo pipefail diff --git a/units/shell-ai.sh b/units/shell-ai.sh new file mode 100755 index 00000000..87a0949a --- /dev/null +++ b/units/shell-ai.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# shell-ai unit: per-host API secrets from Bitwarden -> ~/.config/shell/secrets.sh and fish conf.d. +# Not a chezmoi template: `bitwarden` aborts a whole template on one missing item, while this +# unit writes whatever secrets exist and leaves the files alone when the vault is locked. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +s="$(bw_session)" +if [ -z "$s" ]; then + note "SKIPPED: vault locked or --no-secrets; secret files left untouched" + exit 0 +fi + +trim() { printf '%s' "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'; } +shq() { printf "'%s'" "${1//\'/\'\\\'\'}"; } +bwget() { BW_SESSION="$s" bw get "$@" 2>/dev/null; } + +deepseek="$(trim "$(bwget password deepseek-api-key)")" +forgejo="$(trim "$(bwget notes forgejo | grep 'API Token:' | sed 's/API Token: //')")" +tavily="$(trim "$(bwget password tavily-api-key)")" +meta="$(trim "$(bwget notes meta-api-key)")" +kiro="$(trim "$(bwget notes kiro-api-key)")" + +if [ -z "$deepseek$forgejo$tavily$meta$kiro" ]; then + note "warn: no secrets found in Bitwarden (deepseek-api-key missing?); files left untouched" + exit 0 +fi + +# write_atomic dest: stdin -> temp file in the same directory (mode 0600) -> mv +write_atomic() { + local dest="$1" tmp + mkdir -p "$(dirname "$dest")" || return 1 + tmp="$(umask 077; mktemp "$dest.XXXXXX")" || return 1 + if cat > "$tmp" && chmod 600 "$tmp" && mv -f "$tmp" "$dest"; then return 0; fi + rm -f "$tmp"; return 1 +} + +secrets_sh() { + echo '# Managed by dotfiles playbook — DO NOT EDIT' + echo 'export OMP_KIRO_CONFIG="$HOME/.config/omp/kiro.json"' + [ -z "$deepseek" ] || echo "export DEEPSEEK_API_KEY=$(shq "$deepseek")" + [ -z "$forgejo" ] || echo "export FORGEJO_TOKEN=$(shq "$forgejo")" + [ -z "$tavily" ] || echo "export TAVILY_API_KEY=$(shq "$tavily")" + [ -z "$meta" ] || echo "export META_API_KEY=$(shq "$meta")" + [ -z "$kiro" ] || echo "export KIRO_API_KEY=$(shq "$kiro")" +} + +secrets_fish() { + echo '# Managed by dotfiles playbook — DO NOT EDIT' + [ -z "$deepseek" ] || echo "set -gx DEEPSEEK_API_KEY $(shq "$deepseek")" + [ -z "$forgejo" ] || echo "set -gx FORGEJO_TOKEN $(shq "$forgejo")" + [ -z "$tavily" ] || echo "set -gx TAVILY_API_KEY $(shq "$tavily")" + [ -z "$meta" ] || echo "set -gx META_API_KEY $(shq "$meta")" + [ -z "$kiro" ] || echo "set -gx KIRO_API_KEY $(shq "$kiro")" + cat <<'FISH' +if command -v bw >/dev/null 2>&1; and test -r /tmp/bw_session + set -gx KIRO_API_KEY (env BW_SESSION=(cat /tmp/bw_session) bw get item 2706eda5-e8ad-43df-9838-b4e00167e95d 2>/dev/null | jq -r .notes) + test "$KIRO_API_KEY" = null; and set -e KIRO_API_KEY +end +FISH +} + +write_sh() { secrets_sh | write_atomic "$HOME/.config/shell/secrets.sh"; } +write_fish() { secrets_fish | write_atomic "$HOME/.config/fish/conf.d/secrets.fish"; } + +step "shell secrets file" write_sh +step "fish secrets file" write_fish +finish From 29bbc306f7a00fbe418a35330da5d3b095e6f2fe Mon Sep 17 00:00:00 2001 From: James Reilly Date: Sun, 25 Oct 2026 21:19:53 +0000 Subject: [PATCH 04/62] feat(tailnet): audit/gc/rename devices against inventory; fix role's dead device cleanup --- Justfile | 12 + docs/src/SUMMARY.md | 1 + docs/src/roles/tailscale.md | 4 +- docs/src/tailnet.md | 31 +++ roles/tailscale/tasks/connect.yml | 45 +--- scripts/bw-seed-tailnet.sh | 49 ++++ scripts/inventory_parser.py | 60 +++++ scripts/tailnet.py | 424 ++++++++++++++++++++++++++++++ tests/test_tailnet.py | 127 +++++++++ 9 files changed, 718 insertions(+), 35 deletions(-) create mode 100644 docs/src/tailnet.md create mode 100755 scripts/bw-seed-tailnet.sh create mode 100755 scripts/tailnet.py create mode 100644 tests/test_tailnet.py diff --git a/Justfile b/Justfile index 08c48444..f8079e04 100644 --- a/Justfile +++ b/Justfile @@ -537,6 +537,18 @@ seed-kube: seed-omp-broker: @{{ dotfiles_dir }}/scripts/bw-seed-omp-broker.sh +# Tailnet hygiene (scripts/tailnet.py): audit is read-only; the others dry-run unless `--apply` +tailnet-audit: + @{{ dotfiles_dir }}/scripts/tailnet.py audit + +# gc + rename + inventory alignment, e.g. `just tailnet-sync --apply` +tailnet-sync *args: + @{{ dotfiles_dir }}/scripts/tailnet.py sync {{ args }} + +# Store the Tailscale OAuth client secret (device gc/rename + join key) in Bitwarden +seed-tailnet-oauth: + @{{ dotfiles_dir }}/scripts/bw-seed-tailnet.sh + # Show all online fleet machines' doctor status in parallel doctor-fleet: #!/usr/bin/env bash diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index a7ea7562..89a5e9a7 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -6,6 +6,7 @@ - [Architecture](architecture.md) - [Units](units.md) +- [Tailnet hygiene](tailnet.md) - [Inventory & Groups](inventory.md) - [Secrets with Bitwarden](bitwarden.md) - [Onboarding a New Machine](onboarding.md) diff --git a/docs/src/roles/tailscale.md b/docs/src/roles/tailscale.md index e205f8fa..cdcc8133 100644 --- a/docs/src/roles/tailscale.md +++ b/docs/src/roles/tailscale.md @@ -11,7 +11,7 @@ Installs Tailscale, fetches a reusable auth key from BW, joins the network, conf 1. Configures `systemd-resolved` so MagicDNS works (link `/etc/resolv.conf` to the stub resolver, drop in `DNSStubListenerExtra=::1` for container DNS). 2. Sets `net.ipv4.conf.{all,default}.rp_filter=2` and IP forwarding on, so exit-node and subnet-router traffic actually routes. 3. If `tailscale status` reports connected → updates hostname, prefs, operator, systray, then exits the role. -4. If not connected and the vault is unlocked → fetches `tailscale-apikey` from BW, deletes any stale device with this hostname via the Tailscale API, then `tailscale up --authkey …`. +4. If not connected and the vault is unlocked → fetches `tailscale-apikey` (an OAuth client secret) from BW, runs `scripts/tailnet.py release ` to delete stale devices that still hold the hostname (see [Tailnet hygiene](../tailnet.md)), then `tailscale up --authkey …`. 5. If not connected and the vault is **locked** → emits a one-line warning and ends the role. Joining without a key would either fail noisily or, worse, succeed with an empty key. ## Failure modes @@ -20,7 +20,7 @@ Installs Tailscale, fetches a reusable auth key from BW, joins the network, conf |---|---|---| | Role aborts with "vault locked" warning, host is not on tailnet | First-time apply on this host without BW unlocked | Unlock BW (`bw unlock --raw | tee /tmp/bw_session > /dev/null`), then `just apply-tags tailscale` | | Tailscale connected but MagicDNS names don't resolve | `systemd-resolved` link to `/etc/resolv.conf` was overwritten by NetworkManager / DHCP client | The role re-applies it; just re-run `dots-apply`. To make it sticky, check `network-manager` settings (Bluefin does this correctly out of the box) | -| Duplicate device entries in admin console after rebuild | API-key device cleanup didn't run (vault locked at the time) | Manually delete in [tailscale admin console](https://login.tailscale.com/admin/machines) or unlock vault and re-apply | +| Duplicate device entries / `-1` names after rebuild | The BW credential is dead or lacks scopes, so `tailnet.py release` could not delete the old device | `just seed-tailnet-oauth`, then `just tailnet-sync --apply` | | Joins but no routes advertised | `tailscale_advertise_routes` not set in `host_vars` | Add it: `tailscale_advertise_routes: ""` | ## How to verify diff --git a/docs/src/tailnet.md b/docs/src/tailnet.md new file mode 100644 index 00000000..52345d1f --- /dev/null +++ b/docs/src/tailnet.md @@ -0,0 +1,31 @@ +# Tailnet hygiene + +`scripts/tailnet.py` keeps the tailnet and `inventory.yml` in agreement. Everything is a dry run unless `--apply` is given. + +| Command | What it does | +|---|---| +| `just tailnet-audit` | Read-only report; exit 1 when anything needs doing | +| `just tailnet-sync [--apply]` | GC, then rename, then inventory alignment, planned as one sequence | +| `scripts/tailnet.py gc [--apply] [--days N] [--k8s-days N]` | Delete duplicate devices and stale test VMs | +| `scripts/tailnet.py rename [--apply]` | Rename live devices to their intended name | +| `scripts/tailnet.py fix-inventory [--apply]` | Make `ansible_host` IPs and `tailnet_name` overrides follow the live devices | +| `scripts/tailnet.py release NAME` | Delete offline devices called `NAME` or `NAME-`; the `tailscale` role runs it before joining | + +## Rules + +- A device's tailnet name is the first MagicDNS label. An inventory host's **intended** name is its `tailnet_name` var, else the inventory name. A `tailnet_name` of the form `-` is the auto-suffix Tailscale adds when a stale device still holds ``, so it means "wants ``" and is removed once the rename has happened. +- **Duplicates:** several devices named `` / `-`: the live (else most recently seen) one is kept; older offline ones are deleted. +- **Test VMs:** offline longer than `--days` (default 14), named `gate-*`, `repro-*`, `tuna-v1-preflight-*` or `*-vm`, and not an inventory host. +- **k8s proxies** (`tag:k8s*`) are only deleted with `--k8s-days`. The home cluster is powered down and expected back, so its proxies are deliberately kept by default. +- Never touched: devices that are online, this machine, devices shared in from other tailnets, and the retired `vps` group. +- `fix-inventory` run alone reflects the live state, so it will not drop a `tailnet_name` override before the rename has actually happened. + +## Credential + +Reads fall back to the local `tailscale status --json`; writes need the API. The credential is a Tailscale **OAuth client** (scopes `devices:core` write and `auth_keys` write, tag `tag:fleet`) created at . Store it with `just seed-tailnet-oauth`: the secret is verified against Tailscale before it is written to Bitwarden item `tailscale-apikey`. `TS_OAUTH_CLIENT_SECRET` overrides the Bitwarden lookup. + +The same secret is the join auth key used by the [`tailscale`](roles/tailscale.md) role. The previous cleanup in that role treated it as a classic API key, got a 401 that `failed_when: false` hid, and never deleted anything; that is why re-joined machines came back as `himachal-1` and `kerala-1`. + +## Caveat when renaming + +After `rename --apply`, `himachal-1` stops resolving; use the new name. Do not rename while something is using the old MagicDNS name (an rsync or ssh session, for example). diff --git a/roles/tailscale/tasks/connect.yml b/roles/tailscale/tasks/connect.yml index ea38bd8a..d211b19c 100644 --- a/roles/tailscale/tasks/connect.yml +++ b/roles/tailscale/tasks/connect.yml @@ -23,39 +23,18 @@ PATH: "{{ ansible_env_path }}" BW_SESSION: "{{ bw_session }}" -- name: Remove old tailscale device with same hostname - block: - - name: Fetch tailscale device list - ansible.builtin.uri: - url: https://api.tailscale.com/api/v2/tailnet/-/devices - method: GET - user: "{{ tailscale_apikey.stdout }}" - password: "" - force_basic_auth: true - status_code: 200 - headers: - Content-Type: application/json - register: ts_devices - failed_when: false - no_log: true - - - name: Find old device matching our hostname - ansible.builtin.set_fact: - ts_old_device_id: >- - {{ (ts_devices.json.devices | default([]) - | selectattr('name', 'equalto', inventory_hostname + '.' + tailscale_domain) - | map(attribute='id') | first) | default('') }} - - - name: Delete old tailscale device - ansible.builtin.uri: - url: https://api.tailscale.com/api/v2/device/{{ ts_old_device_id }} - method: DELETE - user: "{{ tailscale_apikey.stdout }}" - password: "" - force_basic_auth: true - status_code: 200 - when: ts_old_device_id | length > 0 - no_log: true +# A stale device that still holds this hostname makes Tailscale join us as +# -1. scripts/tailnet.py exchanges the OAuth client secret for an API +# token (the old inline cleanup used it as a classic API key, got a silent 401, +# and never deleted anything). +- name: Remove stale tailscale devices with this hostname + ansible.builtin.command: python3 {{ playbook_dir }}/scripts/tailnet.py release {{ inventory_hostname }} + environment: + TS_OAUTH_CLIENT_SECRET: "{{ tailscale_apikey.stdout }}" + register: ts_release + changed_when: "'deleted' in ts_release.stdout" + failed_when: false + no_log: true when: tailscale_apikey.stdout | default('') | length > 0 - name: Join tailscale network diff --git a/scripts/bw-seed-tailnet.sh b/scripts/bw-seed-tailnet.sh new file mode 100755 index 00000000..a79661d3 --- /dev/null +++ b/scripts/bw-seed-tailnet.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Store the Tailscale OAuth client secret in Bitwarden item `tailscale-apikey`. +# Used by the tailscale role (as the join auth key) and scripts/tailnet.py (device gc/rename). +# +# Create the client first: https://login.tailscale.com/admin/settings/oauth +# scopes: devices:core (write), auth_keys (write); tag: tag:fleet +# Usage: ./scripts/bw-seed-tailnet.sh (prompts for the secret, no echo) +# printf %s "$SECRET" | ./scripts/bw-seed-tailnet.sh +# The secret is checked against Tailscale before anything is written. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/bw-item.sh" + +if [ -z "${BW_SESSION:-}" ]; then + BW_SESSION=$("$SCRIPT_DIR/bw-unlock.sh") + export BW_SESSION +fi + +if [ -t 0 ]; then + read -rs -p "Tailscale OAuth client secret (tskey-client-...): " secret; echo >&2 +else + secret=$(cat) +fi +secret="${secret//[[:space:]]/}" +case "$secret" in + tskey-client-*) ;; + *) echo "that is not an OAuth client secret (expected tskey-client-...)" >&2; exit 1 ;; +esac +cid=$(cut -d- -f3 <<<"$secret") + +code=$(curl -s -o /dev/null -w '%{http_code}' -X POST https://api.tailscale.com/api/v2/oauth/token \ + --data-urlencode "client_id=$cid" --data-urlencode "client_secret=$secret") +if [ "$code" != 200 ]; then + echo "Tailscale rejected the secret (HTTP $code); nothing written." >&2 + exit 1 +fi + +payload=$(bw get template item \ + | jq --arg tok "$secret" --arg cid "$cid" ' + . + { + name: "tailscale-apikey", + type: 1, + notes: "Tailscale OAuth client (devices:core, auth_keys; tag:fleet). Used as the join auth key by the tailscale role and by scripts/tailnet.py. Managed by scripts/bw-seed-tailnet.sh.", + login: {username: $cid, password: $tok} + }') + +bw_upsert_item "tailscale-apikey" "$payload" 1 +bw sync >/dev/null +echo "Done. Check the tailnet with: just tailnet-audit" diff --git a/scripts/inventory_parser.py b/scripts/inventory_parser.py index 0b7047fa..1a33fd05 100644 --- a/scripts/inventory_parser.py +++ b/scripts/inventory_parser.py @@ -47,6 +47,66 @@ def parse_inventory(text: str): return all_hosts, vps +def parse_host_vars(text: str): + """{host: {var: value}} for the 6-space `key: value` lines under each all.hosts entry. + + Values are stripped of inline comments and quotes; no PyYAML (see parse_inventory). + """ + hosts, section, host = {}, None, None + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + indent = len(line) - len(line.lstrip()) + if indent == 2: + section = line.strip().rstrip(":") + elif indent == 4 and section == "hosts": + host = line.strip().rstrip(":") + hosts[host] = {} + elif indent == 6 and section == "hosts" and host is not None: + m = re.match(r"([A-Za-z0-9_]+):\s*(.*?)\s*(?:#.*)?$", line.strip()) + if m and m.group(2): + hosts[host][m.group(1)] = m.group(2).strip("'\"") + return hosts + + +def set_host_var(text: str, host: str, key: str, value): + """Return `text` with `host`'s `key` set to `value` (or removed when value is None). + + Edits only the host's own block under all.hosts, keeping its inline comment. + A missing key is appended after the block's last variable line. Unknown hosts + leave the text unchanged. + """ + lines = text.split("\n") + start = None + for i, line in enumerate(lines): + if line.startswith(" children:"): + break + if re.fullmatch(rf" {re.escape(host)}:\s*(#.*)?", line): + start = i + break + if start is None: + return text + end = start + 1 + while end < len(lines) and (lines[end].startswith(" ") or not lines[end].strip() or lines[end].lstrip().startswith("#")): + if lines[end].startswith(" ") and not lines[end].startswith(" "): + break + end += 1 + last_var = start + for i in range(start + 1, end): + if lines[i].startswith(" ") and not lines[i].lstrip().startswith("#"): + last_var = i + m = re.match(rf"( {re.escape(key)}:\s*)([^#\n]*?)(\s*(?:#.*)?)$", lines[i]) + if m: + if value is None: + del lines[i] + else: + lines[i] = f"{m.group(1)}{value}{m.group(3) if m.group(3).strip() else ''}" + return "\n".join(lines) + if value is not None: + lines.insert(last_var + 1, f" {key}: {value}") + return "\n".join(lines) + + def register_host(name: str, mtype: str = "desktop", inv_path: str = "inventory.yml") -> bool: """Add `name` to all.hosts and to the `mtype` group. diff --git a/scripts/tailnet.py b/scripts/tailnet.py new file mode 100755 index 00000000..3d095faa --- /dev/null +++ b/scripts/tailnet.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""Tailnet hygiene: garbage-collect stale devices, rename them to the inventory, align inventory.yml. + + tailnet.py audit what is wrong (exit 1 when anything is) + tailnet.py gc [--apply] delete duplicates and stale test VMs + tailnet.py rename [--apply] rename live devices to their inventory (tailnet) name + tailnet.py fix-inventory [--apply] ansible_host IPs / tailnet_name overrides follow the live devices + tailnet.py sync [--apply] gc, then rename, then fix-inventory (planned as one sequence) + tailnet.py release NAME delete offline devices called NAME or NAME- (run before joining) + +Everything is a dry run unless --apply. Reads come from the Tailscale API when a credential is +available (TS_OAUTH_CLIENT_SECRET, or Bitwarden item `tailscale-apikey`), otherwise from the local +`tailscale status --json` (read-only: enough for audit and for planning). Writes need the API. + +Names: a device's tailnet name is its first MagicDNS label. An inventory host's *intended* name is +its `tailnet_name` var, else the inventory name; a `tailnet_name` of the form - is the +auto-suffix Tailscale adds when a stale device still holds , so it is treated as "wants ". +Retired hosts (the `vps` group) and devices shared in from other tailnets are never touched. +""" +import argparse +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timedelta, timezone + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from inventory_parser import parse_host_vars, parse_inventory, set_host_var # noqa: E402 + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +INVENTORY = os.environ.get("DOTFILES_INVENTORY", os.path.join(REPO, "inventory.yml")) +API = "https://api.tailscale.com/api/v2" +BW_ITEM = "tailscale-apikey" +TEST_VM = re.compile(r"(^gate-|^repro-|^tuna-v1-preflight-|.-vm$)") +IP100 = re.compile(r"^100\.\d+\.\d+\.\d+$") + + +class AuthError(Exception): + pass + + +# ── devices ──────────────────────────────────────────────────────────────── + +def _ts(value): + if not value or value.startswith("0001-"): + return None + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def devices_from_status(status, now): + """Normalise `tailscale status --json` (peers + self).""" + suffix = (status.get("MagicDNSSuffix") or (status.get("CurrentTailnet") or {}).get("MagicDNSSuffix") or "").strip(".") + out = [] + entries = [(p, False) for p in (status.get("Peer") or {}).values()] + if status.get("Self"): + entries.append((status["Self"], True)) + for p, is_self in entries: + dns = (p.get("DNSName") or "").rstrip(".") + name = (dns.split(".")[0] if dns else p.get("HostName", "")).lower() + online = bool(p.get("Online")) + seen = now if online else _ts(p.get("LastSeen")) + out.append({ + "id": p.get("ID") or "", + "name": name, + "ip": (p.get("TailscaleIPs") or [""])[0], + "tags": list(p.get("Tags") or []), + "online": online, + "seen": seen, + "self": is_self, + "foreign": bool(suffix and dns and not dns.endswith("." + suffix)), + }) + return out + + +def devices_from_api(payload, now, self_ids=()): + out = [] + for d in payload.get("devices", []): + addrs = d.get("addresses") or [""] + out.append({ + "id": d.get("nodeId") or d.get("id") or "", + "name": (d.get("name") or "").split(".")[0].lower(), + "ip": next((a for a in addrs if IP100.match(a)), addrs[0]), + "tags": list(d.get("tags") or []), + "online": bool(d.get("connectedToControl")), + "seen": now if d.get("connectedToControl") else _ts(d.get("lastSeen")), + "self": (d.get("nodeId") or d.get("id")) in self_ids, + "foreign": bool(d.get("isExternal")), + }) + return out + + +# ── API client ───────────────────────────────────────────────────────────── + +def _secret(): + secret = os.environ.get("TS_OAUTH_CLIENT_SECRET", "") + if secret: + return secret + session = os.environ.get("BW_SESSION", "") + if not session and os.path.exists("/tmp/bw_session"): + session = open("/tmp/bw_session").read().strip() + if not session: + return "" + env = {**os.environ, "BW_SESSION": session} + try: + r = subprocess.run(["bw", "get", "password", BW_ITEM], capture_output=True, text=True, env=env, timeout=60) + except (OSError, subprocess.TimeoutExpired): + return "" + return r.stdout.strip() if r.returncode == 0 else "" + + +class Api: + def __init__(self, secret): + self.secret = secret + self.auth = None + + def _request(self, method, path, data=None, form=None, auth=None): + headers = {"Authorization": auth} if auth else {} + body = None + if form is not None: + body = urllib.parse.urlencode(form).encode() + elif data is not None: + body = json.dumps(data).encode() + headers["Content-Type"] = "application/json" + req = urllib.request.Request(API + path, data=body, method=method, headers=headers) + try: + with urllib.request.urlopen(req, timeout=30) as r: + raw = r.read() + except urllib.error.HTTPError as e: + if e.code == 401: + raise AuthError(f"Tailscale rejected the credential (HTTP 401): {e.read().decode()[:120]}") from None + raise + return json.loads(raw) if raw else {} + + def _bearer(self): + if self.auth: + return self.auth + if self.secret.startswith("tskey-client-"): + cid = self.secret.split("-")[2] + tok = self._request("POST", "/oauth/token", form={"client_id": cid, "client_secret": self.secret}) + self.auth = "Bearer " + tok["access_token"] + else: # classic API key: HTTP basic with an empty password + import base64 + self.auth = "Basic " + base64.b64encode((self.secret + ":").encode()).decode() + return self.auth + + def list_devices(self): + return self._request("GET", "/tailnet/-/devices?fields=all", auth=self._bearer()) + + def delete(self, device_id): + self._request("DELETE", f"/device/{device_id}", auth=self._bearer()) + + def rename(self, device_id, name): + self._request("POST", f"/device/{device_id}/name", data={"name": name}, auth=self._bearer()) + + +AUTH_HELP = ( + "Create an OAuth client at https://login.tailscale.com/admin/settings/oauth (scopes: devices:core write, " + "auth_keys write; tag tag:fleet), then `just seed-tailnet-oauth` to store its secret in Bitwarden item " + f"`{BW_ITEM}` (or export TS_OAUTH_CLIENT_SECRET)." +) + + +def load_devices(source, now): + """(devices, api_or_None, source_label). Falls back from the API to local status for reads.""" + if source in ("auto", "api"): + secret = _secret() + if secret: + api = Api(secret) + try: + me = local_status().get("Self", {}).get("ID") + return devices_from_api(api.list_devices(), now, {me} if me else ()), api, "api" + except AuthError as e: + if source == "api": + raise + print(f"note: {e}\n {AUTH_HELP}\n falling back to local `tailscale status` (read-only)", file=sys.stderr) + elif source == "api": + raise AuthError("no Tailscale credential found. " + AUTH_HELP) + return devices_from_status(local_status(), now), None, "local" + + +def local_status(): + try: + return json.loads(subprocess.run(["tailscale", "status", "--json"], capture_output=True, text=True, timeout=10).stdout) + except (OSError, ValueError, subprocess.TimeoutExpired): + return {} + + +# ── planning (pure) ──────────────────────────────────────────────────────── + +def load_inventory(text): + all_hosts, vps = parse_inventory(text) + hv = parse_host_vars(text) + hosts = {} + for h in sorted(all_hosts - vps): + v = hv.get(h, {}) + tn = v.get("tailnet_name", "").lower() + intended = h.lower() if (not tn or re.fullmatch(rf"{re.escape(h.lower())}-\d+", tn)) else tn + hosts[h] = {"intended": intended, "override": tn, "ansible_host": v.get("ansible_host", "")} + return hosts + + +def _family(devs, name): + return [d for d in devs if not d["foreign"] and (d["name"] == name or re.fullmatch(rf"{re.escape(name)}-\d+", d["name"]))] + + +def _primary(family): + epoch = datetime.min.replace(tzinfo=timezone.utc) + return max(family, key=lambda d: (d["online"], d["seen"] or epoch)) + + +def plan_gc(devs, hosts, now, days=14, k8s_days=None): + """Delete actions: duplicates of an inventory host, stale test VMs, optionally stale k8s proxies.""" + epoch = datetime.min.replace(tzinfo=timezone.utc) + actions, doomed = [], set() + + def add(d, reason): + if d["id"] not in doomed and not d["self"] and not d["online"] and not d["foreign"]: + doomed.add(d["id"]) + actions.append({"op": "delete", "id": d["id"], "name": d["name"], "ip": d["ip"], "reason": reason}) + + inventory_names = {h["intended"] for h in hosts.values()} + for h in hosts.values(): + fam = _family(devs, h["intended"]) + if len(fam) > 1: + primary = _primary(fam) + for d in fam: + if d is not primary and (d["seen"] or epoch) < (primary["seen"] or epoch): + add(d, f"duplicate of {primary['name']}") + for d in devs: + old = d["seen"] is None or now - d["seen"] > timedelta(days=days) + if TEST_VM.search(d["name"]) and d["name"] not in inventory_names and old: + add(d, f"stale test VM (offline > {days}d)") + if k8s_days is not None and any(t.startswith("tag:k8s") for t in d["tags"]): + if d["seen"] is None or now - d["seen"] > timedelta(days=k8s_days): + add(d, f"stale k8s proxy (offline > {k8s_days}d)") + return actions + + +def apply_to_devices(devs, actions): + """The device list as it will be after `actions` (gc deletes, renames).""" + out = [dict(d) for d in devs] + for a in actions: + if a["op"] == "delete": + out = [d for d in out if d["id"] != a["id"]] + elif a["op"] == "rename": + for d in out: + if d["id"] == a["id"]: + d["name"] = a["to"] + return out + + +def plan_rename(devs, hosts): + actions, conflicts = [], [] + for host, h in hosts.items(): + fam = _family(devs, h["intended"]) + if not fam: + continue + primary = _primary(fam) + if primary["name"] == h["intended"]: + continue + holder = [d for d in devs if d["name"] == h["intended"] and not d["foreign"]] + if holder: + conflicts.append(f"{host}: wants {h['intended']} but it is held by {holder[0]['ip']} ({'online' if holder[0]['online'] else 'offline'})") + continue + actions.append({"op": "rename", "id": primary["id"], "from": primary["name"], "to": h["intended"], "host": host}) + return actions, conflicts + + +def plan_inventory(devs, hosts): + """Inventory edits that make inventory.yml agree with the devices as they are in `devs`.""" + actions, notes = [], [] + for host, h in hosts.items(): + fam = _family(devs, h["intended"]) + if not fam: + if h["ansible_host"] not in ("localhost", "127.0.0.1"): + notes.append(f"{host}: no tailnet device named {h['intended']}") + continue + primary = _primary(fam) + if primary["name"] != h["intended"]: + continue # not renamed yet: leave the override and IP alone + if IP100.match(h["ansible_host"]) and primary["ip"] and h["ansible_host"] != primary["ip"]: + actions.append({"op": "inventory", "host": host, "key": "ansible_host", "from": h["ansible_host"], "to": primary["ip"]}) + if h["override"] and h["intended"] == host.lower(): + actions.append({"op": "inventory", "host": host, "key": "tailnet_name", "from": h["override"], "to": None}) + return actions, notes + + +def build_plan(devs, hosts, now, days=14, k8s_days=None): + gc = plan_gc(devs, hosts, now, days, k8s_days) + after_gc = apply_to_devices(devs, gc) + renames, conflicts = plan_rename(after_gc, hosts) + after_rename = apply_to_devices(after_gc, renames) + inv, notes = plan_inventory(after_rename, hosts) + return {"gc": gc, "rename": renames, "inventory": inv, "conflicts": conflicts, "notes": notes} + + +def apply_inventory(text, actions): + for a in actions: + text = set_host_var(text, a["host"], a["key"], a["to"]) + return text + + +# ── CLI ──────────────────────────────────────────────────────────────────── + +def show(plan, kinds): + n = 0 + for a in plan["gc"] if "gc" in kinds else []: + print(f"delete {a['name']:<40} {a['ip']:<16} {a['reason']}") + n += 1 + for a in plan["rename"] if "rename" in kinds else []: + print(f"rename {a['from']} -> {a['to']} (inventory host {a['host']})") + n += 1 + for a in plan["inventory"] if "inventory" in kinds else []: + to = "(remove)" if a["to"] is None else a["to"] + print(f"inventory {a['host']}.{a['key']}: {a['from']} -> {to}") + n += 1 + for c in plan["conflicts"]: + print(f"conflict {c}") + for c in plan["notes"]: + print(f"note {c}") + return n + + +def run_writes(api, plan, kinds, apply_inv): + failed = 0 + for a in plan["gc"] if "gc" in kinds else []: + try: + api.delete(a["id"]) + print(f"deleted {a['name']}") + except (urllib.error.URLError, AuthError) as e: + print(f"FAILED delete {a['name']}: {e}") + failed += 1 + for a in plan["rename"] if "rename" in kinds else []: + try: + api.rename(a["id"], a["to"]) + print(f"renamed {a['from']} -> {a['to']}") + except (urllib.error.URLError, AuthError) as e: + print(f"FAILED rename {a['from']}: {e}") + failed += 1 + if "inventory" in kinds and plan["inventory"] and failed == 0: + apply_inv() + return failed + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("cmd", choices=["audit", "gc", "rename", "fix-inventory", "sync", "release"]) + ap.add_argument("name", nargs="?", help="device name (release)") + ap.add_argument("--apply", action="store_true", help="perform the changes (default: dry run)") + ap.add_argument("--days", type=int, default=14, help="test VMs offline longer than this are stale (default 14)") + ap.add_argument("--k8s-days", type=int, default=None, help="also delete tag:k8s proxies offline longer than this (off by default)") + ap.add_argument("--source", choices=["auto", "api", "local"], default="auto") + args = ap.parse_args(argv) + + now = datetime.now(timezone.utc) + try: + with open(INVENTORY) as f: + text = f.read() + hosts = load_inventory(text) + devs, api, label = load_devices("api" if (args.apply or args.cmd == "release") else args.source, now) + except AuthError as e: + print(f"error: {e}\n {AUTH_HELP}", file=sys.stderr) + return 2 + except urllib.error.URLError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + if args.cmd == "release": + if not args.name: + ap.error("release needs a device name") + name = args.name.lower() + victims = [d for d in _family(devs, name) if not d["self"] and not d["online"]] + for d in victims: + api.delete(d["id"]) + print(f"deleted {d['name']} ({d['ip']})") + return 0 + + plan = build_plan(devs, hosts, now, args.days, args.k8s_days) + kinds = {"audit": {"gc", "rename", "inventory"}, "sync": {"gc", "rename", "inventory"}, "gc": {"gc"}, + "rename": {"rename"}, "fix-inventory": {"inventory"}}[args.cmd] + if args.cmd == "gc": # gc alone: later steps are irrelevant + plan["conflicts"], plan["notes"] = [], [] + if args.cmd == "fix-inventory": # alone it must reflect the live state, not a simulated one + inv, notes = plan_inventory(devs, hosts) + plan = {"gc": [], "rename": [], "inventory": inv, "conflicts": [], "notes": notes} + if args.cmd == "rename": + renames, conflicts = plan_rename(devs, hosts) + plan = {"gc": [], "rename": renames, "inventory": [], "conflicts": conflicts, "notes": []} + + print(f"# {len(devs)} devices from {label}, {len(hosts)} inventory hosts", file=sys.stderr) + count = show(plan, kinds) + if not count and not plan["conflicts"] and not plan["notes"]: + print("tailnet and inventory agree") + if not args.apply or args.cmd == "audit": + return 1 if count or plan["conflicts"] or plan["notes"] else 0 + + def write_inventory(): + with open(INVENTORY, "w") as f: + f.write(apply_inventory(text, plan["inventory"])) + print(f"updated {INVENTORY}") + + if args.cmd == "fix-inventory": + write_inventory() + return 0 + if args.cmd == "sync" and plan["inventory"] and (plan["gc"] or plan["rename"]): + # Inventory edits assume the renames happened; re-plan from the live state afterwards. + failed = run_writes(api, plan, {"gc", "rename"}, lambda: None) + if failed: + return 1 + devs, _, _ = load_devices("api", datetime.now(timezone.utc)) + inv, _ = plan_inventory(devs, hosts) + plan["inventory"] = inv + if inv: + write_inventory() + return 0 + return 1 if run_writes(api, plan, kinds, write_inventory) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_tailnet.py b/tests/test_tailnet.py new file mode 100644 index 00000000..5cd2b61a --- /dev/null +++ b/tests/test_tailnet.py @@ -0,0 +1,127 @@ +import sys +from datetime import datetime, timedelta, timezone + +import pytest + +from conftest import REPO + +sys.path.insert(0, str(REPO / "scripts")) +import tailnet # noqa: E402 +from inventory_parser import parse_host_vars, set_host_var # noqa: E402 + +NOW = datetime(2026, 10, 25, 12, 0, tzinfo=timezone.utc) + +INVENTORY = """\ +--- +all: + hosts: + himachal: + ansible_host: 100.73.3.51 + ansible_connection: ssh + kerala: + ansible_host: 100.96.124.92 # keep this comment + ansible_user: james + tailnet_name: kerala-1 + goa: + ansible_host: localhost + ansible_connection: local + matrix: + ansible_host: 100.73.19.81 + tailnet_name: reilly-asia-matrix + children: + desktop: + hosts: + himachal: + kerala: + vps: + hosts: + matrix: +""" + + +def dev(name, ip, online=False, days_ago=1, tags=(), self_=False, foreign=False): + return {"id": f"id-{name}-{ip}", "name": name, "ip": ip, "tags": list(tags), "online": online, + "seen": NOW if online else NOW - timedelta(days=days_ago), "self": self_, "foreign": foreign} + + +def plan(devs, **kw): + return tailnet.build_plan(devs, tailnet.load_inventory(INVENTORY), NOW, **kw) + + +def test_stale_duplicate_is_deleted_and_live_device_takes_the_name(): + p = plan([dev("himachal", "100.73.3.51", days_ago=24), dev("himachal-1", "100.79.182.70", online=True)]) + assert [a["name"] for a in p["gc"]] == ["himachal"] + assert [(a["from"], a["to"]) for a in p["rename"]] == [("himachal-1", "himachal")] + # inventory follows the live device's address + assert {"op": "inventory", "host": "himachal", "key": "ansible_host", "from": "100.73.3.51", "to": "100.79.182.70"} in p["inventory"] + + +def test_auto_suffix_override_means_the_base_name_and_is_dropped_once_renamed(): + devs = [dev("kerala", "100.67.142.116", days_ago=57), dev("kerala-1", "100.96.124.92", days_ago=1)] + p = plan(devs) + assert [a["name"] for a in p["gc"]] == ["kerala"] + assert [(a["from"], a["to"]) for a in p["rename"]] == [("kerala-1", "kerala")] + assert {"op": "inventory", "host": "kerala", "key": "tailnet_name", "from": "kerala-1", "to": None} in p["inventory"] + # standalone inventory fix must not drop the override before the rename really happened + inv, _ = tailnet.plan_inventory(devs, tailnet.load_inventory(INVENTORY)) + assert inv == [] + + +def test_the_newer_device_is_never_the_one_deleted(): + # the suffixed device is older than the plain-named one: keep the newer, delete the older + p = plan([dev("kerala", "100.1.1.1", days_ago=1), dev("kerala-1", "100.2.2.2", days_ago=30)]) + assert [a["name"] for a in p["gc"]] == ["kerala-1"] + assert p["rename"] == [] + + +def test_online_self_and_foreign_devices_are_never_deleted(): + devs = [dev("himachal", "100.1.1.1", online=True), dev("himachal-1", "100.1.1.2", online=True), + dev("gate-x-vm", "100.1.1.3", days_ago=90, self_=True), dev("gate-y-vm", "100.1.1.4", days_ago=90, foreign=True)] + assert plan(devs)["gc"] == [] + + +def test_test_vms_are_deleted_only_when_stale_and_k8s_proxies_only_on_request(): + devs = [dev("gate-old-vm", "100.1.1.1", days_ago=60, tags=["tag:k8s"]), + dev("gate-new-vm", "100.1.1.2", days_ago=1, tags=["tag:k8s"]), + dev("grafana", "100.1.1.3", days_ago=100, tags=["tag:k8s"])] + assert [a["name"] for a in plan(devs)["gc"]] == ["gate-old-vm"] + assert {a["name"] for a in plan(devs, k8s_days=90)["gc"]} == {"gate-old-vm", "grafana"} + + +def test_two_live_devices_for_one_host_are_left_alone(): + p = plan([dev("kerala", "100.1.1.1", online=True), dev("kerala-1", "100.1.1.2", online=True)]) + assert p["gc"] == [] and p["rename"] == [] + + +def test_retired_vps_hosts_are_ignored(): + p = plan([dev("reilly-asia-matrix", "100.73.19.81", days_ago=50)]) + assert p["gc"] == [] and p["rename"] == [] and p["inventory"] == [] + assert not any("matrix" in n for n in p["notes"]) + + +def test_inventory_edits_keep_comments_and_other_hosts(): + out = tailnet.apply_inventory(INVENTORY, [ + {"host": "kerala", "key": "tailnet_name", "to": None}, + {"host": "kerala", "key": "ansible_host", "to": "100.9.9.9"}, + ]) + assert "tailnet_name: kerala-1" not in out + assert "ansible_host: 100.9.9.9 # keep this comment" in out + assert parse_host_vars(out)["himachal"] == parse_host_vars(INVENTORY)["himachal"] + assert "reilly-asia-matrix" in out + + +def test_set_host_var_appends_a_missing_key_inside_the_block(): + out = set_host_var(INVENTORY, "goa", "tailnet_name", "goa-x") + assert parse_host_vars(out)["goa"]["tailnet_name"] == "goa-x" + assert parse_host_vars(out)["kerala"] == parse_host_vars(INVENTORY)["kerala"] + assert set_host_var(INVENTORY, "nope", "k", "v") == INVENTORY + + +def test_status_normalisation_marks_shared_in_devices_foreign(): + status = {"MagicDNSSuffix": "tn.ts.net", "Self": {"ID": "n1", "DNSName": "me.tn.ts.net.", "TailscaleIPs": ["100.1.1.1"], "Online": True}, + "Peer": {"a": {"ID": "n2", "DNSName": "chuppvm.other.ts.net.", "TailscaleIPs": ["100.1.1.2"], "Online": False, "LastSeen": "2026-04-24T00:00:00Z"}, + "b": {"ID": "n3", "DNSName": "dilli.tn.ts.net.", "TailscaleIPs": ["100.1.1.3"], "Online": False, "LastSeen": "0001-01-01T00:00:00Z", "Tags": ["tag:fleet"]}}} + d = {x["name"]: x for x in tailnet.devices_from_status(status, NOW)} + assert d["me"]["self"] and d["me"]["online"] + assert d["chuppvm"]["foreign"] and not d["dilli"]["foreign"] + assert d["dilli"]["seen"] is None and d["dilli"]["tags"] == ["tag:fleet"] From 81c6e9096bdcc40a166ef1806983bfd4e4d65a01 Mon Sep 17 00:00:00 2001 From: James Reilly Date: Sun, 25 Oct 2026 21:24:22 +0000 Subject: [PATCH 05/62] refactor: move fonts, github, kube and forgejo-registry roles to bash units --- CLAUDE.md | 4 +- CONTEXT.md | 2 +- docs/src/SUMMARY.md | 8 +- docs/src/architecture.md | 12 +- docs/src/index.md | 6 +- docs/src/roles/forgejo_registry.md | 24 ++-- docs/src/roles/github.md | 41 +++--- docs/src/roles/kube.md | 32 +++-- docs/src/roles/shell_fonts.md | 16 +-- docs/src/servers/aws-k8s/cluster.md | 2 +- mise.toml | 16 +++ roles/forgejo_registry/defaults/main.yml | 4 - roles/forgejo_registry/tasks/main.yml | 60 --------- roles/github/tasks/main.yml | 157 ----------------------- roles/kube/defaults/main.yml | 6 - roles/kube/tasks/main.yml | 110 ---------------- roles/shell_fonts/tasks/main.yml | 36 ------ scripts/run-units.sh | 12 +- site.yml | 10 -- tests/test_fonts_unit.py | 40 ++++++ tests/test_forgejo_registry_unit.py | 48 +++++++ tests/test_github_unit.py | 68 ++++++++++ tests/test_kube_unit.py | 70 ++++++++++ tests/unit_helpers.py | 23 ++++ units/chezmoi.sh | 10 +- units/fonts.sh | 31 +++++ units/forgejo-registry.sh | 39 ++++++ units/github.sh | 68 ++++++++++ units/kube.conf | 5 + units/kube.sh | 51 ++++++++ units/lib.sh | 29 ++++- units/shell-ai.sh | 9 -- 32 files changed, 564 insertions(+), 485 deletions(-) delete mode 100644 roles/forgejo_registry/defaults/main.yml delete mode 100644 roles/forgejo_registry/tasks/main.yml delete mode 100644 roles/github/tasks/main.yml delete mode 100644 roles/kube/defaults/main.yml delete mode 100644 roles/kube/tasks/main.yml delete mode 100644 roles/shell_fonts/tasks/main.yml create mode 100644 tests/test_fonts_unit.py create mode 100644 tests/test_forgejo_registry_unit.py create mode 100644 tests/test_github_unit.py create mode 100644 tests/test_kube_unit.py create mode 100644 tests/unit_helpers.py create mode 100755 units/fonts.sh create mode 100755 units/forgejo-registry.sh create mode 100755 units/github.sh create mode 100644 units/kube.conf create mode 100755 units/kube.sh diff --git a/CLAUDE.md b/CLAUDE.md index c13831c8..2fabede9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,10 +50,10 @@ just add-machine NAME # Onboard + bootstrap an already-reachable machine via S `site.yml` runs roles in tagged phases. Notable wiring: - `bitwarden` resolves `BW_SESSION` (env, `/tmp/bw_session`, or interactive unlock), runs `bw sync` with a 15s timeout, and sets a `bw_unlocked` host fact. All BW-using roles gate on `when: bw_unlocked | default(false)`. -- `kube` fetches `~/.kube/config` + `~/.talos/config` from Bitwarden — desktops only. +- the `kube` unit fetches `~/.kube/config` + `~/.talos/config` from Bitwarden — desktops only. - `ssh_keys` round-trips per-machine ed25519 keys through Bitwarden, building cross-machine `authorized_keys`. -After the playbook, `just apply*` runs bash **units** (`units/`, via mise through `scripts/run-units.sh`; see [`docs/src/units.md`](docs/src/units.md)): `omp`, `chezmoi`, `atuin`, `shell-ai`. `home/` is the chezmoi source for shell, git, neovim and atuin config; `ssh_keys` remains an Ansible role. +After the playbook, `just apply*` runs bash **units** (`units/`, via mise through `scripts/run-units.sh`; see [`docs/src/units.md`](docs/src/units.md)): `chezmoi`, `fonts`, `omp`, `atuin`, `shell-ai`, `github`, `kube`, `forgejo-registry`. `home/` is the chezmoi source for shell, git, neovim and atuin config; `ssh_keys` remains an Ansible role. Roles are conditional on inventory group (`desktop`, `server`, `vps`, `test_fleet`, `termux_hosts`; `llm` is dormant/commented out) and on `skip_*` flags / `machine_profile` in `host_vars/`. See [`docs/src/roles/`](docs/src/roles/) for the full reference. diff --git a/CONTEXT.md b/CONTEXT.md index 3ff8f72c..3867b112 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -21,7 +21,7 @@ ## Playbook - **Role** — an Ansible role that configures one concern (e.g. `tailscale`, `proxy`, `ssh_keys`). The interface is `tasks/main.yml`; the implementation is templates, files, and handlers. -- **Unit** — an idempotent bash script in `units/` run in parallel by mise after the playbook (`omp`, `chezmoi`, `atuin`, `shell-ai`). A failing step does not stop the rest of the unit. See `docs/src/units.md`. +- **Unit** — an idempotent bash script in `units/` run in parallel by mise after the playbook (`chezmoi`, `fonts`, `omp`, `atuin`, `shell-ai`, `github`, `kube`, `forgejo-registry`). A failing step does not stop the rest of the unit. See `docs/src/units.md`. - **Phase** — a logical grouping of roles in `site.yml` (Phase 1: system + packages, Phase 2: secrets + auth, Phase 3: desktop, Phase 4: services). Phases are documented in comments, not enforced by the playbook. - **Tag** — an Ansible tag applied to roles so `just apply-tags ` can target a subset. diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index a7ea7562..21c72189 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -60,7 +60,7 @@ ## Dotfiles -- [shell_fonts](roles/shell_fonts.md) +- [shell_fonts (unit)](roles/shell_fonts.md) - [shell_dotfiles (chezmoi)](roles/shell_dotfiles.md) - [shell_atuin (chezmoi + unit)](roles/shell_atuin.md) - [shell_ai (unit)](roles/shell_ai.md) @@ -74,10 +74,10 @@ - [bitwarden](roles/bitwarden.md) - [ssh_keys](roles/ssh_keys.md) - [ssh_mesh](roles/ssh_mesh.md) -- [github](roles/github.md) +- [github (unit)](roles/github.md) - [tailscale](roles/tailscale.md) -- [kube](roles/kube.md) -- [forgejo_registry](roles/forgejo_registry.md) +- [kube (unit)](roles/kube.md) +- [forgejo_registry (unit)](roles/forgejo_registry.md) ## Desktop diff --git a/docs/src/architecture.md b/docs/src/architecture.md index a58fd2c5..703fb22e 100644 --- a/docs/src/architecture.md +++ b/docs/src/architecture.md @@ -62,14 +62,10 @@ Every role in `site.yml` carries one or more tags. Tags fall into two kinds: **p | System + packages | `homebrew` | `packages`, `homebrew` | | System + packages | `termux_packages` | `system`, `packages`, `termux` | | System + packages | `bitwarden` | `secrets`, `bitwarden` | -| System + packages | `shell_fonts` | `dotfiles`, `shell`, `fonts` | | System + packages | `hive_ops` | `hive`, `hive_ops` | | Secrets + auth | `ssh_keys` | `secrets`, `ssh`, `ssh_keys` | | Secrets + auth | `ssh_mesh` | `ssh`, `mesh_check` | -| Secrets + auth | `github` | `secrets`, `github` | | Secrets + auth | `tailscale` | `secrets`, `tailscale` | -| Secrets + auth | `kube` | `secrets`, `kube` | -| Secrets + auth | `forgejo_registry` | `secrets`, `forgejo`, `registry` | | Desktop apps | `flatpak` | `packages`, `desktop`, `flatpak` | | Desktop apps | `bluefin_common` | `desktop`, `bluefin` | | Desktop apps | `gnome` | `desktop`, `gnome` | @@ -85,11 +81,11 @@ Every role in `site.yml` carries one or more tags. Tags fall into two kinds: **p | Services | `server_hardening` | `system`, `hardening`, `security` | | Services | `fleet_facts` | `facts`, `fleet_facts` | -Shell, git, neovim and atuin config, omp, and the shell/atuin secrets are not roles: they are [units](units.md) (`chezmoi`, `omp`, `atuin`, `shell-ai`) that `just apply*` runs after the playbook. The old tags `dotfiles`, `shell`, `git`, `neovim`, `editor`, `atuin`, `ai`, `secrets`, `omp` still select them through `scripts/run-units.sh`. +Shell, git, neovim and atuin config, fonts, omp, the shell/atuin/GitHub/Forgejo secrets and the kubeconfigs are not roles: they are [units](units.md) (`chezmoi`, `fonts`, `omp`, `atuin`, `shell-ai`, `github`, `kube`, `forgejo-registry`) that `just apply*` runs after the playbook. The old tags `dotfiles`, `shell`, `fonts`, `git`, `neovim`, `editor`, `atuin`, `ai`, `secrets`, `github`, `kube`, `forgejo`, `registry`, `omp` still select them through `scripts/run-units.sh`. Common phase-wide selections: -- `--tags secrets` — every BW-backed role only (`bitwarden`, `ssh_keys`, `github`, `tailscale`, `kube`, `forgejo_registry`) +- `--tags secrets` — every BW-backed role only (`bitwarden`, `ssh_keys`, `tailscale`; the `atuin`, `shell-ai`, `github`, `kube` and `forgejo-registry` units also run for this tag) - `--tags desktop` — every desktop-app role (`flatpak`, `bluefin_common`, `gnome`, `zen_browser`, `pipewire_audio`) - `--tags services` — every long-running service role (`syncthing`, `systemd`, `bst_dashboard`, `kirocrew`, `proxy`, `tailscale_cert`) - `--skip-tags secrets` — everything except BW-backed roles. This is `just apply-nosecrets`, which the daily `dotfiles-update.timer` and the `dots` alias run; `bootstrap.sh` and `phone-apply.service` pass `--skip-tags secrets` directly @@ -128,8 +124,8 @@ Every flag `site.yml` reads, all defaulting to `false` (`| default(false)`). Set |------|--------------|---------------------------------|---------------------| | `skip_sudo` | `sudo` | — | `termux` | | `skip_ssh_mesh` | `ssh_mesh` | not `vm-test` | — | -| `skip_kube` | `kube` | desktop, not `vm-test` | `mumbai`, `termux` | -| `skip_forgejo` | `forgejo_registry` | not `vm-test` | — | +| `skip_kube` | `kube` unit | desktop, not `vm-test` | `mumbai`, `termux` | +| `skip_forgejo` | `forgejo-registry` unit | not `vm-test` | — | | `skip_flatpak` | `flatpak` | desktop, not `vm-test` | `mumbai`, `termux` | | `skip_bluefin` | `bluefin_common` | desktop, not `vm-test` | `mumbai`, `termux` | | `skip_gnome` | `gnome` | desktop, not `vm-test` | `mumbai`, `termux` | diff --git a/docs/src/index.md b/docs/src/index.md index 174e2d12..1c87dcae 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -65,12 +65,12 @@ The `llm` group is dormant (commented out in `inventory.yml`). | Phase | Roles | Main tags | |-------|-------|-----------| -| 1 — System + packages + dotfiles | sudo, sshd, apk_packages, homebrew, termux_packages, bitwarden, shell_fonts, hive_ops | `system`, `packages`, `dotfiles` | -| 2 — Secrets + auth | ssh_keys, ssh_mesh, github, tailscale, kube, forgejo_registry | `secrets` | +| 1 — System + packages + dotfiles | sudo, sshd, apk_packages, homebrew, termux_packages, bitwarden, hive_ops | `system`, `packages`, `dotfiles` | +| 2 — Secrets + auth | ssh_keys, ssh_mesh, tailscale | `secrets` | | 3 — Desktop apps | flatpak, bluefin_common, gnome, zen_browser (+ browser_fxa), pipewire_audio | `desktop` | | 4 — Services | syncthing, systemd, bst_dashboard, kirocrew, proxy, tailscale_cert, server_hardening | `services` | -After the playbook, `just apply*` runs the [bash units](units.md) (`chezmoi`, `omp`, `atuin`, `shell-ai`) in parallel through mise. +After the playbook, `just apply*` runs the [bash units](units.md) (`chezmoi`, `fonts`, `omp`, `atuin`, `shell-ai`, `github`, `kube`, `forgejo-registry`) in parallel through mise. Only the `secrets`-tagged work needs Bitwarden. With the vault locked (or `--skip-tags secrets`), phases 1, 3 and 4 run normally and BW-touching tasks diff --git a/docs/src/roles/forgejo_registry.md b/docs/src/roles/forgejo_registry.md index 6b8fc7a0..b300c8bf 100644 --- a/docs/src/roles/forgejo_registry.md +++ b/docs/src/roles/forgejo_registry.md @@ -1,29 +1,21 @@ # forgejo_registry +**Unit:** `forgejo-registry` (`units/forgejo-registry.sh`; see [Units](../units.md)) **Tags:** `secrets`, `forgejo`, `registry` -**Secrets needed:** Yes (Login item named after the registry host) -**Runs on:** All machines except `machine_profile: vm-test` (unless `skip_forgejo: true`) +**Secrets needed:** Yes (login item named after the registry host) +**Runs on:** All machines except `machine_profile: vm-test` or `skip_forgejo: true` in `host_vars` Logs podman in to the private Forgejo container registry. -## What It Does +## What it does -When `bw_unlocked` and `podman` is on `PATH`: - -1. Checks `podman login --get-login {{ forgejo_registry }}` — skips if already logged in -2. Fetches `bw get username` / `bw get password` for the item named `{{ forgejo_registry }}` -3. Runs `podman login … --password-stdin` -4. Warns if the Bitwarden item is missing +1. `SKIPPED:` unless `podman` is installed; no-op when `podman login --get-login ` already succeeds. +2. With an unlocked vault: `bw get username` / `bw get password` for the item named after the registry, then `podman login -u --password-stdin`. A missing item prints `PARTIAL:` and is not a failure. ## Configuration -```yaml -# roles/forgejo_registry/defaults/main.yml -forgejo_registry: forgejo.manatee-basking.ts.net -``` - -The Bitwarden Login item's name must equal this value; its password is a registry/package token. +The registry host is `FORGEJO_REGISTRY` in `units/forgejo-registry.sh` (default `forgejo.manatee-basking.ts.net`, overridable from the environment). The Bitwarden login item's name must equal it; its password is a registry/package token. ## Notes -- Forgejo runs on the home Talos cluster, which is currently powered down — a fresh `podman login` can't succeed until it returns. Hosts already logged in skip the step. +- Forgejo ran on the home cluster, which is powered down, so a fresh `podman login` cannot succeed until it returns. Hosts already logged in skip the step. diff --git a/docs/src/roles/github.md b/docs/src/roles/github.md index b36243e3..9e4a16d9 100644 --- a/docs/src/roles/github.md +++ b/docs/src/roles/github.md @@ -1,38 +1,37 @@ # github -**Tags:** `secrets`, `github` -**Secrets needed:** Yes (only on first auth — once `gh` is authenticated, BW is not required) -**Runs on:** All machines with `gh` installed +**Unit:** `github` (`units/github.sh`; see [Units](../units.md)) +**Tags:** `secrets`, `github` +**Secrets needed:** Only on first auth; once `gh` is authenticated Bitwarden is not used +**Runs on:** All machines with `gh` installed (`SKIPPED:` otherwise) -Logs `gh` in via a BW-stored PAT (only if not already authenticated), then registers this host's SSH key with GitHub for both auth and signing. +Logs `gh` in via a Bitwarden-stored PAT (only if not already authenticated), then registers this host's SSH key with GitHub for both authentication and signing. -## What it does +## Steps -1. Ensures `github.com`'s ed25519 host key is in `~/.ssh/known_hosts` (prevents MITM warnings on first push). -2. `gh auth status` — if already authenticated, skip the token fetch entirely (no BW dependency for steady-state). -3. If not authenticated and BW is unlocked → `bw get password github-token` → `gh auth login --with-token`. If the BW item is missing, emits a one-line warning and continues. -4. Sets `gh config set git_protocol ssh`. -5. Lists existing keys on the GitHub account. For each key type (auth + signing), runs `gh ssh-key add` if this host's key isn't already there. +1. `github known_hosts`: ensure github.com's ed25519 host key is in `~/.ssh/known_hosts`. +2. `gh login`: skipped when `gh auth status` succeeds. Otherwise, with an unlocked vault, `bw get password github-token` → `gh auth login --with-token`. Locked vault or missing item prints a `PARTIAL:` line. +3. `git protocol ssh` (soft): `gh config set git_protocol ssh`. +4. `register SSH keys`: needs a token with `admin:public_key` and `admin:ssh_signing_key` (otherwise a `PARTIAL:` line with the regenerate-PAT link). For each of authentication and signing, `gh ssh-key add ~/.ssh/id_ed25519.pub --title -auth|-signing` unless `gh ssh-key list` already shows the key with that type. ## Failure modes | Symptom | Cause | Fix | |---|---|---| -| `gh auth status` says `not logged in` after apply, vault locked | First run on this host with BW locked | Unlock BW and re-run `just apply-tags github`, or `gh auth login` manually | -| Warning: `No 'github-token' item found` | BW vault doesn't have the PAT | Create a BW *login* item named `github-token` with your PAT as the password | -| Key registration silently fails | `gh ssh-key add` returned non-zero (failed_when:false) — possibly key was just deleted on github.com side | Inspect with `gh ssh-key list` and re-run the role | -| Stale keys clutter github.com | This host's previous key was rotated; the old gh entry stays forever | Delete the old key manually at — no automated cleanup yet | +| `PARTIAL: gh not logged in; vault locked` | First run on this host with Bitwarden locked | `bw unlock`, then `just apply-tags github`, or `gh auth login` | +| `PARTIAL: no 'github-token' item` | Vault has no PAT | Create a login item `github-token` with the PAT as the password | +| `PARTIAL: GitHub token lacks admin:public_key / admin:ssh_signing_key` | PAT scopes | Regenerate the PAT with those scopes and update the item | +| `warn: failed to upload SSH ... key` | `gh ssh-key add` failed | Inspect with `gh ssh-key list` and re-run | -## How to verify +## Verify ```bash -gh auth status # Should say "Logged in" with this account -gh ssh-key list | grep "$(hostname)" # Should show -auth and -signing -git log --show-signature -1 # Latest commit should show "Good \"git\" signature for ..." +gh auth status +gh ssh-key list | grep "$(cat /etc/dotfiles-machine)" # -auth and -signing ``` ## Notes -- Depends on the `ssh_keys` role running first (the key must exist on disk). -- The signing key registration enables the "Verified" badge on commits — only if your local `~/.gitconfig` is also set to sign with the SSH key (the `git` role handles this). -- The PAT in BW needs `admin:public_key`, `write:gpg_key`, and `read:user` scopes for the registrations to succeed. +- Needs `~/.ssh/id_ed25519.pub`, written by the `ssh_keys` role; the units run after the playbook, so a first apply gets both in one pass. +- The "Verified" badge also needs the signing config in `~/.gitconfig` (the chezmoi-managed [git](git.md) config). +- Stale keys from rotated machine keys are not cleaned up: delete them at . diff --git a/docs/src/roles/kube.md b/docs/src/roles/kube.md index d6dd5028..baa75eb7 100644 --- a/docs/src/roles/kube.md +++ b/docs/src/roles/kube.md @@ -1,32 +1,30 @@ # kube -**Tags:** `secrets`, `kube` -**Secrets needed:** Yes -**Runs on:** Desktop group only (unless `skip_kube: true`) +**Unit:** `kube` (`units/kube.sh`; see [Units](../units.md)) +**Tags:** `secrets`, `kube` +**Secrets needed:** Yes +**Runs on:** Desktops only (`SKIPPED:` otherwise, or with `skip_kube: true` / `machine_profile: vm-test` in `host_vars`) -Pulls the kube + Talos configs for both Talos clusters from BW so any desktop in the fleet can talk to them: the home cluster (Bihar + Karnataka — powered down, expected back) and the [AWS cluster](../servers/aws-k8s/cluster.md). +Pulls the kube + Talos configs for both clusters from Bitwarden so any desktop in the fleet can talk to them: the home cluster (Bihar + Karnataka, powered down, expected back) and the [AWS cluster](../servers/aws-k8s/cluster.md) (its nodes were terminated on 2026-10-25; the configs are kept). -## What it does +## Steps -When `bw_unlocked` is true: +With an unlocked vault (a locked vault or `--no-secrets` prints `SKIPPED:` and leaves existing files alone): -1. Ensures `~/.kube/` and `~/.talos/` exist (mode 0700). -2. `bw get notes kubeconfig` → writes to `~/.kube/config` (0600). Skipped if BW returns empty. -3. `bw get notes talosconfig` → writes to `~/.talos/config` (0600). Skipped if BW returns empty. -4. Pins `tls-server-name` on every cluster in `~/.kube/config` to `kube_tls_server_name` (default: bihar's LAN address) — the home API-server cert lacks the tailnet address in its SANs. -5. Emits a one-line warning if either home-cluster note is missing in BW. -6. `bw get notes kubeconfig-aws-migration` → `~/.kube/config-aws-migration` and `bw get notes talosconfig-aws-migration` → `~/.talos/config-aws-migration` (0600, each skipped if empty). Kept as separate files so both clusters stay independently usable. - -When the vault is locked the role no-ops cleanly — existing local configs are preserved. +1. `kube dirs`: `~/.kube/` and `~/.talos/` (0700). +2. `home kubeconfig`: `bw get notes kubeconfig` → `~/.kube/config` (0600). A missing note prints `warn:` and leaves the file as it is. +3. `home talosconfig`: `bw get notes talosconfig` → `~/.talos/config` (0600). +4. `pin tls-server-name`: after writing the home kubeconfig, every cluster in it gets `--tls-server-name=$KUBE_TLS_SERVER_NAME` (`units/kube.conf`, default bihar's LAN address `192.168.0.5`): the home API-server cert lacks the tailnet address in its SANs. +5. `aws kubeconfig` / `aws talosconfig`: `kubeconfig-aws-migration` → `~/.kube/config-aws-migration`, `talosconfig-aws-migration` → `~/.talos/config-aws-migration`. Separate files keep both clusters independently usable. ## Failure modes | Symptom | Cause | Fix | |---|---|---| | `kubectl` returns `Unable to connect to the server` after onboarding | Vault was locked, configs never deployed | Unlock BW, re-run `just apply-tags kube` | -| Warning: `kubeconfig missing in BW` | Vault was never seeded | Run `just seed-kube` from a host that already has a working config | +| `warn: kubeconfig ... missing in BW` | Vault was never seeded | Run `just seed-kube` from a host that already has a working config | | Configs deployed but auth fails | Cluster PKI rotated — local configs are stale | Re-run `just seed-kube` from a healthy host, then re-apply on consumers | -| Role skipped silently on a server host that should have it | `skip_kube: true` in `host_vars/.yml`, or host not in `desktop` group | Adjust inventory or host_vars | +| Unit prints `SKIPPED: not a desktop` on a host that should have it | `skip_kube: true` in `host_vars/.yml`, or host not in the `desktop` group | Adjust inventory or host_vars | ## Seeding BW @@ -48,4 +46,4 @@ KUBECONFIG=~/.kube/config-aws-migration kubectl get nodes # AWS cluster ## Notes - Both files contain cluster PKI material; the repo's `.gitignore` excludes the cluster source files, and these caches live in `~/` only — never commit. -- The role is gated by `is_desktop | bool and not skip_kube`. Servers don't get a kubeconfig — the cluster nodes themselves are Talos and not in the inventory. +- Gated by desktop membership and `skip_kube`. Servers don't get a kubeconfig; the cluster nodes are not in the inventory. diff --git a/docs/src/roles/shell_fonts.md b/docs/src/roles/shell_fonts.md index f794d80e..ad9fe917 100644 --- a/docs/src/roles/shell_fonts.md +++ b/docs/src/roles/shell_fonts.md @@ -1,19 +1,19 @@ # shell_fonts +**Unit:** `fonts` (`units/fonts.sh`; see [Units](../units.md)) **Tags:** `dotfiles`, `shell`, `fonts` **Secrets needed:** No -**Runs on:** All machines (font install only when `is_desktop`) +**Runs on:** All machines (the font install only on desktops) Installs the [JetBrains Mono Nerd Font](https://github.com/ryanoasis/nerd-fonts) for terminals and the prompt. -## What It Does +## Steps -1. Ensures `~/.local/share/fonts/` exists (every host) -2. On desktops, if `JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf` is missing: - downloads the latest `JetBrainsMono.zip` release, unpacks it into - `~/.local/share/fonts/JetBrainsMono/`, and runs `fc-cache -f` +1. `font directory`: `~/.local/share/fonts/` (every host). +2. On desktops, if `JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf` is missing: `install JetBrains Mono Nerd Font` downloads the latest `JetBrainsMono.zip` release into a temp dir, unpacks it into `~/.local/share/fonts/JetBrainsMono/`, then `font cache` (soft) runs `fc-cache -f`. ## Notes -- Idempotent on the presence of the Regular `.ttf` — delete it to force a re-download -- The `gnome` role sets Ptyxis to `JetBrainsMono Nerd Font 12`, which depends on this +- Idempotent on the presence of the Regular `.ttf`; delete it to force a re-download. +- Desktop means: `DOTFILES_IS_DESKTOP`, else `is_desktop` in `host_vars`, else membership of the `desktop` group in `inventory.yml`. +- The `gnome` role sets Ptyxis to `JetBrainsMono Nerd Font 12`, which depends on this. diff --git a/docs/src/servers/aws-k8s/cluster.md b/docs/src/servers/aws-k8s/cluster.md index 9c5c5292..7685b278 100644 --- a/docs/src/servers/aws-k8s/cluster.md +++ b/docs/src/servers/aws-k8s/cluster.md @@ -142,7 +142,7 @@ talosctl --talosconfig ~/.talos/config-aws-migration \ ``` Configs live in Bitwarden as `kubeconfig-aws-migration` / `talosconfig-aws-migration` -and are fetched by the [`kube`](../../roles/kube.md) role into +and are fetched by the [`kube`](../../roles/kube.md) unit into `~/.kube/config-aws-migration` and `~/.talos/config-aws-migration` — kept separate from the home cluster's files so both stay independently usable. diff --git a/mise.toml b/mise.toml index 267f10f8..e180fb14 100644 --- a/mise.toml +++ b/mise.toml @@ -17,3 +17,19 @@ file = "units/atuin.sh" [tasks."unit:shell-ai"] description = "shell-ai: API secrets from Bitwarden into the shell secret files" file = "units/shell-ai.sh" + +[tasks."unit:fonts"] +description = "fonts: JetBrains Mono Nerd Font on desktops" +file = "units/fonts.sh" + +[tasks."unit:github"] +description = "github: gh login from Bitwarden and SSH key registration" +file = "units/github.sh" + +[tasks."unit:kube"] +description = "kube: kubeconfig and talosconfig for the home and AWS clusters from Bitwarden" +file = "units/kube.sh" + +[tasks."unit:forgejo-registry"] +description = "forgejo-registry: podman login to the private Forgejo registry" +file = "units/forgejo-registry.sh" diff --git a/roles/forgejo_registry/defaults/main.yml b/roles/forgejo_registry/defaults/main.yml deleted file mode 100644 index 057ae3af..00000000 --- a/roles/forgejo_registry/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -# Forgejo container registry on the tailnet. The matching Bitwarden login item -# (name == forgejo_registry) holds the username + a registry/package token. -forgejo_registry: forgejo.manatee-basking.ts.net diff --git a/roles/forgejo_registry/tasks/main.yml b/roles/forgejo_registry/tasks/main.yml deleted file mode 100644 index e3541c86..00000000 --- a/roles/forgejo_registry/tasks/main.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# Automate `podman login` to the private Forgejo container registry using the -# credential stored in Bitwarden (item name == {{ forgejo_registry }}). -# Idempotent: skips if already logged in. Gated on an unlocked vault. - -- name: Set up Forgejo registry login - when: bw_unlocked | default(false) - environment: - BW_SESSION: "{{ bw_session | default('') }}" - PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_facts['env']['PATH'] }}" - block: - - name: Check podman is available - ansible.builtin.shell: command -v podman - register: _podman_bin - changed_when: false - failed_when: false - - - name: Check existing Forgejo registry login - ansible.builtin.command: "podman login --get-login {{ forgejo_registry }}" - register: _fj_login - changed_when: false - failed_when: false - when: _podman_bin.rc == 0 - - - name: Fetch Forgejo registry username from Bitwarden - ansible.builtin.command: "bw get username {{ forgejo_registry }}" - register: _fj_user - changed_when: false - no_log: true - failed_when: false - when: _podman_bin.rc == 0 and _fj_login.rc != 0 - - - name: Fetch Forgejo registry password from Bitwarden - ansible.builtin.command: "bw get password {{ forgejo_registry }}" - register: _fj_pass - changed_when: false - no_log: true - failed_when: false - when: _podman_bin.rc == 0 and _fj_login.rc != 0 - - - name: Podman login to Forgejo registry - ansible.builtin.command: - cmd: "podman login {{ forgejo_registry }} -u {{ _fj_user.stdout | trim }} --password-stdin" - stdin: "{{ _fj_pass.stdout }}" - register: _fj_do - changed_when: "'Login Succeeded' in _fj_do.stdout" - no_log: true - when: - - _podman_bin.rc == 0 - - _fj_login.rc != 0 - - _fj_user.rc == 0 and (_fj_user.stdout | length) > 0 - - _fj_pass.rc == 0 and (_fj_pass.stdout | length) > 0 - - - name: Warn if Forgejo credential missing in Bitwarden - ansible.builtin.debug: - msg: "Forgejo registry credential ({{ forgejo_registry }}) not found in Bitwarden — add a login item named '{{ forgejo_registry }}' with the registry token." - when: - - _podman_bin.rc == 0 - - _fj_login.rc != 0 - - (_fj_user.rc | default(1)) != 0 or (_fj_pass.rc | default(1)) != 0 diff --git a/roles/github/tasks/main.yml b/roles/github/tasks/main.yml deleted file mode 100644 index 1e1af3cc..00000000 --- a/roles/github/tasks/main.yml +++ /dev/null @@ -1,157 +0,0 @@ ---- -- name: Check if gh CLI exists - ansible.builtin.stat: - path: "{{ brew_prefix }}/bin/gh" - register: gh_binary - -- name: GitHub authentication and key registration - when: gh_binary.stat.exists - environment: - PATH: "{{ ansible_env_path }}" - BW_SESSION: "{{ bw_session | default('') }}" - block: - # ── Authentication ── - - name: Ensure GitHub host key is in known_hosts - ansible.builtin.known_hosts: - name: github.com - key: "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" - state: present - - - name: Check gh auth status - ansible.builtin.command: gh auth status - register: gh_auth_status - changed_when: false - failed_when: false - - - name: Authenticate with GitHub via Bitwarden token - when: - - gh_auth_status.rc != 0 - - bw_unlocked | default(false) - block: - - name: Fetch GitHub token from Bitwarden - ansible.builtin.command: bw get password github-token - register: gh_token - changed_when: false - failed_when: false - no_log: true - - - name: Warn if GitHub token not found in Bitwarden - ansible.builtin.debug: - msg: >- - No 'github-token' item found in Bitwarden — skipping GitHub auth. - Add a login item named 'github-token' with your PAT as the password, - then re-run 'just apply'. - when: gh_token.rc != 0 or (gh_token.stdout | default('')) | length == 0 - - - name: Login to GitHub with token - ansible.builtin.command: gh auth login --with-token - args: - stdin: "{{ gh_token.stdout }}" - no_log: true - when: - - gh_token.rc == 0 - - (gh_token.stdout | default('')) | length > 0 - - - name: Set git protocol to SSH - ansible.builtin.command: gh config set git_protocol ssh - changed_when: false - - # ── SSH key registration ───────────────────────────────────── - # Checks token scopes upfront so missing permissions produce a - # clear warning instead of a silent HTTP 404 on gh ssh-key add. - - name: Check GitHub token scopes for key management - ansible.builtin.shell: | - set -o pipefail - gh auth status -t 2>&1 | grep -o "Token scopes:.*" || true - register: gh_token_scopes - changed_when: false - failed_when: false - - # Use a fact gate rather than separate when:/failed_when: on each - # downstream task so the scope check runs once and the result - # controls both the gh ssh-key list call and the two upload tasks - # uniformly — otherwise a scope-less token gets two silent 404s. - - name: Set key-scope fact - ansible.builtin.set_fact: - gh_has_key_scopes: >- - {{ ("admin:public_key" in gh_token_scopes.stdout | default("")) - and ("admin:ssh_signing_key" in gh_token_scopes.stdout | default("")) }} - - - name: Warn about missing GitHub token scopes for SSH key upload - ansible.builtin.debug: - msg: >- - ⚠ GitHub token is missing scopes required to upload SSH keys. - Current scopes: {{ gh_token_scopes.stdout | default('unknown') }} - Missing: admin:public_key, admin:ssh_signing_key - → Regenerate the PAT at https://github.com/settings/tokens - with those scopes, then update the 'github-token' Bitwarden item. - when: not gh_has_key_scopes - - - name: Read SSH public key - ansible.builtin.slurp: - src: "{{ ansible_facts['env']['HOME'] }}/.ssh/id_ed25519.pub" - register: ssh_pub_key_raw - - - name: Set SSH public key facts - ansible.builtin.set_fact: - ssh_pub_key_full: "{{ ssh_pub_key_raw.content | b64decode | trim }}" - ssh_key_body: "{{ (ssh_pub_key_raw.content | b64decode).split()[1] }}" - - # Gate on gh_has_key_scopes to avoid the HTTP 404 that gh returns - # when the token lacks admin:public_key / admin:ssh_signing_key. - # Keeps failed_when: false because a transient GitHub API glitch - # shouldn't fail the whole play — the follow-up warn task surfaces it. - - name: List existing GitHub SSH keys - ansible.builtin.command: gh ssh-key list - register: gh_existing_keys - changed_when: false - failed_when: false - when: gh_has_key_scopes | bool - - # Non-TTY `gh ssh-key list` is tab-separated: TITLE KEY ADDED ID TYPE, - # so the same key body can appear once per type; check each separately. - - name: Set existing GitHub key facts - ansible.builtin.set_fact: - gh_key_lines: "{{ gh_existing_keys.stdout_lines | default([]) | select('search', ssh_key_body | regex_escape) | list }}" - - - name: Register SSH authentication key with GitHub - ansible.builtin.command: > - gh ssh-key add "{{ ansible_facts['env']['HOME'] }}/.ssh/id_ed25519.pub" - --title "{{ inventory_hostname }}-auth" - --type authentication - when: - - gh_has_key_scopes | bool - - gh_key_lines | select('search', 'authentication$') | list | length == 0 - register: gh_auth_key_result - failed_when: false - - - name: Warn if SSH auth key upload failed - ansible.builtin.debug: - msg: >- - ⚠ Failed to upload SSH auth key for {{ inventory_hostname }} to GitHub. - stderr: {{ gh_auth_key_result.stderr | default('') }} - when: - - gh_has_key_scopes | bool - - gh_auth_key_result is defined - - gh_auth_key_result.rc | default(0) != 0 - - - name: Register SSH signing key with GitHub - ansible.builtin.command: > - gh ssh-key add "{{ ansible_facts['env']['HOME'] }}/.ssh/id_ed25519.pub" - --title "{{ inventory_hostname }}-signing" - --type signing - when: - - gh_has_key_scopes | bool - - gh_key_lines | select('search', 'signing$') | list | length == 0 - register: gh_sign_key_result - failed_when: false - - - name: Warn if SSH signing key upload failed - ansible.builtin.debug: - msg: >- - ⚠ Failed to upload SSH signing key for {{ inventory_hostname }} to GitHub. - stderr: {{ gh_sign_key_result.stderr | default('') }} - when: - - gh_has_key_scopes | bool - - gh_sign_key_result is defined - - gh_sign_key_result.rc | default(0) != 0 diff --git a/roles/kube/defaults/main.yml b/roles/kube/defaults/main.yml deleted file mode 100644 index 20aa35e7..00000000 --- a/roles/kube/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# Name the Talos API-server cert is actually valid for (a SAN entry). kubeconfig -# dials the control plane over the tailnet IP but verifies TLS against this, so -# `kubectl` works from any tailnet machine without disabling cert verification. -# Control-plane (bihar) LAN IP — present in the API-server cert SANs. -kube_tls_server_name: "192.168.0.5" diff --git a/roles/kube/tasks/main.yml b/roles/kube/tasks/main.yml deleted file mode 100644 index 782c110e..00000000 --- a/roles/kube/tasks/main.yml +++ /dev/null @@ -1,110 +0,0 @@ ---- -# Fetches kubeconfig + talosconfig from Bitwarden and writes them locally. -# Seed Bitwarden with `scripts/bw-seed-kube.sh` from a machine that already -# has working configs. -# Gated on `bw_unlocked` (set by the bitwarden role). - -- name: Deploy kube + talos configs - when: bw_unlocked | default(false) - environment: - BW_SESSION: "{{ bw_session }}" - PATH: "{{ ansible_env_path }}" - block: - - name: Ensure ~/.kube and ~/.talos exist - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/{{ item }}" - state: directory - mode: "0700" - loop: - - .kube - - .talos - - - name: Fetch kubeconfig from Bitwarden - ansible.builtin.command: bw get notes kubeconfig - register: _kubeconfig - changed_when: false - no_log: true - failed_when: false - - - name: Fetch talosconfig from Bitwarden - ansible.builtin.command: bw get notes talosconfig - register: _talosconfig - changed_when: false - no_log: true - failed_when: false - - - name: Write kubeconfig - ansible.builtin.copy: - content: "{{ _kubeconfig.stdout }}" - dest: "{{ ansible_facts['env']['HOME'] }}/.kube/config" - mode: "0600" - when: _kubeconfig.rc == 0 and (_kubeconfig.stdout | length) > 0 - no_log: true - - - name: Write talosconfig - ansible.builtin.copy: - content: "{{ _talosconfig.stdout }}" - dest: "{{ ansible_facts['env']['HOME'] }}/.talos/config" - mode: "0600" - when: _talosconfig.rc == 0 and (_talosconfig.stdout | length) > 0 - no_log: true - - # The kubeconfig connects to the control-plane over its *tailnet* IP, but the - # Talos API-server cert only lists the LAN IP / loopback in its SANs — so TLS - # verification fails ("certificate is valid for …, not 100.x.x.x"). Pin - # tls-server-name to a name the cert IS valid for: kubectl still dials the - # tailnet IP but verifies the cert against this SAN. (Alternative long-term - # fix: add the tailnet IP to machine.certSANs in the Talos config + regen.) - - name: List kubeconfig clusters - ansible.builtin.command: kubectl config get-clusters - register: _clusters - changed_when: false - when: _kubeconfig.rc == 0 and (_kubeconfig.stdout | length) > 0 - - - name: Pin kubeconfig tls-server-name to a valid cert SAN - ansible.builtin.command: - cmd: kubectl config set-cluster {{ item }} --tls-server-name={{ kube_tls_server_name }} - loop: "{{ _clusters.stdout_lines[1:] | default([]) }}" # skip the NAME header row - changed_when: false - when: _kubeconfig.rc == 0 and (_kubeconfig.stdout | length) > 0 - - - name: Warn if either config missing in Bitwarden - ansible.builtin.debug: - msg: >- - {% if _kubeconfig.rc != 0 %}kubeconfig missing in BW — seed it with scripts/bw-seed-kube.sh.{% endif %} - {% if _talosconfig.rc != 0 %}talosconfig missing in BW — seed it with scripts/bw-seed-kube.sh.{% endif %} - when: _kubeconfig.rc != 0 or _talosconfig.rc != 0 - - # AWS migration cluster (matrix/hive workloads) — kept as separate files - # rather than merged into the primary configs above, so both clusters stay - # independently usable. To merge contexts ad hoc: - # KUBECONFIG=~/.kube/config:~/.kube/config-aws-migration kubectl config view --flatten - - name: Fetch AWS migration kubeconfig from Bitwarden - ansible.builtin.command: bw get notes kubeconfig-aws-migration - register: _kubeconfig_aws - changed_when: false - no_log: true - failed_when: false - - - name: Fetch AWS migration talosconfig from Bitwarden - ansible.builtin.command: bw get notes talosconfig-aws-migration - register: _talosconfig_aws - changed_when: false - no_log: true - failed_when: false - - - name: Write AWS migration kubeconfig - ansible.builtin.copy: - content: "{{ _kubeconfig_aws.stdout }}" - dest: "{{ ansible_facts['env']['HOME'] }}/.kube/config-aws-migration" - mode: "0600" - when: _kubeconfig_aws.rc == 0 and (_kubeconfig_aws.stdout | length) > 0 - no_log: true - - - name: Write AWS migration talosconfig - ansible.builtin.copy: - content: "{{ _talosconfig_aws.stdout }}" - dest: "{{ ansible_facts['env']['HOME'] }}/.talos/config-aws-migration" - mode: "0600" - when: _talosconfig_aws.rc == 0 and (_talosconfig_aws.stdout | length) > 0 - no_log: true diff --git a/roles/shell_fonts/tasks/main.yml b/roles/shell_fonts/tasks/main.yml deleted file mode 100644 index 0648833f..00000000 --- a/roles/shell_fonts/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- name: Ensure font directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/share/fonts" - state: directory - mode: "0755" - -- name: Check if JetBrains Mono Nerd Font is installed - ansible.builtin.stat: - path: "{{ ansible_facts['env']['HOME'] }}/.local/share/fonts/JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf" - register: jb_font - -- name: Install JetBrains Mono Nerd Font - when: not jb_font.stat.exists and is_desktop | bool - block: - - name: Download JetBrains Mono Nerd Font - ansible.builtin.get_url: - url: https://github.com/ryanoasis/nerd-fonts/releases/latest/download/JetBrainsMono.zip - dest: /tmp/JetBrainsMonoNerdFont.zip - mode: "0644" - - - name: Create JetBrainsMono font directory - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/share/fonts/JetBrainsMono" - state: directory - mode: "0755" - - - name: Unarchive JetBrains Mono Nerd Font - ansible.builtin.unarchive: - src: /tmp/JetBrainsMonoNerdFont.zip - dest: "{{ ansible_facts['env']['HOME'] }}/.local/share/fonts/JetBrainsMono" - remote_src: true - - - name: Update font cache - ansible.builtin.command: fc-cache -f - changed_when: false diff --git a/scripts/run-units.sh b/scripts/run-units.sh index ad101ec6..4679ea76 100755 --- a/scripts/run-units.sh +++ b/scripts/run-units.sh @@ -3,14 +3,18 @@ # Usage: scripts/run-units.sh [--no-secrets] [--tags a,b] set -uo pipefail -ALL_UNITS="chezmoi omp atuin shell-ai" # run order for the sequential fallback; keep in step with mise.toml +ALL_UNITS="chezmoi fonts omp atuin shell-ai github kube forgejo-registry" # run order for the sequential fallback; keep in step with mise.toml tasks_for_tag() { case "$1" in # Ansible tag -> units (mirrors the old role tags) - dotfiles) echo "chezmoi omp atuin shell-ai" ;; - shell) echo "chezmoi atuin shell-ai" ;; + dotfiles) echo "chezmoi fonts omp atuin shell-ai" ;; + shell) echo "chezmoi fonts atuin shell-ai" ;; + fonts) echo fonts ;; git|neovim|editor) echo chezmoi ;; atuin) echo atuin ;; ai) echo shell-ai ;; - secrets) echo "atuin shell-ai" ;; + secrets) echo "atuin shell-ai github kube forgejo-registry" ;; + github) echo github ;; + kube) echo kube ;; + forgejo|registry) echo forgejo-registry ;; omp) echo omp ;; esac; } diff --git a/site.yml b/site.yml index aff4351a..493a8760 100644 --- a/site.yml +++ b/site.yml @@ -71,8 +71,6 @@ when: inventory_hostname in groups.get('termux_hosts', []) - role: bitwarden tags: [secrets, bitwarden] - - role: shell_fonts - tags: [dotfiles, shell, fonts] - role: hive_ops tags: [hive, hive_ops] @@ -82,16 +80,8 @@ - role: ssh_mesh tags: [ssh, mesh_check] when: machine_profile != 'vm-test' and not skip_ssh_mesh | default(false) - - role: github - tags: [secrets, github] - role: tailscale tags: [secrets, tailscale] - - role: kube - tags: [secrets, kube] - when: is_desktop | bool and machine_profile != 'vm-test' and not skip_kube | default(false) - - role: forgejo_registry - tags: [secrets, forgejo, registry] - when: machine_profile != 'vm-test' and not skip_forgejo | default(false) # Phase 3: Desktop apps - role: flatpak diff --git a/tests/test_fonts_unit.py b/tests/test_fonts_unit.py new file mode 100644 index 00000000..b7c4a020 --- /dev/null +++ b/tests/test_fonts_unit.py @@ -0,0 +1,40 @@ +from pathlib import Path + +from conftest import write_stub +from unit_helpers import calls, run_unit + +CURL = 'echo "curl $*" >> "$HOME/calls"\nout=""; while [ $# -gt 0 ]; do [ "$1" = -o ] && out="$2"; shift; done\n: > "$out"\nexit "${STUB_CURL_RC:-0}"\n' +UNZIP = 'echo "unzip $*" >> "$HOME/calls"\nd=""; while [ $# -gt 0 ]; do [ "$1" = -d ] && d="$2"; shift; done\n: > "$d/JetBrainsMonoNerdFont-Regular.ttf"\n' + + +def setup(stub_env, desktop): + bindir, env = stub_env + write_stub(bindir, "curl", CURL) + write_stub(bindir, "unzip", UNZIP) + write_stub(bindir, "fc-cache", 'echo "fc-cache $*" >> "$HOME/calls"\n') + env["DOTFILES_IS_DESKTOP"] = "1" if desktop else "0" + return env + + +def test_desktop_installs_the_font_once(stub_env): + env = setup(stub_env, True) + r = run_unit("fonts", env) + assert r.returncode == 0, r.stdout + r.stderr + assert (Path(env["HOME"]) / ".local/share/fonts/JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf").exists() + assert "fc-cache -f" in calls(env) + before = calls(env) + assert run_unit("fonts", env).returncode == 0 + assert calls(env) == before # second run downloads nothing + + +def test_non_desktop_downloads_nothing(stub_env): + env = setup(stub_env, False) + r = run_unit("fonts", env) + assert r.returncode == 0 and "curl" not in calls(env) + + +def test_failed_download_fails_the_unit_without_a_partial_install(stub_env): + env = setup(stub_env, True) + r = run_unit("fonts", env, STUB_CURL_RC="22") + assert r.returncode == 1 and "FAIL install JetBrains Mono Nerd Font" in r.stdout + assert not (Path(env["HOME"]) / ".local/share/fonts/JetBrainsMono").exists() diff --git a/tests/test_forgejo_registry_unit.py b/tests/test_forgejo_registry_unit.py new file mode 100644 index 00000000..47ac73b9 --- /dev/null +++ b/tests/test_forgejo_registry_unit.py @@ -0,0 +1,48 @@ +import pytest + +from conftest import write_stub +from unit_helpers import UNLOCKED_BW, calls, run_unit + +CREDS = """\ +if [ "$1" = get ]; then + case "$2" in + username) echo " ci-user "; exit 0 ;; + password) [ -n "$STUB_NO_PASS" ] && exit 1; echo "tok123"; exit 0 ;; + esac +fi +""" + + +@pytest.fixture +def env(stub_env): + bindir, env = stub_env + write_stub(bindir, "bw", UNLOCKED_BW % CREDS) + write_stub(bindir, "podman", 'echo "podman $*" >> "$HOME/calls"\ncase "$1 $2" in "login --get-login") exit "${STUB_LOGGED_IN:-1}" ;; esac\ncat >> "$HOME/stdin" 2>/dev/null\nexit 0\n') + env["BW_SESSION"] = "x" + return env + + +def test_logs_in_with_the_bitwarden_credential_on_stdin(env): + from pathlib import Path + + r = run_unit("forgejo-registry", env) + assert r.returncode == 0, r.stdout + r.stderr + assert "podman login forgejo.manatee-basking.ts.net -u ci-user --password-stdin" in calls(env) + assert Path(env["HOME"], "stdin").read_text() == "tok123" + assert "tok123" not in calls(env) + r.stdout + r.stderr + + +def test_already_logged_in_never_touches_bitwarden(env): + r = run_unit("forgejo-registry", env, STUB_LOGGED_IN="0") + assert r.returncode == 0 and "bw " not in calls(env) and "--password-stdin" not in calls(env) + + +def test_missing_credential_is_partial_not_a_failure(env): + r = run_unit("forgejo-registry", env, STUB_NO_PASS="1") + assert r.returncode == 0 and "PARTIAL:" in r.stdout + assert "--password-stdin" not in calls(env) + + +def test_locked_vault_is_skipped(env): + r = run_unit("forgejo-registry", env, STUB_UNLOCK_RC="1") + assert r.returncode == 0 and "SKIPPED:" in r.stdout diff --git a/tests/test_github_unit.py b/tests/test_github_unit.py new file mode 100644 index 00000000..00ff3627 --- /dev/null +++ b/tests/test_github_unit.py @@ -0,0 +1,68 @@ +import pytest + +from conftest import write_stub +from unit_helpers import UNLOCKED_BW, calls, run_unit + +PUB = "ssh-ed25519 AAAAKEYBODY james@host\n" + +GH = """\ +echo "gh $*" >> "$HOME/calls" +case "$1 $2" in + "auth status") + [ "$3" = "-t" ] && echo " - Token scopes: $STUB_SCOPES" + [ "$3" = "-t" ] || exit "${STUB_AUTHED:-0}" + exit 0 ;; + "auth login") cat > "$HOME/token"; exit 0 ;; + "ssh-key list") printf '%s' "$STUB_KEYS"; exit 0 ;; +esac +exit 0 +""" + + +@pytest.fixture +def env(stub_env): + bindir, env = stub_env + write_stub(bindir, "bw", UNLOCKED_BW % '[ "$1 $2 $3" = "get password github-token" ] && { echo ghp_secret; exit 0; }') + write_stub(bindir, "gh", GH) + ssh = __import__("pathlib").Path(env["HOME"], ".ssh") + ssh.mkdir() + (ssh / "id_ed25519.pub").write_text(PUB) + env.update(BW_SESSION="x", DOTFILES_MACHINE="testbox", STUB_SCOPES="admin:public_key, admin:ssh_signing_key", STUB_KEYS="") + return env + + +def test_logs_in_with_the_bitwarden_token_and_registers_both_keys(env): + from pathlib import Path + + r = run_unit("github", env, STUB_AUTHED="1") + assert r.returncode == 0, r.stdout + r.stderr + assert Path(env["HOME"], "token").read_text() == "ghp_secret" + c = calls(env) + assert "ssh-key add" in c and "--title testbox-auth --type authentication" in c and "--title testbox-signing --type signing" in c + assert "git_protocol ssh" in c + assert "AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" in Path(env["HOME"], ".ssh/known_hosts").read_text() + + +def test_registered_keys_are_not_added_again(env): + keys = "testbox-auth\tssh-ed25519 AAAAKEYBODY\t2026-01-01\t1\tauthentication\ntestbox-signing\tssh-ed25519 AAAAKEYBODY\t2026-01-01\t2\tsigning\n" + r = run_unit("github", env, STUB_KEYS=keys) + assert r.returncode == 0 and "ssh-key add" not in calls(env) + assert "auth login" not in calls(env) # already authenticated + + +def test_only_the_missing_key_type_is_added(env): + keys = "testbox-auth\tssh-ed25519 AAAAKEYBODY\t2026-01-01\t1\tauthentication\n" + run_unit("github", env, STUB_KEYS=keys) + c = calls(env) + assert "--type signing" in c and "--type authentication" not in c + + +def test_token_without_key_scopes_is_partial_and_adds_nothing(env): + r = run_unit("github", env, STUB_SCOPES="repo, read:org") + assert r.returncode == 0 and "PARTIAL:" in r.stdout and "ssh-key add" not in calls(env) + + +def test_locked_vault_does_not_log_in(env): + r = run_unit("github", env, STUB_AUTHED="1", STUB_UNLOCK_RC="1") + assert r.returncode == 0 and "PARTIAL: gh not logged in" in r.stdout + assert "auth login" not in calls(env) diff --git a/tests/test_kube_unit.py b/tests/test_kube_unit.py new file mode 100644 index 00000000..ee8b50e5 --- /dev/null +++ b/tests/test_kube_unit.py @@ -0,0 +1,70 @@ +import stat +from pathlib import Path + +import pytest + +from conftest import write_stub +from unit_helpers import UNLOCKED_BW, calls, run_unit + +NOTES = """\ +if [ "$1 $2" = "get notes" ]; then + case "$3" in + kubeconfig) echo "home-kube"; exit 0 ;; + talosconfig) echo "home-talos"; exit 0 ;; + kubeconfig-aws-migration) echo "aws-kube"; exit 0 ;; + talosconfig-aws-migration) exit 1 ;; + esac +fi +""" + + +@pytest.fixture +def env(stub_env): + bindir, env = stub_env + write_stub(bindir, "bw", UNLOCKED_BW % NOTES) + write_stub(bindir, "kubectl", 'echo "kubectl $*" >> "$HOME/calls"\n[ "$2" = get-clusters ] && printf "NAME\\nhome\\n"\nexit 0\n') + env.update(BW_SESSION="x", DOTFILES_IS_DESKTOP="1") + return env + + +def test_configs_are_written_private_and_the_home_cluster_is_pinned(env): + r = run_unit("kube", env) + assert r.returncode == 0, r.stdout + r.stderr + home = Path(env["HOME"]) + assert (home / ".kube/config").read_text().strip() == "home-kube" + assert (home / ".talos/config").read_text().strip() == "home-talos" + assert (home / ".kube/config-aws-migration").read_text().strip() == "aws-kube" + for f in (".kube/config", ".talos/config", ".kube/config-aws-migration"): + assert stat.S_IMODE((home / f).stat().st_mode) == 0o600 + assert "kubectl config set-cluster home --tls-server-name=192.168.0.5" in calls(env) + + +def test_a_missing_note_warns_and_leaves_no_file(env): + r = run_unit("kube", env) + assert "AWS talosconfig missing in BW" in r.stdout + assert not (Path(env["HOME"]) / ".talos/config-aws-migration").exists() + + +def test_locked_vault_leaves_existing_configs(env): + cfg = Path(env["HOME"], ".kube/config") + cfg.parent.mkdir() + cfg.write_text("previous") + r = run_unit("kube", env, STUB_UNLOCK_RC="1") + assert r.returncode == 0 and "SKIPPED:" in r.stdout + assert cfg.read_text() == "previous" + + +@pytest.mark.parametrize("extra", [{"DOTFILES_IS_DESKTOP": "0"}]) +def test_non_desktops_do_nothing(env, extra): + r = run_unit("kube", env, **extra) + assert "SKIPPED:" in r.stdout + assert not (Path(env["HOME"]) / ".kube").exists() + + +def test_skip_kube_host_var_wins_over_desktop(env, tmp_path): + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "host_vars/phone.yml").write_text("machine_profile: cli-only\nskip_kube: true\n") + (repo / "units").symlink_to(Path(__file__).resolve().parent.parent / "units") + r = run_unit("kube", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="phone") + assert "SKIPPED:" in r.stdout and not (Path(env["HOME"]) / ".kube").exists() diff --git a/tests/unit_helpers.py b/tests/unit_helpers.py new file mode 100644 index 00000000..e1b3679f --- /dev/null +++ b/tests/unit_helpers.py @@ -0,0 +1,23 @@ +"""Shared harness for the bash-unit tests: run units/.sh with stubs and a throwaway HOME.""" +import subprocess +from pathlib import Path + +from conftest import REPO + + +def run_unit(name, env, **extra): + env = {**env, "DOTFILES_DIR": str(REPO), "BREW_PREFIX": "/nonexistent", **extra} + return subprocess.run(["bash", str(REPO / "units" / f"{name}.sh")], env=env, capture_output=True, text=True) + + +def calls(env): + p = Path(env["HOME"], "calls") + return p.read_text() if p.exists() else "" + + +UNLOCKED_BW = """\ +echo "bw $*" >> "$HOME/calls" +[ "$1 $2" = "unlock --check" ] && exit "${STUB_UNLOCK_RC:-0}" +%s +exit 1 +""" diff --git a/units/chezmoi.sh b/units/chezmoi.sh index 2765a44a..dbeb6f88 100755 --- a/units/chezmoi.sh +++ b/units/chezmoi.sh @@ -12,13 +12,9 @@ if ! command -v chezmoi >/dev/null 2>&1; then exit 0 fi -is_desktop=0 -if command -v yq >/dev/null 2>&1; then - [ "$(yq ".all.children.desktop.hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml" 2>/dev/null)" = true ] && is_desktop=1 -else - note "warn: yq missing, assuming non-desktop" -fi -export DOTFILES_MACHINE="$MACHINE" DOTFILES_IS_DESKTOP="$is_desktop" +command -v yq >/dev/null 2>&1 || note "warn: yq missing, assuming non-desktop" +if is_desktop; then desktop=1; else desktop=0; fi +export DOTFILES_MACHINE="$MACHINE" DOTFILES_IS_DESKTOP="$desktop" step "chezmoi apply" chezmoi --source "$DOTFILES_DIR" --keep-going --force --no-tty apply --init finish diff --git a/units/fonts.sh b/units/fonts.sh new file mode 100755 index 00000000..7d114bd4 --- /dev/null +++ b/units/fonts.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# fonts unit: JetBrains Mono Nerd Font for desktop terminals. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +dir="$HOME/.local/share/fonts" +font="$dir/JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf" + +step "font directory" mkdir -p "$dir" +if ! is_desktop; then + note "SKIPPED: not a desktop; no font install" + finish +fi +if [ -f "$font" ]; then + note "ok JetBrains Mono Nerd Font already installed" + finish +fi + +install_font() { + local tmp rc=0 + tmp="$(mktemp -d)" || return 1 + curl -fsSL https://github.com/ryanoasis/nerd-fonts/releases/latest/download/JetBrainsMono.zip -o "$tmp/font.zip" \ + && mkdir -p "$dir/JetBrainsMono" \ + && unzip -oq "$tmp/font.zip" -d "$dir/JetBrainsMono" || rc=1 + rm -rf "$tmp" + return $rc +} +step "install JetBrains Mono Nerd Font" install_font +soft "font cache" fc-cache -f +finish diff --git a/units/forgejo-registry.sh b/units/forgejo-registry.sh new file mode 100755 index 00000000..a46489bf --- /dev/null +++ b/units/forgejo-registry.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# forgejo-registry unit: `podman login` to the private Forgejo container registry from Bitwarden. +# The Bitwarden login item is named after the registry host and holds the username + a registry token. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +FORGEJO_REGISTRY="${FORGEJO_REGISTRY:-forgejo.manatee-basking.ts.net}" + +if skipped_by_profile skip_forgejo; then + note "SKIPPED: skip_forgejo or vm-test" + exit 0 +fi +if ! command -v podman >/dev/null 2>&1; then + note "SKIPPED: podman not installed" + exit 0 +fi +if podman login --get-login "$FORGEJO_REGISTRY" >/dev/null 2>&1; then + note "ok already logged in to $FORGEJO_REGISTRY" + exit 0 +fi +s="$(bw_session)" +if [ -z "$s" ]; then + note "SKIPPED: vault locked or --no-secrets; not logged in to $FORGEJO_REGISTRY" + exit 0 +fi + +registry_login() { + local user pass + user="$(BW_SESSION="$s" bw get username "$FORGEJO_REGISTRY" 2>/dev/null)" + pass="$(BW_SESSION="$s" bw get password "$FORGEJO_REGISTRY" 2>/dev/null)" + if [ -z "$user" ] || [ -z "$pass" ]; then + note "PARTIAL: Forgejo credential ($FORGEJO_REGISTRY) not found in Bitwarden — add a login item with that name holding the registry token" + return 0 + fi + printf '%s' "$pass" | podman login "$FORGEJO_REGISTRY" -u "$(printf '%s' "$user" | tr -d '[:space:]')" --password-stdin +} +step "podman login $FORGEJO_REGISTRY" registry_login +finish diff --git a/units/github.sh b/units/github.sh new file mode 100755 index 00000000..b1e556db --- /dev/null +++ b/units/github.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# github unit: gh login from a Bitwarden token, git over SSH, and this machine's SSH key +# registered with GitHub as an authentication and a signing key. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +GITHUB_HOST_KEY="github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" +pub="$HOME/.ssh/id_ed25519.pub" + +if ! command -v gh >/dev/null 2>&1; then + note "SKIPPED: gh not installed (just packages)" + exit 0 +fi + +known_hosts() { + install -d -m 0700 "$HOME/.ssh" || return 1 + grep -qF "$GITHUB_HOST_KEY" "$HOME/.ssh/known_hosts" 2>/dev/null && return 0 + printf '%s\n' "$GITHUB_HOST_KEY" >> "$HOME/.ssh/known_hosts" +} + +gh_login() { + gh auth status >/dev/null 2>&1 && return 0 + local s tok + s="$(bw_session)" + if [ -z "$s" ]; then + note "PARTIAL: gh not logged in; vault locked (bw unlock, then just apply-tags github)" + return 0 + fi + tok="$(BW_SESSION="$s" bw get password github-token 2>/dev/null)" + if [ -z "$tok" ]; then + note "PARTIAL: no 'github-token' item in Bitwarden; add a login item with your PAT as the password" + return 0 + fi + printf '%s' "$tok" | gh auth login --with-token +} + +# register_key TYPE: add the SSH key as TYPE unless a `... TYPE` line already lists it. +register_key() { + local type="$1" title="$MACHINE-$2" + grep -Eq "${type}\$" <<<"$key_lines" && return 0 + if ! gh ssh-key add "$pub" --title "$title" --type "$type"; then + note "warn: failed to upload SSH $type key for $MACHINE to GitHub" + return 1 + fi +} + +register_keys() { + local scopes key_body + scopes="$(gh auth status -t 2>&1 | grep -o 'Token scopes:.*' || true)" + case "$scopes" in + *admin:public_key*admin:ssh_signing_key*|*admin:ssh_signing_key*admin:public_key*) ;; + *) + note "PARTIAL: GitHub token lacks admin:public_key / admin:ssh_signing_key (have: ${scopes:-unknown}); regenerate the PAT at https://github.com/settings/tokens and update the 'github-token' item" + return 0 ;; + esac + [ -f "$pub" ] || { note "warn: $pub missing; no SSH key to register"; return 1; } + key_body="$(awk '{print $2}' "$pub")" + key_lines="$(gh ssh-key list 2>/dev/null | grep -F "$key_body" || true)" + register_key authentication auth && register_key signing signing +} + +key_lines="" +step "github known_hosts" known_hosts +step "gh login" gh_login +soft "git protocol ssh" gh config set git_protocol ssh +step "register SSH keys" register_keys +finish diff --git a/units/kube.conf b/units/kube.conf new file mode 100644 index 00000000..38f5943a --- /dev/null +++ b/units/kube.conf @@ -0,0 +1,5 @@ +# Sourced by units/kube.sh. +# Name the Talos API-server cert is actually valid for (a SAN entry). kubeconfig dials the +# control plane over the tailnet IP but verifies TLS against this, so `kubectl` works from any +# tailnet machine without disabling cert verification. Control-plane (bihar) LAN IP. +KUBE_TLS_SERVER_NAME="${KUBE_TLS_SERVER_NAME:-192.168.0.5}" diff --git a/units/kube.sh b/units/kube.sh new file mode 100755 index 00000000..3ebfa3e2 --- /dev/null +++ b/units/kube.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# kube unit: kubeconfig + talosconfig for the home and AWS clusters from Bitwarden notes. +# Seed the home notes with scripts/bw-seed-kube.sh. Desktops only. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=units/kube.conf +. "$DOTFILES_DIR/units/kube.conf" + +if ! is_desktop || skipped_by_profile skip_kube; then + note "SKIPPED: not a desktop (or skip_kube / vm-test)" + exit 0 +fi +s="$(bw_session)" +if [ -z "$s" ]; then + note "SKIPPED: vault locked or --no-secrets; existing kube/talos configs left untouched" + exit 0 +fi + +step "kube dirs" install -d -m 0700 "$HOME/.kube" "$HOME/.talos" + +# fetch_note ITEM DEST LABEL: write the note to DEST when Bitwarden has it; prints a warning otherwise. +fetch_note() { + local item="$1" dest="$2" label="$3" val + val="$(BW_SESSION="$s" bw get notes "$item" 2>/dev/null)" + if [ -z "$val" ]; then + note "warn: $label missing in BW (item $item)" + return 0 + fi + printf '%s\n' "$val" | write_atomic "$dest" || return 1 + [ "$dest" != "$HOME/.kube/config" ] || home_written=1 +} + +# The home kubeconfig is rewritten from Bitwarden each run, so the TLS pin is re-applied each time. +pin_tls() { + command -v kubectl >/dev/null 2>&1 || { note "warn: kubectl missing; tls-server-name not pinned"; return 0; } + local c rc=0 + while read -r c; do + [ -n "$c" ] || continue + KUBECONFIG="$HOME/.kube/config" kubectl config set-cluster "$c" --tls-server-name="$KUBE_TLS_SERVER_NAME" >/dev/null || rc=1 + done < <(KUBECONFIG="$HOME/.kube/config" kubectl config get-clusters 2>/dev/null | tail -n +2) + return $rc +} + +home_written=0 +step "home kubeconfig" fetch_note kubeconfig "$HOME/.kube/config" "kubeconfig (seed with scripts/bw-seed-kube.sh)" +step "home talosconfig" fetch_note talosconfig "$HOME/.talos/config" "talosconfig (seed with scripts/bw-seed-kube.sh)" +[ "$home_written" = 0 ] || step "pin tls-server-name" pin_tls +step "aws kubeconfig" fetch_note kubeconfig-aws-migration "$HOME/.kube/config-aws-migration" "AWS kubeconfig" +step "aws talosconfig" fetch_note talosconfig-aws-migration "$HOME/.talos/config-aws-migration" "AWS talosconfig" +finish diff --git a/units/lib.sh b/units/lib.sh index 4f391827..ec78ed7f 100755 --- a/units/lib.sh +++ b/units/lib.sh @@ -25,4 +25,31 @@ bw_session() { [ -n "$s" ] && bw unlock --check --session "$s" >/dev/null 2>&1 && printf '%s' "$s" return 0 } -MACHINE="$(cat /etc/dotfiles-machine 2>/dev/null || hostname)" +MACHINE="${DOTFILES_MACHINE:-$(cat /etc/dotfiles-machine 2>/dev/null || hostname)}" +# host_var KEY [DEFAULT]: KEY from host_vars/$MACHINE.yml (flat `key: value` lines only). +host_var() { + local v + v="$(sed -n "s/^$1:[[:space:]]*\([^#]*[^#[:space:]]\).*/\1/p" "$DOTFILES_DIR/host_vars/$MACHINE.yml" 2>/dev/null | head -n1)" + v="${v//[\"\']/}" + printf '%s' "${v:-${2:-}}" +} +# is_desktop: DOTFILES_IS_DESKTOP, else host_vars `is_desktop`, else membership of the inventory `desktop` group. +is_desktop() { + local o="${DOTFILES_IS_DESKTOP:-}" + [ -n "$o" ] || o="$(host_var is_desktop)" + case "$o" in 1|true) return 0 ;; 0|false) return 1 ;; esac + command -v yq >/dev/null 2>&1 || return 1 + [ "$(yq ".all.children.desktop.hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml" 2>/dev/null)" = true ] +} +# skipped_by_profile FLAG: true when this host opted out (host_vars FLAG: true) or is a vm-test machine. +skipped_by_profile() { + [ "$(host_var "$1")" = true ] || [ "$(host_var machine_profile)" = vm-test ] +} +# write_atomic dest: stdin -> temp file in the same directory (mode 0600) -> mv +write_atomic() { + local dest="$1" tmp + mkdir -p "$(dirname "$dest")" || return 1 + tmp="$(umask 077; mktemp "$dest.XXXXXX")" || return 1 + if cat > "$tmp" && chmod 600 "$tmp" && mv -f "$tmp" "$dest"; then return 0; fi + rm -f "$tmp"; return 1 +} diff --git a/units/shell-ai.sh b/units/shell-ai.sh index 87a0949a..160ce7bb 100755 --- a/units/shell-ai.sh +++ b/units/shell-ai.sh @@ -27,15 +27,6 @@ if [ -z "$deepseek$forgejo$tavily$meta$kiro" ]; then exit 0 fi -# write_atomic dest: stdin -> temp file in the same directory (mode 0600) -> mv -write_atomic() { - local dest="$1" tmp - mkdir -p "$(dirname "$dest")" || return 1 - tmp="$(umask 077; mktemp "$dest.XXXXXX")" || return 1 - if cat > "$tmp" && chmod 600 "$tmp" && mv -f "$tmp" "$dest"; then return 0; fi - rm -f "$tmp"; return 1 -} - secrets_sh() { echo '# Managed by dotfiles playbook — DO NOT EDIT' echo 'export OMP_KIRO_CONFIG="$HOME/.config/omp/kiro.json"' From 47f78a2ee091566a8da4df415a56a11752d40246 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:56:33 +0000 Subject: [PATCH 06/62] fix(omp): target the live spindle broker Read the broker token over Tailscale SSH when seeding Bitwarden, and preserve the vault when the server is unreachable. Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- Justfile | 2 +- docs/src/roles/omp.md | 6 +++--- scripts/bw-seed-omp-broker.sh | 16 ++++++++-------- tests/test_omp_unit.py | 14 +++++++++++++- units/omp.sh | 2 +- units/omp/auth-broker.conf | 6 ++---- 6 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Justfile b/Justfile index 180f1f67..a449dc59 100644 --- a/Justfile +++ b/Justfile @@ -533,7 +533,7 @@ docs-build: seed-kube: @{{ dotfiles_dir }}/scripts/bw-seed-kube.sh -# Seed Bitwarden with the omp auth-broker bearer token (run on the broker host, punjab) +# Seed Bitwarden with the omp auth-broker token via SSH to spindle seed-omp-broker: @{{ dotfiles_dir }}/scripts/bw-seed-omp-broker.sh diff --git a/docs/src/roles/omp.md b/docs/src/roles/omp.md index 37c3ecac..07be64bc 100644 --- a/docs/src/roles/omp.md +++ b/docs/src/roles/omp.md @@ -25,16 +25,16 @@ Each step is independent: a failure prints `FAIL `, fails the unit, and t ## Auth broker -`omp auth-broker` is the fleet's credential vault: provider logins live on one broker, and every other host is a client that holds only a URL and a bearer token. The broker runs on `punjab` as a user unit (`omp-auth-broker.service`, `omp auth-broker serve --bind=:8765`). That unit is created by hand and is **not** managed by this repo. +`omp auth-broker` is the fleet's credential vault: provider logins live on one broker, and every other host is a client that holds only a URL and a bearer token. The broker runs on the OCI `spindle` VPS as a user unit bound to its Tailscale address, `100.113.75.88:8765`. That unit is created by hand and is **not** managed by this repo. | Key in `~/.omp/agent/config.yml` | Source | |---|---| -| `auth.broker.url` | `units/omp/auth-broker.conf` (`OMP_AUTH_BROKER_URL`, punjab's tailnet address, port 8765; keep in step with `inventory.yml`) | +| `auth.broker.url` | `units/omp/auth-broker.conf` (`OMP_AUTH_BROKER_URL`, spindle's Tailscale address, port 8765) | | `auth.broker.token` | Bitwarden password item `omp-auth-broker-token` (`OMP_AUTH_BROKER_BW_ITEM`) | The token is written only when Bitwarden is unlocked; a locked vault or `--no-secrets` leaves the existing token as it is. The config file is `0600`. -To seed the token, run `just seed-omp-broker` **on punjab** with the vault unlocked. It reads `omp auth-broker token` (which does not rotate it) and creates or updates the Bitwarden item. Then run `just apply-tags omp` on each client. Verify with `omp auth-broker status`. If the token is rotated (`omp auth-broker token --regenerate` on punjab), seed again and re-apply on the clients. +To seed the token, run `just seed-omp-broker` on a client with the vault unlocked and SSH access to `ubuntu@100.113.75.88`. It reads the live token over SSH without rotating it and creates or updates the Bitwarden item. `OMP_AUTH_BROKER_SSH` overrides the SSH destination. Then run `just apply-tags omp` on each client and verify with `omp auth-broker status`. If the token is rotated on spindle, seed again and re-apply on the clients. `DEEPSEEK_API_KEY` and the other provider keys are exposed to the shell by the `shell_ai` role's secret files, not by this unit. diff --git a/scripts/bw-seed-omp-broker.sh b/scripts/bw-seed-omp-broker.sh index 845a2974..3c5650c2 100755 --- a/scripts/bw-seed-omp-broker.sh +++ b/scripts/bw-seed-omp-broker.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash -# Seed Bitwarden with the omp auth-broker bearer token. Run on the broker host -# (punjab), where `omp auth-broker token` reads the broker's own token. +# Seed Bitwarden with the live omp auth-broker bearer token over tailnet SSH. +# Run on a client with an unlocked vault and SSH access to ubuntu@spindle. # Usage: ./scripts/bw-seed-omp-broker.sh -# Re-run safely — updates the existing item in place. `omp auth-broker token` -# prints the current token and does not rotate it. Clients fetch the item with -# `just apply-tags omp`. +# Re-run safely — updates the existing item in place. The server command prints +# the current token without rotating it. Clients fetch it with just apply-tags omp. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "$SCRIPT_DIR/bw-item.sh" @@ -17,9 +16,10 @@ fi NAME="${OMP_AUTH_BROKER_BW_ITEM:-omp-auth-broker-token}" -token=$(omp auth-broker token) +broker_host="${OMP_AUTH_BROKER_SSH:-ubuntu@100.113.75.88}" +token=$(ssh -o BatchMode=yes "$broker_host" '/home/ubuntu/.local/bin/omp auth-broker token') if [ -z "$token" ]; then - echo "no token from 'omp auth-broker token' — is this the broker host?" >&2 + echo "no token from broker host $broker_host" >&2 exit 1 fi @@ -29,7 +29,7 @@ payload=$(bw get template item \ . + { name: $name, type: 1, - notes: "Bearer token for the omp auth-broker on punjab. Managed by scripts/bw-seed-omp-broker.sh.", + notes: "Bearer token for the omp auth-broker on spindle. Managed by scripts/bw-seed-omp-broker.sh.", login: {username: "omp-auth-broker", password: $tok} }') diff --git a/tests/test_omp_unit.py b/tests/test_omp_unit.py index f4fbb3c6..71eef93e 100644 --- a/tests/test_omp_unit.py +++ b/tests/test_omp_unit.py @@ -8,7 +8,7 @@ pytestmark = pytest.mark.skipif(shutil.which("yq") is None, reason="yq not installed") -BROKER_URL = "http://100.78.73.8:8765" +BROKER_URL = "http://100.113.75.88:8765" OMP_STUB = """\ case "$1 $2" in @@ -72,3 +72,15 @@ def test_failing_step_does_not_stop_later_steps(env, tmp_path): assert "FAIL claude skill links" in r.stdout assert (home / ".config/omp/kiro.json").is_file() assert broker_url(home) == BROKER_URL + + +def test_seed_does_not_change_vault_when_broker_is_unreachable(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "ssh", "exit 42\n") + write_stub(bindir, "bw", 'touch "$HOME/bw-called"\nexit 0\n') + result = subprocess.run( + ["bash", str(REPO / "scripts/bw-seed-omp-broker.sh")], + env={**env, "BW_SESSION": "test"}, capture_output=True, text=True, + ) + assert result.returncode != 0 + assert not (tmp_path / "home/bw-called").exists() diff --git a/units/omp.sh b/units/omp.sh index 1d6d7b56..b97b34e4 100755 --- a/units/omp.sh +++ b/units/omp.sh @@ -107,7 +107,7 @@ step "routing merged" routing_merge step "broker url" broker_url step "broker token" broker_token if [ "$(yq '.auth.broker | has("token")' "$cfg" 2>/dev/null)" != true ]; then - note "PARTIAL: broker token not written (vault locked or item missing); run: just seed-omp-broker on punjab, then just apply-tags omp" + note "PARTIAL: broker token not written (vault locked or item missing); run: just seed-omp-broker from a client with SSH access to spindle, then just apply-tags omp" fi if have_omp; then soft "broker status" broker_status; else note "skip broker status (omp missing)"; fi finish diff --git a/units/omp/auth-broker.conf b/units/omp/auth-broker.conf index 025c8f15..4fd6cd1e 100644 --- a/units/omp/auth-broker.conf +++ b/units/omp/auth-broker.conf @@ -1,5 +1,3 @@ -# Sourced by units/omp.sh. punjab runs `omp auth-broker serve` on its tailnet -# address (a hand-made user unit, not managed here); keep in step with -# inventory.yml's punjab ansible_host. -OMP_AUTH_BROKER_URL="${OMP_AUTH_BROKER_URL:-http://100.78.73.8:8765}" +# The broker runs on spindle's tailnet address. It is not in inventory.yml. +OMP_AUTH_BROKER_URL="${OMP_AUTH_BROKER_URL:-http://100.113.75.88:8765}" OMP_AUTH_BROKER_BW_ITEM="${OMP_AUTH_BROKER_BW_ITEM:-omp-auth-broker-token}" From 4afab3d348b7bf6026401ee1eda0f871b3c13593 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:57:16 +0000 Subject: [PATCH 07/62] test(omp): run broker safety check without yq Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- tests/test_omp_seed.py | 15 +++++++++++++++ tests/test_omp_unit.py | 12 ------------ 2 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 tests/test_omp_seed.py diff --git a/tests/test_omp_seed.py b/tests/test_omp_seed.py new file mode 100644 index 00000000..564df672 --- /dev/null +++ b/tests/test_omp_seed.py @@ -0,0 +1,15 @@ +import subprocess + +from conftest import REPO, write_stub + + +def test_seed_does_not_change_vault_when_broker_is_unreachable(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "ssh", "exit 42\n") + write_stub(bindir, "bw", 'touch "$HOME/bw-called"\nexit 0\n') + result = subprocess.run( + ["bash", str(REPO / "scripts/bw-seed-omp-broker.sh")], + env={**env, "BW_SESSION": "test"}, capture_output=True, text=True, + ) + assert result.returncode != 0 + assert not (tmp_path / "home/bw-called").exists() diff --git a/tests/test_omp_unit.py b/tests/test_omp_unit.py index 71eef93e..0347c392 100644 --- a/tests/test_omp_unit.py +++ b/tests/test_omp_unit.py @@ -72,15 +72,3 @@ def test_failing_step_does_not_stop_later_steps(env, tmp_path): assert "FAIL claude skill links" in r.stdout assert (home / ".config/omp/kiro.json").is_file() assert broker_url(home) == BROKER_URL - - -def test_seed_does_not_change_vault_when_broker_is_unreachable(stub_env, tmp_path): - bindir, env = stub_env - write_stub(bindir, "ssh", "exit 42\n") - write_stub(bindir, "bw", 'touch "$HOME/bw-called"\nexit 0\n') - result = subprocess.run( - ["bash", str(REPO / "scripts/bw-seed-omp-broker.sh")], - env={**env, "BW_SESSION": "test"}, capture_output=True, text=True, - ) - assert result.returncode != 0 - assert not (tmp_path / "home/bw-called").exists() From 1331429b1a4fea13605e76cbc84aea20bf89ed26 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:11:49 +0000 Subject: [PATCH 08/62] fix: integrate units with onboarding and tagged applies Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- Justfile | 69 ++++++++++--- bootstrap.sh | 85 +++++++++++----- docs/src/architecture.md | 8 +- docs/src/bitwarden.md | 8 +- docs/src/index.md | 6 +- docs/src/onboarding.md | 12 ++- docs/src/roles/omp.md | 2 +- docs/src/roles/systemd.md | 4 + docs/src/units.md | 4 + group_vars/all.yml | 2 + .../systemd/templates/phone-apply.service.j2 | 4 + tests/test_bootstrap_units.py | 86 ++++++++++++++++ tests/test_tagged_apply.py | 98 +++++++++++++++++++ units/lib.sh | 2 +- 14 files changed, 334 insertions(+), 56 deletions(-) create mode 100644 tests/test_bootstrap_units.py create mode 100644 tests/test_tagged_apply.py diff --git a/Justfile b/Justfile index a449dc59..1237f152 100644 --- a/Justfile +++ b/Justfile @@ -15,6 +15,7 @@ _online_hosts := shell('python3 "$1"', justfile_directory() / "scripts/online_ho # Defined once so a change to how the fleet addresses itself is a one-line # edit instead of eight. _play := "ansible-playbook --connection=local -l " + machine + " -e target=" + machine + " -e ansible_connection=local -e ansible_host=127.0.0.1" +_secret_tag_pattern := "secrets|bitwarden|ssh|ssh_keys|github|tailscale|kube|forgejo|registry|atuin|ai|omp" # Apply all config to this machine (unlocks BW interactively if needed) apply *args: @@ -38,12 +39,14 @@ apply *args: SKIP_TAGS="secrets" fi rc=0 + unit_args=() if [ -n "$SKIP_TAGS" ]; then + unit_args+=(--no-secrets) {{ _play }} site.yml --skip-tags "$SKIP_TAGS" $BECOME_ARGS {{ args }} || rc=$? else - {{ _play }} -e "bw_session=${BW_SESSION:-}" site.yml $BECOME_ARGS {{ args }} || rc=$? + {{ _play }} site.yml $BECOME_ARGS {{ args }} || rc=$? fi - urc=0; scripts/run-units.sh || urc=$? + urc=0; scripts/run-units.sh "${unit_args[@]}" || urc=$? [ "$rc" -ne 0 ] || rc=$urc scripts/record-apply.py "$rc" apply "$SKIP_TAGS" exit $rc @@ -52,10 +55,40 @@ apply *args: apply-tags tags: #!/usr/bin/env bash set -uo pipefail - cd {{ dotfiles_dir }} + cd {{ dotfiles_dir }} || exit 1 rc=0 - {{ _play }} -e "bw_session=${BW_SESSION:-}" site.yml --tags {{ tags }} || rc=$? - urc=0; scripts/run-units.sh --tags "{{ tags }}" || urc=$? + ansible_tags="{{ tags }}" + unit_args=(--tags "{{ tags }}") + skip_args=() + needs_secrets=0 + IFS=',' read -ra tag_list <<< "{{ tags }}" + for tag in "${tag_list[@]}"; do + case "$tag" in + {{ _secret_tag_pattern }}) needs_secrets=1 ;; + esac + done + if [ "$needs_secrets" -eq 1 ]; then + # bw-unlock accepts an environment session as-is; reject stale ones first. + if [ -n "${BW_SESSION:-}" ] && ! bw unlock --check >/dev/null 2>&1; then + unset BW_SESSION + fi + if bw_env=$(scripts/bw-resolve.sh local) && eval "$bw_env" && + [ -n "${BW_SESSION:-}" ] && bw unlock --check >/dev/null 2>&1 && + timeout 15 bw sync >/dev/null; then + # Secret roles selected by their own tag still need bw_unlocked set. + ansible_tags="$ansible_tags,bitwarden" + else + echo "Bitwarden unavailable or sync failed; skipping secrets." >&2 + unset BW_SESSION + skip_args=(--skip-tags secrets) + unit_args+=(--no-secrets) + rc=1 + fi + fi + arc=0 + {{ _play }} site.yml --tags "$ansible_tags" "${skip_args[@]}" || arc=$? + [ "$rc" -ne 0 ] || rc=$arc + urc=0; scripts/run-units.sh "${unit_args[@]}" || urc=$? [ "$rc" -ne 0 ] || rc=$urc scripts/record-apply.py "$rc" apply-tags "{{ tags }}" exit $rc @@ -77,9 +110,16 @@ apply-nosecrets *args: apply-remote-tags name tags: #!/usr/bin/env bash set -euo pipefail - # A BW session is only valid on the machine that unlocked it, so resolve - # one on the remote rather than forwarding ours. - REMOTE_BW_SESSION=$({{ dotfiles_dir }}/scripts/bw-resolve.sh remote {{ name }} || true) + # Resolve on the target only for explicitly secret-bearing tags. + REMOTE_BW_SESSION="" + IFS=',' read -ra tag_list <<< "{{ tags }}" + for tag in "${tag_list[@]}"; do + case "$tag" in + {{ _secret_tag_pattern }}) + REMOTE_BW_SESSION=$({{ dotfiles_dir }}/scripts/bw-resolve.sh remote {{ name }} || true) + break ;; + esac + done ssh -t {{ name }} " export PATH=\"\$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:\$PATH\" export BW_SESSION='${REMOTE_BW_SESSION}' @@ -299,8 +339,15 @@ apply-online-tags tags: [ -z "$ONLINE_HOSTS" ] && echo "No remote hosts online." && exit 0 declare -A SESSIONS - for host in $ONLINE_HOSTS; do - SESSIONS[$host]=$(scripts/bw-resolve.sh remote "$host" || true) + IFS=',' read -ra tag_list <<< "{{ tags }}" + for tag in "${tag_list[@]}"; do + case "$tag" in + {{ _secret_tag_pattern }}) + for host in $ONLINE_HOSTS; do + SESSIONS[$host]=$(scripts/bw-resolve.sh remote "$host" || true) + done + break ;; + esac done PIDS=() @@ -313,7 +360,7 @@ apply-online-tags tags: echo "Applying tags '{{ tags }}' to $host (background)..." ssh "$host" " export PATH=\"\$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:\$PATH\" - export BW_SESSION='${SESSIONS[$host]}' + export BW_SESSION='${SESSIONS[$host]:-}' cd ~/.local/share/dotfiles && git pull --ff-only && just apply-tags {{ tags }} " > "/tmp/apply_${host}.log" 2>&1 & PIDS+=("$!:$host") diff --git a/bootstrap.sh b/bootstrap.sh index 3ff87d79..6e1eca13 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -322,7 +322,8 @@ run_phase1() { echo "═══════════════════════════════════════════════════" echo "" - cd "$DOTFILES_DIR" + cd "$DOTFILES_DIR" || return 1 + local rc=0 urc=0 ansible-playbook \ --connection=local \ -l "$MACHINE_NAME" \ @@ -331,7 +332,10 @@ run_phase1() { -e ansible_host=127.0.0.1 \ -e "is_desktop=$is_desktop" \ --skip-tags secrets \ - site.yml + site.yml || rc=$? + scripts/run-units.sh --no-secrets || urc=$? + [ "$rc" -ne 0 ] || rc=$urc + return "$rc" } # ── Bitwarden login ─────────────────────────────────────────────── @@ -346,6 +350,17 @@ setup_bitwarden() { fi local bw_status + # Prefer a usable session, including the cached session from a previous run. + if [ -n "${BW_SESSION:-}" ] && ! bw unlock --check >/dev/null 2>&1; then + unset BW_SESSION + fi + if [ -z "${BW_SESSION:-}" ] && [ -s /tmp/bw_session ]; then + local cached_session + cached_session=$(cat /tmp/bw_session) + if BW_SESSION="$cached_session" bw unlock --check >/dev/null 2>&1; then + export BW_SESSION="$cached_session" + fi + fi bw_status=$(bw status 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['status'])" 2>/dev/null || echo "unauthenticated") echo "" @@ -356,24 +371,24 @@ setup_bitwarden() { echo "═══════════════════════════════════════════════════" echo "" - case "$bw_status" in - unlocked) - BW_SESSION=$(bw unlock --raw 2>/dev/null || true) - ;; - locked) - echo "Unlocking Bitwarden..." - BW_SESSION=$(bw unlock --raw) - ;; - unauthenticated) + if [ -z "${BW_SESSION:-}" ]; then + if [ "$bw_status" = "unauthenticated" ]; then echo "Log in to Bitwarden to complete setup:" - bw login - BW_SESSION=$(bw unlock --raw) - ;; - esac - - export BW_SESSION - rm -f /tmp/bw_session - (umask 077 && printf '%s' "$BW_SESSION" > /tmp/bw_session) + bw login || return 1 + fi + echo "Unlocking Bitwarden..." + BW_SESSION=$(bw unlock --raw) || return 1 + export BW_SESSION + fi + if [ -z "$BW_SESSION" ] || ! bw unlock --check >/dev/null 2>&1; then + echo "Bitwarden session is invalid; secrets setup skipped." >&2 + return 1 + fi + if ! timeout 15 bw sync >/dev/null; then + echo "Bitwarden sync failed; secrets setup skipped." >&2 + return 1 + fi + (umask 077 && printf '%s' "$BW_SESSION" > /tmp/bw_session) || return 1 return 0 } @@ -382,7 +397,8 @@ run_phase2() { local is_desktop="false" [ "$MACHINE_TYPE" = "desktop" ] && is_desktop="true" - cd "$DOTFILES_DIR" + cd "$DOTFILES_DIR" || return 1 + local rc=0 urc=0 ansible-playbook \ --connection=local \ -l "$MACHINE_NAME" \ @@ -390,9 +406,11 @@ run_phase2() { -e ansible_connection=local \ -e ansible_host=127.0.0.1 \ -e "is_desktop=$is_desktop" \ - -e "bw_session=${BW_SESSION:-}" \ --tags secrets \ - site.yml + site.yml || rc=$? + scripts/run-units.sh --tags omp,secrets || urc=$? + [ "$rc" -ne 0 ] || rc=$urc + return "$rc" } # ── Push inventory changes (after GitHub auth is set up) ───────── @@ -434,12 +452,23 @@ main() { ensure_sudo ensure_tailscale install_collections - run_phase1 + local rc=0 phase_rc=0 + run_phase1 || rc=$? - # Phase 2 runs if Bitwarden CLI is now available (installed by Homebrew in Phase 1) + # Phase 2 still runs after a Phase 1 failure, if the vault is available. if setup_bitwarden; then - run_phase2 - push_inventory_changes + run_phase2 || phase_rc=$? + [ "$rc" -ne 0 ] || rc=$phase_rc + if [ "$phase_rc" -eq 0 ]; then + push_inventory_changes || { phase_rc=$?; [ "$rc" -ne 0 ] || rc=$phase_rc; } + fi + else + echo "Bitwarden setup failed; Phase 2 was not run." >&2 + [ "$rc" -ne 0 ] || rc=1 + fi + if [ "$rc" -ne 0 ]; then + echo "Bootstrap failed (exit $rc); address the errors and re-run." >&2 + return "$rc" fi echo "" @@ -453,4 +482,6 @@ main() { echo " To re-apply anytime: just apply" } -main +if [[ -z "${BASH_SOURCE[0]:-}" || "${BASH_SOURCE[0]}" == "$0" ]]; then + main +fi diff --git a/docs/src/architecture.md b/docs/src/architecture.md index 940ef1ee..12f3855b 100644 --- a/docs/src/architecture.md +++ b/docs/src/architecture.md @@ -47,7 +47,7 @@ SSHes to the target, forwards the Bitwarden session token, pulls the latest dotf just apply-tags kube,shell ``` -Runs only roles tagged with the given tags. Useful for deploying specific configs without running the full playbook. `--skip-tags` works the same way in reverse — `just apply-nosecrets` is `--skip-tags secrets` under the hood (see [Bitwarden](bitwarden.md)). +Runs roles and units selected by the given tags. Explicit secret-bearing tags first resolve and sync Bitwarden; unrelated tags do not prompt. `just apply-nosecrets` passes `--skip-tags secrets` to Ansible and `--no-secrets` to units (see [Bitwarden](bitwarden.md)). #### Available Tags @@ -64,7 +64,7 @@ Every role in `site.yml` carries one or more tags. Tags fall into two kinds: **p | System + packages | `bitwarden` | `secrets`, `bitwarden` | | System + packages | `shell_fonts` | `dotfiles`, `shell`, `fonts` | | System + packages | `shell_dotfiles` | `dotfiles`, `shell`, `dotfiles` | -| System + packages | `omp` | `dotfiles`, `omp` | +| System + packages | `omp` unit | `dotfiles`, `omp` | | System + packages | `hive_ops` | `hive`, `hive_ops` | | System + packages | `git` | `dotfiles`, `git` | | System + packages | `neovim` | `dotfiles`, `neovim`, `editor` | @@ -96,7 +96,7 @@ Common phase-wide selections: - `--tags secrets` — every BW-backed role only (`bitwarden`, `shell_atuin`, `shell_ai`, `ssh_keys`, `github`, `tailscale`, `kube`, `forgejo_registry`) - `--tags desktop` — every desktop-app role (`flatpak`, `bluefin_common`, `gnome`, `zen_browser`, `pipewire_audio`) - `--tags services` — every long-running service role (`syncthing`, `systemd`, `bst_dashboard`, `kirocrew`, `proxy`, `tailscale_cert`) -- `--skip-tags secrets` — everything except BW-backed roles. This is `just apply-nosecrets`, which the daily `dotfiles-update.timer` and the `dots` alias run; `bootstrap.sh` and `phone-apply.service` pass `--skip-tags secrets` directly +- `--skip-tags secrets` — everything except BW-backed roles. `just apply-nosecrets`, bootstrap's first phase and `phone-apply.service` also pass `--no-secrets` to units. Tags do not replace the [`skip_*` host_vars flags](#skip-flags) or inventory-group gating described below — a role can be tag-selected and still no-op if its `when:` condition (group membership, `skip_*`, `machine_profile`) evaluates false. @@ -144,7 +144,7 @@ Every flag `site.yml` reads, all defaulting to `false` (`| default(false)`). Set | `skip_proxy` | `proxy` | not `vps`, not `vm-test` | `mumbai`, `punjab`, `termux` | | `skip_tailscale_cert` | `tailscale_cert` | not `vps`, not `vm-test` | `mumbai`, `termux` | -"desktop" is `is_desktop` (membership of the `desktop` group unless overridden); "not `vm-test`" is `machine_profile != 'vm-test'`. Roles without a flag either always run or are gated another way: `termux_packages` (`termux_hosts` group), `omp` (not Termux), `kirocrew` (`kirocrew_enabled: true`), `server_hardening` (`server`/`vps` groups), `retired_services` (not `vps`, not Termux, systemd hosts). +"desktop" is `is_desktop` (membership of the `desktop` group unless overridden); "not `vm-test`" is `machine_profile != 'vm-test'`. Roles without a flag either always run or are gated another way: `termux_packages` (`termux_hosts` group), `kirocrew` (`kirocrew_enabled: true`), `server_hardening` (`server`/`vps` groups), `retired_services` (not `vps`, not Termux, systemd hosts). The `omp` unit skips Termux. ## Secrets Flow diff --git a/docs/src/bitwarden.md b/docs/src/bitwarden.md index 68712e4e..c8de2d48 100644 --- a/docs/src/bitwarden.md +++ b/docs/src/bitwarden.md @@ -5,7 +5,7 @@ All secrets live in [Bitwarden](https://bitwarden.com/) and are resolved at runt ## How It Works 1. The `bitwarden` role runs first in the secrets phase. -2. It resolves a session token in order: `BW_SESSION` env var → `/tmp/bw_session` → interactive `bw unlock`. +2. The apply wrapper can prompt via `scripts/bw-resolve.sh`; the role itself reads `BW_SESSION` from the environment, then falls back to `/tmp/bw_session`. 3. It runs `bw sync` (with a 15 s hard timeout, so a flaky vault server can't hang the play) to refresh the local cached vault. 4. It calls `bw status` and sets a `bw_unlocked` host fact: `true` only if the CLI is installed, a session was resolved, *and* the vault actually unlocks. 5. Every BW-using role gates on `when: bw_unlocked | default(false)`. The end-of-play `post_tasks` summary prints one of: @@ -13,6 +13,8 @@ All secrets live in [Bitwarden](https://bitwarden.com/) and are resolved at runt - `Bitwarden vault was LOCKED — … skipped. To enable: bw unlock …` - `SKIPPED (--skip-tags secrets). Run dots-apply to include …` +For `just apply-tags` with an explicit secret-bearing tag such as `kube`, `ai` or `omp`, the wrapper validates and syncs a session before running, and adds the `bitwarden` tag so legacy secret roles receive `bw_unlocked`. It skips secret work and returns non-zero if the session or sync fails; unrelated tags do not prompt. + ## Remote Apply `just apply-remote ` (and `just apply-remote-tags `): @@ -61,11 +63,11 @@ just apply-nosecrets # git pull + apply with --skip-tags secrets dots # alias: cd to the repo + just apply-nosecrets ``` -The daily systemd timer (`dotfiles-update.service`) calls `apply-nosecrets` for the same reason: it can't unlock the vault non-interactively, so it doesn't try. +The daily systemd timer (`dotfiles-update.service`) calls `apply-nosecrets`: Ansible skips secret-tagged roles and units receive `--no-secrets`, even if a cached session exists. ## No Session? -If Bitwarden can't unlock (first run, no BW CLI installed, vault genuinely locked on this host), every secret-touching task is skipped automatically. Non-secret roles (system, packages, dotfiles, desktop, most services) run normally and the play exits successfully — the end-of-play summary tells you exactly which roles didn't get to do their BW work. +If Bitwarden can't unlock (first run, no BW CLI installed, vault genuinely locked on this host), `just apply` skips secret work and continues with non-secret configuration. A targeted secret apply (`just apply-tags`) also continues non-secret work but exits non-zero, so the requested secret update is not mistaken for success. ## Seeding New Secrets diff --git a/docs/src/index.md b/docs/src/index.md index 20969ce2..bafdd4a9 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -65,11 +65,9 @@ The `llm` group is dormant (commented out in `inventory.yml`). | Phase | Roles | Main tags | |-------|-------|-----------| -| 1 — System + packages + dotfiles | sudo, sshd, apk_packages, homebrew, termux_packages, bitwarden, shell_fonts, shell_dotfiles, omp, hive_ops, git, neovim | `system`, `packages`, `dotfiles` | +| 1 — System + packages + dotfiles | sudo, sshd, apk_packages, homebrew, termux_packages, bitwarden, shell_fonts, shell_dotfiles, hive_ops, git, neovim | `system`, `packages`, `dotfiles` | | 2 — Secrets + auth | shell_atuin, shell_ai, ssh_keys, ssh_mesh, github, tailscale, kube, forgejo_registry | `secrets` | | 3 — Desktop apps | flatpak, bluefin_common, gnome, zen_browser (+ browser_fxa), pipewire_audio | `desktop` | | 4 — Services | syncthing, systemd, bst_dashboard, kirocrew, proxy, tailscale_cert, server_hardening | `services` | -Only the `secrets`-tagged work needs Bitwarden. With the vault locked (or -`--skip-tags secrets`), phases 1, 3 and 4 run normally and BW-touching tasks -skip cleanly. +The [`omp` unit](units.md) runs after Ansible, and can also use Bitwarden for its broker token. `just apply-nosecrets` skips the secret-tagged roles and passes `--no-secrets` to units. diff --git a/docs/src/onboarding.md b/docs/src/onboarding.md index 31236e16..c1ff0a3c 100644 --- a/docs/src/onboarding.md +++ b/docs/src/onboarding.md @@ -63,8 +63,10 @@ The bootstrap script: 1. Ensures python, git, uv and Ansible, then clones the dotfiles repo from [GitHub](https://github.com/hanthor/dotfiles) 2. Registers the machine in `inventory.yml` if needed (prompts for the type when `--type` isn't given) 3. Writes `/etc/dotfiles-machine` so the playbook can self-identify, sets the hostname, sets up sudo and Tailscale -4. Runs `site.yml` with `--skip-tags secrets` (installs [Homebrew](https://brew.sh/) and core packages among the rest) -5. If the Bitwarden CLI is now available and unlocks, runs `site.yml --tags secrets` and pushes any inventory changes +4. Runs `site.yml --skip-tags secrets`, then all bash units with `--no-secrets` (installing [Homebrew](https://brew.sh/) and core packages among the rest) +5. Unlocks and syncs Bitwarden, runs `site.yml --tags secrets`, then the `omp` and `secrets` units, and pushes any inventory changes + +Each phase runs its units even if Ansible fails. Bootstrap continues to Phase 2 when possible, but exits non-zero if either phase or Bitwarden setup fails. ## After Bootstrap Completes @@ -79,9 +81,9 @@ just apply ``` **Why this step matters:** -- Bootstrap runs `site.yml --skip-tags secrets`, which skips Bitwarden integration -- `just apply` runs the full playbook with secrets, unlocking Bitwarden and completing setup -- This pull-in step distributes SSH keys across the fleet, registers with GitHub and Tailscale, and sets up daily/on-login automation +- Bootstrap's first phase deliberately skips secrets; the second phase only covers secret-tagged roles and the broker unit. +- `just apply` reruns the full playbook and units with secrets and completes any work that was skipped. +- This distributes SSH keys across the fleet, registers with GitHub and Tailscale, and sets up daily/on-login automation. Do **not** edit `host_vars/.yml` before the first `just apply` — bootstrap sets sensible defaults, and `just apply` will pull in critical secrets (SSH keys, GitHub tokens, Tailscale auth). Any per-machine customizations can follow once the first apply succeeds; the `skip_*` flags that turn individual roles off are listed in [Architecture → Skip Flags](architecture.md#skip-flags). diff --git a/docs/src/roles/omp.md b/docs/src/roles/omp.md index 07be64bc..76552a46 100644 --- a/docs/src/roles/omp.md +++ b/docs/src/roles/omp.md @@ -51,4 +51,4 @@ Complete the browser sign-in, then select `kiro/auto` with `/model kiro/auto`. K ## Package source -The OMP and Bun Homebrew formulas are in `can1357/tap`; both are core packages for Homebrew-supported hosts. On musl, the official OMP binary installer verifies its SHA-256 and requires `libgcc` and `libstdc++`. `roles/pi/files/package.json` is tracked by Renovate and protected by `deps-guard`; the unit does not install it. +The OMP Homebrew formula is in `can1357/tap`; Bun comes from Homebrew core. Both are core packages for Homebrew-supported hosts. On musl, the official OMP binary installer verifies its SHA-256 and requires `libgcc` and `libstdc++`. `roles/pi/files/package.json` is tracked by Renovate and protected by `deps-guard`; the unit does not install it. diff --git a/docs/src/roles/systemd.md b/docs/src/roles/systemd.md index 6afc4f16..e0139345 100644 --- a/docs/src/roles/systemd.md +++ b/docs/src/roles/systemd.md @@ -15,6 +15,10 @@ Keeps the dotfiles repo up to date automatically. - **Desktops/servers:** Daily timer (`dotfiles-update.timer`) - **Laptops:** Runs on login via `graphical-session.target` (no timer) +### phone-apply (himachal only) + +Pushes non-secret Ansible configuration to `termux` and `mumbai`, then runs the non-secret units on each phone from a temporary archive of the controller's tracked revision. Each operation is attempted independently; an unreachable phone does not fail the service. SSH aliases for the two inventory hosts must be configured on himachal. + ### atuin-daemon Keeps [Atuin](https://atuin.sh/) shell history synced in the background. diff --git a/docs/src/units.md b/docs/src/units.md index 42f5c721..7f98e8ea 100644 --- a/docs/src/units.md +++ b/docs/src/units.md @@ -19,6 +19,10 @@ Everything not listed under `units/` is still an Ansible role. `scripts/run-units.sh [--no-secrets] [--tags a,b]` is the one wrapper the recipes call. It maps Ansible tags to units (`tasks_for_tag`, mirroring the tags of the roles the units replaced), runs the selection with `mise run --continue-on-error --jobs 8 --output prefix --timeout 15m`, and exits with mise's status. Without `mise` on `PATH` it prints `mise not installed; running units sequentially` and runs the selected units one after another, continuing after failures, so hosts without mise (Termux, musl) keep updating. +`just apply-tags` resolves, validates and syncs Bitwarden for explicitly secret-bearing tags (including `omp`), and selects the `bitwarden` role so legacy secret roles receive its facts. If that fails, it skips secrets, still runs the non-secret selection, and returns an error. Tags unrelated to secrets do not prompt for a vault unlock. `just apply-nosecrets` and bootstrap's first phase pass `--no-secrets` to units even if a session is already present. + +Bootstrap runs all units after its first Ansible pass and the `omp,secrets` selection after the secret pass. The phone push timer runs the non-secret units remotely from a temporary archive of the controller's tracked revision, after its Ansible passes. + ## Rules for a unit - Idempotent bash. `set -uo pipefail`, never `set -e`. diff --git a/group_vars/all.yml b/group_vars/all.yml index bf9f9714..11fa59d4 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -70,6 +70,7 @@ ssh_mesh_exclude: # ── Core packages (all machines) ───────────────────────────────── core_brews: - bitwarden-cli + - bun - gh - git-lfs - node # Bluefin ships node OTB, but non-Bluefin hosts (e.g. mumbai, the phone's Debian VM) don't @@ -101,6 +102,7 @@ brew_blocked: core_taps: - ublue-os/experimental-tap core_tap_brews: + - can1357/tap/omp - ublue-os/experimental-tap/bluefin-cli # ── Desktop-only packages ──────────────────────────────────────── # Trimmed to essentials — Bluefin already ships gcc, python, node, etc. diff --git a/roles/systemd/templates/phone-apply.service.j2 b/roles/systemd/templates/phone-apply.service.j2 index 5c409d9a..3bbc3f19 100644 --- a/roles/systemd/templates/phone-apply.service.j2 +++ b/roles/systemd/templates/phone-apply.service.j2 @@ -12,6 +12,10 @@ ExecStartPre=/usr/bin/git pull --ff-only # without failing the service (the leading "-"). ExecStart=-{{ brew_prefix }}/bin/ansible-playbook -i inventory.yml -e target=termux site.yml --skip-tags secrets ExecStart=-{{ brew_prefix }}/bin/ansible-playbook -i inventory.yml -e target=mumbai site.yml --skip-tags secrets +# Run the same tracked revision on each target, even if its Ansible pass failed. +# An archive avoids assuming the phones have a checkout or Ansible installed. +ExecStart=-:/bin/bash -c "set -o pipefail; /usr/bin/git archive HEAD | /usr/bin/ssh -o BatchMode=yes termux 'mkdir -p \"$HOME/.cache\"; d=$(mktemp -d \"$HOME/.cache/dotfiles-phone.XXXXXXXX\") || exit 1; tar -xf - -C \"$d\" && bash \"$d/scripts/run-units.sh\" --no-secrets; rc=$?; rm -rf -- \"$d\"; exit \"$rc\"'" +ExecStart=-:/bin/bash -c "set -o pipefail; /usr/bin/git archive HEAD | /usr/bin/ssh -o BatchMode=yes mumbai 'mkdir -p \"$HOME/.cache\"; d=$(mktemp -d \"$HOME/.cache/dotfiles-phone.XXXXXXXX\") || exit 1; tar -xf - -C \"$d\" && bash \"$d/scripts/run-units.sh\" --no-secrets; rc=$?; rm -rf -- \"$d\"; exit \"$rc\"'" Environment="PATH={{ brew_prefix }}/bin:%h/.local/bin:/usr/local/bin:/usr/bin:/bin" Environment="HOME={{ ansible_facts['env']['HOME'] }}" StandardOutput=journal diff --git a/tests/test_bootstrap_units.py b/tests/test_bootstrap_units.py new file mode 100644 index 00000000..5e1f0058 --- /dev/null +++ b/tests/test_bootstrap_units.py @@ -0,0 +1,86 @@ +import subprocess +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub + + +@pytest.fixture +def phase_env(stub_env, tmp_path): + bindir, env = stub_env + repo = tmp_path / "checkout" + scripts = repo / "scripts" + scripts.mkdir(parents=True) + log = tmp_path / "calls" + write_stub(bindir, "ansible-playbook", 'printf "ansible %s\\n" "$*" >> "$CALLS"\nexit "${ANSIBLE_RC:-0}"\n') + write_stub(scripts, "run-units.sh", 'printf "units %s\\n" "$*" >> "$CALLS"\nexit "${UNITS_RC:-0}"\n') + env.update(CALLS=str(log), TEST_REPO=str(repo), BW_SESSION="test-session") + return env, log + + +def run_phase(env, phase): + return subprocess.run( + ["bash", "-c", 'source "$1/bootstrap.sh" --name test --type desktop; DOTFILES_DIR="$TEST_REPO"; MACHINE_NAME=test; MACHINE_TYPE=desktop; "$2"', + "bash", str(REPO), phase], + env=env, capture_output=True, text=True, + ) + + +@pytest.mark.parametrize("phase,unit_args", [("run_phase1", "--no-secrets"), ("run_phase2", "--tags omp,secrets")]) +@pytest.mark.parametrize("ansible_rc,units_rc,expected", [(0, 0, 0), (7, 0, 7), (0, 9, 9), (7, 9, 7)]) +def test_bootstrap_phases_continue_and_combine_errors(phase_env, phase, unit_args, ansible_rc, units_rc, expected): + env, log = phase_env + env.update(ANSIBLE_RC=str(ansible_rc), UNITS_RC=str(units_rc)) + result = run_phase(env, phase) + assert result.returncode == expected + calls = log.read_text().splitlines() + assert len(calls) == 2 + assert calls[0].startswith("ansible ") + assert calls[1] == f"units {unit_args}" + assert "test-session" not in "\n".join(calls) + if phase == "run_phase1": + assert "--skip-tags secrets" in calls[0] + else: + assert "--tags secrets" in calls[0] + + +def test_bootstrap_runs_second_phase_after_first_fails(stub_env, tmp_path): + _, env = stub_env + env["CALLS"] = str(tmp_path / "calls") + script = r''' +source "$1/bootstrap.sh" --name test --type desktop +for f in ensure_python ensure_git ensure_uv ensure_ansible clone_repo get_machine_name get_machine_type register_in_inventory write_machine_name set_hostname ensure_sudo ensure_tailscale install_collections setup_bitwarden push_inventory_changes; do + eval "$f() { :; }" +done +run_phase1() { echo first >> "$CALLS"; return 7; } +run_phase2() { echo second >> "$CALLS"; return 0; } +main +''' + result = subprocess.run(["bash", "-c", script, "bash", str(REPO)], env=env, capture_output=True, text=True) + assert result.returncode == 7 + assert Path(env["CALLS"]).read_text().splitlines() == ["first", "second"] + assert "Bootstrap complete!" not in result.stdout + + +@pytest.mark.parametrize("no_secrets", [False, True]) +def test_unit_session_respects_no_secrets_and_keeps_session_off_argv(stub_env, tmp_path, no_secrets): + bindir, env = stub_env + log = tmp_path / "bw-calls" + write_stub(bindir, "bw", 'printf "%s\\n" "$*" >> "$CALLS"\n[ "${BW_SESSION:-}" = "test-session" ]\n') + env.update(BW_SESSION="test-session", CALLS=str(log)) + if no_secrets: + env["UNITS_NO_SECRETS"] = "1" + else: + env.pop("UNITS_NO_SECRETS", None) + result = subprocess.run( + ["bash", "-c", 'source "$1/units/lib.sh"; bw_session', "bash", str(REPO)], + env=env, capture_output=True, text=True, + ) + assert result.returncode == 0 + if no_secrets: + assert result.stdout == "" + assert not log.exists() + else: + assert result.stdout == "test-session" + assert log.read_text().splitlines() == ["unlock --check"] diff --git a/tests/test_tagged_apply.py b/tests/test_tagged_apply.py new file mode 100644 index 00000000..202613b4 --- /dev/null +++ b/tests/test_tagged_apply.py @@ -0,0 +1,98 @@ +import shutil +import subprocess +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub + + +pytestmark = pytest.mark.skipif(shutil.which("just") is None, reason="just not installed") + + +@pytest.fixture +def apply_env(stub_env, tmp_path): + _, env = stub_env + home = Path(env["HOME"]) + bindir = home / ".local/bin" + bindir.mkdir(parents=True) + repo = home / ".local/share/dotfiles" + scripts = repo / "scripts" + scripts.mkdir(parents=True) + (scripts / "online_hosts.py").write_text("") + shutil.copyfile(REPO / "Justfile", repo / "Justfile") + log = tmp_path / "calls" + env.update(CALLS=str(log), BW_SESSION="stale") + write_stub(bindir, "bw", ''' +printf 'bw %s\n' "$*" >> "$CALLS" +case "$*" in + 'unlock --check') [ "${BW_SESSION:-}" = valid-session ] ;; + sync) [ "${SYNC_FAIL:-0}" = 0 ] ;; + *) exit 1 ;; +esac +''') + write_stub(bindir, "ansible-playbook", 'printf "ansible %s\\n" "$*" >> "$CALLS"\nexit "${ANSIBLE_RC:-0}"\n') + write_stub(bindir, "git", 'exit 0\n') + write_stub(bindir, "ansible-galaxy", 'exit 0\n') + write_stub(bindir, "sudo", 'exit 0\n') + write_stub(bindir, "ssh", 'printf "ssh\\n" >> "$CALLS"\n') + write_stub(scripts, "bw-resolve.sh", ''' +printf 'resolve\n' >> "$CALLS" +[ "${RESOLVE_FAIL:-0}" = 0 ] || exit 1 +printf 'export BW_SESSION=valid-session\n' +''') + write_stub(scripts, "run-units.sh", 'printf "units %s\\n" "$*" >> "$CALLS"\n') + write_stub(scripts, "record-apply.py", 'exit 0\n') + return env, repo, log + + +def run_apply(env, repo, *args): + return subprocess.run(["just", "-f", str(repo / "Justfile"), *args], env=env, capture_output=True, text=True) + + +def test_secret_tag_validates_syncs_and_initializes_legacy_roles(apply_env): + env, repo, log = apply_env + result = run_apply(env, repo, "apply-tags", "omp,kube") + assert result.returncode == 0, result.stderr + calls = log.read_text().splitlines() + assert calls[:4] == ["bw unlock --check", "resolve", "bw unlock --check", "bw sync"] + assert "--tags omp,kube,bitwarden" in calls[4] + assert calls[5] == "units --tags omp,kube" + assert "valid-session" not in result.stdout + result.stderr + "\n".join(calls) + + +def test_nonsecret_tag_does_not_prompt_or_sync(apply_env): + env, repo, log = apply_env + result = run_apply(env, repo, "apply-tags", "git") + assert result.returncode == 0, result.stderr + calls = log.read_text().splitlines() + assert len(calls) == 2 + assert "--tags git" in calls[0] + assert calls[1] == "units --tags git" + + +def test_remote_nonsecret_tag_does_not_resolve_vault(apply_env): + env, repo, log = apply_env + result = run_apply(env, repo, "apply-remote-tags", "phone", "git") + assert result.returncode == 0, result.stderr + assert log.read_text().splitlines() == ["ssh"] + + +def test_failed_sync_runs_nonsecret_work_and_fails(apply_env): + env, repo, log = apply_env + env["SYNC_FAIL"] = "1" + result = run_apply(env, repo, "apply-tags", "omp,kube") + assert result.returncode != 0 + calls = log.read_text().splitlines() + assert "--skip-tags secrets" in calls[-2] + assert calls[-1] == "units --tags omp,kube --no-secrets" + + +def test_full_apply_skips_vault_units_when_unlock_fails(apply_env): + env, repo, log = apply_env + env["RESOLVE_FAIL"] = "1" + result = run_apply(env, repo, "apply") + assert result.returncode == 0, result.stderr + calls = log.read_text().splitlines() + assert "--skip-tags secrets" in calls[-2] + assert calls[-1] == "units --no-secrets" diff --git a/units/lib.sh b/units/lib.sh index d90541e0..ca312c96 100644 --- a/units/lib.sh +++ b/units/lib.sh @@ -20,6 +20,6 @@ bw_session() { command -v bw >/dev/null 2>&1 || return 0 local s="${BW_SESSION:-}" [ -n "$s" ] || { [ -s /tmp/bw_session ] && s="$(cat /tmp/bw_session)"; } - [ -n "$s" ] && bw unlock --check --session "$s" >/dev/null 2>&1 && printf '%s' "$s" + [ -n "$s" ] && BW_SESSION="$s" bw unlock --check >/dev/null 2>&1 && printf '%s' "$s" return 0 } From 792d8b126975eb1c8cebe6263a8abdfd89806334 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:14:24 +0000 Subject: [PATCH 09/62] fix: preserve Atuin data and harden Chezmoi onboarding Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/roles/neovim.md | 2 +- docs/src/roles/shell_atuin.md | 11 ++- docs/src/roles/shell_dotfiles.md | 2 +- docs/src/units.md | 6 +- .../.chezmoiscripts/run_after_20-nvim-sync.sh | 16 +++- tests/test_atuin_unit.py | 89 ++++++++++++++++++- tests/test_chezmoi_unit.py | 21 +++++ tests/test_nvim_sync.py | 56 ++++++++++++ units/atuin.sh | 46 ++++++++-- units/chezmoi.sh | 10 ++- 10 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 tests/test_chezmoi_unit.py create mode 100644 tests/test_nvim_sync.py diff --git a/docs/src/roles/neovim.md b/docs/src/roles/neovim.md index 556eec1d..27393903 100644 --- a/docs/src/roles/neovim.md +++ b/docs/src/roles/neovim.md @@ -5,7 +5,7 @@ **Secrets needed:** No **Runs on:** All machines with `chezmoi` installed -[Neovim](https://neovim.io/) configuration, deployed from `home/dot_config/nvim/` (`init.lua`, `lua/config/`, `lua/plugins/`) by chezmoi. After each apply, `.chezmoiscripts/run_after_20-nvim-sync.sh` bootstraps lazy.nvim and runs `nvim --headless "+Lazy! sync" +qa` (best effort, 300 s timeout, never fails the apply). +[Neovim](https://neovim.io/) configuration, deployed from `home/dot_config/nvim/` (`init.lua`, `lua/config/`, `lua/plugins/`) by chezmoi. `.chezmoiscripts/run_after_20-nvim-sync.sh` bootstraps lazy.nvim and runs `nvim --headless "+Lazy! sync" +qa` on first apply and when Lua configuration files are added, changed, or removed. It skips synchronization on unrelated applies; failures are best effort and retried on the next apply (300 s timeout, never fails the apply). The last successful fingerprint lives in `~/.cache/dotfiles/nvim-config.cksum`. ## Key Plugins diff --git a/docs/src/roles/shell_atuin.md b/docs/src/roles/shell_atuin.md index 68bf168b..b3aaf0f0 100644 --- a/docs/src/roles/shell_atuin.md +++ b/docs/src/roles/shell_atuin.md @@ -11,17 +11,20 @@ Configures [Atuin](https://atuin.sh/) shell history and logs it in to sync. 1. chezmoi writes `~/.config/atuin/config.toml` from `home/dot_config/atuin/config.toml` — sync to `api.atuin.sh` every 5m, fuzzy search, `filter_mode = "global"` (including the Up-arrow binding) 2. The `atuin` unit checks `atuin status`; if it shows `[Sync]` the Bitwarden round-trip is skipped -3. Otherwise, with an unlocked vault, it reads the `atuin.sh` item (username, password, custom field `key` holding the mnemonic) and runs `atuin login -u … -p … -k …` (resetting `~/.local/share/atuin` first when no key file exists), then a best-effort `atuin sync` -4. Imports existing shell history once (`atuin import auto`, sentinel `~/.local/share/atuin/.history_imported`) +3. Otherwise, with an unlocked vault, it reads the `atuin.sh` item (username, password, custom field `key` holding the mnemonic). Atuin 18.24 or newer receives the password and key through its environment, not command-line arguments. The unit preserves the data directory and its history; a missing key can be written by Atuin during login. If a key already exists, the unit logs in only when `atuin key` matches the vault mnemonic, then runs a best-effort `atuin sync`. +4. Imports existing shell history once (`atuin import auto`, sentinel `~/.local/share/atuin/.history_imported`). If the version or existing key requires manual login, the unit also defers importing until that situation is resolved. ## Output lines - `SKIPPED: atuin not installed` — install it (`core_brews`) - `PARTIAL: atuin not logged in; vault locked` — `bw unlock`, then `just apply-tags atuin` - `PARTIAL: Bitwarden item atuin.sh missing or incomplete` — fix the item (needs username, password, field `key`) +- `PARTIAL: atuin 18.24 or newer required` — upgrade Atuin or run `atuin login` interactively; older versions do not support the non-interactive environment credentials +- `PARTIAL: cannot determine the Atuin key path`, `cannot read the existing Atuin key`, or `existing Atuin key differs` — inspect the local data and resolve the key situation before logging in interactively. A vault key in a different representation can also compare unequal. ## Notes -- The mnemonic is passed to `atuin login -k`, never written to `~/.local/share/atuin/key` directly — atuin expects that file to be a binary key -- `atuin login` receives the password on its command line, visible to `ps` for the duration of the call +- Existing keys are never deleted or replaced by the unit; it does not reset `~/.local/share/atuin` when the key is missing. That directory is kept mode 0700, including after a prior history import. +- Atuin 18.24 resolves its own configured key path, including the newer `~/.atuin/data` layout; the history-import sentinel remains in `~/.local/share/atuin`. +- Alpine's community `atuin` package may be an older offline-only build; sync requires a sync-enabled Atuin client. Check the package version and features before attempting a manual login. - Don't set the Up-arrow filter to `session`: it looks broken in fresh terminals diff --git a/docs/src/roles/shell_dotfiles.md b/docs/src/roles/shell_dotfiles.md index 9261f77b..e86b6a4d 100644 --- a/docs/src/roles/shell_dotfiles.md +++ b/docs/src/roles/shell_dotfiles.md @@ -40,4 +40,4 @@ In fish these are abbreviations (`abbr`), not aliases. ## Notes - Shell secrets (`~/.config/shell/secrets.sh`, `~/.config/fish/conf.d/secrets.fish`) are written by the [`shell-ai` unit](shell_ai.md), not here. -- One failing file does not stop the rest: the unit runs `chezmoi apply --init --keep-going --force --no-tty`. +- One failing file does not stop the rest: the unit runs `chezmoi apply --init --keep-going --force --no-tty`. This overwrites managed files even when locally edited; back up and review existing dotfiles before adoption. See [Units](../units.md#chezmoi). diff --git a/docs/src/units.md b/docs/src/units.md index b4e907ac..fd34425f 100644 --- a/docs/src/units.md +++ b/docs/src/units.md @@ -38,7 +38,11 @@ Everything not listed under `units/` is still an Ansible role. ## chezmoi -File-based config (shell, git, neovim, atuin config) is managed by [chezmoi](https://www.chezmoi.io/) from `home/` (`.chezmoiroot` contains `home`, so `.chezmoi.toml.tmpl`, `.chezmoidata.yaml` and `.chezmoiscripts/` live there). `unit:chezmoi` runs `chezmoi --source --keep-going --force --no-tty apply --init`: `--keep-going` continues past a failing file or script, `--force` overwrites like the old roles did, `--no-tty` keeps the timer from hanging on a prompt, `--init` regenerates the config from `home/.chezmoi.toml.tmpl`. +File-based config (shell, git, neovim, atuin config) is managed by [chezmoi](https://www.chezmoi.io/) from `home/` (`.chezmoiroot` contains `home`, so `.chezmoi.toml.tmpl`, `.chezmoidata.yaml` and `.chezmoiscripts/` live there). `unit:chezmoi` runs `chezmoi --source --keep-going --force --no-tty apply --init`: `--keep-going` continues past a failing file or script, `--force` overwrites like the old roles did, `--no-tty` avoids a TTY, and `--init` regenerates the config from `home/.chezmoi.toml.tmpl`. + +**Back up and review existing dotfiles before the first apply.** The current `--force` will overwrite local edits to managed targets without prompting on every apply. First adoption can replace an existing target even without `--force`: chezmoi does not treat a pre-existing, never-written target as a conflict by default. Review the intended changes with `chezmoi --source diff` (after initializing the config with the correct machine data), and keep a private backup of any local files you need. A safer future cutover should detect both first-adoption and later local conflicts and fail closed for unattended runs, with interactive review; merely removing `--force` does not protect initial adoption. + +If chezmoi is absent the unit fails rather than reporting a successful skip. Homebrew hosts can run `just packages`, Termux users can run `pkg install chezmoi`, and Alpine/postmarketOS users can enable the community repository and run `sudo apk add chezmoi`. - Data: `machine`, `is_desktop` (true when the host is in the `desktop` group of `inventory.yml`), `user_name`, `user_email`, `alias_groups`. - Secrets are **not** chezmoi templates: the `bitwarden` template function aborts a whole template on a missing item. The `shell-ai` and `atuin` units handle secrets. diff --git a/home/.chezmoiscripts/run_after_20-nvim-sync.sh b/home/.chezmoiscripts/run_after_20-nvim-sync.sh index 1c43046a..cf86e7f6 100755 --- a/home/.chezmoiscripts/run_after_20-nvim-sync.sh +++ b/home/.chezmoiscripts/run_after_20-nvim-sync.sh @@ -1,7 +1,17 @@ #!/bin/sh -# Best effort: bootstrap lazy.nvim and sync plugins; never fails the apply. +# Best effort: retry bootstrap/sync until it succeeds, then only on config changes. command -v nvim >/dev/null 2>&1 || exit 0 +config="$HOME/.config/nvim" +[ -d "$config" ] || exit 0 lazy="$HOME/.local/share/nvim/lazy/lazy.nvim" -[ -d "$lazy" ] || git clone --filter=blob:none https://github.com/folke/lazy.nvim.git --branch=stable "$lazy" || true -timeout 300 nvim --headless "+Lazy! sync" +qa 2>&1 | tail -5 +state="$HOME/.cache/dotfiles/nvim-config.cksum" +# Include names as well as contents so additions, deletions and renames count. +fingerprint=$(find "$config" -type f -name '*.lua' -exec cksum {} \; | LC_ALL=C sort | cksum) +if [ -d "$lazy" ] && [ -f "$state" ] && [ "$(cat "$state")" = "$fingerprint" ]; then + exit 0 +fi +[ -d "$lazy" ] || git clone --filter=blob:none https://github.com/folke/lazy.nvim.git --branch=stable "$lazy" || exit 0 +if timeout 300 nvim --headless "+Lazy! sync" +qa; then + mkdir -p "$(dirname "$state")" && printf '%s\n' "$fingerprint" > "$state" +fi exit 0 diff --git a/tests/test_atuin_unit.py b/tests/test_atuin_unit.py index fb6a76f6..399a7f4a 100644 --- a/tests/test_atuin_unit.py +++ b/tests/test_atuin_unit.py @@ -11,13 +11,17 @@ ITEM = '{"login":{"username":"U","password":"P"},"fields":[{"name":"key","value":"K"}]}' -def stubs(bindir, status_out, status_rc=1, unlock_rc=0): +def stubs(bindir, status_out, status_rc=1, unlock_rc=0, version="18.24.0"): write_stub( bindir, "atuin", f"""echo "atuin $*" >> "$HOME/calls" case "$1" in status) echo '{status_out}'; exit {status_rc} ;; + --version) echo 'atuin {version}'; exit 0 ;; + info) echo "key path: $HOME/.local/share/atuin/key"; exit 0 ;; + key) cat "$HOME/.local/share/atuin/key"; exit $? ;; + login) [ "$*" = 'login -u U' ] && [ "$ATUIN_PASSWORD" = P ] && [ "$ATUIN_ENCRYPTION_KEY" = K ]; exit $? ;; esac exit 0 """, @@ -45,7 +49,88 @@ def test_logs_in_with_bitwarden_credentials(stub_env): stubs(bindir, "not logged in") r, calls = run(env) assert r.returncode == 0, r.stdout + r.stderr - assert "atuin login -u U -p P -k K" in calls + assert "atuin login -u U\n" in calls + assert " -p " not in calls and " -k " not in calls + + +def test_missing_key_login_keeps_imported_history_and_permissions(stub_env): + bindir, env = stub_env + data = Path(env["HOME"], ".local/share/atuin") + data.mkdir(parents=True, mode=0o755) + data.chmod(0o755) + (data / "history.db").write_bytes(b"existing history") + (data / "history.db-wal").write_bytes(b"pending history") + (data / ".history_imported").write_text("already imported") + stubs(bindir, "not logged in") + + r, calls = run(env) + + assert r.returncode == 0, r.stdout + r.stderr + assert "atuin login -u U\n" in calls + assert "atuin import" not in calls + assert (data / "history.db").read_bytes() == b"existing history" + assert (data / "history.db-wal").read_bytes() == b"pending history" + assert (data / ".history_imported").read_text() == "already imported" + assert data.stat().st_mode & 0o777 == 0o700 + + +def test_existing_different_key_is_never_replaced(stub_env): + bindir, env = stub_env + data = Path(env["HOME"], ".local/share/atuin") + data.mkdir(parents=True) + (data / "key").write_text("different key") + (data / "history.db").write_bytes(b"existing history") + stubs(bindir, "not logged in") + + r, calls = run(env) + + assert r.returncode == 0 + assert "existing Atuin key differs" in r.stdout + assert "atuin login" not in calls and "atuin import" not in calls + assert (data / "key").read_text() == "different key" + assert (data / "history.db").read_bytes() == b"existing history" + assert not (data / ".history_imported").exists() + + +def test_existing_matching_key_can_log_in(stub_env): + bindir, env = stub_env + data = Path(env["HOME"], ".local/share/atuin") + data.mkdir(parents=True) + (data / "key").write_text("K") + stubs(bindir, "not logged in") + + r, calls = run(env) + + assert r.returncode == 0, r.stdout + r.stderr + assert "atuin login -u U\n" in calls + assert (data / "key").read_text() == "K" + + +def test_old_version_requests_manual_login(stub_env): + bindir, env = stub_env + stubs(bindir, "not logged in", version="18.23.0") + + r, calls = run(env) + + assert r.returncode == 0 + assert "run atuin login interactively" in r.stdout + assert "atuin login" not in calls and "atuin import" not in calls + + +def test_no_secrets_keeps_history_without_touching_vault(stub_env): + bindir, env = stub_env + data = Path(env["HOME"], ".local/share/atuin") + data.mkdir(parents=True) + (data / "history.db").write_bytes(b"existing history") + (data / ".history_imported").touch() + stubs(bindir, "not logged in") + + r, calls = run({**env, "UNITS_NO_SECRETS": "1"}) + + assert r.returncode == 0 + assert "PARTIAL:" in r.stdout + assert "bw " not in calls and "atuin login" not in calls + assert (data / "history.db").read_bytes() == b"existing history" def test_already_logged_in_never_touches_bitwarden(stub_env): diff --git a/tests/test_chezmoi_unit.py b/tests/test_chezmoi_unit.py new file mode 100644 index 00000000..f5552c03 --- /dev/null +++ b/tests/test_chezmoi_unit.py @@ -0,0 +1,21 @@ +import shutil +import subprocess +from pathlib import Path + +from conftest import REPO + + +def test_missing_chezmoi_fails_with_install_guidance(stub_env): + bindir, env = stub_env + # Keep only the utilities used by the shared library, not the host's chezmoi. + for name in ("cat", "dirname", "hostname"): + Path(bindir, name).symlink_to(shutil.which(name)) + env = {**env, "PATH": str(bindir), "BREW_PREFIX": "/nonexistent", "DOTFILES_DIR": str(REPO)} + + result = subprocess.run( + ["/bin/bash", str(REPO / "units/chezmoi.sh")], env=env, capture_output=True, text=True + ) + + assert result.returncode != 0 + assert "chezmoi not installed" in result.stdout + assert any(hint in result.stdout for hint in ("just packages", "pkg install", "apk add")) diff --git a/tests/test_nvim_sync.py b/tests/test_nvim_sync.py new file mode 100644 index 00000000..1952a3f5 --- /dev/null +++ b/tests/test_nvim_sync.py @@ -0,0 +1,56 @@ +import subprocess +from pathlib import Path + +from conftest import REPO, write_stub + + +SCRIPT = REPO / "home/.chezmoiscripts/run_after_20-nvim-sync.sh" + + +def run(env): + return subprocess.run(["sh", str(SCRIPT)], env=env, capture_output=True, text=True) + + +def calls(home): + log = Path(home, "nvim-sync-calls") + return log.read_text().splitlines() if log.exists() else [] + + +def test_sync_runs_on_bootstrap_and_config_change_only(stub_env): + bindir, env = stub_env + home = Path(env["HOME"]) + config = home / ".config/nvim" + config.mkdir(parents=True) + (config / "init.lua").write_text("return 1\n") + write_stub(bindir, "git", 'echo clone >> "$HOME/nvim-sync-calls"\nmkdir -p "${@: -1}"\n') + write_stub(bindir, "nvim", 'echo sync >> "$HOME/nvim-sync-calls"\n') + + assert run(env).returncode == 0 + assert calls(home) == ["clone", "sync"] + assert run(env).returncode == 0 + assert calls(home) == ["clone", "sync"] + + plugin = config / "lua/plugins/new.lua" + plugin.parent.mkdir(parents=True) + plugin.write_text("return {}\n") + assert run(env).returncode == 0 + assert calls(home) == ["clone", "sync", "sync"] + plugin.unlink() + assert run(env).returncode == 0 + assert calls(home) == ["clone", "sync", "sync", "sync"] + + +def test_failed_sync_is_retried(stub_env): + bindir, env = stub_env + home = Path(env["HOME"]) + config = home / ".config/nvim" + config.mkdir(parents=True) + (config / "init.lua").write_text("return 1\n") + lazy = home / ".local/share/nvim/lazy/lazy.nvim" + lazy.mkdir(parents=True) + write_stub(bindir, "nvim", 'echo sync >> "$HOME/nvim-sync-calls"\nexit "${NVIM_RC:-0}"\n') + + assert run({**env, "NVIM_RC": "1"}).returncode == 0 + assert run(env).returncode == 0 + assert run(env).returncode == 0 + assert calls(home) == ["sync", "sync"] diff --git a/units/atuin.sh b/units/atuin.sh index 72c20be7..7b0a67ae 100755 --- a/units/atuin.sh +++ b/units/atuin.sh @@ -6,6 +6,7 @@ set -uo pipefail . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" data="$HOME/.local/share/atuin" +allow_import=1 if ! command -v atuin >/dev/null 2>&1; then note "SKIPPED: atuin not installed (brew install atuin)" @@ -13,6 +14,7 @@ if ! command -v atuin >/dev/null 2>&1; then fi step "atuin data dir" install -d -m 0700 "$data" +[ "$UNIT_FAILED" = 0 ] || finish logged_in() { local out @@ -21,7 +23,7 @@ logged_in() { } atuin_login() { - local s item u p k + local s item u p k version key_path line local_key s="$(bw_session)" if [ -z "$s" ]; then note "PARTIAL: atuin not logged in; vault locked (bw unlock, then just apply-tags atuin)" @@ -35,9 +37,41 @@ atuin_login() { note "PARTIAL: Bitwarden item atuin.sh missing or incomplete; atuin login skipped" return 0 fi - # atuin expects a binary key file; `login -k` regenerates it from the mnemonic, so reset first. - [ -e "$data/key" ] || rm -rf "$data" - atuin login -u "$u" -p "$p" -k "$k" || return 1 + # Atuin gained non-interactive environment credentials in 18.24. Older + # versions need a manual login; never expose the password or key in argv. + version="$(atuin --version 2>/dev/null)" + if [[ ! "$version" =~ ([0-9]+)\.([0-9]+)\.([0-9]+) ]] || + (( 10#${BASH_REMATCH[1]} < 18 || + (10#${BASH_REMATCH[1]} == 18 && 10#${BASH_REMATCH[2]} < 24) )); then + note "PARTIAL: atuin 18.24 or newer required for unattended login; upgrade or run atuin login interactively" + allow_import=0 + return 0 + fi + + # Let Atuin resolve its configured key path, including the newer data layout. + key_path="" + while IFS= read -r line; do + case "$line" in 'key path: '*) key_path="${line#key path: }" ;; esac + done <<< "$(atuin info 2>/dev/null)" + if [ -z "$key_path" ]; then + note "PARTIAL: cannot determine the Atuin key path; inspect the local data and log in interactively" + allow_import=0 + return 0 + fi + if [ -e "$key_path" ] || [ -L "$key_path" ]; then + local_key="$(atuin key 2>/dev/null)" || { + note "PARTIAL: cannot read the existing Atuin key; inspect it and log in interactively" + allow_import=0 + return 0 + } + if [ "$local_key" != "$k" ]; then + note "PARTIAL: existing Atuin key differs from the vault; inspect it and log in interactively" + allow_import=0 + return 0 + fi + fi + # Keep both the history and any existing key; Atuin writes a missing key. + ATUIN_PASSWORD="$p" ATUIN_ENCRYPTION_KEY="$k" atuin login -u "$u" &1 && mkdir -p ~/.local/share/atuin && touch ~/.local/share/atuin/.history_imported' +if [ "$allow_import" = 1 ] && [ ! -e "$data/.history_imported" ]; then + soft "atuin import" sh -c 'atuin import auto 2>&1 && touch "$1/.history_imported"' sh "$data" fi finish diff --git a/units/chezmoi.sh b/units/chezmoi.sh index 2765a44a..4f5c8d91 100755 --- a/units/chezmoi.sh +++ b/units/chezmoi.sh @@ -8,8 +8,14 @@ if ! command -v chezmoi >/dev/null 2>&1 && [ "$IS_TERMUX" = 1 ]; then soft "pkg install chezmoi" pkg install -y chezmoi fi if ! command -v chezmoi >/dev/null 2>&1; then - note "SKIPPED: chezmoi not installed (run: just packages)" - exit 0 + if [ "$IS_TERMUX" = 1 ]; then + note "FAIL: chezmoi not installed; run: pkg install chezmoi" + elif [ "$IS_MUSL" = 1 ]; then + note "FAIL: chezmoi not installed; on Alpine/postmarketOS enable the community repository and run: sudo apk add chezmoi" + else + note "FAIL: chezmoi not installed; run: just packages" + fi + exit 1 fi is_desktop=0 From 2af5c14fb8c89f332ad7917842f82926157183b4 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:15:17 +0000 Subject: [PATCH 10/62] docs: correct Himachal Hive operations status Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- host_vars/himachal.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/host_vars/himachal.yml b/host_vars/himachal.yml index aa7df1dd..453fb2fd 100644 --- a/host_vars/himachal.yml +++ b/host_vars/himachal.yml @@ -8,7 +8,6 @@ phone_apply_enabled: false web_services: [] mic_hardware_volume_pct: 75 -# Hive ops (rotation, watchdog, peak windows) now run as CronJobs inside the -# AWS cluster; himachal no longer drives the Hive. hive_ops removes its old -# user timers on the next apply. +# The AWS cluster was terminated; local Hive ops remain disabled. +# The role removes any old local timers on the next apply. hive_ops_enabled: false From 466b0a00a3102c058e0e86950b898298f2aee112 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:16:55 +0000 Subject: [PATCH 11/62] fix: keep remote Bitwarden sessions out of command lines Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- Justfile | 16 +++++++++++----- docs/src/architecture.md | 2 +- docs/src/bitwarden.md | 4 +--- docs/src/roles/sshd.md | 9 ++------- scripts/bw-resolve.sh | 6 +++--- tests/test_tagged_apply.py | 26 ++++++++++++++++++++++++++ 6 files changed, 44 insertions(+), 19 deletions(-) diff --git a/Justfile b/Justfile index 1237f152..f10d5aaa 100644 --- a/Justfile +++ b/Justfile @@ -120,9 +120,12 @@ apply-remote-tags name tags: break ;; esac done - ssh -t {{ name }} " + EXPECT_SESSION=0; [ -z "$REMOTE_BW_SESSION" ] || EXPECT_SESSION=1 + BW_SESSION="$REMOTE_BW_SESSION" ssh -o SendEnv=BW_SESSION -t {{ name }} " export PATH=\"\$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:\$PATH\" - export BW_SESSION='${REMOTE_BW_SESSION}' + if [ '$EXPECT_SESSION' = 1 ] && [ -z \"\${BW_SESSION:-}\" ]; then + echo 'SSH server did not accept BW_SESSION; remote apply aborted.' >&2; exit 1 + fi cd ~/.local/share/dotfiles && git pull --ff-only && just apply-tags {{ tags }} " @@ -160,10 +163,13 @@ apply-remote name *args: echo "Bitwarden unlocked on {{ name }} (session: ${#APPLY_BW_SESSION} chars)" fi echo "Applying to {{ name }} with forwarded BW session..." - ssh -t {{ name }} " + EXPECT_SESSION=0; [ -z "$APPLY_BW_SESSION" ] || EXPECT_SESSION=1 + BW_SESSION="$APPLY_BW_SESSION" ssh -o SendEnv=BW_SESSION -t {{ name }} " export PATH=\"\$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:\$PATH\" - export BW_SESSION='${APPLY_BW_SESSION}' - [ -n \"\$BW_SESSION\" ] && (umask 077 && printf '%s' \"\$BW_SESSION\" > /tmp/bw_session) + if [ '$EXPECT_SESSION' = 1 ] && [ -z \"\${BW_SESSION:-}\" ]; then + echo 'SSH server did not accept BW_SESSION; remote apply aborted.' >&2; exit 1 + fi + [ -n \"\${BW_SESSION:-}\" ] && (umask 077 && printf '%s' \"\$BW_SESSION\" > /tmp/bw_session) cd ~/.local/share/dotfiles && git pull --ff-only && just apply {{ args }} " just push-terminfo {{ name }} diff --git a/docs/src/architecture.md b/docs/src/architecture.md index 12f3855b..8bb6be1e 100644 --- a/docs/src/architecture.md +++ b/docs/src/architecture.md @@ -151,6 +151,6 @@ Every flag `site.yml` reads, all defaulting to `false` (`| default(false)`). Set 1. Ansible requests a Bitwarden session (env var, cached file, or interactive unlock) 2. The `bitwarden` role resolves `BW_SESSION` and caches it 3. Secrets-tagged roles consume the session token for API calls -4. For remote applies, `scripts/bw-resolve.sh remote ` unlocks the vault on the target and the recipe exports that session inline over `ssh -t` (no `SendEnv`) — see [Remote Apply](bitwarden.md#remote-apply) +4. For remote applies, `scripts/bw-resolve.sh remote ` unlocks the vault on the target and SSH forwards that session using `SendEnv=BW_SESSION`; the target must accept it — see [Remote Apply](bitwarden.md#remote-apply). No secrets are stored in git — the repo is fully public. diff --git a/docs/src/bitwarden.md b/docs/src/bitwarden.md index c8de2d48..22c34724 100644 --- a/docs/src/bitwarden.md +++ b/docs/src/bitwarden.md @@ -24,9 +24,7 @@ For `just apply-tags` with an explicit secret-bearing tag such as `kube`, `ai` o out it tries `bw login --apikey` with the `bw-api-key` item from your local vault; to unlock it pipes the master password (item `James Bitwarden`) over stdin, falling back to an interactive `bw unlock` over `ssh -t`. -2. Runs `ssh -t "export BW_SESSION='…'; …; just apply"` — the session - is exported inline in the remote command, not via `SendEnv`. `apply-remote` - also caches it in the remote's `/tmp/bw_session`. +2. Sends the remote session through SSH `SendEnv=BW_SESSION`, not in the command line. The target SSH daemon must accept `BW_SESSION`; if it rejects a resolved session, the apply aborts. `apply-remote` also caches it in the remote's `/tmp/bw_session`. 3. The `bitwarden` role on the remote runs `bw status`: if it reports `unlocked`, `bw_unlocked` flips to true and secrets work proceeds. diff --git a/docs/src/roles/sshd.md b/docs/src/roles/sshd.md index 15f82757..98ef0fb1 100644 --- a/docs/src/roles/sshd.md +++ b/docs/src/roles/sshd.md @@ -4,10 +4,7 @@ **Secrets needed:** No **Runs on:** All machines -Configures the SSH daemon to accept Bitwarden environment variables from -clients that explicitly use SSH environment forwarding. The current -`just apply-remote` recipe does not depend on this drop-in: it resolves the -session on the target and exports it inside the remote command. +Configures the SSH daemon to accept Bitwarden environment variables. `just apply-remote` resolves the session on the target and forwards it back using `SendEnv=BW_SESSION` without putting it in the remote command line. ## What It Does @@ -17,9 +14,7 @@ Drops a config file at `/etc/ssh/sshd_config.d/99-bw-env.conf`: AcceptEnv BW_SESSION BW_CLIENTID BW_CLIENTSECRET ``` -This drop-in is compatibility for explicit `SendEnv`/`AcceptEnv` workflows. It -is not required by `just apply-remote`, which uses target-side resolution -instead of SSH environment forwarding. +The drop-in is required for session-bearing remote applies; those applies fail if the target rejects `BW_SESSION`. ## Handlers diff --git a/scripts/bw-resolve.sh b/scripts/bw-resolve.sh index e343f3de..a43a55c9 100755 --- a/scripts/bw-resolve.sh +++ b/scripts/bw-resolve.sh @@ -47,8 +47,8 @@ if [ "$mode" = "remote" ]; then # Get local BW session for API key lookup local_session=$("$SCRIPT_DIR/bw-unlock.sh" 2>/dev/null || true) if [ -n "$local_session" ]; then - BW_CLIENTID=$(bw get username bw-api-key --session "$local_session" 2>/dev/null || true) - BW_CLIENTSECRET=$(bw get password bw-api-key --session "$local_session" 2>/dev/null || true) + BW_CLIENTID=$(BW_SESSION="$local_session" bw get username bw-api-key 2>/dev/null || true) + BW_CLIENTSECRET=$(BW_SESSION="$local_session" bw get password bw-api-key 2>/dev/null || true) if [ -n "$BW_CLIENTID" ] && [ -n "$BW_CLIENTSECRET" ]; then # Credentials go over stdin (never on either side's command line), and # bw's chatter goes to stderr so it can't pollute the session that @@ -72,7 +72,7 @@ if [ "$mode" = "remote" ]; then # Try non-interactive unlock using master password from local vault local_session=$("$SCRIPT_DIR/bw-unlock.sh" 2>/dev/null || true) if [ -n "$local_session" ]; then - BW_MASTER_PASS=$(bw get password "James Bitwarden" --session "$local_session" 2>/dev/null || true) + BW_MASTER_PASS=$(BW_SESSION="$local_session" bw get password "James Bitwarden" 2>/dev/null || true) if [ -n "$BW_MASTER_PASS" ]; then echo "Unlocking Bitwarden on $host (non-interactive)..." >&2 # Password is piped over stdin and read into a remote env var via diff --git a/tests/test_tagged_apply.py b/tests/test_tagged_apply.py index 202613b4..f4684ec7 100644 --- a/tests/test_tagged_apply.py +++ b/tests/test_tagged_apply.py @@ -96,3 +96,29 @@ def test_full_apply_skips_vault_units_when_unlock_fails(apply_env): calls = log.read_text().splitlines() assert "--skip-tags secrets" in calls[-2] assert calls[-1] == "units --no-secrets" + + +@pytest.mark.parametrize("drop_session", [False, True]) +def test_remote_secret_apply_rejects_lost_ssh_session(apply_env, drop_session): + env, repo, _ = apply_env + bindir = Path(env["HOME"]) / ".local/bin" + write_stub(repo / "scripts", "bw-resolve.sh", 'printf "sentinel-session"\n') + write_stub(bindir, "ssh", ''' +case "$*" in *sentinel-session*) exit 90 ;; esac +if [ "${DROP_SESSION:-0}" = 1 ]; then unset BW_SESSION; fi +exec bash -c "${!#}" +''') + write_stub(bindir, "just", ''' +[ "${BW_SESSION:-}" = sentinel-session ] || exit 91 +touch "$HOME/remote-applied" +''') + env["DROP_SESSION"] = "1" if drop_session else "0" + result = subprocess.run( + [shutil.which("just"), "-f", str(repo / "Justfile"), "apply-remote-tags", "phone", "omp"], + env=env, capture_output=True, text=True, + ) + assert (result.returncode == 0) is not drop_session + assert (Path(env["HOME"]) / "remote-applied").exists() is not drop_session + assert "sentinel-session" not in result.stdout + result.stderr + if drop_session: + assert "did not accept BW_SESSION" in result.stderr From b66cd4bb12a1f7d8d241d90ba3f0d838e0605685 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:22:10 +0000 Subject: [PATCH 12/62] fix: preserve GitHub config and fail closed on key lookup Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/roles/github.md | 8 +++--- tests/test_github_unit.py | 56 ++++++++++++++++++++++++++++++++++----- units/github.sh | 13 ++++++--- 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/docs/src/roles/github.md b/docs/src/roles/github.md index 9e4a16d9..c8bf9686 100644 --- a/docs/src/roles/github.md +++ b/docs/src/roles/github.md @@ -5,14 +5,15 @@ **Secrets needed:** Only on first auth; once `gh` is authenticated Bitwarden is not used **Runs on:** All machines with `gh` installed (`SKIPPED:` otherwise) -Logs `gh` in via a Bitwarden-stored PAT (only if not already authenticated), then registers this host's SSH key with GitHub for both authentication and signing. +Logs `gh` in via a Bitwarden-stored PAT (only if not already authenticated), then registers this host's SSH key with GitHub for both authentication and signing. With `--no-secrets`, only the local known-hosts step runs; the unit does not invoke `gh` or Bitwarden. ## Steps 1. `github known_hosts`: ensure github.com's ed25519 host key is in `~/.ssh/known_hosts`. 2. `gh login`: skipped when `gh auth status` succeeds. Otherwise, with an unlocked vault, `bw get password github-token` → `gh auth login --with-token`. Locked vault or missing item prints a `PARTIAL:` line. -3. `git protocol ssh` (soft): `gh config set git_protocol ssh`. -4. `register SSH keys`: needs a token with `admin:public_key` and `admin:ssh_signing_key` (otherwise a `PARTIAL:` line with the regenerate-PAT link). For each of authentication and signing, `gh ssh-key add ~/.ssh/id_ed25519.pub --title -auth|-signing` unless `gh ssh-key list` already shows the key with that type. +3. `register SSH keys`: needs a token with `admin:public_key` and `admin:ssh_signing_key` (otherwise a `PARTIAL:` line with the regenerate-PAT link). For each of authentication and signing, `gh ssh-key add ~/.ssh/id_ed25519.pub --title -auth|-signing` unless `gh ssh-key list` already shows the key with that type. If the list fails, the unit fails without registering either key. + +GitHub CLI configuration, including `git_protocol: ssh`, is managed by chezmoi, not this unit. ## Failure modes @@ -21,6 +22,7 @@ Logs `gh` in via a Bitwarden-stored PAT (only if not already authenticated), the | `PARTIAL: gh not logged in; vault locked` | First run on this host with Bitwarden locked | `bw unlock`, then `just apply-tags github`, or `gh auth login` | | `PARTIAL: no 'github-token' item` | Vault has no PAT | Create a login item `github-token` with the PAT as the password | | `PARTIAL: GitHub token lacks admin:public_key / admin:ssh_signing_key` | PAT scopes | Regenerate the PAT with those scopes and update the item | +| `warn: failed to list GitHub SSH keys; registration skipped` | `gh ssh-key list` failed | Check GitHub access and re-run; no keys were added | | `warn: failed to upload SSH ... key` | `gh ssh-key add` failed | Inspect with `gh ssh-key list` and re-run | ## Verify diff --git a/tests/test_github_unit.py b/tests/test_github_unit.py index 00ff3627..d41fb0c3 100644 --- a/tests/test_github_unit.py +++ b/tests/test_github_unit.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from conftest import write_stub @@ -13,7 +15,8 @@ [ "$3" = "-t" ] || exit "${STUB_AUTHED:-0}" exit 0 ;; "auth login") cat > "$HOME/token"; exit 0 ;; - "ssh-key list") printf '%s' "$STUB_KEYS"; exit 0 ;; + "ssh-key list") printf '%s' "$STUB_KEYS"; exit "${STUB_LIST_RC:-0}" ;; + "config set") mkdir -p "$HOME/.config/gh"; echo 'git_protocol: ssh' > "$HOME/.config/gh/config.yml" ;; esac exit 0 """ @@ -24,22 +27,23 @@ def env(stub_env): bindir, env = stub_env write_stub(bindir, "bw", UNLOCKED_BW % '[ "$1 $2 $3" = "get password github-token" ] && { echo ghp_secret; exit 0; }') write_stub(bindir, "gh", GH) - ssh = __import__("pathlib").Path(env["HOME"], ".ssh") + ssh = Path(env["HOME"], ".ssh") ssh.mkdir() (ssh / "id_ed25519.pub").write_text(PUB) - env.update(BW_SESSION="x", DOTFILES_MACHINE="testbox", STUB_SCOPES="admin:public_key, admin:ssh_signing_key", STUB_KEYS="") + env.update( + BW_SESSION="x", DOTFILES_MACHINE="testbox", UNITS_NO_SECRETS="0", + STUB_AUTHED="0", STUB_UNLOCK_RC="0", STUB_LIST_RC="0", + STUB_SCOPES="admin:public_key, admin:ssh_signing_key", STUB_KEYS="", + ) return env def test_logs_in_with_the_bitwarden_token_and_registers_both_keys(env): - from pathlib import Path - r = run_unit("github", env, STUB_AUTHED="1") assert r.returncode == 0, r.stdout + r.stderr assert Path(env["HOME"], "token").read_text() == "ghp_secret" c = calls(env) assert "ssh-key add" in c and "--title testbox-auth --type authentication" in c and "--title testbox-signing --type signing" in c - assert "git_protocol ssh" in c assert "AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" in Path(env["HOME"], ".ssh/known_hosts").read_text() @@ -66,3 +70,43 @@ def test_locked_vault_does_not_log_in(env): r = run_unit("github", env, STUB_AUTHED="1", STUB_UNLOCK_RC="1") assert r.returncode == 0 and "PARTIAL: gh not logged in" in r.stdout assert "auth login" not in calls(env) + + +@pytest.mark.parametrize("authed", ["0", "1"]) +def test_no_secrets_skips_all_github_and_vault_operations(env, authed): + r = run_unit("github", env, UNITS_NO_SECRETS="1", STUB_AUTHED=authed) + assert r.returncode == 0, r.stdout + r.stderr + assert "SKIPPED: GitHub authentication and SSH key registration" in r.stdout + assert calls(env) == "" + assert "github.com ssh-ed25519" in Path(env["HOME"], ".ssh/known_hosts").read_text() + + +def test_no_secrets_preserves_known_hosts_failure(env): + Path(env["HOME"], ".ssh/known_hosts").mkdir() + r = run_unit("github", env, UNITS_NO_SECRETS="1") + assert r.returncode != 0 + assert "FAIL github known_hosts" in r.stdout + assert "SKIPPED: GitHub authentication and SSH key registration" in r.stdout + assert calls(env) == "" + + +@pytest.mark.parametrize("authed", ["0", "1"]) +def test_gh_config_is_left_for_chezmoi(env, authed): + config = Path(env["HOME"], ".config/gh/config.yml") + config.parent.mkdir(parents=True) + original = "version: 1\ngit_protocol: https\nprompt: enabled\n" + config.write_text(original) + r = run_unit("github", env, STUB_AUTHED=authed) + assert r.returncode == 0, r.stdout + r.stderr + assert config.read_text() == original + assert "gh config set" not in calls(env) + + +@pytest.mark.parametrize("partial_output", ["", "testbox-auth\tssh-ed25519 AAAAKEYBODY\tauthentication\n"]) +def test_failed_key_list_never_registers_even_with_partial_output(env, partial_output): + r = run_unit("github", env, STUB_LIST_RC="1", STUB_KEYS=partial_output) + assert r.returncode != 0 + assert "failed to list GitHub SSH keys; registration skipped" in r.stdout + assert "FAIL register SSH keys" in r.stdout + assert "ssh-key list" in calls(env) + assert "ssh-key add" not in calls(env) diff --git a/units/github.sh b/units/github.sh index b1e556db..2cddf328 100755 --- a/units/github.sh +++ b/units/github.sh @@ -46,7 +46,7 @@ register_key() { } register_keys() { - local scopes key_body + local scopes key_body key_list scopes="$(gh auth status -t 2>&1 | grep -o 'Token scopes:.*' || true)" case "$scopes" in *admin:public_key*admin:ssh_signing_key*|*admin:ssh_signing_key*admin:public_key*) ;; @@ -56,13 +56,20 @@ register_keys() { esac [ -f "$pub" ] || { note "warn: $pub missing; no SSH key to register"; return 1; } key_body="$(awk '{print $2}' "$pub")" - key_lines="$(gh ssh-key list 2>/dev/null | grep -F "$key_body" || true)" + if ! key_list="$(gh ssh-key list 2>/dev/null)"; then + note "warn: failed to list GitHub SSH keys; registration skipped" + return 1 + fi + key_lines="$(grep -F -- "$key_body" <<<"$key_list" || true)" register_key authentication auth && register_key signing signing } key_lines="" step "github known_hosts" known_hosts +if [ "${UNITS_NO_SECRETS:-0}" = 1 ]; then + note "SKIPPED: GitHub authentication and SSH key registration (--no-secrets)" + finish +fi step "gh login" gh_login -soft "git protocol ssh" gh config set git_protocol ssh step "register SSH keys" register_keys finish From 845c3f45a8eb4caa46abc4b0bfd4b4319359a163 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:26:18 +0000 Subject: [PATCH 13/62] fix: preserve remote phone identities during unit applies Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- roles/systemd/templates/phone-apply.service.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/systemd/templates/phone-apply.service.j2 b/roles/systemd/templates/phone-apply.service.j2 index 3bbc3f19..d7c23e07 100644 --- a/roles/systemd/templates/phone-apply.service.j2 +++ b/roles/systemd/templates/phone-apply.service.j2 @@ -14,8 +14,8 @@ ExecStart=-{{ brew_prefix }}/bin/ansible-playbook -i inventory.yml -e target=ter ExecStart=-{{ brew_prefix }}/bin/ansible-playbook -i inventory.yml -e target=mumbai site.yml --skip-tags secrets # Run the same tracked revision on each target, even if its Ansible pass failed. # An archive avoids assuming the phones have a checkout or Ansible installed. -ExecStart=-:/bin/bash -c "set -o pipefail; /usr/bin/git archive HEAD | /usr/bin/ssh -o BatchMode=yes termux 'mkdir -p \"$HOME/.cache\"; d=$(mktemp -d \"$HOME/.cache/dotfiles-phone.XXXXXXXX\") || exit 1; tar -xf - -C \"$d\" && bash \"$d/scripts/run-units.sh\" --no-secrets; rc=$?; rm -rf -- \"$d\"; exit \"$rc\"'" -ExecStart=-:/bin/bash -c "set -o pipefail; /usr/bin/git archive HEAD | /usr/bin/ssh -o BatchMode=yes mumbai 'mkdir -p \"$HOME/.cache\"; d=$(mktemp -d \"$HOME/.cache/dotfiles-phone.XXXXXXXX\") || exit 1; tar -xf - -C \"$d\" && bash \"$d/scripts/run-units.sh\" --no-secrets; rc=$?; rm -rf -- \"$d\"; exit \"$rc\"'" +ExecStart=-:/bin/bash -c "set -o pipefail; /usr/bin/git archive HEAD | /usr/bin/ssh -o BatchMode=yes termux 'mkdir -p \"$HOME/.cache\"; d=$(mktemp -d \"$HOME/.cache/dotfiles-phone.XXXXXXXX\") || exit 1; tar -xf - -C \"$d\" && DOTFILES_MACHINE=termux bash \"$d/scripts/run-units.sh\" --no-secrets; rc=$?; rm -rf -- \"$d\"; exit \"$rc\"'" +ExecStart=-:/bin/bash -c "set -o pipefail; /usr/bin/git archive HEAD | /usr/bin/ssh -o BatchMode=yes mumbai 'mkdir -p \"$HOME/.cache\"; d=$(mktemp -d \"$HOME/.cache/dotfiles-phone.XXXXXXXX\") || exit 1; tar -xf - -C \"$d\" && DOTFILES_MACHINE=mumbai bash \"$d/scripts/run-units.sh\" --no-secrets; rc=$?; rm -rf -- \"$d\"; exit \"$rc\"'" Environment="PATH={{ brew_prefix }}/bin:%h/.local/bin:/usr/local/bin:/usr/bin:/bin" Environment="HOME={{ ansible_facts['env']['HOME'] }}" StandardOutput=journal From 6b6cc04c942a61ce78a2b7b6e3ba5dcca11e2fcb Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:45:15 +0000 Subject: [PATCH 14/62] refactor: migrate fleet configuration to native units Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- .ansible-lint | 38 - .github/workflows/ci.yml | 36 +- CLAUDE.md | 31 +- CONTEXT.md | 25 +- CONTRIBUTING.md | 38 +- Justfile | 120 ++-- README.md | 24 +- ansible.cfg | 25 - bootstrap.sh | 98 +-- docs/scripts.md | 18 +- docs/src/architecture.md | 153 +--- docs/src/bitwarden.md | 48 +- docs/src/cluster/hive.md | 291 +++----- docs/src/index.md | 32 +- docs/src/inventory.md | 13 +- docs/src/onboarding.md | 32 +- docs/src/operational-readiness.md | 153 +--- docs/src/roles/apk_packages.md | 18 +- docs/src/roles/bitwarden.md | 54 +- docs/src/roles/bluefin_common.md | 42 +- docs/src/roles/browser_fxa.md | 33 +- docs/src/roles/bst_dashboard.md | 21 +- docs/src/roles/flatpak.md | 37 +- docs/src/roles/fleet_facts.md | 39 +- docs/src/roles/gnome.md | 50 +- docs/src/roles/hive_ops.md | 66 +- docs/src/roles/homebrew.md | 33 +- docs/src/roles/kirocrew.md | 15 +- docs/src/roles/pipewire_audio.md | 87 +-- docs/src/roles/proxy.md | 26 +- docs/src/roles/retired_services.md | 60 +- docs/src/roles/server_hardening.md | 22 +- docs/src/roles/shell_fonts.md | 2 +- docs/src/roles/ssh_keys.md | 54 +- docs/src/roles/ssh_mesh.md | 52 +- docs/src/roles/sshd.md | 25 +- docs/src/roles/sudo.md | 17 +- docs/src/roles/syncthing.md | 17 +- docs/src/roles/systemd.md | 45 +- docs/src/roles/tailscale.md | 37 +- docs/src/roles/tailscale_cert.md | 18 +- docs/src/roles/termux_packages.md | 29 +- docs/src/roles/zen_browser.md | 16 +- docs/src/runbook.md | 57 +- docs/src/scripts.md | 304 +------- docs/src/units.md | 56 +- group_vars/all.yml | 5 - host_vars/himachal.yml | 14 + mise.toml | 82 ++- requirements.yml | 8 - roles/apk_packages/tasks/main.yml | 103 --- roles/bitwarden/tasks/main.yml | 98 --- roles/bluefin_common/tasks/main.yml | 224 ------ roles/browser_fxa/README.md | 77 -- roles/browser_fxa/meta/main.yml | 26 - roles/browser_fxa/tasks/main.yml | 170 ----- roles/bst_dashboard/handlers/main.yml | 5 - roles/bst_dashboard/tasks/main.yml | 32 - roles/flatpak/tasks/main.yml | 87 --- roles/fleet_facts/tasks/main.yml | 118 ---- roles/gnome/handlers/main.yml | 3 - roles/gnome/tasks/main.yml | 159 ----- roles/hive_ops/tasks/main.yml | 119 ---- roles/homebrew/handlers/main.yml | 2 - roles/homebrew/tasks/main.yml | 183 ----- roles/homebrew/templates/Brewfile.j2 | 37 - roles/kirocrew/tasks/main.yml | 57 -- roles/osquery/handlers/main.yml | 7 - roles/osquery/tasks/install.yml | 151 ---- roles/osquery/tasks/main.yml | 31 - roles/osquery/tasks/poller.yml | 97 --- .../osquery/templates/poller-inventory.yml.j2 | 22 - roles/pipewire_audio/files/mic-dsp.conf | 85 --- roles/pipewire_audio/handlers/main.yml | 39 - roles/pipewire_audio/tasks/main.yml | 227 ------ roles/proxy/handlers/main.yml | 13 - roles/proxy/tasks/main.yml | 76 -- roles/retired_services/tasks/main.yml | 98 --- roles/retired_services/vars/main.yml | 11 - roles/server_hardening/handlers/main.yml | 13 - roles/server_hardening/tasks/main.yml | 144 ---- roles/ssh_keys/tasks/main.yml | 306 -------- roles/ssh_keys/templates/ssh_config.j2 | 24 - roles/ssh_mesh/tasks/main.yml | 137 ---- roles/sshd/handlers/main.yml | 9 - roles/sshd/tasks/main.yml | 37 - roles/sudo/tasks/main.yml | 17 - roles/syncthing/handlers/main.yml | 5 - roles/syncthing/tasks/main.yml | 39 - roles/systemd/handlers/main.yml | 13 - roles/systemd/tasks/main.yml | 206 ------ .../systemd/templates/phone-apply.service.j2 | 25 - roles/tailscale/handlers/main.yml | 12 - roles/tailscale/tasks/configure_connected.yml | 61 -- roles/tailscale/tasks/configure_prereqs.yml | 55 -- roles/tailscale/tasks/connect.yml | 72 -- roles/tailscale/tasks/install.yml | 34 - roles/tailscale/tasks/main.yml | 27 - roles/tailscale_cert/handlers/main.yml | 6 - roles/tailscale_cert/tasks/main.yml | 90 --- roles/termux_packages/defaults/main.yml | 30 - roles/termux_packages/tasks/main.yml | 70 -- .../templates/start-sshd.sh.j2 | 10 - roles/zen_browser/tasks/main.yml | 107 --- scripts/online_hosts.py | 5 +- scripts/osquery-fleet.py | 9 +- scripts/record-apply.py | 4 +- scripts/run-units.sh | 143 +++- site.yml | 151 ---- talos-k8s/hive/ops/README.md | 4 +- tests/test_bluefin_sync.py | 2 +- tests/test_bluefin_unit.py | 306 ++++++++ tests/test_bluefin_wallpaper_rotate.py | 2 +- tests/test_bootstrap_units.py | 40 +- tests/test_brew_flatpak_check_mode.py | 287 ++++++++ tests/test_browser_fxa_check_mode.py | 164 +++++ tests/test_browser_fxa_unit.py | 112 +++ tests/test_check_small_packages_vpn.py | 223 ++++++ tests/test_fleet_dotfiles_check.py | 139 ++++ tests/test_fleet_facts_check.py | 48 ++ tests/test_fleet_facts_native.py | 142 ++++ tests/test_forgejo_check_credentials.py | 9 + tests/test_github_check_credentials.py | 21 + tests/test_gnome_unit.py | 257 +++++++ tests/test_hive_ops_migration.py | 90 +++ tests/test_hive_peak.py | 2 +- tests/test_hive_syncthing_check.py | 143 ++++ tests/test_install_extensions.py | 4 +- tests/test_kube_check_credentials.py | 26 + tests/test_laptop_services_migration.py | 135 ++++ tests/test_laptop_system_services_check.py | 315 +++++++++ tests/test_osquery_check.py | 55 ++ tests/test_osquery_drift.py | 2 +- tests/test_osquery_native_inventory.py | 95 +++ tests/test_package_native_units.py | 345 +++++++++ tests/test_pipewire_audio_check_mode.py | 174 +++++ tests/test_pipewire_audio_unit.py | 331 +++++++++ tests/test_platform_native_units.py | 326 +++++++++ tests/test_retired_host_units.py | 20 + tests/test_security_native_units.py | 112 +++ tests/test_server_service_units.py | 221 ++++++ tests/test_server_units_check.py | 148 ++++ tests/test_shell_ai_check_credentials.py | 24 + tests/test_ssh_check_credentials.py | 40 ++ tests/test_ssh_native_units.py | 274 +++++++ tests/test_sudo_sshd_check_units.py | 111 +++ tests/test_syncthing_migration.py | 53 ++ tests/test_system_services_migration.py | 127 ++++ tests/test_tagged_apply.py | 13 +- tests/test_unit_tags.py | 17 + tests/test_units_registry.py | 36 - tests/test_visual_units_check.py | 335 +++++++++ tests/test_zen_browser_unit.py | 212 ++++++ units/apk-packages.conf | 9 + units/apk-packages.sh | 112 +++ units/atuin.sh | 5 + units/bluefin.sh | 345 +++++++++ .../bluefin}/files/bluefin-sync.py | 0 .../files/bluefin-wallpaper-rotate.py | 0 .../bluefin}/files/install-extensions.py | 0 .../bluefin-wallpaper-rotate.service.j2 | 0 .../bluefin-wallpaper-rotate.timer.j2 | 0 units/browser-fxa.sh | 208 ++++++ .../browser-fxa/defaults.yml | 0 .../files => units/browser-fxa}/fxa_signin.py | 0 units/bst-dashboard.sh | 75 ++ .../bst-dashboard}/bst-dashboard.service.j2 | 0 units/chezmoi.sh | 40 ++ .../main.yml => units/flatpak-packages.yml | 0 units/flatpak.sh | 307 ++++++++ units/fleet-facts.py | 307 ++++++++ units/fleet-facts.sh | 15 + .../fleet-facts/defaults.yml | 0 units/fonts.sh | 33 + units/forgejo-registry.sh | 4 + units/github.sh | 36 + units/gnome-avatar.py | 102 +++ units/gnome.sh | 346 +++++++++ .../copyous@boerdereinar.dev/extension.js | 0 .../scalable/actions/character-symbolic.svg | 0 .../actions/check-round-outline-symbolic.svg | 0 .../actions/clipboard-disabled-symbolic.svg | 0 .../scalable/actions/clipboard-symbolic.svg | 0 .../scalable/actions/color-symbolic.svg | 0 .../scalable/actions/duration-symbolic.svg | 0 .../scalable/actions/file-symbolic.svg | 0 .../actions/folder-code-legacy-symbolic.svg | 0 .../scalable/actions/folder-symbolic.svg | 0 .../scalable/actions/image-symbolic.svg | 0 .../actions/incognito-disabled-symbolic.svg | 0 .../scalable/actions/incognito-symbolic.svg | 0 .../scalable/actions/keyboard-symbolic.svg | 0 .../scalable/actions/link-symbolic.svg | 0 .../hicolor/scalable/actions/pin-symbolic.svg | 0 .../actions/search-clipboard-symbolic.svg | 0 .../scalable/actions/settings-symbolic.svg | 0 .../hicolor/scalable/actions/tag-symbolic.svg | 0 .../scalable/actions/text-symbolic.svg | 0 .../lib/common/actions.js | 0 .../lib/common/color.js | 0 .../lib/common/constants.js | 0 .../lib/common/dbus.js | 0 .../lib/common/gjs.js | 0 .../lib/common/glob.js | 0 .../lib/common/icons.js | 0 .../lib/common/settings.js | 0 .../lib/common/sound.js | 0 .../lib/database/database.js | 0 .../lib/database/entryTracker.js | 0 .../lib/database/gda.js | 0 .../lib/database/json.js | 0 .../lib/database/memory.js | 0 .../lib/misc/actor.js | 0 .../lib/misc/clipboard.js | 0 .../lib/misc/compatibility.js | 0 .../lib/misc/keyboard.js | 0 .../copyous@boerdereinar.dev/lib/misc/link.js | 0 .../lib/misc/notifications.js | 0 .../lib/misc/shortcuts.js | 0 .../lib/misc/theme.js | 0 .../lib/preferences/actions/actionDefaults.js | 0 .../lib/preferences/actions/actionDialog.js | 0 .../actions/actionSubMenuDialog.js | 0 .../lib/preferences/actions/actionsGroup.js | 0 .../lib/preferences/actions/actionsPage.js | 0 .../customization/dialogCustomization.js | 0 .../customization/headerCustomization.js | 0 .../customization/itemCustomization.js | 0 .../items/characterItemCustomization.js | 0 .../items/codeItemCustomization.js | 0 .../customization/items/exclusions.js | 0 .../items/fileItemCustomization.js | 0 .../items/imageItemCustomization.js | 0 .../items/linkItemCustomization.js | 0 .../items/textItemCustomization.js | 0 .../customization/itemsCustomization.js | 0 .../lib/preferences/customization/profiles.js | 0 .../customization/themeCustomization.js | 0 .../preferences/dependencies/dependencies.js | 0 .../dependencies/dependenciesSettings.js | 0 .../general/appExclusionSettings.js | 0 .../preferences/general/behaviorSettings.js | 0 .../preferences/general/feedbackSettings.js | 0 .../preferences/general/historySettings.js | 0 .../lib/preferences/general/locationsGroup.js | 0 .../preferences/shortcuts/dialogShortcuts.js | 0 .../preferences/shortcuts/itemShortcuts.js | 0 .../shortcuts/navigationShortcuts.js | 0 .../shortcuts/popupMenuShortcuts.js | 0 .../preferences/shortcuts/searchShortcuts.js | 0 .../lib/preferences/shortcuts/shortcutRow.js | 0 .../lib/preferences/utils.js | 0 .../lib/ui/clipboardDialog.js | 0 .../lib/ui/clipboardScrollContainer.js | 0 .../lib/ui/clipboardScrollView.js | 0 .../lib/ui/components/actionMenu.js | 0 .../lib/ui/components/clipboardItemMenu.js | 0 .../lib/ui/components/codeLabel.js | 0 .../lib/ui/components/contentInfo.js | 0 .../lib/ui/components/contentPreview.js | 0 .../lib/ui/components/editDialog.js | 0 .../lib/ui/components/label.js | 0 .../lib/ui/components/qrCodeDialog.js | 0 .../lib/ui/components/shortcutLabel.js | 0 .../lib/ui/components/tagsItem.js | 0 .../lib/ui/indicator.js | 0 .../lib/ui/items/characterItem.js | 0 .../lib/ui/items/clipboardItem.js | 0 .../lib/ui/items/clipboardItemHeader.js | 0 .../lib/ui/items/codeItem.js | 0 .../lib/ui/items/colorItem.js | 0 .../lib/ui/items/fileItem.js | 0 .../lib/ui/items/filesItem.js | 0 .../lib/ui/items/imageItem.js | 0 .../lib/ui/items/linkItem.js | 0 .../lib/ui/items/statusItem.js | 0 .../lib/ui/items/textItem.js | 0 .../copyous@boerdereinar.dev/lib/ui/layout.js | 0 .../lib/ui/searchEntry.js | 0 .../LC_MESSAGES/copyous@boerdereinar.dev.mo | Bin .../LC_MESSAGES/copyous@boerdereinar.dev.mo | Bin .../LC_MESSAGES/copyous@boerdereinar.dev.mo | Bin .../LC_MESSAGES/copyous@boerdereinar.dev.mo | Bin .../LC_MESSAGES/copyous@boerdereinar.dev.mo | Bin .../LC_MESSAGES/copyous@boerdereinar.dev.mo | Bin .../LC_MESSAGES/copyous@boerdereinar.dev.mo | Bin .../LC_MESSAGES/copyous@boerdereinar.dev.mo | Bin .../copyous@boerdereinar.dev/metadata.json | 0 .../copyous@boerdereinar.dev/prefs.js | 0 .../resources.gresource | Bin .../schemas/gschemas.compiled | Bin ...gnome.shell.extensions.copyous.gschema.xml | 0 .../copyous@boerdereinar.dev/theme.gresource | Bin .../thirdparty/qrcodegen.js | 0 .../extension.js | 0 .../metadata.json | 0 .../papershell@lalovene.github.com/prefs.js | 0 .../schemas/gschemas.compiled | Bin ...me.shell.extensions.papershell.gschema.xml | 0 .../KeybindingsComboRow.ui | 0 .../KeybindingsPane.ui | 0 .../KeybindingsRow.ui | 0 .../paperwm@paperwm.github.com/LICENSE | 0 .../paperwm@paperwm.github.com/Settings.ui | 0 .../WinpropsPane.ui | 0 .../paperwm@paperwm.github.com/WinpropsRow.ui | 0 .../acceleratorparse.js | 0 .../paperwm@paperwm.github.com/app.js | 0 .../paperwm@paperwm.github.com/background.js | 0 .../config/user.css | 0 .../paperwm@paperwm.github.com/config/user.js | 0 .../paperwm@paperwm.github.com/extension.js | 0 .../paperwm@paperwm.github.com/gestures.js | 0 .../paperwm@paperwm.github.com/grab.js | 0 .../paperwm@paperwm.github.com/imports.js | 0 .../paperwm@paperwm.github.com/keybindings.js | 0 .../paperwm@paperwm.github.com/lib.js | 0 .../paperwm@paperwm.github.com/liveAltTab.js | 0 .../paperwm@paperwm.github.com/metadata.json | 0 .../paperwm@paperwm.github.com/minimap.js | 0 .../paperwm@paperwm.github.com/navigator.js | 0 .../overviewlayout.js | 0 .../paperwm@paperwm.github.com/patches.js | 0 .../paperwm@paperwm.github.com/prefs.js | 0 .../prefsKeybinding.js | 0 .../resources/ICONS.info | 0 .../resources/focus-mode-center-symbolic.svg | 0 .../resources/focus-mode-default-symbolic.svg | 0 .../resources/focus-mode-edge-symbolic.svg | 0 .../resources/logo.png | Bin .../resources/open-position-down-symbolic.svg | 0 .../resources/open-position-end-symbolic.svg | 0 .../resources/open-position-left-symbolic.svg | 0 .../open-position-right-symbolic.svg | 0 .../open-position-start-symbolic.svg | 0 .../resources/open-position-up-symbolic.svg | 0 .../resources/prefs.css | 0 .../schemas/gschemas.compiled | Bin ...gnome.shell.extensions.paperwm.gschema.xml | 0 .../paperwm@paperwm.github.com/scratch.js | 0 .../paperwm@paperwm.github.com/settings.js | 0 .../stackoverlay.js | 0 .../paperwm@paperwm.github.com/stylesheet.css | 0 .../paperwm@paperwm.github.com/tiling.js | 0 .../paperwm@paperwm.github.com/topbar.js | 0 .../paperwm@paperwm.github.com/utils.js | 0 .../paperwm@paperwm.github.com/virtTiling.js | 0 .../winpropsPane.js | 0 .../paperwm@paperwm.github.com/workspace.js | 0 .../gnome}/gnome-extensions-dconf.dump | 2 +- .../gnome}/gnome-keybindings.ini | 0 units/hive-ops.sh | 195 +++++ .../files => units/hive-ops}/bin/hive-peak.sh | 0 .../hive-ops}/bin/hive-rotate.sh | 0 .../hive-ops}/bin/hive-tiers.sh | 0 .../main.yml => units/hive-ops/defaults.yml | 0 .../hive-ops}/systemd/hive-peak-pause.service | 0 .../hive-ops}/systemd/hive-peak-pause.timer | 0 .../systemd/hive-peak-resume.service | 0 .../hive-ops}/systemd/hive-peak-resume.timer | 0 .../hive-ops}/systemd/hive-rotate.service | 0 .../hive-ops}/systemd/hive-rotate.timer | 0 .../hive-ops}/systemd/hive-tiers.service | 0 .../hive-ops}/systemd/hive-tiers.timer | 0 .../hive-ops}/systemd/hive-watchdog.service | 0 .../hive-ops}/systemd/hive-watchdog.timer | 0 .../hive-ops}/systemd/pi-peak-start.service | 0 .../hive-ops}/systemd/pi-peak-start.timer | 0 .../hive-ops}/systemd/pi-peak-stop.service | 0 .../hive-ops}/systemd/pi-peak-stop.timer | 0 units/homebrew.sh | 306 ++++++++ units/kirocrew-sockets.py | 27 + units/kirocrew.sh | 53 ++ units/kube.sh | 44 ++ units/laptop-services.sh | 238 +++++++ .../laptop-services/phone-apply.timer | 0 .../laptop-services}/system-mitigator.py | 0 .../laptop-services/system-mitigator.service | 0 units/lib.sh | 4 + units/omp.sh | 5 + units/osquery-check.py | 193 +++++ units/osquery-poller-render.py | 59 ++ units/osquery.sh | 158 +++++ {roles => units}/osquery/defaults/main.yml | 0 .../osquery/files/fleet-osquery.py | 0 .../templates/fleet-osquery-poll.service.j2 | 0 .../templates/fleet-osquery-poll.timer.j2 | 0 .../templates/fleet-osquery-report.service.j2 | 0 .../templates/fleet-osquery-report.timer.j2 | 0 .../osquery/templates/osquery.conf.j2 | 0 .../osquery/templates/osqueryd.service.j2 | 0 units/osquery_native_inventory.py | 214 ++++++ units/package-config.bash | 46 ++ units/pipewire-audio.sh | 611 ++++++++++++++++ .../pipewire-audio}/defaults/main.yml | 0 .../files/51-speaker-dsp-default.lua | 0 .../pipewire-audio}/files/mic-rnnoise.conf | 0 .../pipewire-audio}/files/speaker-biquad.conf | 0 .../pipewire-audio}/files/speaker-dsp.conf | 0 .../templates/mic-dsp-link.service.j2 | 0 units/platform-config.bash | 51 ++ units/proxy.sh | 131 ++++ .../templates => units/proxy}/Caddyfile.j2 | 0 .../files => units/proxy}/caddy.container | 0 units/retired-services.sh | 164 +++++ .../retired-services}/defaults/main.yml | 0 units/server-hardening-file.py | 68 ++ units/server-hardening.sh | 162 +++++ units/service-templates.py | 58 ++ units/shell-ai.sh | 21 + units/ssh-fleet.py | 667 ++++++++++++++++++ units/ssh-keys.sh | 8 + units/ssh-mesh.sh | 8 + units/sshd.sh | 124 ++++ units/sudo.sh | 87 +++ units/syncthing.sh | 108 +++ .../syncthing}/syncthing.service | 0 units/system-services.sh | 245 +++++++ .../system-services}/atuin-daemon.service.j2 | 0 .../dotfiles-update.service.j2 | 0 .../system-services}/dotfiles-update.timer.j2 | 0 units/tailscale-cert.sh | 137 ++++ .../tailscale-cert-renew.service.j2 | 12 + .../tailscale-cert/tailscale-cert-renew.timer | 10 + units/tailscale-connect.py | 91 +++ units/tailscale.sh | 265 +++++++ units/termux-packages.conf | 6 + units/termux-packages.sh | 175 +++++ units/termux-start-sshd | 5 + units/zen-browser.sh | 316 +++++++++ .../files => units/zen-browser}/policies.json | 0 .../files => units/zen-browser}/user.js | 0 432 files changed, 13171 insertions(+), 6223 deletions(-) delete mode 100644 .ansible-lint delete mode 100644 ansible.cfg delete mode 100644 requirements.yml delete mode 100644 roles/apk_packages/tasks/main.yml delete mode 100644 roles/bitwarden/tasks/main.yml delete mode 100644 roles/bluefin_common/tasks/main.yml delete mode 100644 roles/browser_fxa/README.md delete mode 100644 roles/browser_fxa/meta/main.yml delete mode 100644 roles/browser_fxa/tasks/main.yml delete mode 100644 roles/bst_dashboard/handlers/main.yml delete mode 100644 roles/bst_dashboard/tasks/main.yml delete mode 100644 roles/flatpak/tasks/main.yml delete mode 100644 roles/fleet_facts/tasks/main.yml delete mode 100644 roles/gnome/handlers/main.yml delete mode 100644 roles/gnome/tasks/main.yml delete mode 100644 roles/hive_ops/tasks/main.yml delete mode 100644 roles/homebrew/handlers/main.yml delete mode 100644 roles/homebrew/tasks/main.yml delete mode 100644 roles/homebrew/templates/Brewfile.j2 delete mode 100644 roles/kirocrew/tasks/main.yml delete mode 100644 roles/osquery/handlers/main.yml delete mode 100644 roles/osquery/tasks/install.yml delete mode 100644 roles/osquery/tasks/main.yml delete mode 100644 roles/osquery/tasks/poller.yml delete mode 100644 roles/osquery/templates/poller-inventory.yml.j2 delete mode 100644 roles/pipewire_audio/files/mic-dsp.conf delete mode 100644 roles/pipewire_audio/handlers/main.yml delete mode 100644 roles/pipewire_audio/tasks/main.yml delete mode 100644 roles/proxy/handlers/main.yml delete mode 100644 roles/proxy/tasks/main.yml delete mode 100644 roles/retired_services/tasks/main.yml delete mode 100644 roles/retired_services/vars/main.yml delete mode 100644 roles/server_hardening/handlers/main.yml delete mode 100644 roles/server_hardening/tasks/main.yml delete mode 100644 roles/ssh_keys/tasks/main.yml delete mode 100644 roles/ssh_keys/templates/ssh_config.j2 delete mode 100644 roles/ssh_mesh/tasks/main.yml delete mode 100644 roles/sshd/handlers/main.yml delete mode 100644 roles/sshd/tasks/main.yml delete mode 100644 roles/sudo/tasks/main.yml delete mode 100644 roles/syncthing/handlers/main.yml delete mode 100644 roles/syncthing/tasks/main.yml delete mode 100644 roles/systemd/handlers/main.yml delete mode 100644 roles/systemd/tasks/main.yml delete mode 100644 roles/systemd/templates/phone-apply.service.j2 delete mode 100644 roles/tailscale/handlers/main.yml delete mode 100644 roles/tailscale/tasks/configure_connected.yml delete mode 100644 roles/tailscale/tasks/configure_prereqs.yml delete mode 100644 roles/tailscale/tasks/connect.yml delete mode 100644 roles/tailscale/tasks/install.yml delete mode 100644 roles/tailscale/tasks/main.yml delete mode 100644 roles/tailscale_cert/handlers/main.yml delete mode 100644 roles/tailscale_cert/tasks/main.yml delete mode 100644 roles/termux_packages/defaults/main.yml delete mode 100644 roles/termux_packages/tasks/main.yml delete mode 100644 roles/termux_packages/templates/start-sshd.sh.j2 delete mode 100644 roles/zen_browser/tasks/main.yml delete mode 100644 site.yml create mode 100644 tests/test_bluefin_unit.py create mode 100644 tests/test_brew_flatpak_check_mode.py create mode 100644 tests/test_browser_fxa_check_mode.py create mode 100644 tests/test_browser_fxa_unit.py create mode 100644 tests/test_check_small_packages_vpn.py create mode 100644 tests/test_fleet_dotfiles_check.py create mode 100644 tests/test_fleet_facts_check.py create mode 100644 tests/test_fleet_facts_native.py create mode 100644 tests/test_forgejo_check_credentials.py create mode 100644 tests/test_github_check_credentials.py create mode 100644 tests/test_gnome_unit.py create mode 100644 tests/test_hive_ops_migration.py create mode 100644 tests/test_hive_syncthing_check.py create mode 100644 tests/test_kube_check_credentials.py create mode 100644 tests/test_laptop_services_migration.py create mode 100644 tests/test_laptop_system_services_check.py create mode 100644 tests/test_osquery_check.py create mode 100644 tests/test_osquery_native_inventory.py create mode 100644 tests/test_package_native_units.py create mode 100644 tests/test_pipewire_audio_check_mode.py create mode 100644 tests/test_pipewire_audio_unit.py create mode 100644 tests/test_platform_native_units.py create mode 100644 tests/test_retired_host_units.py create mode 100644 tests/test_security_native_units.py create mode 100644 tests/test_server_service_units.py create mode 100644 tests/test_server_units_check.py create mode 100644 tests/test_shell_ai_check_credentials.py create mode 100644 tests/test_ssh_check_credentials.py create mode 100644 tests/test_ssh_native_units.py create mode 100644 tests/test_sudo_sshd_check_units.py create mode 100644 tests/test_syncthing_migration.py create mode 100644 tests/test_system_services_migration.py create mode 100644 tests/test_unit_tags.py delete mode 100644 tests/test_units_registry.py create mode 100644 tests/test_visual_units_check.py create mode 100644 tests/test_zen_browser_unit.py create mode 100644 units/apk-packages.conf create mode 100755 units/apk-packages.sh create mode 100755 units/bluefin.sh rename {roles/bluefin_common => units/bluefin}/files/bluefin-sync.py (100%) rename {roles/bluefin_common => units/bluefin}/files/bluefin-wallpaper-rotate.py (100%) rename {roles/bluefin_common => units/bluefin}/files/install-extensions.py (100%) rename {roles/bluefin_common => units/bluefin}/templates/bluefin-wallpaper-rotate.service.j2 (100%) rename {roles/bluefin_common => units/bluefin}/templates/bluefin-wallpaper-rotate.timer.j2 (100%) create mode 100755 units/browser-fxa.sh rename roles/browser_fxa/defaults/main.yml => units/browser-fxa/defaults.yml (100%) rename {roles/browser_fxa/files => units/browser-fxa}/fxa_signin.py (100%) create mode 100755 units/bst-dashboard.sh rename {roles/bst_dashboard/templates => units/bst-dashboard}/bst-dashboard.service.j2 (100%) rename roles/flatpak/vars/main.yml => units/flatpak-packages.yml (100%) create mode 100755 units/flatpak.sh create mode 100755 units/fleet-facts.py create mode 100755 units/fleet-facts.sh rename roles/fleet_facts/defaults/main.yml => units/fleet-facts/defaults.yml (100%) create mode 100755 units/gnome-avatar.py create mode 100755 units/gnome.sh rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/extension.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/character-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/check-round-outline-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-disabled-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/color-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/duration-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/file-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-code-legacy-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/image-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-disabled-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/keyboard-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/link-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/pin-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/search-clipboard-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/settings-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/tag-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/text-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/common/actions.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/common/color.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/common/constants.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/common/dbus.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/common/gjs.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/common/glob.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/common/icons.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/common/settings.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/common/sound.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/database/database.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/database/entryTracker.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/database/gda.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/database/json.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/database/memory.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/misc/actor.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/misc/clipboard.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/misc/compatibility.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/misc/keyboard.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/misc/link.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/misc/notifications.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/misc/shortcuts.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/misc/theme.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDefaults.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDialog.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionSubMenuDialog.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsGroup.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsPage.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/dialogCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/headerCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/characterItemCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/codeItemCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/exclusions.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/fileItemCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/imageItemCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/linkItemCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/textItemCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemsCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/profiles.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/customization/themeCustomization.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependencies.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependenciesSettings.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/general/appExclusionSettings.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/general/behaviorSettings.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/general/feedbackSettings.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/general/historySettings.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/general/locationsGroup.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/dialogShortcuts.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/itemShortcuts.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/navigationShortcuts.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/popupMenuShortcuts.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/searchShortcuts.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/shortcutRow.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/preferences/utils.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/clipboardDialog.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollContainer.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollView.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/actionMenu.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/clipboardItemMenu.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/codeLabel.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/contentInfo.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/contentPreview.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/editDialog.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/label.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/qrCodeDialog.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/shortcutLabel.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/components/tagsItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/indicator.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/characterItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItemHeader.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/codeItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/colorItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/fileItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/filesItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/imageItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/linkItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/statusItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/items/textItem.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/layout.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/lib/ui/searchEntry.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/locale/de/LC_MESSAGES/copyous@boerdereinar.dev.mo (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/locale/fr/LC_MESSAGES/copyous@boerdereinar.dev.mo (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/locale/it/LC_MESSAGES/copyous@boerdereinar.dev.mo (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/locale/pl/LC_MESSAGES/copyous@boerdereinar.dev.mo (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/locale/pt_BR/LC_MESSAGES/copyous@boerdereinar.dev.mo (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/locale/ru/LC_MESSAGES/copyous@boerdereinar.dev.mo (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/locale/tr/LC_MESSAGES/copyous@boerdereinar.dev.mo (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/locale/zh_CN/LC_MESSAGES/copyous@boerdereinar.dev.mo (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/metadata.json (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/prefs.js (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/resources.gresource (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/schemas/gschemas.compiled (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/schemas/org.gnome.shell.extensions.copyous.gschema.xml (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/theme.gresource (100%) rename {roles/gnome/files => units/gnome}/extensions/copyous@boerdereinar.dev/thirdparty/qrcodegen.js (100%) rename {roles/gnome/files => units/gnome}/extensions/papershell@lalovene.github.com/extension.js (100%) rename {roles/gnome/files => units/gnome}/extensions/papershell@lalovene.github.com/metadata.json (100%) rename {roles/gnome/files => units/gnome}/extensions/papershell@lalovene.github.com/prefs.js (100%) rename {roles/gnome/files => units/gnome}/extensions/papershell@lalovene.github.com/schemas/gschemas.compiled (100%) rename {roles/gnome/files => units/gnome}/extensions/papershell@lalovene.github.com/schemas/org.gnome.shell.extensions.papershell.gschema.xml (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/KeybindingsComboRow.ui (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/KeybindingsPane.ui (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/KeybindingsRow.ui (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/LICENSE (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/Settings.ui (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/WinpropsPane.ui (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/WinpropsRow.ui (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/acceleratorparse.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/app.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/background.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/config/user.css (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/config/user.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/extension.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/gestures.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/grab.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/imports.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/keybindings.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/lib.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/liveAltTab.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/metadata.json (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/minimap.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/navigator.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/overviewlayout.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/patches.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/prefs.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/prefsKeybinding.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/ICONS.info (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/focus-mode-center-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/focus-mode-default-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/focus-mode-edge-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/logo.png (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/open-position-down-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/open-position-end-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/open-position-left-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/open-position-right-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/open-position-start-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/open-position-up-symbolic.svg (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/resources/prefs.css (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/schemas/gschemas.compiled (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/schemas/org.gnome.shell.extensions.paperwm.gschema.xml (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/scratch.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/settings.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/stackoverlay.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/stylesheet.css (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/tiling.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/topbar.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/utils.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/virtTiling.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/winpropsPane.js (100%) rename {roles/gnome/files => units/gnome}/extensions/paperwm@paperwm.github.com/workspace.js (100%) rename {roles/gnome/files => units/gnome}/gnome-extensions-dconf.dump (98%) rename {roles/gnome/files => units/gnome}/gnome-keybindings.ini (100%) create mode 100755 units/hive-ops.sh rename {roles/hive_ops/files => units/hive-ops}/bin/hive-peak.sh (100%) rename {roles/hive_ops/files => units/hive-ops}/bin/hive-rotate.sh (100%) rename {roles/hive_ops/files => units/hive-ops}/bin/hive-tiers.sh (100%) rename roles/hive_ops/defaults/main.yml => units/hive-ops/defaults.yml (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-peak-pause.service (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-peak-pause.timer (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-peak-resume.service (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-peak-resume.timer (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-rotate.service (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-rotate.timer (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-tiers.service (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-tiers.timer (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-watchdog.service (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/hive-watchdog.timer (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/pi-peak-start.service (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/pi-peak-start.timer (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/pi-peak-stop.service (100%) rename {roles/hive_ops/files => units/hive-ops}/systemd/pi-peak-stop.timer (100%) create mode 100755 units/homebrew.sh create mode 100755 units/kirocrew-sockets.py create mode 100755 units/kirocrew.sh create mode 100755 units/laptop-services.sh rename roles/systemd/templates/phone-apply.timer.j2 => units/laptop-services/phone-apply.timer (100%) rename {roles/systemd/files => units/laptop-services}/system-mitigator.py (100%) rename roles/systemd/templates/system-mitigator.service.j2 => units/laptop-services/system-mitigator.service (100%) create mode 100755 units/osquery-check.py create mode 100755 units/osquery-poller-render.py create mode 100755 units/osquery.sh rename {roles => units}/osquery/defaults/main.yml (100%) rename {roles => units}/osquery/files/fleet-osquery.py (100%) mode change 100755 => 100644 rename {roles => units}/osquery/templates/fleet-osquery-poll.service.j2 (100%) rename {roles => units}/osquery/templates/fleet-osquery-poll.timer.j2 (100%) rename {roles => units}/osquery/templates/fleet-osquery-report.service.j2 (100%) rename {roles => units}/osquery/templates/fleet-osquery-report.timer.j2 (100%) rename {roles => units}/osquery/templates/osquery.conf.j2 (100%) rename {roles => units}/osquery/templates/osqueryd.service.j2 (100%) create mode 100644 units/osquery_native_inventory.py create mode 100644 units/package-config.bash create mode 100755 units/pipewire-audio.sh rename {roles/pipewire_audio => units/pipewire-audio}/defaults/main.yml (100%) rename {roles/pipewire_audio => units/pipewire-audio}/files/51-speaker-dsp-default.lua (100%) rename {roles/pipewire_audio => units/pipewire-audio}/files/mic-rnnoise.conf (100%) rename {roles/pipewire_audio => units/pipewire-audio}/files/speaker-biquad.conf (100%) rename {roles/pipewire_audio => units/pipewire-audio}/files/speaker-dsp.conf (100%) rename {roles/pipewire_audio => units/pipewire-audio}/templates/mic-dsp-link.service.j2 (100%) create mode 100644 units/platform-config.bash create mode 100755 units/proxy.sh rename {roles/proxy/templates => units/proxy}/Caddyfile.j2 (100%) rename {roles/proxy/files => units/proxy}/caddy.container (100%) create mode 100755 units/retired-services.sh rename {roles/retired_services => units/retired-services}/defaults/main.yml (100%) create mode 100755 units/server-hardening-file.py create mode 100755 units/server-hardening.sh create mode 100755 units/service-templates.py create mode 100755 units/ssh-fleet.py create mode 100755 units/ssh-keys.sh create mode 100755 units/ssh-mesh.sh create mode 100755 units/sshd.sh create mode 100755 units/sudo.sh create mode 100755 units/syncthing.sh rename {roles/syncthing/files => units/syncthing}/syncthing.service (100%) create mode 100755 units/system-services.sh rename {roles/systemd/templates => units/system-services}/atuin-daemon.service.j2 (100%) rename {roles/systemd/templates => units/system-services}/dotfiles-update.service.j2 (100%) rename {roles/systemd/templates => units/system-services}/dotfiles-update.timer.j2 (100%) create mode 100755 units/tailscale-cert.sh create mode 100644 units/tailscale-cert/tailscale-cert-renew.service.j2 create mode 100644 units/tailscale-cert/tailscale-cert-renew.timer create mode 100755 units/tailscale-connect.py create mode 100755 units/tailscale.sh create mode 100644 units/termux-packages.conf create mode 100755 units/termux-packages.sh create mode 100755 units/termux-start-sshd create mode 100755 units/zen-browser.sh rename {roles/zen_browser/files => units/zen-browser}/policies.json (100%) rename {roles/zen_browser/files => units/zen-browser}/user.js (100%) diff --git a/.ansible-lint b/.ansible-lint deleted file mode 100644 index 6731b04c..00000000 --- a/.ansible-lint +++ /dev/null @@ -1,38 +0,0 @@ ---- -# Project lint policy: keep the noise off, treat operational warnings -# (idempotency, risky shell patterns) as real findings. -# -# Background: var-naming generated ~95 warnings across the role tree — -# almost all of them false-positive complaints about register variables -# like `bw_session` not being prefixed with the role name. Refactoring -# every one of them gives zero operational benefit. The signal that -# matters (no-changed-when, risky-shell-pipe) was buried under that -# noise, so `just lint` exited non-zero and nobody noticed. - -profile: safety - -skip_list: - # `bw_session` and similar legitimately cross role boundaries; the - # role-prefix rule treats them as drift. - - var-naming[no-role-prefix] - # We use ignore_errors deliberately in a handful of places (the - # zen_browser default-handler chain in particular). Demote to warn - # instead of fail. - - ignore-errors - -warn_list: - # FQCN for builtins is a style nit; surface but don't fail. - - fqcn[canonical] - # YAML cosmetic findings — surface but don't fail; real authoring - # mistakes still get visibility from yamllint itself. - - yaml[trailing-spaces] - - yaml[line-length] - - yaml[empty-lines] - - yaml[commas] - - key-order[task] - # Surfaced but not blocking — we deliberately use systemctl/curl/shell - # in a handful of bootstrap-y tasks where the systemd/get_url modules - # don't offer enough control. Refactoring them is on the wishlist, not - # a CI blocker. - - command-instead-of-module - - command-instead-of-shell diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e75ea107..f88357c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,27 +26,13 @@ jobs: - name: Install tools run: | - pip install yamllint ansible-lint + pip install yamllint sudo apt-get install -y shellcheck - name: yamllint run: | yamllint -d "{extends: default, rules: {line-length: disable, document-start: disable, truthy: disable, comments-indentation: disable, comments: disable, braces: disable, indentation: disable, commas: disable, empty-lines: disable}}" . - # ansible-lint would install requirements.yml itself, with no retry; - # galaxy.ansible.com intermittently resets connections (#217). - - name: Install collections - run: | - for i in 1 2 3; do - ansible-galaxy collection install -r requirements.yml && exit 0 - echo "galaxy install failed (attempt $i); retrying" - sleep $((i * 15)) - done - exit 1 - - - name: ansible-lint - run: ansible-lint --offline site.yml - - name: shellcheck run: | find . -name "*.sh" -not -path "./.git/*" -type f -print0 | xargs -0 shellcheck -x -S error @@ -74,6 +60,8 @@ jobs: - uses: astral-sh/setup-uv@v10.3.0 + - uses: extractions/setup-just@v4 + - name: pytest run: uvx --with pyyaml pytest -q tests @@ -85,15 +73,17 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Install Ansible + - name: Install native inventory dependency run: | - pip install ansible-core - ansible-galaxy collection install -r requirements.yml + case "$(uname -m)" in x86_64) arch=amd64 ;; aarch64) arch=arm64 ;; *) exit 1 ;; esac + sudo curl --fail --location --silent --show-error \ + "https://github.com/mikefarah/yq/releases/download/v4.47.2/yq_linux_$arch" -o /usr/local/bin/yq + sudo chmod 755 /usr/local/bin/yq - name: Install actual native osquery and private reporting - run: >- - ansible-playbook -i "$(hostname)," -c local site.yml --tags osquery - --skip-tags secrets -e "target=$(hostname)" -e osquery_poller_enabled=true + run: | + echo punjab | sudo tee /etc/dotfiles-machine >/dev/null + DOTFILES_MACHINE=punjab bash units/osquery.sh - name: Exercise real socket, enabled unit and baseline transitions run: sudo python3 tests/osquery_smoke.py @@ -103,8 +93,8 @@ jobs: - name: Verify poller opt-out stops the schedule and service run: | - ansible-playbook -i "$(hostname)," -c local site.yml --tags osquery \ - --skip-tags secrets -e "target=$(hostname)" -e osquery_poller_enabled=false + yq -i '.osquery_poller_enabled = false' host_vars/punjab.yml + DOTFILES_MACHINE=punjab bash units/osquery.sh test "$(systemctl show fleet-osquery-poll.timer -p ActiveState --value)" = inactive test "$(systemctl show fleet-osquery-poll.timer -p UnitFileState --value)" = disabled test "$(systemctl show fleet-osquery-poll.service -p ActiveState --value)" = inactive diff --git a/CLAUDE.md b/CLAUDE.md index 2fabede9..61ea543a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,10 +4,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this repo is -Ansible-driven dotfiles + infra for a personal fleet (desktops, servers, VPS). Each machine manages itself locally via `just` + `ansible-playbook`. Secrets come from Bitwarden at runtime — the repo is public, no secrets in git. +Native Bash-unit dotfiles + infra for a personal fleet (desktops, servers, VPS). Machines manage themselves locally via `just`, `scripts/run-units.sh`, mise and chezmoi. Secrets come from Bitwarden at runtime — the repo is public, no secrets in git. Three distinct concerns live here: -1. **Workstation config** (`roles/`, `site.yml`, `host_vars/`, `group_vars/`) — shells, packages, browser, SSH, Tailscale, kubeconfig, etc. +1. **Workstation config** (`units/`, `home/`, `host_vars/`, `group_vars/`) — shells, packages, browser, SSH, Tailscale, kubeconfig, etc. 2. **Talos K8s cluster IaC** (`talos-k8s/`) — manifests for the Bihar + Karnataka cluster. Detailed handbook in [`docs/src/servers/talos-k8s/cluster.md`](docs/src/servers/talos-k8s/cluster.md). 3. **TunaOS AWS account IaC** (`aws/`, OpenTofu) — the account is **TunaOS infrastructure**, not the personal fleet: the AWS Talos cluster's VPC/nodes/EIPs, punjab's instance, backups (DLM + S3), IAM, budgets. Handbook: [`docs/src/servers/aws/README.md`](docs/src/servers/aws/README.md). @@ -22,12 +22,13 @@ Three distinct concerns live here: Configs: `~/.kube/config-aws-migration`, `~/.talos/config-aws-migration`. Handbook: [`docs/src/servers/aws-k8s/cluster.md`](docs/src/servers/aws-k8s/cluster.md). -Neither cluster's nodes are Ansible-managed — Talos has no SSH. They are not in +Neither cluster's nodes are managed by fleet units — Talos has no SSH. They are not in `inventory.yml`; access is `talosctl`/`kubectl` only. The two Hetzner VPSes it replaced (`matrix`, `telengana`) are retired but still powered on for burn-in. **Nothing on them may be restarted** — see the [cutover runbook](docs/matrix-cutover-runbook.md). +Native units' shared library skips `matrix` and `telengana` before unit work to protect these hosts; do not rely on that guard as authorization to change or restart them. ## Core commands @@ -36,8 +37,8 @@ just apply # Local apply with Bitwarden secrets (most common) just apply-remote HOST # Apply to a single remote, forwarding BW session over SSH just apply-all # Apply in parallel to all online Tailscale-reachable hosts just apply-tags TAGS # Subset apply, e.g. `just apply-tags kube,shell` -just check # Dry-run -just lint # yamllint + ansible-lint +just check # Native inspection; coverage is subject to integration verification +just lint # YAML lint and native unit shell checks just doctor # Health-check local machine; `doctor-fleet` does all hosts just inventory # nmap LAN + Tailscale crosswalk → table of who's on just seed-kube # Push this machine's kubeconfig + talosconfig to Bitwarden @@ -45,17 +46,15 @@ just onboard NAME TYPE # Register a new machine in inventory.yml just add-machine NAME # Onboard + bootstrap an already-reachable machine via SSH ``` -## Playbook structure +## Native configuration -`site.yml` runs roles in tagged phases. Notable wiring: +`just apply*` invokes `scripts/run-units.sh`, which runs `units/` in prerequisite stages via mise, with a sequential fallback. It selects units by the wrapper's tag map and honors each unit's own platform, profile and host-variable gates. See [`docs/src/units.md`](docs/src/units.md). -- `bitwarden` resolves `BW_SESSION` (env, `/tmp/bw_session`, or interactive unlock), runs `bw sync` with a 15s timeout, and sets a `bw_unlocked` host fact. All BW-using roles gate on `when: bw_unlocked | default(false)`. -- the `kube` unit fetches `~/.kube/config` + `~/.talos/config` from Bitwarden — desktops only. -- `ssh_keys` round-trips per-machine ed25519 keys through Bitwarden, building cross-machine `authorized_keys`. +- Apply recipes resolve and sync Bitwarden; unit helpers use `BW_SESSION` or `/tmp/bw_session` noninteractively, and honor `--no-secrets`. +- The `kube` unit fetches kubeconfig and talosconfig from Bitwarden on eligible desktops. +- `ssh-keys` and `ssh-mesh` handle key lifecycle and authorization. `home/` is the chezmoi source for shell, Git, Neovim and Atuin config. -After the playbook, `just apply*` runs bash **units** (`units/`, via mise through `scripts/run-units.sh`; see [`docs/src/units.md`](docs/src/units.md)): `chezmoi`, `fonts`, `omp`, `atuin`, `shell-ai`, `github`, `kube`, `forgejo-registry`. `home/` is the chezmoi source for shell, git, neovim and atuin config; `ssh_keys` remains an Ansible role. - -Roles are conditional on inventory group (`desktop`, `server`, `vps`, `test_fleet`, `termux_hosts`; `llm` is dormant/commented out) and on `skip_*` flags / `machine_profile` in `host_vars/`. See [`docs/src/roles/`](docs/src/roles/) for the full reference. +Groups in `inventory.yml`, `skip_*` flags and `machine_profile` influence individual units. See [`docs/src/roles/`](docs/src/roles/) for component references. ## Kubeconfig + Talos secrets @@ -73,7 +72,7 @@ Production workloads: Lemonade (AMD-optimized local AI), KubeVirt v1.8.2 + KubeV ## TunaOS AWS account (`aws/`) -- **punjab** (this repo's only AWS-hosted Ansible host) is an EC2 `t3.large` in `us-east-1`; the Talos cluster is in `eu-north-1`; `runs-on` (us-east-2) is its own CloudFormation stack — don't import it. +- **punjab** (this repo's only AWS-hosted fleet host) is an EC2 `t3.large` in `us-east-1`; the Talos cluster is in `eu-north-1`; `runs-on` (us-east-2) is its own CloudFormation stack — don't import it. - `just aws-plan` / `just aws-apply`. A plan against the live account must read "No changes"; console edits are drift — codify or revert them. - State is in S3 (`hanthor-fleet-backups-*/tofu/aws/`) and contains Talos PKI via node `user_data`. `aws/terraform.tfvars` (admin IPs, alert email) is gitignored and lives in Bitwarden note `aws-tofu-tfvars` (`just aws-tfvars` / `just aws-seed-tfvars`). - punjab has **no public inbound** (SG empty): Tailscale for access, SSM Session Manager for break-glass. Its EIP is allowlisted on the cluster admin SG, so kubectl/talosctl work from it. @@ -96,7 +95,7 @@ Renovate and the Hive (hive.reilly.asia, `hanthor-hive-agent`) open PRs; master ## References - [`docs/src/servers/talos-k8s/cluster.md`](docs/src/servers/talos-k8s/cluster.md) — Talos cluster handbook -- [`docs/src/roles/`](docs/src/roles/) — every Ansible role explained +- [`docs/src/roles/`](docs/src/roles/) — component references - [`docs/src/bitwarden.md`](docs/src/bitwarden.md) — BW vault structure - `Justfile` — every task recipe; `just --list` for a menu -- `ansible.cfg`, `site.yml`, `inventory.yml` — the playbook entrypoints +- `inventory.yml`, `scripts/run-units.sh`, `mise.toml` — fleet inventory and unit entrypoints diff --git a/CONTEXT.md b/CONTEXT.md index 3867b112..a64f0d0a 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -8,22 +8,21 @@ - **Machine** — a physical host, VM, or VPS managed by this repo. Each has an entry in `inventory.yml` and `host_vars/.yml`. - **Fleet** — the set of all machines. -- **Group** — a logical subset of machines (`desktop`, `server`, `vps`, `test_fleet`, `termux_hosts`; `llm` is dormant, commented out). Used by Ansible inventory groups and by `site.yml` `when:` conditionals. -- **Hostname** — the machine's short name (e.g. `kanpur`, `punjab`). Used as the inventory hostname, Tailscale hostname, and BW item key suffix. -- **Machine profile** — a proposed replacement for scattered `skip_*` flags. A single typed variable that declares a machine's role (e.g. `headless`, `desktop`, `vm-test`) and derives which components to install. +- **Group** — a logical subset of machines (`desktop`, `server`, `vps`, `test_fleet`, `termux_hosts`; `llm` is dormant). Native units and scripts read inventory membership for their gates. +- **Hostname** — the machine's short name (e.g. `kanpur`, `punjab`). Machine selection uses `DOTFILES_MACHINE`, `/etc/dotfiles-machine`, then hostname; it keys host settings and BW item suffixes. +- **Machine profile** — a host variable used alongside `skip_*` flags; individual units decide how to interpret it, including `vm-test` exclusions. ## Secrets -- **BW item** — a Bitwarden vault entry (login, secure note, SSH key) consumed by a role. Named items (e.g. `tailscale-apikey`, `atuin.sh`, `accounts.firefox.com`). -- **BW credential** — a single field extracted from a BW item (password, username, notes, TOTP, SSH key, custom field). Roles fetch these with `ansible.builtin.command: bw get `, gated on `bw_unlocked`. -- **BW session** — the `BW_SESSION` environment variable that unlocks the vault for the current playbook run. Resolved from env → `/tmp/bw_session` cache → interactive unlock. +- **BW item** — a Bitwarden vault entry (login, secure note, SSH key) consumed by a unit. Named items include `tailscale-apikey`, `atuin.sh`, and `accounts.firefox.com`. +- **BW credential** — a field extracted from a BW item (password, username, notes, TOTP, SSH key, custom field). Units perform their own lookups and handle unavailable items. +- **BW session** — `BW_SESSION` or a cached `/tmp/bw_session` value used by noninteractive unit helpers. Apply recipes may resolve an interactive session; `--no-secrets` suppresses unit vault access. -## Playbook +## Native configuration -- **Role** — an Ansible role that configures one concern (e.g. `tailscale`, `proxy`, `ssh_keys`). The interface is `tasks/main.yml`; the implementation is templates, files, and handlers. -- **Unit** — an idempotent bash script in `units/` run in parallel by mise after the playbook (`chezmoi`, `fonts`, `omp`, `atuin`, `shell-ai`, `github`, `kube`, `forgejo-registry`). A failing step does not stop the rest of the unit. See `docs/src/units.md`. -- **Phase** — a logical grouping of roles in `site.yml` (Phase 1: system + packages, Phase 2: secrets + auth, Phase 3: desktop, Phase 4: services). Phases are documented in comments, not enforced by the playbook. -- **Tag** — an Ansible tag applied to roles so `just apply-tags ` can target a subset. +- **Unit** — a Bash entrypoint in `units/` configuring one concern. `scripts/run-units.sh` orders prerequisite stages and runs independent selections via mise or a sequential fallback. A unit can report multiple step failures independently. See `docs/src/units.md`. +- **Tag** — a selection key mapped to units by `tasks_for_tag` in `scripts/run-units.sh`; unit-specific platform and profile gates still apply. +- **chezmoi source** — `home/`, containing managed shell, Git, Neovim and Atuin files. Secret files and procedural operations remain in units. ## Cluster @@ -33,8 +32,8 @@ ## Architecture (from improve-codebase-architecture) -- **Module** — anything with an interface and an implementation (a role, a script, a Just recipe, an Ansible module). -- **Interface** — everything a caller must know to use the module (task names, variables, tags, error modes, `when:` guards). +- **Module** — anything with an interface and an implementation (a unit, a script, a Just recipe). +- **Interface** — everything a caller must know to use the module (commands, variables, tags, error modes and guards). - **Depth** — leverage at the interface. A module is **deep** when a lot of behaviour sits behind a small interface. - **Seam** — where an interface lives; a place behaviour can be altered without editing in place. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56df67aa..8bc49ba8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,42 +14,40 @@ out and can auto-merge once green. Hive PRs touching anything else are labelled never auto-merged. Details: [Bots, Agents & Guardrails](docs/src/automation.md). Don't widen the carve-outs to anything the fleet timer executes. -## Local checks (mirror CI, `.github/workflows/ci.yml`) +## Local checks ```bash yamllint -d "{extends: default, rules: {line-length: disable, document-start: disable, truthy: disable, comments-indentation: disable, comments: disable, braces: disable, indentation: disable, commas: disable, empty-lines: disable}}" . -ansible-lint site.yml +# The commands below correspond to CI jobs; availability and results depend on your environment. find . -name "*.sh" -not -path "./.git/*" -type f -print0 | xargs -0 shellcheck -x -S error uvx --with pyyaml pytest -q tests just --justfile Justfile --list (cd aws && tofu fmt -check -recursive) ``` -CI also checks JSON validity, `py_compile`s tracked Python, runs -`tofu validate` + `aws/tests/run.sh`, and gofmt/vet/tests `tools/secret-broker/`. +CI also checks JSON validity, compiles tracked Python, runs native osquery smoke jobs, +`tofu validate` + `aws/tests/run.sh`, and Go checks in `tools/secret-broker/`. +See `.github/workflows/ci.yml` for the current jobs. These instructions do not +assert that a particular checkout has passed them. -Before applying a playbook change, dry-run it on a machine you own: +`just check` requests native unit inspection (`UNITS_CHECK=1`). Its coverage and +safety across platforms are subject to integration verification; do not assume +it is a complete dry-run guarantee. `just apply-tags ` runs the selected +units and can change the host. Review [Architecture](docs/src/architecture.md) +and [Units](docs/src/units.md) before applying, especially for privileged work. -```bash -just check # ansible-playbook --check --diff -just apply-tags # apply only the roles you touched -``` - -Tags and `skip_*` host_vars flags are listed in -[Architecture](docs/src/architecture.md#available-tags). - -## Roles and docs +## Units and docs -- New role: `roles//`, wired into the right phase of `site.yml` with - tags and any `when:` gate. -- Document it in `docs/src/roles/.md` and add it to - `docs/src/SUMMARY.md`. +- Add unit entrypoints and assets under `units/`, register them in `mise.toml` + and `scripts/run-units.sh`, and implement platform, profile and privilege gates. +- Document behavior, limitations and secrets in the relevant handbook page. + File-based shell, Git, Neovim and Atuin configuration comes from `home/` via chezmoi. ## Secrets The repo is public. Secrets come from Bitwarden at runtime only — never -hardcode credentials, tokens or keys. BW-using tasks gate on -`bw_unlocked | default(false)`. +hardcode credentials, tokens or keys. Unit vault access uses a noninteractive +session and honors `UNITS_NO_SECRETS=1`; preserve existing secret data when unavailable. ## Don'ts diff --git a/Justfile b/Justfile index 7e78944c..15f62121 100644 --- a/Justfile +++ b/Justfile @@ -1,4 +1,4 @@ -# Dotfiles management with Ansible +# Fleet configuration with mise, bash units and chezmoi # Usage: just dotfiles_dir := env("HOME") / ".local/share/dotfiles" @@ -8,48 +8,32 @@ export PATH := env("HOME") / ".local/bin" + ":/home/linuxbrew/.linuxbrew/bin:" + # Resolve online fleet hosts: intersect tailscale online peers with inventory (excluding vps + self) _online_hosts := shell('python3 "$1"', justfile_directory() / "scripts/online_hosts.py") -# Every recipe below runs site.yml against THIS machine over the local -# connection. The flags are not optional decoration: `-l`/`-e target` pin the -# play to this host, and the ansible_connection/ansible_host overrides keep -# Ansible from re-resolving the inventory address and SSHing to itself. -# Defined once so a change to how the fleet addresses itself is a one-line -# edit instead of eight. -_play := "ansible-playbook --connection=local -l " + machine + " -e target=" + machine + " -e ansible_connection=local -e ansible_host=127.0.0.1" -_secret_tag_pattern := "secrets|bitwarden|ssh|ssh_keys|github|tailscale|kube|forgejo|registry|atuin|ai|omp" +_secret_tag_pattern := "secrets|ssh|ssh_keys|github|tailscale|kube|forgejo|registry|atuin|ai|omp|desktop|browser|zen_browser" # Apply all config to this machine (unlocks BW interactively if needed) apply *args: #!/usr/bin/env bash - set -euo pipefail - cd {{ dotfiles_dir }} - git pull --ff-only - ansible-galaxy collection install -r requirements.yml - - BECOME_ARGS="" - if ! sudo -n true 2>/dev/null; then - echo "Sudo requires a password. Adding --ask-become-pass..." - BECOME_ARGS="--ask-become-pass" - fi - + set -uo pipefail + cd {{ dotfiles_dir }} || exit 1 + git pull --ff-only || exit $? + for arg in {{ args }}; do + case "$arg" in --upgrade) export DOTFILES_UPGRADE=1 ;; *) echo "Unknown apply option: $arg" >&2; exit 2 ;; esac + done SKIP_TAGS="" - if bw_env=$(scripts/bw-resolve.sh local); then - eval "$bw_env" - else - echo "Continuing without secrets (run scripts/bw-unlock.sh manually for details)..." - SKIP_TAGS="secrets" - fi - rc=0 unit_args=() - if [ -n "$SKIP_TAGS" ]; then - unit_args+=(--no-secrets) - {{ _play }} site.yml --skip-tags "$SKIP_TAGS" $BECOME_ARGS {{ args }} || rc=$? + if bw_env=$(scripts/bw-resolve.sh local) && eval "$bw_env" && + [ -n "${BW_SESSION:-}" ] && bw unlock --check >/dev/null 2>&1 && + timeout 15 bw sync >/dev/null; then + : else - {{ _play }} site.yml $BECOME_ARGS {{ args }} || rc=$? + echo "Bitwarden unavailable or sync failed; continuing without secrets." >&2 + unset BW_SESSION + SKIP_TAGS=secrets + unit_args+=(--no-secrets) fi - urc=0; scripts/run-units.sh "${unit_args[@]}" || urc=$? - [ "$rc" -ne 0 ] || rc=$urc + rc=0; scripts/run-units.sh "${unit_args[@]}" || rc=$? scripts/record-apply.py "$rc" apply "$SKIP_TAGS" - exit $rc + exit "$rc" # Apply only specific tags (e.g. just apply-tags kube,shell) apply-tags tags: @@ -57,54 +41,42 @@ apply-tags tags: set -uo pipefail cd {{ dotfiles_dir }} || exit 1 rc=0 - ansible_tags="{{ tags }}" unit_args=(--tags "{{ tags }}") - skip_args=() needs_secrets=0 IFS=',' read -ra tag_list <<< "{{ tags }}" for tag in "${tag_list[@]}"; do - case "$tag" in - {{ _secret_tag_pattern }}) needs_secrets=1 ;; - esac + case "$tag" in {{ _secret_tag_pattern }}) needs_secrets=1 ;; esac done if [ "$needs_secrets" -eq 1 ]; then - # bw-unlock accepts an environment session as-is; reject stale ones first. - if [ -n "${BW_SESSION:-}" ] && ! bw unlock --check >/dev/null 2>&1; then - unset BW_SESSION - fi + if [ -n "${BW_SESSION:-}" ] && ! bw unlock --check >/dev/null 2>&1; then unset BW_SESSION; fi if bw_env=$(scripts/bw-resolve.sh local) && eval "$bw_env" && [ -n "${BW_SESSION:-}" ] && bw unlock --check >/dev/null 2>&1 && timeout 15 bw sync >/dev/null; then - # Secret roles selected by their own tag still need bw_unlocked set. - ansible_tags="$ansible_tags,bitwarden" + : else echo "Bitwarden unavailable or sync failed; skipping secrets." >&2 unset BW_SESSION - skip_args=(--skip-tags secrets) unit_args+=(--no-secrets) rc=1 fi fi - arc=0 - {{ _play }} site.yml --tags "$ansible_tags" "${skip_args[@]}" || arc=$? - [ "$rc" -ne 0 ] || rc=$arc urc=0; scripts/run-units.sh "${unit_args[@]}" || urc=$? [ "$rc" -ne 0 ] || rc=$urc scripts/record-apply.py "$rc" apply-tags "{{ tags }}" - exit $rc + exit "$rc" # Apply without unlocking Bitwarden — fast path for the daily timer and `dots` apply-nosecrets *args: #!/usr/bin/env bash set -uo pipefail - cd {{ dotfiles_dir }} + cd {{ dotfiles_dir }} || exit 1 git pull --ff-only || true - rc=0 - {{ _play }} site.yml --skip-tags secrets {{ args }} || rc=$? - urc=0; scripts/run-units.sh --no-secrets || urc=$? - [ "$rc" -ne 0 ] || rc=$urc + for arg in {{ args }}; do + case "$arg" in --upgrade) export DOTFILES_UPGRADE=1 ;; *) echo "Unknown apply option: $arg" >&2; exit 2 ;; esac + done + rc=0; scripts/run-units.sh --no-secrets || rc=$? scripts/record-apply.py "$rc" apply-nosecrets secrets - exit $rc + exit "$rc" # Apply to a remote machine with specific tags (e.g. just apply-remote-tags bihar kube,shell) apply-remote-tags name tags: @@ -131,18 +103,11 @@ apply-remote-tags name tags: # Verify (and self-heal) passwordless SSH to all online fleet members mesh-check: - cd {{ dotfiles_dir }} && {{ _play }} site.yml --tags mesh_check + cd {{ dotfiles_dir }} && scripts/run-units.sh --tags mesh_check # Apply only dotfile configs (shell, git, tmux, etc.) dotfiles: - #!/usr/bin/env bash - set -uo pipefail - cd {{ dotfiles_dir }} - rc=0 - {{ _play }} site.yml --tags dotfiles || rc=$? - urc=0; scripts/run-units.sh --tags dotfiles || urc=$? - [ "$rc" -ne 0 ] || rc=$urc - exit $rc + cd {{ dotfiles_dir }} && scripts/run-units.sh --tags dotfiles # Run the bash units (units/*.sh) in parallel via mise, e.g. `just units --tags omp` units *args: @@ -150,7 +115,7 @@ units *args: # Apply only packages (Homebrew + Flatpak) packages: - cd {{ dotfiles_dir }} && {{ _play }} site.yml --tags packages + cd {{ dotfiles_dir }} && scripts/run-units.sh --tags packages # Apply to a remote machine, forwarding your local BW session over SSH apply-remote name *args: @@ -438,7 +403,7 @@ update *args: # Pull latest, apply, and upgrade all packages update-all: - just update -e upgrade=true + just update --upgrade # Sync current Homebrew state back to dotfiles variables sync-brews: @@ -453,8 +418,8 @@ flatpak-install app: set -euo pipefail flatpak install --system -y flathub {{ app }} cd {{ dotfiles_dir }} - yq -i '.system_flatpaks += ["{{ app }}"]' roles/flatpak/vars/main.yml - git add roles/flatpak/vars/main.yml + yq -i '.system_flatpaks += ["{{ app }}"]' units/flatpak-packages.yml + git add units/flatpak-packages.yml git commit -m "flatpak: add {{ app }}" git push @@ -470,7 +435,7 @@ nvim-update: vim *args: nvim {{ args }} -# Lint YAML + Ansible playbook (uses tools from Brewfile when available) +# Lint YAML and check native unit shell syntax lint: #!/usr/bin/env bash set -euo pipefail @@ -482,17 +447,14 @@ lint: else echo "yamllint not installed (brew install yamllint) — skipping" fi - if command -v ansible-lint >/dev/null 2>&1; then - echo "→ ansible-lint" - ansible-lint site.yml || rc=$? - else - echo "ansible-lint not installed (uv tool install ansible-lint) — skipping" - fi + for file in bootstrap.sh scripts/run-units.sh units/*.sh; do + bash -n "$file" || rc=$? + done exit $rc -# Dry-run apply (no changes) — quick way to see what would change +# Inspect native configuration without applying changes check *args: - cd {{ dotfiles_dir }} && {{ _play }} site.yml --check --diff {{ args }} + cd {{ dotfiles_dir }} && UNITS_CHECK=1 scripts/run-units.sh {{ args }} # Log omp into all providers (OAuth prompts only for missing ones) omp-login: @@ -513,7 +475,7 @@ doctor: echo "→ Machine identity" if [ -f /etc/dotfiles-machine ]; then pass "machine: $(cat /etc/dotfiles-machine)"; else warn "/etc/dotfiles-machine missing — run bootstrap"; fi echo "→ Core binaries" - for bin in brew bw gh tailscale ansible-playbook just mise chezmoi; do + for bin in brew bw gh tailscale just mise chezmoi yq python3; do if command -v "$bin" >/dev/null 2>&1; then pass "$bin"; else warn "$bin missing"; fi done echo "→ Tailscale" diff --git a/README.md b/README.md index 27bae287..49783d4f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # dotfiles -Ansible-managed dotfiles for every machine I own. One command to go from a fresh install to a fully configured system. No central server — each machine manages itself locally, secrets stay in Bitwarden, and everything stays in sync automatically. +Fleet dotfiles and host configuration managed locally by native Bash units, mise and chezmoi. Secrets are fetched from Bitwarden at runtime. Units have platform, profile and privilege gates; a successful apply does not imply every optional component ran. **Overview, live status and architecture: [reilly.asia/infra](https://reilly.asia/infra/)** · handbook: [reilly.asia/infra/handbook](https://reilly.asia/infra/handbook/) @@ -24,18 +24,18 @@ dots # quick pull + apply without secrets | **GitHub** | Auth + signing keys registered automatically | | **GNOME** | Keyboard shortcuts, Ptyxis terminal config | | **Syncthing** | Systemd user service for file sync | -| **Auto-update** | Systemd timer pulls and applies changes daily | +| **Auto-update** | User systemd timer or laptop login service applies non-secret configuration | --- ## Day-to-day commands ```bash -just apply # Pull latest + full apply (prompts for BW unlock) -just apply-nosecrets # Pull latest + apply without Bitwarden (fast) -just dotfiles # Only shell/git/tmux configs -just packages # Only Homebrew + Flatpak -just check # Dry run — see what would change +just apply # Pull latest + apply (attempts BW unlock) +just apply-nosecrets # Pull latest + apply without Bitwarden +just dotfiles # Run the dotfiles unit selection +just packages # Run the package unit selection +just check # Request native inspection; coverage is being integrated just edit-host # Edit this machine's host_vars ``` @@ -55,18 +55,18 @@ From `inventory.yml`: | Name | Group | What | |------|-------|------| | kanpur | desktop | Laptop (Bluefin) | -| himachal | desktop | Laptop — also drives the Hive (`hive_ops`) | +| himachal | desktop | Laptop | | dilli | desktop | Secondary workstation (Bluefin) | | kerala | desktop | postmarketOS ARM device (musl, `apk`) | | mumbai | desktop | Debian VM on the phone (Android Virtualization Framework), `cli-only` | | punjab | server | Headless Ubuntu agent box (EC2 `t3.large`, us-east-1) | | goa | server | Raspberry Pi 5 (applies locally) | | vm | server | Local dev VM | -| termux | termux_hosts | Raw Termux layer on the phone — `termux_packages` only | +| termux | termux_hosts | Raw Termux layer on the phone (platform-specific unit gates apply) | | test-fleet-fedora, test-fleet-node2 | test_fleet | KubeVirt test VMs (`machine_profile: vm-test`) | | matrix, telengana | vps | **Retired** Hetzner VPSes, powered on for burn-in — restart nothing on them | -Not Ansible-managed (Talos, `talosctl`/`kubectl` only): **bihar** + **karnataka** +Not managed by fleet units (Talos, `talosctl`/`kubectl` only): **bihar** + **karnataka** (home cluster — powered down, in storage, expected back) and the AWS cluster in `eu-north-1`. The `llm` group is dormant (commented out in `inventory.yml`). @@ -74,7 +74,7 @@ Not Ansible-managed (Talos, `talosctl`/`kubectl` only): **bihar** + **karnataka* ## Security -Repo is public — no secrets in git, ever. All secrets are fetched from Bitwarden at runtime. `just apply-remote` unlocks Bitwarden on the target (via `scripts/bw-resolve.sh remote`) and exports that session inline over `ssh -t`, so you drive every host from one terminal. +Repo is public — no secrets in git, ever. Units fetch secrets from Bitwarden at runtime; `--no-secrets` suppresses unit vault access. `just apply-remote` resolves a session on the target and forwards it over SSH using `SendEnv=BW_SESSION`, which the target must accept. Retired `matrix` and `telengana` must not be restarted; the shared unit library skips both before unit work. --- @@ -85,5 +85,5 @@ Full handbook lives in [`docs/`](docs/) and is published at [reilly.asia/infra/h - [Talos cluster handbook](docs/src/servers/talos-k8s/cluster.md) — Bihar + Karnataka (home, offline), Lemonade, KubeVirt - [TunaOS AWS cluster handbook](docs/src/servers/aws-k8s/cluster.md) — Matrix/ESS, the Hive, CFP dashboard (TunaOS infrastructure) - [Adding a new machine](docs/src/onboarding.md) -- [Roles reference](docs/src/roles/) +- [Units and runner](docs/src/units.md) and [component reference](docs/src/roles/) - [Bitwarden vault setup](docs/src/bitwarden.md) diff --git a/ansible.cfg b/ansible.cfg deleted file mode 100644 index 1a8135bd..00000000 --- a/ansible.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[defaults] -inventory = inventory.yml -roles_path = roles -gathering = smart -fact_caching = jsonfile -fact_caching_connection = /tmp/ansible_facts -fact_caching_timeout = 3600 -gather_subset = ["!all", "network"] -forks = 20 -stdout_callback = default -result_format = yaml -callbacks_enabled = ansible.posix.profile_tasks, ansible.posix.profile_roles -interpreter_python = auto_silent -inject_facts_as_vars = False -deprecation_warnings = False - -[privilege_escalation] -become = false - -[ssh_connection] -pipelining = true -# Reuse SSH connections across tasks/plays — large speedup for remote applies. -ssh_args = -o ControlMaster=auto -o ControlPersist=300s -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -control_path_dir = ~/.ansible/cp -retries = 2 diff --git a/bootstrap.sh b/bootstrap.sh index 6e1eca13..f2da7e0f 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -65,52 +65,6 @@ ensure_git() { esac } -# ── Install uv ──────────────────────────────────────────────────── -ensure_uv() { - if command -v uv &>/dev/null || [ -x "$HOME/.local/bin/uv" ]; then - export PATH="$HOME/.local/bin:$PATH" - echo "uv found." - return - fi - echo "Installing uv..." - # Retry up to 3 times — DNS may not be ready immediately on fresh installs - local attempts=0 - local tmp_install - tmp_install=$(mktemp) - while [ $attempts -lt 3 ]; do - if curl -LsSf --max-time 30 https://astral.sh/uv/install.sh -o "$tmp_install" && [ -s "$tmp_install" ] && sh "$tmp_install"; then - rm -f "$tmp_install" - export PATH="$HOME/.local/bin:$PATH" - return - fi - attempts=$((attempts + 1)) - echo "uv install attempt $attempts failed, retrying in 5s..." - sleep 5 - done - rm -f "$tmp_install" - # Fallback: install via pip3 - echo "astral.sh unreachable, falling back to pip3 install uv..." - pip3 install --user uv - export PATH="$HOME/.local/bin:$PATH" -} - -# ── Install Ansible ─────────────────────────────────────────────── -ensure_ansible() { - for p in "$HOME/.local/bin" "/home/linuxbrew/.linuxbrew/bin"; do - if [ -x "$p/ansible-playbook" ]; then - export PATH="$p:$PATH" - echo "Ansible found in $p." - return - fi - done - if command -v ansible-playbook &>/dev/null; then - echo "Ansible found." - return - fi - echo "Installing Ansible via uv..." - uv tool install --with ansible ansible-core - export PATH="$HOME/.local/bin:$PATH" -} # ── Install Tailscale + join Tailnet ────────────────────────────── ensure_tailscale() { @@ -303,39 +257,15 @@ ensure_sudo() { sudo chmod 0440 "$sudo_file" } -# ── Install Galaxy collections ──────────────────────────────────── -install_collections() { - echo "Installing Ansible collections..." - ansible-galaxy collection install -r "$DOTFILES_DIR/requirements.yml" --force-with-deps 2>/dev/null || \ - ansible-galaxy collection install community.general -} # ── Phase 1: system + packages + dotfiles (no secrets) ─────────── run_phase1() { local is_desktop="false" [ "$MACHINE_TYPE" = "desktop" ] && is_desktop="true" - - echo "" - echo "═══════════════════════════════════════════════════" - echo " Phase 1: system, packages, dotfiles" - echo " Machine: $MACHINE_NAME ($MACHINE_TYPE)" - echo "═══════════════════════════════════════════════════" - echo "" - + echo "Phase 1: system, packages and dotfiles (no secrets)" cd "$DOTFILES_DIR" || return 1 - local rc=0 urc=0 - ansible-playbook \ - --connection=local \ - -l "$MACHINE_NAME" \ - -e "target=$MACHINE_NAME" \ - -e ansible_connection=local \ - -e ansible_host=127.0.0.1 \ - -e "is_desktop=$is_desktop" \ - --skip-tags secrets \ - site.yml || rc=$? - scripts/run-units.sh --no-secrets || urc=$? - [ "$rc" -ne 0 ] || rc=$urc - return "$rc" + DOTFILES_MACHINE="$MACHINE_NAME" DOTFILES_IS_DESKTOP="$is_desktop" \ + scripts/run-units.sh --no-secrets } # ── Bitwarden login ─────────────────────────────────────────────── @@ -396,21 +326,9 @@ setup_bitwarden() { run_phase2() { local is_desktop="false" [ "$MACHINE_TYPE" = "desktop" ] && is_desktop="true" - cd "$DOTFILES_DIR" || return 1 - local rc=0 urc=0 - ansible-playbook \ - --connection=local \ - -l "$MACHINE_NAME" \ - -e "target=$MACHINE_NAME" \ - -e ansible_connection=local \ - -e ansible_host=127.0.0.1 \ - -e "is_desktop=$is_desktop" \ - --tags secrets \ - site.yml || rc=$? - scripts/run-units.sh --tags omp,secrets || urc=$? - [ "$rc" -ne 0 ] || rc=$urc - return "$rc" + DOTFILES_MACHINE="$MACHINE_NAME" DOTFILES_IS_DESKTOP="$is_desktop" \ + scripts/run-units.sh --tags omp,secrets } # ── Push inventory changes (after GitHub auth is set up) ───────── @@ -441,8 +359,6 @@ main() { ensure_python ensure_git - ensure_uv - ensure_ansible clone_repo get_machine_name get_machine_type @@ -451,7 +367,6 @@ main() { set_hostname ensure_sudo ensure_tailscale - install_collections local rc=0 phase_rc=0 run_phase1 || rc=$? @@ -476,8 +391,7 @@ main() { echo "║ Bootstrap complete! ║" echo "╚═══════════════════════════════════════════════╝" echo "" - echo " Start a new shell:" - echo " eval \"\$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)\" && exec zsh" + echo " Start a new login shell to load your configuration." echo "" echo " To re-apply anytime: just apply" } diff --git a/docs/scripts.md b/docs/scripts.md index 0b73912d..a9c272b0 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -14,13 +14,13 @@ The `scripts/` directory contains utility scripts for inventory management, Bitw ## Inventory Management -These scripts manage the Ansible inventory and track which machines are available. +These scripts manage fleet inventory and track which machines are available. ### `register-machine.py` Register a new machine in the inventory and create its host variables file. -**Purpose**: Add a new machine to the Ansible inventory, creating the necessary entries in `inventory.yml` and initializing a `host_vars/.yml` file for machine-specific configuration. +**Purpose**: Add a new machine to `inventory.yml` and initialize a `host_vars/.yml` file for machine-specific configuration. **Usage**: ```bash @@ -411,13 +411,13 @@ bw get notes \ ## Application State Tracking -These scripts record the outcome of Ansible apply runs and check if the system is in a converged state. +These scripts record native apply outcomes and report their freshness. ### `record-apply.py` -Record the outcome of an Ansible apply run to `~/.cache/dotfiles/last-apply.json`. +Record an apply outcome to `~/.cache/dotfiles/last-apply.json`. -**Purpose**: Track whether the most recent `ansible-playbook` run succeeded or failed, and with which configuration tags and skips. Used by `just doctor` to report convergence freshness. +**Purpose**: Track the native runner's return code and selection label. Used by `just doctor` to report apply freshness; a successful run can still contain skipped or partial units. **Usage**: ```bash @@ -426,7 +426,7 @@ Record the outcome of an Ansible apply run to `~/.cache/dotfiles/last-apply.json ``` **Arguments**: -- `` — the exit code from the ansible-playbook run (required) +- `` — the apply's exit code (required) - `[label]` — descriptive label (default: none; e.g., `full apply`, `desktop only`) - `[skip-tags]` — comma-separated skip-tags used (e.g., `slow,download`) @@ -435,8 +435,8 @@ Record the outcome of an Ansible apply run to `~/.cache/dotfiles/last-apply.json # Record a successful apply ./scripts/record-apply.py 0 "full apply" -# Record a failed apply with skips -ansible-playbook main.yml --skip-tags slow; ./scripts/record-apply.py $? "desktop" "slow" +# Record a failed native apply +./scripts/record-apply.py 1 "apply-nosecrets" "secrets" ``` **Output** (`~/.cache/dotfiles/last-apply.json`): @@ -500,7 +500,7 @@ or Sync currently installed Homebrew packages to `group_vars/all.yml`. -**Purpose**: Keep the Ansible group variables in sync with the actual installed packages, so you can track package changes and automate Homebrew reinstalls via Ansible. +**Purpose**: Keep group variables in sync with installed Homebrew packages for the native Homebrew unit. **Usage**: ```bash diff --git a/docs/src/architecture.md b/docs/src/architecture.md index d096c171..8f834cc1 100644 --- a/docs/src/architecture.md +++ b/docs/src/architecture.md @@ -1,149 +1,34 @@ # Architecture -## Repository Layout +## Repository layout -``` -dotfiles/ -├── site.yml # Main playbook — entrypoint for all machines -├── inventory.yml # Host inventory + group definitions -├── Justfile # Task runner (just apply, just lint, etc.) -├── aws/ # OpenTofu for the TunaOS AWS account and cluster -├── group_vars/ -│ ├── all.yml # Shared vars: packages, flatpaks, fleet config -│ └── vps.yml # VPS-specific overrides -├── host_vars/ -│ └── .yml # Per-machine settings (is_laptop, skip_*, etc.) -├── roles/ -│ └── / # Each role: tasks/, files/, templates/, vars/ -├── docs/ # This handbook -├── scripts/ # Utility scripts (inventory scan, BW seed) -├── talos-k8s/ # Talos Linux + Kubernetes cluster manifests -└── tools/ # Standalone tools, including the secret-broker -``` +- `Justfile` provides local and remote fleet commands. +- `scripts/run-units.sh` selects and orders native units; `mise.toml` registers them as tasks. +- `units/*.sh` and their helpers/assets configure the host. `home/` is the chezmoi source for shell, Git, Neovim and Atuin configuration. +- `inventory.yml`, `group_vars/` and `host_vars/` hold fleet membership and settings. Some connection keys retain their historical `ansible_` names and are read by native tooling. +- `scripts/` holds operational and Bitwarden helpers. `roles/pi/files/skills/` remains a source of omp skills; not every directory named `roles` is an active Ansible role. +- `aws/` contains OpenTofu for the TunaOS AWS account; `talos-k8s/` contains home-cluster manifests; `docs/` is the handbook. -## How It Works +## Applying configuration -Each machine manages **itself** via `ansible-playbook --connection=local`. There's no central control node. This is an [Ansible](https://www.ansible.com/) pull model — the playbook lives on every machine and each host applies it locally. +`just apply` pulls the checkout, resolves and syncs a Bitwarden session when available, and invokes the units runner. If the vault is unavailable, the apply continues with secrets disabled. `just apply-nosecrets` also invokes the runner with `--no-secrets`; it does not unlock the vault. Neither invokes an Ansible playbook. -### Local Apply +`just apply-tags kube,shell` selects units through the tag map in `scripts/run-units.sh`. Secret-bearing tags attempt to resolve and sync Bitwarden; on failure the selected units still run with secrets disabled, and the recipe returns an error. Tags select units, not a bypass of their platform, inventory, profile or host-variable gates. See [Units](units.md) for staging and failure behavior. -```bash -just apply -``` +`just apply-remote HOST` resolves the session on the target, forwards it over SSH using `SendEnv=BW_SESSION`, pulls the remote checkout and runs `just apply` there. The target SSH daemon must accept the forwarded variable. Phone updates use a separate user timer and a temporary archive, with secrets disabled. -Runs the playbook locally with `--connection=local`. The playbook auto-detects which host it's running on by matching `/etc/dotfiles-machine` or falling back to `localhost`. +`just check` sets `UNITS_CHECK=1` for the native runner. Check coverage and exact safety/accuracy across platforms are still subject to integration verification; do not treat it as a complete substitute for a reviewed apply or a guarantee of no changes. -### Remote Apply +### Available tags -```bash -just apply-remote himachal -``` +The authoritative mapping is `tasks_for_tag` in `scripts/run-units.sh`. Common selections include `system`, `packages`, `dotfiles`, `shell`, `ssh`, `mesh_check`, `secrets`, `desktop`, `services`, `osquery`, `security`, `hive`, and `fleet_facts`, plus individual names such as `omp`, `tailscale`, `gnome`, and `proxy`. A tag selects only the listed units: unselected prerequisites are not implicitly applied. `--no-secrets` suppresses unit vault access, rather than removing all secret-capable units from the selection. -SSHes to the target, forwards the Bitwarden session token, pulls the latest dotfiles, then runs `just apply` locally on the remote machine. +## Conditional execution -### Tagged Apply +Each unit implements its own gates. Depending on the unit these include inventory groups, `skip_*` host variables, `machine_profile`, platform detection, executable availability, or noninteractive root access. For example, SSH mesh skips `skip_ssh_mesh` and `vm-test`; proxy and Tailscale certificates skip VPSes and `vm-test`; package units detect their respective platforms. Consult the unit and its [reference page](roles/) before applying. Machine identity comes from `DOTFILES_MACHINE`, `/etc/dotfiles-machine`, or hostname, in that order. -```bash -just apply-tags kube,shell -``` +The retired `matrix` and `telengana` VPSes remain powered on for burn-in: **do not restart anything on them**. The shared unit library skips both identities before unit work, including package upgrades that could restart services indirectly. This guard does not authorize other changes to those hosts. -Runs roles and units selected by the given tags. Explicit secret-bearing tags first resolve and sync Bitwarden; unrelated tags do not prompt. `just apply-nosecrets` passes `--skip-tags secrets` to Ansible and `--no-secrets` to units (see [Bitwarden](bitwarden.md)). +## Secrets -#### Available Tags - -Every role in `site.yml` carries one or more tags. Tags fall into two kinds: **phase tags** shared by roles in that phase, and **role tags** naming (or further scoping) one specific role. Phases follow the `# Phase N` comments in `site.yml`. - -| Phase | Role | Tags | -|-------|------|------| -| (pre/post tasks) | fact setup, platform detection, secrets summary | `always` — runs on every tagged apply unless `--skip-tags always` | -| (before Phase 1) | `sudo` | `system`, `sudo` | -| System + packages | `sshd` | `system`, `sshd` | -| System + packages | `apk_packages` | `system`, `packages`, `apk` | -| System + packages | `homebrew` | `packages`, `homebrew` | -| System + packages | `termux_packages` | `system`, `packages`, `termux` | -| System + packages | `bitwarden` | `secrets`, `bitwarden` | -| System + packages | `omp` unit | `dotfiles`, `omp` | -| System + packages | `hive_ops` | `hive`, `hive_ops` | -| Secrets + auth | `ssh_keys` | `secrets`, `ssh`, `ssh_keys` | -| Secrets + auth | `ssh_mesh` | `ssh`, `mesh_check` | -| Secrets + auth | `tailscale` | `secrets`, `tailscale` | -| Desktop apps | `flatpak` | `packages`, `desktop`, `flatpak` | -| Desktop apps | `bluefin_common` | `desktop`, `bluefin` | -| Desktop apps | `gnome` | `desktop`, `gnome` | -| Desktop apps | `zen_browser` | `desktop`, `browser`, `zen_browser` | -| Desktop apps | `pipewire_audio` | `desktop`, `audio`, `pipewire` | -| Services | `syncthing` | `services`, `syncthing` | -| Services | `systemd` | `services`, `systemd` | -| Services | `bst_dashboard` | `services`, `bst_dashboard` | -| Services | `kirocrew` | `services`, `kirocrew` | -| Services | `proxy` | `services`, `proxy`, `caddy` | -| Services | `tailscale_cert` | `services`, `tailscale_cert` | -| Services | `retired_services` | `system`, `retired_services` | -| Services | `server_hardening` | `system`, `hardening`, `security` | -| Services | `fleet_facts` | `facts`, `fleet_facts` | - -Shell, git, neovim and atuin config, fonts, omp, the shell/atuin/GitHub/Forgejo secrets and the kubeconfigs are not roles: they are [units](units.md) (`chezmoi`, `fonts`, `omp`, `atuin`, `shell-ai`, `github`, `kube`, `forgejo-registry`) that `just apply*` runs after the playbook. The old tags `dotfiles`, `shell`, `fonts`, `git`, `neovim`, `editor`, `atuin`, `ai`, `secrets`, `github`, `kube`, `forgejo`, `registry`, `omp` still select them through `scripts/run-units.sh`. - -Common phase-wide selections: - -- `--tags secrets` — every BW-backed role only (`bitwarden`, `ssh_keys`, `tailscale`; the `atuin`, `shell-ai`, `github`, `kube` and `forgejo-registry` units also run for this tag) -- `--tags desktop` — every desktop-app role (`flatpak`, `bluefin_common`, `gnome`, `zen_browser`, `pipewire_audio`) -- `--tags services` — every long-running service role (`syncthing`, `systemd`, `bst_dashboard`, `kirocrew`, `proxy`, `tailscale_cert`) -- `--skip-tags secrets` — everything except BW-backed roles. `just apply-nosecrets`, bootstrap's first phase and `phone-apply.service` also pass `--no-secrets` to units. - -Tags do not replace the [`skip_*` host_vars flags](#skip-flags) or inventory-group gating described below — a role can be tag-selected and still no-op if its `when:` condition (group membership, `skip_*`, `machine_profile`) evaluates false. - -## Role Structure - -Each role follows the standard Ansible layout: - -``` -roles// -├── tasks/ -│ └── main.yml # Role entrypoint -├── files/ # Static files to deploy -├── templates/ # Jinja2 templates (with .j2 extension) -├── vars/ # Role-specific variables -│ └── main.yml -└── handlers/ # Role handlers (notify targets) - └── main.yml -``` - -## Conditional Execution - -Roles are gated by: - -- **Inventory groups** — `when: is_desktop | bool` runs only on desktop group members -- **Host vars** — `skip_*: true` flags in `host_vars/.yml` skip specific roles -- **Tags** — `--tags kube,shell` limits which roles execute - -### Skip Flags - -Every flag `site.yml` reads, all defaulting to `false` (`| default(false)`). Set `skip_: true` in `host_vars/.yml` to skip that role on that machine. No role reads a `skip_*` variable itself — the gating is all in `site.yml`. - -| Flag | Role skipped | Other `when:` gates on the role | Set in `host_vars/` | -|------|--------------|---------------------------------|---------------------| -| `skip_sudo` | `sudo` | — | `termux` | -| `skip_ssh_mesh` | `ssh_mesh` | not `vm-test` | — | -| `skip_kube` | `kube` unit | desktop, not `vm-test` | `mumbai`, `termux` | -| `skip_forgejo` | `forgejo-registry` unit | not `vm-test` | — | -| `skip_flatpak` | `flatpak` | desktop, not `vm-test` | `mumbai`, `termux` | -| `skip_bluefin` | `bluefin_common` | desktop, not `vm-test` | `mumbai`, `termux` | -| `skip_gnome` | `gnome` | desktop, not `vm-test` | `mumbai`, `termux` | -| `skip_zen_browser` | `zen_browser` | desktop, not `vm-test` | `mumbai`, `termux` | -| `skip_pipewire_audio` | `pipewire_audio` | desktop, not `vm-test` | `mumbai` | -| `skip_syncthing` | `syncthing` | not `vps`, not `vm-test` | `mumbai`, `termux` | -| `skip_systemd` | `systemd` | — | `termux` | -| `skip_proxy` | `proxy` | not `vps`, not `vm-test` | `mumbai`, `punjab`, `termux` | -| `skip_tailscale_cert` | `tailscale_cert` | not `vps`, not `vm-test` | `mumbai`, `termux` | - -"desktop" is `is_desktop` (membership of the `desktop` group unless overridden); "not `vm-test`" is `machine_profile != 'vm-test'`. Roles without a flag either always run or are gated another way: `termux_packages` (`termux_hosts` group), `kirocrew` (`kirocrew_enabled: true`), `server_hardening` (`server`/`vps` groups), `retired_services` (not `vps`, not Termux, systemd hosts). The `omp` unit skips Termux. - -## Secrets Flow - -1. Ansible requests a Bitwarden session (env var, cached file, or interactive unlock) -2. The `bitwarden` role resolves `BW_SESSION` and caches it -3. Secrets-tagged roles consume the session token for API calls -4. For remote applies, `scripts/bw-resolve.sh remote ` unlocks the vault on the target and SSH forwards that session using `SendEnv=BW_SESSION`; the target must accept it — see [Remote Apply](bitwarden.md#remote-apply). - -No secrets are stored in git — the repo is fully public. +The repository is public. Unit helpers read a usable `BW_SESSION` or `/tmp/bw_session`, without prompting, and honor `UNITS_NO_SECRETS=1`. Apply recipes resolve and sync sessions before starting secret-bearing work. Individual units decide how to handle missing items or a locked vault, generally reporting `PARTIAL` or `SKIPPED` and retaining existing secret files. See [Bitwarden](bitwarden.md) and each unit's documentation for details. Remote session forwarding requires a properly configured SSH server. diff --git a/docs/src/bitwarden.md b/docs/src/bitwarden.md index f367cd9c..439ce65e 100644 --- a/docs/src/bitwarden.md +++ b/docs/src/bitwarden.md @@ -4,16 +4,11 @@ All secrets live in [Bitwarden](https://bitwarden.com/) and are resolved at runt ## How It Works -1. The `bitwarden` role runs first in the secrets phase. -2. The apply wrapper can prompt via `scripts/bw-resolve.sh`; the role itself reads `BW_SESSION` from the environment, then falls back to `/tmp/bw_session`. -3. It runs `bw sync` (with a 15 s hard timeout, so a flaky vault server can't hang the play) to refresh the local cached vault. -4. It calls `bw status` and sets a `bw_unlocked` host fact: `true` only if the CLI is installed, a session was resolved, *and* the vault actually unlocks. -5. Every BW-using role gates on `when: bw_unlocked | default(false)`. The end-of-play `post_tasks` summary prints one of: - - `vault unlocked — all secret roles executed normally` - - `Bitwarden vault was LOCKED — … skipped. To enable: bw unlock …` - - `SKIPPED (--skip-tags secrets). Run dots-apply to include …` +1. `just apply` uses `scripts/bw-resolve.sh local` to resolve a session from the environment, `/tmp/bw_session`, or an interactive unlock. +2. The recipe validates the session and runs `bw sync` with a 15-second timeout before invoking `scripts/run-units.sh`. If resolution or sync fails it continues with `--no-secrets`. +3. Unit helpers validate `BW_SESSION` or `/tmp/bw_session` noninteractively, and honor `UNITS_NO_SECRETS=1`. Units handle unavailable sessions and items independently; inspect their `SKIPPED` and `PARTIAL` output. -For `just apply-tags` with an explicit secret-bearing tag such as `kube`, `ai` or `omp`, the wrapper validates and syncs a session before running, and adds the `bitwarden` tag so legacy secret roles receive `bw_unlocked`. It skips secret work and returns non-zero if the session or sync fails; unrelated tags do not prompt. +For `just apply-tags` with a secret-bearing tag such as `kube`, `ai` or `omp`, the recipe resolves and syncs a session before running. If this fails it runs the selected units with secrets disabled and returns nonzero. Unrelated tags do not prompt. ## Remote Apply @@ -25,30 +20,27 @@ For `just apply-tags` with an explicit secret-bearing tag such as `kube`, `ai` o vault; to unlock it pipes the master password (item `James Bitwarden`) over stdin, falling back to an interactive `bw unlock` over `ssh -t`. 2. Sends the remote session through SSH `SendEnv=BW_SESSION`, not in the command line. The target SSH daemon must accept `BW_SESSION`; if it rejects a resolved session, the apply aborts. `apply-remote` also caches it in the remote's `/tmp/bw_session`. -3. The `bitwarden` role on the remote runs `bw status`: if it reports - `unlocked`, `bw_unlocked` flips to true and secrets work proceeds. +3. The remote apply validates and syncs the session before invoking native units. If a resolved session is not accepted over SSH, the remote apply aborts. -**Caveat — session portability is not guaranteed.** A `BW_SESSION` token is only valid on the machine that unlocked it — that's why the recipes unlock on the remote instead of forwarding yours. If the remote unlock fails, the remote `bw status` returns `locked`, the role prints a one-line warning, and every BW-touching task is skipped cleanly — nothing fails, no files get clobbered with empty values. Unlock the vault directly on that host (`bw unlock`, then write the session to `/tmp/bw_session`) to enable secrets there. +**Caveat:** sessions are host-specific. If remote resolution fails, secret-capable units run with secrets disabled; targeted secret applies return nonzero. Unlock Bitwarden on that host and retry. Each unit determines how to preserve or report existing data; consult its reference. ## Vault Items -Every item a role or script reads (grep `bw get` / `bw list` under `roles/` and `scripts/`): +Common items consumed by units and scripts (the unit sources and their configuration are authoritative): | Bitwarden Item | Type | Used By | |---------------|------|---------| -| `james@` | SSH Key | `ssh_keys` role — per-machine ed25519 keys | -| `tailscale-apikey` | Login (password) | `tailscale` role — key for `tailscale up --authkey` and stale-device cleanup via the API | -| `kubeconfig` | Secure Note | `kube` role — home cluster kubeconfig | -| `talosconfig` | Secure Note | `kube` role — home cluster Talos config | -| `kubeconfig-aws-migration` | Secure Note | `kube` role — AWS cluster kubeconfig | -| `talosconfig-aws-migration` | Secure Note | `kube` role — AWS cluster Talos config | -| `accounts.firefox.com` | Login + TOTP | `browser_fxa` role — Firefox Account credentials | -| `github-token` | Login (password) | `github` role — PAT for `gh auth login` | -| `atuin.sh` | Login + custom field `key` | `atuin` unit — atuin sync login + mnemonic | -| `deepseek-api-key` | Login (password) | `shell-ai` unit — pi `auth.json` | -| `forgejo` | Secure Note (`API Token:` line) | `shell-ai` unit — `FORGEJO_TOKEN` | -| `tavily-api-key` | Login (password) | `shell-ai` unit — `TAVILY_API_KEY` | -| `forgejo.manatee-basking.ts.net` | Login | `forgejo_registry` role — podman registry login | +| `@` | SSH Key | `ssh-keys` unit — per-machine ed25519 keys | +| `tailscale-apikey` | Login (password) | `tailscale` unit — join and stale-device API cleanup | +| `kubeconfig`, `talosconfig` | Secure notes | `kube` unit — home cluster credentials | +| `kubeconfig-aws-migration`, `talosconfig-aws-migration` | Secure notes | `kube` unit — stored AWS cluster credentials | +| `accounts.firefox.com` | Login + TOTP | `browser-fxa` unit — Firefox Account credentials | +| `github-token` | Login (password) | `github` unit — PAT for `gh` | +| `atuin.sh` | Login + custom field `key` | `atuin` unit — history sync | +| `deepseek-api-key`, `tavily-api-key` | Login (password) | `shell-ai` unit — shell environment | +| `forgejo`, `meta-api-key`, `kiro-api-key` | Secure notes | `shell-ai` unit — shell environment | +| `omp-auth-broker-token` | Login (password) | `omp` unit — broker client token (item configurable) | +| `forgejo.manatee-basking.ts.net` | Login | `forgejo-registry` unit — Podman registry login | | `bw-api-key` | Login | `scripts/bw-resolve.sh` — `bw login --apikey` on a remote | | `James Bitwarden` | Login | `scripts/bw-resolve.sh` — non-interactive unlock on a remote | @@ -57,11 +49,11 @@ Every item a role or script reads (grep `bw get` / `bw list` under `roles/` and For fast iteration that doesn't need the vault — packages, dotfiles, desktop config — use the explicit no-secrets path: ```bash -just apply-nosecrets # git pull + apply with --skip-tags secrets +just apply-nosecrets # git pull + native units with --no-secrets dots # alias: cd to the repo + just apply-nosecrets ``` -The daily systemd timer (`dotfiles-update.service`) calls `apply-nosecrets`: Ansible skips secret-tagged roles and units receive `--no-secrets`, even if a cached session exists. +The systemd update service calls `apply-nosecrets`, so units receive `--no-secrets` even if a cached session exists. ## No Session? diff --git a/docs/src/cluster/hive.md b/docs/src/cluster/hive.md index cbb8ab02..a63b4268 100644 --- a/docs/src/cluster/hive.md +++ b/docs/src/cluster/hive.md @@ -1,40 +1,55 @@ -# Hive — 24/7 AI Agent Supervisor - -> **TunaOS infrastructure.** This runs in the TunaOS AWS account and serves -> the TunaOS project (Hive, Matrix, CI), not James's personal fleet. See -> [TunaOS AWS Account & IaC](../servers/aws/README.md). - -> **Now runs on the [AWS Talos cluster](../servers/aws-k8s/cluster.md)** at -> **https://school.tunaos.org** (namespace `hive`, Cloudflare-proxied, on the -> control-plane node; `hive.tunaos.org` is a legacy alias). Every Hive — school, -> reef, hive.reilly.asia — reports to **https://hub.tunaos.org**, the one place to -> see them all. Point `kubectl` at it with -> `export KUBECONFIG=~/.kube/config-aws-migration`. Operator timers (rotation, -> watchdog, peak windows) live in the [`hive_ops`](../roles/hive_ops.md) role on -> himachal. The architecture below was written for the home cluster (bihar, -> Tailscale ingress `hive.manatee-basking.ts.net`), which is powered down — -> treat node names and the Tailscale URL as historical. - -> **Hive** is an open-source AI agent orchestration system running on the Talos K8s cluster. A fleet of specialized agents autonomously maintain the `tuna-os/tunaos` repository — triaging issues, analyzing code, and creating PRs. A governor dynamically adjusts agent pace based on issue queue depth. - -## Table of contents - -1. [Architecture](#1-architecture) -2. [What Hive does](#2-what-hive-does) -3. [Deployment](#3-deployment) -4. [AI backend: pi + DeepSeek](#4-ai-backend-pi--deepseek) -5. [Agents](#5-agents) -6. [ACMM levels](#6-acmm-levels) -7. [Governor](#7-governor) -8. [Configuration](#8-configuration) -9. [Building the image](#9-building-the-image) -10. [Day-to-day operations](#10-day-to-day-operations) -11. [Troubleshooting](#11-troubleshooting) -12. [GitHub App](#12-github-app) +# Hive — AI Agent Supervisor + +## Current deployment: Spindle VPS + +The school and reef Hives and their hub are deployed on the +[Spindle VPS](../servers/spindle-vps/README.md), not Kubernetes. On 2026-10-27, +read-only inspection found the `school`, +`reef`, `hub`, and `hub-front` Docker Compose services running under +`/srv/hives` on `spindle` (`ubuntu@100.113.75.88` over Tailscale). The live +Caddy configuration routes [school.tunaos.org](https://school.tunaos.org) and +[reef.tunaos.org](https://reef.tunaos.org) to their spokes, and +[hub.tunaos.org](https://hub.tunaos.org) to the hub and its frontend. Container +state alone does not establish that the public endpoints or agents are healthy. + +The configured images are `ghcr.io/hivecommons/hive:v6-latest` for the two +spokes and `ghcr.io/hivecommons/hive-hub:v6-latest` for the hub. The Compose +services bind-mount `/srv/hives/bin/hive` over `/usr/local/bin/hive`, so the +image tag alone does not describe the running binary. The observed school and +reef seed configurations use the `omp` backend with +`github-copilot/gpt-6-luna`, not the historical pi/DeepSeek backend below. +Runtime state and model selection may differ from the seed files. + +The old AWS cluster also had a `hive.reilly.asia` spoke; no hanthor container +was present in the inspected Spindle Docker inventory. Its current status was +not established. The former [hive_ops role](../roles/hive_ops.md) targets the +retired cluster, not this Compose deployment. + +For a read-only inventory, run: +```bash +ssh ubuntu@100.113.75.88 'docker ps --filter name=hives-' +``` + +## Archived cluster guide + +> **Historical reference only — do not apply or run the Kubernetes, image-build, +> secret, restart, or state-reset instructions below.** The AWS Talos cluster +> was terminated on 2026-10-25; its Kubernetes resources and CronJobs are not a +> current deployment. The home Talos cluster (bihar and karnataka) is powered +> down temporarily, not decommissioned. Preserve its configurations. The +> sections below describe older home-cluster and AWS configurations at different +> points in time, not the current Spindle service. See the +> [AWS cluster history](../servers/aws-k8s/cluster.md) and +> [historical cluster manifests](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/hive). +> Examples are retained for context, not as instructions for either cluster. + +The earlier home-cluster Hive used pi with DeepSeek, tmux agents, a governor, +and a GitHub App. The architecture and examples below are snapshots of that +period; later AWS configurations and the current Compose deployment differ. --- -## 1. Architecture +## 1. Architecture (historical) ``` ┌──────────────────────────────────────────────────────────────┐ @@ -92,7 +107,7 @@ Agents share a persistent `/data` volume (10Gi PVC) for beads, session state, an --- -## 2. What Hive does +## 2. What Hive did (historical) ### Advisory digest @@ -112,11 +127,11 @@ Agents can contribute facts to a shared knowledge base. Facts are tagged by type --- -## 3. Deployment +## 3. Deployment (historical) ### Namespace and resources -All Hive resources live in the `hive` namespace on the Talos cluster: +The home-cluster deployment used the `hive` namespace on bihar: ``` hive (namespace) @@ -129,7 +144,7 @@ hive (namespace) └── secret/ghcr-auth — GHCR pull credentials ``` -### Files +### Historical files (not the current repository layout) ``` talos-k8s/hive/ @@ -148,7 +163,7 @@ talos-k8s/hive/ ### CI workflow -`.github/workflows/hive-build.yml` — triggers on push to `master`: +The former `.github/workflows/hive-build.yml` workflow did the following: 1. Clones `kubestellar/hive` v2 source 2. Applies `pi-marker.patch` and three sed patches to the Go source: @@ -159,32 +174,17 @@ talos-k8s/hive/ 4. Builds Docker image with pi + pi-wrapper.sh + fd/ripgrep 5. Pushes to `ghcr.io/hanthor/hive:latest` -### Secrets - -```bash -kubectl create secret generic hive-secrets -n hive \ - --from-literal=DEEPSEEK_API_KEY=sk-... \ - --from-literal=GH_APP_ID= \ - --from-literal=GH_APP_INSTALLATION_ID= \ - --from-file=gh-app-key.pem=/path/to/key.pem -``` - -### Deploy - -Each spoke's manifests live in `talos-k8s/hive/spokes//manifest.yaml` -(exported from the cluster; secrets are out of band). The hive container -image is owned by the hive-operator `HiveRelease` controller, so do not roll -images by applying these files. See `talos-k8s/hive/README.md`. - -```bash -kubectl apply -f talos-k8s/hive/spokes/school/manifest.yaml -``` +### Secrets and deployment (historical) -Access: **https://school.tunaos.org** (AWS cluster; legacy alias `hive.tunaos.org`). Fleet-wide view of every Hive: **https://hub.tunaos.org**. The old home-cluster URL was `https://hive.manatee-basking.ts.net`. +The home-cluster pod used a Kubernetes Secret for DeepSeek and GitHub App +credentials. Later, the AWS deployment exported spoke manifests under +`talos-k8s/hive/spokes/` and used the hive-operator to manage images. These +manifests are historical records, **not deployment instructions**. The home +cluster's former Tailscale ingress was `hive.manatee-basking.ts.net`. --- -## 4. AI backend: pi + DeepSeek +## 4. AI backend: pi + DeepSeek (historical) ### Why pi (not goose) @@ -232,9 +232,9 @@ Written at runtime by the wrapper: --- -## 5. Agents +## 5. Agents (historical) -### Current agents (ACMM L5) +### Historical agents (ACMM L5) | Agent | Mode | Cadence | Role | |-------|------|---------|------| @@ -296,7 +296,7 @@ governor: --- -## 6. ACMM levels +## 6. ACMM levels (historical) The ACMM (Autonomous Code Maintenance Maturity) framework defines 6 levels: @@ -309,7 +309,7 @@ The ACMM (Autonomous Code Maintenance Maturity) framework defines 6 levels: | **L5** | Guarded Auto-Merge | Auto-merge on green CI with guardrails | | **L6** | Full Autonomy | Unrestricted agent actions | -Our Hive runs at **L5**. At each level, the Go binary enables additional pack agents and upgrades agent modes. +That historical configuration used **L5**; this is not a statement about the current spokes. ### Changing ACMM level @@ -318,16 +318,12 @@ Our Hive runs at **L5**. At each level, the Go binary enables additional pack ag acmm_level: 5 ``` -After changing, nuke the persistent state to clear stale overrides: - -```bash -kubectl exec -n hive deploy/hive -- sh -c 'echo "{}" > /data/hive-state.json' -kubectl rollout restart deploy/hive -n hive -``` +The old procedure reset persistent state and restarted the pod; do not use it +on the current deployment or on retained backups. --- -## 7. Governor +## 7. Governor (historical) The governor evaluates the issue queue every 5 minutes (`eval_interval_s: 300`) and switches between four modes: @@ -338,7 +334,7 @@ The governor evaluates the issue queue every 5 minutes (`eval_interval_s: 300`) | **busy** | ≤50 issues | Faster scanner, ci-maintainer still active | | **surge** | >50 issues | Scanner every 15m, ci-maintainer paused | -### Current cadences +### Historical cadences ```yaml governor: @@ -371,7 +367,7 @@ The supervisor always runs at 5m regardless of mode. --- -## 8. Configuration +## 8. Configuration (historical) ### Backends (backends.conf) @@ -381,7 +377,7 @@ The `backends.conf` in the ConfigMap defines supported AI backends. Adding a new 2. `backend_binary()` — maps backend name to binary in PATH 3. `backend_perm_flag()` — permission flag (pi returns empty — no permission popups) -Current supported: `claude copilot bob gemini codex amazonq goose aider pi` +The old supported list was `claude copilot bob gemini codex amazonq goose aider pi`. ### Agent configuration (hive.yaml) @@ -402,34 +398,15 @@ Two locations store state that survives pod restarts: - **`/data/hive-state.json`** — Governor mode, agent overrides, kick history, cadence overrides - **`/data/agent-configs/*.yaml`** — Per-agent persistent config (backend, model, mode, enabled, display name) -If the dashboard shows stale backend/model/mode values, these files have overrides from a previous configuration. Nuke them: - -```bash -kubectl exec -n hive deploy/hive -- sh -c ' - echo "{}" > /data/hive-state.json - for f in /data/agent-configs/*.yaml; do - sed -i "s/backend: copilot/backend: pi/" "$f" - sed -i "s/model: claude-sonnet-4-6/model: deepseek-v4-pro/" "$f" - sed -i "s/model: claude-opus-4-6/model: deepseek-v4-pro/" "$f" - done -' -kubectl rollout restart deploy/hive -n hive -``` +Persistent overrides could cause stale values in that deployment. Do not copy +its old state-reset and backend-rewrite commands to the current deployment. --- -## 9. Building the image - -### CI (recommended) +## 9. Building the image (historical) -Push to `master` — the workflow builds and pushes to `ghcr.io/hanthor/hive:latest`. - -### Manual (Kaniko) - -```bash -kubectl apply -f talos-k8s/hive/build-job.yaml -kubectl logs -n hive -f job/hive-build -``` +The former CI workflow built `ghcr.io/hanthor/hive:latest`; an older Kaniko +job was also used. Neither is a current deployment procedure. ### What's in the image @@ -444,76 +421,36 @@ kubectl logs -n hive -f job/hive-build --- -## 10. Day-to-day operations +## 10. Historical operations -### Check agent status +Operators inspected the old pod's health, tmux panes, pi processes and logs +with `kubectl`. Those commands targeted the retired Kubernetes deployment; +the read-only Compose inventory at the top of this page is the current starting +point instead. -```bash -# Public health check (the dashboard itself is login-gated — 401 is normal) -curl -s -o /dev/null -w '%{http_code}\n' https://school.tunaos.org/api/health +### Historical restarts and state resets -# Full status: authenticated, from inside the pod (see the tunaos-hive-checkin skill) -TOKEN=$(kubectl get secret -n hive hive-secrets -o jsonpath='{.data.HIVE_DASHBOARD_TOKEN}' | base64 -d) -kubectl exec -n hive deploy/hive -- curl -sS -H "X-Hive-Internal: $TOKEN" http://127.0.0.1:3002/api/status | jq -``` - -### View agent terminal - -```bash -# List tmux sessions -kubectl exec -n hive deploy/hive -- tmux -S /tmp/tmux-1001/default ls - -# View an agent's terminal -kubectl exec -n hive deploy/hive -- tmux -S /tmp/tmux-1001/default capture-pane -t hive-scanner -p -S -30 -``` - -### Check pi processes - -```bash -kubectl exec -n hive deploy/hive -- ps aux | grep "pi$" -# Should show 9 pi processes (one per active agent) -``` - -### View logs - -```bash -kubectl logs -n hive -l app.kubernetes.io/name=hive -f --tail=100 -``` - -### Restart - -```bash -kubectl rollout restart deploy/hive -n hive -``` - -### Nuke persistent state - -If agents have stale backend/model overrides or wrong modes: - -```bash -kubectl exec -n hive deploy/hive -- sh -c 'echo "{}" > /data/hive-state.json' -# Also check /data/agent-configs/ for stale copilot/goose references -kubectl rollout restart deploy/hive -n hive -``` +The former procedure restarted the pod and sometimes reset persistent state. +It is not a recovery procedure for Spindle; do not reset retained data. --- -## 11. Troubleshooting +## 11. Historical troubleshooting | Symptom | Check | |---------|-------| -| ImagePullBackOff | IPv6 timeouts on ghcr.io. Verify `/etc/hosts` on bihar has a pinned IPv4 entry for `ghcr.io` (` ghcr.io`, from `dig +short A ghcr.io`) | +| ImagePullBackOff | The home cluster once encountered IPv6 timeouts to ghcr.io | | Agent CLI crashed | Check `ps aux \| grep pi` — should see 9 pi processes | -| Agent stuck in ADVISORY mode | Check ACMM level in state file. May need state nuke | -| Dashboard shows old backend/model | Stale overrides in `/data/agent-configs/*.yaml`. Fix with sed + state nuke | +| Agent stuck in ADVISORY mode | Historical state-file ACMM overrides | +| Dashboard shows old backend/model | Historical `/data/agent-configs/*.yaml` overrides | | No kicks sent | Governor eval cycle: 5min. Agent cadences vary per mode | -| Issues not created | Verify GitHub App installed on tuna-os. Check token: `cat /var/run/hive-metrics/gh-app-token.cache` | +| Issues not created | GitHub App installation and authentication were relevant | | "CLI did not reach input prompt" | pi is mid-response when kick arrives. Normal — agent will pick up next kick | | fd/ripgrep download errors | pi can't download tools through MITM proxy. Pre-installed in Docker image | | "backend binary not found" | Missing binary in PATH. Check `backends.conf` and verify wrapper is at `/usr/local/bin/pi` | | "failed to persist config" | Read-only ConfigMap mount. Cosmetic — config is cached in memory | -### Common agent issues +### Historical agent issues **Agent shows bare bash shell:** - pi process died or never started @@ -526,12 +463,11 @@ kubectl rollout restart deploy/hive -n hive - Wait for next governor cycle **Agent mode mismatch:** -- State file has persistent mode override -- Nuke state and restart +- The state file held persistent mode overrides in that deployment. --- -## 12. GitHub App +## 12. GitHub App (historical) Hive authenticates to GitHub via a GitHub App installed on `tuna-os` (App and installation IDs are in the `hive-secrets` Secret). @@ -557,19 +493,17 @@ The entrypoint generates an installation token on startup and caches it at `/var The App gives Hive its own rate limit pool, separate from personal API usage. -### Updating permissions +### Permission changes (historical) -1. Go to: `https://github.com/organizations/tuna-os/settings/apps/hanthor-hive-agent/permissions` -2. Add the needed permissions -3. Re-accept on the installation page -4. Restart hive: `kubectl rollout restart deploy/hive -n hive` +GitHub App permission changes required re-acceptance by the installation. Do +not use the old Kubernetes restart procedure to apply them. -## Images and upgrades +## Images and upgrades on the former AWS cluster -As of 2026-09-24 every Hive runs **stock upstream images** -(`ghcr.io/hivecommons/hive`, `hive-hub`), not the `tuna-os/hive` fork. The fork's -remaining delta was CI/registry plumbing plus two ACMM tweaks nothing here uses, -and it had fallen a major version behind (v4 vs upstream's v5). +On 2026-09-24 the AWS Hives used stock upstream images +(`ghcr.io/hivecommons/hive`, `hive-hub`), not the `tuna-os/hive` fork. +The configuration described below was subsequently superseded by the +`HiveRelease` operator controller before the cluster was terminated. | Hive | Namespace | Branding | |---|---|---| @@ -578,20 +512,11 @@ and it had fallen a major version behind (v4 vs upstream's v5). | school.tunaos.org (legacy alias hive.tunaos.org) | `hive` | SCHOOL | | hub.tunaos.org | `hive-hub` | — | -**`hive-upgrade`** (CronJob in ns `hive`, daily 04:30 America/New_York) keeps -them on upstream's newest `v5.x.y` release. It resolves the release to a -digest, checks the amd64 image exists, and upgrades one Hive at a time, in the -order in the table above. Each target soaks for 10 minutes. A health, branding -or crash failure rolls that target back to its recorded digest, blocklists the -version and stops the run. "Could not measure" (an RBAC or API error) aborts -without rolling back. Results go to Discord. Details, dry-run, pin/block: -[`talos-k8s/hive/upgrade/README.md`](https://github.com/hanthor/dotfiles/blob/master/talos-k8s/hive/upgrade/README.md). - -It checks once a day, not on every release: upstream cuts several releases an -hour, and each Hive restart costs a boot-time GitHub rescan against a shared -App rate limit. Restarting `hive` and `hive-reef` back-to-back exhausted it on -2026-09-24. - -The old `hive-fork-{switch,verify,drift,ai-check}` CronJobs are **suspended** -(annotated with the reason). They targeted the fork and never had the RBAC to -act. +At that time the `hive-upgrade` CronJob checked upstream v5 releases daily, +upgrading one Hive at a time with a soak and rollback. It was later deleted in +favor of the operator's `HiveRelease` controller. The retained +[`upgrade` history](https://github.com/hanthor/dotfiles/blob/master/talos-k8s/hive/upgrade/README.md) +is not an active upgrade runbook. + +The old `hive-fork-{switch,verify,drift,ai-check}` CronJobs had been suspended; +they do not run on the terminated AWS cluster. diff --git a/docs/src/index.md b/docs/src/index.md index 908bf504..b114a8cb 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,6 +1,6 @@ # Dotfiles Fleet Handbook -Ansible-driven dotfiles and infrastructure for a personal fleet. Three distinct concerns live here: +Native-unit dotfiles and infrastructure for a personal fleet. Three distinct concerns live here: 1. **Workstation config** — shells, packages, browser, SSH, Tailscale, kubeconfig, GNOME, and more 2. **Home Talos K8s cluster** — manifests for the Bihar + Karnataka cluster @@ -24,21 +24,21 @@ just apply-tags kube,shell # Apply to a remote machine just apply-remote himachal -# Dry-run to see what would change +# Request native inspection (coverage is subject to integration verification) just check -# Lint all YAML and Ansible +# Lint YAML and native unit shell scripts just lint ``` ## Machine Fleet -Ansible-managed hosts, from `inventory.yml`: +Fleet hosts, from `inventory.yml`: | Machine | Group | Type | Role | |---------|-------|------|------| | kanpur | desktop | Laptop | Portable workstation | -| himachal | desktop | Laptop | Portable workstation; runs `hive_ops` | +| himachal | desktop | Laptop | Portable workstation | | dilli | desktop | Desktop | Secondary workstation | | kerala | desktop | Mobile | postmarketOS ARM device | | mumbai | desktop | VM | Debian VM on the phone, `machine_profile: cli-only` | @@ -50,7 +50,7 @@ Ansible-managed hosts, from `inventory.yml`: | matrix | vps | VPS | **Retired** — see the cutover runbook | | telengana | vps | VPS | **Retired** — see the cutover runbook | -Not Ansible-managed (Talos — `talosctl`/`kubectl` only): +Not managed by fleet units (Talos — `talosctl`/`kubectl` only): - **bihar** (control plane) + **karnataka** (worker, AMD GPU) — the home cluster. Powered down and in storage, expected back. See the [cluster handbook](servers/talos-k8s/cluster.md). @@ -58,16 +58,14 @@ Not Ansible-managed (Talos — `talosctl`/`kubectl` only): The `llm` group is dormant (commented out in `inventory.yml`). -## Playbook Phases +## Native units -`site.yml` groups roles into four commented phases. Tags cut across them — use -`just apply-tags ` to target a subset. +`just apply*` runs the [native units](units.md) via `scripts/run-units.sh`. +Prerequisites run in stages; independent units can run concurrently via mise, +with a sequential fallback. `just apply-tags ` selects units from the +wrapper's tag map, while `just apply-nosecrets` disables unit vault access. +Unit-specific platform, profile and privilege gates still apply. -| Phase | Roles | Main tags | -|-------|-------|-----------| -| 1 — System + packages + dotfiles | sudo, sshd, apk_packages, homebrew, termux_packages, bitwarden, hive_ops | `system`, `packages`, `dotfiles` | -| 2 — Secrets + auth | ssh_keys, ssh_mesh, tailscale | `secrets` | -| 3 — Desktop apps | flatpak, bluefin_common, gnome, zen_browser (+ browser_fxa), pipewire_audio | `desktop` | -| 4 — Services | syncthing, systemd, bst_dashboard, kirocrew, proxy, tailscale_cert, server_hardening | `services` | - -After the playbook, `just apply*` runs the [bash units](units.md) (`chezmoi`, `fonts`, `omp`, `atuin`, `shell-ai`, `github`, `kube`, `forgejo-registry`) in parallel through mise. The `omp` unit also uses Bitwarden for its broker token. `just apply-nosecrets` skips secret-tagged roles and passes `--no-secrets` to units. +The retired `matrix` and `telengana` VPSes remain powered on for burn-in. +The shared unit library skips them; **nothing on either host may be restarted**. +See the [cutover runbook](../matrix-cutover-runbook.md). diff --git a/docs/src/inventory.md b/docs/src/inventory.md index 2db5562c..f09a22a3 100644 --- a/docs/src/inventory.md +++ b/docs/src/inventory.md @@ -60,7 +60,7 @@ test_fleet: ``` ### `termux_hosts` -The raw Termux/Android layer on the phone. Gates the `termux_packages` role. +The raw Termux/Android layer on the phone. Gates the `termux-packages` unit. ```yaml termux_hosts: @@ -88,17 +88,16 @@ skip_flatpak: true # Skip Flatpak installs skip_gnome: true # Skip GNOME config skip_kube: true # Skip kubeconfig deploy skip_proxy: true # Skip Caddy proxy -machine_profile: cli-only # Free-form profile; `vm-test` skips desktop + service roles +machine_profile: cli-only # Free-form profile; some units skip `vm-test` web_services: [] # Service links for the local dashboard ``` -The full set of `skip_*` flags `site.yml` honours: `skip_sudo`, `skip_ssh_mesh`, -`skip_kube`, `skip_forgejo`, `skip_flatpak`, `skip_bluefin`, `skip_gnome`, -`skip_zen_browser`, `skip_pipewire_audio`, `skip_syncthing`, `skip_systemd`, -`skip_proxy`, `skip_tailscale_cert`. +Units read their own `skip_*` flags and profile settings; see the relevant +unit under `units/` and [Architecture](architecture.md#conditional-execution). +The flags are not all interpreted identically. ## Group Variables - **`group_vars/all.yml`** — Homebrew package lists, Flatpak remotes, Firefox Accounts config, GNOME extensions, authorized SSH keys -- **`roles/flatpak/vars/main.yml`** — the `system_flatpaks` list +- **`units/flatpak-packages.yml`** — the `system_flatpaks` list - **`group_vars/vps.yml`** — currently empty diff --git a/docs/src/onboarding.md b/docs/src/onboarding.md index c1ff0a3c..d65968b1 100644 --- a/docs/src/onboarding.md +++ b/docs/src/onboarding.md @@ -60,13 +60,13 @@ curl -fsSL https://raw.githubusercontent.com/hanthor/dotfiles/master/bootstrap.s ``` The bootstrap script: -1. Ensures python, git, uv and Ansible, then clones the dotfiles repo from [GitHub](https://github.com/hanthor/dotfiles) -2. Registers the machine in `inventory.yml` if needed (prompts for the type when `--type` isn't given) -3. Writes `/etc/dotfiles-machine` so the playbook can self-identify, sets the hostname, sets up sudo and Tailscale -4. Runs `site.yml --skip-tags secrets`, then all bash units with `--no-secrets` (installing [Homebrew](https://brew.sh/) and core packages among the rest) -5. Unlocks and syncs Bitwarden, runs `site.yml --tags secrets`, then the `omp` and `secrets` units, and pushes any inventory changes +1. Ensures Python and git, then clones the dotfiles repo from [GitHub](https://github.com/hanthor/dotfiles). +2. Registers the machine in `inventory.yml` if needed (prompts for the type when `--type` isn't given). +3. Writes `/etc/dotfiles-machine`, sets the hostname, and sets up sudo and Tailscale. +4. Runs native units with `--no-secrets` (including package installation). +5. Resolves and syncs Bitwarden, runs the `omp,secrets` unit selection, and pushes any inventory changes. -Each phase runs its units even if Ansible fails. Bootstrap continues to Phase 2 when possible, but exits non-zero if either phase or Bitwarden setup fails. +Bootstrap continues to the secret phase when possible, but reports failure if a phase or Bitwarden setup fails. Review unit output: skips and partial results are not proof that every host component is configured. ## After Bootstrap Completes @@ -81,11 +81,11 @@ just apply ``` **Why this step matters:** -- Bootstrap's first phase deliberately skips secrets; the second phase only covers secret-tagged roles and the broker unit. -- `just apply` reruns the full playbook and units with secrets and completes any work that was skipped. -- This distributes SSH keys across the fleet, registers with GitHub and Tailscale, and sets up daily/on-login automation. +- Bootstrap's first phase disables secrets; the second selects secret-capable units and omp. +- `just apply` reruns the full native selection with secrets where available. +- This can distribute SSH keys, register with GitHub and Tailscale, and set up daily/on-login automation on eligible hosts. -Do **not** edit `host_vars/.yml` before the first `just apply` — bootstrap sets sensible defaults, and `just apply` will pull in critical secrets (SSH keys, GitHub tokens, Tailscale auth). Any per-machine customizations can follow once the first apply succeeds; the `skip_*` flags that turn individual roles off are listed in [Architecture → Skip Flags](architecture.md#skip-flags). +Host-specific gates and overrides belong in `host_vars/.yml`; consult [Architecture](architecture.md#conditional-execution) and the unit implementations when setting them. The details of what happens during this first apply are covered in the [Post-Onboarding](#post-onboarding) section below. @@ -96,7 +96,7 @@ The details of what happens during this first apply are covered in the [Post-Onb | Type | Group | Has Desktop? | Notes | |------|-------|-------------|-------| | `desktop` | desktop | Yes | Laptops are desktops too — set `is_laptop: true` in `host_vars` | -| `server` | server | No | Gets `server_hardening`; desktop roles skip automatically | +| `server` | server | No | Gets `server_hardening`; desktop units skip automatically | | `vps` | vps | No | Also skips `syncthing`, `proxy`, `tailscale_cert` | There is no `laptop` type — a type that doesn't match a group leaves the host in @@ -105,12 +105,12 @@ hand. ## Post-Onboarding -1. The `ssh_keys` role generates a fresh ed25519 key on the new host, then pushes the pub key up to Bitwarden as `james@`. -2. The new host now needs to **trust the rest of the fleet, and the rest of the fleet needs to trust it**. The first apply on the new host downloads every other host's pub key into `~/.ssh/authorized_keys`. The other hosts will pick up the new host's pub key on *their* next full apply (`dots-apply` on each — the timer runs without secrets, so it skips `ssh_keys`). -3. The `github` role registers the SSH key with GitHub for both auth and commit signing. -4. The `tailscale` role joins the machine to the [Tailscale](https://tailscale.com/) network using the auth key in BW. +1. The `ssh-keys` unit restores the host's ed25519 key from Bitwarden, or generates one if a successful unlocked vault listing has no item, and syncs it as `@`. +2. The unit merges available vault public keys into `~/.ssh/authorized_keys` without removing user entries. Other hosts pick up a new vault key on their next unlocked apply. The separate `ssh-mesh` unit also merges committed fleet public keys during no-secrets applies. +3. The `github` unit can register the SSH key with GitHub for authentication and signing. +4. The `tailscale` unit joins a disconnected eligible machine using its Bitwarden key. 5. The daily `dotfiles-update.timer` (servers) or on-login service (laptops) runs `just apply-nosecrets` — secrets stay on the manual `dots-apply` cadence because BW can't be unlocked non-interactively. Run `just doctor` on the new host once it's bootstrapped — it'll tell you whether BW is unlocked, whether the timer is wired up, and the timestamp of the last successful apply. -See [Architecture](architecture.md) for how the playbook runs on each machine and [Secrets with Bitwarden](bitwarden.md) for the details of the BW handshake. +See [Architecture](architecture.md) for native unit execution and [Secrets with Bitwarden](bitwarden.md) for the BW handshake. diff --git a/docs/src/operational-readiness.md b/docs/src/operational-readiness.md index f9394082..20841ae6 100644 --- a/docs/src/operational-readiness.md +++ b/docs/src/operational-readiness.md @@ -1,147 +1,24 @@ # Fleet Operational Readiness -**Status**: Planning phase -**Updated**: October 2026 -**Horizon**: Pre-scaled operations +**Status:** Planning guidance, not a validated recovery guarantee. The fleet uses native units and Git configuration with Bitwarden for runtime secrets. Eligible hosts run non-secret updates on a user timer or laptop login; availability and successful convergence are not guaranteed. The home Talos cluster is temporarily offline in storage, not decommissioned. -This document outlines operational procedures, failure modes, disaster recovery, and scaling strategy for the dotfiles fleet. +## Common operations -## Overview +- **Add a machine:** follow [Onboarding](onboarding.md), using `just onboard ` or `just add-machine `. Inspect unit output and run `just doctor` on the host. A `SKIPPED` or `PARTIAL` line can mean configuration remains incomplete. +- **Rotate credentials:** update the relevant Bitwarden item securely, then run an unlocked apply on affected hosts. Consult the corresponding unit's documentation for rotation safeguards; do not put credentials in git. The [runbook](runbook.md) covers SSH key rotation. +- **Retire a machine:** review dependencies, remove its inventory and tailnet registration when appropriate, and preserve any needed data and Bitwarden items. The `just purge ` recipe removes inventory and host variables, but does not delete vault items. +- **Recover an auto-update:** inspect `systemctl --user status dotfiles-update.timer` (or the laptop login service), and `journalctl --user -u dotfiles-update.service`. Review git status in `~/.local/share/dotfiles`, then run `just apply-nosecrets` when appropriate. -The dotfiles fleet consists of 14+ machines across desktop, server, VPS, and test environments, managed by Ansible with centralized configuration in git and secrets in Bitwarden. Each machine applies changes daily via systemd timer and pulls from git automatically. +**Retired VPS safeguard:** `matrix` and `telengana` remain powered on for burn-in. Nothing on them may be restarted. Native units' shared library skips both; follow the [cutover runbook](../matrix-cutover-runbook.md), not generic retirement or recovery instructions, for those hosts. -**RTO/RPO targets** (provisional): -- Desktop machines: RTO 4 hours (manual reimaging), RPO 1 day (daily sync) -- Server/cluster: RTO 2 hours (orchestrated failover), RPO 15 min (distributed state) +## Failure and recovery considerations -## Common Operations +- Vault unavailable: secret-dependent units may skip or report partial work; existing credentials are not necessarily refreshed. Retry an unlocked apply once access is restored. +- Git unavailable: `apply-nosecrets` tolerates pull failure and runs the existing checkout. Review its age before relying on it. +- Host unreachable: check connectivity and SSH. Do not treat an offline host as successfully updated. +- Suspected secret exposure: revoke or rotate the affected credentials and investigate; public git and client-side checks do not guarantee prevention of leaks. +- Cluster loss: use the appropriate [home](servers/talos-k8s/cluster.md) or [AWS](servers/aws-k8s/cluster.md) handbook and verified backups. Persistent data recovery and failover times are not guaranteed by fleet unit applies. -### Add a New Machine +## Readiness checks -1. Create `host_vars/.yml` with machine profile (desktop/server/vps) -2. Add entry to `inventory.yml` with group assignment -3. Generate SSH key: `just generate-host-key ` -4. Bootstrap from fresh install: `just bootstrap ` -5. Verify secrets resolved and sync completed: `just check ` - -**Expected time**: 20 minutes (automated bootstrap included) - -### Rotate Bitwarden Credentials - -1. Generate new Bitwarden auth token on target machine -2. Update `roles/bitwarden/defaults/main.yml` with new token -3. Test unlock: `just check --vault-password-file ~/.bw_unlock` -4. Apply to all: `just apply` (prompts for BW unlock once, distributes session) - -**Frequency**: Quarterly or after security incident -**Expected time**: 15 minutes - -### Retire a Machine - -1. Comment out entry in `inventory.yml` -2. Remove `host_vars/.yml` -3. Remove from Tailscale admin console -4. Power down machine (do not destroy hardware yet for 30 days) - -**Verification**: Machine no longer appears in `ansible-inventory` - -### Recover from Failed Auto-Update - -If a machine fails daily sync: - -1. SSH to machine: `ssh ` -2. Check systemd timer status: `systemctl status dotfiles-apply.timer` -3. Trigger manual apply: `just apply-nosecrets` -4. Check for git merge conflicts: `cd ~/.dotfiles && git status` -5. If conflicts exist, resolve manually in `host_vars/.yml` then `git pull` - -**Recovery time**: 5 minutes (usually just a git pull) - -## Failure Modes - -| Failure | Detection | Recovery | -|---------|-----------|----------| -| **Machine offline** | Ansible host unreachable | SSH to machine, check network; if unrecoverable, re-bootstrap | -| **Bitwarden down** | Secret resolution fails on apply | Retry on next scheduled timer (1 hour); manual secrets pull if urgent | -| **Git repository unavailable** | Clone/pull fails | Machine continues with cached config; retry on next timer (DNS/network usually recovers in <10 min) | -| **SSH key compromised** | Detected by audit | Regenerate key, update `authorized_keys` on all machines, audit git signing keys | -| **Secrets leaked to git** | Pre-commit hook catches; never committed by design | N/A — design prevents this | -| **Cluster node failure** (Talos) | Kubernetes node status | Automated failover; if persistent, reimage node via `talosctl` | - -## Disaster Recovery - -### Total Machine Loss (Hardware Failure) - -1. Identify replacement hardware or cloud instance -2. Create new `host_vars/.yml` for the replacement -3. Run `just bootstrap ` (same logical name, new hardware) -4. Machine re-joins fleet with same Tailscale identity (rotated keys, same node ID) -5. Services resume from last synced state (1-day RPO for desktops, 15-min for servers) - -**Time to recovery**: 30 minutes + application startup time - -### Talos Cluster Node Loss - -1. Mark node as drained: `kubectl drain ` -2. Reimage node: `talosctl reset ` -3. Rejoin cluster: `talosctl apply-config -n --file controlplane.yaml` -4. Verify node rejoined: `kubectl get nodes` - -**Time to recovery**: 10 minutes - -### Total Cluster Loss - -Not currently addressed. Clusters are rebuilt from infrastructure-as-code (Talos patches, deployments in git), but persistent volumes require separate backup. - -**Action items**: Define backup strategy for etcd and persistent volumes (outside current scope). - -## Scaling Strategy - -### Machines < 20 - -Current model is sustainable. All machines apply daily; no coordination needed. Bitwarden unlocked once per administrator per day. - -### Machines 20–50 - -Projected state: Cost/complexity still manageable. -**Action items**: -- Distribute daily apply times to avoid thundering herd -- Consider `just apply-batch ` for staged rollouts -- Monitor Bitwarden API rate limits (currently unused) - -### Machines > 50 - -Requires architectural change (not in current scope): -- Centralized state server (e.g., Vault instead of Bitwarden) -- Event-driven apply (git webhook → targeted machines) -- Observability layer (currently none) - -## Testing and Validation - -### Pre-Deployment Checklist - -- [ ] Changes tested on `vm` machine first (VM group) -- [ ] No secrets in git: `git diff --cached | grep -i secret` -- [ ] Lint passes: `ansible-lint` -- [ ] All machines in `inventory.yml` have corresponding `host_vars` -- [ ] SSH keys present for all new machines - -### Annual Disaster Recovery Drill - -- [ ] Reimage one desktop machine from scratch -- [ ] Verify Talos cluster node can rejoin -- [ ] Measure actual RTO and compare against targets - -## Known Limitations - -1. **Single Bitwarden vault** — no redundancy. If vault is inaccessible, no new secrets can be resolved (existing cached credentials continue to work for ~24 hours). -2. **Centralized git control** — all machines pull from one repository. If repo is unavailable, machines continue with cached config until next sync succeeds. -3. **No cross-region** — all machines are on one Tailscale network and in same cloud regions. No active-active failover. -4. **Test fleet not automated** — `test_fleet` machines are manual-apply only, not on auto-update timer. - -## Next Steps - -1. **Validate RTO/RPO** — run annual drill and measure actual times -2. **Document cluster recovery** — add procedures for Talos/Kubernetes emergencies -3. **Backup strategy** — design and test etcd/persistent volume backups -4. **Scaling model** — decide on 20–50 machine architecture before reaching that scale +Review changes on an appropriate non-production host, keep secrets out of git, inspect unit-specific gates and backups, and use the checks described in [Contributing](../../CONTRIBUTING.md). `just check` requests native inspection, but its coverage and non-mutating behavior across platforms are subject to integration verification. Record actual recovery times in drills rather than assuming RTO/RPO targets have been met. diff --git a/docs/src/roles/apk_packages.md b/docs/src/roles/apk_packages.md index c055c950..6e3a8a3d 100644 --- a/docs/src/roles/apk_packages.md +++ b/docs/src/roles/apk_packages.md @@ -1,19 +1,5 @@ # apk_packages -**Tags:** `system`, `packages`, `apk` -**Secrets needed:** No -**Runs on:** All machines (skips if not musl/Alpine) +`units/apk-packages.sh` runs only when the shared library detects a musl loader. Package lists live in `units/apk-packages.conf`. It retries `apk update`, installs missing bootstrap and ordinary packages without upgrading installed ones, and installs Bitwarden CLI globally via npm. If a batch install fails it retries packages individually so unrelated packages can still install; optional mise and cache refresh are best effort. -Installs core packages on Alpine/musl-based systems where Homebrew is unavailable. - -## What It Does - -1. Detects musl libc (Alpine, postmarketOS) — skips entirely on glibc systems -2. Installs core CLI tools via `apk`: bash, fish, zsh, git, neovim, tmux, ripgrep, fzf, jq, podman, etc. -3. Installs Bitwarden CLI via npm (not available in apk repos) -4. Enables podman and podman.socket system services -5. Adds the user to the `podman` group - -## When It's Skipped - -On glibc systems (Fedora, Debian, Ubuntu, Bluefin), this role does nothing — Homebrew handles all package management. +The unit attempts to enable Podman and its socket and adds the current user to the `podman` group; a new login may be needed. The shared unit library skips retired `matrix` and `telengana` before package or group changes; nothing on those hosts may be restarted. Privileged operations require root or noninteractive `sudo -n`; no vault secrets are needed. diff --git a/docs/src/roles/bitwarden.md b/docs/src/roles/bitwarden.md index 40ef71ae..95827343 100644 --- a/docs/src/roles/bitwarden.md +++ b/docs/src/roles/bitwarden.md @@ -1,53 +1,7 @@ -# bitwarden +# Bitwarden session resolution -**Tags:** `secrets`, `bitwarden` -**Secrets needed:** Yes (provides secrets to other roles) -**Runs on:** All machines +Bitwarden is handled by `scripts/bw-resolve.sh` and the apply recipes, not a fleet unit or a host fact. `just apply` resolves a local session, validates it and runs `bw sync` with a 15-second timeout before running units. If resolution or sync fails, it continues with `--no-secrets`. `just apply-tags` does this for secret-bearing selections; on failure it still runs the selection without secrets but returns an error. `just apply-nosecrets` does not unlock the vault. -Resolves a Bitwarden session, refreshes the local vault cache, and publishes a single `bw_unlocked` host fact that every other BW-using role consumes. Failure to unlock is **never fatal** to the play; it's reported once at the end and downstream BW operations are skipped cleanly. +Unit helpers use a valid `BW_SESSION` or `/tmp/bw_session` noninteractively, and honor `UNITS_NO_SECRETS=1`. Units handle missing sessions and items individually, often printing `SKIPPED` or `PARTIAL` and preserving existing secret data. A successful apply alone does not establish that all secrets were refreshed. -## What it does - -1. Resolves `BW_SESSION` in this order: environment → `/tmp/bw_session` cache → interactive `bw unlock`. -2. Runs `bw sync` with a hard 15 s timeout (so a flaky vault server can't hang the whole play). -3. Calls `bw status` and sets `bw_unlocked` (boolean host fact) — `true` only if `bw` is installed, a session was resolved, and the vault actually reports `unlocked`. -4. If a session is set but the vault is still locked, emits a one-line debug warning so the operator knows secrets work is being skipped here. On a remote apply, this means target-side session resolution did not unlock the target vault. - -Every BW-using role downstream gates on `when: bw_unlocked | default(false)`. The end-of-play `post_tasks` summary then prints one of three messages: - -| State | Message | -|---|---| -| unlocked | `vault unlocked — all secret roles executed normally` | -| locked | `Bitwarden vault was LOCKED — … skipped. To enable: bw unlock …` | -| `--skip-tags secrets` | `SKIPPED (--skip-tags secrets). Run dots-apply to include …` | - -## Remote applies - -`just apply-remote ` (and `just apply-remote-tags `) resolve -the Bitwarden session on the **target host**; they do not forward the local -host's `BW_SESSION`: - -1. `scripts/bw-resolve.sh remote ` unlocks Bitwarden on the target and - prints that host's session token. If the remote CLI is logged out, it tries - the configured API-key and master-password paths, then falls back to an - interactive unlock over `ssh -t`. -2. The recipe exports the returned `BW_SESSION` inline in the remote command - and caches it in the target's `/tmp/bw_session`. -3. The remote `bitwarden` role runs `bw status`; when it reports `unlocked`, - `bw_unlocked` becomes true and secrets work proceeds. - -A session token is valid only on the host that unlocked it. If target-side -resolution fails, the remote vault remains locked and BW-using tasks skip -cleanly. See the [full remote apply flow](../bitwarden.md#remote-apply). - -## How to verify it ran correctly - -- `just doctor` will show `vault unlocked` if BW is good on this host. -- Check the end-of-play summary line printed by every `just apply*`. -- Look for any task in `ansible-playbook` output with a `[bitwarden : …]` prefix that ended in `failed:` — should never happen; only "skipping" is acceptable. - -## Notes - -- Must run before any other secrets role (it does, by position in `site.yml`). -- The cached `/tmp/bw_session` persists for the user across reboots only as long as `/tmp` survives (most modern systemd boots clear it). The role re-resolves the session on every play. -- `bw sync` failure is non-fatal; downstream operations use the locally cached vault, which may be stale. +For remote applies, `scripts/bw-resolve.sh remote ` attempts to log in and unlock Bitwarden on the target, returning a target-side session. The apply recipe forwards it using SSH `SendEnv=BW_SESSION`; the target must accept it. Tokens are host-specific. See [Secrets with Bitwarden](../bitwarden.md) for vault items and setup. diff --git a/docs/src/roles/bluefin_common.md b/docs/src/roles/bluefin_common.md index 57c56c62..952828cb 100644 --- a/docs/src/roles/bluefin_common.md +++ b/docs/src/roles/bluefin_common.md @@ -1,42 +1,10 @@ -# bluefin_common +# bluefin +**Unit:** `units/bluefin.sh` **Tags:** `desktop`, `bluefin` **Secrets needed:** No -**Runs on:** Desktop group only (skipped on Bluefin hosts unless `force_bluefin: true`) +**Runs on:** desktops, except `skip_bluefin: true` or `machine_profile: vm-test`. -Syncs [Bluefin](https://projectbluefin.io/) branding, fonts, wallpapers, and GNOME extensions from the Project Bluefin OCI image. +With a GNOME session, on non-Bluefin systems or when `force_bluefin: true`, the unit pulls the amd64 `ghcr.io/projectbluefin/common:latest` image with Podman, extracts available branding into `~/.local/share`, adjusts wallpaper XML paths and runs `units/bluefin/files/bluefin-sync.py`. Missing individual image assets are reported as partial. It installs Inter and JetBrains Mono fonts and attempts the Bluefin extension installer; extension installation is best effort, not a guarantee that every extension works in the running GNOME version. -## What It Does - -1. Pulls the `ghcr.io/projectbluefin/common:latest` container image -2. Extracts Bluefin gsettings/dconf overrides, backgrounds, icons, and logos -3. Installs branding assets to `~/.local/share/` -4. Fixes XML background paths to point to home directory -5. Applies dconf settings via a Python script -6. Installs Bluefin-standard fonts (Inter, JetBrains Mono) -7. Installs Bluefin-recommended GNOME extensions -8. Deploys a wallpaper rotation service and timer -9. Deploys the `corral` (Go) VM utility - -## Fonts - -- **[Inter](https://rsms.me/inter/)** — UI font (from [GitHub releases](https://github.com/rsms/inter)) -- **[JetBrains Mono](https://www.jetbrains.com/lp/mono/)** — monospace font for terminals and editors - -## Wallpaper Rotation - -A systemd timer (`bluefin-wallpaper-rotate.timer`) cycles through Bluefin backgrounds daily. The Python script applies the wallpaper via D-Bus. - -## Corral - -Deploys **corral**, the Go VM manager -(CLI + TUI + web). Corral is downloaded from the rolling `binaries` release of -[hanthor/corral](https://github.com/hanthor/corral) (checksum-verified, so it -updates whenever CI publishes a new build). It is no longer embedded in this -repo. - -## Notes - -- Skips by default on actual Bluefin hosts (they already have this) -- Set `force_bluefin: true` to apply on Bluefin (e.g., after an image reset) -- Set `skip_bluefin: true` in `host_vars` to skip entirely +On eligible desktops, including Bluefin itself, it deploys `units/bluefin/files/bluefin-wallpaper-rotate.py` and user systemd service/timer templates, enables the timer and runs a rotation. It downloads the current architecture's corral binary and verifies its SHA256 against the release checksum, removes retired tailvm executables and completions, and configures `/etc/sysctl.d/99-zram.conf` with `vm.swappiness = 150`. The sysctl step requires root or non-interactive sudo; wallpaper and GNOME actions need the invoking user's live D-Bus/systemd session. Bluefin hosts skip only the branding, fonts and extension steps unless forced, not the whole unit. diff --git a/docs/src/roles/browser_fxa.md b/docs/src/roles/browser_fxa.md index 938e9ff9..100237f8 100644 --- a/docs/src/roles/browser_fxa.md +++ b/docs/src/roles/browser_fxa.md @@ -1,31 +1,12 @@ # browser_fxa -**Tags:** none of its own — pulled in by `zen_browser` via `include_role`, so it runs under `desktop`, `browser`, `zen_browser` -**Secrets needed:** Yes (Firefox Account credentials from Bitwarden) -**Runs on:** Desktop group only +**Unit:** `units/browser-fxa.sh` +**Tags:** `desktop`, `browser`, `zen_browser`, `secrets` +**Secrets needed:** Firefox Account credentials in Bitwarden +**Runs on:** desktops, except `skip_zen_browser: true` or `machine_profile: vm-test`. -Automates Firefox Account sign-in for Zen Browser (and optionally Firefox). +This separate unit attempts Firefox Account sign-in for the selected existing browser profile. It reads `browser_fxa_browser`, `fxa_venv_path`, `fxa_bw_item`, and `fxa_bw_username` from host variables, then `group_vars/all.yml`, then `units/browser-fxa/defaults.yml`. The default browser is `zen`; `firefox` checks Flatpak first, then native Firefox, while `firefox-flatpak` checks only Flatpak. Missing `profiles.ini` skips the unit. -## What It Does +It creates a Python venv, installs PyFxA if needed, and deploys `units/browser-fxa/fxa_signin.py` to `~/.local/bin/fxa_signin.py`. With an unlocked Bitwarden session, it searches the configured item (default `accounts.firefox.com`), selects an exact configured username match when supplied, and obtains credentials and an optional TOTP. Existing `signedInUser.json` is chmod 0600; new token files are created with a restrictive umask. Credentials are not printed. -1. Fetches `accounts.firefox.com` credentials + TOTP from Bitwarden -2. Writes `signedInUser.json` to the browser profile directory -3. This pre-authenticates the browser with Firefox Sync - -## Configuration - -```yaml -# group_vars/all.yml -fxa_bw_item: "accounts.firefox.com" -fxa_bw_username: "" -fxa_browsers: - - zen - - firefox - - firefox-flatpak -``` - -## Notes - -- This only enables FxA sign-in, not the full Sync key bundle -- Opening the browser once after deployment may still be needed to complete Sync -- If credentials are unavailable, the task fails silently — manual sign-in works fine +A locked vault or `--no-secrets` leaves sign-in untouched and reports `PARTIAL`; a missing, incomplete or mismatched item also reports `PARTIAL`. Sign-in failure is best effort and may require manual login. The unit attempts sign-in but does not prove browser login or full Sync operation. diff --git a/docs/src/roles/bst_dashboard.md b/docs/src/roles/bst_dashboard.md index 2da9f631..5021a2bb 100644 --- a/docs/src/roles/bst_dashboard.md +++ b/docs/src/roles/bst_dashboard.md @@ -1,22 +1,5 @@ # bst_dashboard -**Tags:** `services`, `bst_dashboard` -**Secrets needed:** No -**Runs on:** Machines with build infrastructure +`units/bst-dashboard.sh` renders `units/bst-dashboard/bst-dashboard.service.j2` with `units/service-templates.py`, installs the resulting user service, reloads the user manager when changed, and enables and starts it. It also enables user linger via root or noninteractive `sudo -n`. It does not install the dashboard project or retrieve secrets. -Deploys a BuildStream build monitoring dashboard. - -## What It Does - -1. Creates dashboard directory based on hostname -2. Deploys dashboard web UI files -3. Configures dashboard to track local BuildStream build queues - -## Access - -Proxied through Caddy at `https://.manatee-basking.ts.net/bst/`. - -## Notes - -- Used to monitor BuildStream build jobs on machines that run builds -- Dashboard shows build queue status, job history, and artifact cache +The unit skips retired `matrix` and `telengana`, and skips when the project directory is missing. On `karnataka` it selects `/var/home/james/dev/kde-linux` and image `aurora`; elsewhere it selects `/var/home/james/dev/dakota` and image `dakota`. Access is through the Caddy `/bst/` route when that proxy is configured. diff --git a/docs/src/roles/flatpak.md b/docs/src/roles/flatpak.md index acc7a286..76c86653 100644 --- a/docs/src/roles/flatpak.md +++ b/docs/src/roles/flatpak.md @@ -1,39 +1,12 @@ # flatpak +**Unit:** `units/flatpak.sh` **Tags:** `packages`, `desktop`, `flatpak` **Secrets needed:** No -**Runs on:** Desktop group only (`skip_flatpak: false`) +**Runs on:** desktops, except `skip_flatpak: true` or `machine_profile: vm-test`. -Installs and manages [Flatpak](https://flatpak.org/) applications from [Flathub](https://flathub.org/). +The unit requires Flatpak and `yq`. It reads system remotes and `system_flatpaks` from `units/flatpak-packages.yml`, repairs a dangling `/var/lib/flatpak` symlink, adds remotes, removes system Firefox and Thunderbird Flatpaks, and installs missing applications from Flathub. A failed batch is retried individually. `DOTFILES_UPGRADE=1` or `true` requests a best-effort system update. System operations require root or non-interactive sudo. -## What It Does +Host-specific `user_flatpak_remotes` and `user_flatpaks` in `host_vars/.yml` are applied with `--user`, separately from the system store. For `himachal`, these configure FlatPark and user Flathub, then install `org.tunaos.blueshell`, `org.tunaos.tavern`, and `org.tunaos.compass` from FlatPark. An already installed user app with a different origin is preserved rather than switched. This installation does not configure or unlock Compass's keyring or establish an application login; those depend on the user's session and app setup. -1. Fixes broken Flatpak symlinks if present -2. Adds the Flathub remote -3. Removes unwanted system flatpaks (Firefox, Thunderbird — replaced by Zen) -4. Installs missing flatpaks in batch (fast path), falling back to individual installs -5. Updates all flatpaks when `upgrade: true` is set -6. Creates Chromium wrapper scripts for CLI compatibility - -## Package List - -Defined in `roles/flatpak/vars/main.yml` as `system_flatpaks` (the Flathub remote is in `flatpak_remotes`, also set in `group_vars/all.yml`). Key apps: - -| App | Flatpak ID | -|-----|-----------| -| Zen Browser | `app.zen_browser.zen` | -| VS Code | `com.visualstudio.code` | -| Ungoogled Chromium | `com.github.Eloston.UngoogledChromium` | -| Flatseal | `com.github.tchx84.Flatseal` | -| GNOME apps | Calculator, Calendar, Clocks, Contacts, Loupe, etc. | -| Vesktop | `dev.vencord.Vesktop` | -| Obsidian | `md.obsidian.Obsidian` | - -## Wrapper Scripts - -Creates `~/.local/bin/chromium`, `~/.local/bin/google-chrome`, and `~/.local/bin/chromium-browser` that exec into the Ungoogled Chromium flatpak — tools that hardcode `google-chrome` or `chromium-browser` still work. - -## Notes - -- Installs system-wide (`--system`) with `become: true` -- Batch install tolerates partial failures — individual retries catch stragglers +The unit also writes `chromium`, `chromium-browser`, `google-chrome`, and `google-chrome-stable` wrappers in `~/.local/bin`, invoking `org.chromium.Chromium` via Flatpak. Wrappers are attempted even if Flatpak is unavailable. diff --git a/docs/src/roles/fleet_facts.md b/docs/src/roles/fleet_facts.md index ccb63597..5f1f0fef 100644 --- a/docs/src/roles/fleet_facts.md +++ b/docs/src/roles/fleet_facts.md @@ -3,10 +3,9 @@ Publishes each machine's **public** device facts for the live fleet view at [reilly.asia/infra](https://reilly.asia/infra/#fleet). -- **Runs on:** every host (opt out with `fleet_facts_publish: false`), last in - the play, including the no-secrets daily timer -- **Tags:** `facts`, `fleet_facts` -- **Needs:** a logged-in `gh` CLI (skipped quietly otherwise) +- **Implementation:** `units/fleet-facts.sh` and `units/fleet-facts.py` +- **Runs on:** registered inventory hosts unless `fleet_facts_publish: false`, including no-secrets applies +- **Needs:** Python, YAML support via PyYAML or `yq`, and an authenticated `gh` CLI. Missing authentication reports `PARTIAL`. Each run writes `facts/.json` to the **`fleet-facts`** branch of this repo through the GitHub contents API. There's no clone and no merge, and @@ -14,19 +13,19 @@ repo through the GitHub contents API. There's no clone and no merge, and | Field | Source | |---|---| -| `os.{name,version,codename,kernel}` | Ansible distribution facts | +| `os.{name,version,codename,kernel}` | `/etc/os-release` or Android `getprop`, and the running kernel | | `hardware.model` | Android `getprop`, else device-tree model (Pi), else DMI product name (placeholders like "System Product Name" dropped) | -| `hardware.{vendor,form_factor,arch,cpu,cpu_threads,memory_gb}` | Ansible hardware facts | +| `hardware.{vendor,form_factor,arch,cpu,cpu_threads,memory_gb}` | DMI, platform information, `/proc/cpuinfo` and `/proc/meminfo` | | `hardware.gpus` | `lspci -mm` display devices (unnamed virtual adapters dropped) | -| `virtualization`, `uptime_days`, `last_apply` | Ansible facts, `~/.cache/dotfiles/last-apply.json` | +| `virtualization`, `uptime_days`, `last_apply` | `systemd-detect-virt`, `/proc/uptime`, `~/.cache/dotfiles/last-apply.json` | **Never published:** IP/MAC addresses, serial numbers, usernames, anything from Bitwarden. ## Private drift monitoring (`osquery`) -The separate `osquery` role watches changes that must **not** go to the public -`fleet-facts` branch. It installs checksum-pinned upstream osquery 5.23.1 under +The separate `units/osquery.sh` unit watches changes that must **not** go to the public +`fleet-facts` branch. Its assets and defaults live under `units/osquery/`. It installs checksum-pinned upstream osquery 5.23.1 under `/opt`, including on immutable desktops: no package layering, writable `/usr`, reboot, Fleet server, enrollment token, or new network listener. @@ -34,8 +33,8 @@ reboot, Fleet server, enrollment token, or new network listener. and x86_64 or aarch64. Nura/Alpine musl (Kerala) and unrooted Android/Termux are unsupported; `osquery_supported: false` records that exclusion. Offline hosts install on their next successful normal apply. `skip_osquery: true` skips the -role; it does not uninstall an already-running daemon. The retired VPS group is -never modified, and Talos nodes are not Ansible-managed. +unit; it does not uninstall an already-running daemon. Inventory VPSes and the +retired VPSes are excluded; Talos nodes are not managed by these units. ### What's watched @@ -67,7 +66,7 @@ just osquery-fleet --host punjab --host dilli just osquery-fleet --json ``` -The controller uses effective Ansible inventory, Tailscale online status and +The controller uses native inventory parsing, Tailscale online status and existing authenticated SSH plus noninteractive sudo. Host-key verification stays enabled. It shows `ok`, `drift`, `offline`, `unsupported`, `excluded`, `stale` or `error`; missing/bad reports and failed reads never become `ok`. Reports older @@ -83,10 +82,10 @@ hosts' report-refresh window, and two minutes after boot. It runs as `ubuntu`, using that account's existing SSH keys, known hosts and noninteractive sudo. It does not depend on an interactive SSH agent or copy credentials into Kubernetes. -The role installs a root-managed collector and a minimal inventory under +The unit installs a root-managed collector and a minimal inventory under `/opt/fleet-osquery` and `/etc/fleet-osquery/inventory.yml`. Only connection and exclusion metadata is copied—not the full effective inventory or Bitwarden facts. -Apply the `osquery` role on Punjab after changing fleet membership or connection +Apply the `osquery` unit on Punjab after changing fleet membership or connection details to refresh this inventory snapshot. Each poll atomically replaces **one** aggregate: @@ -107,7 +106,7 @@ process itself cannot run or write, check the service journal and the snapshot's collection time—an old file is not current health. `osquery_poller_enabled` defaults to false on other hosts. Setting it false on a -previous poller and applying the role stops its timer and in-flight service; +previous poller and applying the unit stops its timer and in-flight service; the last private snapshot remains for inspection. The manual fleet CLI still works from other machines and supports `--output PATH` for explicit private snapshots. @@ -135,13 +134,3 @@ above. **Keep all report output private:** addresses and executable paths are sensitive; do not attach raw JSON to public issues or publish it through `fleet_facts`. No command lines, process environments, credentials or serial numbers are queried. -Check mode reports platform support only; actual installation and baseline -collection require a normal apply. CI exercises the actual role and -`sudo python3 tests/osquery_smoke.py` on x86_64 and aarch64 Ubuntu runners. -That smoke creates a temporary loopback listener and an enabled, never-started -unit, checks pending/accepted/removed drift, and compares a synthetic prior kernel -against the real current kernel. It uses isolated state and removes its fixtures; -the fleet baseline is untouched. -The native CI jobs also enable the persistent poller and run -`tests/osquery_poller_smoke.py`: a real temporary listener must appear in the -aggregate, disappear after removal, and leave the host baseline unchanged. diff --git a/docs/src/roles/gnome.md b/docs/src/roles/gnome.md index fc4d6698..b4e991f0 100644 --- a/docs/src/roles/gnome.md +++ b/docs/src/roles/gnome.md @@ -1,52 +1,12 @@ # gnome +**Unit:** `units/gnome.sh` **Tags:** `desktop`, `gnome` **Secrets needed:** No -**Runs on:** Desktop group only +**Runs on:** desktops, except `skip_gnome: true` or `machine_profile: vm-test`. -Configures [GNOME](https://www.gnome.org/) desktop environment: keybindings, terminal, avatar, extensions, and remote desktop. +The unit installs `units/gnome/gnome-keybindings.ini` in `~/.config/dconf` and loads it into the invoking user's live session. A pending marker retains a changed load request if the session is unavailable. It configures the default Ptyxis profile for Homebrew fish and `JetBrainsMono Nerd Font 12` on a best-effort basis, and downloads the account avatar for installation by `units/gnome-avatar.py`; the avatar step requires root or non-interactive sudo. -## What It Does +The unit copies the bundled Copyous, PaperShell and PaperWM extensions from `units/gnome/extensions/` to the user's extension directory, without deleting unrelated files. It best-effort loads `units/gnome/gnome-extensions-dconf.dump` and sets `gnome_enabled_extensions` from host variables or `group_vars/all.yml`. Setting the enabled list does not install every listed extension or guarantee compatibility or activation in the current GNOME Shell. The separate Bluefin unit also attempts extension installation. -### Keybindings - -Deploys custom keyboard shortcuts via dconf: -- Custom keybindings for terminal, browser, file manager -- Workspace navigation and window management shortcuts - -### Ptyxis Terminal - -Sets the default Ptyxis profile to use `fish` from Homebrew with JetBrains Mono Nerd Font at 12pt. - -### User Avatar - -- Downloads avatar from `https://reilly.asia/profile.png` -- Installs to both AccountsService and GDM cache -- Shows on login screen and user menu - -### GNOME Extensions - -Deploys and enables extensions from `group_vars/all.yml`: - -| Extension | Purpose | -|-----------|---------| -| Caffeine | Disable screen blanking | -| AppIndicator | System tray icons | -| Blur My Shell | Blur effects | -| Dash to Dock | macOS-style dock | -| [GSConnect](https://github.com/GSConnect/gnome-shell-extension-gsconnect) | Phone integration | -| PaperWM | Tiling window manager | -| PaperShell | PaperWM companion | -| Search Light | Spotlight-style search | - -### Remote Desktop - -Enables GNOME Remote Desktop (RDP) via `grdctl` and starts the systemd user service. - -## Extension Sync - -Extensions are synced omnidirectionally — the `gnome_enabled_extensions` list in `group_vars/all.yml` is the canonical list. To update it from a running desktop: - -```bash -gsettings get org.gnome.shell enabled-extensions -``` +RDP enablement and the GNOME Remote Desktop user service are best effort. Session operations require a reachable existing user D-Bus session; the unit avoids creating an ephemeral session over SSH. Enabling RDP does not establish credentials or verify a remote login. diff --git a/docs/src/roles/hive_ops.md b/docs/src/roles/hive_ops.md index b1e6f3bd..0f712f06 100644 --- a/docs/src/roles/hive_ops.md +++ b/docs/src/roles/hive_ops.md @@ -1,67 +1,7 @@ # hive_ops -**Tags:** `hive`, `hive_ops` -**Secrets needed:** No (uses the AWS cluster kubeconfig from the `kube` role) -**Runs on:** Opt-in — exactly one host (`hive_ops_enabled: true`) +`units/hive-ops.sh` manages optional user timers for the [Hive](https://hub.tunaos.org) on the [AWS Talos cluster](../servers/aws-k8s/cluster.md). It requires `yq` and a working user systemd manager, and skips Termux, inventory VPSes, and retired `matrix`/`telengana`. It does not read the vault; the configured kubeconfig is sensitive and must already exist for the helpers to access the cluster. -Drives the [tuna-os Hive](https://school.tunaos.org) (all Hives: [hub.tunaos.org](https://hub.tunaos.org)) running on the -[AWS Talos cluster](../servers/aws-k8s/cluster.md): backend rotation, a -watchdog, capability tiers, and peak/off-peak scheduling. +With `hive_ops_enabled: true`, it installs three helpers from `units/hive-ops/bin/` and service/timer pairs from `units/hive-ops/systemd/`. The default list in `units/hive-ops/defaults.yml` includes rotation, watchdog, tiers, Hive peak pause/resume, and pi peak start/stop. A host may override `hive_ops_timers`, `hive_ops_kubeconfig` (default `~/.kube/config-aws-migration`) and `hive_ops_path`. A drop-in pins KUBECONFIG and PATH for each service. Only enable one controller so timers do not compete. -These previously lived as hand-managed user units on `telengana`, talking to a -local k3s via `sudo -n k3s kubectl`. When telengana was retired they were moved -into this role and retargeted at plain `kubectl` against the AWS cluster. - -## What It Does - -Deploys three scripts to `~/.local/bin/` and seven user timer/service pairs: - -| Timer | Schedule | Does | -|---|---|---| -| `hive-rotate` | every 5 min | Capability-preserving backend rotation — fails agents over between providers on headroom | -| `hive-watchdog` | every 5 min | Health check / unwedge | -| `hive-tiers` | weekly | Refresh capability tiers | -| `hive-peak-pause` / `hive-peak-resume` | 01:00 / 04:00 | Pause agents during the provider's peak-price window | -| `pi-peak-start` / `pi-peak-stop` | 04:00 / 01:00 | Start/stop the local `pi.service` around DeepSeek off-peak pricing | - -Peak windows default to `01:00-04:00,06:00-10:00` for `deepseek` and are -overridable via `HIVE_PEAK_WINDOWS` / `HIVE_PEAK_PROVIDERS`. - -## Why the drop-in exists - -The scripts reach the hive through `kubectl exec` into the pod. Two things -break that under systemd that don't break it interactively: - -1. **`KUBECONFIG` is unset** — there is no shell rc file to source. -2. **The systemd *user* manager's PATH is `/usr/local/bin:/usr/bin:/snap/bin`**, - which excludes Homebrew — where `kubectl` and `jq` actually live. - -Both fail *opaquely*: the scripts capture `kubectl` output with `2>/dev/null`, -so a missing binary surfaces as `ERROR: no hive pod found` rather than -"command not found". The role writes a `.service.d/kubeconfig.conf` drop-in -pinning `KUBECONFIG` and `PATH` for every unit. - -This is why it worked on telengana without a drop-in: `k3s` sits on the -standard PATH, Homebrew does not. - -## Variables - -| Variable | Default | Purpose | -|---|---|---| -| `hive_ops_enabled` | `false` | Opt a host in. Two hosts both rotating would fight each other. | -| `hive_ops_kubeconfig` | `~/.kube/config-aws-migration` | Cluster to drive | -| `hive_ops_path` | `/home/linuxbrew/.linuxbrew/bin:…` | PATH for the units | -| `hive_ops_timers` | 7 timers | Which units to deploy/enable | - -## Notes - -- Prefer `hive-peak.sh status|pause|resume` over hand-rolled curl — the - authentication path is subtle (see the `tunaos-hive-checkin` skill). -- `hive-rotate.sh apply` takes **~2 minutes** — it makes many `kubectl exec` - round-trips. Don't mistake a short timeout for a failure. -- **Enabled on no host (2026-09-24).** Rotation, watchdog and peak windows run - as `CronJob`s inside the AWS cluster (ns `hive`), which is where they belong — - a desktop that sleeps silently skips them. It previously ran on himachal. - Setting `hive_ops_enabled: false` now also **removes** the timers, units, - drop-ins and scripts from a host on its next apply, so a retired host can't - keep fighting the in-cluster jobs. +When disabled, cleanup is attempted only if `hive-watchdog.timer` exists: listed timers are disabled, their unit files and kubeconfig drop-ins removed, and the three helpers removed if cleanup succeeded. Retired hosts and VPSes are excluded even from cleanup. Existing kubeconfigs and cluster data are not removed. diff --git a/docs/src/roles/homebrew.md b/docs/src/roles/homebrew.md index 993c8465..0d3eabdb 100644 --- a/docs/src/roles/homebrew.md +++ b/docs/src/roles/homebrew.md @@ -1,35 +1,12 @@ # homebrew +**Unit:** `units/homebrew.sh` **Tags:** `packages`, `homebrew` **Secrets needed:** No -**Runs on:** All glibc machines (skipped on musl/Alpine) +**Runs on:** glibc hosts; skips Termux and musl. -Installs and manages [Homebrew](https://brew.sh/) packages on Linux. +The native unit installs and manages Homebrew. It defaults to `/home/linuxbrew/.linuxbrew` (`BREW_PREFIX` overrides the prefix), bootstraps Mike Farah `yq`, and reads package lists from `group_vars/all.yml` and host configuration through `units/package-config.bash`. -## What It Does +It generates `~/Brewfile` with core packages and, on desktops, desktop taps, packages, extras and casks; VS Code extensions are included on Darwin. It taps repositories, attempts to trust them, runs `brew bundle install --no-upgrade`, and uninstalls `brew_blocked` formulae. A blocked package in an install list is a configuration error. `DOTFILES_UPGRADE=1` or `true` additionally requests a best-effort `brew upgrade`; installing Python `packaging` is also best effort. -1. Installs Homebrew to `/home/linuxbrew/.linuxbrew` if missing -2. Generates a `Brewfile` from `group_vars/all.yml` package lists -3. Runs `brew bundle` to install/update all packages -4. Ensures Python `packaging` module is available (required by Ansible's systemd module) - -## Package Categories - -| Variable | Where Defined | Scope | -|----------|--------------|-------| -| `core_brews` | `group_vars/all.yml` | All machines | -| `core_tap_brews` | `group_vars/all.yml` | All machines (tapped) | -| `desktop_brews` | `group_vars/all.yml` | Desktop group only | - -## Key Packages - -- **Shells:** fish, zsh, bash-preexec -- **CLI tools:** neovim, tmux, direnv, just, make, ripgrep, fzf, eza, bat, fd, zoxide -- **Bluefin extras:** atuin, starship, carapace, tealdeer (via `bluefin-cli`) -- **Dev tools:** go, kubernetes-cli, helm, k9s, podman-compose, ansible, uv -- **AI/ML:** ollama, ramalama, openai-whisper, pi-coding-agent - -## Notes - -- Homebrew on Linux installs to `/home/linuxbrew/.linuxbrew` — no root needed -- The playbook adds `brew_bin` to `PATH` for all subsequent tasks +Ownership repair and installing a missing Debian/Ubuntu compiler toolchain may require root or non-interactive sudo. Other systems must supply a compiler if missing. The Homebrew installer itself is downloaded and run non-interactively. diff --git a/docs/src/roles/kirocrew.md b/docs/src/roles/kirocrew.md index 5ae10b35..5fc1b4fe 100644 --- a/docs/src/roles/kirocrew.md +++ b/docs/src/roles/kirocrew.md @@ -1,16 +1,5 @@ # kirocrew -Keeps the [Kiro Crew](https://github.com/kirodotdev/KiroCrew) gateway -(dashboard + Slack + cron) healthy on hosts that run it — currently punjab. +`units/kirocrew.sh` manages an existing [Kiro Crew](https://github.com/kirodotdev/KiroCrew) installation only when `kirocrew_enabled: true` is set in host variables. It skips retired `matrix` and `telengana`. If `~/.local/bin/kirocrew` or `/etc/systemd/system/kirocrew.service` is missing, it skips and prints upstream installation guidance; it does not install either. -- **Enable**: `kirocrew_enabled: true` in `host_vars/.yml` -- **Tags**: `services`, `kirocrew` - -The venv (`~/.kiro/crew-venv`) and the systemd unit are owned by Kiro's own -installer (`kirocrew service install`), so the role does **not** template them. -It: - -1. warns if the binary or unit is missing (install per upstream first), -2. keeps `/etc/kirocrew/kirocrew.env` `root:root 0600` — tokens go there, -3. enables and starts `kirocrew.service`, -4. fails the play if the dashboard listens on anything but localhost. +With root or noninteractive `sudo -n`, it makes `/etc/kirocrew/kirocrew.env` root-owned mode 0600, then enables and starts the service only if those permissions succeeded. It checks listeners using `ss` and `units/kirocrew-sockets.py`, failing the unit if the check fails. The existing environment file contains credentials; this unit neither fetches nor rewrites them. diff --git a/docs/src/roles/pipewire_audio.md b/docs/src/roles/pipewire_audio.md index cf16caa8..4b07b3a3 100644 --- a/docs/src/roles/pipewire_audio.md +++ b/docs/src/roles/pipewire_audio.md @@ -1,89 +1,14 @@ # pipewire_audio +**Unit:** `units/pipewire-audio.sh` **Tags:** `desktop`, `audio`, `pipewire` **Secrets needed:** No -**Runs on:** Desktop group only (unless `skip_pipewire_audio: true`) +**Runs on:** desktops, except `skip_pipewire_audio: true` or `machine_profile: vm-test`. -Replaces Easy Effects with native PipeWire `filter-chain` modules using LSP LV2 and LADSPA plugins — no Flatpak, no GUI, no D-Bus service. +The unit deploys native PipeWire filter chains from `units/pipewire-audio/`. It detects the LSP LV2 plugin and chooses the LSP speaker chain or built-in biquad fallback. It installs the RNNoise microphone chain and user `mic-dsp-link.service` when the RNNoise LADSPA plugin is found. `mic_hardware_volume_pct` comes from the environment, host/group variables or unit defaults and must be 0–100. -## What It Does +DSP prerequisites are installed with supported dnf, rpm-ostree or Arch package managers; these operations require root or non-interactive sudo. Other platforms report `PARTIAL`; bootc images must supply the prerequisites themselves. If plugin discovery is inconclusive, existing microphone configuration or speaker configuration is retained rather than assumed absent. -1. Deploys `speaker-dsp.conf` — a PipeWire filter-chain that creates a virtual "Speakers (DSP)" sink -2. Deploys `mic-dsp.conf` — a PipeWire filter-chain that creates a virtual "Microphone (DSP)" source -3. Configures WirePlumber to prefer the DSP nodes as default devices -4. Cleans up any previous Easy Effects installation (Flatpak, autostart, presets) +Only absent or recognized managed audio files are replaced or removed; custom configs and services are preserved and reported. The old managed WirePlumber rule is removed without deleting other rules. Restart requests are persisted under `~/.local/state/pipewire-audio` until the user systemd manager successfully reloads/restarts the affected services. A live user systemd session is required. GNOME over-amplification is best effort. -## Output Chain: Speaker DSP - -Signal path: **16-band Parametric EQ → Bass Enhancer → Exciter → Limiter** (all LSP LV2). - -| Stage | LV2 Plugin | What It Does | -|-------|-----------|--------------| -| `eq` | LSP ParaEQ x16 LR | Corrective loudness curve with subsonic HPF @ 120 Hz | -| `bass` | LSP Bass Enhancer | Harmonic bass synthesis for small speakers | -| `exciter` | LSP Exciter | Crystalizer substitute — adds presence and air | -| `lim` | LSP Limiter | Brickwall peak limiter @ -0.5 dBFS | - -The EQ curve is a direct translation of the Easy Effects 15-band preset — ~2.5 dB cut through low-mids (220-600 Hz), ~3.5 dB peak at 3.2 kHz, tapering to ~1.5 dB boost above 8 kHz. - -**Simplification:** To run just EQ + Limiter (the most critical, crash-proof chain), comment out the `bass` and `exciter` nodes in `speaker-dsp.conf` and remove their links. - -## Input Chain: Microphone DSP - -Signal path: **RNNoise → Compressor** (LADSPA + LSP LV2). - -| Stage | Plugin | What It Does | -|-------|--------|--------------| -| `rnnoise` | librnnoise\_ladspa.so | Real-time neural network noise suppression (VAD-enabled) | -| `comp` | LSP Compressor Stereo | Downward compression, -24 dB threshold, 2:1 ratio | - -## Bootc Image Requirements - -These packages must be in the bootc image (`Containerfile`): - -``` -lsp-plugins-lv2 -noise-suppression-for-voice -``` - -The filter-chain configs reference: -- LSP LV2 bundles under `/usr/lib64/lv2/` (Fedora default LV2 path) -- RNNoise LADSPA at `/usr/lib64/ladspa/librnnoise_ladspa.so` - -## Verifying - -On a target machine after deploy: - -```bash -# Check DSP nodes appeared -pw-cli ls Node | grep -E 'speaker_dsp|mic_dsp' - -# Check defaults -wpctl status | head -20 - -# Watch for plugin loading errors -journalctl --user -u pipewire -f | grep -iE 'error|filter-chain|lv2' - -# Verify LSP plugins are discoverable -lv2ls | grep lsp-plug -lv2info http://lsp-plug.in/plugins/lv2/para_equalizer_x16_lr | head -5 - -# Verify LADSPA plugin -ls /usr/lib64/ladspa/librnnoise_ladspa.so -``` - -## Troubleshooting - -| Symptom | Likely Cause | Fix | -|---------|-------------|-----| -| DSP node doesn't appear | LSP/LADSPA plugins missing from bootc image | Verify with `lv2ls`, `ls /usr/lib64/ladspa/` | -| Port name mismatch | LSP plugin version changed port symbols | Run `lv2info \| grep 'port.*symbol'` and update config | -| Audio routing loop | PipeWire/WirePlumber bug | Don't set the DSP sink as default manually; let WirePlumber handle it | -| No bass enhancer/exciter effect | Plugin UID mismatch | Comment them out in config for a leaner chain | - -## Notes - -- PipeWire must be restarted after deploying configs (handled by Ansible handlers) -- Select "Speakers (DSP)" and "Microphone (DSP)" in GNOME Sound settings if WirePlumber doesn't auto-select them -- The virtual sink approach works because PipeWire prevents self-looping when capture/playback share the same `node.name` -- To skip: set `skip_pipewire_audio: true` in `host_vars`, or exclude the `pipewire` tag +The unit removes the Easy Effects autostart entry and, if present, stops and uninstalls the system Flatpak; user presets and settings are preserved. Installation does not establish that DSP or microphone routing works on a particular device; inspect `wpctl status`, `pw-cli`, and the user PipeWire journal on the target. diff --git a/docs/src/roles/proxy.md b/docs/src/roles/proxy.md index ed22499b..8bb531a5 100644 --- a/docs/src/roles/proxy.md +++ b/docs/src/roles/proxy.md @@ -1,27 +1,7 @@ # proxy (Caddy) -**Tags:** `services`, `proxy`, `caddy` -**Secrets needed:** Yes (TLS certificates) -**Runs on:** All except VPS (`skip_proxy: false`) +`units/proxy.sh` deploys the rootful Caddy quadlet from `units/proxy/caddy.container` and renders `units/proxy/Caddyfile.j2` using `units/service-templates.py` and the host's `web_services` list. Caddy is configured on port 9000; the unit points the default Tailscale Serve HTTPS handler at `127.0.0.1:9000`. Routes include `/service/` redirects and `/bst/`; there is no homepage at `/`. -Deploys [Caddy](https://caddyserver.com/) as a reverse proxy with automatic TLS certificates. +The unit requires `yq`, Python, root or noninteractive `sudo -n`, systemd, Podman and Tailscale. It skips inventory VPSes, retired `matrix`/`telengana`, `skip_proxy: true`, and `machine_profile: vm-test`. It does not fetch vault secrets. The Tailscale Serve configuration can replace an existing default HTTPS handler. -## What It Does - -1. Creates Caddy config directory -2. Deploys `Caddyfile` from template with per-host service routes -3. Proxies internal services to `*.manatee-basking.ts.net` subdomains -4. Uses [Tailscale](https://tailscale.com/) certificates for TLS - -## Proxied Services - -The Caddyfile (listening on `:9000`, fronted by `tailscale serve`) carries: -- `/service/` — a redirect for each entry in the host's `web_services` -- `/bst/` — the BuildStream dashboard (on build machines) - -There is no homepage at `/`. - -## Notes - -- Skip with `skip_proxy: true` in `host_vars` -- Caddyfile is templated per-host — each machine proxies its own services +It writes `/etc/caddy/Caddyfile` and `/etc/containers/systemd/caddy.container` as root-owned 0644 files. Activation is skipped if either deployment fails; otherwise it reloads systemd, restarts Caddy after changes, and enables and starts the service before changing Serve. diff --git a/docs/src/roles/retired_services.md b/docs/src/roles/retired_services.md index d9f1adcb..9cacaf9a 100644 --- a/docs/src/roles/retired_services.md +++ b/docs/src/roles/retired_services.md @@ -1,61 +1,7 @@ # retired_services -**Tags:** `system`, `retired_services` -**Secrets needed:** No -**Runs on:** every systemd host except the `vps` group and Termux (`site.yml` gates on `vps` membership, `is_termux` and `service_mgr`) +`units/retired-services.sh` removes only the explicitly listed obsolete system and user unit files in `units/retired-services/defaults/main.yml`. It skips Termux, non-Linux or non-systemd hosts, all inventory VPSes, and retired `matrix`/`telengana`. It requires `yq`, Python and systemctl; system-scope cleanup additionally needs root or noninteractive `sudo -n`. User cleanup can still proceed when root access is unavailable. -Deleting a role from the repo stops the fleet from *deploying* its services, but -nothing ever removed what it had already installed — so retired podman quadlets -(Prometheus, Grafana, Cockpit, …) kept running on laptops long after their roles -were gone ([#170](https://github.com/hanthor/dotfiles/issues/170)). This role -removes those leftover unit files. +For each listed file that exists, it stops the generated quadlet service (`.container`, `.volume`, `.network`, or `.pod`) or disables and stops a plain unit, then removes the file. A missing generated unit is acceptable; other stop errors preserve the file. It reloads each systemd scope after changes. User paths are relative to `~/.config/containers/systemd/`; system paths are absolute. `caddy.container` is not on the list. -## What It Does - -For every unit file in its lists that exists on the machine: - -1. stops the service generated from it — quadlet service names are the file stem - (`foo.container` → `foo.service`, `foo.volume` → `foo-volume.service`, - `foo.network` → `foo-network.service`); plain `.service`/`.timer` files are - also disabled -2. deletes the unit file -3. runs one `daemon-reload` (system and/or user, whichever scope had leftovers) - -On a machine with none of the files, everything after one `find` over the unit -directories is skipped — the role reports no changes. - -**Only unit files are removed.** Podman volumes (the data of Prometheus, -Grafana, NetBox's Postgres, …), images, and config directories such as -`/etc/cockpit`, `/etc/homepage`, `/etc/netbox` and `~/.config/monitoring` are -left alone; delete them by hand once you're sure nothing is wanted -(`podman volume ls`, `sudo podman volume ls`). - -## Units - -| Role | Removed in | Scope | Unit files | -|---|---|---|---| -| `cockpit` | `2af694a` | system (`/etc/containers/systemd/`) | `cockpit.container` | -| `homepage` | `f0143a6` | system | `homepage.container` | -| `netbox` | `7f3c122` | system | `netbox.container`, `netbox-worker.container`, `netbox-redis.container`, `netbox-postgres.container`, `netbox-postgres.volume`, `netbox-media.volume`, `netbox.network` | -| `netbox_agent` | `7f3c122` | system (`/etc/systemd/system/`) | `netbox-agent.timer`, `netbox-agent.service` | -| `lima` | `f0143a6` | user (`~/.config/containers/systemd/`) | `lima-bootc.container` | -| `monitoring` | `8c9b171` | user | `node-exporter`, `podman-exporter`, `alloy`, `promtail`, `prometheus` (also the hub variant), `perses`, `grafana`, `loki`, `alertmanager`, `pve-exporter`, `matrix-notifier` `.container`; `alloy-data`, `prometheus-data`, `perses-data`, `grafana-data`, `loki-data`, `alertmanager-data` `.volume`; `monitoring.network` | -| `appflowy` | `f7f2a2a` | user | `appflowy-cloud`, `appflowy-gotrue`, `appflowy-s3`, `appflowy-minio`, `appflowy-redis`, `appflowy-db` `.container`; `appflowy-s3-data`, `appflowy-minio-data`, `appflowy-db-data` `.volume`; `appflowy.network` | -| `authentik` | `f7f2a2a` | user | `authentik-server`, `authentik-worker`, `authentik-redis`, `authentik-db` `.container`; `authentik-db-data`, `authentik-media`, `authentik-certs`, `authentik-custom-templates` `.volume`; `authentik.network` | -| `n8n` | `f7f2a2a` | user | `n8n-server`, `n8n-db` `.container`; `n8n-data`, `n8n-db-data` `.volume`; `n8n.network` | - -`caddy.container` is not on the list — the [`proxy`](proxy.md) role still deploys it. - -## Variables - -| Variable | Purpose | -|---|---| -| `retired_services_system_units` | Absolute paths of rootful quadlets / system units to remove | -| `retired_services_user_units` | File names under `~/.config/containers/systemd/` to remove | - -## Notes - -- When you delete a role that installed units, add them here in the same PR. -- User-scope `systemctl` gets `XDG_RUNTIME_DIR=/run/user/` pinned so it - works from shells that don't export it. -- No `skip_` flag — the role is a no-op wherever the files are absent. +Only unit files are removed: Podman volumes, images, configuration directories, and other user data remain untouched. Add a retired service to the canonical list when its deployed unit needs cleanup. diff --git a/docs/src/roles/server_hardening.md b/docs/src/roles/server_hardening.md index 3e369c4a..579cb717 100644 --- a/docs/src/roles/server_hardening.md +++ b/docs/src/roles/server_hardening.md @@ -1,23 +1,5 @@ # server_hardening -**Tags:** `system`, `hardening`, `security` -**Secrets needed:** No -**Runs on:** `server` and `vps` groups (`site.yml` gates on group membership) +`units/server-hardening.sh` runs for inventory `server` or `vps` members, but explicitly skips retired `matrix` and `telengana`. It requires `yq`, Python, and root or noninteractive `sudo -n`; it uses no vault secrets. `units/server-hardening-file.py` performs the managed line edits. -Applies reliability and light security hardening for long-running server hosts. - -## What It Does - -- **journald:** sets `SystemMaxUse=500M` and vacuums journals to that size -- **swap:** creates a 2G `/swapfile` if missing, sets 0600, runs `mkswap` + `swapon` only when it isn't already active, adds it to `/etc/fstab` -- **NTP:** enables and starts `systemd-timesyncd` -- **SSH rate limits:** `MaxStartups 3:50:10`, `MaxSessions 10`, `ClientAliveCountMax 3` in `sshd_config` (reloads sshd) -- **UFW:** turns logging on if UFW is already active (doesn't enable UFW or add rules) -- **Livepatch:** runs `pro enable livepatch` if Ubuntu Pro is attached (best-effort) -- **Reboot check:** prints a warning if `/var/run/reboot-required` exists - -## Notes - -- Does **not** install fail2ban, change SSH auth/cipher settings, set sysctls, or configure automatic updates -- No `skip_` flag — to keep a host out, take it out of `server`/`vps` -- The retired VPSes (`matrix`, `telengana`) are still in `vps`; don't apply to them (see the cutover runbook) — this role reloads sshd and restarts journald +The unit sets journald's maximum size to 500 MiB, restarts journald on change and vacuums old journals. It creates a 2 GiB `/swapfile` if absent, sets root ownership and mode 0600, formats and enables it if not active, and adds an fstab entry. It enables systemd-timesyncd only when present and chrony is not active. It sets SSH `MaxStartups`, `MaxSessions`, and `ClientAliveCountMax`, reloading SSH best-effort on change. It enables UFW logging only when UFW is already active, attempts Ubuntu Pro livepatch when its status indicates it is needed, and reports a pending reboot. It does not configure firewall rules or install fail2ban. diff --git a/docs/src/roles/shell_fonts.md b/docs/src/roles/shell_fonts.md index ad9fe917..5cded07b 100644 --- a/docs/src/roles/shell_fonts.md +++ b/docs/src/roles/shell_fonts.md @@ -16,4 +16,4 @@ Installs the [JetBrains Mono Nerd Font](https://github.com/ryanoasis/nerd-fonts) - Idempotent on the presence of the Regular `.ttf`; delete it to force a re-download. - Desktop means: `DOTFILES_IS_DESKTOP`, else `is_desktop` in `host_vars`, else membership of the `desktop` group in `inventory.yml`. -- The `gnome` role sets Ptyxis to `JetBrainsMono Nerd Font 12`, which depends on this. +- The `gnome` unit sets Ptyxis to `JetBrainsMono Nerd Font 12`, which depends on this. diff --git a/docs/src/roles/ssh_keys.md b/docs/src/roles/ssh_keys.md index 8b34649a..add3183d 100644 --- a/docs/src/roles/ssh_keys.md +++ b/docs/src/roles/ssh_keys.md @@ -1,55 +1,9 @@ # ssh_keys -**Tags:** `secrets`, `ssh`, `ssh_keys` -**Secrets needed:** Yes -**Runs on:** All machines +`units/ssh-keys.sh` invokes the locked shared implementation `units/ssh-fleet.py`. It manages the local ed25519 identity, fleet SSH config, known hosts, Git allowed signers, and Bitwarden-backed authorization. Fleet names and login data come from `group_vars/all.yml`, `inventory.yml`, and host variables; YAML parsing needs `yq` or Python PyYAML. -Manages per-machine ed25519 SSH keys, distributes the fleet's public keys via `authorized_keys`, and maintains the `allowed_signers` list for git commit verification. +If the private key is absent, the unit requires an unlocked vault and a successful item listing before restoring the complete `@` SSH item or generating a new pair when no item exists. It checks restored and local key pairs and refuses conflicting keys. An existing local key is retained; if its public key differs from the vault item, the vault is left unchanged unless `SSH_KEYS_FORCE_PUSH=true` or `ssh_keys_force_push: true` explicitly permits rotation. -**Source of truth is the local disk.** Bitwarden is the durable mirror that lets new/rebuilt machines recover and lets the rest of the fleet stay in sync. +The unit preserves user entries outside its marked SSH config and `authorized_keys` blocks. Vault public keys and `extra_authorized_keys` are merged into the authorization block; previous managed entries remain by default. `SSH_KEYS_FORCE_AUTHORIZED=true` (or the corresponding host variable) permits removing obsolete entries only within that block. A locked vault, disabled secrets, or an empty vault key list leaves vault authorization unchanged. Host-key scans only replace an entry after a usable scan; failed scans leave existing entries intact. Existing signers are retained. -## What it does - -For the local machine's key (`james@` in Bitwarden): - -1. **No key on disk** → fetch from BW; if absent there too, `ssh-keygen` a fresh ed25519 pair. -2. **Local pub key matches BW** → no-op. -3. **Local pub key differs from BW** → leave the Bitwarden item unchanged and warn by default. An intentional rotation must opt in with `dots-apply -e ssh_keys_force_push=true`; only then is the BW item replaced so every other host sees the current key on its next apply. This gate prevents a key generated while the vault was locked from clobbering the stored key. - -Then, regardless of whether the local item changed: - -- Slurps every other `james@` SSH item from BW in a single `bw list` call. -- Rewrites `~/.ssh/authorized_keys` with the union of those + `extra_authorized_keys`. -- Refreshes `~/.ssh/known_hosts` for every fleet host via `ssh-keyscan`, removing stale entries first. -- Updates `~/.ssh/allowed_signers` with this host's pub key for git commit signing. - -If `bw_unlocked` is false (vault locked, BW not installed, etc.), every BW-touching step is skipped and `authorized_keys` is **not** rewritten — preserving the existing file rather than wiping it with an empty list. - -## Failure modes - -| Symptom | Likely cause | Fix | -|---|---|---| -| `Permission denied (publickey)` from a host that worked yesterday | Local key changed but BW still has the old pub key | For an intentional rotation, run `dots-apply -e ssh_keys_force_push=true` on the source host with BW unlocked, then run `dots-apply` on the failing host. If the mismatch was accidental, restore the Bitwarden key locally instead of forcing it. | -| Some hosts missing from `authorized_keys` | Those hosts never round-tripped their key to BW | Run `dots-apply` on each missing host once, with BW unlocked. | -| New ed25519 key generated on a host that should have inherited an existing key | BW item missing/empty for this hostname | Verify the BW entry name matches `james@` exactly. | - -## How to verify - -```bash -# Check this host's key is in BW -bw get item "james@$(hostname)" | jq '.sshKey.publicKey' | grep -F "$(awk '{print $2}' ~/.ssh/id_ed25519.pub)" - -# Check who can SSH in here -cat ~/.ssh/authorized_keys - -# Test SSH to every fleet host -for h in $(yq '.all.hosts | keys | join(" ")' inventory.yml); do - ssh -o BatchMode=yes -o ConnectTimeout=3 "$h" "echo OK" 2>&1 | head -1 -done -``` - -## Notes - -- Every machine SSHes to every other machine without password prompts after one round-trip apply. -- The same key is used for GitHub auth + commit signing (registered by the `github` role). -- The role tolerates a locked vault and never destructively rewrites `authorized_keys` when BW returned no data. +`--no-secrets` skips vault access; a missing identity is deferred rather than generated. Config and other independent steps still run. The separate [ssh_mesh](ssh_mesh.md) unit adds committed fleet public keys and checks connectivity. diff --git a/docs/src/roles/ssh_mesh.md b/docs/src/roles/ssh_mesh.md index 7347e50a..eec3329e 100644 --- a/docs/src/roles/ssh_mesh.md +++ b/docs/src/roles/ssh_mesh.md @@ -1,53 +1,7 @@ # ssh_mesh -Self-healing passwordless SSH mesh across the fleet. +`units/ssh-mesh.sh` invokes `units/ssh-fleet.py` under the same SSH lock as [ssh_keys](ssh_keys.md). It skips `machine_profile: vm-test` and `skip_ssh_mesh: true`. It merges valid `machine_pubkeys` from `group_vars/all.yml` into `~/.ssh/authorized_keys` without removing existing user or vault entries, and warns if the local public key is missing or stale in git. -## What it does +The unit probes online tailnet peers selected by `scripts/online_hosts.py --peers`, intersected with `machines` and excluding the local host, `ssh_mesh_exclude`, and inventory VPSes. It uses inventory port and login overrides, bypasses connection multiplexing, and on failure closes an existing mux, attempts a fresh ed25519 host-key scan, and retries. Existing known-host entries are not removed unless the scan succeeds. Persistent failures fail the unit and name the peers. -1. Ensures the machine has an ed25519 keypair (generates if missing). -2. Merges all fleet public keys from `machine_pubkeys` (in - `group_vars/all.yml`) into `~/.ssh/authorized_keys` — non-destructive - `lineinfile`, so Bitwarden-managed keys and extras survive. -3. Warns if this machine's own public key is missing or stale in git. -4. Probes passwordless SSH to every fleet machine currently **online on - the tailnet**: `scripts/online_hosts.py --peers` (the same matcher as - `just apply-all`) gives the online MagicDNS names, each inventory host - is matched by its `tailnet_name` (or its own name), then ∩ `machines` − - `ssh_mesh_exclude` − the `vps` group. Port and login user come from - `inventory.yml` (`ansible_port`, `ansible_user`). -5. On failure: reaps stale ControlMaster sockets, refreshes the host key - in `known_hosts` (rebuilt machines), and retries. -6. Fails the apply if the mesh is still broken, naming the unreachable - hosts. - -## Why it's self-healing - -- Public keys live **in git**, not only in Bitwarden — safe to publish, - and every apply replants them. A machine missing a peer's key heals on - its next `just apply` or the daily `dotfiles-update.timer` - (`apply-nosecrets`), no vault unlock needed. -- The role is tagged `[ssh, mesh_check]` but **not** `secrets`, so the - daily no-secrets timer runs it. -- Probes bypass SSH multiplexing (`ControlMaster=no`) so a hung mux - socket can't fake an outage, and are capped with `timeout`. - -## Usage - -```bash -just mesh-check # verify + heal from this machine -just apply-remote-tags HOST mesh_check # verify + heal from HOST -``` - -## Adding a machine to the mesh - -Add its public key to `machine_pubkeys` in `group_vars/all.yml` and its -entry to the `machines` dict. If its MagicDNS name differs from its -inventory name, set `tailnet_name` on it in `inventory.yml`. Machines whose -sshd isn't always reachable (phones, port-forwarded test VMs) go in -`ssh_mesh_exclude`: a failed probe fails the apply. - -## Relationship to ssh_keys - -`ssh_keys` (tag `secrets`) still round-trips **private** keys through -Bitwarden and rebuilds `authorized_keys` from the vault when unlocked. -`ssh_mesh` is the vault-free safety net + verification layer on top. +Normally no vault is needed. If the local identity is missing, the shared implementation consults Bitwarden before restoring or creating it; locked vault or `--no-secrets` defers the identity and probes instead of generating an unmirrored key. Add new public keys to `machine_pubkeys` and host entries to `machines`; configure `tailnet_name` and exclusions where needed. diff --git a/docs/src/roles/sshd.md b/docs/src/roles/sshd.md index 98ef0fb1..6d2fa3e5 100644 --- a/docs/src/roles/sshd.md +++ b/docs/src/roles/sshd.md @@ -1,26 +1,5 @@ # sshd -**Tags:** `system`, `sshd` -**Secrets needed:** No -**Runs on:** All machines +`units/sshd.sh` skips hosts without `/etc/ssh/sshd_config.d`. Otherwise it installs `99-bw-env.conf` with `AcceptEnv BW_SESSION BW_CLIENTID BW_CLIENTSECRET` as a best-effort step. If `sshd_harden: true` is set in host variables, it also validates and installs `10-hardening.conf`: root login, password and keyboard-interactive authentication, and X11 forwarding are disabled; authentication attempts and login grace time are limited. -Configures the SSH daemon to accept Bitwarden environment variables. `just apply-remote` resolves the session on the target and forwards it back using `SendEnv=BW_SESSION` without putting it in the remote command line. - -## What It Does - -Drops a config file at `/etc/ssh/sshd_config.d/99-bw-env.conf`: - -``` -AcceptEnv BW_SESSION BW_CLIENTID BW_CLIENTSECRET -``` - -The drop-in is required for session-bearing remote applies; those applies fail if the target rejects `BW_SESSION`. - -## Handlers - -- **Reload sshd** — restarts the SSH service after config changes - -## Notes - -- Only runs if `/etc/ssh/sshd_config.d/` exists (Debian/Ubuntu/Fedora convention) -- On Alpine/musl systems this directory may not exist — the role skips gracefully +Files are staged and installed root-owned (0644); root or noninteractive `sudo -n` is required. After a change the unit attempts to reload `ssh` on Debian/Ubuntu and `sshd` elsewhere. The shared unit library skips retired `matrix` and `telengana` before changes; nothing on those hosts may be restarted. No vault access is required. diff --git a/docs/src/roles/sudo.md b/docs/src/roles/sudo.md index 13dc8407..9d427614 100644 --- a/docs/src/roles/sudo.md +++ b/docs/src/roles/sudo.md @@ -1,18 +1,5 @@ # sudo -**Tags:** `system`, `sudo` -**Secrets needed:** No -**Runs on:** All machines +`units/sudo.sh` installs a passwordless sudo rule for the current user, unless `skip_sudo: true` is set in host variables. It needs root or noninteractive `sudo -n`; it does not use secrets. -Grants passwordless sudo to the current user and cleans up old sudoers entries. - -## What It Does - -1. Writes a sudoers drop-in at `/etc/sudoers.d/zz-` granting `NOPASSWD:ALL` -2. Validates the file with `visudo -cf` before deploying -3. Removes old/conflicting sudoers files from previous naming conventions - -## Notes - -- Required for unattended playbook runs — many tasks use `become: true` -- The `zz-` prefix ensures this file loads last, overriding any earlier restrictions +It validates a staged rule with `visudo -cf` before atomically installing `/etc/sudoers.d/zz-` as root-owned mode 0440. Only after successful validation and installation does it remove the older `` and `99-` rules. The unit validates the username before using it in a path. diff --git a/docs/src/roles/syncthing.md b/docs/src/roles/syncthing.md index 39568197..75e66e17 100644 --- a/docs/src/roles/syncthing.md +++ b/docs/src/roles/syncthing.md @@ -1,19 +1,10 @@ # syncthing +**Unit:** `units/syncthing.sh` **Tags:** `services`, `syncthing` **Secrets needed:** No -**Runs on:** All except VPS +**Runs on:** non-VPS hosts, except Termux, `matrix`, `telengana`, `skip_syncthing: true`, or `machine_profile: vm-test`. -Deploys [Syncthing](https://syncthing.net/) as a systemd user service for continuous file synchronization. +The unit requires `yq` to determine VPS membership. If `/home/linuxbrew/.linuxbrew/bin/syncthing` exists, it installs `units/syncthing/syncthing.service` to `~/.config/systemd/user/`, reloads the user manager when the file changes, and enables and starts the service. It requires a working user systemd manager. -## What It Does - -1. Creates systemd user directory -2. Deploys `syncthing.service` user unit -3. Enables and starts the service - -## Notes - -- Runs as the user's systemd service (not system-wide) -- Web UI available at `http://localhost:8384` -- Skipped on VPS hosts (no file sync needed) +If the binary is absent, it disables and stops an existing user service, including one supplied by a system-wide user-unit directory; if neither service nor binary exists, it skips. It does not configure Syncthing devices or folders or remove their data. diff --git a/docs/src/roles/systemd.md b/docs/src/roles/systemd.md index e0139345..8c8ddc95 100644 --- a/docs/src/roles/systemd.md +++ b/docs/src/roles/systemd.md @@ -1,44 +1,9 @@ -# systemd +# systemd services -**Tags:** `services`, `systemd` -**Secrets needed:** No -**Runs on:** All machines +`units/system-services.sh` and `units/laptop-services.sh` manage system and user services. Both skip Termux, `skip_systemd: true`, and retired `matrix`/`telengana`. They do not need vault secrets. System Podman configuration requires root or noninteractive `sudo -n`; user services require a working user systemd manager. -Deploys systemd user services and timers for background tasks. +When `/usr/bin/podman` exists, system-services creates the `podman` group, adds the current user, installs a socket drop-in with group access (0660), and enables `podman.socket`. It restarts an already-active socket only when the drop-in changed. This applies to laptops too. -## Services Deployed +For non-laptops, system-services renders `units/system-services/` templates for the user `dotfiles-update` service and timer and the Atuin daemon, then enables the timer and daemon. For `is_laptop: true`, laptop-services instead deploys the login-triggered update service and Atuin daemon, removes an obsolete update timer, and respects `dotfiles_auto_update`. The update uses `just apply-nosecrets`. -### dotfiles-update - -Keeps the dotfiles repo up to date automatically. - -- **Desktops/servers:** Daily timer (`dotfiles-update.timer`) -- **Laptops:** Runs on login via `graphical-session.target` (no timer) - -### phone-apply (himachal only) - -Pushes non-secret Ansible configuration to `termux` and `mumbai`, then runs the non-secret units on each phone from a temporary archive of the controller's tracked revision. Each operation is attempted independently; an unreachable phone does not fail the service. SSH aliases for the two inventory hosts must be configured on himachal. - -### atuin-daemon - -Keeps [Atuin](https://atuin.sh/) shell history synced in the background. - -### Podman socket (conditional) - -If podman is installed: -- Creates the `podman` group -- Adds the user to the group -- Configures the podman socket for group access (mode 0660) -- Enables and starts `podman.socket` - -### system-mitigator (himachal only) - -A Python daemon that monitors for Lenovo EC firmware bugs: -- Detects false `BD_PROCHOT` signals that lock CPU cores to 400 MHz -- Watches for `systemd-journald` watchdog loops -- Sends desktop notifications when issues are detected - -## Notes - -- The `dotfiles-update.timer` on laptops is removed during the laptop-specific path -- All services run as user-level systemd units (no root needed) +On himachal, laptop-services also deploys `units/laptop-services/system-mitigator.py` and its user service. Unless `phone_apply_enabled: false`, it deploys a timer and service that archive the tracked checkout and run native units with `--no-secrets` independently over SSH on `termux` and `mumbai`. Unreachable phones do not fail that service. Disabling phone apply removes the timer and service. diff --git a/docs/src/roles/tailscale.md b/docs/src/roles/tailscale.md index e205f8fa..456f01c0 100644 --- a/docs/src/roles/tailscale.md +++ b/docs/src/roles/tailscale.md @@ -1,38 +1,7 @@ # tailscale -**Tags:** `secrets`, `tailscale` -**Secrets needed:** Yes (only to *join* — once connected, BW is not required) -**Runs on:** All machines +`units/tailscale.sh` skips Termux, where Android manages Tailscale. On other hosts it configures the resolved IPv6 stub and Tailscale forwarding/rp_filter sysctls, applies changed sysctls, and manages the resolver link when DNS acceptance is enabled and systemd-resolved is active. When `tailscale_accept_dns: false`, a connected host instead gets static DNS. These system changes require root or noninteractive `sudo -n`. -Installs Tailscale, fetches a reusable auth key from BW, joins the network, configures DNS, and keeps device hostname + advertised routes in sync. +When already connected it sets the hostname, reconciles configured advertised routes, and configures the desktop operator and systray where applicable. When disconnected it installs Tailscale if needed, then joins via `units/tailscale-connect.py`. The helper reads the `tailscale-apikey` Bitwarden item's login password, attempts to remove a matching stale device via the API, and passes the key to Tailscale through a temporary private file rather than argv. Device-list failure is a warning; failure to delete a matching device prevents joining. An unlocked vault is needed for joining; a locked vault or `--no-secrets` produces `PARTIAL` and leaves the host disconnected. -## What it does - -1. Configures `systemd-resolved` so MagicDNS works (link `/etc/resolv.conf` to the stub resolver, drop in `DNSStubListenerExtra=::1` for container DNS). -2. Sets `net.ipv4.conf.{all,default}.rp_filter=2` and IP forwarding on, so exit-node and subnet-router traffic actually routes. -3. If `tailscale status` reports connected → updates hostname, prefs, operator, systray, then exits the role. -4. If not connected and the vault is unlocked → fetches `tailscale-apikey` from BW, deletes any stale device with this hostname via the Tailscale API, then `tailscale up --authkey …`. -5. If not connected and the vault is **locked** → emits a one-line warning and ends the role. Joining without a key would either fail noisily or, worse, succeed with an empty key. - -## Failure modes - -| Symptom | Cause | Fix | -|---|---|---| -| Role aborts with "vault locked" warning, host is not on tailnet | First-time apply on this host without BW unlocked | Unlock BW (`bw unlock --raw | tee /tmp/bw_session > /dev/null`), then `just apply-tags tailscale` | -| Tailscale connected but MagicDNS names don't resolve | `systemd-resolved` link to `/etc/resolv.conf` was overwritten by NetworkManager / DHCP client | The role re-applies it; just re-run `dots-apply`. To make it sticky, check `network-manager` settings (Bluefin does this correctly out of the box) | -| Duplicate device entries in admin console after rebuild | API-key device cleanup didn't run (vault locked at the time) | Manually delete in [tailscale admin console](https://login.tailscale.com/admin/machines) or unlock vault and re-apply | -| Joins but no routes advertised | `tailscale_advertise_routes` not set in `host_vars` | Add it: `tailscale_advertise_routes: ""` | - -## How to verify - -```bash -tailscale status # should show all fleet machines as peers -tailscale ip -4 # this host's 100.x.x.x -host bihar.manatee-basking.ts.net # MagicDNS resolves to a 100.x.x.x -``` - -## Notes - -- All fleet machines are on the `manatee-basking.ts.net` tailnet. -- The auth key is reusable — new machines join without re-authentication. -- The Tailscale API key (`tailscale-apikey` in BW) is the same item used for joining and for device cleanup; rotate by editing the BW item and re-running this role on each host. +Host variables include `tailscale_advertise_routes`, `tailscale_accept_dns`, `tailscale_operator`, and `tailscale_domain` (the domain also falls back to `group_vars/all.yml`). The shared unit library skips retired `matrix` and `telengana` before this unit changes anything; nothing on those hosts may be restarted. diff --git a/docs/src/roles/tailscale_cert.md b/docs/src/roles/tailscale_cert.md index d636ee9e..395e12a6 100644 --- a/docs/src/roles/tailscale_cert.md +++ b/docs/src/roles/tailscale_cert.md @@ -1,19 +1,5 @@ # tailscale_cert -**Tags:** `services`, `tailscale_cert` -**Secrets needed:** No -**Runs on:** All except VPS +`units/tailscale-cert.sh` obtains a Tailscale certificate for `.` and installs a systemd renewal service and timer from `units/tailscale-cert/`, rendered by `units/service-templates.py`. It checks whether `/etc/ssl/tailscale/cert.pem` remains valid for 30 days; otherwise it runs `tailscale cert` with a 720-hour minimum validity. The certificate is root-owned 0644 and its private key root-owned 0640. -Fetches and deploys Tailscale TLS certificates for HTTPS on `*.manatee-basking.ts.net`. - -## What It Does - -1. Creates certificate storage directory -2. Fetches TLS certificates from Tailscale for the machine's MagicDNS name -3. Installs certificates where Caddy and other services can use them - -## Notes - -- Tailscale provides free TLS certificates for nodes on your tailnet -- Certificates auto-renew — no Let's Encrypt needed for internal services -- The Caddy proxy uses these certificates for all `*.manatee-basking.ts.net` services +The unit requires `yq`, Python, OpenSSL, Tailscale, systemd, and root or noninteractive `sudo -n`. It does not retrieve a Bitwarden secret. The host domain overrides `group_vars/all.yml`. It skips inventory VPSes, retired `matrix`/`telengana`, `skip_tailscale_cert: true`, and `machine_profile: vm-test`. The timer is enabled only after both renewal units are installed successfully. diff --git a/docs/src/roles/termux_packages.md b/docs/src/roles/termux_packages.md index 2f25a968..785bf48b 100644 --- a/docs/src/roles/termux_packages.md +++ b/docs/src/roles/termux_packages.md @@ -1,30 +1,5 @@ # termux_packages -**Tags:** `system`, `packages`, `termux` -**Secrets needed:** No -**Runs on:** `termux_hosts` group only (the raw Termux/Android layer on the phone) +`units/termux-packages.sh` runs for inventory members of `termux_hosts`, using Termux's unprivileged `pkg` and npm rather than systemd or root. Defaults are in `units/termux-packages.conf`; `termux_pkgs`, `termux_npm_global`, and `termux_keepalive_period_ms` can be overridden in `group_vars/all.yml` and then `host_vars/.yml`. Reading overrides requires `yq`. -Package and sshd management for Termux, which has no root, no systemd, no Homebrew and bionic libc. - -## What It Does - -1. `pkg update -y` (retried 3×), then `pkg install -y` the `termux_pkgs` list -2. `npm install -g` the `termux_npm_global` list (the pi coding agent) -3. Deploys `~/.termux/boot/start-sshd` (`termux-wake-lock` + `sshd`) for Termux:Boot -4. Starts `sshd` if it isn't running -5. Schedules the same script as a persisted periodic job via - `termux-job-scheduler` (job id 1), so sshd comes back after Android kills Termux - -## Configuration - -`roles/termux_packages/defaults/main.yml`: - -- `termux_pkgs` — openssh, git, python, nodejs, fish, vim, nano, termux-api, curl, unzip, plus `file` and `e2fsprogs` (Ansible's stat/copy modules need them) -- `termux_npm_global` — `@earendil-works/pi-coding-agent` -- `termux_keepalive_period_ms` — `1800000` (30 min; Android's floor is 15) - -## Notes - -- Requires the Termux:Boot and Termux:API companion apps, installed separately (F-Droid/GitHub) -- Claude Code, Codex etc. ship no android-arm64 binary — they live on the `mumbai` phone VM instead -- Connection quirks (remote tmp, scp, bash as shell executable) are pinned in `host_vars/termux.yml` +It retries `pkg update` three times, installs bootstrap and configured packages (falling back to individual installs after a failed batch), and installs configured npm globals. Optional mise installation is best effort. It installs `units/termux-start-sshd` as `~/.termux/boot/start-sshd`, starts sshd if absent, and schedules persisted job 1 if not already pending. Termux:Boot and Termux:API companion apps are separate prerequisites. No vault secrets are used. diff --git a/docs/src/roles/zen_browser.md b/docs/src/roles/zen_browser.md index 38cdc4f5..a7c882f8 100644 --- a/docs/src/roles/zen_browser.md +++ b/docs/src/roles/zen_browser.md @@ -1,18 +1,12 @@ # zen_browser +**Unit:** `units/zen-browser.sh` **Tags:** `desktop`, `browser`, `zen_browser` **Secrets needed:** No -**Runs on:** Desktop group only +**Runs on:** desktops, except `skip_zen_browser: true` or `machine_profile: vm-test`. -Configures [Zen Browser](https://zen-browser.app/) (Firefox-based) as the default browser. +The unit best-effort pins Zen in the GNOME dock, removes Firefox and Thunderbird dock entries, and sets browser MIME defaults when the Zen Flatpak data directory exists. A live user session is needed for dock changes. -## What It Does +If Zen's Flatpak `profiles.ini` is absent, profile and policy steps are skipped. Otherwise it resolves `Profile0` and merges `units/zen-browser/user.js` into a managed block in that profile's `user.js`, retaining unrelated preferences. It merges `units/zen-browser/policies.json` into both the system Flatpak Zen distribution policy and `~/.zen/distribution/policies.json`, preserving unrelated policy keys and list entries. The system policy requires root or non-interactive sudo; malformed or incompatible existing policies fail rather than being discarded. -1. Pins Zen Browser to the GNOME dock -2. Removes Firefox and Thunderbird from the dock if present -3. Sets Zen as the default web browser - -## Notes - -- Zen is installed via Flatpak (`app.zen_browser.zen`) -- Firefox and Thunderbird are intentionally removed from the dock — Zen replaces both +Firefox Account setup is handled by the separate [browser-fxa unit](browser_fxa.md), not by this unit. A successful configuration does not verify a browser login. diff --git a/docs/src/runbook.md b/docs/src/runbook.md index cd50e49e..79be9474 100644 --- a/docs/src/runbook.md +++ b/docs/src/runbook.md @@ -8,13 +8,11 @@ these from memory or commit logs more than twice, add it here. ## I rotated my SSH key on host X — how do I propagate it? An intentional local rotation is explicit: when the local public key differs -from the Bitwarden item, the `ssh_keys` role leaves Bitwarden unchanged and -warns. This protects the stored key if a new key was generated while the vault -was locked. To publish an intentional rotation, on host X with BW unlocked: +from the Bitwarden item, the `ssh-keys` unit leaves Bitwarden unchanged and +warns. To publish an intentional rotation, on host X with BW unlocked: ```bash -# `dots-apply` forwards these arguments to the Ansible playbook. -dots-apply -e ssh_keys_force_push=true +SSH_KEYS_FORCE_PUSH=true just apply-tags ssh_keys ``` That updates the `james@` item in Bitwarden and refreshes this host's @@ -46,22 +44,22 @@ themselves (`scripts/bw-resolve.sh remote`), but that can fail — e.g. no that host: ```bash -# From your laptop, let bw-resolve.sh handle unlock internally: -just apply-remote-tags bitwarden,ssh_keys +# Resolve and apply the SSH key selection on the remote host: +just apply-remote-tags ssh_keys ``` The `scripts/bw-resolve.sh remote ` script (invoked internally) will SSH to the remote, ensure Bitwarden is logged in and unlocked (either via stored API key, master password from your local vault, or interactive prompt), -and return the session token. The `bitwarden` role will then confirm the vault -is unlocked, and downstream secrets roles will run. +and return the session token. The remote apply validates and syncs it before +secret-capable units run; if resolution fails, they run with secrets disabled. --- ## The timer apply failed last night — what do I do? -The user-level `dotfiles-update.service` runs `just apply-nosecrets` daily, -which now writes `~/.cache/dotfiles/last-apply.json`. Quickest signal: +The user-level `dotfiles-update.service` runs `just apply-nosecrets` on its +configured schedule or laptop login, and records `~/.cache/dotfiles/last-apply.json`. ```bash just doctor # `→ Last apply` line shows status + age @@ -76,9 +74,9 @@ journalctl --user -u dotfiles-update.service -e --no-pager | tail -80 The most common causes: - **git pull failure** (DNS / network blip): the recipe tolerates it with - `git pull --ff-only || true`; the ansible run continues with whatever's + `git pull --ff-only || true`; native units continue with whatever is on disk. If the working tree drifted from main, fix manually. -- **a role started failing after a config change**: re-run interactively +- **a unit started failing after a config change**: re-run interactively with `just apply-nosecrets` and read the failure. - **`mkswap: /swapfile is mounted`** (resolved): `server_hardening` now gates `mkswap`/`swapon` on `swapon --show`, so an already-active swapfile @@ -97,11 +95,11 @@ Fix is to re-login (downloads a clean vault): ```bash bw logout bw login # email + master password + 2FA -export BW_SESSION=$(scripts/bw-resolve.sh local) +eval "$(scripts/bw-resolve.sh local)" ``` -The `bitwarden` role now has a 15 s timeout on `bw sync`, which prevents the -race in most cases going forward. +Apply recipes put a 15-second timeout on `bw sync`; this bounds a stalled sync +but is not a guarantee against concurrent Bitwarden CLI races. --- @@ -117,23 +115,23 @@ curl -fsSL https://raw.githubusercontent.com/hanthor/dotfiles/master/bootstrap.s # 3. After bootstrap finishes, on the new machine, get BW into a usable state: bw login # if not logged in -export BW_SESSION=$(scripts/bw-resolve.sh local) # unlocks vault +eval "$(scripts/bw-resolve.sh local)" # unlocks vault # 4. Run the full apply (this pushes the new machine's pubkey to BW): dots-apply -# 5. Back on each existing host (the no-secrets timer skips ssh_keys): +# 5. Back on each existing host (no-secrets applies skip vault synchronization): dots-apply # picks up the new host's pubkey ``` -Verify with `just doctor` on the new host — should be all green and report -"last apply ago". +Inspect `just doctor` and the unit output on the new host; partial or skipped +work may need attention even when the apply exits successfully. --- ## I added a flatpak / brew package — how do I roll it out? -Edit the package list — `system_flatpaks` in `roles/flatpak/vars/main.yml`, +Edit the package list — `system_flatpaks` in `units/flatpak-packages.yml`, or the brew lists (`core_brews`, `desktop_brews`, …) in `group_vars/all.yml`, which render the Brewfile — then commit, push. Every host picks it up on the next `dots-apply` or the next timer fire. @@ -146,18 +144,13 @@ just apply-tags packages --- -## I want to disable a role on one host +## I want to disable a unit on one host -In that host's `host_vars/.yml`, add a `skip_: true` line. The -flags `site.yml` actually checks: - -`skip_sudo`, `skip_ssh_mesh`, `skip_kube`, `skip_forgejo`, `skip_flatpak`, -`skip_bluefin`, `skip_gnome`, `skip_zen_browser`, `skip_pipewire_audio`, -`skip_syncthing`, `skip_systemd`, `skip_proxy`, `skip_tailscale_cert`. - -Roles without a flag are gated some other way — `server_hardening` on -`server`/`vps` membership, `termux_packages` on `termux_hosts`, many roles on -`machine_profile != 'vm-test'`. Grep `site.yml`. +Many units inspect `skip_*` flags in `host_vars/.yml`; others use +inventory membership, `machine_profile`, platform detection or an enable flag. +Read the relevant unit and [Architecture](architecture.md#conditional-execution) +before setting a flag. Retired `matrix` and `telengana` are skipped by the shared +unit library; nothing on either host may be restarted. --- diff --git a/docs/src/scripts.md b/docs/src/scripts.md index 205a30cb..ef53ed9a 100644 --- a/docs/src/scripts.md +++ b/docs/src/scripts.md @@ -1,292 +1,34 @@ # Utility Scripts -The `scripts/` directory contains helper utilities for inventory management, Bitwarden integration, and operational tasks. They are not deployed to machines — they run on your local development machine or during CI. +`scripts/` contains helpers used locally, by remote applies, user services and CI. The native fleet entrypoint is `scripts/run-units.sh`; unit implementations and their assets live in `units/`. -## Inventory Management +## Applying and status -### `register-machine.py` +- `scripts/run-units.sh [--no-secrets] [--tags a,b]` selects units from its tag map, orders prerequisites in stages, and runs independent selections through mise or a sequential fallback. It continues after independent failures; selected dependents may be skipped if a prerequisite fails. `just units` invokes this wrapper. +- `just apply`, `just apply-nosecrets` and `just apply-tags ` invoke the wrapper. The first pulls and resolves Bitwarden, the second disables secret access, and tagged applies resolve Bitwarden only for secret-bearing selections. See [Units](units.md). +- `just check` requests unit inspection with `UNITS_CHECK=1`. Coverage and safety across platforms are subject to integration verification; do not assume a complete or universally non-mutating dry run. +- `scripts/record-apply.py EXIT_CODE [LABEL [SKIP_TAGS]]` records the recipe result, timestamp and git revision in `~/.cache/dotfiles/last-apply.json`. `scripts/last-apply-status.py` reads it for `just doctor` and returns nonzero for failed, stale or unreadable records. A successful record can still include partial or skipped units. -Register a new machine in `inventory.yml` and create its `host_vars` file. +The shared unit library skips retired `matrix` and `telengana` before unit work. **Nothing on those VPSes may be restarted**; pre-run operations and other scripts are not covered by that guard. -**Usage:** -```bash -python3 scripts/register-machine.py [type] [path-to-inventory.yml] -``` +## Inventory -**Arguments:** -- `` — machine name (e.g., `kanpur`, `goa`) -- `[type]` — group/machine type: `desktop`, `server`, `vps`, `termux_hosts` (default: `desktop`) -- `[path-to-inventory.yml]` — inventory file path (default: `inventory.yml`) +- `scripts/register-machine.py [type] [inventory-path]` adds an inventory host and group membership and creates host variables. `just onboard` wraps registration and prints bootstrap instructions. +- `scripts/purge-machine.py [inventory-path]` removes inventory and host variables, not Bitwarden items. `just purge` wraps it. +- `scripts/online_hosts.py` intersects online Tailscale peers with inventory, excluding VPSes and the local host. `--peers` prints online peer names instead; the SSH mesh unit uses it. Tailscale status failure yields no peers. +- `scripts/nmap-inventory.sh [CIDR]` scans the LAN and cross-references Tailscale; it requires nmap and network access. `just inventory` provides the usual entrypoint. -**What it does:** -- Adds the machine to the `all.hosts` section -- Adds it to the specified group under `all.children` -- Sets `ansible_host: localhost` and `ansible_connection: local` -- Creates `host_vars/.yml` with a template +## Bitwarden helpers -**Example:** -```bash -python3 scripts/register-machine.py goa server -``` +- `scripts/bw-resolve.sh local` prints a shell export for a usable local session; use `eval "$(scripts/bw-resolve.sh local)"` to set it. It tries the existing session, cache and interactive unlock. `remote ` attempts to unlock on that host and prints its session token; the apply recipes forward it with SSH `SendEnv=BW_SESSION`. +- `scripts/bw-unlock.sh` resolves a local session and prints the raw token. Avoid exposing token output. Sessions are not guaranteed portable between hosts. +- `scripts/bw-seed-kube.sh` upserts the `kubeconfig` and `talosconfig` secure notes from local files, then syncs. `KUBECONFIG` and `TALOSCONFIG` can override the default paths. These credentials contain cluster PKI; do not commit them. +- `scripts/bw-seed-omp-broker.sh` seeds the omp broker token. See the [omp reference](roles/omp.md). +- `scripts/bw-seed-ssh-keys.sh` is a legacy helper with a hardcoded machine list. It reads private and public keys locally or over SSH, writes SSH-key items and may register GitHub keys, then writes a chezmoi `dot_ssh/authorized_keys` source file. Review its hardcoded hosts and destination before use; it does not use the native fleet unit's vault item naming or managed-block merge. The normal key lifecycle is documented in [ssh_keys](roles/ssh_keys.md). -### `purge-machine.py` +## Other helpers -Remove a machine from `inventory.yml` and its `host_vars` file. Mirror of `register-machine.py`. - -**Usage:** -```bash -python3 scripts/purge-machine.py [path-to-inventory.yml] -``` - -**Important:** Does NOT delete the corresponding Bitwarden SSH key item — you can re-onboard the machine later using the same name. - -**Example:** -```bash -python3 scripts/purge-machine.py matrix -``` - -### `nmap-inventory.sh` - -Scan the LAN using `nmap` and cross-reference with Tailscale to show network status. - -**Usage:** -```bash -./scripts/nmap-inventory.sh [CIDR] -``` - -**Arguments:** -- `[CIDR]` — network range to scan (default: `192.168.0.0/24`). Override with a different subnet. - -**Output columns:** -- IP address -- MAC address -- Hostname (reverse DNS or nmap service detection) -- Tailscale status (online/offline) - -**Prerequisites:** -- `nmap` installed -- Tailscale running and authenticated -- Network access to the scanned range - -**Example:** -```bash -./scripts/nmap-inventory.sh 10.0.0.0/24 -``` - -## Bitwarden Integration - -### `bw-resolve.sh` - -Resolve a usable `BW_SESSION` for local and remote operations. Tries multiple fallback tiers. - -**Usage:** -```bash -./scripts/bw-resolve.sh local -./scripts/bw-resolve.sh remote -``` - -**Modes:** - -- **`local`** — Resolve a session on your local machine. Returns `export BW_SESSION=...` on stdout or exits 1 if unlock fails. - -- **`remote `** — Unlock Bitwarden on a remote machine, export that session inline over `ssh -t`, and return it. Used by `just apply-remote` to drive secret-dependent tasks on the target. - -**Session resolution order:** -1. Existing `BW_SESSION` environment variable (if still valid) -2. Cached session in `/tmp/bw_session` (if not expired) -3. Interactive `bw unlock` prompt (asks for master password) - -**Example — local:** -```bash -eval "$(./scripts/bw-resolve.sh local)" -bw list items # Now BW_SESSION is set -``` - -**Example — remote:** -```bash -./scripts/bw-resolve.sh remote himachal -# Executes commands on himachal with Bitwarden access -``` - -### `bw-unlock.sh` - -Simpler variant of `bw-resolve.sh` — resolves a session on the local machine only. - -**Usage:** -```bash -./scripts/bw-unlock.sh -``` - -**Output:** Prints `` on stdout, or exits 1 if all tiers fail. - -### `bw-seed-ssh-keys.sh` - -Pull SSH public keys from all online machines and seed them into a Bitwarden collection. - -**Usage:** -```bash -./scripts/bw-seed-ssh-keys.sh -``` - -**What it does:** -- Connects to each machine in the online fleet -- Retrieves their `~/.ssh/id_ed25519.pub` -- Creates Bitwarden login items named `james@` with the public key in the password field -- Updates existing items in place (safe to re-run) - -**Prerequisites:** -- SSH access to all machines (keys already in `~/.ssh/authorized_keys`) -- Bitwarden CLI (`bw`) authenticated -- Tailscale or direct network access to all machines - -**Run once:** Before onboarding a new machine, so their SSH keys are available in Bitwarden during setup. - -### `bw-seed-kube.sh` - -Seed kubeconfig and talosconfig from this machine into Bitwarden. - -**Usage:** -```bash -./scripts/bw-seed-kube.sh -``` - -**What it does:** -- Exports your local `~/.kube/config` to Bitwarden as a secure note -- Exports your local `~/.talos/config` to Bitwarden -- Updates existing items in place - -**Prerequisites:** -- kubectl configured with cluster access -- talosctl configured with cluster access -- Bitwarden CLI (`bw`) authenticated - -**Use when:** You've set up cluster access locally and want to share it to other machines. - -## Apply and Status - -### `record-apply.py` - -Record the outcome of a `just apply` run for tracking and status checks. - -**Usage:** -```bash -python3 scripts/record-apply.py [label] [skip-tags] -``` - -**Arguments:** -- `` — exit code from the Ansible apply (0 = success, non-zero = failure) -- `[label]` — optional descriptive label (default: inferred from context) -- `[skip-tags]` — optional comma-separated skip tags (for partial applies) - -**Output:** Writes to `~/.cache/dotfiles/last-apply.json` with: -- Timestamp -- Exit code (0 = success, 1 = failure) -- Label and skip tags (if provided) - -**Used by:** `just doctor` to check health status and flag old or failed applies. - -### `last-apply-status.py` - -Pretty-print the last recorded apply status. Used by `just doctor`. - -**Usage:** -```bash -python3 scripts/last-apply-status.py -``` - -**Exit codes:** -- `0` — apply succeeded and timestamp is recent (< 48 hours) -- `1` — apply failed OR timestamp is stale (≥ 48 hours) - -**Output:** Human-readable status with timestamp, result, and any tags. - -### `online_hosts.py` - -Print the online fleet hosts: inventory hosts whose tailnet name is online, excluding the `vps` group and this machine. A host's tailnet name is its `tailnet_name` in `inventory.yml` (e.g. `termux` → `pixel-9a`), otherwise its inventory name. A peer's name is its MagicDNS short name; the OS HostName is used only when a peer has no DNSName, because HostNames aren't unique (several phones report `localhost`). - -**Usage:** -```bash -python3 scripts/online_hosts.py # inventory names, space-separated -python3 scripts/online_hosts.py --peers # raw online tailnet names, one per line -``` - -**Used by:** `just _online_hosts` (bulk applies, `doctor-fleet`, `push-terminfo`), and `roles/ssh_mesh`, which runs `--peers` on the target via the `script` module. - -**Silently fails:** If Tailscale is not running, returns empty output (so scripts gracefully degrade). - -## Cluster and Infrastructure - -### `kubevirt-node` - -Operations on KubeVirt nodes for the test cluster. - -**Usage:** -```bash -./scripts/kubevirt-node [command] [node-name] -``` - -**Supported commands:** -- `list` — List all KubeVirt test nodes -- `start ` — Power on a node -- `stop ` — Power off a node -- `status ` — Show node status - -**Prerequisites:** -- `kubectl` configured with cluster access -- Permission to manage VirtualMachine resources - -### `sync-brews.sh` - -Sync the Homebrew package state: fetch the current installed packages from Homebrew and update `group_vars/all.yml` with the new list. - -**Usage:** -```bash -./scripts/sync-brews.sh -``` - -**What it does:** -1. Runs `brew list --formulae` and `brew list --casks` -2. Extracts the current package list -3. Updates the `homebrew_packages` and `homebrew_casks` sections in `group_vars/all.yml` -4. Prints a diff so you can review changes - -**Use when:** You've manually installed packages with `brew install` and want to record them in the inventory for automated management. - -### `ess-render-values.py` - -Render Helm values for the AWS Talos cluster (ESS) by applying substitution rules from Bitwarden. - -**Usage:** -```bash -python3 scripts/ess-render-values.py < ess-helm-values-template.yml > values.yml -``` - -**What it does:** -1. Reads the `ess-helm-values` secure note from Bitwarden -2. Applies environment and variable substitutions (e.g., `${DOMAIN}`) -3. Outputs the rendered Helm values to stdout - -**Used by:** Cluster deployment pipelines to generate final Helm configurations. - -**Prerequisites:** -- `bw` CLI authenticated with Bitwarden vault access -- The `ess-helm-values` secure note exists in Bitwarden - -## Prerequisites and Setup - -All scripts assume: -- **Python 3** (for `.py` scripts) -- **Bash** (for `.sh` scripts) -- **Bitwarden CLI** (`bw`) installed and configured when BW scripts are used -- **SSH access** to target machines (for remote operations) -- **Tailscale** installed for machine discovery (for inventory scripts) - -Set up Bitwarden CLI once: -```bash -brew install bitwarden-cli # macOS -# or: apt-get install bw # Linux - -bw login -``` - -Session tokens are cached locally in `/tmp/` for convenience; they expire and are renewed automatically. +- `scripts/sync-brews.sh` compares Homebrew's dumped formulae with `core_brews` and `desktop_brews` in `group_vars/all.yml` and adds missing formulae to `desktop_brews`; review its changes before committing. +- `scripts/kubevirt-node` operates KubeVirt test VMs. The home Talos cluster is currently offline in storage, not decommissioned. +- `scripts/ess-render-values.py` reads sensitive ESS Helm values on stdin and writes transformed values to stdout for the former AWS cluster. It does not retrieve the vault item itself. Do not write secret-bearing output to a tracked file; the AWS Talos cluster is terminated. +- `scripts/osquery-fleet.py` collects private fleet monitoring reports; see [fleet facts and osquery](roles/fleet_facts.md). diff --git a/docs/src/units.md b/docs/src/units.md index 738720b3..d5e9b400 100644 --- a/docs/src/units.md +++ b/docs/src/units.md @@ -1,54 +1,34 @@ # Units -A **unit** is an idempotent bash script in `units/` that configures one independent piece of a machine. Units exist because Ansible aborts a whole role when one task fails: on himachal the `omp` role died at `Deploy omp-kiro-provider config` (`~/.config/omp` did not exist), so the routing merge, `omp-route` and the auth-broker config never ran. In a unit, a failing step is recorded and the next independent step still runs; independent units run in parallel. - -[mise](https://mise.jdx.dev) is the runner: each unit is a `mise.toml` task, and `mise run` supplies the dependency graph, parallel jobs and `--continue-on-error`. - -Everything not listed under `units/` is still an Ansible role. +A unit is a Bash entrypoint in `units/` configuring one concern. Shared helpers in `units/lib.sh` let a unit record failed steps while continuing independent work. Native units replace the fleet playbook; their supporting scripts, templates and files are under `units/`, with chezmoi-managed files under `home/`. ## Running units -| Command | What it does | +| Command | Behavior | |---|---| -| `just units` | Run every unit in parallel (`just units --tags omp`, `--no-secrets` also accepted) | -| `mise run units` | Same, straight through mise | -| `mise run unit:omp` | One unit | -| `mise run --dry-run units` | Show what would run | -| `mise tasks deps units` | Show the task graph | -| `just apply`, `apply-nosecrets`, `apply-tags`, `dotfiles` | Run Ansible first, then `scripts/run-units.sh` (never skipped because Ansible failed; exit codes are combined) | - -`scripts/run-units.sh [--no-secrets] [--tags a,b]` is the one wrapper the recipes call. It maps Ansible tags to units (`tasks_for_tag`, mirroring the tags of the roles the units replaced), runs the selection with `mise run --continue-on-error --jobs 8 --output prefix --timeout 15m`, and exits with mise's status. Without `mise` on `PATH` it prints `mise not installed; running units sequentially` and runs the selected units one after another, continuing after failures, so hosts without mise (Termux, musl) keep updating. - -`just apply-tags` resolves, validates and syncs Bitwarden for explicitly secret-bearing tags (including `omp`), and selects the `bitwarden` role so legacy secret roles receive its facts. If that fails, it skips secrets, still runs the non-secret selection, and returns an error. Tags unrelated to secrets do not prompt for a vault unlock. `just apply-nosecrets` and bootstrap's first phase pass `--no-secrets` to units even if a session is already present. +| `just apply` | Pull, resolve and sync Bitwarden when possible, and run the fleet units. | +| `just apply-nosecrets` | Run units with vault access disabled; pull failure is tolerated. | +| `just apply-tags omp,shell` | Select units mapped from tags; secret-bearing selections attempt vault resolution. | +| `just dotfiles` / `just packages` | Run the corresponding tag selection. | +| `just units --tags omp` | Run the wrapper directly; `--no-secrets` is also accepted. | +| `mise run units` | Run the same wrapper. | +| `mise run unit:omp` | Run one registered unit directly, without wrapper prerequisite ordering. | +| `just check` | Set `UNITS_CHECK=1` and run the native selection in inspection mode. Coverage and safety remain subject to integration verification; do not assume a complete or universally non-mutating dry run. | -Bootstrap runs all units after its first Ansible pass and the `omp,secrets` selection after the secret pass. The phone push timer runs the non-secret units remotely from a temporary archive of the controller's tracked revision, after its Ansible passes. +`scripts/run-units.sh [--no-secrets] [--tags a,b]` is the authoritative tag map and registry. It runs package managers and other prerequisites in stages, then runs independent later units in parallel using mise (`UNITS_JOBS`, default 8), continuing after independent failures. Without mise, stages run sequentially. A failure in a selected SSH keys, Tailscale or Zen unit suppresses its selected dependent units; unselected prerequisites are not implicitly applied. Fleet facts runs last. `SKIPPED` and `PARTIAL` lines describe exclusions or incomplete work and are not, by themselves, proof of a complete configuration. -## Rules for a unit +`just apply-tags` resolves and syncs Bitwarden for explicitly secret-bearing tags; if it fails, selected units run with secrets disabled and the recipe reports failure. The no-secrets path prevents unit vault access even when a session exists. Himachal's phone timer sends a temporary archive of the tracked checkout and runs native units with `--no-secrets` on each target independently. -- Idempotent bash. `set -uo pipefail`, never `set -e`. -- Source `units/lib.sh`: `step "desc" cmd...` (a failure marks the unit failed and continues), `soft "desc" cmd...` (a failure is only a warning), `note "..."`, `finish` (exit 1 only if a `step` failed), `install_if_changed src dest mode`, `bw_session` (a usable Bitwarden session or nothing; never prompts; honours `UNITS_NO_SECRETS=1`). -- mise treats any non-zero exit as failure, so "skipped" and "partial" are printed as `SKIPPED: ...` / `PARTIAL: ...` lines with exit 0. -- Never echo a secret or put one on a command line. +The shared unit library skips identities `matrix` and `telengana` before unit work, including package transactions. These retired VPSes remain powered on for burn-in: nothing on them may be restarted. The guard does not authorize other changes to those hosts. -## Adding a unit +## Implementing a unit -1. Create `units/.sh` (executable) with a `# shellcheck source=units/lib.sh` line above its `. lib.sh` line. -2. Add `[tasks."unit:"]` with `file = "units/.sh"` to `mise.toml`. -3. Add `` to `ALL_UNITS` and to the right `tasks_for_tag` branches in `scripts/run-units.sh`. +Use idempotent Bash with `set -uo pipefail`, source `units/lib.sh` with a shellcheck source directive, and use `step` for recorded failures, `soft` for best-effort work, `note` for status and `finish` for the resulting exit status. `bw_session` is noninteractive and honors `UNITS_NO_SECRETS=1`; do not log secret values. Preserve user data and make platform, profile, privilege and retired-host safeguards explicit. Direct unit invocation does not provide wrapper ordering. -`tests/test_units_registry.py` fails if the three places disagree. - -`units/`, `mise.toml` and `scripts/run-units.sh` are executed by the fleet timer, so they are intentionally not carved out of CODEOWNERS. +Register a new executable `units/.sh` in `mise.toml`, `ALL_UNITS` and the relevant `tasks_for_tag` branches and stage in `scripts/run-units.sh`. Executable configuration paths are not intended to bypass CODEOWNERS review. ## chezmoi -File-based config (shell, git, neovim, atuin config) is managed by [chezmoi](https://www.chezmoi.io/) from `home/` (`.chezmoiroot` contains `home`, so `.chezmoi.toml.tmpl`, `.chezmoidata.yaml` and `.chezmoiscripts/` live there). `unit:chezmoi` runs `chezmoi --source --keep-going --force --no-tty apply --init`: `--keep-going` continues past a failing file or script, `--force` overwrites like the old roles did, `--no-tty` avoids a TTY, and `--init` regenerates the config from `home/.chezmoi.toml.tmpl`. - -**Back up and review existing dotfiles before the first apply.** The current `--force` will overwrite local edits to managed targets without prompting on every apply. First adoption can replace an existing target even without `--force`: chezmoi does not treat a pre-existing, never-written target as a conflict by default. Review the intended changes with `chezmoi --source diff` (after initializing the config with the correct machine data), and keep a private backup of any local files you need. A safer future cutover should detect both first-adoption and later local conflicts and fail closed for unattended runs, with interactive review; merely removing `--force` does not protect initial adoption. - -If chezmoi is absent the unit fails rather than reporting a successful skip. Homebrew hosts can run `just packages`, Termux users can run `pkg install chezmoi`, and Alpine/postmarketOS users can enable the community repository and run `sudo apk add chezmoi`. +`.chezmoiroot` points to `home/`, which contains the source files, `.chezmoi.toml.tmpl`, `.chezmoidata.yaml` and `.chezmoiscripts/`. `units/chezmoi.sh` applies shell, Git, Neovim and Atuin configuration with `--keep-going --force --no-tty apply --init`. It fails if chezmoi is unavailable (on Termux it first attempts a best-effort `pkg install`). Desktop detection honors `DOTFILES_IS_DESKTOP` or host `is_desktop`, then inventory membership; without `yq` the unit warns and assumes non-desktop. -- Data: `machine`, `is_desktop` (true when the host is in the `desktop` group of `inventory.yml`), `user_name`, `user_email`, `alias_groups`. -- Secrets are **not** chezmoi templates: the `bitwarden` template function aborts a whole template on a missing item. The `shell-ai` and `atuin` units handle secrets. -- chezmoi never deletes files it does not manage (no `exact_` directories). -- `ssh_keys` stays an Ansible role (key lifecycle, `authorized_keys` merge). +**Back up and review existing dotfiles before first apply.** `--force` overwrites local edits to managed targets without prompting, including on subsequent runs. Even without `--force`, first adoption of an existing target can replace it. The source uses no `exact_` directories, so unmanaged files are not deleted by directory synchronization. Secrets are handled by units rather than chezmoi templates, so unavailable vault items do not abort unrelated file templates. `units/ssh-keys.sh` and `units/ssh-mesh.sh` manage SSH key lifecycle and authorization separately. diff --git a/group_vars/all.yml b/group_vars/all.yml index c9ce5be9..b2f1a443 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -227,11 +227,6 @@ flatpak_remotes: # Bitwarden item name that holds Firefox Accounts credentials + TOTP fxa_bw_item: "accounts.firefox.com" fxa_bw_username: "jreilly1821@gmail.com" -# Browsers to sign in to via FxA (skips any not installed on the host) -fxa_browsers: - - zen - - firefox - - firefox-flatpak # ── GNOME Extensions ─────────────────────────────────────────────── # Canonical list of enabled extensions, synced across fleet desktops. # To refresh from a source machine: diff --git a/host_vars/himachal.yml b/host_vars/himachal.yml index 453fb2fd..9b3223c3 100644 --- a/host_vars/himachal.yml +++ b/host_vars/himachal.yml @@ -11,3 +11,17 @@ mic_hardware_volume_pct: 75 # The AWS cluster was terminated; local Hive ops remain disabled. # The role removes any old local timers on the next apply. hive_ops_enabled: false + +# User-scoped FlatPark apps; Flathub supplies their runtimes. +user_flatpak_remotes: + - name: flatpark + url: https://dl.flatpark.org/flatpark.flatpakrepo + - name: flathub + url: https://dl.flathub.org/repo/flathub.flatpakrepo +user_flatpaks: + - remote: flatpark + id: org.tunaos.blueshell + - remote: flatpark + id: org.tunaos.tavern + - remote: flatpark + id: org.tunaos.compass diff --git a/mise.toml b/mise.toml index e180fb14..cc9d201c 100644 --- a/mise.toml +++ b/mise.toml @@ -1,6 +1,6 @@ [tasks.units] -description = "Run every unit in parallel; one failing unit does not stop the others" -depends = ["unit:*"] +description = "Run fleet units in dependency order, continuing after independent failures" +run = "scripts/run-units.sh" [tasks."unit:omp"] description = "omp: skills, Kiro plugin, routing, and the auth-broker client" @@ -33,3 +33,81 @@ file = "units/kube.sh" [tasks."unit:forgejo-registry"] description = "forgejo-registry: podman login to the private Forgejo registry" file = "units/forgejo-registry.sh" + +[tasks."unit:sudo"] +file = "units/sudo.sh" + +[tasks."unit:sshd"] +file = "units/sshd.sh" + +[tasks."unit:apk-packages"] +file = "units/apk-packages.sh" + +[tasks."unit:termux-packages"] +file = "units/termux-packages.sh" + +[tasks."unit:homebrew"] +file = "units/homebrew.sh" + +[tasks."unit:flatpak"] +file = "units/flatpak.sh" + +[tasks."unit:ssh-keys"] +file = "units/ssh-keys.sh" + +[tasks."unit:ssh-mesh"] +file = "units/ssh-mesh.sh" + +[tasks."unit:tailscale"] +file = "units/tailscale.sh" + +[tasks."unit:bluefin"] +file = "units/bluefin.sh" + +[tasks."unit:gnome"] +file = "units/gnome.sh" + +[tasks."unit:zen-browser"] +file = "units/zen-browser.sh" + +[tasks."unit:browser-fxa"] +file = "units/browser-fxa.sh" + +[tasks."unit:pipewire-audio"] +file = "units/pipewire-audio.sh" + +[tasks."unit:syncthing"] +file = "units/syncthing.sh" + +[tasks."unit:hive-ops"] +file = "units/hive-ops.sh" + +[tasks."unit:laptop-services"] +file = "units/laptop-services.sh" + +[tasks."unit:system-services"] +file = "units/system-services.sh" + +[tasks."unit:osquery"] +file = "units/osquery.sh" + +[tasks."unit:server-hardening"] +file = "units/server-hardening.sh" + +[tasks."unit:retired-services"] +file = "units/retired-services.sh" + +[tasks."unit:proxy"] +file = "units/proxy.sh" + +[tasks."unit:tailscale-cert"] +file = "units/tailscale-cert.sh" + +[tasks."unit:bst-dashboard"] +file = "units/bst-dashboard.sh" + +[tasks."unit:kirocrew"] +file = "units/kirocrew.sh" + +[tasks."unit:fleet-facts"] +file = "units/fleet-facts.sh" diff --git a/requirements.yml b/requirements.yml deleted file mode 100644 index 682964be..00000000 --- a/requirements.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -collections: - - name: community.general - version: ">=9.0.0" - - name: ansible.posix - version: ">=1.5.0" - -roles: [] diff --git a/roles/apk_packages/tasks/main.yml b/roles/apk_packages/tasks/main.yml deleted file mode 100644 index 1ee88fcf..00000000 --- a/roles/apk_packages/tasks/main.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Install packages via apk on Alpine/musl systems (when Homebrew is not available) -# This role handles PostMarketOS, Alpine, and other musl-based distributions - -- name: Set apk_available fact - ansible.builtin.set_fact: - apk_available: "{{ is_musl }}" - -- name: Skip if apk not available - ansible.builtin.debug: - msg: "Skipping apk packages — musl libc not detected. System uses glibc (using Homebrew instead)." - when: not apk_available - -- name: Update apk package cache - become: true - ansible.builtin.command: apk update - when: apk_available - changed_when: true - register: apk_update_res - until: apk_update_res.rc == 0 - retries: 5 - delay: 5 - failed_when: false - -# Core packages needed for dotfiles configuration on Alpine -- name: Install core APK packages - become: true - community.general.apk: - name: - - bash - - fish - - zsh - - git - - git-lfs - - delta - - vim - - neovim - - tmux - - openssh-client - - curl - - wget - - jq - - yq - - ripgrep - - fd - - fzf - - starship - - bat - - exa - - direnv - - zoxide - - atuin - - shellcheck - - shfmt - - nodejs - - npm - - busybox-extras - - podman - - podman-compose - - crun - - iptables - state: present - update_cache: false # Handled by previous task - when: apk_available - -- name: Install Bitwarden CLI via npm - become: true - ansible.builtin.command: - cmd: npm install -g @bitwarden/cli - when: apk_available - changed_when: false - -- name: Enable podman service - become: true - ansible.builtin.systemd: - name: podman - state: started - enabled: true - when: apk_available - failed_when: false - -- name: Enable podman socket - become: true - ansible.builtin.systemd: - name: podman.socket - state: started - enabled: true - when: apk_available - failed_when: false - -- name: Add user to podman group for rootless containers - become: true - ansible.builtin.user: - name: "{{ ansible_facts['env']['USER'] | default(ansible_facts['user_id']) }}" - groups: podman - append: true - when: apk_available - register: user_podman_group - -- name: Restart user session to apply podman group (optional - login again to use rootless podman) - ansible.builtin.debug: - msg: "User added to podman group. Log out and back in to use rootless podman without sudo." - when: user_podman_group.changed diff --git a/roles/bitwarden/tasks/main.yml b/roles/bitwarden/tasks/main.yml deleted file mode 100644 index 0b0abfd5..00000000 --- a/roles/bitwarden/tasks/main.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- -- name: Check if bw CLI exists - ansible.builtin.command: which bw - register: bw_check - failed_when: false - changed_when: false - -- name: Resolve Bitwarden session - when: bw_check.rc == 0 - block: - # 1. BW_SESSION passed in environment (preferred — set by 'just apply') - - name: Use BW_SESSION from environment - ansible.builtin.set_fact: - bw_session: "{{ lookup('env', 'BW_SESSION') }}" - when: lookup('env', 'BW_SESSION') | length > 0 - - # 2. Cached session from previous run - - name: Check for cached session file - ansible.builtin.stat: - path: /tmp/bw_session - register: bw_session_file - when: bw_session is not defined - - - name: Read cached session - ansible.builtin.slurp: - src: /tmp/bw_session - register: bw_session_cached_raw - when: - - bw_session is not defined - - bw_session_file.stat is defined - - bw_session_file.stat.exists - - - name: Use cached session - ansible.builtin.set_fact: - bw_session: "{{ bw_session_cached_raw.content | b64decode | trim }}" - when: - - bw_session is not defined - - bw_session_cached_raw.content is defined - - # 3. Sync vault to ensure latest items are available. Hard timeout so a - # slow/unresponsive BW server can't block the entire play indefinitely. - # If sync fails, downstream tasks continue against the locally cached vault. - - name: Sync Bitwarden vault - ansible.builtin.command: timeout 15 bw sync - changed_when: false - failed_when: false - when: bw_session is defined - environment: - BW_SESSION: "{{ bw_session }}" - - # 4. Cache session for reuse - - name: Cache session to /tmp/bw_session - ansible.builtin.copy: - content: "{{ bw_session }}" - dest: /tmp/bw_session - mode: "0600" - when: bw_session is defined - no_log: true - - - name: Warn if no Bitwarden session found - ansible.builtin.debug: - msg: > - No BW_SESSION found. Run: export BW_SESSION=$(bw unlock --raw) - then re-run 'just apply'. Skipping secrets. - when: bw_session is not defined - - # 5. Verify the session actually unlocks the vault. Forwarded BW_SESSION - # tokens (e.g. via 'just apply-remote') can be silently rejected by the - # remote host's BW CLI because the encrypted vault state / KDF differs. - # Downstream roles gate on `bw_unlocked` so they degrade cleanly instead - # of issuing API calls with empty credentials. - - name: Check Bitwarden vault state - ansible.builtin.shell: | - set -o pipefail - bw status 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin).get('status','unknown'))" - args: - executable: /bin/bash - register: bw_status - changed_when: false - failed_when: false - environment: - BW_SESSION: "{{ bw_session | default('') }}" - -- name: Set Bitwarden unlocked fact - ansible.builtin.set_fact: - bw_unlocked: "{{ bw_check.rc == 0 and bw_session is defined and ((bw_status | default({})).stdout | default('') | trim) == 'unlocked' }}" - -- name: Warn if BW session present but vault is locked on this host - ansible.builtin.debug: - msg: >- - BW_SESSION is set but `bw status` reports - '{{ (bw_status | default({})).stdout | default('unknown') | trim }}'. - Skipping all secrets work on this host. Unlock the vault directly here - (`bw unlock`, write the session to /tmp/bw_session) to enable secrets. - when: - - bw_check.rc == 0 - - bw_session is defined - - not bw_unlocked diff --git a/roles/bluefin_common/tasks/main.yml b/roles/bluefin_common/tasks/main.yml deleted file mode 100644 index 62626e06..00000000 --- a/roles/bluefin_common/tasks/main.yml +++ /dev/null @@ -1,224 +0,0 @@ ---- -- name: Detect if running GNOME - ansible.builtin.command: gsettings get org.gnome.desktop.interface color-scheme - register: gsettings_check - failed_when: false - changed_when: false - check_mode: false - -- name: Detect if system is Bluefin - ansible.builtin.shell: grep -iq "Bluefin" /etc/os-release - register: bluefin_check - failed_when: false - changed_when: false - check_mode: false - -- name: Get user UID - ansible.builtin.command: id -u - register: user_uid_cmd - changed_when: false - check_mode: false - -- name: Set GNOME/Bluefin facts - ansible.builtin.set_fact: - is_gnome: "{{ gsettings_check.rc == 0 }}" - is_bluefin: "{{ bluefin_check.rc == 0 }}" - dbus_session_bus_address: "unix:path=/run/user/{{ user_uid_cmd.stdout | trim }}/bus" - -- name: Sync Bluefin customizations and branding from OCI image - when: is_gnome and (not is_bluefin or force_bluefin | default(false)) and is_desktop | bool - environment: - DBUS_SESSION_BUS_ADDRESS: "{{ dbus_session_bus_address }}" - block: - - name: Create temporary directory for Bluefin sync - ansible.builtin.tempfile: - state: directory - suffix: bluefin_oci - register: bluefin_temp - - - name: Pull and extract Bluefin common files and branding - ansible.builtin.shell: | - # Use --arch amd64 even on ARM to get the files (they are just text/data) - podman pull --arch amd64 ghcr.io/projectbluefin/common:latest - podman create --name temp-bluefin-sync ghcr.io/projectbluefin/common:latest - - # Structure for extraction - mkdir -p "$SYNC_DIR/dconf" "$SYNC_DIR/backgrounds" "$SYNC_DIR/icons" "$SYNC_DIR/ublue-os" - - # Extract gsettings/dconf - podman cp temp-bluefin-sync:system_files/bluefin/usr/share/glib-2.0/schemas/zz0-bluefin-modifications.gschema.override "$SYNC_DIR/" || true - podman cp temp-bluefin-sync:system_files/bluefin/etc/dconf/db/distro.d/. "$SYNC_DIR/dconf/" || true - - # Extract backgrounds - podman cp temp-bluefin-sync:system_files/bluefin/usr/share/backgrounds/bluefin/. "$SYNC_DIR/backgrounds/" || true - - # Extract icons (logos) - podman cp temp-bluefin-sync:system_files/bluefin/usr/share/icons/hicolor/. "$SYNC_DIR/icons/" || true - - # Extract ublue-os specific assets (logos, fastfetch, etc.) - podman cp temp-bluefin-sync:system_files/bluefin/usr/share/ublue-os/. "$SYNC_DIR/ublue-os/" || true - - podman rm temp-bluefin-sync - environment: - SYNC_DIR: "{{ bluefin_temp.path }}" - changed_when: true - - - name: Ensure local branding directories exist - ansible.builtin.file: - path: "{{ item }}" - state: directory - mode: "0755" - loop: - - "~/.local/share/backgrounds/bluefin" - - "~/.local/share/icons/hicolor" - - "~/.local/share/ublue-os" - - - name: Install Bluefin branding assets locally # noqa: risky-shell-pipe (the | is sed's delimiter, not a pipe) - ansible.builtin.shell: | - cp -r "$SYNC_DIR"/backgrounds/* ~/.local/share/backgrounds/bluefin/ - cp -r "$SYNC_DIR"/icons/* ~/.local/share/icons/hicolor/ - cp -r "$SYNC_DIR"/ublue-os/* ~/.local/share/ublue-os/ - - # Fix XML background paths to point to local home instead of /usr - find ~/.local/share/backgrounds/bluefin/ -name "*.xml" -type f -exec sed -i "s|/usr/share/backgrounds/bluefin/|$HOME/.local/share/backgrounds/bluefin/|g" {} + - environment: - SYNC_DIR: "{{ bluefin_temp.path }}" - changed_when: true - - - name: Apply extracted Bluefin settings via Python script - ansible.builtin.script: - cmd: bluefin-sync.py "{{ bluefin_temp.path }}" - changed_when: true - - - name: Clean up temporary directory - ansible.builtin.file: - path: "{{ bluefin_temp.path }}" - state: absent - -- name: Install Bluefin-standard fonts (Inter, JetBrains Mono) - when: is_gnome and (not is_bluefin or force_bluefin | default(false)) and is_desktop | bool - block: - - name: Ensure font directories exist - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/share/fonts/{{ item }}" - state: directory - mode: "0755" - loop: - - Inter - - JetBrainsMono - - - name: Download Inter font - ansible.builtin.get_url: - url: https://github.com/rsms/inter/releases/latest/download/Inter-4.1.zip - dest: /tmp/Inter.zip - mode: "0644" - - - name: Download JetBrains Mono font - ansible.builtin.get_url: - url: https://github.com/JetBrains/JetBrainsMono/releases/download/v2.304/JetBrainsMono-2.304.zip - dest: /tmp/JetBrainsMono.zip - mode: "0644" - - - name: Unarchive fonts - ansible.builtin.unarchive: - src: "/tmp/{{ item }}.zip" - dest: "{{ ansible_facts['env']['HOME'] }}/.local/share/fonts/{{ item }}" - remote_src: true - loop: - - Inter - - JetBrainsMono - - - name: Update font cache - ansible.builtin.command: fc-cache -f - changed_when: false - -- name: Ensure ~/.local/bin exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/bin" - state: directory - mode: "0755" - -- name: Deploy wallpaper rotation script - ansible.builtin.copy: - src: bluefin-wallpaper-rotate.py - dest: "{{ ansible_facts['env']['HOME'] }}/.local/bin/bluefin-wallpaper-rotate.py" - mode: "0755" - -- name: Install corral (rolling release built by hanthor/corral CI) - ansible.builtin.get_url: - url: "https://github.com/hanthor/corral/releases/download/binaries/corral-linux-{{ ansible_facts.machine | replace('x86_64', 'amd64') | replace('aarch64', 'arm64') }}" - checksum: "sha256:https://github.com/hanthor/corral/releases/download/binaries/SHA256SUMS" - dest: "{{ ansible_facts['env']['HOME'] }}/.local/bin/corral" - mode: "0755" - retries: 2 - delay: 5 - -- name: Remove retired tailvm artifacts (replaced by corral) - ansible.builtin.file: - path: "{{ item }}" - state: absent - loop: - - ~/.local/bin/tailvm-go - - ~/.local/bin/tailvm - - ~/.tailvm-completion.bash - - ~/.tailvm-completion.zsh - - ~/.config/fish/completions/tailvm.fish - -- name: Ensure systemd user directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user" - state: directory - mode: "0755" - -- name: Deploy wallpaper rotation service and timer - ansible.builtin.template: - src: "{{ item }}.j2" - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/{{ item }}" - mode: "0644" - loop: - - bluefin-wallpaper-rotate.service - - bluefin-wallpaper-rotate.timer - register: rotation_systemd - -- name: Reload systemd user daemon - ansible.builtin.systemd: - daemon_reload: true - scope: user - when: rotation_systemd.changed - -- name: Enable and start wallpaper rotation timer - ansible.builtin.systemd: - name: bluefin-wallpaper-rotate.timer - enabled: true - state: started - scope: user - -- name: Run wallpaper rotation once - ansible.builtin.command: "~/.local/bin/bluefin-wallpaper-rotate.py" - environment: - DBUS_SESSION_BUS_ADDRESS: "{{ dbus_session_bus_address }}" - changed_when: true - -- name: Install Bluefin-recommended GNOME extensions - when: is_gnome and (not is_bluefin or force_bluefin | default(false)) and is_desktop | bool - environment: - DBUS_SESSION_BUS_ADDRESS: "{{ dbus_session_bus_address }}" - ansible.builtin.script: - cmd: install-extensions.py - changed_when: false - ignore_errors: true - -- name: Optimize zRAM swappiness for desktop memory management - ansible.builtin.copy: - content: | - # Optimize zRAM swappiness for compressed RAM swap - vm.swappiness = 150 - dest: /etc/sysctl.d/99-zram.conf - mode: "0644" - become: true - register: zram_sysctl - -- name: Apply zRAM swappiness setting immediately - ansible.builtin.command: sysctl -p /etc/sysctl.d/99-zram.conf - become: true - when: zram_sysctl.changed diff --git a/roles/browser_fxa/README.md b/roles/browser_fxa/README.md deleted file mode 100644 index 3e913c85..00000000 --- a/roles/browser_fxa/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# hanthor.browser_fxa - -Sign in to **Firefox Accounts** for [Zen Browser](https://zen-browser.app) or Firefox using [PyFxA](https://github.com/mozilla/PyFxA), with optional [Bitwarden](https://bitwarden.com) credential and TOTP lookup. - -## What it does - -1. Detects the browser profile directory (`profiles.ini` → Install section default) -2. Skips gracefully if the browser is not installed -3. Creates a venv and installs PyFxA -4. Fetches credentials (email, password, TOTP) from Bitwarden (optional) -5. Calls `fxa_signin.py` to authenticate and write `signedInUser.json` -6. Idempotent — skips if already signed in and verified - -## Requirements - -- Python 3.8+ on the target host -- Bitwarden CLI (`bw`) in PATH if using the Bitwarden credential lookup -- `bw_session` variable or `BW_SESSION` set and vault unlocked - -## Role variables - -| Variable | Default | Description | -|---|---|---| -| `browser_fxa_browser` | `"zen"` | Browser to sign in to: `zen`, `firefox`, or `firefox-flatpak` | -| `fxa_bw_item` | `"accounts.firefox.com"` | Bitwarden search term for the FxA item | -| `fxa_bw_username` | `""` | Filter BW results by login.username (useful when multiple items match) | -| `fxa_venv_path` | `~/.local/venvs/fxa` | Path where the PyFxA venv is created | - -## Usage - -### Zen Browser (default) - -```yaml -- hosts: workstations - roles: - - role: hanthor.browser_fxa - vars: - fxa_bw_item: "accounts.firefox.com" - fxa_bw_username: "you@example.com" -``` - -### Firefox - -```yaml -- hosts: workstations - roles: - - role: hanthor.browser_fxa - vars: - browser_fxa_browser: firefox - fxa_bw_item: "accounts.firefox.com" - fxa_bw_username: "you@example.com" -``` - -### Without Bitwarden (supply credentials directly) - -Set `_fxa_email`, `_fxa_password`, and `_fxa_totp` as variables before including the role, -or supply them via `FXA_EMAIL` / `FXA_PASSWORD` / `FXA_TOTP` environment variables and call -`fxa_signin.py` directly. - -## `fxa_signin.py` standalone usage - -The script is also useful outside Ansible: - -```bash -FXA_EMAIL=you@example.com \ -FXA_PASSWORD=hunter2 \ -FXA_TOTP=123456 \ -BROWSER_TYPE=zen \ -python3 fxa_signin.py -``` - -`BROWSER_TYPE` accepts: `zen`, `firefox`, `firefox-flatpak`. -`PROFILE_DIR` overrides auto-detection entirely. - -## License - -GPL-3.0-or-later diff --git a/roles/browser_fxa/meta/main.yml b/roles/browser_fxa/meta/main.yml deleted file mode 100644 index fa1702f3..00000000 --- a/roles/browser_fxa/meta/main.yml +++ /dev/null @@ -1,26 +0,0 @@ -galaxy_info: - author: hanthor - description: >- - Sign in to Firefox Accounts (FxA) for Zen Browser or Firefox using PyFxA, - with optional Bitwarden credential lookup and TOTP support. - company: hanthor - license: GPL-3.0-or-later - min_ansible_version: "2.14" - platforms: - - name: Fedora - versions: all - - name: Ubuntu - versions: all - - name: Debian - versions: all - - name: EL - versions: all - galaxy_tags: - - firefox - - zen_browser - - fxa - - browser - - sync - - bitwarden - -dependencies: [] diff --git a/roles/browser_fxa/tasks/main.yml b/roles/browser_fxa/tasks/main.yml deleted file mode 100644 index cc9a2abe..00000000 --- a/roles/browser_fxa/tasks/main.yml +++ /dev/null @@ -1,170 +0,0 @@ ---- -- name: Check browser profile exists - ansible.builtin.stat: - path: >- - {{ - (browser_fxa_browser == 'zen') - | ternary( - ansible_facts['env']['HOME'] + '/.var/app/app.zen_browser.zen/.zen/profiles.ini', - ansible_facts['env']['HOME'] + '/.var/app/org.mozilla.firefox/.mozilla/firefox/profiles.ini' - ) - }} - register: _fxa_profiles_ini - -- name: Skip if browser is not installed - ansible.builtin.meta: end_role - when: not _fxa_profiles_ini.stat.exists - -- name: Ensure venv directory exists - ansible.builtin.file: - path: "{{ fxa_venv_path | dirname }}" - state: directory - mode: "0755" - -- name: Create venv for PyFxA - ansible.builtin.command: - cmd: python3 -m venv {{ fxa_venv_path }} - creates: "{{ fxa_venv_path }}/bin/python" - -- name: Install PyFxA into venv - ansible.builtin.pip: - name: PyFxA - state: present - virtualenv: "{{ fxa_venv_path }}" - -- name: Copy FxA sign-in script - ansible.builtin.copy: - src: fxa_signin.py - dest: "{{ ansible_facts['env']['HOME'] }}/.local/bin/fxa_signin.py" - mode: "0700" - -# ── Bitwarden credential fetch ──────────────────────────────────────────────── -# Gated on `bw_unlocked` (set by the bitwarden role, which also runs `bw sync`). - -- name: Fetch Firefox Account item from Bitwarden - when: bw_unlocked | default(false) - ansible.builtin.shell: | - bw list items --search "{{ fxa_bw_item }}" 2>/dev/null | python3 -c " - import json, sys - raw = sys.stdin.read().strip() - if not raw: - print('BW_EMPTY', file=sys.stderr) - sys.exit(1) - items = json.loads(raw) - if not items: - print('BW_EMPTY', file=sys.stderr) - sys.exit(1) - username_filter = '{{ fxa_bw_username }}' - # Try exact username match first, then fall back to first item - for item in items: - login_username = item.get('login', {}).get('username', '') - if not username_filter or login_username == username_filter: - print(json.dumps(item)) - sys.exit(0) - # Fallback: return first item with a password if username didn't match - for item in items: - if item.get('login', {}).get('password'): - print(json.dumps(item), file=sys.stderr) - print('BW_USERNAME_MISMATCH:' + ','.join( - i.get('login',{}).get('username','?') for i in items - ), file=sys.stderr) - sys.exit(2) - sys.exit(1) - " - register: _fxa_item_raw - no_log: true - failed_when: false - changed_when: false - environment: - PATH: "{{ ansible_env_path }}" - BW_SESSION: "{{ bw_session | default('') }}" - -- name: Report BW lookup result (diagnostic) - ansible.builtin.debug: - msg: >- - {% if _fxa_item_raw.rc == 0 %} - Found '{{ fxa_bw_item }}' in Bitwarden. - {% elif 'BW_USERNAME_MISMATCH' in (_fxa_item_raw.stderr | default('')) %} - Found '{{ fxa_bw_item }}' but username did not match '{{ fxa_bw_username }}'. - Stored usernames: {{ (_fxa_item_raw.stderr | default('')) | regex_findall('BW_USERNAME_MISMATCH:(.+)') | first | default('unknown') }}. - Set fxa_bw_username in group_vars/all.yml to one of those, or clear it to use the first match. - {% elif 'BW_EMPTY' in (_fxa_item_raw.stderr | default('')) %} - No Bitwarden item found matching '{{ fxa_bw_item }}'. Create a BW login item with that name to enable FxA sign-in. - {% else %} - BW lookup failed (vault locked or bw not installed). rc={{ _fxa_item_raw.rc | default('?') }} - {% endif %} - when: _fxa_item_raw.rc is defined and _fxa_item_raw.rc != 0 - -- name: Extract Firefox Account item ID for TOTP lookup - ansible.builtin.set_fact: - _fxa_item_id: "{{ (_fxa_item_raw.stdout | from_json).id }}" - no_log: true - when: - - _fxa_item_raw.rc is defined and _fxa_item_raw.rc == 0 - - (_fxa_item_raw.stdout | default('')) | length > 0 - -- name: Fetch Firefox Account TOTP from Bitwarden - ansible.builtin.shell: bw get totp "{{ _fxa_item_id }}" 2>&1 - register: _fxa_totp_raw - no_log: true - failed_when: false - changed_when: false - when: - - bw_unlocked | default(false) - - _fxa_item_id is defined - environment: - PATH: "{{ ansible_env_path }}" - BW_SESSION: "{{ bw_session | default('') }}" - -- name: Warn if Firefox Account credentials not found in Bitwarden - ansible.builtin.debug: - msg: >- - Could not fetch Firefox Account credentials from Bitwarden - (item '{{ fxa_bw_item }}' for user '{{ fxa_bw_username }}' not found or vault locked) — skipping FxA sign-in. - when: >- - _fxa_item_raw.rc is not defined - or _fxa_item_raw.rc != 0 - or _fxa_item_id is not defined - -- name: Parse Firefox Account credentials - ansible.builtin.set_fact: - _fxa_email: "{{ (_fxa_item_raw.stdout | from_json).login.username | default('') }}" - _fxa_password: "{{ (_fxa_item_raw.stdout | from_json).login.password | default('') }}" - _fxa_totp: "{{ _fxa_totp_raw.stdout | default('') | regex_replace('[^0-9]', '') }}" - no_log: true - when: - - _fxa_item_id is defined - # TOTP is optional — proceed even if not stored in BW (accounts without 2FA) - -# ── Sign-in ─────────────────────────────────────────────────────────────────── -# Script stdout/stderr is safe to surface (no credentials are logged) so we -# intentionally drop no_log here — silent failure is what made this hard to -# debug previously. Sign-in failure does not abort the play; sync can be -# completed by signing in manually in the browser. - -- name: Sign in to Firefox Accounts - ansible.builtin.command: > - {{ fxa_venv_path }}/bin/python {{ ansible_facts['env']['HOME'] }}/.local/bin/fxa_signin.py - register: _fxa_signin_result - changed_when: "'Wrote signedInUser.json' in _fxa_signin_result.stdout" - failed_when: false - when: - - _fxa_email is defined and (_fxa_email | length) > 0 - - _fxa_password is defined and (_fxa_password | length) > 0 - environment: - FXA_EMAIL: "{{ _fxa_email | default('') }}" - FXA_PASSWORD: "{{ _fxa_password | default('') }}" - FXA_TOTP: "{{ _fxa_totp | default('') }}" - BROWSER_TYPE: "{{ browser_fxa_browser }}" - -- name: Firefox Accounts sign-in result - ansible.builtin.debug: - msg: | - rc={{ _fxa_signin_result.rc | default('skipped') }} - stdout: {{ _fxa_signin_result.stdout | default('') }} - stderr: {{ _fxa_signin_result.stderr | default('') }} - {% if (_fxa_signin_result.rc | default(0)) != 0 %} - Sign-in did not complete. Open Zen/Firefox and sign in manually once; - the role will skip on subsequent runs. - {% endif %} - when: _fxa_signin_result is defined and _fxa_signin_result.skipped is not defined diff --git a/roles/bst_dashboard/handlers/main.yml b/roles/bst_dashboard/handlers/main.yml deleted file mode 100644 index 1397a097..00000000 --- a/roles/bst_dashboard/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Reload systemd user daemon - ansible.builtin.systemd: - daemon_reload: true - scope: user diff --git a/roles/bst_dashboard/tasks/main.yml b/roles/bst_dashboard/tasks/main.yml deleted file mode 100644 index 0cfe2c42..00000000 --- a/roles/bst_dashboard/tasks/main.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -- name: Set dashboard directory based on hostname - ansible.builtin.set_fact: - bst_dashboard_dir: "/var/home/james/dev/{{ 'kde-linux' if inventory_hostname == 'karnataka' else 'dakota' }}" - bst_dashboard_image_name: "{{ 'aurora' if inventory_hostname == 'karnataka' else 'dakota' }}" - -- name: Check if dashboard directory exists - ansible.builtin.stat: - path: "{{ bst_dashboard_dir }}" - register: _bst_dir - -- name: Deploy bst-dashboard service - ansible.builtin.template: - src: bst-dashboard.service.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/bst-dashboard.service" - mode: "0644" - when: _bst_dir.stat.exists - notify: Reload systemd user daemon - -- name: Enable and start bst-dashboard - ansible.builtin.systemd: - name: bst-dashboard.service - scope: user - enabled: true - state: started - when: _bst_dir.stat.exists - -- name: Enable linger for user - ansible.builtin.command: loginctl enable-linger {{ ansible_facts['env']['USER'] }} - become: true - changed_when: false - when: _bst_dir.stat.exists diff --git a/roles/flatpak/tasks/main.yml b/roles/flatpak/tasks/main.yml deleted file mode 100644 index 9013b772..00000000 --- a/roles/flatpak/tasks/main.yml +++ /dev/null @@ -1,87 +0,0 @@ ---- -- name: Fix broken flatpak symlink if present - ansible.builtin.shell: | - if [ -L /var/lib/flatpak ] && [ ! -e /var/lib/flatpak ]; then - rm /var/lib/flatpak - mkdir -p /var/lib/flatpak - fi - become: true - changed_when: false - -- name: Add flathub remote (system) - ansible.builtin.command: flatpak remote-add --if-not-exists --system {{ item.name }} {{ item.url }} - loop: "{{ flatpak_remotes }}" - become: true - changed_when: false - -- name: Remove unwanted flatpaks if present - ansible.builtin.shell: | - REMOVE="" - for pkg in org.mozilla.firefox org.mozilla.Thunderbird; do - flatpak info --system "$pkg" >/dev/null 2>&1 && REMOVE="$REMOVE $pkg" - done - if [ -n "$REMOVE" ]; then - flatpak uninstall --system -y $REMOVE - echo "removed:$REMOVE" - fi - become: true - register: flatpak_remove - changed_when: "'removed:' in flatpak_remove.stdout" - failed_when: false - -- name: Get list of installed system flatpaks - ansible.builtin.command: flatpak list --system --columns=application --app - register: installed_flatpaks - changed_when: false - become: true - failed_when: false - -- name: Set missing flatpaks fact - ansible.builtin.set_fact: - missing_flatpaks: "{{ system_flatpaks | reject('in', installed_flatpaks.stdout_lines) | list }}" - -# Fast path: try all missing apps in one batch command -- name: Install missing system flatpaks (batch) - ansible.builtin.shell: | - flatpak install --system -y flathub {{ missing_flatpaks | join(' ') }} 2>&1 - become: true - register: flatpak_batch - when: missing_flatpaks | length > 0 - changed_when: flatpak_batch.rc == 0 - failed_when: false - -# Slow path: re-run individually only for the apps that failed (keeps the common case fast) -- name: Install flatpaks that failed in batch (individually, tolerating bad IDs) - ansible.builtin.command: flatpak install --system -y flathub {{ item }} - loop: "{{ missing_flatpaks }}" - become: true - register: flatpak_individual - when: missing_flatpaks | length > 0 and flatpak_batch.rc | default(0) != 0 - changed_when: flatpak_individual.rc == 0 - failed_when: false - -- name: Update all flatpaks - ansible.builtin.command: flatpak update --system -y - become: true - when: upgrade | default(false) | bool - changed_when: true - ignore_errors: true - -- name: Ensure ~/.local/bin exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/bin" - state: directory - mode: "0755" - -- name: Create chromium wrapper scripts pointing to flatpak Chromium - ansible.builtin.copy: - dest: "{{ ansible_facts['env']['HOME'] }}/.local/bin/{{ item }}" - content: | - #!/usr/bin/env bash - exec flatpak run org.chromium.Chromium "$@" - mode: "0755" - loop: - - google-chrome - - google-chrome-stable - - chromium-browser - - chromium diff --git a/roles/fleet_facts/tasks/main.yml b/roles/fleet_facts/tasks/main.yml deleted file mode 100644 index 62de9379..00000000 --- a/roles/fleet_facts/tasks/main.yml +++ /dev/null @@ -1,118 +0,0 @@ ---- -# ── Public device facts for reilly.asia/infra ────────────────────────── -# Each host writes facts/.json on the fleet-facts branch through the -# GitHub contents API (the gh CLI's existing login, so it works in the -# no-secrets daily timer too). Only what's safe to publish: OS, hardware -# model, CPU/GPU, memory, uptime, last apply. No addresses, serials or MACs. - -- name: Check gh is available and logged in - ansible.builtin.command: gh auth status - register: fleet_facts_gh - changed_when: false - failed_when: false - environment: - PATH: "{{ ansible_env_path }}" - -- name: Warn that facts cannot be published without gh auth - ansible.builtin.debug: - msg: | - fleet_facts: gh is not authenticated on {{ inventory_hostname }}, so no device - facts were published. Run `just apply` once with Bitwarden unlocked (the daily - no-secrets timer cannot authenticate gh). - when: - - fleet_facts_publish | bool - - fleet_facts_gh.rc != 0 - -- name: Publish device facts - when: - - fleet_facts_publish | bool - - fleet_facts_gh.rc == 0 - block: - # DMI model names are often placeholders on self-built boards, and ARM - # boards (the Pi) and phones don't have DMI at all. - - name: Read device-tree model (ARM boards) - ansible.builtin.slurp: - src: /sys/firmware/devicetree/base/model - register: fleet_facts_dt - failed_when: false - - - name: Read Android device model (Termux) - ansible.builtin.command: getprop ro.product.model - register: fleet_facts_android - changed_when: false - failed_when: false - when: is_termux | default(false) - - - name: Detect GPUs - ansible.builtin.shell: set -o pipefail; lspci -mm 2>/dev/null | grep -iE '"(VGA|3D|Display)' | cut -d'"' -f6,8 | tr '"' ' ' - args: - executable: /bin/bash - register: fleet_facts_gpu - changed_when: false - failed_when: false - - - name: Read last apply outcome - ansible.builtin.slurp: - src: "{{ ansible_facts['env']['HOME'] }}/.cache/dotfiles/last-apply.json" - register: fleet_facts_last_apply - failed_when: false - - - name: Build device facts - vars: - _placeholder_models: ["", "NA", "None", "System Product Name", "To Be Filled By O.E.M.", "Default string", "Not Specified"] - _dmi_model: "{{ (ansible_facts['product_name'] | default('') | trim) if (ansible_facts['product_name'] | default('') | trim) not in _placeholder_models else '' }}" - _dt_model: "{{ (fleet_facts_dt.content | default('') | b64decode | replace('\u0000', '') | trim) if fleet_facts_dt.content is defined else '' }}" - _android_model: "{{ fleet_facts_android.stdout | default('') | trim }}" - _vendor: "{{ ansible_facts['system_vendor'] | default('') | trim }}" - _cpu_list: "{{ ansible_facts['processor'] | default([]) }}" - _cpu: "{{ (_cpu_list | select('search', '[A-Za-z]{3,}') | reject('match', '^(GenuineIntel|AuthenticAMD|[0-9]+)$') | first | default('')) }}" - _last: "{{ (fleet_facts_last_apply.content | b64decode | from_json) if fleet_facts_last_apply.content is defined else {} }}" - ansible.builtin.set_fact: - fleet_facts_doc: - schema: 1 - host: "{{ inventory_hostname }}" - updated: "{{ ansible_facts['date_time']['iso8601'] }}" - groups: "{{ group_names | reject('equalto', 'ungrouped') | list }}" - os: - name: "{{ ansible_facts['distribution'] | default('') }}" - version: "{{ ansible_facts['distribution_version'] | default('') }}" - codename: "{{ ansible_facts['distribution_release'] | default('') }}" - kernel: "{{ ansible_facts['kernel'] | default('') }}" - hardware: - vendor: "{{ '' if _vendor in _placeholder_models else _vendor }}" - model: "{{ _android_model or _dt_model or _dmi_model }}" - form_factor: "{{ ansible_facts['form_factor'] | default('') }}" - arch: "{{ ansible_facts['architecture'] | default('') }}" - cpu: "{{ _cpu | regex_replace('\\s+', ' ') | trim }}" - cpu_threads: "{{ ansible_facts['processor_vcpus'] | default(ansible_facts['processor_count'] | default(0)) | int }}" - # lspci prints bare 'Device 1111' for unnamed virtual adapters (EC2, QEMU). - gpus: "{{ fleet_facts_gpu.stdout_lines | default([]) | map('trim') | select | reject('match', '^(\\S+ )?Device [0-9a-fA-F]{4}$') | list }}" - memory_gb: "{{ ((ansible_facts['memtotal_mb'] | default(0) | int) / 1024) | round(1) }}" - virtualization: "{{ ansible_facts['virtualization_type'] if ansible_facts['virtualization_role'] | default('') == 'guest' else '' }}" - uptime_days: "{{ ((ansible_facts['uptime_seconds'] | default(0) | int) / 86400) | round(1) }}" - last_apply: - at: "{{ _last.timestamp | default(_last.at | default('')) }}" - ok: "{{ (_last.rc | default(_last.exit_code | default(1)) | int) == 0 if _last else '' }}" - - - name: Publish facts to the fleet-facts branch - ansible.builtin.shell: | - set -euo pipefail - path="facts/{{ inventory_hostname }}.json" - sha=$(gh api "repos/{{ fleet_facts_repo }}/contents/$path?ref={{ fleet_facts_branch }}" --jq .sha 2>/dev/null || true) - body=$(jq -n --arg c "$FACTS_B64" --arg b "{{ fleet_facts_branch }}" --arg s "$sha" \ - --arg m "facts: {{ inventory_hostname }}" \ - '{message:$m, content:$c, branch:$b} + (if $s == "" then {} else {sha:$s} end)') - gh api -X PUT "repos/{{ fleet_facts_repo }}/contents/$path" --input - <<<"$body" --jq .commit.sha - args: - executable: /bin/bash - environment: - PATH: "{{ ansible_env_path }}" - FACTS_B64: "{{ fleet_facts_doc | to_nice_json | b64encode }}" - register: fleet_facts_put - changed_when: fleet_facts_put.rc == 0 - failed_when: false - - - name: Report publish failure - ansible.builtin.debug: - msg: "fleet_facts: could not publish ({{ fleet_facts_put.stderr | default('') | truncate(200) }})" - when: fleet_facts_put.rc | default(0) != 0 diff --git a/roles/gnome/handlers/main.yml b/roles/gnome/handlers/main.yml deleted file mode 100644 index b9a96cd9..00000000 --- a/roles/gnome/handlers/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- name: Load GNOME keybindings - ansible.builtin.shell: dconf load /org/gnome/settings-daemon/plugins/media-keys/ < {{ ansible_facts['env']['HOME'] }}/.config/dconf/gnome-keybindings.ini diff --git a/roles/gnome/tasks/main.yml b/roles/gnome/tasks/main.yml deleted file mode 100644 index 60cf2607..00000000 --- a/roles/gnome/tasks/main.yml +++ /dev/null @@ -1,159 +0,0 @@ ---- -# ── GNOME Keybindings ───────────────────────────────────────────── -- name: Ensure dconf config directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/dconf" - state: directory - mode: "0755" - -- name: Copy GNOME keybindings file - ansible.builtin.copy: - src: gnome-keybindings.ini - dest: "{{ ansible_facts['env']['HOME'] }}/.config/dconf/gnome-keybindings.ini" - mode: "0644" - notify: Load GNOME keybindings - -# ── Ptyxis: default profile → brew fish ────────────────────────── -- name: Get Ptyxis default profile UUID - ansible.builtin.command: gsettings get org.gnome.Ptyxis default-profile-uuid - register: ptyxis_uuid - changed_when: false - failed_when: false - when: is_desktop | bool - -- name: Set Ptyxis default profile to use brew fish and Nerd Font - ansible.builtin.shell: | - UUID=$(gsettings get org.gnome.Ptyxis default-profile-uuid | tr -d "'") - CURRENT_CMD=$(gsettings get "org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${UUID}/" custom-command 2>/dev/null | tr -d "'") - CURRENT_FONT=$(gsettings get org.gnome.Ptyxis font-name 2>/dev/null | tr -d "'") - if [ "$CURRENT_CMD" != "{{ brew_prefix }}/bin/fish" ] || [ "$CURRENT_FONT" != "JetBrainsMono Nerd Font 12" ]; then - gsettings set "org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${UUID}/" use-custom-command true - gsettings set "org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${UUID}/" custom-command '{{ brew_prefix }}/bin/fish' - gsettings set org.gnome.Ptyxis use-system-font false - gsettings set org.gnome.Ptyxis font-name 'JetBrainsMono Nerd Font 12' - echo changed - fi - register: ptyxis_set - when: is_desktop | bool and ptyxis_uuid.rc is defined and ptyxis_uuid.rc == 0 - changed_when: "'changed' in ptyxis_set.stdout" - ignore_errors: true - -# ── User avatar ─────────────────────────────────────────────────── -- name: Download user avatar from reilly.asia - ansible.builtin.get_url: - url: https://reilly.asia/profile.png - dest: /var/lib/AccountsService/icons/james - mode: "0644" - force: true - become: true - when: is_desktop | bool - -- name: Check if gdm user exists - ansible.builtin.getent: - database: passwd - key: gdm - register: gdm_user - failed_when: false - become: true - when: is_desktop | bool - -- name: Copy avatar to GDM cache directory - ansible.builtin.copy: - src: /var/lib/AccountsService/icons/james - dest: /var/lib/gdm3/.cache/ - mode: "0644" - force: true - become: true - when: - - is_desktop | bool - - gdm_user.ansible_facts is defined - -- name: Set avatar ownership for GDM - ansible.builtin.file: - path: /var/lib/gdm3/.cache/james - owner: gdm - group: gdm - mode: "0644" - become: true - when: - - is_desktop | bool - - gdm_user.ansible_facts is defined - -- name: Set user avatar in AccountsService - community.general.ini_file: - path: /var/lib/AccountsService/users/james - section: User - option: Icon - value: /var/lib/AccountsService/icons/james - mode: "0644" - create: true - become: true - when: is_desktop | bool - -# ── GNOME Extensions — omnidirectional sync ─────────────────────── -# User extensions: copied from repo; system extensions: RPM-managed (Bluefin) - -- name: Ensure user GNOME extensions directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/share/gnome-shell/extensions" - state: directory - mode: "0755" - when: is_desktop | bool - -- name: Deploy user GNOME extensions - ansible.builtin.copy: - src: "extensions/{{ item }}/" - dest: "{{ ansible_facts['env']['HOME'] }}/.local/share/gnome-shell/extensions/{{ item }}/" - mode: "0644" - directory_mode: "0755" - loop: - - copyous@boerdereinar.dev - - papershell@lalovene.github.com - - paperwm@paperwm.github.com - when: is_desktop | bool - -- name: Load GNOME extension settings via dconf - ansible.builtin.shell: | - dconf load /org/gnome/shell/extensions/ < {{ role_path }}/files/gnome-extensions-dconf.dump - changed_when: false - failed_when: false - when: is_desktop | bool - register: dconf_load - -- name: Warn if dconf load failed (no D-Bus session?) - ansible.builtin.debug: - msg: >- - dconf load for GNOME extensions failed (rc={{ dconf_load.rc }}). - This is normal if SSH'd without a D-Bus session. - Extension settings will apply on next login. - when: - - is_desktop | bool - - dconf_load.rc | default(0) != 0 - -- name: Set enabled GNOME extensions - community.general.dconf: - key: /org/gnome/shell/enabled-extensions - value: "{{ gnome_enabled_extensions }}" - state: present - when: is_desktop | bool - failed_when: false - -# ── GNOME Remote Desktop (RDP) ──────────────────────────────────── -- name: Enable GNOME RDP protocol - ansible.builtin.shell: | - if ! grdctl rdp status 2>/dev/null | grep -q "Status: enabled"; then - grdctl rdp enable && echo changed - fi - register: grdctl_enable - changed_when: "'changed' in grdctl_enable.stdout" - ignore_errors: true - when: is_desktop | bool - -- name: Enable GNOME Remote Desktop service - ansible.builtin.systemd: - name: gnome-remote-desktop.service - state: started - enabled: true - scope: user - when: is_desktop | bool - ignore_errors: true diff --git a/roles/hive_ops/tasks/main.yml b/roles/hive_ops/tasks/main.yml deleted file mode 100644 index 46336f9d..00000000 --- a/roles/hive_ops/tasks/main.yml +++ /dev/null @@ -1,119 +0,0 @@ ---- -# ── Hive operations: rotation, watchdog, peak/off-peak scheduling ────── -# Drives the tuna-os Hive on the AWS Talos cluster (see the tunaos-hive-checkin -# skill). These were previously user units on telengana talking to a local k3s; -# they now use plain kubectl against the AWS cluster's kubeconfig. -# -# Off by default — set `hive_ops_enabled: true` on exactly one host. - -- name: Deploy hive operations - when: hive_ops_enabled | bool - block: - - name: Ensure ~/.local/bin exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/bin" - state: directory - mode: "0755" - - - name: Deploy hive helper scripts - ansible.builtin.copy: - src: "bin/{{ item }}" - dest: "{{ ansible_facts['env']['HOME'] }}/.local/bin/{{ item }}" - mode: "0755" - loop: - - hive-peak.sh - - hive-rotate.sh - - hive-tiers.sh - - - name: Ensure user systemd directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user" - state: directory - mode: "0755" - - - name: Deploy hive systemd units - ansible.builtin.copy: - src: systemd/ - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/" - mode: "0644" - register: hive_units - - # The units run non-interactively, so they can't rely on shell rc files. - # KUBECONFIG has no default outside the shell, and the systemd *user* - # manager's PATH is only /usr/local/bin:/usr/bin:/snap/bin — which does not - # include Homebrew, where kubectl actually lives. Without this the scripts - # fail opaquely ("no hive pod found") because kubectl isn't found at all. - - name: Ensure drop-in directories for hive services - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/{{ item }}.service.d" - state: directory - mode: "0755" - loop: "{{ hive_ops_timers }}" - - - name: Pin KUBECONFIG and PATH for hive services - ansible.builtin.copy: - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/{{ item }}.service.d/kubeconfig.conf" - mode: "0644" - content: | - [Service] - Environment=KUBECONFIG={{ hive_ops_kubeconfig }} - Environment=PATH={{ hive_ops_path }} - loop: "{{ hive_ops_timers }}" - register: hive_dropins - - - name: Reload user systemd - ansible.builtin.systemd: - scope: user - daemon_reload: true - when: hive_units.changed or hive_dropins.changed - - - name: Enable and start hive timers - ansible.builtin.systemd: - name: "{{ item }}.timer" - scope: user - enabled: true - state: started - loop: "{{ hive_ops_timers }}" - -# ── Retire on hosts that are no longer opted in ──────────────────────── -# Flipping hive_ops_enabled off must actually stop the timers, or the old host -# keeps rotating/pausing agents alongside the in-cluster CronJobs. -- name: Check for leftover hive timers - ansible.builtin.stat: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/hive-watchdog.timer" - register: hive_ops_leftover - when: not (hive_ops_enabled | bool) - -- name: Remove hive operations from this host - when: - - not (hive_ops_enabled | bool) - - hive_ops_leftover.stat.exists | default(false) - block: - - name: Stop and disable hive timers - ansible.builtin.systemd: - name: "{{ item }}.timer" - scope: user - enabled: false - state: stopped - loop: "{{ hive_ops_timers }}" - failed_when: false - - - name: Remove hive units and drop-ins - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/{{ item.0 }}{{ item.1 }}" - state: absent - loop: "{{ hive_ops_timers | product(['.timer', '.service', '.service.d']) | list }}" - - - name: Remove hive helper scripts - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.local/bin/{{ item }}" - state: absent - loop: - - hive-peak.sh - - hive-rotate.sh - - hive-tiers.sh - - - name: Reload user systemd after removing hive units - ansible.builtin.systemd: - scope: user - daemon_reload: true diff --git a/roles/homebrew/handlers/main.yml b/roles/homebrew/handlers/main.yml deleted file mode 100644 index 1385adc7..00000000 --- a/roles/homebrew/handlers/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# Handlers for homebrew role (Run brew bundle moved to tasks) diff --git a/roles/homebrew/tasks/main.yml b/roles/homebrew/tasks/main.yml deleted file mode 100644 index 27bfaa21..00000000 --- a/roles/homebrew/tasks/main.yml +++ /dev/null @@ -1,183 +0,0 @@ ---- -# Homebrew requires glibc — skip on musl-based systems (Alpine, postmarketOS) -# and on Termux (Android/bionic libc, no glibc at all). -- name: Set brew_supported fact - ansible.builtin.set_fact: - brew_supported: "{{ not (is_musl or is_termux) }}" - -- name: Warn that Homebrew is not supported on this system - ansible.builtin.debug: - msg: >- - Skipping Homebrew — - {{ 'Termux (Android/bionic libc) detected. Use pkg instead.' if is_termux - else 'musl libc detected (postmarketOS/Alpine). Install packages via apk instead.' }} - when: not brew_supported - -- name: Check if /home/linuxbrew exists - ansible.builtin.stat: - path: /home/linuxbrew - register: linuxbrew_dir - when: brew_supported - -# Some base VM/cloud images pre-create /home/linuxbrew owned by a default -# image user (e.g. Google's AVF Debian image ships it owned by "droid"), -# which silently breaks every `brew` fetch with no clear error pointing here. -- name: Ensure /home/linuxbrew is owned by the current user - ansible.builtin.file: - path: /home/linuxbrew - owner: "{{ ansible_facts['env']['USER'] | default(ansible_facts['user_id']) }}" - recurse: true - become: true - when: brew_supported and linuxbrew_dir.stat.exists and linuxbrew_dir.stat.pw_name != (ansible_facts['env']['USER'] | default(ansible_facts['user_id'])) - -- name: Check for a C compiler - ansible.builtin.command: which cc gcc - register: brew_cc_check - changed_when: false - failed_when: false - when: brew_supported - -# Homebrew-on-Linux needs a real toolchain for any formula without a -# prebuilt bottle for this arch — Bluefin/ostree hosts already ship one, -# but a bare Debian/Ubuntu host (e.g. mumbai, the phone's VM) doesn't. -- name: Install build tools (Debian/Ubuntu, no compiler present) - ansible.builtin.apt: - name: build-essential - state: present - update_cache: true - become: true - when: >- - brew_supported and brew_cc_check.rc != 0 - and ansible_facts['os_family'] | default('') == 'Debian' - -- name: Check if Homebrew is installed - ansible.builtin.stat: - path: /home/linuxbrew/.linuxbrew/bin/brew - register: brew_installed - when: brew_supported - -- name: Install Homebrew - ansible.builtin.shell: | - NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - args: - creates: /home/linuxbrew/.linuxbrew/bin/brew - when: brew_supported and not brew_installed.stat.exists - -- name: Deploy Brewfile - ansible.builtin.template: - src: Brewfile.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/Brewfile" - mode: "0644" - when: brew_supported - -# Newer Homebrew refuses to load formulae from third-party taps until -# explicitly trusted — `brew bundle` fails atomically on the whole -# Brewfile the first time it hits an untrusted tap. This includes taps -# that are never explicitly `tap`-ped in the Brewfile at all, only -# referenced inline via a tap-qualified package name (e.g. -# "charmbracelet/tap/crush") — so the trust list has to be derived from -# every package name, not just core_taps/desktop_taps. -- name: Derive full list of Homebrew taps (explicit + implied by tap-qualified package names) - ansible.builtin.set_fact: - brew_all_taps: >- - {{ - ((core_taps | default([])) + (desktop_taps | default([])) + - (core_brews | default([])) + (core_tap_brews | default([])) + - (desktop_brews | default([])) + (extra_brews | default([])) + - (desktop_casks | default([]))) - | select('match', '^[^/]+/[^/]+/.+$') - | map('regex_replace', '^([^/]+/[^/]+)/.+$', '\1') - | list - | unique - }} - when: brew_supported - -- name: Refuse blocked Homebrew packages (brew_blocked) in any install list - ansible.builtin.assert: - that: blocked_requested | length == 0 - fail_msg: "{{ blocked_requested | join(', ') }} is in brew_blocked (group_vars/all.yml) — remove it from the install lists" - quiet: true - vars: - blocked_requested: >- - {{ - ((core_brews | default([])) + (core_tap_brews | default([])) + - (desktop_brews | default([])) + (extra_brews | default([])) + - (desktop_casks | default([]))) - | map('regex_replace', '^.*/', '') - | intersect(brew_blocked | default([])) - }} - when: brew_supported - -- name: Tap our own Homebrew taps - ansible.builtin.command: - cmd: /home/linuxbrew/.linuxbrew/bin/brew tap {{ item }} - environment: - PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_facts['env']['PATH'] }}" - loop: "{{ brew_all_taps | default([]) }}" - when: brew_supported - changed_when: false - -- name: Trust our own Homebrew taps - ansible.builtin.command: - cmd: /home/linuxbrew/.linuxbrew/bin/brew trust {{ item }} - environment: - PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_facts['env']['PATH'] }}" - loop: "{{ brew_all_taps | default([]) }}" - when: brew_supported - register: brew_trust - changed_when: "'now trusted' in (brew_trust.stdout | default(''))" - failed_when: false - -- name: Run brew bundle - ansible.builtin.command: - cmd: /home/linuxbrew/.linuxbrew/bin/brew bundle install --file={{ ansible_facts['env']['HOME'] }}/Brewfile --no-upgrade - environment: - PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_facts['env']['PATH'] }}" - GIT_CONFIG_COUNT: "1" - GIT_CONFIG_KEY_0: "credential.helper" - GIT_CONFIG_VALUE_0: "" - HOMEBREW_NO_AUTO_UPDATE: "1" - HOMEBREW_NO_ENV_HINTS: "1" - register: brew_bundle - when: brew_supported - changed_when: "'Installing ' in (brew_bundle.stdout | default('')) or 'Tapping ' in (brew_bundle.stdout | default(''))" - failed_when: false - -- name: Warn if brew bundle reported errors - ansible.builtin.debug: - msg: "brew bundle returned {{ brew_bundle.rc }} — last stderr: {{ brew_bundle.stderr_lines[-10:] | default([]) }}" - when: brew_supported and brew_bundle.rc | default(0) != 0 - -- name: Uninstall blocked Homebrew packages - ansible.builtin.command: - cmd: /home/linuxbrew/.linuxbrew/bin/brew uninstall --formula {{ item }} - environment: - PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_facts['env']['PATH'] }}" - HOMEBREW_NO_AUTO_UPDATE: "1" - HOMEBREW_NO_ENV_HINTS: "1" - loop: "{{ brew_blocked | default([]) }}" - register: brew_blocked_uninstall - when: brew_supported - # rc 1 + "No such keg" = already absent. - changed_when: brew_blocked_uninstall.rc == 0 - failed_when: >- - brew_blocked_uninstall.rc != 0 and - 'No such keg' not in (brew_blocked_uninstall.stderr | default('')) - -- name: Upgrade all Homebrew packages - ansible.builtin.command: /home/linuxbrew/.linuxbrew/bin/brew upgrade - environment: - PATH: "/home/linuxbrew/.linuxbrew/bin:{{ ansible_facts['env']['PATH'] }}" - HOMEBREW_NO_AUTO_UPDATE: "1" - HOMEBREW_NO_ENV_HINTS: "1" - register: brew_upgrade - when: brew_supported and (upgrade | default(false) | bool) - changed_when: "'Upgrading ' in (brew_upgrade.stdout | default(''))" - failed_when: false - -- name: Ensure packaging is installed in brew Python (required by Ansible systemd module) - ansible.builtin.command: > - /home/linuxbrew/.linuxbrew/bin/python3 -m pip install packaging --break-system-packages --quiet - when: brew_supported - changed_when: false - failed_when: false diff --git a/roles/homebrew/templates/Brewfile.j2 b/roles/homebrew/templates/Brewfile.j2 deleted file mode 100644 index 3fcff72c..00000000 --- a/roles/homebrew/templates/Brewfile.j2 +++ /dev/null @@ -1,37 +0,0 @@ -# Core packages -{% for tap in core_taps %} -tap "{{ tap }}" -{% endfor %} - -{% for pkg in core_brews %} -brew "{{ pkg }}" -{% endfor %} - -{% for pkg in core_tap_brews %} -brew "{{ pkg }}" -{% endfor %} - -{% if is_desktop | bool %} -# Desktop packages -{% for tap in desktop_taps %} -tap "{{ tap }}" -{% endfor %} - -{% for pkg in desktop_brews %} -brew "{{ pkg }}" -{% endfor %} - -{% for pkg in extra_brews | default([]) %} -brew "{{ pkg }}" -{% endfor %} - -{% for pkg in desktop_casks | default([]) %} -cask "{{ pkg }}" -{% endfor %} - -{% if ansible_facts['os_family'] == 'Darwin' %} -{% for ext in desktop_vscode_extensions %} -vscode "{{ ext }}" -{% endfor %} -{% endif %} -{% endif %} diff --git a/roles/kirocrew/tasks/main.yml b/roles/kirocrew/tasks/main.yml deleted file mode 100644 index 0313743f..00000000 --- a/roles/kirocrew/tasks/main.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -# Kiro Crew gateway (dashboard + Slack + cron) on punjab. The venv and unit are -# owned by Kiro's own installer (`kirocrew service install`), so this role -# doesn't template them — it keeps the service running, keeps its env file -# (where tokens go) root-only, and asserts the dashboard never listens beyond -# localhost. Enable with `kirocrew_enabled: true` in host_vars. - -- name: Check for kirocrew binary - ansible.builtin.stat: - path: "{{ ansible_facts['env']['HOME'] }}/.local/bin/kirocrew" - register: kirocrew_bin - -- name: Check for kirocrew unit - ansible.builtin.stat: - path: /etc/systemd/system/kirocrew.service - register: kirocrew_unit - -- name: Warn when Kiro Crew is not installed - ansible.builtin.debug: - msg: >- - kirocrew_enabled is set but Kiro Crew isn't installed (binary or unit - missing). Install it per https://github.com/kirodotdev/KiroCrew, then - run `sudo ~/.local/bin/kirocrew service install`. - when: not (kirocrew_bin.stat.exists and kirocrew_unit.stat.exists) - -- name: Manage Kiro Crew - when: kirocrew_bin.stat.exists and kirocrew_unit.stat.exists - block: - - name: Keep kirocrew env file root-only - ansible.builtin.file: - path: /etc/kirocrew/kirocrew.env - owner: root - group: root - mode: "0600" - become: true - - - name: Enable and start kirocrew - ansible.builtin.systemd: - name: kirocrew.service - enabled: true - state: started - become: true - - - name: Read kirocrew listening sockets - ansible.builtin.shell: set -o pipefail && ss -Htlnp | grep '"kirocrew"' || true - args: - executable: /bin/bash - register: kirocrew_listen - changed_when: false - become: true - - - name: Assert kirocrew dashboard is localhost-only - ansible.builtin.assert: - that: - - kirocrew_listen.stdout_lines | reject('search', '127\.0\.0\.1:|\[::1\]:') | list | length == 0 - fail_msg: "kirocrew is listening beyond localhost: {{ kirocrew_listen.stdout_lines }}" - quiet: true diff --git a/roles/osquery/handlers/main.yml b/roles/osquery/handlers/main.yml deleted file mode 100644 index 3ac682bf..00000000 --- a/roles/osquery/handlers/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Restart osquery daemon - ansible.builtin.systemd_service: - name: osqueryd.service - state: restarted - daemon_reload: true - become: true diff --git a/roles/osquery/tasks/install.yml b/roles/osquery/tasks/install.yml deleted file mode 100644 index 0095d63c..00000000 --- a/roles/osquery/tasks/install.yml +++ /dev/null @@ -1,151 +0,0 @@ ---- -- name: Provision private native osquery monitoring - become: true - block: - - name: Create persistent executable and download directories - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: root - group: root - mode: "0755" - loop: - - /var/cache/fleet-osquery - - "/opt/osquery-{{ osquery_version }}" - - /opt/fleet-osquery - - - name: Download checksum-pinned upstream osquery archive - ansible.builtin.get_url: - url: >- - https://github.com/osquery/osquery/releases/download/{{ osquery_version }}/{{ osquery_release_archives[ansible_facts['architecture']].name }} - dest: "/var/cache/fleet-osquery/{{ osquery_release_archives[ansible_facts['architecture']].name }}" - checksum: "sha256:{{ osquery_release_archives[ansible_facts['architecture']].sha256 }}" - owner: root - group: root - mode: "0644" - timeout: 120 - - - name: Extract the upstream native binary - ansible.builtin.unarchive: - src: "/var/cache/fleet-osquery/{{ osquery_release_archives[ansible_facts['architecture']].name }}" - dest: "/opt/osquery-{{ osquery_version }}" - remote_src: true - include: - - opt/osquery/bin/osqueryd - extra_opts: - - --strip-components=2 - creates: "/opt/osquery-{{ osquery_version }}/bin/osqueryd" - owner: root - group: root - - - name: Label the persistent native executable for SELinux hosts - ansible.builtin.file: - path: "/opt/osquery-{{ osquery_version }}/bin/osqueryd" - owner: root - group: root - mode: "0755" - setype: bin_t - - - name: Expose osquery interactive mode - ansible.builtin.file: - src: osqueryd - dest: "/opt/osquery-{{ osquery_version }}/bin/osqueryi" - state: link - owner: root - group: root - - - name: Select the pinned osquery release - ansible.builtin.file: - src: "/opt/osquery-{{ osquery_version }}" - dest: /opt/osquery - state: link - owner: root - group: root - notify: Restart osquery daemon - - - name: Create private osquery configuration and state directories - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: root - group: root - mode: "0700" - loop: - - /etc/osquery - - /var/lib/osquery - - /var/log/osquery - - /var/lib/fleet-osquery - - - name: Install the private drift reporter - ansible.builtin.copy: - src: fleet-osquery.py - dest: /opt/fleet-osquery/fleet-osquery - owner: root - group: root - mode: "0755" - setype: bin_t - - - name: Configure bounded local-only osquery scheduling - ansible.builtin.template: - src: osquery.conf.j2 - dest: /etc/osquery/osquery.conf - owner: root - group: root - mode: "0600" - validate: /opt/osquery/bin/osqueryd --disable_extensions --config_check --config_path=%s - notify: Restart osquery daemon - - - name: Install osquery daemon unit - ansible.builtin.template: - src: osqueryd.service.j2 - dest: /etc/systemd/system/osqueryd.service - owner: root - group: root - mode: "0644" - notify: Restart osquery daemon - - - name: Install drift comparison units - ansible.builtin.template: - src: "{{ item }}.j2" - dest: "/etc/systemd/system/{{ item }}" - owner: root - group: root - mode: "0644" - loop: - - fleet-osquery-report.service - - fleet-osquery-report.timer - - # Enable before baseline collection so our own startup timer is included. - - name: Enable drift timer without starting an uninitialized report - ansible.builtin.systemd_service: - name: fleet-osquery-report.timer - daemon_reload: true - enabled: true - - - name: Apply changed daemon configuration - ansible.builtin.meta: flush_handlers - - - name: Enable and start osquery daemon - ansible.builtin.systemd_service: - name: osqueryd.service - enabled: true - state: started - - - name: Establish the initial baseline without accepting later drift - ansible.builtin.command: - argv: - - /opt/fleet-osquery/fleet-osquery - - baseline - - --if-missing - creates: /var/lib/fleet-osquery/baseline.json - - - name: Refresh the private drift report - ansible.builtin.systemd_service: - name: fleet-osquery-report.service - state: started - changed_when: false - - - name: Start periodic drift reporting - ansible.builtin.systemd_service: - name: fleet-osquery-report.timer - state: started diff --git a/roles/osquery/tasks/main.yml b/roles/osquery/tasks/main.yml deleted file mode 100644 index aae81725..00000000 --- a/roles/osquery/tasks/main.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: Determine native osquery support - ansible.builtin.set_fact: - osquery_native_supported: >- - {{ osquery_supported | bool - and ansible_facts['system'] == 'Linux' - and ansible_facts['service_mgr'] | default('') == 'systemd' - and ansible_facts['architecture'] in osquery_release_archives - and not is_musl | default(false) | bool - and not is_termux | default(false) | bool - and inventory_hostname not in groups.get('vps', []) }} - -- name: Explain unsupported osquery platform - ansible.builtin.debug: - msg: >- - osquery not installed: requires native glibc Linux, systemd, root and - x86_64/aarch64. musl/Android and retired VPSes are not modified. - when: not osquery_native_supported | bool - -- name: Explain osquery check-mode boundary - ansible.builtin.debug: - msg: "osquery support detected; release installation and live baseline collection require an apply (not check mode)." - when: osquery_native_supported | bool and ansible_check_mode - -- name: Install native osquery monitoring - ansible.builtin.include_tasks: install.yml - when: osquery_native_supported | bool and not ansible_check_mode - -- name: Manage the always-on fleet poller - ansible.builtin.include_tasks: poller.yml - when: osquery_native_supported | bool and not ansible_check_mode diff --git a/roles/osquery/tasks/poller.yml b/roles/osquery/tasks/poller.yml deleted file mode 100644 index 94a05b5f..00000000 --- a/roles/osquery/tasks/poller.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- -- name: Check for a previously managed fleet poller - ansible.builtin.stat: - path: /etc/systemd/system/fleet-osquery-poll.timer - get_checksum: false - register: osquery_poller_timer - when: not osquery_poller_enabled | bool - -- name: Disable fleet polling when this host is not selected - ansible.builtin.systemd_service: - name: fleet-osquery-poll.timer - enabled: false - state: stopped - become: true - when: - - not osquery_poller_enabled | bool - - osquery_poller_timer.stat.exists - -- name: Stop any remaining poll on an unselected host - ansible.builtin.systemd_service: - name: fleet-osquery-poll.service - state: stopped - become: true - when: - - not osquery_poller_enabled | bool - - osquery_poller_timer.stat.exists - -- name: Configure persistent private fleet polling - become: true - when: osquery_poller_enabled | bool - block: - - name: Verify Ansible inventory tooling under the polling identity - ansible.builtin.command: - argv: [ansible-inventory, --version] - become: false - environment: - PATH: "{{ osquery_poller_path }}" - changed_when: false - - - name: Create root-managed poller configuration directory - ansible.builtin.file: - path: /etc/fleet-osquery - state: directory - owner: root - group: root - mode: "0755" - - - name: Install the collector and its existing discovery helper - ansible.builtin.copy: - src: "{{ playbook_dir }}/scripts/{{ item }}" - dest: "/opt/fleet-osquery/{{ item }}" - owner: root - group: root - mode: "0644" - loop: - - osquery-fleet.py - - online_hosts.py - - - name: Snapshot only fleet connection and exclusion metadata - ansible.builtin.template: - src: poller-inventory.yml.j2 - dest: /etc/fleet-osquery/inventory.yml - owner: root - group: root - mode: "0644" - - - name: Create private aggregate snapshot storage for the polling identity - ansible.builtin.file: - path: "{{ osquery_poller_state_dir }}" - state: directory - owner: "{{ ansible_facts['user_id'] }}" - group: "{{ ansible_facts['user_gid'] }}" - mode: "0700" - - - name: Install system-level fleet polling units - ansible.builtin.template: - src: "{{ item }}.j2" - dest: "/etc/systemd/system/{{ item }}" - owner: root - group: root - mode: "0644" - loop: - - fleet-osquery-poll.service - - fleet-osquery-poll.timer - - - name: Enable persistent fleet polling schedule - ansible.builtin.systemd_service: - name: fleet-osquery-poll.timer - daemon_reload: true - enabled: true - state: started - - - name: Collect the initial fleet snapshot under the existing SSH identity - ansible.builtin.systemd_service: - name: fleet-osquery-poll.service - state: started - changed_when: false diff --git a/roles/osquery/templates/poller-inventory.yml.j2 b/roles/osquery/templates/poller-inventory.yml.j2 deleted file mode 100644 index e6da7973..00000000 --- a/roles/osquery/templates/poller-inventory.yml.j2 +++ /dev/null @@ -1,22 +0,0 @@ -all: - hosts: -{% for host in groups['all'] | sort %} - {{ host | to_json }}: -{% for key in ['ansible_host', 'ansible_user', 'ansible_ssh_user', 'ansible_port', 'ansible_ssh_port', 'ansible_connection', 'tailnet_name', 'osquery_supported'] %} -{% if hostvars[host][key] is defined %} - {{ key }}: {{ hostvars[host][key] | to_json }} -{% endif %} -{% endfor %} -{% endfor %} - children: -{% for group in ['vps', 'test_fleet', 'termux_hosts'] %} - {{ group }}: -{% if groups.get(group, []) %} - hosts: -{% for host in groups[group] | sort %} - {{ host | to_json }}: {} -{% endfor %} -{% else %} - hosts: {} -{% endif %} -{% endfor %} diff --git a/roles/pipewire_audio/files/mic-dsp.conf b/roles/pipewire_audio/files/mic-dsp.conf deleted file mode 100644 index da5de452..00000000 --- a/roles/pipewire_audio/files/mic-dsp.conf +++ /dev/null @@ -1,85 +0,0 @@ -# PipeWire filter-chain — Microphone DSP (input) -# -# Replaces Easy Effects input preset "input-mic-noise-reduction.json". -# -# Chain: RNNoise noise suppression → Compressor -# -# PREREQUISITES (must be in bootc image): -# noise-suppression-for-voice → librnnoise_ladspa.so (LADSPA) -# lsp-plugins-lv2 → Compressor Stereo -# -# VERIFY after deploy: -# ls /usr/lib64/ladspa/librnnoise_ladspa.so -# lv2ls | grep compressor_stereo -# -# LSP port names verified with: -# lv2info http://lsp-plug.in/plugins/lv2/compressor_stereo | grep -A2 'port.*symbol' - -context.modules = [ - { - name = libpipewire-module-filter-chain - args = { - node.description = "Mic DSP" - media.name = "Mic DSP" - - filter.graph = { - nodes = [ - # ── 1. RNNoise noise suppression (LADSPA) ──────────────── - { - type = ladspa - name = rnnoise - plugin = "librnnoise_ladspa.so" - label = "noise_suppressor_stereo" - control = { - "Wet" = 1.0 # 1.0 = full noise suppression - "Release" = 20.0 # fast release for speech (ms) - "VAD Threshold (%)" = 50.0 # voice activity detection threshold - } - } - - # ── 2. Compressor — tames mic level spikes ─────────────── - # Easy Effects preset: downward compression, -24 dB, - # 2:1 ratio, 5 ms attack, 100 ms release. - { - type = lv2 - name = comp - plugin = "http://lsp-plug.in/plugins/lv2/compressor_stereo" - control = { - at = 5.0 # attack (ms) - rt = 100.0 # release (ms) - pth = -24.0 # peak threshold (dB) - ra = 2.0 # ratio - kn = 3.0 # knee (dB), LSP uses absolute value - mk = 0.0 # makeup gain (dB) - scm = 0 # sidechain: feed-forward - } - } - ] - - links = [ - { output = "rnnoise:Out", input = "comp:In" } - ] - } - - # ── Mic input routing ────────────────────────────────────── - # capture: Audio/Sink → receives audio from hardware mic - # playback: Audio/Source → apps record from this virtual mic - # - # Same node.name on both sides = single filter node. - # WirePlumber will link the capture sink to the hardware source - # when this node is set as the default source. - capture.props = { - node.name = "mic_dsp" - node.description = "Microphone (DSP)" - media.class = "Audio/Sink" - priority.session = 2000 - } - playback.props = { - node.name = "mic_dsp" - node.description = "Microphone (DSP)" - media.class = "Audio/Source" - priority.session = 2000 - } - } - } -] diff --git a/roles/pipewire_audio/handlers/main.yml b/roles/pipewire_audio/handlers/main.yml deleted file mode 100644 index 12af3712..00000000 --- a/roles/pipewire_audio/handlers/main.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: Reset pipewire failed state (clear start-limit-hit) - ansible.builtin.command: systemctl --user reset-failed pipewire.service pipewire.socket pipewire-pulse.service pipewire-pulse.socket # noqa: command-instead-of-module - failed_when: false - changed_when: false - listen: Restart pipewire - -- name: Restart pipewire services - ansible.builtin.systemd_service: - name: "{{ item }}" - state: restarted - scope: user - loop: - - pipewire - - pipewire-pulse.socket - failed_when: false - listen: Restart pipewire - -- name: Reset wireplumber failed state - ansible.builtin.command: systemctl --user reset-failed wireplumber.service # noqa: command-instead-of-module - failed_when: false - changed_when: false - listen: Restart wireplumber - -- name: Restart wireplumber service - ansible.builtin.systemd_service: - name: wireplumber - state: restarted - scope: user - failed_when: false - listen: Restart wireplumber - -- name: Restart mic dsp link - ansible.builtin.systemd_service: - name: mic-dsp-link.service - state: restarted - scope: user - daemon_reload: true - failed_when: false diff --git a/roles/pipewire_audio/tasks/main.yml b/roles/pipewire_audio/tasks/main.yml deleted file mode 100644 index 375c41b8..00000000 --- a/roles/pipewire_audio/tasks/main.yml +++ /dev/null @@ -1,227 +0,0 @@ ---- -# ── Platform detection ─────────────────────────────────────────────────── - -- name: Detect if system is bootc-based (container-native, no package manager) - ansible.builtin.stat: - path: /usr/bin/bootc - register: bootc_bin - -- name: Detect if rpm-ostree is available (Fedora Atomic) - ansible.builtin.stat: - path: /usr/bin/rpm-ostree - register: rpm_ostree_bin - -- name: Detect if dnf is available (traditional Fedora) - ansible.builtin.stat: - path: /usr/bin/dnf - register: dnf_bin - -# ── Prerequisite packages (only on mutable systems) ────────────────────── -# On bootc systems packages are baked into the image — we can't install at -# runtime. LV2/LADSPA plugins must be in the Containerfile. - -- name: Install LV2 + LADSPA DSP prerequisites (dnf) - when: dnf_bin.stat.exists | default(false) - become: true - ansible.builtin.dnf: - name: - - lsp-plugins-lv2 - - noise-suppression-for-voice - state: present - -- name: Layer LV2 + LADSPA DSP prerequisites (rpm-ostree) - when: rpm_ostree_bin.stat.exists | default(false) - become: true - ansible.builtin.command: - cmd: rpm-ostree install --apply-live lsp-plugins-lv2 noise-suppression-for-voice - register: rpm_ostree_install - changed_when: "'Already layered' not in rpm_ostree_install.stdout and 'No change' not in rpm_ostree_install.stdout" - failed_when: false - -- name: Install LV2 + LADSPA DSP prerequisites (Arch) - when: ansible_facts['os_family'] == 'Archlinux' - become: true - community.general.pacman: - name: - - lsp-plugins - - noise-suppression-for-voice - state: present - -# ── Plugin availability check ──────────────────────────────────────────── - -- name: Find RNNoise LADSPA plugin (cross-distro paths) - ansible.builtin.shell: > - set -o pipefail; find /usr/lib /usr/lib64 -name librnnoise_ladspa.so 2>/dev/null | head -1 - register: rnnoise_path - failed_when: false - changed_when: false - -- name: Check if LV2 plugins are available (lv2ls may not be installed) - ansible.builtin.command: lv2ls - register: lv2ls_result - failed_when: false - changed_when: false - -- name: Set DSP availability facts - ansible.builtin.set_fact: - has_rnnoise: "{{ rnnoise_path.stdout | default('') | length > 0 }}" - has_lv2_lsp: "{{ 'lsp-plug.in/plugins/lv2/para_equalizer_x16_lr' in lv2ls_result.stdout | default('') }}" - is_bootc: "{{ bootc_bin.stat.exists | default(false) }}" - -# ── PipeWire config directories ────────────────────────────────────────── - -- name: Ensure PipeWire config directories exist - ansible.builtin.file: - path: "{{ item }}" - state: directory - mode: "0755" - loop: - - ~/.config/pipewire/pipewire.conf.d - - ~/.config/pipewire/filter-chain.conf.d - -# ── Speaker DSP ────────────────────────────────────────────────────────── -# Tier 1: LV2-based full chain (para EQ + bass enhancer + exciter + limiter) -# Tier 2: Built-in biquad EQ (zero external deps, works everywhere) -# -# If LV2 plugins are available, speaker-dsp.conf (LV2) takes priority. -# Otherwise speaker-biquad.conf (built-in biquads) is used. - -- name: Deploy speaker DSP (LV2-based, full chain) - when: has_lv2_lsp | bool - ansible.builtin.copy: - src: speaker-dsp.conf - dest: "{{ ansible_facts['env']['HOME'] }}/.config/pipewire/pipewire.conf.d/50-speaker-dsp.conf" - mode: "0644" - notify: Restart pipewire - -- name: Deploy speaker DSP (built-in biquads, zero-dependency fallback) - when: not (has_lv2_lsp | bool) - ansible.builtin.copy: - src: speaker-biquad.conf - dest: "{{ ansible_facts['env']['HOME'] }}/.config/pipewire/pipewire.conf.d/50-speaker-dsp.conf" - mode: "0644" - notify: Restart pipewire - -# ── Mic DSP ────────────────────────────────────────────────────────────── -# Requires librnnoise_ladspa.so (RNNoise noise suppression). -# Only deployed when the LADSPA plugin exists on the system. - -- name: Deploy mic DSP (RNNoise LADSPA) - when: has_rnnoise | bool - ansible.builtin.copy: - src: mic-rnnoise.conf - dest: "{{ ansible_facts['env']['HOME'] }}/.config/pipewire/pipewire.conf.d/60-mic-dsp.conf" - mode: "0644" - notify: Restart pipewire - -- name: Remove mic DSP config (RNNoise unavailable) - when: not (has_rnnoise | bool) - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/pipewire/pipewire.conf.d/60-mic-dsp.conf" - state: absent - notify: Restart pipewire - -# ── LADSPA_PATH for bootc systems ──────────────────────────────────────── -# On immutable systems /usr/lib*/ladspa is read-only. If we need user-space -# LADSPA plugins, pipewire needs LADSPA_PATH to find them. - -- name: Ensure pipewire.service drop-in directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/pipewire.service.d" - state: directory - mode: "0755" - -- name: Ensure pipewire LADSPA_PATH includes user ladspa directory - ansible.builtin.copy: - content: | - [Service] - Environment=LADSPA_PATH=%h/.ladspa:/usr/lib/x86_64-linux-gnu/ladspa:/usr/lib/ladspa:/usr/lib64/ladspa - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/pipewire.service.d/ladspa-path.conf" - mode: "0644" - register: ladspa_path_dropin - notify: Restart pipewire - -- name: Ensure user ladspa directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.ladspa" - state: directory - mode: "0755" - -# ── Mic DSP persistent routing ────────────────────────────────────────── -# WirePlumber 0.5 doesn't auto-route hardware mic to filter-chain. -# Systemd oneshot creates the link at login. - -- name: Deploy mic DSP link service (routes hardware mic through RNNoise) - when: has_rnnoise | bool - ansible.builtin.template: - src: mic-dsp-link.service.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/mic-dsp-link.service" - mode: "0644" - notify: Restart mic dsp link - -- name: Enable mic DSP link service - ansible.builtin.systemd_service: - name: mic-dsp-link.service - enabled: true - state: started - scope: user - daemon_reload: true - when: has_rnnoise | bool - -- name: Disable mic DSP link service (RNNoise unavailable) - ansible.builtin.systemd_service: - name: mic-dsp-link.service - enabled: false - state: stopped - scope: user - when: not (has_rnnoise | bool) - -# ── WirePlumber cleanup (old-format Lua unsupported in WP 0.5) ─────────── - -- name: Remove old-format WirePlumber default-device Lua rule (WP 0.5+) - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/wireplumber/main.lua.d/51-speaker-dsp-default.lua" - state: absent - notify: Restart wireplumber - -- name: Remove empty WirePlumber Lua config directory - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/wireplumber/main.lua.d" - state: absent - -# ── GNOME volume over-amplification ─────────────────────────────────────── -# Allows volume slider to exceed 100% (software amplification). Virtual -# DSP sinks use software volume, so over-100% is safe and useful. - -- name: Enable GNOME over-100% volume (software amplification) - ansible.builtin.command: - cmd: gsettings set org.gnome.desktop.sound allow-volume-above-100-percent true - environment: - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ ansible_facts['user_uid'] }}/bus" - failed_when: false - changed_when: false - -# ── Easy Effects cleanup ───────────────────────────────────────────────── -# Migrated to native PipeWire filter-chain above. - -- name: Remove Easy Effects autostart entry - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/autostart/com.github.wwmm.easyeffects.desktop" - state: absent - -- name: Remove Easy Effects Flatpak app data (presets, gsettings) - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.var/app/com.github.wwmm.easyeffects" - state: absent - -- name: Kill any running Easy Effects processes - ansible.builtin.command: flatpak kill com.github.wwmm.easyeffects - failed_when: false - changed_when: false - -- name: Uninstall Easy Effects Flatpak (system) - ansible.builtin.command: flatpak uninstall --system -y com.github.wwmm.easyeffects - become: true - register: ee_uninstall - changed_when: "'removed' in ee_uninstall.stdout or 'uninstalled' in ee_uninstall.stdout" - failed_when: false diff --git a/roles/proxy/handlers/main.yml b/roles/proxy/handlers/main.yml deleted file mode 100644 index fbd51782..00000000 --- a/roles/proxy/handlers/main.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Reload systemd daemon - ansible.builtin.systemd: - daemon_reload: true - become: true - listen: Reload and restart caddy - -- name: Restart caddy - ansible.builtin.systemd: - name: caddy.service - state: restarted - become: true - listen: Reload and restart caddy diff --git a/roles/proxy/tasks/main.yml b/roles/proxy/tasks/main.yml deleted file mode 100644 index 45c84f7f..00000000 --- a/roles/proxy/tasks/main.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -- name: Create Caddy config directory - ansible.builtin.file: - path: /etc/caddy - state: directory - mode: "0755" - owner: root - group: root - become: true - -- name: Get Tailscale hostname for Caddyfile - ansible.builtin.command: tailscale status --json - register: _ts_status - changed_when: false - failed_when: false - become: true - -- name: Set tailscale hostname fact - ansible.builtin.set_fact: - proxy_tailscale_host: "{{ (_ts_status.stdout | from_json).Self.DNSName | regex_replace('\\.$', '') }}" - when: (_ts_status.stdout | default('')) | length > 0 - -- name: Deploy Caddyfile - ansible.builtin.template: - src: Caddyfile.j2 - dest: /etc/caddy/Caddyfile - mode: "0644" - owner: root - group: root - become: true - notify: Reload and restart caddy - -- name: Deploy Caddy quadlet - ansible.builtin.copy: - src: caddy.container - dest: /etc/containers/systemd/caddy.container - mode: "0644" - owner: root - group: root - become: true - notify: Reload and restart caddy - -- name: Flush handlers - ansible.builtin.meta: flush_handlers - -- name: Reload systemd daemon - ansible.builtin.systemd: - daemon_reload: true - become: true - -- name: Enable and start Caddy - ansible.builtin.systemd: - name: caddy.service - state: started - enabled: true - become: true - -- name: Check Tailscale serve config - ansible.builtin.command: tailscale serve status - register: ts_serve_status - changed_when: false - failed_when: false - become: true - -- name: Point tailscale serve at Caddy (not homepage directly) - ansible.builtin.command: tailscale serve --bg http://127.0.0.1:9000 - when: "'127.0.0.1:9000' not in (ts_serve_status.stdout | default(''))" - become: true - -- name: Remove old direct homepage serve if present - ansible.builtin.command: tailscale serve --https=443 off - when: - - "'127.0.0.1:3000' in (ts_serve_status.stdout | default(''))" - - "'127.0.0.1:9000' not in (ts_serve_status.stdout | default(''))" - become: true - failed_when: false diff --git a/roles/retired_services/tasks/main.yml b/roles/retired_services/tasks/main.yml deleted file mode 100644 index dee6c1b3..00000000 --- a/roles/retired_services/tasks/main.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- -# Stops and removes unit files left by roles that were deleted from this repo. -# Only the unit files go: podman volumes, images and config dirs stay. On a -# machine without any of these files every task after the lookup is skipped. - -# One find over the unit dirs instead of a stat per file: this runs on every -# daily apply, and ~70 module calls is noticeable on a laptop. find warns on -# missing paths, so only hand it the dirs that exist. -- name: Check which unit directories exist - ansible.builtin.stat: - path: "{{ item }}" - get_checksum: false - get_mime: false - get_attributes: false - loop: "{{ _retired_dirs }}" - register: _retired_dir_stat - -- name: Find leftover unit files from retired roles - ansible.builtin.find: - paths: "{{ _retired_dir_stat.results | selectattr('stat.isdir', 'defined') | selectattr('stat.isdir') | map(attribute='item') | list }}" - patterns: "{{ retired_services_system_units | map('basename') | list + retired_services_user_units }}" - file_type: any - register: _retired_found - when: _retired_dir_stat.results | selectattr('stat.isdir', 'defined') | selectattr('stat.isdir') | list | length > 0 - -- name: Collect leftover unit files - ansible.builtin.set_fact: - _retired_system_present: "{{ retired_services_system_units | select('in', _found_paths) | list }}" - _retired_user_present: "{{ _user_paths | select('in', _found_paths) | map('basename') | list }}" - vars: - _found_paths: "{{ _retired_found.files | default([]) | map(attribute='path') | list }}" - _user_paths: "{{ retired_services_user_units | map('regex_replace', '^', _retired_user_dir ~ '/') | list }}" - -- name: Remove leftover system units - when: _retired_system_present | length > 0 - become: true - block: - # Generated quadlet units can't be disabled (removing the file and - # reloading drops their [Install] links); plain units are disabled so no - # dangling timers.target.wants symlink is left behind. - - name: Stop leftover system services - ansible.builtin.systemd: - name: "{{ _unit }}" - state: stopped - enabled: "{{ omit if _ext in retired_services_quadlet_suffix else false }}" - vars: - _stem: "{{ (item | basename | splitext)[0] }}" - _ext: "{{ (item | basename | splitext)[1] }}" - _unit: "{{ (_stem ~ retired_services_quadlet_suffix[_ext] ~ '.service') if _ext in retired_services_quadlet_suffix else item | basename }}" - loop: "{{ _retired_system_present }}" - register: _retired_system_stop - # A file whose unit was never generated (podman gone, bad quadlet) has - # nothing to stop — still remove the file. - failed_when: - - _retired_system_stop is failed - - "'Could not find the requested service' not in (_retired_system_stop.msg | default(''))" - - - name: Remove leftover system unit files - ansible.builtin.file: - path: "{{ item }}" - state: absent - loop: "{{ _retired_system_present }}" - - - name: Reload systemd after removing retired units - ansible.builtin.systemd: - daemon_reload: true - -- name: Remove leftover user quadlets - when: _retired_user_present | length > 0 - # systemctl --user needs the user's runtime dir; it isn't always in the - # environment (e.g. non-login shells), so pin it like zen_browser does. - environment: - XDG_RUNTIME_DIR: "/run/user/{{ ansible_facts['user_uid'] }}" - block: - - name: Stop leftover user services - ansible.builtin.systemd: - name: "{{ _stem ~ retired_services_quadlet_suffix[_ext] ~ '.service' }}" - scope: user - state: stopped - vars: - _stem: "{{ (item | splitext)[0] }}" - _ext: "{{ (item | splitext)[1] }}" - loop: "{{ _retired_user_present }}" - register: _retired_user_stop - failed_when: - - _retired_user_stop is failed - - "'Could not find the requested service' not in (_retired_user_stop.msg | default(''))" - - - name: Remove leftover user quadlet files - ansible.builtin.file: - path: "{{ _retired_user_dir }}/{{ item }}" - state: absent - loop: "{{ _retired_user_present }}" - - - name: Reload user systemd after removing retired quadlets - ansible.builtin.systemd: - scope: user - daemon_reload: true diff --git a/roles/retired_services/vars/main.yml b/roles/retired_services/vars/main.yml deleted file mode 100644 index 4fc8c7f9..00000000 --- a/roles/retired_services/vars/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# Quadlet file extension -> suffix of the service the generator makes from it -# (foo.volume -> foo-volume.service). Anything else is a plain unit file. -retired_services_quadlet_suffix: - .container: "" - .volume: "-volume" - .network: "-network" - .pod: "-pod" - -_retired_user_dir: "{{ ansible_facts['env']['HOME'] }}/.config/containers/systemd" -_retired_dirs: "{{ retired_services_system_units | map('dirname') | unique | list + [_retired_user_dir] }}" diff --git a/roles/server_hardening/handlers/main.yml b/roles/server_hardening/handlers/main.yml deleted file mode 100644 index d2dbe391..00000000 --- a/roles/server_hardening/handlers/main.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Restart journald - ansible.builtin.systemd: - name: systemd-journald - state: restarted - become: true - -# `Reload sshd` is intentionally not defined here. It lives in roles/sshd, which -# runs unconditionally in the same play, so this role's `notify: Reload sshd` -# resolves to that single definition. Defining a second handler with the same -# name would shadow it: Ansible keeps only the last definition, and because this -# role is conditional, that copy inherits the role's `when:` and is skipped on -# non-server hosts -- silently dropping the reload everywhere else. diff --git a/roles/server_hardening/tasks/main.yml b/roles/server_hardening/tasks/main.yml deleted file mode 100644 index bb07cef8..00000000 --- a/roles/server_hardening/tasks/main.yml +++ /dev/null @@ -1,144 +0,0 @@ ---- -# Server hardening — security + reliability for long-running hosts. -# Skips desktops (keyboard-interactive machines with different needs). - -# ── Journald log limits ─────────────────────────────────────────── -- name: Set journald max disk usage - ansible.builtin.lineinfile: - path: /etc/systemd/journald.conf - regexp: '^#?SystemMaxUse=' - line: SystemMaxUse=500M - become: true - notify: Restart journald - -- name: Vacuum journals to stay under limit - ansible.builtin.command: journalctl --vacuum-size=500M - become: true - changed_when: false - -# ── Swap ────────────────────────────────────────────────────────── -- name: Check if swap file exists - ansible.builtin.stat: - path: /swapfile - register: swapfile - -- name: Create 2G swap file - ansible.builtin.command: - cmd: dd if=/dev/zero of=/swapfile bs=1M count=2048 - creates: /swapfile - become: true - when: not swapfile.stat.exists - -- name: Set swap file permissions - ansible.builtin.file: - path: /swapfile - mode: "0600" - owner: root - group: root - become: true - -# Gate format/enable on whether the swapfile is actually active, not on -# file existence — the stat above is taken before creation, so on the run -# that creates the file it's stale, and on every run after that mkswap/ -# swapon would otherwise run unconditionally against an already-active -# swapfile and fail the play. -- name: Check if swap file is already active - ansible.builtin.command: swapon --show=NAME --noheadings - become: true - register: swap_active - changed_when: false - -- name: Format swap file - ansible.builtin.command: mkswap /swapfile - become: true - when: "'/swapfile' not in swap_active.stdout" - -- name: Enable swap file - ansible.builtin.command: swapon /swapfile - become: true - when: "'/swapfile' not in swap_active.stdout" - -- name: Add swap to fstab - ansible.builtin.lineinfile: - path: /etc/fstab - line: /swapfile none swap sw 0 0 - become: true - -# ── NTP time sync ───────────────────────────────────────────────── -# Some images keep time with chrony instead (Ubuntu on EC2 uses it for the -# Amazon Time Sync Service) and don't ship timesyncd at all — and the two -# shouldn't both run. Only enable timesyncd when it exists and chrony isn't -# already doing the job. -- name: Check for chrony - ansible.builtin.command: systemctl is-active chrony chronyd # noqa: command-instead-of-module - register: server_hardening_chrony - changed_when: false - failed_when: false - -- name: Check for systemd-timesyncd - ansible.builtin.stat: - path: /usr/lib/systemd/systemd-timesyncd - register: server_hardening_timesyncd - -- name: Enable systemd-timesyncd - ansible.builtin.systemd: - name: systemd-timesyncd - enabled: true - state: started - become: true - when: - - server_hardening_timesyncd.stat.exists - - "'active' not in server_hardening_chrony.stdout_lines" - -# ── SSH rate limiting ───────────────────────────────────────────── -- name: Harden SSH rate limits - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: "{{ item.regexp }}" - line: "{{ item.line }}" - become: true - loop: - - {regexp: '^#?MaxStartups', line: 'MaxStartups 3:50:10'} - - {regexp: '^#?MaxSessions', line: 'MaxSessions 10'} - - {regexp: '^#?ClientAliveCountMax', line: 'ClientAliveCountMax 3'} - notify: Reload sshd - -# ── UFW logging ─────────────────────────────────────────────────── -- name: Check if UFW is active - ansible.builtin.command: ufw status - register: ufw_status - changed_when: false - failed_when: false - -- name: Enable UFW logging - ansible.builtin.command: ufw logging on - become: true - when: - - ufw_status.rc == 0 - - "'Status: active' in ufw_status.stdout" - -# ── Kernel livepatch (Ubuntu Pro) ───────────────────────────────── -- name: Check if Ubuntu Pro is attached - ansible.builtin.command: pro status --format json - register: pro_status - changed_when: false - failed_when: false - -- name: Enable livepatch - ansible.builtin.command: pro enable livepatch - become: true - when: - - pro_status.rc == 0 - - "'livepatch' not in (pro_status.stdout | default('')) or 'disabled' in (pro_status.stdout | default(''))" - failed_when: false - -# ── Reboot notification ─────────────────────────────────────────── -- name: Check if reboot is needed - ansible.builtin.stat: - path: /var/run/reboot-required - register: reboot_needed - -- name: Warn if reboot is required - ansible.builtin.debug: - msg: "🚨 REBOOT REQUIRED on {{ inventory_hostname }} — kernel was updated and needs a restart." - when: reboot_needed.stat.exists diff --git a/roles/ssh_keys/tasks/main.yml b/roles/ssh_keys/tasks/main.yml deleted file mode 100644 index a97ad3cc..00000000 --- a/roles/ssh_keys/tasks/main.yml +++ /dev/null @@ -1,306 +0,0 @@ ---- -- name: Ensure ~/.ssh directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.ssh" - state: directory - mode: "0700" - -- name: Deploy SSH config - ansible.builtin.template: - src: ssh_config.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.ssh/config" - mode: "0644" - -- name: Ensure SSH ControlPath directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.ssh/cp" - state: directory - mode: "0700" - -- name: Set SSH key path facts - ansible.builtin.set_fact: - priv_key: "{{ ansible_facts['env']['HOME'] }}/.ssh/id_ed25519" - pub_key: "{{ ansible_facts['env']['HOME'] }}/.ssh/id_ed25519.pub" - -- name: Check if private key exists on disk - ansible.builtin.stat: - path: "{{ priv_key }}" - register: priv_key_stat - -# ── Key missing — fetch from BW or generate ── -- name: Fetch SSH key from Bitwarden - ansible.builtin.shell: | - set -euo pipefail - BW_ITEM=$(bw get item "{{ user_login }}@{{ inventory_hostname }}" 2>/dev/null) || exit 1 - echo "$BW_ITEM" | python3 -c " - import sys, json - item = json.load(sys.stdin) - ssh = item.get('sshKey', {}) - priv = ssh.get('privateKey', '') - pub = ssh.get('publicKey', '') - if not priv or not pub: - sys.exit(1) - with open('{{ priv_key }}', 'w') as f: - f.write(priv + ('' if priv.endswith('\n') else '\n')) - with open('{{ pub_key }}', 'w') as f: - f.write(pub + ('' if pub.endswith('\n') else '\n')) - " - chmod 600 "{{ priv_key }}" - chmod 644 "{{ pub_key }}" - args: - executable: /bin/bash - register: bw_fetch_result - failed_when: false - changed_when: bw_fetch_result.rc == 0 - when: not priv_key_stat.stat.exists - environment: - BW_SESSION: "{{ bw_session | default('') }}" - PATH: "{{ ansible_env_path }}" - -- name: Generate new SSH key pair if not on disk or in BW - ansible.builtin.command: > - ssh-keygen -t ed25519 -C "{{ inventory_hostname }}" - -f "{{ priv_key }}" -N "" - when: not priv_key_stat.stat.exists and bw_fetch_result.rc | default(1) != 0 - -# ── Fix permissions ── -- name: Set key permissions - ansible.builtin.file: - path: "{{ item.path }}" - mode: "{{ item.mode }}" - loop: - - {path: "{{ priv_key }}", mode: "0600"} - - {path: "{{ pub_key }}", mode: "0644"} - -# ── Sync local key with Bitwarden ───────────────────────────────── -# Gated on `bw_unlocked` (set by the bitwarden role earlier in the play). -# On every run: -# - if BW has no item → create -# - if BW pubkey ≠ disk → leave BW alone and warn, unless -# ssh_keys_force_push=true. A key generated while the -# vault was locked must never clobber the machine's -# real key in BW; intentional rotations opt in. -# - if BW pubkey == disk → no-op -# Skipped when the vault is locked; next unlocked apply will heal it. -- name: Sync local SSH key to Bitwarden (create if missing, update only when forced) - ansible.builtin.shell: | - python3 - <<'PY' - import json, os, subprocess, sys - - name = os.environ["BW_ITEM_NAME"] - priv = open(os.environ["PRIV_PATH"]).read() - pub_parts = open(os.environ["PUB_PATH"]).read().strip().split() - pub_norm = pub_parts[0] + " " + pub_parts[1] - fp = subprocess.run( - ["ssh-keygen", "-lf", os.environ["PUB_PATH"]], - capture_output=True, text=True, check=True, - ).stdout.split()[1] - - r = subprocess.run(["bw", "get", "item", name], capture_output=True, text=True) - if r.returncode == 0 and r.stdout.strip(): - item = json.loads(r.stdout) - if item.get("sshKey", {}).get("publicKey", "").strip() == pub_norm: - print("MATCH") - sys.exit(0) - if os.environ.get("FORCE_PUSH", "false").lower() != "true": - print("MISMATCH") - sys.exit(0) - item["sshKey"] = {"privateKey": priv, "publicKey": pub_norm, "keyFingerprint": fp} - encoded = subprocess.run( - ["bw", "encode"], input=json.dumps(item), - capture_output=True, text=True, check=True, - ).stdout - subprocess.run( - ["bw", "edit", "item", item["id"]], input=encoded, - capture_output=True, text=True, check=True, - ) - print("UPDATED") - else: - item = { - "type": 5, "name": name, - "sshKey": {"privateKey": priv, "publicKey": pub_norm, "keyFingerprint": fp}, - } - encoded = subprocess.run( - ["bw", "encode"], input=json.dumps(item), - capture_output=True, text=True, check=True, - ).stdout - subprocess.run( - ["bw", "create", "item"], input=encoded, - capture_output=True, text=True, check=True, - ) - print("CREATED") - PY - args: - executable: /bin/bash - register: bw_sync - changed_when: bw_sync.stdout is search("UPDATED|CREATED") - when: bw_unlocked | default(false) - environment: - BW_SESSION: "{{ bw_session | default('') }}" - BW_ITEM_NAME: "{{ user_login }}@{{ inventory_hostname }}" - PRIV_PATH: "{{ priv_key }}" - PUB_PATH: "{{ pub_key }}" - FORCE_PUSH: "{{ ssh_keys_force_push | default(false) | bool | string | lower }}" - PATH: "{{ ansible_env_path }}" - -- name: Warn when local SSH key differs from Bitwarden (not pushed) - ansible.builtin.debug: - msg: >- - Local ~/.ssh/id_ed25519 differs from the key stored in Bitwarden item - {{ user_login }}@{{ inventory_hostname }}; Bitwarden was NOT updated. - If this is an intentional rotation, re-run with -e ssh_keys_force_push=true. - Otherwise restore the Bitwarden key locally (move the local key aside and re-apply). - when: bw_sync.stdout | default('') is search('MISMATCH') - -# ── Update allowed_signers for git commit signing ── -- name: Read public key - ansible.builtin.slurp: - src: "{{ pub_key }}" - register: pub_key_content - -- name: Set allowed_signers facts - ansible.builtin.set_fact: - signers_pub_key: "{{ pub_key_content.content | b64decode | trim }}" - signers_key_body: "{{ (pub_key_content.content | b64decode).split()[1] }}" - -- name: Ensure allowed_signers file exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.ssh/allowed_signers" - state: touch - mode: "0644" - modification_time: preserve - access_time: preserve - -- name: Ensure current key is in allowed_signers (replacing stale entry) - ansible.builtin.lineinfile: - path: "{{ ansible_facts['env']['HOME'] }}/.ssh/allowed_signers" - regexp: "{{ signers_key_body | regex_escape }}" - line: "{{ user_email }} namespaces=\"git\" {{ signers_pub_key }}" - create: true - mode: "0644" - -# ── Manage known_hosts for all fleet machines ──────────────────── -# Scan each remote fleet machine's host key, flush stale entries, then -# add only the current key. Prevents "REMOTE HOST IDENTIFICATION CHANGED" -# warnings when machines get rebuilt. -# Keyed by the same HostName (and Port) that ssh_config.j2 emits, since that -# is what ssh looks up. Non-22 ports (phones, port-forwarded test VMs) are -# recorded as `[host]:port`. -- name: Scan host key for each fleet machine - ansible.builtin.command: >- - ssh-keyscan -t ed25519 -T 5 -p {{ hostvars[item]['ansible_port'] | default(22) }} - {{ machines[item]['hostname'] }} - register: host_key_scan - changed_when: false - failed_when: false - loop: "{{ machines.keys() | list }}" - when: item != inventory_hostname - -- name: Remove stale known_hosts entries - ansible.builtin.known_hosts: - name: "{{ _kh_name }}" - state: absent - vars: - _kh_host: "{{ machines[item.item]['hostname'] }}" - _kh_port: "{{ hostvars[item.item]['ansible_port'] | default(22) | int }}" - _kh_name: "{{ _kh_host if _kh_port | int == 22 else '[' ~ _kh_host ~ ']:' ~ _kh_port }}" - loop: "{{ host_key_scan.results | default([]) }}" - when: - - item.rc is defined - - item.rc == 0 - - item.stdout | length > 0 - loop_control: - label: "{{ item.item }}" - -- name: Add current host key to known_hosts - ansible.builtin.known_hosts: - name: "{{ _kh_name }}" - key: "{{ item.stdout }}" - state: present - hash_host: false - vars: - _kh_host: "{{ machines[item.item]['hostname'] }}" - _kh_port: "{{ hostvars[item.item]['ansible_port'] | default(22) | int }}" - _kh_name: "{{ _kh_host if _kh_port | int == 22 else '[' ~ _kh_host ~ ']:' ~ _kh_port }}" - loop: "{{ host_key_scan.results | default([]) }}" - when: - - item.rc is defined - - item.rc == 0 - - item.stdout | length > 0 - loop_control: - label: "{{ item.item }}" - -# ── Distribute all machine keys to authorized_keys ── -# One bw list call to get all SSH keys, then write authorized_keys in one shot. -# Skipped silently when BW is locked, and the writer below refuses to overwrite -# if BW returned no keys — otherwise a session hiccup would wipe authorized_keys. -- name: Fetch all machine SSH public keys from Bitwarden (batch) - ansible.builtin.shell: | - set -euo pipefail - MACHINE_NAMES="{{ machines.keys() | list | join(' ') }}" - bw list items 2>/dev/null | python3 -c " - import sys, json - items = json.load(sys.stdin) - names = set('{{ user_login }}@' + m for m in '${MACHINE_NAMES}'.split()) - keys = [] - for item in items: - if item.get('name') in names: - pub = item.get('sshKey', {}).get('publicKey', '').strip() - if pub: - keys.append(pub) - print('\n'.join(keys)) - " - args: - executable: /bin/bash - register: all_machine_keys - changed_when: false - failed_when: false - when: bw_unlocked | default(false) - environment: - BW_SESSION: "{{ bw_session | default('') }}" - PATH: "{{ ansible_env_path }}" - -# Self-healing authorized_keys (shrink guard). -# A full overwrite gated only on "≥1 key" silently DROPS keys whenever the BW -# batch is partial (a machine's item momentarily missing, a `bw list` hiccup) — -# that is what repeatedly wiped himachal's key off goa. Instead, UNION the -# desired set with whatever is already authorized: every unlocked run re-adds -# all known machine keys (healing a prior wipe) and NEVER removes a live key. -# Decommissioning a key is now an explicit action: ssh_keys_force_authorized=true. -- name: Read current authorized_keys (shrink guard) - ansible.builtin.slurp: - src: "{{ ansible_facts['env']['HOME'] }}/.ssh/authorized_keys" - register: _existing_ak - failed_when: false - when: - - all_machine_keys.stdout_lines is defined - - all_machine_keys.stdout_lines | length > 0 - -- name: Compute authorized_keys sets - ansible.builtin.set_fact: - _ak_desired: "{{ (all_machine_keys.stdout_lines | default([]) + (extra_authorized_keys | default([]))) | map('trim') | select | unique | list }}" - _ak_current: "{{ ((_existing_ak.content | b64decode).splitlines() | map('trim') | select | list) if (_existing_ak.content is defined and _existing_ak.content is string) else [] }}" - when: - - all_machine_keys.stdout_lines is defined - - all_machine_keys.stdout_lines | length > 0 - -- name: Warn when a live key would have been removed (kept instead) - ansible.builtin.debug: - msg: >- - Kept {{ (_ak_current | difference(_ak_desired)) | length }} authorized key(s) that - are not in the current Bitwarden batch (likely a partial result). Set - ssh_keys_force_authorized=true only when intentionally decommissioning a key. - when: - - all_machine_keys.stdout_lines is defined - - all_machine_keys.stdout_lines | length > 0 - - (_ak_current | difference(_ak_desired)) | length > 0 - - not (ssh_keys_force_authorized | default(false)) - -- name: Write authorized_keys (union — self-healing, never silently removes a key) - ansible.builtin.copy: - dest: "{{ ansible_facts['env']['HOME'] }}/.ssh/authorized_keys" - content: "{{ ((_ak_desired if (ssh_keys_force_authorized | default(false)) else (_ak_current | union(_ak_desired))) | unique | join('\n')) }}\n" - mode: "0600" - when: - - all_machine_keys.stdout_lines is defined - - all_machine_keys.stdout_lines | length > 0 diff --git a/roles/ssh_keys/templates/ssh_config.j2 b/roles/ssh_keys/templates/ssh_config.j2 deleted file mode 100644 index 9d7704b8..00000000 --- a/roles/ssh_keys/templates/ssh_config.j2 +++ /dev/null @@ -1,24 +0,0 @@ -Host * - AddKeysToAgent yes - IdentityFile ~/.ssh/id_ed25519 - ServerAliveInterval 30 - ServerAliveCountMax 3 - # Reuse connections — big speedup for repeated SSH to the same host - ControlMaster auto - ControlPersist 10m - ControlPath ~/.ssh/cp/%C - # Don't hash known_hosts so tab-completion and doctor checks work - HashKnownHosts no - # Compression on slow links; negligible on LAN/Tailscale - Compression yes - -{% for name, info in machines.items() %} -Host {{ name }} - HostName {{ info.hostname }} -{% if hostvars[name] is defined and hostvars[name].ansible_port is defined %} - Port {{ hostvars[name].ansible_port }} -{% endif %} - User {{ hostvars[name].ansible_user | default(user_login) if hostvars[name] is defined else user_login }} - SendEnv BW_SESSION - -{% endfor %} diff --git a/roles/ssh_mesh/tasks/main.yml b/roles/ssh_mesh/tasks/main.yml deleted file mode 100644 index fcd7a8f0..00000000 --- a/roles/ssh_mesh/tasks/main.yml +++ /dev/null @@ -1,137 +0,0 @@ ---- -# ── Self-healing SSH mesh ───────────────────────────────────────── -# Distributes fleet public keys from git (machine_pubkeys in -# group_vars/all.yml — public keys are safe in a public repo) and -# verifies passwordless SSH to every fleet machine currently online -# on the tailnet. Stale known_hosts entries (rebuilt machines) are -# healed automatically. Runs on every apply, including the daily -# no-secrets timer — no Bitwarden needed. - -- name: Ensure SSH key pair exists - ansible.builtin.command: > - ssh-keygen -t ed25519 -C "{{ inventory_hostname }}" - -f "{{ ansible_facts['env']['HOME'] }}/.ssh/id_ed25519" -N "" - args: - creates: "{{ ansible_facts['env']['HOME'] }}/.ssh/id_ed25519" - -- name: Authorize all fleet public keys from git (non-destructive merge) - ansible.builtin.lineinfile: - path: "{{ ansible_facts['env']['HOME'] }}/.ssh/authorized_keys" - line: "{{ item.value }}" - create: true - mode: "0600" - loop: "{{ machine_pubkeys | default({}) | dict2items }}" - loop_control: - label: "{{ item.key }}" - -- name: Read this machine's public key - ansible.builtin.slurp: - src: "{{ ansible_facts['env']['HOME'] }}/.ssh/id_ed25519.pub" - register: mesh_own_pub - -- name: Warn if this machine's public key is not committed to git - ansible.builtin.debug: - msg: >- - WARNING: {{ inventory_hostname }}'s public key is missing or stale in - group_vars/all.yml machine_pubkeys. Other machines cannot self-heal - SSH to this host until it is committed: - {{ mesh_own_pub.content | b64decode | trim }} - when: >- - (machine_pubkeys | default({})).get(inventory_hostname, '').split()[:2] - != (mesh_own_pub.content | b64decode | trim).split()[:2] - -# ── Mesh verification ───────────────────────────────────────────── -# Same matcher as `just apply-all` (scripts/online_hosts.py, under test); -# `script` copies the file to the target, so phone-pushed applies work too. -- name: Get online tailnet peers - ansible.builtin.script: - cmd: "{{ playbook_dir }}/scripts/online_hosts.py --peers" - executable: python3 - register: mesh_online - changed_when: false - failed_when: false - -# A host is online when its tailnet name (inventory `tailnet_name`, else the -# inventory name) is — the same mapping scripts/online_hosts.py applies. -- name: Reset online fleet members - ansible.builtin.set_fact: - mesh_online_hosts: [] - -- name: Map online tailnet peers to inventory hosts - ansible.builtin.set_fact: - mesh_online_hosts: "{{ mesh_online_hosts + [item] }}" - loop: "{{ groups['all'] }}" - when: (hostvars[item].tailnet_name | default(item)) in (mesh_online.stdout_lines | default([])) - -# Retired VPSes are never probed. -- name: Compute mesh targets (online fleet members) - ansible.builtin.set_fact: - mesh_targets: >- - {{ machines.keys() | list - | select('in', mesh_online_hosts) - | difference(ssh_mesh_exclude | default([])) - | difference(groups.get('vps', [])) - | difference([inventory_hostname]) }} - -# ControlMaster=no: a hung mux socket must not poison the probe — we are -# testing the path + key auth, not the local multiplexer. Port and login -# user come from inventory, as in ssh_config.j2. -- name: Probe passwordless SSH to online fleet members - ansible.builtin.command: >- - timeout 15 ssh -o BatchMode=yes -o PasswordAuthentication=no - -o ControlMaster=no -o ControlPath=none - -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 - -p {{ hostvars[item].ansible_port | default(22) }} - {{ hostvars[item].ansible_user | default(user_login) }}@{{ machines[item].hostname }} true - loop: "{{ mesh_targets }}" - register: mesh_probe - changed_when: false - failed_when: false - -- name: Heal stale state and retry failed targets - ansible.builtin.shell: | - set -o pipefail - # reap a stale/hung mux master for this host, then refresh host key - timeout 5 ssh -O exit -p "$PORT" "$TARGET" >/dev/null 2>&1 || true - ssh-keygen -R "$KH_NAME" >/dev/null 2>&1 || true - ssh-keyscan -t ed25519 -T 5 -p "$PORT" "$HOST" >> ~/.ssh/known_hosts 2>/dev/null || true - timeout 15 ssh -o BatchMode=yes -o PasswordAuthentication=no \ - -o ControlMaster=no -o ControlPath=none \ - -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 \ - -p "$PORT" "$TARGET" true - args: - executable: /bin/bash - environment: - HOST: "{{ machines[item].hostname }}" - PORT: "{{ hostvars[item].ansible_port | default(22) }}" - TARGET: "{{ hostvars[item].ansible_user | default(user_login) }}@{{ machines[item].hostname }}" - KH_NAME: >- - {{ machines[item].hostname if (hostvars[item].ansible_port | default(22) | int) == 22 - else '[' ~ machines[item].hostname ~ ']:' ~ hostvars[item].ansible_port }} - loop: >- - {{ mesh_probe.results | default([]) - | rejectattr('rc', 'eq', 0) | map(attribute='item') | list }} - register: mesh_retry - changed_when: true - failed_when: false - -- name: Fail if SSH mesh is still broken - vars: - mesh_failed: >- - {{ mesh_retry.results | default([]) - | rejectattr('rc', 'eq', 0) | map(attribute='item') | list }} - ansible.builtin.fail: - msg: >- - SSH mesh broken from {{ inventory_hostname }} to: - {{ mesh_failed | join(', ') }}. - The remote is missing this machine's key in authorized_keys — it will - self-heal on its next daily apply, or run `just apply-remote-tags - mesh_check` now. Ensure this machine's pubkey is committed in - group_vars/all.yml machine_pubkeys. - when: mesh_failed | length > 0 - -- name: SSH mesh OK - ansible.builtin.debug: - msg: >- - SSH mesh OK from {{ inventory_hostname }} → - {{ mesh_targets | join(', ') if mesh_targets else 'no fleet peers online' }} diff --git a/roles/sshd/handlers/main.yml b/roles/sshd/handlers/main.yml deleted file mode 100644 index fc7562cc..00000000 --- a/roles/sshd/handlers/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# The unit is `ssh` on Debian/Ubuntu and `sshd` everywhere else (Fedora, Arch). -# ignore_errors covers non-systemd hosts (postmarketOS/OpenRC, Termux). -- name: Reload sshd - ansible.builtin.systemd: - name: "{{ 'ssh' if ansible_facts['os_family'] == 'Debian' else 'sshd' }}" - state: reloaded - become: true - ignore_errors: true diff --git a/roles/sshd/tasks/main.yml b/roles/sshd/tasks/main.yml deleted file mode 100644 index 6cbc3508..00000000 --- a/roles/sshd/tasks/main.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Check if /etc/ssh/sshd_config.d directory exists - ansible.builtin.stat: - path: /etc/ssh/sshd_config.d - register: sshd_config_d - -- name: Deploy AcceptEnv drop-in file - ansible.builtin.copy: - content: "AcceptEnv BW_SESSION BW_CLIENTID BW_CLIENTSECRET\n" - dest: /etc/ssh/sshd_config.d/99-bw-env.conf - mode: "0644" - become: true - when: sshd_config_d.stat.isdir is defined and sshd_config_d.stat.isdir - ignore_errors: true - notify: Reload sshd - -# Opt-in hardening for internet-facing hosts (sshd_harden: true in host_vars). -# Deliberately not fleet-wide: reloading sshd on the retired VPSes is off-limits. -- name: Deploy sshd hardening drop-in - ansible.builtin.copy: - content: | - # Managed by dotfiles (sshd role, sshd_harden) - PermitRootLogin no - PasswordAuthentication no - KbdInteractiveAuthentication no - X11Forwarding no - MaxAuthTries 3 - LoginGraceTime 30 - dest: /etc/ssh/sshd_config.d/10-hardening.conf - mode: "0644" - # sshd takes the first value it sees, so 10- beats the distro drop-ins. - validate: /usr/sbin/sshd -t -f %s - become: true - when: - - sshd_harden | default(false) - - sshd_config_d.stat.isdir is defined and sshd_config_d.stat.isdir - notify: Reload sshd diff --git a/roles/sudo/tasks/main.yml b/roles/sudo/tasks/main.yml deleted file mode 100644 index 3902d795..00000000 --- a/roles/sudo/tasks/main.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: Ensure passwordless sudo for current user - ansible.builtin.copy: - content: "{{ ansible_user_id | default(lookup('env', 'USER')) }} ALL=(ALL) NOPASSWD:ALL\n" - dest: "/etc/sudoers.d/zz-{{ ansible_user_id | default(lookup('env', 'USER')) }}" - mode: "0440" - validate: /usr/sbin/visudo -cf %s - become: true - -- name: Remove old sudoers files - ansible.builtin.file: - path: "/etc/sudoers.d/{{ item }}" - state: absent - loop: - - "{{ ansible_user_id | default(lookup('env', 'USER')) }}" - - "99-{{ ansible_user_id | default(lookup('env', 'USER')) }}" - become: true diff --git a/roles/syncthing/handlers/main.yml b/roles/syncthing/handlers/main.yml deleted file mode 100644 index 1397a097..00000000 --- a/roles/syncthing/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Reload systemd user daemon - ansible.builtin.systemd: - daemon_reload: true - scope: user diff --git a/roles/syncthing/tasks/main.yml b/roles/syncthing/tasks/main.yml deleted file mode 100644 index 8181e3d6..00000000 --- a/roles/syncthing/tasks/main.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -# The unit's ExecStart points at the brew binary; hosts without it would -# crash-loop with 203/EXEC, so only deploy/enable when it's installed. -- name: Check for syncthing binary - ansible.builtin.stat: - path: /home/linuxbrew/.linuxbrew/bin/syncthing - register: syncthing_bin - -- name: Create systemd user directory - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user" - state: directory - mode: "0755" - when: syncthing_bin.stat.exists - -- name: Copy syncthing service file - ansible.builtin.copy: - src: syncthing.service - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/syncthing.service" - mode: "0644" - notify: Reload systemd user daemon - when: syncthing_bin.stat.exists - -- name: Enable and start syncthing user service - ansible.builtin.systemd: - name: syncthing.service - scope: user - enabled: true - state: started - when: syncthing_bin.stat.exists - -- name: Stop and disable syncthing user service (binary missing) - ansible.builtin.systemd: - name: syncthing.service - scope: user - enabled: false - state: stopped - failed_when: false - when: not syncthing_bin.stat.exists diff --git a/roles/systemd/handlers/main.yml b/roles/systemd/handlers/main.yml deleted file mode 100644 index 4ec02f2f..00000000 --- a/roles/systemd/handlers/main.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Reload systemd user daemon - ansible.builtin.systemd: - daemon_reload: true - scope: user - -- name: Restart podman socket - ansible.builtin.systemd: - name: podman.socket - scope: system - state: restarted - daemon_reload: true - become: true diff --git a/roles/systemd/tasks/main.yml b/roles/systemd/tasks/main.yml deleted file mode 100644 index 5157a560..00000000 --- a/roles/systemd/tasks/main.yml +++ /dev/null @@ -1,206 +0,0 @@ ---- -# Detect if podman is available on this system -- name: Check if podman is installed - ansible.builtin.stat: - path: /usr/bin/podman - register: podman_available - -- name: Ensure podman group exists - ansible.builtin.group: - name: podman - system: true - state: present - become: true - when: podman_available.stat.exists - -- name: Add user to podman group - ansible.builtin.user: - name: "{{ ansible_facts['env']['USER'] }}" - groups: podman - append: true - become: true - when: podman_available.stat.exists - -- name: Create podman socket drop-in directory - ansible.builtin.file: - path: /etc/systemd/system/podman.socket.d - state: directory - mode: "0755" - become: true - when: podman_available.stat.exists - -- name: Deploy podman socket drop-in for group access - ansible.builtin.copy: - dest: /etc/systemd/system/podman.socket.d/group-access.conf - mode: "0644" - content: | - [Socket] - SocketGroup=podman - SocketMode=0660 - DirectoryMode=0710 - become: true - notify: Restart podman socket - when: podman_available.stat.exists - -- name: Enable and start rootful podman socket - ansible.builtin.systemd: - name: podman.socket - scope: system - enabled: true - state: started - daemon_reload: true - become: true - when: podman_available.stat.exists - -- name: Debug message for systems without podman - ansible.builtin.debug: - msg: "Podman not found on this system, skipping socket configuration" - when: not podman_available.stat.exists - -- name: Create systemd user directory - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user" - state: directory - mode: "0755" - -- name: Deploy dotfiles-update service - ansible.builtin.template: - src: dotfiles-update.service.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/dotfiles-update.service" - mode: "0644" - notify: Reload systemd user daemon - -# Laptops: run on login/boot via graphical-session, no periodic timer -- name: Enable dotfiles-update service on login (laptop) - ansible.builtin.systemd: - name: dotfiles-update.service - scope: user - enabled: true - when: is_laptop | bool and dotfiles_auto_update | default(true) | bool - -- name: Stop and disable dotfiles-update service when automatic updates are off - ansible.builtin.systemd: - name: dotfiles-update.service - scope: user - enabled: false - state: stopped - when: is_laptop | bool and not (dotfiles_auto_update | default(true) | bool) - failed_when: false - -- name: Remove disabled dotfiles-update service unit - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/dotfiles-update.service" - state: absent - when: is_laptop | bool and not (dotfiles_auto_update | default(true) | bool) - notify: Reload systemd user daemon - -- name: Remove dotfiles-update timer on laptop (not needed) - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/dotfiles-update.timer" - state: absent - when: is_laptop | bool - -# Desktops/servers: daily timer -- name: Deploy dotfiles-update timer (non-laptop) - ansible.builtin.template: - src: dotfiles-update.timer.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/dotfiles-update.timer" - mode: "0644" - when: not (is_laptop | bool) - notify: Reload systemd user daemon - -- name: Enable and start dotfiles-update timer (non-laptop) - ansible.builtin.systemd: - name: dotfiles-update.timer - scope: user - enabled: true - state: started - daemon_reload: true - when: not (is_laptop | bool) - ignore_errors: "{{ ansible_check_mode }}" - -- name: Deploy atuin daemon service - ansible.builtin.template: - src: atuin-daemon.service.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/atuin-daemon.service" - mode: "0644" - notify: Reload systemd user daemon - -- name: Enable and start atuin daemon - ansible.builtin.systemd: - name: atuin-daemon.service - scope: user - enabled: true - state: started - -- name: Deploy system-mitigator script (Lenovo 400MHz/Journald health) - ansible.builtin.copy: - src: system-mitigator.py - dest: "{{ ansible_facts['env']['HOME'] }}/.local/bin/system-mitigator.py" - mode: "0755" - when: inventory_hostname == 'himachal' - -- name: Deploy system-mitigator service template - ansible.builtin.template: - src: system-mitigator.service.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/system-mitigator.service" - mode: "0644" - when: inventory_hostname == 'himachal' - notify: Reload systemd user daemon - -- name: Enable and start system-mitigator daemon - ansible.builtin.systemd: - name: system-mitigator.service - scope: user - enabled: true - state: started - when: inventory_hostname == 'himachal' - -# Phone hosts (termux, mumbai) can't reliably self-apply on a schedule — -# Termux can't run Ansible at all (no Rust target for the cryptography -# dependency), and neither host is always reachable (asleep, app killed, -# off tailnet). himachal periodically pushes to them instead, tolerating -# unreachable as the expected common case rather than an error. -- name: Deploy phone-apply service template - ansible.builtin.template: - src: phone-apply.service.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/phone-apply.service" - mode: "0644" - when: inventory_hostname == 'himachal' - notify: Reload systemd user daemon - -- name: Deploy phone-apply timer template - ansible.builtin.template: - src: phone-apply.timer.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/phone-apply.timer" - mode: "0644" - when: inventory_hostname == 'himachal' - notify: Reload systemd user daemon - -- name: Enable and start phone-apply timer - ansible.builtin.systemd: - name: phone-apply.timer - scope: user - enabled: true - state: started - daemon_reload: true - when: inventory_hostname == 'himachal' and phone_apply_enabled | default(true) | bool - -- name: Stop and disable phone-apply timer when automatic pushes are off - ansible.builtin.systemd: - name: phone-apply.timer - scope: user - enabled: false - state: stopped - when: inventory_hostname == 'himachal' and not (phone_apply_enabled | default(true) | bool) - failed_when: false - -- name: Remove disabled phone-apply units - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/{{ item }}" - state: absent - loop: - - phone-apply.service - - phone-apply.timer - when: inventory_hostname == 'himachal' and not (phone_apply_enabled | default(true) | bool) - notify: Reload systemd user daemon diff --git a/roles/systemd/templates/phone-apply.service.j2 b/roles/systemd/templates/phone-apply.service.j2 deleted file mode 100644 index d7c23e07..00000000 --- a/roles/systemd/templates/phone-apply.service.j2 +++ /dev/null @@ -1,25 +0,0 @@ -[Unit] -Description=Periodic push-apply to phone-hosted machines (termux, mumbai) -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -WorkingDirectory={{ ansible_facts['env']['HOME'] }}/.local/share/dotfiles -ExecStartPre=/usr/bin/git pull --ff-only -# Phone hosts are frequently unreachable (asleep, app killed, off tailnet) -# — that's expected, not an error, so each apply is allowed to fail -# without failing the service (the leading "-"). -ExecStart=-{{ brew_prefix }}/bin/ansible-playbook -i inventory.yml -e target=termux site.yml --skip-tags secrets -ExecStart=-{{ brew_prefix }}/bin/ansible-playbook -i inventory.yml -e target=mumbai site.yml --skip-tags secrets -# Run the same tracked revision on each target, even if its Ansible pass failed. -# An archive avoids assuming the phones have a checkout or Ansible installed. -ExecStart=-:/bin/bash -c "set -o pipefail; /usr/bin/git archive HEAD | /usr/bin/ssh -o BatchMode=yes termux 'mkdir -p \"$HOME/.cache\"; d=$(mktemp -d \"$HOME/.cache/dotfiles-phone.XXXXXXXX\") || exit 1; tar -xf - -C \"$d\" && DOTFILES_MACHINE=termux bash \"$d/scripts/run-units.sh\" --no-secrets; rc=$?; rm -rf -- \"$d\"; exit \"$rc\"'" -ExecStart=-:/bin/bash -c "set -o pipefail; /usr/bin/git archive HEAD | /usr/bin/ssh -o BatchMode=yes mumbai 'mkdir -p \"$HOME/.cache\"; d=$(mktemp -d \"$HOME/.cache/dotfiles-phone.XXXXXXXX\") || exit 1; tar -xf - -C \"$d\" && DOTFILES_MACHINE=mumbai bash \"$d/scripts/run-units.sh\" --no-secrets; rc=$?; rm -rf -- \"$d\"; exit \"$rc\"'" -Environment="PATH={{ brew_prefix }}/bin:%h/.local/bin:/usr/local/bin:/usr/bin:/bin" -Environment="HOME={{ ansible_facts['env']['HOME'] }}" -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=default.target diff --git a/roles/tailscale/handlers/main.yml b/roles/tailscale/handlers/main.yml deleted file mode 100644 index c7513310..00000000 --- a/roles/tailscale/handlers/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Restart systemd-resolved - ansible.builtin.systemd: - name: systemd-resolved - state: restarted - become: true - failed_when: false - -- name: Apply sysctl settings - ansible.builtin.command: sysctl -p /etc/sysctl.d/99-tailscale.conf - become: true - changed_when: true diff --git a/roles/tailscale/tasks/configure_connected.yml b/roles/tailscale/tasks/configure_connected.yml deleted file mode 100644 index e4df0aa9..00000000 --- a/roles/tailscale/tasks/configure_connected.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- -# Already-connected machine: sync routes, hostname, DNS, desktop extras. - -- name: Read current Tailscale prefs (for idempotency comparison) - ansible.builtin.command: tailscale debug prefs - register: tailscale_prefs - changed_when: false - failed_when: false - when: tailscale_advertise_routes is defined - -- name: Advertise subnet routes (only when routes differ from current prefs) - ansible.builtin.command: > - tailscale up - --hostname={{ inventory_hostname }} - --advertise-routes={{ tailscale_advertise_routes }} - {% if tailscale_accept_dns is defined and not tailscale_accept_dns %}--accept-dns=false{% endif %} - {% if tailscale_operator is defined %}--operator={{ tailscale_operator }}{% endif %} - become: true - when: - - tailscale_advertise_routes is defined - - tailscale_advertise_routes not in (tailscale_prefs.stdout | default('')) - changed_when: true - -- name: Sync Tailscale hostname to match machine hostname - ansible.builtin.command: tailscale set --hostname={{ inventory_hostname }} - become: true - changed_when: false - -- name: Fix resolv.conf when systemd-resolved is not active - ansible.builtin.copy: - dest: /etc/resolv.conf - content: "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" - owner: root - group: root - mode: "0644" - force: true - become: true - when: - - tailscale_accept_dns is defined - - not tailscale_accept_dns - -- name: Set Tailscale operator to allow user-level management - ansible.builtin.command: tailscale set --operator={{ ansible_facts['env']['USER'] }} - become: true - when: is_desktop | bool - changed_when: false - failed_when: false - -- name: Install Tailscale systray service on desktop machines - ansible.builtin.command: tailscale configure systray --enable-startup=systemd - when: is_desktop | bool - changed_when: false - failed_when: false - -- name: Enable and start Tailscale systray service - ansible.builtin.shell: | - systemctl --user daemon-reload - systemctl --user enable --now tailscale-systray - when: is_desktop | bool - changed_when: false - failed_when: false diff --git a/roles/tailscale/tasks/configure_prereqs.yml b/roles/tailscale/tasks/configure_prereqs.yml deleted file mode 100644 index 71e477cb..00000000 --- a/roles/tailscale/tasks/configure_prereqs.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- -# DNS + sysctl pre-connection setup. Runs on every machine, connected or not. -# Ensures systemd-resolved is configured for Tailscale MagicDNS and rp_filter -# is set for exit node compatibility. - -- name: Create systemd-resolved drop-in directory - ansible.builtin.file: - path: /etc/systemd/resolved.conf.d - state: directory - mode: "0755" - become: true - -- name: Configure systemd-resolved to listen on [::1] for container DNS - ansible.builtin.copy: - dest: /etc/systemd/resolved.conf.d/ipv6-stub.conf - mode: "0644" - content: | - [Resolve] - DNSStubListenerExtra=::1 - become: true - notify: Restart systemd-resolved - -# Only repoint resolv.conf when resolved is actually running — otherwise the -# stub file doesn't exist and we'd break DNS on the host. -- name: Check whether systemd-resolved is active - ansible.builtin.command: systemctl is-active systemd-resolved # noqa: command-instead-of-module - register: tailscale_resolved_active - changed_when: false - failed_when: false - -- name: Point resolv.conf to systemd-resolved stub resolver - ansible.builtin.file: - src: /run/systemd/resolve/stub-resolv.conf - dest: /etc/resolv.conf - state: link - force: true - follow: false - become: true - notify: Restart systemd-resolved - when: - - not (tailscale_accept_dns is defined and not tailscale_accept_dns) - - tailscale_resolved_active.stdout | default('') == 'active' - -# rp_filter=2 (loose mode) is required for exit node traffic to route correctly. -- name: Configure sysctl for Tailscale exit node compatibility - ansible.builtin.copy: - dest: /etc/sysctl.d/99-tailscale.conf - mode: "0644" - content: | - net.ipv4.conf.all.rp_filter = 2 - net.ipv4.conf.default.rp_filter = 2 - net.ipv4.ip_forward = 1 - net.ipv6.conf.all.forwarding = 1 - become: true - notify: Apply sysctl settings diff --git a/roles/tailscale/tasks/connect.yml b/roles/tailscale/tasks/connect.yml deleted file mode 100644 index ea38bd8a..00000000 --- a/roles/tailscale/tasks/connect.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -# Join tailnet: fetch authkey from BW, cleanup old device, run tailscale up. - -# Need BW to join tailscale (auth key lives there). -- name: Warn and skip tailscale join (vault locked) - ansible.builtin.debug: - msg: >- - tailscale is not connected but the Bitwarden vault is locked on this host, - so the auth key can't be fetched. Unlock the vault (`bw unlock`, - `printf '%s' "$session" > /tmp/bw_session`) and re-run. - when: not (bw_unlocked | default(false)) - -- name: End role if vault is locked - ansible.builtin.meta: end_role - when: not (bw_unlocked | default(false)) - -- name: Fetch tailscale auth/API key from Bitwarden - ansible.builtin.shell: set -o pipefail && bw get item tailscale-apikey | python3 -c "import sys,json; print(json.load(sys.stdin)['login']['password'])" - register: tailscale_apikey - changed_when: false - no_log: true - environment: - PATH: "{{ ansible_env_path }}" - BW_SESSION: "{{ bw_session }}" - -- name: Remove old tailscale device with same hostname - block: - - name: Fetch tailscale device list - ansible.builtin.uri: - url: https://api.tailscale.com/api/v2/tailnet/-/devices - method: GET - user: "{{ tailscale_apikey.stdout }}" - password: "" - force_basic_auth: true - status_code: 200 - headers: - Content-Type: application/json - register: ts_devices - failed_when: false - no_log: true - - - name: Find old device matching our hostname - ansible.builtin.set_fact: - ts_old_device_id: >- - {{ (ts_devices.json.devices | default([]) - | selectattr('name', 'equalto', inventory_hostname + '.' + tailscale_domain) - | map(attribute='id') | first) | default('') }} - - - name: Delete old tailscale device - ansible.builtin.uri: - url: https://api.tailscale.com/api/v2/device/{{ ts_old_device_id }} - method: DELETE - user: "{{ tailscale_apikey.stdout }}" - password: "" - force_basic_auth: true - status_code: 200 - when: ts_old_device_id | length > 0 - no_log: true - when: tailscale_apikey.stdout | default('') | length > 0 - -- name: Join tailscale network - ansible.builtin.command: tailscale up --authkey {{ tailscale_apikey.stdout }} --advertise-tags=tag:fleet --accept-routes --hostname {{ inventory_hostname }} - become: true - no_log: true - -- name: Advertise subnet routes after join - ansible.builtin.command: > - tailscale up --advertise-routes={{ tailscale_advertise_routes }} - --reset=false - become: true - when: tailscale_advertise_routes is defined - changed_when: true diff --git a/roles/tailscale/tasks/install.yml b/roles/tailscale/tasks/install.yml deleted file mode 100644 index 697abe86..00000000 --- a/roles/tailscale/tasks/install.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# Tailscale binary installation. Two adapters: ujust (Bluefin) or upstream install script. - -- name: Check if tailscale command exists - ansible.builtin.command: which tailscale - register: _ts_installed - changed_when: false - failed_when: false - -- name: End install phase if already installed - ansible.builtin.meta: end_role - when: _ts_installed.rc == 0 - -- name: Check if ujust exists - ansible.builtin.command: which ujust - register: _ujust_exists - changed_when: false - failed_when: false - -- name: Install tailscale via ujust (Bluefin) - ansible.builtin.command: ujust install-tailscale - when: _ujust_exists.rc == 0 - -- name: Install tailscale via install script (non-Bluefin) - # Download fully, then run: a dropped connection can't execute half a script. - ansible.builtin.shell: | - set -eu - tmp_script=$(mktemp) - trap 'rm -f "$tmp_script"' EXIT - curl -fsSL --max-time 60 https://tailscale.com/install.sh -o "$tmp_script" - [ -s "$tmp_script" ] || { echo "Downloaded script is empty"; exit 1; } - sh "$tmp_script" - become: true - when: _ujust_exists.rc != 0 diff --git a/roles/tailscale/tasks/main.yml b/roles/tailscale/tasks/main.yml deleted file mode 100644 index f8b2daa8..00000000 --- a/roles/tailscale/tasks/main.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -# ── Phase 1: Network prerequisites (always, before connection check) ── -- name: Configure DNS and sysctl prerequisites - ansible.builtin.include_tasks: configure_prereqs.yml - -# ── Phase 2: Check connection state ── -- name: Check if tailscale is already connected - ansible.builtin.command: tailscale status - register: _ts_status - changed_when: false - failed_when: false - -# ── Phase 3a: Already connected — configure routes, hostname, desktop extras ── -- name: Configure connected Tailscale - ansible.builtin.include_tasks: configure_connected.yml - when: _ts_status.rc == 0 - -# ── Phase 3b: Not connected — install, fetch auth, join ── -- name: End role if already connected - ansible.builtin.meta: end_role - when: _ts_status.rc == 0 - -- name: Install Tailscale - ansible.builtin.include_tasks: install.yml - -- name: Join tailnet - ansible.builtin.include_tasks: connect.yml diff --git a/roles/tailscale_cert/handlers/main.yml b/roles/tailscale_cert/handlers/main.yml deleted file mode 100644 index 12188a80..00000000 --- a/roles/tailscale_cert/handlers/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Reload systemd daemon - ansible.builtin.systemd: - daemon_reload: true - scope: system - become: true diff --git a/roles/tailscale_cert/tasks/main.yml b/roles/tailscale_cert/tasks/main.yml deleted file mode 100644 index 8bf6dba0..00000000 --- a/roles/tailscale_cert/tasks/main.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- -- name: Create Tailscale cert directory - ansible.builtin.file: - path: /etc/ssl/tailscale - state: directory - owner: root - group: root - mode: "0755" - become: true - -- name: Check if Tailscale cert is still valid (30+ days) - ansible.builtin.shell: | - openssl x509 -checkend 2592000 -noout \ - -in /etc/ssl/tailscale/cert.pem 2>/dev/null && echo "valid" || echo "renew" - register: cert_validity - changed_when: false - failed_when: false - become: true - -- name: Provision Tailscale TLS certificate - ansible.builtin.command: - cmd: > - tailscale cert - --cert-file /etc/ssl/tailscale/cert.pem - --key-file /etc/ssl/tailscale/key.pem - --min-validity 720h - {{ inventory_hostname }}.{{ tailscale_domain }} - become: true - register: ts_cert_result - changed_when: "'wrote' in ts_cert_result.stderr" - failed_when: ts_cert_result.rc != 0 - when: cert_validity.stdout != 'valid' - -- name: Set cert permissions - ansible.builtin.file: - path: "{{ item.path }}" - owner: root - group: root - mode: "{{ item.mode }}" - become: true - loop: - - { path: /etc/ssl/tailscale/cert.pem, mode: "0644" } - - { path: /etc/ssl/tailscale/key.pem, mode: "0640" } - -- name: Deploy cert renewal systemd service - ansible.builtin.copy: - dest: /etc/systemd/system/tailscale-cert-renew.service - mode: "0644" - content: | - [Unit] - Description=Renew Tailscale TLS certificate - After=tailscaled.service - Wants=tailscaled.service - - [Service] - Type=oneshot - ExecStart=/usr/bin/tailscale cert \ - --cert-file /etc/ssl/tailscale/cert.pem \ - --key-file /etc/ssl/tailscale/key.pem \ - --min-validity 720h \ - {{ inventory_hostname }}.{{ tailscale_domain }} - become: true - notify: Reload systemd daemon - -- name: Deploy cert renewal systemd timer - ansible.builtin.copy: - dest: /etc/systemd/system/tailscale-cert-renew.timer - mode: "0644" - content: | - [Unit] - Description=Weekly Tailscale TLS certificate renewal - - [Timer] - OnCalendar=weekly - RandomizedDelaySec=1h - Persistent=true - - [Install] - WantedBy=timers.target - become: true - notify: Reload systemd daemon - -- name: Enable cert renewal timer - ansible.builtin.systemd: - name: tailscale-cert-renew.timer - scope: system - enabled: true - state: started - daemon_reload: true - become: true diff --git a/roles/termux_packages/defaults/main.yml b/roles/termux_packages/defaults/main.yml deleted file mode 100644 index ef8c2470..00000000 --- a/roles/termux_packages/defaults/main.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -# Captured from a live `pkg list-installed` on the phone (2026-07-20) plus -# what was added this session to get SSH + Ansible + the pi coding agent working. -termux_pkgs: - - openssh - - git - - python - - nodejs - - fish - - vim - - nano - - termux-api - - curl - - unzip - # file + lsattr: Ansible's stat/copy/template modules look these up via - # PATH and hard-fail without them (Termux ships neither by default). - - file - - e2fsprogs - -# Global npm packages usable directly under Termux's bionic libc. -# claude-code, codex, and antigravity-cli ship no android-arm64 native -# binary and cannot run here — they belong on the Debian VM (the -# `mumbai` host). -termux_npm_global: - - "@earendil-works/pi-coding-agent" - -# How often the JobScheduler keep-alive job re-asserts wake-lock + sshd. -# Android N+ enforces a 900000ms (15min) floor for periodic jobs; this is -# deliberately looser to be less aggressive on battery. -termux_keepalive_period_ms: 1800000 diff --git a/roles/termux_packages/tasks/main.yml b/roles/termux_packages/tasks/main.yml deleted file mode 100644 index fdbc0760..00000000 --- a/roles/termux_packages/tasks/main.yml +++ /dev/null @@ -1,70 +0,0 @@ ---- -# Package + sshd management for the raw Termux/Android layer on the phone. -# Only runs for hosts in the `termux` inventory group — see site.yml. - -- name: Update pkg package cache - ansible.builtin.command: pkg update -y - changed_when: true - register: termux_pkg_update - until: termux_pkg_update.rc == 0 - retries: 3 - delay: 5 - -- name: Install Termux packages - ansible.builtin.command: - cmd: "pkg install -y {{ termux_pkgs | join(' ') }}" - changed_when: "'Setting up' in (termux_pkg_install.stdout | default(''))" - register: termux_pkg_install - -- name: Install global npm packages - ansible.builtin.command: - cmd: "npm install -g {{ item }}" - loop: "{{ termux_npm_global }}" - register: termux_npm_install - changed_when: "'added' in (termux_npm_install.stdout | default(''))" - -- name: Ensure Termux:Boot directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.termux/boot" - state: directory - mode: "0755" - -- name: Deploy sshd boot script - ansible.builtin.template: - src: start-sshd.sh.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.termux/boot/start-sshd" - mode: "0755" - -- name: Check if sshd is currently running - ansible.builtin.command: pgrep -x sshd - register: termux_sshd_running - changed_when: false - failed_when: false - -- name: Start sshd if not running - ansible.builtin.command: sshd - when: termux_sshd_running.rc != 0 - -# Termux:Boot only fires at device boot — if Android kills the app in the -# background between opens (common; a phone isn't an always-on machine), -# sshd goes unreachable until the user manually reopens Termux. Android's -# native JobScheduler (exposed via termux-job-scheduler, part of -# Termux:API) survives that: it wakes the app periodically regardless of -# whether it was killed, so we reuse the boot script as a periodic job -# too — same wake-lock + ensure-sshd logic, just on a timer instead of -# only at boot. -- name: Check currently scheduled Termux jobs - ansible.builtin.command: termux-job-scheduler -p - register: termux_jobs_pending - changed_when: false - failed_when: false - -- name: Schedule periodic sshd keep-alive job - ansible.builtin.command: - cmd: >- - termux-job-scheduler - --job-id 1 - --period-ms {{ termux_keepalive_period_ms }} - --persisted true - -s {{ ansible_facts['env']['HOME'] }}/.termux/boot/start-sshd - when: "'Pending Job 1' not in (termux_jobs_pending.stdout | default(''))" diff --git a/roles/termux_packages/templates/start-sshd.sh.j2 b/roles/termux_packages/templates/start-sshd.sh.j2 deleted file mode 100644 index c607ca24..00000000 --- a/roles/termux_packages/templates/start-sshd.sh.j2 +++ /dev/null @@ -1,10 +0,0 @@ -#!/data/data/com.termux/files/usr/bin/sh -# Deployed by termux_packages role. Runs twice over: once at device boot -# via Termux:Boot, and periodically via Android's JobScheduler -# (termux-job-scheduler) so sshd comes back even if Android killed -# Termux in the background between opens — a phone isn't an always-on -# machine. Requires the Termux:Boot and Termux:API companion apps -# installed separately (not available via pkg — install from -# F-Droid/GitHub releases). -termux-wake-lock -sshd diff --git a/roles/zen_browser/tasks/main.yml b/roles/zen_browser/tasks/main.yml deleted file mode 100644 index 15eb4011..00000000 --- a/roles/zen_browser/tasks/main.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- -- name: Add Zen Browser to GNOME dock and remove Firefox/Thunderbird - ansible.builtin.shell: | - python3 << 'EOF' - import subprocess, ast, os, sys - os.environ['DBUS_SESSION_BUS_ADDRESS'] = f"unix:path=/run/user/{os.getuid()}/bus" - os.environ['XDG_RUNTIME_DIR'] = f"/run/user/{os.getuid()}" - result = subprocess.run( - ['gsettings', 'get', 'org.gnome.shell', 'favorite-apps'], - capture_output=True, text=True) - if result.returncode != 0: - print(f"gsettings get failed: {result.stderr}", file=sys.stderr) - sys.exit(1) - apps = ast.literal_eval(result.stdout.strip()) if result.stdout.strip() else [] - remove = {'org.mozilla.firefox.desktop', 'org.mozilla.Thunderbird.desktop'} - apps = [a for a in apps if a not in remove] - zen = 'app.zen_browser.zen.desktop' - if zen not in apps: - apps.insert(0, zen) - r = subprocess.run( - ['gsettings', 'set', 'org.gnome.shell', 'favorite-apps', repr(apps)], - capture_output=True, text=True) - if r.returncode != 0: - print(f"gsettings set failed: {r.stderr}", file=sys.stderr) - sys.exit(1) - print("dock updated") - EOF - register: dock_result - changed_when: "'dock updated' in dock_result.stdout" - failed_when: dock_result.rc != 0 - ignore_errors: true - when: is_desktop | bool - -- name: Set Zen Browser as default browser - ansible.builtin.shell: | - test -d "{{ ansible_facts['env']['HOME'] }}/.var/app/app.zen_browser.zen" || exit 0 - for mime in x-scheme-handler/http x-scheme-handler/https x-scheme-handler/ftp text/html text/xml application/xhtml+xml application/xml; do - if command -v xdg-mime &>/dev/null; then - xdg-mime default app.zen_browser.zen.desktop "$mime" - else - gio mime "$mime" app.zen_browser.zen.desktop - fi - done - changed_when: false - ignore_errors: true - -- name: Check if Zen Browser profiles.ini exists - ansible.builtin.stat: - path: "{{ ansible_facts['env']['HOME'] }}/.var/app/app.zen_browser.zen/.zen/profiles.ini" - register: zen_profiles_ini - -- name: Skip if Zen Browser is not installed - ansible.builtin.meta: end_role - when: not zen_profiles_ini.stat.exists - -- name: Find Zen Browser profile path - ansible.builtin.shell: | - python3 -c " - import configparser - c = configparser.ConfigParser() - c.read('{{ ansible_facts['env']['HOME'] }}/.var/app/app.zen_browser.zen/.zen/profiles.ini') - print(c['Profile0']['Path']) - " - register: zen_profile_path - changed_when: false - -- name: Copy user.js to Zen Browser profile - ansible.builtin.copy: - src: user.js - dest: "{{ ansible_facts['env']['HOME'] }}/.var/app/app.zen_browser.zen/.zen/{{ zen_profile_path.stdout }}/user.js" - mode: "0644" - -- name: Ensure Zen Browser distribution directory exists (Flatpak bundle) - ansible.builtin.file: - path: /var/lib/flatpak/app/app.zen_browser.zen/current/active/files/zen/distribution - state: directory - owner: root - group: root - mode: "0755" - become: true - -- name: Deploy policies.json to Flatpak bundle directory (ensures Bitwarden extension is force-installed) - ansible.builtin.copy: - src: policies.json - dest: /var/lib/flatpak/app/app.zen_browser.zen/current/active/files/zen/distribution/policies.json - owner: root - group: root - mode: "0644" - become: true - -- name: Ensure Zen Browser distribution directory exists - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.zen/distribution" - state: directory - mode: "0755" - -- name: Copy policies.json to Zen Browser user distribution (fallback) - ansible.builtin.copy: - src: policies.json - dest: "{{ ansible_facts['env']['HOME'] }}/.zen/distribution/policies.json" - mode: "0644" - -# ── Firefox Accounts sign-in ────────────────────────────────────────────────── - -- name: Sign in to Firefox Accounts (Zen Browser) - ansible.builtin.include_role: - name: browser_fxa diff --git a/scripts/online_hosts.py b/scripts/online_hosts.py index 10d5dee0..ff86ddca 100755 --- a/scripts/online_hosts.py +++ b/scripts/online_hosts.py @@ -3,9 +3,8 @@ Used by the Justfile's `_online_hosts`. Tailscale failure → empty output. -`--peers` prints just the online tailnet names, one per line, and needs -nothing but this file: roles/ssh_mesh copies it to the target with the -`script` module, so both callers share one matching rule (online_peers). +`--peers` prints just the online tailnet names, one per line; both callers +share the same matching rule (online_peers). """ import json import os diff --git a/scripts/osquery-fleet.py b/scripts/osquery-fleet.py index ffd587f9..f4c8b04f 100755 --- a/scripts/osquery-fleet.py +++ b/scripts/osquery-fleet.py @@ -3,8 +3,8 @@ Offline, unsupported and excluded hosts remain visible in coverage. Drift is a monitoring result, while stale data and collection failures make the CLI fail. -Optional snapshots are private and atomic. Subprocess diagnostics/report extras -are not forwarded: effective Ansible inventory can contain secrets. +Optional snapshots are private and atomic. Inventory diagnostics never expose +host variable values. """ import argparse from datetime import datetime, timedelta, timezone @@ -19,6 +19,8 @@ import tempfile from online_hosts import online_peers +sys.path.append(str(Path(__file__).resolve().parent.parent / "units")) +from osquery_native_inventory import load_inventory INVENTORY = Path(__file__).resolve().parent.parent / "inventory.yml" REPORT_PATH = "/var/lib/fleet-osquery/report.json" @@ -47,9 +49,6 @@ def command_json(command, label, timeout=30): return value -def load_inventory(path): - return command_json(["ansible-inventory", "--inventory", str(path), "--list"], - "ansible-inventory") def load_status(): diff --git a/scripts/record-apply.py b/scripts/record-apply.py index 3c3a9de3..7ef090af 100755 --- a/scripts/record-apply.py +++ b/scripts/record-apply.py @@ -3,8 +3,8 @@ Usage: record-apply.py EXIT_CODE [LABEL [SKIP_TAGS]] -Called from each apply* recipe in the Justfile right after the ansible-playbook -invocation. `just doctor` reads the file to report convergence freshness. +Called from each apply recipe in the Justfile after running native units. +`just doctor` reads the file to report convergence freshness. """ import json import os diff --git a/scripts/run-units.sh b/scripts/run-units.sh index 4679ea76..61debd02 100755 --- a/scripts/run-units.sh +++ b/scripts/run-units.sh @@ -1,21 +1,50 @@ #!/usr/bin/env bash -# Run the bash units (units/*.sh) through mise, in parallel, failure-isolated. +# Run the fleet units in prerequisite order, continuing after independent failures. # Usage: scripts/run-units.sh [--no-secrets] [--tags a,b] set -uo pipefail -ALL_UNITS="chezmoi fonts omp atuin shell-ai github kube forgejo-registry" # run order for the sequential fallback; keep in step with mise.toml -tasks_for_tag() { case "$1" in # Ansible tag -> units (mirrors the old role tags) - dotfiles) echo "chezmoi fonts omp atuin shell-ai" ;; - shell) echo "chezmoi fonts atuin shell-ai" ;; - fonts) echo fonts ;; - git|neovim|editor) echo chezmoi ;; - atuin) echo atuin ;; - ai) echo shell-ai ;; - secrets) echo "atuin shell-ai github kube forgejo-registry" ;; - github) echo github ;; - kube) echo kube ;; - forgejo|registry) echo forgejo-registry ;; - omp) echo omp ;; +ALL_UNITS="sudo sshd apk-packages termux-packages homebrew flatpak chezmoi fonts omp atuin shell-ai ssh-keys ssh-mesh github kube forgejo-registry tailscale bluefin gnome zen-browser browser-fxa pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services proxy tailscale-cert bst-dashboard kirocrew fleet-facts" +tasks_for_tag() { case "$1" in + system) echo "sudo sshd apk-packages termux-packages retired-services server-hardening" ;; + sudo) echo sudo ;; + sshd) echo sshd ;; + packages) echo "apk-packages termux-packages homebrew flatpak" ;; + apk) echo apk-packages ;; + termux) echo termux-packages ;; + homebrew) echo homebrew ;; + flatpak) echo flatpak ;; + dotfiles) echo "chezmoi fonts omp atuin shell-ai" ;; + shell) echo "chezmoi fonts atuin shell-ai" ;; + fonts) echo fonts ;; + git|neovim|editor) echo chezmoi ;; + atuin) echo atuin ;; + ai) echo shell-ai ;; + secrets) echo "atuin shell-ai ssh-keys github kube forgejo-registry tailscale browser-fxa" ;; + ssh|ssh_keys) echo "ssh-keys ssh-mesh" ;; + mesh_check) echo ssh-mesh ;; + github) echo github ;; + kube) echo kube ;; + forgejo|registry) echo forgejo-registry ;; + tailscale) echo tailscale ;; + omp) echo omp ;; + desktop) echo "flatpak bluefin gnome zen-browser browser-fxa pipewire-audio" ;; + bluefin) echo bluefin ;; + gnome) echo gnome ;; + browser|zen_browser) echo "zen-browser browser-fxa" ;; + audio|pipewire) echo pipewire-audio ;; + services) echo "syncthing laptop-services system-services osquery proxy tailscale-cert bst-dashboard kirocrew" ;; + syncthing) echo syncthing ;; + systemd) echo "laptop-services system-services" ;; + osquery) echo osquery ;; + security) echo "osquery server-hardening" ;; + hardening) echo server-hardening ;; + retired_services) echo retired-services ;; + proxy|caddy) echo proxy ;; + tailscale_cert) echo tailscale-cert ;; + bst_dashboard) echo bst-dashboard ;; + kirocrew) echo kirocrew ;; + hive|hive_ops) echo hive-ops ;; + facts|fleet_facts) echo fleet-facts ;; esac; } repo="$(cd "$(dirname "$0")/.." && pwd)" @@ -26,7 +55,10 @@ tags="" while [ $# -gt 0 ]; do case "$1" in --no-secrets) export UNITS_NO_SECRETS=1 ;; - --tags) tags="${2:-}"; shift ;; + --tags) + [ $# -ge 2 ] && [ -n "$2" ] || { echo 'missing value for --tags' >&2; exit 2; } + case "$2" in ,*|*,|*,,*) echo 'empty unit tag' >&2; exit 2 ;; esac + tags="$2"; shift ;; *) echo "usage: $0 [--no-secrets] [--tags a,b]" >&2; exit 2 ;; esac shift @@ -38,29 +70,76 @@ if [ -z "$tags" ]; then else IFS=',' read -ra tag_list <<< "$tags" for t in "${tag_list[@]}"; do - for u in $(tasks_for_tag "$t"); do wanted="$wanted$u "; done + mapped=$(tasks_for_tag "$t") + if [ -z "$mapped" ]; then echo "unknown unit tag: $t" >&2; exit 2; fi + read -ra mapped_units <<< "$mapped" + for u in "${mapped_units[@]}"; do wanted="$wanted$u "; done done fi -selected=() -for u in $ALL_UNITS; do - case "$wanted" in *" $u "*) selected+=("$u") ;; esac -done -[ "${#selected[@]}" -gt 0 ] || exit 0 +selected() { case "$wanted" in *" $1 "*) return 0 ;; *) return 1 ;; esac; } cd "$repo" || exit 1 -if command -v mise >/dev/null 2>&1; then +if [ "${UNITS_CHECK:-0}" = 1 ]; then + # mise trust updates user configuration; do not write it during a check. + have_mise=0 +elif command -v mise >/dev/null 2>&1; then mise trust -q "$repo/mise.toml" >/dev/null 2>&1 || true - args=() - for u in "${selected[@]}"; do - [ "${#args[@]}" -gt 0 ] && args+=(":::") - args+=("unit:$u") - done - exec mise run --continue-on-error --jobs "${UNITS_JOBS:-8}" --output prefix --timeout 15m "${args[@]}" + have_mise=1 +else + echo "mise not installed; running units sequentially" + have_mise=0 fi -echo "mise not installed; running units sequentially (brew install mise for parallel runs)" +# A stage runs selected units in parallel where available. Non-selected +# prerequisites are never implicitly applied by a tag-only invocation. +run_stage() { + local unit rc=0 + local -a units=() args=() + for unit in "$@"; do selected "$unit" && units+=("$unit"); done + [ "${#units[@]}" -gt 0 ] || return 0 + if [ "$have_mise" = 1 ]; then + for unit in "${units[@]}"; do + [ "${#args[@]}" -eq 0 ] || args+=(":::") + args+=("unit:$unit") + done + mise run --continue-on-error --jobs "${UNITS_JOBS:-8}" --output prefix --timeout 15m "${args[@]}" + else + for unit in "${units[@]}"; do bash "$repo/units/$unit.sh" || rc=1; done + return "$rc" + fi +} + rc=0 -for u in "${selected[@]}"; do - bash "$repo/units/$u.sh" || rc=1 +# Package managers and the audio unit must not transact concurrently. Packages +# also provide yq/chezmoi on new machines, before inventory-dependent units. +for unit in sudo apk-packages termux-packages homebrew flatpak sshd; do + run_stage "$unit" || rc=1 +done +ssh_ok=1; run_stage ssh-keys || { ssh_ok=0; rc=1; } +tailscale_ok=1; run_stage tailscale || { tailscale_ok=0; rc=1; } +# Desktop settings and profile creation are ordered to avoid overwriting one +# another; other independent units can still run when one step fails. +for unit in chezmoi bluefin gnome; do + run_stage "$unit" || rc=1 done -exit $rc +zen_ok=1; run_stage zen-browser || { zen_ok=0; rc=1; } + +later=(fonts omp atuin shell-ai kube forgejo-registry pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services bst-dashboard kirocrew) +if [ "$zen_ok" = 1 ]; then + later+=(browser-fxa) +elif selected browser-fxa; then + echo "SKIPPED: browser-fxa requires a configured Zen profile" >&2 +fi +if [ "$ssh_ok" = 1 ]; then + later+=(ssh-mesh github) +else + for unit in ssh-mesh github; do selected "$unit" && echo "SKIPPED: $unit requires a valid SSH identity" >&2; done +fi +if [ "$tailscale_ok" = 1 ]; then + later+=(proxy tailscale-cert) +else + for unit in proxy tailscale-cert; do selected "$unit" && echo "SKIPPED: $unit requires Tailscale" >&2; done +fi +run_stage "${later[@]}" || rc=1 +run_stage fleet-facts || rc=1 +exit "$rc" diff --git a/site.yml b/site.yml deleted file mode 100644 index 493a8760..00000000 --- a/site.yml +++ /dev/null @@ -1,151 +0,0 @@ ---- -- name: Configure workstation - hosts: "{{ target | default(lookup('file', '/etc/dotfiles-machine', errors='ignore') | default('localhost')) }}" - gather_facts: true - - pre_tasks: - - name: Set convenience facts - ansible.builtin.set_fact: - is_desktop: "{{ is_desktop | default(inventory_hostname in groups.get('desktop', [])) }}" - is_laptop: "{{ is_laptop | default(false) }}" - brew_bin: "{{ brew_prefix }}/bin" - machine_profile: "{{ machine_profile | default('') }}" - tags: always - - - name: Ensure brew is in PATH for this run - ansible.builtin.set_fact: - ansible_env_path: "{{ brew_bin }}:{{ ansible_facts['env']['PATH'] }}" - tags: always - - # Platform detection, computed once — apk_packages, homebrew, and - # termux_packages each need to know libc flavor to decide whether to - # run; previously each re-implemented the same stat checks. - - name: Detect musl libc - ansible.builtin.stat: - path: "{{ item }}" - get_mime: false - get_checksum: false - get_attributes: false - register: _musl_stat - loop: - - /lib/ld-musl-x86_64.so.1 - - /lib/ld-musl-aarch64.so.1 - tags: always - - - name: Detect Termux (Android/bionic libc) - ansible.builtin.stat: - path: /data/data/com.termux/files/usr - get_mime: false - get_checksum: false - get_attributes: false - register: _termux_stat - tags: always - - - name: Set platform facts - ansible.builtin.set_fact: - is_musl: "{{ _musl_stat.results | selectattr('stat.exists') | list | length > 0 }}" - is_termux: "{{ _termux_stat.stat.exists }}" - tags: always - - environment: - PATH: "{{ brew_bin | default('/home/linuxbrew/.linuxbrew/bin') }}:/home/linuxbrew/.linuxbrew/bin:/data/data/com.termux/files/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - # Termux's non-interactive module execution doesn't reliably inherit - # HOME (Android has no real getpwuid()-backed home dir for the app - # user), which breaks every "~/..." dest: path in copy/template tasks. - # Re-assert it from the fact already gathered at play start. - HOME: "{{ ansible_facts['env']['HOME'] | default(omit) }}" - - roles: - - role: sudo - tags: [system, sudo] - when: not skip_sudo | default(false) - # Phase 1: System + packages (no secrets needed) - - role: sshd - tags: [system, sshd] - - role: apk_packages - tags: [system, packages, apk] - - role: homebrew - tags: [packages, homebrew] - - role: termux_packages - tags: [system, packages, termux] - when: inventory_hostname in groups.get('termux_hosts', []) - - role: bitwarden - tags: [secrets, bitwarden] - - role: hive_ops - tags: [hive, hive_ops] - - # Phase 2: Secrets + auth - - role: ssh_keys - tags: [secrets, ssh, ssh_keys] - - role: ssh_mesh - tags: [ssh, mesh_check] - when: machine_profile != 'vm-test' and not skip_ssh_mesh | default(false) - - role: tailscale - tags: [secrets, tailscale] - - # Phase 3: Desktop apps - - role: flatpak - tags: [packages, desktop, flatpak] - when: is_desktop | bool and machine_profile != 'vm-test' and not skip_flatpak | default(false) - - role: bluefin_common - tags: [desktop, bluefin] - when: is_desktop | bool and machine_profile != 'vm-test' and not skip_bluefin | default(false) - - role: gnome - tags: [desktop, gnome] - when: is_desktop | bool and machine_profile != 'vm-test' and not skip_gnome | default(false) - - role: zen_browser - tags: [desktop, browser, zen_browser] - when: is_desktop | bool and machine_profile != 'vm-test' and not skip_zen_browser | default(false) - - role: pipewire_audio - tags: [desktop, audio, pipewire] - when: is_desktop | bool and machine_profile != 'vm-test' and not skip_pipewire_audio | default(false) - - # Phase 4: Services - - role: syncthing - tags: [services, syncthing] - when: inventory_hostname not in groups.get('vps', []) and machine_profile != 'vm-test' and not skip_syncthing | default(false) - - role: systemd - tags: [services, systemd] - when: not skip_systemd | default(false) - - role: osquery - tags: [services, security, osquery] - when: inventory_hostname not in groups.get('vps', []) and not skip_osquery | default(false) - - role: bst_dashboard - tags: [services, bst_dashboard] - - role: kirocrew - tags: [services, kirocrew] - when: kirocrew_enabled | default(false) - - role: proxy - tags: [services, proxy, caddy] - when: inventory_hostname not in groups.get('vps', []) and machine_profile != 'vm-test' and not skip_proxy | default(false) - - role: tailscale_cert - tags: [services, tailscale_cert] - when: inventory_hostname not in groups.get('vps', []) and machine_profile != 'vm-test' and not skip_tailscale_cert | default(false) - # Stops and deletes quadlets/units left behind by roles removed from - # this repo. Never on the retired VPSes: nothing there may be stopped. - - role: retired_services - tags: [system, retired_services] - when: inventory_hostname not in groups.get('vps', []) and not is_termux | bool and ansible_facts['service_mgr'] | default('') == 'systemd' - - role: server_hardening - tags: [system, hardening, security] - when: inventory_hostname in groups.get('server', []) or inventory_hostname in groups.get('vps', []) - # Last, so it reports the state this run left behind. - - role: fleet_facts - tags: [facts, fleet_facts] - - post_tasks: - - name: Summarise secrets phase - ansible.builtin.debug: - msg: >- - {% if 'secrets' in (ansible_skip_tags | default([])) %} - [secrets phase] SKIPPED (--skip-tags secrets). Run `dots-apply` to include - ssh_keys, github, kube, tailscale, browser_fxa, and shell secrets. - {% elif not (bw_unlocked | default(false)) %} - [secrets phase] Bitwarden vault was LOCKED — BW-touching tasks in - ssh_keys, github, kube, tailscale, browser_fxa, and shell - were skipped. To enable: `bw unlock`, write the session to - /tmp/bw_session, then re-run. - {% else %} - [secrets phase] vault unlocked — all secret roles executed normally. - {% endif %} - tags: always diff --git a/talos-k8s/hive/ops/README.md b/talos-k8s/hive/ops/README.md index ddf42918..a02b840a 100644 --- a/talos-k8s/hive/ops/README.md +++ b/talos-k8s/hive/ops/README.md @@ -13,8 +13,8 @@ agents that get stuck. | hanthor (hive.reilly.asia) | `hive-hanthor` | `/state/hive-rotate-hanthor` | - `scripts/`: the contents of ConfigMap `hive-ops-scripts`, mounted at `/scripts`. - This directory is the source of truth. The older copies in `roles/hive_ops/files/bin/` - are no longer used (the role is disabled on every host). + This directory is the source of truth for the cluster ConfigMap. Separate host + timer helpers live under `units/hive-ops/` and are controlled by `hive_ops_enabled`. - `cronjobs.yaml`: only the hive-ops CronJobs the operator does not own (school's suspended rotate/watchdog/pace/nudge, kept for rollback). The rest are rendered by `HiveHousekeeping/fleet` in tuna-os/hive-operator (`config/samples/housekeeping.yaml`); `kubectl apply` of an old copy is drift. diff --git a/tests/test_bluefin_sync.py b/tests/test_bluefin_sync.py index f56b2b77..9a2d38a9 100644 --- a/tests/test_bluefin_sync.py +++ b/tests/test_bluefin_sync.py @@ -12,7 +12,7 @@ from conftest import REPO -SRC = REPO / "roles/bluefin_common/files/bluefin-sync.py" +SRC = REPO / "units/bluefin/files/bluefin-sync.py" spec = importlib.util.spec_from_file_location("bluefin_sync", SRC) bluefin_sync = importlib.util.module_from_spec(spec) spec.loader.exec_module(bluefin_sync) diff --git a/tests/test_bluefin_unit.py b/tests/test_bluefin_unit.py new file mode 100644 index 00000000..815e4646 --- /dev/null +++ b/tests/test_bluefin_unit.py @@ -0,0 +1,306 @@ +"""Behavioral coverage for bluefin's host gates and desktop side effects.""" +import os +import shutil +import socket +import subprocess +import sys +import tempfile +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub + + +@pytest.fixture +def desktop(tmp_path, stub_env): + bindir, env = stub_env + home = Path(env["HOME"]) + config = tmp_path / "config" + (config / "host_vars").mkdir(parents=True) + (config / "inventory.yml").write_text("all: {}\n") + (config / "roles").symlink_to(REPO / "roles", target_is_directory=True) + env.update(DOTFILES_DIR=str(config), DOTFILES_MACHINE="test-desktop", + DOTFILES_IS_DESKTOP="true", BREW_PREFIX="/nonexistent", + DBUS_SESSION_BUS_ADDRESS="") + + write_stub(bindir, "grep", '''\ +if [ "${3:-}" = /etc/os-release ]; then exit "${STUB_BLUEFIN_RC:-1}"; fi +exec /usr/bin/grep "$@" +''') + write_stub(bindir, "gsettings", '''\ +echo "gsettings $* bus=${DBUS_SESSION_BUS_ADDRESS:-}" >> "$HOME/calls" +if [ "$1" = get ]; then exit "${STUB_GNOME_RC:-0}"; fi +''') + write_stub(bindir, "podman", '''\ +echo "podman $*" >> "$HOME/calls" +case "$1" in + pull) exit "${STUB_PULL_RC:-0}" ;; + create) echo test-container ;; + cp) + if [ "${STUB_MISSING_ICON:-0}" = 1 ] && [[ "$2" = *icons* ]]; then exit 1; fi + case "$2" in + *gschema.override) printf '[org.gnome.desktop.interface]\\ncolor-scheme=\\x27prefer-dark\\x27\\n' > "$3/zz0-bluefin-modifications.gschema.override" ;; + *dconf*) : ;; + *backgrounds*) printf '/usr/share/backgrounds/bluefin/a.jpg\\n' > "$3/01-bluefin.xml" ;; + *icons*) printf icon > "$3/icon" ;; + *ublue-os*) printf logo > "$3/logo" ;; + esac ;; +esac +''') + write_stub(bindir, "curl", '''\ +url='' out='' +while [ "$#" -gt 0 ]; do + case "$1" in + -o) out="$2"; shift 2 ;; + --retry) shift 2 ;; + http*) url="$1"; shift ;; + *) shift ;; + esac +done +echo "curl $url" >> "$HOME/calls" +case "$url" in + */SHA256SUMS) + read -r hash _ <<< "$(printf 'corral binary' | sha256sum)" + [ "${STUB_BAD_CHECKSUM:-0}" = 1 ] && hash=0000000000000000000000000000000000000000000000000000000000000000 + case "$(uname -m)" in x86_64) arch=amd64 ;; aarch64) arch=arm64 ;; *) arch="$(uname -m)" ;; esac + printf '%s corral-linux-%s\\n' "$hash" "$arch" > "$out" ;; + */corral-linux-*) printf 'corral binary' > "$out" ;; + *) printf 'font archive' > "$out" ;; +esac +''') + write_stub(bindir, "unzip", '''\ +echo "unzip $*" >> "$HOME/calls" +archive="$2"; dest="$4" +printf font > "$dest/$(basename "$archive").font" +''') + write_stub(bindir, "fc-cache", 'echo "fc-cache $*" >> "$HOME/calls"\n') + write_stub(bindir, "systemctl", '''\ +echo "systemctl $*" >> "$HOME/calls" +if [ "${STUB_SYSTEMCTL_FAIL:-}" = "$2" ]; then exit 1; fi +''') + write_stub(bindir, "python3", f'''\ +if [[ "$1" = */install-extensions.py ]]; then + echo "extensions bus=${{DBUS_SESSION_BUS_ADDRESS:-}}" >> "$HOME/calls" + exit "${{STUB_EXTENSIONS_RC:-0}}" +fi +exec "{sys.executable}" "$@" +''') + # Redirect the only privileged destination, including when tests run as root. + write_stub(bindir, "sudo", 'shift\nexec "$@"\n') + write_stub(bindir, "install", '''\ +if [ "${4:-}" = /etc/sysctl.d/99-zram.conf ]; then + echo "install sysctl" >> "$HOME/calls" + exec /usr/bin/install "$1" "$2" "$3" "$HOME/sysctl.conf" +fi +exec /usr/bin/install "$@" +''') + write_stub(bindir, "cmp", '''\ +if [ "${3:-}" = /etc/sysctl.d/99-zram.conf ]; then + exec /usr/bin/cmp "$1" "$2" "$HOME/sysctl.conf" +fi +exec /usr/bin/cmp "$@" +''') + write_stub(bindir, "stat", '''\ +if [ "${3:-}" = /etc/sysctl.d/99-zram.conf ]; then + exec /usr/bin/stat "$1" "$2" "$HOME/sysctl.conf" +fi +exec /usr/bin/stat "$@" +''') + write_stub(bindir, "sysctl", 'echo "sysctl $*" >> "$HOME/calls"\n') + return env, config + + +def run(env, **overrides): + return subprocess.run(["bash", str(REPO / "units/bluefin.sh")], + env={**env, **overrides}, capture_output=True, text=True) + + +def log(env): + path = Path(env["HOME"]) / "calls" + return path.read_text() if path.exists() else "" + + +def hostvars(config, text): + (config / "host_vars/test-desktop.yml").write_text(text) + + +@pytest.mark.parametrize("value", ["skip_bluefin: true\n", "machine_profile: vm-test\n"]) +def test_skip_flags_precede_inventory_lookup(desktop, value): + env, config = desktop + hostvars(config, value) + r = run(env, DOTFILES_IS_DESKTOP="") + assert r.returncode == 0 + assert "SKIPPED" in r.stdout + assert log(env) == "" + + +def test_non_desktop_is_skipped_without_yq(desktop): + env, _ = desktop + r = run(env, DOTFILES_IS_DESKTOP="false") + assert r.returncode == 0 + assert "not a desktop" in r.stdout + assert log(env) == "" + + +def test_missing_yq_is_reported_when_membership_is_unknown(desktop, tmp_path): + env, _ = desktop + bindir = tmp_path / "minimal-bin" + bindir.mkdir() + for name in ("bash", "dirname", "sed", "head"): + (bindir / name).symlink_to(shutil.which(name)) + r = run(env, DOTFILES_IS_DESKTOP="", PATH=str(bindir)) + assert r.returncode == 1 + assert "desktop inventory requires yq" in r.stdout + assert log(env) == "" + + +def test_host_desktop_override_needs_no_inventory_reader(desktop, tmp_path): + env, config = desktop + hostvars(config, "is_desktop: false\n") + bindir = tmp_path / "minimal-bin" + bindir.mkdir() + for name in ("bash", "dirname", "sed", "head"): + (bindir / name).symlink_to(shutil.which(name)) + r = run(env, DOTFILES_IS_DESKTOP="", PATH=str(bindir)) + assert r.returncode == 0 + assert "not a desktop" in r.stdout + assert log(env) == "" + + +def test_inventory_non_desktop_is_a_clean_skip(desktop): + env, _ = desktop + write_stub(Path(env["PATH"].split(":")[0]), "yq", "echo false\n") + r = run(env, DOTFILES_IS_DESKTOP="") + assert r.returncode == 0 + assert "not a desktop" in r.stdout + assert log(env) == "" + + +def test_inventory_reader_errors_are_not_a_successful_skip(desktop): + env, _ = desktop + write_stub(Path(env["PATH"].split(":")[0]), "yq", "exit 2\n") + r = run(env, DOTFILES_IS_DESKTOP="") + assert r.returncode == 1 + assert "desktop inventory lookup" in r.stdout + assert log(env) == "" + + +def test_gnome_sync_and_system_changes_are_idempotent(desktop): + env, _ = desktop + home = Path(env["HOME"]) + retired = home / ".local/bin/tailvm" + retired.parent.mkdir(parents=True) + retired.write_text("old") + first = run(env) + assert first.returncode == 0, first.stdout + first.stderr + assert not retired.exists() + assert (home / ".local/bin/corral").read_bytes() == b"corral binary" + assert (home / ".local/bin/corral").stat().st_mode & 0o777 == 0o755 + xml = (home / ".local/share/backgrounds/bluefin/01-bluefin.xml").read_text() + assert f"{home}/.local/share/backgrounds/bluefin/a.jpg" in xml + assert (home / ".local/share/icons/hicolor/icon").read_text() == "icon" + assert (home / ".local/share/ublue-os/logo").read_text() == "logo" + assert (home / ".local/share/fonts/Inter/Inter.zip.font").is_file() + assert (home / ".local/share/fonts/JetBrainsMono/JetBrainsMono.zip.font").is_file() + service = (home / ".config/systemd/user/bluefin-wallpaper-rotate.service").read_text() + assert f"ExecStart={home}/.local/bin/bluefin-wallpaper-rotate.py" in service + bus = f"unix:path=/run/user/{os.getuid()}/bus" + assert f"Environment=DBUS_SESSION_BUS_ADDRESS={bus}" in service + assert f"extensions bus={bus}" in log(env) + assert "color-scheme 'prefer-dark' bus=" + bus in log(env) + assert "vm.swappiness = 150" in (home / "sysctl.conf").read_text() + before = log(env) + second = run(env) + assert second.returncode == 0, second.stdout + second.stderr + later = log(env)[len(before):] + assert "systemctl --user daemon-reload" not in later + assert "install sysctl" not in later + assert "sysctl -p" not in later + assert "systemctl --user enable --now bluefin-wallpaper-rotate.timer" in later + + +def test_existing_custom_bus_is_used_for_probe_and_service(desktop): + env, _ = desktop + with tempfile.TemporaryDirectory(prefix="bluefin-bus-") as directory: + path = str(Path(directory) / "bus") + with socket.socket(socket.AF_UNIX) as session: + session.bind(path) + bus = f"unix:path={path}" + r = run(env, DBUS_SESSION_BUS_ADDRESS=bus) + assert r.returncode == 0, r.stdout + r.stderr + assert "session bus socket unavailable" not in r.stdout + assert f"gsettings get org.gnome.desktop.interface color-scheme bus={bus}" in log(env) + assert f"extensions bus={bus}" in log(env) + service = (Path(env["HOME"]) / ".config/systemd/user/bluefin-wallpaper-rotate.service").read_text() + assert f"Environment=DBUS_SESSION_BUS_ADDRESS={bus}" in service + + +def test_missing_bus_and_failed_gnome_probe_are_reported(desktop, tmp_path): + env, _ = desktop + r = run(env, DBUS_SESSION_BUS_ADDRESS=f"unix:path={tmp_path}/missing", STUB_GNOME_RC="1") + assert r.returncode == 0, r.stdout + r.stderr + assert "PARTIAL: session bus socket unavailable" in r.stdout + assert "PARTIAL: GNOME session detection failed" in r.stdout + assert "podman pull" not in log(env) + + +@pytest.mark.parametrize("gnome,bluefin,force,sync", [ + ("0", "0", "false", False), + ("0", "0", "true", True), + ("1", "1", "true", False), + ("0", "1", "false", True), +]) +def test_platform_gates_do_not_suppress_unconditional_tasks(desktop, gnome, bluefin, force, sync): + env, config = desktop + hostvars(config, f"force_bluefin: {force}\n") + r = run(env, STUB_GNOME_RC=gnome, STUB_BLUEFIN_RC=bluefin) + assert r.returncode == 0, r.stdout + r.stderr + assert ("podman pull" in log(env)) is sync + assert ("extensions bus=" in log(env)) is sync + assert (Path(env["HOME"]) / ".local/bin/corral").is_file() + assert "sysctl -p" in log(env) + assert "systemctl --user enable --now bluefin-wallpaper-rotate.timer" in log(env) + + +def test_missing_optional_image_asset_is_reported(desktop): + env, _ = desktop + r = run(env, STUB_MISSING_ICON="1") + assert r.returncode == 0, r.stdout + r.stderr + assert "PARTIAL: Bluefin image asset unavailable:" in r.stdout + assert "icons/hicolor" in r.stdout + assert (Path(env["HOME"]) / ".local/share/ublue-os/logo").is_file() + + +def test_os_detection_failure_does_not_apply_wrong_branding(desktop): + env, _ = desktop + r = run(env, STUB_BLUEFIN_RC="2") + assert r.returncode == 1 + assert "detect Bluefin OS" in r.stdout + assert "podman pull" not in log(env) + + +def test_failed_pull_is_visible_and_independent_steps_continue(desktop): + env, _ = desktop + r = run(env, STUB_PULL_RC="1") + assert r.returncode == 1 + assert "FAIL Bluefin branding and settings" in r.stdout + assert (Path(env["HOME"]) / ".local/bin/corral").is_file() + assert "sysctl -p" in log(env) + + +def test_invalid_corral_checksum_never_installs_binary(desktop): + env, _ = desktop + r = run(env, STUB_GNOME_RC="1", STUB_BAD_CHECKSUM="1") + assert r.returncode == 1 + assert "checksum missing or mismatched" in r.stderr + assert not (Path(env["HOME"]) / ".local/bin/corral").exists() + assert "sysctl -p" in log(env) + + +def test_extensions_failure_remains_nonfatal(desktop): + env, _ = desktop + r = run(env, STUB_EXTENSIONS_RC="1") + assert r.returncode == 0, r.stdout + r.stderr + assert "warn GNOME extensions" in r.stdout + assert "sysctl -p" in log(env) diff --git a/tests/test_bluefin_wallpaper_rotate.py b/tests/test_bluefin_wallpaper_rotate.py index d2c0eeeb..0097aca0 100644 --- a/tests/test_bluefin_wallpaper_rotate.py +++ b/tests/test_bluefin_wallpaper_rotate.py @@ -9,7 +9,7 @@ import tempfile # Import the script's logic (we'll create a testable module from it) -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../roles/bluefin_common/files")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../units/bluefin/files")) class WallpaperRotationTests(TestCase): diff --git a/tests/test_bootstrap_units.py b/tests/test_bootstrap_units.py index 5e1f0058..ab7f4756 100644 --- a/tests/test_bootstrap_units.py +++ b/tests/test_bootstrap_units.py @@ -6,51 +6,13 @@ from conftest import REPO, write_stub -@pytest.fixture -def phase_env(stub_env, tmp_path): - bindir, env = stub_env - repo = tmp_path / "checkout" - scripts = repo / "scripts" - scripts.mkdir(parents=True) - log = tmp_path / "calls" - write_stub(bindir, "ansible-playbook", 'printf "ansible %s\\n" "$*" >> "$CALLS"\nexit "${ANSIBLE_RC:-0}"\n') - write_stub(scripts, "run-units.sh", 'printf "units %s\\n" "$*" >> "$CALLS"\nexit "${UNITS_RC:-0}"\n') - env.update(CALLS=str(log), TEST_REPO=str(repo), BW_SESSION="test-session") - return env, log - - -def run_phase(env, phase): - return subprocess.run( - ["bash", "-c", 'source "$1/bootstrap.sh" --name test --type desktop; DOTFILES_DIR="$TEST_REPO"; MACHINE_NAME=test; MACHINE_TYPE=desktop; "$2"', - "bash", str(REPO), phase], - env=env, capture_output=True, text=True, - ) - - -@pytest.mark.parametrize("phase,unit_args", [("run_phase1", "--no-secrets"), ("run_phase2", "--tags omp,secrets")]) -@pytest.mark.parametrize("ansible_rc,units_rc,expected", [(0, 0, 0), (7, 0, 7), (0, 9, 9), (7, 9, 7)]) -def test_bootstrap_phases_continue_and_combine_errors(phase_env, phase, unit_args, ansible_rc, units_rc, expected): - env, log = phase_env - env.update(ANSIBLE_RC=str(ansible_rc), UNITS_RC=str(units_rc)) - result = run_phase(env, phase) - assert result.returncode == expected - calls = log.read_text().splitlines() - assert len(calls) == 2 - assert calls[0].startswith("ansible ") - assert calls[1] == f"units {unit_args}" - assert "test-session" not in "\n".join(calls) - if phase == "run_phase1": - assert "--skip-tags secrets" in calls[0] - else: - assert "--tags secrets" in calls[0] - def test_bootstrap_runs_second_phase_after_first_fails(stub_env, tmp_path): _, env = stub_env env["CALLS"] = str(tmp_path / "calls") script = r''' source "$1/bootstrap.sh" --name test --type desktop -for f in ensure_python ensure_git ensure_uv ensure_ansible clone_repo get_machine_name get_machine_type register_in_inventory write_machine_name set_hostname ensure_sudo ensure_tailscale install_collections setup_bitwarden push_inventory_changes; do +for f in ensure_python ensure_git clone_repo get_machine_name get_machine_type register_in_inventory write_machine_name set_hostname ensure_sudo ensure_tailscale setup_bitwarden push_inventory_changes; do eval "$f() { :; }" done run_phase1() { echo first >> "$CALLS"; return 7; } diff --git a/tests/test_brew_flatpak_check_mode.py b/tests/test_brew_flatpak_check_mode.py new file mode 100644 index 00000000..0613b426 --- /dev/null +++ b/tests/test_brew_flatpak_check_mode.py @@ -0,0 +1,287 @@ +"""Check-mode package inspection without invoking package managers on the host.""" +import json +import os +import stat +import sys +from pathlib import Path + +import pytest +import yaml + +from conftest import write_stub +from test_package_native_units import package_env, run_unit + + +@pytest.fixture +def check_env(package_env, tmp_path): + root, env, log = package_env + bindir = Path(env["PATH"].split(os.pathsep)[0]) + sandbox_log = tmp_path / "sandbox.jsonl" + env.update(UNITS_CHECK="1", SANDBOX_LOG=str(sandbox_log)) + program = f'''#!{sys.executable} +import json, os, sys +args = sys.argv[1:] +with open(os.environ["SANDBOX_LOG"], "a") as f: + f.write(json.dumps(args) + "\\n") +if os.getenv("SANDBOX_FAIL") == "1": + print("sentinel-credential", file=sys.stderr) + sys.exit(1) +required = ["--unshare-all", "--ro-bind", "--tmpfs", "--clearenv"] +if not all(flag in args for flag in required) or args[args.index("--tmpfs") + 1] != "/run": + sys.exit(98) +command = next((i for i, arg in enumerate(args) if arg == "flatpak" or arg.endswith("/bin/brew") or arg.endswith("/bin/python3")), None) +if command is None: + sys.exit(97) +exe, sub = args[command], args[command + 1:] +if exe == "flatpak": + if sub == ["remotes", "--system", "--columns=name"]: + print(os.getenv("SYSTEM_REMOTES", "flathub")) + elif sub == ["list", "--system", "--columns=application", "--app"]: + print(os.getenv("SYSTEM_APPS", "org.mozilla.firefox\\norg.example.Installed")) + elif sub == ["remotes", "--user", "--columns=name"]: + print(os.getenv("USER_REMOTES", "")) + elif sub == ["list", "--user", "--columns=application", "--app"]: + print("\\n".join(json.loads(os.getenv("USER_ORIGINS", "{{}}")))) + elif sub[:3] == ["info", "--user", "--show-origin"]: + origins = json.loads(os.getenv("USER_ORIGINS", "{{}}")) + if sub[-1] not in origins or os.getenv("FAIL_ORIGIN") == sub[-1]: + sys.exit(1) + print(origins[sub[-1]]) + else: + sys.exit(96) +elif exe.endswith("/bin/brew"): + if sub == ["tap"]: + print(os.getenv("BREW_TAPS", "example/core\\nexample/desktop")) + elif sub[:3] == ["info", "--json=v1", "--formula"] and len(sub) == 4: + if os.getenv("BLOCKED_FAIL") == "1": + print("sentinel-credential", file=sys.stderr) + sys.exit(1) + if os.getenv("BLOCKED_MALFORMED") == "1": + print('not-json sentinel-credential') + else: + installed = [{{"version": "1"}}] if os.getenv("BLOCKED_INSTALLED", "1") == "1" else [] + print(json.dumps([{{"name": "canonical-name", "aliases": [sub[-1]], "installed": installed}}])) + elif sub == ["bundle", "check", "--file=-", "--no-upgrade", "--verbose"]: + brewfile = sys.stdin.read() + if 'brew "git"' not in brewfile: + sys.exit(95) + if os.getenv("MISSING_BUNDLE") == "1": + print("brew bundle can't satisfy your Brewfile's dependencies.", file=sys.stderr) + print("→ Formula desktop-pkg needs to be installed.", file=sys.stderr) + print("sentinel-credential", file=sys.stderr) + sys.exit(1) + else: + sys.exit(94) +elif sub[:1] != ["-c"]: + sys.exit(93) +elif os.getenv("PYTHON_MISSING") == "1": + sys.exit(42) +''' + bwrap = bindir / "bwrap" + bwrap.write_text(program) + bwrap.chmod(0o755) + return root, env, log, sandbox_log + + +def sandbox_calls(path): + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + +def tree_state(path): + return { + str(item.relative_to(path)): (stat.S_IMODE(item.stat().st_mode), item.read_bytes() if item.is_file() else None) + for item in path.rglob("*") if not item.is_symlink() + } + + +def test_flatpak_check_reports_scoped_drift_without_mutation(check_env): + root, env, log, sandbox = check_env + (root / "host_vars/laptop.yml").write_text(yaml.safe_dump({ + "user_flatpak_remotes": [{"name": "private", "url": "https://example.test/secret?token=sentinel-credential"}], + "user_flatpaks": [ + {"remote": "private", "id": "org.example.Existing"}, + {"remote": "private", "id": "org.example.New"}, + {"remote": "other", "id": "org.example.New"}, + ], + })) + env["USER_ORIGINS"] = json.dumps({"org.example.Existing": "other"}) + env["DOTFILES_UPGRADE"] = "1" + home = Path(env["HOME"]) + before = tree_state(home) + result = run_unit("flatpak", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "remove system Flatpak org.mozilla.firefox" in result.stdout + assert "install system Flatpak org.example.First" in result.stdout + assert "add user Flatpak remote private" in result.stdout + assert "install user Flatpak org.example.New from private" in result.stdout + assert "org.example.Existing uses other" in result.stdout + assert "org.example.New would use private" in result.stdout + assert "system update requested" in result.stdout + assert "sentinel-credential" not in result.stdout + result.stderr + assert tree_state(home) == before + assert not log.exists() + commands = sandbox_calls(sandbox) + assert commands and all("--unshare-all" in cmd and "--clearenv" in cmd for cmd in commands) + assert all("--ro-bind" in cmd and "--tmpfs" in cmd for cmd in commands) + assert all("install" not in cmd and "uninstall" not in cmd and "remote-add" not in cmd for cmd in commands) + + +def test_flatpak_check_reports_inspection_and_origin_failures(check_env): + root, env, log, sandbox = check_env + (root / "host_vars/laptop.yml").write_text(yaml.safe_dump({ + "user_flatpaks": [{"remote": "private", "id": "org.example.Existing"}], + })) + env["USER_ORIGINS"] = json.dumps({"org.example.Existing": "private"}) + env["FAIL_ORIGIN"] = "org.example.Existing" + result = run_unit("flatpak", env) + assert result.returncode != 0 + assert "Cannot inspect origin" in result.stderr + assert "install user Flatpak org.example.Existing" not in result.stdout + env["SANDBOX_FAIL"] = "1" + result = run_unit("flatpak", env) + assert result.returncode != 0 + assert "read-only sandbox" in result.stderr + assert "sentinel-credential" not in result.stdout + result.stderr + assert not log.exists() + + +def test_flatpak_check_respects_profile_and_never_touches_wrappers(check_env): + root, env, log, sandbox = check_env + (root / "host_vars/laptop.yml").write_text("skip_flatpak: true\n") + result = run_unit("flatpak", env) + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert not sandbox.exists() and not log.exists() + assert not (Path(env["HOME"]) / ".local").exists() + + +def test_homebrew_check_reports_real_plans_and_never_runs_apply(check_env): + _, env, log, sandbox = check_env + env["MISSING_BUNDLE"] = "1" + env["DOTFILES_UPGRADE"] = "1" + home, prefix = Path(env["HOME"]), Path(env["BREW_PREFIX"]) + (home / "Brewfile").write_text("old contents\n") + (home / "Brewfile").chmod(0o600) + before_home, before_prefix = tree_state(home), tree_state(prefix) + result = run_unit("homebrew", env) + assert result.returncode != 0 # bundle uses the same status for drift and inspection errors + assert "update Brewfile" in result.stdout + assert "Brewfile mode to 0644" in result.stdout + assert "tap Homebrew repository coreowner/tap" in result.stdout + assert "dependency desktop-pkg" in result.stdout + assert 'attempt Homebrew tap trust on apply' in result.stdout + assert "uninstall blocked Homebrew formula trash-cli" in result.stdout + assert "Homebrew upgrade requested" in result.stdout + assert "sentinel-credential" not in result.stdout + result.stderr + assert tree_state(home) == before_home and tree_state(prefix) == before_prefix + assert not log.exists() + commands = sandbox_calls(sandbox) + assert commands and all("--unshare-all" in cmd and "--clearenv" in cmd for cmd in commands) + assert all("--ro-bind" in cmd and "--tmpfs" in cmd for cmd in commands) + assert all("install" not in cmd and "uninstall" not in cmd and "upgrade" not in cmd for cmd in commands) + + +def test_homebrew_check_missing_brew_and_failed_sandbox(check_env): + _, env, log, sandbox = check_env + brew = Path(env["BREW_PREFIX"]) / "bin/brew" + original = brew.read_bytes() + brew.unlink() + result = run_unit("homebrew", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "CHANGE: install Homebrew" in result.stdout + assert 'attempt Homebrew tap trust on apply' in result.stdout + assert "install configured Homebrew taps and packages" in result.stdout + assert not sandbox.exists() and not log.exists() + brew.write_bytes(original) + brew.chmod(0o755) + env["SANDBOX_FAIL"] = "1" + result = run_unit("homebrew", env) + assert result.returncode != 0 + assert "Cannot inspect Homebrew" in result.stderr + assert "sentinel-credential" not in result.stdout + result.stderr + assert not log.exists() + + +def test_homebrew_check_never_bootstraps_missing_yq(check_env): + _, env, log, sandbox = check_env + bindir = Path(env["PATH"].split(os.pathsep)[0]) + write_stub(bindir, "yq", 'echo "sentinel-credential" >&2; exit 1\n') + result = run_unit("homebrew", env) + assert result.returncode != 0 + assert "Mike Farah yq is required" in result.stderr + assert "sentinel-credential" not in result.stdout + result.stderr + assert not sandbox.exists() and not log.exists() + + +def test_flatpak_check_matching_system_and_wrappers_are_unchanged(check_env): + _, env, log, _ = check_env + env["DOTFILES_UPGRADE"] = "0" + env["SYSTEM_APPS"] = "\n".join(["org.example.Installed", "org.example.First", "org.example.Second"]) + bindir = Path(env["HOME"]) / ".local/bin" + bindir.mkdir(parents=True) + for name in ("google-chrome", "google-chrome-stable", "chromium-browser", "chromium"): + wrapper = bindir / name + wrapper.write_text('#!/usr/bin/env bash\nexec flatpak run org.chromium.Chromium "$@"\n') + wrapper.chmod(0o755) + before = tree_state(Path(env["HOME"])) + result = run_unit("flatpak", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "CHANGE:" not in result.stdout + assert tree_state(Path(env["HOME"])) == before and not log.exists() + + +def test_homebrew_check_matching_brewfile_and_bundle(check_env): + _, env, log, _ = check_env + env["DOTFILES_UPGRADE"] = "0" + env["BREW_TAPS"] = "example/core\nexample/desktop\ncoreowner/tap\nme/tap" + env["BLOCKED_INSTALLED"] = "0" + bindir = Path(env["PATH"].split(os.pathsep)[0]) + write_stub(bindir, "uname", 'echo Linux\n') + brewfile = Path(env["HOME"]) / "Brewfile" + brewfile.write_text( + '# Core packages\ntap "example/core"\n\n' + 'brew "git"\nbrew "coreowner/tap/formula"\n\n\n' + '# Desktop packages\ntap "example/desktop"\n\n' + 'brew "desktop-pkg"\n\nbrew "me/tap/extra"\n\n' + 'cask "desktop-cask"\n\n' + ) + brewfile.chmod(0o644) + before = tree_state(Path(env["HOME"])) + result = run_unit("homebrew", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "ok Homebrew bundle dependencies" in result.stdout + assert "CHANGE: update Brewfile" not in result.stdout + assert "Brewfile mode" not in result.stdout + assert "uninstall blocked" not in result.stdout + assert tree_state(Path(env["HOME"])) == before and not log.exists() + + +def test_flatpak_check_redacts_invalid_configuration(check_env): + root, env, log, sandbox = check_env + (root / "units/flatpak-packages.yml").write_text("flatpak_remotes: [sentinel-credential\n") + result = run_unit("flatpak", env) + assert result.returncode != 0 + assert "Cannot inspect Flatpak configuration" in result.stderr + assert "sentinel-credential" not in result.stdout + result.stderr + assert not sandbox.exists() and not log.exists() + + +def test_homebrew_check_reports_missing_python_packaging(check_env): + _, env, log, _ = check_env + env["PYTHON_MISSING"] = "1" + result = run_unit("homebrew", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "CHANGE: install Homebrew Python packaging" in result.stdout + assert not log.exists() + + +@pytest.mark.parametrize("failure", ["BLOCKED_FAIL", "BLOCKED_MALFORMED"]) +def test_homebrew_check_blocked_alias_inspection_failure_is_not_absence(check_env, failure): + _, env, log, sandbox = check_env + env[failure] = "1" + result = run_unit("homebrew", env) + assert result.returncode != 0 + assert "Cannot inspect blocked Homebrew formula trash-cli" in result.stderr + assert "uninstall blocked Homebrew formula" not in result.stdout + assert "sentinel-credential" not in result.stdout + result.stderr + assert any(["info", "--json=v1", "--formula", "trash-cli"] == cmd[-4:] for cmd in sandbox_calls(sandbox)) + assert not log.exists() diff --git a/tests/test_browser_fxa_check_mode.py b/tests/test_browser_fxa_check_mode.py new file mode 100644 index 00000000..21929e7f --- /dev/null +++ b/tests/test_browser_fxa_check_mode.py @@ -0,0 +1,164 @@ +import json +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub +from test_browser_fxa_unit import prepare, run + + +def configure_source(tmp_path, env, browser): + root = tmp_path / "source" + (root / "host_vars").mkdir(parents=True) + (root / "units").mkdir() + (root / "units/browser-fxa").symlink_to(REPO / "units/browser-fxa", target_is_directory=True) + (root / "host_vars/himachal.yml").write_text( + f'browser_fxa_browser: "{browser}"\nfxa_bw_username: "jreilly1821@gmail.com"\n' + ) + env["DOTFILES_DIR"] = str(root) + + +def add_profile(base, name): + profile = base / name + profile.mkdir(parents=True) + (base / "profiles.ini").write_text( + f"[Profile0]\nIsRelative=1\nPath={name}\nDefault=1\n" + ) + return profile + + +def snapshot(root): + return { + str(path.relative_to(root)): ( + path.lstat().st_mode & 0o777, + path.read_bytes() if path.is_file() else None, + ) + for path in root.rglob("*") + } + + +@pytest.mark.parametrize("flatpak", [False, True]) +def test_firefox_apply_selects_flatpak_first_then_native(stub_env, tmp_path, flatpak): + env, home, _ = prepare(stub_env) + env.pop("UNITS_CHECK", None) + configure_source(tmp_path, env, "firefox") + native = add_profile(home / ".mozilla/firefox", "native.default") + selected = native + if flatpak: + selected = add_profile(home / ".var/app/org.mozilla.firefox/.mozilla/firefox", "flatpak.default") + result = run(env) + assert result.returncode == 0, result.stdout + result.stderr + assert (selected / "signedInUser.json").read_text() == "session-data" + if flatpak: + assert not (native / "signedInUser.json").exists() + assert (home / "signin-captured").read_text().splitlines()[0] == "jreilly1821@gmail.com" + + +def test_firefox_flatpak_does_not_fall_back_to_native(stub_env, tmp_path): + env, home, _ = prepare(stub_env) + env.pop("UNITS_CHECK", None) + configure_source(tmp_path, env, "firefox-flatpak") + native = add_profile(home / ".mozilla/firefox", "native.default") + result = run(env) + assert result.returncode == 0 and "profile not installed" in result.stdout + assert not (native / "signedInUser.json").exists() + assert not (home / "bw-calls").exists() + + +def prepare_check(stub_env, tmp_path): + env, home, profile = prepare(stub_env) + configure_source(tmp_path, env, "zen") + env.update(UNITS_CHECK="1", CHECK_LOG=str(tmp_path / "check-calls")) + write_stub(home / ".local/venvs/fxa/bin", "python", ''' +if [ "$1" = -B ]; then + [ "${PYFXA_ERROR:-0}" = 0 ] || exit 2 + echo "${PYFXA_STATE:-installed}" +else + echo "unexpected python $*" >> "$CHECK_LOG" + exit 1 +fi +''') + write_stub(Path(env["PATH"].split(":")[0]), "bw", 'echo "unexpected bw $*" >> "$CHECK_LOG"; exit 1\n') + return env, home, profile + + +def test_check_reports_drift_without_mutation_or_secrets(stub_env, tmp_path): + env, home, profile = prepare_check(stub_env, tmp_path) + token = profile / "signedInUser.json" + token.write_text(json.dumps({"accountData": {"verified": False, "sessionToken": "private-token"}})) + token.chmod(0o644) + before = snapshot(home) + result = run(env) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD install Firefox Accounts sign-in script" in result.stdout + assert "WOULD restrict Firefox Accounts token permissions" in result.stdout + assert "WOULD attempt Firefox Accounts sign-in" in result.stdout + assert "private-token" not in result.stdout + result.stderr + assert "s'ec ret" not in result.stdout + result.stderr + assert snapshot(home) == before + assert not Path(env["CHECK_LOG"]).exists() + + +def test_check_verified_profile_and_missing_dependencies(stub_env, tmp_path): + env, home, profile = prepare_check(stub_env, tmp_path) + (home / ".local/venvs/fxa/bin/python").unlink() + dest = home / ".local/bin/fxa_signin.py" + dest.parent.mkdir(parents=True) + dest.write_bytes((REPO / "units/browser-fxa/fxa_signin.py").read_bytes()) + dest.chmod(0o700) + token = profile / "signedInUser.json" + token.write_text(json.dumps({"accountData": {"verified": True, "email": "secret@example.test"}})) + token.chmod(0o600) + before = snapshot(home) + result = run(env) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD create Firefox Accounts venv and install PyFxA" in result.stdout + assert "profile is verified" in result.stdout + assert "WOULD attempt" not in result.stdout + assert "secret@example.test" not in result.stdout + result.stderr + assert snapshot(home) == before + + +def test_check_reports_inspection_errors_without_mutation(stub_env, tmp_path): + env, home, profile = prepare_check(stub_env, tmp_path) + env["PYFXA_ERROR"] = "1" + before = snapshot(home) + result = run(env) + assert result.returncode != 0 and "inspect Firefox Accounts PyFxA" in result.stdout + assert snapshot(home) == before + env.pop("PYFXA_ERROR") + (profile.parent / "profiles.ini").write_text("not an INI file") + before = snapshot(home) + result = run(env) + assert result.returncode != 0 and "inspect Firefox Accounts default profile" in result.stdout + assert snapshot(home) == before + assert not Path(env["CHECK_LOG"]).exists() + + +def test_check_finds_native_firefox_profile(stub_env, tmp_path): + env, home, _ = prepare_check(stub_env, tmp_path) + configure_source_again = Path(env["DOTFILES_DIR"]) / "host_vars/himachal.yml" + configure_source_again.write_text('browser_fxa_browser: "firefox"\n') + profile = add_profile(home / ".mozilla/firefox", "native.default") + before = snapshot(home) + result = run(env) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD attempt Firefox Accounts sign-in" in result.stdout + assert not (profile / "signedInUser.json").exists() + assert snapshot(home) == before + + +def test_check_missing_pyfxa_and_unreadable_token(stub_env, tmp_path): + env, home, profile = prepare_check(stub_env, tmp_path) + env["PYFXA_STATE"] = "missing" + before = snapshot(home) + result = run(env) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD install Firefox Accounts PyFxA" in result.stdout + assert snapshot(home) == before + (profile / "signedInUser.json").mkdir() + before = snapshot(home) + result = run(env) + assert result.returncode != 0 and "FAIL inspect Firefox Accounts token" in result.stdout + assert snapshot(home) == before + assert not Path(env["CHECK_LOG"]).exists() diff --git a/tests/test_browser_fxa_unit.py b/tests/test_browser_fxa_unit.py new file mode 100644 index 00000000..4773f66c --- /dev/null +++ b/tests/test_browser_fxa_unit.py @@ -0,0 +1,112 @@ +import json +import subprocess +from pathlib import Path + +from conftest import REPO, write_stub + + +UNIT = REPO / "units/browser-fxa.sh" + + +def prepare(stub_env): + bindir, env = stub_env + home = Path(env["HOME"]) + env.update(DOTFILES_DIR=str(REPO), DOTFILES_MACHINE="himachal", DOTFILES_IS_DESKTOP="1", BW_SESSION="session") + write_stub(bindir, "yq", ''' +python3 - "$@" <<'PY' +import sys, yaml +key = sys.argv[2].split()[0].lstrip('.') +with open(sys.argv[3]) as f: + data = yaml.safe_load(f) or {} +value = data.get(key, '__UNSET__') +print('__UNSET__' if value is None else value) +PY +''') + write_stub(bindir, "bw", ''' +printf '%s\\n' "$*" >> "$HOME/bw-calls" +case "$1" in + unlock) [ "${BW_LOCKED:-0}" != 1 ] ;; + list) printf '%s' "$BW_ITEMS" ;; + get) printf '%s' '123456' ;; + *) exit 1 ;; +esac +''') + venv_bin = home / ".local/venvs/fxa/bin" + venv_bin.mkdir(parents=True) + write_stub(venv_bin, "python", ''' +[ "$1" != -c ] || exit 0 +printf '%s\\n' "$FXA_EMAIL" "$FXA_PASSWORD" "$FXA_TOTP" > "$HOME/signin-captured" +printf '%s' 'session-data' > "$PROFILE_DIR/signedInUser.json" +''') + base = home / ".var/app/app.zen_browser.zen/.zen" + profile = base / "profile.default" + profile.mkdir(parents=True) + (base / "profiles.ini").write_text("[Profile0]\nName=default\nIsRelative=1\nPath=profile.default\nDefault=1\n") + env["BW_ITEMS"] = json.dumps([ + {"id": "wrong", "login": {"username": "other@example.test", "password": "other-secret"}}, + {"id": "correct", "login": {"username": "jreilly1821@gmail.com", "password": "s'ec ret"}}, + ]) + return env, home, profile + + +def run(env): + return subprocess.run(["bash", str(UNIT)], env=env, text=True, capture_output=True) + + +def test_selects_exact_account_and_keeps_tokens_private(stub_env): + env, home, profile = prepare(stub_env) + result = run(env) + assert result.returncode == 0, result.stdout + result.stderr + assert (home / "signin-captured").read_text().splitlines() == ["jreilly1821@gmail.com", "s'ec ret", "123456"] + assert (profile / "signedInUser.json").read_text() == "session-data" + assert (profile / "signedInUser.json").stat().st_mode & 0o777 == 0o600 + assert "get totp correct" in (home / "bw-calls").read_text() + assert "s'ec ret" not in result.stdout + result.stderr + + +def test_mismatched_account_never_used(stub_env): + env, home, profile = prepare(stub_env) + env["BW_ITEMS"] = json.dumps([{"id": "wrong", "login": {"username": "other", "password": "do-not-leak"}}]) + (profile / "signedInUser.json").write_text("existing") + result = run(env) + assert result.returncode == 0 + assert "username mismatch" in result.stdout + assert "do-not-leak" not in result.stdout + result.stderr + assert (profile / "signedInUser.json").read_text() == "existing" + assert not (home / "signin-captured").exists() + + +def test_no_secrets_preserves_profile_and_never_calls_bitwarden(stub_env): + env, home, profile = prepare(stub_env) + env["UNITS_NO_SECRETS"] = "1" + (profile / "signedInUser.json").write_text("existing") + result = run(env) + assert result.returncode == 0 + assert "PARTIAL:" in result.stdout + assert (profile / "signedInUser.json").read_text() == "existing" + assert not (home / "bw-calls").exists() + assert not (home / "signin-captured").exists() + + +def test_locked_vault_leaves_existing_account_untouched(stub_env): + env, home, profile = prepare(stub_env) + env["BW_LOCKED"] = "1" + (profile / "signedInUser.json").write_text("existing") + result = run(env) + assert result.returncode == 0 + assert "PARTIAL:" in result.stdout + assert (profile / "signedInUser.json").read_text() == "existing" + assert not (home / "signin-captured").exists() + assert "list" not in (home / "bw-calls").read_text() + + +def test_missing_default_profile_is_not_created(stub_env): + env, home, profile = prepare(stub_env) + (profile.parent / "profiles.ini").write_text( + "[Profile0]\nIsRelative=1\nPath=missing.default\nDefault=1\n" + ) + result = run(env) + assert result.returncode == 0 + assert "default profile does not exist" in result.stdout + assert not (profile.parent / "missing.default").exists() + assert not (home / "signin-captured").exists() diff --git a/tests/test_check_small_packages_vpn.py b/tests/test_check_small_packages_vpn.py new file mode 100644 index 00000000..38ca6825 --- /dev/null +++ b/tests/test_check_small_packages_vpn.py @@ -0,0 +1,223 @@ +"""Read-only checks for APK, Termux and Tailscale native units.""" +from pathlib import Path + +from conftest import write_stub +from test_platform_native_units import platform_env, run_unit, log_lines + + +def snapshot(path): + result = {} + for item in path.rglob('*'): + rel = str(item.relative_to(path)) + if item.is_symlink(): + result[rel] = ('link', str(item.readlink())) + elif item.is_file(): + result[rel] = ('file', item.stat().st_mode, item.read_bytes()) + else: + result[rel] = ('directory', item.stat().st_mode) + return result + + +def test_apk_check_reports_packages_services_and_group_without_mutation(platform_env): + _, system, bindir, env, log = platform_env + write_stub(bindir, 'apk', '''\ +echo "apk $*" >> "$LOG" +[ "$1" = info ] && [ "$#" = 1 ] || exit 99 +printf 'chezmoi\nbash\n' +''') + write_stub(bindir, 'systemctl', '''\ +echo "systemctl $*" >> "$LOG" +case "$1:$2" in + is-enabled:podman) echo enabled ;; + is-active:podman) echo active ;; + is-enabled:podman.socket) echo disabled; exit 1 ;; + is-active:podman.socket) echo inactive; exit 3 ;; + *) exit 99 ;; +esac +''') + write_stub(bindir, 'id', '''\ +case "$1" in -nG) echo james ;; -u) echo 1000 ;; -un) echo james ;; *) exit 99 ;; esac +''') + before = snapshot(system), snapshot(Path(env['HOME'])) + result = run_unit(platform_env, 'apk-packages', UNITS_CHECK='1', TEST_MUSL='1') + assert result.returncode == 0, result.stdout + result.stderr + assert 'ok APK package chezmoi' in result.stdout + assert 'WOULD install APK package yq-go' in result.stdout + assert 'ok podman enabled and active' in result.stdout + assert 'WOULD enable and start podman.socket' in result.stdout + assert 'WOULD add james to podman group' in result.stdout + assert 'WOULD run npm install -g @bitwarden/cli' in result.stdout + assert (snapshot(system), snapshot(Path(env['HOME']))) == before + assert log_lines(log) == ['apk info', 'systemctl is-enabled podman', + 'systemctl is-active podman', 'systemctl is-enabled podman.socket', + 'systemctl is-active podman.socket'] + + +def test_apk_check_fails_on_inspection_error_and_skips_retired_services(platform_env): + _, _, bindir, _, log = platform_env + write_stub(bindir, 'apk', 'echo "apk $*" >> "$LOG"; exit 2\n') + result = run_unit(platform_env, 'apk-packages', UNITS_CHECK='1', TEST_MUSL='1', DOTFILES_MACHINE='matrix') + assert result.returncode != 0 + assert 'Cannot inspect installed APK packages' in result.stderr + assert 'SKIPPED: podman services on retired VPS' in result.stdout + assert log_lines(log) == ['apk info'] + write_stub(bindir, 'apk', '[ "$1" = info ] || exit 99; echo bash\n') + write_stub(bindir, 'systemctl', 'exit 1\n') + result = run_unit(platform_env, 'apk-packages', UNITS_CHECK='1', TEST_MUSL='1') + assert result.returncode != 0 + assert 'Cannot inspect podman enablement' in result.stderr + + +def termux_check_stubs(bindir): + write_stub(bindir, 'dpkg-query', '''\ +echo "dpkg-query $*" >> "$LOG" +[ "${DPKG_FAIL:-0}" = 0 ] || exit 2 +printf 'installed python\ninstalled openssh:arm64\nconfig-files git\n' +''') + write_stub(bindir, 'pgrep', '''\ +echo "pgrep $*" >> "$LOG" +exit "${PGREP_RC:-1}" +''') + write_stub(bindir, 'termux-job-scheduler', '''\ +echo "scheduler $*" >> "$LOG" +[ "$1" = -p ] || exit 99 +[ "${SCHED_FAIL:-0}" = 0 ] || exit 2 +[ "${PENDING:-0}" = 0 ] || echo 'Pending Job 1' +''') + for name in ('pkg', 'npm', 'sshd'): + write_stub(bindir, name, 'echo "MUTATION $0 $*" >> "$LOG"; exit 99\n') + + +def test_termux_check_reports_real_drift_without_mutation(platform_env): + _, system, bindir, env, log = platform_env + termux_check_stubs(bindir) + before = snapshot(system), snapshot(Path(env['HOME'])) + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone') + assert result.returncode == 0, result.stdout + result.stderr + assert 'ok Termux package python' in result.stdout + assert 'ok Termux package openssh' in result.stdout + assert 'WOULD install Termux package git' in result.stdout + assert 'WOULD create or chmod Termux boot directory' in result.stdout + assert 'WOULD install or chmod Termux sshd boot script' in result.stdout + assert 'WOULD start Termux sshd' in result.stdout + assert 'WOULD schedule Termux sshd keep-alive every 1800000 ms' in result.stdout + assert 'WOULD run npm install -g @earendil-works/pi-coding-agent' in result.stdout + assert (snapshot(system), snapshot(Path(env['HOME']))) == before + assert not any('MUTATION' in line for line in log_lines(log)) + + +def test_termux_check_detects_existing_boot_and_reports_inspection_failures(platform_env): + root, _, bindir, env, _ = platform_env + termux_check_stubs(bindir) + directory = Path(env['HOME']) / '.termux/boot' + directory.mkdir(parents=True, mode=0o755) + directory.chmod(0o755) + script = directory / 'start-sshd' + script.write_bytes((root / 'units/termux-start-sshd').read_bytes()) + script.chmod(0o755) + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', PENDING='1', PGREP_RC='0') + assert result.returncode == 0, result.stdout + result.stderr + assert 'ok Termux boot directory' in result.stdout + assert 'ok Termux sshd boot script' in result.stdout + assert 'ok Termux sshd running' in result.stdout + assert 'job exists' in result.stdout + for failure in ({'DPKG_FAIL': '1'}, {'SCHED_FAIL': '1'}, {'PGREP_RC': '2'}): + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', **failure) + assert result.returncode != 0 + assert 'Cannot inspect' in result.stderr + + +def tailscale_check_stubs(bindir): + write_stub(bindir, 'tailscale', '''\ +echo "tailscale $*" >> "$LOG" +case "$1:$2" in + status:--json) [ "${STATUS_FAIL:-0}" = 0 ] || exit 2 + printf '{"BackendState":"%s","secret":"ts-secret-test"}\\n' "${TS_STATE:-Running}" ;; + debug:prefs) [ "${PREFS_FAIL:-0}" = 0 ] || exit 2 + printf '%s\\n' '{"Hostname":"other","OperatorUser":"other","AdvertiseRoutes":[],"secret":"ts-secret-test"}' ;; + *) exit 99 ;; +esac +''') + write_stub(bindir, 'systemctl', '''\ +echo "systemctl $*" >> "$LOG" +case "$1" in + is-active) echo "${RESOLVED_STATE:-inactive}"; [ "${RESOLVED_STATE:-inactive}" = active ] && exit 0 || exit 3 ;; + *) exit 99 ;; +esac +''') + for name in ('bw', 'curl', 'sysctl'): + write_stub(bindir, name, 'echo "MUTATION $0 $*" >> "$LOG"; exit 99\n') + + +def test_tailscale_check_reports_connected_drift_without_secrets_or_mutation(platform_env): + root, system, bindir, env, log = platform_env + tailscale_check_stubs(bindir) + (root / 'host_vars/laptop.yml').write_text( + 'tailscale_advertise_routes: "192.168.0.0/24"\ntailscale_accept_dns: false\n') + resolv = system / 'etc/resolv.conf' + resolv.write_text('nameserver 9.9.9.9\n') + before = snapshot(system), snapshot(Path(env['HOME'])) + result = run_unit(platform_env, 'tailscale', UNITS_CHECK='1', BW_SESSION='bw-secret-test') + assert result.returncode == 0, result.stdout + result.stderr + assert 'WOULD configure Tailscale resolved configuration' in result.stdout + assert 'WOULD configure Tailscale sysctl configuration' in result.stdout + assert 'WOULD configure Tailscale static DNS' in result.stdout + assert 'WOULD configure Tailscale advertised routes' in result.stdout + assert 'WOULD set Tailscale hostname' in result.stdout + assert 'WOULD set Tailscale desktop operator' in result.stdout + assert 'WOULD configure Tailscale systray' in result.stdout + assert 'secret-test' not in result.stdout + result.stderr + assert (snapshot(system), snapshot(Path(env['HOME']))) == before + assert log_lines(log) == ['tailscale status --json', 'tailscale debug prefs'] + + +def test_tailscale_check_link_retired_disconnected_and_inspection_errors(platform_env): + _, system, bindir, env, log = platform_env + tailscale_check_stubs(bindir) + resolv = system / 'etc/resolv.conf' + resolv.write_text('nameserver 9.9.9.9\n') + before = snapshot(system), snapshot(Path(env['HOME'])) + result = run_unit(platform_env, 'tailscale', UNITS_CHECK='1', RESOLVED_STATE='active') + assert result.returncode == 0, result.stdout + result.stderr + assert 'WOULD configure Tailscale resolver link' in result.stdout + assert 'WOULD restart systemd-resolved' in result.stdout + assert (snapshot(system), snapshot(Path(env['HOME']))) == before + log.unlink() + result = run_unit(platform_env, 'tailscale', UNITS_CHECK='1', DOTFILES_MACHINE='matrix', TS_STATE='NeedsLogin') + assert result.returncode == 0, result.stdout + result.stderr + assert 'SKIPPED: Tailscale installation and rejoin on retired VPS' in result.stdout + assert 'SKIPPED: resolved restart on retired VPS' in result.stdout + assert 'tailscale debug prefs' not in log_lines(log) + for failure in ({'STATUS_FAIL': '1'}, {'PREFS_FAIL': '1'}): + result = run_unit(platform_env, 'tailscale', UNITS_CHECK='1', **failure) + assert result.returncode != 0 + assert 'inspect Tailscale' in result.stdout or 'Cannot inspect Tailscale' in result.stderr + + +def test_termux_check_rejects_unreadable_overrides_and_skips_non_host(platform_env): + root, _, bindir, _, log = platform_env + termux_check_stubs(bindir) + (root / 'host_vars/phone.yml').write_text('termux_pkgs:\n - custom\n') + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone') + assert result.returncode != 0 + assert 'yq is required' in result.stderr + assert not any('MUTATION' in line for line in log_lines(log)) + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='laptop') + assert result.returncode == 0 + assert 'SKIPPED: not in termux_hosts' in result.stdout + + +def test_tailscale_check_missing_binary_and_termux_skip(platform_env): + _, system, bindir, env, log = platform_env + write_stub(bindir, 'systemctl', 'echo "systemctl $*" >> "$LOG"; echo inactive; exit 3\n') + before = snapshot(system), snapshot(Path(env['HOME'])) + result = run_unit(platform_env, 'tailscale', UNITS_CHECK='1', BW_SESSION='bw-secret-test') + assert result.returncode == 0, result.stdout + result.stderr + assert 'WOULD install Tailscale and join tailnet' in result.stdout + assert 'bw-secret-test' not in result.stdout + result.stderr + assert (snapshot(system), snapshot(Path(env['HOME']))) == before + log.unlink() + result = run_unit(platform_env, 'tailscale', UNITS_CHECK='1', TEST_TERMUX='1') + assert result.returncode == 0 + assert 'SKIPPED: Android Tailscale' in result.stdout + assert not log.exists() diff --git a/tests/test_fleet_dotfiles_check.py b/tests/test_fleet_dotfiles_check.py new file mode 100644 index 00000000..449278b3 --- /dev/null +++ b/tests/test_fleet_dotfiles_check.py @@ -0,0 +1,139 @@ +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub + + +def run(unit, env, **extra): + return subprocess.run( + ["bash", str(REPO / "units" / f"{unit}.sh")], + env={**env, "DOTFILES_DIR": str(REPO), "BREW_PREFIX": "/nonexistent", + "UNITS_CHECK": "1", "DOTFILES_IS_DESKTOP": "1", **extra}, + capture_output=True, text=True, + ) + + +def test_fonts_check_reports_install_without_creating_directory(stub_env): + bindir, env = stub_env + for command in ("curl", "unzip", "fc-cache"): + write_stub(bindir, command, 'echo "unexpected mutation" >&2; exit 99\n') + result = run("fonts", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "create font directory" in result.stdout + assert "install JetBrains" in result.stdout + assert list(Path(env["HOME"]).iterdir()) == [] + + +def test_fonts_check_detects_existing_font_and_nondirectory_error(stub_env): + _, env = stub_env + root = Path(env["HOME"]) + font = root / ".local/share/fonts/JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf" + font.parent.mkdir(parents=True) + font.write_bytes(b"font") + result = run("fonts", env) + assert result.returncode == 0 + assert "already installed" in result.stdout + font.unlink() + font.parent.rmdir() + font.parent.parent.rmdir() + font.parent.parent.write_text("blocked") + result = run("fonts", env) + assert result.returncode != 0 + assert "not a directory" in result.stdout + assert font.parent.parent.read_text() == "blocked" + + +def test_fonts_check_handles_nondesktop_and_inventory_errors(stub_env): + bindir, env = stub_env + result = run("fonts", env, DOTFILES_IS_DESKTOP="0") + assert result.returncode == 0 + assert "create font directory" in result.stdout + assert "not a desktop" in result.stdout + assert "download" not in result.stdout + write_stub(bindir, "yq", "exit 5\n") + result = run("fonts", env, DOTFILES_IS_DESKTOP="", DOTFILES_MACHINE="unknown-test-host") + assert result.returncode != 0 + assert "cannot inspect desktop membership" in result.stdout + assert list(Path(env["HOME"]).iterdir()) == [] + + +def test_chezmoi_check_uses_dry_status_without_exposing_file_contents(stub_env): + bindir, env = stub_env + write_stub(bindir, "chezmoi", '''\ +case " $* " in + *" --init "*) echo 'unsafe init' >&2; exit 99 ;; + *" --dry-run "*" status "*) printf ' M .gitconfig\\n R .chezmoiscripts/example\\n' ;; + *) echo 'unsafe invocation' >&2; exit 99 ;; +esac +''') + target = Path(env["HOME"], ".gitconfig") + target.write_text("private token\n") + result = run("chezmoi", env) + assert result.returncode == 0, result.stdout + result.stderr + assert ".gitconfig" in result.stdout and " R " in result.stdout + assert "private token" not in result.stdout + result.stderr + assert target.read_text() == "private token\n" + assert sorted(p.name for p in Path(env["HOME"]).iterdir()) == [".gitconfig"] + + +def test_chezmoi_check_propagates_inspection_error(stub_env): + bindir, env = stub_env + write_stub(bindir, "chezmoi", "exit 7\n") + result = run("chezmoi", env) + assert result.returncode != 0 + assert "status inspection failed" in result.stdout + assert list(Path(env["HOME"]).iterdir()) == [] + + +def test_unsupported_atuin_and_omp_checks_fail_before_running_them(stub_env): + bindir, env = stub_env + for command in ("atuin", "omp"): + write_stub(bindir, command, 'echo "unsafe invocation" >&2; exit 99\n') + for unit in ("atuin", "omp"): + result = run(unit, env) + assert result.returncode != 0 + assert "cannot be inspected safely" in result.stdout + assert "unsafe invocation" not in result.stderr + assert list(Path(env["HOME"]).iterdir()) == [] + + +def home_snapshot(home): + result = {} + for path in home.rglob("*"): + stat = path.lstat() + if path.is_symlink(): + contents = ("link", os.readlink(path)) + elif path.is_file(): + contents = ("file", path.read_bytes()) + else: + contents = ("directory",) + result[str(path.relative_to(home))] = (stat.st_mode, stat.st_mtime_ns, contents) + return result + + +@pytest.mark.parametrize("existing_state", [False, True]) +def test_real_chezmoi_check_preserves_absent_or_existing_state(stub_env, existing_state): + binary = shutil.which("chezmoi") + if not binary: + pytest.skip("chezmoi not installed") + _, env = stub_env + home = Path(env["HOME"]) + env = {**env, "XDG_CONFIG_HOME": str(home / ".config"), + "XDG_CACHE_HOME": str(home / ".cache"), + "XDG_DATA_HOME": str(home / ".local/share"), + "DOTFILES_MACHINE": "test-host", "DOTFILES_IS_DESKTOP": "1"} + if existing_state: + setup = subprocess.run( + [binary, "--source", str(REPO), "--force", "--no-tty", "apply", "--init", "--exclude=scripts"], + env=env, capture_output=True, text=True, + ) + assert setup.returncode == 0, setup.stdout + setup.stderr + assert (home / ".config/chezmoi/chezmoistate.boltdb").is_file() + before = home_snapshot(home) + result = run("chezmoi", env) + assert result.returncode == 0, result.stdout + result.stderr + assert home_snapshot(home) == before diff --git a/tests/test_fleet_facts_check.py b/tests/test_fleet_facts_check.py new file mode 100644 index 00000000..18839381 --- /dev/null +++ b/tests/test_fleet_facts_check.py @@ -0,0 +1,48 @@ +from types import SimpleNamespace + +import pytest + +from test_fleet_facts_native import config_repo, fake_host, fleet + + +def test_check_plans_publication_without_github_write(tmp_path, monkeypatch, capsys): + repo = config_repo(tmp_path) + monkeypatch.setenv('HOME', str(tmp_path)) + monkeypatch.setattr(fleet.sys, 'argv', ['fleet-facts.py', str(repo), 'laptop', '0', '--check']) + monkeypatch.setattr(fleet, 'config', lambda *args: (True, 'hanthor/dotfiles', 'fleet-facts', ['desktop'])) + monkeypatch.setattr(fleet, 'collect', lambda *args, **kwargs: {'private': 'SECRET'}) + calls = [] + def command(args): + calls.append(args) + return SimpleNamespace(returncode=0, stdout='') + monkeypatch.setattr(fleet, 'command', command) + monkeypatch.setattr(fleet.subprocess, 'run', lambda *args, **kwargs: pytest.fail('must not publish')) + assert fleet.main() == 0 + assert calls == [['gh', 'auth', 'status']] + output = capsys.readouterr().out + assert 'PLAN: publish facts/laptop.json' in output + assert 'SECRET' not in output + + +def test_check_fails_if_auth_inspection_fails(tmp_path, monkeypatch): + repo = config_repo(tmp_path) + monkeypatch.setattr(fleet.sys, 'argv', ['fleet-facts.py', str(repo), 'laptop', '0', '--check']) + monkeypatch.setattr(fleet, 'command', lambda args: None) + monkeypatch.setattr(fleet, 'collect', lambda *args: pytest.fail('must not collect')) + with pytest.raises(RuntimeError, match='gh authentication unavailable'): + fleet.main() + + +def test_check_respects_publish_opt_out(tmp_path, monkeypatch, capsys): + repo = config_repo(tmp_path) + (repo / 'host_vars/laptop.yml').write_text('fleet_facts_publish: false\n') + monkeypatch.setattr(fleet.sys, 'argv', ['fleet-facts.py', str(repo), 'laptop', '0', '--check']) + monkeypatch.setattr(fleet, 'command', lambda args: pytest.fail('must not invoke gh')) + assert fleet.main() == 0 + assert 'SKIPPED' in capsys.readouterr().out + + +def test_strict_collection_reports_failed_gpu_inspection(fake_host): + root, home, _ = fake_host + with pytest.raises(RuntimeError, match='GPU inventory inspection failed'): + fleet.collect('laptop', [], home, root=root, run=lambda args: None, strict=True) diff --git a/tests/test_fleet_facts_native.py b/tests/test_fleet_facts_native.py new file mode 100644 index 00000000..a8ef45f4 --- /dev/null +++ b/tests/test_fleet_facts_native.py @@ -0,0 +1,142 @@ +import base64 +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from conftest import REPO + +SPEC = importlib.util.spec_from_file_location('fleet_facts_native', REPO / 'units/fleet-facts.py') +fleet = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(fleet) + + +def put(root, path, text): + dest = root / path.lstrip('/') + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(text) + + +@pytest.fixture +def fake_host(tmp_path, monkeypatch): + root = tmp_path / 'root' + home = tmp_path / 'home' + home.mkdir() + put(root, '/etc/os-release', 'NAME="Example Linux"\nVERSION_ID="42"\nVERSION_CODENAME=stable\n') + put(root, '/proc/cpuinfo', 'processor : 0\nmodel name : Public CPU 123\nprocessor : 1\nmodel name : Public CPU 123\n') + put(root, '/proc/meminfo', 'MemTotal: 16777216 kB\n') + put(root, '/proc/uptime', '172800.0 0.0\n') + put(root, '/sys/devices/virtual/dmi/id/sys_vendor', 'Example Vendor\n') + put(root, '/sys/devices/virtual/dmi/id/product_name', 'System Product Name\n') + put(root, '/sys/firmware/devicetree/base/model', 'Example Device\0') + put(root, '/sys/devices/virtual/dmi/id/chassis_type', '9\n') + monkeypatch.setattr(fleet.platform, 'release', lambda: '6.9.0') + monkeypatch.setattr(fleet.platform, 'machine', lambda: 'x86_64') + def run(args): + if args[0] == 'lspci': + return SimpleNamespace(returncode=0, stdout='00:02.0 "VGA compatible controller" "Acme" "Graphics"\n00:03.0 "VGA compatible controller" "Acme" "Device 1111"\n') + return SimpleNamespace(returncode=1, stdout='') + return root, home, run + + +def test_collect_public_schema_and_filtered_last_apply(fake_host): + root, home, run = fake_host + last = home / '.cache/dotfiles/last-apply.json' + last.parent.mkdir(parents=True) + last.write_text(json.dumps({'timestamp': '2026-10-27T01:02:03Z', 'exit_code': 0, 'hostname': 'PRIVATE-HOST', 'branch': 'SECRET-BRANCH', 'token': 'SECRET-TOKEN'})) + doc = fleet.collect('laptop', ['desktop'], home, root=root, run=run) + assert doc['schema'] == 1 and doc['host'] == 'laptop' and doc['groups'] == ['desktop'] + assert doc['os'] == {'name': 'Example Linux', 'version': '42', 'codename': 'stable', 'kernel': '6.9.0'} + assert doc['hardware'] == {'vendor': 'Example Vendor', 'model': 'Example Device', 'form_factor': 'Laptop', 'arch': 'x86_64', 'cpu': 'Public CPU 123', 'cpu_threads': 2, 'gpus': ['Acme Graphics'], 'memory_gb': 16.0} + assert doc['uptime_days'] == 2.0 and doc['last_apply'] == {'at': '2026-10-27T01:02:03Z', 'ok': True} + assert 'PRIVATE-HOST' not in json.dumps(doc) and 'SECRET-' not in json.dumps(doc) + + +def test_sensitive_hardware_values_are_never_published(fake_host, capsys): + root, home, run = fake_host + put(root, '/sys/firmware/devicetree/base/model', 'serial number PRIVATE-1234\n') + put(root, '/sys/devices/virtual/dmi/id/product_name', '192.168.1.4\n') + put(root, '/sys/devices/virtual/dmi/id/sys_vendor', 'token=SECRET-TOKEN\n') + doc = fleet.collect('laptop', [], home, root=root, run=run) + content = json.dumps(doc) + assert doc['hardware']['model'] == '' and doc['hardware']['vendor'] == '' + assert 'PRIVATE-1234' not in content and '192.168.1.4' not in content and 'SECRET-TOKEN' not in content + assert 'omitted' in capsys.readouterr().err + + +def test_required_collection_failure_is_not_faked(fake_host): + root, home, run = fake_host + (root / 'proc/meminfo').unlink() + with pytest.raises(RuntimeError, match='required OS or hardware data unavailable'): + fleet.collect('laptop', [], home, root=root, run=run) + + +def test_termux_android_model_and_release(fake_host): + root, home, _ = fake_host + def run(args): + if args == ['getprop', 'ro.build.version.release']: + return SimpleNamespace(returncode=0, stdout='16\n') + if args == ['getprop', 'ro.product.model']: + return SimpleNamespace(returncode=0, stdout='Pixel 9a\n') + return SimpleNamespace(returncode=1, stdout='') + doc = fleet.collect('termux', ['termux_hosts'], home, termux=True, root=root, run=run) + assert doc['os']['name'] == 'Android' and doc['os']['version'] == '16' + assert doc['hardware']['model'] == 'Pixel 9a' + + +def config_repo(tmp_path): + repo = tmp_path / 'repo' + (repo / 'units').mkdir(parents=True) + (repo / 'units/fleet-facts').symlink_to(REPO / 'units/fleet-facts') + (repo / 'group_vars').mkdir() + (repo / 'host_vars').mkdir() + (repo / 'inventory.yml').write_text('all:\n hosts:\n laptop: {}\n children:\n desktop:\n hosts:\n laptop: {}\n') + return repo + + +def test_group_defaults_and_host_override(tmp_path): + repo = config_repo(tmp_path) + (repo / 'group_vars/desktop.yml').write_text('fleet_facts_publish: false\nfleet_facts_branch: desktop-facts\n') + assert fleet.config(repo, 'laptop') == (False, 'hanthor/dotfiles', 'desktop-facts', ['desktop']) + (repo / 'host_vars/laptop.yml').write_text('fleet_facts_publish: true\nfleet_facts_repo: example/fleet\n') + assert fleet.config(repo, 'laptop') == (True, 'example/fleet', 'desktop-facts', ['desktop']) + + +def test_github_contents_write_and_nonfatal_failure(tmp_path, monkeypatch, capsys): + repo = config_repo(tmp_path) + monkeypatch.setenv('HOME', str(tmp_path)) + monkeypatch.setattr(fleet.sys, 'argv', ['fleet-facts.py', str(repo), 'laptop', '0']) + doc = {'schema': 1, 'host': 'laptop', 'hardware': {'model': 'Public'}} + monkeypatch.setattr(fleet, 'collect', lambda *args: doc) + def command(args): + if args[:2] == ['gh', 'auth']: + return SimpleNamespace(returncode=0, stdout='') + assert args == ['gh', 'api', 'repos/hanthor/dotfiles/contents/facts/laptop.json?ref=fleet-facts', '--jq', '.sha'] + return SimpleNamespace(returncode=0, stdout='abc123\n') + monkeypatch.setattr(fleet, 'command', command) + sent = [] + def put_request(args, **kwargs): + sent.append((args, json.loads(kwargs['input']))) + return SimpleNamespace(returncode=1) + monkeypatch.setattr(fleet.subprocess, 'run', put_request) + assert fleet.main() == 0 + args, body = sent[0] + assert args[:5] == ['gh', 'api', '-X', 'PUT', 'repos/hanthor/dotfiles/contents/facts/laptop.json'] + assert body['branch'] == 'fleet-facts' and body['sha'] == 'abc123' + assert json.loads(base64.b64decode(body['content'])) == doc + assert 'could not publish' in capsys.readouterr().err + + +def test_opt_out_and_missing_auth_never_collect(tmp_path, monkeypatch): + repo = config_repo(tmp_path) + monkeypatch.setenv('HOME', str(tmp_path)) + monkeypatch.setattr(fleet.sys, 'argv', ['fleet-facts.py', str(repo), 'laptop', '0']) + monkeypatch.setattr(fleet, 'collect', lambda *args: pytest.fail('must not collect')) + (repo / 'host_vars/laptop.yml').write_text('fleet_facts_publish: false\n') + monkeypatch.setattr(fleet, 'command', lambda args: pytest.fail('must not invoke gh for opt-out')) + assert fleet.main() == 0 + (repo / 'host_vars/laptop.yml').write_text('fleet_facts_publish: true\n') + monkeypatch.setattr(fleet, 'command', lambda args: None) + assert fleet.main() == 0 diff --git a/tests/test_forgejo_check_credentials.py b/tests/test_forgejo_check_credentials.py new file mode 100644 index 00000000..ce4b47af --- /dev/null +++ b/tests/test_forgejo_check_credentials.py @@ -0,0 +1,9 @@ +from test_forgejo_registry_unit import env +from unit_helpers import calls, run_unit + + +def test_check_blocks_credential_helper_and_registry_login(env): + result = run_unit("forgejo-registry", env, UNITS_CHECK="1") + assert result.returncode != 0 + assert "credential helpers may prompt" in result.stdout + assert calls(env) == "" diff --git a/tests/test_github_check_credentials.py b/tests/test_github_check_credentials.py new file mode 100644 index 00000000..0394b481 --- /dev/null +++ b/tests/test_github_check_credentials.py @@ -0,0 +1,21 @@ +from pathlib import Path + +from test_github_unit import env +from unit_helpers import calls, run_unit + + +def test_check_known_hosts_without_github_or_vault_calls(env): + ssh = Path(env["HOME"], ".ssh") + ssh.chmod(0o755) + result = run_unit("github", env, UNITS_CHECK="1") + assert result.returncode != 0 + assert "would set" in result.stdout and "would add GitHub host key" in result.stdout + assert "keyring" in result.stdout and "audit events" in result.stdout + assert not (ssh / "known_hosts").exists() + assert ssh.stat().st_mode & 0o777 == 0o755 and calls(env) == "" + + +def test_check_no_secrets_only_inspects_known_hosts(env): + result = run_unit("github", env, UNITS_CHECK="1", UNITS_NO_SECRETS="1") + assert result.returncode == 0 and "would add GitHub host key" in result.stdout + assert calls(env) == "" and not Path(env["HOME"], ".ssh/known_hosts").exists() diff --git a/tests/test_gnome_unit.py b/tests/test_gnome_unit.py new file mode 100644 index 00000000..5d6c8bff --- /dev/null +++ b/tests/test_gnome_unit.py @@ -0,0 +1,257 @@ +import importlib.util +import os +import shutil +import socket +import stat +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from conftest import REPO, write_stub +from unit_helpers import calls, run_unit + + +@pytest.fixture +def env(stub_env): + bindir, env = stub_env + write_stub(bindir, "yq", ''' +echo "yq $*" >> "$HOME/calls" +[ "${STUB_YQ_FAIL:-0}" = 1 ] && exit 1 +case "$*" in + *'has('* ) echo "${STUB_DESKTOP:-true}" ;; + *'.brew_prefix'*) echo /custom/brew ;; + *'.gnome_enabled_extensions'*host_vars*) echo "${STUB_HOST_EXTENSIONS:-null}" ;; + *'.gnome_enabled_extensions'*) echo '["canonical@example", "other@example"]' ;; + *) exit 1 ;; +esac +''') + write_stub(bindir, "gdbus", 'echo "gdbus $*" >> "$HOME/calls"\nexit "${STUB_BUS_FAIL:-0}"\n') + write_stub(bindir, "dconf", ''' +echo "dconf $* bus=${DBUS_SESSION_BUS_ADDRESS:-unset}" >> "$HOME/calls" +cat > /dev/null +case "$*" in + *media-keys*) exit "${STUB_KEY_FAIL:-0}" ;; + *extensions*) exit "${STUB_EXTENSION_FAIL:-0}" ;; +esac +''') + write_stub(bindir, "gsettings", ''' +echo "gsettings $* bus=${DBUS_SESSION_BUS_ADDRESS:-unset}" >> "$HOME/calls" +case "$*" in + 'get org.gnome.Ptyxis default-profile-uuid') echo "'test-uuid'" ;; + *' custom-command') echo "'${STUB_CURRENT_COMMAND:-/old/fish}'" ;; + 'get org.gnome.Ptyxis font-name') echo "'${STUB_CURRENT_FONT:-old font}'" ;; +esac +exit "${STUB_GSETTINGS_FAIL:-0}" +''') + write_stub(bindir, "curl", ''' +echo "curl $*" >> "$HOME/calls" +[ "${STUB_CURL_FAIL:-0}" = 1 ] && exit 1 +while [ "$#" -gt 0 ]; do + if [ "$1" = -o ]; then printf image > "$2"; exit 0; fi + shift +done +exit 1 +''') + write_stub(bindir, "sudo", 'echo "sudo $*" >> "$HOME/calls"\nexit "${STUB_SUDO_FAIL:-0}"\n') + write_stub(bindir, "python3", 'echo "python3 $*" >> "$HOME/calls"\nexit "${STUB_SUDO_FAIL:-0}"\n') + write_stub(bindir, "grdctl", ''' +echo "grdctl $*" >> "$HOME/calls" +[ "$*" = "rdp status" ] && echo "Status: enabled" +exit "${STUB_GRD_FAIL:-0}" +''') + write_stub(bindir, "systemctl", 'echo "systemctl $* runtime=${XDG_RUNTIME_DIR:-unset}" >> "$HOME/calls"\nexit "${STUB_SYSTEMCTL_FAIL:-0}"\n') + env.update(DOTFILES_IS_DESKTOP="1", DOTFILES_MACHINE="gnome-test", DBUS_SESSION_BUS_ADDRESS="unix:path=/test/user-bus") + return env + + +def test_desktop_uses_canonical_assets_and_live_user_session(env): + result = run_unit("gnome", env) + assert result.returncode == 0, result.stdout + result.stderr + home = Path(env["HOME"]) + files = REPO / "units/gnome" + keys = home / ".config/dconf/gnome-keybindings.ini" + assert keys.read_bytes() == (files / "gnome-keybindings.ini").read_bytes() + assert stat.S_IMODE(keys.stat().st_mode) == 0o644 + extension = home / ".local/share/gnome-shell/extensions/papershell@lalovene.github.com/extension.js" + assert extension.read_bytes() == (files / "extensions/papershell@lalovene.github.com/extension.js").read_bytes() + assert stat.S_IMODE(extension.stat().st_mode) == 0o644 + log = calls(env) + assert "dconf load /org/gnome/settings-daemon/plugins/media-keys/ bus=unix:path=/test/user-bus" in log + assert "dconf load /org/gnome/shell/extensions/ bus=unix:path=/test/user-bus" in log + assert 'gsettings set org.gnome.shell enabled-extensions ["canonical@example", "other@example"]' in log + assert "group_vars/all.yml" in log + assert "custom-command /custom/brew/bin/fish" in log + assert "systemctl --user enable --now gnome-remote-desktop.service" in log + assert "grdctl rdp enable" not in log + assert "gnome-avatar.py" in log + + mtime = extension.stat().st_mtime_ns + first_keys = log.count("dconf load /org/gnome/settings-daemon/plugins/media-keys/") + second = run_unit("gnome", env, STUB_CURRENT_COMMAND="/custom/brew/bin/fish", STUB_CURRENT_FONT="JetBrainsMono Nerd Font 12") + assert second.returncode == 0, second.stdout + second.stderr + assert extension.stat().st_mtime_ns == mtime + assert calls(env).count("dconf load /org/gnome/settings-daemon/plugins/media-keys/") == first_keys + assert calls(env).count("custom-command /custom/brew/bin/fish") == 1 + + +def test_failed_keybinding_load_is_reported_and_retried(env): + result = run_unit("gnome", env, STUB_KEY_FAIL="1") + assert result.returncode == 1 + assert "FAIL load GNOME keybindings in user session" in result.stdout + marker = Path(env["HOME"]) / ".config/dconf/.gnome-keybindings.pending" + assert marker.exists() + assert "dconf load /org/gnome/shell/extensions/" in calls(env) + result = run_unit("gnome", env) + assert result.returncode == 0, result.stdout + result.stderr + assert not marker.exists() + assert calls(env).count("dconf load /org/gnome/settings-daemon/plugins/media-keys/") == 2 + + +@pytest.mark.parametrize("extra", [ + {"DBUS_SESSION_BUS_ADDRESS": "", "XDG_RUNTIME_DIR": ""}, + {"DBUS_SESSION_BUS_ADDRESS": "autolaunch:", "XDG_RUNTIME_DIR": ""}, + {"STUB_BUS_FAIL": "1"}, +]) +def test_missing_or_unreachable_session_is_not_treated_as_success(env, extra): + bash_env = Path(env["HOME"]) / "no-runtime-socket.bash" + bash_env.write_text('function [() { if [[ "${1:-}" == -S && "${2:-}" == "/run/user/$UID/bus" ]]; then return 1; fi; builtin [ "$@"; }\n') + result = run_unit("gnome", env, BASH_ENV=str(bash_env), **extra) + assert result.returncode == 1 + assert "GNOME user D-Bus session unavailable" in result.stdout + assert "FAIL load GNOME keybindings" in result.stdout + assert Path(env["HOME"], ".config/dconf/.gnome-keybindings.pending").exists() + assert "dconf load" not in calls(env) + assert "gsettings " not in calls(env) + assert "systemctl " not in calls(env) + + +def test_existing_runtime_bus_is_used_without_autolaunch(env, tmp_path): + runtime = tmp_path / "runtime" + runtime.mkdir() + with socket.socket(socket.AF_UNIX) as bus: + bus.bind(str(runtime / "bus")) + result = run_unit("gnome", env, DBUS_SESSION_BUS_ADDRESS="", XDG_RUNTIME_DIR=str(runtime)) + assert result.returncode == 0, result.stdout + result.stderr + assert f"bus=unix:path={runtime}/bus" in calls(env) + + +@pytest.mark.parametrize("address", ["", "unix:path=/existing/session"]) +def test_ssh_discovers_user_runtime_without_overriding_inherited_bus(env, address): + # Stub only the socket check so this never depends on the test host's /run. + bash_env = Path(env["HOME"]) / "runtime-socket.bash" + bash_env.write_text('function [() { if [[ "${1:-}" == -S && "${2:-}" == "/run/user/$UID/bus" ]]; then return 0; fi; builtin [ "$@"; }\n') + result = run_unit("gnome", env, BASH_ENV=str(bash_env), XDG_RUNTIME_DIR="", DBUS_SESSION_BUS_ADDRESS=address) + assert result.returncode == 0, result.stdout + result.stderr + runtime = f"/run/user/{os.getuid()}" + expected_bus = address or f"unix:path={runtime}/bus" + assert f"dconf load /org/gnome/shell/extensions/ bus={expected_bus}" in calls(env) + assert f"systemctl --user enable --now gnome-remote-desktop.service runtime={runtime}" in calls(env) + + +def test_optional_session_failures_are_visible(env): + result = run_unit("gnome", env, STUB_EXTENSION_FAIL="1", STUB_GSETTINGS_FAIL="1", STUB_GRD_FAIL="1", STUB_SYSTEMCTL_FAIL="1") + assert result.returncode == 0 + assert "warn Ptyxis profile" in result.stdout + assert "warn load GNOME extension settings" in result.stdout + assert "warn enable GNOME extensions" in result.stdout + assert "warn enable GNOME RDP protocol" in result.stdout + assert "warn enable GNOME Remote Desktop user service" in result.stdout + + +@pytest.mark.parametrize("failure", [{"STUB_CURL_FAIL": "1"}, {"STUB_SUDO_FAIL": "1"}]) +def test_avatar_failure_fails_without_suppressing_other_steps(env, failure): + result = run_unit("gnome", env, **failure) + assert result.returncode == 1 + assert "FAIL AccountsService avatar" in result.stdout + assert "dconf load /org/gnome/shell/extensions/" in calls(env) + + +def test_non_desktop_and_profile_gates(env, tmp_path): + result = run_unit("gnome", env, DOTFILES_IS_DESKTOP="0") + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert not Path(env["HOME"], "calls").exists() + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "host_vars/phone.yml").write_text("skip_gnome: true\n") + result = run_unit("gnome", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="phone") + assert result.returncode == 0 and "SKIPPED" in result.stdout + (repo / "host_vars/phone.yml").write_text("machine_profile: vm-test\n") + result = run_unit("gnome", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="phone") + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert not Path(env["HOME"], "calls").exists() + + +def test_inventory_error_and_missing_yq_do_not_silently_skip(env, tmp_path): + result = run_unit("gnome", env, DOTFILES_IS_DESKTOP="", STUB_YQ_FAIL="1") + assert result.returncode == 1 and "FAIL determine desktop membership" in result.stdout + bindir = tmp_path / "limited-bin" + bindir.mkdir() + for binary in ("bash", "dirname", "sed", "head", "cat"): + (bindir / binary).symlink_to(shutil.which(binary)) + result = run_unit("gnome", env, DOTFILES_IS_DESKTOP="", PATH=str(bindir)) + assert result.returncode == 1 and "FAIL determine desktop membership" in result.stdout + + +def test_host_extension_override_and_configuration_failure(env, tmp_path): + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "host_vars/gnome-test.yml").write_text("brew_prefix: /host/brew\ngnome_enabled_extensions:\n - host@example\n") + (repo / "units").mkdir(parents=True) + (repo / "units/gnome").symlink_to(REPO / "units/gnome") + result = run_unit("gnome", env, DOTFILES_DIR=str(repo), STUB_HOST_EXTENSIONS='["host@example"]') + assert result.returncode == 0, result.stdout + result.stderr + assert 'gsettings set org.gnome.shell enabled-extensions ["host@example"]' in calls(env) + assert "custom-command /host/brew/bin/fish" in calls(env) + result = run_unit("gnome", env, STUB_YQ_FAIL="1") + assert result.returncode == 1 and "FAIL read GNOME configuration" in result.stdout + + +def load_avatar_helper(): + spec = importlib.util.spec_from_file_location("gnome_avatar", REPO / "units/gnome-avatar.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_avatar_updates_accounts_and_gdm_idempotently(tmp_path, monkeypatch): + helper = load_avatar_helper() + icons = tmp_path / "icons" + users = tmp_path / "users" + icons.mkdir() + users.mkdir() + account = users / "james" + account.write_text("# retained\n[User]\nName=James\nIcon=/old\nIcon=/duplicate\n[Other]\nIcon=untouched\n") + source = tmp_path / "source.png" + source.write_bytes(b"avatar") + monkeypatch.setattr(helper, "ICON", icons / "james") + monkeypatch.setattr(helper, "ACCOUNT", account) + monkeypatch.setattr(helper, "GDM_CACHE", tmp_path / "gdm/.cache") + monkeypatch.setattr(helper.pwd, "getpwnam", lambda name: SimpleNamespace(pw_uid=os.getuid())) + monkeypatch.setattr(helper.grp, "getgrnam", lambda name: SimpleNamespace(gr_gid=os.getgid())) + helper.install_avatar(source) + cache = tmp_path / "gdm/.cache/james" + assert cache.read_bytes() == (icons / "james").read_bytes() == b"avatar" + assert account.read_text() == "# retained\n[User]\nName=James\nIcon=/var/lib/AccountsService/icons/james\n[Other]\nIcon=untouched\n" + for path in (cache, icons / "james", account): + assert stat.S_IMODE(path.stat().st_mode) == 0o644 + before = [(p.stat().st_ino, p.stat().st_mtime_ns) for p in (cache, icons / "james", account)] + helper.install_avatar(source) + assert before == [(p.stat().st_ino, p.stat().st_mtime_ns) for p in (cache, icons / "james", account)] + + +def test_avatar_without_gdm_creates_user_section(tmp_path, monkeypatch): + helper = load_avatar_helper() + (tmp_path / "icons").mkdir() + (tmp_path / "users").mkdir() + source = tmp_path / "source.png" + source.write_bytes(b"avatar") + monkeypatch.setattr(helper, "ICON", tmp_path / "icons/james") + monkeypatch.setattr(helper, "ACCOUNT", tmp_path / "users/james") + monkeypatch.setattr(helper, "GDM_CACHE", tmp_path / "gdm/.cache") + def no_gdm(name): + raise KeyError(name) + monkeypatch.setattr(helper.pwd, "getpwnam", no_gdm) + helper.install_avatar(source) + assert (tmp_path / "users/james").read_text() == "[User]\nIcon=/var/lib/AccountsService/icons/james\n" + assert not (tmp_path / "gdm").exists() diff --git a/tests/test_hive_ops_migration.py b/tests/test_hive_ops_migration.py new file mode 100644 index 00000000..f149388c --- /dev/null +++ b/tests/test_hive_ops_migration.py @@ -0,0 +1,90 @@ +import shutil +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub +from unit_helpers import calls, run_unit + +pytestmark = pytest.mark.skipif(shutil.which("yq") is None, reason="yq is required") + + +def hive_repo(tmp_path, host, variables): + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "host_vars" / f"{host}.yml").write_text(variables) + (repo / "units").symlink_to(REPO / "units") + (repo / "inventory.yml").write_text("all:\n children:\n vps:\n hosts:\n matrix: {}\n telengana: {}\n") + return repo + + +def test_opt_in_deploys_aws_dropins_and_timers_without_restart(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = hive_repo(tmp_path, "laptop", "hive_ops_enabled: true\n") + args = dict(DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop") + home = Path(env["HOME"]) + (home / ".kube").mkdir() + (home / ".kube/config").write_text("offline home cluster") + result = run_unit("hive-ops", env, **args) + assert result.returncode == 0, result.stdout + result.stderr + units = home / ".config/systemd/user" + dropin = (units / "hive-watchdog.service.d/kubeconfig.conf").read_text() + assert f"KUBECONFIG={home}/.kube/config-aws-migration" in dropin + assert "/home/linuxbrew/.linuxbrew/bin" in dropin + assert (home / ".kube/config").read_text() == "offline home cluster" + assert (home / ".local/bin/hive-rotate.sh").stat().st_mode & 0o111 + assert "enable --now hive-watchdog.timer" in calls(env) + assert "restart" not in calls(env) + (home / "calls").unlink() + second = run_unit("hive-ops", env, **args) + assert second.returncode == 0, second.stdout + second.stderr + assert "daemon-reload" not in calls(env) + + +def test_opt_out_preserves_unrelated_dropins(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = hive_repo(tmp_path, "laptop", "hive_ops_enabled: false\n") + units = Path(env["HOME"]) / ".config/systemd/user" + dropins = units / "hive-watchdog.service.d" + dropins.mkdir(parents=True) + (units / "hive-watchdog.timer").write_text("old") + (units / "hive-watchdog.service").write_text("old") + (dropins / "kubeconfig.conf").write_text("old") + (dropins / "custom.conf").write_text("unrelated") + result = run_unit("hive-ops", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop") + assert result.returncode == 0, result.stdout + result.stderr + assert not (units / "hive-watchdog.timer").exists() + assert not (dropins / "kubeconfig.conf").exists() + assert (dropins / "custom.conf").read_text() == "unrelated" + assert "disable --now hive-watchdog.timer" in calls(env) + + +def test_disable_failure_does_not_remove_live_timer_or_helpers(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n[[ "$*" != *"disable --now hive-watchdog.timer"* ]]\n') + repo = hive_repo(tmp_path, "laptop", "hive_ops_enabled: false\n") + home = Path(env["HOME"]) + units = home / ".config/systemd/user" + units.mkdir(parents=True) + (units / "hive-watchdog.timer").write_text("old") + helpers = home / ".local/bin" + helpers.mkdir(parents=True) + (helpers / "hive-rotate.sh").write_text("old") + result = run_unit("hive-ops", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop") + assert result.returncode != 0 + assert (units / "hive-watchdog.timer").exists() + assert (helpers / "hive-rotate.sh").exists() + + +def test_retired_vps_keeps_old_hive_units(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = hive_repo(tmp_path, "telengana", "hive_ops_enabled: false\n") + timer = Path(env["HOME"]) / ".config/systemd/user/hive-watchdog.timer" + timer.parent.mkdir(parents=True) + timer.write_text("old") + result = run_unit("hive-ops", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="telengana") + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert timer.read_text() == "old" and calls(env) == "" diff --git a/tests/test_hive_peak.py b/tests/test_hive_peak.py index 4b3fdc23..1d88e1b4 100644 --- a/tests/test_hive_peak.py +++ b/tests/test_hive_peak.py @@ -5,7 +5,7 @@ from conftest import REPO, write_stub -SCRIPT = REPO / "roles/hive_ops/files/bin/hive-peak.sh" +SCRIPT = REPO / "units/hive-ops/bin/hive-peak.sh" # Fake kubectl: one pod, one unexpired owner session, and a dashboard whose # POSTs succeed except for agents listed in $STUB_FAIL. POSTs go to $STUB_LOG. diff --git a/tests/test_hive_syncthing_check.py b/tests/test_hive_syncthing_check.py new file mode 100644 index 00000000..7f30e78b --- /dev/null +++ b/tests/test_hive_syncthing_check.py @@ -0,0 +1,143 @@ +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub +from unit_helpers import run_unit + + +def repo_for(tmp_path, variables): + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "host_vars/laptop.yml").write_text(variables) + (repo / "units").symlink_to(REPO / "units") + (repo / "inventory.yml").write_text("inventory") + return repo + + +def snapshot(home): + return {str(p.relative_to(home)): (p.stat().st_mode, p.read_bytes() if p.is_file() else None) + for p in home.rglob("*")} + + +def stub_yq(bindir): + write_stub(bindir, "yq", ''' +case "$*" in + *'.hive_ops_timers[]'*) echo hive-watchdog ;; + *) echo false ;; +esac +''') + + +def test_hive_check_reports_file_and_service_drift_without_changes(stub_env, tmp_path): + bindir, env = stub_env + stub_yq(bindir) + log = tmp_path / "systemctl-calls" + env["SYSTEMCTL_LOG"] = str(log) + write_stub(bindir, "systemctl", ''' +echo "$*" >> "$SYSTEMCTL_LOG" +case "$*" in + *--property=LoadState*) echo loaded ;; + *--property=UnitFileState*) echo disabled ;; + *--property=ActiveState*) echo inactive ;; + *) exit 99 ;; +esac +''') + repo = repo_for(tmp_path, "hive_ops_enabled: true\n") + home = Path(env["HOME"]) + dest = home / ".local/bin/hive-peak.sh" + dest.parent.mkdir(parents=True) + dest.write_text("old helper") + before = snapshot(home) + result = run_unit("hive-ops", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", UNITS_CHECK="1") + assert result.returncode == 0, result.stdout + result.stderr + assert "CHANGE: update" in result.stdout and "hive-peak.sh" in result.stdout + assert "CHANGE: enable hive-watchdog.timer" in result.stdout + assert "CHANGE: start hive-watchdog.timer" in result.stdout + assert snapshot(home) == before + assert all(" show " in f" {line} " for line in log.read_text().splitlines()) + + +def test_hive_check_reports_opt_out_and_keeps_files(stub_env, tmp_path): + bindir, env = stub_env + stub_yq(bindir) + write_stub(bindir, "systemctl", 'exit 99\n') + repo = repo_for(tmp_path, "hive_ops_enabled: false\n") + home = Path(env["HOME"]) + timer = home / ".config/systemd/user/hive-watchdog.timer" + timer.parent.mkdir(parents=True) + timer.write_text("old") + before = snapshot(home) + result = run_unit("hive-ops", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", UNITS_CHECK="1") + assert result.returncode == 0 + assert "CHANGE: disable and stop hive-watchdog.timer" in result.stdout + assert "CHANGE: remove" in result.stdout + assert snapshot(home) == before + + +def test_hive_check_service_inspection_failure_is_error(stub_env, tmp_path): + bindir, env = stub_env + stub_yq(bindir) + write_stub(bindir, "systemctl", 'exit 1\n') + repo = repo_for(tmp_path, "hive_ops_enabled: true\n") + home = Path(env["HOME"]) + before = snapshot(home) + result = run_unit("hive-ops", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", UNITS_CHECK="1") + assert result.returncode != 0 and "FAIL check hive-watchdog.timer" in result.stdout + assert snapshot(home) == before + + +@pytest.mark.skipif(Path("/home/linuxbrew/.linuxbrew/bin/syncthing").exists(), reason="requires missing system syncthing") +def test_syncthing_check_plans_disable_without_mutation(stub_env, tmp_path): + bindir, env = stub_env + stub_yq(bindir) + write_stub(bindir, "systemctl", 'exit 99\n') + repo = repo_for(tmp_path, "") + home = Path(env["HOME"]) + unit = home / ".config/systemd/user/syncthing.service" + unit.parent.mkdir(parents=True) + unit.write_text("old") + before = snapshot(home) + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", UNITS_CHECK="1") + assert result.returncode == 0 + assert "CHANGE: disable and stop syncthing.service" in result.stdout + assert snapshot(home) == before + + +@pytest.mark.skipif(Path("/home/linuxbrew/.linuxbrew/bin/syncthing").exists(), reason="requires missing system syncthing") +def test_syncthing_check_reports_systemctl_error(stub_env, tmp_path): + bindir, env = stub_env + stub_yq(bindir) + write_stub(bindir, "systemctl", 'exit 1\n') + repo = repo_for(tmp_path, "") + home = Path(env["HOME"]) + before = snapshot(home) + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", UNITS_CHECK="1") + assert result.returncode != 0 and "FAIL check syncthing service" in result.stdout + assert snapshot(home) == before + + +@pytest.mark.skipif(not Path("/home/linuxbrew/.linuxbrew/bin/syncthing").exists(), reason="requires installed system syncthing") +def test_syncthing_check_reports_file_and_service_drift(stub_env, tmp_path): + bindir, env = stub_env + stub_yq(bindir) + write_stub(bindir, "systemctl", ''' +case "$*" in + *--property=LoadState*) echo loaded ;; + *--property=UnitFileState*) echo disabled ;; + *--property=ActiveState*) echo inactive ;; + *) exit 99 ;; +esac +''') + repo = repo_for(tmp_path, "") + home = Path(env["HOME"]) + unit = home / ".config/systemd/user/syncthing.service" + unit.parent.mkdir(parents=True) + unit.write_text("old") + before = snapshot(home) + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", UNITS_CHECK="1") + assert result.returncode == 0, result.stdout + result.stderr + assert "CHANGE: update syncthing.service" in result.stdout + assert "CHANGE: enable syncthing.service" in result.stdout + assert "CHANGE: start syncthing.service" in result.stdout + assert snapshot(home) == before diff --git a/tests/test_install_extensions.py b/tests/test_install_extensions.py index 5f5c74aa..b188c8c7 100644 --- a/tests/test_install_extensions.py +++ b/tests/test_install_extensions.py @@ -13,8 +13,8 @@ SCRIPT = ( Path(__file__).parents[1] - / "roles" - / "bluefin_common" + / "units" + / "bluefin" / "files" / "install-extensions.py" ) diff --git a/tests/test_kube_check_credentials.py b/tests/test_kube_check_credentials.py new file mode 100644 index 00000000..324c1818 --- /dev/null +++ b/tests/test_kube_check_credentials.py @@ -0,0 +1,26 @@ +import stat +from pathlib import Path + +from test_kube_unit import env +from unit_helpers import calls, run_unit + + +def test_check_reports_local_drift_without_accessing_vault(env): + home = Path(env["HOME"]) + config = home / ".kube/config" + config.parent.mkdir() + config.write_text("test-private-config\n") + config.chmod(0o644) + result = run_unit("kube", env, UNITS_CHECK="1") + assert result.returncode != 0 + assert "mode differs from 0600" in result.stdout and "audit events" in result.stdout + assert "test-private-config" not in result.stdout + result.stderr + assert config.read_text() == "test-private-config\n" and stat.S_IMODE(config.stat().st_mode) == 0o644 + assert not (home / ".talos").exists() + assert calls(env) == "" + + +def test_check_without_secrets_has_no_vault_or_filesystem_effects(env): + result = run_unit("kube", env, UNITS_CHECK="1", UNITS_NO_SECRETS="1") + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert calls(env) == "" and not Path(env["HOME"], ".kube").exists() diff --git a/tests/test_laptop_services_migration.py b/tests/test_laptop_services_migration.py new file mode 100644 index 00000000..4445ce71 --- /dev/null +++ b/tests/test_laptop_services_migration.py @@ -0,0 +1,135 @@ +import shlex +import subprocess + +from pathlib import Path + +from conftest import REPO, write_stub +from unit_helpers import calls, run_unit + + +def laptop_repo(tmp_path, host, variables): + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "host_vars" / f"{host}.yml").write_text(variables) + (repo / "units").symlink_to(REPO / "units") + return repo + + +def test_laptop_login_service_and_idempotent_reload(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = laptop_repo(tmp_path, "kanpur", "is_laptop: true\ndotfiles_update_delay_seconds: 90\n") + args = dict(DOTFILES_DIR=str(repo), DOTFILES_MACHINE="kanpur") + first = run_unit("laptop-services", env, **args) + assert first.returncode == 0, first.stdout + first.stderr + home = Path(env["HOME"]) + update = (home / ".config/systemd/user/dotfiles-update.service").read_text() + assert "ExecStartPre=/bin/sleep 90" in update + assert "WantedBy=graphical-session.target" in update + assert "apply-nosecrets" in update + assert "enable dotfiles-update.service" in calls(env) + assert "enable --now dotfiles-update.service" not in calls(env) + assert "enable --now atuin-daemon.service" in calls(env) + (home / "calls").unlink() + second = run_unit("laptop-services", env, **args) + assert second.returncode == 0, second.stdout + second.stderr + assert "daemon-reload" not in calls(env) + + +def test_himachal_optouts_disable_only_owned_timers(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = laptop_repo(tmp_path, "himachal", "is_laptop: true\ndotfiles_auto_update: false\nphone_apply_enabled: false\n") + units = Path(env["HOME"]) / ".config/systemd/user" + units.mkdir(parents=True) + for name in ("dotfiles-update.service", "dotfiles-update.timer", "phone-apply.service", "phone-apply.timer", "unrelated.service"): + (units / name).write_text("old") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="himachal") + assert result.returncode == 0, result.stdout + result.stderr + assert (units / "unrelated.service").read_text() == "old" + assert not (units / "phone-apply.timer").exists() + assert not (units / "phone-apply.service").exists() + assert not (units / "dotfiles-update.service").exists() + assert not (units / "dotfiles-update.timer").exists() + assert "disable --now phone-apply.timer" in calls(env) + assert "enable --now system-mitigator.service" in calls(env) + assert "unrelated.service" not in calls(env) + + +def test_failed_disable_preserves_timer_and_reports_failure(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n[[ "$*" != *"disable --now phone-apply.timer"* ]]\n') + repo = laptop_repo(tmp_path, "himachal", "is_laptop: true\nphone_apply_enabled: false\n") + units = Path(env["HOME"]) / ".config/systemd/user" + units.mkdir(parents=True) + (units / "phone-apply.timer").write_text("old") + (units / "phone-apply.service").write_text("old") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="himachal") + assert result.returncode != 0 + assert (units / "phone-apply.timer").exists() + assert (units / "phone-apply.service").exists() + assert "enable --now system-mitigator.service" in calls(env) + + +def test_himachal_phone_schedule_is_preserved_when_enabled(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = laptop_repo(tmp_path, "himachal", "is_laptop: true\nphone_apply_enabled: true\n") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="himachal") + assert result.returncode == 0, result.stdout + result.stderr + units = Path(env["HOME"]) / ".config/systemd/user" + assert "OnCalendar=*-*-* 00/2:00:00" in (units / "phone-apply.timer").read_text() + service = (units / "phone-apply.service").read_text() + assert "ansible-playbook" not in service + assert "enable --now phone-apply.timer" in calls(env) + + +def test_phone_push_uses_remote_machine_identity_and_cleans_up(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", "exit 0\n") + git = write_stub(bindir, "phone-git", "printf archive\n") + ssh = write_stub( + bindir, "phone-ssh", + 'host="$3"; shift 3\n' + '[ "${OFFLINE_HOST:-}" != "$host" ] || exit 255\n' + 'export HOME="$REMOTE_HOMES/$host"\n' + 'export DOTFILES_MACHINE=localhost\nbash -c "$1"\n', + ) + write_stub(bindir, "tar", 'mkdir -p "$4/scripts"\ncp "$STUB_RUNNER" "$4/scripts/run-units.sh"\n') + runner = tmp_path / "runner.sh" + runner.write_text('printf "%s %s\\n" "$DOTFILES_MACHINE" "$*" >> "$RECORD"\n') + remote_homes = tmp_path / "phones" + for host in ("termux", "mumbai"): + (remote_homes / host).mkdir(parents=True) + env.update(REMOTE_HOMES=str(remote_homes), STUB_RUNNER=str(runner), RECORD=str(tmp_path / "record")) + repo = laptop_repo(tmp_path, "himachal", "is_laptop: true\nphone_apply_enabled: true\n") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="himachal") + assert result.returncode == 0, result.stdout + result.stderr + service = (Path(env["HOME"]) / ".config/systemd/user/phone-apply.service").read_text() + starts = [line for line in service.splitlines() if line.startswith("ExecStart=")] + assert len(starts) == 2 and all(line.startswith("ExecStart=-:/bin/bash") for line in starts) + commands = [line.removeprefix("ExecStart=-:") for line in starts] + for command in commands: + command = command.replace("/usr/bin/git", str(git)).replace("/usr/bin/ssh", str(ssh)) + completed = subprocess.run(shlex.split(command), env=env, capture_output=True, text=True) + assert completed.returncode == 0, completed.stderr + assert (tmp_path / "record").read_text().splitlines() == ["termux --no-secrets", "mumbai --no-secrets"] + for host in ("termux", "mumbai"): + assert list((remote_homes / host / ".cache").iterdir()) == [] + (tmp_path / "record").unlink() + statuses = [] + for command in commands: + command = command.replace("/usr/bin/git", str(git)).replace("/usr/bin/ssh", str(ssh)) + result = subprocess.run(shlex.split(command), env={**env, "OFFLINE_HOST": "termux"}, capture_output=True, text=True) + statuses.append(result.returncode) + assert statuses == [255, 0] + assert (tmp_path / "record").read_text().splitlines() == ["mumbai --no-secrets"] + + +def test_retired_vps_is_never_touched(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = laptop_repo(tmp_path, "telengana", "is_laptop: true\n") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="telengana") + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert calls(env) == "" diff --git a/tests/test_laptop_system_services_check.py b/tests/test_laptop_system_services_check.py new file mode 100644 index 00000000..aa665545 --- /dev/null +++ b/tests/test_laptop_system_services_check.py @@ -0,0 +1,315 @@ +import os +import subprocess +from pathlib import Path + +from conftest import write_stub +from test_laptop_services_migration import laptop_repo +from test_system_services_migration import setup_system_services +from unit_helpers import run_unit + + +def systemctl_stub(bindir): + write_stub(bindir, "systemctl", '''echo "$*" >> "$CONTROL_LOG" +[ "$1" != --user ] || shift +case "$1" in + is-enabled) + if [ "${INSPECTION_ERROR:-0}" = 1 ]; then exit 1; fi + echo "${ENABLED_STATE:-enabled}" + [ "${ENABLED_STATE:-enabled}" = enabled ] && exit 0 + exit 1 ;; + is-active) + echo "${ACTIVE_STATE:-active}" + [ "${ACTIVE_STATE:-active}" = active ] && exit 0 + exit 3 ;; + *) [ "${CHECK_GUARD:-0}" != 1 ] || exit 99 ;; +esac +''') + + +def snapshot(root): + result = {} + for path in sorted(root.rglob("*")): + info = path.lstat() + if path.is_symlink(): + value = ("link", os.readlink(path)) + elif path.is_file(): + value = ("file", path.read_bytes()) + else: + value = ("dir",) + result[str(path.relative_to(root))] = (info.st_mode, value) + return result + + +def run_script(program, env): + return subprocess.run(["bash", str(program)], env=env, text=True, capture_output=True) + + +def check_env(tmp_path, env): + scratch = tmp_path / "scratch" + scratch.mkdir() + return {**env, "CONTROL_LOG": str(tmp_path / "systemctl.log"), + "TMPDIR": str(scratch), "UNITS_CHECK": "1", "CHECK_GUARD": "1"} + + +def assert_read_only_calls(env): + log = Path(env["CONTROL_LOG"]) + for call in log.read_text().splitlines() if log.exists() else []: + words = call.split() + if words and words[0] == "--user": + words.pop(0) + assert words and words[0] in {"is-enabled", "is-active"}, call + assert list(Path(env["TMPDIR"]).iterdir()) == [] + + +def test_laptop_clean_check_and_optout_drift_are_nonmutating(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + repo = laptop_repo(tmp_path, "himachal", "is_laptop: true\n") + env = check_env(tmp_path, original_env) + args = {"DOTFILES_DIR": str(repo), "DOTFILES_MACHINE": "himachal"} + applied = run_unit("laptop-services", {**env, "UNITS_CHECK": "0", "CHECK_GUARD": "0"}, **args) + assert applied.returncode == 0, applied.stdout + applied.stderr + Path(env["CONTROL_LOG"]).unlink() + home = Path(env["HOME"]) + before = snapshot(home) + clean = run_unit("laptop-services", env, **args) + assert clean.returncode == 0, clean.stdout + clean.stderr + assert "would " not in clean.stdout + assert snapshot(home) == before + assert_read_only_calls(env) + + (repo / "host_vars/himachal.yml").write_text( + "is_laptop: true\ndotfiles_auto_update: false\nphone_apply_enabled: false\n" + ) + units = home / ".config/systemd/user" + (units / "dotfiles-update.timer").write_text("obsolete") + (units / "atuin-daemon.service").chmod(0o600) + (home / ".local/bin/system-mitigator.py").chmod(0o600) + before = snapshot(home) + Path(env["CONTROL_LOG"]).unlink() + drift = run_unit("laptop-services", {**env, "ENABLED_STATE": "disabled", "ACTIVE_STATE": "inactive"}, **args) + assert drift.returncode == 0, drift.stdout + drift.stderr + for expected in ("would disable and remove", "dotfiles-update.timer", "dotfiles-update.service", + "phone-apply.timer", "would remove", "phone-apply.service", "would set mode 644", + "would set mode 755", "would enable atuin-daemon.service", "would start atuin-daemon.service"): + assert expected in drift.stdout + assert snapshot(home) == before + assert_read_only_calls(env) + + +def test_laptop_missing_units_and_inspection_failures(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + repo = laptop_repo(tmp_path, "kanpur", "is_laptop: true\n") + env = check_env(tmp_path, original_env) + home = Path(env["HOME"]) + before = snapshot(home) + args = {"DOTFILES_DIR": str(repo), "DOTFILES_MACHINE": "kanpur"} + result = run_unit("laptop-services", {**env, "ENABLED_STATE": "not-found", "ACTIVE_STATE": "inactive"}, **args) + assert result.returncode == 0, result.stdout + result.stderr + assert "would install" in result.stdout and "would start atuin-daemon.service" in result.stdout + assert snapshot(home) == before + assert_read_only_calls(env) + result = run_unit("laptop-services", {**env, "INSPECTION_ERROR": "1"}, **args) + assert result.returncode != 0 + assert "Cannot inspect enabled state" in result.stderr + assert snapshot(home) == before + assert_read_only_calls(env) + + +def test_laptop_comparison_error_is_not_reported_as_drift(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + repo = laptop_repo(tmp_path, "kanpur", "is_laptop: true\n") + env = check_env(tmp_path, original_env) + units = Path(env["HOME"]) / ".config/systemd/user" + units.mkdir(parents=True) + (units / "atuin-daemon.service").write_text("existing") + write_stub(bindir, "cmp", "exit 2\n") + before = snapshot(Path(env["HOME"])) + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="kanpur") + assert result.returncode != 0 and "Cannot compare" in result.stderr + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) + + +def test_system_user_service_check_reports_drift_and_missing_template(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + env = check_env(tmp_path, original_env) + program, env, _ = setup_system_services(tmp_path, env, "server", "", False) + home = Path(env["HOME"]) + before = snapshot(home) + result = run_script(program, {**env, "ENABLED_STATE": "disabled", "ACTIVE_STATE": "inactive"}) + assert result.returncode == 0, result.stdout + result.stderr + for expected in ("would install", "dotfiles-update.service", "dotfiles-update.timer", + "atuin-daemon.service", "would enable dotfiles-update.timer", "would start atuin-daemon.service"): + assert expected in result.stdout + assert snapshot(home) == before + assert_read_only_calls(env) + (Path(env["DOTFILES_DIR"]) / "units/system-services").unlink() + result = run_script(program, env) + assert result.returncode != 0 + assert "FAIL" in result.stdout + assert snapshot(home) == before + assert_read_only_calls(env) + + +def test_system_clean_check_and_manager_error(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + env = check_env(tmp_path, original_env) + program, env, _ = setup_system_services(tmp_path, env, "server", "", False) + applied = run_script(program, {**env, "UNITS_CHECK": "0", "CHECK_GUARD": "0"}) + assert applied.returncode == 0, applied.stdout + applied.stderr + Path(env["CONTROL_LOG"]).unlink() + before = snapshot(Path(env["HOME"])) + clean = run_script(program, env) + assert clean.returncode == 0, clean.stdout + clean.stderr + assert "would " not in clean.stdout + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) + failed = run_script(program, {**env, "INSPECTION_ERROR": "1"}) + assert failed.returncode != 0 and "Cannot inspect enabled state" in failed.stderr + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) + + +def test_podman_check_plans_group_dropin_and_restart_without_mutation(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + write_stub(bindir, "id", '''case "$1" in +-u) echo 1000 ;; +-un) echo tester ;; +-nG) echo users ;; +esac +''') + write_stub(bindir, "getent", 'exit "${GROUP_RC:-2}"\n') + for name in ("sudo", "groupadd", "usermod", "install", "chmod"): + write_stub(bindir, name, 'echo "$0 $*" >> "$MUTATION_LOG"\nexit 99\n') + env = check_env(tmp_path, original_env) + env["MUTATION_LOG"] = str(tmp_path / "mutations") + program, env, dropin = setup_system_services(tmp_path, env, "laptop", "is_laptop: true\n", True) + home_before = snapshot(Path(env["HOME"])) + result = run_script(program, env) + assert result.returncode == 0, result.stdout + result.stderr + for expected in ("would create podman group", "would add tester to podman group", "would create", + "group-access.conf", "would reload system systemd", "would restart podman.socket"): + assert expected in result.stdout + assert not dropin.exists() and not Path(env["MUTATION_LOG"]).exists() + assert snapshot(Path(env["HOME"])) == home_before + assert_read_only_calls(env) + error = run_script(program, {**env, "GROUP_RC": "3"}) + assert error.returncode != 0 and "FAIL configure system podman socket" in error.stdout + assert not dropin.exists() and not Path(env["MUTATION_LOG"]).exists() + assert_read_only_calls(env) + + +def test_podman_clean_check_and_socket_inspection_error(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + write_stub(bindir, "id", '''case "$1" in +-u) echo 1000 ;; +-un) echo tester ;; +-nG) echo "users podman" ;; +esac +''') + write_stub(bindir, "getent", "exit 0\n") + env = check_env(tmp_path, original_env) + program, env, dropin = setup_system_services(tmp_path, env, "laptop", "is_laptop: true\n", True) + dropin.mkdir(parents=True) + dropin.chmod(0o755) + conf = dropin / "group-access.conf" + conf.write_text("[Socket]\nSocketGroup=podman\nSocketMode=0660\nDirectoryMode=0710\n") + conf.chmod(0o644) + before = snapshot(tmp_path / "systemd") + result = run_script(program, env) + assert result.returncode == 0, result.stdout + result.stderr + assert "would " not in result.stdout + assert snapshot(tmp_path / "systemd") == before + assert_read_only_calls(env) + result = run_script(program, {**env, "INSPECTION_ERROR": "1"}) + assert result.returncode != 0 and "Cannot inspect enabled state of podman.socket" in result.stderr + assert snapshot(tmp_path / "systemd") == before + assert_read_only_calls(env) + + +def test_laptop_render_error_fails_without_mutation(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + write_stub(bindir, "cat", '[ "$#" -gt 0 ] || exit 2\nexec /usr/bin/cat "$@"\n') + repo = laptop_repo(tmp_path, "kanpur", "is_laptop: true\n") + env = check_env(tmp_path, original_env) + before = snapshot(Path(env["HOME"])) + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="kanpur") + assert result.returncode != 0 and "FAIL render dotfiles update service" in result.stdout + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) + + +def test_system_comparison_error_is_not_treated_as_file_drift(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + env = check_env(tmp_path, original_env) + program, env, _ = setup_system_services(tmp_path, env, "server", "", False) + units = Path(env["HOME"]) / ".config/systemd/user" + units.mkdir(parents=True) + (units / "dotfiles-update.timer").write_text("existing") + write_stub(bindir, "cmp", "exit 2\n") + before = snapshot(Path(env["HOME"])) + result = run_script(program, env) + assert result.returncode != 0 and "Cannot compare" in result.stderr + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) + + +def test_retired_hosts_skip_check_without_service_access(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + env = check_env(tmp_path, original_env) + program, env, dropin = setup_system_services(tmp_path, env, "matrix", "is_laptop: true\n", True) + before = snapshot(Path(env["HOME"])) + laptop = run_unit("laptop-services", env, DOTFILES_DIR=env["DOTFILES_DIR"], DOTFILES_MACHINE="matrix") + system = run_script(program, env) + assert laptop.returncode == system.returncode == 0 + assert "SKIPPED" in laptop.stdout and "SKIPPED" in system.stdout + assert not Path(env["CONTROL_LOG"]).exists() and not dropin.exists() + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) + + +def test_unreadable_host_variables_fail_both_checks(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + env = check_env(tmp_path, original_env) + program, env, dropin = setup_system_services(tmp_path, env, "laptop", "is_laptop: true\n", True) + hostvars = Path(env["DOTFILES_DIR"]) / "host_vars/laptop.yml" + hostvars.unlink() + hostvars.symlink_to(tmp_path / "missing-host-vars") + before = snapshot(Path(env["HOME"])) + laptop = run_unit("laptop-services", env, DOTFILES_DIR=env["DOTFILES_DIR"], DOTFILES_MACHINE="laptop") + system = run_script(program, env) + assert laptop.returncode != 0 and system.returncode != 0 + assert "FAIL inspect host variables" in laptop.stdout + assert "FAIL inspect host variables" in system.stdout + assert not Path(env["CONTROL_LOG"]).exists() and not dropin.exists() + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) + + +def test_service_drift_does_not_disclose_file_contents(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + env = check_env(tmp_path, original_env) + _, env, _ = setup_system_services(tmp_path, env, "laptop", "is_laptop: true\n", False) + units = Path(env["HOME"]) / ".config/systemd/user" + units.mkdir(parents=True) + secret = "sensitive-unit-content-example" + (units / "atuin-daemon.service").write_text(secret) + before = snapshot(Path(env["HOME"])) + result = run_unit("laptop-services", env, DOTFILES_DIR=env["DOTFILES_DIR"], DOTFILES_MACHINE="laptop") + assert result.returncode == 0, result.stdout + result.stderr + assert "would update" in result.stdout + assert secret not in result.stdout + result.stderr + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) diff --git a/tests/test_osquery_check.py b/tests/test_osquery_check.py new file mode 100644 index 00000000..410d7c08 --- /dev/null +++ b/tests/test_osquery_check.py @@ -0,0 +1,55 @@ +import importlib.util +import os + +import pytest + +from conftest import REPO + +SPEC = importlib.util.spec_from_file_location('osquery_check', REPO / 'units/osquery-check.py') +check = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(check) + + +def test_file_and_symlink_checks_only_report_changes(tmp_path, capsys): + path = tmp_path / 'installed' + path.write_bytes(b'old') + path.chmod(0o600) + link = tmp_path / 'link' + link.symlink_to('old-target') + assert check.file_from_bytes(path, b'new', 0o600) + assert check.symlink(str(link), 'new-target') + assert path.read_bytes() == b'old' + assert os.readlink(link) == 'old-target' + output = capsys.readouterr().out + assert 'update ' + str(path) in output + assert 'set symlink ' + str(link) in output + + +def test_systemd_inspection_reports_drift(monkeypatch, capsys): + monkeypatch.setattr(check, 'exists', lambda path: True) + calls = [] + def systemctl(*args): + calls.append(args) + return 'LoadState=loaded\nUnitFileState=disabled\nActiveState=inactive\n' + monkeypatch.setattr(check, 'systemctl', systemctl) + check.service_state('osqueryd.service') + assert calls == [('show', '--property=LoadState,UnitFileState,ActiveState', 'osqueryd.service')] + output = capsys.readouterr().out + assert 'enable osqueryd.service' in output + assert 'start osqueryd.service' in output + + +def test_systemd_incomplete_inspection_fails(monkeypatch): + monkeypatch.setattr(check, 'exists', lambda path: True) + monkeypatch.setattr(check, 'systemctl', lambda *args: 'LoadState=loaded\n') + with pytest.raises(RuntimeError, match='incomplete systemd'): + check.service_state('osqueryd.service') + + +def test_disabled_poller_service_is_not_disabled(monkeypatch, capsys): + monkeypatch.setattr(check, 'exists', lambda path: True) + monkeypatch.setattr(check, 'systemctl', lambda *args: 'LoadState=loaded\nUnitFileState=enabled\nActiveState=active\n') + check.service_state('fleet-osquery-poll.service', enabled=False, disable=False) + output = capsys.readouterr().out + assert 'stop fleet-osquery-poll.service' in output + assert 'disable fleet-osquery-poll.service' not in output diff --git a/tests/test_osquery_drift.py b/tests/test_osquery_drift.py index 51e5149a..b8aeb76e 100644 --- a/tests/test_osquery_drift.py +++ b/tests/test_osquery_drift.py @@ -15,7 +15,7 @@ from conftest import REPO -SCRIPT = REPO / "roles" / "osquery" / "files" / "fleet-osquery.py" +SCRIPT = REPO / "units" / "osquery" / "files" / "fleet-osquery.py" @pytest.fixture diff --git a/tests/test_osquery_native_inventory.py b/tests/test_osquery_native_inventory.py new file mode 100644 index 00000000..751dcb13 --- /dev/null +++ b/tests/test_osquery_native_inventory.py @@ -0,0 +1,95 @@ +import importlib.util +import json +import sys + +import pytest + +from conftest import REPO + +sys.path.insert(0, str(REPO / "units")) +sys.path.insert(0, str(REPO / "scripts")) +import osquery_native_inventory as native + +spec = importlib.util.spec_from_file_location("osquery_fleet_native_test", REPO / "scripts/osquery-fleet.py") +fleet = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fleet) + + +def put(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + + +def test_effective_variables_nested_groups_and_alias_targets(tmp_path): + path = tmp_path / "inventory.yml" + put(path, {"all": {"vars": {"ansible_user": "base"}, "hosts": { + "controller": {"ansible_connection": "local"}, "phone": {"ansible_host": "100.86.202.86"}}, + "children": {"vps": {"children": {"retired": {"hosts": {"old": {}}}}}, + "mobile": {"vars": {"ansible_port": 2222, "ansible_connection": "local"}, "children": { + "phones": {"hosts": {"phone": {"tailnet_name": "mumbai"}}}}}}}}) + put(tmp_path / "group_vars/all.yml", {"ansible_user": "all-user", "ansible_connection": "ssh", "secret": "do-not-copy"}) + put(tmp_path / "group_vars/mobile.yml", {"ansible_user": "mobile-user", "ansible_port": 2022}) + put(tmp_path / "group_vars/phones.yml", {"ansible_user": "phone-user"}) + put(tmp_path / "host_vars/phone.yml", {"ansible_user": "host-user", "ansible_port": 3333}) + got = native.load_inventory(path) + assert fleet.group_hosts(got, "vps") == {"old"} + variables = got["_meta"]["hostvars"]["phone"] + assert variables == {"ansible_host": "100.86.202.86", "tailnet_name": "mumbai", + "ansible_user": "host-user", "ansible_port": 3333, "ansible_connection": "ssh"} + command = fleet.report_command("phone", variables, {"controller"}) + assert command[command.index("-l") + 1] == "host-user" + assert command[command.index("-p") + 1] == "3333" + assert command[-2] == "100.86.202.86" + assert "do-not-copy" not in repr(got) + + +def test_same_depth_group_priority_and_inventory_host_override(tmp_path): + path = tmp_path / "inventory.yml" + put(path, {"all": {"hosts": {"host": {"ansible_user": "inventory-host"}}, "children": { + "alpha": {"vars": {"ansible_group_priority": 20, "ansible_port": 2020}, "hosts": {"host": {}}}, + "zulu": {"vars": {"ansible_group_priority": 1, "ansible_port": 3030}, "hosts": {"host": {}}}}}}) + values = native.load_inventory(path)["_meta"]["hostvars"]["host"] + assert values["ansible_user"] == "inventory-host" + assert values["ansible_port"] == 2020 + + +def test_cycle_and_bad_inventory_fail_without_leaking_values(tmp_path, capsys): + path = tmp_path / "inventory.yml" + put(path, {"all": {"children": {"parent": {"children": {"all": None}}}}}) + with pytest.raises(ValueError, match="cycle"): + native.load_inventory(path) + put(path, {"all": {"hosts": {"host": {"ansible_host": "{{ super_secret }}"}}}}) + assert fleet.main(["--inventory", str(path)]) == 1 + captured = capsys.readouterr() + assert "super_secret" not in captured.out + captured.err + assert "unresolved templates" in captured.err + + +def test_sanitized_snapshot_preserves_exclusions_without_yaml_dependency(tmp_path): + path = tmp_path / "snapshot.yml" + put(path, {"all": {"hosts": {"kerala": {"osquery_supported": False}}, + "children": {"vps": {"hosts": {"matrix": {}}}}}}) + got = native.load_inventory(path) + assert fleet.group_hosts(got, "vps") == {"matrix"} + assert got["_meta"]["hostvars"]["kerala"]["osquery_supported"] is False + + +def test_host_gate_checks_nested_vps_and_rejects_unknown_machine(tmp_path, capsys): + path = tmp_path / "inventory.yml" + put(path, {"all": {"children": {"vps": {"children": { + "retired": {"hosts": {"old": {}}}}}, "server": {"hosts": {"current": {}}}}}}) + assert native.main(["groups", str(path), "old", "vps", "server"]) == 0 + assert capsys.readouterr().out.strip() == "true false" + assert native.main(["groups", str(path), "mystery", "vps"]) == 1 + assert "absent from inventory" in capsys.readouterr().err + + +def test_osquery_gates_inherit_group_settings_and_host_overrides(tmp_path, capsys): + path = tmp_path / "inventory.yml" + put(path, {"all": {"children": {"server": {"hosts": {"node": {}}, + "vars": {"skip_osquery": True}}}}}) + put(tmp_path / "group_vars/server.yml", {"osquery_supported": False, "osquery_poller_enabled": True}) + put(tmp_path / "host_vars/node.yml", {"skip_osquery": False, "osquery_version": "5.23.1"}) + assert native.main(["osquery-settings", str(path), "node"]) == 0 + assert json.loads(capsys.readouterr().out) == {"skip_osquery": False, "osquery_supported": False, + "osquery_poller_enabled": True, "osquery_version": "5.23.1"} diff --git a/tests/test_package_native_units.py b/tests/test_package_native_units.py new file mode 100644 index 00000000..1f307c70 --- /dev/null +++ b/tests/test_package_native_units.py @@ -0,0 +1,345 @@ +"""Behavioral coverage for native package units; all package commands are isolated.""" +import json +import os +import shutil +import stat +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from conftest import REPO, write_stub + + +@pytest.fixture +def package_env(tmp_path, stub_env): + yq = shutil.which("yq") + if not yq or "github.com/mikefarah/yq" not in subprocess.run( + [yq, "--version"], text=True, capture_output=True + ).stdout: + pytest.skip("Mike Farah yq is required") + bindir, env = stub_env + root = tmp_path / "repo" + root.mkdir() + (root / "units").mkdir() + (root / "units/package-config.bash").symlink_to(REPO / "units/package-config.bash") + (root / "group_vars").mkdir() + (root / "host_vars").mkdir() + (root / "group_vars/all.yml").write_text(yaml.safe_dump({ + "core_taps": ["example/core"], + "core_brews": ["git", "coreowner/tap/formula"], + "core_tap_brews": [], + "desktop_taps": ["example/desktop"], + "desktop_brews": ["desktop-pkg"], + "desktop_casks": ["desktop-cask"], + "desktop_vscode_extensions": ["author.extension"], + "brew_blocked": ["trash-cli"], + "flatpak_remotes": [{"name": "wrong", "url": "https://example.test/wrong"}], + })) + (root / "host_vars/laptop.yml").write_text(yaml.safe_dump({"extra_brews": ["me/tap/extra"]})) + (root / "inventory.yml").write_text(yaml.safe_dump({ + "all": {"children": {"desktop": {"hosts": {"laptop": None}}}} + })) + (root / "units/flatpak-packages.yml").write_text(yaml.safe_dump({ + "flatpak_remotes": [{"name": "flathub", "url": "https://example.test/flathub.flatpakrepo"}], + "system_flatpaks": ["org.example.Installed", "org.example.First", "org.example.Second"], + })) + prefix = tmp_path / "brew" / ".linuxbrew" + (prefix / "bin").mkdir(parents=True) + log = tmp_path / "commands.jsonl" + env.update(DOTFILES_DIR=str(root), DOTFILES_MACHINE="laptop", BREW_PREFIX=str(prefix), + COMMAND_LOG=str(log), INSTALLED_APPS=str(tmp_path / "installed"), + USER_APPS=str(tmp_path / "user-apps.json")) + Path(env["INSTALLED_APPS"]).write_text("org.mozilla.firefox\norg.example.Installed\n") + Path(env["USER_APPS"]).write_text("{}") + write_stub(bindir, "cc", "exit 0\n") + write_stub(bindir, "gcc", "exit 0\n") + # Never invoke host sudo, even if the CI process runs as root. + write_stub(bindir, "id", 'if [ "$1" = -u ]; then echo 1000; else echo tester; fi\n') + write_stub(bindir, "sudo", 'shift; if [ "$1" = flatpak ]; then PACKAGE_SUDO=1 exec "$@"; fi; exit 0\n') + program = f'''#!{sys.executable} +import json, os, sys +name = os.path.basename(sys.argv[0]) +args = sys.argv[1:] +with open(os.environ["COMMAND_LOG"], "a") as f: + f.write(json.dumps({{"name": name, "args": args, + "git_count": os.getenv("GIT_CONFIG_COUNT"), + "git_value": os.getenv("GIT_CONFIG_VALUE_0"), + "sudo": os.getenv("PACKAGE_SUDO")}}) + "\\n") +if name == "brew" and args == ["install", "yq"]: + if os.getenv("FAIL_YQ") == "1": + sys.exit(1) + target = os.getenv("YQ_BINARY") + if target: + os.symlink(target, os.path.join(os.path.dirname(sys.argv[0]), "yq")) +if name == "brew" and args and args[0] == "tap" and os.getenv("FAIL_TAP") == "1": + sys.exit(1) +if name == "brew" and args[:2] == ["uninstall", "--formula"]: + print("Error: No such keg", file=sys.stderr) + sys.exit(1) +if name == "flatpak" and args and args[0] == "list": + if "--user" in args: + if os.getenv("FAIL_USER_LIST") == "1": + sys.exit(1) + print("\\n".join(json.load(open(os.environ["USER_APPS"])))) + else: + print(open(os.environ["INSTALLED_APPS"]).read(), end="") +if name == "flatpak" and args and args[0] == "info" and "--user" in args: + if os.getenv("FAIL_USER_INFO") == args[-1]: + sys.exit(1) + print(json.load(open(os.environ["USER_APPS"]))[args[-1]]) +if name == "flatpak" and args and args[0] == "install" and "--user" in args: + state = json.load(open(os.environ["USER_APPS"])) + state[args[-1]] = args[-2] + with open(os.environ["USER_APPS"], "w") as f: + json.dump(state, f) +if name == "flatpak" and args and args[0] == "install" and os.getenv("FAIL_BATCH") == "1": + if "org.example.Second" in args: + sys.exit(1) +''' + for path in (prefix / "bin/brew", prefix / "bin/python3", bindir / "flatpak"): + path.write_text(program) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + return root, env, log + + +def run_unit(name, env): + return subprocess.run(["bash", str(REPO / "units" / f"{name}.sh")], + env=env, text=True, capture_output=True) + + +def calls(log, name): + if not log.exists(): + return [] + return [entry for entry in map(json.loads, log.read_text().splitlines()) if entry["name"] == name] + + +def isolate_bootstrap_path(env, tmp_path): + original_bin = Path(env["PATH"].split(os.pathsep)[0]) + isolated = tmp_path / "bootstrap-bin" + isolated.mkdir() + for binary in ("bash", "dirname", "stat", "mktemp", "rm", "mkdir", "install", "chmod", "cmp", "grep", "uname", "cp", "cat", "sed", "head"): + target = shutil.which(binary) + assert target is not None + (isolated / binary).symlink_to(target) + for binary in ("id", "sudo", "cc", "gcc"): + (isolated / binary).symlink_to(original_bin / binary) + env["YQ_BINARY"] = shutil.which("yq") + env["PATH"] = str(isolated) + return isolated + + +def test_fresh_homebrew_bootstraps_yq_then_loads_config(package_env, tmp_path): + _, env, log = package_env + bindir = isolate_bootstrap_path(env, tmp_path) + brew = Path(env["BREW_PREFIX"]) / "bin/brew" + template = tmp_path / "brew-template" + template.write_text(brew.read_text()) + brew.unlink() + env["BREW_INSTALL_TEMPLATE"] = str(template) + write_stub(bindir, "curl", '''cat > "$4" <<'INSTALLER' +mkdir -p "$BREW_PREFIX/bin" +cp "$BREW_INSTALL_TEMPLATE" "$BREW_PREFIX/bin/brew" +chmod +x "$BREW_PREFIX/bin/brew" +INSTALLER +''') + result = run_unit("homebrew", env) + assert result.returncode == 0, result.stdout + result.stderr + args = [entry["args"] for entry in calls(log, "brew")] + assert args[0] == ["install", "yq"] + assert any(call[:2] == ["bundle", "install"] for call in args) + assert 'brew "desktop-pkg"' in (Path(env["HOME"]) / "Brewfile").read_text() + + +def test_failed_yq_bootstrap_stops_before_bundle(package_env, tmp_path): + _, env, log = package_env + isolate_bootstrap_path(env, tmp_path) + env["FAIL_YQ"] = "1" + result = run_unit("homebrew", env) + assert result.returncode != 0 + assert "Unable to install yq" in result.stderr + assert [entry["args"] for entry in calls(log, "brew")] == [["install", "yq"]] + assert not (Path(env["HOME"]) / "Brewfile").exists() + + +def test_brew_desktop_lists_taps_blocked_and_no_upgrade(package_env): + _, env, log = package_env + result = run_unit("homebrew", env) + assert result.returncode == 0, result.stdout + result.stderr + brewfile = Path(env["HOME"]) / "Brewfile" + content = brewfile.read_text() + assert 'brew "git"' in content + assert 'brew "desktop-pkg"' in content + assert 'brew "me/tap/extra"' in content + assert 'cask "desktop-cask"' in content + assert stat.S_IMODE(brewfile.stat().st_mode) == 0o644 + entries = calls(log, "brew") + args = [entry["args"] for entry in entries] + for tap in ("example/core", "example/desktop", "coreowner/tap", "me/tap"): + assert ["tap", tap] in args + assert ["trust", tap] in args + bundle = next(entry for entry in entries if entry["args"][:2] == ["bundle", "install"]) + assert "--no-upgrade" in bundle["args"] + assert bundle["git_count"] == "1" and bundle["git_value"] == "" + assert ["uninstall", "--formula", "trash-cli"] in args + assert ["upgrade"] not in args + before = brewfile.stat().st_mtime_ns + result = run_unit("homebrew", env) + assert result.returncode == 0, result.stdout + result.stderr + assert brewfile.stat().st_mtime_ns == before + + +def test_brew_host_profile_and_explicit_upgrade(package_env): + root, env, log = package_env + (root / "host_vars/laptop.yml").write_text("is_desktop: false\nextra_brews: [private-extra]\n") + env["DOTFILES_UPGRADE"] = "1" + result = run_unit("homebrew", env) + assert result.returncode == 0, result.stdout + result.stderr + content = (Path(env["HOME"]) / "Brewfile").read_text() + assert "desktop-pkg" not in content and "private-extra" not in content + assert ["upgrade"] in [entry["args"] for entry in calls(log, "brew")] + + +def test_brew_blocked_host_package_fails_before_any_brew_action(package_env): + root, env, log = package_env + (root / "host_vars/laptop.yml").write_text("extra_brews: [owner/tap/trash-cli]\n") + result = run_unit("homebrew", env) + assert result.returncode != 0 + assert "brew_blocked" in result.stderr + assert not calls(log, "brew") + assert not (Path(env["HOME"]) / "Brewfile").exists() + + +def test_brew_continues_after_tap_failure(package_env): + _, env, log = package_env + env["FAIL_TAP"] = "1" + result = run_unit("homebrew", env) + assert result.returncode != 0 + args = [entry["args"] for entry in calls(log, "brew")] + assert any(call[:2] == ["bundle", "install"] for call in args) + assert ["uninstall", "--formula", "trash-cli"] in args + + +def test_flatpak_system_scope_batch_fallback_and_wrappers(package_env): + _, env, log = package_env + env["FAIL_BATCH"] = "1" + result = run_unit("flatpak", env) + assert result.returncode != 0 + args = [entry["args"] for entry in calls(log, "flatpak")] + assert ["remote-add", "--if-not-exists", "--system", "flathub", "https://example.test/flathub.flatpakrepo"] in args + assert ["uninstall", "--system", "-y", "org.mozilla.firefox"] in args + assert ["install", "--system", "-y", "flathub", "org.example.First", "org.example.Second"] in args + assert ["install", "--system", "-y", "flathub", "org.example.First"] in args + assert not any("org.example.Installed" in call for call in args if call and call[0] == "install") + assert ["install", "--system", "-y", "flathub", "org.example.Second"] in args + assert not any("--user" in call or "update" in call for call in args) + for name in ("google-chrome", "google-chrome-stable", "chromium-browser", "chromium"): + wrapper = Path(env["HOME"]) / ".local/bin" / name + assert wrapper.read_text() == '#!/usr/bin/env bash\nexec flatpak run org.chromium.Chromium "$@"\n' + assert stat.S_IMODE(wrapper.stat().st_mode) == 0o755 + + +def test_user_flatpaks_preserve_origins_and_do_not_use_sudo(package_env): + root, env, log = package_env + (root / "host_vars/laptop.yml").write_text(yaml.safe_dump({ + "user_flatpak_remotes": [{"name": "flatpark", "url": "https://example.test/flatpark.flatpakrepo"}], + "user_flatpaks": [ + {"remote": "flatpark", "id": "org.example.Blueshell"}, + {"remote": "flatpark", "id": "org.example.Tavern"}, + {"remote": "flatpark", "id": "org.example.Compass"}, + ], + })) + Path(env["USER_APPS"]).write_text(json.dumps({ + "org.example.Blueshell": "other-origin", "org.example.Tavern": "flatpark", + })) + result = run_unit("flatpak", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "PRESERVED: user Flatpak org.example.Blueshell uses other-origin" in result.stdout + entries = calls(log, "flatpak") + user = [entry for entry in entries if "--user" in entry["args"]] + args = [entry["args"] for entry in user] + assert ["remote-add", "--if-not-exists", "--user", "flatpark", "https://example.test/flatpark.flatpakrepo"] in args + assert ["install", "--user", "-y", "flatpark", "org.example.Compass"] in args + assert not any(call[0] == "install" and call[-1] in ("org.example.Blueshell", "org.example.Tavern") for call in args) + assert all(entry["sudo"] is None for entry in user) + assert all(entry["sudo"] == "1" for entry in entries if "--system" in entry["args"]) + result = run_unit("flatpak", env) + assert result.returncode == 0, result.stdout + result.stderr + installs = [entry["args"] for entry in calls(log, "flatpak") if entry["args"][:2] == ["install", "--user"]] + assert installs == [["install", "--user", "-y", "flatpark", "org.example.Compass"]] + + +def test_user_flatpak_origin_failure_is_not_an_install(package_env): + root, env, log = package_env + (root / "host_vars/laptop.yml").write_text(yaml.safe_dump({ + "user_flatpaks": [ + {"remote": "flatpark", "id": "org.example.Existing"}, + {"remote": "flatpark", "id": "org.example.New"}, + ], + })) + Path(env["USER_APPS"]).write_text(json.dumps({"org.example.Existing": "another"})) + env["FAIL_USER_INFO"] = "org.example.Existing" + result = run_unit("flatpak", env) + assert result.returncode != 0 + assert "Could not determine origin" in result.stderr + args = [entry["args"] for entry in calls(log, "flatpak")] + assert ["install", "--user", "-y", "flatpark", "org.example.New"] in args + assert not any(call[:2] == ["install", "--user"] and call[-1] == "org.example.Existing" for call in args) + + +def test_invalid_user_flatpak_config_does_not_block_system_apps(package_env): + root, env, log = package_env + (root / "host_vars/laptop.yml").write_text("user_flatpaks:\n - remote: flatpark\n") + result = run_unit("flatpak", env) + assert result.returncode != 0 + assert "Invalid user Flatpak app ID" in result.stderr + args = [entry["args"] for entry in calls(log, "flatpak")] + assert ["install", "--system", "-y", "flathub", "org.example.First", "org.example.Second"] in args + assert not any("--user" in call for call in args) + + +def test_flatpak_profile_gate_and_explicit_update(package_env): + root, env, log = package_env + (root / "host_vars/laptop.yml").write_text("machine_profile: vm-test\n") + result = run_unit("flatpak", env) + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert not calls(log, "flatpak") + (root / "host_vars/laptop.yml").write_text("skip_flatpak: true\n") + result = run_unit("flatpak", env) + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert not calls(log, "flatpak") + (root / "host_vars/laptop.yml").write_text("skip_flatpak: false\n") + env["DOTFILES_UPGRADE"] = "true" + result = run_unit("flatpak", env) + assert result.returncode == 0, result.stdout + result.stderr + assert ["update", "--system", "-y"] in [entry["args"] for entry in calls(log, "flatpak")] + + +def test_missing_or_unusable_yq_fails_without_package_operations(package_env): + _, env, log = package_env + bindir = Path(env["PATH"].split(os.pathsep)[0]) + write_stub(bindir, "yq", 'echo "unusable yq" >&2; exit 1\n') + for unit in ("homebrew", "flatpak"): + result = run_unit(unit, env) + assert result.returncode != 0 + assert ["install", "yq"] in [entry["args"] for entry in calls(log, "brew")] + assert not any(entry["args"][:2] == ["bundle", "install"] for entry in calls(log, "brew")) + assert not calls(log, "flatpak") + + +def test_missing_flatpak_fails_but_still_installs_wrappers(package_env, tmp_path): + _, env, log = package_env + isolated = tmp_path / "isolated" + isolated.mkdir() + for binary in ("bash", "yq", "dirname", "mktemp", "mkdir", "install", "chmod", "cmp", "rm", "sed", "head"): + target = shutil.which(binary) + assert target is not None + (isolated / binary).symlink_to(target) + env["PATH"] = str(isolated) + result = run_unit("flatpak", env) + assert result.returncode != 0 + assert "flatpak is required" in result.stderr + assert (Path(env["HOME"]) / ".local/bin/chromium").exists() + assert not calls(log, "flatpak") diff --git a/tests/test_pipewire_audio_check_mode.py b/tests/test_pipewire_audio_check_mode.py new file mode 100644 index 00000000..3764eda1 --- /dev/null +++ b/tests/test_pipewire_audio_check_mode.py @@ -0,0 +1,174 @@ +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub +from test_pipewire_audio_unit import asset, home, setup +from unit_helpers import run_unit + + +def snapshot(root): + return { + str(path.relative_to(root)): ( + path.lstat().st_mode & 0o777, + path.read_bytes() if path.is_file() else None, + ) + for path in root.rglob("*") + } + + +def prepare_check(stub_env, tmp_path): + env = setup(stub_env) + env.update(UNITS_CHECK="1", CHECK_LOG=str(tmp_path / "audio-calls")) + bindir = Path(env["PATH"]) + write_stub(bindir, "systemctl", ''' +echo "systemctl $*" >> "$CHECK_LOG" +[ "${SESSION_FAIL:-0}" = 0 ] || exit 1 +case "$2" in + show-environment) ;; + show) + [ "${SERVICE_FAIL:-0}" = 0 ] || exit 1 + printf 'LoadState=%s\\nActiveState=%s\\nUnitFileState=%s\\n' "${LOAD_STATE:-loaded}" "${ACTIVE_STATE:-active}" "${ENABLED_STATE:-enabled}" + ;; + *) echo "unexpected mutating systemctl" >> "$CHECK_LOG"; exit 1 ;; +esac +''') + write_stub(bindir, "gsettings", ''' +echo "gsettings $*" >> "$CHECK_LOG" +[ "$1" = get ] || exit 1 +[ "${GSETTINGS_FAIL:-0}" = 0 ] || exit 1 +echo "${GNOME_VOLUME:-true}" +''') + write_stub(bindir, "flatpak", ''' +echo "flatpak $*" >> "$CHECK_LOG" +[ "${FLATPAK_FAIL:-0}" = 0 ] || exit 1 +case "$1" in + ps) [ "${FLATPAK_RUNNING:-0}" = 0 ] || echo com.github.wwmm.easyeffects ;; + list) [ "${FLATPAK_INSTALLED:-0}" = 0 ] || echo com.github.wwmm.easyeffects ;; + *) exit 1 ;; +esac +''') + write_stub(bindir, "sudo", 'echo "unexpected sudo $*" >> "$CHECK_LOG"; exit 1\n') + return env + + +def put(env, path, data, mode=0o644): + target = home(env, path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + target.chmod(mode) + return target + + +def managed_files(env): + put(env, ".config/pipewire/pipewire.conf.d/50-speaker-dsp.conf", asset("speaker-dsp.conf")) + put(env, ".config/pipewire/pipewire.conf.d/60-mic-dsp.conf", asset("mic-rnnoise.conf")) + put(env, ".config/systemd/user/pipewire.service.d/ladspa-path.conf", ( + b"[Service]\nEnvironment=LADSPA_PATH=%h/.ladspa:/usr/lib/x86_64-linux-gnu/ladspa:/usr/lib/ladspa:/usr/lib64/ladspa\n" + )) + template = (REPO / "units/pipewire-audio/templates/mic-dsp-link.service.j2").read_text() + put(env, ".config/systemd/user/mic-dsp-link.service", template.replace( + "{{ mic_hardware_volume_pct | default(100) }}", "75" + ).encode()) + for path in (".config/pipewire/filter-chain.conf.d", ".ladspa", ".local/state/pipewire-audio"): + home(env, path).mkdir(parents=True, exist_ok=True) + + +def test_check_clean_audio_has_no_changes(stub_env, tmp_path): + env = prepare_check(stub_env, tmp_path) + managed_files(env) + before = snapshot(Path(env["HOME"])) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD" not in result.stdout + assert snapshot(Path(env["HOME"])) == before + log = Path(env["CHECK_LOG"]).read_text() + assert "systemctl --user show-environment" in log + assert "unexpected" not in log + + +def test_check_audio_reports_drift_and_preserves_owned_and_custom_files(stub_env, tmp_path): + env = prepare_check(stub_env, tmp_path) + managed_files(env) + speaker = home(env, ".config/pipewire/pipewire.conf.d/50-speaker-dsp.conf") + speaker.write_text("user-owned speaker config") + mic = home(env, ".config/pipewire/pipewire.conf.d/60-mic-dsp.conf") + mic.chmod(0o600) + put(env, ".config/wireplumber/main.lua.d/51-speaker-dsp-default.lua", asset("51-speaker-dsp-default.lua")) + put(env, ".config/autostart/com.github.wwmm.easyeffects.desktop", b"obsolete\n") + home(env, ".local/state/pipewire-audio/mic").touch() + env.update(GNOME_VOLUME="false", FLATPAK_RUNNING="1", FLATPAK_INSTALLED="1", ENABLED_STATE="disabled") + before = snapshot(Path(env["HOME"])) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + for expected in ( + "retained custom audio config", "WOULD install audio config", "WOULD remove audio config", + "WOULD reload user service definitions", "WOULD restart PipeWire", "WOULD restart WirePlumber", + "WOULD enable microphone link service", "WOULD restart microphone link service", + "WOULD enable GNOME over-amplification", "WOULD remove Easy Effects autostart", + "WOULD stop Easy Effects Flatpak", "WOULD uninstall Easy Effects Flatpak", + ): + assert expected in result.stdout + assert snapshot(Path(env["HOME"])) == before + assert "unexpected" not in Path(env["CHECK_LOG"]).read_text() + + +def test_check_absent_plugin_plans_removal_and_disable(stub_env, tmp_path): + env = prepare_check(stub_env, tmp_path) + managed_files(env) + env.update(RNNOISE="0", LV2="0") + before = snapshot(Path(env["HOME"])) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD remove audio config" in result.stdout + assert "WOULD stop microphone link service" in result.stdout + assert "WOULD disable microphone link service" in result.stdout + assert snapshot(Path(env["HOME"])) == before + + +@pytest.mark.parametrize("error", ["FIND_FAIL", "LV2_FAIL", "SESSION_FAIL", "SERVICE_FAIL", "GSETTINGS_FAIL", "FLATPAK_FAIL"]) +def test_check_inspection_errors_fail_without_mutation(stub_env, tmp_path, error): + env = prepare_check(stub_env, tmp_path) + managed_files(env) + env[error] = "1" + before = snapshot(Path(env["HOME"])) + result = run_unit("pipewire-audio", env) + assert result.returncode != 0 and "FAIL" in result.stdout + assert snapshot(Path(env["HOME"])) == before + if Path(env["CHECK_LOG"]).exists(): + assert "unexpected" not in Path(env["CHECK_LOG"]).read_text() + + +def test_check_inspects_missing_packages_and_rpm_errors(stub_env, tmp_path): + env = prepare_check(stub_env, tmp_path) + bindir = Path(env["PATH"]) + write_stub(bindir, "dnf", 'echo "unexpected dnf $*" >> "$CHECK_LOG"; exit 1\n') + write_stub(bindir, "rpm", ''' +[ "${RPM_ERROR:-0}" = 0 ] || { echo "database failure" >&2; exit 1; } +echo "package $2 is not installed" +exit 1 +''') + before = snapshot(Path(env["HOME"])) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD install DSP package lsp-plugins-lv2" in result.stdout + assert "WOULD install DSP package noise-suppression-for-voice" in result.stdout + assert snapshot(Path(env["HOME"])) == before + env["RPM_ERROR"] = "1" + result = run_unit("pipewire-audio", env) + assert result.returncode != 0 and "FAIL inspect DSP package" in result.stdout + assert snapshot(Path(env["HOME"])) == before + assert "unexpected" not in Path(env["CHECK_LOG"]).read_text() + + +def test_check_comparison_failure_is_not_drift(stub_env, tmp_path): + env = prepare_check(stub_env, tmp_path) + managed_files(env) + bindir = Path(env["PATH"]) + (bindir / "cmp").unlink() + write_stub(bindir, "cmp", "exit 2\n") + before = snapshot(Path(env["HOME"])) + result = run_unit("pipewire-audio", env) + assert result.returncode != 0 + assert "FAIL compare audio config" in result.stderr + assert snapshot(Path(env["HOME"])) == before diff --git a/tests/test_pipewire_audio_unit.py b/tests/test_pipewire_audio_unit.py new file mode 100644 index 00000000..b68f6fea --- /dev/null +++ b/tests/test_pipewire_audio_unit.py @@ -0,0 +1,331 @@ +from pathlib import Path +import shutil + +from conftest import REPO, write_stub +from unit_helpers import calls, run_unit + + +def setup(stub_env, machine="himachal"): + bindir, env = stub_env + # Do not let the host's package managers or Flatpak influence the test. + for name in ("bash", "cat", "chmod", "cmp", "dirname", "head", "install", + "mkdir", "mktemp", "rm", "rmdir", "sed", "stat", "touch"): + (bindir / name).symlink_to(shutil.which(name)) + for key in ("MIC_HARDWARE_VOLUME_PCT", "FIND_FAIL", "RNNOISE", "LV2", "LV2_FAIL", "GNOME_VOLUME", + "SESSION_FAIL", "RESTART_FAIL", "MIC_LOAD", "FLATPAK_RUNNING", "FLATPAK_INSTALLED", + "RPM_INSTALLED", "YQ_RESULT", "YQ_RC", "XDG_RUNTIME_DIR", "DBUS_SESSION_BUS_ADDRESS"): + env.pop(key, None) + env.update(PATH=str(bindir), DOTFILES_MACHINE=machine, DOTFILES_IS_DESKTOP="1") + write_stub(bindir, "id", 'echo 1000\n') + # Arch hosts must not run a real pacman transaction during these tests. + write_stub(bindir, "pacman", 'exit 0\n') + write_stub(bindir, "find", '''\ +[ "${FIND_FAIL:-0}" = 0 ] || exit 1 +[ "${RNNOISE:-1}" = 0 ] || echo /usr/lib/ladspa/librnnoise_ladspa.so +''') + write_stub(bindir, "lv2ls", '''\ +[ "${LV2_FAIL:-0}" = 0 ] || exit 1 +[ "${LV2:-1}" = 0 ] || echo http://lsp-plug.in/plugins/lv2/para_equalizer_x16_lr +''') + write_stub(bindir, "gsettings", '''\ +echo "gsettings $*" >> "$HOME/calls" +echo "gsettings-bus $DBUS_SESSION_BUS_ADDRESS" >> "$HOME/calls" +[ "$1" != get ] || echo "${GNOME_VOLUME:-true}" +''') + write_stub(bindir, "systemctl", '''\ +echo "systemctl $*" >> "$HOME/calls" +echo "session $XDG_RUNTIME_DIR $DBUS_SESSION_BUS_ADDRESS" >> "$HOME/calls" +[ "${SESSION_FAIL:-0}" = 0 ] || exit 1 +shift +case "$1" in + list-unit-files|list-units) [ "${MIC_LOAD:-not-found}" = loaded ] && echo mic-dsp-link.service; : ;; + is-enabled) [ -e "$HOME/.mic-enabled" ] ;; + is-active) [ -e "$HOME/.mic-active" ] ;; + enable) touch "$HOME/.mic-enabled" ;; + disable) rm -f "$HOME/.mic-enabled" ;; + start) touch "$HOME/.mic-active" ;; + stop) rm -f "$HOME/.mic-active" ;; + restart) [ "${RESTART_FAIL:-0}" = 0 ] || exit 1 ;; +esac +''') + write_stub(bindir, "flatpak", '''\ +echo "flatpak $*" >> "$HOME/calls" +case "$1" in + ps) [ "${FLATPAK_RUNNING:-0}" = 0 ] || echo com.github.wwmm.easyeffects ;; + list) [ "${FLATPAK_INSTALLED:-0}" = 0 ] || echo com.github.wwmm.easyeffects ;; +esac +''') + write_stub(bindir, "sudo", '''\ +echo "sudo $*" >> "$HOME/calls" +[ "$1" = -n ] && shift +"$@" +''') + return env + + +def home(env, path): + return Path(env["HOME"]) / path + + +def asset(name): + return (REPO / "units/pipewire-audio/files" / name).read_bytes() + + +def test_full_chain_host_volume_services_and_idempotence(stub_env): + env = setup(stub_env) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert home(env, ".config/pipewire/pipewire.conf.d/50-speaker-dsp.conf").read_bytes() == asset("speaker-dsp.conf") + assert home(env, ".config/pipewire/pipewire.conf.d/60-mic-dsp.conf").read_bytes() == asset("mic-rnnoise.conf") + service = home(env, ".config/systemd/user/mic-dsp-link.service").read_text() + assert "set-source-volume $$HW_MIC 75%" in service + assert "{{" not in service + assert "systemctl --user enable mic-dsp-link.service" in calls(env) + assert "systemctl --user start mic-dsp-link.service" in calls(env) + assert "systemctl --user restart pipewire pipewire-pulse.socket" in calls(env) + assert "session /run/user/1000 unix:path=/run/user/1000/bus" in calls(env) + before = calls(env) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + after = calls(env)[len(before):] + assert "daemon-reload" not in after + assert " restart " not in after + assert " start " not in after + assert " enable " not in after + + +def test_builtin_eq_absent_mic_and_user_owned_data_survive(stub_env): + env = setup(stub_env) + config = home(env, ".config/pipewire/pipewire.conf.d/60-mic-dsp.conf") + config.parent.mkdir(parents=True) + config.write_bytes(asset("mic-rnnoise.conf")) + directory = home(env, ".config/wireplumber/main.lua.d") + directory.mkdir(parents=True) + (directory / "51-speaker-dsp-default.lua").write_bytes(asset("51-speaker-dsp-default.lua")) + (directory / "my-rule.lua").write_text("user rule") + preset = home(env, ".var/app/com.github.wwmm.easyeffects/config/preset.json") + preset.parent.mkdir(parents=True) + preset.write_text("user preset") + home(env, ".mic-active").touch() + home(env, ".mic-enabled").touch() + service = home(env, ".config/systemd/user/mic-dsp-link.service") + service.parent.mkdir(parents=True) + template = (REPO / "units/pipewire-audio/templates/mic-dsp-link.service.j2").read_text() + service.write_text(template.replace("{{ mic_hardware_volume_pct | default(100) }}", "75")) + result = run_unit("pipewire-audio", env, RNNOISE="0", LV2="0", MIC_LOAD="loaded") + assert result.returncode == 0, result.stdout + result.stderr + assert home(env, ".config/pipewire/pipewire.conf.d/50-speaker-dsp.conf").read_bytes() == asset("speaker-biquad.conf") + assert not config.exists() + assert not (directory / "51-speaker-dsp-default.lua").exists() + assert (directory / "my-rule.lua").read_text() == "user rule" + assert preset.read_text() == "user preset" + assert "systemctl --user stop mic-dsp-link.service" in calls(env) + assert "systemctl --user disable mic-dsp-link.service" in calls(env) + assert "systemctl --user restart wireplumber" in calls(env) + + +def test_mic_search_failure_preserves_config_and_service(stub_env): + env = setup(stub_env) + config = home(env, ".config/pipewire/pipewire.conf.d/60-mic-dsp.conf") + config.parent.mkdir(parents=True) + config.write_text("existing mic config") + result = run_unit("pipewire-audio", env, FIND_FAIL="1") + assert result.returncode == 1 + assert config.read_text() == "existing mic config" + assert "PARTIAL: microphone DSP unchanged" in result.stdout + assert "mic-dsp-link.service" not in calls(env) + + +def test_session_failure_retries_pending_restarts(stub_env): + env = setup(stub_env) + first = run_unit("pipewire-audio", env, SESSION_FAIL="1") + assert first.returncode == 1 + assert home(env, ".local/state/pipewire-audio/pipewire").exists() + assert home(env, ".local/state/pipewire-audio/mic").exists() + second = run_unit("pipewire-audio", env) + assert second.returncode == 0, second.stdout + second.stderr + assert "systemctl --user restart pipewire pipewire-pulse.socket" in calls(env) + assert not home(env, ".local/state/pipewire-audio/pipewire").exists() + assert not home(env, ".local/state/pipewire-audio/mic").exists() + + +def test_gates_and_missing_inventory_parser(stub_env): + env = setup(stub_env, "mumbai") + result = run_unit("pipewire-audio", env) + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert not calls(env) + env["DOTFILES_MACHINE"] = "test-fleet-fedora" + result = run_unit("pipewire-audio", env) + assert result.returncode == 0 and "SKIPPED" in result.stdout + env["DOTFILES_MACHINE"] = "himachal" + env["DOTFILES_IS_DESKTOP"] = "0" + assert "SKIPPED" in run_unit("pipewire-audio", env).stdout + env.pop("DOTFILES_IS_DESKTOP") + result = run_unit("pipewire-audio", env) + assert result.returncode == 1 and "requires yq" in result.stdout + assert not calls(env) + + +def test_inventory_detection_and_parser_errors(stub_env): + env = setup(stub_env) + env.pop("DOTFILES_IS_DESKTOP") + bindir = Path(env["PATH"]) + write_stub(bindir, "yq", 'echo "${YQ_RESULT:-true}"\nexit "${YQ_RC:-0}"\n') + result = run_unit("pipewire-audio", env, YQ_RESULT="false") + assert result.returncode == 0 and "SKIPPED" in result.stdout + result = run_unit("pipewire-audio", env, YQ_RC="2") + assert result.returncode == 1 and "read desktop inventory" in result.stdout + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_changed_volume_restarts_mic_and_invalid_override_fails(stub_env): + env = setup(stub_env) + assert run_unit("pipewire-audio", env).returncode == 0 + before = calls(env) + result = run_unit("pipewire-audio", env, MIC_HARDWARE_VOLUME_PCT="50") + assert result.returncode == 0, result.stdout + result.stderr + assert "set-source-volume $$HW_MIC 50%" in home(env, ".config/systemd/user/mic-dsp-link.service").read_text() + assert "systemctl --user restart mic-dsp-link.service" in calls(env)[len(before):] + before = calls(env) + result = run_unit("pipewire-audio", env, MIC_HARDWARE_VOLUME_PCT="101") + assert result.returncode == 1 and "invalid mic_hardware_volume_pct" in result.stdout + assert calls(env) == before + + +def test_system_package_and_flatpak_changes(stub_env): + env = setup(stub_env) + bindir = Path(env["PATH"]) + write_stub(bindir, "dnf", 'echo "dnf $*" >> "$HOME/calls"\n') + write_stub(bindir, "rpm", 'exit "${RPM_INSTALLED:-1}"\n') + result = run_unit("pipewire-audio", env, FLATPAK_RUNNING="1", FLATPAK_INSTALLED="1") + assert result.returncode == 0, result.stdout + result.stderr + assert "sudo -n dnf install -y lsp-plugins-lv2 noise-suppression-for-voice" in calls(env) + assert "flatpak kill com.github.wwmm.easyeffects" in calls(env) + assert "sudo -n flatpak uninstall --system -y com.github.wwmm.easyeffects" in calls(env) + before = calls(env) + result = run_unit("pipewire-audio", env, RPM_INSTALLED="0") + assert result.returncode == 0, result.stdout + result.stderr + assert "dnf install" not in calls(env)[len(before):] + + +def test_ostree_package_failure_is_reported_but_audio_continues(stub_env): + env = setup(stub_env) + bindir = Path(env["PATH"]) + write_stub(bindir, "rpm", 'exit 1\n') + write_stub(bindir, "rpm-ostree", 'echo "rpm-ostree $*" >> "$HOME/calls"\nexit 1\n') + result = run_unit("pipewire-audio", env) + assert result.returncode == 1 and "FAIL DSP prerequisite packages" in result.stdout + assert "sudo -n rpm-ostree install --apply-live --idempotent" in calls(env) + assert home(env, ".config/pipewire/pipewire.conf.d/50-speaker-dsp.conf").exists() + + +def test_default_mic_volume_and_gnome_setting(stub_env): + env = setup(stub_env, "kanpur") + result = run_unit("pipewire-audio", env, GNOME_VOLUME="false", DBUS_SESSION_BUS_ADDRESS="unix:path=/custom/bus") + assert result.returncode == 0, result.stdout + result.stderr + service = home(env, ".config/systemd/user/mic-dsp-link.service").read_text() + assert "set-source-volume $$HW_MIC 100%" in service + assert "gsettings set org.gnome.desktop.sound allow-volume-above-100-percent true" in calls(env) + assert "gsettings-bus unix:path=/custom/bus" in calls(env) + + +def test_missing_plugin_and_service_are_not_an_error(stub_env): + env = setup(stub_env) + result = run_unit("pipewire-audio", env, RNNOISE="0") + assert result.returncode == 0, result.stdout + result.stderr + assert "systemctl --user stop mic-dsp-link.service" not in calls(env) + assert "systemctl --user disable mic-dsp-link.service" not in calls(env) + + +def test_custom_audio_files_and_service_are_preserved(stub_env): + env = setup(stub_env) + paths = [ + ".config/pipewire/pipewire.conf.d/50-speaker-dsp.conf", + ".config/pipewire/pipewire.conf.d/60-mic-dsp.conf", + ".config/systemd/user/pipewire.service.d/ladspa-path.conf", + ".config/systemd/user/mic-dsp-link.service", + ".config/wireplumber/main.lua.d/51-speaker-dsp-default.lua", + ] + for path in paths: + target = home(env, path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("custom " + path) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + for path in paths: + assert home(env, path).read_text() == "custom " + path + assert path in result.stdout + assert "systemctl --user start mic-dsp-link.service" not in calls(env) + assert "systemctl --user restart wireplumber" not in calls(env) + result = run_unit("pipewire-audio", env, RNNOISE="0", LV2="0") + assert result.returncode == 0, result.stdout + result.stderr + assert home(env, paths[1]).read_text() == "custom " + paths[1] + assert home(env, paths[0]).read_text() == "custom " + paths[0] + assert "systemctl --user stop mic-dsp-link.service" not in calls(env) + + +def test_custom_service_is_not_started_with_managed_mic_config(stub_env): + env = setup(stub_env) + service = home(env, ".config/systemd/user/mic-dsp-link.service") + service.parent.mkdir(parents=True) + service.write_text("[Service]\nExecStart=/custom/microphone\n") + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert service.read_text() == "[Service]\nExecStart=/custom/microphone\n" + assert "retained custom microphone link service" in result.stdout + assert home(env, ".config/pipewire/pipewire.conf.d/60-mic-dsp.conf").read_bytes() == asset("mic-rnnoise.conf") + assert "systemctl --user enable mic-dsp-link.service" not in calls(env) + assert "systemctl --user start mic-dsp-link.service" not in calls(env) + + +def test_unknown_lv2_preserves_existing_speaker_and_fresh_falls_back(stub_env): + env = setup(stub_env) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + speaker = home(env, ".config/pipewire/pipewire.conf.d/50-speaker-dsp.conf") + assert speaker.read_bytes() == asset("speaker-dsp.conf") + result = run_unit("pipewire-audio", env, LV2_FAIL="1") + assert result.returncode == 0 and "LSP availability is unknown" in result.stdout + assert speaker.read_bytes() == asset("speaker-dsp.conf") + result = run_unit("pipewire-audio", env, LV2="0") + assert result.returncode == 0 and speaker.read_bytes() == asset("speaker-biquad.conf") + speaker.unlink() + Path(env["PATH"], "lv2ls").unlink() + result = run_unit("pipewire-audio", env) + assert result.returncode == 0 and "LSP availability is unknown" in result.stdout + assert speaker.read_bytes() == asset("speaker-biquad.conf") + + +def test_mic_default_and_group_then_host_precedence(stub_env, tmp_path): + env = setup(stub_env, "audio-test") + source = tmp_path / "source" + assets = source / "units/pipewire-audio" + (assets / "defaults").mkdir(parents=True) + (assets / "defaults/main.yml").write_text("mic_hardware_volume_pct: 93\n") + (assets / "files").symlink_to(REPO / "units/pipewire-audio/files", target_is_directory=True) + (assets / "templates").symlink_to(REPO / "units/pipewire-audio/templates", target_is_directory=True) + groups = source / "group_vars" + groups.mkdir() + hosts = source / "host_vars" + hosts.mkdir() + + def apply(expected, **extra): + result = run_unit("pipewire-audio", env, DOTFILES_DIR=str(source), **extra) + assert result.returncode == 0, result.stdout + result.stderr + service = home(env, ".config/systemd/user/mic-dsp-link.service").read_text() + assert f"set-source-volume $$HW_MIC {expected}%" in service + + apply(93) + (groups / "all.yml").write_text("mic_hardware_volume_pct: 64\n") + apply(64) + (groups / "desktop.yml").write_text("mic_hardware_volume_pct: 42\n") + apply(42) + (groups / "special.yml").write_text("mic_hardware_volume_pct: 39\n") + result = run_unit("pipewire-audio", env, DOTFILES_DIR=str(source)) + assert result.returncode == 1 and "requires yq" in result.stderr + write_stub(Path(env["PATH"]), "yq", 'echo true\n') + apply(39) + (hosts / "audio-test.yml").write_text("mic_hardware_volume_pct: 37\n") + apply(37) + apply(29, MIC_HARDWARE_VOLUME_PCT="29") diff --git a/tests/test_platform_native_units.py b/tests/test_platform_native_units.py new file mode 100644 index 00000000..7d5d0d06 --- /dev/null +++ b/tests/test_platform_native_units.py @@ -0,0 +1,326 @@ +"""Behavioral coverage for native platform units; all system paths are sandboxed.""" +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub + + +@pytest.fixture +def platform_env(tmp_path, stub_env): + bindir, env = stub_env + # Do not inherit host-specific tools such as yq, sudo, tailscale or pkg. + for name in ("bash", "dirname", "cat", "sed", "head", "grep", "mktemp", "rm", "chmod", + "cmp", "install", "mkdir", "mv", "ln", "readlink", "stat", "sh"): + target = shutil.which(name) + assert target + (bindir / name).symlink_to(target) + (bindir / "python3").symlink_to(sys.executable) + root = tmp_path / "repo" + units = root / "units" + units.mkdir(parents=True) + (root / "host_vars").mkdir() + (root / "group_vars").mkdir() + (root / "group_vars/all.yml").write_text("tailscale_domain: example.ts.net\n") + (root / "inventory.yml").write_text( + "all:\n children:\n desktop:\n hosts:\n laptop:\n" + " termux_hosts:\n hosts:\n phone:\n" + ) + for name in ("apk-packages.sh", "apk-packages.conf", "termux-packages.sh", "termux-packages.conf", + "termux-start-sshd", "tailscale.sh", "tailscale-connect.py", "sudo.sh", "sshd.sh", + "platform-config.bash", "package-config.bash"): + (units / name).symlink_to(REPO / "units" / name) + lib = (REPO / "units/lib.sh").read_text() + lib = lib.replace("IS_TERMUX=0; [ -d /data/data/com.termux/files/usr ] && IS_TERMUX=1", + 'IS_TERMUX="${TEST_TERMUX:-0}"') + lib = lib.replace("IS_MUSL=0; { [ -e /lib/ld-musl-x86_64.so.1 ] || [ -e /lib/ld-musl-aarch64.so.1 ]; } && IS_MUSL=1", + 'IS_MUSL="${TEST_MUSL:-0}"') + (units / "lib.sh").write_text(lib) + system = tmp_path / "system" + (system / "etc/ssh/sshd_config.d").mkdir(parents=True) + (system / "etc/sudoers.d").mkdir(parents=True) + (system / "etc/os-release").write_text('ID=ubuntu\nID_LIKE=debian\n') + log = tmp_path / "calls" + env.update(PATH=str(bindir), DOTFILES_DIR=str(root), DOTFILES_SYSTEM_ROOT=str(system), + DOTFILES_MACHINE="laptop", USER="james", LOG=str(log), + BREW_PREFIX=str(tmp_path / "no-brew"), TMPDIR=str(tmp_path)) + write_stub(bindir, "id", '''\ +case "$1" in -u) echo 1000 ;; -un) echo james ;; -nG) echo "james podman" ;; *) exit 1 ;; esac +''') + # Model root ownership while still writing only the temporary test directory. + (bindir / "stat").unlink() + write_stub(bindir, "stat", '''\ +if [ "$1" = -c ] && [ "$2" = '%a:%u:%g' ]; then + mode=$(/usr/bin/stat -c %a "$3") || exit 1; echo "$mode:0:0" +else exec /usr/bin/stat "$@"; fi +''') + write_stub(bindir, "sudo", '''\ +[ "$1" = -n ] || exit 1 +shift +case "$1" in + /usr/sbin/visudo) echo visudo >> "$LOG"; exit "${FAIL_VISUDO:-0}" ;; + /usr/sbin/sshd) echo validate-sshd >> "$LOG"; exit "${FAIL_SSHD:-0}" ;; + install) + shift; args=() + while [ "$#" -gt 0 ]; do + case "$1" in -o|-g) shift 2 ;; *) args+=("$1"); shift ;; esac + done + exec /usr/bin/install "${args[@]}" ;; + *) exec "$@" ;; +esac +''') + write_stub(bindir, "systemctl", '''\ +echo "systemctl $*" >> "$LOG" +if [ "$1" = is-active ]; then echo "${RESOLVED_STATE:-inactive}"; fi +''') + write_stub(bindir, "sysctl", 'echo "sysctl $*" >> "$LOG"\n') + write_stub(bindir, "sleep", 'exit 0\n') + return root, system, bindir, env, log + + +def run_unit(fixture, name, **updates): + root, _, _, env, _ = fixture + return subprocess.run(["bash", str(root / "units" / f"{name}.sh")], + env={**env, **updates}, text=True, capture_output=True) + + +def log_lines(path): + return path.read_text().splitlines() if path.exists() else [] + + +def test_sudo_validate_before_replacement_and_cleanup(platform_env): + root, system, _, _, log = platform_env + sudoers = system / "etc/sudoers.d" + (sudoers / "james").write_text("old rule\n") + (sudoers / "99-james").write_text("another old rule\n") + failed = run_unit(platform_env, "sudo", FAIL_VISUDO="1") + assert failed.returncode == 1 + assert (sudoers / "james").read_text() == "old rule\n" + assert (sudoers / "99-james").exists() + assert not (sudoers / "zz-james").exists() + result = run_unit(platform_env, "sudo") + assert result.returncode == 0, result.stdout + result.stderr + dest = sudoers / "zz-james" + assert dest.read_text() == "james ALL=(ALL) NOPASSWD:ALL\n" + assert dest.stat().st_mode & 0o777 == 0o440 + assert not (sudoers / "james").exists() and not (sudoers / "99-james").exists() + (root / "host_vars/phone.yml").write_text("skip_sudo: true\n") + before = log_lines(log) + skipped = run_unit(platform_env, "sudo", DOTFILES_MACHINE="phone", TEST_TERMUX="1") + assert skipped.returncode == 0 and log_lines(log) == before + + +def test_sshd_validates_hardening_and_reloads_once_only_on_change(platform_env): + root, system, _, _, log = platform_env + (root / "host_vars/laptop.yml").write_text("sshd_harden: true\n") + directory = system / "etc/ssh/sshd_config.d" + bad = run_unit(platform_env, "sshd", FAIL_SSHD="1") + assert bad.returncode == 1 + assert (directory / "99-bw-env.conf").read_text() == "AcceptEnv BW_SESSION BW_CLIENTID BW_CLIENTSECRET\n" + assert not (directory / "10-hardening.conf").exists() + assert log_lines(log).count("systemctl reload ssh") == 1 + result = run_unit(platform_env, "sshd") + assert result.returncode == 0, result.stdout + result.stderr + assert "PasswordAuthentication no\n" in (directory / "10-hardening.conf").read_text() + assert log_lines(log).count("systemctl reload ssh") == 2 + assert run_unit(platform_env, "sshd").returncode == 0 + assert log_lines(log).count("systemctl reload ssh") == 2 + (directory / "99-bw-env.conf").write_text("out of date\n") + retired = run_unit(platform_env, "sshd", DOTFILES_MACHINE="matrix") + assert retired.returncode == 0 + assert log_lines(log).count("systemctl reload ssh") == 2 + + +def test_termux_packages_bootstrap_ssh_and_keepalive(platform_env): + _, system, bindir, _, log = platform_env + write_stub(bindir, "pkg", 'echo "pkg $*" >> "$LOG"\n') + write_stub(bindir, "npm", 'echo "npm $*" >> "$LOG"\n') + write_stub(bindir, "pgrep", 'exit "${SSHD_RUNNING:-1}"\n') + write_stub(bindir, "sshd", 'echo sshd-start >> "$LOG"\n') + write_stub(bindir, "termux-job-scheduler", '''\ +if [ "$1" = -p ]; then [ "${PENDING:-0}" = 0 ] || echo 'Pending Job 1'; exit 0; fi +echo "job $*" >> "$LOG" +''') + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1") + assert result.returncode == 0, result.stdout + result.stderr + calls = log_lines(log) + assert calls.index("pkg install -y python yq chezmoi") < next(i for i, x in enumerate(calls) if x.startswith("npm ")) + assert "pkg install -y mise" in calls + assert "npm install -g @earendil-works/pi-coding-agent" in calls + assert "sshd-start" in calls + assert any("--job-id 1 --period-ms 1800000 --persisted true" in line for line in calls) + boot = Path(platform_env[3]["HOME"]) / ".termux/boot/start-sshd" + assert "termux-wake-lock\nsshd\n" in boot.read_text() and boot.stat().st_mode & 0o777 == 0o755 + log.unlink() + assert run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1", + SSHD_RUNNING="0", PENDING="1").returncode == 0 + assert not any(line.startswith("job ") or line == "sshd-start" for line in log_lines(log)) + log.unlink() + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="laptop") + assert result.returncode == 0 and not log_lines(log) + assert system.exists() + + +def test_termux_unavailable_bootstrap_package_does_not_block_ssh(platform_env): + _, _, bindir, env, log = platform_env + write_stub(bindir, "pkg", '''\ +echo "pkg $*" >> "$LOG" +case " $* " in *' yq '*) exit 1 ;; esac +''') + write_stub(bindir, "npm", 'echo "npm $*" >> "$LOG"\n') + write_stub(bindir, "pgrep", 'exit 1\n') + write_stub(bindir, "sshd", 'echo sshd-start >> "$LOG"\n') + write_stub(bindir, "termux-job-scheduler", 'echo "job $*" >> "$LOG"\n') + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1") + assert result.returncode == 1 and "Termux package unavailable or installation failed: yq" in result.stderr + assert (Path(env["HOME"]) / ".termux/boot/start-sshd").exists() + assert "sshd-start" in log_lines(log) + assert any(line.startswith("job --job-id 1") for line in log_lines(log)) + + +def test_apk_failure_isolation_and_bootstrap(platform_env): + _, _, bindir, _, log = platform_env + write_stub(bindir, "apk", '''\ +case "$1" in + info) exit 1 ;; + update) echo 'apk update' >> "$LOG" ;; + add) echo "apk $*" >> "$LOG"; case " $* " in *' exa '*) exit 1 ;; esac ;; +esac +''') + write_stub(bindir, "npm", 'echo "npm $*" >> "$LOG"\n') + result = run_unit(platform_env, "apk-packages", TEST_MUSL="1") + assert result.returncode == 1 and "exa" in result.stderr + calls = log_lines(log) + assert any("chezmoi yq-go python3 py3-yaml" in line for line in calls) + assert any("--no-upgrade" in line and " bash" in line for line in calls) + assert "npm install -g @bitwarden/cli" in calls + assert any("systemctl enable --now podman.socket" in line for line in calls) + log.unlink() + assert run_unit(platform_env, "apk-packages").returncode == 0 + assert not log_lines(log) + + +def tailscale_stubs(fixture): + _, _, bindir, env, _ = fixture + write_stub(bindir, "tailscale", '''\ +echo "tailscale $*" >> "$LOG" +if [ "$1" = status ]; then exit "${TS_CONNECTED:-1}"; fi +if [ "$1" = up ]; then + for arg in "$@"; do case "$arg" in --auth-key=file:*) + file="${arg#--auth-key=file:}" + /usr/bin/stat -c %a "$file" > "$KEY_MODE" + /usr/bin/cat "$file" > "$KEY_CAPTURE" + printf '%s' "$file" > "$KEY_PATH" + ;; esac; done +fi +''') + write_stub(bindir, "bw", '''\ +echo "bw $*" >> "$LOG" +case "$1" in unlock) exit "${BW_LOCKED:-0}" ;; get) printf '%s' '{"login":{"password":"tskey-sensitive-test"}}' ;; esac +''') + tmp = Path(env["TMPDIR"]) + env.update(KEY_MODE=str(tmp / "key-mode"), KEY_CAPTURE=str(tmp / "captured-key"), KEY_PATH=str(tmp / "key-path")) + mock = tmp / "mockpython" + mock.mkdir() + (mock / "sitecustomize.py").write_text('''\ +import io, json, os, urllib.error, urllib.request +class Response(io.BytesIO): + status = 200 + def __enter__(self): return self + def __exit__(self, *args): self.close() +def open_mock(req, timeout): + assert req.get_header("Authorization").startswith("Basic ") + with open(os.environ["LOG"], "a") as f: f.write("api " + req.get_method() + " " + req.full_url + "\\n") + if req.get_method() == "GET": + name = os.environ.get("DEVICE_NAME", "laptop.example.ts.net") + return Response(json.dumps({"devices": [{"name": name, "id": "device-123"}]}).encode()) + if os.environ.get("FAIL_DELETE") == "1": + raise urllib.error.HTTPError(req.full_url, 403, "secret-bearing response", None, None) + return Response(b"{}") +urllib.request.urlopen = open_mock +''') + env["PYTHONPATH"] = str(mock) + + +def test_tailscale_join_keeps_tokens_off_argv_and_no_secrets_skips_vault(platform_env): + _, _, _, env, log = platform_env + tailscale_stubs(platform_env) + no_secrets = run_unit(platform_env, "tailscale", UNITS_NO_SECRETS="1", BW_SESSION="session-sensitive") + assert no_secrets.returncode == 0, no_secrets.stdout + no_secrets.stderr + assert "PARTIAL:" in no_secrets.stdout + assert not any(line.startswith("bw ") or line.startswith("api ") for line in log_lines(log)) + log.unlink() + joined = run_unit(platform_env, "tailscale", BW_SESSION="session-sensitive") + assert joined.returncode == 0, joined.stdout + joined.stderr + calls = log_lines(log) + assert any("api DELETE https://api.tailscale.com/api/v2/device/device-123" in line for line in calls) + assert any("--auth-key=file:" in line and "--advertise-tags=tag:fleet" in line for line in calls) + assert Path(env["KEY_MODE"]).read_text().strip() == "600" + assert Path(env["KEY_CAPTURE"]).read_text() == "tskey-sensitive-test" + assert not Path(Path(env["KEY_PATH"]).read_text()).exists() + assert "tskey-sensitive-test" not in "\n".join(calls) + joined.stdout + joined.stderr + assert "session-sensitive" not in "\n".join(calls) + joined.stdout + joined.stderr + traced = subprocess.run(["bash", "-x", str(platform_env[0] / "units/tailscale.sh")], + env={**env, "BW_SESSION": "session-sensitive"}, text=True, capture_output=True) + assert traced.returncode == 0, traced.stdout + traced.stderr + assert "session-sensitive" not in traced.stdout + traced.stderr + assert "tskey-sensitive-test" not in traced.stdout + traced.stderr + + +def test_tailscale_resolver_routes_and_retired_service_guard(platform_env): + root, system, _, _, log = platform_env + tailscale_stubs(platform_env) + (root / "host_vars/laptop.yml").write_text( + 'tailscale_advertise_routes: "192.168.0.0/24"\ntailscale_accept_dns: false\ntailscale_operator: james\n' + ) + dest = system / "etc/resolv.conf" + external = system / "stub" + external.write_text("existing resolver\n") + dest.symlink_to(external) + result = run_unit(platform_env, "tailscale", TS_CONNECTED="0") + assert result.returncode == 0, result.stdout + result.stderr + assert not dest.is_symlink() and dest.read_text() == "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" + assert external.read_text() == "existing resolver\n" + assert any("--advertise-routes=192.168.0.0/24 --accept-dns=false --operator=james" in line for line in log_lines(log)) + log.unlink() + result = run_unit(platform_env, "tailscale", DOTFILES_MACHINE="matrix", TS_CONNECTED="1", BW_SESSION="session-sensitive") + assert result.returncode == 0 + assert not any("restart" in line or "reload" in line or line.startswith("bw ") for line in log_lines(log)) + + +def test_tailscale_matching_stale_device_and_delete_failure(platform_env): + _, _, _, _, log = platform_env + tailscale_stubs(platform_env) + other = run_unit(platform_env, "tailscale", BW_SESSION="x", DEVICE_NAME="other.example.ts.net") + assert other.returncode == 0, other.stdout + other.stderr + assert not any("api DELETE" in line for line in log_lines(log)) + log.unlink() + failed = run_unit(platform_env, "tailscale", BW_SESSION="x", FAIL_DELETE="1") + assert failed.returncode == 1 + assert "stale Tailscale device" in failed.stderr + assert "secret-bearing response" not in failed.stderr + assert not any(line.startswith("tailscale up") for line in log_lines(log)) + + +def test_tailscale_resolver_only_points_to_active_stub(platform_env): + _, system, _, _, log = platform_env + tailscale_stubs(platform_env) + dest = system / "etc/resolv.conf" + dest.write_text("nameserver 9.9.9.9\n") + inactive = run_unit(platform_env, "tailscale", TS_CONNECTED="0") + assert inactive.returncode == 0, inactive.stdout + inactive.stderr + assert dest.read_text() == "nameserver 9.9.9.9\n" and not dest.is_symlink() + log.unlink() + active = run_unit(platform_env, "tailscale", TS_CONNECTED="0", RESOLVED_STATE="active") + assert active.returncode == 0, active.stdout + active.stderr + assert dest.is_symlink() and os.readlink(dest) == "/run/systemd/resolve/stub-resolv.conf" + assert log_lines(log).count("systemctl restart systemd-resolved") == 1 + log.unlink() + repeat = run_unit(platform_env, "tailscale", TS_CONNECTED="0", RESOLVED_STATE="active") + assert repeat.returncode == 0, repeat.stdout + repeat.stderr + assert "systemctl restart systemd-resolved" not in log_lines(log) diff --git a/tests/test_retired_host_units.py b/tests/test_retired_host_units.py new file mode 100644 index 00000000..71ac9f63 --- /dev/null +++ b/tests/test_retired_host_units.py @@ -0,0 +1,20 @@ +import subprocess + +import pytest + +from conftest import REPO, write_stub + + +@pytest.mark.parametrize("machine", ["matrix", "telengana"]) +@pytest.mark.parametrize("unit", sorted((REPO / "units").glob("*.sh")), ids=lambda p: p.stem) +def test_retired_hosts_never_apply_units(stub_env, tmp_path, machine, unit): + bindir, env = stub_env + log = tmp_path / "mutations" + for command in ("sudo", "systemctl", "brew", "flatpak", "tailscale", "apk", "pkg", "chezmoi"): + write_stub(bindir, command, 'printf "%s\\n" "$0" >> "$MUTATIONS"\nexit 1\n') + env.update(DOTFILES_MACHINE=machine, DOTFILES_SYSTEM_ROOT=str(tmp_path / "system"), MUTATIONS=str(log)) + result = subprocess.run(["bash", str(unit)], env=env, capture_output=True, text=True, timeout=15) + assert result.returncode == 0, result.stderr + assert "retired VPS" in result.stdout + assert not log.exists() + assert not (tmp_path / "system").exists() diff --git a/tests/test_security_native_units.py b/tests/test_security_native_units.py new file mode 100644 index 00000000..abc35686 --- /dev/null +++ b/tests/test_security_native_units.py @@ -0,0 +1,112 @@ +import hashlib +import importlib.util +import subprocess +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub + + +spec = importlib.util.spec_from_file_location("hardening_file", REPO / "units/server-hardening-file.py") +hardening = importlib.util.module_from_spec(spec) +spec.loader.exec_module(hardening) + + +def test_hardening_replaces_last_match_and_preserves_mode_and_symlink(tmp_path): + real = tmp_path / "sshd_config" + link = tmp_path / "sshd_link" + real.write_text("#MaxStartups 10:30:100\nOtherSetting yes\nMaxStartups 8:30:60\n") + real.chmod(0o640) + link.symlink_to(real) + assert hardening.update(link, r"^#?MaxStartups", "MaxStartups 3:50:10") + assert real.read_text() == "#MaxStartups 10:30:100\nOtherSetting yes\nMaxStartups 3:50:10\n" + assert link.is_symlink() and real.stat().st_mode & 0o777 == 0o640 + assert not hardening.update(link, r"^#?MaxStartups", "MaxStartups 3:50:10") + + +def test_fstab_entry_is_added_once_without_clobbering_existing_lines(tmp_path): + fstab = tmp_path / "fstab" + fstab.write_text("/dev/root / ext4 defaults 0 1") + rule = "/swapfile none swap sw 0 0" + assert hardening.update(fstab, "^" + rule + "$", rule) + assert fstab.read_text() == "/dev/root / ext4 defaults 0 1\n" + rule + "\n" + assert not hardening.update(fstab, "^" + rule + "$", rule) + + +def run_retired_function(env, scope, path): + return subprocess.run(["bash", "-c", 'source "$1"; retired_stop "$2" "$3"', "bash", + str(REPO / "units/retired-services.sh"), scope, path], + env=env, text=True, capture_output=True) + + +@pytest.mark.parametrize("filename,unit", [("old.container", "old.service"), + ("old.volume", "old-volume.service"), + ("old.network", "old-network.service")]) +def test_quadlet_stops_generated_service_without_disabling(stub_env, tmp_path, filename, unit): + bindir, env = stub_env + calls = tmp_path / "calls" + env["CALLS"] = str(calls) + write_stub(bindir, "systemctl", 'printf "%s\\n" "$*" >> "$CALLS"\nif [[ "$*" == *show* ]]; then echo loaded; fi\n') + result = run_retired_function(env, "user", filename) + assert result.returncode == 0 + lines = calls.read_text().splitlines() + assert lines[-1] == f"--user stop {unit}" + assert not any("disable" in line for line in lines) + + +def test_missing_quadlet_is_not_stopped_and_unrelated_services_untouched(stub_env, tmp_path): + bindir, env = stub_env + calls = tmp_path / "calls" + env["CALLS"] = str(calls) + write_stub(bindir, "systemctl", 'printf "%s\\n" "$*" >> "$CALLS"\necho not-found\n') + assert run_retired_function(env, "user", "obsolete.container").returncode == 0 + assert calls.read_text().splitlines() == ["--user show --property=LoadState --value obsolete.service"] + + +def test_systemd_failure_is_not_treated_as_missing(stub_env): + bindir, env = stub_env + write_stub(bindir, "systemctl", "exit 1\n") + assert run_retired_function(env, "user", "obsolete.container").returncode != 0 + + +def test_cleanup_preserves_unrelated_units_and_files_that_fail_to_stop(stub_env): + bindir, env = stub_env + unit_dir = Path(env["HOME"]) / ".config/containers/systemd" + unit_dir.mkdir(parents=True) + for name in ("lima-bootc.container", "grafana.container", "my-service.container"): + (unit_dir / name).write_text(name) + write_stub(bindir, "systemctl", 'if [[ "$*" == *show* ]]; then echo loaded; exit 0; fi\n' + 'if [[ "$*" == "--user stop grafana.service" ]]; then exit 1; fi\nexit 0\n') + command = ["bash", "-c", 'source "$1"; retired_cleanup_user lima-bootc.container grafana.container; finish', + "bash", str(REPO / "units/retired-services.sh")] + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 1 + assert not (unit_dir / "lima-bootc.container").exists() + assert (unit_dir / "grafana.container").exists() + assert (unit_dir / "my-service.container").read_text() == "my-service.container" + + +@pytest.mark.parametrize("unit", ["osquery.sh", "server-hardening.sh", "retired-services.sh"]) +@pytest.mark.parametrize("host", ["matrix", "telengana"]) +def test_retired_vps_exits_before_any_privileged_operation(stub_env, unit, host): + bindir, env = stub_env + env.update(DOTFILES_DIR=str(REPO), DOTFILES_MACHINE=host) + for binary in ("systemctl", "sudo", "yq"): + write_stub(bindir, binary, 'echo "must not run" >&2\nexit 99\n') + result = subprocess.run(["bash", str(REPO / "units" / unit)], env=env, capture_output=True, text=True) + assert result.returncode == 0 + assert "SKIPPED" in result.stdout + assert "must not run" not in result.stderr + + +def test_osquery_archive_checksum_rejects_tampering(tmp_path, stub_env): + _, env = stub_env + archive = tmp_path / "release.tar.gz" + archive.write_bytes(b"authentic release") + checksum = hashlib.sha256(archive.read_bytes()).hexdigest() + command = ["bash", "-c", 'source "$1"; osquery_verify_archive "$2" "$3"', "bash", + str(REPO / "units/osquery.sh"), str(archive), checksum] + assert subprocess.run(command, env=env, capture_output=True).returncode == 0 + archive.write_bytes(b"tampered release") + assert subprocess.run(command, env=env, capture_output=True).returncode != 0 diff --git a/tests/test_server_service_units.py b/tests/test_server_service_units.py new file mode 100644 index 00000000..07ec8648 --- /dev/null +++ b/tests/test_server_service_units.py @@ -0,0 +1,221 @@ +import subprocess +import sys + +import pytest + +from conftest import REPO +from unit_helpers import run_unit + + +# System files live in a private fake /etc. The scripts under test still execute +# their real rendering, comparison, privilege, and service-control logic. +FAKE_SYSTEM = r'''#!/usr/bin/env python3 +import json, os, pathlib, sys +name = pathlib.Path(sys.argv[0]).name +args = sys.argv[1:] +root = pathlib.Path(os.environ['FAKE_ROOT']) +log = root / 'calls' +def mapped(arg): + return str(root / arg.lstrip('/')) if arg.startswith('/etc/') else arg +def record(): + with log.open('a') as f: f.write(name + ' ' + ' '.join(args) + '\n') +if name == 'sudo': + if args and args[0] == '-n': args.pop(0) + os.execvp(args[0], args) +if name == 'yq': + expression = args[-2] + import yaml + with open(args[-1]) as f: data = yaml.safe_load(f) + if 'children.vps.hosts' in expression: + print(str(os.environ['MACHINE'] in (data['all']['children']['vps']['hosts'] or {})).lower()) + elif 'web_services' in expression: + print(json.dumps((data or {}).get('web_services') or [])) + elif 'tailscale_domain' in expression: + print((data or {}).get('tailscale_domain') or '') + else: sys.exit(2) + sys.exit(0) +if name in ('systemctl', 'tailscale', 'loginctl'): + record() + if name == 'tailscale' and args[:2] == ['serve', 'status']: + state = root / 'serve' + if state.exists(): print(state.read_text(), end='') + if name == 'tailscale' and args[:2] == ['serve', '--bg']: + (root / 'serve').write_text(args[2]) + if name == 'tailscale' and args[:1] == ['cert']: + pathlib.Path(mapped(args[args.index('--cert-file')+1])).write_text('VALID') + pathlib.Path(mapped(args[args.index('--key-file')+1])).write_text('PRIVATE') + sys.exit(0) +if name == 'openssl': + path = pathlib.Path(mapped(args[args.index('-in') + 1])) + sys.exit(0 if path.exists() and path.read_text() == 'VALID' else 1) +if name == 'chown': sys.exit(0) +if name == 'stat': + try: + mode = pathlib.Path(mapped(args[-1])).stat().st_mode & 0o777 + print(f'{mode:o}:0:0') + except FileNotFoundError: sys.exit(1) + sys.exit(0) +if name == 'install': + if os.environ.get('FAIL_CADDY') and args[-1] == '/etc/caddy/Caddyfile': sys.exit(1) + output = [] + i = 0 + while i < len(args): + if args[i] in ('-o', '-g'): + i += 2 + else: + output.append(mapped(args[i])) + i += 1 + os.execv('/usr/bin/install', ['install'] + output) +if name in ('cmp', 'chmod'): + os.execv('/usr/bin/' + name, [name] + [mapped(x) for x in args]) +sys.exit(2) +''' + + +@pytest.fixture +def fake_system(stub_env, tmp_path): + bindir, env = stub_env + root = tmp_path / 'system' + (root / 'etc/systemd/system').mkdir(parents=True) + env['FAKE_ROOT'] = str(root) + for name in ('sudo', 'yq', 'install', 'stat', 'cmp', 'chmod', 'chown', 'systemctl', 'tailscale', 'openssl', 'loginctl'): + path = bindir / name + path.write_text(FAKE_SYSTEM) + path.chmod(0o755) + repo = tmp_path / 'repo' + (repo / 'host_vars').mkdir(parents=True) + (repo / 'group_vars').mkdir() + (repo / 'group_vars/all.yml').write_text('tailscale_domain: example.ts.net\n') + (repo / 'inventory.yml').write_text('all:\n children:\n vps:\n hosts:\n matrix: {}\n telengana: {}\n other-vps: {}\n') + (repo / 'units').symlink_to(REPO / 'units') + env.update(DOTFILES_DIR=str(repo), DOTFILES_MACHINE='workstation') + return repo, root, env + + +def system_calls(root): + path = root / 'calls' + return path.read_text() if path.exists() else '' + + +def test_proxy_redirects_serve_migration_and_idempotence(fake_system): + repo, root, env = fake_system + (repo / 'host_vars/workstation.yml').write_text('web_services:\n - name: Build / Monitor\n href: https://example.test/monitor\n') + (root / 'serve').write_text('http://127.0.0.1:3000') + result = run_unit('proxy', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, result.stdout + result.stderr + caddy = (root / 'etc/caddy/Caddyfile').read_text() + assert 'handle /service/build---monitor {' in caddy + assert 'redir https://example.test/monitor 302' in caddy + assert 'reverse_proxy 10.88.0.1:8765' in caddy + assert (root / 'serve').read_text() == 'http://127.0.0.1:9000' + assert 'systemctl restart caddy.service' in system_calls(root) + (root / 'calls').unlink() + result = run_unit('proxy', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, result.stdout + result.stderr + assert 'systemctl restart' not in system_calls(root) + assert 'tailscale serve --bg' not in system_calls(root) + + +def test_proxy_failure_does_not_skip_other_configuration(fake_system): + repo, root, env = fake_system + env['FAIL_CADDY'] = '1' + result = run_unit('proxy', env, DOTFILES_DIR=str(repo)) + assert result.returncode != 0 + assert 'FAIL deploy Caddyfile' in result.stdout + assert (root / 'etc/containers/systemd/caddy.container').exists() + assert 'systemctl restart' not in system_calls(root) + assert 'tailscale serve --bg' not in system_calls(root) + + +def test_certificate_renewal_permissions_timer_and_validity(fake_system): + repo, root, env = fake_system + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, result.stdout + result.stderr + cert = root / 'etc/ssl/tailscale/cert.pem' + key = root / 'etc/ssl/tailscale/key.pem' + assert cert.read_text() == 'VALID' and cert.stat().st_mode & 0o777 == 0o644 + assert key.read_text() == 'PRIVATE' and key.stat().st_mode & 0o777 == 0o640 + service = (root / 'etc/systemd/system/tailscale-cert-renew.service').read_text() + timer = (root / 'etc/systemd/system/tailscale-cert-renew.timer').read_text() + assert 'workstation.example.ts.net' in service and '--min-validity 720h' in service + assert 'OnCalendar=weekly' in timer and 'RandomizedDelaySec=1h' in timer and 'Persistent=true' in timer + assert 'systemctl enable --now tailscale-cert-renew.timer' in system_calls(root) + (root / 'calls').unlink() + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, result.stdout + result.stderr + assert 'tailscale cert ' not in system_calls(root) + + +def test_certificate_renews_expired_cert_and_uses_host_domain(fake_system): + repo, root, env = fake_system + (repo / 'host_vars/workstation.yml').write_text('tailscale_domain: override.ts.net\n') + cert = root / 'etc/ssl/tailscale/cert.pem' + cert.parent.mkdir(parents=True) + cert.write_text('EXPIRED') + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, result.stdout + result.stderr + assert cert.read_text() == 'VALID' + assert 'workstation.override.ts.net' in system_calls(root) + assert 'workstation.override.ts.net' in (root / 'etc/systemd/system/tailscale-cert-renew.service').read_text() + + +@pytest.mark.parametrize('unit', ['proxy', 'tailscale-cert', 'bst-dashboard', 'kirocrew']) +@pytest.mark.parametrize('machine', ['matrix', 'telengana']) +def test_retired_vps_are_untouched(fake_system, unit, machine): + repo, root, env = fake_system + env['DOTFILES_MACHINE'] = machine + result = run_unit(unit, env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0 and 'SKIPPED' in result.stdout + assert system_calls(root) == '' + + +@pytest.mark.parametrize('unit', ['proxy', 'tailscale-cert']) +def test_other_vps_and_profile_exclusions(fake_system, unit): + repo, root, env = fake_system + env['DOTFILES_MACHINE'] = 'other-vps' + result = run_unit(unit, env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0 and 'SKIPPED' in result.stdout + env['DOTFILES_MACHINE'] = 'vm' + (repo / 'host_vars/vm.yml').write_text('machine_profile: vm-test\n') + result = run_unit(unit, env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0 and 'SKIPPED' in result.stdout + assert system_calls(root) == '' + + +@pytest.mark.parametrize('unit,flag', [('proxy', 'skip_proxy'), ('tailscale-cert', 'skip_tailscale_cert')]) +def test_per_host_exclusion_prevents_service_changes(fake_system, unit, flag): + repo, root, env = fake_system + (repo / 'host_vars/workstation.yml').write_text(f'{flag}: true\n') + result = run_unit(unit, env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0 and 'SKIPPED' in result.stdout + assert system_calls(root) == '' + + +def test_enabled_kirocrew_without_binary_does_not_change_service(fake_system): + repo, root, env = fake_system + (repo / 'host_vars/workstation.yml').write_text('kirocrew_enabled: true\n') + result = run_unit('kirocrew', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0 and 'binary or system service missing' in result.stdout + assert system_calls(root) == '' + + +def test_dashboard_template_uses_correct_project_and_service_scope(): + renderer = REPO / 'units/service-templates.py' + template = REPO / 'units/bst-dashboard/bst-dashboard.service.j2' + for project, image in [('/var/home/james/dev/kde-linux', 'aurora'), ('/var/home/james/dev/dakota', 'dakota')]: + result = subprocess.run([sys.executable, str(renderer), 'dashboard', str(template), project, image], capture_output=True, text=True, check=True) + assert f'--log /var/tmp/{image}-build.log --target oci/{image}.bst --project {project}' in result.stdout + assert 'WantedBy=default.target' in result.stdout and 'Restart=always' in result.stdout + + +@pytest.mark.parametrize('address,allowed', [('127.0.0.1:8000', True), ('[::1]:8000', True), ('0.0.0.0:8000', False), ('[::]:8000', False)]) +def test_kirocrew_listener_exposure(address, allowed): + line = f'LISTEN 0 128 {address} 0.0.0.0:* users:(("kirocrew",pid=7,fd=5))\n' + result = subprocess.run([sys.executable, str(REPO / 'units/kirocrew-sockets.py')], input=line, capture_output=True, text=True) + assert (result.returncode == 0) is allowed + + +def test_kirocrew_checks_local_address_not_loopback_peer(): + line = 'LISTEN 0 128 0.0.0.0:8000 127.0.0.1:99 users:(("kirocrew",pid=7,fd=5))\n' + result = subprocess.run([sys.executable, str(REPO / 'units/kirocrew-sockets.py')], input=line, capture_output=True, text=True) + assert result.returncode != 0 diff --git a/tests/test_server_units_check.py b/tests/test_server_units_check.py new file mode 100644 index 00000000..c9058473 --- /dev/null +++ b/tests/test_server_units_check.py @@ -0,0 +1,148 @@ +"""Read-only checks for the native server/service units, using a private fake system.""" +import json +import subprocess +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub + + +UNITS = ("server-hardening", "retired-services", "proxy", "tailscale-cert", "bst-dashboard", "kirocrew") + + +@pytest.fixture +def check_system(stub_env, tmp_path): + bindir, env = stub_env + root = tmp_path / "system" + repo = tmp_path / "repo" + units = repo / "units" + units.mkdir(parents=True) + (repo / "host_vars").mkdir() + (repo / "group_vars").mkdir() + (repo / "host_vars/workstation.yml").write_text("kirocrew_enabled: true\n") + (repo / "group_vars/all.yml").write_text("tailscale_domain: example.ts.net\n") + (repo / "inventory.yml").write_text(json.dumps({"all": {"children": { + "server": {"hosts": {"workstation": {}}}, "vps": {"hosts": {}} + }}})) + for name in ("lib.sh", "osquery_native_inventory.py", "server-hardening-file.py", + "service-templates.py", "kirocrew-sockets.py", "retired-services", "proxy", + "tailscale-cert", "bst-dashboard"): + (units / name).symlink_to(REPO / "units" / name) + for name in UNITS: + text = (REPO / "units" / f"{name}.sh").read_text() + for path in ("/etc", "/var/home/james/dev", "/run/systemd/system", + "/usr/lib/systemd/systemd-timesyncd", "/swapfile", "/var/run/reboot-required"): + text = text.replace(path, str(root / path.lstrip("/"))) + (units / f"{name}.sh").write_text(text) + for directory in ("etc/systemd", "etc/ssh", "run/systemd/system", "var/home/james/dev/dakota", + "etc/kirocrew", "etc/systemd/system", "usr/lib/systemd"): + (root / directory).mkdir(parents=True, exist_ok=True) + (root / "etc/systemd/journald.conf").write_text("#SystemMaxUse=1G\n") + (root / "etc/ssh/sshd_config").write_text("#MaxStartups 10:30:100\n") + (root / "etc/fstab").write_text("/dev/root / ext4 defaults 0 1\n") + (root / "usr/lib/systemd/systemd-timesyncd").touch() + (root / "etc/systemd/system/kirocrew.service").touch() + (root / "etc/kirocrew/kirocrew.env").write_text("DO_NOT_PRINT=this-is-a-secret\n") + (root / "etc/kirocrew/kirocrew.env").chmod(0o600) + home = Path(env["HOME"]) + (home / ".local/bin").mkdir(parents=True) + (home / ".local/bin/kirocrew").touch() + old = home / ".config/containers/systemd/grafana.container" + old.parent.mkdir(parents=True) + old.write_text("retired\n") + env.update(DOTFILES_DIR=str(repo), DOTFILES_MACHINE="workstation", UNITS_CHECK="1", + CALLS=str(tmp_path / "calls")) + write_stub(bindir, "sudo", '[ "$1" = -n ] && shift\nexec "$@"\n') + write_stub(bindir, "yq", ''' +case "$*" in + *retired_services_system_units*) ;; + *retired_services_user_units*) echo grafana.container ;; + *children.vps.hosts*) echo false ;; + *web_services*) echo '[]' ;; + *tailscale_domain*) echo example.ts.net ;; + *) exit 2 ;; +esac +''') + write_stub(bindir, "systemctl", ''' +echo "systemctl $*" >> "$CALLS" +[[ "$*" == *show* ]] || exit 99 +[ "${FAIL_SYSTEMCTL:-0}" != 1 ] || exit 1 +if [[ "$*" == *--value* ]]; then echo loaded; else + printf 'LoadState=loaded\\nActiveState=inactive\\nUnitFileState=disabled\\n' +fi +''') + write_stub(bindir, "tailscale", ''' +echo "tailscale $*" >> "$CALLS" +[ "$1 $2" = "serve status" ] || exit 99 +''') + write_stub(bindir, "loginctl", 'echo "loginctl $*" >> "$CALLS"\n[ "$1" = show-user ] || exit 99\necho no\n') + write_stub(bindir, "journalctl", '[ "$1" = --disk-usage ] || exit 99\necho 1G\n') + write_stub(bindir, "swapon", '[ "$1" = --show=NAME ] || exit 99\n') + write_stub(bindir, "ufw", '[ "$1" = status ] || exit 99\necho "Status: inactive"\n') + write_stub(bindir, "pro", '[ "$1" = status ] || exit 99\necho \'{"livepatch": "enabled"}\'\n') + write_stub(bindir, "ss", 'printf "%s\\n" "${SOCKETS:-}"\n') + for command in ("dd", "mkswap", "install", "chown", "chmod"): + write_stub(bindir, command, 'echo "FORBIDDEN $0 $*" >> "$CALLS"\nexit 99\n') + return repo, root, env, old + + +def run_check(check_system, name, **extra): + repo, _, env, _ = check_system + return subprocess.run(["bash", str(repo / "units" / f"{name}.sh")], + env={**env, **extra}, capture_output=True, text=True) + + +@pytest.mark.parametrize("unit,expected", [ + ("server-hardening", "would update journald limit"), + ("retired-services", "would stop grafana.service"), + ("proxy", "would deploy"), + ("tailscale-cert", "would provision Tailscale certificate"), + ("bst-dashboard", "would deploy dashboard user service"), + ("kirocrew", "would start kirocrew"), +]) +def test_check_reports_drift_without_mutation(check_system, unit, expected): + _, root, env, old = check_system + original = {p: p.read_bytes() for p in root.rglob("*") if p.is_file()} + result = run_check(check_system, unit) + assert result.returncode == 0, (result.stdout, result.stderr) + assert expected in result.stdout + assert "this-is-a-secret" not in result.stdout + result.stderr + assert {p: p.read_bytes() for p in root.rglob("*") if p.is_file()} == original + assert old.read_text() == "retired\n" + calls = Path(env["CALLS"]) + if calls.exists(): + for call in calls.read_text().splitlines(): + assert (call.startswith("systemctl ") and "show" in call or + call == "tailscale serve status" or call.startswith("loginctl show-user ")) + + +@pytest.mark.parametrize("unit", UNITS) +def test_check_fails_when_systemd_inspection_fails(check_system, unit): + result = run_check(check_system, unit, FAIL_SYSTEMCTL="1") + assert result.returncode != 0 + assert "FAIL" in result.stdout + + +@pytest.mark.parametrize("unit", UNITS) +def test_retired_vps_remain_untouched_in_check(check_system, unit): + result = run_check(check_system, unit, DOTFILES_MACHINE="matrix") + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert not Path(check_system[2]["CALLS"]).exists() + + +def test_kirocrew_rejects_non_loopback_listener(check_system): + line = 'LISTEN 0 128 0.0.0.0:8000 0.0.0.0:* users:(("kirocrew",pid=7,fd=5))' + result = run_check(check_system, "kirocrew", SOCKETS=line) + assert result.returncode != 0 + assert "listening beyond localhost" in result.stderr + + +def test_hardening_line_check_never_writes(tmp_path): + config = tmp_path / "sshd_config" + config.write_text("#MaxSessions 20\n") + command = ["python3", str(REPO / "units/server-hardening-file.py"), "--check", str(config), + "^#?MaxSessions", "MaxSessions 10"] + result = subprocess.run(command, capture_output=True, text=True) + assert result.returncode == 10 + assert config.read_text() == "#MaxSessions 20\n" diff --git a/tests/test_shell_ai_check_credentials.py b/tests/test_shell_ai_check_credentials.py new file mode 100644 index 00000000..cc1bd5a3 --- /dev/null +++ b/tests/test_shell_ai_check_credentials.py @@ -0,0 +1,24 @@ +from conftest import write_stub +from test_shell_ai_unit import env, home, run + + +def test_check_inspects_modes_without_reading_vault_or_writing_files(env): + sh = home(env) / ".config/shell/secrets.sh" + sh.parent.mkdir(parents=True) + sh.write_text("private-test-value\n") + sh.chmod(0o644) + write_stub(env["PATH"].split(":")[0], "bw", 'touch "$HOME/bw-called"; exit 1\n') + result = run(env, UNITS_CHECK="1") + assert result.returncode != 0 + assert "mode differs from 0600" in result.stdout + assert "audit events" in result.stdout + assert "private-test-value" not in result.stdout + result.stderr + assert sh.read_text() == "private-test-value\n" and sh.stat().st_mode & 0o777 == 0o644 + assert not (home(env) / "bw-called").exists() + assert not (home(env) / ".config/fish").exists() + + +def test_check_with_secrets_disabled_skips_vault(env): + result = run(env, UNITS_CHECK="1", UNITS_NO_SECRETS="1") + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert not (home(env) / ".config").exists() diff --git a/tests/test_ssh_check_credentials.py b/tests/test_ssh_check_credentials.py new file mode 100644 index 00000000..65dbbc5e --- /dev/null +++ b/tests/test_ssh_check_credentials.py @@ -0,0 +1,40 @@ +import pytest + +from test_ssh_native_units import execute, fleet, pytestmark + + +@pytest.mark.parametrize("mode", ["ssh-keys", "ssh-mesh"]) +def test_check_plans_local_ssh_drift_without_writes_or_secret_access(fleet, mode): + home = fleet["home"] + ssh = home / ".ssh" + ssh.mkdir(mode=0o755) + ssh.chmod(0o755) + private = ssh / "id_ed25519" + private.write_text(fleet["private"]) + private.chmod(0o644) + public = ssh / "id_ed25519.pub" + public.write_text(fleet["public"] + "\n") + result = execute(fleet, mode, UNITS_CHECK="1") + assert result.returncode != 0 + assert "would set" in result.stdout + assert fleet["private"] not in result.stdout + result.stderr + assert private.read_text() == fleet["private"] + assert private.stat().st_mode & 0o777 == 0o644 + assert ssh.stat().st_mode & 0o777 == 0o755 + assert not (ssh / ".dotfiles-ssh.lock").exists() + assert not (ssh / "config").exists() + assert not (ssh / "authorized_keys").exists() + assert not (ssh / "known_hosts").exists() + assert not (home / "bw-calls").exists() + assert not (home / "ssh-calls").exists() + if mode == "ssh-keys": + assert "audit events" in result.stdout and "would write" in result.stdout + else: + assert "authentication, retries and host-key refresh" in result.stdout + + +def test_missing_identity_check_does_not_read_vault_or_create_directory(fleet): + result = execute(fleet, "ssh-mesh", UNITS_CHECK="1") + assert result.returncode != 0 and "audit events" in result.stdout + assert not (fleet["home"] / ".ssh").exists() + assert not (fleet["home"] / "bw-calls").exists() diff --git a/tests/test_ssh_native_units.py b/tests/test_ssh_native_units.py new file mode 100644 index 00000000..ac134b9f --- /dev/null +++ b/tests/test_ssh_native_units.py @@ -0,0 +1,274 @@ +import base64 +import hashlib +import hmac +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub + + +pytestmark = pytest.mark.skipif(shutil.which("ssh-keygen") is None, reason="ssh-keygen required") + + +def make_key(path): + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "fixture", "-f", str(path)], check=True) + return path.read_text(), path.with_suffix(".pub").read_text().strip() + + +@pytest.fixture +def fleet(stub_env, tmp_path): + bindir, env = stub_env + home = Path(env["HOME"]) + data = tmp_path / "data" + data.mkdir() + private, public = make_key(tmp_path / "source") + other_private, other_public = make_key(tmp_path / "other") + variables = { + "user_login": "james", "user_email": "james@example.org", + "machines": {"newhost": {"hostname": "newhost"}, "remote": {"hostname": "peer"}, + "retired": {"hostname": "retired"}}, + "machine_pubkeys": {"newhost": public, "remote": other_public}, + "ssh_mesh_exclude": [], "extra_authorized_keys": [], + } + inventory = {"all": {"hosts": {"newhost": {}, "remote": {"ansible_port": 2222, "ansible_user": "alice", "tailnet_name": "peer-dns"}, "retired": {}}, + "children": {"vps": {"hosts": {"retired": {}}}}}} + (data / "all.yml.json").write_text(json.dumps(variables)) + (data / "inventory.yml.json").write_text(json.dumps(inventory)) + (data / "test-fleet-fedora.yml.json").write_text(json.dumps({"machine_profile": "vm-test"})) + item = {"id": "item-1", "type": 5, "name": "james@newhost", + "sshKey": {"privateKey": private, "publicKey": public}} + (home / "items.json").write_text(json.dumps([item])) + (home / "item.json").write_text(json.dumps(item)) + (home / "scan-key").write_text(" ".join(other_public.split()[:2])) + write_stub(bindir, "yq", 'cat "$SSH_FIXTURE/${3##*/}.json"\n') + write_stub(bindir, "bw", '''printf '%s\\n' "$*" >> "$HOME/bw-calls" +case "$1 $2" in + 'unlock --check') [ "$BW_SESSION" = 'secret-session' ] ;; + 'list items') [ "${BW_LIST_FAIL:-0}" != 1 ] && cat "$HOME/items.json" ;; + 'get item') cat "$HOME/item.json" ;; + 'encode ') base64 | tr -d '\\n' ;; + 'edit item'|'create item') cat > "$HOME/encoded-item" ;; + *) exit 1 ;; +esac +''') + write_stub(bindir, "ssh-keyscan", '''printf '%s\\n' "$*" >> "$HOME/scans" +[ "${SCAN_FAIL:-0}" != 1 ] || exit 1 +[ "${!#}" = peer ] || exit 1 +printf '[peer]:2222 %s\\n' "$(cat "$HOME/scan-key")" +''') + write_stub(bindir, "ssh", '''printf '%s\\n' "$*" >> "$HOME/ssh-calls" +[ "$1" = -O ] && exit 0 +[ "${SSH_FAIL:-0}" = 1 ] && exit 1 +if [ ! -e "$HOME/probed" ]; then touch "$HOME/probed"; exit 1; fi +exit 0 +''') + write_stub(bindir, "tailscale", '''printf '%s\\n' '{"Peer":{"one":{"Online":true,"DNSName":"peer-dns.example."},"two":{"Online":true,"DNSName":"retired.example."}}}' +''') + env.update(DOTFILES_DIR=str(REPO), DOTFILES_MACHINE="newhost", SSH_FIXTURE=str(data), + BW_SESSION="secret-session", BREW_PREFIX="/nonexistent") + return {"env": env, "home": home, "data": data, "private": private, "public": public, + "other_private": other_private, "other_public": other_public, "variables": variables} + + +def execute(fleet, mode="ssh-keys", **env): + return subprocess.run(["bash", str(REPO / f"units/{mode}.sh")], env={**fleet["env"], **env}, + text=True, capture_output=True, timeout=30) + + +def test_restores_vault_identity_preserves_user_files_and_idempotency(fleet): + home = fleet["home"] + ssh = home / ".ssh" + ssh.mkdir() + (ssh / "config").write_text("Host personal\n User custom\n") + (ssh / "authorized_keys").write_text("# my unmanaged key\n") + (ssh / "allowed_signers").write_text("other namespaces=\"git\" ssh-ed25519 untouched\n") + salt = b"fixed-test-salt" + digest = hmac.new(salt, b"[peer]:2222", hashlib.sha1).digest() + hashed = "|1|" + base64.b64encode(salt).decode() + "|" + base64.b64encode(digest).decode() + (ssh / "known_hosts").write_text(f"{hashed} {fleet['public']}\n[peer]:2222,unmanaged {fleet['public']}\n") + result = execute(fleet) + assert result.returncode == 0, result.stdout + result.stderr + assert (ssh / "id_ed25519").read_text() == fleet["private"] + assert (ssh / "id_ed25519.pub").read_text().strip() == fleet["public"] + assert (ssh / "id_ed25519").stat().st_mode & 0o777 == 0o600 + assert (ssh / "authorized_keys").stat().st_mode & 0o777 == 0o600 + assert "Host personal\n User custom\n" in (ssh / "config").read_text() + assert "Host remote\n HostName peer\n Port 2222\n User alice\n" in (ssh / "config").read_text() + assert "# my unmanaged key" in (ssh / "authorized_keys").read_text() + assert fleet["public"] in (ssh / "authorized_keys").read_text() + assert "other namespaces=" in (ssh / "allowed_signers").read_text() + known = (ssh / "known_hosts").read_text() + assert hashed not in known and "unmanaged " + fleet["public"] in known + assert "[peer]:2222 " + " ".join(fleet["other_public"].split()[:2]) in known + calls = (home / "bw-calls").read_text() + assert "secret-session" not in calls and "secret-session" not in result.stdout + result.stderr + assert fleet["private"] not in result.stdout + result.stderr + paths = [ssh / name for name in ("config", "authorized_keys", "allowed_signers", "known_hosts")] + before = [(p.read_text(), p.stat().st_mtime_ns) for p in paths] + again = execute(fleet) + assert again.returncode == 0, again.stdout + again.stderr + assert [(p.read_text(), p.stat().st_mtime_ns) for p in paths] == before + + +def test_existing_mismatch_never_overwrites_vault_without_force(fleet): + ssh = fleet["home"] / ".ssh" + ssh.mkdir() + (ssh / "id_ed25519").write_text(fleet["other_private"]) + (ssh / "id_ed25519").chmod(0o600) + (ssh / "id_ed25519.pub").write_text(fleet["other_public"] + "\n") + first = execute(fleet, UNITS_NO_SECRETS="1") + assert first.returncode == 0, first.stdout + first.stderr + assert (ssh / "id_ed25519").read_text() == fleet["other_private"] + local = (ssh / "id_ed25519.pub").read_text().split()[:2] + assert local != fleet["public"].split()[:2] + second = execute(fleet) + assert second.returncode == 0, second.stdout + second.stderr + assert "vault left unchanged" in second.stdout + assert not (fleet["home"] / "encoded-item").exists() + third = execute(fleet, SSH_KEYS_FORCE_PUSH="true") + assert third.returncode == 0, third.stdout + third.stderr + item = json.loads(base64.b64decode((fleet["home"] / "encoded-item").read_bytes())) + assert item["sshKey"]["publicKey"].split() == local + assert item["sshKey"]["privateKey"] == (ssh / "id_ed25519").read_text() + + +def test_unavailable_vault_listing_never_creates_identity(fleet): + result = execute(fleet, BW_LIST_FAIL="1") + assert result.returncode != 0 + assert "listing failed" in result.stdout + assert not (fleet["home"] / ".ssh/id_ed25519").exists() + assert (fleet["home"] / ".ssh/config").exists() + assert not (fleet["home"] / "encoded-item").exists() + + +@pytest.mark.parametrize("mode", ["ssh-keys", "ssh-mesh"]) +@pytest.mark.parametrize("disabled", [True, False]) +def test_missing_identity_waits_for_unlocked_vault(fleet, mode, disabled): + kwargs = {"UNITS_NO_SECRETS": "1"} if disabled else {"BW_SESSION": "locked-session"} + first = execute(fleet, mode, **kwargs) + assert first.returncode == 0, first.stdout + first.stderr + assert "PARTIAL: SSH identity deferred" in first.stdout + ssh = fleet["home"] / ".ssh" + assert not (ssh / "id_ed25519").exists() + assert not (ssh / "id_ed25519.pub").exists() + assert not (fleet["home"] / "encoded-item").exists() + assert not (fleet["home"] / "ssh-calls").exists() + if mode == "ssh-mesh": + assert fleet["other_public"] in (ssh / "authorized_keys").read_text() + second = execute(fleet, mode) + assert second.returncode == 0, second.stdout + second.stderr + assert (ssh / "id_ed25519").read_text() == fleet["private"] + + +def test_mesh_uses_vault_key_merges_git_keys_and_excludes_vps(fleet): + home = fleet["home"] + ssh = home / ".ssh" + ssh.mkdir() + (ssh / "authorized_keys").write_text("# unmanaged\n") + result = execute(fleet, "ssh-mesh") + assert result.returncode == 0, result.stdout + result.stderr + assert (ssh / "id_ed25519").read_text() == fleet["private"] + keys = (ssh / "authorized_keys").read_text() + assert "# unmanaged" in keys and fleet["other_public"] in keys + calls = (home / "ssh-calls").read_text() + assert calls.count("alice@peer true") == 2 + assert "-O exit -p 2222 alice@peer" in calls + assert "retired" not in calls + assert "[peer]:2222" in (ssh / "known_hosts").read_text() + + +def test_mesh_failed_scan_preserves_known_hosts_and_reports_failure(fleet): + ssh = fleet["home"] / ".ssh" + ssh.mkdir() + (ssh / "id_ed25519").write_text(fleet["private"]) + (ssh / "id_ed25519").chmod(0o600) + (ssh / "id_ed25519.pub").write_text(fleet["public"] + "\n") + old = "[peer]:2222 " + fleet["public"] + "\n" + (ssh / "known_hosts").write_text(old) + result = execute(fleet, "ssh-mesh", UNITS_NO_SECRETS="1", SSH_FAIL="1", SCAN_FAIL="1") + assert result.returncode != 0 + assert "SSH mesh broken to: remote" in result.stdout + assert (ssh / "known_hosts").read_text() == old + + +def test_profile_skip_does_not_create_key_or_contact_vault(fleet): + result = execute(fleet, "ssh-mesh", DOTFILES_MACHINE="test-fleet-fedora") + assert result.returncode == 0 + assert "SKIPPED" in result.stdout + assert not (fleet["home"] / ".ssh/id_ed25519").exists() + assert not (fleet["home"] / "bw-calls").exists() + + +def test_parallel_units_restore_one_identity_and_preserve_both_key_sources(fleet): + env = fleet["env"] + units = [subprocess.Popen(["bash", str(REPO / f"units/{name}.sh")], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + for name in ("ssh-keys", "ssh-mesh")] + for unit in units: + stdout, stderr = unit.communicate(timeout=30) + assert unit.returncode == 0, stdout + stderr + ssh = fleet["home"] / ".ssh" + assert (ssh / "id_ed25519").read_text() == fleet["private"] + authorized = (ssh / "authorized_keys").read_text() + assert fleet["public"] in authorized and fleet["other_public"] in authorized + + +def test_missing_vault_item_is_created_and_authorized_immediately(fleet): + home = fleet["home"] + (home / "items.json").write_text("[]") + result = execute(fleet) + assert result.returncode == 0, result.stdout + result.stderr + item = json.loads(base64.b64decode((home / "encoded-item").read_bytes())) + assert item["name"] == "james@newhost" and item["type"] == 5 + local = (home / ".ssh/id_ed25519.pub").read_text().split()[:2] + assert item["sshKey"]["publicKey"].split() == local + assert item["sshKey"]["publicKey"] in (home / ".ssh/authorized_keys").read_text() + + +def test_partial_vault_never_shrinks_keys_without_explicit_force(fleet): + home = fleet["home"] + own = json.loads((home / "item.json").read_text()) + remote = {"id": "remote-1", "name": "james@remote", "sshKey": {"publicKey": fleet["other_public"]}} + (home / "items.json").write_text(json.dumps([own, remote])) + ssh = home / ".ssh" + ssh.mkdir() + (ssh / "authorized_keys").write_text('# user-managed access\n') + first = execute(fleet) + assert first.returncode == 0, first.stdout + first.stderr + (home / "items.json").write_text(json.dumps([own])) + second = execute(fleet) + assert second.returncode == 0, second.stdout + second.stderr + assert fleet["other_public"] in (ssh / "authorized_keys").read_text() + forced = execute(fleet, SSH_KEYS_FORCE_AUTHORIZED="true") + assert forced.returncode == 0, forced.stdout + forced.stderr + content = (ssh / "authorized_keys").read_text() + assert fleet["other_public"] not in content + assert "# user-managed access" in content and fleet["public"] in content + + +def test_mesh_does_not_consult_vault_when_identity_exists(fleet): + ssh = fleet["home"] / ".ssh" + ssh.mkdir() + (ssh / "id_ed25519").write_text(fleet["private"]) + (ssh / "id_ed25519").chmod(0o600) + (ssh / "id_ed25519.pub").write_text(fleet["public"] + "\n") + result = execute(fleet, "ssh-mesh", BW_LIST_FAIL="1") + assert result.returncode == 0, result.stdout + result.stderr + assert not (fleet["home"] / "bw-calls").exists() + + +def test_invalid_vault_key_pair_is_not_installed_or_logged(fleet): + home = fleet["home"] + item = json.loads((home / "item.json").read_text()) + item["sshKey"]["publicKey"] = fleet["other_public"] + (home / "item.json").write_text(json.dumps(item)) + (home / "items.json").write_text(json.dumps([item])) + result = execute(fleet) + assert result.returncode != 0 + assert "does not match" in result.stdout + assert not (home / ".ssh/id_ed25519").exists() + assert fleet["private"] not in result.stdout + result.stderr diff --git a/tests/test_sudo_sshd_check_units.py b/tests/test_sudo_sshd_check_units.py new file mode 100644 index 00000000..bbb85af2 --- /dev/null +++ b/tests/test_sudo_sshd_check_units.py @@ -0,0 +1,111 @@ +"""Read-only checks for the native sudo and SSH daemon units.""" + +import os +import subprocess +from pathlib import Path + +from conftest import write_stub +from test_platform_native_units import platform_env # noqa: F401 + + +def run_check(fixture, name, **updates): + root, _, _, env, _ = fixture + return subprocess.run( + ["bash", str(root / "units" / f"{name}.sh")], + env={**env, "UNITS_CHECK": "1", **updates}, text=True, capture_output=True, + ) + + +def snapshot(path): + return { + str(p.relative_to(path)): ( + "link", os.readlink(p) + ) if p.is_symlink() else ( + "file", p.read_bytes(), p.stat().st_mode + ) if p.is_file() else ("dir", p.stat().st_mode) + for p in path.rglob("*") + } + + +def test_sudo_check_reports_install_and_legacy_removal_without_writes(platform_env): + _, system, _, env, log = platform_env + directory = system / "etc/sudoers.d" + (directory / "james").write_text("old policy\n") + (directory / "zz-james").symlink_to("missing") + before = snapshot(system) + temp_before = set(Path(env["TMPDIR"]).iterdir()) + result = run_check(platform_env, "sudo") + assert result.returncode == 0, result.stderr + assert "would validate sudo policy" in result.stdout + assert "would install sudo policy" in result.stdout + assert "would remove legacy sudo policy" in result.stdout + assert snapshot(system) == before + assert set(Path(env["TMPDIR"]).iterdir()) == temp_before + assert not log.exists() + + +def test_sudo_check_detects_clean_policy_and_inspection_failures(platform_env): + _, system, bindir, _, _ = platform_env + dest = system / "etc/sudoers.d/zz-james" + dest.write_text("james ALL=(ALL) NOPASSWD:ALL\n") + dest.chmod(0o440) + result = run_check(platform_env, "sudo") + assert result.returncode == 0 and "ok sudo policy" in result.stdout + assert "would install" not in result.stdout + write_stub(bindir, "sudo", 'echo "inspection denied" >&2\nexit 1\n') + result = run_check(platform_env, "sudo") + assert result.returncode != 0 and "inspect sudo policy" in result.stdout + assert "would install" not in result.stdout + + +def test_sudo_check_rejects_unreadable_comparison(platform_env): + _, system, bindir, _, _ = platform_env + dest = system / "etc/sudoers.d/zz-james" + dest.write_text("policy\n") + write_stub(bindir, "cmp", 'exit 2\n') + result = run_check(platform_env, "sudo") + assert result.returncode != 0 + assert "Cannot read sudo policy" in result.stderr + + +def test_sshd_check_reports_both_dropins_and_reload_without_writes(platform_env): + root, system, _, env, log = platform_env + (root / "host_vars/laptop.yml").write_text("sshd_harden: true\n") + directory = system / "etc/ssh/sshd_config.d" + (directory / "99-bw-env.conf").write_text("obsolete\n") + before = snapshot(system) + temp_before = set(Path(env["TMPDIR"]).iterdir()) + result = run_check(platform_env, "sshd") + assert result.returncode == 0, result.stderr + assert "would install sshd drop-in" in result.stdout + assert "99-bw-env.conf" in result.stdout and "10-hardening.conf" in result.stdout + assert "would validate sshd drop-in" in result.stdout + assert "would reload sshd" in result.stdout + assert snapshot(system) == before + assert set(Path(env["TMPDIR"]).iterdir()) == temp_before + assert not log.exists() + + +def test_sshd_check_clean_retired_and_inspection_error(platform_env): + root, system, bindir, _, log = platform_env + directory = system / "etc/ssh/sshd_config.d" + dest = directory / "99-bw-env.conf" + dest.write_text("AcceptEnv BW_SESSION BW_CLIENTID BW_CLIENTSECRET\n") + dest.chmod(0o644) + clean = run_check(platform_env, "sshd") + assert clean.returncode == 0 and "ok sshd drop-in" in clean.stdout + assert "would install" not in clean.stdout and "would reload" not in clean.stdout + dest.write_text("out of date\n") + retired = run_check(platform_env, "sshd", DOTFILES_MACHINE="matrix") + assert retired.returncode == 0 and "SKIPPED: sshd reload on retired VPS" in retired.stdout + assert not log.exists() + write_stub(bindir, "cmp", 'exit 2\n') + failed = run_check(platform_env, "sshd") + assert failed.returncode != 0 and "Cannot read sshd drop-in" in failed.stderr + + +def test_sshd_check_absent_dropin_directory_matches_apply_skip(platform_env): + _, system, _, _, _ = platform_env + (system / "etc/ssh/sshd_config.d").rmdir() + result = run_check(platform_env, "sshd") + assert result.returncode == 0 and "SKIPPED" in result.stdout diff --git a/tests/test_syncthing_migration.py b/tests/test_syncthing_migration.py new file mode 100644 index 00000000..4d0b80ba --- /dev/null +++ b/tests/test_syncthing_migration.py @@ -0,0 +1,53 @@ +import shutil +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub +from unit_helpers import calls, run_unit + + +def syncthing_repo(tmp_path, host, variables): + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "host_vars" / f"{host}.yml").write_text(variables) + (repo / "units").symlink_to(REPO / "units") + (repo / "inventory.yml").write_text("all:\n children:\n vps:\n hosts:\n matrix: {}\n telengana: {}\n") + return repo + + +@pytest.mark.parametrize("host,variables", [("matrix", ""), ("laptop", "skip_syncthing: true\n"), ("vm", "machine_profile: vm-test\n")]) +def test_excluded_hosts_never_change_service(stub_env, tmp_path, host, variables): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = syncthing_repo(tmp_path, host, variables) + unit = Path(env["HOME"]) / ".config/systemd/user/syncthing.service" + unit.parent.mkdir(parents=True) + unit.write_text("existing") + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE=host) + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert unit.read_text() == "existing" and calls(env) == "" + + +@pytest.mark.skipif(shutil.which("yq") is None or Path("/home/linuxbrew/.linuxbrew/bin/syncthing").exists(), reason="requires yq and no installed syncthing") +def test_missing_binary_disable_failure_is_reported(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\nexit 1\n') + repo = syncthing_repo(tmp_path, "laptop", "") + unit = Path(env["HOME"]) / ".config/systemd/user/syncthing.service" + unit.parent.mkdir(parents=True) + unit.write_text("existing") + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop") + assert result.returncode != 0 + assert "disable --now syncthing.service" in calls(env) + assert unit.read_text() == "existing" + + +@pytest.mark.skipif(shutil.which("yq") is None or Path("/home/linuxbrew/.linuxbrew/bin/syncthing").exists(), reason="requires yq and no installed syncthing") +def test_missing_binary_stops_system_wide_user_service(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\nif [[ "$*" == *"show --property=LoadState"* ]]; then echo loaded; fi\n') + repo = syncthing_repo(tmp_path, "laptop", "") + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop") + assert result.returncode == 0, result.stdout + result.stderr + assert "disable --now syncthing.service" in calls(env) diff --git a/tests/test_system_services_migration.py b/tests/test_system_services_migration.py new file mode 100644 index 00000000..1e1c67b6 --- /dev/null +++ b/tests/test_system_services_migration.py @@ -0,0 +1,127 @@ +import subprocess +from pathlib import Path + +from conftest import REPO, write_stub +from unit_helpers import calls + + +def setup_system_services(tmp_path, env, host, variables, podman): + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "host_vars" / f"{host}.yml").write_text(variables) + (repo / "units").mkdir() + (repo / "units/system-services").symlink_to(REPO / "units/system-services") + units = tmp_path / "units" + units.mkdir() + (units / "lib.sh").symlink_to(REPO / "units/lib.sh") + binary = tmp_path / "podman" + if podman: + binary.touch() + dropin = tmp_path / "systemd/podman.socket.d" + script = (REPO / "units/system-services.sh").read_text() + script = script.replace("/usr/bin/podman", str(binary)) + script = script.replace("/etc/systemd/system/podman.socket.d", str(dropin)) + program = units / "system-services.sh" + program.write_text(script) + environment = {**env, "DOTFILES_DIR": str(repo), "DOTFILES_MACHINE": host, "STUB_STATE": str(tmp_path)} + return program, environment, dropin + + +def run(program, env): + return subprocess.run(["bash", str(program)], env=env, text=True, capture_output=True) + + +def systemctl_stub(bindir): + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\nif [ "$1" = is-active ]; then exit "${SOCKET_ACTIVE:-0}"; fi\n') + + +def test_nonlaptop_daily_timer_and_user_service_are_idempotent(stub_env, tmp_path): + bindir, env = stub_env + systemctl_stub(bindir) + program, env, _ = setup_system_services(tmp_path, env, "server", "dotfiles_update_delay_seconds: 45\n", False) + result = run(program, env) + assert result.returncode == 0, result.stdout + result.stderr + units = Path(env["HOME"]) / ".config/systemd/user" + service = (units / "dotfiles-update.service").read_text() + timer = (units / "dotfiles-update.timer").read_text() + assert "ExecStartPre=/bin/sleep 45" in service + assert "WantedBy=default.target" in service + assert "graphical-session" not in service + assert "apply-nosecrets" in service + assert "OnCalendar=daily" in timer and "RandomizedDelaySec=30min" in timer + assert "--user enable --now dotfiles-update.timer" in calls(env) + assert "--user enable --now atuin-daemon.service" in calls(env) + (Path(env["HOME"]) / "calls").unlink() + result = run(program, env) + assert result.returncode == 0, result.stdout + result.stderr + assert "daemon-reload" not in calls(env) + assert "restart" not in calls(env) + + +def test_missing_template_does_not_start_stale_update_timer(stub_env, tmp_path): + bindir, env = stub_env + systemctl_stub(bindir) + program, env, _ = setup_system_services(tmp_path, env, "server", "", False) + (Path(env["DOTFILES_DIR"]) / "units/system-services").unlink() + result = run(program, env) + assert result.returncode != 0 + assert "enable --now dotfiles-update.timer" not in calls(env) + + +def test_podman_group_membership_socket_and_no_redundant_restart(stub_env, tmp_path): + bindir, env = stub_env + systemctl_stub(bindir) + write_stub(bindir, "id", '''case "$1" in +-u) echo 0 ;; +-un) echo tester ;; +-nG) if [ -e "$STUB_STATE/member" ]; then echo "users podman"; else echo users; fi ;; +esac +''') + write_stub(bindir, "getent", '[ -e "$STUB_STATE/group" ] || exit 2\n') + write_stub(bindir, "groupadd", 'echo "groupadd $*" >> "$HOME/calls"\ntouch "$STUB_STATE/group"\n') + write_stub(bindir, "usermod", 'echo "usermod $*" >> "$HOME/calls"\ntouch "$STUB_STATE/member"\n') + program, env, dropin = setup_system_services(tmp_path, env, "laptop", "is_laptop: true\n", True) + result = run(program, env) + assert result.returncode == 0, result.stdout + result.stderr + assert (dropin / "group-access.conf").read_text() == "[Socket]\nSocketGroup=podman\nSocketMode=0660\nDirectoryMode=0710\n" + assert "groupadd --system podman" in calls(env) + assert "usermod -a -G podman tester" in calls(env) + assert "restart podman.socket" in calls(env) + assert "--user" not in calls(env) + (Path(env["HOME"]) / "calls").unlink() + result = run(program, env) + assert result.returncode == 0, result.stdout + result.stderr + assert "groupadd" not in calls(env) and "usermod" not in calls(env) + assert "restart podman.socket" not in calls(env) + + +def test_podman_privilege_failure_does_not_suppress_user_services(stub_env, tmp_path): + bindir, env = stub_env + systemctl_stub(bindir) + write_stub(bindir, "id", '''case "$1" in +-u) echo 1000 ;; +-un) echo tester ;; +-nG) echo users ;; +esac +''') + write_stub(bindir, "getent", "exit 2\n") + write_stub(bindir, "sudo", "exit 1\n") + program, env, _ = setup_system_services(tmp_path, env, "server", "", True) + result = run(program, env) + assert result.returncode != 0 + assert "FAIL configure system podman socket" in result.stdout + assert "--user enable --now dotfiles-update.timer" in calls(env) + assert "--user enable --now atuin-daemon.service" in calls(env) + + +def test_retired_vps_and_skip_systemd_never_touch_services(stub_env, tmp_path): + bindir, env = stub_env + systemctl_stub(bindir) + program, env, dropin = setup_system_services(tmp_path, env, "matrix", "", True) + result = run(program, env) + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert calls(env) == "" and not dropin.exists() + (Path(env["DOTFILES_DIR"]) / "host_vars/server.yml").write_text("skip_systemd: true\n") + result = run(program, {**env, "DOTFILES_MACHINE": "server"}) + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert calls(env) == "" and not dropin.exists() diff --git a/tests/test_tagged_apply.py b/tests/test_tagged_apply.py index f4684ec7..5b7dfe04 100644 --- a/tests/test_tagged_apply.py +++ b/tests/test_tagged_apply.py @@ -31,9 +31,7 @@ def apply_env(stub_env, tmp_path): *) exit 1 ;; esac ''') - write_stub(bindir, "ansible-playbook", 'printf "ansible %s\\n" "$*" >> "$CALLS"\nexit "${ANSIBLE_RC:-0}"\n') write_stub(bindir, "git", 'exit 0\n') - write_stub(bindir, "ansible-galaxy", 'exit 0\n') write_stub(bindir, "sudo", 'exit 0\n') write_stub(bindir, "ssh", 'printf "ssh\\n" >> "$CALLS"\n') write_stub(scripts, "bw-resolve.sh", ''' @@ -50,14 +48,13 @@ def run_apply(env, repo, *args): return subprocess.run(["just", "-f", str(repo / "Justfile"), *args], env=env, capture_output=True, text=True) -def test_secret_tag_validates_syncs_and_initializes_legacy_roles(apply_env): +def test_secret_tag_validates_syncs_and_does_not_expose_session(apply_env): env, repo, log = apply_env result = run_apply(env, repo, "apply-tags", "omp,kube") assert result.returncode == 0, result.stderr calls = log.read_text().splitlines() assert calls[:4] == ["bw unlock --check", "resolve", "bw unlock --check", "bw sync"] - assert "--tags omp,kube,bitwarden" in calls[4] - assert calls[5] == "units --tags omp,kube" + assert calls[4] == "units --tags omp,kube" assert "valid-session" not in result.stdout + result.stderr + "\n".join(calls) @@ -66,9 +63,7 @@ def test_nonsecret_tag_does_not_prompt_or_sync(apply_env): result = run_apply(env, repo, "apply-tags", "git") assert result.returncode == 0, result.stderr calls = log.read_text().splitlines() - assert len(calls) == 2 - assert "--tags git" in calls[0] - assert calls[1] == "units --tags git" + assert calls == ["units --tags git"] def test_remote_nonsecret_tag_does_not_resolve_vault(apply_env): @@ -84,7 +79,6 @@ def test_failed_sync_runs_nonsecret_work_and_fails(apply_env): result = run_apply(env, repo, "apply-tags", "omp,kube") assert result.returncode != 0 calls = log.read_text().splitlines() - assert "--skip-tags secrets" in calls[-2] assert calls[-1] == "units --tags omp,kube --no-secrets" @@ -94,7 +88,6 @@ def test_full_apply_skips_vault_units_when_unlock_fails(apply_env): result = run_apply(env, repo, "apply") assert result.returncode == 0, result.stderr calls = log.read_text().splitlines() - assert "--skip-tags secrets" in calls[-2] assert calls[-1] == "units --no-secrets" diff --git a/tests/test_unit_tags.py b/tests/test_unit_tags.py new file mode 100644 index 00000000..1b83b71e --- /dev/null +++ b/tests/test_unit_tags.py @@ -0,0 +1,17 @@ +import subprocess + +import pytest + +from conftest import REPO, write_stub + + +@pytest.mark.parametrize("tags", ["", ",git", "git,", "git,,shell", "not-a-real-tag"]) +def test_invalid_selection_never_runs_units(stub_env, tmp_path, tags): + bindir, env = stub_env + marker = tmp_path / "mise-called" + write_stub(bindir, "mise", 'touch "$MARKER"\n') + env["MARKER"] = str(marker) + result = subprocess.run(["bash", str(REPO / "scripts/run-units.sh"), "--tags", tags], + env=env, capture_output=True, text=True, timeout=15) + assert result.returncode == 2 + assert not marker.exists() diff --git a/tests/test_units_registry.py b/tests/test_units_registry.py deleted file mode 100644 index c1522032..00000000 --- a/tests/test_units_registry.py +++ /dev/null @@ -1,36 +0,0 @@ -import re -import tomllib - -from conftest import REPO - - -def mise_units(): - tasks = tomllib.loads((REPO / "mise.toml").read_text())["tasks"] - return {k.removeprefix("unit:"): v for k, v in tasks.items() if k.startswith("unit:")} - - -def wrapper(): - return (REPO / "scripts/run-units.sh").read_text() - - -def all_units(): - return set(re.search(r'^ALL_UNITS="([^"]*)"', wrapper(), re.M).group(1).split()) - - -def test_every_unit_is_registered_everywhere(): - on_disk = {p.stem for p in (REPO / "units").glob("*.sh")} - {"lib"} - assert set(mise_units()) == on_disk == all_units() - - -def test_mise_task_files_exist(): - for name, task in mise_units().items(): - assert task["file"] == f"units/{name}.sh" - assert (REPO / task["file"]).is_file() - - -def test_tag_map_only_names_known_units(): - case = re.search(r"tasks_for_tag\(\) \{ case .*?esac; \}", wrapper(), re.S).group(0) - named = set() - for echoed in re.findall(r'echo "?([a-z0-9 -]+?)"? ?;;', case): - named.update(echoed.split()) - assert named and named <= all_units() diff --git a/tests/test_visual_units_check.py b/tests/test_visual_units_check.py new file mode 100644 index 00000000..90d7c829 --- /dev/null +++ b/tests/test_visual_units_check.py @@ -0,0 +1,335 @@ +"""Read-only inspection coverage for the visual desktop units.""" +import json +import os +import stat +import sys +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub +from unit_helpers import run_unit + + +@pytest.fixture +def visual(stub_env, tmp_path): + bindir, env = stub_env + home = Path(env['HOME']) + repo = tmp_path / 'repo' + (repo / 'units').mkdir(parents=True) + (repo / 'host_vars').mkdir() + (repo / 'group_vars').mkdir() + for unit in ('bluefin', 'gnome', 'zen-browser'): + (repo / 'units' / unit).symlink_to(REPO / 'units' / unit, target_is_directory=True) + # Keep AccountsService inspection isolated from the host's real account. + (repo / 'units/gnome-avatar.py').write_text('''\ +import os +from pathlib import Path +ICON = Path(os.environ['HOME']) / 'account-icon' +ACCOUNT = Path(os.environ['HOME']) / 'account-data' +GDM_CACHE = Path(os.environ['HOME']) / 'gdm-cache' +def account_with_icon(text): + return '[User]\\nIcon=/example\\n' +''') + system_policy = '/var/lib/flatpak/app/app.zen_browser.zen/current/active/files/zen/distribution/policies.json' + bluefin_wrapper = tmp_path / 'bluefin-check-wrapper.py' + bluefin_wrapper.write_text('''\ +import os +import sys +program = sys.stdin.read().replace("Path('/etc/sysctl.d/99-zram.conf')", 'Path(' + repr(os.environ['CHECK_ZRAM']) + ')') +exec(compile(program, '', 'exec')) +''') + write_stub(bindir, 'python3', f'''\ +args=("$@") +if [ "${{args[0]:-}}" = - ] && [ "${{args[1]:-}}" = "$DOTFILES_DIR/units/bluefin" ]; then + exec '{sys.executable}' "$CHECK_BLUEFIN_WRAPPER" "${{args[@]:1}}" +fi +if [ "${{args[1]:-}}" = '{system_policy}' ]; then args[1]="$CHECK_SYSTEM_POLICY"; fi +exec '{sys.executable}' "${{args[@]}}" +''') + write_stub(bindir, 'yq', '''\ +case "$*" in + *'.brew_prefix'*) echo /custom/brew ;; + *'.gnome_enabled_extensions'*) echo '[]' ;; + *'has('*) echo true ;; + *) exit 1 ;; +esac +''') + write_stub(bindir, 'grep', '''\ +if [ "${2:-}" = Bluefin ] && [ "${3:-}" = /etc/os-release ]; then exit 1; fi +exec /usr/bin/grep "$@" +''') + write_stub(bindir, 'gdbus', '[ "${FAIL_SESSION:-0}" = 0 ]\n') + write_stub(bindir, 'gsettings', '''\ +echo "gsettings $*" >> "$CHECK_LOG" +[ "${FAIL_GSETTINGS:-0}" = 0 ] || exit 1 +case "$*" in + 'get org.gnome.desktop.interface color-scheme') echo "'prefer-dark'" ;; + 'get org.gnome.Ptyxis default-profile-uuid') echo "'profile'" ;; + 'get org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/profile/ custom-command') echo "'/custom/brew/bin/fish'" ;; + 'get org.gnome.Ptyxis font-name') echo "'JetBrainsMono Nerd Font 12'" ;; + 'get org.gnome.shell enabled-extensions') echo '@as []' ;; + 'get org.gnome.shell favorite-apps') if [ -n "${CHECK_DOCK:-}" ]; then echo "$CHECK_DOCK"; else echo "['org.mozilla.firefox.desktop']"; fi ;; + 'get org.gnome.desktop.background picture-uri'|'get org.gnome.desktop.background picture-uri-dark') echo "''" ;; + *) exit 91 ;; +esac +''') + write_stub(bindir, 'dconf', '''\ +echo "dconf $*" >> "$CHECK_LOG" +[ "$1" = dump ] || exit 91 +[ -n "${CHECK_DCONF:-}" ] && cat "$CHECK_DCONF" +exit 0 +''') + write_stub(bindir, 'systemctl', '''\ +echo "systemctl $*" >> "$CHECK_LOG" +[ "$1 $2" = '--user show' ] || exit 91 +[ "${FAIL_SYSTEMCTL:-0}" = 0 ] || exit 1 +printf 'LoadState=loaded\\nUnitFileState=disabled\\nActiveState=inactive\\n' +''') + write_stub(bindir, 'gnome-extensions', '''\ +echo "gnome-extensions $*" >> "$CHECK_LOG" +[ "$1" = list ] || exit 91 +''') + write_stub(bindir, 'grdctl', '''\ +echo "grdctl $*" >> "$CHECK_LOG" +[ "$*" = 'rdp status' ] || exit 91 +printf 'Status: enabled\\nPassword: private-test-password\\n' +''') + write_stub(bindir, 'xdg-mime', '''\ +echo "xdg-mime $*" >> "$CHECK_LOG" +[ "$1 $2" = 'query default' ] || exit 91 +[ "${FAIL_MIME:-0}" = 0 ] || exit 1 +echo "${CHECK_MIME:-other.desktop}" +''') + for name in ('curl', 'wget', 'podman', 'unzip', 'sudo', 'install', 'fc-cache', 'sysctl', 'mktemp', + 'mkdir', 'chmod', 'rm', 'cp', 'mv', 'touch'): + write_stub(bindir, name, 'echo "FORBIDDEN $0 $*" >> "$CHECK_LOG"\nexit 91\n') + env.update(DOTFILES_DIR=str(repo), DOTFILES_MACHINE='visual-check', DOTFILES_IS_DESKTOP='1', + UNITS_CHECK='1', DBUS_SESSION_BUS_ADDRESS='unix:path=/test/bus', + CHECK_LOG=str(tmp_path / 'calls'), CHECK_SYSTEM_POLICY=str(tmp_path / 'system-policy.json'), + CHECK_ZRAM=str(tmp_path / 'zram.conf'), CHECK_BLUEFIN_WRAPPER=str(bluefin_wrapper)) + return env, repo + + +def inspect(unit, env, **extra): + return run_unit(unit, env, DOTFILES_DIR=env['DOTFILES_DIR'], **extra) + + +def snapshot(path): + result = {} + for entry in [path, *sorted(path.rglob('*'))]: + info = entry.lstat() + result[str(entry.relative_to(path))] = ( + stat.S_IMODE(info.st_mode), info.st_mtime_ns, + entry.read_bytes() if entry.is_file() else None, + os.readlink(entry) if entry.is_symlink() else None, + ) + return result + + +def assert_read_only(env, before): + assert snapshot(Path(env['HOME'])) == before + assert not (Path(env['DOTFILES_DIR']) / 'units/__pycache__').exists() + log = Path(env['CHECK_LOG']) + text = log.read_text() if log.exists() else '' + assert 'FORBIDDEN' not in text + assert 'gsettings set ' not in text + assert 'dconf load ' not in text + assert 'systemctl --user enable' not in text + assert 'grdctl rdp enable' not in text + assert 'xdg-mime default' not in text + + +def test_bluefin_reports_local_and_offline_plans_without_mutating(visual): + env, _ = visual + home = Path(env['HOME']) + old = home / '.local/bin/tailvm' + old.parent.mkdir(parents=True) + old.write_text('old binary') + before = snapshot(home) + result = inspect('bluefin', env) + assert result.returncode == 0, result.stdout + result.stderr + assert 'WOULD remove retired tailvm file' in result.stdout + assert 'WOULD update wallpaper rotation script' in result.stdout + assert 'WOULD enable and start wallpaper timer' in result.stdout + assert 'remote image and archives not inspected' in result.stdout + assert 'WOULD install GNOME extension' in result.stdout + assert_read_only(env, before) + + +def test_bluefin_inspection_errors_are_not_success(visual): + env, _ = visual + before = snapshot(Path(env['HOME'])) + result = inspect('bluefin', env, FAIL_SYSTEMCTL='1', DBUS_SESSION_BUS_ADDRESS='autolaunch:') + assert result.returncode != 0 + assert 'FAIL inspect GNOME session' in result.stdout + assert 'FAIL inspect wallpaper timer' in result.stdout + assert_read_only(env, before) + assert 'gsettings ' not in Path(env['CHECK_LOG']).read_text() + + +def test_gnome_reports_deployment_and_session_drift_without_mutation(visual): + env, _ = visual + home = Path(env['HOME']) + before = snapshot(home) + result = inspect('gnome', env) + assert result.returncode == 0, result.stdout + result.stderr + assert 'WOULD update GNOME keybindings file' in result.stdout + assert 'WOULD load GNOME keybindings' in result.stdout + assert 'WOULD create or fix extension' in result.stdout + assert 'WOULD load GNOME extension settings' in result.stdout + assert 'WOULD enable and start GNOME Remote Desktop' in result.stdout + assert 'remote image not inspected' in result.stdout + assert 'private-test-password' not in result.stdout + result.stderr + assert_read_only(env, before) + + +def test_gnome_session_errors_do_not_create_pending_marker(visual): + env, _ = visual + home = Path(env['HOME']) + before = snapshot(home) + result = inspect('gnome', env, FAIL_SESSION='1') + assert result.returncode != 0 + assert 'FAIL inspect GNOME user session' in result.stdout + assert not (home / '.config/dconf/.gnome-keybindings.pending').exists() + assert_read_only(env, before) + + +def test_gnome_invalid_dconf_does_not_disclose_values(visual, tmp_path): + env, _ = visual + data = tmp_path / 'broken-dconf' + data.write_text('[unclosed-secret-value\n') + before = snapshot(Path(env['HOME'])) + result = inspect('gnome', env, CHECK_DCONF=str(data)) + assert result.returncode != 0 + assert 'FAIL inspect GNOME user session' in result.stdout + assert 'unclosed-secret-value' not in result.stdout + result.stderr + assert_read_only(env, before) + assert 'systemctl --user show' in Path(env['CHECK_LOG']).read_text() + + +def zen_profile(home): + base = home / '.var/app/app.zen_browser.zen/.zen' + profile = base / 'abc.default' + profile.mkdir(parents=True) + (base / 'profiles.ini').write_text('[Profile0]\nIsRelative=1\nPath=abc.default\n') + return profile + + +def test_zen_reports_drift_without_writing_preferences_or_policies(visual): + env, _ = visual + home = Path(env['HOME']) + profile = zen_profile(home) + (profile / 'user.js').write_text('user_pref("local", true);\n') + before = snapshot(home) + result = inspect('zen-browser', env) + assert result.returncode == 0, result.stdout + result.stderr + for expected in ('WOULD update GNOME dock', 'WOULD set default browser', + 'WOULD update Zen profile preferences', 'WOULD update Flatpak Zen policy', + 'WOULD update user Zen policy'): + assert expected in result.stdout + assert_read_only(env, before) + + +def test_zen_converged_preferences_and_user_policy_are_recognized(visual): + env, _ = visual + home = Path(env['HOME']) + profile = zen_profile(home) + (profile / 'user.js').write_text((REPO / 'units/zen-browser/user.js').read_text()) + (profile / 'user.js').chmod(0o644) + policy = home / '.zen/distribution/policies.json' + policy.parent.mkdir(parents=True) + data = json.loads((REPO / 'units/zen-browser/policies.json').read_text()) + policy.write_text(json.dumps(data, indent=2, ensure_ascii=False) + '\n') + policy.chmod(0o644) + before = snapshot(home) + result = inspect('zen-browser', env, CHECK_DOCK="['app.zen_browser.zen.desktop']", + CHECK_MIME='app.zen_browser.zen.desktop') + assert result.returncode == 0, result.stdout + result.stderr + assert 'OK GNOME dock' in result.stdout + assert 'OK Zen profile preferences' in result.stdout + assert 'OK user Zen policy' in result.stdout + assert 'WOULD set default browser' not in result.stdout + assert_read_only(env, before) + + +def test_zen_reports_inspection_errors_without_exposing_policy(visual): + env, _ = visual + home = Path(env['HOME']) + profile = zen_profile(home) + (profile / 'user.js').write_text('// BEGIN dotfiles Zen preferences\nprivate-value\n') + policy = home / '.zen/distribution/policies.json' + policy.parent.mkdir(parents=True) + policy.write_text('private-policy-value') + before = snapshot(home) + result = inspect('zen-browser', env, FAIL_MIME='1') + assert result.returncode != 0 + assert 'FAIL inspect default browser' in result.stdout + assert 'FAIL Zen profile preferences' in result.stdout + assert 'FAIL user Zen policy' in result.stdout + assert 'private-value' not in result.stdout + result.stderr + assert 'private-policy-value' not in result.stdout + result.stderr + assert_read_only(env, before) + + +@pytest.mark.parametrize('unit', ['bluefin', 'gnome', 'zen-browser']) +def test_non_desktop_check_has_no_side_effects(visual, unit): + env, _ = visual + before = snapshot(Path(env['HOME'])) + result = inspect(unit, env, DOTFILES_IS_DESKTOP='0') + assert result.returncode == 0 + assert 'not a desktop' in result.stdout + assert_read_only(env, before) + assert not Path(env['CHECK_LOG']).exists() + + +def test_bluefin_missing_source_is_an_inspection_error(visual): + env, repo = visual + assets = repo / 'units/bluefin' + assets.unlink() + assets.mkdir() + before = snapshot(Path(env['HOME'])) + result = inspect('bluefin', env) + assert result.returncode != 0 + assert 'FAIL inspect Bluefin files and wallpaper' in result.stdout + assert_read_only(env, before) + + +def test_zen_missing_profile_still_inspects_dock_without_writes(visual): + env, _ = visual + before = snapshot(Path(env['HOME'])) + result = inspect('zen-browser', env) + assert result.returncode == 0 + assert 'WOULD update GNOME dock' in result.stdout + assert 'profiles.ini not found' in result.stdout + assert_read_only(env, before) + + +def test_zen_does_not_autolaunch_a_session(visual): + env, _ = visual + before = snapshot(Path(env['HOME'])) + result = inspect('zen-browser', env, DBUS_SESSION_BUS_ADDRESS='autolaunch:') + assert result.returncode != 0 + assert 'FAIL inspect GNOME dock' in result.stdout + assert_read_only(env, before) + assert not Path(env['CHECK_LOG']).exists() + + +def test_zen_gio_fallback_checks_default_without_changing_it(visual, tmp_path): + env, _ = visual + bindir = Path(env['PATH'].split(':')[0]) + write_stub(bindir, 'gio', '''\ +echo "gio $*" >> "$CHECK_LOG" +[ "$1" = mime ] && [ "$#" = 2 ] || exit 91 +printf 'Default application for “%s”: app.zen_browser.zen.desktop\\nRegistered applications:\\n' "$2" +''') + bash_env = tmp_path / 'no-xdg-mime.bash' + bash_env.write_text('command() { if [ "$1" = -v ] && [ "$2" = xdg-mime ]; then return 1; fi; builtin command "$@"; }\n') + home = Path(env['HOME']) + zen_profile(home) + before = snapshot(home) + result = inspect('zen-browser', env, BASH_ENV=str(bash_env)) + assert result.returncode == 0, result.stdout + result.stderr + assert 'WOULD set default browser' not in result.stdout + assert Path(env['CHECK_LOG']).read_text().count('gio mime ') == 7 + assert_read_only(env, before) diff --git a/tests/test_zen_browser_unit.py b/tests/test_zen_browser_unit.py new file mode 100644 index 00000000..83618870 --- /dev/null +++ b/tests/test_zen_browser_unit.py @@ -0,0 +1,212 @@ +import json +from pathlib import Path + +from conftest import REPO, write_stub +from unit_helpers import calls, run_unit + + +ZEN_ID = "app.zen_browser.zen.desktop" +ASSETS = REPO / "units/zen-browser" + + +def setup(stub_env, desktop="1"): + bindir, env = stub_env + # Render and compare against a throwaway system policy, even on a host + # that actually has Zen's system-wide Flatpak installed. + system_policy = "/var/lib/flatpak/app/app.zen_browser.zen/current/active/files/zen/distribution/policies.json" + write_stub(bindir, "python3", f'''\ +args=("$@") +if [ "${{args[1]:-}}" = "{system_policy}" ]; then args[1]="$HOME/system-existing-policy.json"; fi +exec /usr/bin/python3 "${{args[@]}}" +''') + write_stub(bindir, "cmp", f'''\ +args=("$@") +if [ "${{args[2]:-}}" = "{system_policy}" ]; then args[2]="$HOME/system-existing-policy.json"; fi +exec /usr/bin/cmp "${{args[@]}}" +''') + write_stub(bindir, "stat", f'''\ +if [ "${{3:-}}" = "{system_policy}" ]; then echo 644:0:0; else exec /usr/bin/stat "$@"; fi +''') + env.update(DOTFILES_IS_DESKTOP=desktop, DOTFILES_MACHINE="zen-unit-test", + DBUS_SESSION_BUS_ADDRESS="", XDG_RUNTIME_DIR="") + write_stub(bindir, "id", 'if [ "$1" = -u ]; then echo 1234; else /usr/bin/id "$@"; fi\n') + write_stub(bindir, "gsettings", '''\ +echo "gsettings $* DBUS=$DBUS_SESSION_BUS_ADDRESS RUNTIME=$XDG_RUNTIME_DIR" >> "$HOME/calls" +if [ "$1" = get ]; then + if [ -f "$HOME/dock" ]; then cat "$HOME/dock"; else echo "['org.mozilla.firefox.desktop', 'other.desktop', 'org.mozilla.Thunderbird.desktop']"; fi +else printf '%s\\n' "$4" > "$HOME/dock"; fi +''') + write_stub(bindir, "xdg-mime", 'echo "xdg-mime $*" >> "$HOME/calls"\nexit "${MIME_RC:-0}"\n') + write_stub(bindir, "sudo", '''\ +echo "sudo $*" >> "$HOME/calls" +[ "${SUDO_RC:-0}" = 0 ] || exit "$SUDO_RC" +case " $* " in *" -d "*) exit 0 ;; esac +for arg in "$@"; do previous="${current:-}"; current="$arg"; done +cp "$previous" "$HOME/system-policy.json" && cp "$previous" "$HOME/system-existing-policy.json" +''') + return env + + +def profile(env, path="abc.default", is_relative=True): + base = Path(env["HOME"]) / ".var/app/app.zen_browser.zen/.zen" + base.mkdir(parents=True, exist_ok=True) + target = base / path if is_relative else Path(path) + target.mkdir(parents=True, exist_ok=True) + (base / "profiles.ini").write_text( + f"[Profile0]\nIsRelative={int(is_relative)}\nPath={path}\n" + ) + return target + + +def test_desktop_preferences_policy_and_dock_are_idempotent(stub_env): + env = setup(stub_env) + target = profile(env) + user_js = target / "user.js" + user_js.write_text('user_pref("custom.preference", 42);\n') + user_policy = Path(env["HOME"]) / ".zen/distribution/policies.json" + user_policy.parent.mkdir(parents=True) + user_policy.write_text(json.dumps({"policies": {"DisableTelemetry": True, + "Extensions": {"Install": ["https://example.test/other.xpi"]}}})) + + result = run_unit("zen-browser", env) + assert result.returncode == 0, result.stdout + result.stderr + contents = user_js.read_text() + assert contents.startswith('user_pref("custom.preference", 42);\n') + assert (ASSETS / "user.js").read_text().rstrip() in contents + assert user_js.stat().st_mode & 0o777 == 0o644 + policy = json.loads(user_policy.read_text())["policies"] + assert policy["DisableTelemetry"] is True + assert policy["Extensions"]["Install"][0] == "https://example.test/other.xpi" + assert len(policy["Extensions"]["Install"]) == 2 + assert policy["ExtensionSettings"]["{446900e4-71c2-419f-a6a7-df9c091e268b}"]["installation_mode"] == "force_installed" + assert json.loads((Path(env["HOME"]) / "system-policy.json").read_text()) == json.loads((ASSETS / "policies.json").read_text()) + log = calls(env) + assert "DBUS=unix:path=/run/user/1234/bus RUNTIME=/run/user/1234" in log + assert (Path(env["HOME"]) / "dock").read_text().strip() == "['app.zen_browser.zen.desktop', 'other.desktop']" + assert log.count("xdg-mime default") == 7 + assert "sudo -n install -d -m 0755 -o root -g root" in log + assert "sudo -n install -m 0644 -o root -g root" in log + + before = user_js.read_bytes(), user_policy.read_bytes() + result = run_unit("zen-browser", env) + assert result.returncode == 0, result.stdout + result.stderr + assert (user_js.read_bytes(), user_policy.read_bytes()) == before + assert calls(env).count("gsettings set") == 1 + assert calls(env).count("sudo -n install -m 0644") == 1 + + +def test_missing_profile_and_host_gates_do_not_change_profiles(stub_env): + env = setup(stub_env) + result = run_unit("zen-browser", env) + assert result.returncode == 0 and "profiles.ini not found" in result.stdout + assert "sudo" not in calls(env) and "xdg-mime" not in calls(env) + for overrides in ({"DOTFILES_IS_DESKTOP": "0"}, + {"DOTFILES_MACHINE": "mumbai"}, + {"DOTFILES_MACHINE": "test-fleet-fedora"}): + (Path(env["HOME"]) / "calls").unlink(missing_ok=True) + result = run_unit("zen-browser", env, **overrides) + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert not calls(env) + + +def test_inventory_errors_are_not_silent_non_desktop(stub_env): + env = setup(stub_env, desktop="") + bindir = stub_env[0] + write_stub(bindir, "yq", 'echo "inventory unavailable" >&2\nexit 2\n') + result = run_unit("zen-browser", env) + assert result.returncode == 1 + assert "FAIL desktop inventory" in result.stdout + assert not calls(env) + + +def test_absolute_profile_and_existing_managed_preferences(stub_env, tmp_path): + env = setup(stub_env) + target = profile(env, str(tmp_path / "absolute profile"), is_relative=False) + user_js = target / "user.js" + user_js.write_text((ASSETS / "user.js").read_text()) + result = run_unit("zen-browser", env) + assert result.returncode == 0, result.stdout + result.stderr + assert user_js.read_text() == (ASSETS / "user.js").read_text() + + +def test_broken_profile_and_sudo_failure_are_reported(stub_env): + env = setup(stub_env) + target = profile(env) + (target.parent / "profiles.ini").write_text("[Profile0]\nPath=missing\n") + result = run_unit("zen-browser", env, SUDO_RC="1") + assert result.returncode == 1 + assert "FAIL Zen profile preferences" in result.stdout + assert "FAIL Flatpak Zen policy" in result.stdout + assert not (target.parent / "missing").exists() + assert (Path(env["HOME"]) / ".zen/distribution/policies.json").exists() + + +def test_malformed_existing_policy_is_not_overwritten(stub_env): + env = setup(stub_env) + profile(env) + policy = Path(env["HOME"]) / ".zen/distribution/policies.json" + policy.parent.mkdir(parents=True) + policy.write_text("not json") + result = run_unit("zen-browser", env) + assert result.returncode == 1 + assert "FAIL user Zen policy" in result.stdout + assert policy.read_text() == "not json" + + +def test_default_browser_failure_is_soft_and_reports_warning(stub_env): + env = setup(stub_env) + profile(env) + result = run_unit("zen-browser", env, MIME_RC="1") + assert result.returncode == 0 + assert "warn default browser" in result.stdout + assert calls(env).count("xdg-mime default") == 7 + + +def test_system_policy_preserves_existing_settings(stub_env): + env = setup(stub_env) + profile(env) + existing = Path(env["HOME"]) / "system-existing-policy.json" + existing.write_text(json.dumps({"policies": {"DisableTelemetry": True, + "ExtensionSettings": {"custom@example": {"installation_mode": "allowed"}}}})) + result = run_unit("zen-browser", env) + assert result.returncode == 0, result.stdout + result.stderr + policies = json.loads(existing.read_text())["policies"] + assert policies["DisableTelemetry"] is True + assert policies["ExtensionSettings"]["custom@example"]["installation_mode"] == "allowed" + assert policies["ExtensionSettings"]["{446900e4-71c2-419f-a6a7-df9c091e268b}"]["installation_mode"] == "force_installed" + + +def test_bad_preference_markers_do_not_destroy_custom_file(stub_env): + env = setup(stub_env) + user_js = profile(env) / "user.js" + original = '// BEGIN dotfiles Zen preferences\nuser_pref("custom", true);\n' + user_js.write_text(original) + result = run_unit("zen-browser", env) + assert result.returncode == 1 and "FAIL Zen profile preferences" in result.stdout + assert user_js.read_text() == original + + +def test_inventory_membership_controls_desktop_gate(stub_env): + env = setup(stub_env, desktop="") + bindir = stub_env[0] + write_stub(bindir, "yq", 'echo "${DESKTOP_RESULT:-false}"\n') + result = run_unit("zen-browser", env) + assert result.returncode == 0 and "not a desktop" in result.stdout + assert not calls(env) + result = run_unit("zen-browser", env, DESKTOP_RESULT="true") + assert result.returncode == 0 and "gsettings get" in calls(env) + + +def test_dock_uses_inherited_session(stub_env): + env = setup(stub_env) + result = run_unit("zen-browser", env, DBUS_SESSION_BUS_ADDRESS="unix:path=/custom/bus", + XDG_RUNTIME_DIR="/custom/runtime") + assert result.returncode == 0 + assert "DBUS=unix:path=/custom/bus RUNTIME=/custom/runtime" in calls(env) + + +def test_dock_bus_falls_back_to_inherited_runtime(stub_env): + env = setup(stub_env) + result = run_unit("zen-browser", env, XDG_RUNTIME_DIR="/custom/runtime") + assert result.returncode == 0 + assert "DBUS=unix:path=/custom/runtime/bus RUNTIME=/custom/runtime" in calls(env) diff --git a/units/apk-packages.conf b/units/apk-packages.conf new file mode 100644 index 00000000..1bc978a3 --- /dev/null +++ b/units/apk-packages.conf @@ -0,0 +1,9 @@ +# The old role's yq is Alpine's yq-go (Mike Farah); core bootstrap is +# independent of legacy packages that may not exist on a given release. +APK_BOOTSTRAP=(chezmoi yq-go python3 py3-yaml) +APK_PACKAGES=( + bash fish zsh git git-lfs delta vim neovim tmux openssh-client curl wget + jq ripgrep fd fzf starship bat exa direnv zoxide atuin shellcheck shfmt + nodejs npm busybox-extras podman podman-compose crun iptables +) +APK_OPTIONAL=(mise) diff --git a/units/apk-packages.sh b/units/apk-packages.sh new file mode 100755 index 00000000..8aee1e64 --- /dev/null +++ b/units/apk-packages.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=units/platform-config.bash +. "$DOTFILES_DIR/units/platform-config.bash" +# shellcheck source=units/apk-packages.conf +. "$DOTFILES_DIR/units/apk-packages.conf" + +if [ "$IS_MUSL" != 1 ]; then + note 'SKIPPED: APK packages require musl' + finish +fi + +check_apk_packages() { + local installed pkg line found + installed="$(apk info 2>/dev/null)" || { echo 'Cannot inspect installed APK packages' >&2; return 1; } + for pkg in "$@"; do + found=0 + while IFS= read -r line; do [ "$line" != "$pkg" ] || found=1; done <<< "$installed" + if [ "$found" = 0 ]; then note "WOULD install APK package $pkg"; else note "ok APK package $pkg"; fi + done +} +check_podman_service() { + local unit="$1" enabled active rc + enabled="$(systemctl is-enabled "$unit" 2>/dev/null)"; rc=$? + case "$enabled:$rc" in + enabled:0|enabled-runtime:0|disabled:1|masked:1|masked-runtime:1|static:0|indirect:0|linked:0|linked-runtime:0|generated:0|transient:0|not-found:1) ;; + *) echo "Cannot inspect $unit enablement" >&2; return 1 ;; + esac + active="$(systemctl is-active "$unit" 2>/dev/null)"; rc=$? + case "$active:$rc" in + active:0|reloading:0|refreshing:0|inactive:3|failed:3|activating:3|deactivating:3) ;; + unknown:4) [ "$enabled" = not-found ] || { echo "Cannot inspect $unit activity" >&2; return 1; } ;; + *) echo "Cannot inspect $unit activity" >&2; return 1 ;; + esac + if [ "$enabled" != enabled ] || [ "$active" != active ]; then + note "WOULD enable and start $unit (enabled=$enabled, active=$active)" + else + note "ok $unit enabled and active" + fi +} +check_podman_group() { + local user groups + user="${USER:-$(id -un)}" + groups="$(id -nG "$user")" || return 1 + case " $groups " in + *' podman '*) note 'ok podman group membership' ;; + *) note "WOULD add $user to podman group" ;; + esac +} +if [ "${UNITS_CHECK:-0}" = 1 ]; then + note 'WOULD update APK package cache (not inspected offline)' + step 'inspect APK packages' check_apk_packages "${APK_BOOTSTRAP[@]}" "${APK_OPTIONAL[@]}" "${APK_PACKAGES[@]}" + note 'WOULD run npm install -g @bitwarden/cli (current registry version not inspected)' + if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then + note 'SKIPPED: podman services on retired VPS' + else + step 'inspect podman service' check_podman_service podman + step 'inspect podman socket' check_podman_service podman.socket + fi + step 'inspect podman group membership' check_podman_group + finish +fi + +update_cache() { + local attempt + for attempt in 1 2 3 4 5; do + platform_root apk update && return 0 + [ "$attempt" = 5 ] || sleep 5 + done + return 1 +} +install_packages() { + # The role's state=present does not upgrade already installed packages. + local pkg rc=0 missing=() + for pkg in "$@"; do + apk info -e "$pkg" >/dev/null 2>&1 || missing+=("$pkg") + done + [ "${#missing[@]}" != 0 ] || return 0 + platform_root apk add --no-upgrade "${missing[@]}" && return 0 + # An unavailable package must not prevent unrelated packages being installed. + for pkg in "${missing[@]}"; do + if ! platform_root apk add --no-upgrade "$pkg"; then + echo "APK package unavailable or installation failed: $pkg" >&2 + rc=1 + fi + done + return "$rc" +} +add_podman_group() { + local user groups + user="${USER:-$(id -un)}" + groups="$(id -nG "$user")" || return 1 + case " $groups " in *' podman '*) return 0 ;; esac + platform_root usermod -a -G podman "$user" || return 1 + note 'User added to podman group; log out and back in to use rootless podman.' +} + +soft 'APK cache update' update_cache +step 'APK bootstrap packages' install_packages "${APK_BOOTSTRAP[@]}" +soft 'APK optional mise' install_packages "${APK_OPTIONAL[@]}" +step 'APK packages' install_packages "${APK_PACKAGES[@]}" +step 'Bitwarden CLI via npm' platform_root npm install -g @bitwarden/cli +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then + note 'SKIPPED: podman services on retired VPS' +else + soft 'podman service' platform_root systemctl enable --now podman + soft 'podman socket' platform_root systemctl enable --now podman.socket +fi +step 'podman group membership' add_podman_group +finish diff --git a/units/atuin.sh b/units/atuin.sh index 7b0a67ae..e8b662fd 100755 --- a/units/atuin.sh +++ b/units/atuin.sh @@ -13,6 +13,11 @@ if ! command -v atuin >/dev/null 2>&1; then exit 0 fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + note "FAIL: atuin login status cannot be inspected safely: atuin status opens and may initialize its SQLite stores" + exit 1 +fi + step "atuin data dir" install -d -m 0700 "$data" [ "$UNIT_FAILED" = 0 ] || finish diff --git a/units/bluefin.sh b/units/bluefin.sh new file mode 100755 index 00000000..13d00bed --- /dev/null +++ b/units/bluefin.sh @@ -0,0 +1,345 @@ +#!/usr/bin/env bash +# Bluefin branding, wallpaper rotation, corral and desktop swappiness. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [ "${UNITS_CHECK:-0}" = 1 ]; then export PYTHONDONTWRITEBYTECODE=1; fi + +assets="$DOTFILES_DIR/units/bluefin" +image=ghcr.io/projectbluefin/common:latest + +if skipped_by_profile skip_bluefin; then + note "SKIPPED: bluefin disabled by host/profile" + finish +fi + +# Unlike is_desktop(), distinguish a missing/broken inventory reader from a +# negative membership result. An explicit host override does not need yq. +desktop="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" +case "$desktop" in + true|1) ;; + false|0) note "SKIPPED: not a desktop"; finish ;; + *) + if ! command -v yq >/dev/null 2>&1; then + step "desktop inventory requires yq" false + finish + fi + if ! desktop="$(yq ".all.children.desktop.hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml")"; then + step "desktop inventory lookup" false + finish + fi + case "$desktop" in + true) ;; + false) note "SKIPPED: not a desktop"; finish ;; + *) step "desktop inventory lookup returned an invalid result" false; finish ;; + esac + ;; +esac + +uid="$(id -u)" || { step "resolve user UID" false; finish; } +bus="${DBUS_SESSION_BUS_ADDRESS:-unix:path=/run/user/$uid/bus}" +export DBUS_SESSION_BUS_ADDRESS="$bus" +# Check simple local socket addresses directly; gsettings also probes other +# D-Bus transports and reports failures below. +case "$bus" in + unix:path=*%*|unix:path=*\;*) ;; + unix:path=*) + bus_socket="${bus#unix:path=}" + bus_socket="${bus_socket%%,*}" + if [ ! -S "$bus_socket" ]; then note "PARTIAL: session bus socket unavailable ($bus_socket)"; fi + ;; +esac +is_gnome=0 +if { [ "${UNITS_CHECK:-0}" != 1 ] || [[ "$bus" != autolaunch:* ]]; } && gsettings get org.gnome.desktop.interface color-scheme >/dev/null 2>&1; then + is_gnome=1 +else + note "PARTIAL: GNOME session detection failed; Bluefin branding, fonts and extensions not applied" +fi +is_bluefin=0 +if grep -iq Bluefin /etc/os-release; then + is_bluefin=1 +else + rc=$? + if [ "$rc" != 1 ]; then step "detect Bluefin OS" false; finish; fi +fi +force_bluefin="$(host_var force_bluefin false)" + +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if [ "$is_gnome" != 1 ]; then step "inspect GNOME session" false; fi + check_bluefin_files() { + python3 - "$assets" "$HOME" "$bus" <<'PY' +import datetime +from pathlib import Path +import subprocess +import sys +sys.excepthook = lambda kind, value, traceback: print('Bluefin inspection failed (' + kind.__name__ + ')', file=sys.stderr) + +assets, home, bus = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3] + +def compare(name, dest, content, mode): + try: + info = dest.stat() + changed = not dest.is_file() or dest.read_bytes() != content or info.st_mode & 0o7777 != mode + except FileNotFoundError: + changed = True + print(('WOULD update ' if changed else 'OK ') + name) + +localbin = home / '.local/bin' +try: + info = localbin.stat() + print(('OK ' if localbin.is_dir() and info.st_mode & 0o7777 == 0o755 else 'WOULD update ') + 'local bin directory') +except FileNotFoundError: + print('WOULD create local bin directory') +compare('wallpaper rotation script', localbin / 'bluefin-wallpaper-rotate.py', + (assets / 'files/bluefin-wallpaper-rotate.py').read_bytes(), 0o755) +corral = localbin / 'corral' +try: + info = corral.stat() + if not corral.is_file() or info.st_mode & 0o7777 != 0o755: + print('WOULD install corral') + else: + print('PLAN refresh corral; remote version/checksum not inspected') +except FileNotFoundError: + print('WOULD install corral') +for path in (localbin / 'tailvm-go', localbin / 'tailvm', home / '.tailvm-completion.bash', + home / '.tailvm-completion.zsh', home / '.config/fish/completions/tailvm.fish'): + if path.exists() or path.is_symlink(): + print('WOULD remove retired tailvm file: ' + str(path)) +text = (assets / 'templates/bluefin-wallpaper-rotate.service.j2').read_text() +if '\n' in bus or '\r' in bus: + raise ValueError('invalid session bus address') +bus_value = bus.replace('%', '%%').replace('\\', '\\\\').replace('"', '\\"') +assignment = 'DBUS_SESSION_BUS_ADDRESS=' + bus_value +if any(c.isspace() or c in '\\"' for c in bus): + assignment = '"' + assignment + '"' +for token, value in (("{{ ansible_facts['env']['HOME'] }}", str(home)), + ('Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ user_uid_cmd.stdout | trim }}/bus', + 'Environment=' + assignment)): + if token not in text: + raise ValueError('missing service template token') + text = text.replace(token, value) +units = home / '.config/systemd/user' +compare('wallpaper service', units / 'bluefin-wallpaper-rotate.service', text.encode(), 0o644) +compare('wallpaper timer unit', units / 'bluefin-wallpaper-rotate.timer', + (assets / 'templates/bluefin-wallpaper-rotate.timer.j2').read_bytes(), 0o644) +compare('zRAM swappiness', Path('/etc/sysctl.d/99-zram.conf'), + b'# Optimize zRAM swappiness for compressed RAM swap\nvm.swappiness = 150\n', 0o644) +wallpaper = home / '.local/share/backgrounds/bluefin' / (datetime.datetime.now().strftime('%m') + '-bluefin.xml') +if wallpaper.exists(): + for key in ('picture-uri', 'picture-uri-dark'): + result = subprocess.run(['gsettings', 'get', 'org.gnome.desktop.background', key], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True) + import ast + current = ast.literal_eval(result.stdout.strip()) + print(('WOULD rotate ' if current != 'file://' + str(wallpaper) else 'OK ') + key) +else: + print('SKIPPED wallpaper rotation: current month wallpaper is absent') +PY + } + check_timer() { + local state line load='?' enabled='?' active='?' + state="$(systemctl --user show --property=LoadState --property=UnitFileState --property=ActiveState bluefin-wallpaper-rotate.timer)" || return 1 + while IFS= read -r line; do + case "$line" in + LoadState=*) load="${line#*=}" ;; + UnitFileState=*) enabled="${line#*=}" ;; + ActiveState=*) active="${line#*=}" ;; + esac + done <<< "$state" + [ "$load" != '?' ] && [ "$enabled" != '?' ] && [ "$active" != '?' ] || return 1 + if [ "$load" = loaded ] && [ "$enabled" = enabled ] && [ "$active" = active ]; then + note "OK wallpaper timer" + else + note "WOULD enable and start wallpaper timer" + fi + } + check_extensions() { + local all enabled uuid + all="$(gnome-extensions list)" || return 1 + enabled="$(gnome-extensions list --enabled)" || return 1 + for uuid in appindicatorsupport@rgcjonas.gmail.com dash-to-dock@micxgx.gmail.com blur-my-shell@aunetx gsconnect@andyholmes.github.io logomenu@aryan_k search-light@icedman.github.com caffeine@patapon.info; do + if [[ $'\n'"$all"$'\n' != *$'\n'"$uuid"$'\n'* ]]; then + note "WOULD install GNOME extension $uuid" + elif [[ $'\n'"$enabled"$'\n' != *$'\n'"$uuid"$'\n'* ]]; then + note "WOULD enable GNOME extension $uuid" + fi + done + } + step "inspect Bluefin files and wallpaper" check_bluefin_files + step "inspect wallpaper timer" check_timer + if [ "$is_gnome" = 1 ] && { [ "$is_bluefin" = 0 ] || [ "$force_bluefin" = true ] || [ "$force_bluefin" = 1 ]; }; then + note "PLAN refresh Bluefin branding and fonts; remote image and archives not inspected" + step "inspect GNOME extensions" check_extensions + fi + finish +fi + +sync_branding() { + local tmp container='' rc=0 extracted=0 source target + tmp="$(mktemp -d)" || return 1 + if podman pull --arch amd64 "$image" && container="$(podman create "$image")" && [ -n "$container" ]; then + mkdir -p "$tmp/dconf" "$tmp/backgrounds" "$tmp/icons" "$tmp/ublue-os" || rc=1 + if [ "$rc" = 0 ]; then + while IFS='|' read -r source target; do + if podman cp "$container:$source" "$tmp/$target"; then + extracted=$((extracted + 1)) + else + note "PARTIAL: Bluefin image asset unavailable: $source" + fi + done <<'ASSETS' +system_files/bluefin/usr/share/glib-2.0/schemas/zz0-bluefin-modifications.gschema.override|. +system_files/bluefin/etc/dconf/db/distro.d/.|dconf/ +system_files/bluefin/usr/share/backgrounds/bluefin/.|backgrounds/ +system_files/bluefin/usr/share/icons/hicolor/.|icons/ +system_files/bluefin/usr/share/ublue-os/.|ublue-os/ +ASSETS + if [ "$extracted" = 0 ]; then + echo "No Bluefin image assets could be extracted" >&2 + rc=1 + else + install -d -m 0755 "$HOME/.local/share/backgrounds/bluefin" "$HOME/.local/share/icons/hicolor" "$HOME/.local/share/ublue-os" \ + && cp -a "$tmp/backgrounds/." "$HOME/.local/share/backgrounds/bluefin/" \ + && cp -a "$tmp/icons/." "$HOME/.local/share/icons/hicolor/" \ + && cp -a "$tmp/ublue-os/." "$HOME/.local/share/ublue-os/" \ + && python3 - "$HOME/.local/share/backgrounds/bluefin" <<'PY' || rc=1 +import os +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +old = b'/usr/share/backgrounds/bluefin/' +new = os.fsencode(str(root) + '/') +for path in root.rglob('*.xml'): + data = path.read_bytes() + updated = data.replace(old, new) + if updated != data: + path.write_bytes(updated) +PY + DBUS_SESSION_BUS_ADDRESS="$bus" python3 "$assets/files/bluefin-sync.py" "$tmp" || rc=1 + fi + fi + else + rc=1 + fi + if [ -n "$container" ]; then podman rm "$container" || rc=1; fi + rm -rf "$tmp" || rc=1 + return "$rc" +} + +install_bluefin_fonts() { + local tmp rc=0 + tmp="$(mktemp -d)" || return 1 + install -d -m 0755 "$HOME/.local/share/fonts/Inter" "$HOME/.local/share/fonts/JetBrainsMono" \ + && curl -fsSL --retry 1 https://github.com/rsms/inter/releases/latest/download/Inter-4.1.zip -o "$tmp/Inter.zip" \ + && curl -fsSL --retry 1 https://github.com/JetBrains/JetBrainsMono/releases/download/v2.304/JetBrainsMono-2.304.zip -o "$tmp/JetBrainsMono.zip" \ + && unzip -ouq "$tmp/Inter.zip" -d "$HOME/.local/share/fonts/Inter" \ + && unzip -ouq "$tmp/JetBrainsMono.zip" -d "$HOME/.local/share/fonts/JetBrainsMono" || rc=1 + rm -rf "$tmp" || rc=1 + return "$rc" +} + +install_corral() { + local arch name tmp hash file actual expected='' rc=0 + arch="$(uname -m)" || return 1 + case "$arch" in x86_64) arch=amd64 ;; aarch64) arch=arm64 ;; esac + name="corral-linux-$arch" + tmp="$(mktemp -d)" || return 1 + if curl -fsSL --retry 1 "https://github.com/hanthor/corral/releases/download/binaries/$name" -o "$tmp/$name" \ + && curl -fsSL --retry 1 https://github.com/hanthor/corral/releases/download/binaries/SHA256SUMS -o "$tmp/SHA256SUMS"; then + while read -r hash file; do + file="${file#\*}" + file="${file#./}" + if [ "$file" = "$name" ]; then expected="$hash"; break; fi + done < "$tmp/SHA256SUMS" + actual="$(sha256sum "$tmp/$name")" || rc=1 + actual="${actual%% *}" + if [[ ! "$expected" =~ ^[[:xdigit:]]{64}$ ]] || [ "${expected,,}" != "${actual,,}" ]; then + echo "Corral checksum missing or mismatched for $name" >&2 + rc=1 + elif [ "$rc" = 0 ]; then + install_if_changed "$tmp/$name" "$HOME/.local/bin/corral" 0755 || rc=1 + fi + else + rc=1 + fi + rm -rf "$tmp" || rc=1 + return "$rc" +} + +remove_tailvm() { + rm -f "$HOME/.local/bin/tailvm-go" "$HOME/.local/bin/tailvm" \ + "$HOME/.tailvm-completion.bash" "$HOME/.tailvm-completion.zsh" \ + "$HOME/.config/fish/completions/tailvm.fish" +} + +rotation_units() { + local tmp service="$HOME/.config/systemd/user/bluefin-wallpaper-rotate.service" + local timer="$HOME/.config/systemd/user/bluefin-wallpaper-rotate.timer" changed=0 rc=0 + tmp="$(mktemp -d)" || return 1 + python3 - "$assets/templates/bluefin-wallpaper-rotate.service.j2" "$tmp/service" "$HOME" "$bus" <<'PY' || rc=1 +import pathlib +import sys + +source, destination, home, bus = sys.argv[1:] +text = pathlib.Path(source).read_text() +if '\n' in bus or '\r' in bus: + raise ValueError('invalid session bus address') +bus_value = bus.replace('%', '%%').replace('\\', '\\\\').replace('"', '\\"') +assignment = 'DBUS_SESSION_BUS_ADDRESS=' + bus_value +if any(c.isspace() or c in '\\"' for c in bus): + assignment = '"' + assignment + '"' +for token, value in (("{{ ansible_facts['env']['HOME'] }}", home), + ('Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ user_uid_cmd.stdout | trim }}/bus', + 'Environment=' + assignment)): + if token not in text: + raise ValueError(f'missing service template token: {token}') + text = text.replace(token, value) +pathlib.Path(destination).write_text(text) +PY + if [ "$rc" = 0 ]; then + install -d -m 0755 "$(dirname "$service")" || rc=1 + if ! cmp -s "$tmp/service" "$service" || [ "$(stat -c %a "$service" 2>/dev/null)" != 644 ]; then changed=1; fi + if ! cmp -s "$assets/templates/bluefin-wallpaper-rotate.timer.j2" "$timer" || [ "$(stat -c %a "$timer" 2>/dev/null)" != 644 ]; then changed=1; fi + if [ "$rc" = 0 ]; then + install_if_changed "$tmp/service" "$service" 0644 \ + && install_if_changed "$assets/templates/bluefin-wallpaper-rotate.timer.j2" "$timer" 0644 || rc=1 + fi + if [ "$rc" = 0 ] && [ "$changed" = 1 ]; then systemctl --user daemon-reload || rc=1; fi + fi + rm -rf "$tmp" || rc=1 + return "$rc" +} + +root_run() { + if [ "$EUID" = 0 ]; then "$@"; else sudo -n "$@"; fi +} + +zram_swappiness() { + local tmp dest=/etc/sysctl.d/99-zram.conf rc=0 + tmp="$(mktemp)" || return 1 + printf '# Optimize zRAM swappiness for compressed RAM swap\nvm.swappiness = 150\n' > "$tmp" || rc=1 + if [ "$rc" = 0 ] && { ! cmp -s "$tmp" "$dest" || [ "$(stat -c %a "$dest" 2>/dev/null)" != 644 ]; }; then + root_run install -m 0644 "$tmp" "$dest" && root_run sysctl -p "$dest" || rc=1 + fi + rm -f "$tmp" || rc=1 + return "$rc" +} + +if [ "$is_gnome" = 1 ] && { [ "$is_bluefin" = 0 ] || [ "$force_bluefin" = true ] || [ "$force_bluefin" = 1 ]; }; then + step "Bluefin branding and settings" sync_branding + step "Bluefin fonts" install_bluefin_fonts + step "font cache" fc-cache -f +fi + +step "local bin directory" install -d -m 0755 "$HOME/.local/bin" +step "wallpaper rotation script" install_if_changed "$assets/files/bluefin-wallpaper-rotate.py" "$HOME/.local/bin/bluefin-wallpaper-rotate.py" 0755 +step "corral" install_corral +step "remove retired tailvm" remove_tailvm +step "wallpaper systemd units" rotation_units +step "wallpaper timer" systemctl --user enable --now bluefin-wallpaper-rotate.timer +step "rotate wallpaper" env DBUS_SESSION_BUS_ADDRESS="$bus" python3 "$HOME/.local/bin/bluefin-wallpaper-rotate.py" +if [ "$is_gnome" = 1 ] && { [ "$is_bluefin" = 0 ] || [ "$force_bluefin" = true ] || [ "$force_bluefin" = 1 ]; }; then + soft "GNOME extensions" env DBUS_SESSION_BUS_ADDRESS="$bus" python3 "$assets/files/install-extensions.py" +fi +step "zRAM swappiness" zram_swappiness +finish diff --git a/roles/bluefin_common/files/bluefin-sync.py b/units/bluefin/files/bluefin-sync.py similarity index 100% rename from roles/bluefin_common/files/bluefin-sync.py rename to units/bluefin/files/bluefin-sync.py diff --git a/roles/bluefin_common/files/bluefin-wallpaper-rotate.py b/units/bluefin/files/bluefin-wallpaper-rotate.py similarity index 100% rename from roles/bluefin_common/files/bluefin-wallpaper-rotate.py rename to units/bluefin/files/bluefin-wallpaper-rotate.py diff --git a/roles/bluefin_common/files/install-extensions.py b/units/bluefin/files/install-extensions.py similarity index 100% rename from roles/bluefin_common/files/install-extensions.py rename to units/bluefin/files/install-extensions.py diff --git a/roles/bluefin_common/templates/bluefin-wallpaper-rotate.service.j2 b/units/bluefin/templates/bluefin-wallpaper-rotate.service.j2 similarity index 100% rename from roles/bluefin_common/templates/bluefin-wallpaper-rotate.service.j2 rename to units/bluefin/templates/bluefin-wallpaper-rotate.service.j2 diff --git a/roles/bluefin_common/templates/bluefin-wallpaper-rotate.timer.j2 b/units/bluefin/templates/bluefin-wallpaper-rotate.timer.j2 similarity index 100% rename from roles/bluefin_common/templates/bluefin-wallpaper-rotate.timer.j2 rename to units/bluefin/templates/bluefin-wallpaper-rotate.timer.j2 diff --git a/units/browser-fxa.sh b/units/browser-fxa.sh new file mode 100755 index 00000000..65bf54da --- /dev/null +++ b/units/browser-fxa.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# Firefox Accounts sign-in previously included by the Zen browser role. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if skipped_by_profile skip_zen_browser; then + note "SKIPPED: Zen Browser disabled by host profile" + exit 0 +fi +# Resolve explicit host settings first; inventory parse failures are not skips. +case "${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" in + 0|false) note "SKIPPED: not a desktop"; exit 0 ;; + 1|true) ;; + '') + if ! command -v yq >/dev/null 2>&1; then + note "FAIL desktop detection: yq is required"; exit 1 + fi + desktop="$(MACHINE="$MACHINE" yq -r '.all.children.desktop.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml")" || { + note "FAIL desktop inventory lookup"; exit 1 + } + case "$desktop" in + true) ;; + false) note "SKIPPED: not a desktop"; exit 0 ;; + *) note "FAIL invalid desktop inventory lookup"; exit 1 ;; + esac ;; + *) note "FAIL invalid desktop override"; exit 1 ;; +esac +if ! command -v yq >/dev/null 2>&1; then + note "FAIL Firefox Accounts settings: yq is required"; exit 1 +fi + +# Read the existing variable sources in Ansible's host/group/default precedence. +setting() { + local key="$1" file value + for file in "$DOTFILES_DIR/host_vars/$MACHINE.yml" "$DOTFILES_DIR/group_vars/all.yml" \ + "$DOTFILES_DIR/units/browser-fxa/defaults.yml"; do + [ -f "$file" ] || continue + value="$(yq -r ".${key} // \"__UNSET__\"" "$file")" || return 1 + if [ "$value" != __UNSET__ ] && [ "$value" != null ]; then + printf '%s' "$value"; return 0 + fi + done + return 1 +} + +browser="$(setting browser_fxa_browser)" || { note "FAIL Firefox Accounts browser setting"; exit 1; } +case "$browser" in + zen) profiles="$HOME/.var/app/app.zen_browser.zen/.zen/profiles.ini" ;; + firefox-flatpak) profiles="$HOME/.var/app/org.mozilla.firefox/.mozilla/firefox/profiles.ini" ;; + firefox) + profiles="$HOME/.var/app/org.mozilla.firefox/.mozilla/firefox/profiles.ini" + [ -e "$profiles" ] || profiles="$HOME/.mozilla/firefox/profiles.ini" + ;; + *) note "FAIL unsupported Firefox Accounts browser: $browser"; exit 1 ;; +esac +if [ ! -f "$profiles" ]; then + if [ "${UNITS_CHECK:-0}" = 1 ] && { [ -e "$profiles" ] || [ -L "$profiles" ]; }; then + note "FAIL inspect Firefox Accounts profiles.ini"; exit 1 + fi + note "SKIPPED: $browser profile not installed"; exit 0 +fi +if [ "${UNITS_CHECK:-0}" = 1 ] && [ ! -r "$profiles" ]; then + note "FAIL inspect Firefox Accounts profiles.ini"; exit 1 +fi +venv="$(setting fxa_venv_path)" || { note "FAIL Firefox Accounts venv setting"; exit 1; } +venv="${venv//"{{ ansible_facts['env']['HOME'] }}"/$HOME}" +case "$venv" in '~/'*) venv="$HOME/${venv#\~/}" ;; esac +case "$venv" in *'{{'*|*'}}'*) note "FAIL unsupported Firefox Accounts venv template"; exit 1 ;; esac + +if [ "${UNITS_CHECK:-0}" = 1 ]; then + check_browser_fxa() { + local src="$DOTFILES_DIR/units/browser-fxa/fxa_signin.py" dest="$HOME/.local/bin/fxa_signin.py" + local rc mode profile account_state + [ -r "$src" ] || { note "FAIL read Firefox Accounts sign-in script"; return 1; } + if [ -e "$dest" ]; then + cmp -s "$src" "$dest"; rc=$? + [ "$rc" -le 1 ] || { note "FAIL compare Firefox Accounts sign-in script"; return 1; } + mode="$(stat -Lc %a "$dest")" || { note "FAIL inspect Firefox Accounts sign-in script mode"; return 1; } + if [ "$rc" = 1 ] || [ "$mode" != 700 ]; then + note "WOULD install Firefox Accounts sign-in script" + fi + else + note "WOULD install Firefox Accounts sign-in script" + fi + if [ ! -x "$venv/bin/python" ]; then + note "WOULD create Firefox Accounts venv and install PyFxA" + else + account_state="$(PYTHONDONTWRITEBYTECODE=1 "$venv/bin/python" -B -c ' +import importlib.machinery, sys +package = importlib.machinery.PathFinder.find_spec("fxa", sys.path) +module = None if package is None or package.submodule_search_locations is None else ( + importlib.machinery.PathFinder.find_spec("fxa.core", package.submodule_search_locations) +) +print("installed" if module is not None else "missing") +' 2>/dev/null)" || { note "FAIL inspect Firefox Accounts PyFxA"; return 1; } + case "$account_state" in + installed) ;; + missing) note "WOULD install Firefox Accounts PyFxA" ;; + *) note "FAIL inspect Firefox Accounts PyFxA"; return 1 ;; + esac + fi + # Use the shipped resolver, without importing the installed (possibly stale) copy. + profile="$(env -u PROFILE_DIR -u ZEN_PROFILE_DIR BROWSER_TYPE="$browser" PYTHONDONTWRITEBYTECODE=1 python3 -B -c ' +import importlib.util, sys +spec = importlib.util.spec_from_file_location("fxa_signin", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +print(module.resolve_profile_dir()) +' "$src" 2>/dev/null)" || { note "FAIL inspect Firefox Accounts default profile"; return 1; } + [ -d "$profile" ] || { note "FAIL Firefox Accounts default profile does not exist"; return 1; } + if [ -e "$profile/signedInUser.json" ]; then + mode="$(stat -Lc %a "$profile/signedInUser.json")" || { note "FAIL inspect Firefox Accounts token mode"; return 1; } + [ "$mode" = 600 ] || note "WOULD restrict Firefox Accounts token permissions" + account_state="$(python3 -B -c ' +import json, sys +try: + with open(sys.argv[1]) as f: + data = json.load(f) + print("verified" if data.get("accountData", {}).get("verified") else "unverified") +except json.JSONDecodeError: + print("unverified") +' "$profile/signedInUser.json" 2>/dev/null)" || { note "FAIL inspect Firefox Accounts token"; return 1; } + else + account_state=unverified + fi + case "$account_state" in + verified) note "Firefox Accounts profile is verified" ;; + unverified) note "WOULD attempt Firefox Accounts sign-in if credentials are available" ;; + *) note "FAIL inspect Firefox Accounts token"; return 1 ;; + esac + note "Firefox Accounts credentials were not inspected" + } + step "check Firefox Accounts" check_browser_fxa + finish +fi + +create_venv() { + [ -x "$venv/bin/python" ] || { mkdir -p "$(dirname "$venv")" && python3 -m venv "$venv"; } +} +install_pyfxa() { + [ -x "$venv/bin/python" ] || return 1 + "$venv/bin/python" -c 'import fxa.core' >/dev/null 2>&1 || "$venv/bin/python" -m pip install PyFxA +} +step "Firefox Accounts venv" create_venv +step "Firefox Accounts PyFxA" install_pyfxa +step "Firefox Accounts sign-in script" install_if_changed "$DOTFILES_DIR/units/browser-fxa/fxa_signin.py" "$HOME/.local/bin/fxa_signin.py" 0700 + +s="$(bw_session)" +if [ -z "$s" ]; then + note "PARTIAL: Firefox Accounts sign-in skipped (vault locked or secrets disabled)" + finish +fi +item_name="$(setting fxa_bw_item)" || { note "FAIL Firefox Accounts item setting"; UNIT_FAILED=1; finish; } +username="$(setting fxa_bw_username)" || { note "FAIL Firefox Accounts username setting"; UNIT_FAILED=1; finish; } +# Only an exact username match is accepted; never leak a mismatched item's contents. +selected="$(BW_SESSION="$s" bw list items --search "$item_name" 2>/dev/null | FXA_MATCH="$username" python3 -c ' +import base64, json, os, sys +try: + items = json.load(sys.stdin) + match = os.environ["FXA_MATCH"] + item = next((i for i in items if not match or (i.get("login") or {}).get("username") == match), None) + if not item: + sys.exit(1) + login = item.get("login") or {} + fields = (item.get("id") or "", login.get("username") or "", login.get("password") or "") + if not all(isinstance(v, str) and v for v in fields): + sys.exit(1) + for value in fields: + print(base64.b64encode(value.encode()).decode()) +except (ValueError, TypeError, AttributeError): + sys.exit(1) +')" || { + note "PARTIAL: Firefox Accounts item missing, incomplete, or username mismatch" + finish +} +mapfile -t fields <<< "$selected" +if [ "${#fields[@]}" -ne 3 ]; then + note "PARTIAL: Firefox Accounts item invalid"; finish +fi +item_id="$(printf '%s' "${fields[0]}" | base64 -d)" || { UNIT_FAILED=1; finish; } +email="$(printf '%s' "${fields[1]}" | base64 -d)" || { UNIT_FAILED=1; finish; } +password="$(printf '%s' "${fields[2]}" | base64 -d)" || { UNIT_FAILED=1; finish; } +totp="$(BW_SESSION="$s" bw get totp "$item_id" 2>/dev/null | tr -cd '0-9')" || totp='' + +sign_in() { + local profile + [ -x "$venv/bin/python" ] && [ -f "$HOME/.local/bin/fxa_signin.py" ] || return 1 + # Resolve the selected existing profile using the same logic as the role's script. + profile="$(env -u PROFILE_DIR -u ZEN_PROFILE_DIR BROWSER_TYPE="$browser" python3 -c ' +import importlib.util, os +p = os.path.join(os.environ["HOME"], ".local/bin/fxa_signin.py") +spec = importlib.util.spec_from_file_location("fxa_signin", p) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +print(module.resolve_profile_dir()) +')" || return 1 + [ -d "$profile" ] || { note "Firefox Accounts default profile does not exist"; return 1; } + # The upstream script opens existing files in place; protect existing and new tokens. + if [ -e "$profile/signedInUser.json" ]; then chmod 600 "$profile/signedInUser.json" || return 1; fi + ( umask 077 + FXA_EMAIL="$email" FXA_PASSWORD="$password" FXA_TOTP="$totp" BROWSER_TYPE="$browser" \ + PROFILE_DIR="$profile" "$venv/bin/python" "$HOME/.local/bin/fxa_signin.py" >/dev/null 2>&1 + ) || { note "Firefox Accounts sign-in failed; sign in manually or retry after checking credentials"; return 1; } +} +# The Ansible role reported sign-in failures without failing the rest of the play. +soft "Firefox Accounts sign-in" sign_in +finish diff --git a/roles/browser_fxa/defaults/main.yml b/units/browser-fxa/defaults.yml similarity index 100% rename from roles/browser_fxa/defaults/main.yml rename to units/browser-fxa/defaults.yml diff --git a/roles/browser_fxa/files/fxa_signin.py b/units/browser-fxa/fxa_signin.py similarity index 100% rename from roles/browser_fxa/files/fxa_signin.py rename to units/browser-fxa/fxa_signin.py diff --git a/units/bst-dashboard.sh b/units/bst-dashboard.sh new file mode 100755 index 00000000..42cb7b8c --- /dev/null +++ b/units/bst-dashboard.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then + note "SKIPPED: dashboard excluded on retired VPS" + finish +fi +if [ "$MACHINE" = karnataka ]; then + dashboard_dir=/var/home/james/dev/kde-linux + image=aurora +else + dashboard_dir=/var/home/james/dev/dakota + image=dakota +fi +if [ ! -e "$dashboard_dir" ]; then + note "SKIPPED: dashboard project does not exist: $dashboard_dir" + finish +fi +as_root() { if [ "$EUID" = 0 ]; then "$@"; else sudo -n "$@"; fi; } +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then + export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus" +fi +tmp=$(mktemp) || { step "create dashboard temporary file" false; finish; } +trap 'rm -f "$tmp"' EXIT +unit="$HOME/.config/systemd/user/bst-dashboard.service" +changed=0 +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if python3 "$DOTFILES_DIR/units/service-templates.py" dashboard "$DOTFILES_DIR/units/bst-dashboard/bst-dashboard.service.j2" "$dashboard_dir" "$image" > "$tmp"; then + if [ ! -e "$unit" ] && [ ! -L "$unit" ]; then + note "would deploy dashboard user service and reload user systemd" + elif mode=$(stat -c %a "$unit"); then + cmp -s "$tmp" "$unit"; rc=$? + case "$rc" in + 0) [ "$mode" = 644 ] && note "ok dashboard user service" || note "would set dashboard service permissions and reload user systemd" ;; + 1) note "would deploy dashboard user service and reload user systemd" ;; + *) step "compare dashboard user service" false ;; + esac + else step "inspect dashboard user service" false; fi + else step "render dashboard user service" false; fi + state=$(systemctl --user show bst-dashboard.service --property=LoadState --property=ActiveState --property=UnitFileState 2>/dev/null); rc=$? + if { [ "$rc" != 0 ] && [[ "$state" != *'LoadState=not-found'* ]]; } || [[ "$state" != *'LoadState='* || "$state" != *'ActiveState='* ]]; then + step "inspect dashboard user service" false + elif [[ "$state" == *'LoadState=not-found'* ]]; then + note "would enable and start dashboard after deployment" + elif [[ "$state" != *'LoadState=loaded'* || "$state" != *'UnitFileState='* ]]; then + step "inspect dashboard user service" false + else + [[ "$state" == *'ActiveState=active'* ]] || note "would start dashboard" + [[ "$state" == *'UnitFileState=enabled'* ]] || note "would enable dashboard" + fi + if linger=$(loginctl show-user "${USER:-$(id -un)}" --property=Linger --value 2>/dev/null); then + case "$linger" in yes) note "ok user linger" ;; no) note "would enable user linger" ;; *) step "inspect user linger" false ;; esac + else step "inspect user linger" false; fi + finish +fi +if python3 "$DOTFILES_DIR/units/service-templates.py" dashboard "$DOTFILES_DIR/units/bst-dashboard/bst-dashboard.service.j2" "$dashboard_dir" "$image" > "$tmp"; then + if [ ! -f "$unit" ] || ! cmp -s "$tmp" "$unit" || [ "$(stat -c %a "$unit" 2>/dev/null)" != 644 ]; then changed=1; fi + if install_if_changed "$tmp" "$unit" 0644; then + note "ok deploy dashboard user service" + if [ "$changed" = 0 ] || systemctl --user daemon-reload; then + step "enable and start dashboard" systemctl --user enable --now bst-dashboard.service + else + step "reload dashboard user systemd" false + fi + else + step "deploy dashboard user service" false + fi +else + step "render dashboard user service" false +fi +step "enable user linger" as_root loginctl enable-linger "${USER:-$(id -un)}" +finish diff --git a/roles/bst_dashboard/templates/bst-dashboard.service.j2 b/units/bst-dashboard/bst-dashboard.service.j2 similarity index 100% rename from roles/bst_dashboard/templates/bst-dashboard.service.j2 rename to units/bst-dashboard/bst-dashboard.service.j2 diff --git a/units/chezmoi.sh b/units/chezmoi.sh index 14f72c41..fdbc5d10 100755 --- a/units/chezmoi.sh +++ b/units/chezmoi.sh @@ -4,6 +4,46 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if ! command -v chezmoi >/dev/null 2>&1; then + note "FAIL: chezmoi not installed; cannot inspect managed files" + exit 1 + fi + desktop_hint="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" + case "$desktop_hint" in + 1|true) desktop=1 ;; + 0|false) desktop=0 ;; + *) + if ! command -v yq >/dev/null 2>&1; then + note "FAIL: yq required to inspect desktop membership" + exit 1 + fi + export DOTFILES_MACHINE="$MACHINE" + membership="$(yq '(.all.children.desktop.hosts // {}) | has(strenv(DOTFILES_MACHINE))' "$DOTFILES_DIR/inventory.yml" 2>/dev/null)" || { + note "FAIL: cannot inspect desktop membership" + exit 1 + } + case "$membership" in true) desktop=1 ;; false) desktop=0 ;; *) note "FAIL: invalid desktop membership"; exit 1 ;; esac + ;; + esac + export DOTFILES_MACHINE="$MACHINE" DOTFILES_IS_DESKTOP="$desktop" + # Supply the same data as .chezmoi.toml.tmpl without --init: older chezmoi + # versions write even during a dry-run init. The config is a pipe, not a file. + machine_json="$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$MACHINE")" || { + note "FAIL: python3 required to encode chezmoi configuration" + exit 1 + } + if [ "$desktop" = 1 ]; then desktop_toml=true; else desktop_toml=false; fi + if ! chezmoi --source "$DOTFILES_DIR" \ + --config <(printf '[data]\nmachine = %s\nis_desktop = %s\n' "$machine_json" "$desktop_toml") --config-format toml \ + --persistent-state "${XDG_CONFIG_HOME:-$HOME/.config}/chezmoi/chezmoistate.boltdb" \ + --dry-run --force --no-tty --no-pager status /dev/null 2>&1 && [ "$IS_TERMUX" = 1 ]; then soft "pkg install chezmoi" pkg install -y chezmoi fi diff --git a/roles/flatpak/vars/main.yml b/units/flatpak-packages.yml similarity index 100% rename from roles/flatpak/vars/main.yml rename to units/flatpak-packages.yml diff --git a/units/flatpak.sh b/units/flatpak.sh new file mode 100755 index 00000000..c357c266 --- /dev/null +++ b/units/flatpak.sh @@ -0,0 +1,307 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "$0")/lib.sh" +# shellcheck source=units/package-config.bash +. "$DOTFILES_DIR/units/package-config.bash" + +if skipped_by_profile skip_flatpak; then + note 'SKIPPED: Flatpak disabled for this host/profile' + exit 0 +fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if ! package_require_yq >/dev/null 2>&1 || ! package_desktop >/dev/null 2>&1; then + echo 'Cannot inspect Flatpak desktop configuration' >&2; exit 1 + fi +else + step 'Flatpak configuration dependencies' package_require_yq + [ "$UNIT_FAILED" = 0 ] || finish + step 'desktop classification' package_desktop + [ "$UNIT_FAILED" = 0 ] || finish +fi +if [ "$PACKAGE_DESKTOP" != 1 ]; then + note 'SKIPPED: Flatpak is only configured on desktops' + exit 0 +fi + +# Shared package list and system remote configuration. +flatpak_vars="$DOTFILES_DIR/units/flatpak-packages.yml" +load_config() { + local kind + package_list system_flatpaks "$flatpak_vars" || return 1 + apps=("${PACKAGE_LIST[@]}") + kind="$(yq -r '.flatpak_remotes | tag' "$flatpak_vars")" || return 1 + [ "$kind" = '!!seq' ] || { echo "$flatpak_vars: flatpak_remotes must be a list" >&2; return 1; } + remotes="$(yq -r '.flatpak_remotes[] | .name + "\t" + .url' "$flatpak_vars")" || return 1 + local name url + while IFS=$'\t' read -r name url; do + [ -n "$name$url" ] || continue + case "$name" in ''|*[!a-zA-Z0-9_.-]*) echo 'Invalid Flatpak remote name' >&2; return 1 ;; esac + case "$url" in http://*|https://*) ;; *) echo 'Invalid Flatpak remote URL' >&2; return 1 ;; esac + case "$url" in *[[:space:]]*) echo 'Invalid Flatpak remote URL' >&2; return 1 ;; esac + done <<< "$remotes" +} +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if ! load_config >/dev/null 2>&1; then echo 'Cannot inspect Flatpak configuration' >&2; exit 1; fi +else + step 'Flatpak configuration' load_config + [ "$UNIT_FAILED" = 0 ] || finish +fi + +fix_store() { + if [ -L /var/lib/flatpak ] && [ ! -e /var/lib/flatpak ]; then + package_as_root rm /var/lib/flatpak && package_as_root mkdir -p /var/lib/flatpak + fi +} +add_remotes() { + local name url rc=0 + while IFS=$'\t' read -r name url; do + [ -n "$name" ] || continue + package_as_root flatpak remote-add --if-not-exists --system "$name" "$url" || rc=1 + done <<< "$remotes" + return "$rc" +} +get_installed() { + installed="$(package_as_root flatpak list --system --columns=application --app)" +} +remove_unwanted() { + local pkg remove=() + for pkg in org.mozilla.firefox org.mozilla.Thunderbird; do + if printf '%s\n' "$installed" | grep -Fxq "$pkg"; then remove+=("$pkg"); fi + done + [ "${#remove[@]}" = 0 ] || package_as_root flatpak uninstall --system -y "${remove[@]}" +} +install_missing() { + local app missing=() rc=0 + for app in "${apps[@]}"; do + if ! printf '%s\n' "$installed" | grep -Fxq "$app"; then missing+=("$app"); fi + done + [ "${#missing[@]}" -gt 0 ] || return 0 + if package_as_root flatpak install --system -y flathub "${missing[@]}"; then return 0; fi + for app in "${missing[@]}"; do + package_as_root flatpak install --system -y flathub "$app" || rc=1 + done + return "$rc" +} + +# Host-specific user installs are independent of the system-wide role lists. +load_user_config() { + local host="$DOTFILES_DIR/host_vars/$MACHINE.yml" kind name url remote app + user_remotes=''; user_apps='' + [ -f "$host" ] || return 0 + kind="$(yq -r '.user_flatpak_remotes | tag' "$host")" || return 1 + case "$kind" in + '!!null') ;; + '!!seq') user_remotes="$(yq -r '.user_flatpak_remotes[] | (.name // "") + "\t" + (.url // "")' "$host")" || return 1 ;; + *) echo "$host: user_flatpak_remotes must be a list" >&2; return 1 ;; + esac + [ -z "$user_remotes" ] || while IFS=$'\t' read -r name url; do + case "$name" in ''|-*|*[!a-zA-Z0-9_.-]*) echo 'Invalid user Flatpak remote name' >&2; return 1 ;; esac + case "$url" in http://*|https://*) ;; *) echo 'Invalid user Flatpak remote URL' >&2; return 1 ;; esac + case "$url" in *[[:space:]]*) echo 'Invalid user Flatpak remote URL' >&2; return 1 ;; esac + done <<< "$user_remotes" + kind="$(yq -r '.user_flatpaks | tag' "$host")" || return 1 + case "$kind" in + '!!null') ;; + '!!seq') user_apps="$(yq -r '.user_flatpaks[] | (.remote // "") + "\t" + (.id // "")' "$host")" || return 1 ;; + *) echo "$host: user_flatpaks must be a list" >&2; return 1 ;; + esac + [ -z "$user_apps" ] || while IFS=$'\t' read -r remote app; do + case "$remote" in ''|-*|*[!a-zA-Z0-9_.-]*) echo 'Invalid user Flatpak app remote' >&2; return 1 ;; esac + case "$app" in ''|-*|*[!a-zA-Z0-9_.-]*) echo 'Invalid user Flatpak app ID' >&2; return 1 ;; esac + done <<< "$user_apps" +} +add_user_remotes() { + local name url rc=0 + while IFS=$'\t' read -r name url; do + [ -n "$name" ] || continue + flatpak remote-add --if-not-exists --user "$name" "$url" || rc=1 + done <<< "$user_remotes" + return "$rc" +} +install_user_apps() { + local installed remote app origin rc=0 + installed="$(flatpak list --user --columns=application --app)" || return 1 + while IFS=$'\t' read -r remote app; do + [ -n "$remote" ] || continue + if printf '%s\n' "$installed" | grep -Fxq "$app"; then + if ! origin="$(flatpak info --user --show-origin "$app")"; then + echo "Could not determine origin of user Flatpak $app" >&2 + rc=1 + elif [ "$origin" != "$remote" ]; then + note "PRESERVED: user Flatpak $app uses $origin, not requested $remote" + fi + elif flatpak install --user -y "$remote" "$app"; then + installed+=$'\n'"$app" + else + rc=1 + fi + done <<< "$user_apps" + return "$rc" +} +install_wrappers() { + local tmp name rc=0 + tmp="$(mktemp)" || return 1 + printf '#!/usr/bin/env bash\nexec flatpak run org.chromium.Chromium "$@"\n' > "$tmp" || { rm -f "$tmp"; return 1; } + for name in google-chrome google-chrome-stable chromium-browser chromium; do + install_if_changed "$tmp" "$HOME/.local/bin/$name" 0755 || rc=1 + done + rm -f "$tmp" + return "$rc" +} + +# Flatpak's ostensibly read-only commands can initialise or update an OSTree +# repository. Confine inspection to a read-only mount and isolated network and +# bus namespaces; never invoke sudo or fall back to an unconfined query. +check_flatpak_query() { + local env_args=() key + for key in XDG_DATA_HOME FLATPAK_USER_DIR FLATPAK_SYSTEM_DIR FLATPAK_CONFIG_DIR FLATPAK_DATA_DIR; do + [ -z "${!key:-}" ] || env_args+=(--setenv "$key" "${!key}") + done + bwrap --unshare-all --new-session --die-with-parent --ro-bind / / \ + --proc /proc --dev /dev --tmpfs /run --clearenv \ + --setenv HOME "$HOME" --setenv PATH "$PATH" "${env_args[@]}" \ + --setenv XDG_RUNTIME_DIR /run --setenv XDG_CACHE_HOME /run/cache \ + --setenv DBUS_SYSTEM_BUS_ADDRESS unix:path=/run/no-bus \ + --setenv DBUS_SESSION_BUS_ADDRESS unix:path=/run/no-bus \ + flatpak "$@" &1 +} + +check_flatpak_list() { + local line + CHECK_LIST="$(check_flatpak_query "$@")" || return 1 + while IFS= read -r line; do + [ -n "$line" ] || continue + case "$line" in *[!a-zA-Z0-9_.-]*) return 1 ;; esac + done <<< "$CHECK_LIST" +} + +check_wrappers() { + local name path mode rc + for name in google-chrome google-chrome-stable chromium-browser chromium; do + path="$HOME/.local/bin/$name" + if [ ! -e "$path" ] && [ ! -L "$path" ]; then + note "CHANGE: install Chromium wrapper $name" + elif [ ! -f "$path" ] || [ ! -r "$path" ]; then + echo "Cannot inspect Chromium wrapper $name" >&2; UNIT_FAILED=1 + else + cmp -s "$path" <(printf '#!/usr/bin/env bash\nexec flatpak run org.chromium.Chromium "$@"\n'); rc=$? + if [ "$rc" = 1 ]; then note "CHANGE: update Chromium wrapper $name"; fi + if [ "$rc" -gt 1 ]; then echo "Cannot compare Chromium wrapper $name" >&2; UNIT_FAILED=1; continue; fi + mode="$(stat -L -c %a "$path" 2>/dev/null || stat -L -f %Lp "$path" 2>/dev/null)" || { + echo "Cannot inspect Chromium wrapper permissions: $name" >&2; UNIT_FAILED=1; continue; + } + [ "$mode" = 755 ] || note "CHANGE: set Chromium wrapper $name mode to 0755" + fi + done +} + +check_flatpak() { + local system_remotes system_apps user_remote_names user_installed name url app remote origin + local -A planned_user_origins=() + check_wrappers + if [ -L /var/lib/flatpak ] && [ ! -e /var/lib/flatpak ]; then + note 'CHANGE: repair dangling Flatpak system store' + fi + if ! command -v flatpak >/dev/null 2>&1; then + echo 'Cannot inspect Flatpak: executable unavailable' >&2; UNIT_FAILED=1; return + fi + if ! command -v bwrap >/dev/null 2>&1; then + echo 'Cannot inspect Flatpak safely: bubblewrap is required' >&2; UNIT_FAILED=1; return + fi + for name in HOME XDG_DATA_HOME FLATPAK_USER_DIR FLATPAK_SYSTEM_DIR FLATPAK_CONFIG_DIR FLATPAK_DATA_DIR; do + case "${!name:-}" in /run|/run/*|/var/run|/var/run/*) + echo 'Cannot inspect Flatpak paths hidden by the sandbox' >&2; UNIT_FAILED=1; return ;; + esac + done + if check_flatpak_list remotes --system --columns=name; then + system_remotes="$CHECK_LIST" + while IFS=$'\t' read -r name url; do + [ -n "$name" ] || continue + if ! printf '%s\n' "$system_remotes" | grep -Fxq -- "$name"; then + note "CHANGE: add system Flatpak remote $name" + fi + done <<< "$remotes" + else + echo 'Cannot inspect system Flatpak remotes in the read-only sandbox' >&2; UNIT_FAILED=1 + fi + if check_flatpak_list list --system --columns=application --app; then + system_apps="$CHECK_LIST" + for app in org.mozilla.firefox org.mozilla.Thunderbird; do + if printf '%s\n' "$system_apps" | grep -Fxq -- "$app"; then note "CHANGE: remove system Flatpak $app"; fi + done + for app in "${apps[@]}"; do + if ! printf '%s\n' "$system_apps" | grep -Fxq -- "$app"; then note "CHANGE: install system Flatpak $app from flathub"; fi + done + else + echo 'Cannot inspect system Flatpak applications in the read-only sandbox' >&2; UNIT_FAILED=1 + fi + if ! load_user_config >/dev/null 2>&1; then + echo 'Cannot inspect user Flatpak configuration' >&2; UNIT_FAILED=1; return + fi + if [ -n "$user_remotes" ]; then + if check_flatpak_list remotes --user --columns=name; then + user_remote_names="$CHECK_LIST" + while IFS=$'\t' read -r name url; do + [ -n "$name" ] || continue + if ! printf '%s\n' "$user_remote_names" | grep -Fxq -- "$name"; then note "CHANGE: add user Flatpak remote $name"; fi + done <<< "$user_remotes" + else + echo 'Cannot inspect user Flatpak remotes in the read-only sandbox' >&2; UNIT_FAILED=1 + fi + fi + if [ -n "$user_apps" ]; then + if check_flatpak_list list --user --columns=application --app; then + user_installed="$CHECK_LIST" + while IFS=$'\t' read -r remote app; do + [ -n "$remote" ] || continue + if printf '%s\n' "$user_installed" | grep -Fxq -- "$app"; then + if [ -n "${planned_user_origins[$app]:-}" ]; then + origin="${planned_user_origins[$app]}" + [ "$origin" = "$remote" ] || note "PRESERVED: user Flatpak $app would use $origin, not requested $remote" + elif check_flatpak_list info --user --show-origin "$app" && [ -n "$CHECK_LIST" ] && [[ "$CHECK_LIST" != *$'\n'* ]]; then + origin="$CHECK_LIST" + [ "$origin" = "$remote" ] || note "PRESERVED: user Flatpak $app uses $origin, not requested $remote" + else + echo "Cannot inspect origin of user Flatpak $app" >&2; UNIT_FAILED=1 + fi + else + note "CHANGE: install user Flatpak $app from $remote" + planned_user_origins[$app]="$remote" + user_installed+=$'\n'"$app" + fi + done <<< "$user_apps" + else + echo 'Cannot inspect user Flatpak applications in the read-only sandbox' >&2; UNIT_FAILED=1 + fi + fi + case "${DOTFILES_UPGRADE:-0}" in 1|true) note 'PLANNED: Flatpak system update requested; available updates not inspected offline' ;; esac +} + +if [ "${UNITS_CHECK:-0}" = 1 ]; then + check_flatpak + finish +fi + +if command -v flatpak >/dev/null 2>&1; then + step 'Flatpak system store' fix_store + step 'Flatpak remotes' add_remotes + if get_installed; then + step 'unwanted system Flatpaks' remove_unwanted + step 'system Flatpak applications' install_missing + else + step 'list system Flatpaks' false + fi + case "${DOTFILES_UPGRADE:-0}" in 1|true) soft 'Flatpak update' package_as_root flatpak update --system -y ;; esac + if load_user_config; then + [ -z "$user_remotes" ] || step 'user Flatpak remotes' add_user_remotes + [ -z "$user_apps" ] || step 'user Flatpak applications' install_user_apps + else + step 'user Flatpak configuration' false + fi +else + note 'flatpak is required for desktop applications' >&2 + step 'Flatpak executable unavailable' false +fi +step 'Chromium Flatpak wrappers' install_wrappers +finish diff --git a/units/fleet-facts.py b/units/fleet-facts.py new file mode 100755 index 00000000..89300b24 --- /dev/null +++ b/units/fleet-facts.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""Collect only public fleet metadata and publish it with the gh contents API.""" +import base64 +import datetime as dt +import json +import os +import platform +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path +from urllib.parse import quote + +PLACEHOLDERS = {'', 'NA', 'None', 'System Product Name', 'To Be Filled By O.E.M.', 'Default string', 'Not Specified'} +SENSITIVE = re.compile( + r'(?i)(?:\b(?:serial|password|token|secret|credential|api[_ -]?key)\b|' + r'\b(?:\d{1,3}\.){3}\d{1,3}\b|\b(?:[0-9a-f]{2}:){5}[0-9a-f]{2}\b|' + r'\b(?:[0-9a-f]{1,4}:){2,}[0-9a-f]{0,4}\b|\b[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}\b|' + r'\b[0-9a-f]{20,}\b|\b\d{12,}\b|[\w.+-]+@[\w.-]+\.[a-z]{2,}|https?://|(?:^|\s)/(?:home|var|etc|root|data)/)' +) +CHASSIS = {3: 'Desktop', 4: 'Low Profile Desktop', 5: 'Pizza Box', 6: 'Mini Tower', 7: 'Tower', 8: 'Portable', 9: 'Laptop', 10: 'Notebook', 11: 'Hand Held', 12: 'Docking Station', 13: 'All In One', 14: 'Sub Notebook', 15: 'Space-Saving', 16: 'Lunch Box', 17: 'Main Server Chassis', 18: 'Expansion Chassis', 19: 'SubChassis', 20: 'Bus Expansion Chassis', 21: 'Peripheral Chassis', 22: 'Storage Chassis', 23: 'Rack Mount Chassis', 24: 'Sealed-Case PC', 30: 'Tablet', 31: 'Convertible', 32: 'Detachable'} + + +def warn(message): + print('fleet_facts: ' + message, file=sys.stderr) + + +def clean(value): + text = ' '.join(str(value or '').replace('\x00', '').split()) + if text in PLACEHOLDERS: + return '' + if len(text) > 200 or SENSITIVE.search(text): + warn('sensitive or invalid hardware/OS text omitted') + return '' + return text + + +def read_yaml(path): + if not path.exists(): + return {} + try: + import yaml + except ImportError: + if not shutil.which('yq'): + raise RuntimeError('install yq or Python PyYAML to read fleet configuration') + result = subprocess.run(['yq', '-o=json', '.', str(path)], capture_output=True, text=True, timeout=20) + if result.returncode: + raise RuntimeError('could not read fleet configuration with yq') + return json.loads(result.stdout) or {} + with path.open(encoding='utf-8') as stream: + return yaml.safe_load(stream) or {} + + +def group_file(repo, group): + path = repo / 'group_vars' / (group + '.yml') + return read_yaml(path) + + +def config(repo, machine): + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_-]*', machine): + raise ValueError('invalid inventory host name') + defaults = read_yaml(repo / 'units/fleet-facts/defaults.yml') + if not all(key in defaults for key in ('fleet_facts_publish', 'fleet_facts_repo', 'fleet_facts_branch')): + raise ValueError('fleet facts defaults are missing or incomplete') + inventory = read_yaml(repo / 'inventory.yml') + if not isinstance(inventory, dict) or not isinstance(inventory.get('all'), dict): + raise ValueError('invalid inventory') + all_group = inventory['all'] + groups = [] + host_values = {} + + def visit(name, node, depth): + if not isinstance(node, dict): + raise ValueError('invalid inventory group') + hosts = node.get('hosts') or {} + children = node.get('children') or {} + if not isinstance(hosts, dict) or not isinstance(children, dict): + raise ValueError('invalid inventory hosts or children') + belongs = machine in hosts + if belongs: + value = hosts[machine] or {} + if not isinstance(value, dict): + raise ValueError('invalid inventory host variables') + host_values.update(value) + for child_name, child in sorted(children.items()): + if not re.fullmatch(r'[A-Za-z0-9_-]+', child_name): + raise ValueError('invalid inventory group name') + belongs = visit(child_name, child or {}, depth + 1) or belongs + if belongs and name not in ('all', 'ungrouped'): + groups.append((depth, name, node.get('vars') or {})) + return belongs + + if not visit('all', all_group, 0): + raise RuntimeError('machine is not in inventory; register it before publishing') + values = dict(defaults) + values.update(all_group.get('vars') or {}) + values.update(group_file(repo, 'all')) + for _, name, variables in sorted(groups): + if not isinstance(variables, dict): + raise ValueError('invalid inventory group variables') + values.update(variables) + values.update(group_file(repo, name)) + values.update(host_values) + values.update(read_yaml(repo / 'host_vars' / (machine + '.yml'))) + enabled = values['fleet_facts_publish'] + if isinstance(enabled, str): + if enabled.lower() not in ('true', 'false', 'yes', 'no', '1', '0'): + raise ValueError('invalid fleet_facts_publish value') + enabled = enabled.lower() in ('true', 'yes', '1') + if not isinstance(enabled, bool): + raise ValueError('invalid fleet_facts_publish value') + repo_name = values['fleet_facts_repo'] + branch = values['fleet_facts_branch'] + if not isinstance(repo_name, str) or not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repo_name): + raise ValueError('invalid fleet facts repository') + if not isinstance(branch, str) or not re.fullmatch(r'[A-Za-z0-9_./-]+', branch): + raise ValueError('invalid fleet facts branch') + return enabled, repo_name, branch, sorted(name for _, name, _ in groups) + + +def optional_file(root, name, strict=False): + try: + return (root / name.lstrip('/')).read_text(encoding='utf-8', errors='replace').strip() + except FileNotFoundError: + return '' + except OSError: + if strict: + raise RuntimeError('could not read an optional hardware attribute') from None + warn('could not read an optional hardware attribute') + return '' + + +def required_file(root, name): + value = optional_file(root, name) + if not value: + raise RuntimeError('required OS or hardware data unavailable: ' + name) + return value + + +def command(args): + try: + return subprocess.run(args, capture_output=True, text=True, timeout=20) + except (OSError, subprocess.TimeoutExpired): + return None + + +def read_os_release(root): + data = required_file(root, '/etc/os-release') + values = {} + for line in data.splitlines(): + if not line or line.startswith('#') or '=' not in line: + continue + key, value = line.split('=', 1) + if key in ('NAME', 'VERSION_ID', 'VERSION_CODENAME', 'UBUNTU_CODENAME'): + parsed = shlex.split(value, comments=False) + if len(parsed) > 1: + raise ValueError('invalid os-release value') + values[key] = parsed[0] if parsed else '' + if not values.get('NAME'): + raise ValueError('os-release has no NAME') + return values + + +def collect(machine, groups, home, termux=False, root=Path('/'), run=command, strict=False): + if termux: + release = run(['getprop', 'ro.build.version.release']) + if release is None or release.returncode or not release.stdout.strip(): + raise RuntimeError('cannot read Android release with getprop') + os_info = {'name': 'Android', 'version': clean(release.stdout), 'codename': ''} + android_model = run(['getprop', 'ro.product.model']) + if android_model is None or android_model.returncode: + if strict: + raise RuntimeError('Android model inspection failed') + warn('Android model unavailable') + model = '' + else: + model = clean(android_model.stdout) + else: + os_release = read_os_release(root) + os_info = {'name': clean(os_release.get('NAME')), 'version': clean(os_release.get('VERSION_ID')), 'codename': clean(os_release.get('VERSION_CODENAME') or os_release.get('UBUNTU_CODENAME'))} + model = '' + os_info['kernel'] = clean(platform.release()) + model = model or clean(optional_file(root, '/sys/firmware/devicetree/base/model', strict)) or clean(optional_file(root, '/sys/devices/virtual/dmi/id/product_name', strict)) + vendor = clean(optional_file(root, '/sys/devices/virtual/dmi/id/sys_vendor', strict)) + chassis = optional_file(root, '/sys/devices/virtual/dmi/id/chassis_type', strict) + form_factor = CHASSIS.get(int(chassis), '') if chassis.isdigit() else '' + cpuinfo = required_file(root, '/proc/cpuinfo') + cpu = '' + cpu_count = 0 + for line in cpuinfo.splitlines(): + key, separator, value = line.partition(':') + if not separator: + continue + if key.strip().lower() == 'processor' and value.strip().isdigit(): + cpu_count += 1 + if not cpu and key.strip().lower() in ('model name', 'hardware', 'processor') and not value.strip().isdigit(): + cpu = clean(value) + cpu_count = cpu_count or os.cpu_count() or 0 + if not cpu: + warn('CPU model unavailable') + meminfo = required_file(root, '/proc/meminfo') + match = re.search(r'^MemTotal:\s*(\d+)\s+kB\s*$', meminfo, re.MULTILINE) + if not match: + raise ValueError('MemTotal missing in /proc/meminfo') + memory_gb = round((int(match.group(1)) // 1024) / 1024, 1) + uptime = required_file(root, '/proc/uptime').split()[0] + uptime_days = round(float(uptime) / 86400, 1) + gpus = [] + pci = run(['lspci', '-mm']) + if pci is None or pci.returncode: + if strict: + raise RuntimeError('GPU inventory inspection failed (lspci missing or failed)') + warn('GPU inventory unavailable (lspci missing or failed)') + else: + for line in pci.stdout.splitlines(): + fields = shlex.split(line) + if len(fields) >= 4 and re.match(r'^(VGA|3D|Display)', fields[1], re.I): + gpu = clean(fields[2] + ' ' + fields[3]) + if gpu and not re.fullmatch(r'(?:\S+ )?Device [0-9a-fA-F]{4}', gpu): + gpus.append(gpu) + virt = run(['systemd-detect-virt', '--vm']) + if virt is None: + if strict: + raise RuntimeError('virtualization inspection unavailable') + warn('virtualization detection unavailable') + virtualization = '' + elif virt.returncode == 0: + virtualization = clean(virt.stdout) + elif virt.returncode == 1: + virtualization = '' + else: + if strict: + raise RuntimeError('virtualization inspection failed') + warn('virtualization detection failed') + virtualization = '' + last_path = home / '.cache/dotfiles/last-apply.json' + if last_path.exists(): + with last_path.open(encoding='utf-8') as stream: + last = json.load(stream) + if not isinstance(last, dict): + raise ValueError('invalid last-apply record') + else: + last = {} + at = last.get('timestamp', last.get('at', '')) + if at: + if not isinstance(at, str): + raise ValueError('invalid last-apply timestamp') + dt.datetime.fromisoformat(at.replace('Z', '+00:00')) + status = int(last.get('rc', last.get('exit_code', 1))) == 0 if last else '' + return { + 'schema': 1, 'host': machine, 'updated': dt.datetime.now(dt.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'groups': groups, 'os': os_info, + 'hardware': {'vendor': vendor, 'model': model, 'form_factor': form_factor, 'arch': clean(platform.machine()), 'cpu': cpu, 'cpu_threads': cpu_count, 'gpus': gpus, 'memory_gb': memory_gb}, + 'virtualization': virtualization, 'uptime_days': uptime_days, 'last_apply': {'at': at, 'ok': status}, + } + + +def main(): + repo, machine, termux = Path(sys.argv[1]), sys.argv[2], sys.argv[3] == '1' + enabled, target, branch, groups = config(repo, machine) + if not enabled: + print('SKIPPED: fleet facts publishing disabled for this host') + return 0 + if len(sys.argv) > 4 and sys.argv[4] == '--check': + auth = command(['gh', 'auth', 'status']) + if auth is None or auth.returncode: + raise RuntimeError('gh authentication unavailable; cannot inspect fleet facts publication') + collect(machine, groups, Path(os.environ['HOME']), termux, strict=True) + # Every apply publishes a new timestamp, even when other facts are unchanged. + print(f'PLAN: publish facts/{machine}.json to {target} branch {branch} (updated timestamp)') + return 0 + auth = command(['gh', 'auth', 'status']) + if auth is None or auth.returncode: + print('PARTIAL: gh is not authenticated; fleet facts not published') + return 0 + facts = collect(machine, groups, Path(os.environ['HOME']), termux) + path = f'repos/{target}/contents/facts/{machine}.json' + prior = command(['gh', 'api', f'{path}?ref={quote(branch, safe="")}', '--jq', '.sha']) + sha = prior.stdout.strip() if prior is not None and prior.returncode == 0 else '' + content = base64.b64encode((json.dumps(facts, indent=2, ensure_ascii=True) + '\n').encode()).decode() + body = {'message': f'facts: {machine}', 'content': content, 'branch': branch} + if sha: + body['sha'] = sha + try: + result = subprocess.run(['gh', 'api', '-X', 'PUT', path, '--input', '-', '--jq', '.commit.sha'], input=json.dumps(body), capture_output=True, text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired): + result = None + if result is None or result.returncode: + warn('could not publish fleet facts (GitHub API request failed)') + else: + print('published fleet facts') + return 0 + + +if __name__ == '__main__': + try: + sys.exit(main()) + except RuntimeError as error: + # RuntimeError messages above are fixed diagnostics, never raw values. + warn('collection/configuration failed: ' + str(error)) + sys.exit(1) + except Exception as error: + # Do not expose YAML parse excerpts, host data, or command output. + warn('collection/configuration failed: ' + error.__class__.__name__) + sys.exit(1) diff --git a/units/fleet-facts.sh b/units/fleet-facts.sh new file mode 100755 index 00000000..30824e97 --- /dev/null +++ b/units/fleet-facts.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then + note "SKIPPED: fleet facts excluded on retired VPS" + finish +fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + step "check fleet facts publication" env PYTHONDONTWRITEBYTECODE=1 python3 "$DOTFILES_DIR/units/fleet-facts.py" "$DOTFILES_DIR" "$MACHINE" "$IS_TERMUX" --check + finish +fi +step "collect and publish fleet facts" python3 "$DOTFILES_DIR/units/fleet-facts.py" "$DOTFILES_DIR" "$MACHINE" "$IS_TERMUX" +finish diff --git a/roles/fleet_facts/defaults/main.yml b/units/fleet-facts/defaults.yml similarity index 100% rename from roles/fleet_facts/defaults/main.yml rename to units/fleet-facts/defaults.yml diff --git a/units/fonts.sh b/units/fonts.sh index 7d114bd4..1db2343c 100755 --- a/units/fonts.sh +++ b/units/fonts.sh @@ -7,6 +7,39 @@ set -uo pipefail dir="$HOME/.local/share/fonts" font="$dir/JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf" +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if { [ -e "$dir" ] || [ -L "$dir" ]; } && [ ! -d "$dir" ]; then + note "FAIL: font directory path is not a directory: $dir" + exit 1 + fi + [ -d "$dir" ] || note "CHANGE: create font directory $dir" + desktop_hint="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" + case "$desktop_hint" in + 1|true) desktop=1 ;; + 0|false) desktop=0 ;; + *) + if ! command -v yq >/dev/null 2>&1; then + note "FAIL: yq required to inspect desktop membership" + exit 1 + fi + export DOTFILES_MACHINE="$MACHINE" + membership="$(yq '(.all.children.desktop.hosts // {}) | has(strenv(DOTFILES_MACHINE))' "$DOTFILES_DIR/inventory.yml" 2>/dev/null)" || { + note "FAIL: cannot inspect desktop membership" + exit 1 + } + case "$membership" in true) desktop=1 ;; false) desktop=0 ;; *) note "FAIL: invalid desktop membership"; exit 1 ;; esac + ;; + esac + if [ "$desktop" = 0 ]; then + note "SKIPPED: not a desktop; no font install" + elif [ -f "$font" ]; then + note "ok JetBrains Mono Nerd Font already installed" + else + note "CHANGE: download and install JetBrains Mono Nerd Font; refresh font cache" + fi + exit 0 +fi + step "font directory" mkdir -p "$dir" if ! is_desktop; then note "SKIPPED: not a desktop; no font install" diff --git a/units/forgejo-registry.sh b/units/forgejo-registry.sh index a46489bf..ccb7a19b 100755 --- a/units/forgejo-registry.sh +++ b/units/forgejo-registry.sh @@ -15,6 +15,10 @@ if ! command -v podman >/dev/null 2>&1; then note "SKIPPED: podman not installed" exit 0 fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + note "FAIL cannot safely inspect $FORGEJO_REGISTRY login: Podman credential helpers may prompt; registry check unsupported" + exit 1 +fi if podman login --get-login "$FORGEJO_REGISTRY" >/dev/null 2>&1; then note "ok already logged in to $FORGEJO_REGISTRY" exit 0 diff --git a/units/github.sh b/units/github.sh index 2cddf328..79b25198 100755 --- a/units/github.sh +++ b/units/github.sh @@ -12,6 +12,42 @@ if ! command -v gh >/dev/null 2>&1; then note "SKIPPED: gh not installed (just packages)" exit 0 fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + check_known_hosts() { + local dir="$HOME/.ssh" path="$HOME/.ssh/known_hosts" mode rc + if [ ! -e "$dir" ]; then + note "would create $dir (mode 0700)" + elif [ ! -d "$dir" ] || ! mode="$(stat -Lc '%a' "$dir")"; then + note "FAIL cannot inspect $dir" + return 1 + elif [ "$mode" != 700 ]; then + note "would set $dir mode 0700" + fi + if [ ! -e "$path" ]; then + note "would add GitHub host key to $path" + return 0 + fi + if [ ! -f "$path" ]; then + note "FAIL cannot inspect $path" + return 1 + fi + grep -qF "$GITHUB_HOST_KEY" "$path" 2>/dev/null; rc=$? + case "$rc" in + 0) note "ok GitHub host key" ;; + 1) note "would add GitHub host key to $path" ;; + *) note "FAIL cannot inspect $path"; return 1 ;; + esac + } + step "check github known_hosts" check_known_hosts + if [ "${UNITS_NO_SECRETS:-0}" = 1 ]; then + note "SKIPPED: GitHub authentication and SSH key registration (--no-secrets)" + else + note "FAIL cannot safely inspect GitHub authentication or registered keys: gh may prompt via the OS keyring and Bitwarden reads may persist audit events" + UNIT_FAILED=1 + fi + finish +fi + known_hosts() { install -d -m 0700 "$HOME/.ssh" || return 1 diff --git a/units/gnome-avatar.py b/units/gnome-avatar.py new file mode 100755 index 00000000..c50d63bc --- /dev/null +++ b/units/gnome-avatar.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Install the desktop avatar and update AccountsService as root.""" + +import grp +import os +from pathlib import Path +import pwd +import re +import sys +import tempfile + +ICON = Path('/var/lib/AccountsService/icons/james') +ACCOUNT = Path('/var/lib/AccountsService/users/james') +GDM_CACHE = Path('/var/lib/gdm3/.cache') +ICON_VALUE = '/var/lib/AccountsService/icons/james' + + +def write_file(path, content, owner=None): + """Replace changed content atomically, retaining existing ownership unless specified.""" + try: + previous = path.stat() + unchanged = path.read_bytes() == content + except FileNotFoundError: + previous = None + unchanged = False + if not unchanged: + fd, name = tempfile.mkstemp(prefix=f'.{path.name}.', dir=path.parent) + try: + with os.fdopen(fd, 'wb') as stream: + stream.write(content) + if owner is not None: + os.chown(name, *owner) + elif previous is not None: + os.chown(name, previous.st_uid, previous.st_gid) + os.chmod(name, 0o644) + os.replace(name, path) + finally: + if os.path.exists(name): + os.unlink(name) + else: + if owner is not None and (previous.st_uid, previous.st_gid) != owner: + os.chown(path, *owner) + if previous.st_mode & 0o7777 != 0o644: + os.chmod(path, 0o644) + + +def account_with_icon(text): + """Set User.Icon without rewriting unrelated account options or comments.""" + lines = text.splitlines(keepends=True) + output = [] + in_user = False + saw_user = False + wrote_icon = False + for line in lines: + section = re.match(r'^\s*\[([^]]+)\]', line) + if section: + if in_user and not wrote_icon: + if output and not output[-1].endswith('\n'): + output[-1] += '\n' + output.append(f'Icon={ICON_VALUE}\n') + wrote_icon = True + in_user = section.group(1) == 'User' + saw_user = saw_user or in_user + if in_user and re.match(r'^\s*Icon\s*=', line): + if not wrote_icon: + output.append(f'Icon={ICON_VALUE}\n') + wrote_icon = True + continue + output.append(line) + if in_user and not wrote_icon: + if output and not output[-1].endswith('\n'): + output[-1] += '\n' + output.append(f'Icon={ICON_VALUE}\n') + elif not saw_user: + if output and not output[-1].endswith('\n'): + output[-1] += '\n' + output.extend(['[User]\n', f'Icon={ICON_VALUE}\n']) + return ''.join(output) + + +def install_avatar(source): + data = source.read_bytes() + write_file(ICON, data) + try: + gdm = pwd.getpwnam('gdm') + except KeyError: + pass + else: + group = grp.getgrnam('gdm') + GDM_CACHE.mkdir(parents=True, exist_ok=True, mode=0o755) + write_file(GDM_CACHE / 'james', data, (gdm.pw_uid, group.gr_gid)) + try: + account = ACCOUNT.read_text() + except FileNotFoundError: + account = '' + write_file(ACCOUNT, account_with_icon(account).encode()) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + raise SystemExit('usage: gnome-avatar.py PROFILE_PNG') + install_avatar(Path(sys.argv[1])) diff --git a/units/gnome.sh b/units/gnome.sh new file mode 100755 index 00000000..a25629d6 --- /dev/null +++ b/units/gnome.sh @@ -0,0 +1,346 @@ +#!/usr/bin/env bash +# GNOME desktop configuration; session commands run as the invoking desktop user. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [ "${UNITS_CHECK:-0}" = 1 ]; then export PYTHONDONTWRITEBYTECODE=1; fi + +if skipped_by_profile skip_gnome; then + note "SKIPPED: skip_gnome or vm-test" + finish +fi + +# Unlike is_desktop(), distinguish a missing/broken inventory reader from a +# negative result: silently skipping a desktop would leave it unconfigured. +desktop="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" +case "$desktop" in + 1|true) ;; + 0|false) note "SKIPPED: not a desktop"; finish ;; + '') + if ! command -v yq >/dev/null 2>&1 || ! desktop="$(yq ".all.children.desktop.hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml")"; then + step "determine desktop membership (yq required)" false + finish + fi + case "$desktop" in + true) ;; + false) note "SKIPPED: not a desktop"; finish ;; + *) step "determine desktop membership (invalid inventory result)" false; finish ;; + esac + ;; + *) step "determine desktop membership (invalid override)" false; finish ;; +esac + +# Do not let GLib autolaunch a throwaway D-Bus session when run over SSH. +# An existing runtime bus belongs to the invoking user and is safe to use. +if [ -z "${XDG_RUNTIME_DIR:-}" ] && [ -S "/run/user/$UID/bus" ]; then + export XDG_RUNTIME_DIR="/run/user/$UID" +fi +if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -n "${XDG_RUNTIME_DIR:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then + export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus" +fi +session_ok=0 +if [ -n "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [[ "$DBUS_SESSION_BUS_ADDRESS" != autolaunch:* ]] && gdbus call --session --dest org.freedesktop.DBus --object-path /org/freedesktop/DBus --method org.freedesktop.DBus.Peer.Ping >/dev/null 2>&1; then + session_ok=1 +else + note "GNOME user D-Bus session unavailable; session settings require a reachable user bus and gdbus" +fi +require_session() { [ "$session_ok" = 1 ]; } + +assets="$DOTFILES_DIR/units/gnome" +dconf_dir="$HOME/.config/dconf" +keybindings="$dconf_dir/gnome-keybindings.ini" +pending="$dconf_dir/.gnome-keybindings.pending" + +install_keybindings() { + mkdir -p "$dconf_dir" && chmod 0755 "$dconf_dir" || return 1 + if ! cmp -s "$assets/gnome-keybindings.ini" "$keybindings" || [ "$(stat -c %a "$keybindings" 2>/dev/null)" != 644 ]; then + # Keep the notification until the live user's dconf load succeeds. + : > "$pending" || return 1 + fi + install_if_changed "$assets/gnome-keybindings.ini" "$keybindings" 0644 +} +load_keybindings() { + require_session && dconf load /org/gnome/settings-daemon/plugins/media-keys/ < "$keybindings" && rm -f "$pending" +} +if [ "${UNITS_CHECK:-0}" != 1 ]; then + if install_keybindings; then + note "ok GNOME keybindings file" + if [ -f "$pending" ]; then + step "load GNOME keybindings in user session" load_keybindings + fi + else + step "GNOME keybindings file" false + fi +fi + +# Values come from the same inventory variables as the former role, not a +# second hard-coded list. Host variables override group variables. +brew_prefix="" +extensions_value="" +read_configuration() { + local host_file="$DOTFILES_DIR/host_vars/$MACHINE.yml" host_extensions="null" + command -v yq >/dev/null 2>&1 || { echo "yq is required to read GNOME configuration" >&2; return 1; } + brew_prefix="$(host_var brew_prefix)" + if [ -z "$brew_prefix" ]; then + brew_prefix="$(yq -r '.brew_prefix' "$DOTFILES_DIR/group_vars/all.yml")" || return 1 + fi + [ -n "$brew_prefix" ] && [ "$brew_prefix" != null ] || return 1 + if [ -f "$host_file" ]; then + host_extensions="$(yq -o=json -I=0 '.gnome_enabled_extensions' "$host_file")" || return 1 + fi + if [ "$host_extensions" != null ]; then + extensions_value="$host_extensions" + else + extensions_value="$(yq -o=json -I=0 '.gnome_enabled_extensions' "$DOTFILES_DIR/group_vars/all.yml")" || return 1 + fi + case "$extensions_value" in \[*\]) ;; *) echo "gnome_enabled_extensions must be a list" >&2; return 1 ;; esac +} +config_ok=0 +if read_configuration; then + config_ok=1 + note "ok read GNOME configuration" +else + step "read GNOME configuration" false +fi + +if [ "${UNITS_CHECK:-0}" = 1 ]; then + check_gnome_files() { + python3 - "$assets" "$HOME" "$DOTFILES_DIR/units/gnome-avatar.py" <<'PY' +import importlib.util +import grp +from pathlib import Path +import pwd +import stat +import sys +sys.excepthook = lambda kind, value, traceback: print('GNOME file inspection failed (' + kind.__name__ + ')', file=sys.stderr) + +assets, home, avatar_script = map(Path, sys.argv[1:]) +def directory(path, label): + try: + info = path.stat() + changed = not stat.S_ISDIR(info.st_mode) or info.st_mode & 0o7777 != 0o755 + except FileNotFoundError: + changed = True + if changed: + print('WOULD create or fix ' + label) +def file(path, data, mode, label): + try: + info = path.stat() + changed = not stat.S_ISREG(info.st_mode) or path.read_bytes() != data or info.st_mode & 0o7777 != mode + except FileNotFoundError: + changed = True + if changed: + print('WOULD update ' + label) + return changed + +dconf = home / '.config/dconf' +directory(dconf, 'dconf directory') +changed = file(dconf / 'gnome-keybindings.ini', (assets / 'gnome-keybindings.ini').read_bytes(), 0o644, 'GNOME keybindings file') +if changed or (dconf / '.gnome-keybindings.pending').is_file(): + print('WOULD load GNOME keybindings in user session') +extensions = home / '.local/share/gnome-shell/extensions' +directory(extensions, 'GNOME extensions directory') +for uuid in ('copyous@boerdereinar.dev', 'papershell@lalovene.github.com', 'paperwm@paperwm.github.com'): + source, target = assets / 'extensions' / uuid, extensions / uuid + if not source.is_dir(): + raise FileNotFoundError('missing extension assets: ' + uuid) + directory(target, 'extension ' + uuid) + for path in source.rglob('*'): + dest = target / path.relative_to(source) + label = 'extension ' + uuid + '/' + str(path.relative_to(source)) + if path.is_dir(): + directory(dest, label) + elif path.is_file(): + file(dest, path.read_bytes(), 0o644, label) + else: + raise ValueError('unsupported extension asset: ' + str(path)) + +# The image is downloaded during apply, so its contents cannot be compared offline. +sys.dont_write_bytecode = True +spec = importlib.util.spec_from_file_location('gnome_avatar', avatar_script) +avatar = importlib.util.module_from_spec(spec) +spec.loader.exec_module(avatar) +try: + account = avatar.ACCOUNT.read_text() +except FileNotFoundError: + account = '' +file(avatar.ACCOUNT, avatar.account_with_icon(account).encode(), 0o644, 'AccountsService account') +try: + info = avatar.ICON.stat() + if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o7777 != 0o644: + print('WOULD update AccountsService avatar') +except FileNotFoundError: + print('WOULD install AccountsService avatar') +try: + gdm = pwd.getpwnam('gdm') +except KeyError: + pass +else: + gid = grp.getgrnam('gdm').gr_gid + try: + info = (avatar.GDM_CACHE / 'james').stat() + if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o7777 != 0o644 or (info.st_uid, info.st_gid) != (gdm.pw_uid, gid): + print('WOULD update GDM avatar metadata') + except FileNotFoundError: + print('WOULD install GDM avatar') + print('PLAN refresh GDM avatar; remote image not inspected') +print('PLAN refresh AccountsService avatar; remote image not inspected') +PY + } + check_gnome_session() { + require_session || return 1 + python3 - "$assets" "$brew_prefix" "$extensions_value" "$config_ok" <<'PY' +import ast +import configparser +import io +import json +from pathlib import Path +import subprocess +import sys +sys.excepthook = lambda kind, value, traceback: print('GNOME session inspection failed (' + kind.__name__ + ')', file=sys.stderr) + +assets, prefix = Path(sys.argv[1]), sys.argv[2] +config_ok = sys.argv[4] == '1' +desired_extensions = json.loads(sys.argv[3]) if config_ok else None +def command(*args): + return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True).stdout.strip() +def settings(schema, key): + return command('gsettings', 'get', schema, key) +def ini(text): + parser = configparser.ConfigParser(interpolation=None, strict=True) + parser.optionxform = str + parser.read_file(io.StringIO(text)) + return parser + +def ptyxis(): + uuid = ast.literal_eval(settings('org.gnome.Ptyxis', 'default-profile-uuid')) + current_cmd = ast.literal_eval(settings('org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/' + uuid + '/', 'custom-command')) + font = ast.literal_eval(settings('org.gnome.Ptyxis', 'font-name')) + if current_cmd != prefix + '/bin/fish' or font != 'JetBrainsMono Nerd Font 12': + print('WOULD configure Ptyxis profile') + +def extension_settings(): + desired = ini((assets / 'gnome-extensions-dconf.dump').read_text()) + actual = ini(command('dconf', 'dump', '/org/gnome/shell/extensions/')) + if any(not actual.has_option(section, key) or actual.get(section, key) != value + for section in desired.sections() for key, value in desired.items(section)): + print('WOULD load GNOME extension settings') + +def enabled_extensions(): + value = settings('org.gnome.shell', 'enabled-extensions') + if value.startswith('@as '): + value = value[4:] + if ast.literal_eval(value) != desired_extensions: + print('WOULD update enabled GNOME extensions') + +def rdp(): + if 'Status: enabled' not in command('grdctl', 'rdp', 'status'): + print('WOULD enable GNOME RDP protocol') + +def remote_service(): + state = command('systemctl', '--user', 'show', '--property=LoadState', '--property=UnitFileState', + '--property=ActiveState', 'gnome-remote-desktop.service') + properties = dict(line.split('=', 1) for line in state.splitlines() if '=' in line) + if not all(key in properties for key in ('LoadState', 'UnitFileState', 'ActiveState')): + raise ValueError('incomplete systemd service state') + if (properties['LoadState'], properties['UnitFileState'], properties['ActiveState']) != ('loaded', 'enabled', 'active'): + print('WOULD enable and start GNOME Remote Desktop user service') + +failed = False +checks = [('extension settings', extension_settings), ('RDP', rdp), ('remote service', remote_service)] +if config_ok: + checks.extend((('Ptyxis profile', ptyxis), ('enabled extensions', enabled_extensions))) +for label, inspect in checks: + try: + inspect() + except Exception as error: + print('FAIL inspect ' + label + ' (' + type(error).__name__ + ')', file=sys.stderr) + failed = True +if failed: + sys.exit(1) +PY + } + step "inspect GNOME files" check_gnome_files + step "inspect GNOME user session" check_gnome_session + finish +fi + +configure_ptyxis() { + local uuid current_cmd current_font + require_session || return 1 + uuid="$(gsettings get org.gnome.Ptyxis default-profile-uuid)" || return 1 + uuid="${uuid//\'/}" + current_cmd="$(gsettings get "org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${uuid}/" custom-command 2>/dev/null)" || current_cmd="" + current_font="$(gsettings get org.gnome.Ptyxis font-name 2>/dev/null)" || current_font="" + current_cmd="${current_cmd//\'/}"; current_font="${current_font//\'/}" + if [ "$current_cmd" != "$brew_prefix/bin/fish" ] || [ "$current_font" != 'JetBrainsMono Nerd Font 12' ]; then + gsettings set "org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${uuid}/" use-custom-command true && + gsettings set "org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${uuid}/" custom-command "$brew_prefix/bin/fish" && + gsettings set org.gnome.Ptyxis use-system-font false && + gsettings set org.gnome.Ptyxis font-name 'JetBrainsMono Nerd Font 12' + fi +} +if [ "$config_ok" = 1 ]; then soft "Ptyxis profile" configure_ptyxis; fi + +install_avatar() { + local tmp rc=0 + tmp="$(mktemp -d)" || return 1 + if curl -fsSL https://reilly.asia/profile.png -o "$tmp/profile.png"; then + if [ "$EUID" = 0 ]; then + python3 "$DOTFILES_DIR/units/gnome-avatar.py" "$tmp/profile.png" || rc=1 + else + sudo -n python3 "$DOTFILES_DIR/units/gnome-avatar.py" "$tmp/profile.png" || rc=1 + fi + else + rc=1 + fi + rm -rf "$tmp" + return "$rc" +} +step "AccountsService avatar" install_avatar + +deploy_extensions() { + local uuid src dest file rel + local -a entries + mkdir -p "$HOME/.local/share/gnome-shell/extensions" && chmod 0755 "$HOME/.local/share/gnome-shell/extensions" || return 1 + shopt -s globstar dotglob nullglob + for uuid in copyous@boerdereinar.dev papershell@lalovene.github.com paperwm@paperwm.github.com; do + src="$assets/extensions/$uuid" + dest="$HOME/.local/share/gnome-shell/extensions/$uuid" + [ -d "$src" ] || { echo "missing extension assets: $src" >&2; return 1; } + mkdir -p "$dest" && chmod 0755 "$dest" || return 1 + entries=("$src"/**) + for file in "${entries[@]}"; do + rel="${file#"$src"/}" + if [ -d "$file" ]; then + mkdir -p "$dest/$rel" && chmod 0755 "$dest/$rel" || return 1 + elif [ -f "$file" ]; then + install_if_changed "$file" "$dest/$rel" 0644 || return 1 + else + echo "unsupported extension asset: $file" >&2 + return 1 + fi + done + done +} +step "deploy user GNOME extensions" deploy_extensions +load_extension_settings() { + require_session && dconf load /org/gnome/shell/extensions/ < "$assets/gnome-extensions-dconf.dump" +} +soft "load GNOME extension settings in user session" load_extension_settings +if [ "$config_ok" = 1 ]; then + enable_extensions() { require_session && gsettings set org.gnome.shell enabled-extensions "$extensions_value"; } + soft "enable GNOME extensions in user session" enable_extensions +fi + +enable_rdp() { + local status + require_session || return 1 + status="$(grdctl rdp status 2>/dev/null)" || status="" + case "$status" in *'Status: enabled'*) return 0 ;; esac + grdctl rdp enable +} +soft "enable GNOME RDP protocol" enable_rdp +enable_remote_service() { require_session && systemctl --user enable --now gnome-remote-desktop.service; } +soft "enable GNOME Remote Desktop user service" enable_remote_service +finish diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/extension.js b/units/gnome/extensions/copyous@boerdereinar.dev/extension.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/extension.js rename to units/gnome/extensions/copyous@boerdereinar.dev/extension.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/character-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/character-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/character-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/character-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/check-round-outline-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/check-round-outline-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/check-round-outline-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/check-round-outline-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-disabled-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-disabled-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-disabled-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-disabled-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/clipboard-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/color-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/color-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/color-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/color-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/duration-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/duration-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/duration-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/duration-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/file-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/file-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/file-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/file-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-code-legacy-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-code-legacy-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-code-legacy-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-code-legacy-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/folder-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/image-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/image-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/image-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/image-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-disabled-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-disabled-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-disabled-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-disabled-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/incognito-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/keyboard-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/keyboard-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/keyboard-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/keyboard-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/link-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/link-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/link-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/link-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/pin-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/pin-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/pin-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/pin-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/search-clipboard-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/search-clipboard-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/search-clipboard-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/search-clipboard-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/settings-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/settings-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/settings-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/settings-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/tag-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/tag-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/tag-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/tag-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/text-symbolic.svg b/units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/text-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/text-symbolic.svg rename to units/gnome/extensions/copyous@boerdereinar.dev/icons/hicolor/scalable/actions/text-symbolic.svg diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/actions.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/common/actions.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/actions.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/common/actions.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/color.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/common/color.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/color.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/common/color.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/constants.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/common/constants.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/constants.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/common/constants.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/dbus.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/common/dbus.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/dbus.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/common/dbus.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/gjs.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/common/gjs.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/gjs.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/common/gjs.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/glob.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/common/glob.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/glob.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/common/glob.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/icons.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/common/icons.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/icons.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/common/icons.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/settings.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/common/settings.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/settings.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/common/settings.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/sound.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/common/sound.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/common/sound.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/common/sound.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/database.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/database/database.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/database.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/database/database.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/entryTracker.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/database/entryTracker.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/entryTracker.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/database/entryTracker.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/gda.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/database/gda.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/gda.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/database/gda.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/json.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/database/json.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/json.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/database/json.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/memory.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/database/memory.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/database/memory.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/database/memory.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/actor.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/actor.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/actor.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/actor.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/clipboard.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/clipboard.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/clipboard.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/clipboard.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/compatibility.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/compatibility.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/compatibility.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/compatibility.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/keyboard.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/keyboard.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/keyboard.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/keyboard.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/link.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/link.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/link.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/link.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/notifications.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/notifications.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/notifications.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/notifications.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/shortcuts.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/shortcuts.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/shortcuts.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/shortcuts.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/theme.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/theme.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/misc/theme.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/misc/theme.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDefaults.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDefaults.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDefaults.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDefaults.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDialog.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDialog.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDialog.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionDialog.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionSubMenuDialog.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionSubMenuDialog.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionSubMenuDialog.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionSubMenuDialog.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsGroup.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsGroup.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsGroup.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsGroup.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsPage.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsPage.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsPage.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/actions/actionsPage.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/dialogCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/dialogCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/dialogCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/dialogCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/headerCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/headerCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/headerCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/headerCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/characterItemCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/characterItemCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/characterItemCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/characterItemCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/codeItemCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/codeItemCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/codeItemCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/codeItemCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/exclusions.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/exclusions.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/exclusions.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/exclusions.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/fileItemCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/fileItemCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/fileItemCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/fileItemCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/imageItemCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/imageItemCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/imageItemCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/imageItemCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/linkItemCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/linkItemCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/linkItemCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/linkItemCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/textItemCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/textItemCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/textItemCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/items/textItemCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemsCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemsCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemsCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/itemsCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/profiles.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/profiles.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/profiles.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/profiles.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/themeCustomization.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/themeCustomization.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/customization/themeCustomization.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/customization/themeCustomization.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependencies.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependencies.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependencies.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependencies.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependenciesSettings.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependenciesSettings.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependenciesSettings.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/dependencies/dependenciesSettings.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/appExclusionSettings.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/appExclusionSettings.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/appExclusionSettings.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/appExclusionSettings.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/behaviorSettings.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/behaviorSettings.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/behaviorSettings.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/behaviorSettings.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/feedbackSettings.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/feedbackSettings.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/feedbackSettings.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/feedbackSettings.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/historySettings.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/historySettings.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/historySettings.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/historySettings.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/locationsGroup.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/locationsGroup.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/general/locationsGroup.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/general/locationsGroup.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/dialogShortcuts.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/dialogShortcuts.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/dialogShortcuts.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/dialogShortcuts.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/itemShortcuts.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/itemShortcuts.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/itemShortcuts.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/itemShortcuts.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/navigationShortcuts.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/navigationShortcuts.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/navigationShortcuts.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/navigationShortcuts.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/popupMenuShortcuts.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/popupMenuShortcuts.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/popupMenuShortcuts.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/popupMenuShortcuts.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/searchShortcuts.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/searchShortcuts.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/searchShortcuts.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/searchShortcuts.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/shortcutRow.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/shortcutRow.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/shortcutRow.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/shortcuts/shortcutRow.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/utils.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/utils.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/preferences/utils.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/preferences/utils.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/clipboardDialog.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/clipboardDialog.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/clipboardDialog.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/clipboardDialog.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollContainer.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollContainer.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollContainer.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollContainer.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollView.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollView.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollView.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/clipboardScrollView.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/actionMenu.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/actionMenu.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/actionMenu.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/actionMenu.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/clipboardItemMenu.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/clipboardItemMenu.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/clipboardItemMenu.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/clipboardItemMenu.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/codeLabel.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/codeLabel.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/codeLabel.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/codeLabel.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/contentInfo.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/contentInfo.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/contentInfo.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/contentInfo.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/contentPreview.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/contentPreview.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/contentPreview.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/contentPreview.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/editDialog.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/editDialog.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/editDialog.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/editDialog.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/label.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/label.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/label.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/label.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/qrCodeDialog.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/qrCodeDialog.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/qrCodeDialog.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/qrCodeDialog.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/shortcutLabel.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/shortcutLabel.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/shortcutLabel.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/shortcutLabel.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/tagsItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/tagsItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/components/tagsItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/components/tagsItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/indicator.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/indicator.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/indicator.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/indicator.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/characterItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/characterItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/characterItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/characterItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItemHeader.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItemHeader.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItemHeader.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/clipboardItemHeader.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/codeItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/codeItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/codeItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/codeItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/colorItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/colorItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/colorItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/colorItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/fileItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/fileItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/fileItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/fileItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/filesItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/filesItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/filesItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/filesItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/imageItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/imageItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/imageItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/imageItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/linkItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/linkItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/linkItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/linkItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/statusItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/statusItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/statusItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/statusItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/textItem.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/textItem.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/items/textItem.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/items/textItem.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/layout.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/layout.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/layout.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/layout.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/searchEntry.js b/units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/searchEntry.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/lib/ui/searchEntry.js rename to units/gnome/extensions/copyous@boerdereinar.dev/lib/ui/searchEntry.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/de/LC_MESSAGES/copyous@boerdereinar.dev.mo b/units/gnome/extensions/copyous@boerdereinar.dev/locale/de/LC_MESSAGES/copyous@boerdereinar.dev.mo similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/de/LC_MESSAGES/copyous@boerdereinar.dev.mo rename to units/gnome/extensions/copyous@boerdereinar.dev/locale/de/LC_MESSAGES/copyous@boerdereinar.dev.mo diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/fr/LC_MESSAGES/copyous@boerdereinar.dev.mo b/units/gnome/extensions/copyous@boerdereinar.dev/locale/fr/LC_MESSAGES/copyous@boerdereinar.dev.mo similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/fr/LC_MESSAGES/copyous@boerdereinar.dev.mo rename to units/gnome/extensions/copyous@boerdereinar.dev/locale/fr/LC_MESSAGES/copyous@boerdereinar.dev.mo diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/it/LC_MESSAGES/copyous@boerdereinar.dev.mo b/units/gnome/extensions/copyous@boerdereinar.dev/locale/it/LC_MESSAGES/copyous@boerdereinar.dev.mo similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/it/LC_MESSAGES/copyous@boerdereinar.dev.mo rename to units/gnome/extensions/copyous@boerdereinar.dev/locale/it/LC_MESSAGES/copyous@boerdereinar.dev.mo diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/pl/LC_MESSAGES/copyous@boerdereinar.dev.mo b/units/gnome/extensions/copyous@boerdereinar.dev/locale/pl/LC_MESSAGES/copyous@boerdereinar.dev.mo similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/pl/LC_MESSAGES/copyous@boerdereinar.dev.mo rename to units/gnome/extensions/copyous@boerdereinar.dev/locale/pl/LC_MESSAGES/copyous@boerdereinar.dev.mo diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/pt_BR/LC_MESSAGES/copyous@boerdereinar.dev.mo b/units/gnome/extensions/copyous@boerdereinar.dev/locale/pt_BR/LC_MESSAGES/copyous@boerdereinar.dev.mo similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/pt_BR/LC_MESSAGES/copyous@boerdereinar.dev.mo rename to units/gnome/extensions/copyous@boerdereinar.dev/locale/pt_BR/LC_MESSAGES/copyous@boerdereinar.dev.mo diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/ru/LC_MESSAGES/copyous@boerdereinar.dev.mo b/units/gnome/extensions/copyous@boerdereinar.dev/locale/ru/LC_MESSAGES/copyous@boerdereinar.dev.mo similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/ru/LC_MESSAGES/copyous@boerdereinar.dev.mo rename to units/gnome/extensions/copyous@boerdereinar.dev/locale/ru/LC_MESSAGES/copyous@boerdereinar.dev.mo diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/tr/LC_MESSAGES/copyous@boerdereinar.dev.mo b/units/gnome/extensions/copyous@boerdereinar.dev/locale/tr/LC_MESSAGES/copyous@boerdereinar.dev.mo similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/tr/LC_MESSAGES/copyous@boerdereinar.dev.mo rename to units/gnome/extensions/copyous@boerdereinar.dev/locale/tr/LC_MESSAGES/copyous@boerdereinar.dev.mo diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/zh_CN/LC_MESSAGES/copyous@boerdereinar.dev.mo b/units/gnome/extensions/copyous@boerdereinar.dev/locale/zh_CN/LC_MESSAGES/copyous@boerdereinar.dev.mo similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/locale/zh_CN/LC_MESSAGES/copyous@boerdereinar.dev.mo rename to units/gnome/extensions/copyous@boerdereinar.dev/locale/zh_CN/LC_MESSAGES/copyous@boerdereinar.dev.mo diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/metadata.json b/units/gnome/extensions/copyous@boerdereinar.dev/metadata.json similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/metadata.json rename to units/gnome/extensions/copyous@boerdereinar.dev/metadata.json diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/prefs.js b/units/gnome/extensions/copyous@boerdereinar.dev/prefs.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/prefs.js rename to units/gnome/extensions/copyous@boerdereinar.dev/prefs.js diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/resources.gresource b/units/gnome/extensions/copyous@boerdereinar.dev/resources.gresource similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/resources.gresource rename to units/gnome/extensions/copyous@boerdereinar.dev/resources.gresource diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/schemas/gschemas.compiled b/units/gnome/extensions/copyous@boerdereinar.dev/schemas/gschemas.compiled similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/schemas/gschemas.compiled rename to units/gnome/extensions/copyous@boerdereinar.dev/schemas/gschemas.compiled diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/schemas/org.gnome.shell.extensions.copyous.gschema.xml b/units/gnome/extensions/copyous@boerdereinar.dev/schemas/org.gnome.shell.extensions.copyous.gschema.xml similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/schemas/org.gnome.shell.extensions.copyous.gschema.xml rename to units/gnome/extensions/copyous@boerdereinar.dev/schemas/org.gnome.shell.extensions.copyous.gschema.xml diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/theme.gresource b/units/gnome/extensions/copyous@boerdereinar.dev/theme.gresource similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/theme.gresource rename to units/gnome/extensions/copyous@boerdereinar.dev/theme.gresource diff --git a/roles/gnome/files/extensions/copyous@boerdereinar.dev/thirdparty/qrcodegen.js b/units/gnome/extensions/copyous@boerdereinar.dev/thirdparty/qrcodegen.js similarity index 100% rename from roles/gnome/files/extensions/copyous@boerdereinar.dev/thirdparty/qrcodegen.js rename to units/gnome/extensions/copyous@boerdereinar.dev/thirdparty/qrcodegen.js diff --git a/roles/gnome/files/extensions/papershell@lalovene.github.com/extension.js b/units/gnome/extensions/papershell@lalovene.github.com/extension.js similarity index 100% rename from roles/gnome/files/extensions/papershell@lalovene.github.com/extension.js rename to units/gnome/extensions/papershell@lalovene.github.com/extension.js diff --git a/roles/gnome/files/extensions/papershell@lalovene.github.com/metadata.json b/units/gnome/extensions/papershell@lalovene.github.com/metadata.json similarity index 100% rename from roles/gnome/files/extensions/papershell@lalovene.github.com/metadata.json rename to units/gnome/extensions/papershell@lalovene.github.com/metadata.json diff --git a/roles/gnome/files/extensions/papershell@lalovene.github.com/prefs.js b/units/gnome/extensions/papershell@lalovene.github.com/prefs.js similarity index 100% rename from roles/gnome/files/extensions/papershell@lalovene.github.com/prefs.js rename to units/gnome/extensions/papershell@lalovene.github.com/prefs.js diff --git a/roles/gnome/files/extensions/papershell@lalovene.github.com/schemas/gschemas.compiled b/units/gnome/extensions/papershell@lalovene.github.com/schemas/gschemas.compiled similarity index 100% rename from roles/gnome/files/extensions/papershell@lalovene.github.com/schemas/gschemas.compiled rename to units/gnome/extensions/papershell@lalovene.github.com/schemas/gschemas.compiled diff --git a/roles/gnome/files/extensions/papershell@lalovene.github.com/schemas/org.gnome.shell.extensions.papershell.gschema.xml b/units/gnome/extensions/papershell@lalovene.github.com/schemas/org.gnome.shell.extensions.papershell.gschema.xml similarity index 100% rename from roles/gnome/files/extensions/papershell@lalovene.github.com/schemas/org.gnome.shell.extensions.papershell.gschema.xml rename to units/gnome/extensions/papershell@lalovene.github.com/schemas/org.gnome.shell.extensions.papershell.gschema.xml diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/KeybindingsComboRow.ui b/units/gnome/extensions/paperwm@paperwm.github.com/KeybindingsComboRow.ui similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/KeybindingsComboRow.ui rename to units/gnome/extensions/paperwm@paperwm.github.com/KeybindingsComboRow.ui diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/KeybindingsPane.ui b/units/gnome/extensions/paperwm@paperwm.github.com/KeybindingsPane.ui similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/KeybindingsPane.ui rename to units/gnome/extensions/paperwm@paperwm.github.com/KeybindingsPane.ui diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/KeybindingsRow.ui b/units/gnome/extensions/paperwm@paperwm.github.com/KeybindingsRow.ui similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/KeybindingsRow.ui rename to units/gnome/extensions/paperwm@paperwm.github.com/KeybindingsRow.ui diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/LICENSE b/units/gnome/extensions/paperwm@paperwm.github.com/LICENSE similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/LICENSE rename to units/gnome/extensions/paperwm@paperwm.github.com/LICENSE diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/Settings.ui b/units/gnome/extensions/paperwm@paperwm.github.com/Settings.ui similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/Settings.ui rename to units/gnome/extensions/paperwm@paperwm.github.com/Settings.ui diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/WinpropsPane.ui b/units/gnome/extensions/paperwm@paperwm.github.com/WinpropsPane.ui similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/WinpropsPane.ui rename to units/gnome/extensions/paperwm@paperwm.github.com/WinpropsPane.ui diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/WinpropsRow.ui b/units/gnome/extensions/paperwm@paperwm.github.com/WinpropsRow.ui similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/WinpropsRow.ui rename to units/gnome/extensions/paperwm@paperwm.github.com/WinpropsRow.ui diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/acceleratorparse.js b/units/gnome/extensions/paperwm@paperwm.github.com/acceleratorparse.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/acceleratorparse.js rename to units/gnome/extensions/paperwm@paperwm.github.com/acceleratorparse.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/app.js b/units/gnome/extensions/paperwm@paperwm.github.com/app.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/app.js rename to units/gnome/extensions/paperwm@paperwm.github.com/app.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/background.js b/units/gnome/extensions/paperwm@paperwm.github.com/background.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/background.js rename to units/gnome/extensions/paperwm@paperwm.github.com/background.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/config/user.css b/units/gnome/extensions/paperwm@paperwm.github.com/config/user.css similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/config/user.css rename to units/gnome/extensions/paperwm@paperwm.github.com/config/user.css diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/config/user.js b/units/gnome/extensions/paperwm@paperwm.github.com/config/user.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/config/user.js rename to units/gnome/extensions/paperwm@paperwm.github.com/config/user.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/extension.js b/units/gnome/extensions/paperwm@paperwm.github.com/extension.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/extension.js rename to units/gnome/extensions/paperwm@paperwm.github.com/extension.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/gestures.js b/units/gnome/extensions/paperwm@paperwm.github.com/gestures.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/gestures.js rename to units/gnome/extensions/paperwm@paperwm.github.com/gestures.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/grab.js b/units/gnome/extensions/paperwm@paperwm.github.com/grab.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/grab.js rename to units/gnome/extensions/paperwm@paperwm.github.com/grab.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/imports.js b/units/gnome/extensions/paperwm@paperwm.github.com/imports.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/imports.js rename to units/gnome/extensions/paperwm@paperwm.github.com/imports.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/keybindings.js b/units/gnome/extensions/paperwm@paperwm.github.com/keybindings.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/keybindings.js rename to units/gnome/extensions/paperwm@paperwm.github.com/keybindings.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/lib.js b/units/gnome/extensions/paperwm@paperwm.github.com/lib.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/lib.js rename to units/gnome/extensions/paperwm@paperwm.github.com/lib.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/liveAltTab.js b/units/gnome/extensions/paperwm@paperwm.github.com/liveAltTab.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/liveAltTab.js rename to units/gnome/extensions/paperwm@paperwm.github.com/liveAltTab.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/metadata.json b/units/gnome/extensions/paperwm@paperwm.github.com/metadata.json similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/metadata.json rename to units/gnome/extensions/paperwm@paperwm.github.com/metadata.json diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/minimap.js b/units/gnome/extensions/paperwm@paperwm.github.com/minimap.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/minimap.js rename to units/gnome/extensions/paperwm@paperwm.github.com/minimap.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/navigator.js b/units/gnome/extensions/paperwm@paperwm.github.com/navigator.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/navigator.js rename to units/gnome/extensions/paperwm@paperwm.github.com/navigator.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/overviewlayout.js b/units/gnome/extensions/paperwm@paperwm.github.com/overviewlayout.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/overviewlayout.js rename to units/gnome/extensions/paperwm@paperwm.github.com/overviewlayout.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/patches.js b/units/gnome/extensions/paperwm@paperwm.github.com/patches.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/patches.js rename to units/gnome/extensions/paperwm@paperwm.github.com/patches.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/prefs.js b/units/gnome/extensions/paperwm@paperwm.github.com/prefs.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/prefs.js rename to units/gnome/extensions/paperwm@paperwm.github.com/prefs.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/prefsKeybinding.js b/units/gnome/extensions/paperwm@paperwm.github.com/prefsKeybinding.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/prefsKeybinding.js rename to units/gnome/extensions/paperwm@paperwm.github.com/prefsKeybinding.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/ICONS.info b/units/gnome/extensions/paperwm@paperwm.github.com/resources/ICONS.info similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/ICONS.info rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/ICONS.info diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/focus-mode-center-symbolic.svg b/units/gnome/extensions/paperwm@paperwm.github.com/resources/focus-mode-center-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/focus-mode-center-symbolic.svg rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/focus-mode-center-symbolic.svg diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/focus-mode-default-symbolic.svg b/units/gnome/extensions/paperwm@paperwm.github.com/resources/focus-mode-default-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/focus-mode-default-symbolic.svg rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/focus-mode-default-symbolic.svg diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/focus-mode-edge-symbolic.svg b/units/gnome/extensions/paperwm@paperwm.github.com/resources/focus-mode-edge-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/focus-mode-edge-symbolic.svg rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/focus-mode-edge-symbolic.svg diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/logo.png b/units/gnome/extensions/paperwm@paperwm.github.com/resources/logo.png similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/logo.png rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/logo.png diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-down-symbolic.svg b/units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-down-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-down-symbolic.svg rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-down-symbolic.svg diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-end-symbolic.svg b/units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-end-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-end-symbolic.svg rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-end-symbolic.svg diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-left-symbolic.svg b/units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-left-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-left-symbolic.svg rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-left-symbolic.svg diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-right-symbolic.svg b/units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-right-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-right-symbolic.svg rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-right-symbolic.svg diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-start-symbolic.svg b/units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-start-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-start-symbolic.svg rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-start-symbolic.svg diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-up-symbolic.svg b/units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-up-symbolic.svg similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/open-position-up-symbolic.svg rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/open-position-up-symbolic.svg diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/prefs.css b/units/gnome/extensions/paperwm@paperwm.github.com/resources/prefs.css similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/resources/prefs.css rename to units/gnome/extensions/paperwm@paperwm.github.com/resources/prefs.css diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/schemas/gschemas.compiled b/units/gnome/extensions/paperwm@paperwm.github.com/schemas/gschemas.compiled similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/schemas/gschemas.compiled rename to units/gnome/extensions/paperwm@paperwm.github.com/schemas/gschemas.compiled diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/schemas/org.gnome.shell.extensions.paperwm.gschema.xml b/units/gnome/extensions/paperwm@paperwm.github.com/schemas/org.gnome.shell.extensions.paperwm.gschema.xml similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/schemas/org.gnome.shell.extensions.paperwm.gschema.xml rename to units/gnome/extensions/paperwm@paperwm.github.com/schemas/org.gnome.shell.extensions.paperwm.gschema.xml diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/scratch.js b/units/gnome/extensions/paperwm@paperwm.github.com/scratch.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/scratch.js rename to units/gnome/extensions/paperwm@paperwm.github.com/scratch.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/settings.js b/units/gnome/extensions/paperwm@paperwm.github.com/settings.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/settings.js rename to units/gnome/extensions/paperwm@paperwm.github.com/settings.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/stackoverlay.js b/units/gnome/extensions/paperwm@paperwm.github.com/stackoverlay.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/stackoverlay.js rename to units/gnome/extensions/paperwm@paperwm.github.com/stackoverlay.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/stylesheet.css b/units/gnome/extensions/paperwm@paperwm.github.com/stylesheet.css similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/stylesheet.css rename to units/gnome/extensions/paperwm@paperwm.github.com/stylesheet.css diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/tiling.js b/units/gnome/extensions/paperwm@paperwm.github.com/tiling.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/tiling.js rename to units/gnome/extensions/paperwm@paperwm.github.com/tiling.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/topbar.js b/units/gnome/extensions/paperwm@paperwm.github.com/topbar.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/topbar.js rename to units/gnome/extensions/paperwm@paperwm.github.com/topbar.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/utils.js b/units/gnome/extensions/paperwm@paperwm.github.com/utils.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/utils.js rename to units/gnome/extensions/paperwm@paperwm.github.com/utils.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/virtTiling.js b/units/gnome/extensions/paperwm@paperwm.github.com/virtTiling.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/virtTiling.js rename to units/gnome/extensions/paperwm@paperwm.github.com/virtTiling.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/winpropsPane.js b/units/gnome/extensions/paperwm@paperwm.github.com/winpropsPane.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/winpropsPane.js rename to units/gnome/extensions/paperwm@paperwm.github.com/winpropsPane.js diff --git a/roles/gnome/files/extensions/paperwm@paperwm.github.com/workspace.js b/units/gnome/extensions/paperwm@paperwm.github.com/workspace.js similarity index 100% rename from roles/gnome/files/extensions/paperwm@paperwm.github.com/workspace.js rename to units/gnome/extensions/paperwm@paperwm.github.com/workspace.js diff --git a/roles/gnome/files/gnome-extensions-dconf.dump b/units/gnome/gnome-extensions-dconf.dump similarity index 98% rename from roles/gnome/files/gnome-extensions-dconf.dump rename to units/gnome/gnome-extensions-dconf.dump index 0879b36b..7c05d9a7 100644 --- a/roles/gnome/files/gnome-extensions-dconf.dump +++ b/units/gnome/gnome-extensions-dconf.dump @@ -1,6 +1,6 @@ # Cleaned GNOME extension dconf settings — synced across fleet. # Source: dumped from dilli, redacted for machine-specific data. -# To refresh from a source machine: dconf dump /org/gnome/shell/extensions/ > roles/gnome/files/gnome-extensions-dconf.dump +# To refresh from a source machine: dconf dump /org/gnome/shell/extensions/ > units/gnome/gnome-extensions-dconf.dump # Then remove [gsconnect*] and [paperwm/workspaces*] sections (per-machine). [Logo-menu] diff --git a/roles/gnome/files/gnome-keybindings.ini b/units/gnome/gnome-keybindings.ini similarity index 100% rename from roles/gnome/files/gnome-keybindings.ini rename to units/gnome/gnome-keybindings.ini diff --git a/units/hive-ops.sh b/units/hive-ops.sh new file mode 100755 index 00000000..729be32d --- /dev/null +++ b/units/hive-ops.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if [ "${UNITS_CHECK:-0}" = 1 ] && { [ -e "$DOTFILES_DIR/host_vars/$MACHINE.yml" ] || [ -L "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; } && { [ ! -f "$DOTFILES_DIR/host_vars/$MACHINE.yml" ] || [ ! -r "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; }; then + step "inspect host variables" false + finish +fi +unitdir="$HOME/.config/systemd/user" +bindir="$HOME/.local/bin" +enabled=$(host_var hive_ops_enabled false) +# Never stop or alter anything on the retired VPSes, even if stale files exist. +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ] || [ "$IS_TERMUX" = 1 ]; then + note "SKIPPED: hive operations excluded on this host" + finish +fi +if [ "$enabled" != true ] && [ ! -f "$unitdir/hive-watchdog.timer" ]; then + note "SKIPPED: hive operations disabled" + finish +fi +if ! command -v yq >/dev/null 2>&1; then + step "check hive host and timer configuration (yq required)" false + finish +fi +vps=$(MACHINE="$MACHINE" yq '.all.children.vps.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml") || { step "check VPS membership" false; finish; } +case "$vps" in + true) note "SKIPPED: hive operations excluded on VPS"; finish ;; + false) ;; + *) step "check VPS membership" false; finish ;; +esac +hostfile="$DOTFILES_DIR/host_vars/$MACHINE.yml" +sourcefile="$DOTFILES_DIR/units/hive-ops/defaults.yml" +if [ -f "$hostfile" ]; then + has_timers=$(yq 'has("hive_ops_timers")' "$hostfile") || { step "read hive timer overrides" false; finish; } + if [ "${UNITS_CHECK:-0}" = 1 ] && [ "$has_timers" != true ] && [ "$has_timers" != false ]; then + step "inspect hive timer overrides" false + finish + fi + [ "$has_timers" = true ] && sourcefile="$hostfile" +fi +timer_text=$(yq -r '.hive_ops_timers[]' "$sourcefile") || { step "read hive timers" false; finish; } +if [ -z "$timer_text" ]; then + step "hive timer list is empty" false + finish +fi +mapfile -t timers <<< "$timer_text" +for timer in "${timers[@]}"; do + if [[ ! "$timer" =~ ^(hive|pi)-[a-z0-9-]+$ ]]; then + step "invalid hive timer name" false + finish + fi +done +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then + export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus" +fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if [ "$enabled" != true ]; then + for timer in "${timers[@]}"; do + [ ! -f "$unitdir/$timer.timer" ] || note "CHANGE: disable and stop $timer.timer" + for file in "$unitdir/$timer.timer" "$unitdir/$timer.service" "$unitdir/$timer.service.d/kubeconfig.conf"; do + [ ! -e "$file" ] && [ ! -L "$file" ] || note "CHANGE: remove $file" + done + done + for helper in hive-peak.sh hive-rotate.sh hive-tiers.sh; do + [ ! -e "$bindir/$helper" ] && [ ! -L "$bindir/$helper" ] || note "CHANGE: remove $bindir/$helper" + done + finish + fi + + # Compare without exposing file contents; drop-ins contain host-specific paths. + check_file() { # source, destination, expected mode + local src="$1" dest="$2" expected="$3" rc mode + if [ ! -f "$src" ] || [ ! -r "$src" ]; then + step "read $src" false + elif [ ! -e "$dest" ] && [ ! -L "$dest" ]; then + note "CHANGE: install $dest" + elif [ ! -f "$dest" ] || [ ! -r "$dest" ]; then + step "inspect $dest" false + else + cmp -s "$src" "$dest"; rc=$? + case "$rc" in + 0) ;; + 1) note "CHANGE: update $dest" ;; + *) step "compare $dest" false ;; + esac + mode=$(stat -c %a "$dest") || { step "inspect mode of $dest" false; return; } + [ "$mode" = "$expected" ] || note "CHANGE: set mode of $dest to $expected" + fi + } + kubeconfig=$(host_var hive_ops_kubeconfig "$HOME/.kube/config-aws-migration") + hive_path=$(host_var hive_ops_path '/home/linuxbrew/.linuxbrew/bin:/usr/local/bin:/usr/bin:/bin') + for helper in hive-peak.sh hive-rotate.sh hive-tiers.sh; do + check_file "$DOTFILES_DIR/units/hive-ops/bin/$helper" "$bindir/$helper" 755 + done + for timer in "${timers[@]}"; do + for suffix in service timer; do + check_file "$DOTFILES_DIR/units/hive-ops/systemd/$timer.$suffix" "$unitdir/$timer.$suffix" 644 + done + dropin="$unitdir/$timer.service.d/kubeconfig.conf" + if [ ! -e "$dropin" ] && [ ! -L "$dropin" ]; then + note "CHANGE: install $dropin" + elif [ ! -f "$dropin" ] || [ ! -r "$dropin" ]; then + step "inspect $dropin" false + else + printf '[Service]\nEnvironment=KUBECONFIG=%s\nEnvironment=PATH=%s\n' "$kubeconfig" "$hive_path" | cmp -s - "$dropin"; rc=$? + case "$rc" in + 0) ;; + 1) note "CHANGE: update $dropin" ;; + *) step "compare $dropin" false ;; + esac + mode=$(stat -c %a "$dropin") || { step "inspect mode of $dropin" false; continue; } + [ "$mode" = 644 ] || note "CHANGE: set mode of $dropin to 644" + fi + state=$(systemctl --user show --property=LoadState --value "$timer.timer") || { step "check $timer.timer" false; continue; } + case "$state" in + not-found) note "CHANGE: enable and start $timer.timer" ;; + loaded) + timer_enabled=$(systemctl --user show --property=UnitFileState --value "$timer.timer") || { step "check $timer.timer enablement" false; continue; } + active=$(systemctl --user show --property=ActiveState --value "$timer.timer") || { step "check $timer.timer activity" false; continue; } + if [ -z "$timer_enabled" ] || [ -z "$active" ]; then + step "inspect $timer.timer state" false + else + [ "$timer_enabled" = enabled ] || note "CHANGE: enable $timer.timer" + [ "$active" = active ] || note "CHANGE: start $timer.timer" + fi + ;; + *) step "check $timer.timer load state" false ;; + esac + done + finish +fi + + +if [ "$enabled" != true ]; then + removed=0 + for timer in "${timers[@]}"; do + if [ -f "$unitdir/$timer.timer" ]; then + if ! systemctl --user disable --now "$timer.timer"; then + step "disable $timer.timer" false + continue + fi + fi + if rm -f "$unitdir/$timer.timer" "$unitdir/$timer.service" "$unitdir/$timer.service.d/kubeconfig.conf"; then + rmdir "$unitdir/$timer.service.d" 2>/dev/null || : + removed=1 + else + step "remove $timer units" false + fi + done + if [ "$UNIT_FAILED" = 0 ]; then + step "remove hive helpers" rm -f "$bindir/hive-peak.sh" "$bindir/hive-rotate.sh" "$bindir/hive-tiers.sh" + fi + [ "$removed" = 0 ] || step "reload user systemd" systemctl --user daemon-reload + finish +fi + +kubeconfig=$(host_var hive_ops_kubeconfig "$HOME/.kube/config-aws-migration") +hive_path=$(host_var hive_ops_path '/home/linuxbrew/.linuxbrew/bin:/usr/local/bin:/usr/bin:/bin') +changed=0 +for helper in hive-peak.sh hive-rotate.sh hive-tiers.sh; do + step "deploy $helper" install_if_changed "$DOTFILES_DIR/units/hive-ops/bin/$helper" "$bindir/$helper" 0755 +done +for timer in "${timers[@]}"; do + for suffix in service timer; do + src="$DOTFILES_DIR/units/hive-ops/systemd/$timer.$suffix" + dest="$unitdir/$timer.$suffix" + if [ ! -f "$dest" ] || ! cmp -s "$src" "$dest"; then changed=1; fi + step "deploy $timer.$suffix" install_if_changed "$src" "$dest" 0644 + done + dropin="$unitdir/$timer.service.d/kubeconfig.conf" + content=$(printf '[Service]\nEnvironment=KUBECONFIG=%s\nEnvironment=PATH=%s\n' "$kubeconfig" "$hive_path") + if [ ! -f "$dropin" ] || [ "$(cat "$dropin")" != "$content" ]; then + changed=1 + if mkdir -p "$(dirname "$dropin")" && printf '%s\n' "$content" > "$dropin" && chmod 0644 "$dropin"; then + note "ok $timer drop-in" + else + step "$timer drop-in" false + fi + else + step "$timer drop-in mode" chmod 0644 "$dropin" + fi +done +[ "$UNIT_FAILED" = 0 ] || finish +if [ "$changed" = 1 ]; then + if ! systemctl --user daemon-reload; then + step "reload user systemd" false + finish + fi +fi +for timer in "${timers[@]}"; do + step "enable and start $timer.timer" systemctl --user enable --now "$timer.timer" +done +finish diff --git a/roles/hive_ops/files/bin/hive-peak.sh b/units/hive-ops/bin/hive-peak.sh similarity index 100% rename from roles/hive_ops/files/bin/hive-peak.sh rename to units/hive-ops/bin/hive-peak.sh diff --git a/roles/hive_ops/files/bin/hive-rotate.sh b/units/hive-ops/bin/hive-rotate.sh similarity index 100% rename from roles/hive_ops/files/bin/hive-rotate.sh rename to units/hive-ops/bin/hive-rotate.sh diff --git a/roles/hive_ops/files/bin/hive-tiers.sh b/units/hive-ops/bin/hive-tiers.sh similarity index 100% rename from roles/hive_ops/files/bin/hive-tiers.sh rename to units/hive-ops/bin/hive-tiers.sh diff --git a/roles/hive_ops/defaults/main.yml b/units/hive-ops/defaults.yml similarity index 100% rename from roles/hive_ops/defaults/main.yml rename to units/hive-ops/defaults.yml diff --git a/roles/hive_ops/files/systemd/hive-peak-pause.service b/units/hive-ops/systemd/hive-peak-pause.service similarity index 100% rename from roles/hive_ops/files/systemd/hive-peak-pause.service rename to units/hive-ops/systemd/hive-peak-pause.service diff --git a/roles/hive_ops/files/systemd/hive-peak-pause.timer b/units/hive-ops/systemd/hive-peak-pause.timer similarity index 100% rename from roles/hive_ops/files/systemd/hive-peak-pause.timer rename to units/hive-ops/systemd/hive-peak-pause.timer diff --git a/roles/hive_ops/files/systemd/hive-peak-resume.service b/units/hive-ops/systemd/hive-peak-resume.service similarity index 100% rename from roles/hive_ops/files/systemd/hive-peak-resume.service rename to units/hive-ops/systemd/hive-peak-resume.service diff --git a/roles/hive_ops/files/systemd/hive-peak-resume.timer b/units/hive-ops/systemd/hive-peak-resume.timer similarity index 100% rename from roles/hive_ops/files/systemd/hive-peak-resume.timer rename to units/hive-ops/systemd/hive-peak-resume.timer diff --git a/roles/hive_ops/files/systemd/hive-rotate.service b/units/hive-ops/systemd/hive-rotate.service similarity index 100% rename from roles/hive_ops/files/systemd/hive-rotate.service rename to units/hive-ops/systemd/hive-rotate.service diff --git a/roles/hive_ops/files/systemd/hive-rotate.timer b/units/hive-ops/systemd/hive-rotate.timer similarity index 100% rename from roles/hive_ops/files/systemd/hive-rotate.timer rename to units/hive-ops/systemd/hive-rotate.timer diff --git a/roles/hive_ops/files/systemd/hive-tiers.service b/units/hive-ops/systemd/hive-tiers.service similarity index 100% rename from roles/hive_ops/files/systemd/hive-tiers.service rename to units/hive-ops/systemd/hive-tiers.service diff --git a/roles/hive_ops/files/systemd/hive-tiers.timer b/units/hive-ops/systemd/hive-tiers.timer similarity index 100% rename from roles/hive_ops/files/systemd/hive-tiers.timer rename to units/hive-ops/systemd/hive-tiers.timer diff --git a/roles/hive_ops/files/systemd/hive-watchdog.service b/units/hive-ops/systemd/hive-watchdog.service similarity index 100% rename from roles/hive_ops/files/systemd/hive-watchdog.service rename to units/hive-ops/systemd/hive-watchdog.service diff --git a/roles/hive_ops/files/systemd/hive-watchdog.timer b/units/hive-ops/systemd/hive-watchdog.timer similarity index 100% rename from roles/hive_ops/files/systemd/hive-watchdog.timer rename to units/hive-ops/systemd/hive-watchdog.timer diff --git a/roles/hive_ops/files/systemd/pi-peak-start.service b/units/hive-ops/systemd/pi-peak-start.service similarity index 100% rename from roles/hive_ops/files/systemd/pi-peak-start.service rename to units/hive-ops/systemd/pi-peak-start.service diff --git a/roles/hive_ops/files/systemd/pi-peak-start.timer b/units/hive-ops/systemd/pi-peak-start.timer similarity index 100% rename from roles/hive_ops/files/systemd/pi-peak-start.timer rename to units/hive-ops/systemd/pi-peak-start.timer diff --git a/roles/hive_ops/files/systemd/pi-peak-stop.service b/units/hive-ops/systemd/pi-peak-stop.service similarity index 100% rename from roles/hive_ops/files/systemd/pi-peak-stop.service rename to units/hive-ops/systemd/pi-peak-stop.service diff --git a/roles/hive_ops/files/systemd/pi-peak-stop.timer b/units/hive-ops/systemd/pi-peak-stop.timer similarity index 100% rename from roles/hive_ops/files/systemd/pi-peak-stop.timer rename to units/hive-ops/systemd/pi-peak-stop.timer diff --git a/units/homebrew.sh b/units/homebrew.sh new file mode 100755 index 00000000..49d971d3 --- /dev/null +++ b/units/homebrew.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "$0")/lib.sh" +# shellcheck source=units/package-config.bash +. "$DOTFILES_DIR/units/package-config.bash" + +if [ "$IS_TERMUX" = 1 ] || [ "$IS_MUSL" = 1 ]; then + note 'SKIPPED: Homebrew requires glibc; use Termux packages or apk on this host' + exit 0 +fi + +brew_prefix="${BREW_PREFIX:-/home/linuxbrew/.linuxbrew}" +brew_bin="$brew_prefix/bin/brew" +PATH="$brew_prefix/bin:$PATH" +export PATH HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_ENV_HINTS=1 + +load_config() { + package_require_yq && package_desktop || return 1 + package_list core_taps || return 1; core_taps=("${PACKAGE_LIST[@]}") + package_list core_brews || return 1; core_brews=("${PACKAGE_LIST[@]}") + package_list core_tap_brews || return 1; core_tap_brews=("${PACKAGE_LIST[@]}") + package_list desktop_taps || return 1; desktop_taps=("${PACKAGE_LIST[@]}") + package_list desktop_brews || return 1; desktop_brews=("${PACKAGE_LIST[@]}") + package_list extra_brews || return 1; extra_brews=("${PACKAGE_LIST[@]}") + package_list desktop_casks || return 1; desktop_casks=("${PACKAGE_LIST[@]}") + package_list desktop_vscode_extensions || return 1; extensions=("${PACKAGE_LIST[@]}") + package_list brew_blocked || return 1; blocked=("${PACKAGE_LIST[@]}") + + local pkg banned + for pkg in "${core_brews[@]}" "${core_tap_brews[@]}" "${desktop_brews[@]}" "${extra_brews[@]}" "${desktop_casks[@]}"; do + for banned in "${blocked[@]}"; do + if [ "${pkg##*/}" = "$banned" ]; then + echo "$pkg is in brew_blocked; remove it from the install lists" >&2 + return 1 + fi + done + done +} + +fix_owner() { + local parent="${brew_prefix%/.linuxbrew}" owner + [ -d "$parent" ] || return 0 + owner="$(stat -c %u "$parent" 2>/dev/null || stat -f %u "$parent")" || return 1 + [ "$owner" = "$(id -u)" ] || package_as_root chown -R "$(id -un)" "$parent" +} + +install_compiler() { + if command -v cc >/dev/null 2>&1 && command -v gcc >/dev/null 2>&1; then return 0; fi + local ID='' ID_LIKE='' + [ -r /etc/os-release ] || { echo 'cc and gcc are required for Homebrew; install a compiler toolchain' >&2; return 1; } + # shellcheck source=/dev/null + . /etc/os-release + case " $ID $ID_LIKE " in + *' debian '*|*' ubuntu '*) package_as_root apt-get update && package_as_root apt-get install -y build-essential ;; + *) echo 'cc and gcc are required for Homebrew; install a compiler toolchain' >&2; return 1 ;; + esac +} + +install_brew() { + [ -x "$brew_bin" ] && return 0 + local script rc + command -v curl >/dev/null 2>&1 || { echo 'curl is required to install Homebrew' >&2; return 1; } + script="$(mktemp)" || return 1 + rc=0 + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o "$script" && NONINTERACTIVE=1 /bin/bash "$script" || rc=$? + rm -f "$script" + [ "$rc" = 0 ] && [ -x "$brew_bin" ] +} + +ensure_yq() { + local version + if command -v yq >/dev/null 2>&1 && version="$(yq --version 2>/dev/null)" && [[ "$version" == *github.com/mikefarah/yq* ]]; then + return 0 + fi + echo 'Installing Mike Farah yq for package configuration' + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=credential.helper GIT_CONFIG_VALUE_0='' "$brew_bin" install yq || { + echo 'Unable to install yq with Homebrew' >&2 + return 1 + } + hash -r + if ! command -v yq >/dev/null 2>&1 || ! version="$(yq --version 2>/dev/null)" || [[ "$version" != *github.com/mikefarah/yq* ]]; then + echo 'Mike Farah yq is unavailable after Homebrew installation' >&2 + return 1 + fi +} + +render_brewfile() { + local pkg + echo '# Core packages' + for pkg in "${core_taps[@]}"; do printf 'tap "%s"\n' "$pkg"; done + echo + for pkg in "${core_brews[@]}"; do printf 'brew "%s"\n' "$pkg"; done + echo + for pkg in "${core_tap_brews[@]}"; do printf 'brew "%s"\n' "$pkg"; done + echo + if [ "$PACKAGE_DESKTOP" = 1 ]; then + echo '# Desktop packages' + for pkg in "${desktop_taps[@]}"; do printf 'tap "%s"\n' "$pkg"; done + echo + for pkg in "${desktop_brews[@]}"; do printf 'brew "%s"\n' "$pkg"; done + echo + for pkg in "${extra_brews[@]}"; do printf 'brew "%s"\n' "$pkg"; done + echo + for pkg in "${desktop_casks[@]}"; do printf 'cask "%s"\n' "$pkg"; done + echo + if [ "$(uname -s)" = Darwin ]; then + for pkg in "${extensions[@]}"; do printf 'vscode "%s"\n' "$pkg"; done + fi + fi +} + +deploy_brewfile() { + local tmp + tmp="$(mktemp "$HOME/.Brewfile.XXXXXX")" || return 1 + render_brewfile > "$tmp" || { rm -f "$tmp"; return 1; } + install_if_changed "$tmp" "$HOME/Brewfile" 0644 + local rc=$? + rm -f "$tmp" + return "$rc" +} + +tap_and_trust() { + local pkg tap seen=' ' rc=0 + for pkg in "${core_taps[@]}" "${desktop_taps[@]}" "${core_brews[@]}" "${core_tap_brews[@]}" "${desktop_brews[@]}" "${extra_brews[@]}" "${desktop_casks[@]}"; do + case "$pkg" in + */*/*) tap="${pkg%/*}" ;; + */*) tap="$pkg" ;; + *) continue ;; + esac + case "$seen" in *" $tap "*) continue ;; esac + seen="$seen$tap " + "$brew_bin" tap "$tap" || rc=1 + # Older Homebrew versions may not support trust, as in the Ansible role. + soft "trust $tap" "$brew_bin" trust "$tap" + done + return "$rc" +} + +bundle() { + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=credential.helper GIT_CONFIG_VALUE_0='' \ + "$brew_bin" bundle install "--file=$HOME/Brewfile" --no-upgrade +} + +uninstall_blocked() { + local pkg err rc=0 + err="$(mktemp)" || return 1 + for pkg in "${blocked[@]}"; do + if ! "$brew_bin" uninstall --formula "$pkg" 2>"$err"; then + if ! grep -Fq 'No such keg' "$err"; then cat "$err" >&2; rc=1; fi + fi + done + rm -f "$err" + return "$rc" +} + +# Homebrew may initialise caches or fetch metadata even for some queries. +# Constrain every Homebrew/Python probe to a read-only, offline namespace. +check_brew_query() { + bwrap --unshare-all --new-session --die-with-parent --ro-bind / / \ + --proc /proc --dev /dev --tmpfs /run --clearenv \ + --setenv HOME "$HOME" --setenv PATH "$PATH" \ + --setenv XDG_RUNTIME_DIR /run --setenv XDG_CACHE_HOME /run/cache \ + --setenv DBUS_SYSTEM_BUS_ADDRESS unix:path=/run/no-bus \ + --setenv DBUS_SESSION_BUS_ADDRESS unix:path=/run/no-bus \ + --setenv HOMEBREW_CACHE /run/brew-cache --setenv HOMEBREW_LOGS /run/brew-logs \ + --setenv HOMEBREW_TEMP /run --setenv HOMEBREW_NO_AUTO_UPDATE 1 \ + --setenv HOMEBREW_NO_ANALYTICS 1 --setenv HOMEBREW_NO_ENV_HINTS 1 \ + --setenv HOMEBREW_NO_BOOTSNAP 1 \ + --setenv GIT_TERMINAL_PROMPT 0 --setenv NONINTERACTIVE 1 \ + --setenv GIT_CONFIG_COUNT 1 --setenv GIT_CONFIG_KEY_0 credential.helper --setenv GIT_CONFIG_VALUE_0 '' \ + --setenv PYTHONDONTWRITEBYTECODE 1 "$@" 2>&1 +} + +check_brew() { + local parent="${brew_prefix%/.linuxbrew}" owner uid version rc mode result pkg tap seen=' ' taps line info kind count + if [ -d "$parent" ]; then + if owner="$(stat -c %u "$parent" 2>/dev/null || stat -f %u "$parent" 2>/dev/null)" && uid="$(id -u)"; then + [ "$owner" = "$uid" ] || note 'CHANGE: repair Homebrew directory ownership' + else + echo 'Cannot inspect Homebrew directory ownership' >&2; UNIT_FAILED=1 + fi + fi + if ! command -v cc >/dev/null 2>&1 || ! command -v gcc >/dev/null 2>&1; then + note 'CHANGE: Homebrew compiler toolchain is missing' + fi + if [ ! -x "$brew_bin" ]; then note 'CHANGE: install Homebrew'; fi + if ! command -v yq >/dev/null 2>&1 || ! version="$(yq --version 2>/dev/null)" || [[ "$version" != *github.com/mikefarah/yq* ]]; then + echo 'Cannot inspect Homebrew configuration: Mike Farah yq is required' >&2 + UNIT_FAILED=1; return + fi + if ! load_config >/dev/null 2>&1; then echo 'Cannot inspect Homebrew configuration' >&2; UNIT_FAILED=1; return; fi + if [ ! -e "$HOME/Brewfile" ] && [ ! -L "$HOME/Brewfile" ]; then + note 'CHANGE: install Brewfile' + elif [ ! -f "$HOME/Brewfile" ] || [ ! -r "$HOME/Brewfile" ]; then + echo 'Cannot inspect Brewfile' >&2; UNIT_FAILED=1 + else + cmp -s "$HOME/Brewfile" <(render_brewfile); rc=$? + if [ "$rc" = 1 ]; then note 'CHANGE: update Brewfile'; fi + if [ "$rc" -gt 1 ]; then echo 'Cannot compare Brewfile' >&2; UNIT_FAILED=1; fi + if mode="$(stat -L -c %a "$HOME/Brewfile" 2>/dev/null || stat -L -f %Lp "$HOME/Brewfile" 2>/dev/null)"; then + [ "$mode" = 644 ] || note 'CHANGE: set Brewfile mode to 0644' + else + echo 'Cannot inspect Brewfile permissions' >&2; UNIT_FAILED=1 + fi + fi + note 'PLANNED: attempt Homebrew tap trust on apply; current trust state not inspected' + if [ ! -x "$brew_bin" ]; then + note 'PLANNED: install configured Homebrew taps and packages after bootstrap' + return + fi + if ! command -v bwrap >/dev/null 2>&1; then + echo 'Cannot inspect Homebrew safely: bubblewrap is required' >&2; UNIT_FAILED=1; return + fi + case "$HOME" in /run|/run/*|/var/run|/var/run/*) + echo 'Cannot inspect Homebrew home hidden by the sandbox' >&2; UNIT_FAILED=1; return ;; + esac + if taps="$(check_brew_query "$brew_bin" tap &2; UNIT_FAILED=1; return ;; esac + done + for pkg in "${core_taps[@]}" "${desktop_taps[@]}" "${core_brews[@]}" "${core_tap_brews[@]}" "${desktop_brews[@]}" "${extra_brews[@]}" "${desktop_casks[@]}"; do + case "$pkg" in */*/*) tap="${pkg%/*}" ;; */*) tap="$pkg" ;; *) continue ;; esac + case "$seen" in *" $tap "*) continue ;; esac + seen="$seen$tap " + if ! printf '%s\n' "$taps" | grep -Fxqi -- "$tap"; then note "CHANGE: tap Homebrew repository $tap"; fi + done + else + echo 'Cannot inspect Homebrew taps in the read-only sandbox' >&2; UNIT_FAILED=1 + fi + if result="$(render_brewfile | check_brew_query "$brew_bin" bundle check --file=- --no-upgrade --verbose)"; then + note 'ok Homebrew bundle dependencies' + else + # Do not print command output: Homebrew diagnostics can contain credentials. + # Bundle uses the same nonzero status for drift and inspection failures. + if [[ "$result" == *"can't satisfy your Brewfile's dependencies"* ]]; then + note 'CHANGE: Homebrew bundle dependencies need installation' + for pkg in "${core_taps[@]}" "${core_brews[@]}" "${core_tap_brews[@]}" "${desktop_taps[@]}" "${desktop_brews[@]}" "${extra_brews[@]}" "${desktop_casks[@]}" "${extensions[@]}"; do + if [[ "$result" == *" $pkg needs to be "* ]]; then note "CHANGE: Homebrew bundle dependency $pkg"; fi + done + else + echo 'Cannot inspect Homebrew bundle dependencies in the read-only sandbox' >&2 + fi + UNIT_FAILED=1 + fi + # Resolve each blocked name through Homebrew itself, including aliases and + # renamed formulae. Unknown/unreadable metadata is an inspection failure, + # not evidence that the formula is absent. + for pkg in "${blocked[@]}"; do + if ! info="$(check_brew_query "$brew_bin" info --json=v1 --formula "$pkg" /dev/null)" || [ "$count" != 1 ] || + ! kind="$(printf '%s' "$info" | yq -p=json -r '.[0].installed | tag' 2>/dev/null)" || [ "$kind" != '!!seq' ] || + ! count="$(printf '%s' "$info" | yq -p=json -r '.[0].installed | length' 2>/dev/null)"; then + echo "Cannot inspect blocked Homebrew formula $pkg in the read-only sandbox" >&2; UNIT_FAILED=1; continue + fi + case "$count" in + ''|*[!0-9]*) echo "Cannot inspect blocked Homebrew formula $pkg" >&2; UNIT_FAILED=1 ;; + 0) ;; + *) note "CHANGE: uninstall blocked Homebrew formula $pkg" ;; + esac + done + if [ -x "$brew_prefix/bin/python3" ]; then + check_brew_query "$brew_prefix/bin/python3" -c 'import importlib.util, sys; sys.exit(0 if importlib.util.find_spec("packaging") else 42)' /dev/null + rc=$? + case "$rc" in + 0) ;; + 42) note 'CHANGE: install Homebrew Python packaging' ;; + *) echo 'Cannot inspect Homebrew Python packaging in the read-only sandbox' >&2; UNIT_FAILED=1 ;; + esac + else + note 'PLANNED: best-effort Python packaging install if Homebrew Python is available' + fi + case "${DOTFILES_UPGRADE:-0}" in 1|true) note 'PLANNED: Homebrew upgrade requested; available upgrades not inspected offline' ;; esac +} + +if [ "${UNITS_CHECK:-0}" = 1 ]; then + check_brew + finish +fi + +step 'Homebrew directory ownership' fix_owner +step 'Homebrew compiler' install_compiler +step 'Homebrew installation' install_brew +if [ ! -x "$brew_bin" ]; then + step 'Homebrew executable unavailable' false + finish +fi +if ensure_yq; then + note 'ok Homebrew yq bootstrap' +else + step 'Homebrew yq bootstrap' false + finish +fi +if load_config; then + note 'ok Homebrew configuration' +else + step 'Homebrew configuration' false + finish +fi +step 'Brewfile' deploy_brewfile +step 'Homebrew taps' tap_and_trust +step 'Homebrew bundle' bundle +step 'blocked Homebrew packages' uninstall_blocked +case "${DOTFILES_UPGRADE:-0}" in 1|true) soft 'Homebrew upgrade' "$brew_bin" upgrade ;; esac +soft 'brew Python packaging' "$brew_prefix/bin/python3" -m pip install packaging --break-system-packages --quiet +finish diff --git a/units/kirocrew-sockets.py b/units/kirocrew-sockets.py new file mode 100755 index 00000000..faeddfeb --- /dev/null +++ b/units/kirocrew-sockets.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Reject non-loopback listening sockets owned by the kirocrew process.""" +import sys + + +def main(): + unsafe = [] + for line in sys.stdin: + if '"kirocrew"' not in line: + continue + columns = line.split() + if len(columns) < 4: + unsafe.append(line.rstrip()) + continue + local = columns[3] + if not (local.startswith('127.0.0.1:') or local.startswith('[::1]:')): + unsafe.append(line.rstrip()) + if unsafe: + print('kirocrew is listening beyond localhost:', file=sys.stderr) + for line in unsafe: + print(line, file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/units/kirocrew.sh b/units/kirocrew.sh new file mode 100755 index 00000000..58881b37 --- /dev/null +++ b/units/kirocrew.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then + note "SKIPPED: kirocrew excluded on retired VPS" + finish +fi +if [ "$(host_var kirocrew_enabled false)" != true ]; then + note "SKIPPED: kirocrew not enabled for this host" + finish +fi +as_root() { if [ "$EUID" = 0 ]; then "$@"; else sudo -n "$@"; fi; } +if [ ! -e "$HOME/.local/bin/kirocrew" ] || [ ! -e /etc/systemd/system/kirocrew.service ]; then + note "SKIPPED: kirocrew binary or system service missing; install from https://github.com/kirodotdev/KiroCrew and run sudo ~/.local/bin/kirocrew service install" + finish +fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if [ "$EUID" != 0 ] && ! sudo -n true; then step "noninteractive root access" false; finish; fi + if mode=$(as_root stat -c '%a:%u:%g' /etc/kirocrew/kirocrew.env 2>/dev/null); then + [ "$mode" = '600:0:0' ] && note "ok kirocrew environment permissions" || note "would secure kirocrew environment ownership and permissions" + else step "inspect kirocrew environment permissions" false; fi + state=$(as_root systemctl show kirocrew.service --property=LoadState --property=ActiveState --property=UnitFileState 2>/dev/null); rc=$? + if { [ "$rc" != 0 ] && [[ "$state" != *'LoadState=not-found'* ]]; } || [[ "$state" != *'LoadState='* || "$state" != *'ActiveState='* ]]; then + step "inspect kirocrew service" false + elif [[ "$state" == *'LoadState=not-found'* ]]; then + note "would enable and start kirocrew after systemd recognizes its service" + elif [[ "$state" != *'LoadState=loaded'* || "$state" != *'UnitFileState='* ]]; then + step "inspect kirocrew service" false + else + [[ "$state" == *'ActiveState=active'* ]] || note "would start kirocrew; its future listeners cannot be inspected without starting it" + [[ "$state" == *'UnitFileState=enabled'* ]] || note "would enable kirocrew" + fi + if as_root ss -Htlnp | python3 "$DOTFILES_DIR/units/kirocrew-sockets.py"; then + note "ok current kirocrew listeners are localhost-only" + else step "inspect current kirocrew listeners" false; fi + finish +fi +secure_env() { + as_root chown root:root /etc/kirocrew/kirocrew.env && as_root chmod 0600 /etc/kirocrew/kirocrew.env +} +if secure_env; then + note "ok secure kirocrew environment" + step "enable and start kirocrew" as_root systemctl enable --now kirocrew.service +else + step "secure kirocrew environment" false +fi +check_listeners() { + as_root ss -Htlnp | python3 "$DOTFILES_DIR/units/kirocrew-sockets.py" +} +step "check kirocrew listens only on localhost" check_listeners +finish diff --git a/units/kube.sh b/units/kube.sh index 3ebfa3e2..26c00b4f 100755 --- a/units/kube.sh +++ b/units/kube.sh @@ -7,10 +7,54 @@ set -uo pipefail # shellcheck source=units/kube.conf . "$DOTFILES_DIR/units/kube.conf" +if [ "${UNITS_CHECK:-0}" = 1 ] && ! skipped_by_profile skip_kube; then + desktop="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" + case "$desktop" in + 1|true|0|false) ;; + *) + if ! command -v yq >/dev/null 2>&1 || ! desktop="$(yq ".all.children.desktop.hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml" 2>/dev/null)"; then + note "FAIL cannot inspect desktop inventory (yq required)" + exit 1 + fi + case "$desktop" in true|false) ;; *) note "FAIL invalid desktop inventory result"; exit 1 ;; esac ;; + esac + DOTFILES_IS_DESKTOP="$desktop" +fi + if ! is_desktop || skipped_by_profile skip_kube; then note "SKIPPED: not a desktop (or skip_kube / vm-test)" exit 0 fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if [ "${UNITS_NO_SECRETS:-0}" = 1 ]; then + note "SKIPPED: secrets disabled; existing kube/talos configs left untouched" + exit 0 + fi + for dir in "$HOME/.kube" "$HOME/.talos"; do + if [ ! -e "$dir" ]; then + note "PARTIAL: $dir absent; apply would create it if the vault is unlocked" + elif [ ! -d "$dir" ] || ! mode="$(stat -Lc '%a' "$dir")"; then + note "FAIL cannot inspect directory $dir" + elif [ "$mode" != 700 ]; then + note "PARTIAL: $dir mode differs from 0700; apply would change it if the vault is unlocked" + else + note "ok $dir mode 0700" + fi + done + for path in "$HOME/.kube/config" "$HOME/.talos/config" "$HOME/.kube/config-aws-migration" "$HOME/.talos/config-aws-migration"; do + if [ -L "$path" ] || [ ! -e "$path" ]; then + note "PARTIAL: $path absent or symlinked; apply would replace it if the note is available" + elif [ ! -f "$path" ] || ! mode="$(stat -c '%a' "$path")"; then + note "FAIL cannot inspect $path" + elif [ "$mode" != 600 ]; then + note "PARTIAL: $path mode differs from 0600; apply would replace it if the note is available" + else + note "ok $path mode 0600" + fi + done + note "FAIL cannot compare kube/talos notes or TLS pin safely: Bitwarden item reads may persist audit events" + exit 1 +fi s="$(bw_session)" if [ -z "$s" ]; then note "SKIPPED: vault locked or --no-secrets; existing kube/talos configs left untouched" diff --git a/units/laptop-services.sh b/units/laptop-services.sh new file mode 100755 index 00000000..dc3163cd --- /dev/null +++ b/units/laptop-services.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [ "${UNITS_CHECK:-0}" = 1 ] && [ "$MACHINE" != matrix ] && [ "$MACHINE" != telengana ] && [ "$IS_TERMUX" != 1 ]; then + hostvars="$DOTFILES_DIR/host_vars/$MACHINE.yml" + if [ -e "$hostvars" ] || [ -L "$hostvars" ]; then + if [ ! -f "$hostvars" ] || [ ! -r "$hostvars" ] || ! cat "$hostvars" >/dev/null 2>&1; then + step "inspect host variables" false; finish + fi + elif [ ! -d "$DOTFILES_DIR/host_vars" ] || [ ! -x "$DOTFILES_DIR/host_vars" ]; then + step "inspect host variables directory" false; finish + fi +fi + +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ] || [ "$IS_TERMUX" = 1 ] || [ "$(host_var skip_systemd false)" = true ] || [ "$(host_var is_laptop false)" != true ]; then + note "SKIPPED: laptop services excluded on this host" + finish +fi +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then + export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus" +fi +unitdir="$HOME/.config/systemd/user" +brew_prefix=$(host_var brew_prefix /home/linuxbrew/.linuxbrew) +delay=$(host_var dotfiles_update_delay_seconds 60) +auto_update=$(host_var dotfiles_auto_update true) +phone_apply=$(host_var phone_apply_enabled true) +if [[ ! "$delay" =~ ^[0-9]+$ ]]; then + step "invalid dotfiles update delay" false + finish +fi +tmp=$(mktemp -d) || { step "create temporary directory" false; finish; } +trap 'rm -rf "$tmp"' EXIT +changed=0 +check_file() { + local src="$1" dest="$2" mode="$3" rc actual parent + CHECK_CHANGED=0 + if [ ! -f "$src" ] || [ ! -r "$src" ]; then + echo "Cannot inspect source: $src" >&2; return 1 + fi + if [ ! -e "$dest" ]; then + parent=$(dirname "$dest") + while [ ! -e "$parent" ] && [ "$parent" != / ]; do parent=$(dirname "$parent"); done + if [ ! -d "$parent" ] || [ ! -x "$parent" ]; then + echo "Cannot inspect destination: $dest" >&2; return 1 + fi + if ! cat "$src" >/dev/null; then echo "Cannot read source: $src" >&2; return 1; fi + CHECK_CHANGED=1 + note "would install $dest" + return 0 + fi + if [ ! -f "$dest" ] || [ ! -r "$dest" ]; then + echo "Cannot inspect destination: $dest" >&2; return 1 + fi + if cmp -s "$src" "$dest"; then :; else + rc=$? + if [ "$rc" != 1 ]; then echo "Cannot compare $dest" >&2; return 1; fi + CHECK_CHANGED=1 + note "would update $dest" + fi + actual=$(stat -Lc '%a' "$dest") || return 1 + [ "$actual" = "$mode" ] || note "would set mode $mode on $dest" +} +check_enabled() { + local unit="$1" start="$2" state rc + if state=$(systemctl --user is-enabled "$unit" 2>/dev/null); then rc=0; else rc=$?; fi + case "$state:$rc" in + enabled:0) ;; + disabled:1|masked:1|masked-runtime:1|not-found:1|not-found:4|static:0|indirect:0|indirect:1|linked:0|linked-runtime:0|enabled-runtime:0|generated:0|transient:0|alias:0) + note "would enable $unit" ;; + *) echo "Cannot inspect enabled state of $unit" >&2; return 1 ;; + esac + [ "$start" = 1 ] || return 0 + if state=$(systemctl --user is-active "$unit" 2>/dev/null); then rc=0; else rc=$?; fi + case "$state:$rc" in + active:0|reloading:0|refreshing:0) ;; + inactive:3|failed:3|activating:3|deactivating:3|unknown:3|unknown:4) + note "would start $unit" ;; + *) echo "Cannot inspect active state of $unit" >&2; return 1 ;; + esac +} +check_remove() { + local dest="$unitdir/$1" parent + if [ -e "$dest" ] || [ -L "$dest" ]; then + changed=1 + note "would disable and remove $dest" + else + parent=$(dirname "$dest") + while [ ! -e "$parent" ] && [ "$parent" != / ]; do parent=$(dirname "$parent"); done + if [ ! -d "$parent" ] || [ ! -x "$parent" ]; then + echo "Cannot inspect destination: $dest" >&2; return 1 + fi + fi +} +install_unit() { + local src="$1" dest="$unitdir/$2" + if [ "${UNITS_CHECK:-0}" = 1 ]; then + check_file "$src" "$dest" 644 || return 1 + [ "$CHECK_CHANGED" != 1 ] || changed=1 + return 0 + fi + if [ ! -f "$dest" ] || ! cmp -s "$src" "$dest"; then changed=1; fi + install_if_changed "$src" "$dest" 0644 +} +remove_unit() { + local name="$1" + if [ "${UNITS_CHECK:-0}" = 1 ]; then check_remove "$name"; return; fi + if [ -e "$unitdir/$name" ] || [ -L "$unitdir/$name" ]; then + systemctl --user disable --now "$name" || return 1 + rm -f "$unitdir/$name" || return 1 + changed=1 + fi +} + +cat > "$tmp/dotfiles-update.service" < "$tmp/atuin-daemon.service" < "$tmp/phone-apply.service" < ' + target) + return True + return False + + +def systemctl(*args): + result = subprocess.run(['systemctl', *args], capture_output=True, text=True, timeout=30) + if result.returncode: + raise RuntimeError('systemd inspection failed') + return result.stdout + + +def service_state(unit, enabled=True, active=True, disable=True): + # The apply installs these unit files before reloading systemd. + if not exists(Path('/etc/systemd/system', unit)): + plan(('enable and start ' if enabled else 'stop ') + unit) + return + values = {} + for line in systemctl('show', '--property=LoadState,UnitFileState,ActiveState', unit).splitlines(): + key, sep, value = line.partition('=') + if sep: + values[key] = value + if not {'LoadState', 'UnitFileState', 'ActiveState'} <= values.keys(): + raise RuntimeError('incomplete systemd unit inspection') + if values['LoadState'] not in ('loaded', 'not-found'): + raise RuntimeError('systemd unit has an invalid load state') + if enabled: + if values['UnitFileState'] != 'enabled': + plan('enable ' + unit) + if active and values['ActiveState'] != 'active': + plan('start ' + unit) + else: + if disable and values['UnitFileState'] in ('enabled', 'enabled-runtime', 'linked', 'linked-runtime', 'alias'): + plan('disable ' + unit) + if values['ActiveState'] not in ('inactive', 'failed'): + plan('stop ' + unit) + + +def main(): + repo, version, archive, checksum, poller, pollpath, state, home, user, uid, gid, apply_path = sys.argv[1:] + repo = Path(repo) + assets = repo / 'units/osquery' + uid, gid = int(uid), int(gid) + systemctl('show', '--property=Version', '--value') + for path in ('/var/cache/fleet-osquery', '/opt/osquery-' + version, '/opt/fleet-osquery'): + metadata(Path(path), 0o755, directory=True) + cache = Path('/var/cache/fleet-osquery') / archive + try: + with cache.open('rb') as stream: + hasher = hashlib.sha256() + for block in iter(lambda: stream.read(1024 * 1024), b''): + hasher.update(block) + digest = hasher.hexdigest() + except FileNotFoundError: + digest = None + if digest != checksum: + if not shutil.which('curl', path=apply_path): + raise RuntimeError('osquery download requires curl') + plan('download and verify osquery release ' + version) + target = Path('/opt/osquery-' + version + '/bin/osqueryd') + if not exists(target) and not shutil.which('tar', path=apply_path): + raise RuntimeError('osquery extraction requires tar') + metadata(target, 0o755) + symlink('/opt/osquery-' + version + '/bin/osqueryi', 'osqueryd') + daemon_changed = symlink('/opt/osquery', '/opt/osquery-' + version) + for path in ('/etc/osquery', '/var/lib/osquery', '/var/log/osquery', '/var/lib/fleet-osquery'): + metadata(Path(path), 0o700, directory=True) + installed_file(assets / 'files/fleet-osquery.py', '/opt/fleet-osquery/fleet-osquery', 0o755) + if installed_file(assets / 'templates/osquery.conf.j2', '/etc/osquery/osquery.conf', 0o600): + daemon_changed = True + for unit in ('osqueryd.service', 'fleet-osquery-report.service', 'fleet-osquery-report.timer'): + if installed_file(assets / ('templates/' + unit + '.j2'), '/etc/systemd/system/' + unit, 0o644) and unit == 'osqueryd.service': + daemon_changed = True + if daemon_changed: + plan('restart osqueryd.service after configuration changes') + if shutil.which('selinuxenabled'): + result = subprocess.run(['selinuxenabled'], capture_output=True, timeout=30) + if result.returncode not in (0, 1): + raise RuntimeError('SELinux status inspection failed') + if result.returncode == 0: + for path in (target, Path('/opt/fleet-osquery/fleet-osquery')): + if exists(path): + result = subprocess.run(['stat', '-c', '%C', str(path)], capture_output=True, text=True, timeout=30) + if result.returncode or len(result.stdout.strip().split(':')) < 3: + raise RuntimeError('SELinux context inspection failed') + if result.stdout.strip().split(':')[2] != 'bin_t': + plan('set SELinux bin_t context on ' + str(path)) + plan('reload systemd and validate osquery configuration during apply') + print('NOTE: configuration validation and release download are not run in check mode') + service_state('osqueryd.service') + service_state('fleet-osquery-report.timer') + if not exists(Path('/var/lib/fleet-osquery/baseline.json')): + plan('generate osquery baseline') + plan('run fleet-osquery-report.service') + if poller != 'true': + if exists(Path('/etc/systemd/system/fleet-osquery-poll.timer')): + service_state('fleet-osquery-poll.timer', enabled=False) + service_state('fleet-osquery-poll.service', enabled=False, disable=False) + return + spec = importlib.util.spec_from_file_location('osquery_poller_render', repo / 'units/osquery-poller-render.py') + renderer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(renderer) + original_path = os.environ.get('PATH', '') + try: + os.environ['PATH'] = pollpath + inventory = renderer.inventory(repo / 'inventory.yml') + finally: + os.environ['PATH'] = original_path + inventory_text = json.dumps(inventory, indent=2, sort_keys=True) + '\n' + service_text = renderer.service(assets / 'templates/fleet-osquery-poll.service.j2', user, str(gid), home, pollpath, state) + for path in ('/etc/fleet-osquery', '/opt/fleet-osquery'): + metadata(Path(path), 0o755, directory=True) + for source in ('scripts/osquery-fleet.py', 'scripts/online_hosts.py', 'units/osquery_native_inventory.py'): + installed_file(repo / source, '/opt/fleet-osquery/' + Path(source).name, 0o644) + file_from_bytes(Path('/etc/fleet-osquery/inventory.yml'), inventory_text.encode(), 0o644) + metadata(Path(state), 0o700, uid, gid, directory=True) + file_from_bytes(Path('/etc/systemd/system/fleet-osquery-poll.service'), service_text.encode(), 0o644) + installed_file(assets / 'templates/fleet-osquery-poll.timer.j2', '/etc/systemd/system/fleet-osquery-poll.timer', 0o644) + service_state('fleet-osquery-poll.timer') + plan('run fleet-osquery-poll.service') + + +if __name__ == '__main__': + try: + main() + except RuntimeError as error: + print('osquery check: ' + str(error), file=sys.stderr) + sys.exit(1) + except (OSError, ValueError, TypeError, KeyError, ImportError, subprocess.SubprocessError) as error: + print('osquery check: inspection failed (' + error.__class__.__name__ + ')', file=sys.stderr) + sys.exit(1) diff --git a/units/osquery-poller-render.py b/units/osquery-poller-render.py new file mode 100755 index 00000000..2a046beb --- /dev/null +++ b/units/osquery-poller-render.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Render the osquery poller's secret-free inventory and systemd service.""" +import json +from pathlib import Path +import sys + + +from osquery_native_inventory import load_inventory +GROUPS = ("vps", "test_fleet", "termux_hosts") + + +def inventory(path): + effective = load_inventory(path) + hosts = effective["_meta"]["hostvars"] + def members(name): + group = effective.get(name, {}) + result = set(group.get("hosts", [])) + for child in group.get("children", []): + result.update(members(child)) + return result + groups = {name: {"hosts": {host: {} for host in sorted(members(name))}} + for name in GROUPS} + return {"all": {"hosts": hosts, "children": groups}} + + +def service(template, user, group, home, path, state): + replacements = { + "{{ ansible_facts['user_id'] }}": user, + "{{ ansible_facts['user_gid'] }}": group, + "{{ ansible_facts['env']['HOME'] }}": home, + "{{ osquery_poller_path }}": path, + "{{ osquery_poller_state_dir }}": state, + } + if any(any(c in value for c in '\n\r"\\') for value in replacements.values()): + raise ValueError("unsupported characters in poller service values") + text = Path(template).read_text(encoding="utf-8") + for old, new in replacements.items(): + text = text.replace(old, new) + if "{{" in text or "{%" in text: + raise ValueError("unexpanded poller template") + return text + + +def main(): + try: + if sys.argv[1] == "inventory" and len(sys.argv) == 3: + print(json.dumps(inventory(sys.argv[2]), indent=2, sort_keys=True)) + elif sys.argv[1] == "service" and len(sys.argv) == 8: + print(service(*sys.argv[2:]), end="") + else: + raise ValueError("invalid arguments") + except (OSError, KeyError, TypeError, ValueError): + print("osquery poller rendering failed; verify inventory tooling and input", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/units/osquery.sh b/units/osquery.sh new file mode 100755 index 00000000..952e6338 --- /dev/null +++ b/units/osquery.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +osquery_verify_archive() { + local archive=$1 expected=$2 + shift 2 + printf '%s %s\n' "$expected" "$archive" | "$@" sha256sum -c --status +} + +osquery_main() { + local vps arch version archive checksum supported poller settings skip_setting supported_setting archives override_path override_state + local -a rootcmd=() + case "$MACHINE" in matrix|telengana) note "SKIPPED: retired VPS"; finish ;; esac + if [ "$IS_TERMUX" = 1 ] || [ "$IS_MUSL" = 1 ] || [ "$(uname -s)" != Linux ] || [ ! -d /run/systemd/system ]; then + note "SKIPPED: osquery requires native glibc Linux and systemd"; finish + fi + if ! command -v yq >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then step "osquery inventory requires yq and python3" false; finish; fi + vps=$(python3 "$DOTFILES_DIR/units/osquery_native_inventory.py" groups "$DOTFILES_DIR/inventory.yml" "$MACHINE" vps) || { step "check VPS membership" false; finish; } + case "$vps" in true) note "SKIPPED: VPS"; finish ;; false) ;; *) step "check VPS membership" false; finish ;; esac + arch=$(uname -m) + case "$arch" in x86_64|aarch64) ;; *) note "SKIPPED: unsupported osquery architecture $arch"; finish ;; esac + local defaults="$DOTFILES_DIR/units/osquery/defaults/main.yml" assets="$DOTFILES_DIR/units/osquery" + settings=$(python3 "$DOTFILES_DIR/units/osquery_native_inventory.py" osquery-settings "$DOTFILES_DIR/inventory.yml" "$MACHINE") || { step "read osquery host settings" false; finish; } + osquery_setting() { yq -p=json -r "$1" <<< "$settings"; } + skip_setting=$(osquery_setting '.skip_osquery') || { step "read osquery skip setting" false; finish; } + supported_setting=$(osquery_setting '.osquery_supported') || { step "read osquery support setting" false; finish; } + poller=$(osquery_setting '.osquery_poller_enabled') || { step "read osquery poller setting" false; finish; } + if [ "$supported_setting" = null ]; then supported_setting=$(yq -r '.osquery_supported' "$defaults") || { step "read osquery support default" false; finish; }; fi + if [ "$poller" = null ]; then poller=$(yq -r '.osquery_poller_enabled' "$defaults") || { step "read osquery poller default" false; finish; }; fi + case "$skip_setting:$supported_setting:$poller" in + true:true:true|true:true:false|true:false:true|true:false:false|null:false:true|null:false:false|false:false:true|false:false:false) + note "SKIPPED: osquery disabled for this host"; finish ;; + null:true:true|null:true:false|false:true:true|false:true:false) ;; + *) step "validate osquery host settings" false; finish ;; + esac + version=$(osquery_setting '.osquery_version') || { step "read osquery version override" false; finish; } + if [ "$version" = null ]; then version=$(yq -r '.osquery_version' "$defaults") || { step "read osquery version" false; finish; }; fi + archives=$(yq -p=json -o=json -I=0 '.osquery_release_archives' <<< "$settings") || { step "read osquery release override" false; finish; } + if [ "$archives" = null ]; then archives=$(yq -o=json -I=0 '.osquery_release_archives' "$defaults") || { step "read osquery release defaults" false; finish; }; fi + archive=$(ARCH="$arch" yq -p=json -r '.[strenv(ARCH)].name' <<< "$archives") || { step "read osquery archive" false; finish; } + checksum=$(ARCH="$arch" yq -p=json -r '.[strenv(ARCH)].sha256' <<< "$archives") || { step "read osquery checksum" false; finish; } + override_path=$(osquery_setting '.osquery_poller_path') || { step "read osquery poller path" false; finish; } + override_state=$(osquery_setting '.osquery_poller_state_dir') || { step "read osquery poller state" false; finish; } + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ ! "$archive" =~ ^osquery-[A-Za-z0-9_.-]+\.tar\.gz$ ]] || [[ ! "$checksum" =~ ^[0-9a-f]{64}$ ]]; then + step "validate osquery release metadata" false; finish + fi + if [ "$(id -u)" != 0 ]; then + if [ "${UNITS_CHECK:-0}" = 1 ]; then + if command -v sudo >/dev/null 2>&1; then rootcmd=(sudo -n); else step "noninteractive root access" false; finish; fi + elif command -v sudo >/dev/null 2>&1 && sudo -n true; then rootcmd=(sudo -n); else step "noninteractive root access" false; finish; fi + fi + for supported in systemctl python3; do + if ! command -v "$supported" >/dev/null 2>&1; then step "osquery dependency $supported" false; finish; fi + done + if [ "${UNITS_CHECK:-0}" = 1 ]; then + if ! command -v sha256sum >/dev/null 2>&1; then step "osquery dependency sha256sum" false; finish; fi + local pollpath="$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/bin:/data/data/com.termux/files/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + local state="$HOME/.local/state/fleet-osquery" user uid gid + if [ "$override_path" != null ]; then pollpath=$override_path; fi + if [ "$override_state" != null ]; then state=$override_state; fi + user=$(id -un) && uid=$(id -u) && gid=$(id -g) || { step "read osquery poller identity" false; finish; } + step "check osquery monitoring and poller" "${rootcmd[@]}" env PYTHONDONTWRITEBYTECODE=1 python3 \ + "$DOTFILES_DIR/units/osquery-check.py" "$DOTFILES_DIR" "$version" "$archive" "$checksum" \ + "$poller" "$pollpath" "$state" "$HOME" "$user" "$uid" "$gid" "$PATH" + finish + fi + + local temp + temp=$(mktemp -d) || { step "create osquery temporary directory" false; finish; } + trap 'rm -rf -- "$temp"' EXIT + osquery_install_file() { + local src=$1 dest=$2 mode=$3 + if "${rootcmd[@]}" cmp -s "$src" "$dest"; then + "${rootcmd[@]}" chmod "$mode" "$dest" && "${rootcmd[@]}" chown root:root "$dest" + return + fi + "${rootcmd[@]}" install -o root -g root -m "$mode" "$src" "$dest" || return 1 + OSQUERY_CHANGED=1 + } + osquery_install() { + local cache="/var/cache/fleet-osquery/$archive" target="/opt/osquery-$version/bin/osqueryd" unit + local OSQUERY_CHANGED=0 daemon_changed=0 + if ! command -v sha256sum >/dev/null 2>&1; then note "osquery requires sha256sum"; return 1; fi + "${rootcmd[@]}" install -d -o root -g root -m 0755 /var/cache/fleet-osquery "/opt/osquery-$version" /opt/fleet-osquery || return 1 + if ! osquery_verify_archive "$cache" "$checksum" "${rootcmd[@]}"; then + if ! command -v curl >/dev/null 2>&1; then note "osquery download requires curl"; return 1; fi + curl --fail --silent --show-error --location --max-time 120 "https://github.com/osquery/osquery/releases/download/$version/$archive" -o "$temp/$archive" || return 1 + osquery_verify_archive "$temp/$archive" "$checksum" || { note "osquery release checksum mismatch"; return 1; } + "${rootcmd[@]}" install -o root -g root -m 0644 "$temp/$archive" "$cache" || return 1 + fi + if [ ! -e "$target" ]; then + if ! command -v tar >/dev/null 2>&1; then note "osquery extraction requires tar"; return 1; fi + "${rootcmd[@]}" tar -xzf "$cache" --no-same-owner --strip-components=2 -C "/opt/osquery-$version" opt/osquery/bin/osqueryd || return 1 + fi + "${rootcmd[@]}" chown root:root "$target" && "${rootcmd[@]}" chmod 0755 "$target" || return 1 + if command -v selinuxenabled >/dev/null 2>&1 && selinuxenabled; then "${rootcmd[@]}" chcon -t bin_t "$target" || return 1; fi + "${rootcmd[@]}" ln -sfn osqueryd "/opt/osquery-$version/bin/osqueryi" || return 1 + if [ "$(readlink /opt/osquery 2>/dev/null)" != "/opt/osquery-$version" ]; then + "${rootcmd[@]}" ln -sfnT "/opt/osquery-$version" /opt/osquery || return 1 + daemon_changed=1 + fi + "${rootcmd[@]}" install -d -o root -g root -m 0700 /etc/osquery /var/lib/osquery /var/log/osquery /var/lib/fleet-osquery || return 1 + osquery_install_file "$assets/files/fleet-osquery.py" /opt/fleet-osquery/fleet-osquery 0755 || return 1 + if command -v selinuxenabled >/dev/null 2>&1 && selinuxenabled; then "${rootcmd[@]}" chcon -t bin_t /opt/fleet-osquery/fleet-osquery || return 1; fi + # The role's config has no substitutions. Validate before replacing it. + "${rootcmd[@]}" /opt/osquery/bin/osqueryd --disable_extensions --config_check --config_path="$assets/templates/osquery.conf.j2" || return 1 + OSQUERY_CHANGED=0 + osquery_install_file "$assets/templates/osquery.conf.j2" /etc/osquery/osquery.conf 0600 || return 1 + [ "$OSQUERY_CHANGED" = 0 ] || daemon_changed=1 + OSQUERY_CHANGED=0 + osquery_install_file "$assets/templates/osqueryd.service.j2" /etc/systemd/system/osqueryd.service 0644 || return 1 + [ "$OSQUERY_CHANGED" = 0 ] || daemon_changed=1 + for unit in fleet-osquery-report.service fleet-osquery-report.timer; do + osquery_install_file "$assets/templates/$unit.j2" "/etc/systemd/system/$unit" 0644 || return 1 + done + "${rootcmd[@]}" systemctl daemon-reload || return 1 + "${rootcmd[@]}" systemctl enable fleet-osquery-report.timer || return 1 + if [ "$daemon_changed" = 1 ]; then "${rootcmd[@]}" systemctl restart osqueryd.service || return 1; fi + "${rootcmd[@]}" systemctl enable --now osqueryd.service || return 1 + if [ ! -e /var/lib/fleet-osquery/baseline.json ]; then "${rootcmd[@]}" /opt/fleet-osquery/fleet-osquery baseline --if-missing || return 1; fi + "${rootcmd[@]}" systemctl start fleet-osquery-report.service || return 1 + "${rootcmd[@]}" systemctl start fleet-osquery-report.timer + } + osquery_poller() { + local pollpath="$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/bin:/data/data/com.termux/files/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + local state="$HOME/.local/state/fleet-osquery" unit OSQUERY_CHANGED=0 + if [ "$override_path" != null ]; then pollpath=$override_path; fi + if [ "$override_state" != null ]; then state=$override_state; fi + if [ "$poller" != true ]; then + if [ -e /etc/systemd/system/fleet-osquery-poll.timer ]; then + "${rootcmd[@]}" systemctl disable --now fleet-osquery-poll.timer || return 1 + "${rootcmd[@]}" systemctl stop fleet-osquery-poll.service || return 1 + fi + return 0 + fi + PATH="$pollpath" python3 "$DOTFILES_DIR/units/osquery-poller-render.py" inventory "$DOTFILES_DIR/inventory.yml" > "$temp/inventory.yml" || return 1 + python3 "$DOTFILES_DIR/units/osquery-poller-render.py" service "$assets/templates/fleet-osquery-poll.service.j2" "$(id -un)" "$(id -g)" "$HOME" "$pollpath" "$state" > "$temp/poll.service" || return 1 + "${rootcmd[@]}" install -d -o root -g root -m 0755 /etc/fleet-osquery /opt/fleet-osquery || return 1 + osquery_install_file "$DOTFILES_DIR/scripts/osquery-fleet.py" /opt/fleet-osquery/osquery-fleet.py 0644 || return 1 + osquery_install_file "$DOTFILES_DIR/scripts/online_hosts.py" /opt/fleet-osquery/online_hosts.py 0644 || return 1 + osquery_install_file "$DOTFILES_DIR/units/osquery_native_inventory.py" /opt/fleet-osquery/osquery_native_inventory.py 0644 || return 1 + osquery_install_file "$temp/inventory.yml" /etc/fleet-osquery/inventory.yml 0644 || return 1 + "${rootcmd[@]}" install -d -o "$(id -u)" -g "$(id -g)" -m 0700 "$state" || return 1 + osquery_install_file "$temp/poll.service" /etc/systemd/system/fleet-osquery-poll.service 0644 || return 1 + unit=fleet-osquery-poll.timer + osquery_install_file "$assets/templates/$unit.j2" "/etc/systemd/system/$unit" 0644 || return 1 + "${rootcmd[@]}" systemctl daemon-reload || return 1 + "${rootcmd[@]}" systemctl enable --now fleet-osquery-poll.timer || return 1 + "${rootcmd[@]}" systemctl start fleet-osquery-poll.service + } + step "osquery monitoring" osquery_install + step "osquery fleet poller" osquery_poller + finish +} + +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then osquery_main "$@"; fi diff --git a/roles/osquery/defaults/main.yml b/units/osquery/defaults/main.yml similarity index 100% rename from roles/osquery/defaults/main.yml rename to units/osquery/defaults/main.yml diff --git a/roles/osquery/files/fleet-osquery.py b/units/osquery/files/fleet-osquery.py old mode 100755 new mode 100644 similarity index 100% rename from roles/osquery/files/fleet-osquery.py rename to units/osquery/files/fleet-osquery.py diff --git a/roles/osquery/templates/fleet-osquery-poll.service.j2 b/units/osquery/templates/fleet-osquery-poll.service.j2 similarity index 100% rename from roles/osquery/templates/fleet-osquery-poll.service.j2 rename to units/osquery/templates/fleet-osquery-poll.service.j2 diff --git a/roles/osquery/templates/fleet-osquery-poll.timer.j2 b/units/osquery/templates/fleet-osquery-poll.timer.j2 similarity index 100% rename from roles/osquery/templates/fleet-osquery-poll.timer.j2 rename to units/osquery/templates/fleet-osquery-poll.timer.j2 diff --git a/roles/osquery/templates/fleet-osquery-report.service.j2 b/units/osquery/templates/fleet-osquery-report.service.j2 similarity index 100% rename from roles/osquery/templates/fleet-osquery-report.service.j2 rename to units/osquery/templates/fleet-osquery-report.service.j2 diff --git a/roles/osquery/templates/fleet-osquery-report.timer.j2 b/units/osquery/templates/fleet-osquery-report.timer.j2 similarity index 100% rename from roles/osquery/templates/fleet-osquery-report.timer.j2 rename to units/osquery/templates/fleet-osquery-report.timer.j2 diff --git a/roles/osquery/templates/osquery.conf.j2 b/units/osquery/templates/osquery.conf.j2 similarity index 100% rename from roles/osquery/templates/osquery.conf.j2 rename to units/osquery/templates/osquery.conf.j2 diff --git a/roles/osquery/templates/osqueryd.service.j2 b/units/osquery/templates/osqueryd.service.j2 similarity index 100% rename from roles/osquery/templates/osqueryd.service.j2 rename to units/osquery/templates/osqueryd.service.j2 diff --git a/units/osquery_native_inventory.py b/units/osquery_native_inventory.py new file mode 100644 index 00000000..1b409e96 --- /dev/null +++ b/units/osquery_native_inventory.py @@ -0,0 +1,214 @@ +"""Resolve the connection variables used by the fleet collector, without Ansible.""" +import json +from pathlib import Path +import re +import subprocess +import sys + + +FIELDS = ("ansible_host", "ansible_user", "ansible_ssh_user", "ansible_port", + "ansible_ssh_port", "ansible_connection", "tailnet_name", "osquery_supported") +NAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*$") + + +def yaml_document(path): + try: + text = path.read_text(encoding="utf-8") + try: + return json.loads(text) + except json.JSONDecodeError: + pass + result = subprocess.run(["yq", "-o=json", "-I=0", ".", str(path)], + capture_output=True, timeout=30, check=True) + return json.loads(result.stdout) + except FileNotFoundError as error: + if error.filename == "yq": + raise ValueError("native inventory requires yq for YAML") from None + raise ValueError("inventory file cannot be read") from None + except (OSError, UnicodeError, ValueError, subprocess.SubprocessError): + raise ValueError("inventory YAML could not be parsed") from None + + +def mapping(value): + if value is None: + return {} + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise ValueError("inventory contains malformed mappings") + return value + + +def name_ok(name): + if not isinstance(name, str) or not NAME.fullmatch(name) or name in (".", ".."): + raise ValueError("inventory contains invalid names") + return name + + +def variable_files(root, directory, name): + base = root / directory + for path in (base / name, base / (name + ".yml"), base / (name + ".yaml"), base / (name + ".json")): + if path.is_file(): + yield path + elif path.is_dir(): + for item in sorted(path.iterdir()): + if item.is_file() and (item.suffix in ("", ".yml", ".yaml", ".json")) and not item.name.startswith("."): + yield item + + +def selected(values, fields): + return {key: values[key] for key in fields if key in values} + + +def contains_template(value): + if isinstance(value, str): + return "{{" in value or "{%" in value + if isinstance(value, dict): + return any(contains_template(item) for item in value.values()) + if isinstance(value, list): + return any(contains_template(item) for item in value) + return False + + +def load_inventory(path, resolve_vars=True, fields=FIELDS): + path = Path(path) + document = mapping(yaml_document(path)) + if not document: + raise ValueError("inventory is empty") + groups, hosts = {}, {} + + def group(name): + name_ok(name) + return groups.setdefault(name, {"hosts": set(), "children": set(), "vars": {}}) + + def add_group(name, definition, depth=0): + if depth > 100: + raise ValueError("inventory contains a group cycle") + entry = group(name) + definition = mapping(definition) + if set(definition) - {"hosts", "children", "vars"}: + raise ValueError("inventory contains invalid group properties") + entry["vars"].update(mapping(definition.get("vars"))) + for host, values in mapping(definition.get("hosts")).items(): + name_ok(host) + entry["hosts"].add(host) + hosts.setdefault(host, {}).update(mapping(values)) + for child, value in mapping(definition.get("children")).items(): + name_ok(child) + entry["children"].add(child) + add_group(child, value, depth + 1) + + group("all") + for name, value in document.items(): + add_group(name, value) + if name != "all": + groups["all"]["children"].add(name) + + depths = {"all": 0} + def visit(name, stack, depth): + if name in stack: + raise ValueError("inventory contains a group cycle") + depths[name] = max(depth, depths.get(name, 0)) + for child in groups[name]["children"]: + visit(child, stack | {name}, depth + 1) + visit("all", set(), 0) + + def priority(name): + value = groups[name]["vars"].get("ansible_group_priority", 1) + if isinstance(value, bool): + raise ValueError("invalid inventory group priority") + try: + return int(value) + except (ValueError, TypeError): + raise ValueError("invalid inventory group priority") from None + + order = sorted(groups, key=lambda g: (depths.get(g, 0), priority(g), g)) + members = {} + def group_members(name): + if name not in members: + members[name] = set(groups[name]["hosts"]) + for child in groups[name]["children"]: + members[name].update(group_members(child)) + return members[name] + for name in order: + group_members(name) + result = {name: {"hosts": sorted(entry["hosts"]), "children": sorted(entry["children"])} + for name, entry in groups.items()} + if not resolve_vars: + result["_meta"] = {"hostvars": {host: {} for host in hosts}} + return result + effective = {host: {} for host in hosts} + # Inventory-defined group vars precede group_vars files; within each + # source, parent groups precede children and siblings use priority/name. + for name in order: + for host in members[name]: + effective[host].update(selected(groups[name]["vars"], fields)) + for name in order: + variables = {} + for source in variable_files(path.parent, "group_vars", name): + variables.update(selected(mapping(yaml_document(source)), fields)) + for host in members[name]: + effective[host].update(variables) + for host in hosts: + effective[host].update(selected(hosts[host], fields)) + for source in variable_files(path.parent, "host_vars", host): + effective[host].update(selected(mapping(yaml_document(source)), fields)) + for value in effective[host].values(): + if contains_template(value): + raise ValueError("inventory connection variables contain unresolved templates") + # Group membership was already validated above. + result["_meta"] = {"hostvars": effective} + return result + + +def group_contains(inventory, group, host): + item = inventory.get(group, {}) + return host in item.get("hosts", []) or any( + group_contains(inventory, child, host) for child in item.get("children", [])) + + +def main(argv): + try: + if len(argv) < 3 or argv[0] not in ("groups", "osquery-settings"): + raise ValueError("invalid inventory query") + host = argv[2] + if argv[0] == "groups": + if len(argv) < 4: + raise ValueError("invalid inventory query") + inventory = load_inventory(argv[1], resolve_vars=False) + if host not in inventory["_meta"]["hostvars"]: + raise ValueError("machine is absent from inventory") + print(" ".join("true" if group_contains(inventory, group, host) else "false" + for group in argv[3:])) + else: + if len(argv) != 3: + raise ValueError("invalid inventory query") + booleans = ("skip_osquery", "osquery_supported", "osquery_poller_enabled") + fields = booleans + ("osquery_version", "osquery_release_archives", + "osquery_poller_path", "osquery_poller_state_dir") + inventory = load_inventory(argv[1], fields=fields) + if host not in inventory["_meta"]["hostvars"]: + raise ValueError("machine is absent from inventory") + values = inventory["_meta"]["hostvars"][host] + for key in booleans: + value = values.get(key) + if value is None: + values[key] = None + elif value is True or str(value).lower() in ("true", "yes", "on", "1"): + values[key] = True + elif value is False or str(value).lower() in ("false", "no", "off", "0"): + values[key] = False + else: + raise ValueError("invalid osquery boolean setting") + for key in ("osquery_version", "osquery_poller_path", "osquery_poller_state_dir"): + if key in values and not isinstance(values[key], str): + raise ValueError("invalid osquery string setting") + if "osquery_release_archives" in values and not isinstance(values["osquery_release_archives"], dict): + raise ValueError("invalid osquery release setting") + print(json.dumps(values)) + except (OSError, ValueError, TypeError, KeyError) as error: + print(f"native inventory: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/units/package-config.bash b/units/package-config.bash new file mode 100644 index 00000000..47302874 --- /dev/null +++ b/units/package-config.bash @@ -0,0 +1,46 @@ +# shellcheck shell=bash +# Helpers for the package units. The repository YAML remains authoritative. +package_require_yq() { + if ! command -v yq >/dev/null 2>&1; then + echo 'yq is required to read package and inventory configuration' >&2 + return 1 + fi +} + +# package_list KEY [FILE]: populate PACKAGE_LIST from a YAML sequence. Group +# defaults are overridden by the host file; FILE selects an authoritative role +# vars file instead (role vars take precedence over group and host vars). +package_list() { + local key="$1" source="${2:-$DOTFILES_DIR/group_vars/all.yml}" host="$DOTFILES_DIR/host_vars/$MACHINE.yml" has value kind item + PACKAGE_LIST=() + if [ "$#" -eq 1 ] && [ -f "$host" ]; then + has="$(yq -r "has(\"$key\")" "$host")" || return 1 + [ "$has" != true ] || source="$host" + fi + kind="$(yq -r ".$key | tag" "$source")" || return 1 + case "$kind" in '!!null') return 0 ;; '!!seq') ;; *) echo "$source: $key must be a list" >&2; return 1 ;; esac + # Package names and IDs cannot contain whitespace or control characters. + value="$(yq -r ".$key | .[]" "$source")" || return 1 + [ -n "$value" ] || return 0 + while IFS= read -r item; do + case "$item" in ''|*[!a-zA-Z0-9_@.+/-]*) echo "$source: invalid value in $key" >&2; return 1 ;; esac + PACKAGE_LIST+=("$item") + done <<< "$value" +} + +# Resolve desktop membership; malformed or missing inputs fail. +package_desktop() { + local value="${DOTFILES_IS_DESKTOP:-}" host="$DOTFILES_DIR/host_vars/$MACHINE.yml" has + if [ -z "$value" ] && [ -f "$host" ]; then + has="$(yq -r 'has("is_desktop")' "$host")" || return 1 + if [ "$has" = true ]; then value="$(yq -r '.is_desktop' "$host")" || return 1; fi + fi + if [ -z "$value" ]; then + value="$(MACHINE="$MACHINE" yq -r '.all.children.desktop.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml")" || return 1 + fi + case "$value" in 1|true) PACKAGE_DESKTOP=1 ;; 0|false) PACKAGE_DESKTOP=0 ;; *) echo 'Invalid desktop classification' >&2; return 1 ;; esac +} + +package_as_root() { + if [ "$(id -u)" = 0 ]; then "$@"; else sudo -n "$@"; fi +} diff --git a/units/pipewire-audio.sh b/units/pipewire-audio.sh new file mode 100755 index 00000000..d2bc1e86 --- /dev/null +++ b/units/pipewire-audio.sh @@ -0,0 +1,611 @@ +#!/usr/bin/env bash +# Native PipeWire DSP and the user-session microphone routing service. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if skipped_by_profile skip_pipewire_audio; then + note "SKIPPED: skip_pipewire_audio / vm-test" + finish +fi +# is_desktop silently treats a missing or broken yq as a non-desktop. Only +# consult inventory when neither the environment nor host_vars decides it. +desktop="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" +case "$desktop" in + 1|true) ;; + 0|false) note "SKIPPED: not a desktop"; finish ;; + '') + if ! command -v yq >/dev/null 2>&1; then + step "desktop inventory requires yq" false + finish + fi + desktop="$(yq ".all.children.desktop.hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml")" || { + step "read desktop inventory" false + finish + } + case "$desktop" in + true) ;; + false) note "SKIPPED: not a desktop"; finish ;; + *) step "invalid desktop inventory result" false; finish ;; + esac + ;; + *) step "invalid is_desktop override" false; finish ;; +esac + +assets="$DOTFILES_DIR/units/pipewire-audio" +conf="$HOME/.config/pipewire/pipewire.conf.d" +user_systemd="$HOME/.config/systemd/user" +state="$HOME/.local/state/pipewire-audio" +# The unit default and flat group/host variables remain the source of truth. +flat_var() { + local value + value="$(sed -n "s/^$2:[[:space:]]*\([^#]*[^#[:space:]]\).*/\1/p" "$1" 2>/dev/null | head -n1)" + value="${value//[\"\']/}" + printf '%s' "$value" +} +resolve_mic_volume() { + local value group file membership present + value="${MIC_HARDWARE_VOLUME_PCT:-$(host_var mic_hardware_volume_pct)}" + if [ -n "$value" ]; then printf '%s' "$value"; return 0; fi + [ -r "$assets/defaults/main.yml" ] || return 1 + value="$(flat_var "$assets/defaults/main.yml" mic_hardware_volume_pct)" + [ -n "$value" ] || return 1 + file="$DOTFILES_DIR/group_vars/all.yml" + if [ -f "$file" ]; then + group="$(flat_var "$file" mic_hardware_volume_pct)" + [ -z "$group" ] || value="$group" + fi + for file in "$DOTFILES_DIR"/group_vars/*.yml; do + [ -f "$file" ] || continue + group="${file##*/}"; group="${group%.yml}" + [ "$group" != all ] || continue + membership="$(flat_var "$file" mic_hardware_volume_pct)" + [ -n "$membership" ] || continue + if [ "$group" = desktop ]; then + value="$membership" + else + command -v yq >/dev/null 2>&1 || { note "mic_hardware_volume_pct in group $group requires yq" >&2; return 1; } + present="$(yq ".all.children[\"$group\"].hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml")" || return 1 + case "$present" in + true) value="$membership" ;; + false) ;; + *) note "unable to resolve group $group membership" >&2; return 1 ;; + esac + fi + done + printf '%s' "$value" +} +if ! mic_volume="$(resolve_mic_volume)"; then + step "resolve mic_hardware_volume_pct" false + finish +fi +# The volume is substituted into a shell command in a systemd unit. +if [[ ! "$mic_volume" =~ ^[0-9]{1,3}$ ]] || (( 10#$mic_volume > 100 )); then + step "invalid mic_hardware_volume_pct (expected 0-100)" false + finish +fi + +as_root() { + if [ "$(id -u)" = 0 ]; then "$@"; else sudo -n "$@"; fi +} + +if [ "${UNITS_CHECK:-0}" = 1 ]; then + check_audio_packages() { + local os_id="" os_like="" pkg result rc manager=0 + if [ -r /etc/os-release ]; then + os_id="$(. /etc/os-release; printf '%s' "${ID:-}")" + os_like="$(. /etc/os-release; printf '%s' "${ID_LIKE:-}")" + fi + if command -v dnf >/dev/null 2>&1 || command -v rpm-ostree >/dev/null 2>&1; then + manager=1 + command -v rpm >/dev/null 2>&1 || { note "FAIL rpm required to inspect DSP packages"; return 1; } + for pkg in lsp-plugins-lv2 noise-suppression-for-voice; do + result="$(LC_ALL=C rpm -q "$pkg" 2>&1)"; rc=$? + if [ "$rc" = 0 ]; then : + elif [ "$rc" = 1 ] && [ "$result" = "package $pkg is not installed" ]; then + note "WOULD install DSP package $pkg" + else + note "FAIL inspect DSP package $pkg"; return 1 + fi + done + fi + if [[ " $os_id $os_like " == *" arch "* ]]; then + manager=1 + command -v pacman >/dev/null 2>&1 || { note "FAIL pacman required to inspect DSP packages"; return 1; } + for pkg in lsp-plugins noise-suppression-for-voice; do + result="$(LC_ALL=C pacman -Q "$pkg" 2>&1)"; rc=$? + if [ "$rc" = 0 ]; then : + elif [ "$rc" = 1 ] && [ "$result" = "error: package '$pkg' was not found" ]; then + note "WOULD install DSP package $pkg" + else + note "FAIL inspect DSP package $pkg"; return 1 + fi + done + fi + if [ "$manager" = 0 ]; then + note "PARTIAL: DSP package prerequisites must be inspected for this distribution" + fi + } + + # Return 0 for equal bytes, 1 for different, and 2 for an inspection error. + check_compare() { + local rc + cmp -s "$1" "$2"; rc=$? + [ "$rc" -le 1 ] || { note "FAIL compare audio config $2" >&2; return 2; } + return "$rc" + } + check_directory() { + local dir="$1" mode + if [ ! -e "$dir" ] && [ ! -L "$dir" ]; then + note "WOULD create audio directory $dir" + elif [ ! -d "$dir" ]; then + note "FAIL audio directory is not a directory: $dir"; return 1 + else + mode="$(stat -Lc %a "$dir")" || return 1 + [ "$mode" = 755 ] || note "WOULD set audio directory permissions: $dir" + fi + } + check_managed_config() { + local src="$1" dest="$2" marker="$3" candidate rc mode owned=0 + shift 3 + custom_config=1 + [ -r "$src" ] || { note "FAIL read canonical audio asset $src"; return 1; } + for candidate in "$@"; do + [ -r "$candidate" ] || { note "FAIL read canonical audio asset $candidate"; return 1; } + done + if [ ! -e "$dest" ] && [ ! -L "$dest" ]; then + owned=1 + elif [ ! -L "$dest" ] && [ -f "$dest" ]; then + for candidate in "$@"; do + check_compare "$candidate" "$dest"; rc=$? + [ "$rc" != 2 ] || return 1 + if [ "$rc" = 0 ]; then owned=1; break; fi + done + fi + if [ "$owned" = 0 ]; then + note "PARTIAL: retained custom audio config $dest" + return 0 + fi + custom_config=0 + if [ ! -e "$dest" ]; then + note "WOULD install audio config $dest" + printf -v "${marker}_changed" '%s' 1 + return 0 + fi + check_compare "$src" "$dest"; rc=$? + [ "$rc" != 2 ] || return 1 + mode="$(stat -c %a "$dest")" || return 1 + if [ "$rc" = 1 ] || [ "$mode" != 644 ]; then + note "WOULD install audio config $dest" + printf -v "${marker}_changed" '%s' 1 + fi + } + check_remove_config() { + local dest="$1" marker="$2" candidate="$3" rc + custom_config=0 + [ -r "$candidate" ] || { note "FAIL read canonical audio asset $candidate"; return 1; } + [ -e "$dest" ] || [ -L "$dest" ] || return 0 + if [ -L "$dest" ] || [ ! -f "$dest" ]; then + custom_config=1 + else + check_compare "$candidate" "$dest"; rc=$? + [ "$rc" != 2 ] || return 1 + [ "$rc" = 0 ] || custom_config=1 + fi + if [ "$custom_config" = 1 ]; then + note "PARTIAL: retained custom audio config $dest" + else + note "WOULD remove audio config $dest" + printf -v "${marker}_changed" '%s' 1 + fi + } + check_audio() { + local rnnoise lsp=unknown lv2_plugins speaker dir mic_service template pattern prefix suffix existing volume + local rc mode dropin old_rule mic_config_custom=1 mic_service_custom=0 + local pipewire_changed=0 wireplumber_changed=0 mic_changed=0 uid properties line load="" active="" enabled="" + local running installed current app + check_audio_packages || return 1 + local paths=() p + for p in /usr/lib /usr/lib64; do [ ! -d "$p" ] || paths+=("$p"); done + [ "${#paths[@]}" -gt 0 ] || { note "FAIL inspect RNNoise plugin"; return 1; } + rnnoise="$(find "${paths[@]}" -name librnnoise_ladspa.so -print -quit)" || { note "FAIL inspect RNNoise plugin"; return 1; } + if command -v lv2ls >/dev/null 2>&1; then + lv2_plugins="$(lv2ls)" || { note "FAIL inspect LSP plugins"; return 1; } + lsp=0 + [[ "$lv2_plugins" != *lsp-plug.in/plugins/lv2/para_equalizer_x16_lr* ]] || lsp=1 + else + note "PARTIAL: LSP availability is unknown (lv2ls unavailable)" + fi + for dir in "$conf" "$HOME/.config/pipewire/filter-chain.conf.d" "$user_systemd/pipewire.service.d" "$HOME/.ladspa" "$state"; do + check_directory "$dir" || return 1 + done + if [ "$lsp" = unknown ] && { [ -e "$conf/50-speaker-dsp.conf" ] || [ -L "$conf/50-speaker-dsp.conf" ]; }; then + note "PARTIAL: retained existing speaker DSP because LSP availability is unknown" + else + if [ "$lsp" = 1 ]; then speaker=speaker-dsp.conf; else speaker=speaker-biquad.conf; fi + check_managed_config "$assets/files/$speaker" "$conf/50-speaker-dsp.conf" pipewire "$assets/files/speaker-dsp.conf" "$assets/files/speaker-biquad.conf" || return 1 + fi + if [ -n "$rnnoise" ]; then + check_managed_config "$assets/files/mic-rnnoise.conf" "$conf/60-mic-dsp.conf" pipewire "$assets/files/mic-rnnoise.conf" || return 1 + else + check_remove_config "$conf/60-mic-dsp.conf" pipewire "$assets/files/mic-rnnoise.conf" || return 1 + fi + mic_config_custom="$custom_config" + dropin="$user_systemd/pipewire.service.d/ladspa-path.conf" + check_managed_config <(printf '%s\n' '[Service]' 'Environment=LADSPA_PATH=%h/.ladspa:/usr/lib/x86_64-linux-gnu/ladspa:/usr/lib/ladspa:/usr/lib64/ladspa') "$dropin" pipewire <(printf '%s\n' '[Service]' 'Environment=LADSPA_PATH=%h/.ladspa:/usr/lib/x86_64-linux-gnu/ladspa:/usr/lib/ladspa:/usr/lib64/ladspa') || return 1 + + mic_service="$user_systemd/mic-dsp-link.service" + template="$(cat "$assets/templates/mic-dsp-link.service.j2")" || { note "FAIL read microphone link service template"; return 1; } + pattern='{{ mic_hardware_volume_pct | default(100) }}' + [[ "$template" == *"$pattern"* ]] || { note "FAIL invalid microphone link service template"; return 1; } + if [ -e "$mic_service" ] || [ -L "$mic_service" ]; then + if [ -f "$mic_service" ] && [ ! -L "$mic_service" ]; then + existing="$(cat "$mic_service")" || { note "FAIL read microphone link service"; return 1; } + prefix="${template%%"$pattern"*}"; suffix="${template#*"$pattern"}" + if [[ "$existing" == "$prefix"*"$suffix" ]]; then + volume="${existing#"$prefix"}"; volume="${volume%"$suffix"}" + [[ "$volume" =~ ^[0-9]+$ ]] || mic_service_custom=1 + else + mic_service_custom=1 + fi + else + mic_service_custom=1 + fi + [ "$mic_service_custom" = 0 ] || note "PARTIAL: retained custom microphone link service $mic_service" + elif [ -z "$rnnoise" ]; then + mic_service_custom=1 + fi + if [ "$mic_config_custom" = 1 ]; then + mic_service_custom=1 + note "PARTIAL: microphone link service unchanged because microphone DSP is not managed" + fi + if [ -n "$rnnoise" ] && [ "$mic_service_custom" = 0 ]; then + template="${template//"$pattern"/$mic_volume}" + if [ -e "$mic_service" ]; then + check_compare <(printf '%s\n' "$template") "$mic_service"; rc=$? + [ "$rc" != 2 ] || return 1 + mode="$(stat -c %a "$mic_service")" || return 1 + else + rc=1; mode="" + fi + if [ "$rc" = 1 ] || [ "$mode" != 644 ]; then + note "WOULD install microphone link service $mic_service" + mic_changed=1 + fi + fi + old_rule="$HOME/.config/wireplumber/main.lua.d/51-speaker-dsp-default.lua" + check_remove_config "$old_rule" wireplumber "$assets/files/51-speaker-dsp-default.lua" || return 1 + dir="${old_rule%/*}" + if [ -d "$dir" ] && [ ! -L "$dir" ]; then + [ -r "$dir" ] && [ -x "$dir" ] || { note "FAIL inspect WirePlumber rule directory"; return 1; } + if ( + shopt -s nullglob dotglob + entries=("$dir"/*) + [ "${#entries[@]}" = 0 ] || { [ "${#entries[@]}" = 1 ] && [ "${entries[0]}" = "$old_rule" ] && [ "$custom_config" = 0 ]; } + ); then + note "WOULD remove empty obsolete WirePlumber rule directory" + fi + fi + + uid="$(id -u)" || return 1 + export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$uid}" + export DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-unix:path=$XDG_RUNTIME_DIR/bus}" + systemctl --user show-environment >/dev/null || { note "FAIL inspect user audio session"; return 1; } + if [ "$pipewire_changed" = 1 ] || [ "$mic_changed" = 1 ] || [ -e "$state/pipewire" ] || [ -e "$state/mic" ]; then + note "WOULD reload user service definitions" + fi + if [ "$pipewire_changed" = 1 ] || [ -e "$state/pipewire" ]; then note "WOULD restart PipeWire services"; fi + if [ "$wireplumber_changed" = 1 ] || [ -e "$state/wireplumber" ]; then note "WOULD restart WirePlumber"; fi + if [ "$mic_service_custom" = 0 ]; then + properties="$(systemctl --user show mic-dsp-link.service --property=LoadState --property=ActiveState --property=UnitFileState)" || { note "FAIL inspect microphone link service"; return 1; } + while IFS= read -r line; do + case "$line" in LoadState=*) load="${line#*=}" ;; ActiveState=*) active="${line#*=}" ;; UnitFileState=*) enabled="${line#*=}" ;; esac + done <<< "$properties" + case "$load" in loaded|not-found|error|masked) ;; *) note "FAIL inspect microphone service load state"; return 1 ;; esac + case "$active" in active|inactive|failed|activating|deactivating|reloading|maintenance) ;; *) note "FAIL inspect microphone service active state"; return 1 ;; esac + case "$enabled" in enabled|enabled-runtime|disabled|static|indirect|generated|transient|masked|masked-runtime|linked|linked-runtime|alias|bad|'') ;; *) note "FAIL inspect microphone service enablement"; return 1 ;; esac + if [ "$load" = error ] || [ "$enabled" = bad ]; then note "FAIL inspect microphone service definition"; return 1; fi + if [ -n "$rnnoise" ]; then + [ "$enabled" = enabled ] || [ "$enabled" = enabled-runtime ] || note "WOULD enable microphone link service" + if [ "$active" != active ]; then note "WOULD start microphone link service" + elif [ "$mic_changed" = 1 ] || [ -e "$state/mic" ]; then note "WOULD restart microphone link service"; fi + elif [ "$load" != not-found ]; then + [ "$active" != active ] || note "WOULD stop microphone link service" + if [ "$enabled" = enabled ] || [ "$enabled" = enabled-runtime ]; then note "WOULD disable microphone link service"; fi + fi + fi + if command -v gsettings >/dev/null 2>&1; then + current="$(gsettings get org.gnome.desktop.sound allow-volume-above-100-percent)" || { note "FAIL inspect GNOME over-amplification"; return 1; } + case "$current" in true) ;; false) note "WOULD enable GNOME over-amplification" ;; *) note "FAIL invalid GNOME over-amplification state"; return 1 ;; esac + else + note "PARTIAL: GNOME over-amplification unavailable" + fi + if [ -e "$HOME/.config/autostart/com.github.wwmm.easyeffects.desktop" ] || [ -L "$HOME/.config/autostart/com.github.wwmm.easyeffects.desktop" ]; then + note "WOULD remove Easy Effects autostart" + fi + if command -v flatpak >/dev/null 2>&1; then + running="$(flatpak ps --columns=application)" || { note "FAIL inspect running Flatpaks"; return 1; } + installed="$(flatpak list --system --app --columns=application)" || { note "FAIL inspect installed Flatpaks"; return 1; } + while IFS= read -r app; do [ "$app" != com.github.wwmm.easyeffects ] || note "WOULD stop Easy Effects Flatpak"; done <<< "$running" + while IFS= read -r app; do [ "$app" != com.github.wwmm.easyeffects ] || note "WOULD uninstall Easy Effects Flatpak"; done <<< "$installed" + fi + } + step "check PipeWire audio" check_audio + finish +fi + +install_packages() { + local os_id="" os_like="" supported=0 + if [ -r /etc/os-release ]; then + # os-release is the system's shell-compatible distribution metadata. + os_id="$(. /etc/os-release; printf '%s' "${ID:-}")" + os_like="$(. /etc/os-release; printf '%s' "${ID_LIKE:-}")" + fi + if command -v dnf >/dev/null 2>&1; then + supported=1 + if ! command -v rpm >/dev/null 2>&1 || ! rpm -q lsp-plugins-lv2 noise-suppression-for-voice >/dev/null 2>&1; then + as_root dnf install -y lsp-plugins-lv2 noise-suppression-for-voice || return 1 + fi + fi + if command -v rpm-ostree >/dev/null 2>&1; then + supported=1 + if ! command -v rpm >/dev/null 2>&1 || ! rpm -q lsp-plugins-lv2 noise-suppression-for-voice >/dev/null 2>&1; then + as_root rpm-ostree install --apply-live --idempotent lsp-plugins-lv2 noise-suppression-for-voice || return 1 + fi + fi + if [[ " $os_id $os_like " == *" arch "* ]]; then + supported=1 + if ! pacman -Q lsp-plugins noise-suppression-for-voice >/dev/null 2>&1; then + as_root pacman -S --needed --noconfirm lsp-plugins noise-suppression-for-voice || return 1 + fi + fi + if [ "$supported" = 0 ]; then + if command -v bootc >/dev/null 2>&1; then + note "PARTIAL: bootc DSP prerequisites must be provided by the image" + else + note "PARTIAL: no supported DSP package manager; install prerequisites for this distribution" + fi + fi +} +step "DSP prerequisite packages" install_packages + +find_rnnoise() { + local paths=() p + for p in /usr/lib /usr/lib64; do [ ! -d "$p" ] || paths+=("$p"); done + [ "${#paths[@]}" -gt 0 ] || { note "No system library directories found" >&2; return 1; } + find "${paths[@]}" -name librnnoise_ladspa.so -print -quit +} +rnnoise="" +if ! rnnoise="$(find_rnnoise)"; then + step "detect RNNoise plugin" false + # A failed search is not proof the plugin is absent: leave the mic alone. + rnnoise=unknown +fi +lsp=unknown +if command -v lv2ls >/dev/null 2>&1; then + if lv2_plugins="$(lv2ls)"; then + lsp=0 + [[ "$lv2_plugins" != *lsp-plug.in/plugins/lv2/para_equalizer_x16_lr* ]] || lsp=1 + else + note "PARTIAL: lv2ls failed; LSP availability is unknown" + fi +else + note "PARTIAL: lv2ls unavailable; LSP availability is unknown" +fi + +step "audio directories" install -d -m 0755 "$conf" "$HOME/.config/pipewire/filter-chain.conf.d" "$user_systemd/pipewire.service.d" "$HOME/.ladspa" "$state" + +# Persist restart requests, so a failed user-session restart is retried next run. +install_config() { + local src="$1" dest="$2" marker="$3" changed=0 + if [ ! -f "$dest" ] || ! cmp -s "$src" "$dest" || [ "$(stat -c %a "$dest")" != 644 ]; then changed=1; fi + if [ "$changed" = 1 ]; then touch "$state/$marker" || return 1; fi + install_if_changed "$src" "$dest" 0644 +} +remove_config() { + local dest="$1" marker="$2" + if [ -e "$dest" ] || [ -L "$dest" ]; then + touch "$state/$marker" && rm -f "$dest" || return 1 + fi +} +# Only replace or remove files whose bytes match an old managed asset. +owned_or_absent() { + local dest="$1" candidate + shift + [ -e "$dest" ] || [ -L "$dest" ] || return 0 + for candidate in "$@"; do [ -r "$candidate" ] || return 2; done + [ ! -L "$dest" ] || return 1 + for candidate in "$@"; do cmp -s "$dest" "$candidate" && return 0; done + return 1 +} +install_managed() { + local src="$1" dest="$2" marker="$3" rc + shift 3 + custom_config=1 + if owned_or_absent "$dest" "$@"; then + install_config "$src" "$dest" "$marker" || return 1 + custom_config=0 + else + rc=$? + [ "$rc" = 1 ] || { note "Missing canonical audio asset for $dest" >&2; return 1; } + note "PARTIAL: retained custom audio config $dest" + fi +} +remove_managed() { + local dest="$1" marker="$2" rc + shift 2 + custom_config=1 + if owned_or_absent "$dest" "$@"; then + remove_config "$dest" "$marker" || return 1 + custom_config=0 + else + rc=$? + [ "$rc" = 1 ] || { note "Missing canonical audio asset for $dest" >&2; return 1; } + note "PARTIAL: retained custom audio config $dest" + fi +} +if [ "$lsp" = unknown ] && { [ -e "$conf/50-speaker-dsp.conf" ] || [ -L "$conf/50-speaker-dsp.conf" ]; }; then + note "PARTIAL: retained existing speaker DSP because LSP availability is unknown" +else + if [ "$lsp" = 1 ]; then speaker=speaker-dsp.conf; else speaker=speaker-biquad.conf; fi + [ "$lsp" != unknown ] || note "PARTIAL: installing built-in speaker EQ because LSP availability is unknown" + step "speaker DSP" install_managed "$assets/files/$speaker" "$conf/50-speaker-dsp.conf" pipewire "$assets/files/speaker-dsp.conf" "$assets/files/speaker-biquad.conf" +fi +mic_config_custom=1 +if [ "$rnnoise" = unknown ]; then + note "PARTIAL: microphone DSP unchanged because plugin detection failed" +elif [ -n "$rnnoise" ]; then + step "microphone DSP" install_managed "$assets/files/mic-rnnoise.conf" "$conf/60-mic-dsp.conf" pipewire "$assets/files/mic-rnnoise.conf" + mic_config_custom="$custom_config" +else + step "remove unavailable microphone DSP" remove_managed "$conf/60-mic-dsp.conf" pipewire "$assets/files/mic-rnnoise.conf" + mic_config_custom="$custom_config" +fi + +write_dropin() { + local tmp + tmp="$(mktemp)" || return 1 + printf '%s\n' '[Service]' 'Environment=LADSPA_PATH=%h/.ladspa:/usr/lib/x86_64-linux-gnu/ladspa:/usr/lib/ladspa:/usr/lib64/ladspa' > "$tmp" || { rm -f "$tmp"; return 1; } + install_managed "$tmp" "$user_systemd/pipewire.service.d/ladspa-path.conf" pipewire "$tmp" + local rc=$? + rm -f "$tmp" + return "$rc" +} +step "PipeWire LADSPA path" write_dropin + +mic_service="$user_systemd/mic-dsp-link.service" +mic_service_custom=0 +managed_mic_service() { + local template pattern prefix suffix existing volume + [ -f "$mic_service" ] && [ ! -L "$mic_service" ] || return 1 + template="$(cat "$assets/templates/mic-dsp-link.service.j2")" || return 2 + pattern='{{ mic_hardware_volume_pct | default(100) }}' + [[ "$template" == *"$pattern"* ]] || return 2 + prefix="${template%%"$pattern"*}" + suffix="${template#*"$pattern"}" + existing="$(cat "$mic_service")" || return 2 + [[ "$existing" == "$prefix"*"$suffix" ]] || return 1 + volume="${existing#"$prefix"}" + volume="${volume%"$suffix"}" + [[ "$volume" =~ ^[0-9]+$ ]] +} +if [ -e "$mic_service" ] || [ -L "$mic_service" ]; then + if managed_mic_service; then :; else + rc=$? + mic_service_custom=1 + if [ "$rc" = 1 ]; then + note "PARTIAL: retained custom microphone link service $mic_service" + else + step "read canonical microphone link service" false + fi + fi +elif [ "$rnnoise" = "" ]; then + # A same-named service elsewhere is not ours to disable. + mic_service_custom=1 +fi +if [ "$mic_config_custom" = 1 ]; then + mic_service_custom=1 + note "PARTIAL: microphone link service unchanged because microphone DSP is not managed" +fi + +write_mic_service() { + local tmp template pattern rc + mic_service_custom=1 + template="$(cat "$assets/templates/mic-dsp-link.service.j2")" || return 1 + pattern='{{ mic_hardware_volume_pct | default(100) }}' + [[ "$template" == *"$pattern"* ]] || return 1 + template="${template//"$pattern"/$mic_volume}" + tmp="$(mktemp)" || return 1 + printf '%s\n' "$template" > "$tmp" || { rm -f "$tmp"; return 1; } + install_config "$tmp" "$mic_service" mic + rc=$? + rm -f "$tmp" + [ "$rc" != 0 ] || mic_service_custom=0 + return "$rc" +} +if [ "$rnnoise" != unknown ] && [ -n "$rnnoise" ] && [ "$mic_service_custom" = 0 ]; then + step "microphone link service" write_mic_service +fi + +old_rule="$HOME/.config/wireplumber/main.lua.d/51-speaker-dsp-default.lua" +step "remove obsolete WirePlumber rule" remove_managed "$old_rule" wireplumber "$assets/files/51-speaker-dsp-default.lua" +# Do not delete other, user-owned WirePlumber rules. +if [ -d "${old_rule%/*}" ]; then rmdir "${old_rule%/*}" 2>/dev/null || :; fi + +# Systemd user managers also work in non-interactive applies with a live session. +uid="$(id -u)" +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$uid}" +export DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-unix:path=$XDG_RUNTIME_DIR/bus}" +sync_user_services() { + local unit_files loaded_units active=0 + systemctl --user show-environment >/dev/null || return 1 + if [ -e "$state/pipewire" ] || [ -e "$state/mic" ]; then + systemctl --user daemon-reload || return 1 + fi + if [ -e "$state/pipewire" ]; then + systemctl --user reset-failed pipewire.service pipewire.socket pipewire-pulse.service pipewire-pulse.socket || note "warn: could not reset all PipeWire failed states" + systemctl --user restart pipewire pipewire-pulse.socket || return 1 + rm -f "$state/pipewire" || return 1 + fi + if [ -e "$state/wireplumber" ]; then + systemctl --user reset-failed wireplumber.service || note "warn: could not reset WirePlumber failed state" + systemctl --user restart wireplumber || return 1 + rm -f "$state/wireplumber" || return 1 + fi + [ "$rnnoise" != unknown ] || return 0 + [ "$mic_service_custom" = 0 ] || return 0 + if [ -n "$rnnoise" ]; then + systemctl --user is-active --quiet mic-dsp-link.service && active=1 + if ! systemctl --user is-enabled --quiet mic-dsp-link.service; then + systemctl --user enable mic-dsp-link.service || return 1 + fi + if [ "$active" = 0 ]; then + systemctl --user start mic-dsp-link.service || return 1 + elif [ -e "$state/mic" ]; then + systemctl --user restart mic-dsp-link.service || return 1 + fi + rm -f "$state/mic" || return 1 + else + unit_files="$(systemctl --user list-unit-files --no-legend --no-pager mic-dsp-link.service)" || return 1 + loaded_units="$(systemctl --user list-units --all --no-legend --no-pager mic-dsp-link.service)" || return 1 + if [ -n "$unit_files" ] || [ -n "$loaded_units" ]; then + if systemctl --user is-active --quiet mic-dsp-link.service; then + systemctl --user stop mic-dsp-link.service || return 1 + fi + if systemctl --user is-enabled --quiet mic-dsp-link.service; then + systemctl --user disable mic-dsp-link.service || return 1 + fi + fi + rm -f "$state/mic" || return 1 + fi +} +step "user audio services" sync_user_services + +set_gnome_volume() { + local current + current="$(gsettings get org.gnome.desktop.sound allow-volume-above-100-percent)" || return 1 + [ "$current" = true ] || gsettings set org.gnome.desktop.sound allow-volume-above-100-percent true +} +# GNOME is optional and the original role explicitly tolerated its absence. +soft "GNOME over-amplification" set_gnome_volume +step "remove Easy Effects autostart" rm -f "$HOME/.config/autostart/com.github.wwmm.easyeffects.desktop" +# Preserve user-owned Easy Effects presets and settings. +cleanup_easyeffects() { + local installed running app found=0 + if ! command -v flatpak >/dev/null 2>&1; then + note "Flatpak unavailable; no Easy Effects Flatpak to remove" + return 0 + fi + running="$(flatpak ps --columns=application)" || return 1 + while IFS= read -r app; do + if [ "$app" = com.github.wwmm.easyeffects ]; then + flatpak kill com.github.wwmm.easyeffects || return 1 + break + fi + done <<< "$running" + installed="$(flatpak list --system --app --columns=application)" || return 1 + while IFS= read -r app; do [ "$app" != com.github.wwmm.easyeffects ] || found=1; done <<< "$installed" + [ "$found" = 0 ] || as_root flatpak uninstall --system -y com.github.wwmm.easyeffects +} +step "remove Easy Effects Flatpak" cleanup_easyeffects +finish diff --git a/roles/pipewire_audio/defaults/main.yml b/units/pipewire-audio/defaults/main.yml similarity index 100% rename from roles/pipewire_audio/defaults/main.yml rename to units/pipewire-audio/defaults/main.yml diff --git a/roles/pipewire_audio/files/51-speaker-dsp-default.lua b/units/pipewire-audio/files/51-speaker-dsp-default.lua similarity index 100% rename from roles/pipewire_audio/files/51-speaker-dsp-default.lua rename to units/pipewire-audio/files/51-speaker-dsp-default.lua diff --git a/roles/pipewire_audio/files/mic-rnnoise.conf b/units/pipewire-audio/files/mic-rnnoise.conf similarity index 100% rename from roles/pipewire_audio/files/mic-rnnoise.conf rename to units/pipewire-audio/files/mic-rnnoise.conf diff --git a/roles/pipewire_audio/files/speaker-biquad.conf b/units/pipewire-audio/files/speaker-biquad.conf similarity index 100% rename from roles/pipewire_audio/files/speaker-biquad.conf rename to units/pipewire-audio/files/speaker-biquad.conf diff --git a/roles/pipewire_audio/files/speaker-dsp.conf b/units/pipewire-audio/files/speaker-dsp.conf similarity index 100% rename from roles/pipewire_audio/files/speaker-dsp.conf rename to units/pipewire-audio/files/speaker-dsp.conf diff --git a/roles/pipewire_audio/templates/mic-dsp-link.service.j2 b/units/pipewire-audio/templates/mic-dsp-link.service.j2 similarity index 100% rename from roles/pipewire_audio/templates/mic-dsp-link.service.j2 rename to units/pipewire-audio/templates/mic-dsp-link.service.j2 diff --git a/units/platform-config.bash b/units/platform-config.bash new file mode 100644 index 00000000..1826220e --- /dev/null +++ b/units/platform-config.bash @@ -0,0 +1,51 @@ +# shellcheck shell=bash +# Query a direct hosts group without requiring yq on an unbootstrapped phone. +# The fallback accepts the repository's block-style inventory, and fails closed +# for a missing group or an unrecognised hosts entry rather than guessing. +platform_group_contains() { + local group="$1" line in_children=0 in_group=0 in_hosts=0 group_seen=0 hosts_seen=0 found=1 + if command -v yq >/dev/null 2>&1; then + local answer + answer="$(GROUP="$group" MACHINE="$MACHINE" yq -r '.all.children[strenv(GROUP)].hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml")" || return 2 + case "$answer" in true) return 0 ;; false) return 1 ;; *) return 2 ;; esac + fi + [ -r "$DOTFILES_DIR/inventory.yml" ] || return 2 + while IFS= read -r line || [ -n "$line" ]; do + [[ "$line" =~ ^[[:space:]]*(#|$) ]] && continue + if [[ "$line" =~ ^' '[^[:space:]] ]]; then + in_children=0; in_group=0; in_hosts=0 + [ "$line" = ' children:' ] && in_children=1 + continue + fi + [ "$in_children" = 1 ] || continue + if [[ "$line" =~ ^' '[^[:space:]] ]]; then + in_group=0; in_hosts=0 + if [ "$line" = " $group:" ]; then in_group=1; group_seen=1; fi + continue + fi + [ "$in_group" = 1 ] || continue + if [[ "$line" =~ ^' '[^[:space:]] ]]; then + in_hosts=0 + if [ "$line" = ' hosts:' ]; then in_hosts=1; hosts_seen=1; fi + continue + fi + [ "$in_hosts" = 1 ] || continue + if [[ "$line" =~ ^' '([^[:space:]:]+):[[:space:]]*(#.*)?$ ]]; then + [ "${BASH_REMATCH[1]}" != "$MACHINE" ] || found=0 + elif [[ "$line" =~ ^' '[^[:space:]] ]]; then + return 2 + fi + done < "$DOTFILES_DIR/inventory.yml" + [ "$group_seen" = 1 ] && [ "$hosts_seen" = 1 ] || return 2 + return "$found" +} + +platform_desktop() { + local value="${DOTFILES_IS_DESKTOP:-}" + [ -n "$value" ] || value="$(host_var is_desktop)" + case "$value" in 1|true) return 0 ;; 0|false) return 1 ;; '') platform_group_contains desktop ;; *) return 2 ;; esac +} + +platform_root() { + if [ "$(id -u)" = 0 ]; then "$@"; else sudo -n "$@"; fi +} diff --git a/units/proxy.sh b/units/proxy.sh new file mode 100755 index 00000000..ffee169d --- /dev/null +++ b/units/proxy.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ] || skipped_by_profile skip_proxy; then + note "SKIPPED: proxy excluded on this host" + finish +fi +if ! command -v yq >/dev/null 2>&1; then + step "check proxy VPS membership (yq required)" false + finish +fi +vps=$(MACHINE="$MACHINE" yq '.all.children.vps.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml") || { step "check proxy VPS membership" false; finish; } +case "$vps" in + true) note "SKIPPED: proxy excluded on VPS"; finish ;; + false) ;; + *) step "check proxy VPS membership" false; finish ;; +esac + +as_root() { if [ "$EUID" = 0 ]; then "$@"; else sudo -n "$@"; fi; } +tmp=$(mktemp -d) || { step "create proxy temporary directory" false; finish; } +trap 'rm -rf "$tmp"' EXIT +changed=0 +install_root_file() { + local src="$1" dest="$2" mode + mode=$(as_root stat -c '%a:%u:%g' "$dest" 2>/dev/null || true) + if as_root cmp -s "$src" "$dest" && [ "$mode" = '644:0:0' ]; then return 0; fi + as_root install -m 0644 -o root -g root "$src" "$dest" || return 1 + changed=1 +} +render_caddyfile() { + local host_file="$DOTFILES_DIR/host_vars/$MACHINE.yml" + if [ -f "$host_file" ]; then + yq -o=json '.web_services // []' "$host_file" > "$tmp/services.json" || return 1 + else + printf '[]\n' > "$tmp/services.json" || return 1 + fi + python3 "$DOTFILES_DIR/units/service-templates.py" proxy "$DOTFILES_DIR/units/proxy/Caddyfile.j2" "$tmp/services.json" > "$tmp/Caddyfile" +} +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if [ "$EUID" != 0 ] && ! sudo -n true; then step "noninteractive root access" false; finish; fi + proxy_check_dir() { + local dir=$1 mode + if as_root test -e "$dir"; then + if mode=$(as_root stat -c '%a:%u:%g' "$dir"); then + if as_root test -d "$dir" && [ "$mode" = '755:0:0' ]; then note "ok directory $dir"; else note "would create or set ownership and permissions on $dir"; fi + else step "inspect directory $dir" false; fi + else note "would create directory $dir"; fi + } + proxy_check_file() { + local src=$1 dest=$2 mode rc + [ -r "$src" ] || { step "read $src" false; return; } + if ! as_root test -e "$dest" && ! as_root test -L "$dest"; then note "would deploy $dest and restart Caddy"; return; fi + mode=$(as_root stat -c '%a:%u:%g' "$dest") || { step "inspect $dest" false; return; } + as_root cmp -s "$src" "$dest"; rc=$? + case "$rc" in + 0) [ "$mode" = '644:0:0' ] && note "ok $dest" || note "would set ownership and permissions on $dest and restart Caddy" ;; + 1) note "would deploy $dest and restart Caddy" ;; + *) step "compare $dest" false ;; + esac + } + proxy_check_dir /etc/caddy + proxy_check_dir /etc/containers/systemd + if render_caddyfile; then proxy_check_file "$tmp/Caddyfile" /etc/caddy/Caddyfile; else step "render Caddyfile" false; fi + proxy_check_file "$DOTFILES_DIR/units/proxy/caddy.container" /etc/containers/systemd/caddy.container + state=$(as_root systemctl show caddy.service --property=LoadState --property=ActiveState --property=UnitFileState 2>/dev/null); rc=$? + if { [ "$rc" != 0 ] && [[ "$state" != *'LoadState=not-found'* ]]; } || [[ "$state" != *'LoadState='* || "$state" != *'ActiveState='* ]]; then + step "inspect Caddy service" false + elif [[ "$state" == *'LoadState=not-found'* ]]; then + note "would reload systemd and enable and start Caddy" + elif [[ "$state" != *'LoadState=loaded'* || "$state" != *'UnitFileState='* ]]; then + step "inspect Caddy service" false + else + [[ "$state" == *'ActiveState=active'* ]] || note "would start Caddy" + case "$state" in *'UnitFileState=enabled'*|*'UnitFileState=generated'*) ;; *) note "would enable Caddy" ;; esac + fi + if serve=$(as_root tailscale serve status 2>/dev/null); then + [[ "$serve" == *127.0.0.1:9000* ]] && note "ok Tailscale serve target" || note "would point Tailscale serve at Caddy" + else step "inspect Tailscale serve status" false; fi + finish +fi +step "create Caddy directories" as_root install -d -m 0755 -o root -g root /etc/caddy /etc/containers/systemd +caddy_ok=0 +quadlet_ok=0 +if render_caddyfile; then + if install_root_file "$tmp/Caddyfile" /etc/caddy/Caddyfile; then + caddy_ok=1 + note "ok deploy Caddyfile" + else + step "deploy Caddyfile" false + fi +else + step "render Caddyfile" false +fi +if install_root_file "$DOTFILES_DIR/units/proxy/caddy.container" /etc/containers/systemd/caddy.container; then + quadlet_ok=1 + note "ok deploy Caddy quadlet" +else + step "deploy Caddy quadlet" false +fi +# Do not switch traffic or restart a working service after a partial deploy. +if [ "$caddy_ok" = 0 ] || [ "$quadlet_ok" = 0 ]; then + note "PARTIAL: Caddy activation skipped after a configuration failure" + finish +fi +if ! as_root systemctl daemon-reload; then + step "reload systemd for Caddy" false + finish +fi +if [ "$changed" = 1 ]; then + if as_root systemctl restart caddy.service; then + note "ok restart Caddy after configuration change" + else + step "restart Caddy after configuration change" false + finish + fi +fi +if as_root systemctl enable --now caddy.service; then + note "ok enable and start Caddy" +else + step "enable and start Caddy" false + finish +fi +serve=$(as_root tailscale serve status 2>/dev/null || true) +if [[ "$serve" != *127.0.0.1:9000* ]]; then + # --bg replaces the existing default HTTPS handler, including the old + # direct homepage target; turning 443 off afterward would remove the new one. + step "point Tailscale serve at Caddy" as_root tailscale serve --bg http://127.0.0.1:9000 +fi +finish diff --git a/roles/proxy/templates/Caddyfile.j2 b/units/proxy/Caddyfile.j2 similarity index 100% rename from roles/proxy/templates/Caddyfile.j2 rename to units/proxy/Caddyfile.j2 diff --git a/roles/proxy/files/caddy.container b/units/proxy/caddy.container similarity index 100% rename from roles/proxy/files/caddy.container rename to units/proxy/caddy.container diff --git a/units/retired-services.sh b/units/retired-services.sh new file mode 100755 index 00000000..3d991af9 --- /dev/null +++ b/units/retired-services.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# Only files explicitly listed in the role's canonical inventory are eligible. +retired_unit_name() { + local name=${1##*/} stem=${1##*/} + stem=${stem%.*} + case "$name" in + *.container) printf '%s.service\n' "$stem" ;; + *.volume) printf '%s-volume.service\n' "$stem" ;; + *.network) printf '%s-network.service\n' "$stem" ;; + *.pod) printf '%s-pod.service\n' "$stem" ;; + *) printf '%s\n' "$name" ;; + esac +} + +# A missing generated unit is normal, but any other systemd error is not. +retired_stop() { + local scope=$1 path=$2 unit state + local -a ctl=(systemctl) + [ "$scope" = user ] && ctl+=(--user) + unit=$(retired_unit_name "$path") + state=$("${ctl[@]}" show --property=LoadState --value "$unit" 2>/dev/null) || { + [ "$state" = not-found ] && return 0 + return 1 + } + [ "$state" = not-found ] && return 0 + [ "$state" = loaded ] || return 1 + case "$path" in + *.container|*.volume|*.network|*.pod) "${ctl[@]}" stop "$unit" ;; + *) "${ctl[@]}" disable --now "$unit" ;; + esac +} + +retired_cleanup_user() { + local file user_changed=0 + for file in "$@"; do + [ -n "$file" ] || continue + file="$HOME/.config/containers/systemd/$file" + [ -e "$file" ] || [ -L "$file" ] || continue + if ! retired_stop user "$file"; then step "stop retired user unit $file" false; continue; fi + if rm -f -- "$file"; then user_changed=1; else step "remove retired user unit $file" false; fi + done + if [ "$user_changed" = 1 ]; then step "reload user systemd" systemctl --user daemon-reload; fi +} + +retired_main() { + local vps file found=0 system_changed=0 user_changed=0 + local -a system_units user_units rootcmd=() + case "$MACHINE" in matrix|telengana) note "SKIPPED: retired VPS"; finish ;; esac + if [ "$IS_TERMUX" = 1 ] || [ "$(uname -s)" != Linux ] || [ ! -d /run/systemd/system ]; then + note "SKIPPED: retired services require native systemd"; finish + fi + if ! command -v yq >/dev/null 2>&1 || ! command -v systemctl >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then + step "retired service dependencies (yq, python3 and systemctl)" false; finish + fi + vps=$(python3 "$DOTFILES_DIR/units/osquery_native_inventory.py" groups "$DOTFILES_DIR/inventory.yml" "$MACHINE" vps) || { step "check VPS membership" false; finish; } + case "$vps" in true) note "SKIPPED: VPS"; finish ;; false) ;; *) step "check VPS membership" false; finish ;; esac + local defaults="$DOTFILES_DIR/units/retired-services/defaults/main.yml" system_list user_list + system_list=$(yq -r '.retired_services_system_units[]' "$defaults") || { step "read retired system units" false; finish; } + user_list=$(yq -r '.retired_services_user_units[]' "$defaults") || { step "read retired user units" false; finish; } + mapfile -t system_units <<< "$system_list" + mapfile -t user_units <<< "$user_list" + export XDG_RUNTIME_DIR="/run/user/$(id -u)" + if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then + export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus" + fi + for file in "${system_units[@]}"; do + [ -n "$file" ] || continue + if [ -e "$file" ] || [ -L "$file" ]; then found=1; break; fi + done + if [ "${UNITS_CHECK:-0}" = 1 ]; then + # lstat distinguishes absent paths from inaccessible parent directories. + retired_verify_paths() { + python3 -c 'import os, sys +failed = False +for path in sys.argv[1:]: + if not path: + continue + try: + os.lstat(path) + except FileNotFoundError: + pass + except OSError as error: + print("cannot inspect retired path %s: %s" % (path, error), file=sys.stderr) + failed = True +sys.exit(1 if failed else 0)' "$@" + } + local -a user_paths=() + for file in "${user_units[@]}"; do + [ -z "$file" ] || user_paths+=("$HOME/.config/containers/systemd/$file") + done + retired_verify_paths "${system_units[@]}" || step "inspect retired system paths" false + retired_verify_paths "${user_paths[@]}" || step "inspect retired user paths" false + retired_inspect() { + local scope=$1 path=$2 unit state rc + local -a ctl=(systemctl) + [ "$scope" = user ] && ctl+=(--user) + unit=$(retired_unit_name "$path") + if [ "$scope" = system ]; then + state=$("${rootcmd[@]}" "${ctl[@]}" show --property=LoadState --value "$unit" 2>/dev/null); rc=$? + else + state=$("${ctl[@]}" show --property=LoadState --value "$unit" 2>/dev/null); rc=$? + fi + if [ "$state" = not-found ]; then + note "would remove retired $scope unit file $path (generated unit absent)" + elif [ "$rc" = 0 ] && [ "$state" = loaded ]; then + case "$path" in + *.container|*.volume|*.network|*.pod) note "would stop $unit and remove retired $scope unit file $path" ;; + *) note "would disable and stop $unit and remove retired $scope unit file $path" ;; + esac + else + step "inspect retired $scope unit $unit" false + return 1 + fi + } + if [ "$found" = 1 ] && [ "$(id -u)" != 0 ]; then + if command -v sudo >/dev/null 2>&1 && sudo -n true; then rootcmd=(sudo -n); else + step "root access for retired system units" false + system_units=() + fi + fi + for file in "${system_units[@]}"; do + [ -e "$file" ] || [ -L "$file" ] || continue + if retired_inspect system "$file"; then system_changed=1; fi + done + for file in "${user_units[@]}"; do + [ -n "$file" ] || continue + file="$HOME/.config/containers/systemd/$file" + [ -e "$file" ] || [ -L "$file" ] || continue + if retired_inspect user "$file"; then user_changed=1; fi + done + [ "$system_changed" = 0 ] || note "would reload system systemd after removal" + [ "$user_changed" = 0 ] || note "would reload user systemd after removal" + [ "$system_changed" != 0 ] || [ "$user_changed" != 0 ] || [ "$UNIT_FAILED" != 0 ] || note "ok no retired unit files found" + finish + fi + if [ "$found" = 1 ]; then + if [ "$(id -u)" != 0 ]; then + if command -v sudo >/dev/null 2>&1 && sudo -n true; then rootcmd=(sudo -n); else + step "root access for retired system units" false + system_units=() + fi + fi + for file in "${system_units[@]}"; do + [ -e "$file" ] || [ -L "$file" ] || continue + # Pass the function itself through sudo, without trusting arbitrary unit names. + if [ "${#rootcmd[@]}" = 0 ]; then + retired_stop system "$file" + else + "${rootcmd[@]}" bash -c "$(declare -f retired_unit_name retired_stop); retired_stop system \"\$1\"" _ "$file" + fi + if [ "$?" -ne 0 ]; then step "stop retired system unit $file" false; continue; fi + if "${rootcmd[@]}" rm -f -- "$file"; then system_changed=1; else step "remove retired system unit $file" false; fi + done + if [ "$system_changed" = 1 ]; then step "reload system systemd" "${rootcmd[@]}" systemctl daemon-reload; fi + fi + retired_cleanup_user "${user_units[@]}" + finish +} + +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then retired_main "$@"; fi diff --git a/roles/retired_services/defaults/main.yml b/units/retired-services/defaults/main.yml similarity index 100% rename from roles/retired_services/defaults/main.yml rename to units/retired-services/defaults/main.yml diff --git a/units/server-hardening-file.py b/units/server-hardening-file.py new file mode 100755 index 00000000..3e7a5280 --- /dev/null +++ b/units/server-hardening-file.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Apply lineinfile-style edits without replacing other settings.""" +import errno +import os +from pathlib import Path +import re +import stat +import sys +import tempfile + + +def update(path, pattern, line, check=False): + path = Path(os.path.realpath(path)) + original = path.read_bytes() + text = original.decode("utf-8") + lines = text.splitlines(keepends=True) + matches = [i for i, current in enumerate(lines) if re.search(pattern, current)] + if matches: + i = matches[-1] + ending = "\n" if lines[i].endswith("\n") else "" + lines[i] = line + ending + elif not any(current.rstrip("\r\n") == line for current in lines): + if lines and not lines[-1].endswith("\n"): + lines[-1] += "\n" + lines.append(line + "\n") + result = "".join(lines).encode("utf-8") + if result == original: + return False + if check: + return True + metadata = path.stat() + fd, temp = tempfile.mkstemp(prefix=".dotfiles-hardening-", dir=path.parent) + try: + with os.fdopen(fd, "wb") as out: + out.write(result) + out.flush() + os.fchmod(out.fileno(), stat.S_IMODE(metadata.st_mode)) + os.fchown(out.fileno(), metadata.st_uid, metadata.st_gid) + # Atomic replacement must retain SELinux labels and other file xattrs. + # If copying a label fails, leave the original in place. + try: + names = os.listxattr(path) + except OSError as error: + if error.errno not in (errno.ENOTSUP, errno.EOPNOTSUPP): + raise + names = [] + for name in names: + os.setxattr(temp, name, os.getxattr(path, name)) + os.replace(temp, path) + finally: + if os.path.exists(temp): + os.unlink(temp) + return True + + +def main(): + try: + check = len(sys.argv) > 1 and sys.argv[1] == "--check" + args = sys.argv[2:] if check else sys.argv[1:] + changed = update(args[0], args[1], args[2], check=check) + except (OSError, UnicodeError, re.error, IndexError) as error: + print(f"hardening file update: {error}", file=sys.stderr) + return 1 + return 10 if changed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/units/server-hardening.sh b/units/server-hardening.sh new file mode 100755 index 00000000..a4a14432 --- /dev/null +++ b/units/server-hardening.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +hardening_main() { + local server vps memberships rc changed=0 swap_output ufw_output pro_output + local -a rootcmd=() + case "$MACHINE" in matrix|telengana) note "SKIPPED: retired VPS must not be modified"; finish ;; esac + if ! command -v yq >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then step "server group membership (yq and python3 required)" false; finish; fi + memberships=$(python3 "$DOTFILES_DIR/units/osquery_native_inventory.py" groups "$DOTFILES_DIR/inventory.yml" "$MACHINE" server vps) || { step "host group membership" false; finish; } + read -r server vps <<< "$memberships" + case "$server:$vps" in true:true|true:false|false:true) ;; false:false) note "SKIPPED: not a server or VPS"; finish ;; *) step "host group membership" false; finish ;; esac + if [ "$(id -u)" != 0 ]; then + if command -v sudo >/dev/null 2>&1 && sudo -n true; then rootcmd=(sudo -n); else step "noninteractive root access" false; finish; fi + fi + + hardening_line() { + "${rootcmd[@]}" python3 "$DOTFILES_DIR/units/server-hardening-file.py" "$@" + } + if [ "${UNITS_CHECK:-0}" = 1 ]; then + local rule pattern line mode chrony_unit chrony_state chrony_active=0 + hardening_inspect_line() { + local description=$1 + shift + hardening_line --check "$@"; rc=$? + case "$rc" in + 0) note "ok $description" ;; + 10) note "would update $description" ;; + *) step "inspect $description" false ;; + esac + } + hardening_inspect_line "journald limit (and restart journald)" /etc/systemd/journald.conf '^#?SystemMaxUse=' 'SystemMaxUse=500M' + if "${rootcmd[@]}" journalctl --disk-usage >/dev/null; then + note "would vacuum journals to 500M" + else + step "inspect journal usage" false + fi + if "${rootcmd[@]}" test -e /swapfile; then + if mode=$("${rootcmd[@]}" stat -c '%a:%u:%g' /swapfile); then + [ "$mode" = '600:0:0' ] && note "ok swap permissions" || note "would set swap ownership and permissions" + else + step "inspect swap permissions" false + fi + else + note "would create 2 GiB /swapfile with root ownership and mode 0600" + fi + if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then + if printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then + note "ok swap enabled" + else + note "would format and enable /swapfile" + fi + else + step "inspect active swap" false + fi + hardening_inspect_line "swap fstab" /etc/fstab '^/swapfile none swap sw 0 0$' '/swapfile none swap sw 0 0' + for chrony_unit in chrony chronyd; do + chrony_state=$(systemctl show "$chrony_unit" --property=LoadState --property=ActiveState 2>/dev/null); rc=$? + if [ "$rc" != 0 ] && [[ "$chrony_state" != *'LoadState=not-found'* ]]; then + step "inspect $chrony_unit" false + elif [[ "$chrony_state" != *'LoadState='* || "$chrony_state" != *'ActiveState='* ]]; then + step "inspect $chrony_unit" false + elif [[ "$chrony_state" != *'LoadState=loaded'* && "$chrony_state" != *'LoadState=not-found'* ]]; then + step "inspect $chrony_unit" false + elif [[ "$chrony_state" == *'ActiveState=active'* ]]; then + chrony_active=1 + fi + done + if [ -e /usr/lib/systemd/systemd-timesyncd ] && [ "$chrony_active" = 0 ]; then + note "would enable and start systemd-timesyncd if chrony is inactive" + fi + for rule in 'MaxStartups|MaxStartups 3:50:10' 'MaxSessions|MaxSessions 10' 'ClientAliveCountMax|ClientAliveCountMax 3'; do + pattern=${rule%%|*}; line=${rule#*|} + hardening_inspect_line "SSH limit $pattern (and reload SSH)" /etc/ssh/sshd_config "^#?$pattern" "$line" + done + if command -v ufw >/dev/null 2>&1; then + if ufw_output=$("${rootcmd[@]}" ufw status 2>/dev/null); then + [[ "$ufw_output" != *'Status: active'* ]] || note "would enable UFW logging" + else + step "inspect UFW status" false + fi + fi + if command -v pro >/dev/null 2>&1; then + if pro_output=$("${rootcmd[@]}" pro status --format json 2>/dev/null); then + if [[ "$pro_output" != *livepatch* ]] || [[ "$pro_output" == *disabled* ]]; then note "would enable livepatch"; fi + else + step "inspect Ubuntu Pro status" false + fi + fi + if [ -e /var/run/reboot-required ]; then note "REBOOT REQUIRED on $MACHINE — kernel was updated and needs a restart."; fi + finish + fi + hardening_line /etc/systemd/journald.conf '^#?SystemMaxUse=' 'SystemMaxUse=500M'; rc=$? + case "$rc" in + 0) note "ok journald limit" ;; + 10) note "ok journald limit"; step "restart journald" "${rootcmd[@]}" systemctl restart systemd-journald ;; + *) step "journald limit" false ;; + esac + step "vacuum journals" "${rootcmd[@]}" journalctl --vacuum-size=500M + + local swap_ready=1 + if [ ! -e /swapfile ]; then + if ! (umask 077; "${rootcmd[@]}" dd if=/dev/zero of=/swapfile bs=1M count=2048); then + step "create swap file" false; swap_ready=0 + fi + fi + if [ "$swap_ready" = 1 ] && [ -e /swapfile ]; then + if ! "${rootcmd[@]}" chown root:root /swapfile; then step "protect swap file" false; swap_ready=0; fi + if ! "${rootcmd[@]}" chmod 0600 /swapfile; then step "set swap permissions" false; swap_ready=0; fi + if [ "$swap_ready" = 1 ]; then + if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then + if ! printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then + if ! "${rootcmd[@]}" mkswap /swapfile; then + step "format swap" false; swap_ready=0 + elif ! "${rootcmd[@]}" swapon /swapfile; then + step "enable swap" false; swap_ready=0 + fi + fi + else + step "check active swap" false; swap_ready=0 + fi + fi + if [ "$swap_ready" = 1 ]; then + hardening_line /etc/fstab '^/swapfile none swap sw 0 0$' '/swapfile none swap sw 0 0'; rc=$? + case "$rc" in 0|10) note "ok swap fstab" ;; *) step "swap fstab" false ;; esac + fi + fi + + local chrony + chrony=$(systemctl is-active chrony chronyd 2>/dev/null) || : + if [ -e /usr/lib/systemd/systemd-timesyncd ] && ! printf '%s\n' "$chrony" | grep -Fxq active; then + step "enable timesyncd" "${rootcmd[@]}" systemctl enable --now systemd-timesyncd + fi + + local rule pattern line + for rule in 'MaxStartups|MaxStartups 3:50:10' 'MaxSessions|MaxSessions 10' 'ClientAliveCountMax|ClientAliveCountMax 3'; do + pattern=${rule%%|*}; line=${rule#*|} + hardening_line /etc/ssh/sshd_config "^#?$pattern" "$line"; rc=$? + case "$rc" in 0) ;; 10) changed=1 ;; *) step "SSH limit $pattern" false ;; esac + done + if [ "$changed" = 1 ]; then + local ssh_unit=sshd + if [ -r /etc/os-release ]; then + # shellcheck disable=SC1091 + . /etc/os-release + case " ${ID:-} ${ID_LIKE:-} " in *' debian '*|*' ubuntu '*) ssh_unit=ssh ;; esac + fi + # The role's shared SSH reload handler ignores errors on non-systemd hosts. + soft "reload SSH" "${rootcmd[@]}" systemctl reload "$ssh_unit" + fi + if ufw_output=$(ufw status 2>/dev/null) && [[ "$ufw_output" == *'Status: active'* ]]; then + step "enable UFW logging" "${rootcmd[@]}" ufw logging on + fi + if pro_output=$(pro status --format json 2>/dev/null) && { [[ "$pro_output" != *livepatch* ]] || [[ "$pro_output" == *disabled* ]]; }; then + soft "enable livepatch" "${rootcmd[@]}" pro enable livepatch + fi + if [ -e /var/run/reboot-required ]; then note "REBOOT REQUIRED on $MACHINE — kernel was updated and needs a restart."; fi + finish +} + +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then hardening_main "$@"; fi diff --git a/units/service-templates.py b/units/service-templates.py new file mode 100755 index 00000000..0ed3407a --- /dev/null +++ b/units/service-templates.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Render the deliberately small native service templates.""" +import json +import re +import sys + + +def replace(template, values): + for key, value in values.items(): + template = template.replace("{{ " + key + " }}", value) + if "{{" in template or "{%" in template: + raise ValueError("unsupported or unexpanded service template expression") + return template + + +def render_proxy(template, services): + pattern = r"\{% for svc in web_services \| default\(\[\]\) %\}\n(.*?)\{% endfor %\}\n" + match = re.search(pattern, template, re.DOTALL) + if match is None or re.search(pattern, template[match.end():], re.DOTALL): + raise ValueError("unsupported proxy service loop") + if not isinstance(services, list): + raise ValueError("web_services must be a list") + body = match.group(1) + key = "{{ svc.name | lower | replace(' ', '-') | replace('/', '-') }}" + href = "{{ svc.href }}" + if key not in body or href not in body: + raise ValueError("unsupported proxy redirect template") + sections = [] + for service in services: + name, url = service["name"], service["href"] + if not isinstance(name, str) or not isinstance(url, str): + raise ValueError("service name and href must be strings") + sections.append(body.replace(key, name.lower().replace(' ', '-').replace('/', '-')).replace(href, url)) + return replace(template[:match.start()] + ''.join(sections) + template[match.end():], {}) + + +def main(): + mode, template_path, *args = sys.argv[1:] + with open(template_path, encoding="utf-8") as source: + template = source.read() + if mode == "proxy": + with open(args[0], encoding="utf-8") as source: + rendered = render_proxy(template, json.load(source)) + elif mode == "dashboard": + rendered = replace(template, {"bst_dashboard_dir": args[0], "bst_dashboard_image_name": args[1]}) + elif mode == "cert": + rendered = replace(template, {"inventory_hostname": args[0], "tailscale_domain": args[1]}) + else: + raise ValueError("unknown service template") + sys.stdout.write(rendered) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, IndexError, TypeError) as error: + print(f"service template: {error}", file=sys.stderr) + sys.exit(1) diff --git a/units/shell-ai.sh b/units/shell-ai.sh index 160ce7bb..fe711734 100755 --- a/units/shell-ai.sh +++ b/units/shell-ai.sh @@ -5,6 +5,26 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if [ "${UNITS_NO_SECRETS:-0}" = 1 ]; then + note "SKIPPED: secrets disabled; secret files left untouched" + exit 0 + fi + for path in "$HOME/.config/shell/secrets.sh" "$HOME/.config/fish/conf.d/secrets.fish"; do + if [ -L "$path" ] || [ ! -e "$path" ]; then + note "PARTIAL: $path absent or symlinked; apply would replace it if secrets are available" + elif [ ! -f "$path" ] || ! mode="$(stat -c '%a' "$path")"; then + note "FAIL cannot inspect $path" + elif [ "$mode" != 600 ]; then + note "PARTIAL: $path mode differs from 0600; apply would replace it if secrets are available" + else + note "ok $path mode 0600" + fi + done + note "FAIL cannot compare shell secrets safely: Bitwarden item reads may persist audit events" + exit 1 +fi + s="$(bw_session)" if [ -z "$s" ]; then @@ -52,6 +72,7 @@ end FISH } + write_sh() { secrets_sh | write_atomic "$HOME/.config/shell/secrets.sh"; } write_fish() { secrets_fish | write_atomic "$HOME/.config/fish/conf.d/secrets.fish"; } diff --git a/units/ssh-fleet.py b/units/ssh-fleet.py new file mode 100755 index 00000000..44f595a2 --- /dev/null +++ b/units/ssh-fleet.py @@ -0,0 +1,667 @@ +#!/usr/bin/env python3 +"""SSH units' shared, locked implementation; fleet data remains in inventory/vars.""" +import base64 +import fcntl +import hashlib +import hmac +import json +import os +import re +import stat +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(os.environ["DOTFILES_DIR"]) +HOME = Path(os.environ["HOME"]) +SSH = HOME / ".ssh" +MACHINE = os.environ["DOTFILES_MACHINE"] +BEGIN = "# BEGIN dotfiles SSH fleet" +END = "# END dotfiles SSH fleet" +AK_BEGIN = "# BEGIN dotfiles Bitwarden SSH keys" +AK_END = "# END dotfiles Bitwarden SSH keys" +CHECK = os.environ.get("UNITS_CHECK") == "1" +PLANNED = {} + + +def run(args, *, data=None, env=None, timeout=30): + return subprocess.run(args, input=data, text=True, capture_output=True, env=env, timeout=timeout) + + +def yaml_file(path): + try: + result = run(["yq", "-o=json", ".", str(path)]) + except FileNotFoundError: + result = None + if result is not None and result.returncode == 0: + value = json.loads(result.stdout) + else: + try: + import yaml + except ImportError as exc: + raise RuntimeError(f"cannot read {path}: install yq or Python PyYAML") from exc + try: + value = yaml.safe_load(path.read_text()) + except yaml.YAMLError as exc: + raise ValueError(f"invalid YAML: {path}") from exc + if not isinstance(value, dict): + raise ValueError(f"invalid YAML mapping: {path}") + return value + + +def atomic(path, content, mode): + if path.is_symlink(): + raise RuntimeError(f"refusing to replace symlink: {path}") + if CHECK: + if path.exists() and not path.is_file(): + raise RuntimeError(f"not a regular file: {path}") + previous = text(path) + if not path.exists() or previous != content or stat.S_IMODE(path.stat().st_mode) != mode: + print(f"would write {path} (mode {mode:04o})") + else: + print(f"ok {path}") + PLANNED[path] = content + return + if path.exists() and path.read_text() == content: + path.chmod(mode) + return + fd, name = tempfile.mkstemp(prefix=".ssh-fleet-", dir=path.parent) + try: + with os.fdopen(fd, "w") as f: + os.fchmod(f.fileno(), mode) + f.write(content) + os.replace(name, path) + finally: + if os.path.exists(name): + os.unlink(name) + + +def text(path): + if CHECK and path in PLANNED: + return PLANNED[path] + return path.read_text() if path.exists() else "" + + +def split_block(original, begin, end): + lines = original.splitlines(keepends=True) + starts = [i for i, line in enumerate(lines) if line.rstrip("\r\n") == begin] + ends = [i for i, line in enumerate(lines) if line.rstrip("\r\n") == end] + if len(starts) != len(ends) or len(starts) > 1 or (starts and starts[0] >= ends[0]): + raise ValueError("malformed managed block") + if not starts: + return original, "", "", False + start, finish = starts[0], ends[0] + return "".join(lines[:start]), "".join(lines[start + 1:finish]), "".join(lines[finish + 1:]), True + + +def replace_block(original, begin, end, lines): + before, _, after, found = split_block(original, begin, end) + block = begin + "\n" + "\n".join(lines) + "\n" + end + "\n" + if not found and before and not before.endswith("\n"): + before += "\n" + return before + block + after + + +def key_parts(key): + if not isinstance(key, str): + raise ValueError("invalid ed25519 public key") + key = key.strip() + if "\n" in key or "\r" in key: + raise ValueError("invalid multiline public key") + parts = key.split() + if len(parts) < 2 or parts[0] != "ssh-ed25519": + raise ValueError("invalid ed25519 public key") + try: + blob = base64.b64decode(parts[1], validate=True) + except ValueError as exc: + raise ValueError("invalid ed25519 public key") from exc + if len(blob) != 51 or blob[:19] != b"\x00\x00\x00\x0bssh-ed25519\x00\x00\x00\x20": + raise ValueError("invalid ed25519 public key") + return parts[:2] + + +def ssh_data(item): + data = item.get("sshKey") or {} + if not isinstance(data, dict): + raise ValueError("invalid Bitwarden SSH item") + return data + +def session_env(): + if os.environ.get("UNITS_NO_SECRETS") == "1": + return None + session = os.environ.get("BW_SESSION", "") + if not session: + try: + session = Path("/tmp/bw_session").read_text().strip() + except OSError: + pass + if not session: + return None + env = dict(os.environ, BW_SESSION=session) + try: + if run(["bw", "unlock", "--check"], env=env).returncode == 0: + return env + except (OSError, subprocess.SubprocessError): + pass + return None + + +class Fleet: + def __init__(self): + self.vars = yaml_file(ROOT / "group_vars/all.yml") + inventory = yaml_file(ROOT / "inventory.yml")["all"] + self.hosts = inventory.get("hosts") or {} + self.vps = set(((inventory.get("children") or {}).get("vps") or {}).get("hosts") or {}) + host_file = ROOT / "host_vars" / (MACHINE + ".yml") + self.host_vars = yaml_file(host_file) if host_file.exists() else {} + self.login = self.host_vars.get("user_login", self.vars["user_login"]) + self.email = self.host_vars.get("user_email", self.vars["user_email"]) + self.machines = self.vars["machines"] + self.env = None + self.items = None + self.key_ready = False + self.key_deferred = False + self.failed = False + + def step(self, label, fn): + try: + fn() + print("ok " + label) + except (OSError, ValueError, KeyError, TypeError, RuntimeError, subprocess.SubprocessError) as exc: + # Never include subprocess stderr or private-key material in output. + print("FAIL " + label + ": " + str(exc)) + self.failed = True + + def unlock(self): + self.env = session_env() + if self.env is None: + print("PARTIAL: vault locked or secrets disabled; Bitwarden SSH sync skipped") + return + result = run(["bw", "list", "items"], env=self.env, timeout=90) + if result.returncode: + raise RuntimeError("Bitwarden item listing failed; refusing to treat it as an empty vault") + items = json.loads(result.stdout) + if not isinstance(items, list) or any(not isinstance(item, dict) for item in items): + raise ValueError("invalid Bitwarden item listing") + self.items = items + + def vault_item(self): + if self.env is None: + raise RuntimeError("Bitwarden session unavailable; refusing to generate a key") + if self.items is None: + raise RuntimeError("Bitwarden listing unavailable; refusing to generate or overwrite a key") + name = f"{self.login}@{MACHINE}" + matches = [i for i in self.items if isinstance(i, dict) and i.get("name") == name] + if len(matches) > 1: + raise RuntimeError("multiple Bitwarden SSH items have this machine name") + if not matches: + return None + result = run(["bw", "get", "item", matches[0].get("id") or name], env=self.env) + if result.returncode: + raise RuntimeError("cannot retrieve machine SSH item from Bitwarden") + item = json.loads(result.stdout) + if not isinstance(item, dict) or item.get("name") != name: + raise ValueError("invalid machine SSH item") + return item + + def ensure_key(self): + private, public = SSH / "id_ed25519", SSH / "id_ed25519.pub" + if private.is_symlink() or public.is_symlink(): + raise RuntimeError("refusing to replace or chmod symlinked SSH keys") + if not private.exists(): + if self.env is None: + self.key_deferred = True + print("PARTIAL: SSH identity deferred until an unlocked Bitwarden listing is available (run an unlocked apply)") + return + item = self.vault_item() + if item is not None: + key = ssh_data(item) + priv, pub = key.get("privateKey"), key.get("publicKey") + if not isinstance(priv, str) or not priv or not isinstance(pub, str) or not pub: + raise RuntimeError("machine's Bitwarden SSH item has no complete key pair") + key_parts(pub) + if public.exists() and key_parts(public.read_text()) != key_parts(pub): + raise RuntimeError("existing public key conflicts with Bitwarden key") + # Validate the private key in a private temporary file before installation. + fd, name = tempfile.mkstemp(prefix=".ssh-key-", dir=SSH) + try: + with os.fdopen(fd, "w") as f: + f.write(priv.rstrip("\n") + "\n") + derived = run(["ssh-keygen", "-y", "-f", name]) + if derived.returncode or key_parts(derived.stdout) != key_parts(pub): + raise RuntimeError("Bitwarden SSH key pair is invalid or does not match") + atomic(public, pub.rstrip("\n") + "\n", 0o644) + os.replace(name, private) + finally: + if os.path.exists(name): + os.unlink(name) + else: + if public.exists(): + raise RuntimeError("public key exists without private key; refusing to overwrite it") + result = run(["ssh-keygen", "-q", "-t", "ed25519", "-C", MACHINE, + "-f", str(private), "-N", ""]) + if result.returncode: + raise RuntimeError("ssh-keygen failed") + private.chmod(0o600) + derived = run(["ssh-keygen", "-y", "-f", str(private)]) + if derived.returncode: + raise RuntimeError("cannot read existing private key") + derived_parts = key_parts(derived.stdout) + if not public.exists(): + atomic(public, " ".join(derived_parts) + " " + MACHINE + "\n", 0o644) + if key_parts(public.read_text()) != derived_parts: + raise RuntimeError("local SSH private and public keys do not match") + private.chmod(0o600) + public.chmod(0o644) + self.key_ready = True + + def sync_key(self): + if self.env is None: + return + if not self.key_ready: + raise RuntimeError("local SSH key unavailable; Bitwarden unchanged") + item = self.vault_item() + private, public = SSH / "id_ed25519", SSH / "id_ed25519.pub" + pub = " ".join(key_parts(public.read_text())) + if item is not None: + saved = ssh_data(item).get("publicKey", "") + if saved and key_parts(saved) == key_parts(pub): + return + force = os.environ.get("SSH_KEYS_FORCE_PUSH", str(self.host_vars.get("ssh_keys_force_push", "false"))) + if force.lower() not in ("true", "1", "yes"): + print("WARNING: local SSH key differs from Bitwarden; vault left unchanged (set SSH_KEYS_FORCE_PUSH=true for an intentional rotation)") + return + fingerprint = run(["ssh-keygen", "-lf", str(public)]) + if fingerprint.returncode or len(fingerprint.stdout.split()) < 2: + raise RuntimeError("cannot determine SSH fingerprint") + if item is None: + item = {"type": 5, "name": f"{self.login}@{MACHINE}"} + item["sshKey"] = {"privateKey": private.read_text(), "publicKey": pub, + "keyFingerprint": fingerprint.stdout.split()[1]} + encoded = run(["bw", "encode"], data=json.dumps(item), env=self.env) + if encoded.returncode: + raise RuntimeError("Bitwarden SSH item encoding failed") + args = ["bw", "create", "item"] if not item.get("id") else ["bw", "edit", "item", item["id"]] + if run(args, data=encoded.stdout, env=self.env).returncode: + raise RuntimeError("Bitwarden SSH item write failed") + # Include the just-created/updated item in this run's authorization + # batch; the original listing was taken before the vault write. + self.items = [old for old in self.items if old.get("name") != item["name"]] + [item] + + def config(self): + path = SSH / "config" + old = text(path) + # Adopt the exact static header from the former role, then only its + # recognizable host stanzas. Other user entries are never removed. + legacy = ("Host *\n AddKeysToAgent yes\n IdentityFile ~/.ssh/id_ed25519\n" + " ServerAliveInterval 30\n ServerAliveCountMax 3\n" + " # Reuse connections — big speedup for repeated SSH to the same host\n" + " ControlMaster auto\n ControlPersist 10m\n ControlPath ~/.ssh/cp/%C\n" + " # Don't hash known_hosts so tab-completion and doctor checks work\n" + " HashKnownHosts no\n # Compression on slow links; negligible on LAN/Tailscale\n" + " Compression yes\n\n") + if old.startswith(legacy): + old = old[len(legacy):] + for name in self.machines: + pattern = (r"(?m)^Host " + re.escape(name) + r"\n HostName [^\n]+\n" + r"(?: Port [0-9]+\n)? User [^\n]+\n SendEnv BW_SESSION\n\n") + old = re.sub(pattern, "", old, count=1) + lines = ["Host *", " AddKeysToAgent yes", " IdentityFile ~/.ssh/id_ed25519", + " ServerAliveInterval 30", " ServerAliveCountMax 3", " ControlMaster auto", + " ControlPersist 10m", " ControlPath ~/.ssh/cp/%C", " HashKnownHosts no", + " Compression yes", ""] + for name, info in self.machines.items(): + host = self.hosts.get(name) or {} + lines += [f"Host {name}", f" HostName {info['hostname']}"] + if "ansible_port" in host: + lines.append(f" Port {host['ansible_port']}") + lines += [f" User {host.get('ansible_user', self.login)}", " SendEnv BW_SESSION", ""] + atomic(path, replace_block(old, BEGIN, END, lines), 0o644) + if CHECK: + check_directory(SSH / "cp", 0o700) + else: + (SSH / "cp").mkdir(mode=0o700, exist_ok=True) + (SSH / "cp").chmod(0o700) + + def signers(self): + if self.key_deferred: + print("PARTIAL: allowed signers deferred until SSH identity is available") + return + if not self.key_ready: + raise RuntimeError("local SSH key unavailable") + pub = text(SSH / "id_ed25519.pub").strip() + key_parts(pub) + line = f'{self.email} namespaces="git" {pub}' + path = SSH / "allowed_signers" + old = text(path) + if line not in old.splitlines(): + old += ("\n" if old and not old.endswith("\n") else "") + line + "\n" + atomic(path, old, 0o644) + + def authorize_vault(self): + if self.env is None: + return + if self.items is None: + raise RuntimeError("Bitwarden listing unavailable; authorized_keys unchanged") + names = {f"{self.login}@{name}" for name in self.machines} + keys = [] + for item in self.items: + if not isinstance(item, dict) or item.get("name") not in names: + continue + pub = ssh_data(item).get("publicKey", "") + if not isinstance(pub, str): + raise ValueError("invalid Bitwarden public key") + pub = pub.strip() + if pub: + key_parts(pub) + keys.append(pub) + if not keys: + print("PARTIAL: no fleet SSH keys in Bitwarden; authorized_keys unchanged") + return + keys += self.host_vars.get("extra_authorized_keys", self.vars.get("extra_authorized_keys", [])) or [] + for key in keys: + if not isinstance(key, str) or "\n" in key or "\r" in key: + raise ValueError("invalid extra authorized key") + path = SSH / "authorized_keys" + old = text(path) + # Force can remove only entries previously managed in our block; older + # unmarked entries are indistinguishable from user keys and stay intact. + force = os.environ.get("SSH_KEYS_FORCE_AUTHORIZED", str(self.host_vars.get("ssh_keys_force_authorized", "false"))) + before, managed, after, _ = split_block(old, AK_BEGIN, AK_END) + previous = managed.splitlines() + if force.lower() not in ("true", "1", "yes"): + keys = previous + keys + outside = before + after + existing = set(outside.splitlines()) + unique = list(dict.fromkeys(k.strip() for k in keys if k.strip() and k.strip() not in existing)) + atomic(path, replace_block(old, AK_BEGIN, AK_END, unique), 0o600) + + def authorize_git(self): + keys = self.vars.get("machine_pubkeys") or {} + path = SSH / "authorized_keys" + old = text(path) + before, _, after, _ = split_block(old, AK_BEGIN, AK_END) + outside = before + after + existing = set(outside.splitlines()) + for key in keys.values(): + key_parts(key) + if key not in existing: + old += ("\n" if old and not old.endswith("\n") else "") + key + "\n" + existing.add(key) + if keys: + atomic(path, old, 0o600) + if self.key_ready: + own = text(SSH / "id_ed25519.pub") + committed = keys.get(MACHINE, "") + if not committed or key_parts(committed) != key_parts(own): + print(f"WARNING: {MACHINE}'s SSH public key is missing or stale in group_vars/all.yml: {own.strip()}") + + def host_data(self, name): + info = self.machines[name] + host = self.hosts.get(name) or {} + return info["hostname"], int(host.get("ansible_port", 22)), host.get("ansible_user", self.login) + + def scan(self, name): + host, port, _ = self.host_data(name) + kh = host if port == 22 else f"[{host}]:{port}" + result = run(["ssh-keyscan", "-t", "ed25519", "-T", "5", "-p", str(port), host], timeout=8) + if result.returncode: + return False + keys = [] + for line in result.stdout.splitlines(): + parts = line.split() + if len(parts) == 3 and parts[0] == kh: + key_parts(" ".join(parts[1:])) + keys.append(line) + if not keys: + return False + path = SSH / "known_hosts" + if path.is_symlink(): + raise RuntimeError("refusing to replace symlinked known_hosts") + old = text(path) + def matches(token): + if not token.startswith("|1|"): + return token.lower() == kh.lower() + try: + _, version, salt, digest = token.split("|") + if version != "1": + return False + actual = hmac.new(base64.b64decode(salt, validate=True), kh.encode(), hashlib.sha1).digest() + return hmac.compare_digest(actual, base64.b64decode(digest, validate=True)) + except ValueError: + return False + + kept, current = [], [] + for line in old.splitlines(keepends=True): + parts = line.split(None, 1) + if len(parts) != 2 or parts[0].startswith(("#", "@")): + kept.append(line) + continue + names = parts[0].split(",") + if not any(matches(token) for token in names): + kept.append(line) + continue + current.append(parts[1].strip()) + remaining = [token for token in names if not matches(token)] + if remaining: + kept.append(",".join(remaining) + " " + parts[1]) + if sorted(current) == sorted(line.split(None, 1)[1] for line in keys): + if CHECK: + atomic(path, old, 0o644) + else: + path.chmod(0o644) + return True + content = "".join(kept) + content += ("\n" if content and not content.endswith("\n") else "") + "\n".join(keys) + "\n" + atomic(path, content, 0o644) + return True + + def known_hosts(self): + for name in self.machines: + if name != MACHINE: + try: + self.scan(name) + except (OSError, ValueError, KeyError, subprocess.SubprocessError, RuntimeError) as exc: + print(f"warn: cannot refresh host key for {name}: {exc}") + + def mesh(self): + if not self.key_ready: + print("PARTIAL: SSH mesh probes deferred until SSH identity is available") + return + result = run(["python3", str(ROOT / "scripts/online_hosts.py"), "--peers"], timeout=15) + if result.returncode: + raise RuntimeError("cannot determine online tailnet peers") + online = set(result.stdout.splitlines()) + excluded = set(self.vars.get("ssh_mesh_exclude") or []) | self.vps | {MACHINE} + targets = [name for name in self.machines if name not in excluded + and (self.hosts.get(name) or {}).get("tailnet_name", name).lower() in online] + failed = [] + for name in targets: + host, port, user = self.host_data(name) + target = f"{user}@{host}" + args = ["ssh", "-o", "BatchMode=yes", "-o", "PasswordAuthentication=no", + "-o", "ControlMaster=no", "-o", "ControlPath=none", "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=8", "-p", str(port), target, "true"] + try: + probe = run(args, timeout=15) + if probe.returncode == 0: + continue + except subprocess.TimeoutExpired: + pass + try: + run(["ssh", "-O", "exit", "-p", str(port), target], timeout=5) + except subprocess.TimeoutExpired: + pass + # Never remove the old known-host entry unless a usable scan succeeds. + try: + self.scan(name) + except (OSError, ValueError, KeyError, subprocess.SubprocessError, RuntimeError) as exc: + print(f"warn: cannot refresh host key for {name}: {exc}") + try: + if run(args, timeout=15).returncode == 0: + continue + except subprocess.TimeoutExpired: + pass + failed.append(name) + if failed: + raise RuntimeError("SSH mesh broken to: " + ", ".join(failed)) + print("SSH mesh OK: " + (", ".join(targets) if targets else "no fleet peers online")) + + +def check_directory(path, mode): + if not path.exists(): + print(f"would create {path} (mode {mode:04o})") + elif not path.is_dir(): + raise RuntimeError(f"not a directory: {path}") + elif stat.S_IMODE(path.stat().st_mode) != mode: + print(f"would set {path} mode {mode:04o}") + else: + print(f"ok {path}") + + +def derive_private_key(private): + # Validate in an anonymous fd without chmodding the key or prompting. + if not hasattr(os, "memfd_create"): + raise RuntimeError("SSH key inspection requires memfd_create") + fd = os.memfd_create("ssh-fleet-key", 0) + try: + os.fchmod(fd, 0o600) + with os.fdopen(os.dup(fd), "w") as f: + f.write(private.rstrip("\n") + "\n") + result = subprocess.run(["ssh-keygen", "-y", "-P", "", "-f", f"/proc/self/fd/{fd}"], + stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=30, pass_fds=(fd,)) + if result.returncode: + raise RuntimeError("cannot validate SSH private key without prompting") + return key_parts(result.stdout) + finally: + os.close(fd) + + +class CheckFleet(Fleet): + def ensure_key(self): + private, public = SSH / "id_ed25519", SSH / "id_ed25519.pub" + if private.is_symlink() or public.is_symlink(): + raise RuntimeError("refusing to replace or chmod symlinked SSH keys") + if not private.exists(): + self.key_deferred = True + print("PARTIAL: SSH identity restoration or generation depends on the vault") + return + if not private.is_file(): + raise RuntimeError(f"not a regular file: {private}") + derived = derive_private_key(private.read_text()) + if stat.S_IMODE(private.stat().st_mode) != 0o600: + print(f"would set {private} mode 0600") + if not public.exists(): + atomic(public, " ".join(derived) + " " + MACHINE + "\n", 0o644) + elif not public.is_file() or key_parts(public.read_text()) != derived: + raise RuntimeError("local SSH private and public keys do not match") + elif stat.S_IMODE(public.stat().st_mode) != 0o644: + print(f"would set {public} mode 0644") + self.key_ready = True + + def known_hosts(self): + failed = False + for name in self.machines: + if name != MACHINE: + try: + if not self.scan(name): + raise RuntimeError("no usable host key returned") + except (OSError, ValueError, KeyError, subprocess.SubprocessError, RuntimeError) as exc: + print(f"FAIL cannot inspect host key for {name}: {exc}") + failed = True + if failed: + raise RuntimeError("one or more SSH host keys could not be inspected") + + def mesh(self): + if not self.key_ready: + print("PARTIAL: SSH mesh probes deferred until SSH identity is available") + return + result = run(["tailscale", "status", "--json"], timeout=15) + if result.returncode: + raise RuntimeError("cannot inspect online tailnet peers") + status = json.loads(result.stdout) + if not isinstance(status, dict): + raise ValueError("invalid tailnet status") + peers = status.get("Peer") or {} + if not isinstance(peers, dict): + raise ValueError("invalid tailnet peer listing") + online = set() + for peer in peers.values(): + if not isinstance(peer, dict): + raise ValueError("invalid tailnet peer listing") + if peer.get("Online"): + online.add(((peer.get("DNSName") or "").split(".")[0] or peer.get("HostName") or "").lower()) + excluded = set(self.vars.get("ssh_mesh_exclude") or []) | self.vps | {MACHINE} + targets = [name for name in self.machines if name not in excluded + and (self.hosts.get(name) or {}).get("tailnet_name", name).lower() in online] + print("online SSH mesh targets: " + (", ".join(targets) if targets else "none")) + if targets: + raise RuntimeError("SSH authentication, retries and host-key refresh cannot safely be probed in check mode") + + +def check_main(mode): + fleet = CheckFleet() + if mode == "mesh" and (fleet.host_vars.get("machine_profile") == "vm-test" or fleet.host_vars.get("skip_ssh_mesh", False)): + print("SKIPPED: SSH mesh disabled for this profile") + return 0 + fleet.step("SSH directory", lambda: check_directory(SSH, 0o700)) + if os.environ.get("UNITS_NO_SECRETS") != "1" and (mode == "keys" or not (SSH / "id_ed25519").exists()): + print("FAIL cannot inspect Bitwarden SSH keys safely: item reads may persist audit events") + fleet.failed = True + fleet.step("SSH key pair", fleet.ensure_key) + if mode == "keys": + fleet.step("SSH config", fleet.config) + if os.environ.get("UNITS_NO_SECRETS") == "1": + print("SKIPPED: Bitwarden SSH synchronization (--no-secrets)") + fleet.step("allowed signers", fleet.signers) + fleet.step("known hosts", fleet.known_hosts) + elif mode == "mesh": + fleet.step("fleet authorized keys", fleet.authorize_git) + fleet.step("mesh targets", fleet.mesh) + else: + raise ValueError("invalid mode") + return int(fleet.failed) + + +def main(mode): + if not re.fullmatch(r"[A-Za-z0-9_.-]+", MACHINE) or MACHINE in (".", ".."): + raise ValueError("invalid machine name") + if CHECK: + return check_main(mode) + SSH.mkdir(mode=0o700, exist_ok=True) + SSH.chmod(0o700) + lock_path = SSH / ".dotfiles-ssh.lock" + fd = os.open(lock_path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "r+") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + fleet = Fleet() + if mode == "mesh" and (fleet.host_vars.get("machine_profile") == "vm-test" or fleet.host_vars.get("skip_ssh_mesh", False)): + print("SKIPPED: SSH mesh disabled for this profile") + return 0 + # Mesh normally needs no vault; only consult it before creating a + # missing identity, preventing a race with the keys unit's restore. + if mode == "keys" or not (SSH / "id_ed25519").exists(): + fleet.step("Bitwarden session", fleet.unlock) + fleet.step("SSH key pair", fleet.ensure_key) + if mode == "keys": + fleet.step("SSH config", fleet.config) + fleet.step("Bitwarden SSH key sync", fleet.sync_key) + fleet.step("allowed signers", fleet.signers) + fleet.step("known hosts", fleet.known_hosts) + fleet.step("Bitwarden authorized keys", fleet.authorize_vault) + elif mode == "mesh": + fleet.step("fleet authorized keys", fleet.authorize_git) + fleet.step("mesh probes", fleet.mesh) + else: + raise ValueError("invalid mode") + return int(fleet.failed) + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1])) + except (OSError, ValueError, KeyError, TypeError, RuntimeError, subprocess.SubprocessError) as exc: + print("FAIL SSH fleet: " + str(exc), file=sys.stderr) + sys.exit(1) diff --git a/units/ssh-keys.sh b/units/ssh-keys.sh new file mode 100755 index 00000000..2fe41c15 --- /dev/null +++ b/units/ssh-keys.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +export DOTFILES_DIR DOTFILES_MACHINE="$MACHINE" +[ "${UNITS_CHECK:-0}" != 1 ] || export PYTHONDONTWRITEBYTECODE=1 +step "SSH key management" python3 "$DOTFILES_DIR/units/ssh-fleet.py" keys +finish diff --git a/units/ssh-mesh.sh b/units/ssh-mesh.sh new file mode 100755 index 00000000..6b2d95a1 --- /dev/null +++ b/units/ssh-mesh.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +export DOTFILES_DIR DOTFILES_MACHINE="$MACHINE" +[ "${UNITS_CHECK:-0}" != 1 ] || export PYTHONDONTWRITEBYTECODE=1 +step "SSH mesh" python3 "$DOTFILES_DIR/units/ssh-fleet.py" mesh +finish diff --git a/units/sshd.sh b/units/sshd.sh new file mode 100755 index 00000000..277cf82a --- /dev/null +++ b/units/sshd.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=units/platform-config.bash +. "$DOTFILES_DIR/units/platform-config.bash" +if [ "${UNITS_CHECK:-0}" = 1 ]; then + host_file="$DOTFILES_DIR/host_vars/$MACHINE.yml" + if [ -e "$host_file" ] && { [ ! -f "$host_file" ] || [ ! -r "$host_file" ]; }; then + step 'read sshd host configuration' false + finish + fi +fi + +root="${DOTFILES_SYSTEM_ROOT:-}" +dir="$root/etc/ssh/sshd_config.d" +if [ "${UNITS_CHECK:-0}" = 1 ]; then + parent="$root/etc/ssh" + if [ -e "$parent" ] && { [ ! -d "$parent" ] || [ ! -x "$parent" ]; }; then + step 'inspect sshd configuration directory' false + finish + fi +fi +if [ ! -d "$dir" ]; then + note 'SKIPPED: sshd drop-in directory is absent' + finish +fi +if [ "${UNITS_CHECK:-0}" = 1 ] && { [ ! -r "$dir" ] || [ ! -x "$dir" ]; }; then + step 'inspect sshd drop-in directory' false + finish +fi +emit_sshd_acceptenv() { printf 'AcceptEnv BW_SESSION BW_CLIENTID BW_CLIENTSECRET\n'; } +emit_sshd_hardening() { + cat <<'EOF' +# Managed by dotfiles (sshd role, sshd_harden) +PermitRootLogin no +PasswordAuthentication no +KbdInteractiveAuthentication no +X11Forwarding no +MaxAuthTries 3 +LoginGraceTime 30 +EOF +} + +if [ "${UNITS_CHECK:-0}" = 1 ]; then + changed=0 + inspect_dropin() { + local dest="$1" generator="$2" validate="$3" state rc + if [ -L "$dest" ] || [ ! -e "$dest" ]; then + changed=1 + elif [ ! -f "$dest" ]; then + echo "Cannot replace non-file sshd drop-in: $dest" >&2 + return 1 + else + state="$(stat -c '%a:%u:%g' "$dest")" || return 1 + if "$generator" | cmp -s - "$dest"; then + if [ "$state" = '644:0:0' ]; then + note "ok sshd drop-in: $dest" + return 0 + fi + else + rc=$? + if [ "$rc" -ne 1 ]; then echo "Cannot read sshd drop-in: $dest" >&2; return 1; fi + fi + changed=1 + fi + if [ "$validate" = 1 ]; then note "would validate sshd drop-in: $dest"; fi + note "would install sshd drop-in: $dest" + } + step 'inspect sshd AcceptEnv drop-in' inspect_dropin "$dir/99-bw-env.conf" emit_sshd_acceptenv 0 + if [ "$(host_var sshd_harden false)" = true ]; then + step 'inspect sshd hardening drop-in' inspect_dropin "$dir/10-hardening.conf" emit_sshd_hardening 1 + fi + if [ "$changed" = 1 ]; then + if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then + note 'SKIPPED: sshd reload on retired VPS' + else + note 'would reload sshd after changes' + fi + fi + finish +fi +tmp="$(mktemp -d)" || { step 'create sshd staging directory' false; finish; } +trap 'rm -rf "$tmp"' EXIT +changed=0 + +install_dropin() { + local candidate="$1" dest="$2" validate="$3" state staged + state="$(stat -c '%a:%u:%g' "$dest" 2>/dev/null)" || state='' + if [ ! -L "$dest" ] && cmp -s "$candidate" "$dest" && [ "$state" = '644:0:0' ]; then return 0; fi + if [ "$validate" = 1 ]; then + platform_root /usr/sbin/sshd -t -f "$candidate" || return 1 + fi + staged="$(platform_root mktemp "$dir/.dotfiles.XXXXXXXX")" || return 1 + if ! platform_root install -m 0644 -o root -g root "$candidate" "$staged" || ! platform_root mv -fT "$staged" "$dest"; then + platform_root rm -f "$staged" + return 1 + fi + changed=1 +} + +emit_sshd_acceptenv > "$tmp/bw-env" +soft 'sshd AcceptEnv drop-in' install_dropin "$tmp/bw-env" "$dir/99-bw-env.conf" 0 +if [ "$(host_var sshd_harden false)" = true ]; then + emit_sshd_hardening > "$tmp/hardening" + step 'sshd validated hardening drop-in' install_dropin "$tmp/hardening" "$dir/10-hardening.conf" 1 +fi + +if [ "$changed" = 1 ]; then + if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then + note 'SKIPPED: sshd reload on retired VPS' + else + service=sshd + if [ -r "$root/etc/os-release" ]; then + os_id=''; os_like='' + # shellcheck source=/dev/null + . "$root/etc/os-release" + os_id="${ID:-}"; os_like="${ID_LIKE:-}" + case " $os_id $os_like " in *' debian '*|*' ubuntu '*) service=ssh ;; esac + fi + soft 'reload sshd' platform_root systemctl reload "$service" + fi +fi +finish diff --git a/units/sudo.sh b/units/sudo.sh new file mode 100755 index 00000000..a5c09153 --- /dev/null +++ b/units/sudo.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=units/platform-config.bash +. "$DOTFILES_DIR/units/platform-config.bash" +if [ "${UNITS_CHECK:-0}" = 1 ]; then + host_file="$DOTFILES_DIR/host_vars/$MACHINE.yml" + if [ -e "$host_file" ] && { [ ! -f "$host_file" ] || [ ! -r "$host_file" ]; }; then + step 'read sudo host configuration' false + finish + fi +fi + +if [ "$(host_var skip_sudo false)" = true ]; then + note 'SKIPPED: sudo disabled for this host' + finish +fi +user="$(id -un)" || { step 'determine sudo user' false; finish; } +case "$user" in ''|*[!a-zA-Z0-9_.-]*) step 'invalid sudo user name' false; finish ;; esac +root="${DOTFILES_SYSTEM_ROOT:-}" +dir="$root/etc/sudoers.d" +if [ "${UNITS_CHECK:-0}" = 1 ]; then + # A non-interactive privileged read is needed for root-owned 0440 sudoers. + # Keep the entire inspection in one invocation so sudo failure cannot be + # mistaken for a missing file or a content mismatch. + check_sudo_policy() { + platform_root bash -s -- "$dir" "$user" <<'CHECK' +dir=$1 user=$2 +if [ ! -d "$dir" ] || [ ! -r "$dir" ] || [ ! -x "$dir" ]; then + echo "Cannot inspect sudoers directory: $dir" >&2 + exit 1 +fi +dest="$dir/zz-$user" +changed=0 +if [ -L "$dest" ] || [ ! -e "$dest" ]; then + changed=1 +elif [ ! -f "$dest" ]; then + echo "Cannot replace non-file sudo policy: $dest" >&2 + exit 1 +else + state=$(stat -c '%a:%u:%g' "$dest") || exit 1 + if printf '%s ALL=(ALL) NOPASSWD:ALL\n' "$user" | cmp -s - "$dest"; then + : + else + rc=$? + if [ "$rc" -ne 1 ]; then echo "Cannot read sudo policy: $dest" >&2; exit 1; fi + changed=1 + fi + [ "$state" = '440:0:0' ] || changed=1 +fi +echo 'would validate sudo policy with visudo' +if [ "$changed" = 1 ]; then echo "would install sudo policy: $dest"; else echo "ok sudo policy: $dest"; fi +for old in "$dir/$user" "$dir/99-$user"; do + if [ -e "$old" ] || [ -L "$old" ]; then + if [ -d "$old" ] && [ ! -L "$old" ]; then + echo "Cannot remove sudo policy directory: $old" >&2 + exit 1 + fi + echo "would remove legacy sudo policy: $old" + fi +done +CHECK + } + step 'inspect sudo policy' check_sudo_policy + finish +fi +tmp="$(mktemp -d)" || { step 'create sudo staging directory' false; finish; } +trap 'rm -rf "$tmp"' EXIT + +install_sudo_policy() { + local candidate="$tmp/sudoers" dest="$dir/zz-$user" state staged + printf '%s ALL=(ALL) NOPASSWD:ALL\n' "$user" > "$candidate" || return 1 + # Always validate before replacing the current rule or deleting older rules. + platform_root /usr/sbin/visudo -cf "$candidate" >/dev/null || return 1 + state="$(stat -c '%a:%u:%g' "$dest" 2>/dev/null)" || state='' + if [ -L "$dest" ] || ! cmp -s "$candidate" "$dest" || [ "$state" != '440:0:0' ]; then + staged="$(platform_root mktemp "$dir/.dotfiles.XXXXXXXX")" || return 1 + if ! platform_root install -m 0440 -o root -g root "$candidate" "$staged" || ! platform_root mv -fT "$staged" "$dest"; then + platform_root rm -f "$staged" + return 1 + fi + fi + platform_root rm -f -- "$dir/$user" "$dir/99-$user" +} +step 'validated passwordless sudo policy' install_sudo_policy +finish diff --git a/units/syncthing.sh b/units/syncthing.sh new file mode 100755 index 00000000..49f9ce78 --- /dev/null +++ b/units/syncthing.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [ "${UNITS_CHECK:-0}" = 1 ] && { [ -e "$DOTFILES_DIR/host_vars/$MACHINE.yml" ] || [ -L "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; } && { [ ! -f "$DOTFILES_DIR/host_vars/$MACHINE.yml" ] || [ ! -r "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; }; then + step "inspect host variables" false + finish +fi + +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ] || [ "$IS_TERMUX" = 1 ] || skipped_by_profile skip_syncthing; then + note "SKIPPED: syncthing excluded on this host" + finish +fi +if ! command -v yq >/dev/null 2>&1; then + step "check VPS membership (yq required)" false + finish +fi +vps=$(MACHINE="$MACHINE" yq '.all.children.vps.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml") || { step "check VPS membership" false; finish; } +case "$vps" in + true) note "SKIPPED: syncthing excluded on VPS"; finish ;; + false) ;; + *) step "check VPS membership" false; finish ;; +esac + +# Remote sessions may not inherit the user's systemd bus environment. +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then + export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus" +fi +unit="$HOME/.config/systemd/user/syncthing.service" +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if [ ! -e /home/linuxbrew/.linuxbrew/bin/syncthing ]; then + if [ -f "$unit" ]; then + note "CHANGE: disable and stop syncthing.service (binary missing)" + else + state=$(systemctl --user show --property=LoadState --value syncthing.service) || { step "check syncthing service" false; finish; } + case "$state" in + not-found) note "SKIPPED: syncthing binary and service not installed" ;; + loaded) note "CHANGE: disable and stop syncthing.service (binary missing)" ;; + *) step "check syncthing service load state" false ;; + esac + fi + finish + fi + src="$DOTFILES_DIR/units/syncthing/syncthing.service" + if [ ! -f "$src" ] || [ ! -r "$src" ]; then + step "read syncthing service source" false + elif [ ! -e "$unit" ] && [ ! -L "$unit" ]; then + note "CHANGE: install syncthing.service" + elif [ ! -f "$unit" ] || [ ! -r "$unit" ]; then + step "inspect syncthing service file" false + else + cmp -s "$src" "$unit"; rc=$? + case "$rc" in + 0) ;; + 1) note "CHANGE: update syncthing.service" ;; + *) step "compare syncthing service file" false ;; + esac + mode=$(stat -c %a "$unit") || { step "inspect syncthing service mode" false; finish; } + [ "$mode" = 644 ] || note "CHANGE: set syncthing.service mode 0644" + fi + state=$(systemctl --user show --property=LoadState --value syncthing.service) || { step "check syncthing service" false; finish; } + case "$state" in + not-found) note "CHANGE: enable and start syncthing.service" ;; + loaded) + enabled=$(systemctl --user show --property=UnitFileState --value syncthing.service) || { step "check syncthing enablement" false; finish; } + active=$(systemctl --user show --property=ActiveState --value syncthing.service) || { step "check syncthing activity" false; finish; } + if [ -z "$enabled" ] || [ -z "$active" ]; then + step "inspect syncthing service state" false + else + [ "$enabled" = enabled ] || note "CHANGE: enable syncthing.service" + [ "$active" = active ] || note "CHANGE: start syncthing.service" + fi + ;; + *) step "check syncthing service load state" false ;; + esac + finish +fi +if [ ! -e /home/linuxbrew/.linuxbrew/bin/syncthing ]; then + # A user service can also come from a system-wide user-unit directory. + if [ -f "$unit" ]; then + step "stop syncthing without binary" systemctl --user disable --now syncthing.service + else + state=$(systemctl --user show --property=LoadState --value syncthing.service) || { step "check syncthing service" false; finish; } + case "$state" in + not-found) note "SKIPPED: syncthing binary and service not installed" ;; + loaded) step "stop syncthing without binary" systemctl --user disable --now syncthing.service ;; + *) step "check syncthing service load state" false ;; + esac + fi + finish +fi +changed=0 +[ -f "$unit" ] && cmp -s "$DOTFILES_DIR/units/syncthing/syncthing.service" "$unit" || changed=1 +if install_if_changed "$DOTFILES_DIR/units/syncthing/syncthing.service" "$unit" 0644; then + note "ok syncthing service file" +else + step "syncthing service file" false + finish +fi +if [ "$changed" = 1 ]; then + if ! systemctl --user daemon-reload; then + step "reload user systemd" false + finish + fi +fi +step "enable and start syncthing" systemctl --user enable --now syncthing.service +finish diff --git a/roles/syncthing/files/syncthing.service b/units/syncthing/syncthing.service similarity index 100% rename from roles/syncthing/files/syncthing.service rename to units/syncthing/syncthing.service diff --git a/units/system-services.sh b/units/system-services.sh new file mode 100755 index 00000000..f6a92891 --- /dev/null +++ b/units/system-services.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [ "${UNITS_CHECK:-0}" = 1 ] && [ "$MACHINE" != matrix ] && [ "$MACHINE" != telengana ] && [ "$IS_TERMUX" != 1 ]; then + hostvars="$DOTFILES_DIR/host_vars/$MACHINE.yml" + if [ -e "$hostvars" ] || [ -L "$hostvars" ]; then + if [ ! -f "$hostvars" ] || [ ! -r "$hostvars" ] || ! cat "$hostvars" >/dev/null 2>&1; then + step "inspect host variables" false; finish + fi + elif [ ! -d "$DOTFILES_DIR/host_vars" ] || [ ! -x "$DOTFILES_DIR/host_vars" ]; then + step "inspect host variables directory" false; finish + fi +fi + +# Nothing on the retired VPSes may be restarted or otherwise altered. +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ] || [ "$IS_TERMUX" = 1 ] || [ "$(host_var skip_systemd false)" = true ]; then + note "SKIPPED: system services excluded on this host" + finish +fi + +root() { + if [ "$(id -u)" = 0 ]; then "$@"; else sudo -n "$@"; fi +} +check_file() { + local src="$1" dest="$2" mode="$3" rc actual parent + CHECK_CHANGED=0 + if [ ! -f "$src" ] || [ ! -r "$src" ]; then + echo "Cannot inspect source: $src" >&2; return 1 + fi + if [ ! -e "$dest" ]; then + parent=$(dirname "$dest") + while [ ! -e "$parent" ] && [ "$parent" != / ]; do parent=$(dirname "$parent"); done + if [ ! -d "$parent" ] || [ ! -x "$parent" ]; then + echo "Cannot inspect destination: $dest" >&2; return 1 + fi + if ! cat "$src" >/dev/null; then echo "Cannot read source: $src" >&2; return 1; fi + CHECK_CHANGED=1 + note "would install $dest" + return 0 + fi + if [ ! -f "$dest" ] || [ ! -r "$dest" ]; then + echo "Cannot inspect destination: $dest" >&2; return 1 + fi + if cmp -s "$src" "$dest"; then :; else + rc=$? + if [ "$rc" != 1 ]; then echo "Cannot compare $dest" >&2; return 1; fi + CHECK_CHANGED=1 + note "would update $dest" + fi + actual=$(stat -Lc '%a' "$dest") || return 1 + [ "$actual" = "$mode" ] || note "would set mode $mode on $dest" +} +check_enabled() { + local scope="$1" unit="$2" state rc + local -a args=() + [ "$scope" != user ] || args+=(--user) + CHECK_ACTIVE=0 + if state=$(systemctl "${args[@]}" is-enabled "$unit" 2>/dev/null); then rc=0; else rc=$?; fi + case "$state:$rc" in + enabled:0) ;; + disabled:1|masked:1|masked-runtime:1|not-found:1|not-found:4|static:0|indirect:0|indirect:1|linked:0|linked-runtime:0|enabled-runtime:0|generated:0|transient:0|alias:0) + note "would enable $unit" ;; + *) echo "Cannot inspect enabled state of $unit" >&2; return 1 ;; + esac + if state=$(systemctl "${args[@]}" is-active "$unit" 2>/dev/null); then rc=0; else rc=$?; fi + case "$state:$rc" in + active:0|reloading:0|refreshing:0) CHECK_ACTIVE=1 ;; + inactive:3|failed:3|activating:3|deactivating:3|unknown:3|unknown:4) + note "would start $unit" ;; + *) echo "Cannot inspect active state of $unit" >&2; return 1 ;; + esac +} +check_podman() { + local login groups rc actual changed + local dest=/etc/systemd/system/podman.socket.d/group-access.conf + local dir=/etc/systemd/system/podman.socket.d + if [ ! -e /usr/bin/podman ]; then + note "SKIPPED: system podman not installed" + return 0 + fi + login=$(id -un) || return 1 + if getent group podman >/dev/null; then :; else + rc=$? + [ "$rc" = 2 ] || return "$rc" + note "would create podman group" + fi + groups=$(id -nG "$login") || return 1 + case " $groups " in *' podman '*) ;; *) note "would add $login to podman group" ;; esac + printf '[Socket]\nSocketGroup=podman\nSocketMode=0660\nDirectoryMode=0710\n' > "$tmp/podman-group.conf" || return 1 + if [ -e "$dir" ]; then + [ -d "$dir" ] || { echo "Cannot inspect directory: $dir" >&2; return 1; } + actual=$(stat -Lc '%a' "$dir") || return 1 + [ "$actual" = 755 ] || note "would set mode 755 on $dir" + else + note "would create $dir" + fi + check_file "$tmp/podman-group.conf" "$dest" 644 || return 1 + changed=$CHECK_CHANGED + check_enabled system podman.socket || return 1 + if [ "$changed" = 1 ]; then + note "would reload system systemd" + [ "$CHECK_ACTIVE" != 1 ] || note "would restart podman.socket" + fi +} +configure_podman() { + if [ "${UNITS_CHECK:-0}" = 1 ]; then check_podman; return; fi + local login groups group_status was_active=0 changed=0 + local dest=/etc/systemd/system/podman.socket.d/group-access.conf + local content="$tmp/podman-group.conf" + if [ ! -e /usr/bin/podman ]; then + note "SKIPPED: system podman not installed" + return 0 + fi + login=$(id -un) || return 1 + if getent group podman >/dev/null; then + : + else + group_status=$? + [ "$group_status" = 2 ] || return "$group_status" + root groupadd --system podman || return 1 + fi + groups=$(id -nG "$login") || return 1 + case " $groups " in + *' podman '*) ;; + *) root usermod -a -G podman "$login" || return 1 ;; + esac + printf '[Socket]\nSocketGroup=podman\nSocketMode=0660\nDirectoryMode=0710\n' > "$content" || return 1 + root install -d -m 0755 /etc/systemd/system/podman.socket.d || return 1 + if [ ! -f "$dest" ] || ! cmp -s "$content" "$dest"; then + changed=1 + root install -m 0644 "$content" "$dest" || return 1 + else + root chmod 0644 "$dest" || return 1 + fi + if root systemctl is-active --quiet podman.socket; then was_active=1; fi + root systemctl daemon-reload || return 1 + root systemctl enable --now podman.socket || return 1 + # A running socket must be reopened for changed ownership/mode to take effect. + # A socket just started already used the new settings and needs no restart. + if [ "$changed" = 1 ] && [ "$was_active" = 1 ]; then + root systemctl restart podman.socket || return 1 + fi +} + +tmp=$(mktemp -d) || { step "create temporary directory" false; finish; } +trap 'rm -rf "$tmp"' EXIT +step "configure system podman socket" configure_podman + +# Laptop user units are owned by laptop-services.sh; the system socket above +# applies to laptops too, so both units can run without touching the same file. +if [ "$(host_var is_laptop false)" = true ]; then + finish +fi + +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then + export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus" +fi +brew_prefix=$(host_var brew_prefix /home/linuxbrew/.linuxbrew) +delay=$(host_var dotfiles_update_delay_seconds 60) +if [[ ! "$delay" =~ ^[0-9]+$ ]]; then + step "invalid dotfiles update delay" false + finish +fi +unitdir="$HOME/.config/systemd/user" +changed=0 +install_unit() { + local src="$1" dest="$unitdir/$2" + if [ "${UNITS_CHECK:-0}" = 1 ]; then + check_file "$src" "$dest" 644 || return 1 + [ "$CHECK_CHANGED" != 1 ] || changed=1 + return 0 + fi + if [ ! -f "$dest" ] || ! cmp -s "$src" "$dest"; then changed=1; fi + install_if_changed "$src" "$dest" 0644 +} + +# Render the non-laptop branches of the migrated service templates without +# requiring Jinja on the host. +render_service() { + local template="$1" dest="$2" line branch=1 + : > "$dest" || return 1 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + '{% if is_laptop | bool %}') branch=0; continue ;; + '{% else %}') branch=1; continue ;; + '{% endif %}') branch=1; continue ;; + esac + [ "$branch" = 1 ] || continue + line=${line//'{{ brew_prefix }}'/"$brew_prefix"} + line=${line//"{{ ansible_facts['env']['HOME'] }}"/"$HOME"} + line=${line//'{{ dotfiles_update_delay_seconds | default(60) }}'/"$delay"} + if [[ "$line" == *'{{'* || "$line" == *'{%'* ]]; then + echo "Unsupported system service template expression" >&2 + return 1 + fi + printf '%s\n' "$line" >> "$dest" || return 1 + done < "$template" +} +templates="$DOTFILES_DIR/units/system-services" +update_ready=0 +timer_ready=0 +atuin_ready=0 +if render_service "$templates/dotfiles-update.service.j2" "$tmp/dotfiles-update.service" && install_unit "$tmp/dotfiles-update.service" dotfiles-update.service; then + update_ready=1 + note "ok deploy dotfiles update service" +else + step "render and deploy dotfiles update service" false +fi +if install_unit "$templates/dotfiles-update.timer.j2" dotfiles-update.timer; then + timer_ready=1 + note "ok deploy dotfiles update timer" +else + step "deploy dotfiles update timer" false +fi +if render_service "$templates/atuin-daemon.service.j2" "$tmp/atuin-daemon.service" && install_unit "$tmp/atuin-daemon.service" atuin-daemon.service; then + atuin_ready=1 + note "ok deploy atuin daemon" +else + step "render and deploy atuin daemon" false +fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + [ "$changed" != 1 ] || note "would reload user systemd" + if [ "$update_ready" = 1 ] && [ "$timer_ready" = 1 ]; then + step "inspect dotfiles update timer" check_enabled user dotfiles-update.timer + fi + if [ "$atuin_ready" = 1 ]; then + step "inspect atuin daemon" check_enabled user atuin-daemon.service + fi + finish +fi +if [ "$changed" = 1 ]; then + if ! systemctl --user daemon-reload; then + step "reload user systemd" false + finish + fi +fi +if [ "$update_ready" = 1 ] && [ "$timer_ready" = 1 ]; then + step "enable and start dotfiles update timer" systemctl --user enable --now dotfiles-update.timer +fi +if [ "$atuin_ready" = 1 ]; then + step "enable and start atuin daemon" systemctl --user enable --now atuin-daemon.service +fi +finish diff --git a/roles/systemd/templates/atuin-daemon.service.j2 b/units/system-services/atuin-daemon.service.j2 similarity index 100% rename from roles/systemd/templates/atuin-daemon.service.j2 rename to units/system-services/atuin-daemon.service.j2 diff --git a/roles/systemd/templates/dotfiles-update.service.j2 b/units/system-services/dotfiles-update.service.j2 similarity index 100% rename from roles/systemd/templates/dotfiles-update.service.j2 rename to units/system-services/dotfiles-update.service.j2 diff --git a/roles/systemd/templates/dotfiles-update.timer.j2 b/units/system-services/dotfiles-update.timer.j2 similarity index 100% rename from roles/systemd/templates/dotfiles-update.timer.j2 rename to units/system-services/dotfiles-update.timer.j2 diff --git a/units/tailscale-cert.sh b/units/tailscale-cert.sh new file mode 100755 index 00000000..3ff0d34c --- /dev/null +++ b/units/tailscale-cert.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ] || skipped_by_profile skip_tailscale_cert; then + note "SKIPPED: Tailscale certificates excluded on this host" + finish +fi +if ! command -v yq >/dev/null 2>&1; then + step "check certificate VPS membership (yq required)" false + finish +fi +vps=$(MACHINE="$MACHINE" yq '.all.children.vps.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml") || { step "check certificate VPS membership" false; finish; } +case "$vps" in + true) note "SKIPPED: Tailscale certificates excluded on VPS"; finish ;; + false) ;; + *) step "check certificate VPS membership" false; finish ;; +esac +as_root() { if [ "$EUID" = 0 ]; then "$@"; else sudo -n "$@"; fi; } +# Host variables override the shared group domain, as in the playbook. +domain="" +if [ -f "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; then + domain=$(yq -r '.tailscale_domain // ""' "$DOTFILES_DIR/host_vars/$MACHINE.yml") || { step "read host certificate domain" false; finish; } +fi +if [ -z "$domain" ]; then + domain=$(yq -r '.tailscale_domain // ""' "$DOTFILES_DIR/group_vars/all.yml") || { step "read certificate domain" false; finish; } +fi +if [[ ! "$MACHINE" =~ ^[[:alnum:]_.-]+$ || ! "$domain" =~ ^[[:alnum:]_.-]+$ ]]; then + step "validate certificate hostname and domain" false + finish +fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + if [ "$EUID" != 0 ] && ! sudo -n true; then step "noninteractive root access" false; finish; fi + tmp=$(mktemp -d) || { step "create certificate temporary directory" false; finish; } + trap 'rm -rf "$tmp"' EXIT + if as_root test -e /etc/ssl/tailscale; then + mode=$(as_root stat -c '%a:%u:%g' /etc/ssl/tailscale) || { step "inspect certificate directory" false; mode=; } + [ -z "$mode" ] || { if as_root test -d /etc/ssl/tailscale && [ "$mode" = '755:0:0' ]; then note "ok certificate directory"; else note "would set certificate directory ownership and permissions"; fi; } + else note "would create certificate directory"; fi + renew=0 + if ! as_root test -e /etc/ssl/tailscale/cert.pem; then + renew=1 + note "would provision Tailscale certificate for $MACHINE.$domain" + elif ! as_root openssl x509 -noout -in /etc/ssl/tailscale/cert.pem >/dev/null 2>&1; then + step "inspect Tailscale certificate" false + elif as_root openssl x509 -checkend 2592000 -noout -in /etc/ssl/tailscale/cert.pem >/dev/null 2>&1; then + note "ok certificate valid for at least 30 days" + else + renew=1 + note "would renew Tailscale certificate for $MACHINE.$domain" + fi + cert_check_mode() { + local path=$1 expected=$2 mode + if as_root test -e "$path"; then + if mode=$(as_root stat -c '%a:%u:%g' "$path"); then + [ "$mode" = "$expected:0:0" ] && note "ok permissions $path" || note "would set ownership and permissions on $path" + else step "inspect permissions $path" false; fi + elif [ "$renew" = 1 ]; then + note "would set ownership and permissions on newly provisioned $path" + else + step "missing $path (apply cannot set its permissions)" false + fi + } + cert_check_mode /etc/ssl/tailscale/cert.pem 644 + cert_check_mode /etc/ssl/tailscale/key.pem 640 + cert_check_file() { + local src=$1 dest=$2 mode rc + [ -r "$src" ] || { step "read $src" false; return; } + if ! as_root test -e "$dest" && ! as_root test -L "$dest"; then note "would deploy $dest"; return; fi + mode=$(as_root stat -c '%a:%u:%g' "$dest") || { step "inspect $dest" false; return; } + as_root cmp -s "$src" "$dest"; rc=$? + case "$rc" in + 0) [ "$mode" = '644:0:0' ] && note "ok $dest" || note "would set ownership and permissions on $dest" ;; + 1) note "would deploy $dest" ;; + *) step "compare $dest" false ;; + esac + } + if python3 "$DOTFILES_DIR/units/service-templates.py" cert "$DOTFILES_DIR/units/tailscale-cert/tailscale-cert-renew.service.j2" "$MACHINE" "$domain" > "$tmp/service" && [ -s "$tmp/service" ]; then + cert_check_file "$tmp/service" /etc/systemd/system/tailscale-cert-renew.service + else step "render certificate renewal service" false; fi + cert_check_file "$DOTFILES_DIR/units/tailscale-cert/tailscale-cert-renew.timer" /etc/systemd/system/tailscale-cert-renew.timer + state=$(as_root systemctl show tailscale-cert-renew.timer --property=LoadState --property=ActiveState --property=UnitFileState 2>/dev/null); rc=$? + if { [ "$rc" != 0 ] && [[ "$state" != *'LoadState=not-found'* ]]; } || [[ "$state" != *'LoadState='* || "$state" != *'ActiveState='* ]]; then + step "inspect certificate renewal timer" false + elif [[ "$state" == *'LoadState=not-found'* ]]; then + note "would reload systemd and enable and start certificate renewal timer" + elif [[ "$state" != *'LoadState=loaded'* || "$state" != *'UnitFileState='* ]]; then + step "inspect certificate renewal timer" false + else + [[ "$state" == *'ActiveState=active'* ]] || note "would start certificate renewal timer" + [[ "$state" == *'UnitFileState=enabled'* ]] || note "would enable certificate renewal timer" + fi + finish +fi +step "create Tailscale certificate directory" as_root install -d -m 0755 -o root -g root /etc/ssl/tailscale +if ! as_root openssl x509 -checkend 2592000 -noout -in /etc/ssl/tailscale/cert.pem >/dev/null 2>&1; then + step "provision Tailscale certificate" as_root tailscale cert --cert-file /etc/ssl/tailscale/cert.pem --key-file /etc/ssl/tailscale/key.pem --min-validity 720h "$MACHINE.$domain" +fi +set_cert_mode() { + as_root chown root:root "$1" && as_root chmod "$2" "$1" +} +step "set certificate permissions" set_cert_mode /etc/ssl/tailscale/cert.pem 0644 +step "set private key permissions" set_cert_mode /etc/ssl/tailscale/key.pem 0640 + +tmp=$(mktemp -d) || { step "create certificate temporary directory" false; finish; } +trap 'rm -rf "$tmp"' EXIT +install_root_file() { + local src="$1" dest="$2" mode + mode=$(as_root stat -c '%a:%u:%g' "$dest" 2>/dev/null || true) + if as_root cmp -s "$src" "$dest" && [ "$mode" = '644:0:0' ]; then return 0; fi + as_root install -m 0644 -o root -g root "$src" "$dest" +} +service_ok=0 +timer_ok=0 +if python3 "$DOTFILES_DIR/units/service-templates.py" cert "$DOTFILES_DIR/units/tailscale-cert/tailscale-cert-renew.service.j2" "$MACHINE" "$domain" > "$tmp/tailscale-cert-renew.service" && + [ -s "$tmp/tailscale-cert-renew.service" ] && + install_root_file "$tmp/tailscale-cert-renew.service" /etc/systemd/system/tailscale-cert-renew.service; then + service_ok=1 + note "ok deploy certificate renewal service" +else + step "deploy certificate renewal service" false +fi +if install_root_file "$DOTFILES_DIR/units/tailscale-cert/tailscale-cert-renew.timer" /etc/systemd/system/tailscale-cert-renew.timer; then + timer_ok=1 + note "ok deploy certificate renewal timer" +else + step "deploy certificate renewal timer" false +fi +if [ "$service_ok" = 1 ] && [ "$timer_ok" = 1 ]; then + if as_root systemctl daemon-reload; then + step "enable and start certificate renewal timer" as_root systemctl enable --now tailscale-cert-renew.timer + else + step "reload systemd for certificate renewal" false + fi +fi +finish diff --git a/units/tailscale-cert/tailscale-cert-renew.service.j2 b/units/tailscale-cert/tailscale-cert-renew.service.j2 new file mode 100644 index 00000000..e2c65295 --- /dev/null +++ b/units/tailscale-cert/tailscale-cert-renew.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Renew Tailscale TLS certificate +After=tailscaled.service +Wants=tailscaled.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/tailscale cert \ + --cert-file /etc/ssl/tailscale/cert.pem \ + --key-file /etc/ssl/tailscale/key.pem \ + --min-validity 720h \ + {{ inventory_hostname }}.{{ tailscale_domain }} diff --git a/units/tailscale-cert/tailscale-cert-renew.timer b/units/tailscale-cert/tailscale-cert-renew.timer new file mode 100644 index 00000000..fab3a05f --- /dev/null +++ b/units/tailscale-cert/tailscale-cert-renew.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly Tailscale TLS certificate renewal + +[Timer] +OnCalendar=weekly +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/units/tailscale-connect.py b/units/tailscale-connect.py new file mode 100755 index 00000000..4a648264 --- /dev/null +++ b/units/tailscale-connect.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Join Tailscale without placing vault or Tailscale credentials in argv.""" +import base64 +import json +import os +import signal +import subprocess +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request + + +def fail(message): + print(message, file=sys.stderr) + return 1 + + +def interrupt(_signal, _frame): + raise InterruptedError() + + +def main(): + if len(sys.argv) != 4: + return fail("Tailscale enrollment requires machine, domain and privilege mode") + machine, domain, root_mode = sys.argv[1:] + try: + result = subprocess.run(["bw", "get", "item", "tailscale-apikey"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + check=True, text=True) + key = json.loads(result.stdout)["login"]["password"] + if not isinstance(key, str) or not key.strip(): + raise ValueError("empty key") + except (OSError, subprocess.SubprocessError, KeyError, TypeError, ValueError): + return fail("Unable to read a nonempty Tailscale key from Bitwarden") + + authorization = "Basic " + base64.b64encode((key + ":").encode()).decode() + headers = {"Authorization": authorization, "Content-Type": "application/json"} + try: + request = urllib.request.Request("https://api.tailscale.com/api/v2/tailnet/-/devices", headers=headers) + with urllib.request.urlopen(request, timeout=30) as response: + devices = json.load(response).get("devices", []) + if not isinstance(devices, list): + raise ValueError("invalid device list") + except (OSError, ValueError, TypeError): + # The old role explicitly treated the list operation as best effort. + print("warn: could not read Tailscale device list; stale-device cleanup skipped") + devices = [] + old = next((device.get("id") for device in devices + if isinstance(device, dict) and device.get("name") == machine + "." + domain + and device.get("id")), None) + if old is not None: + try: + url = "https://api.tailscale.com/api/v2/device/" + urllib.parse.quote(str(old), safe="") + request = urllib.request.Request(url, headers=headers, method="DELETE") + with urllib.request.urlopen(request, timeout=30) as response: + if response.status != 200: + raise ValueError("unexpected response") + except (OSError, ValueError): + return fail("Unable to remove the matching stale Tailscale device; join skipped") + + # The auth key is read from a private file by tailscale; the file and its + # containing directory are removed even on normal termination signals. + old_handlers = {sig: signal.signal(sig, interrupt) for sig in (signal.SIGTERM, signal.SIGINT)} + try: + with tempfile.TemporaryDirectory(prefix="dotfiles-tailscale-") as directory: + path = os.path.join(directory, "authkey") + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as file: + file.write(key) + command = ([] if root_mode == "root" else ["sudo", "-n"]) + [ + "tailscale", "up", "--auth-key=file:" + path, + "--advertise-tags=tag:fleet", "--accept-routes", "--hostname", machine, + ] + environment = os.environ.copy() + environment.pop("BW_SESSION", None) + result = subprocess.run(command, env=environment, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if result.returncode: + return fail("Tailscale join failed; check the daemon and key permissions") + except (OSError, InterruptedError): + return fail("Tailscale join interrupted or could not be started") + finally: + for sig, previous in old_handlers.items(): + signal.signal(sig, previous) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/units/tailscale.sh b/units/tailscale.sh new file mode 100755 index 00000000..cdb25de3 --- /dev/null +++ b/units/tailscale.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +set -uo pipefail +# The session must not be printed even when the caller enables shell tracing. +set +x +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=units/platform-config.bash +. "$DOTFILES_DIR/units/platform-config.bash" + +if [ "$IS_TERMUX" = 1 ]; then + note 'SKIPPED: Android Tailscale is managed outside unprivileged Termux' + finish +fi +root="${DOTFILES_SYSTEM_ROOT:-}" +retired=0 +case "$MACHINE" in matrix|telengana) retired=1 ;; esac +routes="$(host_var tailscale_advertise_routes)" +accept_dns="$(host_var tailscale_accept_dns true)" +operator="$(host_var tailscale_operator)" +domain="$(host_var tailscale_domain)" +if [ -z "$domain" ]; then + domain="$(sed -n 's/^tailscale_domain:[[:space:]]*\([^#]*[^#[:space:]]\).*/\1/p' "$DOTFILES_DIR/group_vars/all.yml" 2>/dev/null | head -n1)" + domain="${domain//[\"\']/}" +fi +resolved_changed=0 +sysctl_changed=0 + +check_tailscale_files() { + local connected="$1" resolved_active="$2" + PYTHONDONTWRITEBYTECODE=1 python3 - "$root" "$accept_dns" "$resolved_active" "$connected" "$retired" <<'PY' +import os, stat, sys +root, accept_dns, active, connected, retired = sys.argv[1:] +def path(name): + return root + name +def directory(name): + try: + info = os.stat(path(name)) + return stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o755 + except FileNotFoundError: + return False +def content(name, expected): + try: + filename = path(name) + info = os.lstat(filename) + if not stat.S_ISREG(info.st_mode): + return False + with open(filename, 'rb') as f: + same = f.read() == expected + return same and stat.S_IMODE(info.st_mode) == 0o644 and info.st_uid == 0 and info.st_gid == 0 + except FileNotFoundError: + return False +def report(label, good): + print(('ok ' if good else 'WOULD configure ') + label) +try: + report('Tailscale resolved directory', directory('/etc/systemd/resolved.conf.d')) + resolved = not content('/etc/systemd/resolved.conf.d/ipv6-stub.conf', b'[Resolve]\nDNSStubListenerExtra=::1\n') + report('Tailscale resolved configuration', not resolved) + if accept_dns != 'false' and active == 'active': + try: + linked = os.readlink(path('/etc/resolv.conf')) == '/run/systemd/resolve/stub-resolv.conf' + except FileNotFoundError: + linked = False + except OSError as e: + if e.errno != 22: raise + linked = False + report('Tailscale resolver link', linked) + resolved = resolved or not linked + report('Tailscale sysctl directory', directory('/etc/sysctl.d')) + sysctl = not content('/etc/sysctl.d/99-tailscale.conf', b'net.ipv4.conf.all.rp_filter = 2\nnet.ipv4.conf.default.rp_filter = 2\nnet.ipv4.ip_forward = 1\nnet.ipv6.conf.all.forwarding = 1\n') + report('Tailscale sysctl configuration', not sysctl) + if resolved: + print('SKIPPED: resolved restart on retired VPS' if retired == '1' else 'WOULD restart systemd-resolved') + if sysctl: + print('WOULD apply Tailscale sysctls') + if connected == '1' and accept_dns == 'false': + report('Tailscale static DNS', content('/etc/resolv.conf', b'nameserver 1.1.1.1\nnameserver 8.8.8.8\n')) +except OSError: + print('Cannot inspect Tailscale system files', file=sys.stderr) + sys.exit(1) +PY +} +check_tailscale_prefs() { + local prefs user + prefs="$(tailscale debug prefs 2>/dev/null)" || { echo 'Cannot inspect Tailscale preferences' >&2; return 1; } + user="${USER:-$(id -un)}" + PYTHONDONTWRITEBYTECODE=1 python3 -c ' +import json, sys +try: + raw = sys.stdin.read() + prefs = json.loads(raw) + if not isinstance(prefs, dict): raise ValueError() + hostname, routes, desktop, user = sys.argv[1:] + print(("ok " if prefs.get("Hostname", "") == hostname else "WOULD set ") + "Tailscale hostname") + if routes: + print(("ok " if routes in raw else "WOULD configure ") + "Tailscale advertised routes") + if desktop == "1": + print(("ok " if prefs.get("OperatorUser", "") == user else "WOULD set ") + "Tailscale desktop operator") +except (ValueError, TypeError): + print("Cannot parse Tailscale preferences", file=sys.stderr) + sys.exit(1) +' "$MACHINE" "$routes" "$1" "$user" <<< "$prefs" +} +if [ "${UNITS_CHECK:-0}" = 1 ]; then + for config in "$DOTFILES_DIR/host_vars/$MACHINE.yml" "$DOTFILES_DIR/group_vars/all.yml"; do + if [ -e "$config" ] || [ -L "$config" ]; then + [ -f "$config" ] && [ -r "$config" ] || step 'inspect Tailscale configuration' false + fi + done + if [ -z "$domain" ]; then note 'PARTIAL: Tailscale domain is unset; joining requires a configured domain'; fi + resolved_active=inactive + if [ "$accept_dns" != false ]; then + resolved_active="$(systemctl is-active systemd-resolved 2>/dev/null)"; rc=$? + case "$resolved_active:$rc" in + active:0|reloading:0|refreshing:0|inactive:3|failed:3|activating:3|deactivating:3) ;; + unknown:4) + load_state="$(systemctl show -p LoadState --value systemd-resolved 2>/dev/null)" || load_state='' + [ "$load_state" = not-found ] || step 'inspect systemd-resolved state' false ;; + *) step 'inspect systemd-resolved state' false ;; + esac + fi + connected=0 + if command -v tailscale >/dev/null 2>&1; then + status="$(tailscale status --json 2>/dev/null)" + if [ "$?" = 0 ] && backend="$(PYTHONDONTWRITEBYTECODE=1 python3 -c ' +import json, sys +try: + value = json.load(sys.stdin)["BackendState"] + if value not in ("Running", "NeedsLogin", "NeedsMachineAuth", "Stopped", "Starting", "NoState"): raise ValueError() + print(value) +except (ValueError, KeyError, TypeError): + sys.exit(1) +' <<< "$status")"; then + if [ "$backend" = Running ]; then connected=1 + elif [ "$retired" = 1 ]; then note 'SKIPPED: Tailscale installation and rejoin on retired VPS' + else note "WOULD join tailnet (Tailscale state: $backend; credentials not inspected)"; fi + else + step 'inspect Tailscale status' false + fi + elif [ "$retired" = 1 ]; then + note 'SKIPPED: Tailscale installation and rejoin on retired VPS' + else + note 'WOULD install Tailscale and join tailnet (credentials not inspected)' + fi + step 'inspect Tailscale system files' check_tailscale_files "$connected" "$resolved_active" + if [ "$connected" = 1 ]; then + desktop=0 + if platform_desktop; then desktop=1; else + rc=$?; [ "$rc" = 1 ] || step 'read desktop inventory group' false + fi + step 'inspect Tailscale preferences' check_tailscale_prefs "$desktop" + if [ "$desktop" = 1 ] && [ "$retired" = 0 ]; then + note 'WOULD configure Tailscale systray, reload user systemd, and enable/start systray service' + fi + fi + finish +fi + +tmp="$(mktemp -d)" || { step 'create Tailscale staging directory' false; finish; } +trap 'rm -rf "$tmp"' EXIT + +# Replace system files atomically, without following a pre-existing symlink. +install_system_file() { + local src="$1" dest="$2" flag="$3" state staged + state="$(stat -c '%a:%u:%g' "$dest" 2>/dev/null)" || state='' + if [ ! -L "$dest" ] && cmp -s "$src" "$dest" && [ "$state" = '644:0:0' ]; then return 0; fi + staged="$(platform_root mktemp "$(dirname "$dest")/.dotfiles.XXXXXXXX")" || return 1 + if ! platform_root install -m 0644 -o root -g root "$src" "$staged" || ! platform_root mv -fT "$staged" "$dest"; then + platform_root rm -f "$staged" + return 1 + fi + case "$flag" in resolved) resolved_changed=1 ;; sysctl) sysctl_changed=1 ;; esac +} +configure_resolved() { + platform_root install -d -m 0755 "$root/etc/systemd/resolved.conf.d" || return 1 + printf '[Resolve]\nDNSStubListenerExtra=::1\n' > "$tmp/resolved" || return 1 + install_system_file "$tmp/resolved" "$root/etc/systemd/resolved.conf.d/ipv6-stub.conf" resolved +} +configure_resolv_link() { + local active + [ "$accept_dns" != false ] || return 0 + active="$(systemctl is-active systemd-resolved 2>/dev/null)" || : + [ "$active" = active ] || return 0 + if [ "$(readlink "$root/etc/resolv.conf" 2>/dev/null)" = /run/systemd/resolve/stub-resolv.conf ]; then return 0; fi + platform_root ln -sfn /run/systemd/resolve/stub-resolv.conf "$root/etc/resolv.conf" || return 1 + resolved_changed=1 +} +configure_sysctl() { + platform_root install -d -m 0755 "$root/etc/sysctl.d" || return 1 + printf '%s\n' 'net.ipv4.conf.all.rp_filter = 2' 'net.ipv4.conf.default.rp_filter = 2' 'net.ipv4.ip_forward = 1' 'net.ipv6.conf.all.forwarding = 1' > "$tmp/sysctl" || return 1 + install_system_file "$tmp/sysctl" "$root/etc/sysctl.d/99-tailscale.conf" sysctl +} +configure_routes() { + local prefs args=() + [ -n "$routes" ] || return 0 + prefs="$(tailscale debug prefs 2>/dev/null)" || : + case "$prefs" in *"$routes"*) return 0 ;; esac + args=(up "--hostname=$MACHINE" "--advertise-routes=$routes") + [ "$accept_dns" != false ] || args+=(--accept-dns=false) + [ -z "$operator" ] || args+=("--operator=$operator") + platform_root tailscale "${args[@]}" +} +configure_static_dns() { + [ "$accept_dns" = false ] || return 0 + printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' > "$tmp/resolv" || return 1 + install_system_file "$tmp/resolv" "$root/etc/resolv.conf" none +} +install_tailscale() { + command -v tailscale >/dev/null 2>&1 && return 0 + if command -v ujust >/dev/null 2>&1; then + ujust install-tailscale + else + curl -fsSL --max-time 60 https://tailscale.com/install.sh -o "$tmp/install.sh" && + [ -s "$tmp/install.sh" ] && platform_root sh "$tmp/install.sh" + fi +} +join_tailnet() { + local session mode + session="$(bw_session)" + if [ -z "$session" ]; then + note 'PARTIAL: Tailscale not connected; vault locked or secrets disabled' + return 0 + fi + [ -n "$domain" ] || { echo 'tailscale_domain is missing' >&2; return 1; } + mode=sudo + [ "$(id -u)" != 0 ] || mode=root + BW_SESSION="$session" python3 "$DOTFILES_DIR/units/tailscale-connect.py" "$MACHINE" "$domain" "$mode" || return 1 + if [ -n "$routes" ]; then + platform_root tailscale up "--advertise-routes=$routes" --reset=false || return 1 + fi +} + +step 'Tailscale resolved configuration' configure_resolved +step 'Tailscale resolver link' configure_resolv_link +step 'Tailscale sysctl configuration' configure_sysctl +if [ "$resolved_changed" = 1 ]; then + if [ "$retired" = 1 ]; then note 'SKIPPED: resolved restart on retired VPS'; + else soft 'restart systemd-resolved' platform_root systemctl restart systemd-resolved; fi +fi +if [ "$sysctl_changed" = 1 ]; then step 'apply Tailscale sysctls' platform_root sysctl -p "$root/etc/sysctl.d/99-tailscale.conf"; fi + +if tailscale status >/dev/null 2>&1; then + step 'Tailscale advertised routes' configure_routes + step 'Tailscale hostname' platform_root tailscale set "--hostname=$MACHINE" + step 'Tailscale static DNS' configure_static_dns + if platform_desktop; then + soft 'Tailscale desktop operator' platform_root tailscale set "--operator=${USER:-$(id -un)}" + if [ "$retired" = 0 ]; then + soft 'Tailscale systray configuration' tailscale configure systray --enable-startup=systemd + soft 'Tailscale systray service' systemctl --user daemon-reload + soft 'Tailscale systray start' systemctl --user enable --now tailscale-systray + fi + else + desktop_rc=$? + [ "$desktop_rc" = 1 ] || step 'read desktop inventory group' false + fi +else + if [ "$retired" = 1 ]; then + note 'SKIPPED: Tailscale installation and rejoin on retired VPS' + else + step 'Tailscale installation' install_tailscale + if command -v tailscale >/dev/null 2>&1; then step 'Tailscale join' join_tailnet + else step 'Tailscale binary available' false; fi + fi +fi +finish diff --git a/units/termux-packages.conf b/units/termux-packages.conf new file mode 100644 index 00000000..8ac5af1b --- /dev/null +++ b/units/termux-packages.conf @@ -0,0 +1,6 @@ +# The former role defaults, plus tools needed before native units can run. +TERMUX_BOOTSTRAP=(python yq chezmoi) +TERMUX_PACKAGES=(openssh git nodejs fish vim nano termux-api curl unzip file e2fsprogs) +TERMUX_OPTIONAL=(mise) +TERMUX_NPM_GLOBAL=('@earendil-works/pi-coding-agent') +TERMUX_KEEPALIVE_PERIOD_MS=1800000 diff --git a/units/termux-packages.sh b/units/termux-packages.sh new file mode 100755 index 00000000..01ffeac6 --- /dev/null +++ b/units/termux-packages.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=units/platform-config.bash +. "$DOTFILES_DIR/units/platform-config.bash" +# shellcheck source=units/termux-packages.conf +. "$DOTFILES_DIR/units/termux-packages.conf" +# shellcheck source=units/package-config.bash +. "$DOTFILES_DIR/units/package-config.bash" + +if platform_group_contains termux_hosts; then :; else + rc=$? + if [ "$rc" = 1 ]; then note 'SKIPPED: not in termux_hosts'; else step 'read Termux inventory group' false; fi + finish +fi + +# Role defaults are overridden by group_vars/all and then host_vars, as before. +# No YAML parser is needed on a freshly bootstrapped phone without overrides. +load_overrides() { + local file key has value + for file in "$DOTFILES_DIR/group_vars/all.yml" "$DOTFILES_DIR/host_vars/$MACHINE.yml"; do + [ -f "$file" ] || continue + if ! grep -Eq '^termux_(pkgs|npm_global|keepalive_period_ms):' "$file"; then continue; fi + package_require_yq || return 1 + for key in termux_pkgs termux_npm_global; do + has="$(yq -r "has(\"$key\")" "$file")" || return 1 + if [ "$has" = true ]; then + package_list "$key" "$file" || return 1 + if [ "$key" = termux_pkgs ]; then TERMUX_PACKAGES=("${PACKAGE_LIST[@]}"); else TERMUX_NPM_GLOBAL=("${PACKAGE_LIST[@]}"); fi + fi + done + has="$(yq -r 'has("termux_keepalive_period_ms")' "$file")" || return 1 + if [ "$has" = true ]; then + value="$(yq -r '.termux_keepalive_period_ms' "$file")" || return 1 + TERMUX_KEEPALIVE_PERIOD_MS="$value" + fi + done + [[ "$TERMUX_KEEPALIVE_PERIOD_MS" =~ ^[0-9]+$ ]] +} +update_packages() { + local attempt + for attempt in 1 2 3; do + pkg update -y && return 0 + [ "$attempt" = 3 ] || sleep 5 + done + return 1 +} +install_packages() { + local pkg rc=0 + [ "$#" != 0 ] || return 0 + pkg install -y "$@" && return 0 + for pkg in "$@"; do + if ! pkg install -y "$pkg"; then + echo "Termux package unavailable or installation failed: $pkg" >&2 + rc=1 + fi + done + return "$rc" +} +ensure_sshd() { + pgrep -x sshd >/dev/null 2>&1 || sshd +} +schedule_keepalive() { + local pending + pending="$(termux-job-scheduler -p 2>/dev/null)" || : + case "$pending" in *'Pending Job 1'*) return 0 ;; esac + termux-job-scheduler --job-id 1 --period-ms "$TERMUX_KEEPALIVE_PERIOD_MS" --persisted true -s "$HOME/.termux/boot/start-sshd" +} + +check_termux_packages() { + local installed pkg line state name found + installed="$(dpkg-query -W -f='${db:Status-Status} ${binary:Package}\n' 2>/dev/null)" || { + echo 'Cannot inspect installed Termux packages' >&2; return 1; + } + for pkg in "$@"; do + found=0 + while IFS=' ' read -r state name; do + name="${name%%:*}" + [ "$state" != installed ] || [ "$name" != "$pkg" ] || found=1 + done <<< "$installed" + if [ "$found" = 0 ]; then note "WOULD install Termux package $pkg"; else note "ok Termux package $pkg"; fi + done +} +check_termux_boot() { + PYTHONDONTWRITEBYTECODE=1 python3 - "$DOTFILES_DIR/units/termux-start-sshd" "$HOME/.termux/boot" <<'PY' +import os, stat, sys +src, directory = sys.argv[1:] +dest = os.path.join(directory, 'start-sshd') +try: + with open(src, 'rb') as f: + expected = f.read() + try: + info = os.stat(directory) + directory_ok = stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o755 + except FileNotFoundError: + directory_ok = False + print(('ok ' if directory_ok else 'WOULD create or chmod ') + 'Termux boot directory') + try: + info = os.stat(dest) + if stat.S_ISREG(info.st_mode): + with open(dest, 'rb') as f: + same = f.read() == expected + same = same and stat.S_IMODE(info.st_mode) == 0o755 + else: + same = False + except FileNotFoundError: + same = False + print(('ok ' if same else 'WOULD install or chmod ') + 'Termux sshd boot script') +except OSError: + print('Cannot inspect Termux boot files', file=sys.stderr) + sys.exit(1) +PY +} +check_termux_sshd() { + local rc + pgrep -x sshd >/dev/null 2>&1; rc=$? + case "$rc" in + 0) note 'ok Termux sshd running' ;; + 1) note 'WOULD start Termux sshd' ;; + *) echo 'Cannot inspect Termux sshd process' >&2; return 1 ;; + esac +} +check_termux_keepalive() { + local pending + pending="$(termux-job-scheduler -p 2>/dev/null)" || { echo 'Cannot inspect Termux scheduled jobs' >&2; return 1; } + case "$pending" in + *'Pending Job 1'*) note 'ok Termux sshd keep-alive job exists (period not checked by apply)' ;; + *) note "WOULD schedule Termux sshd keep-alive every $TERMUX_KEEPALIVE_PERIOD_MS ms" ;; + esac +} +check_termux_overrides() { + local file rc + for file in "$DOTFILES_DIR/group_vars/all.yml" "$DOTFILES_DIR/host_vars/$MACHINE.yml"; do + [ -e "$file" ] || [ -L "$file" ] || continue + [ -f "$file" ] && [ -r "$file" ] || { echo 'Cannot inspect Termux package configuration' >&2; return 1; } + grep -Eq '^termux_(pkgs|npm_global|keepalive_period_ms):' "$file"; rc=$? + [ "$rc" -le 1 ] || { echo 'Cannot inspect Termux package configuration' >&2; return 1; } + done + load_overrides +} +if [ "${UNITS_CHECK:-0}" = 1 ]; then + note 'WOULD update Termux package cache (not inspected offline)' + if check_termux_overrides; then + step 'inspect Termux packages' check_termux_packages "${TERMUX_BOOTSTRAP[@]}" "${TERMUX_OPTIONAL[@]}" "${TERMUX_PACKAGES[@]}" + for npm_package in "${TERMUX_NPM_GLOBAL[@]}"; do + note "WOULD run npm install -g $npm_package (current registry version not inspected)" + done + step 'inspect Termux sshd keep-alive' check_termux_keepalive + else + step 'read Termux package configuration' false + fi + step 'inspect Termux boot files' check_termux_boot + step 'inspect Termux sshd' check_termux_sshd + finish +fi + +step 'Termux package cache' update_packages +step 'Termux bootstrap packages' install_packages "${TERMUX_BOOTSTRAP[@]}" +soft 'Termux optional mise' install_packages "${TERMUX_OPTIONAL[@]}" +config_ok=0 +if load_overrides; then + config_ok=1 + step 'Termux packages' install_packages "${TERMUX_PACKAGES[@]}" + for npm_package in "${TERMUX_NPM_GLOBAL[@]}"; do + step "Termux npm package $npm_package" npm install -g "$npm_package" + done +else + step 'read Termux package configuration' false +fi +step 'Termux boot directory' install -d -m 0755 "$HOME/.termux/boot" +step 'Termux sshd boot script' install_if_changed "$DOTFILES_DIR/units/termux-start-sshd" "$HOME/.termux/boot/start-sshd" 0755 +step 'Termux sshd running' ensure_sshd +[ "$config_ok" = 0 ] || step 'Termux sshd keep-alive' schedule_keepalive +finish diff --git a/units/termux-start-sshd b/units/termux-start-sshd new file mode 100755 index 00000000..d4709521 --- /dev/null +++ b/units/termux-start-sshd @@ -0,0 +1,5 @@ +#!/data/data/com.termux/files/usr/bin/sh +# Run at boot via Termux:Boot and periodically via Android's JobScheduler. +# Termux:Boot and Termux:API are separate companion apps. +termux-wake-lock +sshd diff --git a/units/zen-browser.sh b/units/zen-browser.sh new file mode 100755 index 00000000..7a9c6738 --- /dev/null +++ b/units/zen-browser.sh @@ -0,0 +1,316 @@ +#!/usr/bin/env bash +# Zen Browser desktop integration and policy. Firefox Accounts is a separate unit. +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [ "${UNITS_CHECK:-0}" = 1 ]; then export PYTHONDONTWRITEBYTECODE=1; fi + +if skipped_by_profile skip_zen_browser; then + note "SKIPPED: skip_zen_browser or vm-test" + finish +fi + +# Unlike is_desktop, do not mistake a missing or broken inventory reader for a +# non-desktop host. Explicit host and environment overrides take precedence. +desktop_status() { + local value="${DOTFILES_IS_DESKTOP:-}" + [ -n "$value" ] || value="$(host_var is_desktop)" + case "$value" in + 1|true) return 0 ;; + 0|false) return 1 ;; + esac + if ! command -v yq >/dev/null 2>&1; then + note "FAIL: yq is required to determine desktop inventory membership" >&2 + return 2 + fi + value="$(yq ".all.children.desktop.hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml")" || return 2 + case "$value" in true) return 0 ;; false) return 1 ;; *) + note "FAIL: could not determine desktop inventory membership" >&2; return 2 ;; + esac +} +desktop_status +status=$? +if [ "$status" = 1 ]; then + note "SKIPPED: not a desktop" + finish +elif [ "$status" != 0 ]; then + step "desktop inventory" false + finish +fi + +assets="$DOTFILES_DIR/units/zen-browser" +zen_base="$HOME/.var/app/app.zen_browser.zen/.zen" +flatpak_policy=/var/lib/flatpak/app/app.zen_browser.zen/current/active/files/zen/distribution/policies.json +check_mode="${UNITS_CHECK:-0}" + +update_dock() { + local runtime="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" + DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-unix:path=$runtime/bus}" \ + XDG_RUNTIME_DIR="$runtime" python3 - <<'PY' +import ast +import os +import subprocess +import sys +if os.environ.get('UNITS_CHECK') == '1': + sys.excepthook = lambda kind, value, traceback: print('Zen dock inspection failed (' + kind.__name__ + ')', file=sys.stderr) + +get = subprocess.run(['gsettings', 'get', 'org.gnome.shell', 'favorite-apps'], + capture_output=True, text=True, check=True) +value = get.stdout.strip() +if os.environ.get('UNITS_CHECK') == '1': + if value.startswith('@as '): + value = value[4:] + if not value: + raise ValueError('empty dock state') +apps = ast.literal_eval(value) if value else [] +if os.environ.get('UNITS_CHECK') == '1' and (not isinstance(apps, list) or not all(isinstance(app, str) for app in apps)): + raise ValueError('invalid dock state') +new = [app for app in apps if app not in { + 'org.mozilla.firefox.desktop', 'org.mozilla.Thunderbird.desktop'}] +if 'app.zen_browser.zen.desktop' not in new: + new.insert(0, 'app.zen_browser.zen.desktop') +if os.environ.get('UNITS_CHECK') == '1': + print('WOULD update GNOME dock' if new != apps else 'OK GNOME dock') +elif new != apps: + subprocess.run(['gsettings', 'set', 'org.gnome.shell', 'favorite-apps', repr(new)], check=True) +PY +} +if [ "$check_mode" = 1 ]; then + if [[ "${DBUS_SESSION_BUS_ADDRESS:-}" = autolaunch:* ]]; then + step "inspect GNOME dock (session bus unavailable)" false + else + step "inspect GNOME dock" update_dock + fi +else + soft "GNOME dock" update_dock +fi + +set_defaults() { + local mime rc=0 + for mime in x-scheme-handler/http x-scheme-handler/https x-scheme-handler/ftp text/html text/xml application/xhtml+xml application/xml; do + if command -v xdg-mime >/dev/null 2>&1; then + xdg-mime default app.zen_browser.zen.desktop "$mime" || rc=1 + else + gio mime "$mime" app.zen_browser.zen.desktop || rc=1 + fi + done + return "$rc" +} +check_defaults() { + local mime current rc=0 + for mime in x-scheme-handler/http x-scheme-handler/https x-scheme-handler/ftp text/html text/xml application/xhtml+xml application/xml; do + if command -v xdg-mime >/dev/null 2>&1; then + current="$(xdg-mime query default "$mime")" || { rc=1; continue; } + elif command -v gio >/dev/null 2>&1; then + current="$(LC_ALL=C gio mime "$mime")" || { rc=1; continue; } + current="${current%%$'\n'*}" + case "$current" in + 'Default application for '*) current="${current#*: }" ;; + 'No default applications for '*) current='' ;; + *) rc=1; continue ;; + esac + else + rc=1; continue + fi + if [ "$current" != app.zen_browser.zen.desktop ]; then + note "WOULD set default browser for $mime" + fi + done + return "$rc" +} +if [ "$check_mode" = 1 ]; then + inspect_path() { + python3 - "$1" "$2" <<'PY' +import os +import stat +import sys +try: + info = os.stat(sys.argv[1]) +except FileNotFoundError: + print('absent') +except OSError: + print('Zen path inspection failed', file=sys.stderr) + sys.exit(1) +else: + print('present' if sys.argv[2] == 'exists' or stat.S_ISDIR(info.st_mode) else 'absent') +PY + } + if app_present="$(inspect_path "$HOME/.var/app/app.zen_browser.zen" directory)"; then + if [ "$app_present" = present ]; then step "inspect default browser" check_defaults; fi + else + step "inspect Zen installation" false + fi + if ! profile_present="$(inspect_path "$zen_base/profiles.ini" exists)"; then + step "inspect Zen profiles.ini" false + finish + fi +else + if [ -d "$HOME/.var/app/app.zen_browser.zen" ]; then soft "default browser" set_defaults; fi + if [ -e "$zen_base/profiles.ini" ]; then profile_present=present; else profile_present=absent; fi +fi +if [ "$profile_present" = absent ]; then + note "SKIPPED: Zen profiles.ini not found" + finish +fi + +profile_path() { + python3 - "$zen_base" <<'PY' +import configparser +import os +import sys +if os.environ.get('UNITS_CHECK') == '1': + sys.excepthook = lambda kind, value, traceback: print('Zen profile inspection failed (' + kind.__name__ + ')', file=sys.stderr) + +base = sys.argv[1] +config = configparser.ConfigParser() +with open(os.path.join(base, 'profiles.ini'), encoding='utf-8') as f: + config.read_file(f) +profile = config['Profile0'] +path = profile['Path'] +if profile.get('IsRelative', '1') == '1': + path = os.path.join(base, path) +if not os.path.isdir(path): + raise SystemExit('Zen Profile0 directory does not exist: ' + path) +print(path) +PY +} + +install_preferences() { + local profile + profile="$(profile_path)" || return 1 + python3 - "$assets/user.js" "$profile/user.js" <<'PY' +import os +from pathlib import Path +import sys +import tempfile +if os.environ.get('UNITS_CHECK') == '1': + sys.excepthook = lambda kind, value, traceback: print('Zen preferences inspection failed (' + kind.__name__ + ')', file=sys.stderr) + +source, dest = map(Path, sys.argv[1:]) +managed = source.read_text(encoding='utf-8') +if os.environ.get('UNITS_CHECK') == '1': + try: + info = dest.stat() + except FileNotFoundError: + info = None + exists = info is not None +else: + exists = dest.exists() +old = dest.read_text(encoding='utf-8') if exists else '' +begin = '// BEGIN dotfiles Zen preferences\n' +end = '// END dotfiles Zen preferences\n' +if old == managed: + new = old # Existing installations from the Ansible role. +elif begin in old or end in old: + if old.count(begin) != 1 or old.count(end) != 1 or old.index(begin) > old.index(end): + raise SystemExit('Invalid managed Zen preferences markers in ' + str(dest)) + before, remainder = old.split(begin, 1) + _, after = remainder.split(end, 1) + new = before + begin + managed.rstrip('\n') + '\n' + end + after +else: + separator = '' if not old or old.endswith('\n') else '\n' + new = old + separator + begin + managed.rstrip('\n') + '\n' + end +if os.environ.get('UNITS_CHECK') == '1': + if new != old or info is None or info.st_mode & 0o7777 != 0o644: + print('WOULD update Zen profile preferences') + else: + print('OK Zen profile preferences') + raise SystemExit(0) +if new != old or not dest.exists(): + fd, temp = tempfile.mkstemp(dir=dest.parent, prefix='.user.js.') + try: + with os.fdopen(fd, 'w', encoding='utf-8') as f: + f.write(new) + os.chmod(temp, 0o644) + os.replace(temp, dest) + finally: + if os.path.exists(temp): + os.unlink(temp) +else: + os.chmod(dest, 0o644) +PY +} +step "Zen profile preferences" install_preferences + +# Merge managed policies into each existing file, preserving unrelated local +# policy settings and extension installs. Render before invoking sudo. +render_policy() { + python3 - "$1" "$assets/policies.json" "$2" <<'PY' +import json +from pathlib import Path +import os +import sys +if os.environ.get('UNITS_CHECK') == '1': + sys.excepthook = lambda kind, value, traceback: print('Zen policy inspection failed (' + kind.__name__ + ')', file=sys.stderr) + +existing, managed, output = map(Path, sys.argv[1:]) +with managed.open(encoding='utf-8') as f: + required = json.load(f) +if os.environ.get('UNITS_CHECK') == '1': + try: + info = existing.stat() + except FileNotFoundError: + info = None + exists = info is not None +else: + exists = existing.exists() +if exists: + with existing.open(encoding='utf-8') as f: + current = json.load(f) +else: + current = {} + +def merge(old, new): + if isinstance(old, dict) and isinstance(new, dict): + result = old.copy() + for key, value in new.items(): + result[key] = merge(result[key], value) if key in result else value + return result + if isinstance(old, list) and isinstance(new, list): + return old + [value for value in new if value not in old] + if isinstance(new, (dict, list)): + raise ValueError('Existing Zen policy has an incompatible structure') + return new + +if not isinstance(current, dict): + raise SystemExit('Existing policy must be a JSON object: ' + str(existing)) +rendered = json.dumps(merge(current, required), indent=2, ensure_ascii=False) + '\n' +if os.environ.get('UNITS_CHECK') == '1': + old = existing.read_text(encoding='utf-8') if exists else None + system = output == Path('system') + changed = (old != rendered or info is None or info.st_mode & 0o7777 != 0o644 + or (system and (info.st_uid != 0 or info.st_gid != 0))) + print(('WOULD update ' if changed else 'OK ') + ('Flatpak Zen policy' if system else 'user Zen policy')) +else: + output.write_text(rendered, encoding='utf-8') +PY +} +install_system_policy() { + if [ "$check_mode" = 1 ]; then render_policy "$flatpak_policy" system; return $?; fi + local tmp rc=0 + tmp="$(mktemp)" || return 1 + if render_policy "$flatpak_policy" "$tmp"; then + if [ "$(id -u)" = 0 ]; then + install -d -m 0755 -o root -g root "$(dirname "$flatpak_policy")" && + { { cmp -s "$tmp" "$flatpak_policy" && [ "$(stat -c '%a:%u:%g' "$flatpak_policy")" = '644:0:0' ]; } || install -m 0644 -o root -g root "$tmp" "$flatpak_policy"; } || rc=1 + else + sudo -n install -d -m 0755 -o root -g root "$(dirname "$flatpak_policy")" && + { { cmp -s "$tmp" "$flatpak_policy" && [ "$(stat -c '%a:%u:%g' "$flatpak_policy")" = '644:0:0' ]; } || sudo -n install -m 0644 -o root -g root "$tmp" "$flatpak_policy"; } || rc=1 + fi + else + rc=1 + fi + rm -f "$tmp" + return "$rc" +} +install_user_policy() { + if [ "$check_mode" = 1 ]; then render_policy "$HOME/.zen/distribution/policies.json" user; return $?; fi + local dest="$HOME/.zen/distribution/policies.json" tmp rc=0 + tmp="$(mktemp)" || return 1 + render_policy "$dest" "$tmp" && install -d -m 0755 "$(dirname "$dest")" && install_if_changed "$tmp" "$dest" 0644 || rc=1 + rm -f "$tmp" + return "$rc" +} +step "Flatpak Zen policy" install_system_policy +step "user Zen policy" install_user_policy +finish diff --git a/roles/zen_browser/files/policies.json b/units/zen-browser/policies.json similarity index 100% rename from roles/zen_browser/files/policies.json rename to units/zen-browser/policies.json diff --git a/roles/zen_browser/files/user.js b/units/zen-browser/user.js similarity index 100% rename from roles/zen_browser/files/user.js rename to units/zen-browser/user.js From b208f2beae03f5ac438c66c5de3f9e5a7939a95d Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:52:43 +0000 Subject: [PATCH 15/62] fix: isolate native migration CI fixtures Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- .github/workflows/ci.yml | 21 +++++++++++++++++ docs/src/roles/fleet_facts.md | 6 +++++ tests/conftest.py | 2 ++ tests/test_bluefin_unit.py | 2 +- tests/test_osquery_native_inventory.py | 32 ++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f88357c0..eaba53ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,6 +80,14 @@ jobs: "https://github.com/mikefarah/yq/releases/download/v4.47.2/yq_linux_$arch" -o /usr/local/bin/yq sudo chmod 755 /usr/local/bin/yq + # The hosted runner is not on the fleet tailnet. Keep Punjab's real + # inventory variables and server membership, but do not claim remote + # discovery or remote host health in this local integration smoke. + - name: Select local smoke inventory + run: | + yq -i '.all.hosts |= with_entries(select(.key == "punjab")) | + .all.children = {"server": {"hosts": {"punjab": null}}}' inventory.yml + - name: Install actual native osquery and private reporting run: | echo punjab | sudo tee /etc/dotfiles-machine >/dev/null @@ -99,6 +107,19 @@ jobs: test "$(systemctl show fleet-osquery-poll.timer -p UnitFileState --value)" = disabled test "$(systemctl show fleet-osquery-poll.service -p ActiveState --value)" = inactive + - name: Diagnose osquery smoke failure + if: ${{ failure() }} + run: | + sudo systemctl status fleet-osquery-poll.service fleet-osquery-report.service osqueryd.service --no-pager -l || true + sudo journalctl -b --no-pager -u fleet-osquery-poll.service -u fleet-osquery-report.service -u osqueryd.service -n 200 || true + if [ -f "$HOME/.local/state/fleet-osquery/latest.json" ]; then + echo 'Snapshot error reasons (no report contents):' + jq -r '(.error // empty), (.hosts[]? | select(.status == "error" or .status == "stale") | .reason // empty)' \ + "$HOME/.local/state/fleet-osquery/latest.json" | sort -u || true + fi + command -v tailscale || echo 'tailscale CLI not found' + systemctl is-active tailscaled.service || true + # Renovate may bump pi's npm deps unreviewed (CODEOWNERS carve-out); this # makes sure a PR can't *add* or remove one without a human. deps-guard: diff --git a/docs/src/roles/fleet_facts.md b/docs/src/roles/fleet_facts.md index 5f1f0fef..4b9fcaed 100644 --- a/docs/src/roles/fleet_facts.md +++ b/docs/src/roles/fleet_facts.md @@ -110,6 +110,12 @@ previous poller and applying the unit stops its timer and in-flight service; the last private snapshot remains for inspection. The manual fleet CLI still works from other machines and supports `--output PATH` for explicit private snapshots. +CI exercises the real local reporter and poller using a Punjab-only inventory: +hosted runners are not fleet tailnet members, so this is not a remote discovery +or fleet-health check. The full inventory's poller snapshot and unavailable +discovery behavior are covered separately by unit tests. On smoke failure, CI +prints service journals and sanitized snapshot error reasons, not report contents. + ### Baseline and local investigation First apply records the machine's **existing** state, not an audited known-safe diff --git a/tests/conftest.py b/tests/conftest.py index 89c5588f..e7738967 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,8 @@ def write_stub(bindir, name, body): """Drop an executable bash stub `name` into bindir.""" p = Path(bindir) / name + if p.is_symlink(): + p.unlink() p.write_text("#!/usr/bin/env bash\n" + body) p.chmod(p.stat().st_mode | stat.S_IXUSR) return p diff --git a/tests/test_bluefin_unit.py b/tests/test_bluefin_unit.py index 815e4646..5368cdbe 100644 --- a/tests/test_bluefin_unit.py +++ b/tests/test_bluefin_unit.py @@ -19,7 +19,7 @@ def desktop(tmp_path, stub_env): config = tmp_path / "config" (config / "host_vars").mkdir(parents=True) (config / "inventory.yml").write_text("all: {}\n") - (config / "roles").symlink_to(REPO / "roles", target_is_directory=True) + (config / "units").symlink_to(REPO / "units", target_is_directory=True) env.update(DOTFILES_DIR=str(config), DOTFILES_MACHINE="test-desktop", DOTFILES_IS_DESKTOP="true", BREW_PREFIX="/nonexistent", DBUS_SESSION_BUS_ADDRESS="") diff --git a/tests/test_osquery_native_inventory.py b/tests/test_osquery_native_inventory.py index 751dcb13..e1897df5 100644 --- a/tests/test_osquery_native_inventory.py +++ b/tests/test_osquery_native_inventory.py @@ -1,6 +1,7 @@ import importlib.util import json import sys +import shutil import pytest @@ -74,6 +75,37 @@ def test_sanitized_snapshot_preserves_exclusions_without_yaml_dependency(tmp_pat assert got["_meta"]["hostvars"]["kerala"]["osquery_supported"] is False +@pytest.mark.skipif(shutil.which("yq") is None, reason="yq is required to parse the real inventory") +def test_real_inventory_poller_snapshot_preserves_effective_hosts_and_exclusions(tmp_path): + renderer_spec = importlib.util.spec_from_file_location( + "osquery_poller_renderer", REPO / "units/osquery-poller-render.py") + renderer = importlib.util.module_from_spec(renderer_spec) + renderer_spec.loader.exec_module(renderer) + source = REPO / "inventory.yml" + expected = native.load_inventory(source) + snapshot = tmp_path / "inventory.yml" + put(snapshot, renderer.inventory(source)) + installed = native.load_inventory(snapshot) + + assert installed["_meta"]["hostvars"] == expected["_meta"]["hostvars"] + for group in ("vps", "test_fleet", "termux_hosts"): + assert fleet.group_hosts(installed, group) == fleet.group_hosts(expected, group) + + +@pytest.mark.skipif(shutil.which("yq") is None, reason="yq is required to parse the real inventory") +def test_real_inventory_unavailable_discovery_is_not_offline(monkeypatch): + inventory = native.load_inventory(REPO / "inventory.yml") + calls = [] + monkeypatch.setattr(fleet, "command_json", lambda command, label: calls.append(command) or {}) + result = fleet.collect_fleet(inventory, {}, identities={"punjab"}, + discovery_error="Tailscale could not run") + by_host = {host["inventory_host"]: host for host in result["hosts"]} + for host in ("mumbai", "dilli"): + assert by_host[host]["status"] == "error" + assert by_host[host]["reason"] == "Tailscale could not run" + assert not any(host["status"] == "offline" for host in result["hosts"]) + assert calls == [["sudo", "-n", "cat", fleet.REPORT_PATH]] + def test_host_gate_checks_nested_vps_and_rejects_unknown_machine(tmp_path, capsys): path = tmp_path / "inventory.yml" put(path, {"all": {"children": {"vps": {"children": { From e279eaa3b06baf1761d93785c7ee108b9ae0257c Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:02:16 +0000 Subject: [PATCH 16/62] fix: preserve onboarding identity and isolate dependent operations Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- Justfile | 17 ++++- bootstrap.sh | 28 ++++++--- docs/src/bitwarden.md | 4 +- docs/src/roles/github.md | 6 +- docs/src/roles/homebrew.md | 4 +- docs/src/roles/kube.md | 1 + docs/src/roles/ssh_keys.md | 2 + docs/src/units.md | 2 +- scripts/run-units.sh | 4 +- tests/test_bootstrap_units.py | 59 ++++++++++++++++++ tests/test_chezmoi_unit.py | 77 ++++++++++++++++++++++- tests/test_github_unit.py | 20 +++++- tests/test_kube_unit.py | 35 +++++++++++ tests/test_package_native_units.py | 46 ++++++++++++++ tests/test_ssh_native_units.py | 94 ++++++++++++++++++++++++++++ tests/test_tagged_apply.py | 69 +++++++++++++++++++++ units/chezmoi.sh | 42 +++++++------ units/github.sh | 28 +++++---- units/homebrew.sh | 25 +++++--- units/kube.sh | 32 ++++++---- units/package-config.bash | 21 +++++-- units/ssh-fleet.py | 99 +++++++++++++++++++++++++++--- 22 files changed, 627 insertions(+), 88 deletions(-) diff --git a/Justfile b/Justfile index 15f62121..acce1767 100644 --- a/Justfile +++ b/Justfile @@ -8,7 +8,7 @@ export PATH := env("HOME") / ".local/bin" + ":/home/linuxbrew/.linuxbrew/bin:" + # Resolve online fleet hosts: intersect tailscale online peers with inventory (excluding vps + self) _online_hosts := shell('python3 "$1"', justfile_directory() / "scripts/online_hosts.py") -_secret_tag_pattern := "secrets|ssh|ssh_keys|github|tailscale|kube|forgejo|registry|atuin|ai|omp|desktop|browser|zen_browser" +_secret_tag_pattern := "secrets|ssh|ssh_keys|github|tailscale|kube|forgejo|registry|atuin|ai|omp|desktop|browser|zen_browser|shell|dotfiles" # Apply all config to this machine (unlocks BW interactively if needed) apply *args: @@ -134,7 +134,20 @@ apply-remote name *args: if [ '$EXPECT_SESSION' = 1 ] && [ -z \"\${BW_SESSION:-}\" ]; then echo 'SSH server did not accept BW_SESSION; remote apply aborted.' >&2; exit 1 fi - [ -n \"\${BW_SESSION:-}\" ] && (umask 077 && printf '%s' \"\$BW_SESSION\" > /tmp/bw_session) + if [ -n \"\${BW_SESSION:-}\" ]; then + python3 -c ' + import os, tempfile + path = \"/tmp/bw_session\" + fd, temp = tempfile.mkstemp(prefix=\".bw_session.\", dir=\"/tmp\") + try: + with os.fdopen(fd, \"w\") as f: + f.write(os.environ[\"BW_SESSION\"]) + os.replace(temp, path) + finally: + if os.path.exists(temp): + os.unlink(temp) + ' || exit 1 + fi cd ~/.local/share/dotfiles && git pull --ff-only && just apply {{ args }} " just push-terminfo {{ name }} diff --git a/bootstrap.sh b/bootstrap.sh index f2da7e0f..38c2d6a9 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -318,7 +318,18 @@ setup_bitwarden() { echo "Bitwarden sync failed; secrets setup skipped." >&2 return 1 fi - (umask 077 && printf '%s' "$BW_SESSION" > /tmp/bw_session) || return 1 + python3 -c ' +import os, tempfile +path = "/tmp/bw_session" +fd, temp = tempfile.mkstemp(prefix=".bw_session.", dir="/tmp") +try: + with os.fdopen(fd, "w") as f: + f.write(os.environ["BW_SESSION"]) + os.replace(temp, path) +finally: + if os.path.exists(temp): + os.unlink(temp) +' || return 1 return 0 } @@ -335,18 +346,21 @@ run_phase2() { push_inventory_changes() { if [ "$INVENTORY_MODIFIED" != "true" ]; then return; fi - cd "$DOTFILES_DIR" + cd "$DOTFILES_DIR" || return $? # Switch to SSH remote now that GitHub auth is configured - git remote set-url origin git@github.com:hanthor/dotfiles.git + git remote set-url origin git@github.com:hanthor/dotfiles.git || return $? - git add inventory.yml "host_vars/$MACHINE_NAME.yml" 2>/dev/null || true + git add inventory.yml "host_vars/$MACHINE_NAME.yml" || return $? if git diff --cached --quiet; then - return # Nothing to commit (already pushed from another machine) + return 0 # Nothing to commit (already pushed from another machine) + else + local diff_rc=$? + [ "$diff_rc" -eq 1 ] || return "$diff_rc" fi - git commit -m "inventory: add $MACHINE_NAME ($MACHINE_TYPE)" - git push + git commit -m "inventory: add $MACHINE_NAME ($MACHINE_TYPE)" || return $? + git push || return $? echo "✓ Inventory pushed to GitHub." } diff --git a/docs/src/bitwarden.md b/docs/src/bitwarden.md index 1b80e0d2..4d4d68e2 100644 --- a/docs/src/bitwarden.md +++ b/docs/src/bitwarden.md @@ -8,7 +8,7 @@ All secrets live in [Bitwarden](https://bitwarden.com/) and are resolved at runt 2. The recipe validates the session and runs `bw sync` with a 15-second timeout before invoking `scripts/run-units.sh`. If resolution or sync fails it continues with `--no-secrets`. 3. Unit helpers validate `BW_SESSION` or `/tmp/bw_session` noninteractively, and honor `UNITS_NO_SECRETS=1`. Units handle unavailable sessions and items independently; inspect their `SKIPPED` and `PARTIAL` output. -For `just apply-tags` with a secret-bearing tag such as `kube`, `ai` or `omp`, the recipe resolves and syncs a session before running. If this fails it runs the selected units with secrets disabled and returns nonzero. Unrelated tags do not prompt. +For `just apply-tags` with a secret-bearing tag such as `kube`, `ai`, `omp`, `shell` or `dotfiles`, the recipe resolves and syncs a session before running. If this fails it runs the selected units with secrets disabled and returns nonzero. Unrelated tags do not prompt. ## Remote Apply @@ -19,7 +19,7 @@ For `just apply-tags` with a secret-bearing tag such as `kube`, `ai` or `omp`, t out it tries `bw login --apikey` with the `bw-api-key` item from your local vault; to unlock it pipes the master password (item `James Bitwarden`) over stdin, falling back to an interactive `bw unlock` over `ssh -t`. -2. Sends a resolved remote session through SSH `SendEnv=BW_SESSION`, not in the command line. The target SSH daemon must accept `BW_SESSION`; if it rejects a resolved session, the apply aborts. `apply-remote` also caches it in the remote's `/tmp/bw_session`. +2. Sends a resolved remote session through SSH `SendEnv=BW_SESSION`, not in the command line. The target SSH daemon must accept `BW_SESSION`; if it rejects a resolved session, the apply aborts. `apply-remote` also atomically replaces the remote's `/tmp/bw_session` with a mode-0600 file; it never writes through an existing symlink and aborts if caching fails. Bootstrap uses the same private replacement when caching a validated session. 3. The remote full apply, or a targeted apply with a secret-bearing tag, resolves and syncs a session before invoking native units. Other targeted applies do not prompt for one. **Caveat:** sessions are host-specific. If remote resolution fails, the target apply still attempts its own local session resolution and sync; if that fails, it runs units with secrets disabled, and targeted secret applies return nonzero. Each unit determines how to preserve or report existing data; consult its reference. diff --git a/docs/src/roles/github.md b/docs/src/roles/github.md index c8bf9686..a4502233 100644 --- a/docs/src/roles/github.md +++ b/docs/src/roles/github.md @@ -11,7 +11,7 @@ Logs `gh` in via a Bitwarden-stored PAT (only if not already authenticated), the 1. `github known_hosts`: ensure github.com's ed25519 host key is in `~/.ssh/known_hosts`. 2. `gh login`: skipped when `gh auth status` succeeds. Otherwise, with an unlocked vault, `bw get password github-token` → `gh auth login --with-token`. Locked vault or missing item prints a `PARTIAL:` line. -3. `register SSH keys`: needs a token with `admin:public_key` and `admin:ssh_signing_key` (otherwise a `PARTIAL:` line with the regenerate-PAT link). For each of authentication and signing, `gh ssh-key add ~/.ssh/id_ed25519.pub --title -auth|-signing` unless `gh ssh-key list` already shows the key with that type. If the list fails, the unit fails without registering either key. +3. `register SSH keys`: needs a token with `admin:public_key` and `admin:ssh_signing_key` (otherwise a `PARTIAL:` line with the regenerate-PAT link). Authentication and signing keys are listed separately using GitHub's API, so an account with no keys is supported. Each missing type is added with `gh ssh-key add ~/.ssh/id_ed25519.pub --title -auth|-signing`. A list or upload failure does not prevent attempting the other type, and the unit reports failure. GitHub CLI configuration, including `git_protocol: ssh`, is managed by chezmoi, not this unit. @@ -22,7 +22,7 @@ GitHub CLI configuration, including `git_protocol: ssh`, is managed by chezmoi, | `PARTIAL: gh not logged in; vault locked` | First run on this host with Bitwarden locked | `bw unlock`, then `just apply-tags github`, or `gh auth login` | | `PARTIAL: no 'github-token' item` | Vault has no PAT | Create a login item `github-token` with the PAT as the password | | `PARTIAL: GitHub token lacks admin:public_key / admin:ssh_signing_key` | PAT scopes | Regenerate the PAT with those scopes and update the item | -| `warn: failed to list GitHub SSH keys; registration skipped` | `gh ssh-key list` failed | Check GitHub access and re-run; no keys were added | +| `warn: failed to list GitHub SSH ... keys; registration skipped` | GitHub API lookup failed | Check GitHub access and re-run; the unverified key type was not added | | `warn: failed to upload SSH ... key` | `gh ssh-key add` failed | Inspect with `gh ssh-key list` and re-run | ## Verify @@ -34,6 +34,6 @@ gh ssh-key list | grep "$(cat /etc/dotfiles-machine)" # -auth and . diff --git a/docs/src/roles/homebrew.md b/docs/src/roles/homebrew.md index 0d3eabdb..edd83871 100644 --- a/docs/src/roles/homebrew.md +++ b/docs/src/roles/homebrew.md @@ -5,8 +5,8 @@ **Secrets needed:** No **Runs on:** glibc hosts; skips Termux and musl. -The native unit installs and manages Homebrew. It defaults to `/home/linuxbrew/.linuxbrew` (`BREW_PREFIX` overrides the prefix), bootstraps Mike Farah `yq`, and reads package lists from `group_vars/all.yml` and host configuration through `units/package-config.bash`. +The native unit installs and manages Homebrew. It defaults to `/home/linuxbrew/.linuxbrew` (`BREW_PREFIX` overrides the prefix), bootstraps Mike Farah `yq`, and reads package lists from `group_vars/all.yml` and host configuration through `units/package-config.bash`. Package units validate the `yq` implementation and can select the Homebrew binary when an incompatible `yq` precedes it on `PATH`. -It generates `~/Brewfile` with core packages and, on desktops, desktop taps, packages, extras and casks; VS Code extensions are included on Darwin. It taps repositories, attempts to trust them, runs `brew bundle install --no-upgrade`, and uninstalls `brew_blocked` formulae. A blocked package in an install list is a configuration error. `DOTFILES_UPGRADE=1` or `true` additionally requests a best-effort `brew upgrade`; installing Python `packaging` is also best effort. +It generates `~/Brewfile` with core packages and, on desktops, desktop taps, packages, extras and casks; VS Code extensions are included on Darwin. It taps repositories, attempts to trust them, runs `brew bundle install --no-upgrade`, and uninstalls `brew_blocked` formulae. If Brewfile deployment fails, bundle is skipped rather than using a stale file, but independent steps continue. A blocked package in an install list is a configuration error. `DOTFILES_UPGRADE=1` or `true` additionally requests a best-effort `brew upgrade`; installing Python `packaging` is also best effort. Ownership repair and installing a missing Debian/Ubuntu compiler toolchain may require root or non-interactive sudo. Other systems must supply a compiler if missing. The Homebrew installer itself is downloaded and run non-interactively. diff --git a/docs/src/roles/kube.md b/docs/src/roles/kube.md index 6aa5a921..20344a42 100644 --- a/docs/src/roles/kube.md +++ b/docs/src/roles/kube.md @@ -27,6 +27,7 @@ A missing or empty note prints `warn:` and leaves its existing file untouched. | `warn: kubeconfig ... missing in BW` | The note is unavailable or empty | Check the vault; if it needs seeding, run `just seed-kube` from a host with the intended home configs | | Configs deployed but auth fails when the home cluster is running | Cluster PKI may have rotated — local configs are stale | Re-run `just seed-kube` from a healthy host, then re-apply on consumers | | Unit prints `SKIPPED: not a desktop` on a host that should have it | `skip_kube: true` in `host_vars/.yml`, or host not in the `desktop` group | Adjust inventory or host_vars | +| `FAIL cannot inspect desktop inventory` | Missing `yq`, unreadable inventory, or malformed desktop hosts | Install `yq` and fix `inventory.yml`; no configs are changed while classification is unknown | ## Seeding BW diff --git a/docs/src/roles/ssh_keys.md b/docs/src/roles/ssh_keys.md index add3183d..931118d8 100644 --- a/docs/src/roles/ssh_keys.md +++ b/docs/src/roles/ssh_keys.md @@ -4,6 +4,8 @@ If the private key is absent, the unit requires an unlocked vault and a successful item listing before restoring the complete `@` SSH item or generating a new pair when no item exists. It checks restored and local key pairs and refuses conflicting keys. An existing local key is retained; if its public key differs from the vault item, the vault is left unchanged unless `SSH_KEYS_FORCE_PUSH=true` or `ssh_keys_force_push: true` explicitly permits rotation. +Passphrase-protected OpenSSH identities are not prompted for. Existing identities and vault restores are retained when the public key matches the encrypted key's public envelope, but the private contents cannot be verified without the passphrase. The envelope is unauthenticated, so the unit will not create or replace a Bitwarden key from that unverified identity, even with forced rotation. A missing, invalid or mismatched public key remains an error. + The unit preserves user entries outside its marked SSH config and `authorized_keys` blocks. Vault public keys and `extra_authorized_keys` are merged into the authorization block; previous managed entries remain by default. `SSH_KEYS_FORCE_AUTHORIZED=true` (or the corresponding host variable) permits removing obsolete entries only within that block. A locked vault, disabled secrets, or an empty vault key list leaves vault authorization unchanged. Host-key scans only replace an entry after a usable scan; failed scans leave existing entries intact. Existing signers are retained. `--no-secrets` skips vault access; a missing identity is deferred rather than generated. Config and other independent steps still run. The separate [ssh_mesh](ssh_mesh.md) unit adds committed fleet public keys and checks connectivity. diff --git a/docs/src/units.md b/docs/src/units.md index d5e9b400..fc01f8ca 100644 --- a/docs/src/units.md +++ b/docs/src/units.md @@ -29,6 +29,6 @@ Register a new executable `units/.sh` in `mise.toml`, `ALL_UNITS` and the ## chezmoi -`.chezmoiroot` points to `home/`, which contains the source files, `.chezmoi.toml.tmpl`, `.chezmoidata.yaml` and `.chezmoiscripts/`. `units/chezmoi.sh` applies shell, Git, Neovim and Atuin configuration with `--keep-going --force --no-tty apply --init`. It fails if chezmoi is unavailable (on Termux it first attempts a best-effort `pkg install`). Desktop detection honors `DOTFILES_IS_DESKTOP` or host `is_desktop`, then inventory membership; without `yq` the unit warns and assumes non-desktop. +`.chezmoiroot` points to `home/`, which contains the source files, `.chezmoi.toml.tmpl`, `.chezmoidata.yaml` and `.chezmoiscripts/`. `units/chezmoi.sh` applies shell, Git, Neovim and Atuin configuration with `--keep-going --force --no-tty apply --init`. It fails if chezmoi is unavailable (on Termux it first attempts a best-effort `pkg install`). Desktop detection honors `DOTFILES_IS_DESKTOP` or host `is_desktop`, then inventory membership. If the inventory lookup cannot be validated, including when Mike Farah `yq` is unavailable, the unit fails without applying files. **Back up and review existing dotfiles before first apply.** `--force` overwrites local edits to managed targets without prompting, including on subsequent runs. Even without `--force`, first adoption of an existing target can replace it. The source uses no `exact_` directories, so unmanaged files are not deleted by directory synchronization. Secrets are handled by units rather than chezmoi templates, so unavailable vault items do not abort unrelated file templates. `units/ssh-keys.sh` and `units/ssh-mesh.sh` manage SSH key lifecycle and authorization separately. diff --git a/scripts/run-units.sh b/scripts/run-units.sh index 61debd02..a9d6d7a5 100755 --- a/scripts/run-units.sh +++ b/scripts/run-units.sh @@ -131,7 +131,9 @@ elif selected browser-fxa; then echo "SKIPPED: browser-fxa requires a configured Zen profile" >&2 fi if [ "$ssh_ok" = 1 ]; then - later+=(ssh-mesh github) + # Both units update known_hosts; serialize them to avoid lost updates. + run_stage ssh-mesh || rc=1 + run_stage github || rc=1 else for unit in ssh-mesh github; do selected "$unit" && echo "SKIPPED: $unit requires a valid SSH identity" >&2; done fi diff --git a/tests/test_bootstrap_units.py b/tests/test_bootstrap_units.py index ab7f4756..219315ac 100644 --- a/tests/test_bootstrap_units.py +++ b/tests/test_bootstrap_units.py @@ -46,3 +46,62 @@ def test_unit_session_respects_no_secrets_and_keeps_session_off_argv(stub_env, t else: assert result.stdout == "test-session" assert log.read_text().splitlines() == ["unlock --check"] + + +@pytest.mark.parametrize("failure", ["remote", "add", "diff", "commit", "push"]) +def test_inventory_push_propagates_git_failures_inside_or(stub_env, tmp_path, failure): + bindir, env = stub_env + env.update(CALLS=str(tmp_path / "git-calls"), FAIL_GIT=failure, TEST_REPO=str(tmp_path)) + write_stub(bindir, "git", ''' +printf '%s\\n' "$1" >> "$CALLS" +if [ "$1" = "$FAIL_GIT" ]; then exit 7; fi +if [ "$1" = diff ]; then exit 1; fi +''') + script = r''' +source "$1/bootstrap.sh" --name newhost --type desktop +DOTFILES_DIR="$TEST_REPO" +INVENTORY_MODIFIED=true +rc=0 +push_inventory_changes || rc=$? +exit "$rc" +''' + result = subprocess.run(["bash", "-c", script, "bash", str(REPO)], env=env, capture_output=True, text=True) + assert result.returncode == 7 + calls = Path(env["CALLS"]).read_text().splitlines() + assert calls[-1] == failure + assert "Inventory pushed" not in result.stdout + + +@pytest.mark.parametrize("existing", ["insecure", "symlink", "directory"]) +def test_bootstrap_cache_is_private_and_never_follows_symlinks(stub_env, tmp_path, existing): + bindir, env = stub_env + cache = tmp_path / "bw_session" + target = tmp_path / "target" + target.write_text("untouched") + if existing == "insecure": + cache.write_text("old") + cache.chmod(0o666) + elif existing == "symlink": + cache.symlink_to(target) + else: + cache.mkdir() + write_stub(bindir, "bw", ''' +case "$1" in + status) printf '{"status":"unlocked"}\\n' ;; + unlock|sync) exit 0 ;; + *) exit 1 ;; +esac +''') + env["BW_SESSION"] = "private-session-sentinel" + bootstrap = tmp_path / "bootstrap.sh" + bootstrap.write_text((REPO / "bootstrap.sh").read_text().replace("/tmp/bw_session", str(cache)).replace('dir="/tmp"', f'dir="{tmp_path}"')) + result = subprocess.run(["bash", "-c", 'source "$1"; setup_bitwarden', "bash", str(bootstrap)], + env=env, capture_output=True, text=True) + assert (result.returncode != 0) is (existing == "directory") + assert target.read_text() == "untouched" + assert "private-session-sentinel" not in result.stdout + result.stderr + if existing != "directory": + assert not cache.is_symlink() + assert cache.read_text() == env["BW_SESSION"] + assert cache.stat().st_mode & 0o777 == 0o600 + assert not list(tmp_path.glob(".bw_session.*")) diff --git a/tests/test_chezmoi_unit.py b/tests/test_chezmoi_unit.py index f5552c03..752950e8 100644 --- a/tests/test_chezmoi_unit.py +++ b/tests/test_chezmoi_unit.py @@ -2,7 +2,9 @@ import subprocess from pathlib import Path -from conftest import REPO +import pytest + +from conftest import REPO, write_stub def test_missing_chezmoi_fails_with_install_guidance(stub_env): @@ -19,3 +21,76 @@ def test_missing_chezmoi_fails_with_install_guidance(stub_env): assert result.returncode != 0 assert "chezmoi not installed" in result.stdout assert any(hint in result.stdout for hint in ("just packages", "pkg install", "apk add")) + + +@pytest.mark.parametrize("check", [False, True]) +@pytest.mark.parametrize("failure", ["missing", "malformed", "command"]) +def test_unknown_desktop_never_runs_chezmoi(stub_env, tmp_path, check, failure): + bindir, env = stub_env + root = tmp_path / "repo" + (root / "host_vars").mkdir(parents=True) + log = tmp_path / "chezmoi.log" + write_stub(bindir, "chezmoi", 'echo "$*" >> "$LOG"\n') + write_stub(bindir, "yq", '''\ +if [ "$1" = --version ]; then echo 'github.com/mikefarah/yq'; exit 0; fi +if [ "$FAILURE" = command ]; then exit 1; fi +if [ "$FAILURE" = malformed ]; then echo '!!seq'; exit 0; fi +exit 1 +''') + env.update(DOTFILES_DIR=str(root), DOTFILES_MACHINE="unknown", LOG=str(log), + FAILURE=failure, UNITS_CHECK="1" if check else "0") + result = subprocess.run(["bash", str(REPO / "units/chezmoi.sh")], + env=env, capture_output=True, text=True) + assert result.returncode != 0 + assert "FAIL: cannot" in result.stdout + assert not log.exists() + + +@pytest.mark.parametrize("check", [False, True]) +@pytest.mark.parametrize("override,expected", [("true", "1"), ("false", "0")]) +def test_explicit_desktop_override_needs_no_inventory(stub_env, tmp_path, check, override, expected): + bindir, env = stub_env + log = tmp_path / "chezmoi.log" + write_stub(bindir, "chezmoi", 'echo "$DOTFILES_IS_DESKTOP $*" >> "$LOG"\n') + env.update(DOTFILES_DIR=str(tmp_path), DOTFILES_MACHINE="unknown", LOG=str(log), + DOTFILES_IS_DESKTOP=override, UNITS_CHECK="1" if check else "0") + result = subprocess.run(["bash", str(REPO / "units/chezmoi.sh")], + env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + assert log.read_text().startswith(expected + " ") + assert (" status" if check else " apply --init") in log.read_text() + + +def test_host_desktop_override_without_inventory(stub_env, tmp_path): + bindir, env = stub_env + (tmp_path / "host_vars").mkdir() + (tmp_path / "host_vars/laptop.yml").write_text("is_desktop: true\n") + log = tmp_path / "chezmoi.log" + write_stub(bindir, "chezmoi", 'echo "$DOTFILES_IS_DESKTOP" >> "$LOG"\n') + env.update(DOTFILES_DIR=str(tmp_path), DOTFILES_MACHINE="laptop", LOG=str(log)) + result = subprocess.run(["bash", str(REPO / "units/chezmoi.sh")], + env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + assert log.read_text() == "1\n" + + +@pytest.mark.parametrize("membership", ["true", "false", "invalid"]) +def test_inventory_membership_must_be_boolean(stub_env, tmp_path, membership): + bindir, env = stub_env + (tmp_path / "host_vars").mkdir() + log = tmp_path / "chezmoi.log" + write_stub(bindir, "chezmoi", 'echo "$DOTFILES_IS_DESKTOP" >> "$LOG"\n') + write_stub(bindir, "yq", '''\ +if [ "$1" = --version ]; then echo 'github.com/mikefarah/yq'; exit 0; fi +case "$*" in *'| tag'*) echo '!!map' ;; *) echo "$MEMBERSHIP" ;; esac +''') + env.update(DOTFILES_DIR=str(tmp_path), DOTFILES_MACHINE="laptop", LOG=str(log), + MEMBERSHIP=membership) + result = subprocess.run(["bash", str(REPO / "units/chezmoi.sh")], + env=env, capture_output=True, text=True) + if membership == "invalid": + assert result.returncode != 0 + assert not log.exists() + else: + assert result.returncode == 0, result.stdout + result.stderr + assert log.read_text() == ("1\n" if membership == "true" else "0\n") diff --git a/tests/test_github_unit.py b/tests/test_github_unit.py index d41fb0c3..d7a0e656 100644 --- a/tests/test_github_unit.py +++ b/tests/test_github_unit.py @@ -15,7 +15,15 @@ [ "$3" = "-t" ] || exit "${STUB_AUTHED:-0}" exit 0 ;; "auth login") cat > "$HOME/token"; exit 0 ;; - "ssh-key list") printf '%s' "$STUB_KEYS"; exit "${STUB_LIST_RC:-0}" ;; + "api --paginate") + case "$*" in *user/ssh_signing_keys*) kind=signing ;; *) kind=authentication ;; esac + printf '%s' "$STUB_KEYS" | awk -v kind="$kind" '$NF == kind { print $2, $3 }' + exit "${STUB_LIST_RC:-0}" ;; + "ssh-key add") + case "$*" in + *"--type authentication"*) [ "${STUB_FAIL_AUTH:-0}" = 0 ] || exit 1 ;; + *"--type signing"*) touch "$HOME/signing-registered" ;; + esac ;; "config set") mkdir -p "$HOME/.config/gh"; echo 'git_protocol: ssh' > "$HOME/.config/gh/config.yml" ;; esac exit 0 @@ -61,6 +69,12 @@ def test_only_the_missing_key_type_is_added(env): assert "--type signing" in c and "--type authentication" not in c +def test_signing_key_is_registered_even_if_authentication_key_fails(env): + result = run_unit("github", env, STUB_FAIL_AUTH="1") + assert result.returncode != 0 + assert Path(env["HOME"], "signing-registered").exists() + + def test_token_without_key_scopes_is_partial_and_adds_nothing(env): r = run_unit("github", env, STUB_SCOPES="repo, read:org") assert r.returncode == 0 and "PARTIAL:" in r.stdout and "ssh-key add" not in calls(env) @@ -106,7 +120,7 @@ def test_gh_config_is_left_for_chezmoi(env, authed): def test_failed_key_list_never_registers_even_with_partial_output(env, partial_output): r = run_unit("github", env, STUB_LIST_RC="1", STUB_KEYS=partial_output) assert r.returncode != 0 - assert "failed to list GitHub SSH keys; registration skipped" in r.stdout + assert "registration skipped" in r.stdout assert "FAIL register SSH keys" in r.stdout - assert "ssh-key list" in calls(env) + assert "gh api --paginate" in calls(env) assert "ssh-key add" not in calls(env) diff --git a/tests/test_kube_unit.py b/tests/test_kube_unit.py index ee8b50e5..49be9f5f 100644 --- a/tests/test_kube_unit.py +++ b/tests/test_kube_unit.py @@ -68,3 +68,38 @@ def test_skip_kube_host_var_wins_over_desktop(env, tmp_path): (repo / "units").symlink_to(Path(__file__).resolve().parent.parent / "units") r = run_unit("kube", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="phone") assert "SKIPPED:" in r.stdout and not (Path(env["HOME"]) / ".kube").exists() + + +@pytest.mark.parametrize("result", ["failure", "malformed", "invalid"]) +def test_unknown_desktop_inventory_fails_without_reading_secrets(env, tmp_path, result): + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "units").symlink_to(Path(__file__).resolve().parent.parent / "units") + bindir = Path(env["PATH"].split(":")[0]) + write_stub(bindir, "yq", '''\ +if [ "$LOOKUP" = failure ]; then exit 1; fi +case "$*" in + *'| tag'*) [ "$LOOKUP" = malformed ] && echo '!!seq' || echo '!!map' ;; + *) echo invalid ;; +esac +''') + r = run_unit("kube", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="unknown", + DOTFILES_IS_DESKTOP="", LOOKUP=result) + assert r.returncode != 0 + assert "FAIL" in r.stdout + assert not (Path(env["HOME"]) / ".kube").exists() + assert calls(env) == "" + + +@pytest.mark.parametrize("profile", ["skip_kube: true\n", "machine_profile: vm-test\n"]) +def test_profile_skips_without_desktop_lookup(env, tmp_path, profile): + repo = tmp_path / "repo" + (repo / "host_vars").mkdir(parents=True) + (repo / "host_vars/phone.yml").write_text(profile) + (repo / "units").symlink_to(Path(__file__).resolve().parent.parent / "units") + bindir = Path(env["PATH"].split(":")[0]) + write_stub(bindir, "yq", 'exit 1\n') + r = run_unit("kube", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="phone", + DOTFILES_IS_DESKTOP="") + assert r.returncode == 0 and "SKIPPED" in r.stdout + assert calls(env) == "" diff --git a/tests/test_package_native_units.py b/tests/test_package_native_units.py index 1f307c70..e8f0a6ad 100644 --- a/tests/test_package_native_units.py +++ b/tests/test_package_native_units.py @@ -153,6 +153,17 @@ def test_fresh_homebrew_bootstraps_yq_then_loads_config(package_env, tmp_path): assert 'brew "desktop-pkg"' in (Path(env["HOME"]) / "Brewfile").read_text() +def test_homebrew_uses_existing_valid_yq_without_installing(package_env, tmp_path): + _, env, log = package_env + bindir = isolate_bootstrap_path(env, tmp_path) + (bindir / "yq").symlink_to(env["YQ_BINARY"]) + result = run_unit("homebrew", env) + assert result.returncode == 0, result.stdout + result.stderr + args = [entry["args"] for entry in calls(log, "brew")] + assert ["install", "yq"] not in args + assert any(call[:2] == ["bundle", "install"] for call in args) + + def test_failed_yq_bootstrap_stops_before_bundle(package_env, tmp_path): _, env, log = package_env isolate_bootstrap_path(env, tmp_path) @@ -222,6 +233,41 @@ def test_brew_continues_after_tap_failure(package_env): assert ["uninstall", "--formula", "trash-cli"] in args +def test_failed_brewfile_deployment_skips_stale_bundle_but_continues(package_env): + _, env, log = package_env + bindir = Path(env["PATH"].split(os.pathsep)[0]) + brewfile = Path(env["HOME"]) / "Brewfile" + brewfile.write_text('brew "stale-package"\n') + write_stub(bindir, "install", 'exit 1\n') + result = run_unit("homebrew", env) + assert result.returncode != 0 + assert 'SKIPPED: Homebrew bundle' in result.stdout + assert brewfile.read_text() == 'brew "stale-package"\n' + args = [entry["args"] for entry in calls(log, "brew")] + assert not any(call[:2] == ["bundle", "install"] for call in args) + assert any(call[:1] == ["tap"] for call in args) + assert ["uninstall", "--formula", "trash-cli"] in args + + +def test_flatpak_selects_brew_yq_when_path_yq_is_incompatible(package_env): + _, env, log = package_env + bindir = Path(env["PATH"].split(os.pathsep)[0]) + write_stub(bindir, "yq", 'echo "another yq"; exit 0\n') + (Path(env["BREW_PREFIX"]) / "bin/yq").symlink_to(shutil.which("yq")) + result = run_unit("flatpak", env) + assert result.returncode == 0, result.stdout + result.stderr + assert calls(log, "flatpak") + + +def test_flatpak_rejects_incompatible_yq_even_when_it_exits_successfully(package_env): + _, env, log = package_env + bindir = Path(env["PATH"].split(os.pathsep)[0]) + write_stub(bindir, "yq", 'echo "not Mike Farah yq"; exit 0\n') + result = run_unit("flatpak", env) + assert result.returncode != 0 + assert not calls(log, "flatpak") + + def test_flatpak_system_scope_batch_fallback_and_wrappers(package_env): _, env, log = package_env env["FAIL_BATCH"] = "1" diff --git a/tests/test_ssh_native_units.py b/tests/test_ssh_native_units.py index ac134b9f..599a01b0 100644 --- a/tests/test_ssh_native_units.py +++ b/tests/test_ssh_native_units.py @@ -272,3 +272,97 @@ def test_invalid_vault_key_pair_is_not_installed_or_logged(fleet): assert "does not match" in result.stdout assert not (home / ".ssh/id_ed25519").exists() assert fleet["private"] not in result.stdout + result.stderr + + +@pytest.mark.parametrize("mismatch", [False, True]) +def test_restores_encrypted_vault_identity_without_prompting(fleet, tmp_path, mismatch): + source = tmp_path / "encrypted-vault-key" + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "test-passphrase", "-f", str(source)], + check=True, capture_output=True) + private = source.read_text() + public = source.with_suffix(".pub").read_text().strip() + home = fleet["home"] + item = json.loads((home / "item.json").read_text()) + item["sshKey"] = {"privateKey": private, "publicKey": fleet["other_public"] if mismatch else public} + (home / "item.json").write_text(json.dumps(item)) + (home / "items.json").write_text(json.dumps([item])) + result = execute(fleet) + if mismatch: + assert result.returncode != 0 + assert not (home / ".ssh/id_ed25519").exists() + else: + assert result.returncode == 0, result.stdout + result.stderr + assert (home / ".ssh/id_ed25519").read_text() == private + assert (home / ".ssh/id_ed25519.pub").read_text().strip() == public + assert not (home / "encoded-item").exists() + assert private not in result.stdout + result.stderr + + +@pytest.mark.parametrize("contents,parsed,valid", [ + ("---\n", "null", True), + ("# onboarding\n--- # empty\n", "null", True), + ("- unexpected\n", '["unexpected"]', False), + ("null\n", "null", False), +]) +def test_host_vars_accepts_empty_document_but_rejects_nonmapping(fleet, tmp_path, contents, parsed, valid): + root = tmp_path / "fleet-root" + (root / "host_vars").mkdir(parents=True) + (root / "host_vars/newhost.yml").write_text(contents) + (fleet["data"] / "newhost.yml.json").write_text(parsed) + script = ''' +import importlib.util, sys +spec = importlib.util.spec_from_file_location("ssh_fleet", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +module.Fleet() +''' + result = subprocess.run(["python3", "-c", script, str(REPO / "units/ssh-fleet.py")], + env={**fleet["env"], "DOTFILES_DIR": str(root)}, capture_output=True, text=True) + assert (result.returncode == 0) is valid + if not valid: + assert "invalid YAML mapping" in result.stderr + + +@pytest.mark.parametrize("mismatch", [False, True]) +@pytest.mark.parametrize("check", [False, True]) +def test_encrypted_identity_never_prompts_or_overwrites_vault(fleet, mismatch, check): + ssh = fleet["home"] / ".ssh" + ssh.mkdir() + private = ssh / "id_ed25519" + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "test-passphrase", "-f", str(private)], + check=True, capture_output=True) + public = ssh / "id_ed25519.pub" + if mismatch: + public.write_text(fleet["other_public"] + "\n") + private_before, public_before = private.read_bytes(), public.read_bytes() + result = execute(fleet, SSH_KEYS_FORCE_PUSH="true", **({"UNITS_CHECK": "1"} if check else {})) + if mismatch: + assert result.returncode != 0 + assert "local SSH private and public keys do not match" in result.stdout + else: + assert "encrypted SSH key envelope matches" in result.stdout + if not check: + assert result.returncode == 0, result.stdout + result.stderr + assert "Bitwarden key left unchanged" in result.stdout + assert private.read_bytes() == private_before and public.read_bytes() == public_before + assert not (fleet["home"] / "encoded-item").exists() + assert "test-passphrase" not in result.stdout + result.stderr + assert private_before.decode() not in result.stdout + result.stderr + + +def test_encrypted_identity_without_public_key_is_not_replaced(fleet): + ssh = fleet["home"] / ".ssh" + ssh.mkdir() + private = ssh / "id_ed25519" + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "test-passphrase", "-f", str(private)], + check=True, capture_output=True) + before = private.read_bytes() + public = ssh / "id_ed25519.pub" + public.unlink() + result = execute(fleet, SSH_KEYS_FORCE_PUSH="true") + assert result.returncode != 0 + assert "requires an existing public key" in result.stdout + assert private.read_bytes() == before + assert not public.exists() + assert not (fleet["home"] / "encoded-item").exists() + assert before.decode() not in result.stdout + result.stderr diff --git a/tests/test_tagged_apply.py b/tests/test_tagged_apply.py index 5b7dfe04..d022f2a8 100644 --- a/tests/test_tagged_apply.py +++ b/tests/test_tagged_apply.py @@ -115,3 +115,72 @@ def test_remote_secret_apply_rejects_lost_ssh_session(apply_env, drop_session): assert "sentinel-session" not in result.stdout + result.stderr if drop_session: assert "did not accept BW_SESSION" in result.stderr + + +@pytest.mark.parametrize("tag", ["shell", "dotfiles"]) +def test_secret_bearing_aggregate_tags_resolve_locally(apply_env, tag): + env, repo, log = apply_env + result = run_apply(env, repo, "apply-tags", tag) + assert result.returncode == 0, result.stderr + assert log.read_text().splitlines() == [ + "bw unlock --check", "resolve", "bw unlock --check", "bw sync", f"units --tags {tag}" + ] + assert "valid-session" not in result.stdout + result.stderr + + +@pytest.mark.parametrize("tag", ["shell", "dotfiles"]) +def test_secret_bearing_aggregate_tags_fail_closed_on_sync_error(apply_env, tag): + env, repo, log = apply_env + env["SYNC_FAIL"] = "1" + result = run_apply(env, repo, "apply-tags", tag) + assert result.returncode != 0 + assert log.read_text().splitlines()[-1] == f"units --tags {tag} --no-secrets" + + + +@pytest.mark.parametrize("tag", ["shell", "dotfiles"]) +def test_secret_bearing_aggregate_tags_resolve_remotely(apply_env, tag): + env, repo, log = apply_env + result = run_apply(env, repo, "apply-remote-tags", "phone", tag) + assert result.returncode == 0, result.stderr + assert log.read_text().splitlines() == ["resolve", "ssh"] + + +@pytest.mark.parametrize("existing", ["insecure", "symlink", "directory"]) +def test_remote_apply_cache_is_private_and_never_follows_symlinks(apply_env, tmp_path, existing): + env, repo, _ = apply_env + cache = tmp_path / "bw_session" + target = tmp_path / "target" + target.write_text("untouched") + if existing == "insecure": + cache.write_text("old") + cache.chmod(0o666) + elif existing == "symlink": + cache.symlink_to(target) + else: + cache.mkdir() + justfile = repo / "Justfile" + justfile.write_text(justfile.read_text().replace("/tmp/bw_session", str(cache)).replace( + r'dir=\"/tmp\"', f'dir=\\"{tmp_path}\\"')) + bindir = Path(env["HOME"]) / ".local/bin" + write_stub(repo / "scripts", "bw-resolve.sh", 'printf "sentinel-session"\n') + write_stub(bindir, "ssh", ''' +case "$*" in *sentinel-session*) exit 90 ;; esac +exec bash -c "${!#}" +''') + write_stub(bindir, "just", ''' +[ "$1" = push-terminfo ] && exit 0 +[ "${BW_SESSION:-}" = sentinel-session ] || exit 91 +touch "$HOME/remote-applied" +''') + result = subprocess.run([shutil.which("just"), "-f", str(justfile), "apply-remote", "phone"], + cwd=repo, env=env, capture_output=True, text=True) + assert (result.returncode != 0) is (existing == "directory") + assert (Path(env["HOME"]) / "remote-applied").exists() is (existing != "directory") + assert target.read_text() == "untouched" + assert "sentinel-session" not in result.stdout + result.stderr + if existing != "directory": + assert not cache.is_symlink() + assert cache.read_text() == "sentinel-session" + assert cache.stat().st_mode & 0o777 == 0o600 + assert not list(tmp_path.glob(".bw_session.*")) diff --git a/units/chezmoi.sh b/units/chezmoi.sh index fdbc5d10..4c54a796 100755 --- a/units/chezmoi.sh +++ b/units/chezmoi.sh @@ -3,29 +3,31 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=units/package-config.bash +. "$(dirname "${BASH_SOURCE[0]}")/package-config.bash" + +resolve_desktop() { + local hint="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" + case "$hint" in + 1|true) desktop=1 ;; + 0|false) desktop=0 ;; + '') + package_require_yq && package_desktop || return 1 + desktop="$PACKAGE_DESKTOP" + ;; + *) echo 'Invalid desktop classification' >&2; return 1 ;; + esac +} if [ "${UNITS_CHECK:-0}" = 1 ]; then if ! command -v chezmoi >/dev/null 2>&1; then note "FAIL: chezmoi not installed; cannot inspect managed files" exit 1 fi - desktop_hint="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" - case "$desktop_hint" in - 1|true) desktop=1 ;; - 0|false) desktop=0 ;; - *) - if ! command -v yq >/dev/null 2>&1; then - note "FAIL: yq required to inspect desktop membership" - exit 1 - fi - export DOTFILES_MACHINE="$MACHINE" - membership="$(yq '(.all.children.desktop.hosts // {}) | has(strenv(DOTFILES_MACHINE))' "$DOTFILES_DIR/inventory.yml" 2>/dev/null)" || { - note "FAIL: cannot inspect desktop membership" - exit 1 - } - case "$membership" in true) desktop=1 ;; false) desktop=0 ;; *) note "FAIL: invalid desktop membership"; exit 1 ;; esac - ;; - esac + if ! resolve_desktop; then + note "FAIL: cannot inspect desktop membership" + exit 1 + fi export DOTFILES_MACHINE="$MACHINE" DOTFILES_IS_DESKTOP="$desktop" # Supply the same data as .chezmoi.toml.tmpl without --init: older chezmoi # versions write even during a dry-run init. The config is a pipe, not a file. @@ -58,8 +60,10 @@ if ! command -v chezmoi >/dev/null 2>&1; then exit 1 fi -command -v yq >/dev/null 2>&1 || note "warn: yq missing, assuming non-desktop" -if is_desktop; then desktop=1; else desktop=0; fi +if ! resolve_desktop; then + note "FAIL: cannot determine desktop membership; chezmoi apply skipped" + exit 1 +fi export DOTFILES_MACHINE="$MACHINE" DOTFILES_IS_DESKTOP="$desktop" step "chezmoi apply" chezmoi --source "$DOTFILES_DIR" --keep-going --force --no-tty apply --init diff --git a/units/github.sh b/units/github.sh index 79b25198..b8182a46 100755 --- a/units/github.sh +++ b/units/github.sh @@ -71,10 +71,19 @@ gh_login() { printf '%s' "$tok" | gh auth login --with-token } -# register_key TYPE: add the SSH key as TYPE unless a `... TYPE` line already lists it. +# Query each key type separately: `gh ssh-key list` exits nonzero for an empty account. register_key() { - local type="$1" title="$MACHINE-$2" - grep -Eq "${type}\$" <<<"$key_lines" && return 0 + local type="$1" title="$MACHINE-$2" endpoint keys + case "$type" in + authentication) endpoint=user/keys ;; + signing) endpoint=user/ssh_signing_keys ;; + *) return 1 ;; + esac + if ! keys="$(gh api --paginate --jq '.[].key' "$endpoint" 2>/dev/null)"; then + note "warn: failed to list GitHub SSH $type keys; registration skipped" + return 1 + fi + if awk -v key="$key_body" '$2 == key { found=1 } END { exit !found }' <<<"$keys"; then return 0; fi if ! gh ssh-key add "$pub" --title "$title" --type "$type"; then note "warn: failed to upload SSH $type key for $MACHINE to GitHub" return 1 @@ -82,7 +91,7 @@ register_key() { } register_keys() { - local scopes key_body key_list + local scopes key_body scopes="$(gh auth status -t 2>&1 | grep -o 'Token scopes:.*' || true)" case "$scopes" in *admin:public_key*admin:ssh_signing_key*|*admin:ssh_signing_key*admin:public_key*) ;; @@ -92,15 +101,12 @@ register_keys() { esac [ -f "$pub" ] || { note "warn: $pub missing; no SSH key to register"; return 1; } key_body="$(awk '{print $2}' "$pub")" - if ! key_list="$(gh ssh-key list 2>/dev/null)"; then - note "warn: failed to list GitHub SSH keys; registration skipped" - return 1 - fi - key_lines="$(grep -F -- "$key_body" <<<"$key_list" || true)" - register_key authentication auth && register_key signing signing + local rc=0 + register_key authentication auth || rc=1 + register_key signing signing || rc=1 + return "$rc" } -key_lines="" step "github known_hosts" known_hosts if [ "${UNITS_NO_SECRETS:-0}" = 1 ]; then note "SKIPPED: GitHub authentication and SSH key registration (--no-secrets)" diff --git a/units/homebrew.sh b/units/homebrew.sh index 49d971d3..00466631 100755 --- a/units/homebrew.sh +++ b/units/homebrew.sh @@ -69,17 +69,14 @@ install_brew() { } ensure_yq() { - local version - if command -v yq >/dev/null 2>&1 && version="$(yq --version 2>/dev/null)" && [[ "$version" == *github.com/mikefarah/yq* ]]; then - return 0 - fi + if package_require_yq >/dev/null 2>&1; then return 0; fi echo 'Installing Mike Farah yq for package configuration' GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=credential.helper GIT_CONFIG_VALUE_0='' "$brew_bin" install yq || { echo 'Unable to install yq with Homebrew' >&2 return 1 } hash -r - if ! command -v yq >/dev/null 2>&1 || ! version="$(yq --version 2>/dev/null)" || [[ "$version" != *github.com/mikefarah/yq* ]]; then + if ! package_require_yq >/dev/null 2>&1; then echo 'Mike Farah yq is unavailable after Homebrew installation' >&2 return 1 fi @@ -173,7 +170,7 @@ check_brew_query() { } check_brew() { - local parent="${brew_prefix%/.linuxbrew}" owner uid version rc mode result pkg tap seen=' ' taps line info kind count + local parent="${brew_prefix%/.linuxbrew}" owner uid rc mode result pkg tap seen=' ' taps line info kind count if [ -d "$parent" ]; then if owner="$(stat -c %u "$parent" 2>/dev/null || stat -f %u "$parent" 2>/dev/null)" && uid="$(id -u)"; then [ "$owner" = "$uid" ] || note 'CHANGE: repair Homebrew directory ownership' @@ -185,7 +182,7 @@ check_brew() { note 'CHANGE: Homebrew compiler toolchain is missing' fi if [ ! -x "$brew_bin" ]; then note 'CHANGE: install Homebrew'; fi - if ! command -v yq >/dev/null 2>&1 || ! version="$(yq --version 2>/dev/null)" || [[ "$version" != *github.com/mikefarah/yq* ]]; then + if ! package_require_yq >/dev/null 2>&1; then echo 'Cannot inspect Homebrew configuration: Mike Farah yq is required' >&2 UNIT_FAILED=1; return fi @@ -297,9 +294,19 @@ else step 'Homebrew configuration' false finish fi -step 'Brewfile' deploy_brewfile +if deploy_brewfile; then + note 'ok Brewfile' + brewfile_ready=1 +else + step 'Brewfile' false + brewfile_ready=0 +fi step 'Homebrew taps' tap_and_trust -step 'Homebrew bundle' bundle +if [ "$brewfile_ready" = 1 ]; then + step 'Homebrew bundle' bundle +else + note 'SKIPPED: Homebrew bundle (Brewfile deployment failed)' +fi step 'blocked Homebrew packages' uninstall_blocked case "${DOTFILES_UPGRADE:-0}" in 1|true) soft 'Homebrew upgrade' "$brew_bin" upgrade ;; esac soft 'brew Python packaging' "$brew_prefix/bin/python3" -m pip install packaging --break-system-packages --quiet diff --git a/units/kube.sh b/units/kube.sh index 26c00b4f..e0194d0f 100755 --- a/units/kube.sh +++ b/units/kube.sh @@ -7,21 +7,27 @@ set -uo pipefail # shellcheck source=units/kube.conf . "$DOTFILES_DIR/units/kube.conf" -if [ "${UNITS_CHECK:-0}" = 1 ] && ! skipped_by_profile skip_kube; then - desktop="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" - case "$desktop" in - 1|true|0|false) ;; - *) - if ! command -v yq >/dev/null 2>&1 || ! desktop="$(yq ".all.children.desktop.hosts | has(\"$MACHINE\")" "$DOTFILES_DIR/inventory.yml" 2>/dev/null)"; then - note "FAIL cannot inspect desktop inventory (yq required)" - exit 1 - fi - case "$desktop" in true|false) ;; *) note "FAIL invalid desktop inventory result"; exit 1 ;; esac ;; - esac - DOTFILES_IS_DESKTOP="$desktop" +if skipped_by_profile skip_kube; then + note "SKIPPED: not a desktop (or skip_kube / vm-test)" + exit 0 fi -if ! is_desktop || skipped_by_profile skip_kube; then +desktop="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" +case "$desktop" in + 1|true|0|false) ;; + '') + if ! command -v yq >/dev/null 2>&1 || + ! kind="$(yq -r '.all.children.desktop.hosts | tag' "$DOTFILES_DIR/inventory.yml" 2>/dev/null)" || + [ "$kind" != '!!map' ] || + ! desktop="$(MACHINE="$MACHINE" yq -r '.all.children.desktop.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml" 2>/dev/null)"; then + note "FAIL cannot inspect desktop inventory (yq and valid inventory required)" + exit 1 + fi + case "$desktop" in true|false) ;; *) note "FAIL invalid desktop inventory result"; exit 1 ;; esac ;; + *) note "FAIL invalid desktop classification"; exit 1 ;; +esac + +if [ "$desktop" = 0 ] || [ "$desktop" = false ]; then note "SKIPPED: not a desktop (or skip_kube / vm-test)" exit 0 fi diff --git a/units/package-config.bash b/units/package-config.bash index 47302874..1a6134fb 100644 --- a/units/package-config.bash +++ b/units/package-config.bash @@ -1,12 +1,20 @@ # shellcheck shell=bash # Helpers for the package units. The repository YAML remains authoritative. package_require_yq() { - if ! command -v yq >/dev/null 2>&1; then - echo 'yq is required to read package and inventory configuration' >&2 - return 1 - fi + local candidate version + for candidate in "$(type -P yq || :)" "${BREW_PREFIX:-/home/linuxbrew/.linuxbrew}/bin/yq"; do + [ -n "$candidate" ] && [ -x "$candidate" ] || continue + if version="$("$candidate" --version 2>/dev/null)" && [[ "$version" == *github.com/mikefarah/yq* ]]; then + PACKAGE_YQ="$candidate" + return 0 + fi + done + echo 'Mike Farah yq is required to read package and inventory configuration' >&2 + return 1 } +yq() { "$PACKAGE_YQ" "$@"; } + # package_list KEY [FILE]: populate PACKAGE_LIST from a YAML sequence. Group # defaults are overridden by the host file; FILE selects an authoritative role # vars file instead (role vars take precedence over group and host vars). @@ -30,12 +38,15 @@ package_list() { # Resolve desktop membership; malformed or missing inputs fail. package_desktop() { - local value="${DOTFILES_IS_DESKTOP:-}" host="$DOTFILES_DIR/host_vars/$MACHINE.yml" has + local value="${DOTFILES_IS_DESKTOP:-}" host="$DOTFILES_DIR/host_vars/$MACHINE.yml" has kind if [ -z "$value" ] && [ -f "$host" ]; then has="$(yq -r 'has("is_desktop")' "$host")" || return 1 + case "$has" in true|false) ;; *) echo 'Invalid host desktop configuration' >&2; return 1 ;; esac if [ "$has" = true ]; then value="$(yq -r '.is_desktop' "$host")" || return 1; fi fi if [ -z "$value" ]; then + kind="$(yq -r '.all.children.desktop.hosts | tag' "$DOTFILES_DIR/inventory.yml")" || return 1 + [ "$kind" = '!!map' ] || { echo 'Invalid desktop inventory hosts' >&2; return 1; } value="$(MACHINE="$MACHINE" yq -r '.all.children.desktop.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml")" || return 1 fi case "$value" in 1|true) PACKAGE_DESKTOP=1 ;; 0|false) PACKAGE_DESKTOP=0 ;; *) echo 'Invalid desktop classification' >&2; return 1 ;; esac diff --git a/units/ssh-fleet.py b/units/ssh-fleet.py index 44f595a2..38c3b91a 100755 --- a/units/ssh-fleet.py +++ b/units/ssh-fleet.py @@ -29,7 +29,11 @@ def run(args, *, data=None, env=None, timeout=30): return subprocess.run(args, input=data, text=True, capture_output=True, env=env, timeout=timeout) -def yaml_file(path): +def yaml_file(path, *, allow_empty=False): + if allow_empty: + lines = [line.split("#", 1)[0].strip() for line in path.read_text().splitlines()] + if [line for line in lines if line] in ([], ["---"]): + return {} try: result = run(["yq", "-o=json", ".", str(path)]) except FileNotFoundError: @@ -121,6 +125,75 @@ def key_parts(key): return parts[:2] +def encrypted_public_key(private): + # OpenSSH exposes an unauthenticated public key in its encrypted envelope. + # Matching it catches a mismatched .pub, but cannot validate the ciphertext. + lines = private.strip().splitlines() + if len(lines) < 3 or lines[0] != "-----BEGIN OPENSSH PRIVATE KEY-----" or lines[-1] != "-----END OPENSSH PRIVATE KEY-----": + raise RuntimeError("cannot inspect encrypted SSH private key") + try: + blob = base64.b64decode("".join(lines[1:-1]), validate=True) + magic = b"openssh-key-v1\x00" + if not blob.startswith(magic): + raise ValueError() + pos = len(magic) + + def field(): + nonlocal pos + if pos + 4 > len(blob): + raise ValueError() + size = int.from_bytes(blob[pos:pos + 4], "big") + pos += 4 + if size > len(blob) - pos: + raise ValueError() + value = blob[pos:pos + size] + pos += size + return value + + cipher, kdf, options = field(), field(), field() + ciphers = { + b"aes128-ctr": (16, 0), b"aes192-ctr": (16, 0), b"aes256-ctr": (16, 0), + b"aes128-cbc": (16, 0), b"aes192-cbc": (16, 0), b"aes256-cbc": (16, 0), + b"3des-cbc": (8, 0), b"aes128-gcm@openssh.com": (16, 16), + b"aes256-gcm@openssh.com": (16, 16), b"chacha20-poly1305@openssh.com": (8, 16), + } + if cipher not in ciphers or kdf != b"bcrypt" or len(options) < 9 or pos + 4 > len(blob): + raise ValueError() + salt_len = int.from_bytes(options[:4], "big") + if not salt_len or len(options) != 8 + salt_len or not int.from_bytes(options[-4:], "big"): + raise ValueError() + count = int.from_bytes(blob[pos:pos + 4], "big") + pos += 4 + if count != 1: + raise ValueError() + public = field() + encrypted = field() + block_size, tag_size = ciphers[cipher] + if len(encrypted) < block_size or len(encrypted) % block_size or len(blob) - pos != tag_size: + raise ValueError() + return key_parts("ssh-ed25519 " + base64.b64encode(public).decode("ascii")) + except (ValueError, UnicodeError) as exc: + raise RuntimeError("cannot inspect encrypted SSH private key") from exc + + +def inspect_private_key(private, public, *, private_path=None): + try: + if private_path is not None: + result = run(["ssh-keygen", "-y", "-P", "", "-f", str(private_path)]) + if result.returncode: + raise RuntimeError("cannot validate SSH private key without prompting") + return key_parts(result.stdout), True + return derive_private_key(private), True + except RuntimeError: + if not public.exists(): + raise RuntimeError("encrypted or invalid SSH private key requires an existing public key") + parts = encrypted_public_key(private) + if key_parts(public.read_text()) != parts: + raise RuntimeError("local SSH private and public keys do not match") + print("PARTIAL: encrypted SSH key envelope matches; private contents cannot be verified without a passphrase") + return parts, False + + def ssh_data(item): data = item.get("sshKey") or {} if not isinstance(data, dict): @@ -154,13 +227,14 @@ def __init__(self): self.hosts = inventory.get("hosts") or {} self.vps = set(((inventory.get("children") or {}).get("vps") or {}).get("hosts") or {}) host_file = ROOT / "host_vars" / (MACHINE + ".yml") - self.host_vars = yaml_file(host_file) if host_file.exists() else {} + self.host_vars = yaml_file(host_file, allow_empty=True) if host_file.exists() else {} self.login = self.host_vars.get("user_login", self.vars["user_login"]) self.email = self.host_vars.get("user_email", self.vars["user_email"]) self.machines = self.vars["machines"] self.env = None self.items = None self.key_ready = False + self.key_verified = False self.key_deferred = False self.failed = False @@ -228,8 +302,15 @@ def ensure_key(self): try: with os.fdopen(fd, "w") as f: f.write(priv.rstrip("\n") + "\n") - derived = run(["ssh-keygen", "-y", "-f", name]) - if derived.returncode or key_parts(derived.stdout) != key_parts(pub): + derived = run(["ssh-keygen", "-y", "-P", "", "-f", name]) + if derived.returncode: + try: + restored_public = encrypted_public_key(priv) + except RuntimeError as exc: + raise RuntimeError("Bitwarden SSH key pair is invalid or does not match") from exc + else: + restored_public = key_parts(derived.stdout) + if restored_public != key_parts(pub): raise RuntimeError("Bitwarden SSH key pair is invalid or does not match") atomic(public, pub.rstrip("\n") + "\n", 0o644) os.replace(name, private) @@ -244,10 +325,7 @@ def ensure_key(self): if result.returncode: raise RuntimeError("ssh-keygen failed") private.chmod(0o600) - derived = run(["ssh-keygen", "-y", "-f", str(private)]) - if derived.returncode: - raise RuntimeError("cannot read existing private key") - derived_parts = key_parts(derived.stdout) + derived_parts, self.key_verified = inspect_private_key(private.read_text(), public, private_path=private) if not public.exists(): atomic(public, " ".join(derived_parts) + " " + MACHINE + "\n", 0o644) if key_parts(public.read_text()) != derived_parts: @@ -272,6 +350,9 @@ def sync_key(self): if force.lower() not in ("true", "1", "yes"): print("WARNING: local SSH key differs from Bitwarden; vault left unchanged (set SSH_KEYS_FORCE_PUSH=true for an intentional rotation)") return + if not self.key_verified: + print("PARTIAL: encrypted SSH key not verified; Bitwarden key left unchanged") + return fingerprint = run(["ssh-keygen", "-lf", str(public)]) if fingerprint.returncode or len(fingerprint.stdout.split()) < 2: raise RuntimeError("cannot determine SSH fingerprint") @@ -550,7 +631,7 @@ def ensure_key(self): return if not private.is_file(): raise RuntimeError(f"not a regular file: {private}") - derived = derive_private_key(private.read_text()) + derived, self.key_verified = inspect_private_key(private.read_text(), public) if stat.S_IMODE(private.stat().st_mode) != 0o600: print(f"would set {private} mode 0600") if not public.exists(): From a68deea177162f1ee846aea1d893c8f9b6bdfdc5 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:06:27 +0000 Subject: [PATCH 17/62] fix: preserve platform detection during native onboarding Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/roles/shell_fonts.md | 2 +- tests/test_bootstrap_units.py | 2 +- tests/test_fonts_unit.py | 8 ++++++++ units/fonts.sh | 33 ++++++++++++++------------------- units/package-config.bash | 4 +++- units/platform-config.bash | 2 +- 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/docs/src/roles/shell_fonts.md b/docs/src/roles/shell_fonts.md index 5cded07b..506b80a4 100644 --- a/docs/src/roles/shell_fonts.md +++ b/docs/src/roles/shell_fonts.md @@ -15,5 +15,5 @@ Installs the [JetBrains Mono Nerd Font](https://github.com/ryanoasis/nerd-fonts) ## Notes - Idempotent on the presence of the Regular `.ttf`; delete it to force a re-download. -- Desktop means: `DOTFILES_IS_DESKTOP`, else `is_desktop` in `host_vars`, else membership of the `desktop` group in `inventory.yml`. +- Desktop means: `DOTFILES_IS_DESKTOP`, else `is_desktop` in `host_vars`, else membership of the `desktop` group in `inventory.yml`. An invalid override or unreadable inventory fails the unit rather than reporting a non-desktop skip. - The `gnome` unit sets Ptyxis to `JetBrainsMono Nerd Font 12`, which depends on this. diff --git a/tests/test_bootstrap_units.py b/tests/test_bootstrap_units.py index 219315ac..f8cf1333 100644 --- a/tests/test_bootstrap_units.py +++ b/tests/test_bootstrap_units.py @@ -95,7 +95,7 @@ def test_bootstrap_cache_is_private_and_never_follows_symlinks(stub_env, tmp_pat env["BW_SESSION"] = "private-session-sentinel" bootstrap = tmp_path / "bootstrap.sh" bootstrap.write_text((REPO / "bootstrap.sh").read_text().replace("/tmp/bw_session", str(cache)).replace('dir="/tmp"', f'dir="{tmp_path}"')) - result = subprocess.run(["bash", "-c", 'source "$1"; setup_bitwarden', "bash", str(bootstrap)], + result = subprocess.run(["bash", "-c", 'source "$1" --name test --type desktop; setup_bitwarden', "bash", str(bootstrap)], env=env, capture_output=True, text=True) assert (result.returncode != 0) is (existing == "directory") assert target.read_text() == "untouched" diff --git a/tests/test_fonts_unit.py b/tests/test_fonts_unit.py index b7c4a020..cc446a42 100644 --- a/tests/test_fonts_unit.py +++ b/tests/test_fonts_unit.py @@ -38,3 +38,11 @@ def test_failed_download_fails_the_unit_without_a_partial_install(stub_env): r = run_unit("fonts", env, STUB_CURL_RC="22") assert r.returncode == 1 and "FAIL install JetBrains Mono Nerd Font" in r.stdout assert not (Path(env["HOME"]) / ".local/share/fonts/JetBrainsMono").exists() + + +def test_failed_desktop_lookup_does_not_skip_font_configuration(stub_env): + bindir, env = stub_env + write_stub(bindir, "yq", "exit 2\n") + result = run_unit("fonts", env, DOTFILES_IS_DESKTOP="", DOTFILES_MACHINE="unknown-test-host") + assert result.returncode != 0 + assert "curl" not in calls(env) diff --git a/units/fonts.sh b/units/fonts.sh index 1db2343c..c6bb31db 100755 --- a/units/fonts.sh +++ b/units/fonts.sh @@ -3,6 +3,8 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=units/platform-config.bash +. "$DOTFILES_DIR/units/platform-config.bash" dir="$HOME/.local/share/fonts" font="$dir/JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf" @@ -13,23 +15,12 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then exit 1 fi [ -d "$dir" ] || note "CHANGE: create font directory $dir" - desktop_hint="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" - case "$desktop_hint" in - 1|true) desktop=1 ;; - 0|false) desktop=0 ;; - *) - if ! command -v yq >/dev/null 2>&1; then - note "FAIL: yq required to inspect desktop membership" - exit 1 - fi - export DOTFILES_MACHINE="$MACHINE" - membership="$(yq '(.all.children.desktop.hosts // {}) | has(strenv(DOTFILES_MACHINE))' "$DOTFILES_DIR/inventory.yml" 2>/dev/null)" || { - note "FAIL: cannot inspect desktop membership" - exit 1 - } - case "$membership" in true) desktop=1 ;; false) desktop=0 ;; *) note "FAIL: invalid desktop membership"; exit 1 ;; esac - ;; - esac + if platform_desktop; then + desktop=1 + else + rc=$? + if [ "$rc" = 1 ]; then desktop=0; else note "FAIL: cannot inspect desktop membership"; exit 1; fi + fi if [ "$desktop" = 0 ]; then note "SKIPPED: not a desktop; no font install" elif [ -f "$font" ]; then @@ -41,8 +32,12 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then fi step "font directory" mkdir -p "$dir" -if ! is_desktop; then - note "SKIPPED: not a desktop; no font install" +if platform_desktop; then + : +else + rc=$? + if [ "$rc" = 1 ]; then note "SKIPPED: not a desktop; no font install" + else step "desktop membership" false; fi finish fi if [ -f "$font" ]; then diff --git a/units/package-config.bash b/units/package-config.bash index 1a6134fb..55164377 100644 --- a/units/package-config.bash +++ b/units/package-config.bash @@ -13,7 +13,9 @@ package_require_yq() { return 1 } -yq() { "$PACKAGE_YQ" "$@"; } +yq() { + if [ -n "${PACKAGE_YQ:-}" ]; then "$PACKAGE_YQ" "$@"; else command yq "$@"; fi +} # package_list KEY [FILE]: populate PACKAGE_LIST from a YAML sequence. Group # defaults are overridden by the host file; FILE selects an authoritative role diff --git a/units/platform-config.bash b/units/platform-config.bash index 1826220e..1d960d74 100644 --- a/units/platform-config.bash +++ b/units/platform-config.bash @@ -4,7 +4,7 @@ # for a missing group or an unrecognised hosts entry rather than guessing. platform_group_contains() { local group="$1" line in_children=0 in_group=0 in_hosts=0 group_seen=0 hosts_seen=0 found=1 - if command -v yq >/dev/null 2>&1; then + if type -P yq >/dev/null 2>&1; then local answer answer="$(GROUP="$group" MACHINE="$MACHINE" yq -r '.all.children[strenv(GROUP)].hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml")" || return 2 case "$answer" in true) return 0 ;; false) return 1 ;; *) return 2 ;; esac From 6b842b0543ae584a6735bbfb77aec87c65f5d952 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:09:55 +0000 Subject: [PATCH 18/62] docs: clarify offline and retired fleet services Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/desktop/dilli/README.md | 2 +- docs/src/roles/appflowy.md | 4 ++-- docs/src/roles/authentik.md | 4 ++-- docs/src/roles/n8n.md | 4 ++-- docs/src/vps/matrix/README.md | 2 +- docs/src/vps/telengana/README.md | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/src/desktop/dilli/README.md b/docs/src/desktop/dilli/README.md index 3a08f9df..e150005c 100644 --- a/docs/src/desktop/dilli/README.md +++ b/docs/src/desktop/dilli/README.md @@ -19,4 +19,4 @@ Secondary workstation in the hanthor fleet. ## Notes - [PaperWM](https://github.com/paperwm/PaperWM), Papershell, Copyous extensions synced from fleet -- Brew packages managed via dotfiles playbook +- Brew packages managed via native dotfiles units diff --git a/docs/src/roles/appflowy.md b/docs/src/roles/appflowy.md index 0b24d278..36753ec5 100644 --- a/docs/src/roles/appflowy.md +++ b/docs/src/roles/appflowy.md @@ -2,8 +2,8 @@ **Migrated to Kubernetes (2026-06-12).** -This Ansible role (Podman Quadlets on bihar) has been removed. AppFlowy Cloud now runs in the Talos K8s cluster. +The former Ansible role (Podman Quadlets on bihar) was removed. Its Kubernetes manifests are retained for the home Talos cluster, which is currently offline. See [`talos-k8s/appflowy/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/appflowy) for the commercial Helm chart deployment manifest and operational instructions. -Access: `https://appflowy.manatee-basking.ts.net` +Historical access: `https://appflowy.manatee-basking.ts.net` (unavailable while the cluster is offline). diff --git a/docs/src/roles/authentik.md b/docs/src/roles/authentik.md index ac7671a9..b5bcb116 100644 --- a/docs/src/roles/authentik.md +++ b/docs/src/roles/authentik.md @@ -2,8 +2,8 @@ **Migrated to Kubernetes (2026-06-12).** -This Ansible role (Podman Quadlets on bihar) has been removed. Authentik now runs in the Talos K8s cluster. +The former Ansible role (Podman Quadlets on bihar) was removed. Its Kubernetes manifests are retained for the home Talos cluster, which is currently offline. See [`talos-k8s/authentik/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/authentik) for the Helm-based deployment manifest and operational instructions. -Access: `https://auth.manatee-basking.ts.net` +Historical access: `https://auth.manatee-basking.ts.net` (unavailable while the cluster is offline). diff --git a/docs/src/roles/n8n.md b/docs/src/roles/n8n.md index dddcbdd5..635a9d01 100644 --- a/docs/src/roles/n8n.md +++ b/docs/src/roles/n8n.md @@ -2,8 +2,8 @@ **Migrated to Kubernetes (2026-06-12).** -This Ansible role (Podman Quadlet on bihar) has been removed. n8n now runs in the Talos K8s cluster. +The former Ansible role (Podman Quadlet on bihar) was removed. Its Kubernetes manifests are retained for the home Talos cluster, which is currently offline. See [`talos-k8s/n8n/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/n8n) for the plain-K8s deployment manifest and operational instructions. -Access: `https://n8n.manatee-basking.ts.net` +Historical access: `https://n8n.manatee-basking.ts.net` (unavailable while the cluster is offline). diff --git a/docs/src/vps/matrix/README.md b/docs/src/vps/matrix/README.md index 622afe36..6acc1cb2 100644 --- a/docs/src/vps/matrix/README.md +++ b/docs/src/vps/matrix/README.md @@ -15,7 +15,7 @@ Former VPS node in the fleet (`vps` group in `inventory.yml`) that hosted the - [Synapse](https://github.com/element-hq/synapse) + MAS on a single-node [Kubernetes](https://kubernetes.io/) install, with [PostgreSQL](https://www.postgresql.org/) as the database -- Managed by the fleet playbook (daily cron apply, no secrets/homebrew) +- Previously managed by the fleet playbook (daily cron apply, no secrets/homebrew) ## Retirement diff --git a/docs/src/vps/telengana/README.md b/docs/src/vps/telengana/README.md index 3e4dea0d..e8c279f3 100644 --- a/docs/src/vps/telengana/README.md +++ b/docs/src/vps/telengana/README.md @@ -11,7 +11,7 @@ Former VPS node in the fleet (`vps` group in `inventory.yml`). - Ubuntu 24.04 LTS, x86_64 - A single-node k3s hosting the tuna-os Hive -- Managed by the fleet playbook (daily cron apply, no secrets/homebrew) +- Previously managed by the fleet playbook (daily cron apply, no secrets/homebrew) ## Lessons carried forward From a3c3b94e114b6f3fc2eef7cf8fa4d04657190d14 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:20:35 +0000 Subject: [PATCH 19/62] test: provide verifiable device presence in enrollment fixture Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- tests/test_platform_native_units.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_platform_native_units.py b/tests/test_platform_native_units.py index 03a55570..9172cc68 100644 --- a/tests/test_platform_native_units.py +++ b/tests/test_platform_native_units.py @@ -263,8 +263,8 @@ def open_mock(req, timeout): "id": "device-suffix", "connectedToControl": False}, {"name": "laptop-1.example.ts.net", "id": "device-online", "connectedToControl": True}, {"name": "laptop-2.example.ts.net", "id": "self-123", "connectedToControl": False}, - {"name": "laptop-3.example.ts.net", "id": "device-foreign", "isExternal": True}, - {"name": "other.example.ts.net", "id": "device-other"}, + {"name": "laptop-3.example.ts.net", "id": "device-foreign", "isExternal": True, "connectedToControl": False}, + {"name": "other.example.ts.net", "id": "device-other", "connectedToControl": False}, ]}).encode()) if os.environ.get("FAIL_DELETE") == "1": raise urllib.error.HTTPError(url, 403, "secret-bearing response", None, None) From 547284a37c4f9bcb2df5bf00d2665a629eb6d270 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:30:57 +0000 Subject: [PATCH 20/62] Onboard Karnataka on Utah and correct historical fleet docs Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- CLAUDE.md | 21 +-- README.md | 11 +- docs/src/SUMMARY.md | 6 +- docs/src/cluster/hive.md | 12 +- docs/src/index.md | 11 +- docs/src/inventory.md | 13 +- docs/src/network.md | 15 ++- docs/src/roles/appflowy.md | 8 +- docs/src/roles/authentik.md | 8 +- docs/src/roles/gnome.md | 2 +- docs/src/roles/kube.md | 14 +- docs/src/roles/n8n.md | 8 +- docs/src/servers/home-k8s-images.md | 14 +- docs/src/servers/talos-k8s/bihar/README.md | 32 +---- docs/src/servers/talos-k8s/cluster.md | 125 ++++++++++-------- .../src/servers/talos-k8s/karnataka/README.md | 26 ++-- docs/src/talos-k8s.md | 31 +++-- inventory.yml | 11 +- scripts/tailnet.py | 5 +- tests/test_tailnet.py | 10 ++ tests/test_visual_units_check.py | 4 +- units/gnome.sh | 13 +- 22 files changed, 213 insertions(+), 187 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d1c22bf2..5b8250ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,23 +8,18 @@ Native Bash-unit dotfiles + infra for a personal fleet (desktops, servers, VPS). Three distinct concerns live here: 1. **Workstation config** (`units/`, `home/`, `host_vars/`, `group_vars/`) — shells, packages, browser, SSH, Tailscale, kubeconfig, etc. -2. **Talos K8s cluster IaC** (`talos-k8s/`) — manifests for the Bihar + Karnataka cluster. Detailed handbook in [`docs/src/servers/talos-k8s/cluster.md`](docs/src/servers/talos-k8s/cluster.md). +2. **Historical Talos K8s cluster IaC** (`talos-k8s/`) — retained home-cluster manifests and recovery reference in [`docs/src/servers/talos-k8s/cluster.md`](docs/src/servers/talos-k8s/cluster.md). 3. **TunaOS AWS account IaC** (`aws/`, OpenTofu) — the account is **TunaOS infrastructure**, not the personal fleet: punjab's instance, backups (S3), IAM, budgets, and (terminated, still in the code) the AWS Talos cluster's VPC/nodes/EIPs. Credits are exhausted: everything left here costs real money. Handbook: [`docs/src/servers/aws/README.md`](docs/src/servers/aws/README.md). -### Two Talos clusters, not one +### Cluster recovery and current hosts -- **Home** (`bihar` + `karnataka`) — on the LAN. Configs: `~/.kube/config`, `~/.talos/config`. - **Currently powered down and in storage; expected back.** Treat it as - temporarily offline, not decommissioned — don't delete its configs, Bitwarden - notes, or `talos-k8s/` manifests, and expect `just doctor`/`kubectl` against it - to fail until it returns. +- **Home:** Karnataka runs `ghcr.io/projectbluefin/utah:testing` and defaults to `multi-user.target`; it is in the fleet server inventory. Bihar was recently online but is currently unreachable; its OS and services are unverified. Retain the former home cluster's `~/.kube/config`, `~/.talos/config`, Bitwarden notes and `talos-k8s/` manifests as recovery material; don't assume the cluster is running or operate historical recovery commands on these machines. - **AWS** (`eu-north-1`, built 2026-08-27) — **terminated 2026-10-25** after credits were exhausted. It previously ran Matrix, the Hives and the CFP dashboard. The S3 bucket `hanthor-fleet-backups-*` and the himachal-1 mirror (`~/backups/tunaos-aws-2026-10-25`) were retained; verify individual AWS resources rather than assuming they are gone. OpenTofu still describes the old cluster and can propose recreating resources. Configs retained as recovery material: `~/.kube/config-aws-migration`, `~/.talos/config-aws-migration`. Handbook: [`docs/src/servers/aws-k8s/cluster.md`](docs/src/servers/aws-k8s/cluster.md). - **Spindle VPS** — Matrix, standalone LiveKit, School, Reef and Hub run on an Oracle Always Free A1 VM in `eu-frankfurt-1`, 2 OCPU / 12 GB, joined to the tailnet and locked down. The kit and runbook are in [`docs/src/servers/spindle-vps/`](docs/src/servers/spindle-vps/README.md); Oracle API key/config are in Bitwarden. Cloudflare DNS failover is manual, not automatic. -Neither cluster's nodes are managed by fleet units — Talos has no SSH. They are not in -`inventory.yml`; access is `talosctl`/`kubectl` only. +The former AWS Talos nodes are not in `inventory.yml`. Karnataka is an SSH-managed Utah fleet host, not a current Talos node. The two Hetzner VPSes it replaced (`matrix`, `telengana`) are retired but still powered on for burn-in. **Nothing on them may be restarted** — see the @@ -63,13 +58,9 @@ Groups in `inventory.yml`, `skip_*` flags and `machine_profile` influence indivi `talos-k8s/.gitignore` excludes `controlplane.yaml`, `worker.yaml`, and `talosconfig` from git. -## Talos cluster +## Historical home Talos cluster -Two nodes: **bihar** (control plane, Intel) and **karnataka** (worker, AMD Strix Halo APU). Talos `v1.13.2`, K8s `v1.36.1`, flannel CNI. AMD GPU exposed to Kubernetes via the Image Factory schematic. - -Before shutdown, workloads included Lemonade (AMD-optimized local AI), KubeVirt v1.8.2 + KubeVirt Manager, and the Tailscale Operator (Ingress to `*.manatee-basking.ts.net`). - -**The cluster handbook is [`docs/src/servers/talos-k8s/cluster.md`](docs/src/servers/talos-k8s/cluster.md)** — hardware, network, reinstall, troubleshooting. +The former Bihar control plane and Karnataka worker ran Talos `v1.13.2`, Kubernetes `v1.36.1`, Lemonade and KubeVirt. This is historical architecture, not current workload status. The [cluster handbook](docs/src/servers/talos-k8s/cluster.md) preserves recovery instructions and warns against applying them to repurposed nodes. ## TunaOS AWS account (`aws/`) diff --git a/README.md b/README.md index 0e51b60d..3e276d24 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,12 @@ From `inventory.yml`: | test-fleet-fedora, test-fleet-node2 | test_fleet | KubeVirt test VMs (`machine_profile: vm-test`) | | matrix, telengana | vps | **Retired** Hetzner VPSes, powered on for burn-in — restart nothing on them | -The home Talos cluster, **bihar** + **karnataka**, is not managed by fleet units -(access is via `talosctl`/`kubectl`): it is powered down and in storage, expected -back. The AWS Talos cluster was terminated on 2026-10-25. +**Bihar** was recently online but is currently unreachable; its OS and services +are unverified. **Karnataka** runs `projectbluefin/utah:testing` with +`multi-user.target` as its default and is listed in the server inventory. +The home Talos manifests, configs and Bitwarden notes are historical recovery +material, not evidence of their current state. The AWS Talos cluster was +terminated on 2026-10-25. [Spindle](docs/src/servers/spindle-vps/README.md), an OCI VPS in Frankfurt running Matrix, standalone LiveKit and the School, Reef and Hub Hives, is not listed in this inventory. The `llm` group is dormant (commented out in `inventory.yml`). @@ -85,7 +88,7 @@ Repo is public — no secrets in git, ever. Units fetch secrets from Bitwarden a Full handbook lives in [`docs/`](docs/) and is published at [reilly.asia/infra/handbook](https://reilly.asia/infra/handbook/) — run `just docs` to serve it locally. -- [Talos cluster handbook](docs/src/servers/talos-k8s/cluster.md) — Bihar + Karnataka (home, offline), Lemonade, KubeVirt +- [Home Talos cluster history](docs/src/servers/talos-k8s/cluster.md) — historical configuration and recovery reference - [Spindle VPS](docs/src/servers/spindle-vps/README.md) — Matrix, LiveKit and Hives on OCI - [TunaOS AWS cluster handbook](docs/src/servers/aws-k8s/cluster.md) — archived cluster history - [Adding a new machine](docs/src/onboarding.md) diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index ec275c67..f336d6ef 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -27,9 +27,9 @@ - [Kerala](desktop/kerala/README.md) - [Servers]() - [Home Kubernetes Images & ISOs](servers/home-k8s-images.md) - - [Talos K8s Cluster (offline)](servers/talos-k8s/cluster.md) - - [Bihar](servers/talos-k8s/bihar/README.md) - - [Karnataka](servers/talos-k8s/karnataka/README.md) + - [Home Talos Cluster (historical reference)](servers/talos-k8s/cluster.md) + - [Bihar (current state unverified)](servers/talos-k8s/bihar/README.md) + - [Karnataka (Utah)](servers/talos-k8s/karnataka/README.md) - [Goa](servers/goa/README.md) - [Spindle VPS (matrix.reilly.asia)](servers/spindle-vps/README.md) - [VPS]() diff --git a/docs/src/cluster/hive.md b/docs/src/cluster/hive.md index a63b4268..f9902cf8 100644 --- a/docs/src/cluster/hive.md +++ b/docs/src/cluster/hive.md @@ -36,12 +36,14 @@ ssh ubuntu@100.113.75.88 'docker ps --filter name=hives-' > **Historical reference only — do not apply or run the Kubernetes, image-build, > secret, restart, or state-reset instructions below.** The AWS Talos cluster > was terminated on 2026-10-25; its Kubernetes resources and CronJobs are not a -> current deployment. The home Talos cluster (bihar and karnataka) is powered -> down temporarily, not decommissioned. Preserve its configurations. The +> current deployment. Bihar was recently online but is currently unreachable; +> its OS and services are unverified. Karnataka runs Utah. Preserve the +> home cluster's configurations as historical recovery material, not proof of a +> current deployment. Do not run these commands against repurposed nodes. The > sections below describe older home-cluster and AWS configurations at different -> points in time, not the current Spindle service. See the -> [AWS cluster history](../servers/aws-k8s/cluster.md) and -> [historical cluster manifests](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/hive). +> points in time, not the current Spindle service; Kubernetes URLs are historical, +> not running endpoints. See the [AWS cluster history](../servers/aws-k8s/cluster.md) +> and [historical cluster manifests](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/hive). > Examples are retained for context, not as instructions for either cluster. The earlier home-cluster Hive used pi with DeepSeek, tmux agents, a governor, diff --git a/docs/src/index.md b/docs/src/index.md index cd8c0f32..9a546ac7 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -3,7 +3,7 @@ Native-unit dotfiles and infrastructure for a personal fleet. Three distinct concerns live here: 1. **Workstation config** — shells, packages, browser, SSH, Tailscale, kubeconfig, GNOME, and more -2. **Home Talos K8s cluster** — manifests for the Bihar + Karnataka cluster +2. **Home Talos history** — retained manifests and recovery material for the former Bihar + Karnataka cluster 3. **Hosted infrastructure** — OCI spindle and AWS account configuration; the AWS Talos cluster is terminated > **This handbook is public.** It covers architecture, decisions and runbooks. @@ -48,10 +48,13 @@ Fleet hosts, from `inventory.yml`: | matrix | vps | VPS | **Retired** — see the cutover runbook | | telengana | vps | VPS | **Retired** — see the cutover runbook | -Not managed by fleet units (Talos — `talosctl`/`kubectl` only): +Other hosts and historical infrastructure: -- **bihar** (control plane) + **karnataka** (worker, AMD GPU) — the home cluster. - Powered down and in storage, expected back. See the [cluster handbook](servers/talos-k8s/cluster.md). +- **Bihar** was recently online but is currently unreachable; its OS and services are unverified. +- **Karnataka** runs `projectbluefin/utah:testing`, defaults to `multi-user.target`, + and is listed in the server inventory. +- The [home Talos handbook](servers/talos-k8s/cluster.md), manifests, configs and + Bitwarden notes are historical recovery material, not proof of current state. - The [AWS Talos cluster](servers/aws-k8s/cluster.md) was terminated on 2026-10-25; its handbook is historical. [Spindle](servers/spindle-vps/README.md) is an OCI Frankfurt VPS running Matrix, diff --git a/docs/src/inventory.md b/docs/src/inventory.md index e35a0ffb..89c23850 100644 --- a/docs/src/inventory.md +++ b/docs/src/inventory.md @@ -28,6 +28,7 @@ server: punjab: # EC2 agent box (us-east-1; current instance status unverified) vm: # Local dev VM goa: # Raspberry Pi 5 (ARM) + karnataka: # Utah host; multi-user.target by default ``` Server roles: `server_hardening`, plus the service roles. @@ -71,14 +72,16 @@ termux_hosts: ``` ### `llm` (dormant) -Commented out in `inventory.yml` — its only member, karnataka, is now a Talos -worker. +Commented out in `inventory.yml`; its only listed member is karnataka. ### Not in the inventory -**bihar** and **karnataka** are Talos nodes (no SSH) and are managed with -`talosctl`/`kubectl` only. The home cluster is powered down and in storage, -expected back. The terminated AWS cluster's nodes were not in the inventory either. +**Bihar** was recently online but is currently unreachable; its OS and services +are unverified. It is not in the inventory. +The [home Talos reference](servers/talos-k8s/cluster.md), manifests, configs and +Bitwarden notes are retained for historical recovery, not evidence of either +machine's current state. The terminated AWS cluster's nodes were not in the +inventory either. ## Per-Machine Configuration diff --git a/docs/src/network.md b/docs/src/network.md index a1e137c0..dd6f05e3 100644 --- a/docs/src/network.md +++ b/docs/src/network.md @@ -10,8 +10,8 @@ crosswalk. | Name | Role | |------|------| | gateway | Router (DNS, DHCP) — see [Gateway](network/gateway/README.md) | -| bihar | Talos K8s control plane (powered down, expected back) — reserved address | -| karnataka | Talos K8s worker, AMD GPU (powered down, expected back) — reserved address | +| bihar | OS and services unverified; currently unreachable — reserved address | +| karnataka | Utah host with AMD GPU; default target `multi-user.target` — reserved address | | goa | Fleet control, Debian ARM (Raspberry Pi) — reserved address | | phone | Android phone (reserved address) | | printer | Network printer (DHCP) | @@ -20,10 +20,11 @@ crosswalk. | esp32 | Unidentified IoT device (static address) | | dilli | Laptop (WiFi, when online) | -## Fleet nodes (Tailscale MagicDNS) +## Fleet nodes and known names -Hosts are reached by MagicDNS name; `ansible_host` in `inventory.yml` holds the -tailnet address where one is needed. +For inventory hosts, `ansible_host` in `inventory.yml` holds the tailnet address +where one is needed. A listed name or reserved address does not establish that +the host or MagicDNS is currently reachable. | Hostname | LAN | Group | |----------|-----|-------| @@ -37,5 +38,5 @@ tailnet address where one is needed. | termux (ssh `:8022`) | phone | termux_hosts | | matrix | remote | vps — retired | | telengana | remote | vps — retired | -| bihar | reserved | Talos, not in inventory — offline | -| karnataka | reserved | Talos, not in inventory — offline | +| bihar | reserved | not in inventory — currently unreachable, OS and services unverified | +| karnataka | reserved | server — Utah; Tailscale `100.66.158.127`; default target `multi-user.target` | diff --git a/docs/src/roles/appflowy.md b/docs/src/roles/appflowy.md index 36753ec5..f2e61d2b 100644 --- a/docs/src/roles/appflowy.md +++ b/docs/src/roles/appflowy.md @@ -1,9 +1,9 @@ # appflowy -**Migrated to Kubernetes (2026-06-12).** +**Historical Kubernetes migration (2026-06-12).** -The former Ansible role (Podman Quadlets on bihar) was removed. Its Kubernetes manifests are retained for the home Talos cluster, which is currently offline. +The former Ansible role (Podman Quadlets on bihar) was removed. Its Kubernetes manifests are retained as historical recovery material; they do not establish that AppFlowy runs now. Bihar's OS and services are unverified, and Karnataka runs Utah. -See [`talos-k8s/appflowy/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/appflowy) for the commercial Helm chart deployment manifest and operational instructions. +See [`talos-k8s/appflowy/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/appflowy) for the historical commercial Helm chart manifest. Do not apply it to repurposed nodes. -Historical access: `https://appflowy.manatee-basking.ts.net` (unavailable while the cluster is offline). +Former Kubernetes URL: `https://appflowy.manatee-basking.ts.net` (historical, not a running endpoint). diff --git a/docs/src/roles/authentik.md b/docs/src/roles/authentik.md index b5bcb116..296e6ff6 100644 --- a/docs/src/roles/authentik.md +++ b/docs/src/roles/authentik.md @@ -1,9 +1,9 @@ # authentik -**Migrated to Kubernetes (2026-06-12).** +**Historical Kubernetes migration (2026-06-12).** -The former Ansible role (Podman Quadlets on bihar) was removed. Its Kubernetes manifests are retained for the home Talos cluster, which is currently offline. +The former Ansible role (Podman Quadlets on bihar) was removed. Its Kubernetes manifests are retained as historical recovery material; they do not establish that Authentik runs now. Bihar's OS and services are unverified, and Karnataka runs Utah. -See [`talos-k8s/authentik/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/authentik) for the Helm-based deployment manifest and operational instructions. +See [`talos-k8s/authentik/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/authentik) for the historical Helm-based manifest. Do not apply it to repurposed nodes. -Historical access: `https://auth.manatee-basking.ts.net` (unavailable while the cluster is offline). +Former Kubernetes URL: `https://auth.manatee-basking.ts.net` (historical, not a running endpoint). diff --git a/docs/src/roles/gnome.md b/docs/src/roles/gnome.md index b4e991f0..de91d549 100644 --- a/docs/src/roles/gnome.md +++ b/docs/src/roles/gnome.md @@ -5,7 +5,7 @@ **Secrets needed:** No **Runs on:** desktops, except `skip_gnome: true` or `machine_profile: vm-test`. -The unit installs `units/gnome/gnome-keybindings.ini` in `~/.config/dconf` and loads it into the invoking user's live session. A pending marker retains a changed load request if the session is unavailable. It configures the default Ptyxis profile for Homebrew fish and `JetBrainsMono Nerd Font 12` on a best-effort basis, and downloads the account avatar for installation by `units/gnome-avatar.py`; the avatar step requires root or non-interactive sudo. +The unit installs `units/gnome/gnome-keybindings.ini` in `~/.config/dconf` and loads it into the invoking user's live session. A pending marker retains a changed load request if the session is unavailable. It disables touchpad tap-to-click, configures the default Ptyxis profile for Homebrew fish and `JetBrainsMono Nerd Font 12` on a best-effort basis, and downloads the account avatar for installation by `units/gnome-avatar.py`; the avatar step requires root or non-interactive sudo. The unit copies the bundled Copyous, PaperShell and PaperWM extensions from `units/gnome/extensions/` to the user's extension directory, without deleting unrelated files. It best-effort loads `units/gnome/gnome-extensions-dconf.dump` and sets `gnome_enabled_extensions` from host variables or `group_vars/all.yml`. Setting the enabled list does not install every listed extension or guarantee compatibility or activation in the current GNOME Shell. The separate Bluefin unit also attempts extension installation. diff --git a/docs/src/roles/kube.md b/docs/src/roles/kube.md index 20344a42..515c3bcb 100644 --- a/docs/src/roles/kube.md +++ b/docs/src/roles/kube.md @@ -5,7 +5,7 @@ **Secrets needed:** Yes **Runs on:** Desktops only (`SKIPPED:` otherwise, or with `skip_kube: true` / `machine_profile: vm-test` in `host_vars`) -Pulls kube + Talos configs from Bitwarden for the home cluster (Bihar + Karnataka — powered down, expected back) and the historical [AWS cluster](../servers/aws-k8s/cluster.md), terminated 2026-10-25. Neither cluster is currently reachable; the AWS configs are retained separately. +Pulls retained kube + Talos configs from Bitwarden for the former home cluster and the historical [AWS cluster](../servers/aws-k8s/cluster.md), terminated 2026-10-25. Bihar was recently online but is currently unreachable; its OS and services are unverified. Karnataka runs Utah. Preserve the home configs and notes as recovery material; they do not prove a cluster is running or reachable. ## Steps @@ -23,9 +23,9 @@ A missing or empty note prints `warn:` and leaves its existing file untouched. | Symptom | Cause | Fix | |---|---|---| -| `kubectl` returns `Unable to connect to the server` | The home cluster is powered down, or the vault was locked and configs were never deployed | Home-cluster connectivity returns only after it is powered on; if configs are missing, unlock BW and re-run `just apply-tags kube` | +| `kubectl` returns `Unable to connect to the server` | The historical endpoint may no longer host the cluster, or configs may be missing | Do not assume powering on a machine restores the cluster; inspect the intended target and, if configs are missing, unlock BW and re-run `just apply-tags kube` | | `warn: kubeconfig ... missing in BW` | The note is unavailable or empty | Check the vault; if it needs seeding, run `just seed-kube` from a host with the intended home configs | -| Configs deployed but auth fails when the home cluster is running | Cluster PKI may have rotated — local configs are stale | Re-run `just seed-kube` from a healthy host, then re-apply on consumers | +| Configs deployed but auth fails against a deliberately restored cluster | Cluster PKI may have rotated — local configs are stale | Confirm the intended cluster and configs before re-seeding, then re-apply on consumers | | Unit prints `SKIPPED: not a desktop` on a host that should have it | `skip_kube: true` in `host_vars/.yml`, or host not in the `desktop` group | Adjust inventory or host_vars | | `FAIL cannot inspect desktop inventory` | Missing `yq`, unreadable inventory, or malformed desktop hosts | Install `yq` and fix `inventory.yml`; no configs are changed while classification is unknown | @@ -40,15 +40,15 @@ This packages `~/.kube/config` and `~/.talos/config` into BW notes named `kubeco ## How to verify -Both commands below require the home cluster to be powered on; they fail while -it remains in storage. The former AWS cluster cannot be queried. +These historical commands require a deliberately restored, reachable home +Talos cluster; do not assume one exists. The former AWS cluster cannot be queried. ```bash -kubectl get nodes # home cluster: bihar + karnataka (fails while it's powered down) +kubectl get nodes # historical home cluster talosctl --talosconfig ~/.talos/config get members ``` ## Notes - These files contain cluster PKI material; the repo's `.gitignore` excludes the cluster source files, and these caches live in `~/` only — never commit. -- The unit is gated by desktop membership and `skip_kube` / `machine_profile: vm-test`. Servers don't get a kubeconfig; the Talos cluster nodes are not in the inventory. +- The unit is gated by desktop membership and `skip_kube` / `machine_profile: vm-test`. Servers don't get a kubeconfig; the former Talos cluster nodes were not Ansible-managed. diff --git a/docs/src/roles/n8n.md b/docs/src/roles/n8n.md index 635a9d01..2c3dd105 100644 --- a/docs/src/roles/n8n.md +++ b/docs/src/roles/n8n.md @@ -1,9 +1,9 @@ # n8n -**Migrated to Kubernetes (2026-06-12).** +**Historical Kubernetes migration (2026-06-12).** -The former Ansible role (Podman Quadlet on bihar) was removed. Its Kubernetes manifests are retained for the home Talos cluster, which is currently offline. +The former Ansible role (Podman Quadlet on bihar) was removed. Its Kubernetes manifests are retained as historical recovery material; they do not establish that n8n runs now. Bihar's OS and services are unverified, and Karnataka runs Utah. -See [`talos-k8s/n8n/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/n8n) for the plain-K8s deployment manifest and operational instructions. +See [`talos-k8s/n8n/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s/n8n) for the historical Kubernetes manifest. Do not apply it to repurposed nodes. -Historical access: `https://n8n.manatee-basking.ts.net` (unavailable while the cluster is offline). +Former Kubernetes URL: `https://n8n.manatee-basking.ts.net` (historical, not a running endpoint). diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index d04a7f6d..6ccc293a 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -1,30 +1,30 @@ # Home Kubernetes Server Images -Bootable container images for the home K8s cluster (bihar, karnataka, starter). Built from **Fedora Hummingbird + GNOME desktop + K3s**, with graphical target disabled by default. +These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka currently runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` as its default; Bihar's current OS and services are unverified. Image build and installer acceptance are under repair: verify provenance and acceptance before installing or switching any host. ## Image Variants | Variant | Role | CPU/GPU | Image URI | Installer ISO | |---------|------|---------|-----------|----------------| -| **bihar** | K3s control plane | Intel | `ghcr.io/hanthor/home-k8s-bihar:latest` | [Release](https://github.com/hanthor/dotfiles/releases) | -| **karnataka** | K3s worker (AMD APU + GPU) | AMD Strix Halo | `ghcr.io/hanthor/home-k8s-karnataka:latest` | [Release](https://github.com/hanthor/dotfiles/releases) | -| **starter** | Disposable test/staging node | Any | `ghcr.io/hanthor/home-k8s-starter:latest` | [Release](https://github.com/hanthor/dotfiles/releases) | +| **bihar** | Proposed K3s control plane | Intel | `ghcr.io/hanthor/home-k8s-bihar:testing` | [Release](https://github.com/hanthor/dotfiles/releases) | +| **karnataka** | Proposed K3s worker | AMD Strix Halo | `ghcr.io/hanthor/home-k8s-karnataka:testing` | [Release](https://github.com/hanthor/dotfiles/releases) | +| **starter** | Disposable test/staging node | Any | `ghcr.io/hanthor/home-k8s-starter:testing` | [Release](https://github.com/hanthor/dotfiles/releases) | ## Image Composition -Each image includes: +The intended images include the following; existing published tags may contain older builds, so inspect their provenance first: - **Base:** `projectbluefin/utah` (Hummingbird + GNOME 51 desktop + Bluefin theming) - **K3s:** Kubernetes v1.36.1 + SELinux policy - **Storage:** iSCSI utils, NFS utils, Longhorn tools - **Networking:** NetworkManager, Tailscale, iptables-nft -- **GPU:** AMD firmware + amdgpu kernel driver (built into boot image via Talos Image Factory) +- **GPU:** AMD firmware + amdgpu kernel driver included by the image build - **Filesystems:** ext4, xfs, btrfs (all tools included) - **Default state:** Multi-user target (text console), GDM masked ## Obtaining Images -### Latest Release (Recommended) +### Released artifacts Download dated ISO releases from [GitHub Releases](https://github.com/hanthor/dotfiles/releases): - `{bihar,karnataka,starter}-installer.iso` — offline bootc-installer-tui ISOs diff --git a/docs/src/servers/talos-k8s/bihar/README.md b/docs/src/servers/talos-k8s/bihar/README.md index 374c95b9..269a0459 100644 --- a/docs/src/servers/talos-k8s/bihar/README.md +++ b/docs/src/servers/talos-k8s/bihar/README.md @@ -1,36 +1,16 @@ # Bihar -Kubernetes control plane. [Talos Linux](https://www.talos.dev/) node — not in `inventory.yml`, managed with `talosctl`/`kubectl` only. - -> **Powered down and in storage, expected back.** See the [cluster handbook](../cluster.md). +Bihar was recently online but is currently unreachable. Its current OS and +services are unverified. It is not listed in `inventory.yml`. ## Hardware - Arch: x86_64 (Intel) - Motherboard: ASRock - LAN: reserved address on the home /24 -- Role: Control plane (Talos K8s) -- Tailscale: `bihar` (MagicDNS) - -## OS - -[Talos Linux](https://www.talos.dev/) v1.13.2 ([Kubernetes](https://kubernetes.io/) v1.36.1) - -## Services - -Bihar no longer serves anything directly — the old per-host Caddy paths -(`bihar.manatee-basking.ts.net/grafana`, `/auth`, `/n8n`, …) predate Talos. -Cluster services are reached through the Tailscale Operator; see the -[ingress list in the cluster handbook](../cluster.md#tailscale--manatee-baskingtsnet). - -## Networking - -Tailscale runs as a Talos system extension, configured by -[`talos-k8s/networking/tailscale-bihar.yaml`](https://github.com/hanthor/dotfiles/blob/master/talos-k8s/networking/tailscale-bihar.yaml): - -- Advertises the home LAN /24 as a subnet route -- Tailscale DNS: disabled (`TS_ACCEPT_DNS=false`) -## See also +## Historical configuration -- [Talos cluster handbook](../cluster.md) +Bihar was the control plane of the home Talos cluster. The [cluster handbook](../cluster.md), +Talos manifests and configs, and Bitwarden notes are retained as historical +recovery material; they do not establish its current OS or services. diff --git a/docs/src/servers/talos-k8s/cluster.md b/docs/src/servers/talos-k8s/cluster.md index 5ed88c70..0219adb3 100644 --- a/docs/src/servers/talos-k8s/cluster.md +++ b/docs/src/servers/talos-k8s/cluster.md @@ -1,29 +1,31 @@ -# Cluster Handbook — Talos Linux on Bihar + Karnataka +# Historical Cluster Handbook — Talos Linux on Bihar + Karnataka -> **Status: powered down and in storage (as of 2026-08-27), expected back.** -> Temporarily offline, *not* decommissioned — keep its configs, Bitwarden notes -> (`kubeconfig` / `talosconfig`), and `talos-k8s/` manifests. `kubectl` and -> `just doctor` will fail against it until it returns. Matrix and the School, -> Reef and Hub Hives now run on the [OCI Spindle VPS](../spindle-vps/README.md). -> The [AWS Talos cluster](../aws-k8s/cluster.md) was terminated on 2026-10-25. +> **Historical architecture and recovery reference, not a current deployment.** +> Bihar was recently online but is currently unreachable; its OS and services +> are unverified. Karnataka runs `projectbluefin/utah:testing` and defaults to +> `multi-user.target`. Neither this handbook nor retained manifests establish +> what services either machine runs now. > -> **When it returns**, before applying anything under `talos-k8s/ai/`: -> run [`talos-k8s/ai/migrate-pv-path.sh`](https://github.com/hanthor/dotfiles/blob/master/talos-k8s/ai/migrate-pv-path.sh) -> first. `lemonade.yaml` already points its PVs at `/var/lib/…`, but the models -> are still under `/var/tmp/…` on karnataka. Applying the manifest first -> orphans them (#29). Then follow the prerequisites in -> [`talos-k8s/ai/README.md`](https://github.com/hanthor/dotfiles/blob/master/talos-k8s/ai/README.md) -> for the backup job and run `lemonade-warmup.sh` (#26, #27). +> **Do not run the reinstall, reset, bootstrap, configuration, migration, or +> workload commands below against repurposed nodes.** They can wipe disks or +> alter systems. They are historical examples for a separately planned recovery, +> after identifying the target and safeguarding data. Kubernetes service URLs +> below are historical, not running service endpoints. > -> **Home Hummingbird migration: prepared, not deployed.** The build/install and -> activation procedure is in [section 12](#12-hummingbirdbootc-migration-home-only). -> The home machines remain offline/in storage; the Talos recovery instructions -> below and all existing home/AWS credentials remain valid recovery material. - +> Preserve `talos-k8s/`, the home `kubeconfig` / `talosconfig` files and +> Bitwarden notes, and the separate AWS recovery material. Matrix and the School, +> Reef and Hub Hives are documented on the [OCI Spindle VPS](../spindle-vps/README.md). +> The [AWS Talos cluster](../aws-k8s/cluster.md) was terminated on 2026-10-25. +> +> For a deliberate future Talos recovery, inspect and back up the actual PV data +> before applying `talos-k8s/ai/lemonade.yaml`: the manifest uses `/var/lib/…`, +> while the old data was recorded under `/var/tmp/…`. See +> [`migrate-pv-path.sh`](https://github.com/hanthor/dotfiles/blob/master/talos-k8s/ai/migrate-pv-path.sh) +> and the [AI prerequisites](https://github.com/hanthor/dotfiles/blob/master/talos-k8s/ai/README.md); +> neither the location nor survival of those models is verified now. -> **Two-node [Talos Linux](https://www.talos.dev/) [Kubernetes](https://kubernetes.io/) cluster.** -> AMD Strix Halo GPU acceleration · [Lemonade](https://lemonade-sdk.github.io/) · [KubeVirt](https://kubevirt.io/) · [Tailscale](https://tailscale.com/) ingress. -> Everything you need to bring this cluster back from zero. +The following describes the former two-node Talos/Kubernetes cluster, its AMD +GPU workloads, and its recovery procedures as historical reference. ## Table of contents @@ -71,10 +73,10 @@ └──────────────────────────────────────────────────────────────┘ ``` -The cluster is **not diskless** — Talos installs itself onto `/dev/nvme0n1` on each node and persists. Reboots survive. +The former cluster was **not diskless** — Talos installed onto `/dev/nvme0n1` on each node and persisted across reboots. This does not establish current disk contents. -**Source of truth for manifests:** [`talos-k8s/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s) in this repo. -**Source of truth for secrets** (`talosconfig`, `controlplane.yaml`, `worker.yaml`, `kubeconfig`): Bitwarden, fetched onto your workstation by the `kube` role. +**Retained manifests:** [`talos-k8s/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s) in this repo. +**Retained secrets** (`talosconfig`, `controlplane.yaml`, `worker.yaml`, `kubeconfig`): Bitwarden, fetched onto workstations by the `kube` unit. Preserve them as recovery material. --- @@ -108,16 +110,16 @@ The cluster is **not diskless** — Talos installs itself onto `/dev/nvme0n1` on DHCP/DNS are served by the home router, which hands out fixed addresses from its address-reservation table — see [Gateway](../../network/gateway/README.md). Older revisions of this page said dnsmasq on bihar; that predates Talos, which can't run it on the host. -| Host | Address | Role | +| Host | Address | Former role | |-------------|----------|----------------------------| | bihar | reserved | K8s control plane, home services | | karnataka | reserved | K8s worker, LLM host | | goa | reserved | Raspberry Pi, fleet control | | kvm | reserved | NanoKVM (plugged into karnataka) | -The commands below use `` / `` for the nodes' LAN -addresses. Run `just inventory` to rescan with nmap and get the current values -(they are also the endpoints in the talosconfig). +The historical commands use `` / `` for the nodes' LAN +addresses. The retained talosconfig endpoints and router reservations are not +evidence that a Talos API is present at those addresses now. ### Internet — data-capped @@ -130,9 +132,9 @@ until the next billing cycle. ### Tailscale — `manatee-basking.ts.net` -Cluster ingress goes through the [Tailscale Operator](https://tailscale.com/kb/1236/kubernetes-operator). Services with an `Ingress` resource targeting `tailscale` ingressClassName get a `.manatee-basking.ts.net` URL automatically. +The former cluster used the [Tailscale Operator](https://tailscale.com/kb/1236/kubernetes-operator) for ingress. The following URLs are historical, not running Kubernetes services. -| Service | URL | +| Former service | Historical URL | |---------------------|-----| | Cluster homepage | `https://home.manatee-basking.ts.net` | | Lemonade (AI) | `https://lemonade.manatee-basking.ts.net/v1` | @@ -159,14 +161,14 @@ Cluster ingress goes through the [Tailscale Operator](https://tailscale.com/kb/1 Talos ships with an immutable root filesystem, so kernel modules cannot be added at runtime. The AMD GPU driver and the Longhorn iSCSI tooling are **baked into the boot image** via the [Talos Image Factory](https://factory.talos.dev). -**Schematic ID (current, 2026-06-11):** `3a33ec6dfc8cfd61d2a3db3caf97894f31e913952d71ce3c3fbbe565a3f08339` +**Historical schematic ID (2026-06-11):** `3a33ec6dfc8cfd61d2a3db3caf97894f31e913952d71ce3c3fbbe565a3f08339` **Extensions baked in:** - `siderolabs/amdgpu` — AMD GPU kernel driver - `siderolabs/iscsi-tools` — iscsid, for Longhorn block storage - `siderolabs/util-linux-tools` — fstrim/nsenter, for Longhorn -Both nodes run this schematic. Earlier schematics were `e5912b95…` (amdgpu +Both nodes formerly used this schematic. Earlier schematics were `e5912b95…` (amdgpu only) and, in older docs, `b6ab12e…`. Storage details: [`talos-k8s/longhorn/`](https://github.com/hanthor/dotfiles/blob/master/talos-k8s/longhorn/README.md). ```yaml @@ -205,6 +207,8 @@ The mount is automatically propagated into the kubelet namespace. ## 5. Cluster bootstrap +> **Historical example:** do not apply these configs or bootstrap repurposed nodes. + Configs are generated once with `talosctl gen config`, edited, and applied to each node. The generated files contain **cluster PKI secrets** (machine token, bootstrap token, shared secret) and are **not committed** — they live in Bitwarden. ### One-time generation @@ -272,6 +276,8 @@ resources: ## 7. Production workloads +The following workloads and addresses describe the former cluster, not currently running services. + All manifests live in [`talos-k8s/`](https://github.com/hanthor/dotfiles/tree/master/talos-k8s), organized by purpose: ``` @@ -309,13 +315,13 @@ Key env overrides (RDNA 3.5 needs special handling): | `HSA_XNACK` | `1` | Enables unified memory | | `HSA_FORCE_FINE_GRAIN_PCIE` | `1` | Fine-grained VM access | -**Access:** +**Former access:** - LAN: `http://:31305/v1` (NodePort) - Tailscale: `https://lemonade.manatee-basking.ts.net/v1` -Model weights cached on `karnataka:/var/tmp/lemonade-cache` (HuggingFace) and `/var/tmp/lemonade-models` (llama models) via local-storage PersistentVolumes. +Model weights were cached under `karnataka:/var/tmp/lemonade-cache` (HuggingFace) and `/var/tmp/lemonade-models` (llama models) via local-storage PersistentVolumes; their current state is unverified. -**Web UI:** Point a browser at `http://:31305` for the built-in control panel to download models, configure endpoints, and test chat/vision/image generation. +The former web UI was at `http://:31305`; this is not a running-service claim. ### KubeVirt v1.8.2 @@ -336,9 +342,9 @@ Manifest: deployed from upstream bundle. Also see [`talos-k8s/infrastructure/`]( kubectl apply -f https://raw.githubusercontent.com/kubevirt-manager/kubevirt-manager/main/kubernetes/bundled.yaml ``` -Reachable at `https://kubevirt-manager.manatee-basking.ts.net` via the Tailscale ingress (manifest in cluster — not in this repo since it's a stock bundle). +Formerly reachable at `https://kubevirt-manager.manatee-basking.ts.net` via Tailscale ingress (stock bundle, not in this repo). This URL is historical. -Port-forward as fallback: +Former port-forward fallback: ```bash kubectl port-forward svc/kubevirt-manager 8080:8080 -n kubevirt-manager @@ -399,9 +405,11 @@ The operator namespace is `tailscale`. Auth keys are configured during operator ## 9. Day-to-day operations +These commands are retained for a deliberately restored Talos cluster; they do not establish that one exists now. Do not reboot or change repurposed machines. + ### From your workstation -The `kube` role syncs `~/.kube/config` and `~/.talos/config` from Bitwarden, so on any desktop: +The `kube` unit syncs the retained `~/.kube/config` and `~/.talos/config` from Bitwarden. Historical examples: ```bash kubectl get pods -A @@ -432,7 +440,12 @@ talosctl -n reboot # graceful reboot ## 10. Reinstall procedure -If a node is wiped or replaced: +> **Destructive historical procedure: do not run against repurposed nodes.** +> Reinstalling or bootstrapping can destroy existing systems and data. Only use +> as reference for a separately planned recovery after confirming the target, +> disk identity, and independently restorable backups. + +The former procedure for a wiped or replaced node was: 1. **Boot Talos ISO** on the node (or PXE if you set that up separately). Default Talos boots into maintenance mode and accepts an apply-config. 2. **Apply config**: @@ -522,10 +535,10 @@ talosctl -n reset # nuclear: wipes node, requires re-appl - All pods on karnataka stuck in `Terminating`/`Pending` - Ping to karnataka still works (node is up, just CRI is dead) -**Root cause:** Memory pressure from simultaneous large container builds exhausts -karnataka's 62GB unified memory. The node runs Lemonade (48-56Gi), Argo Workflow image +**Historical root cause:** Memory pressure from simultaneous large container builds exhausted +karnataka's 62GB unified memory. The former Talos node ran Lemonade (48-56Gi), Argo Workflow image builds, Forgejo Actions (podman-in-Docker), KubeVirt, and monitoring — when -multiple 4GB image builds run concurrently, containerd OOMs and crashes. +multiple 4GB image builds ran concurrently, containerd OOMed and crashed. **Recovery:** ```bash @@ -550,13 +563,13 @@ After reboot (~2 min), all pods recover automatically. PVC-backed services ### Scope and acceptance status -This is a replacement OS and **new K3s cluster** for the two home nodes, not a -Talos upgrade. Bihar is the Intel server/control plane; Karnataka is the AMD -worker. **No physical migration has been performed.** The home cluster is still -powered down/in storage. Nothing here changes the terminated -[AWS Talos cluster's](../aws-k8s/cluster.md) recovery material; Corral -acceptance uses separate disposable KubeVirt VMs, not Talos node changes. -Sections 1–11 remain the Talos recovery handbook, not instructions for the new OS. +This section records a proposed replacement OS and **new K3s cluster**, not a +Talos upgrade or evidence of current services. Bihar's present OS and services +are unverified. Karnataka runs `projectbluefin/utah:testing`, not the proposed +K3s worker. Do not install, switch, activate, or configure these +images on repurposed nodes. The procedures below and sections 1–11 are retained +for historical context and recovery planning only. The terminated +[AWS Talos cluster's](../aws-k8s/cluster.md) recovery material remains separate. Image builds alone do not prove installation. The GitHub Actions acceptance workflow builds all three images and offline ISOs, drives the **real @@ -629,8 +642,8 @@ unit and a real KubeVirt guest on physical Bihar before cutover. The source is `server-images/Containerfile`, `compose.sh`, `nodes/`, and `rootfs/`. All three targets use the digest-pinned Hummingbird Community `os-minimal` base, -without a desktop. The Bihar/Karnataka images remain the direct installation -path for those machines; `starter` is optional for future x86_64 AMD/Intel nodes. +without a desktop. The Bihar/Karnataka images were proposed as direct installation +paths; `starter` was optional for future x86_64 AMD/Intel nodes. These are not current deployment instructions. Explicit AMD/Intel GPU and CPU, Intel Wi-Fi, Realtek, MediaTek, Atheros, and Broadcom firmware subpackages come through the digest-pinned Utah package factory. That local repository is authenticated by its OCI digest, **not RPM signatures**. @@ -688,10 +701,10 @@ and containerd), user homes under `/var/home`, and these retained workload paths | `/var/lib/lemonade-cache` | Lemonade cache, matching the current home PV manifest | | `/var/lib/lemonade-models` | Lemonade models, matching the current home PV manifest | -The current `talos-k8s/ai/lemonade.yaml` uses `/var/lib/lemonade-*`. The offline -Talos node may still hold the data under `/var/tmp/lemonade-*`: back up those -source directories before wiping and restore into `/var/lib/lemonade-*` on the -new node. Creating an empty PV directory does not migrate the models. +The retained `talos-k8s/ai/lemonade.yaml` uses `/var/lib/lemonade-*`. The former +Talos node was recorded with data under `/var/tmp/lemonade-*`; its present +contents are unverified. Before any recovery, inspect and back up source data +independently. Creating an empty PV directory does not migrate models. ### Back up before building a physical migration plan diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index f98939dd..8c9f67e8 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -1,29 +1,19 @@ # Karnataka -Kubernetes worker node with AMD GPU. [Talos Linux](https://www.talos.dev/) node — not in `inventory.yml`, managed with `talosctl`/`kubectl` only. - -> **Powered down and in storage, expected back.** See the [cluster handbook](../cluster.md). +Karnataka runs `ghcr.io/projectbluefin/utah:testing`, is in the fleet's +`server` inventory group, and uses `multi-user.target` as its default. SSH is +available over Tailscale at `100.66.158.127`. No current workloads are assumed. ## Hardware - Arch: x86_64 - System: [Framework Computer](https://frame.work/) - CPU: [AMD Strix Halo APU](https://www.amd.com/en/products/processors/laptop/ryzen-ai-max-plus.html) -- GPU: AMD integrated (exposed via [Talos Image Factory](https://www.talos.dev/latest/talos-guides/install/boot-assets/) schematic with [`siderolabs/amdgpu`](https://github.com/siderolabs/extensions/tree/main/amdgpu)) +- GPU: AMD integrated - LAN: reserved address on the home /24 -- Role: Worker (Talos K8s) -- Tailscale IP: via MagicDNS - -## OS - -[Talos Linux](https://www.talos.dev/) v1.13.2 ([Kubernetes](https://kubernetes.io/) v1.36.1) - -## K8s Workloads - -- **[Lemonade](https://lemonade-sdk.github.io/)** — AMD-optimized local AI runtime ([omni-modal](https://lemonade-sdk.github.io/docs/category/endpoints): chat, vision, image gen, speech, transcription) on iGPU -- **[KubeVirt](https://kubevirt.io/) v1.8.2** — VM workloads -- **[Tailscale Operator](https://tailscale.com/kb/1236/kubernetes-operator)** — Ingress to `*.manatee-basking.ts.net` -## See also +## Historical configuration -- [Talos cluster handbook](../cluster.md) +Karnataka was the worker of the home Talos cluster. The [cluster handbook](../cluster.md), +Talos manifests and configs, and Bitwarden notes are retained as historical +recovery material; they do not establish its current OS or workloads. diff --git a/docs/src/talos-k8s.md b/docs/src/talos-k8s.md index 46a339e0..b27979b6 100644 --- a/docs/src/talos-k8s.md +++ b/docs/src/talos-k8s.md @@ -1,17 +1,26 @@ -# Talos Kubernetes Cluster +# Historical Talos Kubernetes Cluster -Two-node Kubernetes cluster running on Talos Linux, managed from the `talos-k8s/` directory. +> **Historical reference, not a current deployment.** Bihar was recently online +> but is currently unreachable; its OS and services are unverified. Karnataka is +> running `projectbluefin/utah:testing`, with `multi-user.target` as its default. +> Retain `talos-k8s/`, cluster configs and Bitwarden +> notes as recovery material, not proof either machine still runs Talos. +> +> **Do not run historical reinstall, configuration or workload commands against +> repurposed nodes.** They can destroy data or change current systems. Kubernetes +> service URLs in the handbook are historical, not running endpoints. -> **Powered down and in storage, expected back** — commands below fail until it returns. Matrix and School, Reef and Hub run on the [OCI Spindle VPS](servers/spindle-vps/README.md); the [AWS Talos cluster](servers/aws-k8s/cluster.md) was terminated on 2026-10-25. +The former two-node Talos Linux cluster was managed from `talos-k8s/`. The +[AWS Talos cluster](servers/aws-k8s/cluster.md) was terminated on 2026-10-25. -## Hardware +## Historical hardware roles | Node | Role | Hardware | |------|------|----------| | bihar | Control plane | Intel (bare metal) | | karnataka | Worker | AMD Strix Halo APU (GPU-enabled) | -## Software Stack +## Historical software stack | Component | Version | Notes | |-----------|---------|-------| @@ -39,11 +48,11 @@ talos-k8s/ └── testing-lab/ ci/cd test infra ``` -## Running Workloads +## Former workloads ### Lemonade -[Lemonade](https://lemonade-sdk.github.io/) — AMD-optimized local AI runtime providing omni-modal endpoints (chat, vision, image gen, speech, transcription) on karnataka's integrated AMD GPU (`amdgpu` driver baked into the Talos image via [Image Factory](https://www.talos.dev/latest/talos-guides/install/boot-assets/)). +[Lemonade](https://lemonade-sdk.github.io/) was an AMD-optimized local AI runtime providing omni-modal endpoints (chat, vision, image gen, speech, transcription) on karnataka's integrated AMD GPU (`amdgpu` driver baked into the Talos image via [Image Factory](https://www.talos.dev/latest/talos-guides/install/boot-assets/)). Its current operation is not established by this historical manifest. Manifest: `talos-k8s/ai/lemonade.yaml` @@ -51,20 +60,22 @@ Manifest: `talos-k8s/ai/lemonade.yaml` See the [cluster handbook](servers/talos-k8s/cluster.md) for the full list — Forgejo, Hive, Authentik, n8n, AppFlowy Cloud, Grafana, Corral, and more. -## Cluster Access +## Historical cluster access + +These commands apply only to a deliberately restored cluster; the retained configs do not prove a cluster is reachable. ```bash # kubectl (from kubeconfig in Bitwarden) kubectl get nodes -# Talos CLI (LAN addresses; off-LAN they're reachable via bihar's subnet route) +# Talos CLI (former LAN endpoints; routing is not established now) talosctl -n version # bihar talosctl -n version # karnataka ``` ## Image Factory -Karnataka uses a custom Talos image from Image Factory that includes the `siderolabs/amdgpu` system extension for GPU passthrough to containers. +Karnataka's former Talos installation used a custom Image Factory image including the `siderolabs/amdgpu` system extension for GPU passthrough to containers. ## Handbook diff --git a/inventory.yml b/inventory.yml index ab322567..189ed41c 100644 --- a/inventory.yml +++ b/inventory.yml @@ -35,9 +35,11 @@ all: vm: ansible_host: localhost ansible_connection: local - # ── Talos K8s cluster (not ansible-managed — talosctl / kubectl only) ── - # karnataka: Talos worker — AMD Strix Halo APU, 192.168.0.6 - # bihar: Talos control-plane — Intel, 192.168.0.5 + # Bihar's current OS is unverified; its former Talos configuration is retained. + karnataka: + ansible_host: 100.66.158.127 + ansible_user: james + ansible_connection: ssh matrix: ansible_host: 100.73.19.81 ansible_connection: ssh @@ -73,7 +75,7 @@ all: kanpur: himachal: dilli: - # llm group is dormant — karnataka (former LLM host) is now a Talos worker + # llm group is dormant. # llm: # hosts: # karnataka: @@ -82,6 +84,7 @@ all: punjab: vm: goa: + karnataka: test_fleet: hosts: test-fleet-fedora: diff --git a/scripts/tailnet.py b/scripts/tailnet.py index a3b1f57a..3346a02e 100755 --- a/scripts/tailnet.py +++ b/scripts/tailnet.py @@ -449,7 +449,10 @@ def main(argv=None): else: for host, h in hosts.items(): intended = h["intended"] - if is_retired(intended, protected) or any(is_retired(r, {intended}) for r in protected): + # A protected home node can be inventoried under its own name; + # it remains protected from release and garbage collection. + reserved = retired | (HOME_CLUSTER_NAMES - {host.lower()}) + if is_retired(intended, reserved) or any(is_retired(r, {intended}) for r in reserved): print(f"error: {host} conflicts with a protected name", file=sys.stderr) return 2 devs, api, label = load_devices("api" if (args.apply or args.cmd == "release") else args.source, now) diff --git a/tests/test_tailnet.py b/tests/test_tailnet.py index 287a057c..df1507e1 100644 --- a/tests/test_tailnet.py +++ b/tests/test_tailnet.py @@ -261,6 +261,16 @@ def test_release_refuses_offline_home_cluster_nodes(fake_api, name): assert api.reads == 0 and api.deletes == [] +def test_managed_karnataka_is_audited_without_releasing_its_device(fake_api, capsys): + api, inventory = fake_api + inventory.write_text(INVENTORY.replace(" telengana:\n", " karnataka:\n ansible_host: 100.1.1.1\n telengana:\n", 1)) + api.devices = [api_device("karnataka", "node", connectedToControl=True)] + tailnet.main(["audit"]) + assert api.reads == 1 + assert "conflicts with a protected name" not in capsys.readouterr().err + assert api.deletes == [] + + def test_gc_keeps_home_cluster_nodes_with_explicit_k8s_gc(fake_api): api, _ = fake_api api.devices = [api_device("bihar", "bihar-node", tags=["tag:k8s"]), diff --git a/tests/test_visual_units_check.py b/tests/test_visual_units_check.py index 90d7c829..e0e3ec22 100644 --- a/tests/test_visual_units_check.py +++ b/tests/test_visual_units_check.py @@ -65,6 +65,7 @@ def account_with_icon(text): [ "${FAIL_GSETTINGS:-0}" = 0 ] || exit 1 case "$*" in 'get org.gnome.desktop.interface color-scheme') echo "'prefer-dark'" ;; + 'get org.gnome.desktop.peripherals.touchpad tap-to-click') echo "${CHECK_TAP_TO_CLICK:-false}" ;; 'get org.gnome.Ptyxis default-profile-uuid') echo "'profile'" ;; 'get org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/profile/ custom-command') echo "'/custom/brew/bin/fish'" ;; 'get org.gnome.Ptyxis font-name') echo "'JetBrainsMono Nerd Font 12'" ;; @@ -172,13 +173,14 @@ def test_gnome_reports_deployment_and_session_drift_without_mutation(visual): env, _ = visual home = Path(env['HOME']) before = snapshot(home) - result = inspect('gnome', env) + result = inspect('gnome', env, CHECK_TAP_TO_CLICK='true') assert result.returncode == 0, result.stdout + result.stderr assert 'WOULD update GNOME keybindings file' in result.stdout assert 'WOULD load GNOME keybindings' in result.stdout assert 'WOULD create or fix extension' in result.stdout assert 'WOULD load GNOME extension settings' in result.stdout assert 'WOULD enable and start GNOME Remote Desktop' in result.stdout + assert 'WOULD disable touchpad tap-to-click' in result.stdout assert 'remote image not inspected' in result.stdout assert 'private-test-password' not in result.stdout + result.stderr assert_read_only(env, before) diff --git a/units/gnome.sh b/units/gnome.sh index a25629d6..404df9f7 100755 --- a/units/gnome.sh +++ b/units/gnome.sh @@ -237,6 +237,10 @@ def rdp(): if 'Status: enabled' not in command('grdctl', 'rdp', 'status'): print('WOULD enable GNOME RDP protocol') +def touchpad(): + if settings('org.gnome.desktop.peripherals.touchpad', 'tap-to-click') != 'false': + print('WOULD disable touchpad tap-to-click') + def remote_service(): state = command('systemctl', '--user', 'show', '--property=LoadState', '--property=UnitFileState', '--property=ActiveState', 'gnome-remote-desktop.service') @@ -247,7 +251,7 @@ def remote_service(): print('WOULD enable and start GNOME Remote Desktop user service') failed = False -checks = [('extension settings', extension_settings), ('RDP', rdp), ('remote service', remote_service)] +checks = [('extension settings', extension_settings), ('RDP', rdp), ('remote service', remote_service), ('touchpad', touchpad)] if config_ok: checks.extend((('Ptyxis profile', ptyxis), ('enabled extensions', enabled_extensions))) for label, inspect in checks: @@ -264,6 +268,13 @@ PY step "inspect GNOME user session" check_gnome_session finish fi +disable_tap_to_click() { + require_session || return 1 + [ "$(gsettings get org.gnome.desktop.peripherals.touchpad tap-to-click)" = false ] || + gsettings set org.gnome.desktop.peripherals.touchpad tap-to-click false +} +soft "disable touchpad tap-to-click" disable_tap_to_click + configure_ptyxis() { local uuid current_cmd current_font From ef67d3a78db723569b474ca4f9152f45122f1bac Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:38:04 +0000 Subject: [PATCH 21/62] Preserve active swap on immutable fleet hosts Restore the Homebrew tag and clarify protected device behavior. Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/roles/server_hardening.md | 6 +- docs/src/tailnet.md | 4 +- scripts/run-units.sh | 1 + scripts/tailnet.py | 6 +- tests/test_server_units_check.py | 137 ++++++++++++++++++++++++++++- units/server-hardening.sh | 99 +++++++++++++-------- 6 files changed, 210 insertions(+), 43 deletions(-) diff --git a/docs/src/roles/server_hardening.md b/docs/src/roles/server_hardening.md index 579cb717..74f74c68 100644 --- a/docs/src/roles/server_hardening.md +++ b/docs/src/roles/server_hardening.md @@ -1,5 +1,7 @@ # server_hardening -`units/server-hardening.sh` runs for inventory `server` or `vps` members, but explicitly skips retired `matrix` and `telengana`. It requires `yq`, Python, and root or noninteractive `sudo -n`; it uses no vault secrets. `units/server-hardening-file.py` performs the managed line edits. +`units/server-hardening.sh` runs for inventory `server` or `vps` members, but explicitly skips retired `matrix` and `telengana`. It requires `yq`, Python, `findmnt`, and root or noninteractive `sudo -n`; it uses no vault secrets. `units/server-hardening-file.py` performs the managed line edits. -The unit sets journald's maximum size to 500 MiB, restarts journald on change and vacuums old journals. It creates a 2 GiB `/swapfile` if absent, sets root ownership and mode 0600, formats and enables it if not active, and adds an fstab entry. It enables systemd-timesyncd only when present and chrony is not active. It sets SSH `MaxStartups`, `MaxSessions`, and `ClientAliveCountMax`, reloading SSH best-effort on change. It enables UFW logging only when UFW is already active, attempts Ubuntu Pro livepatch when its status indicates it is needed, and reports a pending reboot. It does not configure firewall rules or install fail2ban. +The unit sets journald's maximum size to 500 MiB, restarts journald on change and vacuums old journals. On writable roots it creates a 2 GiB `/swapfile` if absent, sets root ownership and mode 0600, formats and enables it if not active, and adds an fstab entry. On read-only roots it preserves existing active swap (including zram) without changing `/swapfile` or fstab; if no swap is active, it reports failure. Apply and check modes both report failures when root mount options or the required active-swap status cannot be inspected, and continue the other hardening steps. + +It enables systemd-timesyncd only when present and chrony is not active. It sets SSH `MaxStartups`, `MaxSessions`, and `ClientAliveCountMax`, reloading SSH best-effort on change. It enables UFW logging only when UFW is already active, attempts Ubuntu Pro livepatch when its status indicates it is needed, and reports a pending reboot. It does not configure firewall rules or install fail2ban. diff --git a/docs/src/tailnet.md b/docs/src/tailnet.md index 25c72b39..4ef55b41 100644 --- a/docs/src/tailnet.md +++ b/docs/src/tailnet.md @@ -16,8 +16,8 @@ - A device's tailnet name is the first MagicDNS label. An inventory host's **intended** name is its `tailnet_name` var, else the inventory name. A `tailnet_name` of the form `-` is the auto-suffix Tailscale adds when a stale device still holds ``, so it means "wants ``" and is removed once the rename has happened. - **Duplicates:** several devices named `` / `-`: the live (else most recently seen) one is kept; older offline ones are deleted. - **Test VMs:** offline longer than `--days` (default 14), named `gate-*`, `repro-*`, `tuna-v1-preflight-*` or `*-vm`, and not an inventory host. -- **k8s proxies** (`tag:k8s*`) cannot be distinguished reliably by cluster ownership. `--k8s-days` reports stale proxies but never deletes them; an apply that encounters one refuses all changes. Proxies matching the test-VM or duplicate rules are likewise protected, as are proxies passed to `release`. The home cluster is powered down and expected back. -- Never deleted: devices that are online, this machine, devices shared in from other tailnets, the home cluster nodes `bihar` and `karnataka`, and the retired `vps` group. Both VPS inventory names and `tailnet_name` aliases are protected, including numbered suffixes; `release` refuses these names without contacting the API. Rename can change an online device's MagicDNS name. +- **k8s proxies** (`tag:k8s*`) cannot be distinguished reliably by cluster ownership. `--k8s-days` reports stale proxies but never deletes them; an apply that encounters one refuses all changes. Proxies matching the test-VM or duplicate rules are likewise protected, as are proxies passed to `release`. Retained home-cluster state remains recovery material. +- Never deleted: devices that are online, this machine, devices shared in from other tailnets, protected `bihar` and `karnataka` device names, and the retired `vps` group. Karnataka is now a managed Utah host; protection retains its former cluster identity without excluding it from inventory. Both VPS inventory names and `tailnet_name` aliases are protected, including numbered suffixes; `release` refuses these names without contacting the API. Rename can change an online device's MagicDNS name. - `fix-inventory` run alone reflects the live state, so it will not drop a `tailnet_name` override before the rename has actually happened. - Before planning or writing, every device must have a valid ID and name and an explicitly boolean presence state. Missing or non-boolean `connectedToControl`, invalid or duplicate IDs, and invalid names make the whole snapshot unverifiable: the command reports the problem and makes no plan or changes. Read-only audit reports the problem rather than claiming agreement. A fresh snapshot is checked again before the inventory is updated after `sync`. diff --git a/scripts/run-units.sh b/scripts/run-units.sh index a9d6d7a5..c97983c3 100755 --- a/scripts/run-units.sh +++ b/scripts/run-units.sh @@ -9,6 +9,7 @@ tasks_for_tag() { case "$1" in sudo) echo sudo ;; sshd) echo sshd ;; packages) echo "apk-packages termux-packages homebrew flatpak" ;; + homebrew) echo homebrew ;; apk) echo apk-packages ;; termux) echo termux-packages ;; homebrew) echo homebrew ;; diff --git a/scripts/tailnet.py b/scripts/tailnet.py index 3346a02e..586e3324 100755 --- a/scripts/tailnet.py +++ b/scripts/tailnet.py @@ -8,14 +8,14 @@ tailnet.py sync [--apply] gc, then rename, then fix-inventory (planned as one sequence) tailnet.py release NAME delete offline devices called NAME or NAME- (run before joining) -Everything is a dry run unless --apply. Reads come from the Tailscale API when a credential is +Except for release, commands are dry runs unless --apply. Reads come from the Tailscale API when a credential is available (TS_OAUTH_CLIENT_SECRET, or Bitwarden item `tailscale-apikey`), otherwise from the local `tailscale status --json` (read-only: enough for audit and for planning). Writes need the API. Names: a device's tailnet name is its first MagicDNS label. An inventory host's *intended* name is its `tailnet_name` var, else the inventory name; a `tailnet_name` of the form - is the auto-suffix Tailscale adds when a stale device still holds , so it is treated as "wants ". -Retired hosts (the `vps` group), home cluster nodes, devices shared in from other tailnets, +Retired hosts (the `vps` group), protected Bihar/Karnataka device names, devices shared in from other tailnets, and Kubernetes proxies of unverifiable cluster ownership are never deleted. """ import argparse @@ -40,7 +40,7 @@ IP100 = re.compile(r"^100\.\d+\.\d+\.\d+$") DEVICE_ID = re.compile(r"[A-Za-z0-9_-]+\Z") DNS_LABEL = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\Z") -# The home Talos cluster is offline temporarily, not retired. +# Protect former home-node names even though Karnataka is now fleet-managed Utah. HOME_CLUSTER_NAMES = {"bihar", "karnataka"} diff --git a/tests/test_server_units_check.py b/tests/test_server_units_check.py index c9058473..0294d48b 100644 --- a/tests/test_server_units_check.py +++ b/tests/test_server_units_check.py @@ -78,7 +78,8 @@ def check_system(stub_env, tmp_path): ''') write_stub(bindir, "loginctl", 'echo "loginctl $*" >> "$CALLS"\n[ "$1" = show-user ] || exit 99\necho no\n') write_stub(bindir, "journalctl", '[ "$1" = --disk-usage ] || exit 99\necho 1G\n') - write_stub(bindir, "swapon", '[ "$1" = --show=NAME ] || exit 99\n') + write_stub(bindir, "findmnt", '[ "${FAIL_FINDMNT:-0}" != 1 ] || exit 1\nprintf "%s\\n" "${ROOT_OPTIONS:-rw,relatime}"\n') + write_stub(bindir, "swapon", '[ "$1" = --show=NAME ] || exit 99\n[ "${FAIL_SWAPON:-0}" != 1 ] || exit 1\nprintf "%s\\n" "${ACTIVE_SWAP:-}"\n') write_stub(bindir, "ufw", '[ "$1" = status ] || exit 99\necho "Status: inactive"\n') write_stub(bindir, "pro", '[ "$1" = status ] || exit 99\necho \'{"livepatch": "enabled"}\'\n') write_stub(bindir, "ss", 'printf "%s\\n" "${SOCKETS:-}"\n') @@ -93,6 +94,140 @@ def run_check(check_system, name, **extra): env={**env, **extra}, capture_output=True, text=True) +def prepare_hardening_commands(check_system): + _, _, env, _ = check_system + bindir = Path(env["PATH"].split(":", 1)[0]) + write_stub(bindir, "systemctl", ''' +echo "systemctl $*" >> "$CALLS" +if [[ "$*" == *show* ]]; then + printf 'LoadState=loaded\\nActiveState=inactive\\nUnitFileState=disabled\\n' +elif [ "$1" = is-active ]; then + exit 1 +fi +''') + write_stub(bindir, "journalctl", 'echo "journalctl $*" >> "$CALLS"\n') + write_stub(bindir, "swapon", ''' +echo "swapon $*" >> "$CALLS" +if [ "$1" = --show=NAME ]; then + [ "${FAIL_SWAPON:-0}" != 1 ] || exit 1 + printf "%s\\n" "${ACTIVE_SWAP:-}" +fi +''') + write_stub(bindir, "dd", ''' +echo "dd $*" >> "$CALLS" +for arg in "$@"; do case "$arg" in of=*) : > "${arg#of=}" ;; esac; done +''') + for command in ("chown", "chmod", "mkswap"): + write_stub(bindir, command, f'echo "{command} $*" >> "$CALLS"\n') + + +@pytest.mark.parametrize("check", [True, False]) +@pytest.mark.parametrize("existing_swapfile", [True, False]) +@pytest.mark.parametrize("active", [True, False]) +def test_readonly_root_preserves_swap_and_continues_hardening(check_system, check, existing_swapfile, active): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + swapfile = root / "swapfile" + if existing_swapfile: + swapfile.write_text("existing swapfile\n") + swapfile.chmod(0o640) + fstab = root / "etc/fstab" + if existing_swapfile: + fstab.write_text(fstab.read_text() + "/swapfile none swap sw 0 0\n") + original_fstab = fstab.read_bytes() + result = run_check(check_system, "server-hardening", UNITS_CHECK="1" if check else "0", + ROOT_OPTIONS="ro,relatime", ACTIVE_SWAP="/dev/zram0" if active else " ") + assert result.returncode == (0 if active else 1), (result.stdout, result.stderr) + assert ("active swap preserved on read-only root" if active else + "FAIL active swap required on read-only root") in result.stdout + assert fstab.read_bytes() == original_fstab + assert swapfile.exists() == existing_swapfile + if existing_swapfile: + assert swapfile.read_text() == "existing swapfile\n" + assert swapfile.stat().st_mode & 0o777 == 0o640 + calls = Path(env["CALLS"]).read_text().splitlines() + assert [line for line in calls if line.startswith("swapon ")] == ["swapon --show=NAME --noheadings"] + assert not any(line.startswith(("dd ", "chown ", "chmod ", "mkswap ")) for line in calls) + if check: + assert "would update SSH limit MaxSessions" in result.stdout + else: + assert "MaxSessions 10" in (root / "etc/ssh/sshd_config").read_text() + + +@pytest.mark.parametrize("check", [True, False]) +@pytest.mark.parametrize("failure", ["swap", "mount", "unknown_mount"]) +def test_swap_inspection_failures_are_reported_without_blocking_hardening(check_system, check, failure): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + extra = {"ROOT_OPTIONS": "ro,relatime"} + if failure == "swap": + extra["FAIL_SWAPON"] = "1" + expected = "FAIL inspect active swap on read-only root" + else: + extra.update({"FAIL_FINDMNT": "1"} if failure == "mount" else {"ROOT_OPTIONS": "noexec"}) + expected = "FAIL inspect root mount options" + fstab = root / "etc/fstab" + original_fstab = fstab.read_bytes() + result = run_check(check_system, "server-hardening", UNITS_CHECK="1" if check else "0", **extra) + assert result.returncode == 1, (result.stdout, result.stderr) + assert expected in result.stdout + assert fstab.read_bytes() == original_fstab + assert not (root / "swapfile").exists() + calls = Path(env["CALLS"]).read_text().splitlines() + assert not any(line.startswith(("dd ", "chown ", "chmod ", "mkswap ")) for line in calls) + if check: + assert "would update SSH limit MaxSessions" in result.stdout + else: + assert "MaxSessions 10" in (root / "etc/ssh/sshd_config").read_text() + + +@pytest.mark.parametrize("check", [True, False]) +def test_writable_root_still_manages_swapfile(check_system, check): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + result = run_check(check_system, "server-hardening", UNITS_CHECK="1" if check else "0", + ROOT_OPTIONS="rw,relatime", ACTIVE_SWAP="/dev/zram0") + assert result.returncode == 0, (result.stdout, result.stderr) + swapfile = root / "swapfile" + fstab = (root / "etc/fstab").read_text() + calls = Path(env["CALLS"]).read_text().splitlines() + if check: + assert "would create 2 GiB" in result.stdout + assert "would format and enable" in result.stdout + assert "would update swap fstab" in result.stdout + assert not swapfile.exists() and str(swapfile) not in fstab + else: + assert swapfile.exists() + assert f"{swapfile} none swap sw 0 0" in fstab + assert any(line.startswith("dd ") for line in calls) + assert f"chown root:root {swapfile}" in calls + assert f"chmod 0600 {swapfile}" in calls + assert f"mkswap {swapfile}" in calls + assert f"swapon {swapfile}" in calls + + +@pytest.mark.parametrize("check", [True, False]) +def test_writable_root_preserves_active_swapfile(check_system, check): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + swapfile = root / "swapfile" + swapfile.write_text("existing swapfile\n") + result = run_check(check_system, "server-hardening", UNITS_CHECK="1" if check else "0", + ROOT_OPTIONS="rw,relatime", ACTIVE_SWAP=str(swapfile)) + assert result.returncode == 0, (result.stdout, result.stderr) + assert swapfile.read_text() == "existing swapfile\n" + calls = Path(env["CALLS"]).read_text().splitlines() + assert not any(line.startswith(("dd ", "mkswap ")) for line in calls) + assert [line for line in calls if line.startswith("swapon ")] == ["swapon --show=NAME --noheadings"] + if check: + assert "ok swap enabled" in result.stdout + assert str(swapfile) not in (root / "etc/fstab").read_text() + else: + assert f"chown root:root {swapfile}" in calls + assert f"chmod 0600 {swapfile}" in calls + assert f"{swapfile} none swap sw 0 0" in (root / "etc/fstab").read_text() + + @pytest.mark.parametrize("unit,expected", [ ("server-hardening", "would update journald limit"), ("retired-services", "would stop grafana.service"), diff --git a/units/server-hardening.sh b/units/server-hardening.sh index a4a14432..6f6fd7e5 100755 --- a/units/server-hardening.sh +++ b/units/server-hardening.sh @@ -18,6 +18,27 @@ hardening_main() { hardening_line() { "${rootcmd[@]}" python3 "$DOTFILES_DIR/units/server-hardening-file.py" "$@" } + local root_options swap_root=unknown + if root_options=$(findmnt -n -o OPTIONS --mountpoint /); then + case ",$root_options," in + *,ro,*) swap_root=readonly ;; + *,rw,*) swap_root=writable ;; + *) step "inspect root mount options" false ;; + esac + else + step "inspect root mount options" false + fi + hardening_immutable_swap() { + if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then + if [[ "$swap_output" =~ [^[:space:]] ]]; then + note "ok active swap preserved on read-only root" + else + step "active swap required on read-only root" false + fi + else + step "inspect active swap on read-only root" false + fi + } if [ "${UNITS_CHECK:-0}" = 1 ]; then local rule pattern line mode chrony_unit chrony_state chrony_active=0 hardening_inspect_line() { @@ -36,25 +57,29 @@ hardening_main() { else step "inspect journal usage" false fi - if "${rootcmd[@]}" test -e /swapfile; then - if mode=$("${rootcmd[@]}" stat -c '%a:%u:%g' /swapfile); then - [ "$mode" = '600:0:0' ] && note "ok swap permissions" || note "would set swap ownership and permissions" + if [ "$swap_root" = readonly ]; then + hardening_immutable_swap + elif [ "$swap_root" = writable ]; then + if "${rootcmd[@]}" test -e /swapfile; then + if mode=$("${rootcmd[@]}" stat -c '%a:%u:%g' /swapfile); then + [ "$mode" = '600:0:0' ] && note "ok swap permissions" || note "would set swap ownership and permissions" + else + step "inspect swap permissions" false + fi else - step "inspect swap permissions" false + note "would create 2 GiB /swapfile with root ownership and mode 0600" fi - else - note "would create 2 GiB /swapfile with root ownership and mode 0600" - fi - if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then - if printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then - note "ok swap enabled" + if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then + if printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then + note "ok swap enabled" + else + note "would format and enable /swapfile" + fi else - note "would format and enable /swapfile" + step "inspect active swap" false fi - else - step "inspect active swap" false + hardening_inspect_line "swap fstab" /etc/fstab '^/swapfile none swap sw 0 0$' '/swapfile none swap sw 0 0' fi - hardening_inspect_line "swap fstab" /etc/fstab '^/swapfile none swap sw 0 0$' '/swapfile none swap sw 0 0' for chrony_unit in chrony chronyd; do chrony_state=$(systemctl show "$chrony_unit" --property=LoadState --property=ActiveState 2>/dev/null); rc=$? if [ "$rc" != 0 ] && [[ "$chrony_state" != *'LoadState=not-found'* ]]; then @@ -99,31 +124,35 @@ hardening_main() { esac step "vacuum journals" "${rootcmd[@]}" journalctl --vacuum-size=500M - local swap_ready=1 - if [ ! -e /swapfile ]; then - if ! (umask 077; "${rootcmd[@]}" dd if=/dev/zero of=/swapfile bs=1M count=2048); then - step "create swap file" false; swap_ready=0 + if [ "$swap_root" = readonly ]; then + hardening_immutable_swap + elif [ "$swap_root" = writable ]; then + local swap_ready=1 + if [ ! -e /swapfile ]; then + if ! (umask 077; "${rootcmd[@]}" dd if=/dev/zero of=/swapfile bs=1M count=2048); then + step "create swap file" false; swap_ready=0 + fi fi - fi - if [ "$swap_ready" = 1 ] && [ -e /swapfile ]; then - if ! "${rootcmd[@]}" chown root:root /swapfile; then step "protect swap file" false; swap_ready=0; fi - if ! "${rootcmd[@]}" chmod 0600 /swapfile; then step "set swap permissions" false; swap_ready=0; fi - if [ "$swap_ready" = 1 ]; then - if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then - if ! printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then - if ! "${rootcmd[@]}" mkswap /swapfile; then - step "format swap" false; swap_ready=0 - elif ! "${rootcmd[@]}" swapon /swapfile; then - step "enable swap" false; swap_ready=0 + if [ "$swap_ready" = 1 ] && [ -e /swapfile ]; then + if ! "${rootcmd[@]}" chown root:root /swapfile; then step "protect swap file" false; swap_ready=0; fi + if ! "${rootcmd[@]}" chmod 0600 /swapfile; then step "set swap permissions" false; swap_ready=0; fi + if [ "$swap_ready" = 1 ]; then + if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then + if ! printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then + if ! "${rootcmd[@]}" mkswap /swapfile; then + step "format swap" false; swap_ready=0 + elif ! "${rootcmd[@]}" swapon /swapfile; then + step "enable swap" false; swap_ready=0 + fi fi + else + step "check active swap" false; swap_ready=0 fi - else - step "check active swap" false; swap_ready=0 fi - fi - if [ "$swap_ready" = 1 ]; then - hardening_line /etc/fstab '^/swapfile none swap sw 0 0$' '/swapfile none swap sw 0 0'; rc=$? - case "$rc" in 0|10) note "ok swap fstab" ;; *) step "swap fstab" false ;; esac + if [ "$swap_ready" = 1 ]; then + hardening_line /etc/fstab '^/swapfile none swap sw 0 0$' '/swapfile none swap sw 0 0'; rc=$? + case "$rc" in 0|10) note "ok swap fstab" ;; *) step "swap fstab" false ;; esac + fi fi fi From de22a093b9c4df22a3e9c485aac1493b13a76fab Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:41:38 +0000 Subject: [PATCH 22/62] Support journald drop-in on immutable Utah hosts Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/roles/server_hardening.md | 4 +- tests/test_server_units_check.py | 116 +++++++++++++++++++++++++++++ units/server-hardening.sh | 58 ++++++++++++++- 3 files changed, 175 insertions(+), 3 deletions(-) diff --git a/docs/src/roles/server_hardening.md b/docs/src/roles/server_hardening.md index 74f74c68..c83986c2 100644 --- a/docs/src/roles/server_hardening.md +++ b/docs/src/roles/server_hardening.md @@ -2,6 +2,8 @@ `units/server-hardening.sh` runs for inventory `server` or `vps` members, but explicitly skips retired `matrix` and `telengana`. It requires `yq`, Python, `findmnt`, and root or noninteractive `sudo -n`; it uses no vault secrets. `units/server-hardening-file.py` performs the managed line edits. -The unit sets journald's maximum size to 500 MiB, restarts journald on change and vacuums old journals. On writable roots it creates a 2 GiB `/swapfile` if absent, sets root ownership and mode 0600, formats and enables it if not active, and adds an fstab entry. On read-only roots it preserves existing active swap (including zram) without changing `/swapfile` or fstab; if no swap is active, it reports failure. Apply and check modes both report failures when root mount options or the required active-swap status cannot be inspected, and continue the other hardening steps. +The unit sets journald's maximum size to 500 MiB, restarts journald on change and vacuums old journals. It edits `/etc/systemd/journald.conf` when present; otherwise it manages `/etc/systemd/journald.conf.d/90-dotfiles-server-hardening.conf`, preserving the vendor config and other drop-ins. Check mode reports the pending drop-in without creating it. A symlinked or invalid managed drop-in, or an unusable drop-in directory, is reported as a failure rather than overwritten. + +On writable roots it creates a 2 GiB `/swapfile` if absent, sets root ownership and mode 0600, formats and enables it if not active, and adds an fstab entry. On read-only roots it preserves existing active swap (including zram) without changing `/swapfile` or fstab; if no swap is active, it reports failure. Apply and check modes both report failures when root mount options or the required active-swap status cannot be inspected, and continue the other hardening steps. It enables systemd-timesyncd only when present and chrony is not active. It sets SSH `MaxStartups`, `MaxSessions`, and `ClientAliveCountMax`, reloading SSH best-effort on change. It enables UFW logging only when UFW is already active, attempts Ubuntu Pro livepatch when its status indicates it is needed, and reports a pending reboot. It does not configure firewall rules or install fail2ban. diff --git a/tests/test_server_units_check.py b/tests/test_server_units_check.py index 0294d48b..0acd0a62 100644 --- a/tests/test_server_units_check.py +++ b/tests/test_server_units_check.py @@ -121,6 +121,122 @@ def prepare_hardening_commands(check_system): write_stub(bindir, command, f'echo "{command} $*" >> "$CALLS"\n') +@pytest.mark.parametrize("check", [True, False]) +def test_missing_journald_dropin_directory(check_system, check): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + (root / "etc/systemd/journald.conf").unlink() + directory = root / "etc/systemd/journald.conf.d" + result = run_check(check_system, "server-hardening", UNITS_CHECK="1" if check else "0", + ROOT_OPTIONS="ro,relatime", ACTIVE_SWAP="/dev/zram0") + assert result.returncode == 0, (result.stdout, result.stderr) + calls = Path(env["CALLS"]).read_text().splitlines() + if check: + assert "would update journald limit" in result.stdout + assert not directory.exists() + assert "systemctl restart systemd-journald" not in calls + else: + assert (directory / "90-dotfiles-server-hardening.conf").read_text() == "[Journal]\nSystemMaxUse=500M\n" + assert calls.count("systemctl restart systemd-journald") == 1 + + +@pytest.mark.parametrize("check", [True, False]) +@pytest.mark.parametrize("existing", ["absent", "current", "outdated"]) +def test_missing_journald_main_uses_preserving_dropin(check_system, check, existing): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + (root / "etc/systemd/journald.conf").unlink() + vendor = root / "usr/lib/systemd/journald.conf" + vendor.write_text("[Journal]\nSystemMaxUse=2G\n") + directory = root / "etc/systemd/journald.conf.d" + directory.mkdir() + other = directory / "20-other.conf" + other.write_text("[Journal]\nStorage=persistent\n") + managed = directory / "90-dotfiles-server-hardening.conf" + if existing != "absent": + managed.write_text("[Journal]\n# keep this setting\nStorage=auto\nSystemMaxUse=" + + ("500M\n" if existing == "current" else "1G\n")) + managed.chmod(0o640) + before = managed.read_bytes() if managed.exists() else None + result = run_check(check_system, "server-hardening", UNITS_CHECK="1" if check else "0", + ROOT_OPTIONS="ro,relatime", ACTIVE_SWAP="/dev/zram0") + assert result.returncode == 0, (result.stdout, result.stderr) + assert vendor.read_text() == "[Journal]\nSystemMaxUse=2G\n" + assert other.read_text() == "[Journal]\nStorage=persistent\n" + assert not (root / "etc/systemd/journald.conf").exists() + calls = Path(env["CALLS"]).read_text().splitlines() + restarts = [line for line in calls if line == "systemctl restart systemd-journald"] + if check: + assert ("ok journald limit" if existing == "current" else "would update journald limit") in result.stdout + assert (managed.read_bytes() if managed.exists() else None) == before + assert not restarts + else: + assert managed.read_text() == ("[Journal]\nSystemMaxUse=500M\n" if existing == "absent" else + "[Journal]\n# keep this setting\nStorage=auto\nSystemMaxUse=500M\n") + assert managed.stat().st_mode & 0o777 == (0o644 if existing == "absent" else 0o640) + assert len(restarts) == (0 if existing == "current" else 1) + result = run_check(check_system, "server-hardening", UNITS_CHECK="0", + ROOT_OPTIONS="ro,relatime", ACTIVE_SWAP="/dev/zram0") + assert result.returncode == 0, (result.stdout, result.stderr) + calls = Path(env["CALLS"]).read_text().splitlines() + assert len([line for line in calls if line == "systemctl restart systemd-journald"]) == len(restarts) + + +@pytest.mark.parametrize("check", [True, False]) +@pytest.mark.parametrize("invalid", ["missing_section", "wrong_section", "limit_before_section", "directory", "symlink"]) +def test_invalid_journald_dropin_fails_without_stopping_hardening(check_system, check, invalid): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + (root / "etc/systemd/journald.conf").unlink() + directory = root / "etc/systemd/journald.conf.d" + if invalid == "directory": + directory.write_text("not a directory\n") + original = directory.read_bytes() + else: + directory.mkdir() + managed = directory / "90-dotfiles-server-hardening.conf" + if invalid == "symlink": + vendor = root / "usr/lib/systemd/journald.conf" + vendor.write_text("[Journal]\nSystemMaxUse=1G\n") + managed.symlink_to(vendor) + else: + content = {"missing_section": "SystemMaxUse=1G\n", "wrong_section": "[Other]\nSystemMaxUse=1G\n", + "limit_before_section": "SystemMaxUse=1G\n[Journal]\n"}[invalid] + managed.write_text(content) + original = managed.read_bytes() + result = run_check(check_system, "server-hardening", UNITS_CHECK="1" if check else "0", + ROOT_OPTIONS="ro,relatime", ACTIVE_SWAP="/dev/zram0") + assert result.returncode == 1, (result.stdout, result.stderr) + assert "FAIL" in result.stdout and "journald limit" in result.stdout + assert (directory if invalid == "directory" else managed).read_bytes() == original + if invalid == "symlink": + assert managed.is_symlink() and vendor.read_bytes() == original + assert "systemctl restart systemd-journald" not in Path(env["CALLS"]).read_text().splitlines() + if check: + assert "would update SSH limit MaxSessions" in result.stdout + else: + assert "MaxSessions 10" in (root / "etc/ssh/sshd_config").read_text() + + +def test_existing_journald_main_remains_the_managed_file(check_system): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + main = root / "etc/systemd/journald.conf" + directory = root / "etc/systemd/journald.conf.d" + directory.mkdir() + other = directory / "20-other.conf" + other.write_text("[Journal]\nStorage=persistent\n") + for _ in range(2): + result = run_check(check_system, "server-hardening", UNITS_CHECK="0", + ROOT_OPTIONS="ro,relatime", ACTIVE_SWAP="/dev/zram0") + assert result.returncode == 0, (result.stdout, result.stderr) + assert main.read_text() == "SystemMaxUse=500M\n" + assert other.read_text() == "[Journal]\nStorage=persistent\n" + assert not (directory / "90-dotfiles-server-hardening.conf").exists() + calls = Path(env["CALLS"]).read_text().splitlines() + assert calls.count("systemctl restart systemd-journald") == 1 + + @pytest.mark.parametrize("check", [True, False]) @pytest.mark.parametrize("existing_swapfile", [True, False]) @pytest.mark.parametrize("active", [True, False]) diff --git a/units/server-hardening.sh b/units/server-hardening.sh index 6f6fd7e5..803ce676 100755 --- a/units/server-hardening.sh +++ b/units/server-hardening.sh @@ -18,6 +18,55 @@ hardening_main() { hardening_line() { "${rootcmd[@]}" python3 "$DOTFILES_DIR/units/server-hardening-file.py" "$@" } + hardening_journald() { + local dropin=/etc/systemd/journald.conf.d/90-dotfiles-server-hardening.conf + if "${rootcmd[@]}" test -e /etc/systemd/journald.conf || "${rootcmd[@]}" test -L /etc/systemd/journald.conf; then + hardening_line "$@" /etc/systemd/journald.conf '^#?SystemMaxUse=' 'SystemMaxUse=500M' + return $? + fi + # Never follow a pre-existing link to a file outside our managed drop-in. + if "${rootcmd[@]}" test -L "$dropin"; then return 1; fi + if "${rootcmd[@]}" test -e "$dropin"; then + # Only edit a valid journald drop-in; do not make a directive effective in the wrong section. + if ! "${rootcmd[@]}" python3 -c ' +import re, sys +with open(sys.argv[1], encoding="utf-8") as f: + journal = False + for line in f: + if re.match(r"^\s*\[", line): + if not re.fullmatch(r"\s*\[Journal\]\s*", line): + sys.exit("invalid journald drop-in section") + journal = True + if re.match(r"^#?SystemMaxUse=", line) and not journal: + sys.exit("journald limit is outside [Journal]") + if not journal: + sys.exit("journald drop-in has no [Journal] section") +' "$dropin"; then return 1; fi + hardening_line "$@" "$dropin" '^#?SystemMaxUse=' 'SystemMaxUse=500M' + return $? + fi + if { "${rootcmd[@]}" test -e "${dropin%/*}" || "${rootcmd[@]}" test -L "${dropin%/*}"; } && + ! "${rootcmd[@]}" test -d "${dropin%/*}"; then + return 1 + fi + [ "${1:-}" != --check ] || return 10 + # Stage in the same directory and link without replacing a concurrently created file. + "${rootcmd[@]}" python3 -c ' +import os, sys, tempfile +path = sys.argv[1] +directory = os.path.dirname(path) +os.makedirs(directory, mode=0o755, exist_ok=True) +fd, temp = tempfile.mkstemp(prefix=".dotfiles-journald-", dir=directory) +try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write("[Journal]\nSystemMaxUse=500M\n") + os.fchmod(f.fileno(), 0o644) + os.link(temp, path) +finally: + os.unlink(temp) +' "$dropin" || return 1 + return 10 + } local root_options swap_root=unknown if root_options=$(findmnt -n -o OPTIONS --mountpoint /); then case ",$root_options," in @@ -51,7 +100,12 @@ hardening_main() { *) step "inspect $description" false ;; esac } - hardening_inspect_line "journald limit (and restart journald)" /etc/systemd/journald.conf '^#?SystemMaxUse=' 'SystemMaxUse=500M' + hardening_journald --check; rc=$? + case "$rc" in + 0) note "ok journald limit (and restart journald)" ;; + 10) note "would update journald limit (and restart journald)" ;; + *) step "inspect journald limit (and restart journald)" false ;; + esac if "${rootcmd[@]}" journalctl --disk-usage >/dev/null; then note "would vacuum journals to 500M" else @@ -116,7 +170,7 @@ hardening_main() { if [ -e /var/run/reboot-required ]; then note "REBOOT REQUIRED on $MACHINE — kernel was updated and needs a restart."; fi finish fi - hardening_line /etc/systemd/journald.conf '^#?SystemMaxUse=' 'SystemMaxUse=500M'; rc=$? + hardening_journald; rc=$? case "$rc" in 0) note "ok journald limit" ;; 10) note "ok journald limit"; step "restart journald" "${rootcmd[@]}" systemctl restart systemd-journald ;; From d7dad0c67605299cf3542195773adda0c24544a6 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:52:02 +0000 Subject: [PATCH 23/62] Harden Karnataka K3s startup Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- .../src/servers/talos-k8s/karnataka/README.md | 23 +++++++++++++++-- home-k3s/config.yaml | 12 +++++++++ home-k3s/install-host.sh | 21 ++++++++++++++++ home-k3s/k3s.service | 25 +++++++++++++++++++ home-k3s/resolv.conf | 4 +++ home-k3s/wait-tailnet.sh | 11 ++++++++ 6 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 home-k3s/config.yaml create mode 100755 home-k3s/install-host.sh create mode 100644 home-k3s/k3s.service create mode 100644 home-k3s/resolv.conf create mode 100755 home-k3s/wait-tailnet.sh diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index 8c9f67e8..d25acbe0 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -2,7 +2,9 @@ Karnataka runs `ghcr.io/projectbluefin/utah:testing`, is in the fleet's `server` inventory group, and uses `multi-user.target` as its default. SSH is -available over Tailscale at `100.66.158.127`. No current workloads are assumed. +available over Tailscale at `100.66.158.127`. A single-node K3s cluster runs +KubeVirt, CDI, and Corral. The `corral-vms/punjab` VM is a staging guest; it +is not the EC2 host and has not received its data or services. ## Hardware @@ -10,7 +12,24 @@ available over Tailscale at `100.66.158.127`. No current workloads are assumed. - System: [Framework Computer](https://frame.work/) - CPU: [AMD Strix Halo APU](https://www.amd.com/en/products/processors/laptop/ryzen-ai-max-plus.html) - GPU: AMD integrated -- LAN: reserved address on the home /24 +- LAN: DHCP on the home network + +## K3s operation + +The host-specific files are in `home-k3s/`. K3s listens on the Tailscale +address, and its service waits for that address before starting. The service +loads `br_netfilter` and `overlay` when available. Run `systemctl status k3s` +and `sudo /var/usrlocal/bin/k3s kubectl --server https://100.66.158.127:6443 get pods -A` +to inspect it. Reloading the service definition does not restart K3s. + +Host SELinux is enforcing and `container-selinux` is installed. The running +Utah image does not include `k3s-selinux`, and its policy store is read-only; +K3s SELinux container labeling remains disabled. Our proposed image build +includes the policy, but has not been installed on this host. Do not enable +K3s SELinux until the policy and workload labeling are verified. Metrics-server +cannot currently reach the kubelet over the Tailscale address; node metrics +are unavailable. The default local-path volumes are node-local, without +snapshot or expansion support; deleting a claim deletes its data. ## Historical configuration diff --git a/home-k3s/config.yaml b/home-k3s/config.yaml new file mode 100644 index 00000000..6801b3ce --- /dev/null +++ b/home-k3s/config.yaml @@ -0,0 +1,12 @@ +# Karnataka's existing tailnet address. K3s API and kubelet do not bind the public NIC. +bind-address: 100.66.158.127 +advertise-address: 100.66.158.127 +node-ip: 100.66.158.127 +flannel-iface: tailscale0 +resolv-conf: /etc/rancher/k3s/resolv.conf +kubelet-arg: + - address=100.66.158.127 +disable: + - traefik + - servicelb +write-kubeconfig-mode: "0600" diff --git a/home-k3s/install-host.sh b/home-k3s/install-host.sh new file mode 100755 index 00000000..042d6177 --- /dev/null +++ b/home-k3s/install-host.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Run as root on Karnataka. Leaves Utah, host firewall and Tailscale identity intact. +# K3s state stays in its default writable /var/lib/rancher/k3s. The upstream +# SELinux module cannot be added to Utah's read-only policy store at runtime; +# K3s SELinux container labeling is not enabled. Host SELinux stays enforcing. +set -euo pipefail +cd "$(dirname "$0")" +[ "$(id -u)" = 0 ] || { echo 'Run as root' >&2; exit 1; } +[ "$(hostname -s)" = karnataka ] || { echo 'Wrong host' >&2; exit 1; } +ip -4 addr show dev tailscale0 | grep -q '100\.66\.158\.127/32' || { echo 'Expected Tailscale address is absent' >&2; exit 1; } +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT +curl -fL --retry 3 'https://github.com/k3s-io/k3s/releases/download/v1.36.5%2Bk3s1/k3s' -o "$tmp" +printf '%s %s\n' d73847bcd3c5fccef0115b372e2f9a91f3032dc84bbf71518a4617565294d313 "$tmp" | sha256sum -c - +install -D -m 0755 "$tmp" /var/usrlocal/bin/k3s +install -D -m 0755 wait-tailnet.sh /var/usrlocal/libexec/k3s-wait-tailnet +install -D -m 0600 config.yaml /etc/rancher/k3s/config.yaml +install -D -m 0644 k3s.service /etc/systemd/system/k3s.service +install -D -m 0644 resolv.conf /etc/rancher/k3s/resolv.conf +systemctl daemon-reload +systemctl enable --now k3s.service diff --git a/home-k3s/k3s.service b/home-k3s/k3s.service new file mode 100644 index 00000000..eef18552 --- /dev/null +++ b/home-k3s/k3s.service @@ -0,0 +1,25 @@ +[Unit] +Description=K3s single-node server on Karnataka +Documentation=https://docs.k3s.io/ +Wants=network-online.target tailscaled.service +After=network-online.target tailscaled.service + +[Service] +Type=notify +Environment=PATH=/var/usrlocal/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin +ExecStartPre=/var/usrlocal/libexec/k3s-wait-tailnet +ExecStartPre=-/usr/sbin/modprobe br_netfilter +ExecStartPre=-/usr/sbin/modprobe overlay +ExecStart=/var/usrlocal/bin/k3s server +KillMode=process +Delegate=yes +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/home-k3s/resolv.conf b/home-k3s/resolv.conf new file mode 100644 index 00000000..2c38d459 --- /dev/null +++ b/home-k3s/resolv.conf @@ -0,0 +1,4 @@ +# Public DNS for Kubernetes pods; host Tailscale MagicDNS is not reachable +# from the pod network. Host DNS configuration remains unchanged. +nameserver 1.1.1.1 +nameserver 1.0.0.1 diff --git a/home-k3s/wait-tailnet.sh b/home-k3s/wait-tailnet.sh new file mode 100755 index 00000000..5067e188 --- /dev/null +++ b/home-k3s/wait-tailnet.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# The K3s API binds to this address; tailscaled can start before it exists. +set -u +for _ in $(seq 1 60); do + if ip -4 -o addr show dev tailscale0 2>/dev/null | grep -Fq 'inet 100.66.158.127/32 '; then + exit 0 + fi + sleep 1 +done +echo 'K3s cannot start: expected Tailscale address is not present' >&2 +exit 1 From c3bcfcceb9d1db677f0ae559ef9186000ff52fb3 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:05:52 +0000 Subject: [PATCH 24/62] Document Karnataka cluster firewall Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/talos-k8s/karnataka/README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index d25acbe0..b80dcccf 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -26,10 +26,17 @@ Host SELinux is enforcing and `container-selinux` is installed. The running Utah image does not include `k3s-selinux`, and its policy store is read-only; K3s SELinux container labeling remains disabled. Our proposed image build includes the policy, but has not been installed on this host. Do not enable -K3s SELinux until the policy and workload labeling are verified. Metrics-server -cannot currently reach the kubelet over the Tailscale address; node metrics -are unavailable. The default local-path volumes are node-local, without -snapshot or expansion support; deleting a claim deletes its data. +K3s SELinux until the policy and workload labeling are verified. The default +local-path volumes are node-local, without snapshot or expansion support; +deleting a claim deletes its data. + +Firewalld assigns `cni0` to the `internal` zone, permitting forwarding within +that zone. A rich rule permits `10.42.0.0/16` to reach the Tailscale address on +TCP port 10250 for kubelet metrics. The `internal` zone's default services, +including SSH, are also reachable from pods; do not attach an untrusted +interface to that zone. The public LAN interface remains in `public`. Both +bridge rules are present in the permanent configuration; verify them after +network or firewall changes with `firewall-cmd --zone=internal --list-all`. ## Historical configuration From 6b381e26f52c331701f521c570d5ad309afb747b Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:06:21 +0000 Subject: [PATCH 25/62] Record K3s network policy limitation Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/talos-k8s/karnataka/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index b80dcccf..aa20e244 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -38,6 +38,11 @@ interface to that zone. The public LAN interface remains in `public`. Both bridge rules are present in the permanent configuration; verify them after network or firewall changes with `firewall-cmd --zone=internal --list-all`. +K3s logged `iptables-save` nftables parsing errors during network-policy +reconciliation. No NetworkPolicy objects were present when checked. Policy +enforcement has not been verified; do not rely on it for isolation until the +iptables/nftables incompatibility is resolved and enforcement is tested. + ## Historical configuration Karnataka was the worker of the home Talos cluster. The [cluster handbook](../cluster.md), From bb953aa570ea4a286e645dda3225f853245df66c Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:57:24 +0000 Subject: [PATCH 26/62] Harden Karnataka fleet and cluster management Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- CLAUDE.md | 6 +- docs/src/roles/systemd.md | 2 +- docs/src/servers/home-k8s-images.md | 29 +-- docs/src/servers/talos-k8s/cluster.md | 55 +++-- .../src/servers/talos-k8s/karnataka/README.md | 19 +- docs/src/talos-k8s.md | 10 +- home-k3s/apply-stack.sh | 32 +++ home-k3s/corral-web.yaml | 223 ++++++++++++++++++ home-k3s/firewall.sh | 79 +++++++ home-k3s/install-host.sh | 23 +- home-k3s/local-path-profile.yaml | 11 + tests/test_home_k3s_bootstrap.py | 61 +++++ tests/test_home_k3s_firewall.py | 143 +++++++++++ tests/test_home_k3s_stack.py | 49 ++++ tests/test_laptop_system_services_check.py | 20 +- tests/test_system_services_migration.py | 29 +++ units/laptop-services.sh | 6 +- units/system-services.sh | 25 +- vm-migrations/punjab.yaml | 73 ++++++ 19 files changed, 836 insertions(+), 59 deletions(-) create mode 100755 home-k3s/apply-stack.sh create mode 100644 home-k3s/corral-web.yaml create mode 100755 home-k3s/firewall.sh create mode 100644 home-k3s/local-path-profile.yaml create mode 100644 tests/test_home_k3s_bootstrap.py create mode 100644 tests/test_home_k3s_firewall.py create mode 100644 tests/test_home_k3s_stack.py create mode 100644 vm-migrations/punjab.yaml diff --git a/CLAUDE.md b/CLAUDE.md index 5b8250ff..8143aef0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ Three distinct concerns live here: ### Cluster recovery and current hosts -- **Home:** Karnataka runs `ghcr.io/projectbluefin/utah:testing` and defaults to `multi-user.target`; it is in the fleet server inventory. Bihar was recently online but is currently unreachable; its OS and services are unverified. Retain the former home cluster's `~/.kube/config`, `~/.talos/config`, Bitwarden notes and `talos-k8s/` manifests as recovery material; don't assume the cluster is running or operate historical recovery commands on these machines. +- **Home:** Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` as its default and a single-node K3s `v1.36.5+k3s1` server bound to its tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are installed; `corral-vms/punjab` is a staging VM, not a migrated replacement for active EC2 Punjab. Bihar's OS and services are unverified. Retain the former home cluster's `~/.kube/config`, `~/.talos/config`, Bitwarden notes and `talos-k8s/` manifests as recovery material; don't operate historical recovery commands on these machines. - **AWS** (`eu-north-1`, built 2026-08-27) — **terminated 2026-10-25** after credits were exhausted. It previously ran Matrix, the Hives and the CFP dashboard. The S3 bucket `hanthor-fleet-backups-*` and the himachal-1 mirror (`~/backups/tunaos-aws-2026-10-25`) were retained; verify individual AWS resources rather than assuming they are gone. OpenTofu still describes the old cluster and can propose recreating resources. Configs retained as recovery material: `~/.kube/config-aws-migration`, `~/.talos/config-aws-migration`. Handbook: [`docs/src/servers/aws-k8s/cluster.md`](docs/src/servers/aws-k8s/cluster.md). @@ -67,7 +67,7 @@ The former Bihar control plane and Karnataka worker ran Talos `v1.13.2`, Kuberne - **punjab** (this repo's AWS-hosted fleet host) is an EC2 `t3.large` in `us-east-1`; the former Talos cluster in `eu-north-1` is terminated. `runs-on` (us-east-2) is its own CloudFormation stack — don't import it. - **Do not blindly apply `aws/`:** its configuration still describes the terminated cluster and may propose recreating paid infrastructure. Inspect live resources and review plans before changes; credits are exhausted. - State is in S3 (`hanthor-fleet-backups-*/tofu/aws/`) and contains Talos PKI. `aws/terraform.tfvars` is gitignored and lives in Bitwarden note `aws-tofu-tfvars` (`just aws-tfvars` / `just aws-seed-tfvars`). -- punjab uses Tailscale for access and SSM Session Manager for break-glass. Preserve its data and the S3 backups until a verified replacement and an explicit cleanup decision exist. +- EC2 punjab remains active and uses Tailscale for access and SSM Session Manager for break-glass. Do not stop it; preserve its data and the S3 backups until a verified replacement and an explicit cleanup decision exist. - The former cluster uploaded Postgres dumps to `s3://hanthor-fleet-backups-*/postgres/`; the `postgres-backup-writer` key is in Bitwarden, never in tofu state. - `prevent_destroy` in configuration does not prevent creation of resources missing from the account. Never put Talos `user_data` in the config. @@ -79,7 +79,7 @@ Renovate and the Hive (hive.reilly.asia, `hanthor-hive-agent`) open PRs; master - **Don't commit `talosconfig`, `controlplane.yaml`, `worker.yaml`, or `~/.kube/config`.** `talos-k8s/.gitignore` covers the first three; the kubeconfig isn't in the repo path at all. - **Don't reintroduce NetBox.** LAN inventory comes from `just inventory`. -- **Don't add files to `karnataka/`** — that directory was deleted in cleanup. Workstation-specific config goes in `host_vars/karnataka.yml`; cluster manifests go in `talos-k8s/`. +- **Don't add files to `karnataka/`** — that directory was deleted in cleanup. Workstation-specific config goes in `host_vars/karnataka.yml`; `talos-k8s/` retains historical cluster manifests, not instructions to apply them to the current K3s server. - **Don't rely on host sudo in automated/pi sessions** — `sudo -v` doesn't carry over across TTYs. - **This repo is public.** Secrets flow through Bitwarden only — never hardcode credentials. - **Don't commit tofu state, plans or `aws/terraform.tfvars`.** `aws/.gitignore` covers them. diff --git a/docs/src/roles/systemd.md b/docs/src/roles/systemd.md index 8c8ddc95..88f7e51f 100644 --- a/docs/src/roles/systemd.md +++ b/docs/src/roles/systemd.md @@ -4,6 +4,6 @@ When `/usr/bin/podman` exists, system-services creates the `podman` group, adds the current user, installs a socket drop-in with group access (0660), and enables `podman.socket`. It restarts an already-active socket only when the drop-in changed. This applies to laptops too. -For non-laptops, system-services renders `units/system-services/` templates for the user `dotfiles-update` service and timer and the Atuin daemon, then enables the timer and daemon. For `is_laptop: true`, laptop-services instead deploys the login-triggered update service and Atuin daemon, removes an obsolete update timer, and respects `dotfiles_auto_update`. The update uses `just apply-nosecrets`. +For non-laptops, system-services renders `units/system-services/` templates for the user `dotfiles-update` service and timer and the Atuin daemon, then enables the timer and daemon. It enables user linger through root or noninteractive `sudo -n` so the timer can run without an interactive login. Check mode reports missing linger without changing it. For `is_laptop: true`, laptop-services instead deploys the login-triggered update service and Atuin daemon, removes an obsolete update timer, and respects `dotfiles_auto_update`. The update uses `just apply-nosecrets`. On himachal, laptop-services also deploys `units/laptop-services/system-mitigator.py` and its user service. Unless `phone_apply_enabled: false`, it deploys a timer and service that archive the tracked checkout and run native units with `--no-secrets` independently over SSH on `termux` and `mumbai`. Unreachable phones do not fail that service. Disabling phone apply removes the timer and service. diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index 6ccc293a..85ab17ab 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -1,6 +1,6 @@ # Home Kubernetes Server Images -These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka currently runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` as its default; Bihar's current OS and services are unverified. Image build and installer acceptance are under repair: verify provenance and acceptance before installing or switching any host. +These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to tailnet address `100.66.158.127`; KubeVirt `v1.9.1`, CDI and Corral are installed. The `corral-vms/punjab` VM is staging, not a migrated replacement for active EC2 Punjab. Bihar's current OS and services are unverified. Image build and installer acceptance are under repair: verify provenance and acceptance before any future installation or switch. ## Image Variants @@ -47,7 +47,9 @@ just server-installer bihar ghcr.io/hanthor/home-k8s-bihar:testing artifacts/bih Both are public (no login required). -## Installation +## Proposed Installation (Not for Current Hosts) + +The following is retained for planning and disposable testing, not a rollout instruction. Do not install, switch, reboot, or activate these profiles on Karnataka or other existing hosts: installation can wipe disks, and the proposed Karnataka agent conflicts with its current K3s server. Any future change requires a separately approved plan and independently verified backups. Preserve the historical Talos and AWS recovery material. ### From Installer ISO @@ -58,16 +60,19 @@ Both are public (no login required). ``` 3. **Select image:** Custom URL → `oci:/usr/share/home-k8s/payload` 4. **Choose filesystem:** ext4 (default), xfs, or btrfs -5. **Set hostname:** `bihar`, `karnataka`, or `starter` (enables K3s role) +5. **Set hostname:** `bihar`, `karnataka`, or `starter` (selects the proposed K3s role) 6. **Create admin user:** `nodeadmin` (or any name) 7. **Confirm and install** -The TUI automatically configures: +The intended installed configuration includes: - Disk partitioning (EFI + /boot + root) -- K3s service enablement (based on hostname) +- The proposed hostname-based K3s role (gated until explicit activation) - Tailscale daemon (unenrolled at install time) - SELinux enforcing mode +Verify these against the specific installer and artifact; the proposed image's +SELinux policy is not the current Utah/K3s configuration. + After reboot, the node boots to **text console** (multi-user.target). K3s is gated (disabled) until you manually enable it. ### From OCI Archive (bootc switch) @@ -99,7 +104,7 @@ All three mkfs tools are included. Choose at install time or format a custom lay | **xfs** | High-throughput, servers | Fast journal | Requires `mkfs.xfs` (included) | | **btrfs** | Snapshots, COW | Modern, complex | Requires `mkfs.btrfs` + `btrfs` CLI | -Deployed images support all three without additional tools. If a filesystem option fails to appear in the TUI, verify the mkfs tool exists: `which mkfs.xfs` etc. +The intended images include all three tools; verify the actual artifact. If a filesystem option fails to appear in the TUI, verify the mkfs tool exists: `which mkfs.xfs` etc. ## Graphical Target (Desktop) @@ -130,7 +135,7 @@ sudo toggle-graphical --reboot ## K3s Enablement -K3s is **installed but gated (disabled)** at boot. Roles are determined by hostname: +In the proposed images, K3s is **installed but gated (disabled)** at boot. These hostname roles do not describe the current Karnataka server: | Hostname | K3s Role | Service | Enabled at boot | |----------|----------|---------|-----------------| @@ -171,11 +176,7 @@ kubectl get nodes -o wide ### iSCSI (Longhorn) -Both nodes are configured for Longhorn iSCSI workloads: -- `iscsid.socket` running -- `iscsi-initiator-utils` installed -- Unique initiator name generated at first boot -- `/var/lib/longhorn/` mounted to second NVMe (karnataka only) +The proposed images intend to support Longhorn iSCSI workloads. Verify actual service, package and storage configuration on any target; do not assume a second Karnataka NVMe is usable (historically the WD Black was failing). ### Temporary Disks @@ -264,7 +265,7 @@ sudo tailscale up ## See Also -- [Talos K8s Cluster Handbook](./talos-k8s/cluster.md) — Bihar & karnataka hardware, networking, advanced operations -- [AWS K8s Cluster Handbook](./aws-k8s/cluster.md) — Production cluster migration info +- [Talos K8s Cluster Handbook](./talos-k8s/cluster.md) — historical hardware, networking and recovery reference +- [AWS K8s Cluster Handbook](./aws-k8s/cluster.md) — historical cluster and recovery material - [Roles Documentation](../roles/) — Individual role configurations - [GitHub Releases](https://github.com/hanthor/dotfiles/releases) — Download ISOs and archives diff --git a/docs/src/servers/talos-k8s/cluster.md b/docs/src/servers/talos-k8s/cluster.md index 0219adb3..d31403fc 100644 --- a/docs/src/servers/talos-k8s/cluster.md +++ b/docs/src/servers/talos-k8s/cluster.md @@ -1,10 +1,15 @@ # Historical Cluster Handbook — Talos Linux on Bihar + Karnataka > **Historical architecture and recovery reference, not a current deployment.** -> Bihar was recently online but is currently unreachable; its OS and services -> are unverified. Karnataka runs `projectbluefin/utah:testing` and defaults to -> `multi-user.target`. Neither this handbook nor retained manifests establish -> what services either machine runs now. +> Bihar's OS and services are unverified. Karnataka runs `projectbluefin/utah:testing` +> with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to +> tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are installed; +> `corral-vms/punjab` is staging, not a migrated replacement for active EC2 Punjab. +> On the current server, firewalld puts `cni0` in the internal zone and accepts +> pod subnet `10.42.0.0/16` to kubelet port `10250` on that tailnet address; +> internal-zone services, including SSH, are accessible from pods. NetworkPolicy +> enforcement is unverified. Utah is enforcing SELinux, but `k3s-selinux` is +> absent and K3s container SELinux is disabled. > > **Do not run the reinstall, reset, bootstrap, configuration, migration, or > workload commands below against repurposed nodes.** They can wipe disks or @@ -182,11 +187,12 @@ After install, the node exposes `/dev/dri/card0`, `/dev/dri/renderD128`, and `/d > **Talos version note:** Pinned to `v1.13.2`. `v1.13.3` had multi-arch registry pull errors on AMD64 at the time of writing. -### User volume for /var/mnt/storage +### Historical user volume for /var/mnt/storage -nvme0n1 is the **Crucial P3** (healthy, boot disk). The WD Black failed with I/O errors -and was replaced as the boot disk. Forgejo / local-path PVCs use a `directory`-type -user volume on the Crucial P3's EPHEMERAL partition: +At the time, nvme0n1 was the **Crucial P3** (boot disk). The WD Black had failed +with I/O errors and was replaced as the boot disk. Forgejo / local-path PVCs used +a `directory`-type user volume on the Crucial P3's EPHEMERAL partition. Current +disk health and contents are not established by this record: ```bash # Apply the user volume config (idempotent): @@ -483,9 +489,9 @@ The former procedure for a wiped or replaced node was: If only the worker was reset, the existing kubeconfig still works — only steps 1, 2, 5, and 6 are needed for that node, plus re-applying any DaemonSet pods. -> **WD Black failure:** Do not use the WD Black (`/dev/nvme1n1`) for installs or user volumes. -> Install only to the Crucial P3 (`/dev/nvme0n1`). Storage uses a directory-type volume -> on the Crucial P3's EPHEMERAL partition. +> **Historical WD Black failure:** The former procedure excluded `/dev/nvme1n1` +> and selected the Crucial P3 (`/dev/nvme0n1`). Do not treat this as a current +> disk selection: verify identity, health, contents and backups before any recovery. --- @@ -563,13 +569,13 @@ After reboot (~2 min), all pods recover automatically. PVC-backed services ### Scope and acceptance status -This section records a proposed replacement OS and **new K3s cluster**, not a -Talos upgrade or evidence of current services. Bihar's present OS and services -are unverified. Karnataka runs `projectbluefin/utah:testing`, not the proposed -K3s worker. Do not install, switch, activate, or configure these -images on repurposed nodes. The procedures below and sections 1–11 are retained -for historical context and recovery planning only. The terminated -[AWS Talos cluster's](../aws-k8s/cluster.md) recovery material remains separate. +This section records a proposed replacement OS and **new two-node K3s cluster**, +not a Talos upgrade or instructions for the current single-node K3s server on +Utah. Bihar's present OS and services are unverified. Do not install, switch, +activate, or configure these images on Karnataka or other existing hosts. The +procedures below and sections 1–11 are retained for historical context and recovery +planning only. The terminated [AWS Talos cluster's](../aws-k8s/cluster.md) +recovery material remains separate. Image builds alone do not prove installation. The GitHub Actions acceptance workflow builds all three images and offline ISOs, drives the **real @@ -720,12 +726,13 @@ Keep the old Bitwarden secure notes `kubeconfig` / `talosconfig` and their `kubeconfig-aws-migration` / `talosconfig-aws-migration` notes and files unchanged. Use a new home kubeconfig below; do not overwrite any of these. -**Karnataka's `/dev/nvme1n1` (WD Black) is failing: never select it.** -Only the healthy `/dev/nvme0n1` is a candidate, **after off-node backups and an -explicit physical operator choice**. Identify disks by model, serial, size, -and current contents at the console (`lsblk -o NAME,SIZE,MODEL,SERIAL,FSTYPE,MOUNTPOINTS`); -do not assume device enumeration is sufficient. Apply the same backup and -operator-choice gate to Bihar's intended `/dev/nvme0n1`. +**Karnataka's `/dev/nvme1n1` (WD Black) was reported failing: never select it +without an independent assessment.** The former plan named `/dev/nvme0n1` as a +candidate, not as permission to overwrite the current Utah/K3s server. Any future +disk choice requires off-node backups and an explicit physical operator decision. +Identify disks by model, serial, size, and current contents at the console +(`lsblk -o NAME,SIZE,MODEL,SERIAL,FSTYPE,MOUNTPOINTS`); do not assume device +enumeration or historical health. Apply the same gate to Bihar. ### Build node images and optional starter media diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index aa20e244..7ab92440 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -6,6 +6,10 @@ available over Tailscale at `100.66.158.127`. A single-node K3s cluster runs KubeVirt, CDI, and Corral. The `corral-vms/punjab` VM is a staging guest; it is not the EC2 host and has not received its data or services. +As last checked, the fleet update timer is not installed and user linger is +disabled. The native-unit changes are under review; do not enable unattended +updates from the unmerged checkout. + ## Hardware - Arch: x86_64 @@ -22,6 +26,14 @@ loads `br_netfilter` and `overlay` when available. Run `systemctl status k3s` and `sudo /var/usrlocal/bin/k3s kubectl --server https://100.66.158.127:6443 get pods -A` to inspect it. Reloading the service definition does not restart K3s. +`home-k3s/install-host.sh` reuses the pinned binary and refuses to replace a +different binary while K3s is active. It does not restart an active service; +changes to server configuration take effect on the next start. The separate +`home-k3s/apply-stack.sh` uses conflict-checked server-side apply and is not +transactional: review ownership conflicts before retrying. The storage profile +and Corral manifests passed server-side dry-run against the current cluster; +operator and resource reapplication has not been exercised on it. + Host SELinux is enforcing and `container-selinux` is installed. The running Utah image does not include `k3s-selinux`, and its policy store is read-only; K3s SELinux container labeling remains disabled. Our proposed image build @@ -35,8 +47,11 @@ that zone. A rich rule permits `10.42.0.0/16` to reach the Tailscale address on TCP port 10250 for kubelet metrics. The `internal` zone's default services, including SSH, are also reachable from pods; do not attach an untrusted interface to that zone. The public LAN interface remains in `public`. Both -bridge rules are present in the permanent configuration; verify them after -network or firewall changes with `firewall-cmd --zone=internal --list-all`. +bridge rules are present in the permanent configuration. On Karnataka, +`sudo bash home-k3s/firewall.sh --check` inspects both runtime and permanent +state without changes. `--apply` is an explicit repair operation; it does not +reload firewalld and refuses unrelated interfaces or source bindings in the +`internal` zone. Keep a recovery path available for any firewall change. K3s logged `iptables-save` nftables parsing errors during network-policy reconciliation. No NetworkPolicy objects were present when checked. Policy diff --git a/docs/src/talos-k8s.md b/docs/src/talos-k8s.md index b27979b6..58953f1f 100644 --- a/docs/src/talos-k8s.md +++ b/docs/src/talos-k8s.md @@ -1,9 +1,11 @@ # Historical Talos Kubernetes Cluster -> **Historical reference, not a current deployment.** Bihar was recently online -> but is currently unreachable; its OS and services are unverified. Karnataka is -> running `projectbluefin/utah:testing`, with `multi-user.target` as its default. -> Retain `talos-k8s/`, cluster configs and Bitwarden +> **Historical reference, not a current deployment.** Bihar's OS and services +> are unverified. Karnataka runs `projectbluefin/utah:testing` with +> `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server on its +> tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are +> installed; `corral-vms/punjab` is a staging VM, not a migrated replacement +> for active EC2 Punjab. Retain `talos-k8s/`, cluster configs and Bitwarden > notes as recovery material, not proof either machine still runs Talos. > > **Do not run historical reinstall, configuration or workload commands against diff --git a/home-k3s/apply-stack.sh b/home-k3s/apply-stack.sh new file mode 100755 index 00000000..aaf53ce0 --- /dev/null +++ b/home-k3s/apply-stack.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Run as root on Karnataka, against this cluster only. Upstream asset digests +# are from the GitHub release metadata; the Corral image is pinned by digest. +# Server-side apply fails on ownership conflicts rather than forcing over live +# settings. Apply is not transactional; review any conflict before retrying. +set -euo pipefail +cd "$(dirname "$0")" +[ "$(id -u)" = 0 ] || { echo 'Run as root' >&2; exit 1; } +[ "$(hostname -s)" = karnataka ] || { echo 'Wrong host' >&2; exit 1; } +k=(/var/usrlocal/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml --server https://100.66.158.127:6443) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +fetch() { + local name=$1 hash=$2 url=$3 + curl -fsSL --retry 3 "$url" -o "$tmp/$name" + printf '%s %s\n' "$hash" "$tmp/$name" | sha256sum -c - +} +fetch kubevirt-operator.yaml 9ac45a436c49d32e6fc21d542bcb7600b970705297b9e6752f3ef9b0a1d292da https://github.com/kubevirt/kubevirt/releases/download/v1.9.1/kubevirt-operator.yaml +fetch kubevirt-cr.yaml 43106136dbce3312bdbfdeae612aacafc6c12da518d233f90645b4685d84a2af https://github.com/kubevirt/kubevirt/releases/download/v1.9.1/kubevirt-cr.yaml +fetch cdi-operator.yaml c7d92bd1b2ee1a3952a5990000437c08d58109f31c0bd48ab6188ec27a13e224 https://github.com/kubevirt/containerized-data-importer/releases/download/v1.66.1/cdi-operator.yaml +fetch cdi-cr.yaml a497f90de608c1df26f9ee4095289373f17132d74a2042ca03e00c17c964f8a7 https://github.com/kubevirt/containerized-data-importer/releases/download/v1.66.1/cdi-cr.yaml +"${k[@]}" apply --server-side --field-manager=home-k3s -f "$tmp/kubevirt-operator.yaml" +"${k[@]}" wait --for=condition=Established --timeout=120s crd/kubevirts.kubevirt.io +"${k[@]}" apply --server-side --field-manager=home-k3s -f "$tmp/kubevirt-cr.yaml" +"${k[@]}" wait -n kubevirt kubevirt/kubevirt --for=condition=Available --timeout=10m +"${k[@]}" apply --server-side --field-manager=home-k3s -f "$tmp/cdi-operator.yaml" +"${k[@]}" wait --for=condition=Established --timeout=120s crd/cdis.cdi.kubevirt.io +"${k[@]}" apply --server-side --field-manager=home-k3s -f "$tmp/cdi-cr.yaml" +"${k[@]}" wait cdi/cdi --for=condition=Available --timeout=10m +"${k[@]}" wait --for=create --timeout=120s storageprofile/local-path +"${k[@]}" apply --server-side --field-manager=home-k3s -f local-path-profile.yaml +"${k[@]}" apply --server-side --field-manager=home-k3s -f corral-web.yaml diff --git a/home-k3s/corral-web.yaml b/home-k3s/corral-web.yaml new file mode 100644 index 00000000..d311829d --- /dev/null +++ b/home-k3s/corral-web.yaml @@ -0,0 +1,223 @@ +# Based on tuna-os/corral commit 8cff2423a2ba6704348ef225e2ab66438112411b deploy/corral-web.yaml. +# Ingress intentionally omitted; access through an authenticated SSH port-forward only. +# Corral web UI — Proxmox-style dashboard for KubeVirt, served in-cluster. +# Image: built from the corral repo Containerfile, pushed to +# ghcr.io/tuna-os/corral (public — no pull secret needed). +--- +apiVersion: v1 +kind: Namespace +metadata: + name: corral + labels: + pod-security.kubernetes.io/enforce: privileged +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: corral-web + namespace: corral +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: corral-web +rules: + - apiGroups: ["kubevirt.io"] + resources: ["virtualmachines", "virtualmachineinstances"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["subresources.kubevirt.io"] + resources: + - virtualmachines/start + - virtualmachines/stop + - virtualmachines/restart + - virtualmachines/migrate + - virtualmachines/addvolume + - virtualmachines/removevolume + - virtualmachineinstances/vnc + - virtualmachineinstances/console + - virtualmachineinstances/pause + - virtualmachineinstances/unpause + - virtualmachineinstances/guestosinfo + - virtualmachineinstances/filesystemlist + - virtualmachineinstances/userlist + verbs: ["get", "update"] + # Snapshot / restore / clone / export (Proxmox-parity ops) + - apiGroups: ["snapshot.kubevirt.io"] + resources: ["virtualmachinesnapshots", "virtualmachinerestores"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["clone.kubevirt.io"] + resources: ["virtualmachineclones"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["export.kubevirt.io"] + resources: ["virtualmachineexports"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["cdi.kubevirt.io"] + resources: ["datavolumes"] + verbs: ["get", "list", "create", "delete"] + # PVC: create (new disks), patch (online expand), delete (cleanup) + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "create", "patch", "update", "delete"] + # Capability detection (storage classes + snapshot classes) + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots"] + verbs: ["get", "list"] + # Export download: virtctl vmexport --port-forward needs the token secret + + # a port-forward to the exporter pod. + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "delete"] + # User-defined custom image/ISO sources persist in a ConfigMap. + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + # Bootc builder reads pod logs to extract build variables. + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods/portforward"] + verbs: ["create"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "create", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] + # VDI claims (#226, #227): one Lease per desktop pool member. The opt-in + # broker (CORRAL_VDI_BROKER) claims, renews, and releases them. + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "create", "update", "delete"] + # Events tab (Proxmox-style task log) + live metrics on the summary + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list"] + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] + # Instancetype/preference pickers in the create wizard + - apiGroups: ["instancetype.kubevirt.io"] + resources: ["virtualmachineclusterinstancetypes", "virtualmachineclusterpreferences"] + verbs: ["get", "list"] + # Bootc builder creates jobs to build VM disks on-cluster. + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "create", "delete"] + # Tailscale operator detection (ApplyProxy): tailscaleOperatorPresent() checks + # for the `tailscale` IngressClass the operator registers. + - apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list"] + # Per-VM Tailscale operator proxy (ApplyProxy): create the SA/Role/RoleBinding/ + # Service/Deployment that expose a VM on the tailnet; VM deletion cleans them up. + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["services", "serviceaccounts"] + verbs: ["get", "list", "create", "update", "patch", "delete"] + # Per-VM proxy + schedule RBAC (ApplyProxy/DeleteProxy, cronops): the only + # Role/RoleBinding objects corral-web ever creates are namespaced ones named + # corral--proxy or corral-sched — never ClusterRoles or ClusterRoleBindings, + # and the proxy Role it grants holds only vmi get + vnc get (see + # GenerateProxyRBAC). Cluster scope (rather than a Role in the corral + # namespace) is required because VM namespaces are dynamic per create (-n); + # there is no static namespace list to bind to. `list` is deliberately + # absent: no code path lists roles/rolebindings (apply + get + delete only). + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: corral-web +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: corral-web +subjects: + - kind: ServiceAccount + name: corral-web + namespace: corral +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: corral-web + namespace: corral + labels: + app: corral-web +spec: + replicas: 1 + selector: + matchLabels: + app: corral-web + template: + metadata: + labels: + app: corral-web + spec: + serviceAccountName: corral-web + containers: + - name: corral-web + image: ghcr.io/tuna-os/corral@sha256:b0dcfc6972a050b2ac48f00c192844f1c59bb061591839cf4c330e8136c3880f + imagePullPolicy: IfNotPresent + # --metrics starts the background fleet collector and serves + # Prometheus text exposition on /metrics (ADR-0011, pkg/web/prometheus.go). + # Without it the endpoint exists but always renders corral_collection_success 0: + # the collector that would populate it never runs. + args: ["--metrics"] + env: + # This deployment manages its Kubernetes cluster by default. Other + # configured contexts remain visible in the aggregated inventory. + - name: CORRAL_DEFAULT_BACKEND + value: kubevirt + # Default VM namespace (code default is corral-vms). + # Override with CORRAL_NAMESPACE=custom-ns if needed. + - name: CORRAL_NAMESPACE + value: corral-vms + # The UI is open inside the cluster. Only trusted cluster workloads + # and SSH administrators may access it; do not expose this service. + ports: + - containerPort: 8006 + name: http + livenessProbe: + httpGet: + path: /healthz + port: 8006 + initialDelaySeconds: 2 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: 8006 + initialDelaySeconds: 2 + periodSeconds: 5 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: corral-web + namespace: corral +spec: + type: ClusterIP + selector: + app: corral-web + ports: + - name: http + port: 80 + targetPort: 8006 diff --git a/home-k3s/firewall.sh b/home-k3s/firewall.sh new file mode 100755 index 00000000..8faeeaf9 --- /dev/null +++ b/home-k3s/firewall.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Explicitly manage only cni0 in firewalld's existing internal zone. The +# internal zone's services, including SSH, remain reachable from pods. +# Neither mode reloads firewalld or changes public/tailscale interfaces. +set -euo pipefail + +case "${1:-}" in + --apply|--check) mode=$1 ;; + *) echo "Usage: $0 --apply|--check" >&2; exit 2 ;; +esac +[ "$#" -eq 1 ] || { echo "Usage: $0 --apply|--check" >&2; exit 2; } +[ "$(id -u)" = 0 ] || { echo 'Run as root' >&2; exit 1; } +[ "$(hostname -s)" = karnataka ] || { echo 'Wrong host' >&2; exit 1; } + +zone=internal +rule='rule family="ipv4" source address="10.42.0.0/16" destination address="100.66.158.127/32" port port="10250" protocol="tcp" accept' +firewall-cmd --state >/dev/null + +# A source binding would classify packets by address even on a public NIC. +# Also refuse unrelated interfaces or a default zone that implicitly includes +# them. Check both stores before making any change. +preflight() { + local scope=$1 interfaces sources iface + local -a opts=() names=() + [ "$scope" = runtime ] || opts=(--permanent) + interfaces=$(firewall-cmd "${opts[@]}" --zone="$zone" --list-interfaces) + sources=$(firewall-cmd "${opts[@]}" --zone="$zone" --list-sources) + [ -z "$sources" ] || { echo "$scope: internal has source bindings; refusing" >&2; return 1; } + read -r -a names <<< "$interfaces" + for iface in "${names[@]}"; do + [ "$iface" = cni0 ] || { echo "$scope: internal contains unrelated interface $iface; refusing" >&2; return 1; } + done +} +[ "$(firewall-cmd --get-default-zone)" != "$zone" ] || { + echo 'internal is the default zone; refusing' >&2; exit 1; +} +preflight runtime +preflight permanent + +# A false query exits 1; other errors must not be mistaken for missing rules. +ensure() { + local query=$1 add=$2 rc + shift 2 + if firewall-cmd --quiet "$@" "$query"; then + return 0 + else + rc=$? + [ "$rc" -eq 1 ] || return "$rc" + fi + if [ "$mode" = --check ]; then + echo "Missing $query ($*)" >&2 + return 1 + fi + firewall-cmd "$@" "$add" +} + +check_or_apply() { + local scope + local -a opts + for scope in permanent runtime; do + opts=() + [ "$scope" = runtime ] || opts=(--permanent) + # Install the bounded rule and intra-zone forwarding before binding cni0. + ensure "--query-rich-rule=$rule" "--add-rich-rule=$rule" "${opts[@]}" --zone="$zone" + ensure --query-forward --add-forward "${opts[@]}" --zone="$zone" + ensure --query-interface=cni0 --change-interface=cni0 "${opts[@]}" --zone="$zone" + done +} +check_or_apply +if [ "$mode" = --apply ]; then + # Verify the actual runtime and permanent state, not just command success. + [ "$(firewall-cmd --get-default-zone)" != "$zone" ] || { echo 'internal became the default zone' >&2; exit 1; } + preflight runtime + preflight permanent + mode=--check + check_or_apply +fi + +echo 'internal cni0 forwarding and kubelet rule are present in runtime and permanent configuration' diff --git a/home-k3s/install-host.sh b/home-k3s/install-host.sh index 042d6177..014c9a8d 100755 --- a/home-k3s/install-host.sh +++ b/home-k3s/install-host.sh @@ -7,15 +7,26 @@ set -euo pipefail cd "$(dirname "$0")" [ "$(id -u)" = 0 ] || { echo 'Run as root' >&2; exit 1; } [ "$(hostname -s)" = karnataka ] || { echo 'Wrong host' >&2; exit 1; } -ip -4 addr show dev tailscale0 | grep -q '100\.66\.158\.127/32' || { echo 'Expected Tailscale address is absent' >&2; exit 1; } -tmp=$(mktemp) -trap 'rm -f "$tmp"' EXIT -curl -fL --retry 3 'https://github.com/k3s-io/k3s/releases/download/v1.36.5%2Bk3s1/k3s' -o "$tmp" -printf '%s %s\n' d73847bcd3c5fccef0115b372e2f9a91f3032dc84bbf71518a4617565294d313 "$tmp" | sha256sum -c - -install -D -m 0755 "$tmp" /var/usrlocal/bin/k3s +[ "$(uname -m)" = x86_64 ] || { echo 'K3s asset requires x86_64' >&2; exit 1; } +ip -4 -o addr show dev tailscale0 | grep -Fq 'inet 100.66.158.127/32 ' || { echo 'Expected Tailscale address is absent' >&2; exit 1; } +binary=/var/usrlocal/bin/k3s +checksum=d73847bcd3c5fccef0115b372e2f9a91f3032dc84bbf71518a4617565294d313 +if ! printf '%s %s\n' "$checksum" "$binary" | sha256sum --check --status; then + if systemctl is-active --quiet k3s.service; then + echo 'Refusing to replace a different K3s binary while the service is running' >&2 + exit 1 + fi + tmp=$(mktemp) + trap 'rm -f "$tmp"' EXIT + curl -fL --retry 3 'https://github.com/k3s-io/k3s/releases/download/v1.36.5%2Bk3s1/k3s' -o "$tmp" + printf '%s %s\n' "$checksum" "$tmp" | sha256sum -c - + install -D -m 0755 "$tmp" "$binary" +fi install -D -m 0755 wait-tailnet.sh /var/usrlocal/libexec/k3s-wait-tailnet install -D -m 0600 config.yaml /etc/rancher/k3s/config.yaml install -D -m 0644 k3s.service /etc/systemd/system/k3s.service install -D -m 0644 resolv.conf /etc/rancher/k3s/resolv.conf systemctl daemon-reload systemctl enable --now k3s.service +echo 'Existing K3s processes are not restarted; changed service or server configuration takes effect on the next start.' +echo 'Manage and verify the host firewall separately with firewall.sh --apply and --check.' diff --git a/home-k3s/local-path-profile.yaml b/home-k3s/local-path-profile.yaml new file mode 100644 index 00000000..4b276f69 --- /dev/null +++ b/home-k3s/local-path-profile.yaml @@ -0,0 +1,11 @@ +# CDI has no built-in profile for rancher.io/local-path. This provisioner +# supports filesystem RWO, not block volumes, RWX, expansion, or snapshots. +apiVersion: cdi.kubevirt.io/v1beta1 +kind: StorageProfile +metadata: + name: local-path +spec: + claimPropertySets: + - accessModes: [ReadWriteOnce] + volumeMode: Filesystem + cloneStrategy: copy diff --git a/tests/test_home_k3s_bootstrap.py b/tests/test_home_k3s_bootstrap.py new file mode 100644 index 00000000..bbdc3549 --- /dev/null +++ b/tests/test_home_k3s_bootstrap.py @@ -0,0 +1,61 @@ +import subprocess + +from conftest import REPO, write_stub + + +SCRIPT = REPO / "home-k3s" / "install-host.sh" + + +def run_install(tmp_path, stub_env, *, matches, active): + bindir, env = stub_env + write_stub(bindir, "id", "echo 0\n") + write_stub(bindir, "hostname", "echo karnataka\n") + write_stub(bindir, "uname", "echo x86_64\n") + write_stub(bindir, "ip", "echo '1: tailscale0 inet 100.66.158.127/32 scope global tailscale0'\n") + write_stub( + bindir, + "sha256sum", + '''cat >/dev/null +if [ "${1:-}" = --check ]; then exit "$MATCH_EXIT"; fi +exit 0 +''', + ) + write_stub( + bindir, + "systemctl", + '''echo "systemctl $*" >> "$INSTALL_LOG" +if [ "${1:-}" = is-active ]; then exit "$ACTIVE_EXIT"; fi +''', + ) + write_stub(bindir, "install", 'echo "install $*" >> "$INSTALL_LOG"\n') + write_stub( + bindir, + "curl", + '''echo "curl $*" >> "$INSTALL_LOG" +while [ "$#" -gt 0 ]; do + if [ "$1" = -o ]; then shift; echo asset > "$1"; break; fi + shift +done +''', + ) + log = tmp_path / "install.log" + env.update(INSTALL_LOG=str(log), MATCH_EXIT="0" if matches else "1", ACTIVE_EXIT="0" if active else "3") + result = subprocess.run(["bash", str(SCRIPT)], env=env, text=True, capture_output=True) + return result, log.read_text().splitlines() if log.exists() else [] + + +def test_matching_binary_does_not_download_or_replace_running_k3s(tmp_path, stub_env): + result, calls = run_install(tmp_path, stub_env, matches=True, active=True) + assert result.returncode == 0, result.stderr + assert not any(call.startswith("curl ") for call in calls) + assert not any(call.startswith("install ") and call.endswith(" /var/usrlocal/bin/k3s") for call in calls) + assert not any("restart" in call for call in calls) + + +def test_different_binary_is_not_replaced_while_running(tmp_path, stub_env): + result, calls = run_install(tmp_path, stub_env, matches=False, active=True) + assert result.returncode != 0 + assert "Refusing to replace" in result.stderr + assert calls == ["systemctl is-active --quiet k3s.service"] + + diff --git a/tests/test_home_k3s_firewall.py b/tests/test_home_k3s_firewall.py new file mode 100644 index 00000000..9f06a4f1 --- /dev/null +++ b/tests/test_home_k3s_firewall.py @@ -0,0 +1,143 @@ +import json +import subprocess + +import pytest + +from conftest import REPO, write_stub + + +SCRIPT = REPO / "home-k3s" / "firewall.sh" + + +@pytest.fixture +def firewall(tmp_path, stub_env): + bindir, env = stub_env + write_stub(bindir, "id", 'echo 0\n') + write_stub(bindir, "hostname", 'echo karnataka\n') + write_stub( + bindir, + "firewall-cmd", + r'''exec python3 - "$@" <<'PY' +import json, os, sys +p = os.environ["FIREWALL_STATE"] +s = json.load(open(p)) +args = sys.argv[1:] +scope = "permanent" if "--permanent" in args else "runtime" +zone = s[scope] +if "--state" in args: + print("running") +elif "--get-default-zone" in args: + print(s["default"]) +elif "--list-interfaces" in args: + print(" ".join(zone["interfaces"])) +elif "--list-sources" in args: + print(" ".join(zone["sources"])) +else: + op = next(a for a in args if a.startswith(("--query-", "--add-", "--change-"))) + command, _, value = op.partition("=") + key = {"--query-rich-rule": "rule", "--add-rich-rule": "rule", + "--query-forward": "forward", "--add-forward": "forward", + "--query-interface": "interfaces", "--change-interface": "interfaces"}[command] + if command.startswith("--query-"): + if os.environ.get("FAIL_QUERY"): + sys.exit(2) + present = value in zone[key] if key == "interfaces" else zone[key] + if key == "rule" and present: + present = value == zone[key] + sys.exit(0 if present else 1) + s["changes"].append([scope, op]) + if key == "interfaces": + zone[key].append(value) + else: + zone[key] = value if key == "rule" else True + with open(p, "w") as f: + json.dump(s, f) +PY +''', + ) + path = tmp_path / "firewall.json" + state = { + "default": "public", + "runtime": {"interfaces": [], "sources": [], "forward": False, "rule": ""}, + "permanent": {"interfaces": [], "sources": [], "forward": False, "rule": ""}, + "changes": [], + } + path.write_text(json.dumps(state)) + env["FIREWALL_STATE"] = str(path) + + def run(mode): + result = subprocess.run(["bash", str(SCRIPT), mode], env=env, text=True, capture_output=True) + return result, json.loads(path.read_text()) + + return path, env, run + + +def test_apply_is_scoped_and_idempotent(firewall): + _, _, run = firewall + result, state = run("--apply") + assert result.returncode == 0, result.stderr + for scope in ("runtime", "permanent"): + zone = state[scope] + assert zone["interfaces"] == ["cni0"] + assert zone["sources"] == [] + assert zone["forward"] is True + result, second = run("--apply") + assert result.returncode == 0, result.stderr + assert second == state + result, checked = run("--check") + assert result.returncode == 0, result.stderr + assert checked == state + + +def test_check_reports_missing_without_changes(firewall): + _, _, run = firewall + result, state = run("--check") + assert result.returncode != 0 + assert "Missing" in result.stderr + assert not state["changes"] + + + +def test_check_accepts_host_rule_from_real_firewalld(firewall): + path, _, run = firewall + state = json.loads(path.read_text()) + for scope in ("runtime", "permanent"): + state[scope].update( + interfaces=["cni0"], forward=True, + rule='rule family="ipv4" source address="10.42.0.0/16" ' + 'destination address="100.66.158.127/32" port port="10250" protocol="tcp" accept', + ) + path.write_text(json.dumps(state)) + result, after = run("--check") + assert result.returncode == 0, result.stderr + assert after == state + +@pytest.mark.parametrize("scope", ["runtime", "permanent"]) +@pytest.mark.parametrize("field,value", [("interfaces", "eth0"), ("sources", "10.42.0.0/16")]) +def test_foreign_interface_or_source_refused_before_any_changes(firewall, scope, field, value): + path, _, run = firewall + state = json.loads(path.read_text()) + state[scope][field] = [value] + path.write_text(json.dumps(state)) + result, after = run("--apply") + assert result.returncode != 0 + assert "refusing" in result.stderr + assert after == state + + +def test_internal_default_zone_refused(firewall): + path, _, run = firewall + state = json.loads(path.read_text()) + state["default"] = "internal" + path.write_text(json.dumps(state)) + result, after = run("--apply") + assert result.returncode != 0 + assert after == state + + +def test_query_error_is_not_treated_as_missing(firewall): + _, env, run = firewall + env["FAIL_QUERY"] = "1" + result, state = run("--apply") + assert result.returncode == 2 + assert not state["changes"] diff --git a/tests/test_home_k3s_stack.py b/tests/test_home_k3s_stack.py new file mode 100644 index 00000000..337b43b3 --- /dev/null +++ b/tests/test_home_k3s_stack.py @@ -0,0 +1,49 @@ +import subprocess + +from conftest import REPO, write_stub + + +ASSETS = REPO / "home-k3s" + + +def run_stack(tmp_path, stub_env, fail_apply=""): + bindir, env = stub_env + write_stub(bindir, "id", "echo 0\n") + write_stub(bindir, "hostname", "echo karnataka\n") + write_stub( + bindir, + "curl", + '''while [ "$#" -gt 0 ]; do + if [ "$1" = -o ]; then shift; : > "$1"; break; fi + shift +done +''', + ) + write_stub(bindir, "sha256sum", "cat >/dev/null\n") + write_stub( + bindir, + "k3s", + '''echo "$*" >> "$K3S_LOG" +if [ -n "$FAIL_APPLY" ] && [[ "$*" == *"apply --server-side"*"$FAIL_APPLY"* ]]; then + echo 'simulated ownership conflict' >&2 + exit 1 +fi +''', + ) + log = tmp_path / "k3s.log" + env.update(K3S_LOG=str(log), FAIL_APPLY=fail_apply) + script = tmp_path / "apply-stack.sh" + original = (ASSETS / "apply-stack.sh").read_text() + script.write_text(original.replace("k=(/var/usrlocal/bin/k3s kubectl ", "k=(k3s kubectl ", 1)) + result = subprocess.run(["bash", str(script)], env=env, text=True, capture_output=True) + return result, log.read_text().splitlines() if log.exists() else [] + + + +def test_ownership_conflict_stops_subsequent_changes(tmp_path, stub_env): + result, calls = run_stack(tmp_path, stub_env, "kubevirt-cr.yaml") + assert result.returncode != 0 + assert "simulated ownership conflict" in result.stderr + assert not any("cdi-operator.yaml" in call for call in calls) + + diff --git a/tests/test_laptop_system_services_check.py b/tests/test_laptop_system_services_check.py index aa665545..c69d785e 100644 --- a/tests/test_laptop_system_services_check.py +++ b/tests/test_laptop_system_services_check.py @@ -16,11 +16,11 @@ def systemctl_stub(bindir): if [ "${INSPECTION_ERROR:-0}" = 1 ]; then exit 1; fi echo "${ENABLED_STATE:-enabled}" [ "${ENABLED_STATE:-enabled}" = enabled ] && exit 0 - exit 1 ;; + exit "${ENABLED_RC:-1}" ;; is-active) echo "${ACTIVE_STATE:-active}" [ "${ACTIVE_STATE:-active}" = active ] && exit 0 - exit 3 ;; + exit "${ACTIVE_RC:-3}" ;; *) [ "${CHECK_GUARD:-0}" != 1 ] || exit 99 ;; esac ''') @@ -105,7 +105,7 @@ def test_laptop_missing_units_and_inspection_failures(stub_env, tmp_path): home = Path(env["HOME"]) before = snapshot(home) args = {"DOTFILES_DIR": str(repo), "DOTFILES_MACHINE": "kanpur"} - result = run_unit("laptop-services", {**env, "ENABLED_STATE": "not-found", "ACTIVE_STATE": "inactive"}, **args) + result = run_unit("laptop-services", {**env, "ENABLED_STATE": "not-found", "ENABLED_RC": "4", "ACTIVE_STATE": "inactive", "ACTIVE_RC": "4"}, **args) assert result.returncode == 0, result.stdout + result.stderr assert "would install" in result.stdout and "would start atuin-daemon.service" in result.stdout assert snapshot(home) == before @@ -155,6 +155,20 @@ def test_system_user_service_check_reports_drift_and_missing_template(stub_env, assert_read_only_calls(env) +def test_system_check_handles_absent_unit_but_rejects_manager_error(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + env = check_env(tmp_path, original_env) + program, env, _ = setup_system_services(tmp_path, env, "server", "", False) + missing = run_script(program, {**env, "ENABLED_STATE": "not-found", "ENABLED_RC": "4", "ACTIVE_STATE": "inactive", "ACTIVE_RC": "4"}) + assert missing.returncode == 0, missing.stdout + missing.stderr + assert "would start dotfiles-update.timer" in missing.stdout + failed = run_script(program, {**env, "ACTIVE_STATE": "inactive", "ACTIVE_RC": "4"}) + assert failed.returncode != 0 + assert "Cannot inspect active state" in failed.stderr + assert_read_only_calls(env) + + def test_system_clean_check_and_manager_error(stub_env, tmp_path): bindir, original_env = stub_env systemctl_stub(bindir) diff --git a/tests/test_system_services_migration.py b/tests/test_system_services_migration.py index 1e1c67b6..3a166847 100644 --- a/tests/test_system_services_migration.py +++ b/tests/test_system_services_migration.py @@ -6,6 +6,7 @@ def setup_system_services(tmp_path, env, host, variables, podman): + write_stub(Path(env["PATH"].split(":", 1)[0]), "loginctl", 'echo yes\n') repo = tmp_path / "repo" (repo / "host_vars").mkdir(parents=True) (repo / "host_vars" / f"{host}.yml").write_text(variables) @@ -58,6 +59,34 @@ def test_nonlaptop_daily_timer_and_user_service_are_idempotent(stub_env, tmp_pat assert "restart" not in calls(env) +def test_headless_timer_enables_linger_without_mutating_check_mode(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", '''echo "$*" >> "$HOME/calls" +case "$1 $2" in + "--user is-enabled") echo enabled ;; + "--user is-active") echo active ;; +esac +''') + program, env, _ = setup_system_services(tmp_path, env, "server", "", False) + write_stub(bindir, "sudo", 'test "$1" = -n && shift\nexec "$@"\n') + write_stub(bindir, "loginctl", '''case "$1" in + show-user) if [ -e "$STUB_STATE/linger" ]; then echo yes; else echo no; fi ;; + enable-linger) touch "$STUB_STATE/linger" ;; + *) exit 1 ;; +esac +''') + check = run(program, {**env, "UNITS_CHECK": "1"}) + assert check.returncode == 0, check.stdout + check.stderr + assert "would enable user linger" in check.stdout + assert not (tmp_path / "linger").exists() + applied = run(program, env) + assert applied.returncode == 0, applied.stdout + applied.stderr + assert (tmp_path / "linger").exists() + second = run(program, env) + assert second.returncode == 0, second.stdout + second.stderr + assert "ok ensure user linger" in second.stdout + + def test_missing_template_does_not_start_stale_update_timer(stub_env, tmp_path): bindir, env = stub_env systemctl_stub(bindir) diff --git a/units/laptop-services.sh b/units/laptop-services.sh index dc3163cd..6e880543 100755 --- a/units/laptop-services.sh +++ b/units/laptop-services.sh @@ -63,7 +63,7 @@ check_file() { [ "$actual" = "$mode" ] || note "would set mode $mode on $dest" } check_enabled() { - local unit="$1" start="$2" state rc + local unit="$1" start="$2" state enabled_state rc if state=$(systemctl --user is-enabled "$unit" 2>/dev/null); then rc=0; else rc=$?; fi case "$state:$rc" in enabled:0) ;; @@ -71,10 +71,14 @@ check_enabled() { note "would enable $unit" ;; *) echo "Cannot inspect enabled state of $unit" >&2; return 1 ;; esac + enabled_state=$state [ "$start" = 1 ] || return 0 if state=$(systemctl --user is-active "$unit" 2>/dev/null); then rc=0; else rc=$?; fi case "$state:$rc" in active:0|reloading:0|refreshing:0) ;; + inactive:4) + [ "$enabled_state" = not-found ] || { echo "Cannot inspect active state of $unit" >&2; return 1; } + note "would start $unit" ;; inactive:3|failed:3|activating:3|deactivating:3|unknown:3|unknown:4) note "would start $unit" ;; *) echo "Cannot inspect active state of $unit" >&2; return 1 ;; diff --git a/units/system-services.sh b/units/system-services.sh index f6a92891..e0fe278f 100755 --- a/units/system-services.sh +++ b/units/system-services.sh @@ -52,7 +52,7 @@ check_file() { [ "$actual" = "$mode" ] || note "would set mode $mode on $dest" } check_enabled() { - local scope="$1" unit="$2" state rc + local scope="$1" unit="$2" state enabled_state rc local -a args=() [ "$scope" != user ] || args+=(--user) CHECK_ACTIVE=0 @@ -63,9 +63,13 @@ check_enabled() { note "would enable $unit" ;; *) echo "Cannot inspect enabled state of $unit" >&2; return 1 ;; esac + enabled_state=$state if state=$(systemctl "${args[@]}" is-active "$unit" 2>/dev/null); then rc=0; else rc=$?; fi case "$state:$rc" in active:0|reloading:0|refreshing:0) CHECK_ACTIVE=1 ;; + inactive:4) + [ "$enabled_state" = not-found ] || { echo "Cannot inspect active state of $unit" >&2; return 1; } + note "would start $unit" ;; inactive:3|failed:3|activating:3|deactivating:3|unknown:3|unknown:4) note "would start $unit" ;; *) echo "Cannot inspect active state of $unit" >&2; return 1 ;; @@ -153,6 +157,22 @@ if [ "$(host_var is_laptop false)" = true ]; then finish fi +ensure_timer_linger() { + local user state + user=$(id -un) || return 1 + state=$(loginctl show-user "$user" --property=Linger --value 2>/dev/null) || return 1 + case "$state" in + yes) return 0 ;; + no) + if [ "${UNITS_CHECK:-0}" = 1 ]; then + note "would enable user linger" + else + root loginctl enable-linger "$user" + fi ;; + *) echo "Cannot inspect user linger for $user" >&2; return 1 ;; + esac +} + export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus" @@ -220,6 +240,9 @@ if render_service "$templates/atuin-daemon.service.j2" "$tmp/atuin-daemon.servic else step "render and deploy atuin daemon" false fi +if [ "$update_ready" = 1 ] && [ "$timer_ready" = 1 ]; then + step "ensure user linger" ensure_timer_linger +fi if [ "${UNITS_CHECK:-0}" = 1 ]; then [ "$changed" != 1 ] || note "would reload user systemd" if [ "$update_ready" = 1 ] && [ "$timer_ready" = 1 ]; then diff --git a/vm-migrations/punjab.yaml b/vm-migrations/punjab.yaml new file mode 100644 index 00000000..9927623d --- /dev/null +++ b/vm-migrations/punjab.yaml @@ -0,0 +1,73 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: corral-vms +--- +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: punjab + namespace: corral-vms + labels: + app.kubernetes.io/name: punjab +spec: + runStrategy: Always + dataVolumeTemplates: + - metadata: + name: punjab-root + spec: + source: + http: + url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img + storage: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 320Gi + template: + metadata: + labels: + kubevirt.io/domain: punjab + spec: + domain: + cpu: + cores: 4 + resources: + requests: + memory: 12Gi + devices: + disks: + - name: rootdisk + disk: + bus: virtio + - name: cloudinit + disk: + bus: virtio + interfaces: + - name: default + masquerade: {} + networks: + - name: default + pod: {} + volumes: + - name: rootdisk + dataVolume: + name: punjab-root + - name: cloudinit + cloudInitNoCloud: + userData: | + #cloud-config + hostname: punjab-migration + manage_etc_hosts: true + ssh_pwauth: false + users: + - name: ubuntu + groups: [adm, sudo] + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + lock_passwd: true + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKTteKkOac3kryqONYQB91nN9tuVTxogiiRG5LRmiiIH punjab From 0a58f0cc9498a1a4c8b4c8988f00fabf0bbbcb8d Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:58:24 +0000 Subject: [PATCH 27/62] Document Corral cluster trust boundary Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/talos-k8s/karnataka/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index 7ab92440..d211df13 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -34,6 +34,10 @@ transactional: review ownership conflicts before retrying. The storage profile and Corral manifests passed server-side dry-run against the current cluster; operator and resource reapplication has not been exercised on it. +Corral has no authentication in this deployment. Its ClusterIP is reachable +from cluster workloads and its service account can manage cluster resources; +only run trusted workloads, and do not expose the dashboard publicly. + Host SELinux is enforcing and `container-selinux` is installed. The running Utah image does not include `k3s-selinux`, and its policy store is read-only; K3s SELinux container labeling remains disabled. Our proposed image build From 7b9cd26b4c2e5cbefc2a857de517eaae9033550f Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:59:18 +0000 Subject: [PATCH 28/62] Pin an available Utah image base Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/talos-k8s/cluster.md | 8 +++++--- server-images/Containerfile | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/src/servers/talos-k8s/cluster.md b/docs/src/servers/talos-k8s/cluster.md index d31403fc..0783232a 100644 --- a/docs/src/servers/talos-k8s/cluster.md +++ b/docs/src/servers/talos-k8s/cluster.md @@ -647,9 +647,11 @@ unit and a real KubeVirt guest on physical Bihar before cutover. ### Image contents and state The source is `server-images/Containerfile`, `compose.sh`, `nodes/`, and `rootfs/`. -All three targets use the digest-pinned Hummingbird Community `os-minimal` base, -without a desktop. The Bihar/Karnataka images were proposed as direct installation -paths; `starter` was optional for future x86_64 AMD/Intel nodes. These are not current deployment instructions. +The earlier targets used a digest-pinned Hummingbird Community `os-minimal` +base without a desktop. The proposed targets now use Utah; see the +[current image guide](../home-k8s-images.md). Bihar/Karnataka images were +proposed as direct installation paths; `starter` was optional for future +x86_64 AMD/Intel nodes. These are not current deployment instructions. Explicit AMD/Intel GPU and CPU, Intel Wi-Fi, Realtek, MediaTek, Atheros, and Broadcom firmware subpackages come through the digest-pinned Utah package factory. That local repository is authenticated by its OCI digest, **not RPM signatures**. diff --git a/server-images/Containerfile b/server-images/Containerfile index d42ab564..0aaf1395 100644 --- a/server-images/Containerfile +++ b/server-images/Containerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=ghcr.io/projectbluefin/utah:latest +ARG BASE_IMAGE=ghcr.io/projectbluefin/utah@sha256:d291c8831ac330547e3f99f4e0b566416f050b7ca14a5c2fdd43c04f47f7d1b7 ARG PACKAGE_IMAGE=ghcr.io/projectbluefin/utah-packages@sha256:d257e97a0057e37da47995bb142c180e2352960ab13bd44594215616a395b717 FROM ${PACKAGE_IMAGE} AS packages FROM ${BASE_IMAGE} AS common From acc82ad6815b29c73126674f04cb2555bda33672 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:06:30 +0000 Subject: [PATCH 29/62] Include required K3s storage utilities Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- server-images/compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server-images/compose.sh b/server-images/compose.sh index c5b25bdb..42ce4f49 100755 --- a/server-images/compose.sh +++ b/server-images/compose.sh @@ -10,7 +10,7 @@ dnf5 -y --enable-repo=utah-packages --enable-repo=fedora-storage \ iwlwifi-dvm-firmware iwlwifi-mvm-firmware \ realtek-firmware mt7xxx-firmware atheros-firmware brcmfmac-firmware \ NetworkManager-wifi wpa_supplicant tailscale iptables-nft \ - xfsprogs btrfs-progs + xfsprogs btrfs-progs iscsi-initiator-utils nfs-utils curl --fail --location \ "https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s" -o /usr/bin/k3s From 75ea2e0dfff6bbe8f732276f21e92f8fe9a237aa Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:11:43 +0000 Subject: [PATCH 30/62] Unmask installer serial console on Utah Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- server-images/installer/configure.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server-images/installer/configure.sh b/server-images/installer/configure.sh index 9ed1aa5f..5fa02e34 100755 --- a/server-images/installer/configure.sh +++ b/server-images/installer/configure.sh @@ -6,6 +6,8 @@ dnf5 -y --enablerepo=fedora-storage --setopt=install_weak_deps=False \ install dracut-live systemd-boot-unsigned # These masks belong to live media only, never to the embedded server payload. systemctl mask k3s@server.service k3s@agent.service +# Utah masks serial getty; live-media acceptance and recovery need it. +systemctl unmask getty@tty1.service serial-getty@ttyS0.service systemctl enable getty@tty1.service serial-getty@ttyS0.service mkdir -p /var/roothome kernel=$(basename /usr/lib/modules/*) From 29ccd71e4796b19df64ecd417cdf5892e1b1b929 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:18:25 +0000 Subject: [PATCH 31/62] Clear vendor serial mask in installer image Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- server-images/installer/configure.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server-images/installer/configure.sh b/server-images/installer/configure.sh index 5fa02e34..26a6e987 100755 --- a/server-images/installer/configure.sh +++ b/server-images/installer/configure.sh @@ -6,8 +6,11 @@ dnf5 -y --enablerepo=fedora-storage --setopt=install_weak_deps=False \ install dracut-live systemd-boot-unsigned # These masks belong to live media only, never to the embedded server payload. systemctl mask k3s@server.service k3s@agent.service -# Utah masks serial getty; live-media acceptance and recovery need it. +# Utah masks this instance in both /etc and /usr; only the live image needs it. systemctl unmask getty@tty1.service serial-getty@ttyS0.service +if [ "$(readlink /usr/lib/systemd/system/serial-getty@ttyS0.service || :)" = /dev/null ]; then + rm /usr/lib/systemd/system/serial-getty@ttyS0.service +fi systemctl enable getty@tty1.service serial-getty@ttyS0.service mkdir -p /var/roothome kernel=$(basename /usr/lib/modules/*) From 5ec9a4b291d1663d112fe77146c8c153b58c598b Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:23:03 +0000 Subject: [PATCH 32/62] Provide live initramfs build scratch Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- server-images/installer/configure.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server-images/installer/configure.sh b/server-images/installer/configure.sh index 26a6e987..f45f2543 100755 --- a/server-images/installer/configure.sh +++ b/server-images/installer/configure.sh @@ -13,6 +13,8 @@ if [ "$(readlink /usr/lib/systemd/system/serial-getty@ttyS0.service || :)" = /de fi systemctl enable getty@tty1.service serial-getty@ttyS0.service mkdir -p /var/roothome +# The bootc payload clears /var; dracut requires a writable build scratch area. +install -d -m 1777 /var/tmp kernel=$(basename /usr/lib/modules/*) dracut --force --reproducible --no-hostonly --add dmsquash-live --omit 'tpm2-tss pcsc' --add-drivers 'squashfs overlay loop iso9660 sr_mod cdrom' "/usr/lib/modules/$kernel/initramfs.img" "$kernel" 2>&1 | tee /tmp/dracut-installer.log if grep -q 'dracut\[E\]: FAILED' /tmp/dracut-installer.log; then From 1a5986cb7c647c79a9acf3e492223f397ff2be93 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:35:34 +0000 Subject: [PATCH 33/62] Document Utah installer acceptance blocker Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/home-k8s-images.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index 85ab17ab..1e16833f 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -2,6 +2,12 @@ These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to tailnet address `100.66.158.127`; KubeVirt `v1.9.1`, CDI and Corral are installed. The `corral-vms/punjab` VM is staging, not a migrated replacement for active EC2 Punjab. Bihar's current OS and services are unverified. Image build and installer acceptance are under repair: verify provenance and acceptance before any future installation or switch. +The Utah-based images built in CI, but installer acceptance stopped at its +2 GiB size guard: the starter ISO was about 5.94 GiB. This build has not +passed installation or cluster acceptance. The guard protects +the existing publication workflow and must not simply be removed. See +[issue #351](https://github.com/hanthor/dotfiles/issues/351). + ## Image Variants | Variant | Role | CPU/GPU | Image URI | Installer ISO | @@ -16,7 +22,7 @@ The intended images include the following; existing published tags may contain o - **Base:** `projectbluefin/utah` (Hummingbird + GNOME 51 desktop + Bluefin theming) - **K3s:** Kubernetes v1.36.1 + SELinux policy -- **Storage:** iSCSI utils, NFS utils, Longhorn tools +- **Storage:** iSCSI and NFS utilities - **Networking:** NetworkManager, Tailscale, iptables-nft - **GPU:** AMD firmware + amdgpu kernel driver included by the image build - **Filesystems:** ext4, xfs, btrfs (all tools included) From d3d02b3c64beddfd9d84c9c354d38685d2c82fbf Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:27:37 +0000 Subject: [PATCH 34/62] Allow larger Utah media in acceptance Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- .github/workflows/server-images.yml | 13 ++++++------- docs/src/servers/home-k8s-images.md | 8 ++++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/server-images.yml b/.github/workflows/server-images.yml index de89f49d..3ca2b5e2 100644 --- a/.github/workflows/server-images.yml +++ b/.github/workflows/server-images.yml @@ -146,16 +146,15 @@ jobs: - name: Install starter and accept a real bootc switch run: | - # The recorded acceptance runner had 85 GiB free on a 145 GiB root - # filesystem. Keep sequential media construction: the extra retained - # ISO is bounded below 2 GiB, and the starter disk/registry are removed - # before the existing two-node acceptance. No third VM disk overlaps. + # Keep media construction sequential to bound runner disk usage. + # Utah includes a desktop and its offline payload, so allow up to + # 8 GiB per ISO/archive here. Publication has a separate asset limit. df -h "$ARTIFACTS" | tee "$ARTIFACTS/starter-capacity.log" just server-installer starter \ "ghcr.io/$REGISTRY_OWNER/home-k8s-starter:testing" \ "$ARTIFACTS/starter-installer.iso" \ 2>&1 | tee "$ARTIFACTS/starter-build-installer.log" - test "$(stat --format=%s "$ARTIFACTS/starter-installer.iso")" -lt 2147483648 + test "$(stat --format=%s "$ARTIFACTS/starter-installer.iso")" -lt 8589934592 uv run server-images/test-vm.py --artifacts "$ARTIFACTS" \ --accel kvm install starter \ 2>&1 | tee "$ARTIFACTS/starter-install-driver.log" @@ -164,7 +163,7 @@ jobs: podman save --format oci-archive \ --output "$ARTIFACTS/home-k8s-bihar.oci.tar" \ localhost/home-k8s-bihar:testing - test "$(stat --format=%s "$ARTIFACTS/home-k8s-bihar.oci.tar")" -lt 2147483648 + test "$(stat --format=%s "$ARTIFACTS/home-k8s-bihar.oci.tar")" -lt 8589934592 podman pull "$REGISTRY_FIXTURE_IMAGE" registry="home-k8s-starter-switch-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" podman run --detach --name "$registry" --pull=never \ @@ -233,7 +232,7 @@ jobs: just server-installer "$node" \ "ghcr.io/$REGISTRY_OWNER/home-k8s-$node:testing" "$iso" \ 2>&1 | tee "$ARTIFACTS/$node-build-installer.log" - test "$(stat --format=%s "$iso")" -lt 2147483648 + test "$(stat --format=%s "$iso")" -lt 8589934592 uv run server-images/test-vm.py --artifacts "$ARTIFACTS" \ --accel kvm install "$node" \ 2>&1 | tee "$ARTIFACTS/$node-install-driver.log" diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index 1e16833f..a0116480 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -2,10 +2,10 @@ These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to tailnet address `100.66.158.127`; KubeVirt `v1.9.1`, CDI and Corral are installed. The `corral-vms/punjab` VM is staging, not a migrated replacement for active EC2 Punjab. Bihar's current OS and services are unverified. Image build and installer acceptance are under repair: verify provenance and acceptance before any future installation or switch. -The Utah-based images built in CI, but installer acceptance stopped at its -2 GiB size guard: the starter ISO was about 5.94 GiB. This build has not -passed installation or cluster acceptance. The guard protects -the existing publication workflow and must not simply be removed. See +The Utah-based images built in CI; the starter ISO was about 5.94 GiB. +Acceptance allows media up to 8 GiB, but the publication workflow still +rejects release assets of 2 GiB or more. This build has not yet passed +installation or cluster acceptance. See [issue #351](https://github.com/hanthor/dotfiles/issues/351). ## Image Variants From 76818a2fff82e45e05ebbc201700aabf429da892 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:53:51 +0000 Subject: [PATCH 35/62] Record Utah installer boot failure Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/home-k8s-images.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index a0116480..4dcbe276 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -4,9 +4,10 @@ These are proposed bootable K3s images, not the current OS or workloads of eithe The Utah-based images built in CI; the starter ISO was about 5.94 GiB. Acceptance allows media up to 8 GiB, but the publication workflow still -rejects release assets of 2 GiB or more. This build has not yet passed -installation or cluster acceptance. See -[issue #351](https://github.com/hanthor/dotfiles/issues/351). +rejects release assets of 2 GiB or more ([issue #351](https://github.com/hanthor/dotfiles/issues/351)). +In [CI](https://github.com/hanthor/dotfiles/actions/runs/40602288941), the +starter installer completed, but its installed VM never presented a serial +login within ten minutes; installation and cluster acceptance did not pass. ## Image Variants From dc6340e8f1f0c43c90b9296881e87ae44fb31453 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:06:25 +0000 Subject: [PATCH 36/62] Complete native fleet migration parity Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- Justfile | 83 +++--- bootstrap.sh | 168 ++++++------ docs/src/automation.md | 11 +- docs/src/onboarding.md | 81 ++++-- docs/src/roles/apk_packages.md | 2 +- docs/src/roles/bluefin_common.md | 4 +- docs/src/roles/browser_fxa.md | 4 +- docs/src/roles/bst_dashboard.md | 2 +- docs/src/roles/flatpak.md | 4 +- docs/src/roles/fleet_facts.md | 4 + docs/src/roles/forgejo_registry.md | 2 +- docs/src/roles/github.md | 3 +- docs/src/roles/gnome.md | 2 +- docs/src/roles/homebrew.md | 2 +- docs/src/roles/kirocrew.md | 2 +- docs/src/roles/kube.md | 5 +- docs/src/roles/omp.md | 2 +- docs/src/roles/pipewire_audio.md | 8 +- docs/src/roles/proxy.md | 4 +- docs/src/roles/server_hardening.md | 4 +- docs/src/roles/shell_ai.md | 3 +- docs/src/roles/shell_atuin.md | 4 +- docs/src/roles/shell_dotfiles.md | 3 +- docs/src/roles/shell_fonts.md | 4 +- docs/src/roles/ssh_keys.md | 6 +- docs/src/roles/ssh_mesh.md | 2 +- docs/src/roles/sshd.md | 2 +- docs/src/roles/sudo.md | 4 +- docs/src/roles/syncthing.md | 4 +- docs/src/roles/systemd.md | 4 +- docs/src/roles/tailscale.md | 4 + docs/src/roles/tailscale_cert.md | 8 +- docs/src/roles/termux_packages.md | 4 +- docs/src/roles/zen_browser.md | 2 +- docs/src/runbook.md | 5 +- docs/src/scripts.md | 2 +- .../src/servers/talos-k8s/karnataka/README.md | 11 + group_vars/all.yml | 15 +- inventory.yml | 3 +- scripts/inventory_parser.py | 36 ++- scripts/onboard-machine.sh | 71 +++++ scripts/register-machine.py | 51 +++- scripts/run-units.sh | 12 +- tests/test_atuin_unit.py | 31 ++- tests/test_bluefin_sync.py | 14 + tests/test_bluefin_unit.py | 73 ++++++ tests/test_bluefin_wallpaper_rotate.py | 176 +++++-------- tests/test_bootstrap_units.py | 91 +++++-- tests/test_brew_flatpak_check_mode.py | 48 +++- tests/test_browser_fxa_check_mode.py | 16 +- tests/test_browser_fxa_unit.py | 96 +++++++ tests/test_check_small_packages_vpn.py | 22 +- tests/test_chezmoi_unit.py | 49 ++++ tests/test_fonts_unit.py | 33 ++- tests/test_forgejo_registry_unit.py | 34 +++ tests/test_github_unit.py | 66 +++++ tests/test_gnome_unit.py | 94 +++++++ tests/test_inventory_scripts.py | 20 +- tests/test_kube_unit.py | 44 ++++ tests/test_laptop_services_migration.py | 66 +++++ tests/test_laptop_system_services_check.py | 43 +++- tests/test_onboarding_pr.py | 89 +++++++ tests/test_osquery_native_inventory.py | 47 ++++ tests/test_package_native_units.py | 58 ++++- tests/test_pipewire_audio_check_mode.py | 43 ++++ tests/test_pipewire_audio_unit.py | 130 +++++++++- tests/test_platform_native_units.py | 94 ++++++- tests/test_proxy_cert_parity.py | 243 ++++++++++++++++++ tests/test_server_app_parity.py | 214 +++++++++++++++ tests/test_server_service_units.py | 57 +++- tests/test_server_units_check.py | 13 +- tests/test_shell_ai_unit.py | 60 ++++- tests/test_ssh_native_units.py | 110 +++++++- tests/test_sudo_sshd_check_units.py | 56 ++++ tests/test_syncthing_migration.py | 111 +++++++- tests/test_system_services_migration.py | 19 ++ tests/test_visual_units_check.py | 2 + tests/test_zen_browser_unit.py | 10 + units/apk-packages.conf | 2 +- units/atuin.sh | 16 +- units/bluefin.sh | 37 ++- units/bluefin/files/bluefin-sync.py | 2 + .../bluefin/files/bluefin-wallpaper-rotate.py | 30 ++- units/bluefin/files/install-extensions.py | 7 + units/browser-fxa.sh | 32 ++- units/browser-fxa/fxa_signin.py | 50 ++-- units/bst-dashboard.sh | 19 +- units/chezmoi.sh | 12 +- units/flatpak.sh | 62 +++-- units/fonts.sh | 48 ++-- units/forgejo-registry.sh | 8 +- units/github.sh | 52 +++- units/gnome-avatar.py | 16 +- units/gnome.sh | 40 ++- units/homebrew.sh | 8 +- units/kirocrew-sockets.py | 34 ++- units/kirocrew.sh | 29 ++- units/kube.sh | 15 +- units/laptop-services.sh | 82 ++++-- units/osquery.sh | 20 +- units/osquery_native_inventory.py | 26 +- units/pipewire-audio.sh | 85 +++--- .../templates/mic-dsp-link.service.j2 | 4 +- units/proxy-inventory.py | 46 ++++ units/proxy-serve.py | 54 ++++ units/proxy.sh | 45 ++-- units/server-hardening-file.py | 30 ++- units/server-hardening.sh | 108 +++++--- units/shell-ai.sh | 48 ++-- units/ssh-fleet.py | 33 ++- units/sshd.sh | 10 +- units/sudo.sh | 35 ++- units/syncthing.sh | 65 +++-- units/system-services.sh | 21 +- units/system-services/atuin-daemon.service.j2 | 2 +- .../dotfiles-update.service.j2 | 4 +- units/tailscale-cert.sh | 86 +++++-- units/tailscale-cert/renew.sh | 67 +++++ .../tailscale-cert-renew.service.j2 | 6 +- units/termux-packages.conf | 2 +- units/termux-packages.sh | 86 +++++-- units/termux-start-sshd | 9 +- units/zen-browser.sh | 13 +- 123 files changed, 3721 insertions(+), 814 deletions(-) create mode 100755 scripts/onboard-machine.sh create mode 100644 tests/test_onboarding_pr.py create mode 100644 tests/test_proxy_cert_parity.py create mode 100644 tests/test_server_app_parity.py create mode 100755 units/proxy-inventory.py create mode 100755 units/proxy-serve.py create mode 100755 units/tailscale-cert/renew.sh diff --git a/Justfile b/Justfile index 65339206..ed2b66e8 100644 --- a/Justfile +++ b/Justfile @@ -165,35 +165,27 @@ purge name: git diff --cached --quiet || git commit -m "inventory: remove {{ name }}" git push -# Register a new machine in inventory and print bootstrap instructions - -# Usage: just onboard [desktop|server|vps] +# Propose a machine registration for review, then print bootstrap instructions. +# Usage: just onboard [desktop|server|vps|termux_hosts] onboard name type="desktop": #!/usr/bin/env bash set -euo pipefail - cd {{ dotfiles_dir }} - git pull --ff-only - - python3 scripts/register-machine.py {{ name }} {{ type }} inventory.yml - - if [ ! -f "host_vars/{{ name }}.yml" ]; then - echo "---" > "host_vars/{{ name }}.yml" - fi - - git add inventory.yml host_vars/{{ name }}.yml - git diff --cached --quiet || git commit -m "inventory: add {{ name }} ({{ type }})" - git push - + cd {{quote(justfile_directory())}} + name={{quote(name)}} + type={{quote(type)}} + rc=0 + scripts/onboard-machine.sh "$name" "$type" || rc=$? + case "$rc" in 0|3) ;; *) exit "$rc" ;; esac echo "" + if [ "$rc" -eq 3 ]; then echo "After the PR is reviewed and merged:"; fi echo "Bootstrap the new machine by running this on it:" - echo "" - echo " curl -fsSL https://raw.githubusercontent.com/hanthor/dotfiles/master/bootstrap.sh | bash -s -- --name {{ name }}" - echo "" - echo "Or if it's already reachable over Tailscale:" - echo " just add-machine {{ name }}" - echo "" - echo "First get it on the tailnet by scanning a QR with your phone (run ON it):" - echo " just tailscale-qr {{ name }}" + echo " curl -fsSL https://raw.githubusercontent.com/hanthor/dotfiles/master/bootstrap.sh | bash -s -- --name $name --type $type" + echo "Or, if it is reachable over Tailscale: just add-machine $name $type" + if [ "$type" = termux_hosts ]; then + echo "Join the tailnet using the Android Tailscale app." + else + echo "First join the tailnet on the machine: just tailscale-qr $name" + fi # Phase-0 bootstrap-of-trust with NO auth key and NO Bitwarden: runs interactive # `tailscale up` and renders the login URL as a QR; scan it with your phone and @@ -379,36 +371,27 @@ push-terminfo host="": online: @echo "Online fleet hosts: {{ _online_hosts }}" -# Bootstrap a new machine over SSH — registers it in inventory then runs bootstrap.sh -# Usage: just add-machine kerala (defaults to desktop) -# just add-machine kerala server +# Register through a reviewed PR, then bootstrap a reachable machine over SSH. +# Usage: just add-machine [desktop|server|vps|termux_hosts] add-machine name type="desktop": #!/usr/bin/env bash set -euo pipefail - cd {{ dotfiles_dir }} - git pull --ff-only - - # Register in inventory if not already present - if ! grep -q "^ {{ name }}:" inventory.yml 2>/dev/null; then - echo "→ Registering {{ name }} ({{ type }}) in inventory..." - python3 scripts/register-machine.py {{ name }} {{ type }} inventory.yml - if [ ! -f "host_vars/{{ name }}.yml" ]; then - echo "---" > "host_vars/{{ name }}.yml" - fi - git add inventory.yml "host_vars/{{ name }}.yml" - git diff --cached --quiet || git commit -m "inventory: add {{ name }} ({{ type }})" - git push - echo "✓ Inventory updated and pushed." - else - echo "✓ {{ name }} already in inventory." + cd {{quote(justfile_directory())}} + name={{quote(name)}} + type={{quote(type)}} + rc=0 + scripts/onboard-machine.sh "$name" "$type" || rc=$? + if [ "$rc" -eq 3 ]; then + echo "Bootstrap not started. Review and merge the PR, then rerun just add-machine $name $type." >&2 + exit 1 fi - - # Bootstrap on the remote machine. - # -t allocates a PTY so /dev/tty is available for any remaining prompts. - echo "→ Bootstrapping {{ name }}..." - ssh -t {{ name }} \ - "curl -fsSL https://raw.githubusercontent.com/hanthor/dotfiles/master/bootstrap.sh \ - | bash -s -- --name {{ name }} --type {{ type }}" + [ "$rc" -eq 0 ] || exit "$rc" + echo "Bootstrapping $name over SSH..." + remote_env="" + if [ "$type" = termux_hosts ]; then + remote_env='HOME=/data/data/com.termux/files/home PREFIX=/data/data/com.termux/files/usr TMPDIR=/data/data/com.termux/files/usr/tmp PATH=/data/data/com.termux/files/usr/bin:$PATH' + fi + ssh -t -- "$name" "$remote_env bash -o pipefail -c 'curl -fsSL https://raw.githubusercontent.com/hanthor/dotfiles/master/bootstrap.sh | bash -s -- --name $name --type $type'" # Pull latest changes and apply update *args: diff --git a/bootstrap.sh b/bootstrap.sh index 38c2d6a9..265e5041 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -5,13 +5,15 @@ # curl ... | bash -s -- --name kanpur --type desktop # ./bootstrap.sh --name kanpur --type desktop set -euo pipefail +if [ -d /data/data/com.termux/files/usr ] && { [ -z "${HOME:-}" ] || [ "$HOME" = /data ]; }; then + export HOME=/data/data/com.termux/files/home +fi REPO_URL="https://github.com/hanthor/dotfiles.git" DOTFILES_DIR="$HOME/.local/share/dotfiles" MACHINE_FILE="/etc/dotfiles-machine" MACHINE_NAME="" -MACHINE_TYPE="" # desktop | server | vps -INVENTORY_MODIFIED=false +MACHINE_TYPE="" # desktop | server | vps | termux_hosts # ── Parse arguments ─────────────────────────────────────────────── while [[ $# -gt 0 ]]; do @@ -22,6 +24,9 @@ while [[ $# -gt 0 ]]; do esac done +is_termux() { [ -d /data/data/com.termux/files/usr ]; } +if is_termux; then export TMPDIR="${TMPDIR:-/data/data/com.termux/files/usr/tmp}"; fi + # ── Detect OS ───────────────────────────────────────────────────── detect_os() { if [ -f /usr/lib/os-release ]; then @@ -42,6 +47,7 @@ ensure_python() { return fi echo "Installing Python3..." + if is_termux; then pkg install -y python; return $?; fi case "$OS" in fedora|rhel|centos) sudo dnf install -y python3 ;; debian|ubuntu) sudo apt-get update && sudo apt-get install -y python3 ;; @@ -56,6 +62,7 @@ ensure_python() { ensure_git() { if command -v git &>/dev/null; then return; fi echo "Installing git..." + if is_termux; then pkg install -y git; return $?; fi case "$OS" in fedora|rhel|centos) sudo dnf install -y git ;; debian|ubuntu) sudo apt-get update && sudo apt-get install -y git ;; @@ -105,9 +112,8 @@ ensure_tailscale() { # ── Clone dotfiles ──────────────────────────────────────────────── clone_repo() { - if [ -d "$DOTFILES_DIR/.git" ]; then - echo "Dotfiles repo exists, pulling latest..." - git -C "$DOTFILES_DIR" pull --ff-only + if [ -e "$DOTFILES_DIR/.git" ]; then + echo "Existing dotfiles checkout left unchanged; update it manually after registration is merged." else echo "Cloning dotfiles..." git clone "$REPO_URL" "$DOTFILES_DIR" @@ -147,23 +153,22 @@ get_machine_type() { # Already set via --type flag if [ -n "$MACHINE_TYPE" ]; then return; fi - # Detect from inventory if machine is already registered - if grep -q "^ $MACHINE_NAME:" "$DOTFILES_DIR/inventory.yml" 2>/dev/null; then - MACHINE_TYPE=$(python3 - < 1: + print('Machine belongs to multiple type groups; pass --type explicitly.', file=sys.stderr) + sys.exit(1) +print(next(iter(groups), '')) PYEOF -) + ) || return 1 + if [ -n "$detected" ]; then + MACHINE_TYPE=$detected echo "Detected machine type from inventory: $MACHINE_TYPE" return fi @@ -191,36 +196,32 @@ PYEOF esac } -# ── Register machine in inventory (local) ──────────────────────── +# ── Require reviewed registration before changing the machine ──── register_in_inventory() { - if grep -q "^ $MACHINE_NAME:" "$DOTFILES_DIR/inventory.yml" 2>/dev/null; then - echo "Machine already in inventory." - return + local rc=0 reviewed + python3 "$DOTFILES_DIR/scripts/register-machine.py" --check "$MACHINE_NAME" "$MACHINE_TYPE" "$DOTFILES_DIR/inventory.yml" || rc=$? + case "$rc" in + 0) ;; + 3) + echo "Machine is not registered in this checkout. Registration must be reviewed and merged before bootstrap." >&2 + echo "On an authenticated fleet checkout run: just onboard $MACHINE_NAME $MACHINE_TYPE" >&2 + echo "After merging, update this checkout and rerun bootstrap." >&2 + return 1 ;; + *) return "$rc" ;; + esac + git -C "$DOTFILES_DIR" fetch origin '+refs/heads/master:refs/remotes/origin/master' || return $? + reviewed=$(mktemp) || return $? + if ! git -C "$DOTFILES_DIR" show refs/remotes/origin/master:inventory.yml > "$reviewed"; then + rm -f -- "$reviewed"; return 1 fi - - echo "Adding $MACHINE_NAME to inventory as $MACHINE_TYPE..." - python3 - < "$DOTFILES_DIR/host_vars/$MACHINE_NAME.yml" + rc=0 + python3 "$DOTFILES_DIR/scripts/register-machine.py" --check "$MACHINE_NAME" "$MACHINE_TYPE" "$reviewed" || rc=$? + rm -f -- "$reviewed" + if [ "$rc" -ne 0 ]; then + echo "Registration is not present in the requested group on origin/master; review and merge it before bootstrap." >&2 + return 1 fi - - INVENTORY_MODIFIED=true - echo "✓ Registered in inventory (will be pushed after GitHub auth)." + echo "Machine registration is present on origin/master." } write_machine_name() { @@ -275,7 +276,11 @@ setup_bitwarden() { if ! command -v bw &>/dev/null; then echo "" echo "NOTE: Bitwarden CLI not yet available." - echo " Run 'just apply' after setup to complete secrets." + if is_termux; then + echo " Run 'DOTFILES_MACHINE=$MACHINE_NAME BW_SESSION_CACHE=\$PREFIX/tmp/bw_session just apply' after setup." + else + echo " Run 'just apply' after setup to complete secrets." + fi return 1 fi @@ -284,7 +289,7 @@ setup_bitwarden() { if [ -n "${BW_SESSION:-}" ] && ! bw unlock --check >/dev/null 2>&1; then unset BW_SESSION fi - if [ -z "${BW_SESSION:-}" ] && [ -s /tmp/bw_session ]; then + if ! is_termux && [ -z "${BW_SESSION:-}" ] && [ -s /tmp/bw_session ]; then local cached_session cached_session=$(cat /tmp/bw_session) if BW_SESSION="$cached_session" bw unlock --check >/dev/null 2>&1; then @@ -318,6 +323,8 @@ setup_bitwarden() { echo "Bitwarden sync failed; secrets setup skipped." >&2 return 1 fi + # Termux has no writable /tmp; units inherit BW_SESSION for this run. + if is_termux; then return 0; fi python3 -c ' import os, tempfile path = "/tmp/bw_session" @@ -342,27 +349,6 @@ run_phase2() { scripts/run-units.sh --tags omp,secrets } -# ── Push inventory changes (after GitHub auth is set up) ───────── -push_inventory_changes() { - if [ "$INVENTORY_MODIFIED" != "true" ]; then return; fi - - cd "$DOTFILES_DIR" || return $? - - # Switch to SSH remote now that GitHub auth is configured - git remote set-url origin git@github.com:hanthor/dotfiles.git || return $? - - git add inventory.yml "host_vars/$MACHINE_NAME.yml" || return $? - if git diff --cached --quiet; then - return 0 # Nothing to commit (already pushed from another machine) - else - local diff_rc=$? - [ "$diff_rc" -eq 1 ] || return "$diff_rc" - fi - - git commit -m "inventory: add $MACHINE_NAME ($MACHINE_TYPE)" || return $? - git push || return $? - echo "✓ Inventory pushed to GitHub." -} # ── Main ────────────────────────────────────────────────────────── main() { @@ -371,16 +357,29 @@ main() { echo "╚═══════════════════════════════════════════════╝" echo "" - ensure_python - ensure_git - clone_repo - get_machine_name - get_machine_type - register_in_inventory - write_machine_name - set_hostname - ensure_sudo - ensure_tailscale + get_machine_name || return $? + case "$MACHINE_NAME" in matrix|telengana) echo "Retired VPSes must not be modified." >&2; return 1 ;; esac + if [[ ! "$MACHINE_NAME" =~ ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ ]] || [ "${#MACHINE_NAME}" -gt 63 ]; then + echo "Machine name must be a lowercase DNS hostname." >&2; return 1 + fi + ensure_python || return $? + ensure_git || return $? + clone_repo || return $? + get_machine_type || return $? + if is_termux; then + [ "$MACHINE_TYPE" = termux_hosts ] || { echo "Termux requires a reviewed termux_hosts identity." >&2; return 1; } + elif [ "$MACHINE_TYPE" = termux_hosts ]; then + echo "termux_hosts identity requires the Termux environment." >&2; return 1 + fi + register_in_inventory || return $? + if is_termux; then + echo "Termux: leaving Android hostname, Tailscale and system privileges unchanged." + else + write_machine_name + set_hostname + ensure_sudo + ensure_tailscale + fi local rc=0 phase_rc=0 run_phase1 || rc=$? @@ -388,9 +387,6 @@ main() { if setup_bitwarden; then run_phase2 || phase_rc=$? [ "$rc" -ne 0 ] || rc=$phase_rc - if [ "$phase_rc" -eq 0 ]; then - push_inventory_changes || { phase_rc=$?; [ "$rc" -ne 0 ] || rc=$phase_rc; } - fi else echo "Bitwarden setup failed; Phase 2 was not run." >&2 [ "$rc" -ne 0 ] || rc=1 @@ -407,7 +403,11 @@ main() { echo "" echo " Start a new login shell to load your configuration." echo "" - echo " To re-apply anytime: just apply" + if is_termux; then + echo " To re-apply: DOTFILES_MACHINE=$MACHINE_NAME BW_SESSION_CACHE=\$PREFIX/tmp/bw_session just apply" + else + echo " To re-apply anytime: just apply" + fi } if [[ -z "${BASH_SOURCE[0]:-}" || "${BASH_SOURCE[0]}" == "$0" ]]; then diff --git a/docs/src/automation.md b/docs/src/automation.md index 3eead2b7..2544a801 100644 --- a/docs/src/automation.md +++ b/docs/src/automation.md @@ -1,9 +1,8 @@ # Self-improving repo: bots, agents and guardrails -This repo is edited by bots, and every fleet machine pulls `master` daily and -applies it **with sudo**. A bad merge therefore reaches every machine within -24h. The automation below is set up so that routine improvements land without -you, and anything that executes on the fleet still waits for you. +This repo is edited by bots. Machines with the fleet update timer enabled pull +`master` and apply configuration with sudo. A bad merge can therefore reach +those machines unattended. Executable fleet changes require owner review. ## Who changes what @@ -12,7 +11,7 @@ you, and anything that executes on the fleet still waits for you. | **Renovate** | minor/patch/digest bumps (pi npm deps, Go, GitHub Actions, OpenTofu providers), weekly lockfile maintenance | yes, once CI is green | | **Renovate** (cluster) | image/chart bumps in `talos-k8s/`, grouped weekly, label `deploy-needed` | **no** — manifests are applied by hand, so a merge would only make git lie about the cluster | | **Hive** ([hive.reilly.asia](https://hive.reilly.asia), `hanthor-hive-agent`, ACMM L5) | its agents open PRs labelled `hold`; the Hive itself only merges PRs labelled `lgtm` | only if the PR stays inside the carve-outs below | -| **You** | direct pushes, `just onboard`, etc. | admin bypass — unaffected | +| **You** | review onboarding and other owned changes | admin bypass is available, but onboarding uses a PR | ## The gate @@ -33,7 +32,7 @@ Three pieces, and none of them trusts the PR itself: `deps-guard` lets Renovate change **versions** in `roles/pi/files/package.json` but fails if a PR adds or removes a dependency. The manifest is tracked by -repository automation; the `omp` role copies `roles/pi/files/skills/` and +repository automation; the `omp` unit copies `roles/pi/files/skills/` and installs OMP plugins separately. It does not run `npm install` from this manifest on fleet machines. diff --git a/docs/src/onboarding.md b/docs/src/onboarding.md index d65968b1..a7c3f577 100644 --- a/docs/src/onboarding.md +++ b/docs/src/onboarding.md @@ -34,24 +34,30 @@ Two paths depending on whether the machine is already reachable. ## Path A: Machine Already Online (Tailscale + SSH) ```bash -# From any machine with the dotfiles repo: +# From an authenticated checkout with git and gh: just add-machine ``` -This: -1. If the machine isn't in `inventory.yml` yet, adds it (`ansible_host: localhost`, `ansible_connection: local`) and puts it in the `` group -2. Creates an empty `host_vars/.yml` — add per-machine knobs there afterwards -3. Commits and pushes -4. SSHes in (`ssh -t `) and runs `bootstrap.sh --name --type ` there — the same bootstrap as Path B +For a new machine this proposes an inventory-only PR from `origin/master` in a +temporary worktree and stops without connecting to the host. Review the group, +connection details and `host_vars`, wait for required checks and owner approval, +and merge the PR. Then rerun `just add-machine `; it verifies the +registration on `origin/master` and SSHes in to run bootstrap. An already +registered host skips the PR step. The caller's checkout, index and existing +branches are not changed. The new machine must already be reachable over SSH. ## Path B: Fresh Install ```bash -# From any machine with the dotfiles repo: +# From an authenticated checkout with git and gh: just onboard ``` -This prints a bootstrap command to run **on the new machine**: +Review and merge the registration PR before using the printed command **on the +new machine**. The registration helper uses a temporary worktree and does not +pull, switch branches, stage or commit the caller's work. If a registration PR +already exists, it prints its URL; an existing remote onboarding branch without +an open PR is left untouched for manual inspection. ```bash # On the new machine: @@ -59,14 +65,19 @@ curl -fsSL https://raw.githubusercontent.com/hanthor/dotfiles/master/bootstrap.s | bash -s -- --name --type ``` -The bootstrap script: -1. Ensures Python and git, then clones the dotfiles repo from [GitHub](https://github.com/hanthor/dotfiles). -2. Registers the machine in `inventory.yml` if needed (prompts for the type when `--type` isn't given). -3. Writes `/etc/dotfiles-machine`, sets the hostname, and sets up sudo and Tailscale. -4. Runs native units with `--no-secrets` (including package installation). -5. Resolves and syncs Bitwarden, runs the `omp,secrets` unit selection, and pushes any inventory changes. +Bootstrap first ensures Python and git, then verifies that its local inventory +and the current `origin/master` contain the machine in the requested group +before changing identity, sudo or Tailscale, or running units. If registration is +missing, run `just onboard` from an authenticated fleet checkout, review and +merge the PR, then rerun it. +An existing checkout is never pulled or overwritten by bootstrap: update it +yourself after the merge and rerun. A fresh machine clones `master`. -Bootstrap continues to the secret phase when possible, but reports failure if a phase or Bitwarden setup fails. Review unit output: skips and partial results are not proof that every host component is configured. +After that gate, Linux bootstrap writes `/etc/dotfiles-machine`, sets hostname, +sudo and Tailscale, then runs the native no-secrets and secrets phases. Termux +uses the unprivileged path described below. Bootstrap reports failure if a phase +or Bitwarden setup fails. Review unit output: skips and partial results are not +proof that every host component is configured. ## After Bootstrap Completes @@ -91,17 +102,49 @@ The details of what happens during this first apply are covered in the [Post-Onb ## Machine Types -`` is the inventory group to join. `bootstrap.sh` offers `desktop`, `server` and `vps`: +`` is the inventory group to join. Bootstrap supports these groups: | Type | Group | Has Desktop? | Notes | |------|-------|-------------|-------| | `desktop` | desktop | Yes | Laptops are desktops too — set `is_laptop: true` in `host_vars` | | `server` | server | No | Gets `server_hardening`; desktop units skip automatically | | `vps` | vps | No | Also skips `syncthing`, `proxy`, `tailscale_cert` | +| `termux_hosts` | termux_hosts | No | Existing reviewed raw Termux identity; Android manages Tailscale | -There is no `laptop` type — a type that doesn't match a group leaves the host in -no group. `test_fleet` and `termux_hosts` hosts are added to `inventory.yml` by -hand. +There is no `laptop` type. Unknown types and invalid hostnames are rejected. +New `test_fleet` and `termux_hosts` identities require a manually reviewed +inventory PR; Termux registrations need the correct Android Tailscale name, +SSH user and port rather than the normal localhost defaults. + +## Raw Termux on the phone + +The existing inventory alias is `termux` (`termux_hosts`, port 8022 and +tailnet name `pixel-9a`); `mumbai` is the separate Debian VM and remains a +desktop. Join the tailnet using the Android Tailscale app. In Termux, install +the initial commands and run the already-reviewed identity locally: + +```bash +pkg update && pkg install git python curl openssh +curl -fsSL https://raw.githubusercontent.com/hanthor/dotfiles/master/bootstrap.sh \ + | bash -s -- --name termux --type termux_hosts +``` + +For SSH onboarding, first start `sshd` in Termux and arrange authenticated SSH +access through the fleet's existing `termux` SSH alias; then run `just +add-machine termux termux_hosts` from a fleet checkout. Bootstrap checks the +local and merged inventory, runs `pkg` and the native units without root, and +does not modify the Android hostname, Tailscale app or `/etc`. Termux:Boot and +Termux:API companion apps are needed for the boot script and scheduler. + +Termux cannot persist `/etc/dotfiles-machine`; the bootstrap phases pass the +reviewed identity explicitly. For later local applies, use +`DOTFILES_MACHINE=termux BW_SESSION_CACHE="$PREFIX/tmp/bw_session" just apply`. +If Bitwarden CLI is unavailable or locked, the no-secrets phase still runs but +bootstrap reports the secrets phase incomplete; provide a usable CLI session +and rerun. Bootstrap keeps its Termux session in the process environment rather +than writing `/tmp/bw_session`; the later apply command above uses a private +cache under Termux's writable prefix. The phone push timer also supplies the +identity explicitly. ## Post-Onboarding diff --git a/docs/src/roles/apk_packages.md b/docs/src/roles/apk_packages.md index 6e3a8a3d..64a25de7 100644 --- a/docs/src/roles/apk_packages.md +++ b/docs/src/roles/apk_packages.md @@ -1,5 +1,5 @@ # apk_packages -`units/apk-packages.sh` runs only when the shared library detects a musl loader. Package lists live in `units/apk-packages.conf`. It retries `apk update`, installs missing bootstrap and ordinary packages without upgrading installed ones, and installs Bitwarden CLI globally via npm. If a batch install fails it retries packages individually so unrelated packages can still install; optional mise and cache refresh are best effort. +`units/apk-packages.sh` runs only when the shared library detects a musl loader. Package lists live in `units/apk-packages.conf`. It retries `apk update`, installs missing bootstrap and ordinary packages without upgrading installed ones, and installs Bitwarden CLI globally via npm. Bootstrap includes `just` from Alpine's community repository for fleet commands and update services; an initial installation of `just` is still needed to invoke those commands before the unit has run. If a batch install fails it retries packages individually so unrelated packages can still install; optional mise and cache refresh are best effort. The unit attempts to enable Podman and its socket and adds the current user to the `podman` group; a new login may be needed. The shared unit library skips retired `matrix` and `telengana` before package or group changes; nothing on those hosts may be restarted. Privileged operations require root or noninteractive `sudo -n`; no vault secrets are needed. diff --git a/docs/src/roles/bluefin_common.md b/docs/src/roles/bluefin_common.md index 952828cb..69ed1952 100644 --- a/docs/src/roles/bluefin_common.md +++ b/docs/src/roles/bluefin_common.md @@ -5,6 +5,6 @@ **Secrets needed:** No **Runs on:** desktops, except `skip_bluefin: true` or `machine_profile: vm-test`. -With a GNOME session, on non-Bluefin systems or when `force_bluefin: true`, the unit pulls the amd64 `ghcr.io/projectbluefin/common:latest` image with Podman, extracts available branding into `~/.local/share`, adjusts wallpaper XML paths and runs `units/bluefin/files/bluefin-sync.py`. Missing individual image assets are reported as partial. It installs Inter and JetBrains Mono fonts and attempts the Bluefin extension installer; extension installation is best effort, not a guarantee that every extension works in the running GNOME version. +With a GNOME session, on non-Bluefin systems or when `force_bluefin: true`, the unit pulls the amd64 `ghcr.io/projectbluefin/common:latest` image with Podman, extracts available branding into `~/.local/share`, adjusts extracted wallpaper XML paths and runs `units/bluefin/files/bluefin-sync.py`. Other local wallpaper XML files are left untouched. Missing individual image assets are reported as partial. It installs Inter and JetBrains Mono fonts and attempts the Bluefin extension installer; extension installation is best effort, not a guarantee that every extension works in the running GNOME version. An existing user extension that GNOME does not recognize is left in place rather than overwritten. -On eligible desktops, including Bluefin itself, it deploys `units/bluefin/files/bluefin-wallpaper-rotate.py` and user systemd service/timer templates, enables the timer and runs a rotation. It downloads the current architecture's corral binary and verifies its SHA256 against the release checksum, removes retired tailvm executables and completions, and configures `/etc/sysctl.d/99-zram.conf` with `vm.swappiness = 150`. The sysctl step requires root or non-interactive sudo; wallpaper and GNOME actions need the invoking user's live D-Bus/systemd session. Bluefin hosts skip only the branding, fonts and extension steps unless forced, not the whole unit. +On eligible desktops, including Bluefin itself, it deploys `units/bluefin/files/bluefin-wallpaper-rotate.py` and user systemd service/timer templates, enables the timer and runs a rotation. Rotation prefers the local monthly wallpaper and falls back to Bluefin's `/usr/share/backgrounds/bluefin` copy. It skips if neither exists, but reports failure if either GNOME background setting cannot be applied. It downloads the current architecture's corral binary and verifies its SHA256 against the release checksum, removes retired tailvm executables and completions, and configures `/etc/sysctl.d/99-zram.conf` with `vm.swappiness = 150`. The sysctl step requires root or non-interactive sudo; wallpaper and GNOME actions need the invoking user's live D-Bus/systemd session. Bluefin hosts skip only the branding, fonts and extension steps unless forced, not the whole unit. diff --git a/docs/src/roles/browser_fxa.md b/docs/src/roles/browser_fxa.md index 100237f8..cd8e56a3 100644 --- a/docs/src/roles/browser_fxa.md +++ b/docs/src/roles/browser_fxa.md @@ -7,6 +7,6 @@ This separate unit attempts Firefox Account sign-in for the selected existing browser profile. It reads `browser_fxa_browser`, `fxa_venv_path`, `fxa_bw_item`, and `fxa_bw_username` from host variables, then `group_vars/all.yml`, then `units/browser-fxa/defaults.yml`. The default browser is `zen`; `firefox` checks Flatpak first, then native Firefox, while `firefox-flatpak` checks only Flatpak. Missing `profiles.ini` skips the unit. -It creates a Python venv, installs PyFxA if needed, and deploys `units/browser-fxa/fxa_signin.py` to `~/.local/bin/fxa_signin.py`. With an unlocked Bitwarden session, it searches the configured item (default `accounts.firefox.com`), selects an exact configured username match when supplied, and obtains credentials and an optional TOTP. Existing `signedInUser.json` is chmod 0600; new token files are created with a restrictive umask. Credentials are not printed. +It creates a Python venv, installs PyFxA if needed, and deploys `units/browser-fxa/fxa_signin.py` to `~/.local/bin/fxa_signin.py`. With an unlocked Bitwarden session, it searches the configured item (default `accounts.firefox.com`), selects an exact configured username match when supplied, and obtains credentials and an optional TOTP. Existing regular `signedInUser.json` files are chmod 0600 but never replaced, even if unverified or malformed; complete new token files are published exclusively with mode 0600. Token symlinks are refused rather than followed. Credentials are not printed. -A locked vault or `--no-secrets` leaves sign-in untouched and reports `PARTIAL`; a missing, incomplete or mismatched item also reports `PARTIAL`. Sign-in failure is best effort and may require manual login. The unit attempts sign-in but does not prove browser login or full Sync operation. +A locked vault or `--no-secrets` leaves sign-in untouched and reports `PARTIAL`; a missing, incomplete or mismatched item also reports `PARTIAL`. Sign-in failure or unverified existing account data may require manual login. The unit attempts sign-in but does not prove browser login or full Sync operation. diff --git a/docs/src/roles/bst_dashboard.md b/docs/src/roles/bst_dashboard.md index 5021a2bb..d1af9a30 100644 --- a/docs/src/roles/bst_dashboard.md +++ b/docs/src/roles/bst_dashboard.md @@ -1,5 +1,5 @@ # bst_dashboard -`units/bst-dashboard.sh` renders `units/bst-dashboard/bst-dashboard.service.j2` with `units/service-templates.py`, installs the resulting user service, reloads the user manager when changed, and enables and starts it. It also enables user linger via root or noninteractive `sudo -n`. It does not install the dashboard project or retrieve secrets. +`units/bst-dashboard.sh` renders `units/bst-dashboard/bst-dashboard.service.j2` with `units/service-templates.py`, installs the resulting user service, reloads the user manager when changed, and enables and starts it. A changed service definition is started or restarted after the reload so the running dashboard uses the new configuration; a permissions-only change does not restart it. It also enables user linger via root or noninteractive `sudo -n`. It does not install the dashboard project or retrieve secrets. The unit skips retired `matrix` and `telengana`, and skips when the project directory is missing. On `karnataka` it selects `/var/home/james/dev/kde-linux` and image `aurora`; elsewhere it selects `/var/home/james/dev/dakota` and image `dakota`. Access is through the Caddy `/bst/` route when that proxy is configured. diff --git a/docs/src/roles/flatpak.md b/docs/src/roles/flatpak.md index 76c86653..66a5853a 100644 --- a/docs/src/roles/flatpak.md +++ b/docs/src/roles/flatpak.md @@ -5,8 +5,8 @@ **Secrets needed:** No **Runs on:** desktops, except `skip_flatpak: true` or `machine_profile: vm-test`. -The unit requires Flatpak and `yq`. It reads system remotes and `system_flatpaks` from `units/flatpak-packages.yml`, repairs a dangling `/var/lib/flatpak` symlink, adds remotes, removes system Firefox and Thunderbird Flatpaks, and installs missing applications from Flathub. A failed batch is retried individually. `DOTFILES_UPGRADE=1` or `true` requests a best-effort system update. System operations require root or non-interactive sudo. +The unit requires Flatpak and `yq`. It reads system remotes and `system_flatpaks` from `units/flatpak-packages.yml`, adds remotes, removes system Firefox and Thunderbird Flatpaks, and installs missing applications from Flathub. A failed batch is retried individually. A dangling `/var/lib/flatpak` symlink is preserved: restore its target before retrying system operations. User operations and wrapper deployment remain independent. `DOTFILES_UPGRADE=1` or `true` requests a best-effort system update. System operations require root or non-interactive sudo. Host-specific `user_flatpak_remotes` and `user_flatpaks` in `host_vars/.yml` are applied with `--user`, separately from the system store. For `himachal`, these configure FlatPark and user Flathub, then install `org.tunaos.blueshell`, `org.tunaos.tavern`, and `org.tunaos.compass` from FlatPark. An already installed user app with a different origin is preserved rather than switched. This installation does not configure or unlock Compass's keyring or establish an application login; those depend on the user's session and app setup. -The unit also writes `chromium`, `chromium-browser`, `google-chrome`, and `google-chrome-stable` wrappers in `~/.local/bin`, invoking `org.chromium.Chromium` via Flatpak. Wrappers are attempted even if Flatpak is unavailable. +The unit also writes `chromium`, `chromium-browser`, `google-chrome`, and `google-chrome-stable` wrappers in `~/.local/bin`, invoking `org.chromium.Chromium` via Flatpak. Existing custom files and symlinks are preserved; matching managed wrappers have their mode corrected. Wrappers are attempted even if Flatpak is unavailable. diff --git a/docs/src/roles/fleet_facts.md b/docs/src/roles/fleet_facts.md index 4b9fcaed..2db54b99 100644 --- a/docs/src/roles/fleet_facts.md +++ b/docs/src/roles/fleet_facts.md @@ -81,6 +81,10 @@ Punjab has `osquery_poller_enabled: true`. Its system-level hosts' report-refresh window, and two minutes after boot. It runs as `ubuntu`, using that account's existing SSH keys, known hosts and noninteractive sudo. It does not depend on an interactive SSH agent or copy credentials into Kubernetes. +The poller's default `PATH` includes the applying user's `~/.local/bin`, the +configured Homebrew prefix and the apply environment's `PATH`, so tools installed +outside the standard system directories remain available. `osquery_poller_path` +overrides that default. The unit installs a root-managed collector and a minimal inventory under `/opt/fleet-osquery` and `/etc/fleet-osquery/inventory.yml`. Only connection and diff --git a/docs/src/roles/forgejo_registry.md b/docs/src/roles/forgejo_registry.md index b300c8bf..5fd6a442 100644 --- a/docs/src/roles/forgejo_registry.md +++ b/docs/src/roles/forgejo_registry.md @@ -10,7 +10,7 @@ Logs podman in to the private Forgejo container registry. ## What it does 1. `SKIPPED:` unless `podman` is installed; no-op when `podman login --get-login ` already succeeds. -2. With an unlocked vault: `bw get username` / `bw get password` for the item named after the registry, then `podman login -u --password-stdin`. A missing item prints `PARTIAL:` and is not a failure. +2. With an unlocked vault: `bw get username` / `bw get password` for the item named after the registry, then `podman login -u --password-stdin`. The username is trimmed at its edges, preserving internal whitespace. A missing, empty, or failed credential read prints `PARTIAL:` and is not a failure; partial output from a failed read is never used. ## Configuration diff --git a/docs/src/roles/github.md b/docs/src/roles/github.md index a4502233..bd885aae 100644 --- a/docs/src/roles/github.md +++ b/docs/src/roles/github.md @@ -9,7 +9,7 @@ Logs `gh` in via a Bitwarden-stored PAT (only if not already authenticated), the ## Steps -1. `github known_hosts`: ensure github.com's ed25519 host key is in `~/.ssh/known_hosts`. +1. `github known_hosts`: ensure github.com's pinned ed25519 host key is in `~/.ssh/known_hosts`, including when the existing entry is hashed. A conflicting ed25519 key, a revoked pinned key, or a symlinked known-hosts file fails closed without changing the file; inspect and resolve it before retrying. 2. `gh login`: skipped when `gh auth status` succeeds. Otherwise, with an unlocked vault, `bw get password github-token` → `gh auth login --with-token`. Locked vault or missing item prints a `PARTIAL:` line. 3. `register SSH keys`: needs a token with `admin:public_key` and `admin:ssh_signing_key` (otherwise a `PARTIAL:` line with the regenerate-PAT link). Authentication and signing keys are listed separately using GitHub's API, so an account with no keys is supported. Each missing type is added with `gh ssh-key add ~/.ssh/id_ed25519.pub --title -auth|-signing`. A list or upload failure does not prevent attempting the other type, and the unit reports failure. @@ -23,6 +23,7 @@ GitHub CLI configuration, including `git_protocol: ssh`, is managed by chezmoi, | `PARTIAL: no 'github-token' item` | Vault has no PAT | Create a login item `github-token` with the PAT as the password | | `PARTIAL: GitHub token lacks admin:public_key / admin:ssh_signing_key` | PAT scopes | Regenerate the PAT with those scopes and update the item | | `warn: failed to list GitHub SSH ... keys; registration skipped` | GitHub API lookup failed | Check GitHub access and re-run; the unverified key type was not added | +| `FAIL conflicting GitHub ed25519 host key` | Existing host key differs from the pinned key | Verify GitHub's published host-key fingerprints and inspect the existing entry before removing or replacing it | | `warn: failed to upload SSH ... key` | `gh ssh-key add` failed | Inspect with `gh ssh-key list` and re-run | ## Verify diff --git a/docs/src/roles/gnome.md b/docs/src/roles/gnome.md index de91d549..2b293cc2 100644 --- a/docs/src/roles/gnome.md +++ b/docs/src/roles/gnome.md @@ -5,7 +5,7 @@ **Secrets needed:** No **Runs on:** desktops, except `skip_gnome: true` or `machine_profile: vm-test`. -The unit installs `units/gnome/gnome-keybindings.ini` in `~/.config/dconf` and loads it into the invoking user's live session. A pending marker retains a changed load request if the session is unavailable. It disables touchpad tap-to-click, configures the default Ptyxis profile for Homebrew fish and `JetBrainsMono Nerd Font 12` on a best-effort basis, and downloads the account avatar for installation by `units/gnome-avatar.py`; the avatar step requires root or non-interactive sudo. +The unit installs `units/gnome/gnome-keybindings.ini` in `~/.config/dconf` and loads it into the invoking user's live session. A pending marker retains a changed load request if the session is unavailable. It disables touchpad tap-to-click and configures the default Ptyxis profile for Homebrew fish and `JetBrainsMono Nerd Font 12` on a best-effort basis, including enabling the custom command and disabling the system font even when the command and font already match. Check mode reports drift in all four settings. It downloads the account avatar for installation by `units/gnome-avatar.py`; the avatar step requires root or non-interactive sudo. A newly created GDM cache directory is owned by GDM; an existing root-owned 0755 directory is also repaired (the ownership left by earlier unit runs). Other cache directory ownership and permissions, including ownership of symlink targets, and unrelated contents are left intact. The unit copies the bundled Copyous, PaperShell and PaperWM extensions from `units/gnome/extensions/` to the user's extension directory, without deleting unrelated files. It best-effort loads `units/gnome/gnome-extensions-dconf.dump` and sets `gnome_enabled_extensions` from host variables or `group_vars/all.yml`. Setting the enabled list does not install every listed extension or guarantee compatibility or activation in the current GNOME Shell. The separate Bluefin unit also attempts extension installation. diff --git a/docs/src/roles/homebrew.md b/docs/src/roles/homebrew.md index edd83871..5154b8d4 100644 --- a/docs/src/roles/homebrew.md +++ b/docs/src/roles/homebrew.md @@ -7,6 +7,6 @@ The native unit installs and manages Homebrew. It defaults to `/home/linuxbrew/.linuxbrew` (`BREW_PREFIX` overrides the prefix), bootstraps Mike Farah `yq`, and reads package lists from `group_vars/all.yml` and host configuration through `units/package-config.bash`. Package units validate the `yq` implementation and can select the Homebrew binary when an incompatible `yq` precedes it on `PATH`. -It generates `~/Brewfile` with core packages and, on desktops, desktop taps, packages, extras and casks; VS Code extensions are included on Darwin. It taps repositories, attempts to trust them, runs `brew bundle install --no-upgrade`, and uninstalls `brew_blocked` formulae. If Brewfile deployment fails, bundle is skipped rather than using a stale file, but independent steps continue. A blocked package in an install list is a configuration error. `DOTFILES_UPGRADE=1` or `true` additionally requests a best-effort `brew upgrade`; installing Python `packaging` is also best effort. +It generates `~/Brewfile` with core packages and, on desktops, desktop taps, packages, extras and casks; VS Code extensions are included on Darwin. It taps and attempts to trust repositories referenced by the applicable core or desktop lists, runs `brew bundle install --no-upgrade`, and uninstalls `brew_blocked` formulae. Desktop-only taps are not added on non-desktop hosts. If Brewfile deployment fails, bundle is skipped rather than using a stale file, but independent steps continue. A blocked package in an install list is a configuration error. `DOTFILES_UPGRADE=1` or `true` additionally requests a best-effort `brew upgrade`; installing Python `packaging` is also best effort. Ownership repair and installing a missing Debian/Ubuntu compiler toolchain may require root or non-interactive sudo. Other systems must supply a compiler if missing. The Homebrew installer itself is downloaded and run non-interactively. diff --git a/docs/src/roles/kirocrew.md b/docs/src/roles/kirocrew.md index 5fc1b4fe..761e3644 100644 --- a/docs/src/roles/kirocrew.md +++ b/docs/src/roles/kirocrew.md @@ -2,4 +2,4 @@ `units/kirocrew.sh` manages an existing [Kiro Crew](https://github.com/kirodotdev/KiroCrew) installation only when `kirocrew_enabled: true` is set in host variables. It skips retired `matrix` and `telengana`. If `~/.local/bin/kirocrew` or `/etc/systemd/system/kirocrew.service` is missing, it skips and prints upstream installation guidance; it does not install either. -With root or noninteractive `sudo -n`, it makes `/etc/kirocrew/kirocrew.env` root-owned mode 0600, then enables and starts the service only if those permissions succeeded. It checks listeners using `ss` and `units/kirocrew-sockets.py`, failing the unit if the check fails. The existing environment file contains credentials; this unit neither fetches nor rewrites them. +With root or noninteractive `sudo -n`, it makes the existing regular, non-symlink `/etc/kirocrew/kirocrew.env` root-owned mode 0600, then enables and starts the service only if those permissions succeeded. It checks listeners using `ss` and `units/kirocrew-sockets.py`, matching socket owner PIDs to the service's systemd cgroup (including child cgroups), rather than relying on a process name that may be `python3`. An active service with no attributable listener fails the check instead of claiming loopback-only binding. Check mode cannot determine an inactive service's future bindings and says so. The existing environment file contains credentials; this unit neither fetches nor rewrites them. diff --git a/docs/src/roles/kube.md b/docs/src/roles/kube.md index 515c3bcb..c965410a 100644 --- a/docs/src/roles/kube.md +++ b/docs/src/roles/kube.md @@ -14,10 +14,10 @@ With an unlocked vault (a locked vault or `--no-secrets` prints `SKIPPED:` and l 1. `kube dirs`: `~/.kube/` and `~/.talos/` (0700). 2. `home kubeconfig`: `bw get notes kubeconfig` → `~/.kube/config` (0600). 3. `home talosconfig`: `bw get notes talosconfig` → `~/.talos/config` (0600). -4. `pin tls-server-name`: after writing the home kubeconfig, every cluster in it gets `--tls-server-name=$KUBE_TLS_SERVER_NAME` (`units/kube.conf`, default bihar's LAN address `192.168.0.5`): the home API-server cert lacks the tailnet address in its SANs. If `kubectl` is missing, the unit warns and skips the pin. +4. `pin tls-server-name`: after writing the home kubeconfig, every cluster in it gets `--tls-server-name=$KUBE_TLS_SERVER_NAME` (`units/kube.conf`, default bihar's LAN address `192.168.0.5`): the home API-server cert lacks the tailnet address in its SANs. If `kubectl` is missing, the unit fails and reports that the pin was not applied. 5. `aws kubeconfig` / `aws talosconfig`: `kubeconfig-aws-migration` → `~/.kube/config-aws-migration`, `talosconfig-aws-migration` → `~/.talos/config-aws-migration` (0600). These historical files remain separate from the home configs and no longer access the terminated cluster. -A missing or empty note prints `warn:` and leaves its existing file untouched. +A missing, empty, or failed note read prints `warn:` and leaves its existing file untouched, even if the failed command emitted partial output. If `kubectl config get-clusters` fails after a home config is written, the unit fails instead of silently reporting a successful TLS pin; inspect the config and re-run after fixing `kubectl`. ## Failure modes @@ -28,6 +28,7 @@ A missing or empty note prints `warn:` and leaves its existing file untouched. | Configs deployed but auth fails against a deliberately restored cluster | Cluster PKI may have rotated — local configs are stale | Confirm the intended cluster and configs before re-seeding, then re-apply on consumers | | Unit prints `SKIPPED: not a desktop` on a host that should have it | `skip_kube: true` in `host_vars/.yml`, or host not in the `desktop` group | Adjust inventory or host_vars | | `FAIL cannot inspect desktop inventory` | Missing `yq`, unreadable inventory, or malformed desktop hosts | Install `yq` and fix `inventory.yml`; no configs are changed while classification is unknown | +| `FAIL cannot list home kubeconfig clusters` | `kubectl` could not read the deployed home config | Inspect the config and local `kubectl` error, then retry; the TLS pin was not applied | ## Seeding BW diff --git a/docs/src/roles/omp.md b/docs/src/roles/omp.md index d419408f..3c590cd0 100644 --- a/docs/src/roles/omp.md +++ b/docs/src/roles/omp.md @@ -1,7 +1,7 @@ # omp **Unit:** `units/omp.sh` (mise task `unit:omp`; see [Units](../units.md)) -**Tags:** `dotfiles`, `omp` (`just apply-tags omp` runs it after the Ansible pass) +**Tags:** `dotfiles`, `omp` (`just apply-tags omp` runs the native unit) **Secrets needed:** No for the install and routing; the auth-broker token comes from Bitwarden when the vault is unlocked **Runs on:** All supported hosts except raw Termux (the unit prints `SKIPPED: omp has no Android build`) diff --git a/docs/src/roles/pipewire_audio.md b/docs/src/roles/pipewire_audio.md index 4b07b3a3..c7d4a900 100644 --- a/docs/src/roles/pipewire_audio.md +++ b/docs/src/roles/pipewire_audio.md @@ -5,10 +5,12 @@ **Secrets needed:** No **Runs on:** desktops, except `skip_pipewire_audio: true` or `machine_profile: vm-test`. -The unit deploys native PipeWire filter chains from `units/pipewire-audio/`. It detects the LSP LV2 plugin and chooses the LSP speaker chain or built-in biquad fallback. It installs the RNNoise microphone chain and user `mic-dsp-link.service` when the RNNoise LADSPA plugin is found. `mic_hardware_volume_pct` comes from the environment, host/group variables or unit defaults and must be 0–100. +The unit deploys native PipeWire filter chains from `units/pipewire-audio/`. It detects the LSP LV2 plugin and chooses the LSP speaker chain or built-in biquad fallback. It installs the RNNoise microphone chain and user `mic-dsp-link.service` when the RNNoise LADSPA plugin is found. The service extracts the source name from `pactl`'s tab-separated output and escapes the volume percent for systemd; the unmodified Ansible service is recognized and upgraded. `mic_hardware_volume_pct` comes from the environment, host/group variables or unit defaults and must be 0–100. -DSP prerequisites are installed with supported dnf, rpm-ostree or Arch package managers; these operations require root or non-interactive sudo. Other platforms report `PARTIAL`; bootc images must supply the prerequisites themselves. If plugin discovery is inconclusive, existing microphone configuration or speaker configuration is retained rather than assumed absent. +DSP prerequisites are installed with supported dnf, rpm-ostree or Arch package managers; these operations require root or non-interactive sudo. When rpm-ostree and dnf are both present, rpm-ostree is used. Other platforms report `PARTIAL`; bootc images must supply the prerequisites themselves and the unit does not attempt host package changes. If plugin discovery is inconclusive, existing microphone configuration or speaker configuration is retained rather than assumed absent. -Only absent or recognized managed audio files are replaced or removed; custom configs and services are preserved and reported. The old managed WirePlumber rule is removed without deleting other rules. Restart requests are persisted under `~/.local/state/pipewire-audio` until the user systemd manager successfully reloads/restarts the affected services. A live user systemd session is required. GNOME over-amplification is best effort. +Only absent or recognized managed audio files are replaced or removed; custom configs and services are preserved and reported. The old managed WirePlumber rule is removed without deleting other rules. Restart requests are persisted under `~/.local/state/pipewire-audio` until the user systemd manager successfully reloads/restarts the affected services. After a PipeWire or WirePlumber restart, the managed microphone link service is also restarted to recreate its audio links; failed restarts are retried on the next apply. A live user systemd session is required. GNOME over-amplification is best effort. The unit removes the Easy Effects autostart entry and, if present, stops and uninstalls the system Flatpak; user presets and settings are preserved. Installation does not establish that DSP or microphone routing works on a particular device; inspect `wpctl status`, `pw-cli`, and the user PipeWire journal on the target. + +The link service selects the first ALSA input, assumes stereo capture ports, and waits up to five seconds for it. It tolerates missing devices and link failures, so an active oneshot does not prove routing succeeded; it does not automatically relink after later hotplug or an audio-daemon restart outside this unit. diff --git a/docs/src/roles/proxy.md b/docs/src/roles/proxy.md index 8bb531a5..c920595d 100644 --- a/docs/src/roles/proxy.md +++ b/docs/src/roles/proxy.md @@ -1,7 +1,7 @@ # proxy (Caddy) -`units/proxy.sh` deploys the rootful Caddy quadlet from `units/proxy/caddy.container` and renders `units/proxy/Caddyfile.j2` using `units/service-templates.py` and the host's `web_services` list. Caddy is configured on port 9000; the unit points the default Tailscale Serve HTTPS handler at `127.0.0.1:9000`. Routes include `/service/` redirects and `/bst/`; there is no homepage at `/`. +`units/proxy.sh` deploys the rootful Caddy quadlet from `units/proxy/caddy.container` and renders `units/proxy/Caddyfile.j2` using `units/service-templates.py` and the effective inventory `web_services` list (group settings followed by host overrides). Caddy is configured on port 9000; the unit points the default Tailscale Serve HTTPS handler at `127.0.0.1:9000`. Routes include `/service/` redirects and `/bst/`; there is no homepage at `/`. -The unit requires `yq`, Python, root or noninteractive `sudo -n`, systemd, Podman and Tailscale. It skips inventory VPSes, retired `matrix`/`telengana`, `skip_proxy: true`, and `machine_profile: vm-test`. It does not fetch vault secrets. The Tailscale Serve configuration can replace an existing default HTTPS handler. +The unit requires `yq`, Python, root or noninteractive `sudo -n`, systemd, Podman and Tailscale. It skips inventory VPSes, including members of nested VPS groups, retired `matrix`/`telengana`, `skip_proxy: true`, and `machine_profile: vm-test`. Unknown hosts or invalid inventory settings fail closed. It does not fetch vault secrets. The Tailscale Serve configuration can replace an existing default HTTPS handler. The unit checks the node's DNS name and the root handler on port 443 using Tailscale's JSON status; an unrelated handler pointing to Caddy does not count. If either status cannot be read or parsed, Serve is left unchanged and the unit fails. It writes `/etc/caddy/Caddyfile` and `/etc/containers/systemd/caddy.container` as root-owned 0644 files. Activation is skipped if either deployment fails; otherwise it reloads systemd, restarts Caddy after changes, and enables and starts the service before changing Serve. diff --git a/docs/src/roles/server_hardening.md b/docs/src/roles/server_hardening.md index c83986c2..a6a34abe 100644 --- a/docs/src/roles/server_hardening.md +++ b/docs/src/roles/server_hardening.md @@ -4,6 +4,6 @@ The unit sets journald's maximum size to 500 MiB, restarts journald on change and vacuums old journals. It edits `/etc/systemd/journald.conf` when present; otherwise it manages `/etc/systemd/journald.conf.d/90-dotfiles-server-hardening.conf`, preserving the vendor config and other drop-ins. Check mode reports the pending drop-in without creating it. A symlinked or invalid managed drop-in, or an unusable drop-in directory, is reported as a failure rather than overwritten. -On writable roots it creates a 2 GiB `/swapfile` if absent, sets root ownership and mode 0600, formats and enables it if not active, and adds an fstab entry. On read-only roots it preserves existing active swap (including zram) without changing `/swapfile` or fstab; if no swap is active, it reports failure. Apply and check modes both report failures when root mount options or the required active-swap status cannot be inspected, and continue the other hardening steps. +On writable roots it creates a 2 GiB `/swapfile` if absent, sets root ownership and mode 0600, formats and enables the newly created file, and adds an fstab entry. An existing inactive regular file must already have a swap signature (`blkid`); it is enabled without reformatting. The unit refuses symlinks, non-regular files, and unrecognized existing contents instead of changing or formatting them. A partially created file after an interrupted creation requires manual inspection and recovery. On read-only roots it preserves existing active swap (including zram) without changing `/swapfile` or fstab; if no swap is active, it reports failure. Apply and check modes both report failures when root mount options or the required active-swap status cannot be inspected, and continue the other hardening steps. -It enables systemd-timesyncd only when present and chrony is not active. It sets SSH `MaxStartups`, `MaxSessions`, and `ClientAliveCountMax`, reloading SSH best-effort on change. It enables UFW logging only when UFW is already active, attempts Ubuntu Pro livepatch when its status indicates it is needed, and reports a pending reboot. It does not configure firewall rules or install fail2ban. +It enables systemd-timesyncd only when present and chrony is not active. It prepends a marked global SSH block for `MaxStartups`, `MaxSessions`, and `ClientAliveCountMax`, ahead of any `Include`, existing directives, or `Match` sections, so OpenSSH's first-value-wins rules do not silently defeat these limits. Other configuration, including per-`Match` overrides, is preserved; a malformed managed block is reported rather than overwritten. SSH reload remains best-effort on change. It enables UFW logging only when UFW is already active, attempts Ubuntu Pro livepatch when its status indicates it is needed, and reports a pending reboot. It does not configure firewall rules or install fail2ban. diff --git a/docs/src/roles/shell_ai.md b/docs/src/roles/shell_ai.md index 08bd2988..f74ba6a4 100644 --- a/docs/src/roles/shell_ai.md +++ b/docs/src/roles/shell_ai.md @@ -17,7 +17,7 @@ With an unlocked vault (`bw_session`; never prompts): - `bw get notes meta-api-key` → `META_API_KEY` - `bw get notes kiro-api-key` → `KIRO_API_KEY` -The values present are written (single-quoted, 0600, via temp file + `mv`) to `~/.config/shell/secrets.sh` (bash/zsh; also exports `OMP_KIRO_CONFIG`) and `~/.config/fish/conf.d/secrets.fish` (fish; ends with a runtime Kiro key refresh from `/tmp/bw_session`). +The values present are written (shell-specific single-quote escaping, 0600, via temp file + `mv`) to `~/.config/shell/secrets.sh` (bash/zsh; also exports `OMP_KIRO_CONFIG`) and `~/.config/fish/conf.d/secrets.fish` (fish; attempts a runtime Kiro key refresh from `/tmp/bw_session`). Fish keeps the deployed key when the refresh fails, is empty, or is not a string; the session is passed through the environment rather than command arguments. ## Output lines @@ -27,5 +27,6 @@ The values present are written (single-quoted, 0600, via temp file + `mv`) to `~ ## Notes - Not a chezmoi template on purpose: `bitwarden` template functions abort the whole template when one item is missing, while the unit writes whatever secrets exist. +- A failed Bitwarden read is ignored even if it produced partial output; other successfully read items are still deployed. - A locked vault leaves any previously written secret files untouched — nothing is overwritten with empty values - `TAVILY_API_KEY` is read by the `tavily-search` skill deployed by the [`omp`](omp.md) unit diff --git a/docs/src/roles/shell_atuin.md b/docs/src/roles/shell_atuin.md index b3aaf0f0..3a6f092a 100644 --- a/docs/src/roles/shell_atuin.md +++ b/docs/src/roles/shell_atuin.md @@ -10,9 +10,11 @@ Configures [Atuin](https://atuin.sh/) shell history and logs it in to sync. ## What It Does 1. chezmoi writes `~/.config/atuin/config.toml` from `home/dot_config/atuin/config.toml` — sync to `api.atuin.sh` every 5m, fuzzy search, `filter_mode = "global"` (including the Up-arrow binding) + `just apply-tags atuin` and `just apply-tags secrets` apply only this + chezmoi file, without unrelated dotfiles or chezmoi scripts. 2. The `atuin` unit checks `atuin status`; if it shows `[Sync]` the Bitwarden round-trip is skipped 3. Otherwise, with an unlocked vault, it reads the `atuin.sh` item (username, password, custom field `key` holding the mnemonic). Atuin 18.24 or newer receives the password and key through its environment, not command-line arguments. The unit preserves the data directory and its history; a missing key can be written by Atuin during login. If a key already exists, the unit logs in only when `atuin key` matches the vault mnemonic, then runs a best-effort `atuin sync`. -4. Imports existing shell history once (`atuin import auto`, sentinel `~/.local/share/atuin/.history_imported`). If the version or existing key requires manual login, the unit also defers importing until that situation is resolved. +4. Imports existing shell history once (`atuin import auto`, sentinel `~/.local/share/atuin/.history_imported`). If the version, key lookup, existing key, or login attempt requires manual attention, the unit defers importing until that situation is resolved. A failed login also fails the unit. ## Output lines diff --git a/docs/src/roles/shell_dotfiles.md b/docs/src/roles/shell_dotfiles.md index e86b6a4d..5f42b899 100644 --- a/docs/src/roles/shell_dotfiles.md +++ b/docs/src/roles/shell_dotfiles.md @@ -21,7 +21,7 @@ No longer an Ansible role: the shell, prompt, multiplexer and terminal configs a ## Template data -`home/.chezmoi.toml.tmpl` (regenerated by `apply --init`) provides `machine` and `is_desktop` (from `DOTFILES_MACHINE` / `DOTFILES_IS_DESKTOP`, exported by the unit; `is_desktop` is true when the host is in the `desktop` group of `inventory.yml`). `home/.chezmoidata.yaml` provides `user_name`, `user_email` and `alias_groups`, the single source for the bash/zsh and fish aliases. Add a plain alias there, not in the templates. +`home/.chezmoi.toml.tmpl` (regenerated by `apply --init`) provides `machine` and `is_desktop` from the unit. Desktop classification uses `DOTFILES_IS_DESKTOP`, then the host's `is_desktop` override, then the `desktop` group in `inventory.yml`; it fails rather than guessing if the classification cannot be read. `home/.chezmoidata.yaml` provides `user_name`, `user_email` and `alias_groups`, the single source for the bash/zsh and fish aliases. Add a plain alias there, not in the templates. ## Adding a file @@ -41,3 +41,4 @@ In fish these are abbreviations (`abbr`), not aliases. - Shell secrets (`~/.config/shell/secrets.sh`, `~/.config/fish/conf.d/secrets.fish`) are written by the [`shell-ai` unit](shell_ai.md), not here. - One failing file does not stop the rest: the unit runs `chezmoi apply --init --keep-going --force --no-tty`. This overwrites managed files even when locally edited; back up and review existing dotfiles before adoption. See [Units](../units.md#chezmoi). +- The `atuin` and `secrets` tags apply only `~/.config/atuin/config.toml` through chezmoi, without running its scripts or overwriting other managed files. When combined with a tag requesting full chezmoi apply (such as `shell` or `git`), the full apply takes precedence. diff --git a/docs/src/roles/shell_fonts.md b/docs/src/roles/shell_fonts.md index 506b80a4..b73a097c 100644 --- a/docs/src/roles/shell_fonts.md +++ b/docs/src/roles/shell_fonts.md @@ -10,10 +10,10 @@ Installs the [JetBrains Mono Nerd Font](https://github.com/ryanoasis/nerd-fonts) ## Steps 1. `font directory`: `~/.local/share/fonts/` (every host). -2. On desktops, if `JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf` is missing: `install JetBrains Mono Nerd Font` downloads the latest `JetBrainsMono.zip` release into a temp dir, unpacks it into `~/.local/share/fonts/JetBrainsMono/`, then `font cache` (soft) runs `fc-cache -f`. +2. On desktops, if `JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf` is missing: download the latest `JetBrainsMono.zip` into a temporary directory, extract it there, and copy it into `~/.local/share/fonts/JetBrainsMono/`. Other font files are preserved. Run `fc-cache -f` after installation; a cache failure fails the unit and is retried on the next apply without downloading again. ## Notes -- Idempotent on the presence of the Regular `.ttf`; delete it to force a re-download. +- Idempotent on the presence of the Regular `.ttf`; delete it to force a re-download. Failed extraction cannot leave that file as a false success marker; a pending cache refresh is recorded in `~/.local/share/fonts/.dotfiles-jetbrainsmono-cache-pending` until it succeeds. - Desktop means: `DOTFILES_IS_DESKTOP`, else `is_desktop` in `host_vars`, else membership of the `desktop` group in `inventory.yml`. An invalid override or unreadable inventory fails the unit rather than reporting a non-desktop skip. - The `gnome` unit sets Ptyxis to `JetBrainsMono Nerd Font 12`, which depends on this. diff --git a/docs/src/roles/ssh_keys.md b/docs/src/roles/ssh_keys.md index 931118d8..05fe0676 100644 --- a/docs/src/roles/ssh_keys.md +++ b/docs/src/roles/ssh_keys.md @@ -1,11 +1,11 @@ # ssh_keys -`units/ssh-keys.sh` invokes the locked shared implementation `units/ssh-fleet.py`. It manages the local ed25519 identity, fleet SSH config, known hosts, Git allowed signers, and Bitwarden-backed authorization. Fleet names and login data come from `group_vars/all.yml`, `inventory.yml`, and host variables; YAML parsing needs `yq` or Python PyYAML. +`units/ssh-keys.sh` invokes the locked shared implementation `units/ssh-fleet.py`. It manages the local ed25519 identity, fleet SSH config, known hosts, Git allowed signers, and Bitwarden-backed authorization. Fleet names and login data come from `inventory.yml`, group and host variables. Nested group membership and effective connection variables are resolved by `units/osquery_native_inventory.py`; YAML parsing needs `yq` or Python PyYAML. -If the private key is absent, the unit requires an unlocked vault and a successful item listing before restoring the complete `@` SSH item or generating a new pair when no item exists. It checks restored and local key pairs and refuses conflicting keys. An existing local key is retained; if its public key differs from the vault item, the vault is left unchanged unless `SSH_KEYS_FORCE_PUSH=true` or `ssh_keys_force_push: true` explicitly permits rotation. +If the private key is absent, the unit requires an unlocked vault and a successful item listing before restoring the complete `@` SSH item or generating a new pair when no item exists. It checks restored and local key pairs and refuses conflicting keys. An existing local key is retained; if its public key differs from the vault item, the vault is left unchanged unless `SSH_KEYS_FORCE_PUSH=true` or `ssh_keys_force_push: true` in host or group variables explicitly permits rotation. Passphrase-protected OpenSSH identities are not prompted for. Existing identities and vault restores are retained when the public key matches the encrypted key's public envelope, but the private contents cannot be verified without the passphrase. The envelope is unauthenticated, so the unit will not create or replace a Bitwarden key from that unverified identity, even with forced rotation. A missing, invalid or mismatched public key remains an error. -The unit preserves user entries outside its marked SSH config and `authorized_keys` blocks. Vault public keys and `extra_authorized_keys` are merged into the authorization block; previous managed entries remain by default. `SSH_KEYS_FORCE_AUTHORIZED=true` (or the corresponding host variable) permits removing obsolete entries only within that block. A locked vault, disabled secrets, or an empty vault key list leaves vault authorization unchanged. Host-key scans only replace an entry after a usable scan; failed scans leave existing entries intact. Existing signers are retained. +The unit preserves user entries outside its marked SSH config and `authorized_keys` blocks. Vault public keys and `extra_authorized_keys` are merged into the authorization block; previous managed entries remain by default. `SSH_KEYS_FORCE_AUTHORIZED=true` (or `ssh_keys_force_authorized: true` in host or group variables) permits removing obsolete entries only within that block. A locked vault, disabled secrets, or an empty vault key list leaves vault authorization unchanged. Host-key scans only replace an entry after a usable scan; failed scans leave existing entries intact. The retired `matrix` and `telengana` hosts are never scanned. Existing signers are retained. `--no-secrets` skips vault access; a missing identity is deferred rather than generated. Config and other independent steps still run. The separate [ssh_mesh](ssh_mesh.md) unit adds committed fleet public keys and checks connectivity. diff --git a/docs/src/roles/ssh_mesh.md b/docs/src/roles/ssh_mesh.md index eec3329e..dde9dead 100644 --- a/docs/src/roles/ssh_mesh.md +++ b/docs/src/roles/ssh_mesh.md @@ -2,6 +2,6 @@ `units/ssh-mesh.sh` invokes `units/ssh-fleet.py` under the same SSH lock as [ssh_keys](ssh_keys.md). It skips `machine_profile: vm-test` and `skip_ssh_mesh: true`. It merges valid `machine_pubkeys` from `group_vars/all.yml` into `~/.ssh/authorized_keys` without removing existing user or vault entries, and warns if the local public key is missing or stale in git. -The unit probes online tailnet peers selected by `scripts/online_hosts.py --peers`, intersected with `machines` and excluding the local host, `ssh_mesh_exclude`, and inventory VPSes. It uses inventory port and login overrides, bypasses connection multiplexing, and on failure closes an existing mux, attempts a fresh ed25519 host-key scan, and retries. Existing known-host entries are not removed unless the scan succeeds. Persistent failures fail the unit and name the peers. +The unit probes online tailnet peers selected by `scripts/online_hosts.py --peers`, intersected with `machines` and excluding the local host, `ssh_mesh_exclude`, inventory VPSes (including nested members), and the retired `matrix` and `telengana` hosts. A host-specific `ssh_mesh_exclude` overrides the group default. It uses effective inventory group and host port, login and tailnet-name overrides, bypasses connection multiplexing, and on failure closes an existing mux, attempts a fresh ed25519 host-key scan, and retries. Existing known-host entries are not removed unless the scan succeeds. Persistent failures fail the unit and name the peers. Normally no vault is needed. If the local identity is missing, the shared implementation consults Bitwarden before restoring or creating it; locked vault or `--no-secrets` defers the identity and probes instead of generating an unmirrored key. Add new public keys to `machine_pubkeys` and host entries to `machines`; configure `tailnet_name` and exclusions where needed. diff --git a/docs/src/roles/sshd.md b/docs/src/roles/sshd.md index 6d2fa3e5..34bc2324 100644 --- a/docs/src/roles/sshd.md +++ b/docs/src/roles/sshd.md @@ -2,4 +2,4 @@ `units/sshd.sh` skips hosts without `/etc/ssh/sshd_config.d`. Otherwise it installs `99-bw-env.conf` with `AcceptEnv BW_SESSION BW_CLIENTID BW_CLIENTSECRET` as a best-effort step. If `sshd_harden: true` is set in host variables, it also validates and installs `10-hardening.conf`: root login, password and keyboard-interactive authentication, and X11 forwarding are disabled; authentication attempts and login grace time are limited. -Files are staged and installed root-owned (0644); root or noninteractive `sudo -n` is required. After a change the unit attempts to reload `ssh` on Debian/Ubuntu and `sshd` elsewhere. The shared unit library skips retired `matrix` and `telengana` before changes; nothing on those hosts may be restarted. No vault access is required. +Files are staged and installed root-owned (0644); root or noninteractive `sudo -n` is required. The unit fails before making changes if a host-variable file exists but cannot be read, rather than silently ignoring a hardening opt-in. After a change the unit attempts to reload `ssh` on Debian/Ubuntu and `sshd` elsewhere. The shared unit library skips retired `matrix` and `telengana` before changes; nothing on those hosts may be restarted. No vault access is required. diff --git a/docs/src/roles/sudo.md b/docs/src/roles/sudo.md index 9d427614..60d74e1e 100644 --- a/docs/src/roles/sudo.md +++ b/docs/src/roles/sudo.md @@ -1,5 +1,5 @@ # sudo -`units/sudo.sh` installs a passwordless sudo rule for the current user, unless `skip_sudo: true` is set in host variables. It needs root or noninteractive `sudo -n`; it does not use secrets. +`units/sudo.sh` installs a passwordless sudo rule for the current user, unless `skip_sudo: true` is set in host variables. If a host-variable file exists but cannot be read, it fails rather than ignoring an opt-out. It needs root or noninteractive `sudo -n`; it does not use secrets. -It validates a staged rule with `visudo -cf` before atomically installing `/etc/sudoers.d/zz-` as root-owned mode 0440. Only after successful validation and installation does it remove the older `` and `99-` rules. The unit validates the username before using it in a path. +It validates a staged rule with `visudo -cf` before atomically installing `/etc/sudoers.d/zz-` as root-owned mode 0440. Existing policy is inspected with root privileges, so a private sudoers directory or file does not cause a needless rewrite. An inspection failure preserves the current and legacy rules. Only after successful validation and installation does it remove the older `` and `99-` rules. The unit validates the username before using it in a path. diff --git a/docs/src/roles/syncthing.md b/docs/src/roles/syncthing.md index 75e66e17..352029f2 100644 --- a/docs/src/roles/syncthing.md +++ b/docs/src/roles/syncthing.md @@ -3,8 +3,8 @@ **Unit:** `units/syncthing.sh` **Tags:** `services`, `syncthing` **Secrets needed:** No -**Runs on:** non-VPS hosts, except Termux, `matrix`, `telengana`, `skip_syncthing: true`, or `machine_profile: vm-test`. +**Runs on:** non-VPS hosts, except Termux, `matrix`, `telengana`, `skip_syncthing: true`, or `machine_profile: vm-test`. An unreadable host-variable file fails the unit rather than bypassing an opt-out. -The unit requires `yq` to determine VPS membership. If `/home/linuxbrew/.linuxbrew/bin/syncthing` exists, it installs `units/syncthing/syncthing.service` to `~/.config/systemd/user/`, reloads the user manager when the file changes, and enables and starts the service. It requires a working user systemd manager. +The unit determines VPS membership from inventory with `yq` when available, or the repository's block-style inventory fallback on an unbootstrapped host. It fails closed if membership cannot be determined. If `/home/linuxbrew/.linuxbrew/bin/syncthing` exists, it installs `units/syncthing/syncthing.service` to `~/.config/systemd/user/`, reloads the user manager before enabling and starting the service, and retries the reload on subsequent applies if an earlier one failed. A destination symlink is replaced rather than followed, leaving its target untouched. It requires a working user systemd manager. If the binary is absent, it disables and stops an existing user service, including one supplied by a system-wide user-unit directory; if neither service nor binary exists, it skips. It does not configure Syncthing devices or folders or remove their data. diff --git a/docs/src/roles/systemd.md b/docs/src/roles/systemd.md index 88f7e51f..5606d239 100644 --- a/docs/src/roles/systemd.md +++ b/docs/src/roles/systemd.md @@ -1,9 +1,9 @@ # systemd services -`units/system-services.sh` and `units/laptop-services.sh` manage system and user services. Both skip Termux, `skip_systemd: true`, and retired `matrix`/`telengana`. They do not need vault secrets. System Podman configuration requires root or noninteractive `sudo -n`; user services require a working user systemd manager. +`units/system-services.sh` and `units/laptop-services.sh` manage system and user services. Both skip Termux, `skip_systemd: true`, and retired `matrix`/`telengana`. They fail before touching services if an existing host variables file is unreadable, rather than assuming default host settings. They do not need vault secrets. System Podman configuration requires root or noninteractive `sudo -n`; user services require a working user systemd manager. When `/usr/bin/podman` exists, system-services creates the `podman` group, adds the current user, installs a socket drop-in with group access (0660), and enables `podman.socket`. It restarts an already-active socket only when the drop-in changed. This applies to laptops too. -For non-laptops, system-services renders `units/system-services/` templates for the user `dotfiles-update` service and timer and the Atuin daemon, then enables the timer and daemon. It enables user linger through root or noninteractive `sudo -n` so the timer can run without an interactive login. Check mode reports missing linger without changing it. For `is_laptop: true`, laptop-services instead deploys the login-triggered update service and Atuin daemon, removes an obsolete update timer, and respects `dotfiles_auto_update`. The update uses `just apply-nosecrets`. +For non-laptops, system-services renders `units/system-services/` templates for the user `dotfiles-update` service and timer and the Atuin daemon, then enables the timer and daemon. It enables user linger through root or noninteractive `sudo -n` so the timer can run without an interactive login. Check mode reports missing linger without changing it. For `is_laptop: true`, laptop-services instead deploys the login-triggered update service and Atuin daemon, removes an obsolete update timer, and respects `dotfiles_auto_update`. The update uses `just apply-nosecrets`. Both services resolve `just` and `atuin` through their configured PATH, allowing system packages on musl as well as Homebrew. Switching laptop classification removes the previous managed update-service enablement link without deleting unrelated custom links. Check mode reports that removal without making it. On himachal, laptop-services also deploys `units/laptop-services/system-mitigator.py` and its user service. Unless `phone_apply_enabled: false`, it deploys a timer and service that archive the tracked checkout and run native units with `--no-secrets` independently over SSH on `termux` and `mumbai`. Unreachable phones do not fail that service. Disabling phone apply removes the timer and service. diff --git a/docs/src/roles/tailscale.md b/docs/src/roles/tailscale.md index 3a078dfc..98206225 100644 --- a/docs/src/roles/tailscale.md +++ b/docs/src/roles/tailscale.md @@ -7,3 +7,7 @@ When already connected it sets the hostname, reconciles configured advertised ro Host variables include `tailscale_advertise_routes`, `tailscale_accept_dns`, `tailscale_operator`, and `tailscale_domain` (the domain also falls back to `group_vars/all.yml`). The shared unit library skips retired `matrix` and `telengana` before this unit changes anything; nothing on those hosts may be restarted. Tailnet auditing, hostname release and OAuth credential setup are documented in [Tailnet hygiene](../tailnet.md). + +## Resolver recovery + +The unit repoints `/etc/resolv.conf` to the resolved stub when resolved is active and restarts resolved if the link changes. It does **not** restart `tailscaled`: doing so can disconnect a remote apply. A running daemon that previously selected another DNS manager may need to be restarted to detect the new layout. Arrange console or other independent access before restarting it, then check `journalctl -u tailscaled` for the selected DNS mode and `resolvectl status` for per-link upstreams. Test both a public name and a MagicDNS name through the system resolver and through `100.100.100.100`; working system lookups alone do not prove Tailscale's forwarder has an upstream. See [Tailscale's Linux DNS guidance](https://tailscale.com/kb/1188/linux-dns) and [upstream issue 19062](https://github.com/tailscale/tailscale/issues/19062). diff --git a/docs/src/roles/tailscale_cert.md b/docs/src/roles/tailscale_cert.md index 395e12a6..2e429e89 100644 --- a/docs/src/roles/tailscale_cert.md +++ b/docs/src/roles/tailscale_cert.md @@ -1,5 +1,9 @@ # tailscale_cert -`units/tailscale-cert.sh` obtains a Tailscale certificate for `.` and installs a systemd renewal service and timer from `units/tailscale-cert/`, rendered by `units/service-templates.py`. It checks whether `/etc/ssl/tailscale/cert.pem` remains valid for 30 days; otherwise it runs `tailscale cert` with a 720-hour minimum validity. The certificate is root-owned 0644 and its private key root-owned 0640. +`units/tailscale-cert.sh` obtains a Tailscale certificate for `.` and installs a systemd renewal service, timer and helper from `units/tailscale-cert/`. It renews when `/etc/ssl/tailscale/cert.pem` has less than 30 days of validity or its private key is absent or empty. The certificate is root-owned 0644 and its private key root-owned 0640. -The unit requires `yq`, Python, OpenSSL, Tailscale, systemd, and root or noninteractive `sudo -n`. It does not retrieve a Bitwarden secret. The host domain overrides `group_vars/all.yml`. It skips inventory VPSes, retired `matrix`/`telengana`, `skip_tailscale_cert: true`, and `machine_profile: vm-test`. The timer is enabled only after both renewal units are installed successfully. +A valid existing certificate must cover the configured hostname and match the private key. A mismatch fails without replacing the existing pair; resolve the mismatch explicitly before applying again. + +Both initial provisioning and the timer use the same helper: it serializes renewals with `flock`, requests the pair in a private temporary directory with a 720-hour minimum validity, checks the hostname, expiry and key match, and only replaces the existing files after a successful request. If committing the certificate fails, it restores the prior key; if restoration itself fails, the private backup is retained in the temporary directory and its path is reported. Symlinked or non-regular certificate paths are refused. The two file replacements cannot be atomic as a pair, so consumers may briefly observe the new key before the new certificate. + +The unit requires `yq`, Python, OpenSSL, Tailscale, `flock`, systemd, and root or noninteractive `sudo -n`. It does not retrieve a Bitwarden secret. Effective inventory group settings are honored and host settings take precedence. It skips inventory VPSes (including nested VPS groups), retired `matrix`/`telengana`, `skip_tailscale_cert: true`, and `machine_profile: vm-test`; unknown hosts and invalid inventory settings fail closed. The timer is enabled only after the renewal helper and both systemd units are installed successfully. diff --git a/docs/src/roles/termux_packages.md b/docs/src/roles/termux_packages.md index 785bf48b..45c248bf 100644 --- a/docs/src/roles/termux_packages.md +++ b/docs/src/roles/termux_packages.md @@ -1,5 +1,5 @@ # termux_packages -`units/termux-packages.sh` runs for inventory members of `termux_hosts`, using Termux's unprivileged `pkg` and npm rather than systemd or root. Defaults are in `units/termux-packages.conf`; `termux_pkgs`, `termux_npm_global`, and `termux_keepalive_period_ms` can be overridden in `group_vars/all.yml` and then `host_vars/.yml`. Reading overrides requires `yq`. +`units/termux-packages.sh` runs only in the Termux environment for inventory members of `termux_hosts`, using Termux's unprivileged `pkg` and npm rather than systemd or root. Defaults are in `units/termux-packages.conf`; `termux_pkgs`, `termux_npm_global`, and `termux_keepalive_period_ms` can be overridden in `group_vars/all.yml` and then `host_vars/.yml`. Reading overrides requires `yq`. -It retries `pkg update` three times, installs bootstrap and configured packages (falling back to individual installs after a failed batch), and installs configured npm globals. Optional mise installation is best effort. It installs `units/termux-start-sshd` as `~/.termux/boot/start-sshd`, starts sshd if absent, and schedules persisted job 1 if not already pending. Termux:Boot and Termux:API companion apps are separate prerequisites. No vault secrets are used. +It retries `pkg update` three times, installs bootstrap and configured packages (falling back to individual installs after a failed batch), and installs configured npm globals. Bootstrap includes `just` for fleet commands; an initial installation of `just` is still needed to invoke those commands before the unit has run. Optional mise installation is best effort. It installs `units/termux-start-sshd` as `~/.termux/boot/start-sshd`, starts sshd if absent, and schedules persisted job 1 if not already pending. Existing Ansible and prior native managed boot scripts are recognized; customized, non-regular or symlinked scripts are preserved. Scheduling is skipped if deployment or the scheduler query fails. Existing job 1 is left unchanged. Termux:Boot and Termux:API companion apps are separate prerequisites. No vault secrets are used. diff --git a/docs/src/roles/zen_browser.md b/docs/src/roles/zen_browser.md index a7c882f8..13165c2a 100644 --- a/docs/src/roles/zen_browser.md +++ b/docs/src/roles/zen_browser.md @@ -5,7 +5,7 @@ **Secrets needed:** No **Runs on:** desktops, except `skip_zen_browser: true` or `machine_profile: vm-test`. -The unit best-effort pins Zen in the GNOME dock, removes Firefox and Thunderbird dock entries, and sets browser MIME defaults when the Zen Flatpak data directory exists. A live user session is needed for dock changes. +The unit best-effort pins Zen in the GNOME dock, including an initially empty favorites list, removes Firefox and Thunderbird dock entries, and sets browser MIME defaults when the Zen Flatpak data directory exists. A live user session is needed for dock changes. If Zen's Flatpak `profiles.ini` is absent, profile and policy steps are skipped. Otherwise it resolves `Profile0` and merges `units/zen-browser/user.js` into a managed block in that profile's `user.js`, retaining unrelated preferences. It merges `units/zen-browser/policies.json` into both the system Flatpak Zen distribution policy and `~/.zen/distribution/policies.json`, preserving unrelated policy keys and list entries. The system policy requires root or non-interactive sudo; malformed or incompatible existing policies fail rather than being discarded. diff --git a/docs/src/runbook.md b/docs/src/runbook.md index 79be9474..2303c1a1 100644 --- a/docs/src/runbook.md +++ b/docs/src/runbook.md @@ -106,10 +106,11 @@ but is not a guarantee against concurrent Bitwarden CLI races. ## I want to onboard a new machine — full sequence ```bash -# 1. From an existing fleet machine (with BW unlocked): +# 1. From an authenticated fleet checkout: just onboard desktop # or server / vps +# Review the registration PR, wait for required checks and approval, and merge it. -# 2. Copy the printed bootstrap command, then on the new machine: +# 2. After merge, run the printed bootstrap command on the new machine: curl -fsSL https://raw.githubusercontent.com/hanthor/dotfiles/master/bootstrap.sh \ | bash -s -- --name --type desktop diff --git a/docs/src/scripts.md b/docs/src/scripts.md index 7a49056a..027b5674 100644 --- a/docs/src/scripts.md +++ b/docs/src/scripts.md @@ -13,7 +13,7 @@ The shared unit library skips retired `matrix` and `telengana` before unit work. ## Inventory -- `scripts/register-machine.py [type] [inventory-path]` adds an inventory host and group membership and creates host variables. `just onboard` wraps registration and prints bootstrap instructions. +- `scripts/register-machine.py [type] [inventory-path]` adds an inventory host and group membership; `--check` verifies existing membership without writing. `just onboard` proposes the change and a host-vars file in an isolated worktree, opens a PR, and prints bootstrap instructions to use after review and merge. - `scripts/purge-machine.py [inventory-path]` removes inventory and host variables, not Bitwarden items. `just purge` wraps it. - `scripts/online_hosts.py` intersects online Tailscale peers with inventory, excluding VPSes and the local host. `--peers` prints online peer names instead; the SSH mesh unit uses it. Tailscale status failure yields no peers. - `scripts/nmap-inventory.sh [CIDR]` scans the LAN and cross-references Tailscale; it requires nmap and network access. `just inventory` provides the usual entrypoint. diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index d211df13..7465572c 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -6,6 +6,17 @@ available over Tailscale at `100.66.158.127`. A single-node K3s cluster runs KubeVirt, CDI, and Corral. The `corral-vms/punjab` VM is a staging guest; it is not the EC2 host and has not received its data or services. +NetworkManager, Tailscale and SSH are enabled at boot. Ethernet is preferred; +a locally stored Wi-Fi profile autoconnects with a higher route metric. Wi-Fi +internet and SSH over Tailscale were verified with Ethernet connected; cable +failure and reboot failover have not been tested. The Tailscale device key +expires 2027-04-22 and may require reauthentication. Keep a console recovery +path before testing loss of the wired link. + +Host DNS uses the systemd-resolved stub. Public DNS and MagicDNS resolved after +repair, but Tailscale later reported that its forwarder lacked upstream DNS; +inspect `tailscale status` if lookups fail. K3s uses a separate pod resolver. + As last checked, the fleet update timer is not installed and user linger is disabled. The native-unit changes are under review; do not enable unattended updates from the unmerged checkout. diff --git a/group_vars/all.yml b/group_vars/all.yml index b2f1a443..3c5dc1fb 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -4,11 +4,8 @@ user_name: James Reilly user_email: jreilly1821@gmail.com user_login: james brew_prefix: /home/linuxbrew/.linuxbrew -# Every inventory host reachable over SSH, keyed by inventory name, with the -# address ssh should use (MagicDNS name unless noted). Port and login user -# come from inventory.yml (ansible_port / ansible_user) — ssh_config.j2 and -# ssh_mesh read them from there. Talos nodes (bihar, karnataka) are absent on -# purpose: Talos has no SSH. +# SSH aliases are keyed by inventory name; connection users and ports come +# from inventory.yml. Bihar's current OS is unverified, so it is not added. machines: punjab: hostname: punjab @@ -17,9 +14,11 @@ machines: termux: hostname: pixel-9a himachal: - hostname: himachal + hostname: himachal-1 + karnataka: + hostname: karnataka kerala: - hostname: kerala + hostname: kerala-1 dilli: hostname: dilli kanpur: @@ -39,7 +38,7 @@ machines: hostname: 127.0.0.1 # Fleet SSH public keys, committed to git (public keys only — safe in a -# public repo). The ssh_mesh role merges these into authorized_keys on +# public repo). The ssh-mesh unit merges these into authorized_keys on # every apply (incl. the daily no-secrets timer) so the mesh self-heals # without Bitwarden. Each machine warns on apply if its own key is # missing/stale here. diff --git a/inventory.yml b/inventory.yml index 189ed41c..c3fd0605 100644 --- a/inventory.yml +++ b/inventory.yml @@ -48,8 +48,9 @@ all: ansible_host: 100.101.234.32 ansible_connection: ssh himachal: - ansible_host: 100.73.3.51 + ansible_host: 100.79.182.70 ansible_connection: ssh + tailnet_name: himachal-1 dilli: ansible_host: dilli ansible_user: james diff --git a/scripts/inventory_parser.py b/scripts/inventory_parser.py index 1a33fd05..413fc050 100644 --- a/scripts/inventory_parser.py +++ b/scripts/inventory_parser.py @@ -13,6 +13,33 @@ class UnknownGroupError(ValueError): """Raised when a machine type has no matching group in the inventory.""" +def validate_host_name(name: str) -> None: + """Require a single DNS hostname, safe in inventory, paths and SSH.""" + if len(name) > 63 or not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", name): + raise ValueError("machine name must be a lowercase DNS hostname") + + +def host_groups(text: str, host: str): + """Return direct children groups containing a host in this inventory layout.""" + groups, section, group, in_hosts = set(), None, None, False + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + indent = len(line) - len(line.lstrip()) + match = re.match(r"([A-Za-z0-9_-]+):(?:\s*(?:\{\})?)?(?:\s*#.*)?$", line.strip()) + if not match: + continue + key = match.group(1) + if indent == 2: + section, group, in_hosts = key, None, False + elif section == "children" and indent == 4: + group, in_hosts = key, False + elif section == "children" and indent == 6: + in_hosts = key == "hosts" + elif section == "children" and indent == 8 and in_hosts and key == host: + groups.add(group) + return groups + def _read(inv_path: str) -> str: with open(inv_path, "r", encoding="utf-8") as f: @@ -39,7 +66,9 @@ def parse_inventory(text: str): if indent == 2: section = key elif indent == 4 and section == "hosts": - all_hosts.add(key) + match = re.match(r"([A-Za-z0-9_-]+):(?:\s*(?:\{\})?)?(?:\s*#.*)?$", line.strip()) + if match: + all_hosts.add(match.group(1)) elif indent == 4 and section == "children": group = key elif indent == 8 and section == "children" and group == "vps": @@ -113,14 +142,17 @@ def register_host(name: str, mtype: str = "desktop", inv_path: str = "inventory. Returns False (writing nothing) when the host is already present. Raises UnknownGroupError, again writing nothing, when `mtype` names no group. """ + validate_host_name(name) content = _read(inv_path) - if f" {name}:" in content: + if name in parse_inventory(content)[0]: return False group_marker = f" {mtype}:\n hosts:\n" if group_marker not in content: raise UnknownGroupError(mtype) + if "all:\n hosts:\n" not in content: + raise ValueError("inventory has no supported all.hosts section") host_entry = f" {name}:\n ansible_host: localhost\n ansible_connection: local\n" content = content.replace("all:\n hosts:\n", f"all:\n hosts:\n{host_entry}", 1) diff --git a/scripts/onboard-machine.sh b/scripts/onboard-machine.sh new file mode 100755 index 00000000..8242006c --- /dev/null +++ b/scripts/onboard-machine.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Propose an inventory-only registration without modifying the caller's checkout. +set -euo pipefail +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +name=${1:?machine name required} +type=${2:-desktop} +case "$name" in matrix|telengana) echo "Retired VPSes must not be modified." >&2; exit 2 ;; esac +# Validate before using the name in paths, branch names or commands. +python3 - "$repo/scripts" "$name" "$type" <<'PY' +import sys +sys.path.insert(0, sys.argv[1]) +from inventory_parser import validate_host_name +try: + validate_host_name(sys.argv[2]) + if sys.argv[3] not in ('desktop', 'server', 'vps', 'termux_hosts'): + raise ValueError('machine type must be desktop, server, vps, or termux_hosts') +except ValueError as error: + print(error, file=sys.stderr) + sys.exit(2) +PY + +git -C "$repo" fetch origin '+refs/heads/master:refs/remotes/origin/master' +base=$(git -C "$repo" rev-parse refs/remotes/origin/master) +tmp=$(mktemp -d) +worktree="$tmp/worktree" +cleanup() { + if [ -e "$worktree/.git" ]; then git -C "$repo" worktree remove --force "$worktree" || true; fi + rm -rf -- "$tmp" +} +trap cleanup EXIT +git -C "$repo" worktree add --detach "$worktree" "$base" +rc=0 +python3 "$repo/scripts/register-machine.py" --check "$name" "$type" "$worktree/inventory.yml" || rc=$? +case "$rc" in + 0) echo "$name is already registered as $type on origin/master."; exit 0 ;; + 3) ;; + *) exit "$rc" ;; +esac +if [ "$type" = termux_hosts ]; then + echo "New Termux identities need an individually reviewed inventory PR with the correct Android Tailscale name, SSH user and port; add the host to termux_hosts, merge, then rerun." >&2 + exit 1 +fi +command -v gh >/dev/null 2>&1 || { echo "GitHub CLI (gh) is required to propose registration; use an authenticated fleet checkout." >&2; exit 1; } +branch="onboarding/$name" +pr=$(cd "$worktree" && gh pr list --base master --head "$branch" --state open --json url --jq '.[0].url // empty') +if [ -n "$pr" ]; then + echo "Registration awaits review and merge: $pr" + exit 3 +fi +rc=0 +git -C "$repo" ls-remote --exit-code origin "refs/heads/$branch" >/dev/null || rc=$? +if [ "$rc" -eq 0 ]; then + echo "Remote branch $branch already exists without an open PR; inspect it rather than overwriting it." >&2 + exit 1 +elif [ "$rc" -ne 2 ]; then + echo "Could not inspect remote registration branch." >&2 + exit "$rc" +fi +python3 "$repo/scripts/register-machine.py" "$name" "$type" "$worktree/inventory.yml" +mkdir -p "$worktree/host_vars" +if [ ! -e "$worktree/host_vars/$name.yml" ]; then printf '%s\n' '---' > "$worktree/host_vars/$name.yml"; fi +git -C "$worktree" add -- inventory.yml "host_vars/$name.yml" +git -C "$worktree" commit -m "inventory: add $name ($type)" +# The empty lease requires the remote branch not to exist; never replace anyone's branch. +git -C "$worktree" push --force-with-lease="refs/heads/$branch:" origin "HEAD:refs/heads/$branch" +if ! (cd "$worktree" && gh pr create --base master --head "$branch" --title "inventory: add $name ($type)" --body "Register $name as a $type. Review the inventory and host-specific settings before merging." ); then + echo "PR creation failed; branch $branch was retained. Inspect it and create the PR with gh pr create --base master --head $branch." >&2 + exit 1 +fi +echo "Review and merge the registration PR, then rerun the onboarding command." +exit 3 diff --git a/scripts/register-machine.py b/scripts/register-machine.py index c4435a9f..1a0fd4a2 100644 --- a/scripts/register-machine.py +++ b/scripts/register-machine.py @@ -1,22 +1,45 @@ #!/usr/bin/env python3 -"""Register a new machine in inventory.yml and create its host_vars file.""" +"""Register a machine or check its reviewed inventory membership.""" import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from inventory_parser import UnknownGroupError, register_host +from inventory_parser import UnknownGroupError, host_groups, parse_inventory, register_host, validate_host_name -name = sys.argv[1] -mtype = sys.argv[2] if len(sys.argv) > 2 else "desktop" -inv_path = sys.argv[3] if len(sys.argv) > 3 else "inventory.yml" -try: - added = register_host(name, mtype, inv_path) -except UnknownGroupError: - print(f" Unknown machine type/group '{mtype}' in {inv_path} (e.g. desktop, server, vps)", file=sys.stderr) - sys.exit(1) +def main(argv): + check = bool(argv and argv[0] == "--check") + if check: + argv = argv[1:] + if not 1 <= len(argv) <= 3: + print("usage: register-machine.py [--check] NAME [TYPE] [INVENTORY]", file=sys.stderr) + return 2 + name = argv[0] + mtype = argv[1] if len(argv) > 1 else "desktop" + inv_path = argv[2] if len(argv) > 2 else "inventory.yml" + try: + validate_host_name(name) + if check: + if mtype not in ("desktop", "server", "vps", "termux_hosts"): + raise ValueError("machine type must be desktop, server, vps, or termux_hosts") + with open(inv_path, encoding="utf-8") as source: + content = source.read() + if name not in parse_inventory(content)[0]: + return 3 + groups = host_groups(content, name) + if mtype not in groups: + raise ValueError(f"{name} is not in the {mtype} group") + return 0 + added = register_host(name, mtype, inv_path) + except (OSError, UnknownGroupError, ValueError) as error: + print(f"registration failed: {error}", file=sys.stderr) + return 2 + if added: + print(f" Added {name} to {mtype} group in {inv_path}") + else: + print(f" {name} is already in inventory.") + return 0 -if added: - print(f" Added {name} to {mtype} group in {inv_path}") -else: - print(f" {name} is already in inventory.") + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/run-units.sh b/scripts/run-units.sh index c97983c3..37b19517 100755 --- a/scripts/run-units.sh +++ b/scripts/run-units.sh @@ -18,9 +18,9 @@ tasks_for_tag() { case "$1" in shell) echo "chezmoi fonts atuin shell-ai" ;; fonts) echo fonts ;; git|neovim|editor) echo chezmoi ;; - atuin) echo atuin ;; + atuin) echo "chezmoi atuin" ;; ai) echo shell-ai ;; - secrets) echo "atuin shell-ai ssh-keys github kube forgejo-registry tailscale browser-fxa" ;; + secrets) echo "chezmoi atuin shell-ai ssh-keys github kube forgejo-registry tailscale browser-fxa" ;; ssh|ssh_keys) echo "ssh-keys ssh-mesh" ;; mesh_check) echo ssh-mesh ;; github) echo github ;; @@ -66,18 +66,26 @@ while [ $# -gt 0 ]; do done wanted=" " +chezmoi_full=0 if [ -z "$tags" ]; then + chezmoi_full=1 wanted=" $ALL_UNITS " else IFS=',' read -ra tag_list <<< "$tags" for t in "${tag_list[@]}"; do mapped=$(tasks_for_tag "$t") + case "$t" in dotfiles|shell|git|neovim|editor) chezmoi_full=1 ;; esac if [ -z "$mapped" ]; then echo "unknown unit tag: $t" >&2; exit 2; fi read -ra mapped_units <<< "$mapped" for u in "${mapped_units[@]}"; do wanted="$wanted$u "; done done fi selected() { case "$wanted" in *" $1 "*) return 0 ;; *) return 1 ;; esac; } +if selected chezmoi && [ "$chezmoi_full" = 0 ]; then + export DOTFILES_CHEZMOI_SCOPE=atuin +else + unset DOTFILES_CHEZMOI_SCOPE +fi cd "$repo" || exit 1 if [ "${UNITS_CHECK:-0}" = 1 ]; then diff --git a/tests/test_atuin_unit.py b/tests/test_atuin_unit.py index 399a7f4a..c930ac0b 100644 --- a/tests/test_atuin_unit.py +++ b/tests/test_atuin_unit.py @@ -19,9 +19,9 @@ def stubs(bindir, status_out, status_rc=1, unlock_rc=0, version="18.24.0"): case "$1" in status) echo '{status_out}'; exit {status_rc} ;; --version) echo 'atuin {version}'; exit 0 ;; - info) echo "key path: $HOME/.local/share/atuin/key"; exit 0 ;; + info) echo "key path: $HOME/.local/share/atuin/key"; exit "${{STUB_INFO_RC:-0}}" ;; key) cat "$HOME/.local/share/atuin/key"; exit $? ;; - login) [ "$*" = 'login -u U' ] && [ "$ATUIN_PASSWORD" = P ] && [ "$ATUIN_ENCRYPTION_KEY" = K ]; exit $? ;; + login) [ "$*" = 'login -u U' ] && [ "$ATUIN_PASSWORD" = P ] && [ "$ATUIN_ENCRYPTION_KEY" = K ] && [ "${{STUB_LOGIN_RC:-0}}" = 0 ]; exit $? ;; esac exit 0 """, @@ -31,7 +31,7 @@ def stubs(bindir, status_out, status_rc=1, unlock_rc=0, version="18.24.0"): "bw", f"""echo "bw $*" >> "$HOME/calls" [ "$1 $2" = "unlock --check" ] && exit {unlock_rc} -[ "$1 $2 $3" = "get item atuin.sh" ] && echo '{ITEM}' +[ "$1 $2 $3" = "get item atuin.sh" ] && {{ echo '{ITEM}'; exit "${{STUB_ITEM_RC:-0}}"; }} exit 0 """, ) @@ -156,3 +156,28 @@ def test_missing_atuin_is_skipped(stub_env): r, _ = run(env) assert r.returncode == 0 assert "SKIPPED:" in r.stdout + + +def test_failed_info_does_not_use_partial_key_path_or_import(stub_env): + bindir, env = stub_env + stubs(bindir, "not logged in") + result, calls = run({**env, "STUB_INFO_RC": "1"}) + assert result.returncode == 0 and "cannot determine the Atuin key path" in result.stdout + assert "atuin login" not in calls and "atuin import" not in calls + + +def test_failed_login_does_not_import_history(stub_env): + bindir, env = stub_env + stubs(bindir, "not logged in") + result, calls = run({**env, "STUB_LOGIN_RC": "1"}) + assert result.returncode != 0 and "FAIL atuin login" in result.stdout + assert "atuin login" in calls and "atuin import" not in calls + assert not Path(env["HOME"], ".local/share/atuin/.history_imported").exists() + + +def test_failed_vault_read_does_not_use_partial_credentials(stub_env): + bindir, env = stub_env + stubs(bindir, "not logged in") + result, calls = run({**env, "STUB_ITEM_RC": "1"}) + assert result.returncode == 0 and "missing or incomplete" in result.stdout + assert "atuin login" not in calls diff --git a/tests/test_bluefin_sync.py b/tests/test_bluefin_sync.py index 9a2d38a9..85ed3577 100644 --- a/tests/test_bluefin_sync.py +++ b/tests/test_bluefin_sync.py @@ -110,6 +110,20 @@ def test_redirects_system_background_path_to_local(self, tmp_path): assert "/usr/share/backgrounds/bluefin" not in value assert ".local/share/backgrounds/bluefin" in value + def test_background_uri_encodes_special_characters_in_home(self, tmp_path): + home = tmp_path / 'home & 100%' + override = tmp_path / 'bg.gschema.override' + override.write_text( + "[org.gnome.desktop.background]\n" + "picture-uri='file:///usr/share/backgrounds/bluefin/foo.webp'\n" + ) + with mock.patch.dict('os.environ', {'HOME': str(home)}), \ + mock.patch.object(bluefin_sync, 'run_cmd') as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stderr='') + bluefin_sync.apply_gschema_override(str(override)) + uri = (home / '.local/share/backgrounds/bluefin/foo.webp').as_uri() + assert mock_run.call_args[0][0][-1] == f"'{uri}'" + def test_adds_caffeine_to_enabled_extensions_when_missing(self, tmp_path): """caffeine@patapon.info is appended when enabled-extensions omits it.""" override = tmp_path / "ext.gschema.override" diff --git a/tests/test_bluefin_unit.py b/tests/test_bluefin_unit.py index 5368cdbe..1b5cbe4c 100644 --- a/tests/test_bluefin_unit.py +++ b/tests/test_bluefin_unit.py @@ -186,6 +186,31 @@ def test_inventory_reader_errors_are_not_a_successful_skip(desktop): assert log(env) == "" +def test_unrecognized_existing_extension_is_preserved(tmp_path, monkeypatch, capsys): + import importlib.util + + monkeypatch.setenv('HOME', str(tmp_path)) + script = REPO / 'units/bluefin/files/install-extensions.py' + spec = importlib.util.spec_from_file_location('bluefin_extensions', script) + installer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(installer) + uuid = 'example@example.com' + extension = Path(installer.USER_EXT_DIR) / uuid + extension.mkdir() + customized = extension / 'extension.js' + customized.write_text('custom extension') + monkeypatch.setattr(installer.subprocess, 'run', + lambda args, **kwargs: subprocess.CompletedProcess(args, 1)) + + def unexpected_download(*args, **kwargs): + raise AssertionError('existing extension must not be downloaded over') + monkeypatch.setattr(installer.urllib.request, 'urlopen', unexpected_download) + + installer.install_ext(uuid) + assert customized.read_text() == 'custom extension' + assert 'preserving existing installation' in capsys.readouterr().out + + def test_gnome_sync_and_system_changes_are_idempotent(desktop): env, _ = desktop home = Path(env["HOME"]) @@ -219,6 +244,54 @@ def test_gnome_sync_and_system_changes_are_idempotent(desktop): assert "sysctl -p" not in later assert "systemctl --user enable --now bluefin-wallpaper-rotate.timer" in later +def test_branding_preserves_unmanaged_xml_and_escapes_home(desktop, tmp_path): + import xml.etree.ElementTree as ET + + env, _ = desktop + home = tmp_path / 'home & 100% "user"' + home.mkdir() + env['HOME'] = str(home) + backgrounds = home / '.local/share/backgrounds/bluefin' + backgrounds.mkdir(parents=True) + custom = backgrounds / 'custom.xml' + original = '/usr/share/backgrounds/bluefin/custom.jpg\n' + custom.write_text(original) + + result = run(env) + assert result.returncode == 0, result.stdout + result.stderr + assert custom.read_text() == original + assert ET.parse(backgrounds / '01-bluefin.xml').getroot().text == str(backgrounds / 'a.jpg') + service = (home / '.config/systemd/user/bluefin-wallpaper-rotate.service').read_text() + executable = str(home / '.local/bin/bluefin-wallpaper-rotate.py') + escaped = executable.replace('%', '%%').replace('\\', '\\\\').replace('"', '\\"') + assert f'ExecStart="{escaped}"' in service + + +def test_check_mode_matches_escaped_service_and_wallpaper_uri(desktop, tmp_path): + import datetime + + env, _ = desktop + home = tmp_path / 'home & 100% "user"' + home.mkdir() + env['HOME'] = str(home) + applied = run(env) + assert applied.returncode == 0, applied.stdout + applied.stderr + + wallpaper = home / '.local/share/backgrounds/bluefin' / (datetime.datetime.now().strftime('%m') + '-bluefin.xml') + wallpaper.touch(exist_ok=True) + bindir = Path(env['PATH'].split(':')[0]) + write_stub(bindir, 'gsettings', '''\ +if [ "$2" = org.gnome.desktop.background ]; then printf "'%s'\\n" "$EXPECTED_URI"; fi +''') + write_stub(bindir, 'systemctl', '''\ +printf 'LoadState=loaded\\nUnitFileState=enabled\\nActiveState=active\\n' +''') + checked = run(env, UNITS_CHECK='1', STUB_BLUEFIN_RC='0', EXPECTED_URI=wallpaper.as_uri()) + assert checked.returncode == 0, checked.stdout + checked.stderr + assert 'OK wallpaper service' in checked.stdout + assert 'OK picture-uri\n' in checked.stdout + assert 'OK picture-uri-dark\n' in checked.stdout + def test_existing_custom_bus_is_used_for_probe_and_service(desktop): env, _ = desktop diff --git a/tests/test_bluefin_wallpaper_rotate.py b/tests/test_bluefin_wallpaper_rotate.py index 0097aca0..d6aebf48 100644 --- a/tests/test_bluefin_wallpaper_rotate.py +++ b/tests/test_bluefin_wallpaper_rotate.py @@ -1,123 +1,83 @@ -"""Tests for bluefin-wallpaper-rotate.py wallpaper rotation logic.""" - +"""Behavioral coverage for the installed monthly wallpaper script.""" +import datetime +import importlib.util +import subprocess import sys -import os -from datetime import datetime from pathlib import Path -from unittest import TestCase -from unittest.mock import patch, MagicMock, call -import tempfile +from unittest import mock + +import pytest + +from conftest import REPO, write_stub -# Import the script's logic (we'll create a testable module from it) -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../units/bluefin/files")) +SCRIPT = REPO / "units/bluefin/files/bluefin-wallpaper-rotate.py" +spec = importlib.util.spec_from_file_location('bluefin_wallpaper', SCRIPT) +wallpaper_script = importlib.util.module_from_spec(spec) +spec.loader.exec_module(wallpaper_script) -class WallpaperRotationTests(TestCase): - """Test cases for bluefin-wallpaper-rotate.py.""" - def test_wallpaper_name_format(self): - """Verify wallpaper name matches expected format (MM-bluefin.xml).""" - with patch("datetime.datetime") as mock_dt: - mock_dt.now.return_value = MagicMock( - strftime=MagicMock(return_value="03") - ) - # Simulate the wallpaper naming logic - month = mock_dt.now().strftime("%m") - wallpaper_name = f"{month}-bluefin.xml" - self.assertEqual(wallpaper_name, "03-bluefin.xml") +@pytest.fixture +def wallpaper_env(tmp_path, stub_env): + bindir, env = stub_env + home = tmp_path / 'home & 100%' + home.mkdir() + env['HOME'] = str(home) + write_stub(bindir, 'gsettings', '''\ +printf '%s\\n' "$3 $4" >> "$HOME/calls" +if [ "${FAIL_KEY:-}" = "$3" ]; then exit 1; fi +''') + return env, home - def test_wallpaper_path_expansion(self): - """Test that wallpaper paths are properly expanded.""" - with tempfile.TemporaryDirectory() as tmpdir: - with patch.dict(os.environ, {"HOME": tmpdir}): - with patch("os.path.expanduser") as mock_expand: - mock_expand.return_value = f"{tmpdir}/.local/share/backgrounds/bluefin/01-bluefin.xml" - expanded = mock_expand( - "~/.local/share/backgrounds/bluefin/01-bluefin.xml" - ) - self.assertTrue(expanded.endswith("01-bluefin.xml")) - def test_wallpaper_existence_check(self): - """Test that missing wallpaper is skipped gracefully.""" - with tempfile.TemporaryDirectory() as tmpdir: - nonexistent_path = os.path.join(tmpdir, "nonexistent.xml") - self.assertFalse(os.path.exists(nonexistent_path)) +def run(env, **overrides): + return subprocess.run([sys.executable, str(SCRIPT)], env={**env, **overrides}, + capture_output=True, text=True) - def test_wallpaper_file_exists(self): - """Test that existing wallpaper file is detected.""" - with tempfile.TemporaryDirectory() as tmpdir: - wallpaper_path = os.path.join(tmpdir, "03-bluefin.xml") - Path(wallpaper_path).touch() - self.assertTrue(os.path.exists(wallpaper_path)) - @patch("subprocess.run") - def test_gsettings_light_variant_call(self, mock_run): - """Test that gsettings is called for light variant.""" - uri = "file:///home/user/.local/share/backgrounds/bluefin/03-bluefin.xml" - - # Simulate the gsettings call for light variant - expected_call = [ - "gsettings", "set", "org.gnome.desktop.background", - "picture-uri", uri - ] - - mock_run(expected_call) - mock_run.assert_called_once_with(expected_call) +def test_missing_wallpaper_skips_without_changing_settings(wallpaper_env, tmp_path, monkeypatch, capsys): + _, home = wallpaper_env + monkeypatch.setenv('HOME', str(home)) + monkeypatch.setattr(wallpaper_script, 'SYSTEM_BACKGROUNDS', tmp_path / 'absent') + runner = mock.Mock() + monkeypatch.setattr(wallpaper_script.subprocess, 'run', runner) + assert wallpaper_script.set_wallpaper() == 0 + assert 'not found, skipping rotation' in capsys.readouterr().out + runner.assert_not_called() - @patch("subprocess.run") - def test_gsettings_dark_variant_call(self, mock_run): - """Test that gsettings is called for dark variant.""" - uri = "file:///home/user/.local/share/backgrounds/bluefin/03-bluefin.xml" - - # Simulate the gsettings call for dark variant - expected_call = [ - "gsettings", "set", "org.gnome.desktop.background", - "picture-uri-dark", uri - ] - - mock_run(expected_call) - mock_run.assert_called_once_with(expected_call) - @patch("subprocess.run") - def test_both_gsettings_variants_called(self, mock_run): - """Test that both light and dark variants are set.""" - uri = "file:///home/user/.local/share/backgrounds/bluefin/06-bluefin.xml" - - # Simulate both calls - light_call = ["gsettings", "set", "org.gnome.desktop.background", "picture-uri", uri] - dark_call = ["gsettings", "set", "org.gnome.desktop.background", "picture-uri-dark", uri] - - mock_run(light_call) - mock_run(dark_call) - - self.assertEqual(mock_run.call_count, 2) - calls = [call(light_call), call(dark_call)] - mock_run.assert_has_calls(calls) +def test_system_wallpaper_is_used_when_no_local_copy(wallpaper_env, tmp_path, monkeypatch): + _, home = wallpaper_env + monkeypatch.setenv('HOME', str(home)) + system = tmp_path / 'system-backgrounds' + system.mkdir() + name = datetime.datetime.now().strftime('%m') + '-bluefin.xml' + system_wallpaper = system / name + system_wallpaper.touch() + monkeypatch.setattr(wallpaper_script, 'SYSTEM_BACKGROUNDS', system) + runner = mock.Mock(return_value=subprocess.CompletedProcess([], 0)) + monkeypatch.setattr(wallpaper_script.subprocess, 'run', runner) - @patch("datetime.datetime") - def test_all_months_generate_valid_names(self, mock_dt_class): - """Test that each month generates a valid wallpaper name.""" - for month_num in range(1, 13): - mock_dt = MagicMock() - mock_dt.strftime.return_value = f"{month_num:02d}" - mock_dt_class.now.return_value = mock_dt - - month = mock_dt_class.now().strftime("%m") - wallpaper_name = f"{month}-bluefin.xml" - - # Verify format: MM-bluefin.xml - self.assertRegex(wallpaper_name, r"^\d{2}-bluefin\.xml$") - self.assertIn(month, wallpaper_name) + assert wallpaper_script.set_wallpaper() == 0 + assert [call.args[0][-1] for call in runner.call_args_list] == [system_wallpaper.as_uri()] * 2 + local = home / '.local/share/backgrounds/bluefin' / name + local.parent.mkdir(parents=True) + local.touch() + runner.reset_mock() + assert wallpaper_script.set_wallpaper() == 0 + assert [call.args[0][-1] for call in runner.call_args_list] == [local.as_uri()] * 2 - def test_uri_file_prefix(self): - """Test that wallpaper URIs are properly prefixed with file://.""" - local_path = "/home/user/.local/share/backgrounds/bluefin/01-bluefin.xml" - uri = f"file://{local_path}" - self.assertTrue(uri.startswith("file://")) - self.assertIn(local_path, uri) - @patch("subprocess.run") - def test_missing_wallpaper_no_gsettings_call(self, mock_run): - """Test that gsettings is not called when wallpaper is missing.""" - # When wallpaper doesn't exist, no gsettings calls should be made - mock_run.assert_not_called() +@pytest.mark.parametrize('failed_key', ['', 'picture-uri', 'picture-uri-dark']) +def test_rotates_both_variants_and_reports_failures(wallpaper_env, failed_key): + env, home = wallpaper_env + wallpaper = home / '.local/share/backgrounds/bluefin' / (datetime.datetime.now().strftime('%m') + '-bluefin.xml') + wallpaper.parent.mkdir(parents=True) + wallpaper.touch() + result = run(env, FAIL_KEY=failed_key) + assert result.returncode == (1 if failed_key else 0) + uri = wallpaper.as_uri() + assert '%20' in uri and '%25' in uri + assert (home / 'calls').read_text().splitlines() == [ + f'picture-uri {uri}', f'picture-uri-dark {uri}', + ] diff --git a/tests/test_bootstrap_units.py b/tests/test_bootstrap_units.py index f8cf1333..4f6e207e 100644 --- a/tests/test_bootstrap_units.py +++ b/tests/test_bootstrap_units.py @@ -1,3 +1,4 @@ +import shutil import subprocess from pathlib import Path @@ -12,7 +13,7 @@ def test_bootstrap_runs_second_phase_after_first_fails(stub_env, tmp_path): env["CALLS"] = str(tmp_path / "calls") script = r''' source "$1/bootstrap.sh" --name test --type desktop -for f in ensure_python ensure_git clone_repo get_machine_name get_machine_type register_in_inventory write_machine_name set_hostname ensure_sudo ensure_tailscale setup_bitwarden push_inventory_changes; do +for f in ensure_python ensure_git clone_repo get_machine_name get_machine_type register_in_inventory write_machine_name set_hostname ensure_sudo ensure_tailscale setup_bitwarden; do eval "$f() { :; }" done run_phase1() { echo first >> "$CALLS"; return 7; } @@ -48,28 +49,84 @@ def test_unit_session_respects_no_secrets_and_keeps_session_off_argv(stub_env, t assert log.read_text().splitlines() == ["unlock --check"] -@pytest.mark.parametrize("failure", ["remote", "add", "diff", "commit", "push"]) -def test_inventory_push_propagates_git_failures_inside_or(stub_env, tmp_path, failure): +@pytest.mark.parametrize("local_registered,remote_registered", [(False, False), (True, False)]) +def test_bootstrap_refuses_unreviewed_registration(stub_env, tmp_path, local_registered, remote_registered): bindir, env = stub_env - env.update(CALLS=str(tmp_path / "git-calls"), FAIL_GIT=failure, TEST_REPO=str(tmp_path)) - write_stub(bindir, "git", ''' -printf '%s\\n' "$1" >> "$CALLS" -if [ "$1" = "$FAIL_GIT" ]; then exit 7; fi -if [ "$1" = diff ]; then exit 1; fi -''') + scripts = tmp_path / "scripts" + scripts.mkdir() + for name in ("register-machine.py", "inventory_parser.py"): + shutil.copy(REPO / "scripts" / name, scripts / name) + def inventory(registered): + host = " newhost:\n ansible_host: localhost\n" if registered else "" + member = " newhost:\n" if registered else "" + return f"all:\n hosts:\n{host} children:\n desktop:\n hosts:\n{member}" + (tmp_path / "inventory.yml").write_text(inventory(local_registered)) + remote = tmp_path / "remote-inventory" + remote.write_text(inventory(remote_registered)) + env.update(TEST_REPO=str(tmp_path), REMOTE_INVENTORY=str(remote)) + write_stub(bindir, "git", 'case "$*" in *"show refs/remotes/origin/master:inventory.yml"*) cat "$REMOTE_INVENTORY" ;; esac\n') script = r''' source "$1/bootstrap.sh" --name newhost --type desktop DOTFILES_DIR="$TEST_REPO" -INVENTORY_MODIFIED=true -rc=0 -push_inventory_changes || rc=$? -exit "$rc" +register_in_inventory ''' result = subprocess.run(["bash", "-c", script, "bash", str(REPO)], env=env, capture_output=True, text=True) - assert result.returncode == 7 - calls = Path(env["CALLS"]).read_text().splitlines() - assert calls[-1] == failure - assert "Inventory pushed" not in result.stdout + assert result.returncode != 0 + assert "review" in result.stderr.lower() + assert (tmp_path / "inventory.yml").read_text() == inventory(local_registered) + + +def test_existing_checkout_is_not_pulled(stub_env, tmp_path): + bindir, env = stub_env + (tmp_path / ".git").mkdir() + env.update(TEST_REPO=str(tmp_path), CALLS=str(tmp_path / "calls")) + write_stub(bindir, "git", 'echo touched >> "$CALLS"\n') + script = 'source "$1/bootstrap.sh" --name test --type desktop; DOTFILES_DIR="$TEST_REPO"; clone_repo' + result = subprocess.run(["bash", "-c", script, "bash", str(REPO)], env=env, capture_output=True, text=True) + assert result.returncode == 0 + assert not Path(env["CALLS"]).exists() + + +def test_termux_bootstrap_preserves_android_and_propagates_identity(stub_env, tmp_path): + _, env = stub_env + scripts = tmp_path / "scripts" + scripts.mkdir() + runner = scripts / "run-units.sh" + runner.write_text('#!/bin/sh\nprintf "%s %s %s\\n" "$DOTFILES_MACHINE" "$DOTFILES_IS_DESKTOP" "$*" >> "$CALLS"\n') + runner.chmod(0o755) + env.update(TEST_REPO=str(tmp_path), CALLS=str(tmp_path / "calls")) + script = r''' +source "$1/bootstrap.sh" --name termux --type termux_hosts +DOTFILES_DIR="$TEST_REPO" +is_termux() { return 0; } +for f in ensure_python ensure_git clone_repo get_machine_name get_machine_type register_in_inventory setup_bitwarden; do + eval "$f() { :; }" +done +for f in write_machine_name set_hostname ensure_sudo ensure_tailscale; do + eval "$f() { echo unexpected-root-operation >> \"\$CALLS\"; return 1; }" +done +main +''' + result = subprocess.run(["bash", "-c", script, "bash", str(REPO)], env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + assert Path(env["CALLS"]).read_text().splitlines() == [ + "termux false --no-secrets", "termux false --tags omp,secrets"] + + +def test_termux_bitwarden_session_does_not_write_tmp_cache(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "bw", ''' +case "$1" in status) printf '{"status":"unlocked"}\\n' ;; unlock|sync) exit 0 ;; *) exit 1 ;; esac +''') + env["BW_SESSION"] = "private-session-sentinel" + cache = tmp_path / "bw_session" + bootstrap = tmp_path / "bootstrap.sh" + bootstrap.write_text((REPO / "bootstrap.sh").read_text().replace("/tmp/bw_session", str(cache))) + script = 'source "$1" --name termux --type termux_hosts; is_termux() { return 0; }; setup_bitwarden' + result = subprocess.run(["bash", "-c", script, "bash", str(bootstrap)], env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + assert not cache.exists() + assert env["BW_SESSION"] not in result.stdout + result.stderr @pytest.mark.parametrize("existing", ["insecure", "symlink", "directory"]) diff --git a/tests/test_brew_flatpak_check_mode.py b/tests/test_brew_flatpak_check_mode.py index 0613b426..056ba26f 100644 --- a/tests/test_brew_flatpak_check_mode.py +++ b/tests/test_brew_flatpak_check_mode.py @@ -9,7 +9,7 @@ import yaml from conftest import write_stub -from test_package_native_units import package_env, run_unit +from test_package_native_units import package_env, run_flatpak_with_dangling_store, run_unit @pytest.fixture @@ -180,6 +180,18 @@ def test_homebrew_check_reports_real_plans_and_never_runs_apply(check_env): assert all("install" not in cmd and "uninstall" not in cmd and "upgrade" not in cmd for cmd in commands) +def test_homebrew_check_skips_desktop_only_taps_for_server(check_env): + root, env, log, _ = check_env + (root / "host_vars/laptop.yml").write_text("is_desktop: false\nextra_brews: [private/tap/tool]\n") + env["BREW_TAPS"] = "example/core" + result = run_unit("homebrew", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "tap Homebrew repository coreowner/tap" in result.stdout + assert "tap Homebrew repository example/desktop" not in result.stdout + assert "tap Homebrew repository private/tap" not in result.stdout + assert not log.exists() + + def test_homebrew_check_missing_brew_and_failed_sandbox(check_env): _, env, log, sandbox = check_env brew = Path(env["BREW_PREFIX"]) / "bin/brew" @@ -229,6 +241,40 @@ def test_flatpak_check_matching_system_and_wrappers_are_unchanged(check_env): assert tree_state(Path(env["HOME"])) == before and not log.exists() +def test_flatpak_check_preserves_custom_launchers_and_symlinks(check_env, tmp_path): + _, env, log, _ = check_env + bindir = Path(env["HOME"]) / ".local/bin" + bindir.mkdir(parents=True) + custom = bindir / "google-chrome" + custom.write_text("custom launcher\n") + target = tmp_path / "target" + target.write_text("target\n") + link = bindir / "google-chrome-stable" + link.symlink_to(target) + before = tree_state(Path(env["HOME"])) + result = run_unit("flatpak", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "PRESERVED: existing Chromium launcher google-chrome" in result.stdout + assert "PRESERVED: existing Chromium launcher google-chrome-stable" in result.stdout + assert "CHANGE: install Chromium wrapper chromium" in result.stdout + assert tree_state(Path(env["HOME"])) == before and not log.exists() + assert link.is_symlink() and link.readlink() == target and target.read_text() == "target\n" + + + +def test_flatpak_check_preserves_dangling_store(check_env, tmp_path): + root, env, log, sandbox = check_env + (root / "host_vars/laptop.yml").write_text(yaml.safe_dump({ + "user_flatpaks": [{"remote": "flatpark", "id": "org.example.User"}], + })) + result, store = run_flatpak_with_dangling_store(root, env, tmp_path) + assert result.returncode != 0 and "dangling symlink" in result.stderr + assert "SKIPPED: system Flatpak inspection" in result.stdout + assert store.is_symlink() and not store.exists() and not log.exists() + commands = sandbox_calls(sandbox) + assert any("--user" in cmd for cmd in commands) + assert not any("--system" in cmd for cmd in commands) + def test_homebrew_check_matching_brewfile_and_bundle(check_env): _, env, log, _ = check_env env["DOTFILES_UPGRADE"] = "0" diff --git a/tests/test_browser_fxa_check_mode.py b/tests/test_browser_fxa_check_mode.py index 21929e7f..0c18786b 100644 --- a/tests/test_browser_fxa_check_mode.py +++ b/tests/test_browser_fxa_check_mode.py @@ -92,7 +92,8 @@ def test_check_reports_drift_without_mutation_or_secrets(stub_env, tmp_path): assert result.returncode == 0, result.stdout + result.stderr assert "WOULD install Firefox Accounts sign-in script" in result.stdout assert "WOULD restrict Firefox Accounts token permissions" in result.stdout - assert "WOULD attempt Firefox Accounts sign-in" in result.stdout + assert "data exists but is not verified; it will be left untouched" in result.stdout + assert "WOULD attempt Firefox Accounts sign-in" not in result.stdout assert "private-token" not in result.stdout + result.stderr assert "s'ec ret" not in result.stdout + result.stderr assert snapshot(home) == before @@ -162,3 +163,16 @@ def test_check_missing_pyfxa_and_unreadable_token(stub_env, tmp_path): assert result.returncode != 0 and "FAIL inspect Firefox Accounts token" in result.stdout assert snapshot(home) == before assert not Path(env["CHECK_LOG"]).exists() + + +def test_check_refuses_token_symlink(stub_env, tmp_path): + env, home, profile = prepare_check(stub_env, tmp_path) + target = home / "account-data" + target.write_text("private") + target.chmod(0o644) + (profile / "signedInUser.json").symlink_to(target) + before = snapshot(home) + result = run(env) + assert result.returncode != 0 and "token is a symlink" in result.stdout + assert snapshot(home) == before + assert not Path(env["CHECK_LOG"]).exists() diff --git a/tests/test_browser_fxa_unit.py b/tests/test_browser_fxa_unit.py index 4773f66c..65064854 100644 --- a/tests/test_browser_fxa_unit.py +++ b/tests/test_browser_fxa_unit.py @@ -1,7 +1,11 @@ import json +import os import subprocess +import sys from pathlib import Path +import pytest + from conftest import REPO, write_stub @@ -110,3 +114,95 @@ def test_missing_default_profile_is_not_created(stub_env): assert "default profile does not exist" in result.stdout assert not (profile.parent / "missing.default").exists() assert not (home / "signin-captured").exists() + + +def test_failed_helper_install_does_not_run_stale_copy(stub_env): + env, home, profile = prepare(stub_env) + helper = home / ".local/bin/fxa_signin.py" + helper.parent.mkdir(parents=True) + helper.write_text("old helper") + token = profile / "signedInUser.json" + token.write_text("existing") + write_stub(stub_env[0], "install", 'exit 1\n') + result = run(env) + assert result.returncode != 0 + assert "helper installation failed" in result.stdout + assert helper.read_text() == "old helper" + assert token.read_text() == "existing" + assert not (home / "signin-captured").exists() + assert not (home / "bw-calls").exists() + + +def test_token_symlink_is_not_followed(stub_env): + env, home, profile = prepare(stub_env) + target = home / "unrelated" + target.write_text("custom data") + target.chmod(0o644) + (profile / "signedInUser.json").symlink_to(target) + result = run(env) + assert result.returncode == 0 + assert "token is a symlink" in result.stdout + assert target.read_text() == "custom data" + assert target.stat().st_mode & 0o777 == 0o644 + assert not (home / "signin-captured").exists() + + +@pytest.mark.parametrize("data,expected", [ + ('{"accountData": {"verified": true}}', 0), + ('{"accountData": {"verified": false, "sessionToken": "keep"}}', 1), + ('not valid JSON', 1), + ('[]', 1), +]) +def test_signin_helper_preserves_existing_account_data(tmp_path, data, expected): + profile = tmp_path / "profile" + profile.mkdir() + token = profile / "signedInUser.json" + token.write_text(data) + env = dict(os.environ, PROFILE_DIR=str(profile), FXA_EMAIL="person@example.test", + FXA_PASSWORD="test-password", PYTHONPATH=str(tmp_path)) + result = subprocess.run([sys.executable, str(REPO / "units/browser-fxa/fxa_signin.py")], + env=env, text=True, capture_output=True) + assert result.returncode == expected + assert token.read_text() == data + assert "test-password" not in result.stdout + result.stderr + + +@pytest.mark.parametrize("race", [False, True]) +def test_signin_helper_publishes_private_token_without_overwriting_race(tmp_path, race): + profile = tmp_path / "profile" + profile.mkdir() + package = tmp_path / "fxa" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "core.py").write_text('''\ +import os +from pathlib import Path + +class Session: + token = b"token" + uid = "uid" + +class Client: + def __init__(self, url): + pass + + def login(self, email, password): + if os.environ.get("CREATE_RACE") == "1": + (Path(os.environ["PROFILE_DIR"]) / "signedInUser.json").write_text("browser data") + return Session() +''') + env = dict(os.environ, PROFILE_DIR=str(profile), FXA_EMAIL="person@example.test", + FXA_PASSWORD="test-password", FXA_TOTP="", PYTHONPATH=str(tmp_path), + CREATE_RACE="1" if race else "0") + result = subprocess.run([sys.executable, str(REPO / "units/browser-fxa/fxa_signin.py")], + env=env, text=True, capture_output=True) + token = profile / "signedInUser.json" + if race: + assert result.returncode != 0 + assert token.read_text() == "browser data" + else: + assert result.returncode == 0, result.stderr + assert json.loads(token.read_text())["accountData"]["sessionToken"] == b"token".hex() + assert token.stat().st_mode & 0o777 == 0o600 + assert not list(profile.glob(".signedInUser.*")) + assert "test-password" not in result.stdout + result.stderr diff --git a/tests/test_check_small_packages_vpn.py b/tests/test_check_small_packages_vpn.py index b5ca6fb1..54d17d79 100644 --- a/tests/test_check_small_packages_vpn.py +++ b/tests/test_check_small_packages_vpn.py @@ -81,7 +81,7 @@ def termux_check_stubs(bindir): echo "scheduler $*" >> "$LOG" [ "$1" = -p ] || exit 99 [ "${SCHED_FAIL:-0}" = 0 ] || exit 2 -[ "${PENDING:-0}" = 0 ] || echo 'Pending Job 1' +[ "${PENDING:-0}" = 0 ] || echo "Pending Job ${PENDING}" ''') for name in ('pkg', 'npm', 'sshd'): write_stub(bindir, name, 'echo "MUTATION $0 $*" >> "$LOG"; exit 99\n') @@ -91,7 +91,7 @@ def test_termux_check_reports_real_drift_without_mutation(platform_env): _, system, bindir, env, log = platform_env termux_check_stubs(bindir) before = snapshot(system), snapshot(Path(env['HOME'])) - result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone') + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', TEST_TERMUX='1') assert result.returncode == 0, result.stdout + result.stderr assert 'ok Termux package python' in result.stdout assert 'ok Termux package openssh' in result.stdout @@ -114,16 +114,28 @@ def test_termux_check_detects_existing_boot_and_reports_inspection_failures(plat script = directory / 'start-sshd' script.write_bytes((root / 'units/termux-start-sshd').read_bytes()) script.chmod(0o755) - result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', PENDING='1', PGREP_RC='0') + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', TEST_TERMUX='1', PENDING='1', PGREP_RC='0') assert result.returncode == 0, result.stdout + result.stderr assert 'ok Termux boot directory' in result.stdout assert 'ok Termux sshd boot script' in result.stdout assert 'ok Termux sshd running' in result.stdout assert 'job exists' in result.stdout for failure in ({'DPKG_FAIL': '1'}, {'SCHED_FAIL': '1'}, {'PGREP_RC': '2'}): - result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', **failure) + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', TEST_TERMUX='1', **failure) assert result.returncode != 0 assert 'Cannot inspect' in result.stderr + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', TEST_TERMUX='1', PENDING='10') + assert result.returncode == 0 + assert 'WOULD schedule Termux sshd keep-alive' in result.stdout + script.unlink() + script.symlink_to(directory / 'other-script') + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', TEST_TERMUX='1') + assert result.returncode != 0 and 'Cannot replace non-regular' in result.stderr + script.unlink() + script.write_text('custom script\n') + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', TEST_TERMUX='1') + assert result.returncode != 0 and 'Cannot replace customized' in result.stderr + assert script.read_text() == 'custom script\n' def tailscale_check_stubs(bindir): @@ -191,7 +203,7 @@ def test_termux_check_rejects_unreadable_overrides_and_skips_non_host(platform_e root, _, bindir, _, log = platform_env termux_check_stubs(bindir) (root / 'host_vars/phone.yml').write_text('termux_pkgs:\n - custom\n') - result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone') + result = run_unit(platform_env, 'termux-packages', UNITS_CHECK='1', DOTFILES_MACHINE='phone', TEST_TERMUX='1') assert result.returncode != 0 assert 'yq is required' in result.stderr assert not any('MUTATION' in line for line in log_lines(log)) diff --git a/tests/test_chezmoi_unit.py b/tests/test_chezmoi_unit.py index 752950e8..4d4f4dca 100644 --- a/tests/test_chezmoi_unit.py +++ b/tests/test_chezmoi_unit.py @@ -94,3 +94,52 @@ def test_inventory_membership_must_be_boolean(stub_env, tmp_path, membership): else: assert result.returncode == 0, result.stdout + result.stderr assert log.read_text() == ("1\n" if membership == "true" else "0\n") + + +@pytest.mark.parametrize("check", [False, True]) +def test_atuin_scope_targets_only_its_file_without_scripts(stub_env, tmp_path, check): + bindir, env = stub_env + log = tmp_path / "chezmoi.log" + write_stub(bindir, "chezmoi", 'printf "%s\\n" "$@" > "$LOG"\n') + env.update(DOTFILES_DIR=str(REPO), DOTFILES_MACHINE="laptop", DOTFILES_IS_DESKTOP="0", + DOTFILES_CHEZMOI_SCOPE="atuin", LOG=str(log), UNITS_CHECK="1" if check else "0") + result = subprocess.run(["bash", str(REPO / "units/chezmoi.sh")], + env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + args = log.read_text().splitlines() + assert ("status" if check else "apply") in args + assert "--exclude=scripts" in args and "--parent-dirs" in args + assert "--recursive=false" in args + assert args[-1] == str(Path(env["HOME"]) / ".config/atuin/config.toml") + + +@pytest.mark.parametrize("check", [False, True]) +def test_unknown_scope_fails_closed(stub_env, tmp_path, check): + bindir, env = stub_env + log = tmp_path / "chezmoi.log" + write_stub(bindir, "chezmoi", 'echo called > "$LOG"\n') + env.update(DOTFILES_DIR=str(REPO), DOTFILES_CHEZMOI_SCOPE="other", LOG=str(log), + UNITS_CHECK="1" if check else "0") + result = subprocess.run(["bash", str(REPO / "units/chezmoi.sh")], + env=env, capture_output=True, text=True) + assert result.returncode != 0 and "invalid chezmoi scope" in result.stdout + assert not log.exists() + + +@pytest.mark.skipif(shutil.which("chezmoi") is None, reason="chezmoi is not installed") +def test_atuin_scope_preserves_unrelated_files_on_fresh_home(stub_env): + _, env = stub_env + home = Path(env["HOME"]) + bashrc = home / ".bashrc" + bashrc.write_text("user settings\n") + env.update(DOTFILES_DIR=str(REPO), DOTFILES_MACHINE="laptop", DOTFILES_IS_DESKTOP="0", + DOTFILES_CHEZMOI_SCOPE="atuin") + result = subprocess.run(["bash", str(REPO / "units/chezmoi.sh")], + env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + assert (home / ".config/atuin/config.toml").read_bytes() == ( + REPO / "home/dot_config/atuin/config.toml" + ).read_bytes() + assert bashrc.read_text() == "user settings\n" + assert not (home / ".config/nvim").exists() + assert not (home / ".config/shell").exists() diff --git a/tests/test_fonts_unit.py b/tests/test_fonts_unit.py index cc446a42..1e5c7520 100644 --- a/tests/test_fonts_unit.py +++ b/tests/test_fonts_unit.py @@ -4,14 +4,14 @@ from unit_helpers import calls, run_unit CURL = 'echo "curl $*" >> "$HOME/calls"\nout=""; while [ $# -gt 0 ]; do [ "$1" = -o ] && out="$2"; shift; done\n: > "$out"\nexit "${STUB_CURL_RC:-0}"\n' -UNZIP = 'echo "unzip $*" >> "$HOME/calls"\nd=""; while [ $# -gt 0 ]; do [ "$1" = -d ] && d="$2"; shift; done\n: > "$d/JetBrainsMonoNerdFont-Regular.ttf"\n' +UNZIP = 'echo "unzip $*" >> "$HOME/calls"\nd=""; while [ $# -gt 0 ]; do [ "$1" = -d ] && d="$2"; shift; done\n: > "$d/JetBrainsMonoNerdFont-Regular.ttf"\nexit "${STUB_UNZIP_RC:-0}"\n' def setup(stub_env, desktop): bindir, env = stub_env write_stub(bindir, "curl", CURL) write_stub(bindir, "unzip", UNZIP) - write_stub(bindir, "fc-cache", 'echo "fc-cache $*" >> "$HOME/calls"\n') + write_stub(bindir, "fc-cache", 'echo "fc-cache $*" >> "$HOME/calls"\nexit "${STUB_FC_RC:-0}"\n') env["DOTFILES_IS_DESKTOP"] = "1" if desktop else "0" return env @@ -40,6 +40,35 @@ def test_failed_download_fails_the_unit_without_a_partial_install(stub_env): assert not (Path(env["HOME"]) / ".local/share/fonts/JetBrainsMono").exists() +def test_failed_extraction_cannot_leave_a_false_install_marker(stub_env): + env = setup(stub_env, True) + font_dir = Path(env["HOME"]) / ".local/share/fonts/JetBrainsMono" + font_dir.mkdir(parents=True) + (font_dir / "custom.ttf").write_text("custom font") + result = run_unit("fonts", env, STUB_UNZIP_RC="1") + assert result.returncode == 1 + assert not (font_dir / "JetBrainsMonoNerdFont-Regular.ttf").exists() + assert (font_dir / "custom.ttf").read_text() == "custom font" + assert run_unit("fonts", env).returncode == 0 + assert (font_dir / "JetBrainsMonoNerdFont-Regular.ttf").exists() + + +def test_failed_cache_refresh_is_reported_and_retried_without_downloading(stub_env): + env = setup(stub_env, True) + result = run_unit("fonts", env, STUB_FC_RC="1") + assert result.returncode == 1 and "FAIL font cache" in result.stdout + before = calls(env) + result = run_unit("fonts", env, UNITS_CHECK="1") + assert result.returncode == 0 and "retry JetBrains Mono font cache" in result.stdout + assert calls(env) == before + assert run_unit("fonts", env).returncode == 0 + after = calls(env) + assert after.count("curl ") == before.count("curl ") + assert after.count("fc-cache -f") == 2 + assert run_unit("fonts", env).returncode == 0 + assert calls(env) == after + + def test_failed_desktop_lookup_does_not_skip_font_configuration(stub_env): bindir, env = stub_env write_stub(bindir, "yq", "exit 2\n") diff --git a/tests/test_forgejo_registry_unit.py b/tests/test_forgejo_registry_unit.py index 47ac73b9..7db3d84b 100644 --- a/tests/test_forgejo_registry_unit.py +++ b/tests/test_forgejo_registry_unit.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from conftest import write_stub @@ -46,3 +48,35 @@ def test_missing_credential_is_partial_not_a_failure(env): def test_locked_vault_is_skipped(env): r = run_unit("forgejo-registry", env, STUB_UNLOCK_RC="1") assert r.returncode == 0 and "SKIPPED:" in r.stdout + + +@pytest.mark.parametrize("failed", ["username", "password"]) +def test_failed_credential_read_ignores_partial_stdout(env, failed): + bindir = Path(env["PATH"].split(":")[0]) + write_stub(bindir, "bw", UNLOCKED_BW % ''' +if [ "$1" = get ]; then + printf 'partial-value\\n' + [ "$2" != "$STUB_FAILED_FIELD" ] + exit $? +fi +''') + result = run_unit("forgejo-registry", env, STUB_FAILED_FIELD=failed) + assert result.returncode == 0 and "PARTIAL:" in result.stdout + assert "--password-stdin" not in calls(env) + assert "partial-value" not in result.stdout + result.stderr + + +def test_username_trims_edges_without_removing_internal_whitespace(env): + bindir = Path(env["PATH"].split(":")[0]) + write_stub(bindir, "bw", UNLOCKED_BW % ''' +if [ "$1 $2" = "get username" ]; then printf ' ci user \\n'; exit 0; fi +if [ "$1 $2" = "get password" ]; then printf 'token\\n'; exit 0; fi +''') + write_stub(bindir, "podman", ''' +if [ "$2" = --get-login ]; then exit 1; fi +printf '%s' "$4" > "$HOME/username" +cat > /dev/null +''') + result = run_unit("forgejo-registry", env) + assert result.returncode == 0, result.stdout + result.stderr + assert Path(env["HOME"], "username").read_text() == "ci user" diff --git a/tests/test_github_unit.py b/tests/test_github_unit.py index d7a0e656..fe679b1d 100644 --- a/tests/test_github_unit.py +++ b/tests/test_github_unit.py @@ -1,3 +1,5 @@ +import shutil +import subprocess from pathlib import Path import pytest @@ -124,3 +126,67 @@ def test_failed_key_list_never_registers_even_with_partial_output(env, partial_o assert "FAIL register SSH keys" in r.stdout assert "gh api --paginate" in calls(env) assert "ssh-key add" not in calls(env) + + +def test_failed_token_read_does_not_log_in_with_partial_output(env): + bindir = Path(env["PATH"].split(":")[0]) + write_stub(bindir, "bw", UNLOCKED_BW % ''' +if [ "$1 $2 $3" = "get password github-token" ]; then + echo partial-token + exit 1 +fi +''') + result = run_unit("github", env, STUB_AUTHED="1") + assert result.returncode == 0 and "no 'github-token'" in result.stdout + assert "gh auth login" not in calls(env) + assert "partial-token" not in result.stdout + result.stderr + + +def test_unrelated_matching_comment_does_not_count_as_github_key(env): + known = Path(env["HOME"], ".ssh/known_hosts") + known.write_text("other.example ssh-ed25519 AAAA github.com ssh-ed25519 " + "AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\n") + result = run_unit("github", env, UNITS_NO_SECRETS="1") + assert result.returncode == 0, result.stdout + result.stderr + assert "\ngithub.com ssh-ed25519 " in known.read_text() + + +@pytest.mark.parametrize("check", ["0", "1"]) +def test_conflicting_github_host_key_is_not_trusted_or_replaced(env, check): + known = Path(env["HOME"], ".ssh/known_hosts") + original = "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJm\n" + known.write_text(original) + result = run_unit("github", env, UNITS_NO_SECRETS="1", UNITS_CHECK=check) + assert result.returncode != 0 and "conflicting GitHub" in result.stdout + assert known.read_text() == original + + +@pytest.mark.skipif(shutil.which("ssh-keygen") is None, reason="ssh-keygen not installed") +def test_hashed_existing_github_key_is_not_duplicated(env): + known = Path(env["HOME"], ".ssh/known_hosts") + known.write_text("github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\n") + subprocess.run(["ssh-keygen", "-H", "-f", str(known)], check=True, capture_output=True) + original = known.read_text() + result = run_unit("github", env, UNITS_NO_SECRETS="1") + assert result.returncode == 0, result.stdout + result.stderr + assert known.read_text() == original + + +@pytest.mark.parametrize("check", ["0", "1"]) +def test_revoked_pinned_key_is_not_readded(env, check): + known = Path(env["HOME"], ".ssh/known_hosts") + original = "@revoked github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\n" + known.write_text(original) + result = run_unit("github", env, UNITS_NO_SECRETS="1", UNITS_CHECK=check) + assert result.returncode != 0 and "revoked" in result.stdout + assert known.read_text() == original + + +def test_symlinked_known_hosts_is_not_modified(env): + known = Path(env["HOME"], ".ssh/known_hosts") + target = Path(env["HOME"], "custom-known-hosts") + target.write_text("user settings\n") + known.symlink_to(target) + result = run_unit("github", env, UNITS_NO_SECRETS="1") + assert result.returncode != 0 and "symlinked" in result.stdout + assert known.is_symlink() and target.read_text() == "user settings\n" diff --git a/tests/test_gnome_unit.py b/tests/test_gnome_unit.py index 5d6c8bff..e1d46e1e 100644 --- a/tests/test_gnome_unit.py +++ b/tests/test_gnome_unit.py @@ -10,6 +10,7 @@ from conftest import REPO, write_stub from unit_helpers import calls, run_unit +from test_visual_units_check import assert_read_only, inspect, snapshot, visual @pytest.fixture @@ -40,7 +41,9 @@ def env(stub_env): case "$*" in 'get org.gnome.Ptyxis default-profile-uuid') echo "'test-uuid'" ;; *' custom-command') echo "'${STUB_CURRENT_COMMAND:-/old/fish}'" ;; + *' use-custom-command') echo "${STUB_CUSTOM_ENABLED:-false}" ;; 'get org.gnome.Ptyxis font-name') echo "'${STUB_CURRENT_FONT:-old font}'" ;; + 'get org.gnome.Ptyxis use-system-font') echo "${STUB_SYSTEM_FONT:-true}" ;; esac exit "${STUB_GSETTINGS_FAIL:-0}" ''') @@ -95,6 +98,34 @@ def test_desktop_uses_canonical_assets_and_live_user_session(env): assert calls(env).count("custom-command /custom/brew/bin/fish") == 1 +@pytest.mark.parametrize("custom,system,expected", [ + ("false", "false", "use-custom-command true"), + ("true", "true", "use-system-font false"), +]) +def test_ptyxis_repairs_disabled_settings_even_when_values_match(env, custom, system, expected): + result = run_unit("gnome", env, STUB_CURRENT_COMMAND="/custom/brew/bin/fish", + STUB_CURRENT_FONT="JetBrainsMono Nerd Font 12", + STUB_CUSTOM_ENABLED=custom, STUB_SYSTEM_FONT=system) + assert result.returncode == 0, result.stdout + result.stderr + log = calls(env) + assert expected in log + assert "set org.gnome.Ptyxis font-name" not in log + assert "custom-command /custom/brew/bin/fish" not in log + + +@pytest.mark.parametrize("extra", [ + {"CHECK_CUSTOM_ENABLED": "false"}, + {"CHECK_SYSTEM_FONT": "true"}, +]) +def test_ptyxis_check_detects_disabled_settings_without_mutation(visual, extra): + env, _ = visual + before = snapshot(Path(env["HOME"])) + result = inspect("gnome", env, **extra) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD configure Ptyxis profile" in result.stdout + assert_read_only(env, before) + + def test_failed_keybinding_load_is_reported_and_retried(env): result = run_unit("gnome", env, STUB_KEY_FAIL="1") assert result.returncode == 1 @@ -255,3 +286,66 @@ def no_gdm(name): helper.install_avatar(source) assert (tmp_path / "users/james").read_text() == "[User]\nIcon=/var/lib/AccountsService/icons/james\n" assert not (tmp_path / "gdm").exists() + + +@pytest.mark.parametrize("existing", [False, True]) +def test_avatar_only_chowns_new_gdm_cache_directory(tmp_path, monkeypatch, existing): + helper = load_avatar_helper() + icons, users = tmp_path / "icons", tmp_path / "users" + icons.mkdir() + users.mkdir() + cache = tmp_path / "gdm/.cache" + if existing: + cache.mkdir(parents=True) + (cache / "custom").write_text("keep") + source = tmp_path / "source.png" + source.write_bytes(b"avatar") + monkeypatch.setattr(helper, "ICON", icons / "james") + monkeypatch.setattr(helper, "ACCOUNT", users / "james") + monkeypatch.setattr(helper, "GDM_CACHE", cache) + monkeypatch.setattr(helper.pwd, "getpwnam", lambda name: SimpleNamespace(pw_uid=os.getuid())) + monkeypatch.setattr(helper.grp, "getgrnam", lambda name: SimpleNamespace(gr_gid=os.getgid())) + chown = os.chown + chowned = [] + def record_chown(path, uid, gid): + chowned.append(Path(path)) + chown(path, uid, gid) + monkeypatch.setattr(helper.os, "chown", record_chown) + helper.install_avatar(source) + assert (cache in chowned) is not existing + if existing: + assert (cache / "custom").read_text() == "keep" + + +@pytest.mark.parametrize("kind,mode,uid,repair", [ + (stat.S_IFDIR, 0o755, 0, True), + (stat.S_IFDIR, 0o700, 0, False), + (stat.S_IFDIR, 0o755, 99, False), + (stat.S_IFLNK, 0o755, 0, False), +]) +def test_avatar_repairs_only_legacy_cache_metadata(tmp_path, monkeypatch, kind, mode, uid, repair): + helper = load_avatar_helper() + icons, users = tmp_path / "icons", tmp_path / "users" + icons.mkdir() + users.mkdir() + cache = tmp_path / "gdm/.cache" + cache.mkdir(parents=True) + (cache / "custom").write_text("keep") + source = tmp_path / "source.png" + source.write_bytes(b"avatar") + monkeypatch.setattr(helper, "ICON", icons / "james") + monkeypatch.setattr(helper, "ACCOUNT", users / "james") + monkeypatch.setattr(helper, "GDM_CACHE", cache) + monkeypatch.setattr(helper.pwd, "getpwnam", lambda name: SimpleNamespace(pw_uid=42)) + monkeypatch.setattr(helper.grp, "getgrnam", lambda name: SimpleNamespace(gr_gid=43)) + original_lstat = Path.lstat + def cache_lstat(path): + if path == cache: + return SimpleNamespace(st_mode=kind | mode, st_uid=uid) + return original_lstat(path) + monkeypatch.setattr(Path, "lstat", cache_lstat) + chowned = [] + monkeypatch.setattr(helper.os, "chown", lambda path, user, group: chowned.append((Path(path), user, group))) + helper.install_avatar(source) + assert ((cache, 42, 43) in chowned) is repair + assert (cache / "custom").read_text() == "keep" diff --git a/tests/test_inventory_scripts.py b/tests/test_inventory_scripts.py index ec9f9b8b..7057cf13 100644 --- a/tests/test_inventory_scripts.py +++ b/tests/test_inventory_scripts.py @@ -12,7 +12,7 @@ scripts_dir = Path(__file__).parent.parent / "scripts" sys.path.insert(0, str(scripts_dir)) -from inventory_parser import parse_inventory, register_host, purge_host, UnknownGroupError +from inventory_parser import parse_inventory, register_host, purge_host, UnknownGroupError, host_groups class TestParseInventory(unittest.TestCase): @@ -136,6 +136,24 @@ def test_register_host_in_group(self) -> None: # Verify host2 is under laptop self.assertIn(" laptop:\n hosts:\n host2:", content) + def test_group_name_is_not_mistaken_for_existing_host(self) -> None: + self.assertTrue(register_host("vps", "desktop", str(self.inv_path))) + data = yaml.safe_load(self.inv_path.read_text()) + self.assertIn("vps", data["all"]["hosts"]) + self.assertIn("vps", data["all"]["children"]["desktop"]["hosts"]) + + def test_invalid_name_cannot_mutate_inventory(self) -> None: + before = self.inv_path.read_text() + with self.assertRaises(ValueError): + register_host("../escape", "desktop", str(self.inv_path)) + self.assertEqual(self.inv_path.read_text(), before) + + def test_group_detection_is_not_limited_to_first_300_characters(self) -> None: + hosts = "".join(f" item{i}: {{}}\n" for i in range(50)) + text = f"all:\n hosts:\n target: {{}}\n children:\n server:\n hosts:\n{hosts} target: {{}} # entry\n" + self.assertEqual(host_groups(text, "target"), {"server"}) + self.assertEqual(parse_inventory(text)[0], {"target"}) + class TestPurgeHost(unittest.TestCase): """Test purge_host function.""" diff --git a/tests/test_kube_unit.py b/tests/test_kube_unit.py index 49be9f5f..b17f6519 100644 --- a/tests/test_kube_unit.py +++ b/tests/test_kube_unit.py @@ -103,3 +103,47 @@ def test_profile_skips_without_desktop_lookup(env, tmp_path, profile): DOTFILES_IS_DESKTOP="") assert r.returncode == 0 and "SKIPPED" in r.stdout assert calls(env) == "" + + +def test_failed_note_read_preserves_existing_config_even_with_stdout(env): + cfg = Path(env["HOME"], ".kube/config") + cfg.parent.mkdir() + cfg.write_text("previous\n") + bindir = Path(env["PATH"].split(":")[0]) + write_stub(bindir, "bw", UNLOCKED_BW % ''' +if [ "$1 $2 $3" = "get notes kubeconfig" ]; then + echo partial-private-config + exit 1 +fi +''') + result = run_unit("kube", env) + assert result.returncode == 0 + assert cfg.read_text() == "previous\n" + assert "set-cluster" not in calls(env) + assert "partial-private-config" not in result.stdout + result.stderr + + +def test_failed_cluster_listing_reports_failure_without_pinning_partial_output(env): + bindir = Path(env["PATH"].split(":")[0]) + write_stub(bindir, "kubectl", ''' +echo "kubectl $*" >> "$HOME/calls" +if [ "$2" = get-clusters ]; then printf 'NAME\\nhome\\n'; exit 1; fi +exit 0 +''') + result = run_unit("kube", env) + assert result.returncode != 0 and "cannot list home kubeconfig clusters" in result.stdout + assert "set-cluster" not in calls(env) + assert Path(env["HOME"], ".kube/config-aws-migration").read_text().strip() == "aws-kube" + + +def test_missing_kubectl_reports_unpinned_config(env, tmp_path): + bash_env = tmp_path / "bash-env" + bash_env.write_text(''' +command() { + if [ "$1" = -v ] && [ "$2" = kubectl ]; then return 1; fi + builtin command "$@" +} +''') + result = run_unit("kube", env, BASH_ENV=str(bash_env)) + assert result.returncode != 0 and "kubectl missing; tls-server-name not pinned" in result.stdout + assert Path(env["HOME"], ".kube/config-aws-migration").read_text().strip() == "aws-kube" diff --git a/tests/test_laptop_services_migration.py b/tests/test_laptop_services_migration.py index e35f00e1..f443c25e 100644 --- a/tests/test_laptop_services_migration.py +++ b/tests/test_laptop_services_migration.py @@ -27,6 +27,8 @@ def test_laptop_login_service_and_idempotent_reload(stub_env, tmp_path): assert "ExecStartPre=/bin/sleep 90" in update assert "WantedBy=graphical-session.target" in update assert "apply-nosecrets" in update + assert "ExecStart=/usr/bin/env just " in update + assert "ExecStart=/usr/bin/env atuin daemon start --force" in (home / ".config/systemd/user/atuin-daemon.service").read_text() assert "enable dotfiles-update.service" in calls(env) assert "enable --now dotfiles-update.service" not in calls(env) assert "enable --now atuin-daemon.service" in calls(env) @@ -71,6 +73,20 @@ def test_failed_disable_preserves_timer_and_reports_failure(stub_env, tmp_path): assert "enable --now system-mitigator.service" in calls(env) +def test_failed_old_timer_disable_does_not_enable_login_update(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n[[ "$*" != *"disable --now dotfiles-update.timer"* ]]\n') + repo = laptop_repo(tmp_path, "kanpur", "is_laptop: true\n") + units = Path(env["HOME"]) / ".config/systemd/user" + units.mkdir(parents=True) + (units / "dotfiles-update.timer").write_text("old") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="kanpur") + assert result.returncode != 0 + assert (units / "dotfiles-update.timer").read_text() == "old" + assert "enable dotfiles-update.service" not in calls(env) + assert "enable --now atuin-daemon.service" in calls(env) + + def test_himachal_phone_schedule_is_preserved_when_enabled(stub_env, tmp_path): bindir, env = stub_env write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') @@ -124,6 +140,56 @@ def test_phone_push_uses_remote_machine_identity_and_cleans_up(stub_env, tmp_pat assert (tmp_path / "record").read_text().splitlines() == ["mumbai --no-secrets"] +def test_missing_phone_timer_source_never_starts_stale_timer(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = laptop_repo(tmp_path, "himachal", "is_laptop: true\nphone_apply_enabled: true\n") + (repo / "units").unlink() + sources = repo / "units/laptop-services" + sources.mkdir(parents=True) + for name in ("system-mitigator.py", "system-mitigator.service"): + (sources / name).symlink_to(REPO / "units/laptop-services" / name) + units = Path(env["HOME"]) / ".config/systemd/user" + units.mkdir(parents=True) + (units / "phone-apply.timer").write_text("old") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="himachal") + assert result.returncode != 0 + assert (units / "phone-apply.timer").read_text() == "old" + assert "enable --now phone-apply.timer" not in calls(env) + assert "enable --now system-mitigator.service" in calls(env) + + +def test_missing_mitigator_script_never_starts_stale_service(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = laptop_repo(tmp_path, "himachal", "is_laptop: true\nphone_apply_enabled: false\n") + (repo / "units").unlink() + sources = repo / "units/laptop-services" + sources.mkdir(parents=True) + (sources / "system-mitigator.service").symlink_to(REPO / "units/laptop-services/system-mitigator.service") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="himachal") + assert result.returncode != 0 + assert "enable --now system-mitigator.service" not in calls(env) + assert "enable --now atuin-daemon.service" in calls(env) + + +def test_laptop_removes_only_managed_old_default_target_link(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = laptop_repo(tmp_path, "kanpur", "is_laptop: true\n") + units = Path(env["HOME"]) / ".config/systemd/user" + link = units / "default.target.wants/dotfiles-update.service" + link.parent.mkdir(parents=True) + link.symlink_to("../dotfiles-update.service") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="kanpur") + assert result.returncode == 0, result.stdout + result.stderr + assert not link.is_symlink() + link.symlink_to("/opt/custom/dotfiles-update.service") + result = run_unit("laptop-services", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="kanpur") + assert result.returncode == 0, result.stdout + result.stderr + assert link.is_symlink() and link.readlink() == Path("/opt/custom/dotfiles-update.service") + + def test_retired_vps_is_never_touched(stub_env, tmp_path): bindir, env = stub_env write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') diff --git a/tests/test_laptop_system_services_check.py b/tests/test_laptop_system_services_check.py index c69d785e..5af9d282 100644 --- a/tests/test_laptop_system_services_check.py +++ b/tests/test_laptop_system_services_check.py @@ -301,15 +301,46 @@ def test_unreadable_host_variables_fail_both_checks(stub_env, tmp_path): hostvars.unlink() hostvars.symlink_to(tmp_path / "missing-host-vars") before = snapshot(Path(env["HOME"])) - laptop = run_unit("laptop-services", env, DOTFILES_DIR=env["DOTFILES_DIR"], DOTFILES_MACHINE="laptop") - system = run_script(program, env) - assert laptop.returncode != 0 and system.returncode != 0 - assert "FAIL inspect host variables" in laptop.stdout - assert "FAIL inspect host variables" in system.stdout - assert not Path(env["CONTROL_LOG"]).exists() and not dropin.exists() + for mode in ("1", "0"): + mode_env = {**env, "UNITS_CHECK": mode, "CHECK_GUARD": "1" if mode == "1" else "0"} + laptop = run_unit("laptop-services", mode_env, DOTFILES_DIR=env["DOTFILES_DIR"], DOTFILES_MACHINE="laptop") + system = run_script(program, mode_env) + assert laptop.returncode != 0 and system.returncode != 0 + assert "FAIL inspect host variables" in laptop.stdout + assert "FAIL inspect host variables" in system.stdout + assert not Path(env["CONTROL_LOG"]).exists() and not dropin.exists() + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) + + + +def test_transition_check_reports_old_link_without_removing_it(stub_env, tmp_path): + bindir, original_env = stub_env + systemctl_stub(bindir) + env = check_env(tmp_path, original_env) + program, env, _ = setup_system_services(tmp_path, env, "laptop", "is_laptop: true\n", False) + units = Path(env["HOME"]) / ".config/systemd/user" + link = units / "default.target.wants/dotfiles-update.service" + link.parent.mkdir(parents=True) + link.symlink_to("../dotfiles-update.service") + before = snapshot(Path(env["HOME"])) + result = run_unit("laptop-services", env, DOTFILES_DIR=env["DOTFILES_DIR"], DOTFILES_MACHINE="laptop") + assert result.returncode == 0, result.stdout + result.stderr + assert "would remove" in result.stdout and "default.target.wants" in result.stdout assert snapshot(Path(env["HOME"])) == before assert_read_only_calls(env) + link.unlink() + link = units / "graphical-session.target.wants/dotfiles-update.service" + link.parent.mkdir(parents=True) + link.symlink_to("../dotfiles-update.service") + (Path(env["DOTFILES_DIR"]) / "host_vars/laptop.yml").write_text("is_laptop: false\n") + before = snapshot(Path(env["HOME"])) + result = run_script(program, env) + assert result.returncode == 0, result.stdout + result.stderr + assert "would remove" in result.stdout and "graphical-session.target.wants" in result.stdout + assert snapshot(Path(env["HOME"])) == before + assert_read_only_calls(env) def test_service_drift_does_not_disclose_file_contents(stub_env, tmp_path): bindir, original_env = stub_env diff --git a/tests/test_onboarding_pr.py b/tests/test_onboarding_pr.py new file mode 100644 index 00000000..b56be909 --- /dev/null +++ b/tests/test_onboarding_pr.py @@ -0,0 +1,89 @@ +import os +import shutil +import subprocess + +from conftest import REPO, write_stub + + +def git(*args, cwd=None): + return subprocess.run(['git', *map(str, args)], cwd=cwd, check=True, + capture_output=True, text=True).stdout.strip() + + +def test_registration_pr_does_not_change_checkout_or_push_master(tmp_path, stub_env): + bindir, env = stub_env + repo, remote = tmp_path / 'repo', tmp_path / 'remote.git' + git('init', '--initial-branch=master', repo) + git('config', 'user.name', 'Test', cwd=repo) + git('config', 'user.email', 'test@example.invalid', cwd=repo) + (repo / 'scripts').mkdir() + for name in ('onboard-machine.sh', 'register-machine.py', 'inventory_parser.py'): + shutil.copy2(REPO / 'scripts' / name, repo / 'scripts' / name) + (repo / 'inventory.yml').write_text( + 'all:\n hosts:\n existing:\n termux:\n ansible_port: 8022\n tailnet_name: pixel-9a\n' + ' children:\n desktop:\n hosts:\n existing:\n' + ' server:\n hosts:\n vps:\n hosts:\n termux_hosts:\n hosts:\n termux:\n') + (repo / 'notes').write_text('committed\n') + git('add', '.', cwd=repo) + git('commit', '-m', 'base', cwd=repo) + git('clone', '--bare', repo, remote) + git('remote', 'add', 'origin', remote, cwd=repo) + (repo / 'notes').write_text('staged user work\n') + git('add', 'notes', cwd=repo) + with (repo / 'inventory.yml').open('a') as stream: + stream.write('# uncommitted user work\n') + before = git('status', '--porcelain', cwd=repo) + inventory_before = (repo / 'inventory.yml').read_bytes() + env['GH_STATE'] = str(tmp_path / 'pr-open') + write_stub(bindir, 'gh', ''' +case "$*" in + 'pr list '*) [ ! -f "$GH_STATE" ] || echo 'https://example.invalid/pr/1' ;; + 'pr create '*) touch "$GH_STATE"; echo 'https://example.invalid/pr/1' ;; + *) exit 2 ;; +esac +''') + termux = subprocess.run(['bash', str(repo / 'scripts/onboard-machine.sh'), 'termux', 'termux_hosts'], + env=env, capture_output=True, text=True) + assert termux.returncode == 0, termux.stdout + termux.stderr + new_mobile = subprocess.run(['bash', str(repo / 'scripts/onboard-machine.sh'), 'new-phone', 'termux_hosts'], + env=env, capture_output=True, text=True) + assert new_mobile.returncode != 0 + assert 'individually reviewed inventory PR' in new_mobile.stderr + assert git('status', '--porcelain', cwd=repo) == before + assert not os.path.exists(env['GH_STATE']) + command = ['bash', str(repo / 'scripts/onboard-machine.sh'), 'newhost', 'desktop'] + first = subprocess.run(command, env=env, capture_output=True, text=True) + assert first.returncode == 3, first.stdout + first.stderr + assert git('status', '--porcelain', cwd=repo) == before + assert (repo / 'inventory.yml').read_bytes() == inventory_before + assert 'newhost:' not in git('--git-dir', remote, 'show', 'master:inventory.yml') + branch_inventory = git('--git-dir', remote, 'show', 'onboarding/newhost:inventory.yml') + assert ' newhost:\n' in branch_inventory + assert ' desktop:\n hosts:\n newhost:' in branch_inventory + assert git('--git-dir', remote, 'show', 'onboarding/newhost:host_vars/newhost.yml') == '---' + assert git('--git-dir', remote, 'diff', '--name-only', 'master', 'onboarding/newhost') == 'host_vars/newhost.yml\ninventory.yml' + + second = subprocess.run(command, env=env, capture_output=True, text=True) + assert second.returncode == 3 + assert 'awaits review and merge' in second.stdout + assert git('status', '--porcelain', cwd=repo) == before + + reviewed = git('--git-dir', remote, 'rev-parse', 'onboarding/newhost') + git('--git-dir', remote, 'update-ref', 'refs/heads/master', reviewed) + after_merge = subprocess.run(command, env=env, capture_output=True, text=True) + assert after_merge.returncode == 0, after_merge.stdout + after_merge.stderr + assert git('status', '--porcelain', cwd=repo) == before + wrong_type = subprocess.run(command[:-1] + ['server'], env=env, capture_output=True, text=True) + assert wrong_type.returncode != 0 + assert 'not in the server group' in wrong_type.stderr + assert git('status', '--porcelain', cwd=repo) == before + + +def test_retired_host_is_rejected_before_fetch(stub_env, tmp_path): + bindir, env = stub_env + env['CALLS'] = str(tmp_path / 'calls') + write_stub(bindir, 'git', 'echo called >> "$CALLS"\nexit 1\n') + result = subprocess.run(['bash', str(REPO / 'scripts/onboard-machine.sh'), 'matrix', 'vps'], + env=env, capture_output=True, text=True) + assert result.returncode != 0 + assert not os.path.exists(env['CALLS']) diff --git a/tests/test_osquery_native_inventory.py b/tests/test_osquery_native_inventory.py index e1897df5..8de1b2af 100644 --- a/tests/test_osquery_native_inventory.py +++ b/tests/test_osquery_native_inventory.py @@ -2,6 +2,7 @@ import json import sys import shutil +import subprocess import pytest @@ -125,3 +126,49 @@ def test_osquery_gates_inherit_group_settings_and_host_overrides(tmp_path, capsy assert native.main(["osquery-settings", str(path), "node"]) == 0 assert json.loads(capsys.readouterr().out) == {"skip_osquery": False, "osquery_supported": False, "osquery_poller_enabled": True, "osquery_version": "5.23.1"} + + +def test_poller_retains_apply_path_for_nonstandard_tool_installations(tmp_path, stub_env): + _, env = stub_env + custom_bin = tmp_path / "custom-bin" + custom_bin.mkdir() + tailscale = custom_bin / "tailscale" + tailscale.write_text("#!/bin/sh\nprintf 'custom tailscale\\n'\n") + tailscale.chmod(0o755) + env.update(DOTFILES_MACHINE="workstation", BREW_PREFIX=str(tmp_path / "brew"), + PATH=f"{custom_bin}:/usr/bin:/bin") + result = subprocess.run( + ["bash", "-c", 'source "$1"; osquery_default_poll_path', "bash", + str(REPO / "units/osquery.sh")], env=env, capture_output=True, text=True, check=True) + pollpath = result.stdout + assert pollpath.startswith(f"{env['HOME']}/.local/bin:{env['BREW_PREFIX']}/bin:") + tool = subprocess.run(["tailscale"], env={**env, "PATH": pollpath}, + capture_output=True, text=True, check=True) + assert tool.stdout == "custom tailscale\n" + + renderer_spec = importlib.util.spec_from_file_location( + "osquery_poller_renderer_path", REPO / "units/osquery-poller-render.py") + renderer = importlib.util.module_from_spec(renderer_spec) + renderer_spec.loader.exec_module(renderer) + service = renderer.service(REPO / "units/osquery/templates/fleet-osquery-poll.service.j2", + "user", "1000", env["HOME"], pollpath, str(tmp_path / "state")) + assert f'Environment="PATH={pollpath}"' in service + + +def test_yaml_fallback_without_yq_preserves_group_precedence_and_sanitizes_errors(tmp_path, monkeypatch): + pytest.importorskip("yaml") + path = tmp_path / "inventory.yml" + path.write_text("all:\n children:\n server:\n vars:\n ansible_user: group-user\n hosts:\n host: {}\n") + (tmp_path / "host_vars").mkdir() + (tmp_path / "host_vars/host.yml").write_text("ansible_user: host-user\n") + + def missing_yq(*args, **kwargs): + raise FileNotFoundError("yq") + + monkeypatch.setattr(native.subprocess, "run", missing_yq) + result = native.load_inventory(path) + assert result["_meta"]["hostvars"]["host"]["ansible_user"] == "host-user" + path.write_text("secret-value: [unclosed\n") + with pytest.raises(ValueError) as error: + native.load_inventory(path) + assert "secret-value" not in str(error.value) diff --git a/tests/test_package_native_units.py b/tests/test_package_native_units.py index e8f0a6ad..8d48e26a 100644 --- a/tests/test_package_native_units.py +++ b/tests/test_package_native_units.py @@ -110,6 +110,16 @@ def run_unit(name, env): env=env, text=True, capture_output=True) +def run_flatpak_with_dangling_store(root, env, tmp_path): + store = tmp_path / "flatpak-store" + store.symlink_to(tmp_path / "detached-store") + (root / "units/lib.sh").symlink_to(REPO / "units/lib.sh") + script = root / "units/flatpak-test.sh" + script.write_text((REPO / "units/flatpak.sh").read_text().replace("/var/lib/flatpak", str(store))) + result = subprocess.run(["bash", str(script)], env=env, text=True, capture_output=True) + return result, store + + def calls(log, name): if not log.exists(): return [] @@ -204,13 +214,17 @@ def test_brew_desktop_lists_taps_blocked_and_no_upgrade(package_env): def test_brew_host_profile_and_explicit_upgrade(package_env): root, env, log = package_env - (root / "host_vars/laptop.yml").write_text("is_desktop: false\nextra_brews: [private-extra]\n") + (root / "host_vars/laptop.yml").write_text("is_desktop: false\nextra_brews: [private/tap/private-extra]\n") env["DOTFILES_UPGRADE"] = "1" result = run_unit("homebrew", env) assert result.returncode == 0, result.stdout + result.stderr content = (Path(env["HOME"]) / "Brewfile").read_text() assert "desktop-pkg" not in content and "private-extra" not in content - assert ["upgrade"] in [entry["args"] for entry in calls(log, "brew")] + args = [entry["args"] for entry in calls(log, "brew")] + assert ["upgrade"] in args + assert ["tap", "example/core"] in args and ["tap", "coreowner/tap"] in args + assert ["tap", "example/desktop"] not in args and ["trust", "example/desktop"] not in args + assert ["tap", "private/tap"] not in args def test_brew_blocked_host_package_fails_before_any_brew_action(package_env): @@ -287,6 +301,46 @@ def test_flatpak_system_scope_batch_fallback_and_wrappers(package_env): assert stat.S_IMODE(wrapper.stat().st_mode) == 0o755 +def test_flatpak_preserves_existing_custom_launchers_and_symlinks(package_env, tmp_path): + _, env, _ = package_env + bindir = Path(env["HOME"]) / ".local/bin" + bindir.mkdir(parents=True) + custom = bindir / "google-chrome" + custom.write_text("user launcher\n") + custom.chmod(0o700) + target = tmp_path / "user-target" + target.write_text("user target\n") + target.chmod(0o600) + link = bindir / "google-chrome-stable" + link.symlink_to(target) + managed = bindir / "chromium-browser" + managed.write_text('#!/usr/bin/env bash\nexec flatpak run org.chromium.Chromium "$@"\n') + managed.chmod(0o600) + result = run_unit("flatpak", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "PRESERVED: existing Chromium launcher google-chrome" in result.stdout + assert custom.read_text() == "user launcher\n" and stat.S_IMODE(custom.stat().st_mode) == 0o700 + assert link.is_symlink() and link.readlink() == target + assert target.read_text() == "user target\n" and stat.S_IMODE(target.stat().st_mode) == 0o600 + assert stat.S_IMODE(managed.stat().st_mode) == 0o755 + assert (bindir / "chromium").read_text() == managed.read_text() + + +def test_flatpak_preserves_dangling_store_and_continues_user_operations(package_env, tmp_path): + root, env, log = package_env + (root / "host_vars/laptop.yml").write_text(yaml.safe_dump({ + "user_flatpaks": [{"remote": "flatpark", "id": "org.example.User"}], + })) + env["DOTFILES_UPGRADE"] = "1" + result, store = run_flatpak_with_dangling_store(root, env, tmp_path) + assert result.returncode != 0 and "dangling symlink" in result.stderr + assert store.is_symlink() and not store.exists() + args = [entry["args"] for entry in calls(log, "flatpak")] + assert not any("--system" in call for call in args) + assert ["install", "--user", "-y", "flatpark", "org.example.User"] in args + assert (Path(env["HOME"]) / ".local/bin/chromium").exists() + + def test_user_flatpaks_preserve_origins_and_do_not_use_sudo(package_env): root, env, log = package_env (root / "host_vars/laptop.yml").write_text(yaml.safe_dump({ diff --git a/tests/test_pipewire_audio_check_mode.py b/tests/test_pipewire_audio_check_mode.py index 3764eda1..06c6b4fd 100644 --- a/tests/test_pipewire_audio_check_mode.py +++ b/tests/test_pipewire_audio_check_mode.py @@ -126,6 +126,49 @@ def test_check_absent_plugin_plans_removal_and_disable(stub_env, tmp_path): assert snapshot(Path(env["HOME"])) == before +def test_check_plans_original_microphone_service_migration(stub_env, tmp_path): + env = prepare_check(stub_env, tmp_path) + managed_files(env) + service = home(env, ".config/systemd/user/mic-dsp-link.service") + original = service.read_text().replace("cut -f 2", 'awk "{print $$2}"').replace("%%", "%") + service.write_text(original) + before = snapshot(Path(env["HOME"])) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD install microphone link service" in result.stdout + assert "WOULD restart microphone link service" in result.stdout + assert snapshot(Path(env["HOME"])) == before + assert "unexpected" not in Path(env["CHECK_LOG"]).read_text() + + +@pytest.mark.parametrize("daemon", ["pipewire", "wireplumber"]) +def test_check_daemon_restart_plans_managed_microphone_relink(stub_env, tmp_path, daemon): + env = prepare_check(stub_env, tmp_path) + managed_files(env) + home(env, f".local/state/pipewire-audio/{daemon}").touch() + before = snapshot(Path(env["HOME"])) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "WOULD restart microphone link service" in result.stdout + assert snapshot(Path(env["HOME"])) == before + assert "unexpected" not in Path(env["CHECK_LOG"]).read_text() + + +def test_check_bootc_does_not_plan_package_installation(stub_env, tmp_path): + env = prepare_check(stub_env, tmp_path) + bindir = Path(env["PATH"]) + write_stub(bindir, "bootc", 'exit 0\n') + write_stub(bindir, "dnf", 'echo "unexpected dnf $*" >> "$CHECK_LOG"; exit 1\n') + write_stub(bindir, "rpm", 'echo "unexpected rpm $*" >> "$CHECK_LOG"; exit 1\n') + before = snapshot(Path(env["HOME"])) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "bootc DSP prerequisites must be provided by the image" in result.stdout + assert "WOULD install DSP package" not in result.stdout + assert snapshot(Path(env["HOME"])) == before + assert "unexpected" not in Path(env["CHECK_LOG"]).read_text() + + @pytest.mark.parametrize("error", ["FIND_FAIL", "LV2_FAIL", "SESSION_FAIL", "SERVICE_FAIL", "GSETTINGS_FAIL", "FLATPAK_FAIL"]) def test_check_inspection_errors_fail_without_mutation(stub_env, tmp_path, error): env = prepare_check(stub_env, tmp_path) diff --git a/tests/test_pipewire_audio_unit.py b/tests/test_pipewire_audio_unit.py index b68f6fea..257c3eda 100644 --- a/tests/test_pipewire_audio_unit.py +++ b/tests/test_pipewire_audio_unit.py @@ -1,5 +1,7 @@ from pathlib import Path import shutil +import shlex +import subprocess from conftest import REPO, write_stub from unit_helpers import calls, run_unit @@ -78,7 +80,7 @@ def test_full_chain_host_volume_services_and_idempotence(stub_env): assert home(env, ".config/pipewire/pipewire.conf.d/50-speaker-dsp.conf").read_bytes() == asset("speaker-dsp.conf") assert home(env, ".config/pipewire/pipewire.conf.d/60-mic-dsp.conf").read_bytes() == asset("mic-rnnoise.conf") service = home(env, ".config/systemd/user/mic-dsp-link.service").read_text() - assert "set-source-volume $$HW_MIC 75%" in service + assert "set-source-volume $$HW_MIC 75%%" in service assert "{{" not in service assert "systemctl --user enable mic-dsp-link.service" in calls(env) assert "systemctl --user start mic-dsp-link.service" in calls(env) @@ -94,6 +96,50 @@ def test_full_chain_host_volume_services_and_idempotence(stub_env): assert " enable " not in after +def test_original_ansible_microphone_service_is_migrated(stub_env): + env = setup(stub_env) + service = home(env, ".config/systemd/user/mic-dsp-link.service") + service.parent.mkdir(parents=True) + template = (REPO / "units/pipewire-audio/templates/mic-dsp-link.service.j2").read_text() + original = template.replace("cut -f 2", 'awk "{print $$2}"').replace("%%", "%") + service.write_text(original.replace("{{ mic_hardware_volume_pct | default(100) }}", "75")) + home(env, ".mic-active").touch() + home(env, ".mic-enabled").touch() + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert service.read_text() == template.replace("{{ mic_hardware_volume_pct | default(100) }}", "75") + assert "systemctl --user restart mic-dsp-link.service" in calls(env) + + +def test_rendered_microphone_command_routes_named_source(stub_env): + env = setup(stub_env) + bindir = Path(env["PATH"]) + for name in ("grep", "cut"): + (bindir / name).symlink_to(shutil.which(name)) + write_stub(bindir, "pactl", '''\ +if [ "$*" = "list short sources" ]; then + printf '42\\talsa_input.pci-test.analog-stereo\\tPipeWire\\ts16le 2ch 48000Hz\\tRUNNING\\n' +else + echo "pactl $*" >> "$HOME/mic-calls" +fi +''') + write_stub(bindir, "pw-link", 'echo "pw-link $*" >> "$HOME/mic-calls"\n') + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + service = home(env, ".config/systemd/user/mic-dsp-link.service").read_text() + assert "$$HW_MIC 75%%" in service + command = service.split("ExecStart=", 1)[1].split("\nRemainAfterExit=", 1)[0].replace("\\\n", "") + # Model systemd's literal specifier and dollar escaping before sh executes. + args = shlex.split(command.replace("%%", "%").replace("$$", "$")) + result = subprocess.run(args, env=env, text=True, capture_output=True, check=False) + assert result.returncode == 0, result.stderr + logged = home(env, "mic-calls").read_text().splitlines() + assert "pactl set-source-volume alsa_input.pci-test.analog-stereo 75%" in logged + assert "pw-link alsa_input.pci-test.analog-stereo:capture_FL effect_input.rnnoise:input_FL" in logged + assert "pw-link alsa_input.pci-test.analog-stereo:capture_FR effect_input.rnnoise:input_FR" in logged + assert "pactl set-default-source effect_output.rnnoise" in logged + + def test_builtin_eq_absent_mic_and_user_owned_data_survive(stub_env): env = setup(stub_env) config = home(env, ".config/pipewire/pipewire.conf.d/60-mic-dsp.conf") @@ -136,6 +182,24 @@ def test_mic_search_failure_preserves_config_and_service(stub_env): assert "mic-dsp-link.service" not in calls(env) +def test_search_failure_defers_relink_after_daemon_restart(stub_env): + env = setup(stub_env) + assert run_unit("pipewire-audio", env).returncode == 0 + state = home(env, ".local/state/pipewire-audio") + (state / "pipewire").touch() + before = calls(env) + result = run_unit("pipewire-audio", env, FIND_FAIL="1") + assert result.returncode == 1 + assert "systemctl --user restart pipewire" in calls(env)[len(before):] + assert "mic-dsp-link.service" not in calls(env)[len(before):] + assert (state / "mic").exists() + before = calls(env) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "systemctl --user restart mic-dsp-link.service" in calls(env)[len(before):] + assert not (state / "mic").exists() + + def test_session_failure_retries_pending_restarts(stub_env): env = setup(stub_env) first = run_unit("pipewire-audio", env, SESSION_FAIL="1") @@ -149,6 +213,44 @@ def test_session_failure_retries_pending_restarts(stub_env): assert not home(env, ".local/state/pipewire-audio/mic").exists() +def test_audio_daemon_restarts_relink_managed_microphone_and_retry(stub_env): + env = setup(stub_env) + assert run_unit("pipewire-audio", env).returncode == 0 + state = home(env, ".local/state/pipewire-audio") + for daemon in ("pipewire", "wireplumber"): + (state / daemon).touch() + before = calls(env) + result = run_unit("pipewire-audio", env, RESTART_FAIL="1") + assert result.returncode == 1 + assert (state / daemon).exists() + assert (state / "mic").exists() + assert "systemctl --user restart mic-dsp-link.service" not in calls(env)[len(before):] + before = calls(env) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + actions = calls(env)[len(before):] + assert f"systemctl --user restart {daemon}" in actions + assert "systemctl --user restart mic-dsp-link.service" in actions + assert actions.index(f"systemctl --user restart {daemon}") < actions.index("systemctl --user restart mic-dsp-link.service") + assert not (state / daemon).exists() + assert not (state / "mic").exists() + + +def test_audio_daemon_restart_does_not_run_custom_microphone_service(stub_env): + env = setup(stub_env) + assert run_unit("pipewire-audio", env).returncode == 0 + service = home(env, ".config/systemd/user/mic-dsp-link.service") + service.write_text("[Service]\nExecStart=/custom/microphone\n") + state = home(env, ".local/state/pipewire-audio") + (state / "pipewire").touch() + before = calls(env) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "systemctl --user restart pipewire" in calls(env)[len(before):] + assert "mic-dsp-link.service" not in calls(env)[len(before):] + assert not (state / "mic").exists() + + def test_gates_and_missing_inventory_parser(stub_env): env = setup(stub_env, "mumbai") result = run_unit("pipewire-audio", env) @@ -185,7 +287,7 @@ def test_changed_volume_restarts_mic_and_invalid_override_fails(stub_env): before = calls(env) result = run_unit("pipewire-audio", env, MIC_HARDWARE_VOLUME_PCT="50") assert result.returncode == 0, result.stdout + result.stderr - assert "set-source-volume $$HW_MIC 50%" in home(env, ".config/systemd/user/mic-dsp-link.service").read_text() + assert "set-source-volume $$HW_MIC 50%%" in home(env, ".config/systemd/user/mic-dsp-link.service").read_text() assert "systemctl --user restart mic-dsp-link.service" in calls(env)[len(before):] before = calls(env) result = run_unit("pipewire-audio", env, MIC_HARDWARE_VOLUME_PCT="101") @@ -220,12 +322,32 @@ def test_ostree_package_failure_is_reported_but_audio_continues(stub_env): assert home(env, ".config/pipewire/pipewire.conf.d/50-speaker-dsp.conf").exists() +def test_bootc_never_installs_packages_and_ostree_takes_priority(stub_env): + env = setup(stub_env) + bindir = Path(env["PATH"]) + write_stub(bindir, "rpm", 'exit 1\n') + write_stub(bindir, "dnf", 'echo "dnf $*" >> "$HOME/calls"\n') + write_stub(bindir, "rpm-ostree", 'echo "rpm-ostree $*" >> "$HOME/calls"\n') + write_stub(bindir, "bootc", 'exit 0\n') + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "bootc DSP prerequisites must be provided by the image" in result.stdout + assert "dnf install" not in calls(env) + assert "rpm-ostree install" not in calls(env) + (bindir / "bootc").unlink() + before = calls(env) + result = run_unit("pipewire-audio", env) + assert result.returncode == 0, result.stdout + result.stderr + assert "rpm-ostree install --apply-live --idempotent" in calls(env)[len(before):] + assert "dnf install" not in calls(env)[len(before):] + + def test_default_mic_volume_and_gnome_setting(stub_env): env = setup(stub_env, "kanpur") result = run_unit("pipewire-audio", env, GNOME_VOLUME="false", DBUS_SESSION_BUS_ADDRESS="unix:path=/custom/bus") assert result.returncode == 0, result.stdout + result.stderr service = home(env, ".config/systemd/user/mic-dsp-link.service").read_text() - assert "set-source-volume $$HW_MIC 100%" in service + assert "set-source-volume $$HW_MIC 100%%" in service assert "gsettings set org.gnome.desktop.sound allow-volume-above-100-percent true" in calls(env) assert "gsettings-bus unix:path=/custom/bus" in calls(env) @@ -314,7 +436,7 @@ def apply(expected, **extra): result = run_unit("pipewire-audio", env, DOTFILES_DIR=str(source), **extra) assert result.returncode == 0, result.stdout + result.stderr service = home(env, ".config/systemd/user/mic-dsp-link.service").read_text() - assert f"set-source-volume $$HW_MIC {expected}%" in service + assert f"set-source-volume $$HW_MIC {expected}%%" in service apply(93) (groups / "all.yml").write_text("mic_hardware_volume_pct: 64\n") diff --git a/tests/test_platform_native_units.py b/tests/test_platform_native_units.py index 9172cc68..6270c770 100644 --- a/tests/test_platform_native_units.py +++ b/tests/test_platform_native_units.py @@ -153,7 +153,7 @@ def test_termux_packages_bootstrap_ssh_and_keepalive(platform_env): result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1") assert result.returncode == 0, result.stdout + result.stderr calls = log_lines(log) - assert calls.index("pkg install -y python yq chezmoi") < next(i for i, x in enumerate(calls) if x.startswith("npm ")) + assert calls.index("pkg install -y python yq chezmoi just") < next(i for i, x in enumerate(calls) if x.startswith("npm ")) assert "pkg install -y mise" in calls assert "npm install -g @earendil-works/pi-coding-agent" in calls assert "sshd-start" in calls @@ -170,6 +170,96 @@ def test_termux_packages_bootstrap_ssh_and_keepalive(platform_env): assert system.exists() +def test_termux_packages_require_both_inventory_and_termux(platform_env): + _, _, bindir, _, log = platform_env + write_stub(bindir, "pkg", 'echo "pkg $*" >> "$LOG"\n') + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone") + assert result.returncode == 0 and "require the Termux environment" in result.stdout + assert not log_lines(log) + + +def test_termux_keepalive_exact_job_id_and_inspection_failures(platform_env): + _, _, bindir, env, log = platform_env + write_stub(bindir, "pkg", 'exit 0\n') + write_stub(bindir, "npm", 'exit 0\n') + write_stub(bindir, "pgrep", 'exit "${PGREP_RC:-0}"\n') + write_stub(bindir, "sshd", 'echo sshd-start >> "$LOG"\n') + write_stub(bindir, "termux-job-scheduler", '''\ +if [ "$1" = -p ]; then + [ "${SCHED_FAIL:-0}" = 0 ] || exit 1 + echo 'Pending Job 10' +else echo "job $*" >> "$LOG"; fi +''') + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1") + assert result.returncode == 0, result.stdout + result.stderr + assert any(line.startswith("job --job-id 1 ") and line.endswith("/.termux/boot/start-sshd") for line in log_lines(log)) + log.unlink() + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1", SCHED_FAIL="1", PGREP_RC="2") + assert result.returncode != 0 + assert "Cannot inspect Termux scheduled jobs" in result.stderr + assert "Cannot inspect Termux sshd process" in result.stderr + assert not log_lines(log) + + +def test_termux_does_not_schedule_stale_boot_script(platform_env): + _, _, bindir, env, log = platform_env + write_stub(bindir, "pkg", 'exit 0\n') + write_stub(bindir, "npm", 'exit 0\n') + write_stub(bindir, "pgrep", 'exit 0\n') + write_stub(bindir, "termux-job-scheduler", 'echo "job $*" >> "$LOG"\n') + directory = Path(env["HOME"]) / ".termux/boot" + directory.mkdir(parents=True) + (directory / "start-sshd").mkdir() + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1") + assert result.returncode != 0 + assert not log_lines(log) + assert (directory / "start-sshd").is_dir() + + +def test_termux_preserves_custom_boot_and_accepts_prior_native_script(platform_env): + _, _, bindir, env, log = platform_env + write_stub(bindir, "pkg", 'exit 0\n') + write_stub(bindir, "npm", 'exit 0\n') + write_stub(bindir, "pgrep", 'exit 0\n') + write_stub(bindir, "termux-job-scheduler", 'if [ "$1" != -p ]; then echo scheduled >> "$LOG"; fi\n') + script = Path(env["HOME"]) / ".termux/boot/start-sshd" + script.parent.mkdir(parents=True) + old = ("#!/data/data/com.termux/files/usr/bin/sh\n" + "# Run at boot via Termux:Boot and periodically via Android's JobScheduler.\n" + "# Termux:Boot and Termux:API are separate companion apps.\n" + "termux-wake-lock\nsshd\n") + script.write_text(old) + script.chmod(0o600) + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1") + assert result.returncode == 0, result.stdout + result.stderr + assert script.read_text() == old and script.stat().st_mode & 0o777 == 0o755 + assert log_lines(log) == ["scheduled"] + log.unlink() + script.write_text("custom script\n") + script.chmod(0o600) + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1") + assert result.returncode != 0 and "Cannot replace customized" in result.stderr + assert script.read_text() == "custom script\n" and script.stat().st_mode & 0o777 == 0o600 + assert not log_lines(log) + + +def test_termux_unreadable_overrides_do_not_apply_defaults(platform_env): + root, _, bindir, env, log = platform_env + (root / "host_vars/phone.yml").symlink_to(root / "missing-overrides.yml") + write_stub(bindir, "pkg", 'echo "pkg $*" >> "$LOG"\n') + write_stub(bindir, "npm", 'echo "npm $*" >> "$LOG"\n') + write_stub(bindir, "pgrep", 'exit 1\n') + write_stub(bindir, "sshd", 'echo sshd-start >> "$LOG"\n') + write_stub(bindir, "termux-job-scheduler", 'echo "job $*" >> "$LOG"\n') + result = run_unit(platform_env, "termux-packages", DOTFILES_MACHINE="phone", TEST_TERMUX="1") + assert result.returncode != 0 and "Cannot read Termux package configuration" in result.stderr + calls = log_lines(log) + assert "pkg install -y python yq chezmoi just" in calls + assert not any(line.startswith("npm ") or line.startswith("job ") for line in calls) + assert "sshd-start" in calls + assert (Path(env["HOME"]) / ".termux/boot/start-sshd").is_file() + + def test_termux_unavailable_bootstrap_package_does_not_block_ssh(platform_env): _, _, bindir, env, log = platform_env write_stub(bindir, "pkg", '''\ @@ -200,7 +290,7 @@ def test_apk_failure_isolation_and_bootstrap(platform_env): result = run_unit(platform_env, "apk-packages", TEST_MUSL="1") assert result.returncode == 1 and "exa" in result.stderr calls = log_lines(log) - assert any("chezmoi yq-go python3 py3-yaml" in line for line in calls) + assert any("chezmoi yq-go python3 py3-yaml just" in line for line in calls) assert any("--no-upgrade" in line and " bash" in line for line in calls) assert "npm install -g @bitwarden/cli" in calls assert any("systemctl enable --now podman.socket" in line for line in calls) diff --git a/tests/test_proxy_cert_parity.py b/tests/test_proxy_cert_parity.py new file mode 100644 index 00000000..e86854dc --- /dev/null +++ b/tests/test_proxy_cert_parity.py @@ -0,0 +1,243 @@ +import json +import subprocess +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub +from test_server_units_check import check_system, run_check +from test_server_service_units import fake_system, run_unit, system_calls + + +@pytest.mark.parametrize('unit', ['proxy', 'tailscale-cert']) +@pytest.mark.parametrize('check', ['0', '1']) +def test_nested_vps_never_mutated(fake_system, unit, check): + repo, root, env = fake_system + (repo / 'inventory.yml').write_text(json.dumps({'all': {'children': { + 'vps': {'children': {'hosted': {'hosts': {'workstation': {}}}}} + }}})) + result = run_unit(unit, env, DOTFILES_DIR=str(repo), UNITS_CHECK=check) + assert result.returncode == 0 and 'SKIPPED' in result.stdout + assert system_calls(root) == '' + assert not (root / 'etc/caddy').exists() + assert not (root / 'etc/ssl/tailscale').exists() + +@pytest.mark.parametrize('check', ['0', '1']) +def test_retired_cleanup_skips_nested_vps(check_system, check): + repo, _, env, old = check_system + (repo / 'inventory.yml').write_text(json.dumps({'all': {'children': { + 'vps': {'children': {'hosted': {'hosts': {'workstation': {}}}}} + }}})) + result = run_check(check_system, 'retired-services', UNITS_CHECK=check) + assert result.returncode == 0 and 'SKIPPED' in result.stdout + assert old.read_text() == 'retired\n' + assert not Path(env['CALLS']).exists() + + + +@pytest.mark.parametrize('unit', ['proxy', 'tailscale-cert']) +def test_unknown_host_fails_before_mutating(fake_system, unit): + repo, root, env = fake_system + result = run_unit(unit, env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE='unknown') + assert result.returncode != 0 + assert system_calls(root) == '' + + +def test_group_settings_and_host_precedence(fake_system): + repo, root, env = fake_system + (repo / 'inventory.yml').write_text(json.dumps({'all': {'children': { + 'desktop': {'hosts': {'workstation': {}}}, 'vps': {'hosts': {}} + }}})) + (repo / 'group_vars/desktop.yml').write_text( + 'tailscale_domain: desktop.ts.net\nweb_services:\n' + ' - name: Group App\n href: https://group.example/\n') + result = run_unit('proxy', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, (result.stdout, result.stderr) + assert 'redir https://group.example/ 302' in (root / 'etc/caddy/Caddyfile').read_text() + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, (result.stdout, result.stderr) + assert 'workstation.desktop.ts.net' in (root / 'etc/systemd/system/tailscale-cert-renew.service').read_text() + (repo / 'host_vars/workstation.yml').write_text('skip_proxy: true\nskip_tailscale_cert: true\n') + (root / 'calls').unlink() + for unit in ('proxy', 'tailscale-cert'): + result = run_unit(unit, env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0 and 'SKIPPED' in result.stdout + assert system_calls(root) == '' + + +@pytest.mark.parametrize('unit,flag', [('proxy', 'skip_proxy'), ('tailscale-cert', 'skip_tailscale_cert')]) +def test_inherited_skip_and_host_override(fake_system, unit, flag): + repo, root, env = fake_system + (repo / 'inventory.yml').write_text(json.dumps({'all': {'children': { + 'desktop': {'hosts': {'workstation': {}}}, 'vps': {'hosts': {}} + }}})) + (repo / 'group_vars/desktop.yml').write_text(f'{flag}: true\n') + result = run_unit(unit, env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0 and 'SKIPPED' in result.stdout + assert system_calls(root) == '' + (repo / 'host_vars/workstation.yml').write_text(f'{flag}: false\n') + result = run_unit(unit, env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, (result.stdout, result.stderr) + assert system_calls(root) + + +def test_unreadable_serve_does_not_replace_configuration(fake_system): + repo, root, env = fake_system + (root / 'serve').write_text('existing target') + result = run_unit('proxy', env, DOTFILES_DIR=str(repo), FAIL_SERVE='1') + assert result.returncode != 0 + assert 'inspect Tailscale serve status' in result.stdout + assert (root / 'serve').read_text() == 'existing target' + assert 'tailscale serve --bg' not in system_calls(root) + + +@pytest.mark.parametrize('other', ['workstation.example.ts.net:8443', 'other.example.ts.net:443']) +def test_unrelated_serve_target_does_not_hide_default_drift(fake_system, other): + repo, root, env = fake_system + (root / 'serve_json').write_text(json.dumps({ + 'TCP': {'443': {'HTTPS': True}}, + 'Web': { + 'workstation.example.ts.net:443': {'Handlers': {'/': {'Proxy': 'http://127.0.0.1:3000'}}}, + other: {'Handlers': {'/': {'Proxy': 'http://127.0.0.1:9000'}}}, + }, + })) + result = run_unit('proxy', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, (result.stdout, result.stderr) + assert (root / 'serve').read_text() == 'http://127.0.0.1:9000' + assert 'tailscale serve --bg' in system_calls(root) + + +def test_invalid_serve_json_does_not_replace_configuration(fake_system): + repo, root, env = fake_system + (root / 'serve').write_text('existing target') + (root / 'serve_json').write_text('{broken') + result = run_unit('proxy', env, DOTFILES_DIR=str(repo)) + assert result.returncode != 0 + assert (root / 'serve').read_text() == 'existing target' + assert 'tailscale serve --bg' not in system_calls(root) + + +def test_failed_certificate_request_preserves_existing_pair(fake_system): + repo, root, env = fake_system + directory = root / 'etc/ssl/tailscale' + directory.mkdir(parents=True) + (directory / 'cert.pem').write_text('EXPIRED') + (directory / 'key.pem').write_text('ORIGINAL PRIVATE KEY') + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo), FAIL_CERT='1') + assert result.returncode != 0 + assert (directory / 'cert.pem').read_text() == 'EXPIRED' + assert (directory / 'key.pem').read_text() == 'ORIGINAL PRIVATE KEY' + assert 'systemctl enable' not in system_calls(root) + + +def test_valid_certificate_with_missing_key_is_reprovisioned(fake_system): + repo, root, env = fake_system + directory = root / 'etc/ssl/tailscale' + directory.mkdir(parents=True) + (directory / 'cert.pem').write_text('VALID') + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo)) + assert result.returncode == 0, (result.stdout, result.stderr) + assert (directory / 'key.pem').read_text() == 'PRIVATE' + assert 'tailscale cert ' in system_calls(root) + + +def test_missing_key_check_reports_renewal_without_writing(fake_system): + repo, root, env = fake_system + directory = root / 'etc/ssl/tailscale' + directory.mkdir(parents=True) + (directory / 'cert.pem').write_text('VALID') + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo), UNITS_CHECK='1') + assert result.returncode == 0, (result.stdout, result.stderr) + assert 'private key missing or empty' in result.stdout + assert not (directory / 'key.pem').exists() + assert 'tailscale cert ' not in system_calls(root) + + +@pytest.mark.parametrize('bad', ['BAD_HOST', 'BAD_KEY']) +@pytest.mark.parametrize('check', ['0', '1']) +def test_existing_mismatched_certificate_is_preserved(fake_system, bad, check): + repo, root, env = fake_system + directory = root / 'etc/ssl/tailscale' + directory.mkdir(parents=True) + (directory / 'cert.pem').write_text('VALID') + (directory / 'key.pem').write_text('PRIVATE') + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo), UNITS_CHECK=check, **{bad: '1'}) + assert result.returncode != 0 + assert 'existing pair preserved' in result.stdout + assert (directory / 'cert.pem').read_text() == 'VALID' + assert (directory / 'key.pem').read_text() == 'PRIVATE' + assert 'tailscale cert ' not in system_calls(root) + + +@pytest.mark.parametrize('bad', ['BAD_HOST', 'BAD_KEY']) +def test_invalid_issued_pair_does_not_replace_old_pair(fake_system, bad): + repo, root, env = fake_system + directory = root / 'etc/ssl/tailscale' + directory.mkdir(parents=True) + (directory / 'cert.pem').write_text('EXPIRED') + (directory / 'key.pem').write_text('OLD KEY') + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo), **{bad: '1'}) + assert result.returncode != 0 + assert (directory / 'cert.pem').read_text() == 'EXPIRED' + assert (directory / 'key.pem').read_text() == 'OLD KEY' + + +@pytest.mark.parametrize('check', ['0', '1']) +def test_certificate_directory_file_is_preserved(fake_system, check): + repo, root, env = fake_system + path = root / 'etc/ssl/tailscale' + path.parent.mkdir(parents=True) + path.write_text('user data') + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo), UNITS_CHECK=check) + assert result.returncode != 0 + assert path.read_text() == 'user data' + assert system_calls(root) == '' + + +@pytest.mark.parametrize('path', ['cert.pem', 'key.pem']) +def test_certificate_symlinks_are_not_followed(fake_system, tmp_path, path): + repo, root, env = fake_system + directory = root / 'etc/ssl/tailscale' + directory.mkdir(parents=True) + target = tmp_path / 'user-owned' + target.write_text('leave me alone') + (directory / path).symlink_to(target) + result = run_unit('tailscale-cert', env, DOTFILES_DIR=str(repo)) + assert result.returncode != 0 + assert target.read_text() == 'leave me alone' + assert (directory / path).is_symlink() + assert system_calls(root) == '' + + +def test_renewal_restores_key_if_certificate_commit_fails(stub_env, tmp_path): + bindir, env = stub_env + directory = tmp_path / 'certs' + directory.mkdir() + (directory / 'cert.pem').write_text('OLD CERT') + (directory / 'key.pem').write_text('OLD KEY') + write_stub(bindir, 'tailscale', ''' +while [ "$#" -gt 0 ]; do + case "$1" in + --cert-file) printf 'NEW CERT' > "$2"; shift 2 ;; + --key-file) printf 'NEW KEY' > "$2"; shift 2 ;; + *) shift ;; + esac +done +''') + write_stub(bindir, 'openssl', ''' +case "$*" in + *' -checkhost '*) printf 'Hostname %s does match certificate\\n' "$3" ;; + *' -pubkey '*|*' -pubout '*) echo PUBLIC ;; +esac +''') + write_stub(bindir, 'mv', ''' +case "${*: -1}" in */cert.pem) exit 1 ;; esac +exec /usr/bin/mv "$@" +''') + script = REPO / 'units/tailscale-cert/renew.sh' + result = subprocess.run(['bash', str(script), str(directory), 'host.example.ts.net'], + env=env, capture_output=True, text=True) + assert result.returncode != 0 + assert (directory / 'cert.pem').read_text() == 'OLD CERT' + assert (directory / 'key.pem').read_text() == 'OLD KEY' + assert not list(directory.glob('.renew.????????')) diff --git a/tests/test_server_app_parity.py b/tests/test_server_app_parity.py new file mode 100644 index 00000000..0dd95f80 --- /dev/null +++ b/tests/test_server_app_parity.py @@ -0,0 +1,214 @@ +"""Focused parity and safety regressions for the server application units.""" +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import REPO, write_stub +from test_server_units_check import check_system, prepare_hardening_commands, run_check + + +@pytest.mark.parametrize("check", [True, False]) +@pytest.mark.parametrize("signature,valid", [("swap", True), ("ext4", False), ("", False)]) +def test_existing_inactive_swap_is_never_reformatted(check_system, check, signature, valid): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + bindir = Path(env["PATH"].split(":", 1)[0]) + write_stub(bindir, "blkid", 'printf "%s\\n" "$SWAP_SIGNATURE"\n[ -n "$SWAP_SIGNATURE" ]\n') + swap = root / "swapfile" + swap.write_text("existing data") + fstab = root / "etc/fstab" + before = fstab.read_bytes() + result = run_check(check_system, "server-hardening", UNITS_CHECK="1" if check else "0", + ROOT_OPTIONS="rw,relatime", ACTIVE_SWAP="/dev/zram0", SWAP_SIGNATURE=signature) + assert result.returncode == (0 if valid else 1), (result.stdout, result.stderr) + assert swap.read_text() == "existing data" + calls = Path(env["CALLS"]).read_text().splitlines() + assert not any(line.startswith(("dd ", "mkswap ")) for line in calls) + if not valid: + assert "refusing to format" in result.stdout + assert fstab.read_bytes() == before + assert not any(line.startswith(("chown ", "chmod ")) for line in calls) + elif check: + assert "would enable existing swapfile" in result.stdout + assert fstab.read_bytes() == before + else: + assert f"swapon {swap}" in calls + assert f"{swap} none swap sw 0 0" in fstab.read_text() + + +@pytest.mark.parametrize("check", [True, False]) +def test_symlink_swap_is_rejected_without_touching_target(check_system, check): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + target = root / "other-data" + target.write_text("keep") + (root / "swapfile").symlink_to(target) + fstab = root / "etc/fstab" + before = fstab.read_bytes() + result = run_check(check_system, "server-hardening", UNITS_CHECK="1" if check else "0", + ROOT_OPTIONS="rw,relatime", ACTIVE_SWAP="/dev/zram0") + assert result.returncode == 1 + assert "must be a regular file, not a symlink" in result.stdout + assert target.read_text() == "keep" and fstab.read_bytes() == before + calls = Path(env["CALLS"]).read_text().splitlines() + assert not any(line.startswith(("dd ", "chown ", "chmod ", "mkswap ")) for line in calls) + + +def test_privileged_firewall_and_pro_status_checks(check_system): + _, _, env, _ = check_system + prepare_hardening_commands(check_system) + bindir = Path(env["PATH"].split(":", 1)[0]) + write_stub(bindir, "id", 'if [ "$1" = -u ]; then echo 1000; else exec /usr/bin/id "$@"; fi\n') + write_stub(bindir, "sudo", '[ "$1" = -n ] && shift\nexport FROM_SUDO=1\nexec "$@"\n') + write_stub(bindir, "ufw", ''' +echo "ufw $* ${FROM_SUDO:-no}" >> "$CALLS" +[ "${FROM_SUDO:-}" = 1 ] || exit 1 +if [ "$1" = status ]; then echo 'Status: active'; fi +''') + write_stub(bindir, "pro", ''' +echo "pro $* ${FROM_SUDO:-no}" >> "$CALLS" +[ "${FROM_SUDO:-}" = 1 ] || exit 1 +if [ "$1" = status ]; then echo '{"livepatch":"disabled"}'; fi +''') + result = run_check(check_system, "server-hardening", UNITS_CHECK="0", ROOT_OPTIONS="ro,relatime", ACTIVE_SWAP="/dev/zram0") + assert result.returncode == 0, (result.stdout, result.stderr) + calls = Path(env["CALLS"]).read_text().splitlines() + assert "ufw status 1" in calls and "ufw logging on 1" in calls + assert "pro status --format json 1" in calls and "pro enable livepatch 1" in calls + + +def test_ssh_limits_precede_includes_and_match_without_losing_configuration(tmp_path): + config = tmp_path / "sshd_config" + original = "Include /etc/ssh/sshd_config.d/*.conf\nMaxSessions 25\nMatch User deploy\n MaxSessions 2\n" + config.write_text(original) + script = str(REPO / "units/server-hardening-file.py") + check = subprocess.run([sys.executable, script, "--check", "--ssh", str(config)], capture_output=True, text=True) + assert check.returncode == 10 and config.read_text() == original + result = subprocess.run([sys.executable, script, "--ssh", str(config)], capture_output=True, text=True) + assert result.returncode == 10 + updated = config.read_text() + assert updated.startswith("# BEGIN dotfiles server-hardening\nMaxStartups 3:50:10\nMaxSessions 10\nClientAliveCountMax 3\n") + assert updated.endswith(original) + assert subprocess.run([sys.executable, script, "--ssh", str(config)], capture_output=True).returncode == 0 + + +def test_malformed_ssh_managed_block_is_not_replaced(tmp_path): + config = tmp_path / "sshd_config" + original = "# BEGIN dotfiles server-hardening\nMaxSessions 30\n" + config.write_text(original) + result = subprocess.run([sys.executable, str(REPO / "units/server-hardening-file.py"), "--ssh", str(config)], capture_output=True) + assert result.returncode != 0 and config.read_text() == original + + +def test_dashboard_changed_unit_restarts_only_after_reload(check_system): + _, _, env, _ = check_system + prepare_hardening_commands(check_system) + bindir = Path(env["PATH"].split(":", 1)[0]) + write_stub(bindir, "install", 'exec /usr/bin/install "$@"\n') + write_stub(bindir, "loginctl", 'echo "loginctl $*" >> "$CALLS"\nif [ "$1" = show-user ]; then echo no; fi\n') + service = Path(env["HOME"]) / ".config/systemd/user/bst-dashboard.service" + service.parent.mkdir(parents=True, exist_ok=True) + service.write_text("[Service]\nExecStart=/old/dashboard\n") + check = run_check(check_system, "bst-dashboard", UNITS_CHECK="1") + assert check.returncode == 0 and "start or restart dashboard" in check.stdout + assert service.read_text() == "[Service]\nExecStart=/old/dashboard\n" + Path(env["CALLS"]).unlink() + result = run_check(check_system, "bst-dashboard", UNITS_CHECK="0") + assert result.returncode == 0, (result.stdout, result.stderr) + calls = Path(env["CALLS"]).read_text().splitlines() + assert calls.index("systemctl --user daemon-reload") < calls.index("systemctl --user restart bst-dashboard.service") + assert "--target oci/dakota.bst" in service.read_text() + Path(env["CALLS"]).unlink() + result = run_check(check_system, "bst-dashboard", UNITS_CHECK="0") + assert result.returncode == 0 + calls = Path(env["CALLS"]).read_text().splitlines() + assert "systemctl --user restart bst-dashboard.service" not in calls + assert "systemctl --user daemon-reload" not in calls + +def test_dashboard_enable_failure_does_not_restart(check_system): + _, _, env, _ = check_system + prepare_hardening_commands(check_system) + bindir = Path(env["PATH"].split(":", 1)[0]) + write_stub(bindir, "install", 'exec /usr/bin/install "$@"\n') + write_stub(bindir, "loginctl", 'exit 0\n') + write_stub(bindir, "systemctl", ''' +echo "systemctl $*" >> "$CALLS" +if [ "$1 $2" = "--user enable" ]; then exit 1; fi +''') + result = run_check(check_system, "bst-dashboard", UNITS_CHECK="0") + assert result.returncode != 0 and "FAIL enable dashboard" in result.stdout + calls = Path(env["CALLS"]).read_text().splitlines() + assert "systemctl --user restart bst-dashboard.service" not in calls + + +@pytest.mark.parametrize("address,allowed", [("127.0.0.1:5476", True), ("[::1]:5476", True), ("0.0.0.0:5476", False), ("[::]:5476", False)]) +def test_python_process_listeners_are_checked_by_cgroup(tmp_path, address, allowed): + proc = tmp_path / "proc" + (proc / "42").mkdir(parents=True) + (proc / "42/cgroup").write_text("0::/system.slice/kirocrew.service/workers\n") + (proc / "43").mkdir() + (proc / "43/cgroup").write_text("0::/system.slice/other.service\n") + lines = (f'LISTEN 0 128 {address} 0.0.0.0:* users:(("python3",pid=42,fd=5))\n' + 'LISTEN 0 128 0.0.0.0:80 0.0.0.0:* users:(("python3",pid=43,fd=5))\n') + result = subprocess.run([sys.executable, str(REPO / "units/kirocrew-sockets.py"), "--cgroup", "/system.slice/kirocrew.service", "--proc-root", str(proc)], input=lines, capture_output=True, text=True) + assert (result.returncode == 0) is allowed + + +def test_no_attributable_listener_does_not_claim_safety(tmp_path): + result = subprocess.run([sys.executable, str(REPO / "units/kirocrew-sockets.py"), "--cgroup", "/system.slice/kirocrew.service", "--proc-root", str(tmp_path)], input="", capture_output=True, text=True) + assert result.returncode != 0 and "cannot be verified" in result.stderr + + +def test_unrelated_named_process_cannot_prove_service_safety(tmp_path): + (tmp_path / "43").mkdir() + (tmp_path / "43/cgroup").write_text("0::/system.slice/other.service\n") + line = 'LISTEN 0 128 127.0.0.1:5476 0.0.0.0:* users:(("kirocrew",pid=43,fd=5))\n' + result = subprocess.run([sys.executable, str(REPO / "units/kirocrew-sockets.py"), "--cgroup", "/system.slice/kirocrew.service", "--proc-root", str(tmp_path)], input=line, capture_output=True, text=True) + assert result.returncode != 0 and "cannot be verified" in result.stderr + + +@pytest.mark.parametrize("check", [True, False]) +@pytest.mark.parametrize("address,allowed", [("127.0.0.1:5476", True), ("0.0.0.0:5476", False)]) +def test_kirocrew_unit_checks_python_service_cgroup(check_system, check, address, allowed): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + bindir = Path(env["PATH"].split(":", 1)[0]) + proc = root / "proc" + (proc / "42").mkdir(parents=True) + (proc / "42/cgroup").write_text("0::/system.slice/kirocrew.service\n") + write_stub(bindir, "python3", ''' +script=$1; shift +exec /usr/bin/python3 "$script" "$@" --proc-root "$FAKE_PROC" +''') + write_stub(bindir, "systemctl", ''' +echo "systemctl $*" >> "$CALLS" +if [[ "$*" == *ControlGroup* ]]; then echo /system.slice/kirocrew.service +elif [[ "$*" == *show* ]]; then printf 'LoadState=loaded\\nActiveState=active\\nUnitFileState=enabled\\n' +fi +''') + line = f'LISTEN 0 128 {address} 0.0.0.0:* users:(("python3",pid=42,fd=5))' + result = run_check(check_system, "kirocrew", UNITS_CHECK="1" if check else "0", FAKE_PROC=str(proc), SOCKETS=line) + assert (result.returncode == 0) is allowed, (result.stdout, result.stderr) + calls = Path(env["CALLS"]).read_text().splitlines() + assert any("ControlGroup" in line for line in calls) + assert ("systemctl enable --now kirocrew.service" in calls) is (not check) + + +@pytest.mark.parametrize("check", [True, False]) +def test_kirocrew_refuses_symlinked_credentials(check_system, check): + _, root, env, _ = check_system + prepare_hardening_commands(check_system) + envfile = root / "etc/kirocrew/kirocrew.env" + target = root / "credentials" + target.write_text("SECRET=not-for-output\n") + envfile.unlink() + envfile.symlink_to(target) + result = run_check(check_system, "kirocrew", UNITS_CHECK="1" if check else "0") + assert result.returncode != 0 + assert "not-for-output" not in result.stdout + result.stderr + assert target.read_text() == "SECRET=not-for-output\n" + calls = Path(env["CALLS"]).read_text().splitlines() + assert not any("enable --now kirocrew.service" in line for line in calls) + assert not any(line.startswith(("chown ", "chmod ")) for line in calls) diff --git a/tests/test_server_service_units.py b/tests/test_server_service_units.py index 07ec8648..d850e9a9 100644 --- a/tests/test_server_service_units.py +++ b/tests/test_server_service_units.py @@ -4,7 +4,7 @@ import pytest from conftest import REPO -from unit_helpers import run_unit +from unit_helpers import run_unit as run_common_unit # System files live in a private fake /etc. The scripts under test still execute @@ -26,7 +26,9 @@ def record(): expression = args[-2] import yaml with open(args[-1]) as f: data = yaml.safe_load(f) - if 'children.vps.hosts' in expression: + if expression == '.': + print(json.dumps(data)) + elif 'children.vps.hosts' in expression: print(str(os.environ['MACHINE'] in (data['all']['children']['vps']['hosts'] or {})).lower()) elif 'web_services' in expression: print(json.dumps((data or {}).get('web_services') or [])) @@ -36,18 +38,41 @@ def record(): sys.exit(0) if name in ('systemctl', 'tailscale', 'loginctl'): record() + if name == 'systemctl' and args[:1] == ['show']: + print('LoadState=loaded\nActiveState=active\nUnitFileState=enabled') + if name == 'tailscale' and args[:2] == ['status', '--json']: + print(json.dumps({'Self': {'DNSName': 'workstation.example.ts.net.'}})) if name == 'tailscale' and args[:2] == ['serve', 'status']: + if os.environ.get('FAIL_SERVE'): sys.exit(1) state = root / 'serve' - if state.exists(): print(state.read_text(), end='') + custom = root / 'serve_json' + if custom.exists(): + print(custom.read_text()) + elif state.exists(): + print(json.dumps({'TCP': {'443': {'HTTPS': True}}, 'Web': { + 'workstation.example.ts.net:443': {'Handlers': {'/': {'Proxy': state.read_text()}}}}})) + else: + print('{}') if name == 'tailscale' and args[:2] == ['serve', '--bg']: (root / 'serve').write_text(args[2]) + (root / 'serve_json').unlink(missing_ok=True) if name == 'tailscale' and args[:1] == ['cert']: pathlib.Path(mapped(args[args.index('--cert-file')+1])).write_text('VALID') + if os.environ.get('FAIL_CERT'): sys.exit(1) pathlib.Path(mapped(args[args.index('--key-file')+1])).write_text('PRIVATE') sys.exit(0) if name == 'openssl': path = pathlib.Path(mapped(args[args.index('-in') + 1])) - sys.exit(0 if path.exists() and path.read_text() == 'VALID' else 1) + if not path.exists(): sys.exit(1) + if args[0] == 'pkey': + print('OTHER PUBLIC' if os.environ.get('BAD_KEY') else 'PUBLIC') + sys.exit(0) + if path.read_text() != 'VALID': sys.exit(1) + if '-checkhost' in args: + hostname = args[args.index('-checkhost') + 1] + print(f"Hostname {hostname} does {'NOT ' if os.environ.get('BAD_HOST') else ''}match certificate") + if '-pubkey' in args: print('PUBLIC') + sys.exit(0) if name == 'chown': sys.exit(0) if name == 'stat': try: @@ -56,7 +81,7 @@ def record(): except FileNotFoundError: sys.exit(1) sys.exit(0) if name == 'install': - if os.environ.get('FAIL_CADDY') and args[-1] == '/etc/caddy/Caddyfile': sys.exit(1) + if os.environ.get('FAIL_CADDY') and args[-1].endswith('/etc/caddy/Caddyfile'): sys.exit(1) output = [] i = 0 while i < len(args): @@ -86,12 +111,27 @@ def fake_system(stub_env, tmp_path): (repo / 'host_vars').mkdir(parents=True) (repo / 'group_vars').mkdir() (repo / 'group_vars/all.yml').write_text('tailscale_domain: example.ts.net\n') - (repo / 'inventory.yml').write_text('all:\n children:\n vps:\n hosts:\n matrix: {}\n telengana: {}\n other-vps: {}\n') - (repo / 'units').symlink_to(REPO / 'units') + (repo / 'inventory.yml').write_text('all:\n hosts:\n workstation: {}\n vm: {}\n children:\n vps:\n hosts:\n matrix: {}\n telengana: {}\n other-vps: {}\n') + units = repo / 'units' + units.mkdir() + for name in ('lib.sh', 'proxy-inventory.py', 'proxy-serve.py', 'osquery_native_inventory.py', 'service-templates.py', + 'proxy', 'tailscale-cert', 'bst-dashboard', 'kirocrew-sockets.py'): + (units / name).symlink_to(REPO / 'units' / name) + for name in ('proxy', 'tailscale-cert'): + script = (REPO / 'units' / f'{name}.sh').read_text() + (units / f'{name}.sh').write_text(script.replace('/etc/', str(root / 'etc') + '/')) env.update(DOTFILES_DIR=str(repo), DOTFILES_MACHINE='workstation') return repo, root, env +def run_unit(name, env, **extra): + if name not in ('proxy', 'tailscale-cert'): + return run_common_unit(name, env, **extra) + repo = extra.get('DOTFILES_DIR', env['DOTFILES_DIR']) + return subprocess.run(['bash', str(repo + '/units/' + name + '.sh')], + env={**env, **extra}, capture_output=True, text=True) + + def system_calls(root): path = root / 'calls' return path.read_text() if path.exists() else '' @@ -137,7 +177,8 @@ def test_certificate_renewal_permissions_timer_and_validity(fake_system): assert key.read_text() == 'PRIVATE' and key.stat().st_mode & 0o777 == 0o640 service = (root / 'etc/systemd/system/tailscale-cert-renew.service').read_text() timer = (root / 'etc/systemd/system/tailscale-cert-renew.timer').read_text() - assert 'workstation.example.ts.net' in service and '--min-validity 720h' in service + assert 'workstation.example.ts.net' in service and 'tailscale-cert-renew.sh' in service + assert '--min-validity 720h' in (root / 'etc/systemd/system/tailscale-cert-renew.sh').read_text() assert 'OnCalendar=weekly' in timer and 'RandomizedDelaySec=1h' in timer and 'Persistent=true' in timer assert 'systemctl enable --now tailscale-cert-renew.timer' in system_calls(root) (root / 'calls').unlink() diff --git a/tests/test_server_units_check.py b/tests/test_server_units_check.py index 0acd0a62..2b1ec6bb 100644 --- a/tests/test_server_units_check.py +++ b/tests/test_server_units_check.py @@ -25,7 +25,7 @@ def check_system(stub_env, tmp_path): (repo / "inventory.yml").write_text(json.dumps({"all": {"children": { "server": {"hosts": {"workstation": {}}}, "vps": {"hosts": {}} }}})) - for name in ("lib.sh", "osquery_native_inventory.py", "server-hardening-file.py", + for name in ("lib.sh", "osquery_native_inventory.py", "proxy-inventory.py", "proxy-serve.py", "server-hardening-file.py", "service-templates.py", "kirocrew-sockets.py", "retired-services", "proxy", "tailscale-cert", "bst-dashboard"): (units / name).symlink_to(REPO / "units" / name) @@ -55,6 +55,9 @@ def check_system(stub_env, tmp_path): CALLS=str(tmp_path / "calls")) write_stub(bindir, "sudo", '[ "$1" = -n ] && shift\nexec "$@"\n') write_stub(bindir, "yq", ''' +if [ "$1" = -o=json ] && [ "$2" = -I=0 ]; then + exec python3 -c 'import json,sys,yaml; print(json.dumps(yaml.safe_load(open(sys.argv[1]))))' "$4" +fi case "$*" in *retired_services_system_units*) ;; *retired_services_user_units*) echo grafana.container ;; @@ -74,7 +77,11 @@ def check_system(stub_env, tmp_path): ''') write_stub(bindir, "tailscale", ''' echo "tailscale $*" >> "$CALLS" -[ "$1 $2" = "serve status" ] || exit 99 +case "$*" in + 'status --json') echo '{"Self":{"DNSName":"workstation.example.ts.net."}}' ;; + 'serve status --json') echo '{}' ;; + *) exit 99 ;; +esac ''') write_stub(bindir, "loginctl", 'echo "loginctl $*" >> "$CALLS"\n[ "$1" = show-user ] || exit 99\necho no\n') write_stub(bindir, "journalctl", '[ "$1" = --disk-usage ] || exit 99\necho 1G\n') @@ -365,7 +372,7 @@ def test_check_reports_drift_without_mutation(check_system, unit, expected): if calls.exists(): for call in calls.read_text().splitlines(): assert (call.startswith("systemctl ") and "show" in call or - call == "tailscale serve status" or call.startswith("loginctl show-user ")) + call in ("tailscale status --json", "tailscale serve status --json") or call.startswith("loginctl show-user ")) @pytest.mark.parametrize("unit", UNITS) diff --git a/tests/test_shell_ai_unit.py b/tests/test_shell_ai_unit.py index 77464762..e0bf53b1 100644 --- a/tests/test_shell_ai_unit.py +++ b/tests/test_shell_ai_unit.py @@ -1,10 +1,11 @@ +import shutil import subprocess import pytest from conftest import REPO, write_stub -NASTY = "pa'ss\"w$ord" +NASTY = "pa'ss\\word\"$ord" BW_STUB = """\ if [ "$1 $2" = "unlock --check" ]; then exit "${STUB_UNLOCK_RC:-0}"; fi @@ -72,3 +73,60 @@ def test_locked_or_nosecrets_leaves_files_untouched(env, extra): assert r.returncode == 0 assert "SKIPPED:" in r.stdout assert sh.read_text() == "previous\n" + + +def test_failed_secret_read_does_not_export_partial_stdout(env): + bindir = env["PATH"].split(":")[0] + write_stub(bindir, "bw", ''' +if [ "$1 $2" = "unlock --check" ]; then exit 0; fi +case "$2 $3" in + "password deepseek-api-key") echo partial-secret; exit 1 ;; + "password tavily-api-key") echo tav456; exit 0 ;; +esac +exit 1 +''') + result = run(env) + assert result.returncode == 0 + assert "DEEPSEEK_API_KEY" not in (home(env) / ".config/shell/secrets.sh").read_text() + assert "partial-secret" not in result.stdout + result.stderr + + +@pytest.mark.skipif(shutil.which("fish") is None, reason="fish not installed") +def test_fish_secret_round_trips_quotes_and_backslashes(env): + result = run(env) + assert result.returncode == 0, result.stdout + result.stderr + source = str(home(env) / ".config/fish/conf.d/secrets.fish") + fish = subprocess.run(["fish", "--no-config", "-c", 'source "$SECRET_FILE"; printf "%s" "$DEEPSEEK_API_KEY"'], + env={**env, "SECRET_FILE": source}, capture_output=True, text=True) + assert fish.returncode == 0, fish.stderr + assert fish.stdout == NASTY + + +@pytest.mark.skipif(shutil.which("fish") is None or shutil.which("jq") is None, + reason="fish and jq required") +@pytest.mark.parametrize("mode,expected", [("failed", "stored-kiro"), ("null", "stored-kiro"), + ("valid", "refreshed-kiro")]) +def test_fish_refresh_keeps_stored_key_when_vault_read_fails(env, tmp_path, mode, expected): + result = run(env) + assert result.returncode == 0, result.stdout + result.stderr + source = home(env) / ".config/fish/conf.d/secrets.fish" + text = source.read_text().replace("/tmp/bw_session", "$TEST_BW_SESSION_FILE") + source.write_text("set -gx KIRO_API_KEY stored-kiro\n" + text) + session = tmp_path / "session" + session.write_text("session-value") + write_stub(env["PATH"].split(":")[0], "bw", ''' +printf '%s' "$BW_SESSION" > "$HOME/refresh-session" +printf '%s' "$*" > "$HOME/refresh-args" +case "$REFRESH_MODE" in + failed) echo '{"notes":"bad-partial"}'; exit 1 ;; + null) echo '{"notes":null}'; exit 0 ;; + valid) echo '{"notes":"refreshed-kiro"}'; exit 0 ;; +esac +''') + fish = subprocess.run(["fish", "--no-config", "-c", 'source "$SECRET_FILE"; printf "%s" "$KIRO_API_KEY"'], + env={**env, "SECRET_FILE": str(source), "TEST_BW_SESSION_FILE": str(session), + "REFRESH_MODE": mode}, capture_output=True, text=True) + assert fish.returncode == 0, fish.stderr + assert fish.stdout == expected + assert (home(env) / "refresh-session").read_text() == "session-value" + assert "session-value" not in (home(env) / "refresh-args").read_text() diff --git a/tests/test_ssh_native_units.py b/tests/test_ssh_native_units.py index 599a01b0..3440b775 100644 --- a/tests/test_ssh_native_units.py +++ b/tests/test_ssh_native_units.py @@ -39,6 +39,14 @@ def fleet(stub_env, tmp_path): (data / "all.yml.json").write_text(json.dumps(variables)) (data / "inventory.yml.json").write_text(json.dumps(inventory)) (data / "test-fleet-fedora.yml.json").write_text(json.dumps({"machine_profile": "vm-test"})) + root = tmp_path / "repo" + (root / "group_vars").mkdir(parents=True) + (root / "host_vars").mkdir() + (root / "units").symlink_to(REPO / "units") + (root / "scripts").symlink_to(REPO / "scripts") + (root / "group_vars/all.yml").symlink_to(data / "all.yml.json") + (root / "inventory.yml").symlink_to(data / "inventory.yml.json") + (root / "host_vars/test-fleet-fedora.yml").symlink_to(data / "test-fleet-fedora.yml.json") item = {"id": "item-1", "type": 5, "name": "james@newhost", "sshKey": {"privateKey": private, "publicKey": public}} (home / "items.json").write_text(json.dumps([item])) @@ -68,7 +76,7 @@ def fleet(stub_env, tmp_path): ''') write_stub(bindir, "tailscale", '''printf '%s\\n' '{"Peer":{"one":{"Online":true,"DNSName":"peer-dns.example."},"two":{"Online":true,"DNSName":"retired.example."}}}' ''') - env.update(DOTFILES_DIR=str(REPO), DOTFILES_MACHINE="newhost", SSH_FIXTURE=str(data), + env.update(DOTFILES_DIR=str(root), DOTFILES_MACHINE="newhost", SSH_FIXTURE=str(data), BW_SESSION="secret-session", BREW_PREFIX="/nonexistent") return {"env": env, "home": home, "data": data, "private": private, "public": public, "other_private": other_private, "other_public": other_public, "variables": variables} @@ -307,6 +315,9 @@ def test_restores_encrypted_vault_identity_without_prompting(fleet, tmp_path, mi def test_host_vars_accepts_empty_document_but_rejects_nonmapping(fleet, tmp_path, contents, parsed, valid): root = tmp_path / "fleet-root" (root / "host_vars").mkdir(parents=True) + (root / "group_vars").mkdir() + (root / "group_vars/all.yml").symlink_to(fleet["data"] / "all.yml.json") + (root / "inventory.yml").symlink_to(fleet["data"] / "inventory.yml.json") (root / "host_vars/newhost.yml").write_text(contents) (fleet["data"] / "newhost.yml.json").write_text(parsed) script = ''' @@ -366,3 +377,100 @@ def test_encrypted_identity_without_public_key_is_not_replaced(fleet): assert not public.exists() assert not (fleet["home"] / "encoded-item").exists() assert before.decode() not in result.stdout + result.stderr + + +@pytest.mark.parametrize("check", [False, True]) +def test_ssh_host_key_scans_never_contact_retired_vps(fleet, check): + variables = fleet["variables"] + variables["machines"].pop("retired") + variables["machines"].update({"matrix": {"hostname": "matrix.reilly.asia"}, + "telengana": {"hostname": "telengana"}}) + (fleet["data"] / "all.yml.json").write_text(json.dumps(variables)) + result = execute(fleet, UNITS_NO_SECRETS="1", UNITS_CHECK="1" if check else "0") + assert result.returncode == 0, result.stdout + result.stderr + scans = (fleet["home"] / "scans").read_text().splitlines() + assert len(scans) == 1 and scans[0].endswith(" peer") + + +@pytest.mark.parametrize("check", [False, True]) +@pytest.mark.parametrize("excluded", [False, True]) +def test_host_specific_mesh_exclusions_override_group_defaults(fleet, tmp_path, check, excluded): + root = tmp_path / "host-override" + root.mkdir() + (root / "units").symlink_to(REPO / "units") + (root / "scripts").symlink_to(REPO / "scripts") + (root / "group_vars").mkdir() + (root / "group_vars/all.yml").symlink_to(fleet["data"] / "all.yml.json") + (root / "inventory.yml").symlink_to(fleet["data"] / "inventory.yml.json") + (root / "host_vars").mkdir() + host_exclude = ["remote"] if excluded else [] + fleet["variables"]["ssh_mesh_exclude"] = [] if excluded else ["remote"] + (fleet["data"] / "all.yml.json").write_text(json.dumps(fleet["variables"])) + (root / "host_vars/newhost.yml").write_text(json.dumps({"ssh_mesh_exclude": host_exclude})) + (fleet["data"] / "newhost.yml.json").write_text(json.dumps({"ssh_mesh_exclude": host_exclude})) + ssh = fleet["home"] / ".ssh" + ssh.mkdir() + (ssh / "id_ed25519").write_text(fleet["private"]) + (ssh / "id_ed25519").chmod(0o600) + (ssh / "id_ed25519.pub").write_text(fleet["public"] + "\n") + result = execute(fleet, "ssh-mesh", DOTFILES_DIR=str(root), UNITS_NO_SECRETS="1", + UNITS_CHECK="1" if check else "0") + if excluded: + assert result.returncode == 0, result.stdout + result.stderr + assert not (fleet["home"] / "ssh-calls").exists() + elif check: + assert result.returncode != 0 and "online SSH mesh targets: remote" in result.stdout + assert not (fleet["home"] / "ssh-calls").exists() + else: + assert result.returncode == 0, result.stdout + result.stderr + assert "alice@peer" in (fleet["home"] / "ssh-calls").read_text() + + +def test_group_force_push_allows_explicit_identity_rotation(fleet): + fleet["variables"]["ssh_keys_force_push"] = True + (fleet["data"] / "all.yml.json").write_text(json.dumps(fleet["variables"])) + ssh = fleet["home"] / ".ssh" + ssh.mkdir() + (ssh / "id_ed25519").write_text(fleet["other_private"]) + (ssh / "id_ed25519").chmod(0o600) + (ssh / "id_ed25519.pub").write_text(fleet["other_public"] + "\n") + result = execute(fleet) + assert result.returncode == 0, result.stdout + result.stderr + item = json.loads(base64.b64decode((fleet["home"] / "encoded-item").read_bytes())) + assert item["sshKey"]["publicKey"].split() == fleet["other_public"].split()[:2] + + +def test_group_force_authorized_removes_only_managed_obsolete_keys(fleet): + fleet["variables"]["ssh_keys_force_authorized"] = True + (fleet["data"] / "all.yml.json").write_text(json.dumps(fleet["variables"])) + ssh = fleet["home"] / ".ssh" + ssh.mkdir() + (ssh / "authorized_keys").write_text( + "# user-managed access\n# BEGIN dotfiles Bitwarden SSH keys\n" + + fleet["other_public"] + "\n# END dotfiles Bitwarden SSH keys\n" + ) + result = execute(fleet) + assert result.returncode == 0, result.stdout + result.stderr + content = (ssh / "authorized_keys").read_text() + assert fleet["other_public"] not in content + assert "# user-managed access" in content and fleet["public"] in content + + +def test_nested_inventory_group_vars_and_vps_are_respected(fleet): + inventory = {"all": {"hosts": {"newhost": {}}, "children": { + "fleet": {"vars": {"ansible_port": 2022, "tailnet_name": "peer-dns"}, + "children": {"workstations": {"hosts": {"remote": {}}}}}, + "vps": {"children": {"old_servers": {"hosts": {"retired": {}}}}}, + }}} + (fleet["data"] / "inventory.yml.json").write_text(json.dumps(inventory)) + root = Path(fleet["env"]["DOTFILES_DIR"]) + (root / "group_vars/fleet.yml").write_text(json.dumps({"ansible_user": "group-user", "ansible_port": 2222})) + (root / "host_vars/remote.yml").write_text(json.dumps({"ansible_user": "host-user"})) + result = execute(fleet) + assert result.returncode == 0, result.stdout + result.stderr + config = (fleet["home"] / ".ssh/config").read_text() + assert "Host remote\n HostName peer\n Port 2222\n User host-user\n" in config + result = execute(fleet, "ssh-mesh") + assert result.returncode == 0, result.stdout + result.stderr + calls = (fleet["home"] / "ssh-calls").read_text() + assert "host-user@peer" in calls and "retired" not in calls diff --git a/tests/test_sudo_sshd_check_units.py b/tests/test_sudo_sshd_check_units.py index d7c7bc22..fdfe5d17 100644 --- a/tests/test_sudo_sshd_check_units.py +++ b/tests/test_sudo_sshd_check_units.py @@ -4,6 +4,8 @@ import subprocess from pathlib import Path +import pytest + from conftest import write_stub from test_platform_native_units import platform_env # noqa: F401 @@ -68,6 +70,60 @@ def test_sudo_check_rejects_unreadable_comparison(platform_env): assert "Cannot read sudo policy" in result.stderr +def test_sudo_apply_uses_privileged_inspection_and_preserves_old_rules_on_error(platform_env): + root, system, bindir, env, log = platform_env + directory = system / "etc/sudoers.d" + dest = directory / "zz-james" + dest.write_text("james ALL=(ALL) NOPASSWD:ALL\n") + dest.chmod(0o440) + write_stub(bindir, "sudo", '''\ +[ "$1" = -n ] || exit 1 +shift +case "$1" in + /usr/sbin/visudo) exit 0 ;; + bash) shift; SUDO_INSPECT=1 exec bash "$@" ;; + *) echo "$*" >> "$LOG"; exec "$@" ;; +esac +''') + write_stub(bindir, "stat", '''\ +[ "${SUDO_INSPECT:-0}" = 1 ] || exit 2 +mode=$(/usr/bin/stat -c %a "$3") || exit 1 +echo "$mode:0:0" +''') + write_stub(bindir, "cmp", '''\ +[ "${SUDO_INSPECT:-0}" = 1 ] || exit 2 +[ "${CMP_ERROR:-0}" = 0 ] || exit 2 +exec /usr/bin/cmp "$@" +''') + command = ["bash", str(root / "units/sudo.sh")] + clean = subprocess.run(command, env=env, text=True, capture_output=True) + assert clean.returncode == 0, clean.stdout + clean.stderr + assert not any(line.startswith(("install ", "mv ")) for line in log.read_text().splitlines()) + old = directory / "99-james" + old.write_text("keep this rule\n") + failed = subprocess.run(command, env={**env, "CMP_ERROR": "1"}, text=True, capture_output=True) + assert failed.returncode != 0 and "Cannot read sudo policy" in failed.stderr + assert old.read_text() == "keep this rule\n" and dest.read_text() == "james ALL=(ALL) NOPASSWD:ALL\n" + + +@pytest.mark.parametrize("name", ["sudo", "sshd"]) +@pytest.mark.parametrize("check", [False, True]) +@pytest.mark.parametrize("bad_file", ["directory", "dangling-link"]) +def test_access_units_fail_closed_on_unreadable_host_configuration(platform_env, name, check, bad_file): + root, system, _, env, log = platform_env + host_file = root / "host_vars/laptop.yml" + if bad_file == "directory": + host_file.mkdir() + else: + host_file.symlink_to("missing-host-vars") + before = snapshot(system) + result = subprocess.run(["bash", str(root / "units" / f"{name}.sh")], + env={**env, "UNITS_CHECK": "1" if check else "0"}, + text=True, capture_output=True) + assert result.returncode != 0 and f"read {name} host configuration" in result.stdout + assert snapshot(system) == before and not log.exists() + + def test_sshd_check_reports_both_dropins_and_reload_without_writes(platform_env): root, system, _, env, log = platform_env (root / "host_vars/laptop.yml").write_text("sshd_harden: true\n") diff --git a/tests/test_syncthing_migration.py b/tests/test_syncthing_migration.py index 4d0b80ba..907a49e7 100644 --- a/tests/test_syncthing_migration.py +++ b/tests/test_syncthing_migration.py @@ -12,7 +12,7 @@ def syncthing_repo(tmp_path, host, variables): (repo / "host_vars").mkdir(parents=True) (repo / "host_vars" / f"{host}.yml").write_text(variables) (repo / "units").symlink_to(REPO / "units") - (repo / "inventory.yml").write_text("all:\n children:\n vps:\n hosts:\n matrix: {}\n telengana: {}\n") + (repo / "inventory.yml").write_text("all:\n children:\n vps:\n hosts:\n matrix:\n telengana:\n vps-host:\n") return repo @@ -29,25 +29,126 @@ def test_excluded_hosts_never_change_service(stub_env, tmp_path, host, variables assert unit.read_text() == "existing" and calls(env) == "" -@pytest.mark.skipif(shutil.which("yq") is None or Path("/home/linuxbrew/.linuxbrew/bin/syncthing").exists(), reason="requires yq and no installed syncthing") def test_missing_binary_disable_failure_is_reported(stub_env, tmp_path): bindir, env = stub_env + write_stub(bindir, "yq", 'echo false\n') write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\nexit 1\n') repo = syncthing_repo(tmp_path, "laptop", "") unit = Path(env["HOME"]) / ".config/systemd/user/syncthing.service" unit.parent.mkdir(parents=True) unit.write_text("existing") - result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop") + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", DOTFILES_SYSTEM_ROOT=str(tmp_path / "system")) assert result.returncode != 0 assert "disable --now syncthing.service" in calls(env) assert unit.read_text() == "existing" -@pytest.mark.skipif(shutil.which("yq") is None or Path("/home/linuxbrew/.linuxbrew/bin/syncthing").exists(), reason="requires yq and no installed syncthing") def test_missing_binary_stops_system_wide_user_service(stub_env, tmp_path): bindir, env = stub_env + write_stub(bindir, "yq", 'echo false\n') write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\nif [[ "$*" == *"show --property=LoadState"* ]]; then echo loaded; fi\n') repo = syncthing_repo(tmp_path, "laptop", "") - result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop") + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", DOTFILES_SYSTEM_ROOT=str(tmp_path / "system")) assert result.returncode == 0, result.stdout + result.stderr assert "disable --now syncthing.service" in calls(env) + + +def test_service_symlink_is_replaced_without_touching_target(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "yq", 'echo false\n') + write_stub(bindir, "systemctl", '''\ +echo "$*" >> "$HOME/calls" +case "$*" in + *--property=LoadState*) echo loaded ;; + *--property=UnitFileState*) echo enabled ;; + *--property=ActiveState*) echo active ;; +esac +''') + repo = syncthing_repo(tmp_path, "laptop", "") + system = tmp_path / "system" + binary = system / "home/linuxbrew/.linuxbrew/bin/syncthing" + binary.parent.mkdir(parents=True) + binary.touch() + unit = Path(env["HOME"]) / ".config/systemd/user/syncthing.service" + unit.parent.mkdir(parents=True) + target = tmp_path / "user-data" + target.write_text("do not overwrite\n") + target.chmod(0o600) + unit.symlink_to(target) + args = dict(DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", DOTFILES_SYSTEM_ROOT=str(system)) + check = run_unit("syncthing", env, **args, UNITS_CHECK="1") + assert check.returncode == 0 and "replace syncthing.service symlink" in check.stdout + assert unit.is_symlink() and target.read_text() == "do not overwrite\n" + result = run_unit("syncthing", env, **args) + assert result.returncode == 0, result.stdout + result.stderr + assert not unit.is_symlink() + assert unit.read_bytes() == (REPO / "units/syncthing/syncthing.service").read_bytes() + assert unit.stat().st_mode & 0o777 == 0o644 + assert target.read_text() == "do not overwrite\n" and target.stat().st_mode & 0o777 == 0o600 + + +def test_failed_reload_is_retried_with_unchanged_unit(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "yq", 'echo false\n') + write_stub(bindir, "systemctl", '''\ +echo "$*" >> "$HOME/calls" +if [ "$2" = daemon-reload ]; then exit "${FAIL_RELOAD:-0}"; fi +''') + repo = syncthing_repo(tmp_path, "laptop", "") + system = tmp_path / "system" + binary = system / "home/linuxbrew/.linuxbrew/bin/syncthing" + binary.parent.mkdir(parents=True) + binary.touch() + args = dict(DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", DOTFILES_SYSTEM_ROOT=str(system)) + first = run_unit("syncthing", env, **args, FAIL_RELOAD="1") + assert first.returncode != 0 and "enable --now" not in calls(env) + second = run_unit("syncthing", env, **args) + assert second.returncode == 0, second.stdout + second.stderr + assert calls(env).splitlines() == ["--user daemon-reload", "--user daemon-reload", "--user enable --now syncthing.service"] + + +@pytest.mark.parametrize("host,expected", [("vps-host", "excluded on VPS"), ("laptop", "binary and service not installed")]) +def test_inventory_gate_works_before_yq_is_installed(stub_env, tmp_path, host, expected): + bindir, env = stub_env + for name in ("bash", "dirname", "sed", "head", "id"): + (bindir / name).symlink_to(shutil.which(name)) + env["PATH"] = str(bindir) + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\necho not-found\n') + repo = syncthing_repo(tmp_path, host, "") + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE=host, + DOTFILES_SYSTEM_ROOT=str(tmp_path / "system")) + assert result.returncode == 0 and expected in result.stdout, result.stdout + result.stderr + if host == "vps-host": + assert calls(env) == "" + + +def test_inventory_gate_fails_closed_when_inventory_invalid(stub_env, tmp_path): + bindir, env = stub_env + write_stub(bindir, "yq", 'exit 1\n') + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = syncthing_repo(tmp_path, "laptop", "") + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop") + assert result.returncode != 0 and "check VPS membership" in result.stdout + assert calls(env) == "" + + +@pytest.mark.parametrize("check", [False, True]) +@pytest.mark.parametrize("bad_file", ["directory", "dangling-link"]) +def test_syncthing_fails_closed_on_unreadable_host_configuration(stub_env, tmp_path, check, bad_file): + bindir, env = stub_env + write_stub(bindir, "yq", 'echo yq >> "$HOME/calls"; echo false\n') + write_stub(bindir, "systemctl", 'echo "$*" >> "$HOME/calls"\n') + repo = syncthing_repo(tmp_path, "laptop", "") + host_file = repo / "host_vars/laptop.yml" + host_file.unlink() + if bad_file == "directory": + host_file.mkdir() + else: + host_file.symlink_to("missing-host-vars") + unit = Path(env["HOME"]) / ".config/systemd/user/syncthing.service" + unit.parent.mkdir(parents=True) + unit.write_text("existing") + result = run_unit("syncthing", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="laptop", + UNITS_CHECK="1" if check else "0") + assert result.returncode != 0 and "inspect host variables" in result.stdout + assert unit.read_text() == "existing" and calls(env) == "" diff --git a/tests/test_system_services_migration.py b/tests/test_system_services_migration.py index 3a166847..dc0b6082 100644 --- a/tests/test_system_services_migration.py +++ b/tests/test_system_services_migration.py @@ -49,6 +49,8 @@ def test_nonlaptop_daily_timer_and_user_service_are_idempotent(stub_env, tmp_pat assert "WantedBy=default.target" in service assert "graphical-session" not in service assert "apply-nosecrets" in service + assert "ExecStart=/usr/bin/env just " in service + assert "ExecStart=/usr/bin/env atuin daemon start --force" in (units / "atuin-daemon.service").read_text() assert "OnCalendar=daily" in timer and "RandomizedDelaySec=30min" in timer assert "--user enable --now dotfiles-update.timer" in calls(env) assert "--user enable --now atuin-daemon.service" in calls(env) @@ -59,6 +61,23 @@ def test_nonlaptop_daily_timer_and_user_service_are_idempotent(stub_env, tmp_pat assert "restart" not in calls(env) +def test_nonlaptop_removes_only_managed_old_graphical_target_link(stub_env, tmp_path): + bindir, env = stub_env + systemctl_stub(bindir) + program, env, _ = setup_system_services(tmp_path, env, "server", "", False) + units = Path(env["HOME"]) / ".config/systemd/user" + link = units / "graphical-session.target.wants/dotfiles-update.service" + link.parent.mkdir(parents=True) + link.symlink_to("../dotfiles-update.service") + result = run(program, env) + assert result.returncode == 0, result.stdout + result.stderr + assert not link.is_symlink() + link.symlink_to("/opt/custom/dotfiles-update.service") + result = run(program, env) + assert result.returncode == 0, result.stdout + result.stderr + assert link.is_symlink() and link.readlink() == Path("/opt/custom/dotfiles-update.service") + + def test_headless_timer_enables_linger_without_mutating_check_mode(stub_env, tmp_path): bindir, env = stub_env write_stub(bindir, "systemctl", '''echo "$*" >> "$HOME/calls" diff --git a/tests/test_visual_units_check.py b/tests/test_visual_units_check.py index e0e3ec22..675a5410 100644 --- a/tests/test_visual_units_check.py +++ b/tests/test_visual_units_check.py @@ -68,7 +68,9 @@ def account_with_icon(text): 'get org.gnome.desktop.peripherals.touchpad tap-to-click') echo "${CHECK_TAP_TO_CLICK:-false}" ;; 'get org.gnome.Ptyxis default-profile-uuid') echo "'profile'" ;; 'get org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/profile/ custom-command') echo "'/custom/brew/bin/fish'" ;; + 'get org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/profile/ use-custom-command') echo "${CHECK_CUSTOM_ENABLED:-true}" ;; 'get org.gnome.Ptyxis font-name') echo "'JetBrainsMono Nerd Font 12'" ;; + 'get org.gnome.Ptyxis use-system-font') echo "${CHECK_SYSTEM_FONT:-false}" ;; 'get org.gnome.shell enabled-extensions') echo '@as []' ;; 'get org.gnome.shell favorite-apps') if [ -n "${CHECK_DOCK:-}" ]; then echo "$CHECK_DOCK"; else echo "['org.mozilla.firefox.desktop']"; fi ;; 'get org.gnome.desktop.background picture-uri'|'get org.gnome.desktop.background picture-uri-dark') echo "''" ;; diff --git a/tests/test_zen_browser_unit.py b/tests/test_zen_browser_unit.py index 83618870..654187cf 100644 --- a/tests/test_zen_browser_unit.py +++ b/tests/test_zen_browser_unit.py @@ -210,3 +210,13 @@ def test_dock_bus_falls_back_to_inherited_runtime(stub_env): result = run_unit("zen-browser", env, XDG_RUNTIME_DIR="/custom/runtime") assert result.returncode == 0 assert "DBUS=unix:path=/custom/runtime/bus RUNTIME=/custom/runtime" in calls(env) + + +def test_empty_gnome_favorites_are_populated(stub_env): + env = setup(stub_env) + dock = Path(env["HOME"]) / "dock" + dock.write_text("@as []\n") + result = run_unit("zen-browser", env) + assert result.returncode == 0, result.stdout + result.stderr + assert dock.read_text().strip() == "['app.zen_browser.zen.desktop']" + assert "gsettings set" in calls(env) diff --git a/units/apk-packages.conf b/units/apk-packages.conf index 1bc978a3..2abb2f39 100644 --- a/units/apk-packages.conf +++ b/units/apk-packages.conf @@ -1,6 +1,6 @@ # The old role's yq is Alpine's yq-go (Mike Farah); core bootstrap is # independent of legacy packages that may not exist on a given release. -APK_BOOTSTRAP=(chezmoi yq-go python3 py3-yaml) +APK_BOOTSTRAP=(chezmoi yq-go python3 py3-yaml just) APK_PACKAGES=( bash fish zsh git git-lfs delta vim neovim tmux openssh-client curl wget jq ripgrep fd fzf starship bat exa direnv zoxide atuin shellcheck shfmt diff --git a/units/atuin.sh b/units/atuin.sh index e8b662fd..bbfd1d40 100755 --- a/units/atuin.sh +++ b/units/atuin.sh @@ -28,13 +28,13 @@ logged_in() { } atuin_login() { - local s item u p k version key_path line local_key + local s item u p k version key_path line local_key info s="$(bw_session)" if [ -z "$s" ]; then note "PARTIAL: atuin not logged in; vault locked (bw unlock, then just apply-tags atuin)" return 0 fi - item="$(BW_SESSION="$s" bw get item atuin.sh 2>/dev/null)" + item="$(BW_SESSION="$s" bw get item atuin.sh 2>/dev/null)" || item="" u="$(printf '%s' "$item" | jq -r '.login.username // empty' 2>/dev/null)" p="$(printf '%s' "$item" | jq -r '.login.password // empty' 2>/dev/null)" k="$(printf '%s' "$item" | jq -r '.fields[]? | select(.name=="key") | .value' 2>/dev/null | head -n1)" @@ -55,9 +55,14 @@ atuin_login() { # Let Atuin resolve its configured key path, including the newer data layout. key_path="" + if ! info="$(atuin info 2>/dev/null)"; then + note "PARTIAL: cannot determine the Atuin key path; inspect the local data and log in interactively" + allow_import=0 + return 0 + fi while IFS= read -r line; do case "$line" in 'key path: '*) key_path="${line#key path: }" ;; esac - done <<< "$(atuin info 2>/dev/null)" + done <<< "$info" if [ -z "$key_path" ]; then note "PARTIAL: cannot determine the Atuin key path; inspect the local data and log in interactively" allow_import=0 @@ -76,7 +81,10 @@ atuin_login() { fi fi # Keep both the history and any existing key; Atuin writes a missing key. - ATUIN_PASSWORD="$p" ATUIN_ENCRYPTION_KEY="$k" atuin login -u "$u" &2 rc=1 else - install -d -m 0755 "$HOME/.local/share/backgrounds/bluefin" "$HOME/.local/share/icons/hicolor" "$HOME/.local/share/ublue-os" \ - && cp -a "$tmp/backgrounds/." "$HOME/.local/share/backgrounds/bluefin/" \ - && cp -a "$tmp/icons/." "$HOME/.local/share/icons/hicolor/" \ - && cp -a "$tmp/ublue-os/." "$HOME/.local/share/ublue-os/" \ - && python3 - "$HOME/.local/share/backgrounds/bluefin" <<'PY' || rc=1 + python3 - "$tmp/backgrounds" "$HOME/.local/share/backgrounds/bluefin" <<'PY' || rc=1 import os import pathlib import sys +from xml.sax.saxutils import escape root = pathlib.Path(sys.argv[1]) old = b'/usr/share/backgrounds/bluefin/' -new = os.fsencode(str(root) + '/') +new = os.fsencode(escape(sys.argv[2] + '/', {'"': '"', "'": '''})) for path in root.rglob('*.xml'): + if path.is_symlink(): + continue data = path.read_bytes() updated = data.replace(old, new) if updated != data: path.write_bytes(updated) PY + if [ "$rc" = 0 ]; then + install -d -m 0755 "$HOME/.local/share/backgrounds/bluefin" "$HOME/.local/share/icons/hicolor" "$HOME/.local/share/ublue-os" \ + && cp -a "$tmp/backgrounds/." "$HOME/.local/share/backgrounds/bluefin/" \ + && cp -a "$tmp/icons/." "$HOME/.local/share/icons/hicolor/" \ + && cp -a "$tmp/ublue-os/." "$HOME/.local/share/ublue-os/" || rc=1 + fi DBUS_SESSION_BUS_ADDRESS="$bus" python3 "$assets/files/bluefin-sync.py" "$tmp" || rc=1 fi fi @@ -288,7 +301,13 @@ bus_value = bus.replace('%', '%%').replace('\\', '\\\\').replace('"', '\\"') assignment = 'DBUS_SESSION_BUS_ADDRESS=' + bus_value if any(c.isspace() or c in '\\"' for c in bus): assignment = '"' + assignment + '"' -for token, value in (("{{ ansible_facts['env']['HOME'] }}", home), +executable = home + '/.local/bin/bluefin-wallpaper-rotate.py' +if any(ord(c) < 32 or ord(c) == 127 for c in executable): + raise ValueError('invalid wallpaper executable path') +executable_value = executable.replace('%', '%%').replace('\\', '\\\\').replace('"', '\\"') +if any(c.isspace() or c in '\\"' for c in executable): + executable_value = '"' + executable_value + '"' +for token, value in (("{{ ansible_facts['env']['HOME'] }}/.local/bin/bluefin-wallpaper-rotate.py", executable_value), ('Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ user_uid_cmd.stdout | trim }}/bus', 'Environment=' + assignment)): if token not in text: diff --git a/units/bluefin/files/bluefin-sync.py b/units/bluefin/files/bluefin-sync.py index 7a2ea6f1..8db66a32 100644 --- a/units/bluefin/files/bluefin-sync.py +++ b/units/bluefin/files/bluefin-sync.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +from pathlib import Path import subprocess import configparser import re @@ -32,6 +33,7 @@ def apply_gschema_override(file_path): # Redirect system background paths to local ones if 'usr/share/backgrounds/bluefin' in val: local_path = os.path.expanduser('~/.local/share/backgrounds/bluefin') + val = val.replace('file:///usr/share/backgrounds/bluefin', Path(local_path).as_uri()) val = val.replace('/usr/share/backgrounds/bluefin', local_path) print(f" Redirected to local path: {val}") diff --git a/units/bluefin/files/bluefin-wallpaper-rotate.py b/units/bluefin/files/bluefin-wallpaper-rotate.py index ccddfc33..3bbb1861 100644 --- a/units/bluefin/files/bluefin-wallpaper-rotate.py +++ b/units/bluefin/files/bluefin-wallpaper-rotate.py @@ -1,24 +1,32 @@ #!/usr/bin/env python3 import datetime +from pathlib import Path import subprocess -import os +import sys + +SYSTEM_BACKGROUNDS = Path('/usr/share/backgrounds/bluefin') def set_wallpaper(): month = datetime.datetime.now().strftime("%m") # Bluefin wallpapers are named 01-bluefin.xml, 02-bluefin.xml, etc. wallpaper_name = f"{month}-bluefin.xml" - local_path = os.path.expanduser(f"~/.local/share/backgrounds/bluefin/{wallpaper_name}") - - if not os.path.exists(local_path): + local_path = Path.home() / '.local/share/backgrounds/bluefin' / wallpaper_name + if not local_path.exists(): + local_path = SYSTEM_BACKGROUNDS / wallpaper_name + + if not local_path.exists(): print(f"Wallpaper {local_path} not found, skipping rotation.") - return + return 0 - uri = f"file://{local_path}" + uri = local_path.as_uri() print(f"Rotating wallpaper to {uri}") - - # Set both light and dark variants - subprocess.run(['gsettings', 'set', 'org.gnome.desktop.background', 'picture-uri', uri]) - subprocess.run(['gsettings', 'set', 'org.gnome.desktop.background', 'picture-uri-dark', uri]) + + # Attempt both variants, but report a failure if either could not be set. + failed = False + for key in ('picture-uri', 'picture-uri-dark'): + result = subprocess.run(['gsettings', 'set', 'org.gnome.desktop.background', key, uri]) + failed |= result.returncode != 0 + return int(failed) if __name__ == "__main__": - set_wallpaper() + sys.exit(set_wallpaper()) diff --git a/units/bluefin/files/install-extensions.py b/units/bluefin/files/install-extensions.py index 0bd96753..979353b8 100644 --- a/units/bluefin/files/install-extensions.py +++ b/units/bluefin/files/install-extensions.py @@ -25,6 +25,13 @@ def install_ext(uuid): subprocess.run(['gnome-extensions', 'enable', uuid]) return + # An unrecognized extension may be incompatible or waiting for a Shell + # restart. Do not overwrite an existing (possibly customized) installation. + dest = os.path.join(USER_EXT_DIR, uuid) + if os.path.lexists(dest): + print(f"Extension {uuid} is not recognized; preserving existing installation at {dest}.") + return + print(f"Installing {uuid}...") try: # Fetch extension metadata to find the latest version diff --git a/units/browser-fxa.sh b/units/browser-fxa.sh index 65bf54da..54e7d740 100755 --- a/units/browser-fxa.sh +++ b/units/browser-fxa.sh @@ -109,7 +109,9 @@ spec.loader.exec_module(module) print(module.resolve_profile_dir()) ' "$src" 2>/dev/null)" || { note "FAIL inspect Firefox Accounts default profile"; return 1; } [ -d "$profile" ] || { note "FAIL Firefox Accounts default profile does not exist"; return 1; } - if [ -e "$profile/signedInUser.json" ]; then + if [ -L "$profile/signedInUser.json" ]; then + note "FAIL Firefox Accounts token is a symlink"; return 1 + elif [ -e "$profile/signedInUser.json" ]; then mode="$(stat -Lc %a "$profile/signedInUser.json")" || { note "FAIL inspect Firefox Accounts token mode"; return 1; } [ "$mode" = 600 ] || note "WOULD restrict Firefox Accounts token permissions" account_state="$(python3 -B -c ' @@ -117,16 +119,18 @@ import json, sys try: with open(sys.argv[1]) as f: data = json.load(f) - print("verified" if data.get("accountData", {}).get("verified") else "unverified") -except json.JSONDecodeError: - print("unverified") + account = data.get("accountData") if isinstance(data, dict) else None + print("verified" if isinstance(account, dict) and account.get("verified") else "existing") +except (json.JSONDecodeError, UnicodeDecodeError): + print("existing") ' "$profile/signedInUser.json" 2>/dev/null)" || { note "FAIL inspect Firefox Accounts token"; return 1; } else - account_state=unverified + account_state=absent fi case "$account_state" in verified) note "Firefox Accounts profile is verified" ;; - unverified) note "WOULD attempt Firefox Accounts sign-in if credentials are available" ;; + existing) note "Firefox Accounts data exists but is not verified; it will be left untouched" ;; + absent) note "WOULD attempt Firefox Accounts sign-in if credentials are available" ;; *) note "FAIL inspect Firefox Accounts token"; return 1 ;; esac note "Firefox Accounts credentials were not inspected" @@ -144,7 +148,16 @@ install_pyfxa() { } step "Firefox Accounts venv" create_venv step "Firefox Accounts PyFxA" install_pyfxa -step "Firefox Accounts sign-in script" install_if_changed "$DOTFILES_DIR/units/browser-fxa/fxa_signin.py" "$HOME/.local/bin/fxa_signin.py" 0700 +helper_installed=0 +install_helper() { + install_if_changed "$DOTFILES_DIR/units/browser-fxa/fxa_signin.py" "$HOME/.local/bin/fxa_signin.py" 0700 || return 1 + helper_installed=1 +} +step "Firefox Accounts sign-in script" install_helper +if [ "$helper_installed" != 1 ]; then + note "Firefox Accounts sign-in skipped: helper installation failed" + finish +fi s="$(bw_session)" if [ -z "$s" ]; then @@ -196,7 +209,10 @@ spec.loader.exec_module(module) print(module.resolve_profile_dir()) ')" || return 1 [ -d "$profile" ] || { note "Firefox Accounts default profile does not exist"; return 1; } - # The upstream script opens existing files in place; protect existing and new tokens. + # Never follow a token symlink, including a dangling one. + if [ -L "$profile/signedInUser.json" ]; then + note "Firefox Accounts token is a symlink; leaving it untouched"; return 1 + fi if [ -e "$profile/signedInUser.json" ]; then chmod 600 "$profile/signedInUser.json" || return 1; fi ( umask 077 FXA_EMAIL="$email" FXA_PASSWORD="$password" FXA_TOTP="$totp" BROWSER_TYPE="$browser" \ diff --git a/units/browser-fxa/fxa_signin.py b/units/browser-fxa/fxa_signin.py index 7218da0a..54fb258d 100644 --- a/units/browser-fxa/fxa_signin.py +++ b/units/browser-fxa/fxa_signin.py @@ -10,13 +10,14 @@ BROWSER_TYPE - "zen" (default), "firefox", or "firefox-flatpak" PROFILE_DIR - Override profile directory directly (skips auto-detection) -Idempotent: exits 0 without changes if signedInUser.json already exists and is verified. +Existing account data is never replaced; verified accounts are left signed in. """ import sys import os import json import configparser +import tempfile BROWSER_BASES = { "zen": os.path.expanduser("~/.var/app/app.zen_browser.zen/.zen"), @@ -86,16 +87,20 @@ def main(): signed_in_path = os.path.join(profile_dir, "signedInUser.json") - if os.path.exists(signed_in_path): + if os.path.lexists(signed_in_path): + if os.path.islink(signed_in_path): + print("ERROR: Existing account data is a symlink; leaving it untouched", file=sys.stderr) + sys.exit(1) try: with open(signed_in_path) as f: data = json.load(f) - if data.get("accountData", {}).get("verified"): - existing_email = data["accountData"].get("email", "unknown") - print(f"Already signed in as {existing_email} — skipping", file=sys.stderr) + if isinstance(data, dict) and isinstance(data.get("accountData"), dict) and data["accountData"].get("verified"): + print("Already signed in — skipping", file=sys.stderr) sys.exit(0) - except (json.JSONDecodeError, KeyError): - pass # Malformed — overwrite it + except (json.JSONDecodeError, UnicodeDecodeError): + pass + print("ERROR: Existing account data is not verified; leaving it untouched. Sign in manually.", file=sys.stderr) + sys.exit(1) try: import fxa.core @@ -119,16 +124,27 @@ def main(): token = token.hex() os.makedirs(profile_dir, exist_ok=True) - with open(signed_in_path, "w") as f: - json.dump({ - "version": 1, - "accountData": { - "email": email, - "sessionToken": token, - "uid": session.uid, - "verified": True, - }, - }, f) + # Publish a complete private file only if no browser process has created one + # since the check above. Never truncate another account's data. + fd, temporary = tempfile.mkstemp(prefix=".signedInUser.", dir=profile_dir) + try: + with os.fdopen(fd, "w") as f: + json.dump({ + "version": 1, + "accountData": { + "email": email, + "sessionToken": token, + "uid": session.uid, + "verified": True, + }, + }, f) + try: + os.link(temporary, signed_in_path) + except FileExistsError: + print("ERROR: Account data appeared during sign-in; leaving it untouched", file=sys.stderr) + sys.exit(1) + finally: + os.unlink(temporary) print(f"Wrote signedInUser.json for {email} — restart the browser to begin syncing") diff --git a/units/bst-dashboard.sh b/units/bst-dashboard.sh index 42cb7b8c..cb9800cb 100755 --- a/units/bst-dashboard.sh +++ b/units/bst-dashboard.sh @@ -27,15 +27,16 @@ tmp=$(mktemp) || { step "create dashboard temporary file" false; finish; } trap 'rm -f "$tmp"' EXIT unit="$HOME/.config/systemd/user/bst-dashboard.service" changed=0 +content_changed=0 if [ "${UNITS_CHECK:-0}" = 1 ]; then if python3 "$DOTFILES_DIR/units/service-templates.py" dashboard "$DOTFILES_DIR/units/bst-dashboard/bst-dashboard.service.j2" "$dashboard_dir" "$image" > "$tmp"; then if [ ! -e "$unit" ] && [ ! -L "$unit" ]; then - note "would deploy dashboard user service and reload user systemd" + note "would deploy dashboard user service, reload user systemd, and start or restart dashboard" elif mode=$(stat -c %a "$unit"); then cmp -s "$tmp" "$unit"; rc=$? case "$rc" in 0) [ "$mode" = 644 ] && note "ok dashboard user service" || note "would set dashboard service permissions and reload user systemd" ;; - 1) note "would deploy dashboard user service and reload user systemd" ;; + 1) note "would deploy dashboard user service, reload user systemd, and start or restart dashboard" ;; *) step "compare dashboard user service" false ;; esac else step "inspect dashboard user service" false; fi @@ -57,11 +58,21 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then finish fi if python3 "$DOTFILES_DIR/units/service-templates.py" dashboard "$DOTFILES_DIR/units/bst-dashboard/bst-dashboard.service.j2" "$dashboard_dir" "$image" > "$tmp"; then - if [ ! -f "$unit" ] || ! cmp -s "$tmp" "$unit" || [ "$(stat -c %a "$unit" 2>/dev/null)" != 644 ]; then changed=1; fi + if [ ! -f "$unit" ] || ! cmp -s "$tmp" "$unit"; then content_changed=1; changed=1; fi + if [ "$(stat -c %a "$unit" 2>/dev/null)" != 644 ]; then changed=1; fi if install_if_changed "$tmp" "$unit" 0644; then note "ok deploy dashboard user service" if [ "$changed" = 0 ] || systemctl --user daemon-reload; then - step "enable and start dashboard" systemctl --user enable --now bst-dashboard.service + if [ "$content_changed" = 1 ]; then + if systemctl --user enable bst-dashboard.service; then + note "ok enable dashboard" + step "start or restart dashboard" systemctl --user restart bst-dashboard.service + else + step "enable dashboard" false + fi + else + step "enable and start dashboard" systemctl --user enable --now bst-dashboard.service + fi else step "reload dashboard user systemd" false fi diff --git a/units/chezmoi.sh b/units/chezmoi.sh index 4c54a796..679e7714 100755 --- a/units/chezmoi.sh +++ b/units/chezmoi.sh @@ -6,6 +6,14 @@ set -uo pipefail # shellcheck source=units/package-config.bash . "$(dirname "${BASH_SOURCE[0]}")/package-config.bash" +# Tag-only Atuin applies must not overwrite unrelated dotfiles or run hooks. +scope_args=() +case "${DOTFILES_CHEZMOI_SCOPE:-}" in + '') ;; + atuin) scope_args=(--exclude=scripts --parent-dirs --recursive=false "$HOME/.config/atuin/config.toml") ;; + *) note "FAIL: invalid chezmoi scope"; exit 1 ;; +esac + resolve_desktop() { local hint="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" case "$hint" in @@ -39,7 +47,7 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then if ! chezmoi --source "$DOTFILES_DIR" \ --config <(printf '[data]\nmachine = %s\nis_desktop = %s\n' "$machine_json" "$desktop_toml") --config-format toml \ --persistent-state "${XDG_CONFIG_HOME:-$HOME/.config}/chezmoi/chezmoistate.boltdb" \ - --dry-run --force --no-tty --no-pager status &2 + return 1 fi } add_remotes() { @@ -140,11 +141,22 @@ install_user_apps() { return "$rc" } install_wrappers() { - local tmp name rc=0 + local tmp name path rc=0 tmp="$(mktemp)" || return 1 printf '#!/usr/bin/env bash\nexec flatpak run org.chromium.Chromium "$@"\n' > "$tmp" || { rm -f "$tmp"; return 1; } for name in google-chrome google-chrome-stable chromium-browser chromium; do - install_if_changed "$tmp" "$HOME/.local/bin/$name" 0755 || rc=1 + path="$HOME/.local/bin/$name" + if [ -L "$path" ]; then + note "PRESERVED: existing Chromium launcher $name" + elif [ -e "$path" ]; then + if [ -f "$path" ] && cmp -s "$tmp" "$path"; then + chmod 0755 "$path" || rc=1 + else + note "PRESERVED: existing Chromium launcher $name" + fi + else + install_if_changed "$tmp" "$path" 0755 || rc=1 + fi done rm -f "$tmp" return "$rc" @@ -180,13 +192,16 @@ check_wrappers() { local name path mode rc for name in google-chrome google-chrome-stable chromium-browser chromium; do path="$HOME/.local/bin/$name" - if [ ! -e "$path" ] && [ ! -L "$path" ]; then + if [ -L "$path" ]; then + note "PRESERVED: existing Chromium launcher $name" + continue + elif [ ! -e "$path" ]; then note "CHANGE: install Chromium wrapper $name" elif [ ! -f "$path" ] || [ ! -r "$path" ]; then - echo "Cannot inspect Chromium wrapper $name" >&2; UNIT_FAILED=1 + note "PRESERVED: existing Chromium launcher $name" else cmp -s "$path" <(printf '#!/usr/bin/env bash\nexec flatpak run org.chromium.Chromium "$@"\n'); rc=$? - if [ "$rc" = 1 ]; then note "CHANGE: update Chromium wrapper $name"; fi + if [ "$rc" = 1 ]; then note "PRESERVED: existing Chromium launcher $name"; continue; fi if [ "$rc" -gt 1 ]; then echo "Cannot compare Chromium wrapper $name" >&2; UNIT_FAILED=1; continue; fi mode="$(stat -L -c %a "$path" 2>/dev/null || stat -L -f %Lp "$path" 2>/dev/null)" || { echo "Cannot inspect Chromium wrapper permissions: $name" >&2; UNIT_FAILED=1; continue; @@ -197,12 +212,10 @@ check_wrappers() { } check_flatpak() { - local system_remotes system_apps user_remote_names user_installed name url app remote origin + local system_remotes system_apps user_remote_names user_installed name url app remote origin system_store_ok=1 local -A planned_user_origins=() check_wrappers - if [ -L /var/lib/flatpak ] && [ ! -e /var/lib/flatpak ]; then - note 'CHANGE: repair dangling Flatpak system store' - fi + if ! check_store; then UNIT_FAILED=1; system_store_ok=0; fi if ! command -v flatpak >/dev/null 2>&1; then echo 'Cannot inspect Flatpak: executable unavailable' >&2; UNIT_FAILED=1; return fi @@ -214,6 +227,7 @@ check_flatpak() { echo 'Cannot inspect Flatpak paths hidden by the sandbox' >&2; UNIT_FAILED=1; return ;; esac done + if [ "$system_store_ok" = 1 ]; then if check_flatpak_list remotes --system --columns=name; then system_remotes="$CHECK_LIST" while IFS=$'\t' read -r name url; do @@ -236,6 +250,9 @@ check_flatpak() { else echo 'Cannot inspect system Flatpak applications in the read-only sandbox' >&2; UNIT_FAILED=1 fi + else + note 'SKIPPED: system Flatpak inspection (store unavailable)' + fi if ! load_user_config >/dev/null 2>&1; then echo 'Cannot inspect user Flatpak configuration' >&2; UNIT_FAILED=1; return fi @@ -275,7 +292,9 @@ check_flatpak() { echo 'Cannot inspect user Flatpak applications in the read-only sandbox' >&2; UNIT_FAILED=1 fi fi - case "${DOTFILES_UPGRADE:-0}" in 1|true) note 'PLANNED: Flatpak system update requested; available updates not inspected offline' ;; esac + if [ "$system_store_ok" = 1 ]; then + case "${DOTFILES_UPGRADE:-0}" in 1|true) note 'PLANNED: Flatpak system update requested; available updates not inspected offline' ;; esac + fi } if [ "${UNITS_CHECK:-0}" = 1 ]; then @@ -284,15 +303,20 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then fi if command -v flatpak >/dev/null 2>&1; then - step 'Flatpak system store' fix_store - step 'Flatpak remotes' add_remotes - if get_installed; then - step 'unwanted system Flatpaks' remove_unwanted - step 'system Flatpak applications' install_missing + if check_store; then + note 'ok Flatpak system store' + step 'Flatpak remotes' add_remotes + if get_installed; then + step 'unwanted system Flatpaks' remove_unwanted + step 'system Flatpak applications' install_missing + else + step 'list system Flatpaks' false + fi + case "${DOTFILES_UPGRADE:-0}" in 1|true) soft 'Flatpak update' package_as_root flatpak update --system -y ;; esac else - step 'list system Flatpaks' false + step 'Flatpak system store' false + note 'SKIPPED: system Flatpak operations (store unavailable)' fi - case "${DOTFILES_UPGRADE:-0}" in 1|true) soft 'Flatpak update' package_as_root flatpak update --system -y ;; esac if load_user_config; then [ -z "$user_remotes" ] || step 'user Flatpak remotes' add_user_remotes [ -z "$user_apps" ] || step 'user Flatpak applications' install_user_apps diff --git a/units/fonts.sh b/units/fonts.sh index c6bb31db..6736309c 100755 --- a/units/fonts.sh +++ b/units/fonts.sh @@ -8,6 +8,7 @@ set -uo pipefail dir="$HOME/.local/share/fonts" font="$dir/JetBrainsMono/JetBrainsMonoNerdFont-Regular.ttf" +pending="$dir/.dotfiles-jetbrainsmono-cache-pending" if [ "${UNITS_CHECK:-0}" = 1 ]; then if { [ -e "$dir" ] || [ -L "$dir" ]; } && [ ! -d "$dir" ]; then @@ -23,15 +24,17 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then fi if [ "$desktop" = 0 ]; then note "SKIPPED: not a desktop; no font install" - elif [ -f "$font" ]; then - note "ok JetBrains Mono Nerd Font already installed" - else + elif [ ! -f "$font" ]; then note "CHANGE: download and install JetBrains Mono Nerd Font; refresh font cache" + elif [ -e "$pending" ]; then + note "CHANGE: retry JetBrains Mono font cache refresh" + else + note "ok JetBrains Mono Nerd Font already installed" fi exit 0 fi -step "font directory" mkdir -p "$dir" +step "font directory" sh -c 'mkdir -p "$1" && chmod 0755 "$1"' sh "$dir" if platform_desktop; then : else @@ -40,20 +43,31 @@ else else step "desktop membership" false; fi finish fi -if [ -f "$font" ]; then +if [ ! -f "$font" ]; then + install_font() { + local tmp rc=0 + tmp="$(mktemp -d)" || return 1 + # Extract before touching the destination: a failed unzip must not leave the + # Regular font behind as a false success marker on the next apply. + curl -fsSL https://github.com/ryanoasis/nerd-fonts/releases/latest/download/JetBrainsMono.zip -o "$tmp/font.zip" \ + && mkdir "$tmp/extracted" \ + && unzip -oq "$tmp/font.zip" -d "$tmp/extracted" \ + && [ -f "$tmp/extracted/JetBrainsMonoNerdFont-Regular.ttf" ] \ + && : > "$pending" \ + && mkdir -p "$dir/JetBrainsMono" \ + && chmod 0755 "$dir/JetBrainsMono" \ + && cp -a "$tmp/extracted/." "$dir/JetBrainsMono/" || rc=1 + if [ "$rc" != 0 ] && [ -e "$pending" ]; then rm -f "$font"; fi + rm -rf "$tmp" + return "$rc" + } + step "install JetBrains Mono Nerd Font" install_font +else note "ok JetBrains Mono Nerd Font already installed" - finish fi -install_font() { - local tmp rc=0 - tmp="$(mktemp -d)" || return 1 - curl -fsSL https://github.com/ryanoasis/nerd-fonts/releases/latest/download/JetBrainsMono.zip -o "$tmp/font.zip" \ - && mkdir -p "$dir/JetBrainsMono" \ - && unzip -oq "$tmp/font.zip" -d "$dir/JetBrainsMono" || rc=1 - rm -rf "$tmp" - return $rc -} -step "install JetBrains Mono Nerd Font" install_font -soft "font cache" fc-cache -f +if [ -f "$font" ] && [ -e "$pending" ]; then + refresh_font_cache() { fc-cache -f && rm -f "$pending"; } + step "font cache" refresh_font_cache +fi finish diff --git a/units/forgejo-registry.sh b/units/forgejo-registry.sh index ccb7a19b..abdf5326 100755 --- a/units/forgejo-registry.sh +++ b/units/forgejo-registry.sh @@ -31,13 +31,15 @@ fi registry_login() { local user pass - user="$(BW_SESSION="$s" bw get username "$FORGEJO_REGISTRY" 2>/dev/null)" - pass="$(BW_SESSION="$s" bw get password "$FORGEJO_REGISTRY" 2>/dev/null)" + user="$(BW_SESSION="$s" bw get username "$FORGEJO_REGISTRY" 2>/dev/null)" || user="" + pass="$(BW_SESSION="$s" bw get password "$FORGEJO_REGISTRY" 2>/dev/null)" || pass="" + user="${user#"${user%%[![:space:]]*}"}" + user="${user%"${user##*[![:space:]]}"}" if [ -z "$user" ] || [ -z "$pass" ]; then note "PARTIAL: Forgejo credential ($FORGEJO_REGISTRY) not found in Bitwarden — add a login item with that name holding the registry token" return 0 fi - printf '%s' "$pass" | podman login "$FORGEJO_REGISTRY" -u "$(printf '%s' "$user" | tr -d '[:space:]')" --password-stdin + printf '%s' "$pass" | podman login "$FORGEJO_REGISTRY" -u "$user" --password-stdin } step "podman login $FORGEJO_REGISTRY" registry_login finish diff --git a/units/github.sh b/units/github.sh index b8182a46..140a27e7 100755 --- a/units/github.sh +++ b/units/github.sh @@ -8,6 +8,35 @@ set -uo pipefail GITHUB_HOST_KEY="github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" pub="$HOME/.ssh/id_ed25519.pub" +# ssh-keygen resolves hashed and multi-host entries, unlike substring matching. +# Refuse conflicting ed25519 or revoked pinned keys rather than silently trusting both. +github_host_key_status() { + local path="$HOME/.ssh/known_hosts" entries rc host type body rest found=1 + [ ! -L "$path" ] || { note "FAIL refusing symlinked $path"; return 2; } + [ -e "$path" ] || return 1 + [ -f "$path" ] || { note "FAIL cannot inspect $path"; return 2; } + entries="$(ssh-keygen -F github.com -f "$path" 2>/dev/null)"; rc=$? + [ "$rc" -le 1 ] || { note "FAIL cannot inspect $path"; return 2; } + while read -r host type body rest; do + case "$host" in + ''|'#'*) continue ;; + @cert-authority) continue ;; + @revoked) + if [ "$body" = ssh-ed25519 ] && [ "${rest%% *}" = "${GITHUB_HOST_KEY##* }" ]; then + note "FAIL pinned GitHub host key is revoked in $path"; return 2 + fi + continue ;; + esac + [ "$type" = ssh-ed25519 ] || continue + if [ "$body" != "${GITHUB_HOST_KEY##* }" ]; then + note "FAIL conflicting GitHub ed25519 host key in $path; inspect it before proceeding" + return 2 + fi + found=0 + done <<< "$entries" + return "$found" +} + if ! command -v gh >/dev/null 2>&1; then note "SKIPPED: gh not installed (just packages)" exit 0 @@ -23,19 +52,11 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then elif [ "$mode" != 700 ]; then note "would set $dir mode 0700" fi - if [ ! -e "$path" ]; then - note "would add GitHub host key to $path" - return 0 - fi - if [ ! -f "$path" ]; then - note "FAIL cannot inspect $path" - return 1 - fi - grep -qF "$GITHUB_HOST_KEY" "$path" 2>/dev/null; rc=$? + github_host_key_status; rc=$? case "$rc" in 0) note "ok GitHub host key" ;; 1) note "would add GitHub host key to $path" ;; - *) note "FAIL cannot inspect $path"; return 1 ;; + *) return 1 ;; esac } step "check github known_hosts" check_known_hosts @@ -50,9 +71,14 @@ fi known_hosts() { + local rc install -d -m 0700 "$HOME/.ssh" || return 1 - grep -qF "$GITHUB_HOST_KEY" "$HOME/.ssh/known_hosts" 2>/dev/null && return 0 - printf '%s\n' "$GITHUB_HOST_KEY" >> "$HOME/.ssh/known_hosts" + github_host_key_status; rc=$? + case "$rc" in + 0) return 0 ;; + 1) printf '%s\n' "$GITHUB_HOST_KEY" >> "$HOME/.ssh/known_hosts" ;; + *) return 1 ;; + esac } gh_login() { @@ -63,7 +89,7 @@ gh_login() { note "PARTIAL: gh not logged in; vault locked (bw unlock, then just apply-tags github)" return 0 fi - tok="$(BW_SESSION="$s" bw get password github-token 2>/dev/null)" + tok="$(BW_SESSION="$s" bw get password github-token 2>/dev/null)" || tok="" if [ -z "$tok" ]; then note "PARTIAL: no 'github-token' item in Bitwarden; add a login item with your PAT as the password" return 0 diff --git a/units/gnome-avatar.py b/units/gnome-avatar.py index c50d63bc..b8458d28 100755 --- a/units/gnome-avatar.py +++ b/units/gnome-avatar.py @@ -6,6 +6,7 @@ from pathlib import Path import pwd import re +import stat import sys import tempfile @@ -87,7 +88,20 @@ def install_avatar(source): pass else: group = grp.getgrnam('gdm') - GDM_CACHE.mkdir(parents=True, exist_ok=True, mode=0o755) + GDM_CACHE.parent.mkdir(parents=True, exist_ok=True, mode=0o755) + try: + GDM_CACHE.mkdir(mode=0o755) + except FileExistsError: + if not GDM_CACHE.is_dir(): + raise + # Repair the root-owned 0755 cache created by earlier unit runs, + # without changing custom permissions, owners or symlink targets. + info = GDM_CACHE.lstat() + if (stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and gdm.pw_uid != 0 + and stat.S_IMODE(info.st_mode) == 0o755): + os.chown(GDM_CACHE, gdm.pw_uid, group.gr_gid) + else: + os.chown(GDM_CACHE, gdm.pw_uid, group.gr_gid) write_file(GDM_CACHE / 'james', data, (gdm.pw_uid, group.gr_gid)) try: account = ACCOUNT.read_text() diff --git a/units/gnome.sh b/units/gnome.sh index 404df9f7..18e43ce6 100755 --- a/units/gnome.sh +++ b/units/gnome.sh @@ -177,6 +177,13 @@ except KeyError: pass else: gid = grp.getgrnam('gdm').gr_gid + try: + info = avatar.GDM_CACHE.lstat() + if (stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and gdm.pw_uid != 0 + and stat.S_IMODE(info.st_mode) == 0o755): + print('WOULD repair GDM cache directory ownership') + except FileNotFoundError: + print('WOULD create GDM cache directory') try: info = (avatar.GDM_CACHE / 'james').stat() if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o7777 != 0o644 or (info.st_uid, info.st_gid) != (gdm.pw_uid, gid): @@ -214,9 +221,13 @@ def ini(text): def ptyxis(): uuid = ast.literal_eval(settings('org.gnome.Ptyxis', 'default-profile-uuid')) - current_cmd = ast.literal_eval(settings('org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/' + uuid + '/', 'custom-command')) + profile = 'org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/' + uuid + '/' + current_cmd = ast.literal_eval(settings(profile, 'custom-command')) + custom_enabled = settings(profile, 'use-custom-command') font = ast.literal_eval(settings('org.gnome.Ptyxis', 'font-name')) - if current_cmd != prefix + '/bin/fish' or font != 'JetBrainsMono Nerd Font 12': + system_font = settings('org.gnome.Ptyxis', 'use-system-font') + if (current_cmd != prefix + '/bin/fish' or custom_enabled != 'true' + or font != 'JetBrainsMono Nerd Font 12' or system_font != 'false'): print('WOULD configure Ptyxis profile') def extension_settings(): @@ -277,18 +288,27 @@ soft "disable touchpad tap-to-click" disable_tap_to_click configure_ptyxis() { - local uuid current_cmd current_font + local uuid profile current_cmd current_font custom_enabled system_font require_session || return 1 uuid="$(gsettings get org.gnome.Ptyxis default-profile-uuid)" || return 1 uuid="${uuid//\'/}" - current_cmd="$(gsettings get "org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${uuid}/" custom-command 2>/dev/null)" || current_cmd="" - current_font="$(gsettings get org.gnome.Ptyxis font-name 2>/dev/null)" || current_font="" + profile="org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${uuid}/" + current_cmd="$(gsettings get "$profile" custom-command)" || return 1 + custom_enabled="$(gsettings get "$profile" use-custom-command)" || return 1 + current_font="$(gsettings get org.gnome.Ptyxis font-name)" || return 1 + system_font="$(gsettings get org.gnome.Ptyxis use-system-font)" || return 1 current_cmd="${current_cmd//\'/}"; current_font="${current_font//\'/}" - if [ "$current_cmd" != "$brew_prefix/bin/fish" ] || [ "$current_font" != 'JetBrainsMono Nerd Font 12' ]; then - gsettings set "org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${uuid}/" use-custom-command true && - gsettings set "org.gnome.Ptyxis.Profile:/org/gnome/ptyxis/profiles/${uuid}/" custom-command "$brew_prefix/bin/fish" && - gsettings set org.gnome.Ptyxis use-system-font false && - gsettings set org.gnome.Ptyxis font-name 'JetBrainsMono Nerd Font 12' + if [ "$custom_enabled" != true ]; then + gsettings set "$profile" use-custom-command true || return 1 + fi + if [ "$current_cmd" != "$brew_prefix/bin/fish" ]; then + gsettings set "$profile" custom-command "$brew_prefix/bin/fish" || return 1 + fi + if [ "$system_font" != false ]; then + gsettings set org.gnome.Ptyxis use-system-font false || return 1 + fi + if [ "$current_font" != 'JetBrainsMono Nerd Font 12' ]; then + gsettings set org.gnome.Ptyxis font-name 'JetBrainsMono Nerd Font 12' || return 1 fi } if [ "$config_ok" = 1 ]; then soft "Ptyxis profile" configure_ptyxis; fi diff --git a/units/homebrew.sh b/units/homebrew.sh index 00466631..8ad29c57 100755 --- a/units/homebrew.sh +++ b/units/homebrew.sh @@ -36,6 +36,10 @@ load_config() { fi done done + brew_tap_inputs=("${core_taps[@]}" "${core_brews[@]}" "${core_tap_brews[@]}") + if [ "$PACKAGE_DESKTOP" = 1 ]; then + brew_tap_inputs+=("${desktop_taps[@]}" "${desktop_brews[@]}" "${extra_brews[@]}" "${desktop_casks[@]}") + fi } fix_owner() { @@ -119,7 +123,7 @@ deploy_brewfile() { tap_and_trust() { local pkg tap seen=' ' rc=0 - for pkg in "${core_taps[@]}" "${desktop_taps[@]}" "${core_brews[@]}" "${core_tap_brews[@]}" "${desktop_brews[@]}" "${extra_brews[@]}" "${desktop_casks[@]}"; do + for pkg in "${brew_tap_inputs[@]}"; do case "$pkg" in */*/*) tap="${pkg%/*}" ;; */*) tap="$pkg" ;; @@ -216,7 +220,7 @@ check_brew() { for line in $taps; do case "$line" in *[!a-zA-Z0-9_./-]*) echo 'Cannot inspect Homebrew taps' >&2; UNIT_FAILED=1; return ;; esac done - for pkg in "${core_taps[@]}" "${desktop_taps[@]}" "${core_brews[@]}" "${core_tap_brews[@]}" "${desktop_brews[@]}" "${extra_brews[@]}" "${desktop_casks[@]}"; do + for pkg in "${brew_tap_inputs[@]}"; do case "$pkg" in */*/*) tap="${pkg%/*}" ;; */*) tap="$pkg" ;; *) continue ;; esac case "$seen" in *" $tap "*) continue ;; esac seen="$seen$tap " diff --git a/units/kirocrew-sockets.py b/units/kirocrew-sockets.py index faeddfeb..09ba6fd5 100755 --- a/units/kirocrew-sockets.py +++ b/units/kirocrew-sockets.py @@ -1,13 +1,40 @@ #!/usr/bin/env python3 -"""Reject non-loopback listening sockets owned by the kirocrew process.""" +"""Reject non-loopback sockets belonging to Kiro Crew's systemd cgroup.""" +import argparse +from pathlib import Path +import re import sys def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cgroup") + parser.add_argument("--proc-root", type=Path, default=Path("/proc")) + args = parser.parse_args() + if args.cgroup is not None and (not args.cgroup.startswith("/") or args.cgroup == "/"): + print("invalid kirocrew service cgroup", file=sys.stderr) + return 1 unsafe = [] + owned = 0 for line in sys.stdin: - if '"kirocrew"' not in line: + belongs = args.cgroup is None and '"kirocrew"' in line + if args.cgroup is not None: + for pid in re.findall(r"\bpid=(\d+)", line): + try: + entries = (args.proc_root / pid / "cgroup").read_text().splitlines() + except FileNotFoundError: + # The process exited while ss was enumerating sockets. + continue + except OSError as error: + print(f"cannot inspect listener cgroup: {error}", file=sys.stderr) + return 1 + paths = [entry.split(":", 2)[-1] for entry in entries] + if any(path == args.cgroup or path.startswith(args.cgroup + "/") for path in paths): + belongs = True + break + if not belongs: continue + owned += 1 columns = line.split() if len(columns) < 4: unsafe.append(line.rstrip()) @@ -20,6 +47,9 @@ def main(): for line in unsafe: print(line, file=sys.stderr) return 1 + if args.cgroup is not None and not owned: + print('no listeners attributable to kirocrew; localhost-only binding cannot be verified', file=sys.stderr) + return 1 return 0 diff --git a/units/kirocrew.sh b/units/kirocrew.sh index 58881b37..50bcd81b 100755 --- a/units/kirocrew.sh +++ b/units/kirocrew.sh @@ -16,9 +16,23 @@ if [ ! -e "$HOME/.local/bin/kirocrew" ] || [ ! -e /etc/systemd/system/kirocrew.s note "SKIPPED: kirocrew binary or system service missing; install from https://github.com/kirodotdev/KiroCrew and run sudo ~/.local/bin/kirocrew service install" finish fi +check_listeners() { + local cgroup + if [ "${1:-}" = active ]; then + cgroup=$(as_root systemctl show kirocrew.service --property=ControlGroup --value) || return 1 + as_root ss -Htlnp | as_root python3 "$DOTFILES_DIR/units/kirocrew-sockets.py" --cgroup "$cgroup" + else + as_root ss -Htlnp | as_root python3 "$DOTFILES_DIR/units/kirocrew-sockets.py" + fi +} +safe_env_file() { + ! as_root test -L /etc/kirocrew/kirocrew.env && as_root test -f /etc/kirocrew/kirocrew.env +} if [ "${UNITS_CHECK:-0}" = 1 ]; then if [ "$EUID" != 0 ] && ! sudo -n true; then step "noninteractive root access" false; finish; fi - if mode=$(as_root stat -c '%a:%u:%g' /etc/kirocrew/kirocrew.env 2>/dev/null); then + if ! safe_env_file; then + step "inspect kirocrew environment (must be a regular file, not a symlink)" false + elif mode=$(as_root stat -c '%a:%u:%g' /etc/kirocrew/kirocrew.env 2>/dev/null); then [ "$mode" = '600:0:0' ] && note "ok kirocrew environment permissions" || note "would secure kirocrew environment ownership and permissions" else step "inspect kirocrew environment permissions" false; fi state=$(as_root systemctl show kirocrew.service --property=LoadState --property=ActiveState --property=UnitFileState 2>/dev/null); rc=$? @@ -32,13 +46,15 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then [[ "$state" == *'ActiveState=active'* ]] || note "would start kirocrew; its future listeners cannot be inspected without starting it" [[ "$state" == *'UnitFileState=enabled'* ]] || note "would enable kirocrew" fi - if as_root ss -Htlnp | python3 "$DOTFILES_DIR/units/kirocrew-sockets.py"; then - note "ok current kirocrew listeners are localhost-only" + if [[ "$state" == *'ActiveState=active'* ]]; then + step "inspect current kirocrew listeners" check_listeners active + elif check_listeners; then + note "current kirocrew-named listeners are localhost-only; inactive service binding is unverified" else step "inspect current kirocrew listeners" false; fi finish fi secure_env() { - as_root chown root:root /etc/kirocrew/kirocrew.env && as_root chmod 0600 /etc/kirocrew/kirocrew.env + safe_env_file && as_root chown root:root /etc/kirocrew/kirocrew.env && as_root chmod 0600 /etc/kirocrew/kirocrew.env } if secure_env; then note "ok secure kirocrew environment" @@ -46,8 +62,5 @@ if secure_env; then else step "secure kirocrew environment" false fi -check_listeners() { - as_root ss -Htlnp | python3 "$DOTFILES_DIR/units/kirocrew-sockets.py" -} -step "check kirocrew listens only on localhost" check_listeners +step "check kirocrew listens only on localhost" check_listeners active finish diff --git a/units/kube.sh b/units/kube.sh index e0194d0f..112ff703 100755 --- a/units/kube.sh +++ b/units/kube.sh @@ -72,7 +72,7 @@ step "kube dirs" install -d -m 0700 "$HOME/.kube" "$HOME/.talos" # fetch_note ITEM DEST LABEL: write the note to DEST when Bitwarden has it; prints a warning otherwise. fetch_note() { local item="$1" dest="$2" label="$3" val - val="$(BW_SESSION="$s" bw get notes "$item" 2>/dev/null)" + val="$(BW_SESSION="$s" bw get notes "$item" 2>/dev/null)" || val="" if [ -z "$val" ]; then note "warn: $label missing in BW (item $item)" return 0 @@ -83,12 +83,17 @@ fetch_note() { # The home kubeconfig is rewritten from Bitwarden each run, so the TLS pin is re-applied each time. pin_tls() { - command -v kubectl >/dev/null 2>&1 || { note "warn: kubectl missing; tls-server-name not pinned"; return 0; } - local c rc=0 - while read -r c; do + command -v kubectl >/dev/null 2>&1 || { note "FAIL kubectl missing; tls-server-name not pinned"; return 1; } + local c clusters header=1 rc=0 + if ! clusters="$(KUBECONFIG="$HOME/.kube/config" kubectl config get-clusters 2>/dev/null)"; then + note "FAIL cannot list home kubeconfig clusters; TLS pin not applied" + return 1 + fi + while IFS= read -r c; do + if [ "$header" = 1 ]; then header=0; continue; fi [ -n "$c" ] || continue KUBECONFIG="$HOME/.kube/config" kubectl config set-cluster "$c" --tls-server-name="$KUBE_TLS_SERVER_NAME" >/dev/null || rc=1 - done < <(KUBECONFIG="$HOME/.kube/config" kubectl config get-clusters 2>/dev/null | tail -n +2) + done <<< "$clusters" return $rc } diff --git a/units/laptop-services.sh b/units/laptop-services.sh index 6e880543..bdb70c1b 100755 --- a/units/laptop-services.sh +++ b/units/laptop-services.sh @@ -2,7 +2,7 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -if [ "${UNITS_CHECK:-0}" = 1 ] && [ "$MACHINE" != matrix ] && [ "$MACHINE" != telengana ] && [ "$IS_TERMUX" != 1 ]; then +if [ "$MACHINE" != matrix ] && [ "$MACHINE" != telengana ] && [ "$IS_TERMUX" != 1 ]; then hostvars="$DOTFILES_DIR/host_vars/$MACHINE.yml" if [ -e "$hostvars" ] || [ -L "$hostvars" ]; then if [ ! -f "$hostvars" ] || [ ! -r "$hostvars" ] || ! cat "$hostvars" >/dev/null 2>&1; then @@ -116,8 +116,31 @@ remove_unit() { changed=1 fi } +# The old non-laptop service could be enabled on default.target. Remove only +# its managed link; leaving it would also run the laptop update before login. +remove_old_update_link() { + local link="$unitdir/default.target.wants/dotfiles-update.service" target + [ -L "$link" ] || return 0 + target=$(readlink "$link") || return 1 + case "$target" in + ../dotfiles-update.service|"$unitdir/dotfiles-update.service") + if [ "${UNITS_CHECK:-0}" = 1 ]; then note "would remove $link"; else rm -f "$link" || return 1; fi + changed=1 ;; + esac +} +deploy() { + local description="$1"; shift + if "$@"; then + note "ok $description" + else + step "$description" false + return 1 + fi +} +update_ready=0 atuin_ready=0 mitigator_script_ready=0 mitigator_ready=0 phone_service_ready=0 phone_timer_ready=0 +timer_removed=0 -cat > "$tmp/dotfiles-update.service" < "$tmp/dotfiles-update.service" < "$tmp/atuin-daemon.service" < "$tmp/atuin-daemon.service" < "$tmp/phone-apply.service" < "$tmp/phone-apply.service" </dev/null 2>&1; then step "osquery dependency sha256sum" false; finish; fi - local pollpath="$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/bin:/data/data/com.termux/files/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - local state="$HOME/.local/state/fleet-osquery" user uid gid - if [ "$override_path" != null ]; then pollpath=$override_path; fi - if [ "$override_state" != null ]; then state=$override_state; fi + local user uid gid user=$(id -un) && uid=$(id -u) && gid=$(id -g) || { step "read osquery poller identity" false; finish; } step "check osquery monitoring and poller" "${rootcmd[@]}" env PYTHONDONTWRITEBYTECODE=1 python3 \ "$DOTFILES_DIR/units/osquery-check.py" "$DOTFILES_DIR" "$version" "$archive" "$checksum" \ @@ -124,10 +130,6 @@ osquery_main() { "${rootcmd[@]}" systemctl start fleet-osquery-report.timer } osquery_poller() { - local pollpath="$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/bin:/data/data/com.termux/files/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - local state="$HOME/.local/state/fleet-osquery" unit OSQUERY_CHANGED=0 - if [ "$override_path" != null ]; then pollpath=$override_path; fi - if [ "$override_state" != null ]; then state=$override_state; fi if [ "$poller" != true ]; then if [ -e /etc/systemd/system/fleet-osquery-poll.timer ]; then "${rootcmd[@]}" systemctl disable --now fleet-osquery-poll.timer || return 1 diff --git a/units/osquery_native_inventory.py b/units/osquery_native_inventory.py index 1b409e96..a3a52ce8 100644 --- a/units/osquery_native_inventory.py +++ b/units/osquery_native_inventory.py @@ -14,19 +14,25 @@ def yaml_document(path): try: text = path.read_text(encoding="utf-8") - try: - return json.loads(text) - except json.JSONDecodeError: - pass + except (OSError, UnicodeError): + raise ValueError("inventory file cannot be read") from None + try: + return json.loads(text) + except json.JSONDecodeError: + pass + try: result = subprocess.run(["yq", "-o=json", "-I=0", ".", str(path)], capture_output=True, timeout=30, check=True) return json.loads(result.stdout) - except FileNotFoundError as error: - if error.filename == "yq": - raise ValueError("native inventory requires yq for YAML") from None - raise ValueError("inventory file cannot be read") from None - except (OSError, UnicodeError, ValueError, subprocess.SubprocessError): - raise ValueError("inventory YAML could not be parsed") from None + except (OSError, ValueError, subprocess.SubprocessError): + try: + import yaml + except ImportError: + raise ValueError("native inventory requires yq or Python PyYAML for YAML") from None + try: + return yaml.safe_load(text) + except yaml.YAMLError: + raise ValueError("inventory YAML could not be parsed") from None def mapping(value): diff --git a/units/pipewire-audio.sh b/units/pipewire-audio.sh index d2bc1e86..2d3e296e 100755 --- a/units/pipewire-audio.sh +++ b/units/pipewire-audio.sh @@ -89,6 +89,25 @@ as_root() { if [ "$(id -u)" = 0 ]; then "$@"; else sudo -n "$@"; fi } +# Recognize the original Ansible template as well as the corrected template, +# varying only the rendered numeric volume; anything else belongs to the user. +match_mic_service() { + local dest="$1" template="$2" pattern='{{ mic_hardware_volume_pct | default(100) }}' + local legacy candidate prefix suffix existing volume + [[ "$template" == *"$pattern"* ]] || return 2 + [ -f "$dest" ] && [ ! -L "$dest" ] || return 1 + existing="$(cat "$dest")" || return 2 + legacy="${template/'cut -f 2'/'awk "{print $$2}"'}" + legacy="${legacy//'%%'/'%'}" + for candidate in "$template" "$legacy"; do + prefix="${candidate%%"$pattern"*}"; suffix="${candidate#*"$pattern"}" + [[ "$existing" == "$prefix"*"$suffix" ]] || continue + volume="${existing#"$prefix"}"; volume="${volume%"$suffix"}" + [[ "$volume" =~ ^[0-9]+$ ]] && return 0 + done + return 1 +} + if [ "${UNITS_CHECK:-0}" = 1 ]; then check_audio_packages() { local os_id="" os_like="" pkg result rc manager=0 @@ -96,6 +115,10 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then os_id="$(. /etc/os-release; printf '%s' "${ID:-}")" os_like="$(. /etc/os-release; printf '%s' "${ID_LIKE:-}")" fi + if command -v bootc >/dev/null 2>&1; then + note "PARTIAL: bootc DSP prerequisites must be provided by the image" + return 0 + fi if command -v dnf >/dev/null 2>&1 || command -v rpm-ostree >/dev/null 2>&1; then manager=1 command -v rpm >/dev/null 2>&1 || { note "FAIL rpm required to inspect DSP packages"; return 1; } @@ -200,7 +223,7 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then fi } check_audio() { - local rnnoise lsp=unknown lv2_plugins speaker dir mic_service template pattern prefix suffix existing volume + local rnnoise lsp=unknown lv2_plugins speaker dir mic_service template pattern local rc mode dropin old_rule mic_config_custom=1 mic_service_custom=0 local pipewire_changed=0 wireplumber_changed=0 mic_changed=0 uid properties line load="" active="" enabled="" local running installed current app @@ -239,16 +262,9 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then pattern='{{ mic_hardware_volume_pct | default(100) }}' [[ "$template" == *"$pattern"* ]] || { note "FAIL invalid microphone link service template"; return 1; } if [ -e "$mic_service" ] || [ -L "$mic_service" ]; then - if [ -f "$mic_service" ] && [ ! -L "$mic_service" ]; then - existing="$(cat "$mic_service")" || { note "FAIL read microphone link service"; return 1; } - prefix="${template%%"$pattern"*}"; suffix="${template#*"$pattern"}" - if [[ "$existing" == "$prefix"*"$suffix" ]]; then - volume="${existing#"$prefix"}"; volume="${volume%"$suffix"}" - [[ "$volume" =~ ^[0-9]+$ ]] || mic_service_custom=1 - else - mic_service_custom=1 - fi - else + if match_mic_service "$mic_service" "$template"; then :; else + rc=$? + [ "$rc" = 1 ] || { note "FAIL read microphone link service"; return 1; } mic_service_custom=1 fi [ "$mic_service_custom" = 0 ] || note "PARTIAL: retained custom microphone link service $mic_service" @@ -308,7 +324,7 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then if [ -n "$rnnoise" ]; then [ "$enabled" = enabled ] || [ "$enabled" = enabled-runtime ] || note "WOULD enable microphone link service" if [ "$active" != active ]; then note "WOULD start microphone link service" - elif [ "$mic_changed" = 1 ] || [ -e "$state/mic" ]; then note "WOULD restart microphone link service"; fi + elif [ "$mic_changed" = 1 ] || [ "$pipewire_changed" = 1 ] || [ "$wireplumber_changed" = 1 ] || [ -e "$state/mic" ] || [ -e "$state/pipewire" ] || [ -e "$state/wireplumber" ]; then note "WOULD restart microphone link service"; fi elif [ "$load" != not-found ]; then [ "$active" != active ] || note "WOULD stop microphone link service" if [ "$enabled" = enabled ] || [ "$enabled" = enabled-runtime ]; then note "WOULD disable microphone link service"; fi @@ -341,17 +357,22 @@ install_packages() { os_id="$(. /etc/os-release; printf '%s' "${ID:-}")" os_like="$(. /etc/os-release; printf '%s' "${ID_LIKE:-}")" fi - if command -v dnf >/dev/null 2>&1; then - supported=1 - if ! command -v rpm >/dev/null 2>&1 || ! rpm -q lsp-plugins-lv2 noise-suppression-for-voice >/dev/null 2>&1; then - as_root dnf install -y lsp-plugins-lv2 noise-suppression-for-voice || return 1 - fi + if command -v bootc >/dev/null 2>&1; then + note "PARTIAL: bootc DSP prerequisites must be provided by the image" + return 0 fi + # On Atomic systems dnf may also be installed, but only rpm-ostree can + # layer packages onto the host. Never run both transactions. if command -v rpm-ostree >/dev/null 2>&1; then supported=1 if ! command -v rpm >/dev/null 2>&1 || ! rpm -q lsp-plugins-lv2 noise-suppression-for-voice >/dev/null 2>&1; then as_root rpm-ostree install --apply-live --idempotent lsp-plugins-lv2 noise-suppression-for-voice || return 1 fi + elif command -v dnf >/dev/null 2>&1; then + supported=1 + if ! command -v rpm >/dev/null 2>&1 || ! rpm -q lsp-plugins-lv2 noise-suppression-for-voice >/dev/null 2>&1; then + as_root dnf install -y lsp-plugins-lv2 noise-suppression-for-voice || return 1 + fi fi if [[ " $os_id $os_like " == *" arch "* ]]; then supported=1 @@ -360,11 +381,7 @@ install_packages() { fi fi if [ "$supported" = 0 ]; then - if command -v bootc >/dev/null 2>&1; then - note "PARTIAL: bootc DSP prerequisites must be provided by the image" - else - note "PARTIAL: no supported DSP package manager; install prerequisites for this distribution" - fi + note "PARTIAL: no supported DSP package manager; install prerequisites for this distribution" fi } step "DSP prerequisite packages" install_packages @@ -475,22 +492,14 @@ step "PipeWire LADSPA path" write_dropin mic_service="$user_systemd/mic-dsp-link.service" mic_service_custom=0 +mic_service_owned=0 managed_mic_service() { - local template pattern prefix suffix existing volume - [ -f "$mic_service" ] && [ ! -L "$mic_service" ] || return 1 + local template template="$(cat "$assets/templates/mic-dsp-link.service.j2")" || return 2 - pattern='{{ mic_hardware_volume_pct | default(100) }}' - [[ "$template" == *"$pattern"* ]] || return 2 - prefix="${template%%"$pattern"*}" - suffix="${template#*"$pattern"}" - existing="$(cat "$mic_service")" || return 2 - [[ "$existing" == "$prefix"*"$suffix" ]] || return 1 - volume="${existing#"$prefix"}" - volume="${volume%"$suffix"}" - [[ "$volume" =~ ^[0-9]+$ ]] + match_mic_service "$mic_service" "$template" } if [ -e "$mic_service" ] || [ -L "$mic_service" ]; then - if managed_mic_service; then :; else + if managed_mic_service; then mic_service_owned=1; else rc=$? mic_service_custom=1 if [ "$rc" = 1 ]; then @@ -542,6 +551,14 @@ sync_user_services() { if [ -e "$state/pipewire" ] || [ -e "$state/mic" ]; then systemctl --user daemon-reload || return 1 fi + # Restarting either audio daemon destroys the manually created mic links. + # Persist the relink before restarting, so failed/interrupted applies retry it. + if [ -e "$state/pipewire" ] || [ -e "$state/wireplumber" ]; then + if { [ "$rnnoise" != unknown ] && [ -n "$rnnoise" ] && [ "$mic_service_custom" = 0 ]; } || + { [ "$rnnoise" = unknown ] && [ "$mic_service_owned" = 1 ] && [ ! -L "$conf/60-mic-dsp.conf" ] && [ -f "$conf/60-mic-dsp.conf" ] && cmp -s "$conf/60-mic-dsp.conf" "$assets/files/mic-rnnoise.conf"; }; then + touch "$state/mic" || return 1 + fi + fi if [ -e "$state/pipewire" ]; then systemctl --user reset-failed pipewire.service pipewire.socket pipewire-pulse.service pipewire-pulse.socket || note "warn: could not reset all PipeWire failed states" systemctl --user restart pipewire pipewire-pulse.socket || return 1 diff --git a/units/pipewire-audio/templates/mic-dsp-link.service.j2 b/units/pipewire-audio/templates/mic-dsp-link.service.j2 index e62144fc..4914a6b8 100644 --- a/units/pipewire-audio/templates/mic-dsp-link.service.j2 +++ b/units/pipewire-audio/templates/mic-dsp-link.service.j2 @@ -7,12 +7,12 @@ Wants=wireplumber.service Type=oneshot ExecStart=/bin/sh -c '\ for i in 1 2 3 4 5; do \ - HW_MIC=$$(pactl list short sources 2>/dev/null | grep alsa_input | grep -v monitor | head -1 | awk "{print $$2}"); \ + HW_MIC=$$(pactl list short sources 2>/dev/null | grep alsa_input | grep -v monitor | head -1 | cut -f 2); \ [ -n "$$HW_MIC" ] && break; \ sleep 1; \ done; \ [ -n "$$HW_MIC" ] || exit 0; \ -pactl set-source-volume $$HW_MIC {{ mic_hardware_volume_pct | default(100) }}% 2>/dev/null || true; \ +pactl set-source-volume $$HW_MIC {{ mic_hardware_volume_pct | default(100) }}%% 2>/dev/null || true; \ pw-link $$HW_MIC:capture_FL effect_input.rnnoise:input_FL 2>/dev/null; \ pw-link $$HW_MIC:capture_FR effect_input.rnnoise:input_FR 2>/dev/null; \ pactl set-default-source effect_output.rnnoise 2>/dev/null || true; \ diff --git a/units/proxy-inventory.py b/units/proxy-inventory.py new file mode 100755 index 00000000..1f369b35 --- /dev/null +++ b/units/proxy-inventory.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Resolve proxy and certificate settings with inventory group precedence.""" +import json +import sys + +from osquery_native_inventory import group_contains, load_inventory + + +def main(mode, inventory_path, machine): + if mode not in ("proxy", "cert"): + raise ValueError("invalid service inventory query") + # Check membership before loading settings or doing anything to the host. + inventory = load_inventory(inventory_path, resolve_vars=False) + if machine not in inventory["_meta"]["hostvars"]: + raise ValueError("machine is absent from inventory") + if group_contains(inventory, "vps", machine): + print("skip") + return + flag = "skip_proxy" if mode == "proxy" else "skip_tailscale_cert" + field = "web_services" if mode == "proxy" else "tailscale_domain" + inventory = load_inventory(inventory_path, fields=(flag, "machine_profile", field)) + values = inventory["_meta"]["hostvars"][machine] + skipped = values.get(flag, False) + profile = values.get("machine_profile", "") + if not isinstance(skipped, bool) or not isinstance(profile, str): + raise ValueError("invalid service exclusion setting") + if skipped or profile == "vm-test": + print("skip") + return + value = values.get(field, [] if mode == "proxy" else "") + if mode == "proxy" and value is None: + value = [] + if mode == "cert" and not isinstance(value, str): + raise ValueError("invalid Tailscale certificate domain") + print("run") + print(json.dumps(value, separators=(",", ":")) if mode == "proxy" else value) + + +if __name__ == "__main__": + try: + if len(sys.argv) != 4: + raise ValueError("invalid service inventory query") + main(*sys.argv[1:]) + except (OSError, ValueError, TypeError, KeyError) as error: + print(f"service inventory: {error}", file=sys.stderr) + sys.exit(1) diff --git a/units/proxy-serve.py b/units/proxy-serve.py new file mode 100755 index 00000000..758e74fa --- /dev/null +++ b/units/proxy-serve.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Check the node's default HTTPS Serve handler, not unrelated handlers.""" +import json +import sys +from urllib.parse import urlsplit + + +def mapping(value): + if not isinstance(value, dict): + raise ValueError("unexpected Tailscale status structure") + return value + + +def main(status_path, serve_path): + with open(status_path, encoding="utf-8") as source: + status = mapping(json.load(source)) + with open(serve_path, encoding="utf-8") as source: + serve = json.load(source) + if serve is None: + serve = {} + serve = mapping(serve) + name = mapping(status.get("Self")).get("DNSName") + if not isinstance(name, str) or not name.rstrip("."): + raise ValueError("Tailscale did not report the node DNS name") + web = mapping(serve.get("Web", {})) + tcp = mapping(serve.get("TCP", {})) + endpoint = web.get(name.rstrip(".") + ":443") + port = tcp.get("443") + if endpoint is None or port is None: + print("false") + return + handlers = mapping(mapping(endpoint).get("Handlers", {})) + root = handlers.get("/") + if root is None: + print("false") + return + target = mapping(root).get("Proxy", "") + if not isinstance(target, str): + raise ValueError("unexpected Tailscale proxy target") + url = urlsplit(target) + matches = (mapping(port).get("HTTPS") is True and url.scheme == "http" + and url.hostname == "127.0.0.1" and url.port == 9000 + and url.path in ("", "/") and not url.query and not url.fragment + and url.username is None and url.password is None) + print("true" if matches else "false") + +if __name__ == "__main__": + try: + if len(sys.argv) != 3: + raise ValueError("invalid Serve status query") + main(*sys.argv[1:]) + except (OSError, ValueError, TypeError) as error: + print(f"Tailscale Serve status: {error}", file=sys.stderr) + sys.exit(1) diff --git a/units/proxy.sh b/units/proxy.sh index ffee169d..d4eb345f 100755 --- a/units/proxy.sh +++ b/units/proxy.sh @@ -3,19 +3,15 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ] || skipped_by_profile skip_proxy; then +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then note "SKIPPED: proxy excluded on this host" finish fi -if ! command -v yq >/dev/null 2>&1; then - step "check proxy VPS membership (yq required)" false - finish -fi -vps=$(MACHINE="$MACHINE" yq '.all.children.vps.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml") || { step "check proxy VPS membership" false; finish; } -case "$vps" in - true) note "SKIPPED: proxy excluded on VPS"; finish ;; - false) ;; - *) step "check proxy VPS membership" false; finish ;; +settings=$(python3 "$DOTFILES_DIR/units/proxy-inventory.py" proxy "$DOTFILES_DIR/inventory.yml" "$MACHINE") || { step "resolve proxy inventory settings" false; finish; } +case "${settings%%$'\n'*}" in + skip) note "SKIPPED: proxy excluded on this host"; finish ;; + run) services=${settings#*$'\n'} ;; + *) step "resolve proxy inventory settings" false; finish ;; esac as_root() { if [ "$EUID" = 0 ]; then "$@"; else sudo -n "$@"; fi; } @@ -30,14 +26,14 @@ install_root_file() { changed=1 } render_caddyfile() { - local host_file="$DOTFILES_DIR/host_vars/$MACHINE.yml" - if [ -f "$host_file" ]; then - yq -o=json '.web_services // []' "$host_file" > "$tmp/services.json" || return 1 - else - printf '[]\n' > "$tmp/services.json" || return 1 - fi + printf '%s\n' "$services" > "$tmp/services.json" || return 1 python3 "$DOTFILES_DIR/units/service-templates.py" proxy "$DOTFILES_DIR/units/proxy/Caddyfile.j2" "$tmp/services.json" > "$tmp/Caddyfile" } +inspect_serve() { + as_root tailscale status --json > "$tmp/status.json" || return 1 + as_root tailscale serve status --json > "$tmp/serve.json" || return 1 + python3 "$DOTFILES_DIR/units/proxy-serve.py" "$tmp/status.json" "$tmp/serve.json" +} if [ "${UNITS_CHECK:-0}" = 1 ]; then if [ "$EUID" != 0 ] && ! sudo -n true; then step "noninteractive root access" false; finish; fi proxy_check_dir() { @@ -75,8 +71,12 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then [[ "$state" == *'ActiveState=active'* ]] || note "would start Caddy" case "$state" in *'UnitFileState=enabled'*|*'UnitFileState=generated'*) ;; *) note "would enable Caddy" ;; esac fi - if serve=$(as_root tailscale serve status 2>/dev/null); then - [[ "$serve" == *127.0.0.1:9000* ]] && note "ok Tailscale serve target" || note "would point Tailscale serve at Caddy" + if serve=$(inspect_serve); then + case "$serve" in + true) note "ok Tailscale serve target" ;; + false) note "would point Tailscale serve at Caddy" ;; + *) step "inspect Tailscale serve status" false ;; + esac else step "inspect Tailscale serve status" false; fi finish fi @@ -122,10 +122,15 @@ else step "enable and start Caddy" false finish fi -serve=$(as_root tailscale serve status 2>/dev/null || true) -if [[ "$serve" != *127.0.0.1:9000* ]]; then +if ! serve=$(inspect_serve); then + step "inspect Tailscale serve status" false + finish +fi +if [ "$serve" = false ]; then # --bg replaces the existing default HTTPS handler, including the old # direct homepage target; turning 443 off afterward would remove the new one. step "point Tailscale serve at Caddy" as_root tailscale serve --bg http://127.0.0.1:9000 +elif [ "$serve" != true ]; then + step "inspect Tailscale serve status" false fi finish diff --git a/units/server-hardening-file.py b/units/server-hardening-file.py index 3e7a5280..185fa989 100755 --- a/units/server-hardening-file.py +++ b/units/server-hardening-file.py @@ -24,6 +24,29 @@ def update(path, pattern, line, check=False): lines[-1] += "\n" lines.append(line + "\n") result = "".join(lines).encode("utf-8") + return replace(path, original, result, check) + + +SSH_BLOCK = ("# BEGIN dotfiles server-hardening\n" + "MaxStartups 3:50:10\n" + "MaxSessions 10\n" + "ClientAliveCountMax 3\n" + "# END dotfiles server-hardening\n") + + +def update_ssh(path, check=False): + path = Path(os.path.realpath(path)) + original = path.read_bytes() + text = original.decode("utf-8") + if text.startswith(SSH_BLOCK): + return False + # Refuse to guess how to repair a malformed or manually edited block. + if "# BEGIN dotfiles server-hardening" in text or "# END dotfiles server-hardening" in text: + raise ValueError("SSH hardening block is malformed or not at the start of the file") + return replace(path, original, (SSH_BLOCK + text).encode("utf-8"), check) + + +def replace(path, original, result, check): if result == original: return False if check: @@ -57,8 +80,11 @@ def main(): try: check = len(sys.argv) > 1 and sys.argv[1] == "--check" args = sys.argv[2:] if check else sys.argv[1:] - changed = update(args[0], args[1], args[2], check=check) - except (OSError, UnicodeError, re.error, IndexError) as error: + if args[0] == "--ssh": + changed = update_ssh(args[1], check=check) + else: + changed = update(args[0], args[1], args[2], check=check) + except (OSError, UnicodeError, re.error, IndexError, ValueError) as error: print(f"hardening file update: {error}", file=sys.stderr) return 1 return 10 if changed else 0 diff --git a/units/server-hardening.sh b/units/server-hardening.sh index 803ce676..2de9011b 100755 --- a/units/server-hardening.sh +++ b/units/server-hardening.sh @@ -89,7 +89,7 @@ finally: fi } if [ "${UNITS_CHECK:-0}" = 1 ]; then - local rule pattern line mode chrony_unit chrony_state chrony_active=0 + local mode chrony_unit chrony_state chrony_active=0 hardening_inspect_line() { local description=$1 shift @@ -114,25 +114,41 @@ finally: if [ "$swap_root" = readonly ]; then hardening_immutable_swap elif [ "$swap_root" = writable ]; then - if "${rootcmd[@]}" test -e /swapfile; then - if mode=$("${rootcmd[@]}" stat -c '%a:%u:%g' /swapfile); then - [ "$mode" = '600:0:0' ] && note "ok swap permissions" || note "would set swap ownership and permissions" - else - step "inspect swap permissions" false - fi - else - note "would create 2 GiB /swapfile with root ownership and mode 0600" + local swap_exists=0 swap_active=0 swap_safe=1 swap_type + if "${rootcmd[@]}" test -L /swapfile || { "${rootcmd[@]}" test -e /swapfile && ! "${rootcmd[@]}" test -f /swapfile; }; then + step "inspect swap file (must be a regular file, not a symlink)" false; swap_safe=0 + elif "${rootcmd[@]}" test -e /swapfile; then + swap_exists=1 fi if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then - if printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then + if printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then swap_active=1; fi + else + step "inspect active swap" false; swap_safe=0 + fi + if [ "$swap_safe" = 1 ] && [ "$swap_exists" = 1 ] && [ "$swap_active" = 0 ]; then + if ! swap_type=$("${rootcmd[@]}" blkid -p -s TYPE -o value /swapfile) || [ "$swap_type" != swap ]; then + step "inspect existing swap signature (refusing to format)" false; swap_safe=0 + fi + fi + if [ "$swap_safe" = 1 ]; then + if [ "$swap_exists" = 1 ]; then + if mode=$("${rootcmd[@]}" stat -c '%a:%u:%g' /swapfile); then + [ "$mode" = '600:0:0' ] && note "ok swap permissions" || note "would set swap ownership and permissions" + else + step "inspect swap permissions" false + fi + else + note "would create 2 GiB /swapfile with root ownership and mode 0600" + fi + if [ "$swap_active" = 1 ]; then note "ok swap enabled" + elif [ "$swap_exists" = 1 ]; then + note "would enable existing swapfile" else note "would format and enable /swapfile" fi - else - step "inspect active swap" false + hardening_inspect_line "swap fstab" /etc/fstab '^/swapfile none swap sw 0 0$' '/swapfile none swap sw 0 0' fi - hardening_inspect_line "swap fstab" /etc/fstab '^/swapfile none swap sw 0 0$' '/swapfile none swap sw 0 0' fi for chrony_unit in chrony chronyd; do chrony_state=$(systemctl show "$chrony_unit" --property=LoadState --property=ActiveState 2>/dev/null); rc=$? @@ -149,10 +165,12 @@ finally: if [ -e /usr/lib/systemd/systemd-timesyncd ] && [ "$chrony_active" = 0 ]; then note "would enable and start systemd-timesyncd if chrony is inactive" fi - for rule in 'MaxStartups|MaxStartups 3:50:10' 'MaxSessions|MaxSessions 10' 'ClientAliveCountMax|ClientAliveCountMax 3'; do - pattern=${rule%%|*}; line=${rule#*|} - hardening_inspect_line "SSH limit $pattern (and reload SSH)" /etc/ssh/sshd_config "^#?$pattern" "$line" - done + hardening_line --check --ssh /etc/ssh/sshd_config; rc=$? + case "$rc" in + 0) note "ok SSH limits (and reload SSH)" ;; + 10) note "would update SSH limit MaxSessions and other global limits (and reload SSH)" ;; + *) step "inspect SSH limits" false ;; + esac if command -v ufw >/dev/null 2>&1; then if ufw_output=$("${rootcmd[@]}" ufw status 2>/dev/null); then [[ "$ufw_output" != *'Status: active'* ]] || note "would enable UFW logging" @@ -181,26 +199,40 @@ finally: if [ "$swap_root" = readonly ]; then hardening_immutable_swap elif [ "$swap_root" = writable ]; then - local swap_ready=1 - if [ ! -e /swapfile ]; then - if ! (umask 077; "${rootcmd[@]}" dd if=/dev/zero of=/swapfile bs=1M count=2048); then + local swap_ready=1 swap_created=0 swap_active=0 swap_type + if "${rootcmd[@]}" test -L /swapfile || { "${rootcmd[@]}" test -e /swapfile && ! "${rootcmd[@]}" test -f /swapfile; }; then + step "inspect swap file (must be a regular file, not a symlink)" false; swap_ready=0 + fi + if [ "$swap_ready" = 1 ]; then + if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then + if printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then swap_active=1; fi + else + step "check active swap" false; swap_ready=0 + fi + fi + if [ "$swap_ready" = 1 ] && [ -e /swapfile ] && [ "$swap_active" = 0 ]; then + if ! swap_type=$("${rootcmd[@]}" blkid -p -s TYPE -o value /swapfile) || [ "$swap_type" != swap ]; then + step "inspect existing swap signature (refusing to format)" false; swap_ready=0 + fi + fi + if [ "$swap_ready" = 1 ] && [ ! -e /swapfile ]; then + # Reserve the path exclusively before writing; never truncate an existing file. + if ! "${rootcmd[@]}" python3 -c 'import os, sys; fd = os.open(sys.argv[1], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600); os.close(fd)' /swapfile; then + step "reserve swap file" false; swap_ready=0 + elif ! (umask 077; "${rootcmd[@]}" dd if=/dev/zero of=/swapfile bs=1M count=2048 conv=notrunc); then step "create swap file" false; swap_ready=0 + else + swap_created=1 fi fi - if [ "$swap_ready" = 1 ] && [ -e /swapfile ]; then + if [ "$swap_ready" = 1 ]; then if ! "${rootcmd[@]}" chown root:root /swapfile; then step "protect swap file" false; swap_ready=0; fi if ! "${rootcmd[@]}" chmod 0600 /swapfile; then step "set swap permissions" false; swap_ready=0; fi - if [ "$swap_ready" = 1 ]; then - if swap_output=$("${rootcmd[@]}" swapon --show=NAME --noheadings); then - if ! printf '%s\n' "$swap_output" | grep -Fxq /swapfile; then - if ! "${rootcmd[@]}" mkswap /swapfile; then - step "format swap" false; swap_ready=0 - elif ! "${rootcmd[@]}" swapon /swapfile; then - step "enable swap" false; swap_ready=0 - fi - fi - else - step "check active swap" false; swap_ready=0 + if [ "$swap_ready" = 1 ] && [ "$swap_active" = 0 ]; then + if [ "$swap_created" = 1 ] && ! "${rootcmd[@]}" mkswap /swapfile; then + step "format swap" false; swap_ready=0 + elif ! "${rootcmd[@]}" swapon /swapfile; then + step "enable swap" false; swap_ready=0 fi fi if [ "$swap_ready" = 1 ]; then @@ -216,12 +248,8 @@ finally: step "enable timesyncd" "${rootcmd[@]}" systemctl enable --now systemd-timesyncd fi - local rule pattern line - for rule in 'MaxStartups|MaxStartups 3:50:10' 'MaxSessions|MaxSessions 10' 'ClientAliveCountMax|ClientAliveCountMax 3'; do - pattern=${rule%%|*}; line=${rule#*|} - hardening_line /etc/ssh/sshd_config "^#?$pattern" "$line"; rc=$? - case "$rc" in 0) ;; 10) changed=1 ;; *) step "SSH limit $pattern" false ;; esac - done + hardening_line --ssh /etc/ssh/sshd_config; rc=$? + case "$rc" in 0) ;; 10) changed=1 ;; *) step "SSH limits" false ;; esac if [ "$changed" = 1 ]; then local ssh_unit=sshd if [ -r /etc/os-release ]; then @@ -232,10 +260,10 @@ finally: # The role's shared SSH reload handler ignores errors on non-systemd hosts. soft "reload SSH" "${rootcmd[@]}" systemctl reload "$ssh_unit" fi - if ufw_output=$(ufw status 2>/dev/null) && [[ "$ufw_output" == *'Status: active'* ]]; then + if ufw_output=$("${rootcmd[@]}" ufw status 2>/dev/null) && [[ "$ufw_output" == *'Status: active'* ]]; then step "enable UFW logging" "${rootcmd[@]}" ufw logging on fi - if pro_output=$(pro status --format json 2>/dev/null) && { [[ "$pro_output" != *livepatch* ]] || [[ "$pro_output" == *disabled* ]]; }; then + if pro_output=$("${rootcmd[@]}" pro status --format json 2>/dev/null) && { [[ "$pro_output" != *livepatch* ]] || [[ "$pro_output" == *disabled* ]]; }; then soft "enable livepatch" "${rootcmd[@]}" pro enable livepatch fi if [ -e /var/run/reboot-required ]; then note "REBOOT REQUIRED on $MACHINE — kernel was updated and needs a restart."; fi diff --git a/units/shell-ai.sh b/units/shell-ai.sh index fe711734..0cd5e286 100755 --- a/units/shell-ai.sh +++ b/units/shell-ai.sh @@ -34,13 +34,19 @@ fi trim() { printf '%s' "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'; } shq() { printf "'%s'" "${1//\'/\'\\\'\'}"; } +fishq() { local value="${1//\\/\\\\}"; printf "'%s'" "${value//\'/\\\'}"; } bwget() { BW_SESSION="$s" bw get "$@" 2>/dev/null; } -deepseek="$(trim "$(bwget password deepseek-api-key)")" -forgejo="$(trim "$(bwget notes forgejo | grep 'API Token:' | sed 's/API Token: //')")" -tavily="$(trim "$(bwget password tavily-api-key)")" -meta="$(trim "$(bwget notes meta-api-key)")" -kiro="$(trim "$(bwget notes kiro-api-key)")" +deepseek="$(bwget password deepseek-api-key)" || deepseek="" +forgejo="$(bwget notes forgejo)" || forgejo="" +tavily="$(bwget password tavily-api-key)" || tavily="" +meta="$(bwget notes meta-api-key)" || meta="" +kiro="$(bwget notes kiro-api-key)" || kiro="" +deepseek="$(trim "$deepseek")" +forgejo="$(trim "$(printf '%s\n' "$forgejo" | grep 'API Token:' | sed 's/API Token: //')")" +tavily="$(trim "$tavily")" +meta="$(trim "$meta")" +kiro="$(trim "$kiro")" if [ -z "$deepseek$forgejo$tavily$meta$kiro" ]; then note "warn: no secrets found in Bitwarden (deepseek-api-key missing?); files left untouched" @@ -50,24 +56,30 @@ fi secrets_sh() { echo '# Managed by dotfiles playbook — DO NOT EDIT' echo 'export OMP_KIRO_CONFIG="$HOME/.config/omp/kiro.json"' - [ -z "$deepseek" ] || echo "export DEEPSEEK_API_KEY=$(shq "$deepseek")" - [ -z "$forgejo" ] || echo "export FORGEJO_TOKEN=$(shq "$forgejo")" - [ -z "$tavily" ] || echo "export TAVILY_API_KEY=$(shq "$tavily")" - [ -z "$meta" ] || echo "export META_API_KEY=$(shq "$meta")" - [ -z "$kiro" ] || echo "export KIRO_API_KEY=$(shq "$kiro")" + [ -z "$deepseek" ] || printf 'export DEEPSEEK_API_KEY=%s\n' "$(shq "$deepseek")" + [ -z "$forgejo" ] || printf 'export FORGEJO_TOKEN=%s\n' "$(shq "$forgejo")" + [ -z "$tavily" ] || printf 'export TAVILY_API_KEY=%s\n' "$(shq "$tavily")" + [ -z "$meta" ] || printf 'export META_API_KEY=%s\n' "$(shq "$meta")" + [ -z "$kiro" ] || printf 'export KIRO_API_KEY=%s\n' "$(shq "$kiro")" } secrets_fish() { echo '# Managed by dotfiles playbook — DO NOT EDIT' - [ -z "$deepseek" ] || echo "set -gx DEEPSEEK_API_KEY $(shq "$deepseek")" - [ -z "$forgejo" ] || echo "set -gx FORGEJO_TOKEN $(shq "$forgejo")" - [ -z "$tavily" ] || echo "set -gx TAVILY_API_KEY $(shq "$tavily")" - [ -z "$meta" ] || echo "set -gx META_API_KEY $(shq "$meta")" - [ -z "$kiro" ] || echo "set -gx KIRO_API_KEY $(shq "$kiro")" + [ -z "$deepseek" ] || printf 'set -gx DEEPSEEK_API_KEY %s\n' "$(fishq "$deepseek")" + [ -z "$forgejo" ] || printf 'set -gx FORGEJO_TOKEN %s\n' "$(fishq "$forgejo")" + [ -z "$tavily" ] || printf 'set -gx TAVILY_API_KEY %s\n' "$(fishq "$tavily")" + [ -z "$meta" ] || printf 'set -gx META_API_KEY %s\n' "$(fishq "$meta")" + [ -z "$kiro" ] || printf 'set -gx KIRO_API_KEY %s\n' "$(fishq "$kiro")" cat <<'FISH' -if command -v bw >/dev/null 2>&1; and test -r /tmp/bw_session - set -gx KIRO_API_KEY (env BW_SESSION=(cat /tmp/bw_session) bw get item 2706eda5-e8ad-43df-9838-b4e00167e95d 2>/dev/null | jq -r .notes) - test "$KIRO_API_KEY" = null; and set -e KIRO_API_KEY +if command -v bw >/dev/null 2>&1; and command -v jq >/dev/null 2>&1; and test -r /tmp/bw_session + begin + set -lx BW_SESSION (cat /tmp/bw_session) + if set -l kiro_item (bw get item 2706eda5-e8ad-43df-9838-b4e00167e95d 2>/dev/null | string collect; test $pipestatus[1] -eq 0) + if set -l kiro_key (printf '%s' "$kiro_item" | jq -er '.notes | select(type == "string" and length > 0)' 2>/dev/null | string collect; test $pipestatus[2] -eq 0) + set -gx KIRO_API_KEY "$kiro_key" + end + end + end end FISH } diff --git a/units/ssh-fleet.py b/units/ssh-fleet.py index 38c3b91a..51d9f0e8 100755 --- a/units/ssh-fleet.py +++ b/units/ssh-fleet.py @@ -13,6 +13,9 @@ import tempfile from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from osquery_native_inventory import group_contains, load_inventory + ROOT = Path(os.environ["DOTFILES_DIR"]) HOME = Path(os.environ["HOME"]) SSH = HOME / ".ssh" @@ -21,6 +24,7 @@ END = "# END dotfiles SSH fleet" AK_BEGIN = "# BEGIN dotfiles Bitwarden SSH keys" AK_END = "# END dotfiles Bitwarden SSH keys" +RETIRED = {"matrix", "telengana"} CHECK = os.environ.get("UNITS_CHECK") == "1" PLANNED = {} @@ -223,13 +227,18 @@ def session_env(): class Fleet: def __init__(self): self.vars = yaml_file(ROOT / "group_vars/all.yml") - inventory = yaml_file(ROOT / "inventory.yml")["all"] - self.hosts = inventory.get("hosts") or {} - self.vps = set(((inventory.get("children") or {}).get("vps") or {}).get("hosts") or {}) host_file = ROOT / "host_vars" / (MACHINE + ".yml") - self.host_vars = yaml_file(host_file, allow_empty=True) if host_file.exists() else {} - self.login = self.host_vars.get("user_login", self.vars["user_login"]) - self.email = self.host_vars.get("user_email", self.vars["user_email"]) + local_vars = yaml_file(host_file, allow_empty=True) if host_file.exists() else {} + fields = ("ansible_user", "ansible_port", "tailnet_name", "user_login", "user_email", + "machines", "machine_pubkeys", "ssh_mesh_exclude", "extra_authorized_keys", + "ssh_keys_force_push", "ssh_keys_force_authorized", "machine_profile", "skip_ssh_mesh") + inventory = load_inventory(ROOT / "inventory.yml", fields=fields) + self.hosts = inventory["_meta"]["hostvars"] + self.host_vars = self.hosts.get(MACHINE, local_vars) + self.vars.update(self.host_vars) + self.vps = {name for name in self.hosts if group_contains(inventory, "vps", name)} + self.login = self.vars["user_login"] + self.email = self.vars["user_email"] self.machines = self.vars["machines"] self.env = None self.items = None @@ -346,7 +355,7 @@ def sync_key(self): saved = ssh_data(item).get("publicKey", "") if saved and key_parts(saved) == key_parts(pub): return - force = os.environ.get("SSH_KEYS_FORCE_PUSH", str(self.host_vars.get("ssh_keys_force_push", "false"))) + force = os.environ.get("SSH_KEYS_FORCE_PUSH", str(self.host_vars.get("ssh_keys_force_push", self.vars.get("ssh_keys_force_push", "false")))) if force.lower() not in ("true", "1", "yes"): print("WARNING: local SSH key differs from Bitwarden; vault left unchanged (set SSH_KEYS_FORCE_PUSH=true for an intentional rotation)") return @@ -448,7 +457,7 @@ def authorize_vault(self): old = text(path) # Force can remove only entries previously managed in our block; older # unmarked entries are indistinguishable from user keys and stay intact. - force = os.environ.get("SSH_KEYS_FORCE_AUTHORIZED", str(self.host_vars.get("ssh_keys_force_authorized", "false"))) + force = os.environ.get("SSH_KEYS_FORCE_AUTHORIZED", str(self.host_vars.get("ssh_keys_force_authorized", self.vars.get("ssh_keys_force_authorized", "false")))) before, managed, after, _ = split_block(old, AK_BEGIN, AK_END) previous = managed.splitlines() if force.lower() not in ("true", "1", "yes"): @@ -540,7 +549,7 @@ def matches(token): def known_hosts(self): for name in self.machines: - if name != MACHINE: + if name != MACHINE and name not in RETIRED: try: self.scan(name) except (OSError, ValueError, KeyError, subprocess.SubprocessError, RuntimeError) as exc: @@ -554,7 +563,7 @@ def mesh(self): if result.returncode: raise RuntimeError("cannot determine online tailnet peers") online = set(result.stdout.splitlines()) - excluded = set(self.vars.get("ssh_mesh_exclude") or []) | self.vps | {MACHINE} + excluded = set(self.host_vars.get("ssh_mesh_exclude", self.vars.get("ssh_mesh_exclude")) or []) | self.vps | RETIRED | {MACHINE} targets = [name for name in self.machines if name not in excluded and (self.hosts.get(name) or {}).get("tailnet_name", name).lower() in online] failed = [] @@ -645,7 +654,7 @@ def ensure_key(self): def known_hosts(self): failed = False for name in self.machines: - if name != MACHINE: + if name != MACHINE and name not in RETIRED: try: if not self.scan(name): raise RuntimeError("no usable host key returned") @@ -674,7 +683,7 @@ def mesh(self): raise ValueError("invalid tailnet peer listing") if peer.get("Online"): online.add(((peer.get("DNSName") or "").split(".")[0] or peer.get("HostName") or "").lower()) - excluded = set(self.vars.get("ssh_mesh_exclude") or []) | self.vps | {MACHINE} + excluded = set(self.host_vars.get("ssh_mesh_exclude", self.vars.get("ssh_mesh_exclude")) or []) | self.vps | RETIRED | {MACHINE} targets = [name for name in self.machines if name not in excluded and (self.hosts.get(name) or {}).get("tailnet_name", name).lower() in online] print("online SSH mesh targets: " + (", ".join(targets) if targets else "none")) diff --git a/units/sshd.sh b/units/sshd.sh index 277cf82a..21a438fa 100755 --- a/units/sshd.sh +++ b/units/sshd.sh @@ -4,12 +4,10 @@ set -uo pipefail . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" # shellcheck source=units/platform-config.bash . "$DOTFILES_DIR/units/platform-config.bash" -if [ "${UNITS_CHECK:-0}" = 1 ]; then - host_file="$DOTFILES_DIR/host_vars/$MACHINE.yml" - if [ -e "$host_file" ] && { [ ! -f "$host_file" ] || [ ! -r "$host_file" ]; }; then - step 'read sshd host configuration' false - finish - fi +host_file="$DOTFILES_DIR/host_vars/$MACHINE.yml" +if { [ -e "$host_file" ] || [ -L "$host_file" ]; } && { [ ! -f "$host_file" ] || [ ! -r "$host_file" ]; }; then + step 'read sshd host configuration' false + finish fi root="${DOTFILES_SYSTEM_ROOT:-}" diff --git a/units/sudo.sh b/units/sudo.sh index a5c09153..5f7a5af1 100755 --- a/units/sudo.sh +++ b/units/sudo.sh @@ -4,12 +4,10 @@ set -uo pipefail . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" # shellcheck source=units/platform-config.bash . "$DOTFILES_DIR/units/platform-config.bash" -if [ "${UNITS_CHECK:-0}" = 1 ]; then - host_file="$DOTFILES_DIR/host_vars/$MACHINE.yml" - if [ -e "$host_file" ] && { [ ! -f "$host_file" ] || [ ! -r "$host_file" ]; }; then - step 'read sudo host configuration' false - finish - fi +host_file="$DOTFILES_DIR/host_vars/$MACHINE.yml" +if { [ -e "$host_file" ] || [ -L "$host_file" ]; } && { [ ! -f "$host_file" ] || [ ! -r "$host_file" ]; }; then + step 'read sudo host configuration' false + finish fi if [ "$(host_var skip_sudo false)" = true ]; then @@ -69,12 +67,31 @@ tmp="$(mktemp -d)" || { step 'create sudo staging directory' false; finish; } trap 'rm -rf "$tmp"' EXIT install_sudo_policy() { - local candidate="$tmp/sudoers" dest="$dir/zz-$user" state staged + local candidate="$tmp/sudoers" dest="$dir/zz-$user" staged rc printf '%s ALL=(ALL) NOPASSWD:ALL\n' "$user" > "$candidate" || return 1 # Always validate before replacing the current rule or deleting older rules. platform_root /usr/sbin/visudo -cf "$candidate" >/dev/null || return 1 - state="$(stat -c '%a:%u:%g' "$dest" 2>/dev/null)" || state='' - if [ -L "$dest" ] || ! cmp -s "$candidate" "$dest" || [ "$state" != '440:0:0' ]; then + # sudoers directories and 0440 files are commonly unreadable to the user. + # Distinguish a real mismatch from a privileged inspection failure. + if platform_root bash -s -- "$candidate" "$dest" <<'INSPECT' +candidate=$1 dest=$2 +if [ -L "$dest" ] || [ ! -e "$dest" ]; then exit 1; fi +if [ ! -f "$dest" ]; then echo "Cannot replace non-file sudo policy: $dest" >&2; exit 2; fi +state=$(stat -c '%a:%u:%g' "$dest") || exit 2 +if cmp -s "$candidate" "$dest"; then + [ "$state" = '440:0:0' ] && exit 0 + exit 1 +else + rc=$? + if [ "$rc" -ne 1 ]; then echo "Cannot read sudo policy: $dest" >&2; exit 2; fi + exit 1 +fi +INSPECT + then + : + else + rc=$? + [ "$rc" = 1 ] || return "$rc" staged="$(platform_root mktemp "$dir/.dotfiles.XXXXXXXX")" || return 1 if ! platform_root install -m 0440 -o root -g root "$candidate" "$staged" || ! platform_root mv -fT "$staged" "$dest"; then platform_root rm -f "$staged" diff --git a/units/syncthing.sh b/units/syncthing.sh index 49f9ce78..762da623 100755 --- a/units/syncthing.sh +++ b/units/syncthing.sh @@ -2,7 +2,9 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -if [ "${UNITS_CHECK:-0}" = 1 ] && { [ -e "$DOTFILES_DIR/host_vars/$MACHINE.yml" ] || [ -L "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; } && { [ ! -f "$DOTFILES_DIR/host_vars/$MACHINE.yml" ] || [ ! -r "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; }; then +# shellcheck source=units/platform-config.bash +. "$DOTFILES_DIR/units/platform-config.bash" +if { [ -e "$DOTFILES_DIR/host_vars/$MACHINE.yml" ] || [ -L "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; } && { [ ! -f "$DOTFILES_DIR/host_vars/$MACHINE.yml" ] || [ ! -r "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; }; then step "inspect host variables" false finish fi @@ -11,25 +13,26 @@ if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ] || [ "$IS_TERMUX" = 1 ] note "SKIPPED: syncthing excluded on this host" finish fi -if ! command -v yq >/dev/null 2>&1; then - step "check VPS membership (yq required)" false +if platform_group_contains vps; then + note "SKIPPED: syncthing excluded on VPS" finish +else + rc=$? + if [ "$rc" -ne 1 ]; then + step "check VPS membership" false + finish + fi fi -vps=$(MACHINE="$MACHINE" yq '.all.children.vps.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml") || { step "check VPS membership" false; finish; } -case "$vps" in - true) note "SKIPPED: syncthing excluded on VPS"; finish ;; - false) ;; - *) step "check VPS membership" false; finish ;; -esac # Remote sessions may not inherit the user's systemd bus environment. export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus" fi +binary="${DOTFILES_SYSTEM_ROOT:-}/home/linuxbrew/.linuxbrew/bin/syncthing" unit="$HOME/.config/systemd/user/syncthing.service" if [ "${UNITS_CHECK:-0}" = 1 ]; then - if [ ! -e /home/linuxbrew/.linuxbrew/bin/syncthing ]; then + if [ ! -e "$binary" ]; then if [ -f "$unit" ]; then note "CHANGE: disable and stop syncthing.service (binary missing)" else @@ -47,6 +50,8 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then step "read syncthing service source" false elif [ ! -e "$unit" ] && [ ! -L "$unit" ]; then note "CHANGE: install syncthing.service" + elif [ -L "$unit" ]; then + note "CHANGE: replace syncthing.service symlink" elif [ ! -f "$unit" ] || [ ! -r "$unit" ]; then step "inspect syncthing service file" false else @@ -76,7 +81,7 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then esac finish fi -if [ ! -e /home/linuxbrew/.linuxbrew/bin/syncthing ]; then +if [ ! -e "$binary" ]; then # A user service can also come from a system-wide user-unit directory. if [ -f "$unit" ]; then step "stop syncthing without binary" systemctl --user disable --now syncthing.service @@ -90,19 +95,37 @@ if [ ! -e /home/linuxbrew/.linuxbrew/bin/syncthing ]; then fi finish fi -changed=0 -[ -f "$unit" ] && cmp -s "$DOTFILES_DIR/units/syncthing/syncthing.service" "$unit" || changed=1 -if install_if_changed "$DOTFILES_DIR/units/syncthing/syncthing.service" "$unit" 0644; then - note "ok syncthing service file" -else +install_syncthing_service() { + local src="$DOTFILES_DIR/units/syncthing/syncthing.service" dir staged rc + [ -f "$src" ] && [ -r "$src" ] || return 1 + if [ ! -L "$unit" ] && [ -e "$unit" ]; then + [ -f "$unit" ] || return 1 + if cmp -s "$src" "$unit"; then + chmod 0644 "$unit" + return + else + rc=$? + [ "$rc" = 1 ] || return "$rc" + fi + fi + dir="$(dirname "$unit")" + mkdir -p "$dir" || return 1 + staged="$(mktemp "$dir/.syncthing.service.XXXXXXXX")" || return 1 + if ! install -m 0644 "$src" "$staged" || ! mv -fT "$staged" "$unit"; then + rm -f "$staged" + return 1 + fi +} +if ! install_syncthing_service; then step "syncthing service file" false finish fi -if [ "$changed" = 1 ]; then - if ! systemctl --user daemon-reload; then - step "reload user systemd" false - finish - fi +note "ok syncthing service file" +# Reload even when the file is unchanged: a previous reload may have failed +# after installation, leaving the user manager with a stale unit definition. +if ! systemctl --user daemon-reload; then + step "reload user systemd" false + finish fi step "enable and start syncthing" systemctl --user enable --now syncthing.service finish diff --git a/units/system-services.sh b/units/system-services.sh index e0fe278f..355095ba 100755 --- a/units/system-services.sh +++ b/units/system-services.sh @@ -2,7 +2,7 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -if [ "${UNITS_CHECK:-0}" = 1 ] && [ "$MACHINE" != matrix ] && [ "$MACHINE" != telengana ] && [ "$IS_TERMUX" != 1 ]; then +if [ "$MACHINE" != matrix ] && [ "$MACHINE" != telengana ] && [ "$IS_TERMUX" != 1 ]; then hostvars="$DOTFILES_DIR/host_vars/$MACHINE.yml" if [ -e "$hostvars" ] || [ -L "$hostvars" ]; then if [ ! -f "$hostvars" ] || [ ! -r "$hostvars" ] || ! cat "$hostvars" >/dev/null 2>&1; then @@ -195,6 +195,18 @@ install_unit() { if [ ! -f "$dest" ] || ! cmp -s "$src" "$dest"; then changed=1; fi install_if_changed "$src" "$dest" 0644 } +# A former laptop must not retain login-triggered updates as well as its timer. +# Only remove the link when it points at the unit managed here. +remove_old_update_link() { + local link="$unitdir/graphical-session.target.wants/dotfiles-update.service" target + [ -L "$link" ] || return 0 + target=$(readlink "$link") || return 1 + case "$target" in + ../dotfiles-update.service|"$unitdir/dotfiles-update.service") + if [ "${UNITS_CHECK:-0}" = 1 ]; then note "would remove $link"; else rm -f "$link" || return 1; fi + changed=1 ;; + esac +} # Render the non-laptop branches of the migrated service templates without # requiring Jinja on the host. @@ -241,7 +253,12 @@ else step "render and deploy atuin daemon" false fi if [ "$update_ready" = 1 ] && [ "$timer_ready" = 1 ]; then - step "ensure user linger" ensure_timer_linger + if ! remove_old_update_link; then + step "remove obsolete graphical update link" false + update_ready=0 + else + step "ensure user linger" ensure_timer_linger + fi fi if [ "${UNITS_CHECK:-0}" = 1 ]; then [ "$changed" != 1 ] || note "would reload user systemd" diff --git a/units/system-services/atuin-daemon.service.j2 b/units/system-services/atuin-daemon.service.j2 index b97dd204..17726c79 100644 --- a/units/system-services/atuin-daemon.service.j2 +++ b/units/system-services/atuin-daemon.service.j2 @@ -3,7 +3,7 @@ Description=Atuin shell history sync daemon After=network.target [Service] -ExecStart={{ brew_prefix }}/bin/atuin daemon start --force +ExecStart=/usr/bin/env atuin daemon start --force Environment="PATH={{ brew_prefix }}/bin:%h/.local/bin:/usr/local/bin:/usr/bin:/bin" Restart=on-failure RestartSec=5 diff --git a/units/system-services/dotfiles-update.service.j2 b/units/system-services/dotfiles-update.service.j2 index 88e47307..10b2797c 100644 --- a/units/system-services/dotfiles-update.service.j2 +++ b/units/system-services/dotfiles-update.service.j2 @@ -13,10 +13,10 @@ Wants=network-online.target Type=oneshot # Wait for network to settle and desktop to stabilise before pulling ExecStartPre=/bin/sleep {{ dotfiles_update_delay_seconds | default(60) }} -# Pull latest repo changes, then run ansible locally (no ansible-pull clone overhead). +# Pull latest repo changes, then run native units locally. # Timer cannot unlock Bitwarden non-interactively, so we explicitly skip secrets. # SSH key sync happens on manual `dots-apply` instead. -ExecStart={{ brew_prefix }}/bin/just -f {{ ansible_facts['env']['HOME'] }}/.local/share/dotfiles/Justfile apply-nosecrets +ExecStart=/usr/bin/env just -f {{ ansible_facts['env']['HOME'] }}/.local/share/dotfiles/Justfile apply-nosecrets # Also update podman containers while we're at it ExecStartPost=-/usr/bin/podman auto-update --user ExecStartPost=-/usr/bin/sudo /usr/bin/podman auto-update diff --git a/units/tailscale-cert.sh b/units/tailscale-cert.sh index 3ff0d34c..96dab595 100755 --- a/units/tailscale-cert.sh +++ b/units/tailscale-cert.sh @@ -3,33 +3,40 @@ set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ] || skipped_by_profile skip_tailscale_cert; then +if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then note "SKIPPED: Tailscale certificates excluded on this host" finish fi -if ! command -v yq >/dev/null 2>&1; then - step "check certificate VPS membership (yq required)" false - finish -fi -vps=$(MACHINE="$MACHINE" yq '.all.children.vps.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml") || { step "check certificate VPS membership" false; finish; } -case "$vps" in - true) note "SKIPPED: Tailscale certificates excluded on VPS"; finish ;; - false) ;; - *) step "check certificate VPS membership" false; finish ;; +settings=$(python3 "$DOTFILES_DIR/units/proxy-inventory.py" cert "$DOTFILES_DIR/inventory.yml" "$MACHINE") || { step "resolve certificate inventory settings" false; finish; } +case "${settings%%$'\n'*}" in + skip) note "SKIPPED: Tailscale certificates excluded on this host"; finish ;; + run) domain=${settings#*$'\n'} ;; + *) step "resolve certificate inventory settings" false; finish ;; esac as_root() { if [ "$EUID" = 0 ]; then "$@"; else sudo -n "$@"; fi; } -# Host variables override the shared group domain, as in the playbook. -domain="" -if [ -f "$DOTFILES_DIR/host_vars/$MACHINE.yml" ]; then - domain=$(yq -r '.tailscale_domain // ""' "$DOTFILES_DIR/host_vars/$MACHINE.yml") || { step "read host certificate domain" false; finish; } -fi -if [ -z "$domain" ]; then - domain=$(yq -r '.tailscale_domain // ""' "$DOTFILES_DIR/group_vars/all.yml") || { step "read certificate domain" false; finish; } -fi if [[ ! "$MACHINE" =~ ^[[:alnum:]_.-]+$ || ! "$domain" =~ ^[[:alnum:]_.-]+$ ]]; then step "validate certificate hostname and domain" false finish fi +# Do not follow user-supplied symlinks or overwrite special files as root. +for path in /etc/ssl/tailscale /etc/ssl/tailscale/cert.pem /etc/ssl/tailscale/key.pem; do + if as_root test -L "$path"; then step "refuse symlink certificate path $path" false; finish; fi +done +if as_root test -e /etc/ssl/tailscale && ! as_root test -d /etc/ssl/tailscale; then + step "refuse non-directory certificate path /etc/ssl/tailscale" false + finish +fi +for path in /etc/ssl/tailscale/cert.pem /etc/ssl/tailscale/key.pem; do + if as_root test -e "$path" && ! as_root test -f "$path"; then step "refuse non-regular certificate path $path" false; finish; fi +done +cert_identity_ok() { + local cert_pub key_pub host_check + host_check=$(LC_ALL=C as_root openssl x509 -checkhost "$MACHINE.$domain" -noout -in /etc/ssl/tailscale/cert.pem 2>/dev/null) || return 1 + [ "$host_check" = "Hostname $MACHINE.$domain does match certificate" ] || return 1 + cert_pub=$(as_root openssl x509 -pubkey -noout -in /etc/ssl/tailscale/cert.pem 2>/dev/null) || return 1 + key_pub=$(as_root openssl pkey -pubout -passin pass: -in /etc/ssl/tailscale/key.pem 2>/dev/null) || return 1 + [ -n "$cert_pub" ] && [ "$cert_pub" = "$key_pub" ] +} if [ "${UNITS_CHECK:-0}" = 1 ]; then if [ "$EUID" != 0 ] && ! sudo -n true; then step "noninteractive root access" false; finish; fi tmp=$(mktemp -d) || { step "create certificate temporary directory" false; finish; } @@ -44,8 +51,15 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then note "would provision Tailscale certificate for $MACHINE.$domain" elif ! as_root openssl x509 -noout -in /etc/ssl/tailscale/cert.pem >/dev/null 2>&1; then step "inspect Tailscale certificate" false + elif ! as_root test -s /etc/ssl/tailscale/key.pem; then + renew=1 + note "would renew Tailscale certificate for $MACHINE.$domain (private key missing or empty)" elif as_root openssl x509 -checkend 2592000 -noout -in /etc/ssl/tailscale/cert.pem >/dev/null 2>&1; then - note "ok certificate valid for at least 30 days" + if cert_identity_ok; then + note "ok certificate valid for at least 30 days and matches hostname and private key" + else + step "verify certificate hostname and private key (existing pair preserved)" false + fi else renew=1 note "would renew Tailscale certificate for $MACHINE.$domain" @@ -65,17 +79,18 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then cert_check_mode /etc/ssl/tailscale/cert.pem 644 cert_check_mode /etc/ssl/tailscale/key.pem 640 cert_check_file() { - local src=$1 dest=$2 mode rc + local src=$1 dest=$2 expected=${3:-644} mode rc [ -r "$src" ] || { step "read $src" false; return; } if ! as_root test -e "$dest" && ! as_root test -L "$dest"; then note "would deploy $dest"; return; fi mode=$(as_root stat -c '%a:%u:%g' "$dest") || { step "inspect $dest" false; return; } as_root cmp -s "$src" "$dest"; rc=$? case "$rc" in - 0) [ "$mode" = '644:0:0' ] && note "ok $dest" || note "would set ownership and permissions on $dest" ;; + 0) [ "$mode" = "$expected:0:0" ] && note "ok $dest" || note "would set ownership and permissions on $dest" ;; 1) note "would deploy $dest" ;; *) step "compare $dest" false ;; esac } + cert_check_file "$DOTFILES_DIR/units/tailscale-cert/renew.sh" /etc/systemd/system/tailscale-cert-renew.sh 755 if python3 "$DOTFILES_DIR/units/service-templates.py" cert "$DOTFILES_DIR/units/tailscale-cert/tailscale-cert-renew.service.j2" "$MACHINE" "$domain" > "$tmp/service" && [ -s "$tmp/service" ]; then cert_check_file "$tmp/service" /etc/systemd/system/tailscale-cert-renew.service else step "render certificate renewal service" false; fi @@ -93,9 +108,21 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then fi finish fi -step "create Tailscale certificate directory" as_root install -d -m 0755 -o root -g root /etc/ssl/tailscale -if ! as_root openssl x509 -checkend 2592000 -noout -in /etc/ssl/tailscale/cert.pem >/dev/null 2>&1; then - step "provision Tailscale certificate" as_root tailscale cert --cert-file /etc/ssl/tailscale/cert.pem --key-file /etc/ssl/tailscale/key.pem --min-validity 720h "$MACHINE.$domain" +if ! as_root install -d -m 0755 -o root -g root /etc/ssl/tailscale; then + step "create Tailscale certificate directory" false + finish +fi +if ! as_root test -s /etc/ssl/tailscale/key.pem || ! as_root openssl x509 -checkend 2592000 -noout -in /etc/ssl/tailscale/cert.pem >/dev/null 2>&1; then + if ! as_root bash "$DOTFILES_DIR/units/tailscale-cert/renew.sh" /etc/ssl/tailscale "$MACHINE.$domain"; then + step "provision Tailscale certificate" false + finish + fi + note "ok provision Tailscale certificate" +else + if ! cert_identity_ok; then + step "verify certificate hostname and private key (existing pair preserved)" false + finish + fi fi set_cert_mode() { as_root chown root:root "$1" && as_root chmod "$2" "$1" @@ -106,11 +133,16 @@ step "set private key permissions" set_cert_mode /etc/ssl/tailscale/key.pem 0640 tmp=$(mktemp -d) || { step "create certificate temporary directory" false; finish; } trap 'rm -rf "$tmp"' EXIT install_root_file() { - local src="$1" dest="$2" mode + local src="$1" dest="$2" expected="${3:-644}" mode mode=$(as_root stat -c '%a:%u:%g' "$dest" 2>/dev/null || true) - if as_root cmp -s "$src" "$dest" && [ "$mode" = '644:0:0' ]; then return 0; fi - as_root install -m 0644 -o root -g root "$src" "$dest" + if as_root cmp -s "$src" "$dest" && [ "$mode" = "$expected:0:0" ]; then return 0; fi + as_root install -m "0$expected" -o root -g root "$src" "$dest" } +if ! install_root_file "$DOTFILES_DIR/units/tailscale-cert/renew.sh" /etc/systemd/system/tailscale-cert-renew.sh 755; then + step "deploy certificate renewal helper" false + finish +fi +note "ok deploy certificate renewal helper" service_ok=0 timer_ok=0 if python3 "$DOTFILES_DIR/units/service-templates.py" cert "$DOTFILES_DIR/units/tailscale-cert/tailscale-cert-renew.service.j2" "$MACHINE" "$domain" > "$tmp/tailscale-cert-renew.service" && diff --git a/units/tailscale-cert/renew.sh b/units/tailscale-cert/renew.sh new file mode 100755 index 00000000..5550de05 --- /dev/null +++ b/units/tailscale-cert/renew.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Run as root by the unit and timer. Never write directly to an existing pair. +set -u +if [ "$#" -ne 2 ]; then + echo "usage: renew.sh CERT_DIRECTORY HOSTNAME" >&2 + exit 1 +fi +dir=$1 +host=$2 +cert=$dir/cert.pem +key=$dir/key.pem +if [ ! -d "$dir" ] || [ -L "$dir" ]; then + echo "certificate directory is not a real directory" >&2 + exit 1 +fi +umask 077 +if [ -L "$dir/.renew.lock" ] || { [ -e "$dir/.renew.lock" ] && [ ! -f "$dir/.renew.lock" ]; }; then + echo "refusing non-regular renewal lock" >&2 + exit 1 +fi +exec 9>>"$dir/.renew.lock" || exit 1 +flock -x 9 || exit 1 +for path in "$cert" "$key"; do + if [ -L "$path" ] || { [ -e "$path" ] && [ ! -f "$path" ]; }; then + echo "refusing to replace non-regular certificate path: $path" >&2 + exit 1 + fi +done +tmp=$(mktemp -d "$dir/.renew.XXXXXXXX") || exit 1 +trap 'rm -rf -- "$tmp"' EXIT +if ! tailscale cert --cert-file "$tmp/cert.pem" --key-file "$tmp/key.pem" --min-validity 720h "$host"; then + exit 1 +fi +if [ ! -s "$tmp/cert.pem" ] || [ ! -s "$tmp/key.pem" ] || [ -L "$tmp/cert.pem" ] || [ -L "$tmp/key.pem" ]; then + echo "Tailscale did not produce a certificate and private key" >&2 + exit 1 +fi +host_check=$(LC_ALL=C openssl x509 -checkhost "$host" -noout -in "$tmp/cert.pem" 2>/dev/null) || exit 1 +if [ "$host_check" != "Hostname $host does match certificate" ] || + ! openssl x509 -checkend 0 -noout -in "$tmp/cert.pem" >/dev/null 2>&1; then + echo "Tailscale returned a certificate with the wrong hostname or expiry" >&2 + exit 1 +fi +cert_pub=$(openssl x509 -pubkey -noout -in "$tmp/cert.pem" 2>/dev/null) || exit 1 +key_pub=$(openssl pkey -pubout -passin pass: -in "$tmp/key.pem" 2>/dev/null) || exit 1 +if [ -z "$cert_pub" ] || [ "$cert_pub" != "$key_pub" ]; then + echo "Tailscale returned a certificate and private key that do not match" >&2 + exit 1 +fi +chmod 0644 "$tmp/cert.pem" && chmod 0640 "$tmp/key.pem" || exit 1 +had_key=0 +if [ -e "$key" ]; then + cp -p -- "$key" "$tmp/old-key.pem" || exit 1 + had_key=1 +fi +mv -fT -- "$tmp/key.pem" "$key" || exit 1 +if ! mv -fT -- "$tmp/cert.pem" "$cert"; then + if [ "$had_key" = 1 ]; then + if ! mv -fT -- "$tmp/old-key.pem" "$key"; then + trap - EXIT + echo "failed to restore previous private key; backup retained at $tmp/old-key.pem" >&2 + fi + else + rm -f -- "$key" || echo "failed to remove new private key" >&2 + fi + exit 1 +fi diff --git a/units/tailscale-cert/tailscale-cert-renew.service.j2 b/units/tailscale-cert/tailscale-cert-renew.service.j2 index e2c65295..64aba610 100644 --- a/units/tailscale-cert/tailscale-cert-renew.service.j2 +++ b/units/tailscale-cert/tailscale-cert-renew.service.j2 @@ -5,8 +5,4 @@ Wants=tailscaled.service [Service] Type=oneshot -ExecStart=/usr/bin/tailscale cert \ - --cert-file /etc/ssl/tailscale/cert.pem \ - --key-file /etc/ssl/tailscale/key.pem \ - --min-validity 720h \ - {{ inventory_hostname }}.{{ tailscale_domain }} +ExecStart=/etc/systemd/system/tailscale-cert-renew.sh /etc/ssl/tailscale {{ inventory_hostname }}.{{ tailscale_domain }} diff --git a/units/termux-packages.conf b/units/termux-packages.conf index 8ac5af1b..f5e71b6e 100644 --- a/units/termux-packages.conf +++ b/units/termux-packages.conf @@ -1,5 +1,5 @@ # The former role defaults, plus tools needed before native units can run. -TERMUX_BOOTSTRAP=(python yq chezmoi) +TERMUX_BOOTSTRAP=(python yq chezmoi just) TERMUX_PACKAGES=(openssh git nodejs fish vim nano termux-api curl unzip file e2fsprogs) TERMUX_OPTIONAL=(mise) TERMUX_NPM_GLOBAL=('@earendil-works/pi-coding-agent') diff --git a/units/termux-packages.sh b/units/termux-packages.sh index 01ffeac6..8c578ab5 100755 --- a/units/termux-packages.sh +++ b/units/termux-packages.sh @@ -14,14 +14,20 @@ if platform_group_contains termux_hosts; then :; else if [ "$rc" = 1 ]; then note 'SKIPPED: not in termux_hosts'; else step 'read Termux inventory group' false; fi finish fi +if [ "$IS_TERMUX" != 1 ]; then + note 'SKIPPED: Termux packages require the Termux environment' + finish +fi # Role defaults are overridden by group_vars/all and then host_vars, as before. # No YAML parser is needed on a freshly bootstrapped phone without overrides. load_overrides() { - local file key has value + local file key has value rc for file in "$DOTFILES_DIR/group_vars/all.yml" "$DOTFILES_DIR/host_vars/$MACHINE.yml"; do - [ -f "$file" ] || continue - if ! grep -Eq '^termux_(pkgs|npm_global|keepalive_period_ms):' "$file"; then continue; fi + [ -e "$file" ] || [ -L "$file" ] || continue + [ -f "$file" ] && [ -r "$file" ] || { echo 'Cannot read Termux package configuration' >&2; return 1; } + grep -Eq '^termux_(pkgs|npm_global|keepalive_period_ms):' "$file"; rc=$? + case "$rc" in 0) ;; 1) continue ;; *) echo 'Cannot read Termux package configuration' >&2; return 1 ;; esac package_require_yq || return 1 for key in termux_pkgs termux_npm_global; do has="$(yq -r "has(\"$key\")" "$file")" || return 1 @@ -59,15 +65,27 @@ install_packages() { return "$rc" } ensure_sshd() { - pgrep -x sshd >/dev/null 2>&1 || sshd + local rc + pgrep -x sshd >/dev/null 2>&1; rc=$? + case "$rc" in 0) return 0 ;; 1) sshd ;; *) echo 'Cannot inspect Termux sshd process' >&2; return 1 ;; esac +} +pending_keepalive() { + [[ "$1" =~ (^|[^[:alnum:]_])Pending[[:space:]]Job[[:space:]]1([^[:digit:]]|$) ]] } schedule_keepalive() { local pending - pending="$(termux-job-scheduler -p 2>/dev/null)" || : - case "$pending" in *'Pending Job 1'*) return 0 ;; esac + pending="$(termux-job-scheduler -p 2>/dev/null)" || { echo 'Cannot inspect Termux scheduled jobs' >&2; return 1; } + pending_keepalive "$pending" && return 0 termux-job-scheduler --job-id 1 --period-ms "$TERMUX_KEEPALIVE_PERIOD_MS" --persisted true -s "$HOME/.termux/boot/start-sshd" } +known_boot_script() { + cmp -s "$1" "$DOTFILES_DIR/units/termux-start-sshd" || cmp -s "$1" <(printf '%s\n' \ + '#!/data/data/com.termux/files/usr/bin/sh' \ + '# Run at boot via Termux:Boot and periodically via Android'"'"'s JobScheduler.' \ + '# Termux:Boot and Termux:API are separate companion apps.' \ + 'termux-wake-lock' 'sshd') +} check_termux_packages() { local installed pkg line state name found installed="$(dpkg-query -W -f='${db:Status-Status} ${binary:Package}\n' 2>/dev/null)" || { @@ -83,27 +101,28 @@ check_termux_packages() { done } check_termux_boot() { - PYTHONDONTWRITEBYTECODE=1 python3 - "$DOTFILES_DIR/units/termux-start-sshd" "$HOME/.termux/boot" <<'PY' + local known=0 + if [ -f "$HOME/.termux/boot/start-sshd" ] && known_boot_script "$HOME/.termux/boot/start-sshd"; then known=1; fi + PYTHONDONTWRITEBYTECODE=1 python3 - "$HOME/.termux/boot" "$known" <<'PY' import os, stat, sys -src, directory = sys.argv[1:] +directory, known = sys.argv[1:] dest = os.path.join(directory, 'start-sshd') try: - with open(src, 'rb') as f: - expected = f.read() try: info = os.stat(directory) directory_ok = stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o755 except FileNotFoundError: directory_ok = False print(('ok ' if directory_ok else 'WOULD create or chmod ') + 'Termux boot directory') + if os.path.islink(dest) or (os.path.lexists(dest) and not os.path.isfile(dest)): + print('Cannot replace non-regular Termux sshd boot script', file=sys.stderr) + sys.exit(1) try: info = os.stat(dest) - if stat.S_ISREG(info.st_mode): - with open(dest, 'rb') as f: - same = f.read() == expected - same = same and stat.S_IMODE(info.st_mode) == 0o755 - else: - same = False + if known != '1': + print('Cannot replace customized Termux sshd boot script', file=sys.stderr) + sys.exit(1) + same = stat.S_IMODE(info.st_mode) == 0o755 except FileNotFoundError: same = False print(('ok ' if same else 'WOULD install or chmod ') + 'Termux sshd boot script') @@ -124,10 +143,11 @@ check_termux_sshd() { check_termux_keepalive() { local pending pending="$(termux-job-scheduler -p 2>/dev/null)" || { echo 'Cannot inspect Termux scheduled jobs' >&2; return 1; } - case "$pending" in - *'Pending Job 1'*) note 'ok Termux sshd keep-alive job exists (period not checked by apply)' ;; - *) note "WOULD schedule Termux sshd keep-alive every $TERMUX_KEEPALIVE_PERIOD_MS ms" ;; - esac + if pending_keepalive "$pending"; then + note 'ok Termux sshd keep-alive job exists (period not checked by apply)' + else + note "WOULD schedule Termux sshd keep-alive every $TERMUX_KEEPALIVE_PERIOD_MS ms" + fi } check_termux_overrides() { local file rc @@ -169,7 +189,29 @@ else step 'read Termux package configuration' false fi step 'Termux boot directory' install -d -m 0755 "$HOME/.termux/boot" -step 'Termux sshd boot script' install_if_changed "$DOTFILES_DIR/units/termux-start-sshd" "$HOME/.termux/boot/start-sshd" 0755 +boot_ready=0 +boot_script="$HOME/.termux/boot/start-sshd" +if [ -L "$boot_script" ] || { [ -e "$boot_script" ] && [ ! -f "$boot_script" ]; }; then + echo 'Cannot replace non-regular Termux sshd boot script' >&2 + step 'Termux sshd boot script' false +elif [ -e "$boot_script" ] && ! known_boot_script "$boot_script"; then + echo 'Cannot replace customized Termux sshd boot script' >&2 + step 'Termux sshd boot script' false +elif [ -e "$boot_script" ]; then + if chmod 0755 "$boot_script"; then + note 'ok Termux sshd boot script' + boot_ready=1 + else + step 'Termux sshd boot script' false + fi +elif install_if_changed "$DOTFILES_DIR/units/termux-start-sshd" "$boot_script" 0755; then + note 'ok Termux sshd boot script' + boot_ready=1 +else + step 'Termux sshd boot script' false +fi step 'Termux sshd running' ensure_sshd -[ "$config_ok" = 0 ] || step 'Termux sshd keep-alive' schedule_keepalive +if [ "$config_ok" = 1 ] && [ "$boot_ready" = 1 ]; then + step 'Termux sshd keep-alive' schedule_keepalive +fi finish diff --git a/units/termux-start-sshd b/units/termux-start-sshd index d4709521..c607ca24 100755 --- a/units/termux-start-sshd +++ b/units/termux-start-sshd @@ -1,5 +1,10 @@ #!/data/data/com.termux/files/usr/bin/sh -# Run at boot via Termux:Boot and periodically via Android's JobScheduler. -# Termux:Boot and Termux:API are separate companion apps. +# Deployed by termux_packages role. Runs twice over: once at device boot +# via Termux:Boot, and periodically via Android's JobScheduler +# (termux-job-scheduler) so sshd comes back even if Android killed +# Termux in the background between opens — a phone isn't an always-on +# machine. Requires the Termux:Boot and Termux:API companion apps +# installed separately (not available via pkg — install from +# F-Droid/GitHub releases). termux-wake-lock sshd diff --git a/units/zen-browser.sh b/units/zen-browser.sh index 7a9c6738..c791c1a0 100755 --- a/units/zen-browser.sh +++ b/units/zen-browser.sh @@ -57,13 +57,12 @@ if os.environ.get('UNITS_CHECK') == '1': get = subprocess.run(['gsettings', 'get', 'org.gnome.shell', 'favorite-apps'], capture_output=True, text=True, check=True) value = get.stdout.strip() -if os.environ.get('UNITS_CHECK') == '1': - if value.startswith('@as '): - value = value[4:] - if not value: - raise ValueError('empty dock state') -apps = ast.literal_eval(value) if value else [] -if os.environ.get('UNITS_CHECK') == '1' and (not isinstance(apps, list) or not all(isinstance(app, str) for app in apps)): +if value.startswith('@as '): + value = value[4:] +if not value: + raise ValueError('empty dock state') +apps = ast.literal_eval(value) +if not isinstance(apps, list) or not all(isinstance(app, str) for app in apps): raise ValueError('invalid dock state') new = [app for app in apps if app not in { 'org.mozilla.firefox.desktop', 'org.mozilla.Thunderbird.desktop'}] From a5019b15e6b64effb5663bf91af0f9c2e859795a Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:12:45 +0000 Subject: [PATCH 37/62] Secure fleet SSH trust and fix onboarding registration Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- docs/src/roles/ssh_keys.md | 10 ++- docs/src/roles/ssh_mesh.md | 4 +- docs/src/scripts.md | 2 +- scripts/bw-seed-ssh-keys.sh | 4 +- scripts/inventory_parser.py | 12 ++- scripts/onboard-machine.sh | 2 + tests/test_inventory_scripts.py | 12 +++ tests/test_onboarding_pr.py | 2 + tests/test_ssh_check_credentials.py | 2 +- tests/test_ssh_native_units.py | 135 ++++++++++++++++++++++++++-- units/ssh-fleet.py | 117 +++++------------------- 12 files changed, 191 insertions(+), 113 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eaba53ac..69599e2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: - uses: extractions/setup-just@v4 - name: pytest - run: uvx --with pyyaml pytest -q tests + run: uvx --with pyyaml --with paramiko pytest -q tests osquery-smoke: strategy: diff --git a/docs/src/roles/ssh_keys.md b/docs/src/roles/ssh_keys.md index 05fe0676..05ad37bc 100644 --- a/docs/src/roles/ssh_keys.md +++ b/docs/src/roles/ssh_keys.md @@ -6,6 +6,14 @@ If the private key is absent, the unit requires an unlocked vault and a successf Passphrase-protected OpenSSH identities are not prompted for. Existing identities and vault restores are retained when the public key matches the encrypted key's public envelope, but the private contents cannot be verified without the passphrase. The envelope is unauthenticated, so the unit will not create or replace a Bitwarden key from that unverified identity, even with forced rotation. A missing, invalid or mismatched public key remains an error. -The unit preserves user entries outside its marked SSH config and `authorized_keys` blocks. Vault public keys and `extra_authorized_keys` are merged into the authorization block; previous managed entries remain by default. `SSH_KEYS_FORCE_AUTHORIZED=true` (or `ssh_keys_force_authorized: true` in host or group variables) permits removing obsolete entries only within that block. A locked vault, disabled secrets, or an empty vault key list leaves vault authorization unchanged. Host-key scans only replace an entry after a usable scan; failed scans leave existing entries intact. The retired `matrix` and `telengana` hosts are never scanned. Existing signers are retained. +The unit preserves user entries outside its marked SSH config and `authorized_keys` blocks. Vault public keys and `extra_authorized_keys` are merged into the Bitwarden block; committed keys are managed separately by [ssh_mesh](ssh_mesh.md). Previous managed entries remain by default. `SSH_KEYS_FORCE_AUTHORIZED=true` (or `ssh_keys_force_authorized: true` in host or group variables) permits removing obsolete entries in each managed block when that unit runs; a locked vault or an empty vault key list leaves the Bitwarden block unchanged. Existing signers are retained. + +## Host trust + +The unit never scans or changes `known_hosts`; a network keyscan cannot authenticate a host. Managed SSH config uses `StrictHostKeyChecking yes`, and mesh probes enforce it explicitly. Before connecting to a new host, obtain its host-key fingerprint over an independent authenticated channel (such as its console), compare it with the candidate key, and add only the verified key to `~/.ssh/known_hosts` under the actual hostname and port (`[host]:port` for a non-default port). If a key changes, do not overwrite the old pin until the change has been independently verified; then replace it explicitly. User-supplied SSH settings can override the managed default. + +## Revoking access + +Remove the key from every source that should no longer authorize it, including `machine_pubkeys`, the corresponding Bitwarden SSH item and `extra_authorized_keys`. With an unlocked vault and a nonempty fleet key list, run both SSH units with `SSH_KEYS_FORCE_AUTHORIZED=true`; `just apply-tags ssh_keys` selects both. Inspect `authorized_keys` afterwards: old unmarked entries, including entries written by earlier versions, are deliberately never attributed or deleted automatically. If the verified obsolete key also appears outside the marked blocks, manually remove each such line after confirming it is not independently required. Keep other user-managed lines unchanged. `--no-secrets` skips vault access; a missing identity is deferred rather than generated. Config and other independent steps still run. The separate [ssh_mesh](ssh_mesh.md) unit adds committed fleet public keys and checks connectivity. diff --git a/docs/src/roles/ssh_mesh.md b/docs/src/roles/ssh_mesh.md index dde9dead..5fafa778 100644 --- a/docs/src/roles/ssh_mesh.md +++ b/docs/src/roles/ssh_mesh.md @@ -1,7 +1,7 @@ # ssh_mesh -`units/ssh-mesh.sh` invokes `units/ssh-fleet.py` under the same SSH lock as [ssh_keys](ssh_keys.md). It skips `machine_profile: vm-test` and `skip_ssh_mesh: true`. It merges valid `machine_pubkeys` from `group_vars/all.yml` into `~/.ssh/authorized_keys` without removing existing user or vault entries, and warns if the local public key is missing or stale in git. +`units/ssh-mesh.sh` invokes `units/ssh-fleet.py` under the same SSH lock as [ssh_keys](ssh_keys.md). It skips `machine_profile: vm-test` and `skip_ssh_mesh: true`. It merges valid `machine_pubkeys` from `group_vars/all.yml` into a marked block in `~/.ssh/authorized_keys`, retaining previous managed keys unless explicitly forced and preserving unmarked user and vault entries. It warns if the local public key is missing or stale in git. See [ssh_keys](ssh_keys.md#revoking-access) for revocation, including unmarked legacy entries. -The unit probes online tailnet peers selected by `scripts/online_hosts.py --peers`, intersected with `machines` and excluding the local host, `ssh_mesh_exclude`, inventory VPSes (including nested members), and the retired `matrix` and `telengana` hosts. A host-specific `ssh_mesh_exclude` overrides the group default. It uses effective inventory group and host port, login and tailnet-name overrides, bypasses connection multiplexing, and on failure closes an existing mux, attempts a fresh ed25519 host-key scan, and retries. Existing known-host entries are not removed unless the scan succeeds. Persistent failures fail the unit and name the peers. +The unit probes online tailnet peers selected by `scripts/online_hosts.py --peers`, intersected with `machines` and excluding the local host, `ssh_mesh_exclude`, inventory VPSes (including nested members), and the retired `matrix` and `telengana` hosts. A host-specific `ssh_mesh_exclude` overrides the group default. It uses effective inventory group and host port, login and tailnet-name overrides and bypasses connection multiplexing. Probes require an independently verified host key with `StrictHostKeyChecking=yes`; missing or changed pins fail closed. On failure it closes an existing mux and retries without modifying `known_hosts`. Persistent failures fail the unit and name the peers. Normally no vault is needed. If the local identity is missing, the shared implementation consults Bitwarden before restoring or creating it; locked vault or `--no-secrets` defers the identity and probes instead of generating an unmirrored key. Add new public keys to `machine_pubkeys` and host entries to `machines`; configure `tailnet_name` and exclusions where needed. diff --git a/docs/src/scripts.md b/docs/src/scripts.md index 027b5674..752ae3d6 100644 --- a/docs/src/scripts.md +++ b/docs/src/scripts.md @@ -24,7 +24,7 @@ The shared unit library skips retired `matrix` and `telengana` before unit work. - `scripts/bw-unlock.sh` resolves a local session and prints the raw token. Avoid exposing token output. Sessions are not guaranteed portable between hosts. - `scripts/bw-seed-kube.sh` upserts the `kubeconfig` and `talosconfig` secure notes from local files, then syncs. `KUBECONFIG` and `TALOSCONFIG` can override the default paths. These credentials contain cluster PKI; do not commit them. - `scripts/bw-seed-omp-broker.sh` seeds the omp broker token. See the [omp reference](roles/omp.md). -- `scripts/bw-seed-ssh-keys.sh` is a legacy helper with a hardcoded machine list. It reads private and public keys locally or over SSH, writes SSH-key items and may register GitHub keys, then writes a chezmoi `dot_ssh/authorized_keys` source file. Review its hardcoded hosts and destination before use; it does not use the native fleet unit's vault item naming or managed-block merge. The normal key lifecycle is documented in [ssh_keys](roles/ssh_keys.md). +- `scripts/bw-seed-ssh-keys.sh` is a legacy helper with a hardcoded machine list. It reads private and public keys locally or over SSH, writes SSH-key items and may register GitHub keys, then writes a chezmoi `dot_ssh/authorized_keys` source file. Remote host keys must already be verified and trusted. Review its hardcoded hosts and destination before use; it does not use the native fleet unit's vault item naming or managed-block merge. The normal key lifecycle is documented in [ssh_keys](roles/ssh_keys.md). ## Other helpers diff --git a/scripts/bw-seed-ssh-keys.sh b/scripts/bw-seed-ssh-keys.sh index c87bf069..acfb368b 100755 --- a/scripts/bw-seed-ssh-keys.sh +++ b/scripts/bw-seed-ssh-keys.sh @@ -128,7 +128,7 @@ fetch_keys() { public_key=$(cat "${priv_path}.pub") else # Fetch remotely via SSH - local ssh_opts=(-o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new -o BatchMode=yes) + local ssh_opts=(-o ConnectTimeout=5 -o StrictHostKeyChecking=yes -o BatchMode=yes) local remote_cmd=' priv="$HOME/.ssh/'"$machine"'_id_ed25519" [ ! -f "$priv" ] && priv="$HOME/.ssh/id_ed25519" @@ -143,7 +143,7 @@ fetch_keys() { local output if ! output=$(ssh "${ssh_opts[@]}" "${SSH_USER}@${host}" "$remote_cmd" 2>/dev/null); then - echo " ✗ SSH unreachable — skipping." + echo " ✗ SSH unreachable or host key not trusted — verify the host key before retrying." return fi diff --git a/scripts/inventory_parser.py b/scripts/inventory_parser.py index 413fc050..17590bd2 100644 --- a/scripts/inventory_parser.py +++ b/scripts/inventory_parser.py @@ -151,11 +151,17 @@ def register_host(name: str, mtype: str = "desktop", inv_path: str = "inventory. group_marker = f" {mtype}:\n hosts:\n" if group_marker not in content: raise UnknownGroupError(mtype) - if "all:\n hosts:\n" not in content: - raise ValueError("inventory has no supported all.hosts section") + # Allow comments and blank lines between all: and hosts:, but never insert + # into another group or accept an ambiguous all.hosts section. + host_sections = list(re.finditer( + r"^all:[ \t]*(?:\#.*)?\n(?:[ \t]*(?:\#.*)?\n)* hosts:[ \t]*(?:\#.*)?\n", + content, re.MULTILINE)) + if len(host_sections) != 1: + raise ValueError("inventory has no unambiguous supported all.hosts section") host_entry = f" {name}:\n ansible_host: localhost\n ansible_connection: local\n" - content = content.replace("all:\n hosts:\n", f"all:\n hosts:\n{host_entry}", 1) + offset = host_sections[0].end() + content = content[:offset] + host_entry + content[offset:] content = content.replace(group_marker, f"{group_marker} {name}:\n", 1) _write(inv_path, content) diff --git a/scripts/onboard-machine.sh b/scripts/onboard-machine.sh index 8242006c..c2622570 100755 --- a/scripts/onboard-machine.sh +++ b/scripts/onboard-machine.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Propose an inventory-only registration without modifying the caller's checkout. set -euo pipefail +# Validation imports checkout modules; leave the caller's tree untouched. +export PYTHONDONTWRITEBYTECODE=1 repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) name=${1:?machine name required} type=${2:-desktop} diff --git a/tests/test_inventory_scripts.py b/tests/test_inventory_scripts.py index 7057cf13..f9d1a5e1 100644 --- a/tests/test_inventory_scripts.py +++ b/tests/test_inventory_scripts.py @@ -287,6 +287,18 @@ def test_register_preserves_comments(self) -> None: content = self.inv_path.read_text() self.assertIn("# Production hosts", content) self.assertIn("# Development and test environment", content) + inventory = yaml.safe_load(content) + self.assertIn("prod1", inventory["all"]["hosts"]) + self.assertEqual(inventory["all"]["hosts"]["newhost"], { + "ansible_host": "localhost", "ansible_connection": "local"}) + self.assertIn("newhost", inventory["all"]["children"]["desktop"]["hosts"]) + + def test_registration_rejects_missing_all_hosts_without_writing(self) -> None: + content = "all:\n children:\n desktop:\n hosts:\n" + self.inv_path.write_text(content) + with self.assertRaises(ValueError): + register_host("newhost", "desktop", str(self.inv_path)) + self.assertEqual(self.inv_path.read_text(), content) def test_purge_preserves_comments(self) -> None: """purge_host preserves comments in file.""" diff --git a/tests/test_onboarding_pr.py b/tests/test_onboarding_pr.py index b56be909..c57879a6 100644 --- a/tests/test_onboarding_pr.py +++ b/tests/test_onboarding_pr.py @@ -45,6 +45,7 @@ def test_registration_pr_does_not_change_checkout_or_push_master(tmp_path, stub_ termux = subprocess.run(['bash', str(repo / 'scripts/onboard-machine.sh'), 'termux', 'termux_hosts'], env=env, capture_output=True, text=True) assert termux.returncode == 0, termux.stdout + termux.stderr + assert not (repo / 'scripts/__pycache__').exists() new_mobile = subprocess.run(['bash', str(repo / 'scripts/onboard-machine.sh'), 'new-phone', 'termux_hosts'], env=env, capture_output=True, text=True) assert new_mobile.returncode != 0 @@ -54,6 +55,7 @@ def test_registration_pr_does_not_change_checkout_or_push_master(tmp_path, stub_ command = ['bash', str(repo / 'scripts/onboard-machine.sh'), 'newhost', 'desktop'] first = subprocess.run(command, env=env, capture_output=True, text=True) assert first.returncode == 3, first.stdout + first.stderr + assert not (repo / 'scripts/__pycache__').exists() assert git('status', '--porcelain', cwd=repo) == before assert (repo / 'inventory.yml').read_bytes() == inventory_before assert 'newhost:' not in git('--git-dir', remote, 'show', 'master:inventory.yml') diff --git a/tests/test_ssh_check_credentials.py b/tests/test_ssh_check_credentials.py index 65dbbc5e..4a43e31c 100644 --- a/tests/test_ssh_check_credentials.py +++ b/tests/test_ssh_check_credentials.py @@ -30,7 +30,7 @@ def test_check_plans_local_ssh_drift_without_writes_or_secret_access(fleet, mode if mode == "ssh-keys": assert "audit events" in result.stdout and "would write" in result.stdout else: - assert "authentication, retries and host-key refresh" in result.stdout + assert "authentication and retries" in result.stdout def test_missing_identity_check_does_not_read_vault_or_create_directory(fleet): diff --git a/tests/test_ssh_native_units.py b/tests/test_ssh_native_units.py index 3440b775..b858df8d 100644 --- a/tests/test_ssh_native_units.py +++ b/tests/test_ssh_native_units.py @@ -110,8 +110,9 @@ def test_restores_vault_identity_preserves_user_files_and_idempotency(fleet): assert fleet["public"] in (ssh / "authorized_keys").read_text() assert "other namespaces=" in (ssh / "allowed_signers").read_text() known = (ssh / "known_hosts").read_text() - assert hashed not in known and "unmanaged " + fleet["public"] in known - assert "[peer]:2222 " + " ".join(fleet["other_public"].split()[:2]) in known + assert known == f"{hashed} {fleet['public']}\n[peer]:2222,unmanaged {fleet['public']}\n" + assert not (home / "scans").exists() + assert "out-of-band verification" in result.stdout calls = (home / "bw-calls").read_text() assert "secret-session" not in calls and "secret-session" not in result.stdout + result.stderr assert fleet["private"] not in result.stdout + result.stderr @@ -186,10 +187,12 @@ def test_mesh_uses_vault_key_merges_git_keys_and_excludes_vps(fleet): assert calls.count("alice@peer true") == 2 assert "-O exit -p 2222 alice@peer" in calls assert "retired" not in calls - assert "[peer]:2222" in (ssh / "known_hosts").read_text() + assert "StrictHostKeyChecking=yes" in calls + assert not (ssh / "known_hosts").exists() + assert not (home / "scans").exists() -def test_mesh_failed_scan_preserves_known_hosts_and_reports_failure(fleet): +def test_mesh_authentication_failure_preserves_known_hosts_and_reports_failure(fleet): ssh = fleet["home"] / ".ssh" ssh.mkdir() (ssh / "id_ed25519").write_text(fleet["private"]) @@ -197,7 +200,7 @@ def test_mesh_failed_scan_preserves_known_hosts_and_reports_failure(fleet): (ssh / "id_ed25519.pub").write_text(fleet["public"] + "\n") old = "[peer]:2222 " + fleet["public"] + "\n" (ssh / "known_hosts").write_text(old) - result = execute(fleet, "ssh-mesh", UNITS_NO_SECRETS="1", SSH_FAIL="1", SCAN_FAIL="1") + result = execute(fleet, "ssh-mesh", UNITS_NO_SECRETS="1", SSH_FAIL="1") assert result.returncode != 0 assert "SSH mesh broken to: remote" in result.stdout assert (ssh / "known_hosts").read_text() == old @@ -380,7 +383,7 @@ def test_encrypted_identity_without_public_key_is_not_replaced(fleet): @pytest.mark.parametrize("check", [False, True]) -def test_ssh_host_key_scans_never_contact_retired_vps(fleet, check): +def test_ssh_unit_never_scans_unverified_host_keys(fleet, check): variables = fleet["variables"] variables["machines"].pop("retired") variables["machines"].update({"matrix": {"hostname": "matrix.reilly.asia"}, @@ -388,8 +391,8 @@ def test_ssh_host_key_scans_never_contact_retired_vps(fleet, check): (fleet["data"] / "all.yml.json").write_text(json.dumps(variables)) result = execute(fleet, UNITS_NO_SECRETS="1", UNITS_CHECK="1" if check else "0") assert result.returncode == 0, result.stdout + result.stderr - scans = (fleet["home"] / "scans").read_text().splitlines() - assert len(scans) == 1 and scans[0].endswith(" peer") + assert not (fleet["home"] / "scans").exists() + assert "out-of-band verification" in result.stdout @pytest.mark.parametrize("check", [False, True]) @@ -474,3 +477,119 @@ def test_nested_inventory_group_vars_and_vps_are_respected(fleet): assert result.returncode == 0, result.stdout + result.stderr calls = (fleet["home"] / "ssh-calls").read_text() assert "host-user@peer" in calls and "retired" not in calls + + +def test_managed_ssh_config_requires_verified_host_keys(fleet): + ssh_client = shutil.which("ssh") + if not ssh_client: + pytest.skip("OpenSSH client required") + result = execute(fleet, UNITS_NO_SECRETS="1") + assert result.returncode == 0, result.stdout + result.stderr + config = fleet["home"] / ".ssh/config" + effective = subprocess.run([ssh_client, "-G", "-F", str(config), "remote"], + text=True, capture_output=True, check=True) + assert "stricthostkeychecking true" in effective.stdout.splitlines() + + +def test_committed_key_revocation_preserves_unowned_and_vault_keys(fleet): + home = fleet["home"] + ssh = home / ".ssh" + ssh.mkdir() + legacy = fleet["other_public"] + " legacy-user-entry" + (ssh / "authorized_keys").write_text(legacy + "\n") + first = execute(fleet, "ssh-mesh") + assert first.returncode == 0, first.stdout + first.stderr + contents = (ssh / "authorized_keys").read_text() + assert "# BEGIN dotfiles committed SSH keys" in contents + assert fleet["other_public"] + "\n" in contents + vault_key = {"id": "remote-1", "name": "james@remote", "sshKey": {"publicKey": fleet["other_public"]}} + (home / "items.json").write_text(json.dumps([json.loads((home / "item.json").read_text()), vault_key])) + assert execute(fleet).returncode == 0 + del fleet["variables"]["machine_pubkeys"]["remote"] + (fleet["data"] / "all.yml.json").write_text(json.dumps(fleet["variables"])) + assert execute(fleet, "ssh-mesh").returncode == 0 + assert fleet["other_public"] + "\n" in (ssh / "authorized_keys").read_text() + assert execute(fleet, "ssh-mesh", SSH_KEYS_FORCE_AUTHORIZED="true").returncode == 0 + contents = (ssh / "authorized_keys").read_text() + git_block = contents.split("# BEGIN dotfiles committed SSH keys\n", 1)[1].split("# END dotfiles committed SSH keys", 1)[0] + vault_block = contents.split("# BEGIN dotfiles Bitwarden SSH keys\n", 1)[1].split("# END dotfiles Bitwarden SSH keys", 1)[0] + assert fleet["other_public"] not in git_block + assert fleet["other_public"] in vault_block and legacy in contents + own = json.loads((home / "item.json").read_text()) + (home / "items.json").write_text(json.dumps([own])) + assert execute(fleet, SSH_KEYS_FORCE_AUTHORIZED="true").returncode == 0 + contents = (ssh / "authorized_keys").read_text() + assert legacy in contents + vault_block = contents.split("# BEGIN dotfiles Bitwarden SSH keys\n", 1)[1].split("# END dotfiles Bitwarden SSH keys", 1)[0] + assert fleet["other_public"] not in vault_block + + +@pytest.mark.parametrize("pinned", [False, True]) +def test_mesh_rejects_unverified_and_changed_host_keys(fleet, tmp_path, pinned): + paramiko = pytest.importorskip("paramiko") + import socket + import threading + + real_ssh = shutil.which("ssh") + if not real_ssh: + pytest.skip("OpenSSH client required") + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(3) + listener.settimeout(20) + port = listener.getsockname()[1] + attempts = [] + + class Server(paramiko.ServerInterface): + def check_auth_none(self, username): + attempts.append(username) + return paramiko.AUTH_FAILED + + def check_auth_publickey(self, username, key): + attempts.append(username) + return paramiko.AUTH_FAILED + + def serve(): + for _ in range(2): + try: + client, _ = listener.accept() + transport = paramiko.Transport(client) + transport.add_server_key(paramiko.Ed25519Key.from_private_key_file(str(tmp_path / "other"))) + try: + transport.start_server(server=Server()) + transport.accept(5) + except (EOFError, OSError, paramiko.SSHException): + pass + finally: + transport.close() + except OSError: + break + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + ssh = fleet["home"] / ".ssh" + ssh.mkdir() + (ssh / "id_ed25519").write_text(fleet["private"]) + (ssh / "id_ed25519").chmod(0o600) + (ssh / "id_ed25519.pub").write_text(fleet["public"] + "\n") + known = ssh / "known_hosts" + original = f"[127.0.0.1]:{port} {fleet['public']}\n" if pinned else "" + if pinned: + known.write_text(original) + (ssh / "config").write_text(f"Host *\n UserKnownHostsFile {known}\n GlobalKnownHostsFile /dev/null\n") + fleet["variables"]["machines"]["remote"]["hostname"] = "127.0.0.1" + (fleet["data"] / "all.yml.json").write_text(json.dumps(fleet["variables"])) + inventory = json.loads((fleet["data"] / "inventory.yml.json").read_text()) + inventory["all"]["hosts"]["remote"]["ansible_port"] = port + (fleet["data"] / "inventory.yml.json").write_text(json.dumps(inventory)) + bindir = Path(fleet["env"]["PATH"].split(":", 1)[0]) + write_stub(bindir, "ssh", 'exec "$REAL_SSH" -F "$HOME/.ssh/config" "$@"\n') + try: + result = execute(fleet, "ssh-mesh", UNITS_NO_SECRETS="1", REAL_SSH=real_ssh) + finally: + listener.close() + thread.join(timeout=3) + assert result.returncode != 0 and "SSH mesh broken to: remote" in result.stdout + assert not attempts + assert known.read_text() == original if pinned else not known.exists() + assert not (fleet["home"] / "scans").exists() diff --git a/units/ssh-fleet.py b/units/ssh-fleet.py index 51d9f0e8..ac71db44 100755 --- a/units/ssh-fleet.py +++ b/units/ssh-fleet.py @@ -2,8 +2,6 @@ """SSH units' shared, locked implementation; fleet data remains in inventory/vars.""" import base64 import fcntl -import hashlib -import hmac import json import os import re @@ -24,6 +22,8 @@ END = "# END dotfiles SSH fleet" AK_BEGIN = "# BEGIN dotfiles Bitwarden SSH keys" AK_END = "# END dotfiles Bitwarden SSH keys" +GIT_AK_BEGIN = "# BEGIN dotfiles committed SSH keys" +GIT_AK_END = "# END dotfiles committed SSH keys" RETIRED = {"matrix", "telengana"} CHECK = os.environ.get("UNITS_CHECK") == "1" PLANNED = {} @@ -400,7 +400,7 @@ def config(self): lines = ["Host *", " AddKeysToAgent yes", " IdentityFile ~/.ssh/id_ed25519", " ServerAliveInterval 30", " ServerAliveCountMax 3", " ControlMaster auto", " ControlPersist 10m", " ControlPath ~/.ssh/cp/%C", " HashKnownHosts no", - " Compression yes", ""] + " Compression yes", " StrictHostKeyChecking yes", ""] for name, info in self.machines.items(): host = self.hosts.get(name) or {} lines += [f"Host {name}", f" HostName {info['hostname']}"] @@ -462,25 +462,29 @@ def authorize_vault(self): previous = managed.splitlines() if force.lower() not in ("true", "1", "yes"): keys = previous + keys - outside = before + after - existing = set(outside.splitlines()) - unique = list(dict.fromkeys(k.strip() for k in keys if k.strip() and k.strip() not in existing)) + before_git, _, after_git, _ = split_block(before + after, GIT_AK_BEGIN, GIT_AK_END) + unowned = set((before_git + after_git).splitlines()) + unique = list(dict.fromkeys(k.strip() for k in keys if k.strip() and k.strip() not in unowned)) atomic(path, replace_block(old, AK_BEGIN, AK_END, unique), 0o600) def authorize_git(self): keys = self.vars.get("machine_pubkeys") or {} path = SSH / "authorized_keys" old = text(path) - before, _, after, _ = split_block(old, AK_BEGIN, AK_END) - outside = before + after - existing = set(outside.splitlines()) + before, managed, after, _ = split_block(old, GIT_AK_BEGIN, GIT_AK_END) for key in keys.values(): key_parts(key) - if key not in existing: - old += ("\n" if old and not old.endswith("\n") else "") + key + "\n" - existing.add(key) - if keys: - atomic(path, old, 0o600) + force = os.environ.get("SSH_KEYS_FORCE_AUTHORIZED", str(self.host_vars.get("ssh_keys_force_authorized", self.vars.get("ssh_keys_force_authorized", "false")))) + desired = list(keys.values()) + if force.lower() not in ("true", "1", "yes"): + desired = managed.splitlines() + desired + # Leave unmarked entries untouched; legacy lines cannot safely be + # distinguished from deliberately user-managed access. + before_vault, _, after_vault, _ = split_block(before + after, AK_BEGIN, AK_END) + unowned = set((before_vault + after_vault).splitlines()) + unique = list(dict.fromkeys(k.strip() for k in desired if k.strip() and k.strip() not in unowned)) + if keys or managed: + atomic(path, replace_block(old, GIT_AK_BEGIN, GIT_AK_END, unique), 0o600) if self.key_ready: own = text(SSH / "id_ed25519.pub") committed = keys.get(MACHINE, "") @@ -492,68 +496,10 @@ def host_data(self, name): host = self.hosts.get(name) or {} return info["hostname"], int(host.get("ansible_port", 22)), host.get("ansible_user", self.login) - def scan(self, name): - host, port, _ = self.host_data(name) - kh = host if port == 22 else f"[{host}]:{port}" - result = run(["ssh-keyscan", "-t", "ed25519", "-T", "5", "-p", str(port), host], timeout=8) - if result.returncode: - return False - keys = [] - for line in result.stdout.splitlines(): - parts = line.split() - if len(parts) == 3 and parts[0] == kh: - key_parts(" ".join(parts[1:])) - keys.append(line) - if not keys: - return False - path = SSH / "known_hosts" - if path.is_symlink(): - raise RuntimeError("refusing to replace symlinked known_hosts") - old = text(path) - def matches(token): - if not token.startswith("|1|"): - return token.lower() == kh.lower() - try: - _, version, salt, digest = token.split("|") - if version != "1": - return False - actual = hmac.new(base64.b64decode(salt, validate=True), kh.encode(), hashlib.sha1).digest() - return hmac.compare_digest(actual, base64.b64decode(digest, validate=True)) - except ValueError: - return False - - kept, current = [], [] - for line in old.splitlines(keepends=True): - parts = line.split(None, 1) - if len(parts) != 2 or parts[0].startswith(("#", "@")): - kept.append(line) - continue - names = parts[0].split(",") - if not any(matches(token) for token in names): - kept.append(line) - continue - current.append(parts[1].strip()) - remaining = [token for token in names if not matches(token)] - if remaining: - kept.append(",".join(remaining) + " " + parts[1]) - if sorted(current) == sorted(line.split(None, 1)[1] for line in keys): - if CHECK: - atomic(path, old, 0o644) - else: - path.chmod(0o644) - return True - content = "".join(kept) - content += ("\n" if content and not content.endswith("\n") else "") + "\n".join(keys) + "\n" - atomic(path, content, 0o644) - return True - def known_hosts(self): - for name in self.machines: - if name != MACHINE and name not in RETIRED: - try: - self.scan(name) - except (OSError, ValueError, KeyError, subprocess.SubprocessError, RuntimeError) as exc: - print(f"warn: cannot refresh host key for {name}: {exc}") + # Network keyscan is unauthenticated: neither first-use nor changed + # keys can be trusted without out-of-band operator verification. + print("PARTIAL: SSH host keys require out-of-band verification; known_hosts left unchanged") def mesh(self): if not self.key_ready: @@ -571,7 +517,7 @@ def mesh(self): host, port, user = self.host_data(name) target = f"{user}@{host}" args = ["ssh", "-o", "BatchMode=yes", "-o", "PasswordAuthentication=no", - "-o", "ControlMaster=no", "-o", "ControlPath=none", "-o", "StrictHostKeyChecking=accept-new", + "-o", "ControlMaster=no", "-o", "ControlPath=none", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=8", "-p", str(port), target, "true"] try: probe = run(args, timeout=15) @@ -583,11 +529,6 @@ def mesh(self): run(["ssh", "-O", "exit", "-p", str(port), target], timeout=5) except subprocess.TimeoutExpired: pass - # Never remove the old known-host entry unless a usable scan succeeds. - try: - self.scan(name) - except (OSError, ValueError, KeyError, subprocess.SubprocessError, RuntimeError) as exc: - print(f"warn: cannot refresh host key for {name}: {exc}") try: if run(args, timeout=15).returncode == 0: continue @@ -651,18 +592,6 @@ def ensure_key(self): print(f"would set {public} mode 0644") self.key_ready = True - def known_hosts(self): - failed = False - for name in self.machines: - if name != MACHINE and name not in RETIRED: - try: - if not self.scan(name): - raise RuntimeError("no usable host key returned") - except (OSError, ValueError, KeyError, subprocess.SubprocessError, RuntimeError) as exc: - print(f"FAIL cannot inspect host key for {name}: {exc}") - failed = True - if failed: - raise RuntimeError("one or more SSH host keys could not be inspected") def mesh(self): if not self.key_ready: @@ -688,7 +617,7 @@ def mesh(self): and (self.hosts.get(name) or {}).get("tailnet_name", name).lower() in online] print("online SSH mesh targets: " + (", ".join(targets) if targets else "none")) if targets: - raise RuntimeError("SSH authentication, retries and host-key refresh cannot safely be probed in check mode") + raise RuntimeError("SSH authentication and retries cannot safely be probed in check mode") def check_main(mode): From 2606126bbe7a4c89191e4bd2c483422a52e93ba4 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:37:55 +0000 Subject: [PATCH 38/62] Prepare bounded K3s egress and restore serial recovery Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/home-k8s-images.md | 9 +- .../src/servers/talos-k8s/karnataka/README.md | 136 +++++++++++++-- home-k3s/firewall.sh | 81 ++++++++- server-images/compose.sh | 7 + server-images/installer/configure.sh | 8 +- .../hardware.conf | 4 + server-images/test-vm.py | 4 + tests/test_home_k3s_firewall.py | 164 +++++++++++++++--- 8 files changed, 365 insertions(+), 48 deletions(-) create mode 100644 server-images/rootfs/usr/lib/systemd/system/serial-getty@ttyS0.service.d/hardware.conf diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index 4dcbe276..b9881bb1 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -5,9 +5,12 @@ These are proposed bootable K3s images, not the current OS or workloads of eithe The Utah-based images built in CI; the starter ISO was about 5.94 GiB. Acceptance allows media up to 8 GiB, but the publication workflow still rejects release assets of 2 GiB or more ([issue #351](https://github.com/hanthor/dotfiles/issues/351)). -In [CI](https://github.com/hanthor/dotfiles/actions/runs/40602288941), the +In [CI](https://github.com/hanthor/dotfiles/actions/runs/40605226540), the starter installer completed, but its installed VM never presented a serial login within ten minutes; installation and cluster acceptance did not pass. +Utah's installed image retained a masked serial getty even though live media +unmasked it. The image build now enables a normal authenticated serial getty; +this change still needs a complete acceptance run before use. ## Image Variants @@ -28,6 +31,10 @@ The intended images include the following; existing published tags may contain o - **GPU:** AMD firmware + amdgpu kernel driver included by the image build - **Filesystems:** ext4, xfs, btrfs (all tools included) - **Default state:** Multi-user target (text console), GDM masked +- **Serial recovery:** `serial-getty@ttyS0` uses normal authentication and + starts only when the kernel reports a detected UART. Root autologin is + restricted to the live installer, not the installed system. Physical serial + recovery remains unverified. ## Obtaining Images diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index 7465572c..96749590 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -57,16 +57,132 @@ K3s SELinux until the policy and workload labeling are verified. The default local-path volumes are node-local, without snapshot or expansion support; deleting a claim deletes its data. -Firewalld assigns `cni0` to the `internal` zone, permitting forwarding within -that zone. A rich rule permits `10.42.0.0/16` to reach the Tailscale address on -TCP port 10250 for kubelet metrics. The `internal` zone's default services, -including SSH, are also reachable from pods; do not attach an untrusted -interface to that zone. The public LAN interface remains in `public`. Both -bridge rules are present in the permanent configuration. On Karnataka, -`sudo bash home-k3s/firewall.sh --check` inspects both runtime and permanent -state without changes. `--apply` is an explicit repair operation; it does not -reload firewalld and refuses unrelated interfaces or source bindings in the -`internal` zone. Keep a recovery path available for any firewall change. +Firewalld assigns `cni0` to `internal`, permitting forwarding within that +zone. A rich rule permits `10.42.0.0/16` to reach the Tailscale address on TCP +10250 for kubelet metrics. The `internal` zone's default services, including +SSH, are also reachable from pods; do not attach an untrusted interface to that +zone. The LAN interfaces are in `public`. Both bridge rules were present in +the permanent configuration when last inspected. + +A runtime-only experiment enabling the existing `gateway-lan-to-world` policy +made pod HTTPS work. That policy accepts all traffic from `internal`, `home` +and `trusted` to both `public` and `external`, so it was disabled again and +must not be enabled as the fix. `home-k3s/firewall.sh` instead defines a +`k3s-pod-egress` policy: `internal` ingress, `public` egress, `CONTINUE` +target, and one IPv4 accept rule for source `10.42.0.0/16`. K3s supplies pod +masquerading; the policy does not alter NAT or host INPUT rules. It accepts all +protocols and ports from that CIDR, not just HTTPS: `public` includes the LAN +as well as Internet routes, so pods can reach LAN addresses. Source addresses +can be spoofed; this is not per-pod isolation. IPv6, other pod CIDRs and +non-`public` egress zones are not covered. Other firewalld/K3s rules may also +allow traffic; this policy is not a comprehensive egress firewall. + +The new policy has **not been applied or verified on Karnataka**. The host +has no console recovery, and a rollback timer has not been proven there. Do +not treat the checked-in policy as a persistent fix until the guarded rollout +below succeeds. `sudo bash home-k3s/firewall.sh --check` inspects both stores +without changes. `--apply` refuses foreign interface/source bindings and policy +drift instead of taking over a broader policy. Firewalld 2.5.2 can create a +policy only in the permanent store, so the first `--apply` stages it there but +returns nonzero while the policy is absent at runtime. It never reloads +firewalld. Interrupted creation may leave a partial policy; a later apply can +finish it. Other policies are not changed. + +### Guarded rollout and rollback + +First establish console recovery, retain an existing Tailscale SSH session, +and make sure a second independent SSH session can connect. Do not reload or +restart firewalld. From a trusted checkout on Karnataka, prove that a root +systemd transient timer actually fires before relying on one (the probe only +creates a marker in `/run`): + +```bash +sudo test ! -e /run/k3s-fw-probe +sudo systemd-run --unit=k3s-fw-probe --on-active=5s /usr/bin/touch /run/k3s-fw-probe +sleep 10 +sudo test -f /run/k3s-fw-probe +sudo rm /run/k3s-fw-probe +``` + +Inspect `sudo firewall-cmd --list-all-zones` and `sudo firewall-cmd +--list-all-policies`, then repeat both with `--permanent`. Source bindings +take precedence over interface bindings; investigate any source overlapping +the pod CIDR or any other forwarding policy that could change the expected +scope. The script checks the `internal` bindings and its own policy, not all +other policies. Stop if the resulting exposure is not understood. + +Stop if the marker does not appear. Before scheduling rollback, require the +policy to be absent in **both** stores and the existing bridge setup already +complete, so the rollback below reverts every change this apply can make: + +```bash +sudo bash -s <<'CHECK' +set -euo pipefail +rule='rule family="ipv4" source address="10.42.0.0/16" destination address="100.66.158.127/32" port port="10250" protocol="tcp" accept' +firewall-cmd --state >/dev/null +[[ $(firewall-cmd --get-default-zone) != internal ]] +for scope in runtime permanent; do + opts=(); [[ $scope == runtime ]] || opts=(--permanent) + [[ $(firewall-cmd "${opts[@]}" --zone=internal --list-interfaces) == cni0 ]] + [[ -z $(firewall-cmd "${opts[@]}" --zone=internal --list-sources) ]] + [[ " $(firewall-cmd "${opts[@]}" --get-policies) " != *' k3s-pod-egress '* ]] + firewall-cmd --quiet "${opts[@]}" --zone=internal --query-interface=cni0 + firewall-cmd --quiet "${opts[@]}" --zone=internal --query-forward + firewall-cmd --quiet "${opts[@]}" --zone=internal --query-rich-rule="$rule" +done +CHECK +``` + +If any prerequisite fails, stop and investigate; this rollback is only for a +new permanent policy, not for changes to the bridge zone. Schedule the +rollback and confirm the timer is active **before** staging. It deletes only +the new permanent policy and does not reload firewalld: + +```bash +sudo systemd-run --unit=k3s-egress-rollback --on-active=15m \ + /usr/bin/firewall-cmd --permanent --delete-policy=k3s-pod-egress +sudo systemctl is-active --quiet k3s-egress-rollback.timer +sudo systemctl list-timers k3s-egress-rollback.timer +``` + +Stop if the timer is not active. Then run `sudo bash home-k3s/firewall.sh +--apply`. On first staging, a nonzero result and `Permanent k3s-pod-egress +staged; runtime policy is absent` are expected; any other error means stop and +leave the timer armed. Inspect the staged policy and permanent configuration: + +```bash +sudo firewall-cmd --permanent --info-policy=k3s-pod-egress +sudo firewall-cmd --check-config +sudo bash home-k3s/firewall.sh --check +``` + +The policy must show target `CONTINUE`, priority `-1`, only `internal` ingress, +only `public` egress, and only the documented IPv4 rich rule; the last command +must report `Missing policy k3s-pod-egress (runtime)` and exit nonzero. If +anything else fails, leave the timer armed. Otherwise cancel the timer well +before it expires with `sudo systemctl stop k3s-egress-rollback.timer` and +confirm it is inactive. If it fires, inspect `journalctl -u +k3s-egress-rollback.service` and confirm the policy is absent with `sudo +firewall-cmd --permanent --get-policies`. If rollback fails, use the console +to run `sudo firewall-cmd --permanent --delete-policy=k3s-pod-egress`. + +The staged policy will only become active when firewalld next starts from its +permanent configuration. Do not reload or restart it live. Arrange a controlled +reboot only with console recovery available, then confirm a **new** Tailscale +SSH connection, run `sudo bash home-k3s/firewall.sh --check`, and test pod DNS +plus HTTPS, for example with a disposable pod: + +```bash +sudo /var/usrlocal/bin/k3s kubectl --server https://100.66.158.127:6443 \ + run firewall-egress-probe --rm -i --restart=Never --image=curlimages/curl:8.10.1 \ + --command -- curl -4 -fsS --max-time 15 -o /dev/null https://example.com/ +``` + +If the post-boot checks fail, use the console, delete the permanent policy with +`sudo firewall-cmd --permanent --delete-policy=k3s-pod-egress`, and reboot +under console supervision to remove it from runtime. Removing a permanent +policy does not change runtime. Do not use `--runtime-to-permanent`, which +would overwrite unrelated permanent configuration. K3s logged `iptables-save` nftables parsing errors during network-policy reconciliation. No NetworkPolicy objects were present when checked. Policy diff --git a/home-k3s/firewall.sh b/home-k3s/firewall.sh index 8faeeaf9..d6f9206e 100755 --- a/home-k3s/firewall.sh +++ b/home-k3s/firewall.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# Explicitly manage only cni0 in firewalld's existing internal zone. The -# internal zone's services, including SSH, remain reachable from pods. -# Neither mode reloads firewalld or changes public/tailscale interfaces. +# Manage cni0 and a source-scoped pod egress policy without reloading firewalld. +# The internal zone's existing services, including SSH, remain reachable +# from pods. Public and Tailscale interface assignments are not changed. set -euo pipefail case "${1:-}" in @@ -14,6 +14,8 @@ esac zone=internal rule='rule family="ipv4" source address="10.42.0.0/16" destination address="100.66.158.127/32" port port="10250" protocol="tcp" accept' +policy=k3s-pod-egress +egress_rule='rule family="ipv4" source address="10.42.0.0/16" accept' firewall-cmd --state >/dev/null # A source binding would classify packets by address even on a public NIC. @@ -37,6 +39,38 @@ preflight() { preflight runtime preflight permanent +# Refuse to take over an existing policy whose effects extend beyond this +# script's contract. Partially created policies are safe to finish: their +# target is CONTINUE and each populated field must already be the desired one. +policy_preflight() { + local scope=$1 policies info line target=0 priority=0 disabled=0 + local -a opts=() + [ "$scope" = runtime ] || opts=(--permanent) + policies=$(firewall-cmd "${opts[@]}" --get-policies) + [[ " $policies " == *" $policy "* ]] || return 0 + info=$(firewall-cmd "${opts[@]}" --info-policy="$policy") + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + case "$line" in + "$policy"|"$policy (active)"|'') ;; + 'disable: no') disabled=1 ;; + 'priority: -1') priority=1 ;; + 'target: CONTINUE') target=1 ;; + 'ingress-zones:'|'ingress-zones: internal'|'egress-zones:'|'egress-zones: public'|\ + 'services:'|'ports:'|'protocols:'|'masquerade: no'|'forward-ports:'|\ + 'source-ports:'|'icmp-blocks:'|'rich rules:'|"$egress_rule") ;; + *) echo "$scope: unexpected $policy setting: $line; refusing" >&2; return 1 ;; + esac + done <<< "$info" + if [ "$target" != 1 ] || [ "$priority" != 1 ] || [ "$disabled" != 1 ]; then + echo "$scope: cannot establish safe $policy defaults; refusing" >&2 + return 1 + fi +} +policy_preflight runtime +policy_preflight permanent + # A false query exits 1; other errors must not be mistaken for missing rules. ensure() { local query=$1 add=$2 rc @@ -67,13 +101,52 @@ check_or_apply() { done } check_or_apply + +check_or_apply_policy() { + local scope policies + local -a opts + for scope in permanent runtime; do + opts=() + [ "$scope" = runtime ] || opts=(--permanent) + policies=$(firewall-cmd "${opts[@]}" --get-policies) + if [[ " $policies " != *" $policy "* ]]; then + if [ "$mode" = --check ]; then + echo "Missing policy $policy ($scope)" >&2 + return 1 + fi + if [ "$scope" = runtime ]; then + echo "Permanent $policy staged; runtime policy is absent. Activate only with console recovery; no reload was performed." >&2 + return 1 + fi + # firewalld can create a policy only in permanent configuration. + firewall-cmd --permanent --new-policy="$policy" + fi + # Install the bounded rule before activating the policy with both zones. + ensure "--query-rich-rule=$egress_rule" "--add-rich-rule=$egress_rule" "${opts[@]}" --policy="$policy" + ensure --query-ingress-zone=internal --add-ingress-zone=internal "${opts[@]}" --policy="$policy" + ensure --query-egress-zone=public --add-egress-zone=public "${opts[@]}" --policy="$policy" + # Check that successful mutations actually produced the bounded policy, + # including when the runtime policy has not been loaded yet. + policy_preflight "$scope" + local previous_mode=$mode + mode=--check + ensure "--query-rich-rule=$egress_rule" "--add-rich-rule=$egress_rule" "${opts[@]}" --policy="$policy" + ensure --query-ingress-zone=internal --add-ingress-zone=internal "${opts[@]}" --policy="$policy" + ensure --query-egress-zone=public --add-egress-zone=public "${opts[@]}" --policy="$policy" + mode=$previous_mode + done +} +check_or_apply_policy if [ "$mode" = --apply ]; then # Verify the actual runtime and permanent state, not just command success. [ "$(firewall-cmd --get-default-zone)" != "$zone" ] || { echo 'internal became the default zone' >&2; exit 1; } preflight runtime preflight permanent + policy_preflight runtime + policy_preflight permanent mode=--check check_or_apply + check_or_apply_policy fi -echo 'internal cni0 forwarding and kubelet rule are present in runtime and permanent configuration' +echo 'internal cni0 forwarding, kubelet rule and pod egress policy are present in runtime and permanent configuration' diff --git a/server-images/compose.sh b/server-images/compose.sh index 42ce4f49..ca4b8e77 100755 --- a/server-images/compose.sh +++ b/server-images/compose.sh @@ -36,6 +36,13 @@ setsebool -P container_use_devices on /usr/bin/k3s --version firewall-offline-cmd --check-config systemctl enable NetworkManager.service NetworkManager-wait-online.service sshd.service firewalld.service tailscaled.service iscsi-init.service iscsid.service +# Utah masks this instance in both /etc and /usr. Installed server images need +# an authenticated serial console for headless recovery, not live autologin. +systemctl unmask serial-getty@ttyS0.service +if [ "$(readlink /usr/lib/systemd/system/serial-getty@ttyS0.service || :)" = /dev/null ]; then + rm /usr/lib/systemd/system/serial-getty@ttyS0.service +fi +systemctl enable serial-getty@ttyS0.service # Reboots of the only control plane must be deliberate and preceded by backups. systemctl mask bootc-fetch-apply-updates.timer bootc-fetch-apply-updates.service diff --git a/server-images/installer/configure.sh b/server-images/installer/configure.sh index f45f2543..a7b060fe 100755 --- a/server-images/installer/configure.sh +++ b/server-images/installer/configure.sh @@ -6,11 +6,9 @@ dnf5 -y --enablerepo=fedora-storage --setopt=install_weak_deps=False \ install dracut-live systemd-boot-unsigned # These masks belong to live media only, never to the embedded server payload. systemctl mask k3s@server.service k3s@agent.service -# Utah masks this instance in both /etc and /usr; only the live image needs it. -systemctl unmask getty@tty1.service serial-getty@ttyS0.service -if [ "$(readlink /usr/lib/systemd/system/serial-getty@ttyS0.service || :)" = /dev/null ]; then - rm /usr/lib/systemd/system/serial-getty@ttyS0.service -fi +# Keep the physical console and serial console available on live media. The +# payload already enables a normal serial getty; root autologin is live-only. +systemctl unmask getty@tty1.service systemctl enable getty@tty1.service serial-getty@ttyS0.service mkdir -p /var/roothome # The bootc payload clears /var; dracut requires a writable build scratch area. diff --git a/server-images/rootfs/usr/lib/systemd/system/serial-getty@ttyS0.service.d/hardware.conf b/server-images/rootfs/usr/lib/systemd/system/serial-getty@ttyS0.service.d/hardware.conf new file mode 100644 index 00000000..8cab0eac --- /dev/null +++ b/server-images/rootfs/usr/lib/systemd/system/serial-getty@ttyS0.service.d/hardware.conf @@ -0,0 +1,4 @@ +# Utah masks ttyS0 because absent UARTs can otherwise respawn agetty forever. +# /dev/ttyS0 may exist even when the kernel reports no UART at that port. +[Service] +ExecCondition=/usr/bin/sh -c '/usr/bin/grep -q "^0: uart:" /proc/tty/driver/serial && ! /usr/bin/grep -q "^0: uart:unknown" /proc/tty/driver/serial' diff --git a/server-images/test-vm.py b/server-images/test-vm.py index 17095afb..eb631eb9 100755 --- a/server-images/test-vm.py +++ b/server-images/test-vm.py @@ -218,6 +218,8 @@ def switch_state_script(): test "$(cat /proc/sys/kernel/hostname)" = starter-smoke test ! -e /etc/bootc-installer/live-iso-mode test ! -e /usr/bin/bootc-installer-tui +test ! -e /etc/systemd/system/serial-getty@ttyS0.service.d/installer.conf +systemctl is-active serial-getty@ttyS0.service ''' + inactive_k3s() + '''bootc status --format json > /run/home-k8s-bootc-status.json printf '\\nBOOTC_BEGIN\\n' cat /run/home-k8s-bootc-status.json @@ -424,6 +426,8 @@ def install(args): test "$(getenforce)" = Enforcing test ! -e /etc/bootc-installer/live-iso-mode test ! -e /usr/bin/bootc-installer-tui +test ! -e /etc/systemd/system/serial-getty@ttyS0.service.d/installer.conf +systemctl is-active serial-getty@ttyS0.service {inactive_k3s()} {role_check} systemctl is-active tailscaled diff --git a/tests/test_home_k3s_firewall.py b/tests/test_home_k3s_firewall.py index 9f06a4f1..3f8d335b 100644 --- a/tests/test_home_k3s_firewall.py +++ b/tests/test_home_k3s_firewall.py @@ -7,6 +7,21 @@ SCRIPT = REPO / "home-k3s" / "firewall.sh" +POLICY = "k3s-pod-egress" +EGRESS_RULE = 'rule family="ipv4" source address="10.42.0.0/16" accept' +KUBELET_RULE = ( + 'rule family="ipv4" source address="10.42.0.0/16" ' + 'destination address="100.66.158.127/32" port port="10250" protocol="tcp" accept' +) + + +def empty_policy(): + return { + "target": "CONTINUE", "priority": "-1", "disable": "no", + "ingress-zones": [], "egress-zones": [], "services": [], "ports": [], + "protocols": [], "masquerade": "no", "forward-ports": [], + "source-ports": [], "icmp-blocks": [], "rich rules": [], + } @pytest.fixture @@ -24,10 +39,40 @@ def firewall(tmp_path, stub_env): args = sys.argv[1:] scope = "permanent" if "--permanent" in args else "runtime" zone = s[scope] +policy_name = next((a.split("=", 1)[1] for a in args if a.startswith("--policy=")), None) if "--state" in args: print("running") elif "--get-default-zone" in args: print(s["default"]) +elif "--get-policies" in args: + print(" ".join(zone["policies"])) +elif any(a.startswith("--info-policy=") for a in args): + name = next(a.split("=", 1)[1] for a in args if a.startswith("--info-policy=")) + policy = zone["policies"][name] + active = bool(policy["ingress-zones"] and policy["egress-zones"]) + print(name + (" (active)" if active else "")) + for key in ("disable", "priority", "target", "ingress-zones", "egress-zones", + "services", "ports", "protocols", "masquerade", "forward-ports", + "source-ports", "icmp-blocks", "rich rules"): + value = policy[key] + if key == "rich rules": + print(" rich rules: " + ("\n\t" + "\n\t".join(value) if value else "")) + else: + print(" " + key + ": " + (" ".join(value) if isinstance(value, list) else value)) +elif any(a.startswith("--new-policy=") for a in args): + if scope != "permanent": + sys.exit(2) + op = next(a for a in args if a.startswith("--new-policy=")) + name = op.split("=", 1)[1] + zone["policies"][name] = { + "target": "CONTINUE", "priority": "-1", "disable": "no", + "ingress-zones": [], "egress-zones": [], "services": [], "ports": [], + "protocols": [], "masquerade": "no", "forward-ports": [], + "source-ports": [], "icmp-blocks": [], "rich rules": [], + } + s["changes"].append([scope, op]) + with open(p, "w") as f: + json.dump(s, f) elif "--list-interfaces" in args: print(" ".join(zone["interfaces"])) elif "--list-sources" in args: @@ -35,21 +80,32 @@ def firewall(tmp_path, stub_env): else: op = next(a for a in args if a.startswith(("--query-", "--add-", "--change-"))) command, _, value = op.partition("=") - key = {"--query-rich-rule": "rule", "--add-rich-rule": "rule", - "--query-forward": "forward", "--add-forward": "forward", - "--query-interface": "interfaces", "--change-interface": "interfaces"}[command] - if command.startswith("--query-"): - if os.environ.get("FAIL_QUERY"): - sys.exit(2) - present = value in zone[key] if key == "interfaces" else zone[key] - if key == "rule" and present: - present = value == zone[key] - sys.exit(0 if present else 1) - s["changes"].append([scope, op]) - if key == "interfaces": - zone[key].append(value) + if policy_name: + policy = zone["policies"][policy_name] + key = {"--query-rich-rule": "rich rules", "--add-rich-rule": "rich rules", + "--query-ingress-zone": "ingress-zones", "--add-ingress-zone": "ingress-zones", + "--query-egress-zone": "egress-zones", "--add-egress-zone": "egress-zones"}[command] + if command.startswith("--query-"): + if os.environ.get("FAIL_QUERY"): + sys.exit(2) + sys.exit(0 if value in policy[key] else 1) + policy[key].append(value) else: - zone[key] = value if key == "rule" else True + key = {"--query-rich-rule": "rule", "--add-rich-rule": "rule", + "--query-forward": "forward", "--add-forward": "forward", + "--query-interface": "interfaces", "--change-interface": "interfaces"}[command] + if command.startswith("--query-"): + if os.environ.get("FAIL_QUERY"): + sys.exit(2) + present = value in zone[key] if key == "interfaces" else zone[key] + if key == "rule" and present: + present = value == zone[key] + sys.exit(0 if present else 1) + if key == "interfaces": + zone[key].append(value) + else: + zone[key] = value if key == "rule" else True + s["changes"].append([scope, op]) with open(p, "w") as f: json.dump(s, f) PY @@ -58,8 +114,8 @@ def firewall(tmp_path, stub_env): path = tmp_path / "firewall.json" state = { "default": "public", - "runtime": {"interfaces": [], "sources": [], "forward": False, "rule": ""}, - "permanent": {"interfaces": [], "sources": [], "forward": False, "rule": ""}, + "runtime": {"interfaces": [], "sources": [], "forward": False, "rule": "", "policies": {}}, + "permanent": {"interfaces": [], "sources": [], "forward": False, "rule": "", "policies": {}}, "changes": [], } path.write_text(json.dumps(state)) @@ -73,20 +129,38 @@ def run(mode): def test_apply_is_scoped_and_idempotent(firewall): - _, _, run = firewall + path, _, run = firewall result, state = run("--apply") - assert result.returncode == 0, result.stderr + assert result.returncode != 0 + assert "runtime policy is absent" in result.stderr + expected = empty_policy() + expected.update({"ingress-zones": ["internal"], "egress-zones": ["public"], + "rich rules": [EGRESS_RULE]}) for scope in ("runtime", "permanent"): zone = state[scope] assert zone["interfaces"] == ["cni0"] assert zone["sources"] == [] assert zone["forward"] is True - result, second = run("--apply") - assert result.returncode == 0, result.stderr - assert second == state + assert state["permanent"]["policies"] == {POLICY: expected} + assert state["runtime"]["policies"] == {} + ops = [op for scope, op in state["changes"] if scope == "permanent"] + assert ops.index("--add-rich-rule=" + EGRESS_RULE) < ops.index("--add-egress-zone=public") + result, unchanged = run("--apply") + assert result.returncode != 0 + assert unchanged == state + result, checked = run("--check") + assert result.returncode != 0 + assert "Missing policy k3s-pod-egress (runtime)" in result.stderr + assert checked == state + # Model firewalld starting from its permanent configuration. + state["runtime"]["policies"][POLICY] = expected + path.write_text(json.dumps(state)) result, checked = run("--check") assert result.returncode == 0, result.stderr assert checked == state + result, unchanged = run("--apply") + assert result.returncode == 0, result.stderr + assert unchanged == state def test_check_reports_missing_without_changes(firewall): @@ -97,21 +171,21 @@ def test_check_reports_missing_without_changes(firewall): assert not state["changes"] - -def test_check_accepts_host_rule_from_real_firewalld(firewall): +def test_check_accepts_host_rule_and_egress_policy(firewall): path, _, run = firewall state = json.loads(path.read_text()) + policy = empty_policy() + policy.update({"ingress-zones": ["internal"], "egress-zones": ["public"], + "rich rules": [EGRESS_RULE]}) for scope in ("runtime", "permanent"): - state[scope].update( - interfaces=["cni0"], forward=True, - rule='rule family="ipv4" source address="10.42.0.0/16" ' - 'destination address="100.66.158.127/32" port port="10250" protocol="tcp" accept', - ) + state[scope].update(interfaces=["cni0"], forward=True, rule=KUBELET_RULE, + policies={POLICY: policy}) path.write_text(json.dumps(state)) result, after = run("--check") assert result.returncode == 0, result.stderr assert after == state + @pytest.mark.parametrize("scope", ["runtime", "permanent"]) @pytest.mark.parametrize("field,value", [("interfaces", "eth0"), ("sources", "10.42.0.0/16")]) def test_foreign_interface_or_source_refused_before_any_changes(firewall, scope, field, value): @@ -125,6 +199,40 @@ def test_foreign_interface_or_source_refused_before_any_changes(firewall, scope, assert after == state +@pytest.mark.parametrize("scope", ["runtime", "permanent"]) +@pytest.mark.parametrize("field,value", [ + ("target", "ACCEPT"), ("priority", "1000"), ("disable", "yes"), + ("ingress-zones", ["internal", "home"]), ("egress-zones", ["ANY"]), + ("services", ["ssh"]), ("masquerade", "yes"), ("rich rules", ["rule accept"]), +]) +def test_foreign_policy_refused_before_any_changes(firewall, scope, field, value): + path, _, run = firewall + state = json.loads(path.read_text()) + policy = empty_policy() + policy[field] = value + state[scope]["policies"][POLICY] = policy + path.write_text(json.dumps(state)) + result, after = run("--apply") + assert result.returncode != 0 + assert "refusing" in result.stderr + assert after == state + + +def test_partial_policy_is_completed(firewall): + path, _, run = firewall + state = json.loads(path.read_text()) + policy = empty_policy() + policy["rich rules"] = [EGRESS_RULE] + state["permanent"]["policies"][POLICY] = policy + path.write_text(json.dumps(state)) + result, after = run("--apply") + assert result.returncode != 0 + assert "runtime policy is absent" in result.stderr + assert after["permanent"]["policies"][POLICY]["ingress-zones"] == ["internal"] + assert after["permanent"]["policies"][POLICY]["egress-zones"] == ["public"] + assert after["permanent"]["policies"][POLICY]["rich rules"] == [EGRESS_RULE] + + def test_internal_default_zone_refused(firewall): path, _, run = firewall state = json.loads(path.read_text()) From 7c9a279364f0099d362581d8906c0091ffc12d13 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:47:24 +0000 Subject: [PATCH 39/62] Add guarded runtime bridge for pod egress Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- .../src/servers/talos-k8s/karnataka/README.md | 55 +++++ home-k3s/firewall-egress-bridge.sh | 203 ++++++++++++++++++ tests/test_home_k3s_firewall.py | 181 ++++++++++++++++ 3 files changed, 439 insertions(+) create mode 100755 home-k3s/firewall-egress-bridge.sh diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index 96749590..c51ed3c3 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -88,6 +88,61 @@ returns nonzero while the policy is absent at runtime. It never reloads firewalld. Interrupted creation may leave a partial policy; a later apply can finish it. Other policies are not changed. +### Optional temporary runtime bridge + +`home-k3s/firewall-egress-bridge.sh` is a separate, opt-in bridge until the +dedicated permanent policy can be activated. It has **not** been run on +Karnataka. It requires the exact observed disabled `gateway-lan-to-world` +baseline, `internal` containing only `cni0`, `internal` not being the default, +and **no source bindings in any zone in either store**. It narrows the runtime +policy to `internal` → `public`, adds negative-priority drops for non-pod IPv4 +sources and all IPv6, and only then enables it. It does not modify the vendor +policy's permanent configuration or load the staged dedicated policy. + +Unlike the dedicated policy's `CONTINUE` target, the temporary policy's target +is `ACCEPT`; the drop rules restrict that acceptance to `10.42.0.0/16` IPv4. +They can also drop traffic that a later policy would otherwise allow, and +earlier policies or established-connection handling may bypass them. Pod IPs +can be spoofed, all protocols and LAN destinations in `public` are allowed, +and this is not per-pod isolation. Inspect all other policies and both stores' +zone bindings as described below before considering it. A change to source +bindings or `internal` after activation invalidates the assumptions. + +Only with console recovery, a retained Tailscale SSH session, and the systemd +timer probe below proven successful, run: + +```bash +sudo bash home-k3s/firewall-egress-bridge.sh --apply +sudo bash home-k3s/firewall-egress-bridge.sh --status +``` + +The helper schedules and checks a 15-minute rollback timer before any firewall +mutation. It uses a root-owned snapshot under `/run/k3s-egress-bridge`; do not +remove it while the timer is armed. On a command failure it attempts immediate +rollback, leaving the timer armed to retry. The timer disables the gateway +policy first, removes only the two bridge rules, restores the original zone +membership, and checks the disabled baseline. If rollback cannot be verified, +use console recovery and inspect `journalctl -u +k3s-egress-bridge-rollback.service`; never reload firewalld to recover SSH. + +While the timer remains armed, verify a **new** SSH connection and pod DNS and +HTTPS with the disposable pod command below. After those checks and before the +timer expires, an operator may explicitly stop automatic rollback: + +```bash +sudo bash home-k3s/firewall-egress-bridge.sh --commit +sudo bash home-k3s/firewall-egress-bridge.sh --status +``` + +`--commit` verifies the active narrowed policy and bindings before stopping +the timer; it does not persist the bridge. To revert after committing or after +the timer fires, run `sudo bash home-k3s/firewall-egress-bridge.sh --cleanup`; +this restores and verifies the disabled baseline before stopping any timer. +If the timer has already fired, cleanup removes its managed state. On reboot +`/run` and this runtime-only bridge disappear; if the dedicated policy was +staged, it is then loaded from permanent configuration and must be checked +separately with `firewall.sh --check`. + ### Guarded rollout and rollback First establish console recovery, retain an existing Tailscale SSH session, diff --git a/home-k3s/firewall-egress-bridge.sh b/home-k3s/firewall-egress-bridge.sh new file mode 100755 index 00000000..74af696a --- /dev/null +++ b/home-k3s/firewall-egress-bridge.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# Opt-in, temporary runtime bridge. Never reloads firewalld or edits permanent state. +set -Eeuo pipefail + +case "${1:-}" in + --apply|--status|--commit|--cleanup|--timer-rollback) mode=$1 ;; + *) echo "Usage: $0 --apply|--status|--commit|--cleanup" >&2; exit 2 ;; +esac +[ "$#" -eq 1 ] || { echo "Usage: $0 --apply|--status|--commit|--cleanup" >&2; exit 2; } +[ "$(id -u)" = 0 ] || { echo 'Run as root' >&2; exit 1; } +[ "$(hostname -s)" = karnataka ] || { echo 'Wrong host' >&2; exit 1; } + +policy=gateway-lan-to-world +timer=k3s-egress-bridge-rollback.timer +unit=k3s-egress-bridge-rollback +state=${K3S_BRIDGE_STATE_DIR:-/run/k3s-egress-bridge} +rule4='rule priority="-1" family="ipv4" source NOT address="10.42.0.0/16" drop' +rule6='rule priority="-1" family="ipv6" source address="::/0" drop' +fw() { timeout 20 firewall-cmd "$@"; } + +# Compare lists as sets without accepting duplicates or additional entries. +same_words() { + local actual=$1 expected=$2 word + local -a got=() want=() + read -r -a got <<< "$actual" + read -r -a want <<< "$expected" + [ "${#got[@]}" = "${#want[@]}" ] || return 1 + for word in "${want[@]}"; do + [[ " $actual " == *" $word "* ]] || return 1 + done +} + +zone_preflight() { + local scope zone zones interfaces sources + local -a opts=() + [ "$(fw --get-default-zone)" != internal ] || { echo 'internal is the default zone; refusing' >&2; return 1; } + for scope in runtime permanent; do + opts=(); [ "$scope" = runtime ] || opts=(--permanent) + interfaces=$(fw "${opts[@]}" --zone=internal --list-interfaces) + [ "$interfaces" = cni0 ] || { echo "$scope: internal must contain only cni0; refusing" >&2; return 1; } + zones=$(fw "${opts[@]}" --get-zones) + [ -n "$zones" ] || { echo "$scope: no zones returned; refusing" >&2; return 1; } + for zone in $zones; do + sources=$(fw "${opts[@]}" --zone="$zone" --list-sources) + [ -z "$sources" ] || { echo "$scope: $zone has source bindings; refusing" >&2; return 1; } + done + done +} + +# Inspect all printed fields, not just the two rules. Unknown or additional +# settings are drift and must not be enabled or committed by this helper. +policy_state() { + local expected=$1 info line ingress='' egress='' services='' + local target=0 priority=0 disabled=0 seen4=0 seen6=0 + info=$(fw --info-policy="$policy") + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + case "$line" in + "$policy"|"$policy (active)"|"$policy (disabled)"|'') ;; + 'target: ACCEPT') target=1 ;; + 'priority: 1000') priority=1 ;; + 'disable: yes') [ "$expected" != active ] || return 1; disabled=1 ;; + 'disable: no') [ "$expected" = active ] || return 1; disabled=1 ;; + 'ingress-zones:'*) ingress=${line#ingress-zones:}; ingress=${ingress# } ;; + 'egress-zones:'*) egress=${line#egress-zones:}; egress=${egress# } ;; + 'services:'*) services=${line#services:}; services=${services# } ;; + 'ports:'|'protocols:'|'masquerade: no'|'forward-ports:'|'source-ports:'|'icmp-blocks:'|'rich rules:') ;; + "$rule4") seen4=$((seen4 + 1)) ;; + "$rule6") seen6=$((seen6 + 1)) ;; + *) echo "Unexpected $policy setting: $line" >&2; return 1 ;; + esac + done <<< "$info" + [ "$target" = 1 ] && [ "$priority" = 1 ] && [ "$disabled" = 1 ] && same_words "$services" 'ftp tftp' || return 1 + if [ "$expected" = baseline ]; then + same_words "$ingress" 'internal home trusted' && same_words "$egress" 'external public' && [ "$seen4" = 0 ] && [ "$seen6" = 0 ] + else + same_words "$ingress" internal && same_words "$egress" public && [ "$seen4" = 1 ] && [ "$seen6" = 1 ] + fi || { echo "$policy does not match expected $expected state" >&2; return 1; } +} + +# Query false is status 1; never mistake an actual error for false. +ensure_present() { + local query=$1 add=$2 rc + if fw --quiet --policy="$policy" "$query"; then return 0; else rc=$?; fi + [ "$rc" = 1 ] || return "$rc" + fw --policy="$policy" "$add" +} +ensure_absent() { + local query=$1 remove=$2 rc + if fw --quiet --policy="$policy" "$query"; then + fw --policy="$policy" "$remove" + else + rc=$?; [ "$rc" = 1 ] || return "$rc" + fi +} +rollback() { + # Disable first, before widening zones or removing either protective rule. + ensure_present --query-disable --add-disable || return 1 + ensure_absent "--query-rich-rule=$rule4" "--remove-rich-rule=$rule4" || return 1 + ensure_absent "--query-rich-rule=$rule6" "--remove-rich-rule=$rule6" || return 1 + ensure_present --query-ingress-zone=home --add-ingress-zone=home || return 1 + ensure_present --query-ingress-zone=trusted --add-ingress-zone=trusted || return 1 + ensure_present --query-egress-zone=external --add-egress-zone=external || return 1 + policy_state baseline || { echo 'Rollback incomplete; policy is disabled, inspect it from the console' >&2; return 1; } +} + +fw --state >/dev/null +if [ "$mode" = --status ]; then + if [ -d "$state" ]; then + if systemctl is-active --quiet "$timer"; then echo 'Rollback timer: armed'; else echo 'Rollback timer: not armed'; fi + [ ! -e "$state/committed" ] || echo 'Bridge: committed until firewalld restarts or cleanup' + else + echo 'Bridge: no managed runtime state' + fi + fw --info-policy="$policy" + exit 0 +fi + +if [ "$mode" = --apply ]; then + zone_preflight + policy_state baseline || { echo 'Gateway baseline differs; refusing' >&2; exit 1; } + systemctl is-active --quiet "$timer" && { echo 'Rollback timer already active; refusing' >&2; exit 1; } + systemctl is-active --quiet "$unit.service" && { echo 'Rollback service still running; refusing' >&2; exit 1; } + mkdir -m 0700 -- "$state" || { echo 'Bridge state already exists; inspect --status and use --cleanup' >&2; exit 1; } + # The timer runs a private root-owned snapshot, not a mutable checkout. + if ! install -m 0700 -- "$(readlink -f -- "$0")" "$state/helper"; then + rmdir -- "$state" + echo 'Could not prepare rollback snapshot; no firewall changes made' >&2; exit 1 + fi +else + [ -f "$state/helper" ] || { echo 'No managed bridge state; refusing' >&2; exit 1; } +fi +exec 9>"$state/lock" +flock -w 300 9 || { echo 'Could not acquire bridge lock' >&2; exit 1; } + +case "$mode" in + --apply) + # Arm and verify the timer before any runtime firewall mutation. + if ! systemd-run --unit="$unit" --on-active=15m --setenv="K3S_BRIDGE_STATE_DIR=$state" \ + /bin/bash "$state/helper" --timer-rollback; then + echo 'Could not schedule rollback; no firewall changes made' >&2; exit 1 + fi + if ! systemctl is-active --quiet "$timer"; then + echo 'Rollback timer not active; no firewall changes made' >&2; exit 1 + fi + # Do not let the timer touch unrelated drift if no mutation was attempted. + zone_preflight + policy_state baseline + : > "$state/dirty" + on_error() { + local rc=$? + trap - ERR + echo 'Bridge apply failed; restoring disabled gateway baseline' >&2 + if rollback; then + rm -f -- "$state/dirty" + else + echo 'Rollback failed; timer remains armed; use console recovery' >&2 + fi + exit "$rc" + } + trap on_error ERR + fw --policy="$policy" "--add-rich-rule=$rule4" + fw --policy="$policy" "--add-rich-rule=$rule6" + fw --policy="$policy" --remove-ingress-zone=home + fw --policy="$policy" --remove-ingress-zone=trusted + fw --policy="$policy" --remove-egress-zone=external + policy_state prepared + zone_preflight + fw --policy="$policy" --remove-disable + policy_state active + zone_preflight + trap - ERR + echo 'Bridge active; rollback timer is armed. Verify a new SSH session and pod egress before --commit.' + ;; + --commit) + [ -e "$state/dirty" ] || { echo 'No active managed bridge; refusing commit' >&2; exit 1; } + systemctl is-active --quiet "$timer" || { echo 'Rollback timer is not armed; refusing commit' >&2; exit 1; } + zone_preflight + policy_state active || { echo 'Bridge drifted; refusing commit' >&2; exit 1; } + # If the timer already launched, its locked callback sees this marker. + : > "$state/committed" + if ! systemctl stop "$timer"; then + rm -f -- "$state/committed" + echo 'Could not stop rollback timer; bridge remains uncommitted' >&2; exit 1 + fi + echo 'Bridge committed until firewalld restarts; use --cleanup to restore the baseline.' + ;; + --cleanup) + if [ -e "$state/dirty" ]; then rollback; else policy_state baseline; fi + if systemctl is-active --quiet "$timer"; then systemctl stop "$timer"; fi + rm -f -- "$state/dirty" "$state/committed" "$state/helper" "$state/lock" + rmdir -- "$state" + echo 'Gateway baseline restored; rollback timer stopped.' + ;; + --timer-rollback) + [ ! -e "$state/committed" ] || exit 0 + [ -e "$state/dirty" ] || exit 0 + rollback + rm -f -- "$state/dirty" + echo 'Rollback timer restored the gateway baseline.' + ;; +esac diff --git a/tests/test_home_k3s_firewall.py b/tests/test_home_k3s_firewall.py index 3f8d335b..52abde39 100644 --- a/tests/test_home_k3s_firewall.py +++ b/tests/test_home_k3s_firewall.py @@ -1,5 +1,6 @@ import json import subprocess +from pathlib import Path import pytest @@ -249,3 +250,183 @@ def test_query_error_is_not_treated_as_missing(firewall): result, state = run("--apply") assert result.returncode == 2 assert not state["changes"] + + +BRIDGE_SCRIPT = REPO / "home-k3s" / "firewall-egress-bridge.sh" +RULE4 = 'rule priority="-1" family="ipv4" source NOT address="10.42.0.0/16" drop' +RULE6 = 'rule priority="-1" family="ipv6" source address="::/0" drop' + + +@pytest.fixture +def bridge(tmp_path, stub_env): + bindir, env = stub_env + write_stub(bindir, "id", 'echo 0\n') + write_stub(bindir, "hostname", 'echo karnataka\n') + write_stub(bindir, "systemd-run", 'printf "timer\\n" >> "$BRIDGE_LOG"\ntouch "$TIMER_FILE"\n') + write_stub(bindir, "systemctl", r'''case "$1" in + is-active) test -f "$TIMER_FILE" ;; + stop) rm -f "$TIMER_FILE" ;; + *) exit 2 ;; +esac +''') + write_stub(bindir, "firewall-cmd", r'''exec python3 - "$@" <<'PY' +import json, os, sys +p = os.environ["BRIDGE_FIREWALL_STATE"] +s = json.load(open(p)) +args = sys.argv[1:] +scope = "permanent" if "--permanent" in args else "runtime" +policy = s["policy"] +zone = next((a.split("=", 1)[1] for a in args if a.startswith("--zone=")), "") +if "--state" in args: + print("running") +elif "--get-default-zone" in args: + print(s["default"]) +elif "--get-zones" in args: + print("internal public home trusted external") +elif "--list-interfaces" in args: + print(" ".join(s[scope]["interfaces"])) +elif "--list-sources" in args: + print(" ".join(s[scope]["sources"].get(zone, []))) +elif any(a.startswith("--info-policy=") for a in args): + print("gateway-lan-to-world" + (" (disabled)" if policy["disable"] == "yes" else " (active)")) + for key in ("disable", "priority", "target", "ingress-zones", "egress-zones", + "services", "ports", "protocols", "masquerade", "forward-ports", + "source-ports", "icmp-blocks", "rich rules"): + value = policy[key] + if key == "rich rules": + print(" rich rules: " + ("\n\t" + "\n\t".join(value) if value else "")) + else: + print(" " + key + ": " + (" ".join(value) if isinstance(value, list) else value)) +else: + op = next(a for a in args if a.startswith(("--query-", "--add-", "--remove-"))) + command, _, value = op.partition("=") + key = {"--query-disable": "disable", "--add-disable": "disable", "--remove-disable": "disable", + "--query-rich-rule": "rich rules", "--add-rich-rule": "rich rules", "--remove-rich-rule": "rich rules", + "--query-ingress-zone": "ingress-zones", "--add-ingress-zone": "ingress-zones", "--remove-ingress-zone": "ingress-zones", + "--query-egress-zone": "egress-zones", "--add-egress-zone": "egress-zones", "--remove-egress-zone": "egress-zones"}[command] + if command.startswith("--query-"): + sys.exit(0 if (policy[key] == "yes" if key == "disable" else value in policy[key]) else 1) + with open(os.environ["BRIDGE_LOG"], "a") as f: + f.write(op + "\n") + if os.environ.get("FAIL_OP") == command and not os.path.exists(os.environ["FAIL_ONCE"]): + open(os.environ["FAIL_ONCE"], "w").close() + sys.exit(2) + if key == "disable": + policy[key] = "yes" if command == "--add-disable" else "no" + elif command.startswith("--add-"): + policy[key].append(value) + else: + policy[key].remove(value) + with open(p, "w") as f: + json.dump(s, f) +PY +''') + path = tmp_path / "bridge-firewall.json" + policy = empty_policy() + policy.update({"target": "ACCEPT", "priority": "1000", "disable": "yes", + "ingress-zones": ["internal", "home", "trusted"], + "egress-zones": ["external", "public"], "services": ["ftp", "tftp"]}) + path.write_text(json.dumps({ + "default": "public", "policy": policy, + "runtime": {"interfaces": ["cni0"], "sources": {}}, + "permanent": {"interfaces": ["cni0"], "sources": {}}, + })) + env.update({"BRIDGE_FIREWALL_STATE": str(path), "BRIDGE_LOG": str(tmp_path / "ops"), + "TIMER_FILE": str(tmp_path / "timer"), "FAIL_ONCE": str(tmp_path / "failed"), + "K3S_BRIDGE_STATE_DIR": str(tmp_path / "managed")}) + + def run(mode): + result = subprocess.run(["bash", str(BRIDGE_SCRIPT), mode], env=env, text=True, capture_output=True) + return result, json.loads(path.read_text()) + + return path, env, run + + +def test_bridge_arms_before_changes_and_timer_restores(bridge): + path, env, run = bridge + original = json.loads(path.read_text()) + result, state = run("--apply") + assert result.returncode == 0, result.stderr + policy = state["policy"] + assert policy["disable"] == "no" + assert policy["ingress-zones"] == ["internal"] + assert policy["egress-zones"] == ["public"] + assert set(policy["rich rules"]) == {RULE4, RULE6} + operations = Path(env["BRIDGE_LOG"]).read_text().splitlines() + assert operations[0] == "timer" + assert operations.index("--add-rich-rule=" + RULE4) < operations.index("--remove-disable") + assert operations.index("--add-rich-rule=" + RULE6) < operations.index("--remove-disable") + result, restored = run("--timer-rollback") + assert result.returncode == 0, result.stderr + assert set(restored["policy"]["ingress-zones"]) == set(original["policy"]["ingress-zones"]) + assert set(restored["policy"]["egress-zones"]) == set(original["policy"]["egress-zones"]) + assert restored["policy"]["disable"] == "yes" + assert restored["policy"]["rich rules"] == [] + result, _ = run("--cleanup") + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("scope", ["runtime", "permanent"]) +def test_bridge_refuses_source_bindings_in_other_zones(bridge, scope): + path, _, run = bridge + state = json.loads(path.read_text()) + state[scope]["sources"]["public"] = ["10.42.0.0/16"] + path.write_text(json.dumps(state)) + result, unchanged = run("--apply") + assert result.returncode != 0 + assert "source bindings" in result.stderr + assert unchanged == state + + +def test_bridge_failure_restores_before_returning(bridge): + path, env, run = bridge + original = json.loads(path.read_text()) + env["FAIL_OP"] = "--remove-egress-zone" + result, state = run("--apply") + assert result.returncode != 0 + assert state["policy"]["disable"] == "yes" + assert set(state["policy"]["ingress-zones"]) == set(original["policy"]["ingress-zones"]) + assert set(state["policy"]["egress-zones"]) == set(original["policy"]["egress-zones"]) + assert state["policy"]["rich rules"] == [] + + +def test_bridge_commit_keeps_policy_until_cleanup(bridge): + _, _, run = bridge + result, _ = run("--apply") + assert result.returncode == 0, result.stderr + result, active = run("--commit") + assert result.returncode == 0, result.stderr + result, unchanged = run("--timer-rollback") + assert result.returncode == 0, result.stderr + assert unchanged == active + result, restored = run("--cleanup") + assert result.returncode == 0, result.stderr + assert restored["policy"]["disable"] == "yes" + assert restored["policy"]["rich rules"] == [] + + +@pytest.mark.parametrize("field,value", [ + ("target", "CONTINUE"), ("priority", "-1"), ("disable", "no"), + ("ingress-zones", ["internal", "home", "trusted", "public"]), + ("rich rules", [RULE4]), +]) +def test_bridge_refuses_policy_drift_without_mutation(bridge, field, value): + path, env, run = bridge + state = json.loads(path.read_text()) + state["policy"][field] = value + path.write_text(json.dumps(state)) + result, unchanged = run("--apply") + assert result.returncode != 0 + assert unchanged == state + assert not Path(env["TIMER_FILE"]).exists() + + +def test_bridge_never_removes_guards_if_disabling_fails(bridge): + _, env, run = bridge + result, active = run("--apply") + assert result.returncode == 0, result.stderr + env["FAIL_OP"] = "--add-disable" + result, unchanged = run("--cleanup") + assert result.returncode != 0 + assert unchanged == active + assert Path(env["TIMER_FILE"]).exists() From 88bb15345727943fed0a63f7cfe34a12fac02c0e Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:55:36 +0000 Subject: [PATCH 40/62] Fix serial guard and record verified egress Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/home-k8s-images.md | 6 +++-- .../src/servers/talos-k8s/karnataka/README.md | 27 ++++++++++--------- .../hardware.conf | 5 ++-- server-images/test-vm.py | 2 ++ 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index b9881bb1..bad77148 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -9,8 +9,10 @@ In [CI](https://github.com/hanthor/dotfiles/actions/runs/40605226540), the starter installer completed, but its installed VM never presented a serial login within ten minutes; installation and cluster acceptance did not pass. Utah's installed image retained a masked serial getty even though live media -unmasked it. The image build now enables a normal authenticated serial getty; -this change still needs a complete acceptance run before use. +unmasked it. A subsequent [CI run](https://github.com/hanthor/dotfiles/actions/runs/40622083319) +blocked the live serial getty while probing `/proc/tty/driver/serial`. The +image now enables an authenticated serial getty guarded by the kernel's sysfs +UART type; this change still needs a complete acceptance run before use. ## Image Variants diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index c51ed3c3..09f85e7a 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -64,11 +64,10 @@ SSH, are also reachable from pods; do not attach an untrusted interface to that zone. The LAN interfaces are in `public`. Both bridge rules were present in the permanent configuration when last inspected. -A runtime-only experiment enabling the existing `gateway-lan-to-world` policy -made pod HTTPS work. That policy accepts all traffic from `internal`, `home` -and `trusted` to both `public` and `external`, so it was disabled again and -must not be enabled as the fix. `home-k3s/firewall.sh` instead defines a -`k3s-pod-egress` policy: `internal` ingress, `public` egress, `CONTINUE` +The broad `gateway-lan-to-world` baseline accepts traffic from `internal`, +`home` and `trusted` to both `public` and `external` and must not be enabled +unchanged. `home-k3s/firewall.sh` defines a dedicated `k3s-pod-egress` policy: +`internal` ingress, `public` egress, `CONTINUE` target, and one IPv4 accept rule for source `10.42.0.0/16`. K3s supplies pod masquerading; the policy does not alter NAT or host INPUT rules. It accepts all protocols and ports from that CIDR, not just HTTPS: `public` includes the LAN @@ -77,10 +76,11 @@ can be spoofed; this is not per-pod isolation. IPv6, other pod CIDRs and non-`public` egress zones are not covered. Other firewalld/K3s rules may also allow traffic; this policy is not a comprehensive egress firewall. -The new policy has **not been applied or verified on Karnataka**. The host -has no console recovery, and a rollback timer has not been proven there. Do -not treat the checked-in policy as a persistent fix until the guarded rollout -below succeeds. `sudo bash home-k3s/firewall.sh --check` inspects both stores +On 2026-10-28 the dedicated policy was staged in Karnataka's permanent store +without reloading firewalld. Its first activation on a real firewall restart +is unverified. In isolated firewalld 2.5.2 namespaces, staging, reload, and +`--check` succeeded; traffic from `10.42.0.2` forwarded while `10.43.0.2` did +not. `sudo bash home-k3s/firewall.sh --check` inspects both stores without changes. `--apply` refuses foreign interface/source bindings and policy drift instead of taking over a broader policy. Firewalld 2.5.2 can create a policy only in the permanent store, so the first `--apply` stages it there but @@ -90,9 +90,12 @@ finish it. Other policies are not changed. ### Optional temporary runtime bridge -`home-k3s/firewall-egress-bridge.sh` is a separate, opt-in bridge until the -dedicated permanent policy can be activated. It has **not** been run on -Karnataka. It requires the exact observed disabled `gateway-lan-to-world` +`home-k3s/firewall-egress-bridge.sh` provides a separate, opt-in bridge until +the dedicated permanent policy activates. On 2026-10-28 it was enabled on +Karnataka with its rollback timer armed; a new SSH session from Himachal and +pod HTTPS succeeded before the bridge was committed. The timer was stopped on +commit; its automatic rollback has not been exercised on the live host. The +bridge requires the exact observed disabled `gateway-lan-to-world` baseline, `internal` containing only `cni0`, `internal` not being the default, and **no source bindings in any zone in either store**. It narrows the runtime policy to `internal` → `public`, adds negative-priority drops for non-pod IPv4 diff --git a/server-images/rootfs/usr/lib/systemd/system/serial-getty@ttyS0.service.d/hardware.conf b/server-images/rootfs/usr/lib/systemd/system/serial-getty@ttyS0.service.d/hardware.conf index 8cab0eac..c45e1428 100644 --- a/server-images/rootfs/usr/lib/systemd/system/serial-getty@ttyS0.service.d/hardware.conf +++ b/server-images/rootfs/usr/lib/systemd/system/serial-getty@ttyS0.service.d/hardware.conf @@ -1,4 +1,5 @@ # Utah masks ttyS0 because absent UARTs can otherwise respawn agetty forever. -# /dev/ttyS0 may exist even when the kernel reports no UART at that port. +# sysfs exposes the kernel's serial type: PORT_UNKNOWN is zero. Unlike +# /proc/tty/driver/serial, this attribute is readable by systemd services. [Service] -ExecCondition=/usr/bin/sh -c '/usr/bin/grep -q "^0: uart:" /proc/tty/driver/serial && ! /usr/bin/grep -q "^0: uart:unknown" /proc/tty/driver/serial' +ExecCondition=/usr/bin/grep -qxv 0 /sys/class/tty/ttyS0/type diff --git a/server-images/test-vm.py b/server-images/test-vm.py index eb631eb9..92635803 100755 --- a/server-images/test-vm.py +++ b/server-images/test-vm.py @@ -219,6 +219,7 @@ def switch_state_script(): test ! -e /etc/bootc-installer/live-iso-mode test ! -e /usr/bin/bootc-installer-tui test ! -e /etc/systemd/system/serial-getty@ttyS0.service.d/installer.conf +test "$(cat /sys/class/tty/ttyS0/type)" -gt 0 systemctl is-active serial-getty@ttyS0.service ''' + inactive_k3s() + '''bootc status --format json > /run/home-k8s-bootc-status.json printf '\\nBOOTC_BEGIN\\n' @@ -427,6 +428,7 @@ def install(args): test ! -e /etc/bootc-installer/live-iso-mode test ! -e /usr/bin/bootc-installer-tui test ! -e /etc/systemd/system/serial-getty@ttyS0.service.d/installer.conf +test "$(cat /sys/class/tty/ttyS0/type)" -gt 0 systemctl is-active serial-getty@ttyS0.service {inactive_k3s()} {role_check} From 46778131b09fe5a0edb13af5c04461818cbbe577 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:42:38 +0000 Subject: [PATCH 41/62] Document Punjab staging and recovery status Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/punjab/README.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index ba8b22e6..c4816c90 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -9,6 +9,20 @@ same AWS account as the former [AWS Talos cluster](../aws-k8s/cluster.md), but is an ordinary fleet member, not a cluster node. The Talos cluster was terminated; Matrix and the Hives now run on the [OCI VPS](../spindle-vps/README.md). +## Migration status + +The Karnataka `corral-vms/punjab` VM is a staging guest, not the EC2 host. +Its host key was verified against its console; Tailscale enrollment under the +separate name `punjab-migration` requires approval. Data is being copied into +root-restricted `/srv/punjab-migration/` on the guest, not activated as its +home, service state or Tailscale identity. The source remains running. + +EC2 snapshot `snap-0bc709516bb003e9a` was created on 2026-10-28 while Punjab +was live; it is crash-consistent, not a verified application-consistent restore. +Do not stop the source until transfers, final synchronization, recovery and +service checks are complete. Existing tmux processes cannot move live. +S3 contains older backups with no offsite copy and remains retained. + ## Hardware - EC2 `t3.large` (2 vCPU, 8 GiB, unlimited CPU credits), `us-east-1d` @@ -47,9 +61,9 @@ Ubuntu 24.04 LTS (stock `ubuntu-noble-24.04-amd64-server` AMI), AWS kernel. (needs `session-manager-plugin`; the instance ID is in the OpenTofu state). - **Cluster**: `kubectl`/`talosctl` work from here with `~/.kube/config-aws-migration` / `~/.talos/config-aws-migration`. -- **AWS CLI**: prefer the scoped admin user; it can plan the whole config - (read-only IAM). Log in as root (`aws login`) only for IAM changes, and don't - leave a root session lying around on a shared box. +- **AWS CLI**: the scoped admin user can inspect EC2, S3 and costs, but was + denied CloudFormation, ECS, Secrets Manager, API Gateway and SQS inventory. + An authorized administrator session is needed to audit those resources. ## Host hardening From 6e33edb5d3a70a2970303c17ec36546e5f6cbc46 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:20:36 +0000 Subject: [PATCH 42/62] Document Himachal native dotfiles validation Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/desktop/himachal/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/src/desktop/himachal/README.md b/docs/src/desktop/himachal/README.md index a67782a7..7dabce71 100644 --- a/docs/src/desktop/himachal/README.md +++ b/docs/src/desktop/himachal/README.md @@ -11,6 +11,19 @@ Laptop workstation in the hanthor fleet. still specifies `100.73.3.51`; it should not be assumed to identify the reachable node without checking. +## Native dotfiles validation + +On 2026-10-28, the PR #348 branch was applied with `--no-secrets --tags +dotfiles` to `himachal-1`. Bash and headless Neovim started, and the omp broker +status check succeeded. The apply also ran the Neovim plugin sync script. +Atuin was not logged in; its secret-bearing login was not tested. Unlock the +local Bitwarden vault and apply the `atuin` tag to configure sync. + +A mode-0600 pre-apply backup is at +`~/backups/dotfiles-pre-native-20261028T041910Z.tgz`. It includes the omp +configuration and must be treated as secret-bearing. This validation does not +establish that the full fleet or automatic updates have been migrated. + ## Services - **Dotfiles updates:** `host_vars/himachal.yml` sets `is_laptop: true` and From d211ed6b9191872c99359379524d0988dac3131c Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:20:52 +0000 Subject: [PATCH 43/62] Record home installer acceptance result Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/home-k8s-images.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index bad77148..8741837b 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -1,6 +1,6 @@ # Home Kubernetes Server Images -These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to tailnet address `100.66.158.127`; KubeVirt `v1.9.1`, CDI and Corral are installed. The `corral-vms/punjab` VM is staging, not a migrated replacement for active EC2 Punjab. Bihar's current OS and services are unverified. Image build and installer acceptance are under repair: verify provenance and acceptance before any future installation or switch. +These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to tailnet address `100.66.158.127`; KubeVirt `v1.9.1`, CDI and Corral are installed. The `corral-vms/punjab` VM is staging, not a migrated replacement for active EC2 Punjab. Bihar's current OS and services are unverified. Verify image provenance and physical recovery before any future installation or switch. The Utah-based images built in CI; the starter ISO was about 5.94 GiB. Acceptance allows media up to 8 GiB, but the publication workflow still @@ -11,8 +11,10 @@ login within ten minutes; installation and cluster acceptance did not pass. Utah's installed image retained a masked serial getty even though live media unmasked it. A subsequent [CI run](https://github.com/hanthor/dotfiles/actions/runs/40622083319) blocked the live serial getty while probing `/proc/tty/driver/serial`. The -image now enables an authenticated serial getty guarded by the kernel's sysfs -UART type; this change still needs a complete acceptance run before use. +image now guards the authenticated serial getty using the kernel's sysfs UART +type. A subsequent [CI run](https://github.com/hanthor/dotfiles/actions/runs/40623765935) +passed installer acceptance, including authenticated serial login, bootc switch +and a two-node K3s check. Physical serial recovery remains unverified. ## Image Variants From 11b42c27c64894257d7f238064a09b8f92cdd7b5 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:19:41 +0000 Subject: [PATCH 44/62] Record completed Punjab home staging copy Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/punjab/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index c4816c90..c78db64f 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -13,9 +13,11 @@ terminated; Matrix and the Hives now run on the [OCI VPS](../spindle-vps/README. The Karnataka `corral-vms/punjab` VM is a staging guest, not the EC2 host. Its host key was verified against its console; Tailscale enrollment under the -separate name `punjab-migration` requires approval. Data is being copied into -root-restricted `/srv/punjab-migration/` on the guest, not activated as its -home, service state or Tailscale identity. The source remains running. +separate name `punjab-migration` requires approval. The initial home copy +completed on 2026-10-28 into root-restricted `/srv/punjab-migration/` on the +guest; it is not activated as its home, service state or Tailscale identity. +The source remains running, so the copy needs final synchronization and +validation before cutover. EC2 snapshot `snap-0bc709516bb003e9a` was created on 2026-10-28 while Punjab was live; it is crash-consistent, not a verified application-consistent restore. From 3aa6fb837d729bf1ed517e309248bb2388eeb936 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:08:48 +0000 Subject: [PATCH 45/62] Document staging access and isolated gateway smoke Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/punjab/README.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index c78db64f..d229c978 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -12,12 +12,18 @@ terminated; Matrix and the Hives now run on the [OCI VPS](../spindle-vps/README. ## Migration status The Karnataka `corral-vms/punjab` VM is a staging guest, not the EC2 host. -Its host key was verified against its console; Tailscale enrollment under the -separate name `punjab-migration` requires approval. The initial home copy -completed on 2026-10-28 into root-restricted `/srv/punjab-migration/` on the -guest; it is not activated as its home, service state or Tailscale identity. -The source remains running, so the copy needs final synchronization and -validation before cutover. +Its host key was verified against its console. The separate Tailscale device +`punjab-migration` is online at `100.103.187.46`; direct SSH was verified +against that same host key. The initial home copy completed on 2026-10-28 into +root-restricted `/srv/punjab-migration/` on the guest; it is not activated as +its home, service state or Tailscale identity. The source remains running, so +the copy needs final synchronization and validation before cutover. + +The staged Kiro Crew 0.10.0 executable and dashboard were exercised in an +isolated network namespace with a disposable home overlay, cron and tunnels +disabled. The gateway reported ready and its loopback dashboard returned HTTP +200. This is not a test of live credentials, background jobs or migration of +the running source service. EC2 snapshot `snap-0bc709516bb003e9a` was created on 2026-10-28 while Punjab was live; it is crash-consistent, not a verified application-consistent restore. From 1bcfb3a6499a58f0b0e7d1f948c31c033e55daa0 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:09:26 +0000 Subject: [PATCH 46/62] Record Himachal file and backup checks Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/desktop/himachal/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/src/desktop/himachal/README.md b/docs/src/desktop/himachal/README.md index 7dabce71..e51505f2 100644 --- a/docs/src/desktop/himachal/README.md +++ b/docs/src/desktop/himachal/README.md @@ -14,15 +14,18 @@ Laptop workstation in the hanthor fleet. ## Native dotfiles validation On 2026-10-28, the PR #348 branch was applied with `--no-secrets --tags -dotfiles` to `himachal-1`. Bash and headless Neovim started, and the omp broker -status check succeeded. The apply also ran the Neovim plugin sync script. +dotfiles` to `himachal-1`. Bash, Fish and headless Neovim started, the Git +identity matched, and the omp broker status check succeeded. The apply also ran +the Neovim plugin sync script. After correcting an overly restrictive apply +umask, chezmoi reported only its two scheduled scripts. Atuin was not logged in; its secret-bearing login was not tested. Unlock the local Bitwarden vault and apply the `atuin` tag to configure sync. A mode-0600 pre-apply backup is at -`~/backups/dotfiles-pre-native-20261028T041910Z.tgz`. It includes the omp -configuration and must be treated as secret-bearing. This validation does not -establish that the full fleet or automatic updates have been migrated. +`~/backups/dotfiles-pre-native-20261028T041910Z.tgz`. Shell secret files, GitHub +hosts and the omp broker configuration were excluded; protect the backup as +potentially sensitive. This validation does not establish that the full fleet +or automatic updates have been migrated. ## Services From 6d1ee98162e473cda3efb4938299f582c2f82172 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:40:22 +0000 Subject: [PATCH 47/62] Record staging connectivity checks Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/punjab/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index d229c978..d47506ab 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -18,6 +18,8 @@ against that same host key. The initial home copy completed on 2026-10-28 into root-restricted `/srv/punjab-migration/` on the guest; it is not activated as its home, service state or Tailscale identity. The source remains running, so the copy needs final synchronization and validation before cutover. +The guest resolved and reached `api.github.com` over HTTPS and connected to +the Spindle broker's TCP port. These checks do not prove broker authentication. The staged Kiro Crew 0.10.0 executable and dashboard were exercised in an isolated network namespace with a disposable home overlay, cron and tunnels From a02abd51efbb6d7a7f6c4ddee979e11395234eb1 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:41:06 +0000 Subject: [PATCH 48/62] Record Punjab staging comparison Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/punjab/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index d47506ab..a7802e6e 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -18,6 +18,10 @@ against that same host key. The initial home copy completed on 2026-10-28 into root-restricted `/srv/punjab-migration/` on the guest; it is not activated as its home, service state or Tailscale identity. The source remains running, so the copy needs final synchronization and validation before cutover. +A metadata-based dry run after the initial copy found 577 regular files to +transfer (about 996 MB) and 485 new filesystem entries. A follow-up sync is +running; the comparison did not checksum all contents or check for deletions. + The guest resolved and reached `api.github.com` over HTTPS and connected to the Spindle broker's TCP port. These checks do not prove broker authentication. From 62e27932730fd656ce5243e38abde44be5984a0f Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:56:42 +0000 Subject: [PATCH 49/62] Record completed Punjab follow-up sync Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/punjab/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index a7802e6e..f25f36a0 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -19,8 +19,9 @@ root-restricted `/srv/punjab-migration/` on the guest; it is not activated as its home, service state or Tailscale identity. The source remains running, so the copy needs final synchronization and validation before cutover. A metadata-based dry run after the initial copy found 577 regular files to -transfer (about 996 MB) and 485 new filesystem entries. A follow-up sync is -running; the comparison did not checksum all contents or check for deletions. +transfer (about 996 MB) and 485 new filesystem entries. The follow-up sync +completed successfully. The comparison did not checksum all contents or +check for deletions, and the live source can still change. The guest resolved and reached `api.github.com` over HTTPS and connected to the Spindle broker's TCP port. These checks do not prove broker authentication. From be8c861c086ef637c4ff39347acb35c09ec413d4 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:32:15 +0000 Subject: [PATCH 50/62] Remove Kiro Crew and prepare Punjab replacement inventory Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/SUMMARY.md | 1 - docs/src/roles/kirocrew.md | 5 --- docs/src/servers/aws/README.md | 25 +++++++++++ docs/src/servers/punjab/README.md | 13 ++---- host_vars/punjab.yml | 9 ++-- inventory.yml | 6 ++- mise.toml | 3 -- scripts/run-units.sh | 7 ++- tests/test_server_app_parity.py | 71 ------------------------------ tests/test_server_service_units.py | 25 +---------- tests/test_server_units_check.py | 22 ++------- units/kirocrew-sockets.py | 57 ------------------------ units/kirocrew.sh | 66 --------------------------- 13 files changed, 44 insertions(+), 266 deletions(-) delete mode 100644 docs/src/roles/kirocrew.md delete mode 100755 units/kirocrew-sockets.py delete mode 100755 units/kirocrew.sh diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index f336d6ef..b4a41cb6 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -96,7 +96,6 @@ - [proxy (Caddy)](roles/proxy.md) - [tailscale_cert](roles/tailscale_cert.md) - [bst_dashboard](roles/bst_dashboard.md) -- [kirocrew](roles/kirocrew.md) ## Removed / migrated diff --git a/docs/src/roles/kirocrew.md b/docs/src/roles/kirocrew.md deleted file mode 100644 index 761e3644..00000000 --- a/docs/src/roles/kirocrew.md +++ /dev/null @@ -1,5 +0,0 @@ -# kirocrew - -`units/kirocrew.sh` manages an existing [Kiro Crew](https://github.com/kirodotdev/KiroCrew) installation only when `kirocrew_enabled: true` is set in host variables. It skips retired `matrix` and `telengana`. If `~/.local/bin/kirocrew` or `/etc/systemd/system/kirocrew.service` is missing, it skips and prints upstream installation guidance; it does not install either. - -With root or noninteractive `sudo -n`, it makes the existing regular, non-symlink `/etc/kirocrew/kirocrew.env` root-owned mode 0600, then enables and starts the service only if those permissions succeeded. It checks listeners using `ss` and `units/kirocrew-sockets.py`, matching socket owner PIDs to the service's systemd cgroup (including child cgroups), rather than relying on a process name that may be `python3`. An active service with no attributable listener fails the check instead of claiming loopback-only binding. Check mode cannot determine an inactive service's future bindings and says so. The existing environment file contains credentials; this unit neither fetches nor rewrites them. diff --git a/docs/src/servers/aws/README.md b/docs/src/servers/aws/README.md index b0a695d0..e72bc409 100644 --- a/docs/src/servers/aws/README.md +++ b/docs/src/servers/aws/README.md @@ -61,6 +61,31 @@ does not itself create resources, but applying proposed creates can restore infrastructure and incur charges. `aws/terraform.tfvars` remains sensitive; `just aws-seed-tfvars` stores it in Bitwarden after an intentional edit. +### Punjab replacement transition (2026-10-28) + +The fleet inventory's logical `punjab` host now connects to the replacement +guest on Karnataka at `100.103.187.46`, using the verified `ubuntu` account. +Its separate Tailscale name remains **`punjab-migration`** until the service +cutover and identity rename are confirmed; the inventory's `tailnet_name` +override keeps fleet discovery aligned with that staging identity. This is +a connection-target change, not evidence that service migration is complete +or that the original EC2 instance has stopped. + +`aws/punjab.tf` still describes the original EC2 instance at its historical +Tailscale address `100.78.73.8`, plus its EIP, security group, key pair, SSM +role/profile and DLM policy. It does **not** manage the replacement guest. +Those resource declarations and `prevent_destroy` remain unchanged. The +replacement retains AWS administration tools and its existing SSH-hardening +and no-proxy policies; moving it off EC2 does not require removing those tools. + +Retirement must reconcile live AWS resources, OpenTofu state and configuration +separately: deleted resources still declared here can be proposed for recreation, +while stopped instances can retain billable volumes, EIPs and snapshots. +`prevent_destroy` is not a stop/retirement mechanism and does not protect against +recreation. Retain the S3 backups and state; do not use a blanket destroy to +resolve this drift. Verify actual retirement status before updating the +historical inventory above. + ## IAM & access - **Root** historically had MFA and no access keys. Verify current access diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index f25f36a0..bf2047e3 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -26,11 +26,9 @@ check for deletions, and the live source can still change. The guest resolved and reached `api.github.com` over HTTPS and connected to the Spindle broker's TCP port. These checks do not prove broker authentication. -The staged Kiro Crew 0.10.0 executable and dashboard were exercised in an -isolated network namespace with a disposable home overlay, cron and tunnels -disabled. The gateway reported ready and its loopback dashboard returned HTTP -200. This is not a test of live credentials, background jobs or migration of -the running source service. +Kiro Crew was retired at the owner's request. Its source service is disabled +and masked; its private recovery data is retained, not activated on the +replacement. The Kiro CLI and omp integrations are separate and remain. EC2 snapshot `snap-0bc709516bb003e9a` was created on 2026-10-28 while Punjab was live; it is crash-consistent, not a verified application-consistent restore. @@ -59,10 +57,6 @@ Ubuntu 24.04 LTS (stock `ubuntu-noble-24.04-amd64-server` AMI), AWS kernel. ## Role - Agent host: Claude Code, Kiro, `pi`, run in tmux -- **Kiro Crew gateway** — `kirocrew.service` (system unit, runs as `ubuntu`, - binary `~/.local/bin/kirocrew`, env `/etc/kirocrew/kirocrew.env`). Installed - with Kiro's own installer; the [`kirocrew`](../../roles/kirocrew.md) role keeps - it running, its env file root-only, and its dashboard on localhost. - AWS admin box: `awscli` + `opentofu` (via `extra_brews`) for the [AWS IaC](../aws/README.md) @@ -85,7 +79,6 @@ Ubuntu 24.04 LTS (stock `ubuntu-noble-24.04-amd64-server` AMI), AWS kernel. - sshd drop-in (`sshd_harden: true`): no root login, no passwords/kbd-interactive, no X11, `MaxAuthTries 3`. - `unattended-upgrades` on (Ubuntu default). -- Kiro Crew dashboard binds localhost only — the `kirocrew` role asserts it. ## Ansible notes (`host_vars/punjab.yml`) diff --git a/host_vars/punjab.yml b/host_vars/punjab.yml index 25a4f7b1..b3fe5b5a 100644 --- a/host_vars/punjab.yml +++ b/host_vars/punjab.yml @@ -1,13 +1,12 @@ -# Headless agent box: no tailnet Caddy proxy (needs podman quadlets, absent on stock Ubuntu) +# Headless agent box: keep the existing no-proxy policy during migration. skip_proxy: true -# Runs the OpenTofu config in aws/ (the AWS account's IaC) — see docs/src/servers/aws/README.md +# AWS administration tools remain useful on the Karnataka replacement. +# aws/ still describes the old EC2 resources; see docs/src/servers/aws/README.md. extra_brews: - awscli - session-manager-plugin - opentofu -# Internet-facing EC2 box: stricter sshd drop-in (sshd role) +# Retain stricter sshd policy on the replacement (sshd role). sshd_harden: true -# Kiro Crew gateway (kirocrew role) -kirocrew_enabled: true # Always-on fleet observer; reuse this account's existing Tailscale/SSH access. osquery_poller_enabled: true diff --git a/inventory.yml b/inventory.yml index c3fd0605..9449859c 100644 --- a/inventory.yml +++ b/inventory.yml @@ -4,9 +4,11 @@ all: # `tailnet_name`: the host's MagicDNS name when it isn't the inventory # name; scripts/online_hosts.py matches online peers by it. punjab: - ansible_host: 100.78.73.8 - ansible_user: ubuntu # stock Ubuntu AMI login; there is no `james` here + # Karnataka replacement; retain its staging identity until rename is confirmed. + ansible_host: 100.103.187.46 + ansible_user: ubuntu ansible_connection: ssh + tailnet_name: punjab-migration # This targets the Debian VM running via Android's Virtualization # Framework on the phone (tailnet node name "mumbai"), not the raw # Termux/Android layer itself — that's the separate `termux` host below. diff --git a/mise.toml b/mise.toml index cc9d201c..d3136b4d 100644 --- a/mise.toml +++ b/mise.toml @@ -106,8 +106,5 @@ file = "units/tailscale-cert.sh" [tasks."unit:bst-dashboard"] file = "units/bst-dashboard.sh" -[tasks."unit:kirocrew"] -file = "units/kirocrew.sh" - [tasks."unit:fleet-facts"] file = "units/fleet-facts.sh" diff --git a/scripts/run-units.sh b/scripts/run-units.sh index 37b19517..b9097fee 100755 --- a/scripts/run-units.sh +++ b/scripts/run-units.sh @@ -3,7 +3,7 @@ # Usage: scripts/run-units.sh [--no-secrets] [--tags a,b] set -uo pipefail -ALL_UNITS="sudo sshd apk-packages termux-packages homebrew flatpak chezmoi fonts omp atuin shell-ai ssh-keys ssh-mesh github kube forgejo-registry tailscale bluefin gnome zen-browser browser-fxa pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services proxy tailscale-cert bst-dashboard kirocrew fleet-facts" +ALL_UNITS="sudo sshd apk-packages termux-packages homebrew flatpak chezmoi fonts omp atuin shell-ai ssh-keys ssh-mesh github kube forgejo-registry tailscale bluefin gnome zen-browser browser-fxa pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services proxy tailscale-cert bst-dashboard fleet-facts" tasks_for_tag() { case "$1" in system) echo "sudo sshd apk-packages termux-packages retired-services server-hardening" ;; sudo) echo sudo ;; @@ -33,7 +33,7 @@ tasks_for_tag() { case "$1" in gnome) echo gnome ;; browser|zen_browser) echo "zen-browser browser-fxa" ;; audio|pipewire) echo pipewire-audio ;; - services) echo "syncthing laptop-services system-services osquery proxy tailscale-cert bst-dashboard kirocrew" ;; + services) echo "syncthing laptop-services system-services osquery proxy tailscale-cert bst-dashboard" ;; syncthing) echo syncthing ;; systemd) echo "laptop-services system-services" ;; osquery) echo osquery ;; @@ -43,7 +43,6 @@ tasks_for_tag() { case "$1" in proxy|caddy) echo proxy ;; tailscale_cert) echo tailscale-cert ;; bst_dashboard) echo bst-dashboard ;; - kirocrew) echo kirocrew ;; hive|hive_ops) echo hive-ops ;; facts|fleet_facts) echo fleet-facts ;; esac; } @@ -133,7 +132,7 @@ for unit in chezmoi bluefin gnome; do done zen_ok=1; run_stage zen-browser || { zen_ok=0; rc=1; } -later=(fonts omp atuin shell-ai kube forgejo-registry pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services bst-dashboard kirocrew) +later=(fonts omp atuin shell-ai kube forgejo-registry pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services bst-dashboard) if [ "$zen_ok" = 1 ]; then later+=(browser-fxa) elif selected browser-fxa; then diff --git a/tests/test_server_app_parity.py b/tests/test_server_app_parity.py index 0dd95f80..c6ea40cb 100644 --- a/tests/test_server_app_parity.py +++ b/tests/test_server_app_parity.py @@ -141,74 +141,3 @@ def test_dashboard_enable_failure_does_not_restart(check_system): assert result.returncode != 0 and "FAIL enable dashboard" in result.stdout calls = Path(env["CALLS"]).read_text().splitlines() assert "systemctl --user restart bst-dashboard.service" not in calls - - -@pytest.mark.parametrize("address,allowed", [("127.0.0.1:5476", True), ("[::1]:5476", True), ("0.0.0.0:5476", False), ("[::]:5476", False)]) -def test_python_process_listeners_are_checked_by_cgroup(tmp_path, address, allowed): - proc = tmp_path / "proc" - (proc / "42").mkdir(parents=True) - (proc / "42/cgroup").write_text("0::/system.slice/kirocrew.service/workers\n") - (proc / "43").mkdir() - (proc / "43/cgroup").write_text("0::/system.slice/other.service\n") - lines = (f'LISTEN 0 128 {address} 0.0.0.0:* users:(("python3",pid=42,fd=5))\n' - 'LISTEN 0 128 0.0.0.0:80 0.0.0.0:* users:(("python3",pid=43,fd=5))\n') - result = subprocess.run([sys.executable, str(REPO / "units/kirocrew-sockets.py"), "--cgroup", "/system.slice/kirocrew.service", "--proc-root", str(proc)], input=lines, capture_output=True, text=True) - assert (result.returncode == 0) is allowed - - -def test_no_attributable_listener_does_not_claim_safety(tmp_path): - result = subprocess.run([sys.executable, str(REPO / "units/kirocrew-sockets.py"), "--cgroup", "/system.slice/kirocrew.service", "--proc-root", str(tmp_path)], input="", capture_output=True, text=True) - assert result.returncode != 0 and "cannot be verified" in result.stderr - - -def test_unrelated_named_process_cannot_prove_service_safety(tmp_path): - (tmp_path / "43").mkdir() - (tmp_path / "43/cgroup").write_text("0::/system.slice/other.service\n") - line = 'LISTEN 0 128 127.0.0.1:5476 0.0.0.0:* users:(("kirocrew",pid=43,fd=5))\n' - result = subprocess.run([sys.executable, str(REPO / "units/kirocrew-sockets.py"), "--cgroup", "/system.slice/kirocrew.service", "--proc-root", str(tmp_path)], input=line, capture_output=True, text=True) - assert result.returncode != 0 and "cannot be verified" in result.stderr - - -@pytest.mark.parametrize("check", [True, False]) -@pytest.mark.parametrize("address,allowed", [("127.0.0.1:5476", True), ("0.0.0.0:5476", False)]) -def test_kirocrew_unit_checks_python_service_cgroup(check_system, check, address, allowed): - _, root, env, _ = check_system - prepare_hardening_commands(check_system) - bindir = Path(env["PATH"].split(":", 1)[0]) - proc = root / "proc" - (proc / "42").mkdir(parents=True) - (proc / "42/cgroup").write_text("0::/system.slice/kirocrew.service\n") - write_stub(bindir, "python3", ''' -script=$1; shift -exec /usr/bin/python3 "$script" "$@" --proc-root "$FAKE_PROC" -''') - write_stub(bindir, "systemctl", ''' -echo "systemctl $*" >> "$CALLS" -if [[ "$*" == *ControlGroup* ]]; then echo /system.slice/kirocrew.service -elif [[ "$*" == *show* ]]; then printf 'LoadState=loaded\\nActiveState=active\\nUnitFileState=enabled\\n' -fi -''') - line = f'LISTEN 0 128 {address} 0.0.0.0:* users:(("python3",pid=42,fd=5))' - result = run_check(check_system, "kirocrew", UNITS_CHECK="1" if check else "0", FAKE_PROC=str(proc), SOCKETS=line) - assert (result.returncode == 0) is allowed, (result.stdout, result.stderr) - calls = Path(env["CALLS"]).read_text().splitlines() - assert any("ControlGroup" in line for line in calls) - assert ("systemctl enable --now kirocrew.service" in calls) is (not check) - - -@pytest.mark.parametrize("check", [True, False]) -def test_kirocrew_refuses_symlinked_credentials(check_system, check): - _, root, env, _ = check_system - prepare_hardening_commands(check_system) - envfile = root / "etc/kirocrew/kirocrew.env" - target = root / "credentials" - target.write_text("SECRET=not-for-output\n") - envfile.unlink() - envfile.symlink_to(target) - result = run_check(check_system, "kirocrew", UNITS_CHECK="1" if check else "0") - assert result.returncode != 0 - assert "not-for-output" not in result.stdout + result.stderr - assert target.read_text() == "SECRET=not-for-output\n" - calls = Path(env["CALLS"]).read_text().splitlines() - assert not any("enable --now kirocrew.service" in line for line in calls) - assert not any(line.startswith(("chown ", "chmod ")) for line in calls) diff --git a/tests/test_server_service_units.py b/tests/test_server_service_units.py index d850e9a9..4a6019df 100644 --- a/tests/test_server_service_units.py +++ b/tests/test_server_service_units.py @@ -115,7 +115,7 @@ def fake_system(stub_env, tmp_path): units = repo / 'units' units.mkdir() for name in ('lib.sh', 'proxy-inventory.py', 'proxy-serve.py', 'osquery_native_inventory.py', 'service-templates.py', - 'proxy', 'tailscale-cert', 'bst-dashboard', 'kirocrew-sockets.py'): + 'proxy', 'tailscale-cert', 'bst-dashboard'): (units / name).symlink_to(REPO / 'units' / name) for name in ('proxy', 'tailscale-cert'): script = (REPO / 'units' / f'{name}.sh').read_text() @@ -200,7 +200,7 @@ def test_certificate_renews_expired_cert_and_uses_host_domain(fake_system): assert 'workstation.override.ts.net' in (root / 'etc/systemd/system/tailscale-cert-renew.service').read_text() -@pytest.mark.parametrize('unit', ['proxy', 'tailscale-cert', 'bst-dashboard', 'kirocrew']) +@pytest.mark.parametrize('unit', ['proxy', 'tailscale-cert', 'bst-dashboard']) @pytest.mark.parametrize('machine', ['matrix', 'telengana']) def test_retired_vps_are_untouched(fake_system, unit, machine): repo, root, env = fake_system @@ -232,14 +232,6 @@ def test_per_host_exclusion_prevents_service_changes(fake_system, unit, flag): assert system_calls(root) == '' -def test_enabled_kirocrew_without_binary_does_not_change_service(fake_system): - repo, root, env = fake_system - (repo / 'host_vars/workstation.yml').write_text('kirocrew_enabled: true\n') - result = run_unit('kirocrew', env, DOTFILES_DIR=str(repo)) - assert result.returncode == 0 and 'binary or system service missing' in result.stdout - assert system_calls(root) == '' - - def test_dashboard_template_uses_correct_project_and_service_scope(): renderer = REPO / 'units/service-templates.py' template = REPO / 'units/bst-dashboard/bst-dashboard.service.j2' @@ -247,16 +239,3 @@ def test_dashboard_template_uses_correct_project_and_service_scope(): result = subprocess.run([sys.executable, str(renderer), 'dashboard', str(template), project, image], capture_output=True, text=True, check=True) assert f'--log /var/tmp/{image}-build.log --target oci/{image}.bst --project {project}' in result.stdout assert 'WantedBy=default.target' in result.stdout and 'Restart=always' in result.stdout - - -@pytest.mark.parametrize('address,allowed', [('127.0.0.1:8000', True), ('[::1]:8000', True), ('0.0.0.0:8000', False), ('[::]:8000', False)]) -def test_kirocrew_listener_exposure(address, allowed): - line = f'LISTEN 0 128 {address} 0.0.0.0:* users:(("kirocrew",pid=7,fd=5))\n' - result = subprocess.run([sys.executable, str(REPO / 'units/kirocrew-sockets.py')], input=line, capture_output=True, text=True) - assert (result.returncode == 0) is allowed - - -def test_kirocrew_checks_local_address_not_loopback_peer(): - line = 'LISTEN 0 128 0.0.0.0:8000 127.0.0.1:99 users:(("kirocrew",pid=7,fd=5))\n' - result = subprocess.run([sys.executable, str(REPO / 'units/kirocrew-sockets.py')], input=line, capture_output=True, text=True) - assert result.returncode != 0 diff --git a/tests/test_server_units_check.py b/tests/test_server_units_check.py index 2b1ec6bb..ec7fe3de 100644 --- a/tests/test_server_units_check.py +++ b/tests/test_server_units_check.py @@ -8,7 +8,7 @@ from conftest import REPO, write_stub -UNITS = ("server-hardening", "retired-services", "proxy", "tailscale-cert", "bst-dashboard", "kirocrew") +UNITS = ("server-hardening", "retired-services", "proxy", "tailscale-cert", "bst-dashboard") @pytest.fixture @@ -20,13 +20,12 @@ def check_system(stub_env, tmp_path): units.mkdir(parents=True) (repo / "host_vars").mkdir() (repo / "group_vars").mkdir() - (repo / "host_vars/workstation.yml").write_text("kirocrew_enabled: true\n") (repo / "group_vars/all.yml").write_text("tailscale_domain: example.ts.net\n") (repo / "inventory.yml").write_text(json.dumps({"all": {"children": { "server": {"hosts": {"workstation": {}}}, "vps": {"hosts": {}} }}})) for name in ("lib.sh", "osquery_native_inventory.py", "proxy-inventory.py", "proxy-serve.py", "server-hardening-file.py", - "service-templates.py", "kirocrew-sockets.py", "retired-services", "proxy", + "service-templates.py", "retired-services", "proxy", "tailscale-cert", "bst-dashboard"): (units / name).symlink_to(REPO / "units" / name) for name in UNITS: @@ -36,18 +35,13 @@ def check_system(stub_env, tmp_path): text = text.replace(path, str(root / path.lstrip("/"))) (units / f"{name}.sh").write_text(text) for directory in ("etc/systemd", "etc/ssh", "run/systemd/system", "var/home/james/dev/dakota", - "etc/kirocrew", "etc/systemd/system", "usr/lib/systemd"): + "etc/systemd/system", "usr/lib/systemd"): (root / directory).mkdir(parents=True, exist_ok=True) (root / "etc/systemd/journald.conf").write_text("#SystemMaxUse=1G\n") (root / "etc/ssh/sshd_config").write_text("#MaxStartups 10:30:100\n") (root / "etc/fstab").write_text("/dev/root / ext4 defaults 0 1\n") (root / "usr/lib/systemd/systemd-timesyncd").touch() - (root / "etc/systemd/system/kirocrew.service").touch() - (root / "etc/kirocrew/kirocrew.env").write_text("DO_NOT_PRINT=this-is-a-secret\n") - (root / "etc/kirocrew/kirocrew.env").chmod(0o600) home = Path(env["HOME"]) - (home / ".local/bin").mkdir(parents=True) - (home / ".local/bin/kirocrew").touch() old = home / ".config/containers/systemd/grafana.container" old.parent.mkdir(parents=True) old.write_text("retired\n") @@ -89,7 +83,6 @@ def check_system(stub_env, tmp_path): write_stub(bindir, "swapon", '[ "$1" = --show=NAME ] || exit 99\n[ "${FAIL_SWAPON:-0}" != 1 ] || exit 1\nprintf "%s\\n" "${ACTIVE_SWAP:-}"\n') write_stub(bindir, "ufw", '[ "$1" = status ] || exit 99\necho "Status: inactive"\n') write_stub(bindir, "pro", '[ "$1" = status ] || exit 99\necho \'{"livepatch": "enabled"}\'\n') - write_stub(bindir, "ss", 'printf "%s\\n" "${SOCKETS:-}"\n') for command in ("dd", "mkswap", "install", "chown", "chmod"): write_stub(bindir, command, 'echo "FORBIDDEN $0 $*" >> "$CALLS"\nexit 99\n') return repo, root, env, old @@ -357,7 +350,6 @@ def test_writable_root_preserves_active_swapfile(check_system, check): ("proxy", "would deploy"), ("tailscale-cert", "would provision Tailscale certificate"), ("bst-dashboard", "would deploy dashboard user service"), - ("kirocrew", "would start kirocrew"), ]) def test_check_reports_drift_without_mutation(check_system, unit, expected): _, root, env, old = check_system @@ -365,7 +357,6 @@ def test_check_reports_drift_without_mutation(check_system, unit, expected): result = run_check(check_system, unit) assert result.returncode == 0, (result.stdout, result.stderr) assert expected in result.stdout - assert "this-is-a-secret" not in result.stdout + result.stderr assert {p: p.read_bytes() for p in root.rglob("*") if p.is_file()} == original assert old.read_text() == "retired\n" calls = Path(env["CALLS"]) @@ -389,13 +380,6 @@ def test_retired_vps_remain_untouched_in_check(check_system, unit): assert not Path(check_system[2]["CALLS"]).exists() -def test_kirocrew_rejects_non_loopback_listener(check_system): - line = 'LISTEN 0 128 0.0.0.0:8000 0.0.0.0:* users:(("kirocrew",pid=7,fd=5))' - result = run_check(check_system, "kirocrew", SOCKETS=line) - assert result.returncode != 0 - assert "listening beyond localhost" in result.stderr - - def test_hardening_line_check_never_writes(tmp_path): config = tmp_path / "sshd_config" config.write_text("#MaxSessions 20\n") diff --git a/units/kirocrew-sockets.py b/units/kirocrew-sockets.py deleted file mode 100755 index 09ba6fd5..00000000 --- a/units/kirocrew-sockets.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -"""Reject non-loopback sockets belonging to Kiro Crew's systemd cgroup.""" -import argparse -from pathlib import Path -import re -import sys - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--cgroup") - parser.add_argument("--proc-root", type=Path, default=Path("/proc")) - args = parser.parse_args() - if args.cgroup is not None and (not args.cgroup.startswith("/") or args.cgroup == "/"): - print("invalid kirocrew service cgroup", file=sys.stderr) - return 1 - unsafe = [] - owned = 0 - for line in sys.stdin: - belongs = args.cgroup is None and '"kirocrew"' in line - if args.cgroup is not None: - for pid in re.findall(r"\bpid=(\d+)", line): - try: - entries = (args.proc_root / pid / "cgroup").read_text().splitlines() - except FileNotFoundError: - # The process exited while ss was enumerating sockets. - continue - except OSError as error: - print(f"cannot inspect listener cgroup: {error}", file=sys.stderr) - return 1 - paths = [entry.split(":", 2)[-1] for entry in entries] - if any(path == args.cgroup or path.startswith(args.cgroup + "/") for path in paths): - belongs = True - break - if not belongs: - continue - owned += 1 - columns = line.split() - if len(columns) < 4: - unsafe.append(line.rstrip()) - continue - local = columns[3] - if not (local.startswith('127.0.0.1:') or local.startswith('[::1]:')): - unsafe.append(line.rstrip()) - if unsafe: - print('kirocrew is listening beyond localhost:', file=sys.stderr) - for line in unsafe: - print(line, file=sys.stderr) - return 1 - if args.cgroup is not None and not owned: - print('no listeners attributable to kirocrew; localhost-only binding cannot be verified', file=sys.stderr) - return 1 - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/units/kirocrew.sh b/units/kirocrew.sh deleted file mode 100755 index 50bcd81b..00000000 --- a/units/kirocrew.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -set -uo pipefail -# shellcheck source=units/lib.sh -. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" - -if [ "$MACHINE" = matrix ] || [ "$MACHINE" = telengana ]; then - note "SKIPPED: kirocrew excluded on retired VPS" - finish -fi -if [ "$(host_var kirocrew_enabled false)" != true ]; then - note "SKIPPED: kirocrew not enabled for this host" - finish -fi -as_root() { if [ "$EUID" = 0 ]; then "$@"; else sudo -n "$@"; fi; } -if [ ! -e "$HOME/.local/bin/kirocrew" ] || [ ! -e /etc/systemd/system/kirocrew.service ]; then - note "SKIPPED: kirocrew binary or system service missing; install from https://github.com/kirodotdev/KiroCrew and run sudo ~/.local/bin/kirocrew service install" - finish -fi -check_listeners() { - local cgroup - if [ "${1:-}" = active ]; then - cgroup=$(as_root systemctl show kirocrew.service --property=ControlGroup --value) || return 1 - as_root ss -Htlnp | as_root python3 "$DOTFILES_DIR/units/kirocrew-sockets.py" --cgroup "$cgroup" - else - as_root ss -Htlnp | as_root python3 "$DOTFILES_DIR/units/kirocrew-sockets.py" - fi -} -safe_env_file() { - ! as_root test -L /etc/kirocrew/kirocrew.env && as_root test -f /etc/kirocrew/kirocrew.env -} -if [ "${UNITS_CHECK:-0}" = 1 ]; then - if [ "$EUID" != 0 ] && ! sudo -n true; then step "noninteractive root access" false; finish; fi - if ! safe_env_file; then - step "inspect kirocrew environment (must be a regular file, not a symlink)" false - elif mode=$(as_root stat -c '%a:%u:%g' /etc/kirocrew/kirocrew.env 2>/dev/null); then - [ "$mode" = '600:0:0' ] && note "ok kirocrew environment permissions" || note "would secure kirocrew environment ownership and permissions" - else step "inspect kirocrew environment permissions" false; fi - state=$(as_root systemctl show kirocrew.service --property=LoadState --property=ActiveState --property=UnitFileState 2>/dev/null); rc=$? - if { [ "$rc" != 0 ] && [[ "$state" != *'LoadState=not-found'* ]]; } || [[ "$state" != *'LoadState='* || "$state" != *'ActiveState='* ]]; then - step "inspect kirocrew service" false - elif [[ "$state" == *'LoadState=not-found'* ]]; then - note "would enable and start kirocrew after systemd recognizes its service" - elif [[ "$state" != *'LoadState=loaded'* || "$state" != *'UnitFileState='* ]]; then - step "inspect kirocrew service" false - else - [[ "$state" == *'ActiveState=active'* ]] || note "would start kirocrew; its future listeners cannot be inspected without starting it" - [[ "$state" == *'UnitFileState=enabled'* ]] || note "would enable kirocrew" - fi - if [[ "$state" == *'ActiveState=active'* ]]; then - step "inspect current kirocrew listeners" check_listeners active - elif check_listeners; then - note "current kirocrew-named listeners are localhost-only; inactive service binding is unverified" - else step "inspect current kirocrew listeners" false; fi - finish -fi -secure_env() { - safe_env_file && as_root chown root:root /etc/kirocrew/kirocrew.env && as_root chmod 0600 /etc/kirocrew/kirocrew.env -} -if secure_env; then - note "ok secure kirocrew environment" - step "enable and start kirocrew" as_root systemctl enable --now kirocrew.service -else - step "secure kirocrew environment" false -fi -step "check kirocrew listens only on localhost" check_listeners active -finish From b4a1978edfb1b3af335a5df687c6b1694054fbed Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:39:31 +0000 Subject: [PATCH 51/62] Freeze recovery snapshots and document runner retirement Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- aws/backups.tf | 2 +- aws/punjab.tf | 2 +- docs/src/servers/aws/README.md | 20 ++++++++++++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/aws/backups.tf b/aws/backups.tf index d08e71ec..76262254 100644 --- a/aws/backups.tf +++ b/aws/backups.tf @@ -23,7 +23,7 @@ resource "aws_iam_role_policy_attachment" "dlm" { resource "aws_dlm_lifecycle_policy" "fleet_daily" { description = "Fleet node root volumes daily and weekly snapshots" execution_role_arn = aws_iam_role.dlm.arn - state = "ENABLED" + state = "DISABLED" # Retired cluster: freeze the retained recovery set. policy_details { resource_types = ["VOLUME"] diff --git a/aws/punjab.tf b/aws/punjab.tf index 644c66fb..e5f6654b 100644 --- a/aws/punjab.tf +++ b/aws/punjab.tf @@ -127,7 +127,7 @@ resource "aws_dlm_lifecycle_policy" "punjab" { provider = aws.us_east_1 description = "punjab root volume daily and weekly snapshots" execution_role_arn = aws_iam_role.dlm.arn - state = "ENABLED" + state = "DISABLED" # Migration: retain snapshots without scheduled churn. policy_details { resource_types = ["VOLUME"] diff --git a/docs/src/servers/aws/README.md b/docs/src/servers/aws/README.md index e72bc409..f7f7cc7f 100644 --- a/docs/src/servers/aws/README.md +++ b/docs/src/servers/aws/README.md @@ -20,10 +20,26 @@ and prior deployment, not a verified live resource inventory. |---|---|---| | `eu-north-1` | Terminated Talos cluster — VPC, nodes, Elastic IPs, security groups, pgdata volume | `aws/cluster.tf` | | `eu-north-1` | Backup configuration — DLM policy and retained fleet-backups bucket | `aws/backups.tf` | -| `us-east-1` | [punjab](../punjab/README.md) EC2 agent box; running as of 2026-10-27 | `aws/punjab.tf` | -| `us-east-2` | `runs-on` GitHub Actions runners; current stack status unverified | CloudFormation stack `runs-on` | +| `us-east-1` | Punjab EC2 retirement in progress; root volume and five recovery snapshots retained | `aws/punjab.tf` | +| `us-east-2` | `runs-on` stack deleted 2026-10-28; cache/log buckets retained | Former CloudFormation stack `runs-on` | | global | IAM, backup, account and budget configuration | `aws/iam.tf`, `aws/backups.tf`, `aws/account.tf`, `aws/budgets.tf` | +## Verified retirement inventory (2026-10-28) + +An administrator inventory covered all 17 enabled regions. Punjab was the only +remaining EC2 instance; no NAT gateways, load balancers or RDS instances were +found. The `runs-on` Fargate manager and six Lambda functions were retired with +its CloudFormation stack; deletion reached `DELETE_COMPLETE`, with no ECS +clusters or Lambda functions remaining in `us-east-2`. + +All three S3 buckets remain. The runner cache and logging buckets retained all +observed object versions; their lifecycle expiration was disabled. Private +runner recovery material is root-only on the replacement under +`/srv/punjab-migration/runs-on-retirement/`. The fleet-backups bucket was not +modified. Both DLM policies are disabled to freeze the recovery snapshot set; +the five completed Punjab snapshots remain. Retained storage still incurs +charges; this is not a zero-bill claim. + ## OpenTofu layout ``` From fae5fff2813237541366af4395e4027e831ee217 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:49:00 +0000 Subject: [PATCH 52/62] Set verified Himachal SSH user for replacement fleet polling Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- inventory.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/inventory.yml b/inventory.yml index 9449859c..c5da09e9 100644 --- a/inventory.yml +++ b/inventory.yml @@ -51,6 +51,7 @@ all: ansible_connection: ssh himachal: ansible_host: 100.79.182.70 + ansible_user: james ansible_connection: ssh tailnet_name: himachal-1 dilli: From e06591bc5df7159f483d19fe3a2493bf06a6c557 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:13:16 +0000 Subject: [PATCH 53/62] Cut Punjab over to Karnataka with reboot-safe networking Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- aws/cluster.tf | 11 -- aws/punjab.tf | 22 ++-- aws/tests/aws.tftest.hcl | 9 -- docs/src/servers/aws/README.md | 42 ++++---- docs/src/servers/punjab/README.md | 174 +++++++++++++++--------------- inventory.yml | 3 +- vm-migrations/punjab.yaml | 8 +- 7 files changed, 125 insertions(+), 144 deletions(-) diff --git a/aws/cluster.tf b/aws/cluster.tf index 91eb87bb..69a963e9 100644 --- a/aws/cluster.tf +++ b/aws/cluster.tf @@ -127,17 +127,6 @@ resource "aws_security_group" "admin_bootstrap" { } } - # punjab's Elastic IP — the AWS admin box. - dynamic "ingress" { - for_each = [6443, 50000] - content { - description = "punjab EIP" - from_port = ingress.value - to_port = ingress.value - protocol = "tcp" - cidr_blocks = ["${aws_eip.punjab.public_ip}/32"] - } - } dynamic "ingress" { for_each = length(var.cluster_admin_pmtu_cidrs) > 0 ? [1] : [] diff --git a/aws/punjab.tf b/aws/punjab.tf index e5f6654b..56c4f8cf 100644 --- a/aws/punjab.tf +++ b/aws/punjab.tf @@ -1,8 +1,6 @@ -# punjab — headless agent/dev box, Ansible-managed like any other fleet host -# (inventory.yml, host_vars/punjab.yml). Stock Ubuntu 24.04, reached over -# Tailscale at 100.78.73.8. No public inbound by default: break-glass access is -# SSM Session Manager (`aws ssm start-session --target `), which needs no -# open port. var.punjab_ssh_ingress can re-open 22 temporarily if SSM is down. +# Retained EC2 Punjab recovery host. The active fleet host is now the +# Karnataka VM at 100.103.187.46. The desired recovery state is stopped, +# retaining its original volume and SSM role but no reserved public IP. resource "aws_key_pair" "punjab" { provider = aws.us_east_1 @@ -109,16 +107,10 @@ resource "aws_instance" "punjab" { } } -# Stable public IP: it's allowlisted on the Talos cluster's admin SG so punjab -# can run talosctl/kubectl against it (an ephemeral IP would change on every -# stop/start). -resource "aws_eip" "punjab" { - provider = aws.us_east_1 - domain = "vpc" - instance = aws_instance.punjab.id - tags = { - Name = "punjab" - } +resource "aws_ec2_instance_state" "punjab_retired" { + provider = aws.us_east_1 + instance_id = aws_instance.punjab.id + state = "stopped" } # DLM is regional, so punjab (us-east-1) needs its own policy. Same schedule diff --git a/aws/tests/aws.tftest.hcl b/aws/tests/aws.tftest.hcl index d468905d..e774af4b 100644 --- a/aws/tests/aws.tftest.hcl +++ b/aws/tests/aws.tftest.hcl @@ -30,9 +30,6 @@ mock_provider "aws" { mock_resource "aws_iam_policy" { defaults = { arn = "arn:aws:iam::123456789012:policy/mock" } } - mock_resource "aws_eip" { - defaults = { public_ip = "192.0.2.10" } - } } variables { @@ -62,7 +59,6 @@ run "admin_ingress_without_pmtu" { for port in [6443, 50000] : length([ for r in aws_security_group.admin_bootstrap.ingress : r if r.from_port == port && r.to_port == port && r.protocol == "tcp" - && r.description != "punjab EIP" ]) == 3 ]) error_message = "each port must have one rule per cluster_admin_ingress entry" @@ -210,11 +206,6 @@ run "punjab_hardening" { error_message = "punjab's root volume must be in the snapshot policy" } - assert { - condition = length([for r in aws_security_group.admin_bootstrap.ingress : r if r.description == "punjab EIP"]) == 2 - error_message = "punjab's EIP must be allowlisted on 6443 and 50000" - } - assert { condition = aws_ebs_encryption_by_default.us_east_1.enabled && aws_ebs_encryption_by_default.eu_north_1.enabled error_message = "EBS encryption by default must stay on in both regions" diff --git a/docs/src/servers/aws/README.md b/docs/src/servers/aws/README.md index f7f7cc7f..0d8172a0 100644 --- a/docs/src/servers/aws/README.md +++ b/docs/src/servers/aws/README.md @@ -47,7 +47,7 @@ aws/ ├── versions.tf # provider (eu-north-1 default, us_east_1 alias) + S3 backend ├── cluster.tf # VPC/subnet/IGW/routes, 3 SGs, 2 Talos nodes, EIPs, pgdata ├── backups.tf # DLM role + policy, fleet-backups bucket (+ versioning/SSE/lifecycle) -├── punjab.tf # key pair, SG, instance, EIP, SSM role, DLM +├── punjab.tf # retained stopped instance, key pair, SG, SSM role, disabled DLM ├── account.tf # EBS encryption by default (both regions) ├── iam.tf # scoped admin user + its policies ├── budgets.tf # 5 budgets, one for_each @@ -81,26 +81,26 @@ infrastructure and incur charges. `aws/terraform.tfvars` remains sensitive; The fleet inventory's logical `punjab` host now connects to the replacement guest on Karnataka at `100.103.187.46`, using the verified `ubuntu` account. -Its separate Tailscale name remains **`punjab-migration`** until the service -cutover and identity rename are confirmed; the inventory's `tailnet_name` -override keeps fleet discovery aligned with that staging identity. This is -a connection-target change, not evidence that service migration is complete -or that the original EC2 instance has stopped. - -`aws/punjab.tf` still describes the original EC2 instance at its historical -Tailscale address `100.78.73.8`, plus its EIP, security group, key pair, SSM -role/profile and DLM policy. It does **not** manage the replacement guest. -Those resource declarations and `prevent_destroy` remain unchanged. The -replacement retains AWS administration tools and its existing SSH-hardening -and no-proxy policies; moving it off EC2 does not require removing those tools. - -Retirement must reconcile live AWS resources, OpenTofu state and configuration -separately: deleted resources still declared here can be proposed for recreation, -while stopped instances can retain billable volumes, EIPs and snapshots. -`prevent_destroy` is not a stop/retirement mechanism and does not protect against -recreation. Retain the S3 backups and state; do not use a blanket destroy to -resolve this drift. Verify actual retirement status before updating the -historical inventory above. +The replacement now owns the Tailscale name **`punjab`**; the old node is +`punjab-aws-retired`. Authenticated SSH, broker access, the native no-secrets +apply and monitoring were exercised on the replacement, including reboot +persistence. Atuin's pre-existing cloud account/key mismatch remains unresolved; +local history and its key were retained. + +`aws/punjab.tf` retains the original EC2 instance, volume protection, security +group, key pair and SSM role/profile for recovery. Its desired instance state +is explicitly stopped, the EIP declaration and former cluster allowlist +dependency are removed, and DLM is disabled. It does **not** manage the +replacement guest. The replacement retains AWS administration tools and its +SSH-hardening and no-proxy policies. + +The shutdown handoff's root-only receipt on the replacement is +`/srv/punjab-migration/aws-cutover-receipt.json`; it records observed EC2 +shutdown and EIP release, rather than treating a config edit as proof. +The former cluster's broader declarations still require separate state/config +reconciliation: deleted resources declared here can be proposed for recreation. +`prevent_destroy` does not protect against recreation. Retain S3 backups, +recovery storage and state; do not use a blanket destroy to resolve that drift. ## IAM & access diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index bf2047e3..dc19248c 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -1,87 +1,91 @@ # Punjab -> **TunaOS infrastructure.** This runs in the TunaOS AWS account and serves -> the TunaOS project (Hive, Matrix, CI), not James's personal fleet. See -> [TunaOS AWS Account & IaC](../aws/README.md). - -Headless agent/dev box — the fleet's AWS-hosted Ansible host. It runs in the -same AWS account as the former [AWS Talos cluster](../aws-k8s/cluster.md), -but is an ordinary fleet member, not a cluster node. The Talos cluster was -terminated; Matrix and the Hives now run on the [OCI VPS](../spindle-vps/README.md). - -## Migration status - -The Karnataka `corral-vms/punjab` VM is a staging guest, not the EC2 host. -Its host key was verified against its console. The separate Tailscale device -`punjab-migration` is online at `100.103.187.46`; direct SSH was verified -against that same host key. The initial home copy completed on 2026-10-28 into -root-restricted `/srv/punjab-migration/` on the guest; it is not activated as -its home, service state or Tailscale identity. The source remains running, so -the copy needs final synchronization and validation before cutover. -A metadata-based dry run after the initial copy found 577 regular files to -transfer (about 996 MB) and 485 new filesystem entries. The follow-up sync -completed successfully. The comparison did not checksum all contents or -check for deletions, and the live source can still change. - -The guest resolved and reached `api.github.com` over HTTPS and connected to -the Spindle broker's TCP port. These checks do not prove broker authentication. - -Kiro Crew was retired at the owner's request. Its source service is disabled -and masked; its private recovery data is retained, not activated on the -replacement. The Kiro CLI and omp integrations are separate and remain. - -EC2 snapshot `snap-0bc709516bb003e9a` was created on 2026-10-28 while Punjab -was live; it is crash-consistent, not a verified application-consistent restore. -Do not stop the source until transfers, final synchronization, recovery and -service checks are complete. Existing tmux processes cannot move live. -S3 contains older backups with no offsite copy and remains retained. - -## Hardware - -- EC2 `t3.large` (2 vCPU, 8 GiB, unlimited CPU credits), `us-east-1d` -- 250 GB gp3 root volume (unencrypted — predates EBS encryption-by-default; - snapshotted daily ×7 / weekly ×4 by DLM) -- Default VPC, stable Elastic IP -- IMDSv2 required, hop limit 1; termination protection on -- Instance role: Session Manager only -- Tailscale: `punjab` (MagicDNS) - -The instance, its security group and key pair are codified in -[`aws/punjab.tf`](https://github.com/hanthor/dotfiles/blob/master/aws/punjab.tf) — -see [AWS Account](../aws/README.md). - -## OS - -Ubuntu 24.04 LTS (stock `ubuntu-noble-24.04-amd64-server` AMI), AWS kernel. - -## Role - -- Agent host: Claude Code, Kiro, `pi`, run in tmux -- AWS admin box: `awscli` + `opentofu` (via `extra_brews`) for the - [AWS IaC](../aws/README.md) - -## Access - -- **Normal**: `ssh punjab` over Tailscale. -- **Break-glass** (Tailscale down): SSM Session Manager, no open port needed: - ```bash - aws ssm start-session --region us-east-1 --target - ``` - (needs `session-manager-plugin`; the instance ID is in the OpenTofu state). -- **Cluster**: `kubectl`/`talosctl` work from here with - `~/.kube/config-aws-migration` / `~/.talos/config-aws-migration`. -- **AWS CLI**: the scoped admin user can inspect EC2, S3 and costs, but was - denied CloudFormation, ECS, Secrets Manager, API Gateway and SQS inventory. - An authorized administrator session is needed to audit those resources. - -## Host hardening - -- sshd drop-in (`sshd_harden: true`): no root login, no passwords/kbd-interactive, - no X11, `MaxAuthTries 3`. -- `unattended-upgrades` on (Ubuntu default). - -## Ansible notes (`host_vars/punjab.yml`) - -- `skip_proxy: true` — no podman on stock Ubuntu, so no Caddy quadlet. -- Not a desktop: no GNOME/flatpak/browser roles. -- `syncthing` only runs if its brew binary exists (it currently doesn't). +Punjab is the headless agent/dev VM on Karnataka. Matrix and the Hives run +separately on the [OCI spindle VPS](../spindle-vps/README.md). + +## Active replacement + +- KubeVirt VM: `corral-vms/punjab`, on Karnataka's local-path storage. +- Ubuntu 24.04, 4 vCPUs, 12 GiB RAM, 320 GiB virtual root disk. +- Tailscale: **punjab**, `100.103.187.46`; SSH user **ubuntu**. +- Connect with `ssh ubuntu@punjab` or `ssh ubuntu@100.103.187.46`. +- `/etc/dotfiles-machine` is `punjab`. The guest keeps its own SSH host keys, + machine ID and Tailscale identity; none were copied from EC2. +- The old EC2 tailnet node is named `punjab-aws-retired`. + +The manifest is [`vm-migrations/punjab.yaml`](https://github.com/hanthor/dotfiles/blob/master/vm-migrations/punjab.yaml). +Its cloud-init network data uses DHCPv4 only. The masquerade interface's MTU +is below IPv6's minimum; requesting DHCPv6 caused a network-online timeout +on reboot. Full VM startup and a subsequent in-guest reboot were exercised; +authenticated SSH, broker access and the enabled services returned afterward. + +Karnataka and its local disk are now a dependency of this host. This is not +HA storage; losing Karnataka or that disk requires recovery from backups. + +## Configuration and services + +The canonical checkout is `~/.local/share/dotfiles`, deployed from +`feat/native-laptop-units` while [PR #348](https://github.com/hanthor/dotfiles/pull/348) +is under review. Native Bash units run through mise; no Ansible apply is used +by the replacement's update service. The user timer and Atuin daemon use +linger so they can run without an SSH login. +The real `just apply-nosecrets` completed successfully on the replacement; +secret-dependent setup reports partial/skipped rather than pretending to run. + +- omp authenticates through the Spindle broker at `http://100.113.75.88:8765`. + The obsolete shell override pointing at EC2 was removed by chezmoi adoption. +- Atuin's local history and key were copied and their history database matched + the source at cutover. Cloud sync was already logged out; the local key + differs from the Bitwarden item. Neither key nor history was overwritten. + Resolve that account/key mismatch before attempting cloud sync. +- osquery's daemon, local report timer and fleet poll timer run on the VM. + The temporary staging-hostname baseline is retained privately; a new baseline + was accepted for the final `punjab` identity. Missing reporters on the reachable + Karnataka and Himachal nodes were provisioned during validation. +- SSH password/root login is disabled by the native sshd hardening configuration. +- `skip_proxy: true` remains set; the VM is not a desktop. + +**Kiro Crew is retired.** Its service is disabled/masked, the launcher and +fleet integration are removed, and its private recovery data is quarantined. +Kiro CLI and omp integrations are separate and remain available. + +## Preserved migration data + +`/srv/punjab-migration/` is root-only and contains: + +- `source-system/`: the EC2 system-state recovery copy, not a replacement for + the VM's live networking, machine identity or system directories. +- `guest-original-home/`: the pre-cutover guest home. +- `kirocrew-retired/`: quarantined application data/runtime files. +- `terminal-recovery.tgz`: five tmux panes' scrollback, working-directory + inventory and the former user crontab. Existing processes cannot move live. +- `runs-on-retirement/`: private runner stack/configuration/secret recovery + material and retention evidence. Never publish these files. + +The old home copy is now `/home/ubuntu`; Homebrew is `/home/linuxbrew`. +The old source checkout, including its branch and uncommitted work, is retained +at `~/.local/share/dotfiles-ec2-recovery`. Its four modified tracked files were +also copied into the active checkout. Other old temporary worktrees remain +in the system recovery copy; their original absolute Git paths may need repair +before reuse. + +## AWS recovery and shutdown handoff + +The former EC2 instance is `i-085690e02ca98c95a` in `us-east-1`. Its intended +recovery state is **stopped**, not terminated. The 250 GiB root volume +`vol-09fdb248c28e850ca` and five completed snapshots are retained. Snapshot +`snap-0bc709516bb003e9a` was taken while EC2 was live: it is crash-consistent, +not proof of an application-consistent restore. + +The shutdown runs from the replacement after its checks, not from a process +that needs EC2 to stay online. The root-only receipt at +`/srv/punjab-migration/aws-cutover-receipt.json` records the observed EC2 and +EIP retirement result; consult it and AWS rather than assuming completion +from a documentation entry. The EIP allocation to release is +`eipalloc-00f40f76478a3ea02`. + +The `runs-on` stack reached `DELETE_COMPLETE`; its two S3 buckets and all +observed versions were retained. The fleet-backups S3 bucket remains intact. +Both DLM policies are disabled. Storage still incurs charges. The former +cluster's broader IaC remains historical: follow the warnings in the +[AWS handbook](../aws/README.md), not a blanket `tofu apply` or `destroy`. diff --git a/inventory.yml b/inventory.yml index c5da09e9..ae9ee198 100644 --- a/inventory.yml +++ b/inventory.yml @@ -4,11 +4,10 @@ all: # `tailnet_name`: the host's MagicDNS name when it isn't the inventory # name; scripts/online_hosts.py matches online peers by it. punjab: - # Karnataka replacement; retain its staging identity until rename is confirmed. + # Karnataka replacement; the EC2 node is named punjab-aws-retired. ansible_host: 100.103.187.46 ansible_user: ubuntu ansible_connection: ssh - tailnet_name: punjab-migration # This targets the Debian VM running via Android's Virtualization # Framework on the phone (tailnet node name "mumbai"), not the raw # Termux/Android layer itself — that's the separate `termux` host below. diff --git a/vm-migrations/punjab.yaml b/vm-migrations/punjab.yaml index 9927623d..bf0c897b 100644 --- a/vm-migrations/punjab.yaml +++ b/vm-migrations/punjab.yaml @@ -58,9 +58,15 @@ spec: name: punjab-root - name: cloudinit cloudInitNoCloud: + networkData: | + version: 2 + ethernets: + enp1s0: + dhcp4: true + dhcp6: false userData: | #cloud-config - hostname: punjab-migration + hostname: punjab manage_etc_hosts: true ssh_pwauth: false users: From 9d59ca930d6a7626bc525cba16b726c8a5270553 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:22:09 +0000 Subject: [PATCH 54/62] Document verified replacement SSH identity and vault authorization Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/punjab/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index dc19248c..1bafbf1d 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -13,6 +13,14 @@ separately on the [OCI spindle VPS](../spindle-vps/README.md). machine ID and Tailscale identity; none were copied from EC2. - The old EC2 tailnet node is named `punjab-aws-retired`. +The console-verified ED25519 host-key fingerprint is +`SHA256:4B/Ho1ZjG9yuA0nugLv0ss5ET2FuOp9iy+FwrfIeto8`. +Clients that remember EC2's key for `punjab` must verify this replacement key +before replacing that alias; do not disable strict host-key checking. +Vault-backed SSH key synchronization and authorized-key installation succeeded. +Himachal was offline during the final client-to-VM check, so that direction +was not exercised. + The manifest is [`vm-migrations/punjab.yaml`](https://github.com/hanthor/dotfiles/blob/master/vm-migrations/punjab.yaml). Its cloud-init network data uses DHCPv4 only. The masquerade interface's MTU is below IPv6's minimum; requesting DHCPv6 caused a network-online timeout From d9d3c59445f82675aef570a1afc0cc28409227d9 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:51:02 +0000 Subject: [PATCH 55/62] Record completed Punjab shutdown and final fleet lifecycle Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- CLAUDE.md | 6 +++--- aws/versions.tf | 2 +- docs/src/servers/aws/README.md | 9 +++++---- docs/src/servers/home-k8s-images.md | 2 +- docs/src/servers/punjab/README.md | 16 ++++++++-------- docs/src/servers/talos-k8s/cluster.md | 3 ++- docs/src/servers/talos-k8s/karnataka/README.md | 5 +++-- docs/src/talos-k8s.md | 7 ++++--- 8 files changed, 27 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8143aef0..b8779996 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ Three distinct concerns live here: ### Cluster recovery and current hosts -- **Home:** Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` as its default and a single-node K3s `v1.36.5+k3s1` server bound to its tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are installed; `corral-vms/punjab` is a staging VM, not a migrated replacement for active EC2 Punjab. Bihar's OS and services are unverified. Retain the former home cluster's `~/.kube/config`, `~/.talos/config`, Bitwarden notes and `talos-k8s/` manifests as recovery material; don't operate historical recovery commands on these machines. +- **Home:** Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` as its default and a single-node K3s `v1.36.5+k3s1` server bound to its tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are installed; `corral-vms/punjab` is now the active Punjab replacement, reached as `ubuntu@100.103.187.46` (tailnet name `punjab`). Reboot persistence, broker access and native no-secrets apply passed at cutover. Bihar's OS and services are unverified. Retain the former home cluster's `~/.kube/config`, `~/.talos/config`, Bitwarden notes and `talos-k8s/` manifests as recovery material; don't operate historical recovery commands on these machines. - **AWS** (`eu-north-1`, built 2026-08-27) — **terminated 2026-10-25** after credits were exhausted. It previously ran Matrix, the Hives and the CFP dashboard. The S3 bucket `hanthor-fleet-backups-*` and the himachal-1 mirror (`~/backups/tunaos-aws-2026-10-25`) were retained; verify individual AWS resources rather than assuming they are gone. OpenTofu still describes the old cluster and can propose recreating resources. Configs retained as recovery material: `~/.kube/config-aws-migration`, `~/.talos/config-aws-migration`. Handbook: [`docs/src/servers/aws-k8s/cluster.md`](docs/src/servers/aws-k8s/cluster.md). @@ -64,10 +64,10 @@ The former Bihar control plane and Karnataka worker ran Talos `v1.13.2`, Kuberne ## TunaOS AWS account (`aws/`) -- **punjab** (this repo's AWS-hosted fleet host) is an EC2 `t3.large` in `us-east-1`; the former Talos cluster in `eu-north-1` is terminated. `runs-on` (us-east-2) is its own CloudFormation stack — don't import it. +- **punjab** is now a Karnataka-hosted VM. Its former EC2 `t3.large` in `us-east-1` was stopped on 2026-10-28 and its EIP released; its root volume and recovery snapshots remain. The former Talos cluster in `eu-north-1` is terminated. The separate `runs-on` CloudFormation stack in `us-east-2` reached `DELETE_COMPLETE`; its two S3 buckets and object versions were retained. - **Do not blindly apply `aws/`:** its configuration still describes the terminated cluster and may propose recreating paid infrastructure. Inspect live resources and review plans before changes; credits are exhausted. - State is in S3 (`hanthor-fleet-backups-*/tofu/aws/`) and contains Talos PKI. `aws/terraform.tfvars` is gitignored and lives in Bitwarden note `aws-tofu-tfvars` (`just aws-tfvars` / `just aws-seed-tfvars`). -- EC2 punjab remains active and uses Tailscale for access and SSM Session Manager for break-glass. Do not stop it; preserve its data and the S3 backups until a verified replacement and an explicit cleanup decision exist. +- Do not restart EC2 Punjab or recreate retired runners without an explicit recovery decision. Preserve the retained root volume, snapshots and S3 backups; storage still incurs charges. The root-only cutover receipt on the replacement is `/srv/punjab-migration/aws-cutover-receipt.json`; the successful shutdown service journal records EC2 stopped and EIP released. - The former cluster uploaded Postgres dumps to `s3://hanthor-fleet-backups-*/postgres/`; the `postgres-backup-writer` key is in Bitwarden, never in tofu state. - `prevent_destroy` in configuration does not prevent creation of resources missing from the account. Never put Talos `user_data` in the config. diff --git a/aws/versions.tf b/aws/versions.tf index 96761fd1..d34841ee 100644 --- a/aws/versions.tf +++ b/aws/versions.tf @@ -25,7 +25,7 @@ provider "aws" { region = "eu-north-1" } -# punjab (this repo's AWS-hosted fleet member) lives in us-east-1. +# The retained EC2 Punjab recovery resources live in us-east-1. provider "aws" { alias = "us_east_1" region = "us-east-1" diff --git a/docs/src/servers/aws/README.md b/docs/src/servers/aws/README.md index 0d8172a0..d72270c0 100644 --- a/docs/src/servers/aws/README.md +++ b/docs/src/servers/aws/README.md @@ -20,7 +20,7 @@ and prior deployment, not a verified live resource inventory. |---|---|---| | `eu-north-1` | Terminated Talos cluster — VPC, nodes, Elastic IPs, security groups, pgdata volume | `aws/cluster.tf` | | `eu-north-1` | Backup configuration — DLM policy and retained fleet-backups bucket | `aws/backups.tf` | -| `us-east-1` | Punjab EC2 retirement in progress; root volume and five recovery snapshots retained | `aws/punjab.tf` | +| `us-east-1` | Punjab EC2 stopped; EIP released; root volume and five recovery snapshots retained | `aws/punjab.tf` | | `us-east-2` | `runs-on` stack deleted 2026-10-28; cache/log buckets retained | Former CloudFormation stack `runs-on` | | global | IAM, backup, account and budget configuration | `aws/iam.tf`, `aws/backups.tf`, `aws/account.tf`, `aws/budgets.tf` | @@ -94,9 +94,10 @@ dependency are removed, and DLM is disabled. It does **not** manage the replacement guest. The replacement retains AWS administration tools and its SSH-hardening and no-proxy policies. -The shutdown handoff's root-only receipt on the replacement is -`/srv/punjab-migration/aws-cutover-receipt.json`; it records observed EC2 -shutdown and EIP release, rather than treating a config edit as proof. +The replacement's shutdown service completed successfully at 19:27:15 UTC, +observing EC2 stopped and releasing its EIP before logging out of AWS. Its +root-only receipt is `/srv/punjab-migration/aws-cutover-receipt.json`, with +systemd journal provenance retained rather than treating a config edit as proof. The former cluster's broader declarations still require separate state/config reconciliation: deleted resources declared here can be proposed for recreation. `prevent_destroy` does not protect against recreation. Retain S3 backups, diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index 8741837b..1c39ff25 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -1,6 +1,6 @@ # Home Kubernetes Server Images -These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to tailnet address `100.66.158.127`; KubeVirt `v1.9.1`, CDI and Corral are installed. The `corral-vms/punjab` VM is staging, not a migrated replacement for active EC2 Punjab. Bihar's current OS and services are unverified. Verify image provenance and physical recovery before any future installation or switch. +These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to tailnet address `100.66.158.127`; KubeVirt `v1.9.1`, CDI and Corral are installed. The `corral-vms/punjab` VM is now the active Punjab replacement at `100.103.187.46`; its former EC2 host is stopped with recovery storage retained. Bihar's current OS and services are unverified. Verify image provenance and physical recovery before any future installation or switch. The Utah-based images built in CI; the starter ISO was about 5.94 GiB. Acceptance allows media up to 8 GiB, but the publication workflow still diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index 1bafbf1d..8858a348 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -79,18 +79,18 @@ before reuse. ## AWS recovery and shutdown handoff -The former EC2 instance is `i-085690e02ca98c95a` in `us-east-1`. Its intended -recovery state is **stopped**, not terminated. The 250 GiB root volume +The former EC2 instance `i-085690e02ca98c95a` in `us-east-1` reached **stopped** +at the cutover, not terminated. The 250 GiB root volume `vol-09fdb248c28e850ca` and five completed snapshots are retained. Snapshot `snap-0bc709516bb003e9a` was taken while EC2 was live: it is crash-consistent, not proof of an application-consistent restore. -The shutdown runs from the replacement after its checks, not from a process -that needs EC2 to stay online. The root-only receipt at -`/srv/punjab-migration/aws-cutover-receipt.json` records the observed EC2 and -EIP retirement result; consult it and AWS rather than assuming completion -from a documentation entry. The EIP allocation to release is -`eipalloc-00f40f76478a3ea02`. +The replacement's `punjab-aws-cutover.service` completed successfully at +2026-10-28 19:27:15 UTC, after observing EC2 stopped and releasing EIP allocation +`eipalloc-00f40f76478a3ea02`. It then logged out of the AWS CLI. The root-only +receipt at `/srv/punjab-migration/aws-cutover-receipt.json` preserves that +result with journal provenance; a later check-only authentication failure is +retained separately. Consult the receipt and AWS for current resource state. The `runs-on` stack reached `DELETE_COMPLETE`; its two S3 buckets and all observed versions were retained. The fleet-backups S3 bucket remains intact. diff --git a/docs/src/servers/talos-k8s/cluster.md b/docs/src/servers/talos-k8s/cluster.md index 0783232a..385f0261 100644 --- a/docs/src/servers/talos-k8s/cluster.md +++ b/docs/src/servers/talos-k8s/cluster.md @@ -4,7 +4,8 @@ > Bihar's OS and services are unverified. Karnataka runs `projectbluefin/utah:testing` > with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to > tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are installed; -> `corral-vms/punjab` is staging, not a migrated replacement for active EC2 Punjab. +> `corral-vms/punjab` is now the active Punjab replacement (`100.103.187.46`); +> its former EC2 host is stopped, with recovery storage retained. > On the current server, firewalld puts `cni0` in the internal zone and accepts > pod subnet `10.42.0.0/16` to kubelet port `10250` on that tailnet address; > internal-zone services, including SSH, are accessible from pods. NetworkPolicy diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index 09f85e7a..b99f9ebd 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -3,8 +3,9 @@ Karnataka runs `ghcr.io/projectbluefin/utah:testing`, is in the fleet's `server` inventory group, and uses `multi-user.target` as its default. SSH is available over Tailscale at `100.66.158.127`. A single-node K3s cluster runs -KubeVirt, CDI, and Corral. The `corral-vms/punjab` VM is a staging guest; it -is not the EC2 host and has not received its data or services. +KubeVirt, CDI, and Corral. The `corral-vms/punjab` VM now runs the migrated +Punjab environment at `100.103.187.46` (tailnet name `punjab`). Its former EC2 +host is stopped; its root volume and recovery snapshots are retained. NetworkManager, Tailscale and SSH are enabled at boot. Ethernet is preferred; a locally stored Wi-Fi profile autoconnects with a higher route metric. Wi-Fi diff --git a/docs/src/talos-k8s.md b/docs/src/talos-k8s.md index 58953f1f..6be3b2ae 100644 --- a/docs/src/talos-k8s.md +++ b/docs/src/talos-k8s.md @@ -4,9 +4,10 @@ > are unverified. Karnataka runs `projectbluefin/utah:testing` with > `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server on its > tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are -> installed; `corral-vms/punjab` is a staging VM, not a migrated replacement -> for active EC2 Punjab. Retain `talos-k8s/`, cluster configs and Bitwarden -> notes as recovery material, not proof either machine still runs Talos. +> installed; `corral-vms/punjab` is now the active Punjab replacement at +> `100.103.187.46`. The former EC2 host is stopped with recovery storage retained. +> Retain `talos-k8s/`, cluster configs and Bitwarden notes as recovery material, +> not proof either machine still runs Talos. > > **Do not run historical reinstall, configuration or workload commands against > repurposed nodes.** They can destroy data or change current systems. Kubernetes From 64a5a2324f2b07b3b6d5bd436e0ba0ec7e917a9c Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:15:37 +0000 Subject: [PATCH 56/62] Repair Atuin authentication checks and document lossless sync recovery Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/roles/apk_packages.md | 7 +++++ docs/src/roles/shell_atuin.md | 36 ++++++++++++++++++++++++-- docs/src/servers/aws/README.md | 5 ++-- docs/src/servers/punjab/README.md | 8 +++--- tests/test_atuin_unit.py | 2 +- tests/test_check_small_packages_vpn.py | 3 ++- units/apk-packages.conf | 2 +- units/atuin.sh | 4 +-- 8 files changed, 53 insertions(+), 14 deletions(-) diff --git a/docs/src/roles/apk_packages.md b/docs/src/roles/apk_packages.md index 64a25de7..6024d4a5 100644 --- a/docs/src/roles/apk_packages.md +++ b/docs/src/roles/apk_packages.md @@ -2,4 +2,11 @@ `units/apk-packages.sh` runs only when the shared library detects a musl loader. Package lists live in `units/apk-packages.conf`. It retries `apk update`, installs missing bootstrap and ordinary packages without upgrading installed ones, and installs Bitwarden CLI globally via npm. Bootstrap includes `just` from Alpine's community repository for fleet commands and update services; an initial installation of `just` is still needed to invoke those commands before the unit has run. If a batch install fails it retries packages individually so unrelated packages can still install; optional mise and cache refresh are best effort. +Atuin uses Alpine's `atuin-sync` package, which includes login and history synchronization. The plain `atuin` package is an offline-only client; having it installed does not satisfy the sync-capable package requirement. + +During the Atuin recovery on Kerala, apk-tools 3.0.8 rejected the unit's existing +`--no-upgrade` option. The sync client was installed directly with native +`apk add atuin-sync`; the complete package unit was not claimed as validated +on that apk-tools version. + The unit attempts to enable Podman and its socket and adds the current user to the `podman` group; a new login may be needed. The shared unit library skips retired `matrix` and `telengana` before package or group changes; nothing on those hosts may be restarted. Privileged operations require root or noninteractive `sudo -n`; no vault secrets are needed. diff --git a/docs/src/roles/shell_atuin.md b/docs/src/roles/shell_atuin.md index 3a6f092a..41505f84 100644 --- a/docs/src/roles/shell_atuin.md +++ b/docs/src/roles/shell_atuin.md @@ -12,7 +12,7 @@ Configures [Atuin](https://atuin.sh/) shell history and logs it in to sync. 1. chezmoi writes `~/.config/atuin/config.toml` from `home/dot_config/atuin/config.toml` — sync to `api.atuin.sh` every 5m, fuzzy search, `filter_mode = "global"` (including the Up-arrow binding) `just apply-tags atuin` and `just apply-tags secrets` apply only this chezmoi file, without unrelated dotfiles or chezmoi scripts. -2. The `atuin` unit checks `atuin status`; if it shows `[Sync]` the Bitwarden round-trip is skipped +2. The `atuin` unit checks `atuin status`; a successful exit skips the Bitwarden round-trip. It does not depend on a particular human-readable heading. 3. Otherwise, with an unlocked vault, it reads the `atuin.sh` item (username, password, custom field `key` holding the mnemonic). Atuin 18.24 or newer receives the password and key through its environment, not command-line arguments. The unit preserves the data directory and its history; a missing key can be written by Atuin during login. If a key already exists, the unit logs in only when `atuin key` matches the vault mnemonic, then runs a best-effort `atuin sync`. 4. Imports existing shell history once (`atuin import auto`, sentinel `~/.local/share/atuin/.history_imported`). If the version, key lookup, existing key, or login attempt requires manual attention, the unit defers importing until that situation is resolved. A failed login also fails the unit. @@ -28,5 +28,37 @@ Configures [Atuin](https://atuin.sh/) shell history and logs it in to sync. - Existing keys are never deleted or replaced by the unit; it does not reset `~/.local/share/atuin` when the key is missing. That directory is kept mode 0700, including after a prior history import. - Atuin 18.24 resolves its own configured key path, including the newer `~/.atuin/data` layout; the history-import sentinel remains in `~/.local/share/atuin`. -- Alpine's community `atuin` package may be an older offline-only build; sync requires a sync-enabled Atuin client. Check the package version and features before attempting a manual login. +- Alpine requires the sync-enabled `atuin-sync` package selected by `units/apk-packages.conf`; the plain `atuin` package is offline-only. Versions before 18.24 require native interactive login rather than credentials in command-line arguments. - Don't set the Up-arrow filter to `session`: it looks broken in fresh terminals + +## History/key recovery (2026-10-28) + +Punjab, Karnataka and Kerala were reconciled under the existing Bitwarden +`atuin.sh` item's `key`, which decrypted the readable cloud history. Prior +Punjab and Kerala keys remain in hidden fields +`recovery-punjab-key-2026-10-28` and `recovery-kerala-key-2026-10-28`. +Do not overwrite a mismatched client key or discard its database before recovery. + +The clean history contains 23,760 recovered entries plus one real sync +verification command. Original readable cloud, Punjab and Kerala history IDs +were preserved; additional raw shell history was imported, without broadly +deleting repeated commands. Undated Bash imports identify their timestamp as +import time in their intent metadata. SQLite integrity, record encryption and +an independent full cloud download passed. All three reachable clients then +synced using the canonical key; Himachal was offline and was not modified. + +The owner approved replacing the mixed-key cloud store. Its 245 records under +11 unavailable keys were preserved as ciphertext, not claimed as decrypted or +silently discarded. Original databases and keys are private under +`~/backups/atuin-recovery-20261028T195643Z/` on Punjab. The verified age-encrypted +original snapshot archive is also at +`ubuntu@spindle:~/backups/atuin-recovery-20261028T195643Z/`. +Retain these archives: keys recovered from an old/offline host may unlock more +history later. The original live directory is retained as `live-data-before-switch/`. + +Back up live SQLite databases with SQLite's backup API, not by copying only +their `.db` files while WAL files exist. Test recovery on copies; `store rebuild +history` merges rather than clears its target, and `store verify` can print +failure while returning exit zero. Require explicit verification success. +`store push --force` replaces the entire remote store and requires a verified +recovery archive and an explicit replacement decision; it is not a routine sync. diff --git a/docs/src/servers/aws/README.md b/docs/src/servers/aws/README.md index d72270c0..35826737 100644 --- a/docs/src/servers/aws/README.md +++ b/docs/src/servers/aws/README.md @@ -84,8 +84,9 @@ guest on Karnataka at `100.103.187.46`, using the verified `ubuntu` account. The replacement now owns the Tailscale name **`punjab`**; the old node is `punjab-aws-retired`. Authenticated SSH, broker access, the native no-secrets apply and monitoring were exercised on the replacement, including reboot -persistence. Atuin's pre-existing cloud account/key mismatch remains unresolved; -local history and its key were retained. +persistence. Atuin's later key/history recovery restored sync on Punjab, +Karnataka and Kerala under the canonical vault key, with original data retained +in private recovery archives; see [Atuin](../../roles/shell_atuin.md#historykey-recovery-2026-10-28). `aws/punjab.tf` retains the original EC2 instance, volume protection, security group, key pair and SSM role/profile for recovery. Its desired instance state diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index 8858a348..90421699 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -42,10 +42,10 @@ secret-dependent setup reports partial/skipped rather than pretending to run. - omp authenticates through the Spindle broker at `http://100.113.75.88:8765`. The obsolete shell override pointing at EC2 was removed by chezmoi adoption. -- Atuin's local history and key were copied and their history database matched - the source at cutover. Cloud sync was already logged out; the local key - differs from the Bitwarden item. Neither key nor history was overwritten. - Resolve that account/key mismatch before attempting cloud sync. +- Atuin history was subsequently merged with readable cloud, Kerala and raw + shell history under the canonical Bitwarden key. Punjab, Karnataka and Kerala + now sync successfully with 23,761 history entries. Prior keys, databases and + opaque encrypted records remain backed up; see [Atuin recovery](../../roles/shell_atuin.md#historykey-recovery-2026-10-28). - osquery's daemon, local report timer and fleet poll timer run on the VM. The temporary staging-hostname baseline is retained privately; a new baseline was accepted for the final `punjab` identity. Missing reporters on the reachable diff --git a/tests/test_atuin_unit.py b/tests/test_atuin_unit.py index c930ac0b..e839a89b 100644 --- a/tests/test_atuin_unit.py +++ b/tests/test_atuin_unit.py @@ -135,7 +135,7 @@ def test_no_secrets_keeps_history_without_touching_vault(stub_env): def test_already_logged_in_never_touches_bitwarden(stub_env): bindir, env = stub_env - stubs(bindir, "[Sync] ok", status_rc=0) + stubs(bindir, "", status_rc=0) r, calls = run(env) assert r.returncode == 0 assert "login" not in calls and "bw " not in calls diff --git a/tests/test_check_small_packages_vpn.py b/tests/test_check_small_packages_vpn.py index 54d17d79..b755a0b4 100644 --- a/tests/test_check_small_packages_vpn.py +++ b/tests/test_check_small_packages_vpn.py @@ -23,7 +23,7 @@ def test_apk_check_reports_packages_services_and_group_without_mutation(platform write_stub(bindir, 'apk', '''\ echo "apk $*" >> "$LOG" [ "$1" = info ] && [ "$#" = 1 ] || exit 99 -printf 'chezmoi\nbash\n' +printf 'chezmoi\nbash\natuin\n' ''') write_stub(bindir, 'systemctl', '''\ echo "systemctl $*" >> "$LOG" @@ -43,6 +43,7 @@ def test_apk_check_reports_packages_services_and_group_without_mutation(platform assert result.returncode == 0, result.stdout + result.stderr assert 'ok APK package chezmoi' in result.stdout assert 'WOULD install APK package yq-go' in result.stdout + assert 'WOULD install APK package atuin-sync' in result.stdout assert 'ok podman enabled and active' in result.stdout assert 'WOULD enable and start podman.socket' in result.stdout assert 'WOULD add james to podman group' in result.stdout diff --git a/units/apk-packages.conf b/units/apk-packages.conf index 2abb2f39..51ce56cb 100644 --- a/units/apk-packages.conf +++ b/units/apk-packages.conf @@ -3,7 +3,7 @@ APK_BOOTSTRAP=(chezmoi yq-go python3 py3-yaml just) APK_PACKAGES=( bash fish zsh git git-lfs delta vim neovim tmux openssh-client curl wget - jq ripgrep fd fzf starship bat exa direnv zoxide atuin shellcheck shfmt + jq ripgrep fd fzf starship bat exa direnv zoxide atuin-sync shellcheck shfmt nodejs npm busybox-extras podman podman-compose crun iptables ) APK_OPTIONAL=(mise) diff --git a/units/atuin.sh b/units/atuin.sh index bbfd1d40..6ac17216 100755 --- a/units/atuin.sh +++ b/units/atuin.sh @@ -22,9 +22,7 @@ step "atuin data dir" install -d -m 0700 "$data" [ "$UNIT_FAILED" = 0 ] || finish logged_in() { - local out - out="$(atuin status 2>&1)" || return 1 - case "$out" in *"[Sync]"*) return 0 ;; *) return 1 ;; esac + atuin status >/dev/null 2>&1 } atuin_login() { From 5864a8a741a7b04a4d505f8e0f70f5153588f029 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:18:49 +0000 Subject: [PATCH 57/62] Document safe recovery before old Atuin clients resume sync Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/roles/shell_atuin.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/src/roles/shell_atuin.md b/docs/src/roles/shell_atuin.md index 41505f84..37f7b232 100644 --- a/docs/src/roles/shell_atuin.md +++ b/docs/src/roles/shell_atuin.md @@ -56,6 +56,11 @@ original snapshot archive is also at Retain these archives: keys recovered from an old/offline host may unlock more history later. The original live directory is retained as `live-data-before-switch/`. +Before resuming sync on an old/offline client, back up its databases and key, +recover any unsynced history into the canonical store, then give that client a +fresh verified store. Do not let an old cached mixed-key store upload again: +even a client with the correct account key may carry other hosts' opaque records. + Back up live SQLite databases with SQLite's backup API, not by copying only their `.db` files while WAL files exist. Test recovery on copies; `store rebuild history` merges rather than clears its target, and `store verify` can print From d048cbe455e430a8580963db9861fbeb51a75ab9 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:30:42 +0000 Subject: [PATCH 58/62] Record account-wide AWS shutdown verification and Kiro cancellation Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- docs/src/servers/aws/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/src/servers/aws/README.md b/docs/src/servers/aws/README.md index 35826737..6761009b 100644 --- a/docs/src/servers/aws/README.md +++ b/docs/src/servers/aws/README.md @@ -40,6 +40,30 @@ modified. Both DLM policies are disabled to freeze the recovery snapshot set; the five completed Punjab snapshots remain. Retained storage still incurs charges; this is not a zero-bill claim. +## Shutdown re-verification (2026-10-29) + +Authenticated live inventory covered all 17 enabled regions in this account. +The only remaining EC2 instance was Punjab, already stopped. No running +compute, ECS/EKS clusters, Lambda functions, RDS databases, load balancers, +NAT gateways or allocated Elastic IPs were found. Auto Scaling groups, open +Spot requests, Spot Fleets and EC2 Fleets were also empty in every region. +App Runner and Lightsail endpoint failures were checked against AWS's public +SSM region-availability registry; those service/region pairs are unsupported. +No additional stop/delete operation was necessary. + +The migrated VM passed authenticated SSH using its recorded host key and a +fresh omp request explicitly using `anthropic/claude-opus-5-5`, independent of +AWS/Kiro. The old 250 GiB gp3 root volume, five snapshots and three S3 buckets +remain intact; both DLM policies remain disabled. Retained storage is billable. + +Kiro's live subscription API reports both Enterprise Power and Pro Plus as +`CANCELLED`, expiring **2026-10-31 23:59:59.999 UTC**. Historical cancellation +requests used end-of-period removal on October 25. Cost Explorer still shows +Power charges of about $6.45/day for October 27–29; these do not establish an +active renewal. [Kiro cancellation documentation](https://kiro.dev/docs/enterprise/subscription-management/) +explains month-end cancellation and separate SSO-assignment cleanup. Do not +delete identities or application assignments as a billing workaround. + ## OpenTofu layout ``` From dcbbfbc43deb0d46d585bf8834ed47db92030134 Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:31:17 +0000 Subject: [PATCH 59/62] Provision Karnataka K3s access across the active fleet Preserve historical cluster contexts and vault notes, scope API firewall access to the tailnet, and document verified client rollout. Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- CLAUDE.md | 4 +- Justfile | 2 +- docs/src/SUMMARY.md | 4 +- docs/src/bitwarden.md | 13 +- docs/src/roles/kube.md | 93 +++-- docs/src/servers/talos-k8s/cluster.md | 7 +- .../src/servers/talos-k8s/karnataka/README.md | 165 +++++++- docs/src/units.md | 6 + group_vars/all.yml | 3 +- home-k3s/firewall.sh | 33 +- host_vars/mumbai.yml | 1 - host_vars/termux.yml | 1 - mise.toml | 2 +- scripts/bw-seed-kube.sh | 4 +- tests/test_home_k3s_firewall.py | 68 +++- tests/test_kube_seed.py | 38 ++ tests/test_kube_unit.py | 352 +++++++++++++----- units/apk-packages.conf | 2 +- units/kube.sh | 161 +++++--- units/termux-packages.conf | 4 +- 20 files changed, 747 insertions(+), 216 deletions(-) create mode 100644 tests/test_kube_seed.py diff --git a/CLAUDE.md b/CLAUDE.md index b8779996..7859bf28 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,14 +47,14 @@ just add-machine NAME # Onboard + bootstrap an already-reachable machine via S `just apply*` invokes `scripts/run-units.sh`, which runs `units/` in prerequisite stages via mise, with a sequential fallback. It selects units by the wrapper's tag map and honors each unit's own platform, profile and host-variable gates. See [`docs/src/units.md`](docs/src/units.md). - Apply recipes resolve and sync Bitwarden; unit helpers use `BW_SESSION` or `/tmp/bw_session` noninteractively, and honor `--no-secrets`. -- The `kube` unit fetches kubeconfig and talosconfig from Bitwarden on eligible desktops. +- The `kube` unit provisions the Karnataka K3s default on active Linux/macOS computers, including servers and Termux, while preserving historical Talos recovery configs. Offline clients need a secret-enabled apply when reachable. - `ssh-keys` and `ssh-mesh` handle key lifecycle and authorization. `home/` is the chezmoi source for shell, Git, Neovim and Atuin config. Groups in `inventory.yml`, `skip_*` flags and `machine_profile` influence individual units. See [`docs/src/roles/`](docs/src/roles/) for component references. ## Kubeconfig + Talos secrets -`~/.kube/config` and `~/.talos/config` contain cluster PKI — they live in **Bitwarden as secure notes** named `kubeconfig` and `talosconfig` (home cluster). The AWS cluster's are `kubeconfig-aws-migration` / `talosconfig-aws-migration`, written to `~/.kube/config-aws-migration` / `~/.talos/config-aws-migration`. Pull all four via `just apply-tags kube`; `just seed-kube` only seeds the two home-cluster notes. +The active K3s note is **`kubeconfig-karnataka-k3s`**: it seeds `~/.kube/config-karnataka-k3s`, merges into `~/.kube/config`, and selects context `karnataka-k3s` at `https://100.66.158.127:6443`. All configs contain cluster PKI; keep them private. Historical home notes `kubeconfig` / `talosconfig` seed `~/.kube/config-home-talos` / `~/.talos/config`; AWS notes `kubeconfig-aws-migration` / `talosconfig-aws-migration` retain their separate recovery files. `just apply-tags kube` retrieves these notes; `just seed-kube` only seeds the historical home notes from `config-home-talos` and talosconfig, never the merged K3s default. See [`docs/src/roles/kube.md`](docs/src/roles/kube.md). `talos-k8s/.gitignore` excludes `controlplane.yaml`, `worker.yaml`, and `talosconfig` from git. diff --git a/Justfile b/Justfile index ed2b66e8..ead1b4ff 100644 --- a/Justfile +++ b/Justfile @@ -540,7 +540,7 @@ docs-build: fi cd {{ dotfiles_dir }}/docs && mdbook build -# Seed Bitwarden with this machine's kubeconfig + talosconfig (so other machines can fetch) +# Seed retained home Talos notes from config-home-talos + talosconfig (not the merged K3s default) seed-kube: @{{ dotfiles_dir }}/scripts/bw-seed-kube.sh diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index b4a41cb6..ff4ca90c 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -29,7 +29,7 @@ - [Home Kubernetes Images & ISOs](servers/home-k8s-images.md) - [Home Talos Cluster (historical reference)](servers/talos-k8s/cluster.md) - [Bihar (current state unverified)](servers/talos-k8s/bihar/README.md) - - [Karnataka (Utah)](servers/talos-k8s/karnataka/README.md) + - [Karnataka (Utah / active K3s)](servers/talos-k8s/karnataka/README.md) - [Goa](servers/goa/README.md) - [Spindle VPS (matrix.reilly.asia)](servers/spindle-vps/README.md) - [VPS]() @@ -41,7 +41,7 @@ - [TunaOS AWS Account & IaC](servers/aws/README.md) - [Talos Cluster (terminated; archive)](servers/aws-k8s/cluster.md) - [Hive Agent Fleet](cluster/hive.md) - - [Punjab (agent box)](servers/punjab/README.md) + - [Punjab (active Karnataka VM)](servers/punjab/README.md) # Roles Reference diff --git a/docs/src/bitwarden.md b/docs/src/bitwarden.md index 4d4d68e2..d8697dbb 100644 --- a/docs/src/bitwarden.md +++ b/docs/src/bitwarden.md @@ -32,7 +32,8 @@ Common items consumed by units and scripts (the unit sources and their configura |---------------|------|---------| | `@` | SSH Key | `ssh-keys` unit — per-machine ed25519 keys | | `tailscale-apikey` | Login (password) | `tailscale` unit — join and stale-device API cleanup | -| `kubeconfig`, `talosconfig` | Secure notes | `kube` unit — home cluster credentials | +| `kubeconfig-karnataka-k3s` | Secure note (`notes` field) | `kube` unit — active Karnataka K3s credentials, context `karnataka-k3s`, merged as the fleet default | +| `kubeconfig`, `talosconfig` | Secure notes | `kube` unit — retained home Talos recovery credentials; never repurpose for K3s | | `kubeconfig-aws-migration`, `talosconfig-aws-migration` | Secure notes | `kube` unit — retained credentials for the terminated AWS cluster | | `accounts.firefox.com` | Login + TOTP | `browser-fxa` unit — Firefox Account credentials | | `github-token` | Login (password) | `github` unit — PAT for `gh` | @@ -62,9 +63,13 @@ If Bitwarden can't unlock (first run, no BW CLI installed, vault genuinely locke ## Seeding New Secrets ```bash -# Push kubeconfig + talosconfig (home cluster) to Bitwarden from a machine that has them: +# Push retained home-only configs to their historical Bitwarden notes: just seed-kube ``` -`seed-kube` only handles the two home-cluster notes; the `*-aws-migration` -notes are maintained by hand. +`seed-kube` reads `~/.kube/config-home-talos` and `~/.talos/config`, not the +merged K3s default `~/.kube/config`. It only handles the two retained home +Talos notes. The `*-aws-migration` notes remain recovery material maintained +by hand; the separate `kubeconfig-karnataka-k3s` note is provisioned privately +from the current K3s server. See the [kube unit](roles/kube.md#bitwarden-contract) +for its exact format. Do not upload active K3s credentials into historical notes. diff --git a/docs/src/roles/kube.md b/docs/src/roles/kube.md index c965410a..5387344e 100644 --- a/docs/src/roles/kube.md +++ b/docs/src/roles/kube.md @@ -3,53 +3,80 @@ **Unit:** `kube` (`units/kube.sh`; see [Units](../units.md)) **Tags:** `secrets`, `kube` **Secrets needed:** Yes -**Runs on:** Desktops only (`SKIPPED:` otherwise, or with `skip_kube: true` / `machine_profile: vm-test` in `host_vars`) +**Runs on:** Active Linux and macOS computers, including servers, the phone's Debian VM and Termux. Explicit `skip_kube: true` / `machine_profile: vm-test` exclusions still apply. Retired `matrix` and `telengana` are never modified. -Pulls retained kube + Talos configs from Bitwarden for the former home cluster and the historical [AWS cluster](../servers/aws-k8s/cluster.md), terminated 2026-10-25. Bihar was recently online but is currently unreachable; its OS and services are unverified. Karnataka runs Utah. Preserve the home configs and notes as recovery material; they do not prove a cluster is running or reachable. +[Karnataka K3s](../servers/talos-k8s/karnataka/README.md#current-k3s-cluster) is the fleet's active default cluster, reached over Tailscale at `https://100.66.158.127:6443`. The former home Talos cluster and terminated AWS cluster remain recovery material, not current workload targets. Their Bitwarden notes and local configs must not be repurposed for K3s. -## Steps +## Prerequisites and rollout -With an unlocked vault (a locked vault or `--no-secrets` prints `SKIPPED:` and leaves existing files alone): +`kubectl` is a core package: Homebrew `kubernetes-cli` on supported glibc Linux/macOS hosts, `kubectl` through APK on Alpine and through `pkg` on Termux. Termux also installs `@bitwarden/cli` through its existing npm package unit. Desktop inventory membership is not required. -1. `kube dirs`: `~/.kube/` and `~/.talos/` (0700). -2. `home kubeconfig`: `bw get notes kubeconfig` → `~/.kube/config` (0600). -3. `home talosconfig`: `bw get notes talosconfig` → `~/.talos/config` (0600). -4. `pin tls-server-name`: after writing the home kubeconfig, every cluster in it gets `--tls-server-name=$KUBE_TLS_SERVER_NAME` (`units/kube.conf`, default bihar's LAN address `192.168.0.5`): the home API-server cert lacks the tailnet address in its SANs. If `kubectl` is missing, the unit fails and reports that the pin was not applied. -5. `aws kubeconfig` / `aws talosconfig`: `kubeconfig-aws-migration` → `~/.kube/config-aws-migration`, `talosconfig-aws-migration` → `~/.talos/config-aws-migration` (0600). These historical files remain separate from the home configs and no longer access the terminated cluster. - -A missing, empty, or failed note read prints `warn:` and leaves its existing file untouched, even if the failed command emitted partial output. If `kubectl config get-clusters` fails after a home config is written, the unit fails instead of silently reporting a successful TLS pin; inspect the config and re-run after fixing `kubectl`. +```bash +# First apply, including missing package prerequisites: +just apply-tags packages,kube -## Failure modes +# Subsequent credential refresh/default-context selection: +just apply-tags kube +``` -| Symptom | Cause | Fix | -|---|---|---| -| `kubectl` returns `Unable to connect to the server` | The historical endpoint may no longer host the cluster, or configs may be missing | Do not assume powering on a machine restores the cluster; inspect the intended target and, if configs are missing, unlock BW and re-run `just apply-tags kube` | -| `warn: kubeconfig ... missing in BW` | The note is unavailable or empty | Check the vault; if it needs seeding, run `just seed-kube` from a host with the intended home configs | -| Configs deployed but auth fails against a deliberately restored cluster | Cluster PKI may have rotated — local configs are stale | Confirm the intended cluster and configs before re-seeding, then re-apply on consumers | -| Unit prints `SKIPPED: not a desktop` on a host that should have it | `skip_kube: true` in `host_vars/.yml`, or host not in the `desktop` group | Adjust inventory or host_vars | -| `FAIL cannot inspect desktop inventory` | Missing `yq`, unreadable inventory, or malformed desktop hosts | Install `yq` and fix `inventory.yml`; no configs are changed while classification is unknown | -| `FAIL cannot list home kubeconfig clusters` | `kubectl` could not read the deployed home config | Inspect the config and local `kubectl` error, then retry; the TLS pin was not applied | +An individual remote may use `just apply-remote HOST` after reviewing the other units it applies. Do not target retired hosts. Offline computers receive the configuration on their next secret-enabled apply; the no-secrets timer cannot distribute credentials. -## Seeding BW +For a production host where installing Bitwarden/Node or running the full package unit is inappropriate, install only a supported `kubectl` binary and privately provision the standalone note contents as `~/.kube/config-karnataka-k3s` (directory 0700, regular file 0600). Then, from the checkout, explicitly apply only the local cache: ```bash -# From a machine that already has working configs: -just seed-kube +KUBE_CACHED_ONLY=1 UNITS_NO_SECRETS=1 bash units/kube.sh ``` -This packages `~/.kube/config` and `~/.talos/config` into BW notes named `kubeconfig` and `talosconfig`. It does **not** seed the historical `*-aws-migration` notes. After seeding, every desktop's next `dots-apply` (or `just apply-tags kube`) pulls the home configs down. +This opt-in never invokes Bitwarden or seeds recovery notes. It validates and merges the local cache, preserves the existing default's contexts and migration backup, and selects `karnataka-k3s`. A missing cache fails without changing the default. It does not refresh credentials; provision renewed cache contents privately when needed. Ordinary no-secrets/timer runs still skip without changing configs. + +## Bitwarden contract + +Create a distinct **secure note** named `kubeconfig-karnataka-k3s`. Its standard `notes` field must contain a complete standalone kubeconfig: + +- Context, cluster and user names are all `karnataka-k3s`; `current-context` is `karnataka-k3s`. +- The server is `https://100.66.158.127:6443`, not localhost. +- Embed the cluster CA and client certificate/key, and use normal certificate verification. Do not set `insecure-skip-tls-verify`. +- Store and transform credentials only in private files or pipes. Never paste the note, client key or raw kubeconfig into logs, chat or this public repository. + +The unit reads `bw get notes kubeconfig-karnataka-k3s`; it never edits vault notes. The historical `kubeconfig`, `talosconfig`, `kubeconfig-aws-migration` and `talosconfig-aws-migration` notes retain their existing meanings. + +## Apply behavior + +Except for the explicit cached-only mode above, a locked vault or `--no-secrets` prints `SKIPPED:` and leaves existing files alone. Otherwise: -## How to verify +1. Require `kubectl`, then create `~/.kube/` and `~/.talos/` with mode 0700. +2. Seed missing recovery caches only: home `kubeconfig` → `~/.kube/config-home-talos`, `talosconfig` → `~/.talos/config`, and the historical AWS notes → the matching `config-aws-migration` files in those directories. Existing recovery files are preserved, with permissions restricted to 0600. The newly seeded home kubeconfig receives the historical TLS server-name pin from `units/kube.conf` (default `192.168.0.5`), not the K3s kubeconfig. +3. Fetch K3s into a private temporary directory and select the required `karnataka-k3s` context. A failed or empty fetch may use the last successfully installed `~/.kube/config-karnataka-k3s` cache; without that cache the unit fails without changing the default config. +4. Merge K3s, the existing `~/.kube/config` and home recovery contexts into a private config, then select `karnataka-k3s`. Existing contexts take precedence over recovery-note contexts; the reserved K3s names take precedence over their older local copies. AWS recovery stays in its existing separate file. +5. On first migration of an existing default config, preserve its original bytes in `~/.kube/config-pre-karnataka-k3s` (0600). Atomically install the dedicated K3s cache and merged default config, both 0600. Subsequent applies preserve the original backup and restore K3s as the default without deleting historical or custom contexts. -These historical commands require a deliberately restored, reachable home -Talos cluster; do not assume one exists. The former AWS cluster cannot be queried. +Symlinks and non-regular destination files are rejected instead of overwritten. Merge/context-selection failures leave the existing default untouched. Temporary credential files are private and removed on exit; command output containing kubeconfig data is never logged. + +`just check` inspects only local existence and permissions, without reading Bitwarden items or mutating files. It reports that credential/default-context comparison cannot safely be completed offline; it does not claim to have verified cluster access. + +## Retained home-note seeding + +`just seed-kube` now reads `~/.kube/config-home-talos` and `~/.talos/config`, not the merged default kubeconfig. It only updates the retained home notes; it does not seed K3s or historical AWS notes. For deliberate home-cluster recovery from another private source file, set `KUBECONFIG=/absolute/path/to/home-only-config` explicitly. Never seed the historical home note from the merged K3s default. + +## Failure modes + +| Symptom | Cause | Fix | +|---|---|---| +| `kubectl missing; apply packages before kube` | A tag-only invocation omitted package prerequisites | Run `just apply-tags packages,kube` on the intended active client | +| Karnataka note unavailable and no cached config exists | Note missing, empty, failed read or unavailable to this account | Provision the distinct K3s note, unlock/sync the vault and reapply; do not replace historical notes | +| K3s config lacks a usable `karnataka-k3s` context | Note uses the unrenamed K3s `default` context or is malformed | Correct the standalone K3s note using the private provisioning workflow | +| Cannot merge kubeconfigs | An existing config is invalid or references unavailable credential files | Repair the private source config; its default file is left untouched on merge failure | +| `kubectl` cannot reach Karnataka | Client tailnet routing, server firewall or K3s may be unavailable | Inspect connectivity and the current server; do not run historical Talos recovery commands | +| `SKIPPED: skip_kube / vm-test` | Explicit host opt-out or test-machine profile | Review the intended host policy; changing desktop membership has no effect | + +## Checking active access + +Run these on the intended client after rollout; they do not print credentials: ```bash -kubectl get nodes # historical home cluster -talosctl --talosconfig ~/.talos/config get members +kubectl config current-context # karnataka-k3s +kubectl --context karnataka-k3s get nodes +kubectl --context karnataka-k3s get pods -A ``` -## Notes - -- These files contain cluster PKI material; the repo's `.gitignore` excludes the cluster source files, and these caches live in `~/` only — never commit. -- The unit is gated by desktop membership and `skip_kube` / `machine_profile: vm-test`. Servers don't get a kubeconfig; the former Talos cluster nodes were not Ansible-managed. +Historical Talos commands require a deliberately restored cluster. Do not interpret retained contexts as evidence that the old cluster is running, and do not apply historical recovery commands to repurposed Karnataka. diff --git a/docs/src/servers/talos-k8s/cluster.md b/docs/src/servers/talos-k8s/cluster.md index 385f0261..e59ad8a2 100644 --- a/docs/src/servers/talos-k8s/cluster.md +++ b/docs/src/servers/talos-k8s/cluster.md @@ -4,8 +4,11 @@ > Bihar's OS and services are unverified. Karnataka runs `projectbluefin/utah:testing` > with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to > tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are installed; -> `corral-vms/punjab` is now the active Punjab replacement (`100.103.187.46`); -> its former EC2 host is stopped, with recovery storage retained. +> `corral-vms/punjab` is now the active Punjab replacement (`100.103.187.46`). +> See [current Karnataka K3s operation and client access](karnataka/README.md#current-k3s-cluster) +> instead of using this handbook for the live cluster. Its client context is +> `karnataka-k3s`, provisioned from the separate Bitwarden note +> `kubeconfig-karnataka-k3s`; the former Talos credentials are recovery-only. > On the current server, firewalld puts `cni0` in the internal zone and accepts > pod subnet `10.42.0.0/16` to kubelet port `10250` on that tailnet address; > internal-zone services, including SSH, are accessible from pods. NetworkPolicy diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index b99f9ebd..d14940c8 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -1,11 +1,13 @@ -# Karnataka +# Karnataka — Utah and the active K3s cluster Karnataka runs `ghcr.io/projectbluefin/utah:testing`, is in the fleet's `server` inventory group, and uses `multi-user.target` as its default. SSH is available over Tailscale at `100.66.158.127`. A single-node K3s cluster runs KubeVirt, CDI, and Corral. The `corral-vms/punjab` VM now runs the migrated -Punjab environment at `100.103.187.46` (tailnet name `punjab`). Its former EC2 -host is stopped; its root volume and recovery snapshots are retained. +Punjab environment at `100.103.187.46` (tailnet name `punjab`). This is the +[active replacement](../../punjab/README.md), not a staging VM or an EC2 host. +For the former EC2 host's retirement and retained recovery artifacts, consult +the [AWS account handbook](../../aws/README.md). NetworkManager, Tailscale and SSH are enabled at boot. Ethernet is preferred; a locally stored Wi-Fi profile autoconnects with a higher route metric. Wi-Fi @@ -30,13 +32,136 @@ updates from the unmerged checkout. - GPU: AMD integrated - LAN: DHCP on the home network +## Current K3s cluster + +Read-only inspection on **2026-10-29** found: + +| Component | Live state | +|-----------|------------| +| API endpoint | `https://100.66.158.127:6443` over Tailscale | +| Client context | `karnataka-k3s` | +| Node | `karnataka`, `Ready`, control-plane, internal IP `100.66.158.127`, no external IP | +| Kubernetes | K3s `v1.36.5+k3s1`, containerd `2.3.4-k3s1.36` | +| Host image | Bluefin `testing-20261024-9acbde2`, kernel `7.2.11-200.fc44.x86_64` | +| Core services | CoreDNS, metrics-server and local-path-provisioner; deployments available | +| Virtualization | KubeVirt `v1.9.1` and CDI `v1.66.1`, both `Deployed` | +| Corral | `corral/corral-web`, available, ClusterIP service on port 80 | +| Storage | Default `local-path`, `rancher.io/local-path`, `WaitForFirstConsumer`, reclaim policy `Delete`, no expansion | +| Persistent claims | Bound `corral-vms/punjab-root` and `karnataka-cdi-readiness/cirros-readiness` | +| Punjab | `corral-vms/punjab` VM and VMI `Running`, ready on `karnataka`; guest tailnet IP `100.103.187.46` | + +All services returned by inspection were ClusterIP services (including one +headless metrics service); none advertised a LoadBalancer or NodePort. +The inspected deployments and virt-handler DaemonSet were ready. This is a +single-node cluster, not an HA control plane: host or local-disk loss affects +both Kubernetes and Punjab. A bound PVC is not a backup. + +The API listener is bound to `100.66.158.127:6443` and loopback, not all host +interfaces. The kubelet listener is `100.66.158.127:10250`. The serving +certificate inspected through the host included `100.66.158.127` and +`karnataka` as subject alternative names, was issued by the K3s server CA, +and expires 2027-10-27. Clients must use the embedded CA in the vault-backed +kubeconfig, not `--insecure-skip-tls-verify`. Tailscale routing and access +policy must permit each client to reach TCP 6443; merely reaching SSH does +not prove API connectivity. Do not publish the API or Corral on the Internet. + +### Fleet client provisioning + +The native [`kube` unit](../../../roles/kube.md) provisions the active cluster +on all active fleet computers, including servers, from the separate Bitwarden +secure note **`kubeconfig-karnataka-k3s`**. The note's standard notes field +contains a complete kubeconfig with cluster, user and context named +`karnataka-k3s`, the endpoint above, and embedded CA/client credentials. +Treat it as a secret; never print it or commit it. + +With a valid Bitwarden session, use the normal fleet entry points: + +```bash +just apply-tags packages,kube +# From the fleet controller, apply to each active reachable host: +just apply-remote HOST +``` + +The unit writes `~/.kube/config-karnataka-k3s` with mode `0600`, merges into +`~/.kube/config` without discarding historical contexts, and selects +`karnataka-k3s` on each apply. The first existing default config is backed up +once to `~/.kube/config-pre-karnataka-k3s`. An explicitly set `KUBECONFIG` +environment variable can override the default file; unset it or point to +the intended config before using these commands. + +For production hosts where only client access is required, an operator can +securely preseed the standalone cache above and install `kubectl`, then run +the narrower cached-only path instead of applying unrelated fleet units: + +```bash +KUBE_CACHED_ONLY=1 UNITS_NO_SECRETS=1 bash units/kube.sh +``` + +This mode merges/selects the cached context without invoking Bitwarden or +seeding recovery notes. Ordinary `--no-secrets` without this explicit mode +remains no-write; it does not provision a client. + +Keep the historical home Talos `kubeconfig` / `talosconfig` Bitwarden notes +and their files, and the separate AWS recovery configs. They are not inputs +for this live cluster and must not be overwritten with K3s credentials. +The [Talos handbook](../cluster.md) describes historical recovery only; +its reinstall, reset, bootstrap and workload commands do not apply here. +The terminated [AWS Talos cluster](../../aws-k8s/cluster.md) is also separate. + +### Verified client rollout (2026-10-29) + +Punjab, Karnataka, Kerala and Spindle now use `karnataka-k3s` as their default +context. Each passed a real authenticated `/readyz` request and listed the +Ready `karnataka` node with normal CA verification. Config files are `0600`; +the rollout compared existing contexts, clusters and users to confirm they +were retained. Private pre-rollout copies are under each user's +`~/.local/state/dotfiles/k3s-rollout-20261029T021854Z/`. + +Mumbai, Himachal, Dilli and Kanpur were unreachable during rollout. Termux +was on the tailnet but its configured SSH port 8022 refused connections. +Their repository policy/package changes are prepared, but those machines +have **not** received the live configuration; apply once they are reachable. +Retired `matrix` and `telengana` were not modified. + +### Read-only client smoke commands + +After provisioning and while connected to the tailnet, these commands read +cluster state without creating a pod, changing context or modifying resources: + +```bash +kubectl --context karnataka-k3s get --raw=/readyz +kubectl --context karnataka-k3s get nodes -o wide +kubectl --context karnataka-k3s get deployments,daemonsets -A +kubectl --context karnataka-k3s get services -A +kubectl --context karnataka-k3s get storageclasses,persistentvolumes +kubectl --context karnataka-k3s get persistentvolumeclaims -A +kubectl --context karnataka-k3s get virtualmachines,virtualmachineinstances -n corral-vms +``` + +For an isolated check of the cached credentials rather than the merged file, +add `--kubeconfig "$HOME/.kube/config-karnataka-k3s"` to a command. +Do not use `kubectl config view --raw`, dump Secret objects, or paste +credentials into diagnostics. If the API is unavailable, retain SSH and +inspect `systemctl status k3s` before considering any repair or restart; +Punjab depends on this server. + ## K3s operation -The host-specific files are in `home-k3s/`. K3s listens on the Tailscale -address, and its service waits for that address before starting. The service -loads `br_netfilter` and `overlay` when available. Run `systemctl status k3s` -and `sudo /var/usrlocal/bin/k3s kubectl --server https://100.66.158.127:6443 get pods -A` -to inspect it. Reloading the service definition does not restart K3s. +The host-specific files are in `home-k3s/`. The enabled, active systemd unit +is `/etc/systemd/system/k3s.service`, with +`ExecStart=/var/usrlocal/bin/k3s server`. It orders after +`network-online.target` and `tailscaled.service` and runs +`/var/usrlocal/libexec/k3s-wait-tailnet` before starting, then attempts to load +`br_netfilter` and `overlay`. The binary is pinned at +`/var/usrlocal/bin/k3s`. For read-only inspection on the host: + +```bash +systemctl status k3s +sudo /var/usrlocal/bin/k3s kubectl --server https://100.66.158.127:6443 get pods -A +``` + +Reloading the service definition does not restart K3s. Do not restart it as +part of client provisioning. `home-k3s/install-host.sh` reuses the pinned binary and refuses to replace a different binary while K3s is active. It does not restart an active service; @@ -65,6 +190,30 @@ SSH, are also reachable from pods; do not attach an untrusted interface to that zone. The LAN interfaces are in `public`. Both bridge rules were present in the permanent configuration when last inspected. +Read-only inspection on 2026-10-29 found `tailscale0` unassigned to a +firewalld zone, so incoming API connections used default `public`. That zone +allowed SSH but not TCP 6443; firewalld rejected API connections despite +Tailscale's earlier input-chain acceptance. `home-k3s/firewall.sh` now manages +a scoped API rich rule in `public`, in both runtime and permanent stores: +IPv4 source `100.64.0.0/10`, destination `100.66.158.127/32`, TCP 6443 only. +This is separate from the pod-egress policy and does not open other host +services, change zone/interface assignments, reload firewalld or restart K3s. +Tailscale's existing anti-spoofing input rules reject that source range on +non-Tailscale interfaces; retain those rules. + +The helper refuses changes if the default zone is no longer `public`, if +`tailscale0` is explicitly assigned elsewhere, or if any zone has source +bindings in either store (source classification can override interfaces). +Resolve such drift deliberately rather than opening another zone blindly. +`--check` requires the API rule in both stores; `--apply` checks it again +after changes. A successful firewall check is not proof of client API +connectivity: perform the read-only smoke commands above from each client. +The API rule was subsequently applied and verified in both stores, restoring +client API access without a reload. The full helper still returned nonzero +at the pre-existing absent runtime pod-egress policy described below; it was +not force-activated. This does not invalidate the separate API rule or the +authenticated client checks, and is not a claim that the full helper passed. + The broad `gateway-lan-to-world` baseline accepts traffic from `internal`, `home` and `trusted` to both `public` and `external` and must not be enabled unchanged. `home-k3s/firewall.sh` defines a dedicated `k3s-pod-egress` policy: diff --git a/docs/src/units.md b/docs/src/units.md index fc01f8ca..3c76d66c 100644 --- a/docs/src/units.md +++ b/docs/src/units.md @@ -32,3 +32,9 @@ Register a new executable `units/.sh` in `mise.toml`, `ALL_UNITS` and the `.chezmoiroot` points to `home/`, which contains the source files, `.chezmoi.toml.tmpl`, `.chezmoidata.yaml` and `.chezmoiscripts/`. `units/chezmoi.sh` applies shell, Git, Neovim and Atuin configuration with `--keep-going --force --no-tty apply --init`. It fails if chezmoi is unavailable (on Termux it first attempts a best-effort `pkg install`). Desktop detection honors `DOTFILES_IS_DESKTOP` or host `is_desktop`, then inventory membership. If the inventory lookup cannot be validated, including when Mike Farah `yq` is unavailable, the unit fails without applying files. **Back up and review existing dotfiles before first apply.** `--force` overwrites local edits to managed targets without prompting, including on subsequent runs. Even without `--force`, first adoption of an existing target can replace it. The source uses no `exact_` directories, so unmanaged files are not deleted by directory synchronization. Secrets are handled by units rather than chezmoi templates, so unavailable vault items do not abort unrelated file templates. `units/ssh-keys.sh` and `units/ssh-mesh.sh` manage SSH key lifecycle and authorization separately. + +## Kubernetes access + +The [`kube` unit](roles/kube.md) configures Karnataka K3s on active Linux and macOS computers, including servers, the phone's Debian VM and Termux. It honors explicit `skip_kube` / `vm-test` exclusions and the retired-host guard, not desktop membership. `kubectl` comes from core Homebrew (`kubernetes-cli`), APK or Termux packages. On a fresh client use `just apply-tags packages,kube`; `just apply-tags kube` alone does not install missing prerequisites. + +The separate Bitwarden note `kubeconfig-karnataka-k3s` supplies the active credentials. Applies merge the `karnataka-k3s` context into `~/.kube/config` and select it by default, preserving other contexts and retained Talos/AWS recovery material. Locked-vault and no-secrets applies leave existing configs untouched. A deliberate `KUBE_CACHED_ONLY=1` invocation can instead merge an already provisioned private cache without Bitwarden; see the [client rollout instructions](roles/kube.md#prerequisites-and-rollout). diff --git a/group_vars/all.yml b/group_vars/all.yml index 3c5dc1fb..7bde554f 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -88,6 +88,8 @@ core_brews: - shfmt - unzip - yq + # kubectl is required on servers and phone VMs as well as desktops. + - kubernetes-cli # atuin is also in bluefin-cli, but the brew copy keeps non-Bluefin glibc hosts (Ubuntu, Debian) covered - atuin # bluefin-cli provides: atuin, bat, eza, fd, fzf, starship, carapace, ripgrep, tealdeer, zoxide, colordiff, television, ugrep @@ -117,7 +119,6 @@ desktop_brews: - glab - ramalama - k9s - - kubernetes-cli - kubectx - krew - talosctl diff --git a/home-k3s/firewall.sh b/home-k3s/firewall.sh index d6f9206e..f4808067 100755 --- a/home-k3s/firewall.sh +++ b/home-k3s/firewall.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Manage cni0 and a source-scoped pod egress policy without reloading firewalld. +# Manage cni0, pod egress and tailnet API access without reloading firewalld. # The internal zone's existing services, including SSH, remain reachable # from pods. Public and Tailscale interface assignments are not changed. set -euo pipefail @@ -16,6 +16,8 @@ zone=internal rule='rule family="ipv4" source address="10.42.0.0/16" destination address="100.66.158.127/32" port port="10250" protocol="tcp" accept' policy=k3s-pod-egress egress_rule='rule family="ipv4" source address="10.42.0.0/16" accept' +api_zone=public +api_rule='rule family="ipv4" source address="100.64.0.0/10" destination address="100.66.158.127/32" port port="6443" protocol="tcp" accept' firewall-cmd --state >/dev/null # A source binding would classify packets by address even on a public NIC. @@ -39,6 +41,30 @@ preflight() { preflight runtime preflight permanent +# The API rule belongs in tailscale0's effective zone. Refuse classification +# drift rather than adding an ineffective rule or moving interfaces/sources. +# Source bindings take precedence over interfaces, so inspect every zone. +api_preflight() { + local scope=$1 zones candidate interfaces sources + local -a opts=() + [ "$scope" = runtime ] || opts=(--permanent) + [ "$(firewall-cmd --get-default-zone)" = "$api_zone" ] || { + echo "$scope: API access requires the public default zone; refusing" >&2; return 1; + } + zones=$(firewall-cmd "${opts[@]}" --get-zones) + for candidate in $zones; do + sources=$(firewall-cmd "${opts[@]}" --zone="$candidate" --list-sources) + [ -z "$sources" ] || { echo "$scope: $candidate has source bindings; API classification uncertain; refusing" >&2; return 1; } + interfaces=$(firewall-cmd "${opts[@]}" --zone="$candidate" --list-interfaces) + if [[ " $interfaces " == *" tailscale0 "* ]] && [ "$candidate" != "$api_zone" ]; then + echo "$scope: tailscale0 is in $candidate, not public; refusing" >&2 + return 1 + fi + done +} +api_preflight runtime +api_preflight permanent + # Refuse to take over an existing policy whose effects extend beyond this # script's contract. Partially created policies are safe to finish: their # target is CONTINUE and each populated field must already be the desired one. @@ -94,6 +120,7 @@ check_or_apply() { for scope in permanent runtime; do opts=() [ "$scope" = runtime ] || opts=(--permanent) + ensure "--query-rich-rule=$api_rule" "--add-rich-rule=$api_rule" "${opts[@]}" --zone="$api_zone" # Install the bounded rule and intra-zone forwarding before binding cni0. ensure "--query-rich-rule=$rule" "--add-rich-rule=$rule" "${opts[@]}" --zone="$zone" ensure --query-forward --add-forward "${opts[@]}" --zone="$zone" @@ -142,6 +169,8 @@ if [ "$mode" = --apply ]; then [ "$(firewall-cmd --get-default-zone)" != "$zone" ] || { echo 'internal became the default zone' >&2; exit 1; } preflight runtime preflight permanent + api_preflight runtime + api_preflight permanent policy_preflight runtime policy_preflight permanent mode=--check @@ -149,4 +178,4 @@ if [ "$mode" = --apply ]; then check_or_apply_policy fi -echo 'internal cni0 forwarding, kubelet rule and pod egress policy are present in runtime and permanent configuration' +echo 'internal cni0 forwarding, kubelet rule, scoped tailnet API rule and pod egress policy are present in runtime and permanent configuration; client connectivity must be checked separately' diff --git a/host_vars/mumbai.yml b/host_vars/mumbai.yml index 9e44a039..a961ae96 100644 --- a/host_vars/mumbai.yml +++ b/host_vars/mumbai.yml @@ -2,7 +2,6 @@ is_laptop: false machine_profile: cli-only # Skip everything heavy — phone VM, keep it lightweight -skip_kube: true skip_flatpak: true skip_bluefin: true skip_gnome: true diff --git a/host_vars/termux.yml b/host_vars/termux.yml index 570d2292..f50a8989 100644 --- a/host_vars/termux.yml +++ b/host_vars/termux.yml @@ -22,7 +22,6 @@ machine_profile: cli-only # No root, no sudo binary, no systemd, no glibc — none of these apply. skip_sudo: true skip_systemd: true -skip_kube: true skip_flatpak: true skip_bluefin: true skip_gnome: true diff --git a/mise.toml b/mise.toml index d3136b4d..b330c29c 100644 --- a/mise.toml +++ b/mise.toml @@ -27,7 +27,7 @@ description = "github: gh login from Bitwarden and SSH key registration" file = "units/github.sh" [tasks."unit:kube"] -description = "kube: kubeconfig and talosconfig for the home and AWS clusters from Bitwarden" +description = "kube: Karnataka K3s fleet default and retained Talos recovery configs from Bitwarden" file = "units/kube.sh" [tasks."unit:forgejo-registry"] diff --git a/scripts/bw-seed-kube.sh b/scripts/bw-seed-kube.sh index 3ef2e541..6bb818d7 100755 --- a/scripts/bw-seed-kube.sh +++ b/scripts/bw-seed-kube.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Seed Bitwarden with kubeconfig + talosconfig from this machine. +# Seed retained home Talos notes; never seed from the merged active K3s config. # Usage: ./scripts/bw-seed-kube.sh # Re-run safely — updates existing items in place. set -euo pipefail @@ -12,7 +12,7 @@ if [ -z "${BW_SESSION:-}" ]; then export BW_SESSION fi -KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}" +KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config-home-talos}" TALOSCONFIG_PATH="${TALOSCONFIG:-$HOME/.talos/config}" seed() { diff --git a/tests/test_home_k3s_firewall.py b/tests/test_home_k3s_firewall.py index 52abde39..90cd8f97 100644 --- a/tests/test_home_k3s_firewall.py +++ b/tests/test_home_k3s_firewall.py @@ -14,6 +14,10 @@ 'rule family="ipv4" source address="10.42.0.0/16" ' 'destination address="100.66.158.127/32" port port="10250" protocol="tcp" accept' ) +API_RULE = ( + 'rule family="ipv4" source address="100.64.0.0/10" ' + 'destination address="100.66.158.127/32" port port="6443" protocol="tcp" accept' +) def empty_policy(): @@ -41,10 +45,14 @@ def firewall(tmp_path, stub_env): scope = "permanent" if "--permanent" in args else "runtime" zone = s[scope] policy_name = next((a.split("=", 1)[1] for a in args if a.startswith("--policy=")), None) +if "--zone=public" in args: + zone = zone["public"] if "--state" in args: print("running") elif "--get-default-zone" in args: print(s["default"]) +elif "--get-zones" in args: + print("internal public") elif "--get-policies" in args: print(" ".join(zone["policies"])) elif any(a.startswith("--info-policy=") for a in args): @@ -105,7 +113,8 @@ def firewall(tmp_path, stub_env): if key == "interfaces": zone[key].append(value) else: - zone[key] = value if key == "rule" else True + if not (key == "rule" and "--zone=public" in args and os.environ.get("IGNORE_API_ADD")): + zone[key] = value if key == "rule" else True s["changes"].append([scope, op]) with open(p, "w") as f: json.dump(s, f) @@ -119,6 +128,8 @@ def firewall(tmp_path, stub_env): "permanent": {"interfaces": [], "sources": [], "forward": False, "rule": "", "policies": {}}, "changes": [], } + for scope in ("runtime", "permanent"): + state[scope]["public"] = {"interfaces": [], "sources": [], "rule": ""} path.write_text(json.dumps(state)) env["FIREWALL_STATE"] = str(path) @@ -142,6 +153,8 @@ def test_apply_is_scoped_and_idempotent(firewall): assert zone["interfaces"] == ["cni0"] assert zone["sources"] == [] assert zone["forward"] is True + assert zone["public"] == {"interfaces": [], "sources": [], "rule": API_RULE} + assert zone["rule"] == KUBELET_RULE assert state["permanent"]["policies"] == {POLICY: expected} assert state["runtime"]["policies"] == {} ops = [op for scope, op in state["changes"] if scope == "permanent"] @@ -172,7 +185,8 @@ def test_check_reports_missing_without_changes(firewall): assert not state["changes"] -def test_check_accepts_host_rule_and_egress_policy(firewall): +@pytest.mark.parametrize("tailnet_interfaces", [[], ["tailscale0"]]) +def test_check_accepts_host_rule_and_egress_policy(firewall, tailnet_interfaces): path, _, run = firewall state = json.loads(path.read_text()) policy = empty_policy() @@ -181,6 +195,8 @@ def test_check_accepts_host_rule_and_egress_policy(firewall): for scope in ("runtime", "permanent"): state[scope].update(interfaces=["cni0"], forward=True, rule=KUBELET_RULE, policies={POLICY: policy}) + state[scope]["public"]["rule"] = API_RULE + state[scope]["public"]["interfaces"] = tailnet_interfaces path.write_text(json.dumps(state)) result, after = run("--check") assert result.returncode == 0, result.stderr @@ -252,6 +268,54 @@ def test_query_error_is_not_treated_as_missing(firewall): assert not state["changes"] +@pytest.mark.parametrize("scope", ["runtime", "permanent"]) +def test_missing_api_rule_is_reported_without_changes(firewall, scope): + path, _, run = firewall + run("--apply") + state = json.loads(path.read_text()) + state[scope]["public"]["rule"] = "" + path.write_text(json.dumps(state)) + result, after = run("--check") + assert result.returncode != 0 + assert "6443" in result.stderr + assert after == state + + +@pytest.mark.parametrize("scope", ["runtime", "permanent"]) +@pytest.mark.parametrize("drift", ["default", "interface", "source"]) +def test_api_zone_classification_drift_refused_before_changes(firewall, scope, drift): + path, _, run = firewall + state = json.loads(path.read_text()) + if drift == "default": + state["default"] = "external" + elif drift == "interface": + state[scope]["interfaces"] = ["tailscale0"] + else: + state[scope]["public"]["sources"] = ["100.103.187.46/32"] + path.write_text(json.dumps(state)) + result, after = run("--apply") + assert result.returncode != 0 + assert "refusing" in result.stderr + assert after == state + + +def test_apply_verifies_api_rule_was_actually_installed(firewall): + path, env, run = firewall + policy = empty_policy() + policy.update({"ingress-zones": ["internal"], "egress-zones": ["public"], + "rich rules": [EGRESS_RULE]}) + state = json.loads(path.read_text()) + for scope in ("runtime", "permanent"): + state[scope]["policies"][POLICY] = policy + path.write_text(json.dumps(state)) + env["IGNORE_API_ADD"] = "1" + result, after = run("--apply") + assert result.returncode != 0 + assert "6443" in result.stderr + for scope in ("runtime", "permanent"): + assert not after[scope]["public"]["rule"] + + BRIDGE_SCRIPT = REPO / "home-k3s" / "firewall-egress-bridge.sh" RULE4 = 'rule priority="-1" family="ipv4" source NOT address="10.42.0.0/16" drop' RULE6 = 'rule priority="-1" family="ipv6" source address="::/0" drop' diff --git a/tests/test_kube_seed.py b/tests/test_kube_seed.py new file mode 100644 index 00000000..34336ae6 --- /dev/null +++ b/tests/test_kube_seed.py @@ -0,0 +1,38 @@ +import json +import subprocess +from pathlib import Path + +from conftest import REPO, write_stub + + +def test_seed_kube_defaults_to_retained_home_cache_not_merged_default(stub_env): + bindir, env = stub_env + home = Path(env["HOME"]) + (home / ".kube").mkdir() + (home / ".talos").mkdir() + (home / ".kube/config").write_text("active-k3s-private-config") + (home / ".kube/config-home-talos").write_text("retained-home-private-config") + (home / ".talos/config").write_text("retained-talos-private-config") + write_stub(bindir, "bw", r''' +case "$1 $2" in + "get template") echo '{}' ;; + "list items") echo '[]' ;; + "encode ") cat ;; + "create item") + payload="$(cat)" + name="$(printf '%s' "$payload" | jq -r .name)" + printf '%s' "$payload" > "$HOME/$name-seeded.json" + ;; + "sync ") exit 0 ;; + *) exit 1 ;; +esac +''') + env.pop("KUBECONFIG", None) + env.pop("TALOSCONFIG", None) + result = subprocess.run(["bash", str(REPO / "scripts/bw-seed-kube.sh")], + env={**env, "BW_SESSION": "test-session"}, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + assert json.loads((home / "kubeconfig-seeded.json").read_text())["notes"] == "retained-home-private-config" + assert json.loads((home / "talosconfig-seeded.json").read_text())["notes"] == "retained-talos-private-config" + assert "private-config" not in result.stdout + result.stderr + assert not (home / "kubeconfig-karnataka-k3s-seeded.json").exists() diff --git a/tests/test_kube_unit.py b/tests/test_kube_unit.py index b17f6519..861ed83f 100644 --- a/tests/test_kube_unit.py +++ b/tests/test_kube_unit.py @@ -1,142 +1,304 @@ +import json import stat from pathlib import Path import pytest -from conftest import write_stub +from conftest import REPO, write_stub from unit_helpers import UNLOCKED_BW, calls, run_unit -NOTES = """\ + +def kubeconfig(name, server="https://example.invalid:6443"): + return { + "apiVersion": "v1", "kind": "Config", + "clusters": [{"name": name, "cluster": {"server": server}}], + "users": [{"name": name, "user": {"token": "test-private-token"}}], + "contexts": [{"name": name, "context": {"cluster": name, "user": name}}], + "current-context": name, + } + + +# Models kubectl's local config operations, including first-file merge precedence. +# Only synthetic JSON kubeconfigs are used; there is no cluster/vault access. +KUBECTL = r''' +import json, os, sys +from pathlib import Path +args = sys.argv[1:] +with open(Path(os.environ["HOME"]) / "calls", "a") as log: + log.write("kubectl " + " ".join(args) + "\n") +paths = os.environ["KUBECONFIG"].split(":") +try: + configs = [json.loads(Path(path).read_text()) for path in paths] + config = configs[0] + if args[:2] == ["config", "get-clusters"]: + print("NAME") + for item in config["clusters"]: + print(item["name"]) + if os.environ.get("FAIL_LIST") == "1": + sys.exit(1) + elif args[:2] == ["config", "set-cluster"]: + for item in config["clusters"]: + if item["name"] == args[2]: + item["cluster"]["tls-server-name"] = args[3].split("=", 1)[1] + Path(paths[0]).write_text(json.dumps(config)) + elif args[:2] == ["config", "get-contexts"]: + for item in config["contexts"]: + if item["name"] == args[2]: + print(item["name"]) + elif args[:2] == ["config", "use-context"]: + if os.environ.get("FAIL_SELECT") == "1" or not any(item["name"] == args[2] for item in config["contexts"]): + print("test-private-invalid-config", file=sys.stderr) + sys.exit(1) + config["current-context"] = args[2] + Path(paths[0]).write_text(json.dumps(config)) + elif args[:2] == ["config", "view"]: + if os.environ.get("FAIL_MERGE") == "1": + print("test-private-partial-config") + print("test-private-merge-error", file=sys.stderr) + sys.exit(1) + merged = {"apiVersion": "v1", "kind": "Config", "current-context": config["current-context"]} + for field in ("clusters", "users", "contexts"): + items = {} + for source in configs: + for item in source[field]: + items.setdefault(item["name"], item) + merged[field] = list(items.values()) + print(json.dumps(merged)) + else: + sys.exit(1) +except (ValueError, KeyError, OSError): + print("test-private-invalid-config", file=sys.stderr) + sys.exit(1) +''' + + +@pytest.fixture +def env(stub_env): + bindir, env = stub_env + home = Path(env["HOME"]) + (home / "home-note").write_text(json.dumps(kubeconfig("home"))) + (home / "k3s-note").write_text(json.dumps(kubeconfig("karnataka-k3s", "https://100.66.158.127:6443"))) + write_stub(bindir, "bw", UNLOCKED_BW % r''' if [ "$1 $2" = "get notes" ]; then case "$3" in - kubeconfig) echo "home-kube"; exit 0 ;; + kubeconfig) cat "$HOME/home-note"; exit 0 ;; + kubeconfig-karnataka-k3s) + [ "${NO_K3S_NOTE:-0}" = 1 ] && { echo test-private-partial-config; exit 1; } + cat "$HOME/k3s-note"; exit 0 ;; talosconfig) echo "home-talos"; exit 0 ;; kubeconfig-aws-migration) echo "aws-kube"; exit 0 ;; talosconfig-aws-migration) exit 1 ;; esac fi -""" - - -@pytest.fixture -def env(stub_env): - bindir, env = stub_env - write_stub(bindir, "bw", UNLOCKED_BW % NOTES) - write_stub(bindir, "kubectl", 'echo "kubectl $*" >> "$HOME/calls"\n[ "$2" = get-clusters ] && printf "NAME\\nhome\\n"\nexit 0\n') +''') + kubectl = home / "kubectl-stub.py" + kubectl.write_text(KUBECTL) + write_stub(bindir, "kubectl", f'exec python3 "{kubectl}" "$@"\n') env.update(BW_SESSION="x", DOTFILES_IS_DESKTOP="1") return env -def test_configs_are_written_private_and_the_home_cluster_is_pinned(env): - r = run_unit("kube", env) - assert r.returncode == 0, r.stdout + r.stderr +def config(env): + return json.loads(Path(env["HOME"], ".kube/config").read_text()) + + +def existing_config(env, name="custom"): + path = Path(env["HOME"], ".kube/config") + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(kubeconfig(name))) + return path + + +def test_configs_are_private_and_k3s_is_default_with_home_recovery(env): + result = run_unit("kube", env) + assert result.returncode == 0, result.stdout + result.stderr home = Path(env["HOME"]) - assert (home / ".kube/config").read_text().strip() == "home-kube" + assert config(env)["current-context"] == "karnataka-k3s" + assert {item["name"] for item in config(env)["contexts"]} == {"home", "karnataka-k3s"} + clusters = {item["name"]: item["cluster"] for item in config(env)["clusters"]} + assert clusters["home"]["tls-server-name"] == "192.168.0.5" + assert "tls-server-name" not in clusters["karnataka-k3s"] assert (home / ".talos/config").read_text().strip() == "home-talos" assert (home / ".kube/config-aws-migration").read_text().strip() == "aws-kube" - for f in (".kube/config", ".talos/config", ".kube/config-aws-migration"): - assert stat.S_IMODE((home / f).stat().st_mode) == 0o600 - assert "kubectl config set-cluster home --tls-server-name=192.168.0.5" in calls(env) + for name in (".kube/config", ".kube/config-home-talos", ".kube/config-karnataka-k3s", ".talos/config", ".kube/config-aws-migration"): + assert stat.S_IMODE((home / name).stat().st_mode) == 0o600 + assert "test-private-token" not in result.stdout + result.stderr + assert not list((home / ".kube").glob(".fleet.*")) -def test_a_missing_note_warns_and_leaves_no_file(env): - r = run_unit("kube", env) - assert "AWS talosconfig missing in BW" in r.stdout - assert not (Path(env["HOME"]) / ".talos/config-aws-migration").exists() +def test_existing_contexts_and_original_bytes_survive_repeated_applies(env): + path = existing_config(env) + original = path.read_bytes() + first = run_unit("kube", env) + assert first.returncode == 0, first.stdout + first.stderr + merged = path.read_bytes() + second = run_unit("kube", env) + assert second.returncode == 0, second.stdout + second.stderr + assert path.read_bytes() == merged + assert Path(env["HOME"], ".kube/config-pre-karnataka-k3s").read_bytes() == original + assert {item["name"] for item in config(env)["contexts"]} == {"custom", "home", "karnataka-k3s"} + assert config(env)["current-context"] == "karnataka-k3s" -def test_locked_vault_leaves_existing_configs(env): - cfg = Path(env["HOME"], ".kube/config") - cfg.parent.mkdir() - cfg.write_text("previous") - r = run_unit("kube", env, STUB_UNLOCK_RC="1") - assert r.returncode == 0 and "SKIPPED:" in r.stdout - assert cfg.read_text() == "previous" +def test_later_apply_restores_k3s_default_without_losing_custom_contexts(env): + path = existing_config(env) + assert run_unit("kube", env).returncode == 0 + changed = config(env) + changed["current-context"] = "custom" + path.write_text(json.dumps(changed)) + result = run_unit("kube", env) + assert result.returncode == 0, result.stdout + result.stderr + assert config(env)["current-context"] == "karnataka-k3s" + assert "custom" in {item["name"] for item in config(env)["contexts"]} -@pytest.mark.parametrize("extra", [{"DOTFILES_IS_DESKTOP": "0"}]) -def test_non_desktops_do_nothing(env, extra): - r = run_unit("kube", env, **extra) - assert "SKIPPED:" in r.stdout - assert not (Path(env["HOME"]) / ".kube").exists() +def test_fresh_install_does_not_later_backup_merged_config_as_legacy(env): + assert run_unit("kube", env).returncode == 0 + assert run_unit("kube", env).returncode == 0 + assert not Path(env["HOME"], ".kube/config-pre-karnataka-k3s").exists() -def test_skip_kube_host_var_wins_over_desktop(env, tmp_path): - repo = tmp_path / "repo" - (repo / "host_vars").mkdir(parents=True) - (repo / "host_vars/phone.yml").write_text("machine_profile: cli-only\nskip_kube: true\n") - (repo / "units").symlink_to(Path(__file__).resolve().parent.parent / "units") - r = run_unit("kube", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="phone") - assert "SKIPPED:" in r.stdout and not (Path(env["HOME"]) / ".kube").exists() +def test_explicit_cached_only_rollout_merges_without_bw_and_preserves_original(env): + path = existing_config(env) + original = path.read_bytes() + cache = path.parent / "config-karnataka-k3s" + cache.write_text(Path(env["HOME"], "k3s-note").read_text()) + cache.chmod(0o600) + bindir = Path(env["PATH"].split(":")[0]) + write_stub(bindir, "bw", 'echo unexpected-bw-call >> "$HOME/calls"\nexit 99\n') + result = run_unit("kube", env, KUBE_CACHED_ONLY="1", UNITS_NO_SECRETS="1") + assert result.returncode == 0, result.stdout + result.stderr + assert config(env)["current-context"] == "karnataka-k3s" + assert {item["name"] for item in config(env)["contexts"]} == {"custom", "karnataka-k3s"} + assert (path.parent / "config-pre-karnataka-k3s").read_bytes() == original + assert "bw" not in calls(env) + assert not Path(env["HOME"], ".talos").exists() -@pytest.mark.parametrize("result", ["failure", "malformed", "invalid"]) -def test_unknown_desktop_inventory_fails_without_reading_secrets(env, tmp_path, result): - repo = tmp_path / "repo" - (repo / "host_vars").mkdir(parents=True) - (repo / "units").symlink_to(Path(__file__).resolve().parent.parent / "units") - bindir = Path(env["PATH"].split(":")[0]) - write_stub(bindir, "yq", '''\ -if [ "$LOOKUP" = failure ]; then exit 1; fi -case "$*" in - *'| tag'*) [ "$LOOKUP" = malformed ] && echo '!!seq' || echo '!!map' ;; - *) echo invalid ;; -esac -''') - r = run_unit("kube", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="unknown", - DOTFILES_IS_DESKTOP="", LOOKUP=result) - assert r.returncode != 0 - assert "FAIL" in r.stdout - assert not (Path(env["HOME"]) / ".kube").exists() - assert calls(env) == "" +def test_cached_only_rollout_without_cache_preserves_default_and_never_reads_bw(env): + path = existing_config(env) + original = path.read_bytes() + result = run_unit("kube", env, KUBE_CACHED_ONLY="1", UNITS_NO_SECRETS="1") + assert result.returncode != 0 and "no cached config exists" in result.stdout + assert path.read_bytes() == original + assert "bw " not in calls(env) + + +def test_recovery_caches_are_not_overwritten_or_read_from_vault(env): + home = Path(env["HOME"]) + for name, content in ((".talos/config", "retained-talos"), (".kube/config-aws-migration", "retained-aws"), (".talos/config-aws-migration", "retained-aws-talos")): + path = home / name + path.parent.mkdir(exist_ok=True) + path.write_text(content) + result = run_unit("kube", env) + assert result.returncode == 0, result.stdout + result.stderr + assert (home / ".talos/config").read_text() == "retained-talos" + assert (home / ".kube/config-aws-migration").read_text() == "retained-aws" + assert (home / ".talos/config-aws-migration").read_text() == "retained-aws-talos" + assert "bw get notes talosconfig" not in calls(env) + assert "bw get notes kubeconfig-aws-migration" not in calls(env) + + +def test_a_missing_recovery_note_warns_and_leaves_no_file(env): + result = run_unit("kube", env) + assert result.returncode == 0 + assert "talosconfig-aws-migration missing in BW" in result.stdout + assert not Path(env["HOME"], ".talos/config-aws-migration").exists() + + +@pytest.mark.parametrize("extra", [{"STUB_UNLOCK_RC": "1"}, {"UNITS_NO_SECRETS": "1"}]) +def test_locked_or_disabled_vault_leaves_existing_configs(env, extra): + path = existing_config(env) + original = path.read_bytes() + result = run_unit("kube", env, **extra) + assert result.returncode == 0 and "SKIPPED:" in result.stdout + assert path.read_bytes() == original + + +@pytest.mark.parametrize("machine", ["punjab", "karnataka", "spindle", "mumbai", "termux", "unknown-active-server"]) +def test_all_active_computers_are_eligible_without_desktop_lookup(env, machine): + result = run_unit("kube", env, DOTFILES_IS_DESKTOP="0", DOTFILES_MACHINE=machine) + assert result.returncode == 0, result.stdout + result.stderr + assert config(env)["current-context"] == "karnataka-k3s" + + +@pytest.mark.parametrize("machine", ["matrix", "telengana"]) +def test_retired_hosts_are_never_touched(env, machine): + result = run_unit("kube", env, DOTFILES_MACHINE=machine) + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert calls(env) == "" and not Path(env["HOME"], ".kube").exists() @pytest.mark.parametrize("profile", ["skip_kube: true\n", "machine_profile: vm-test\n"]) -def test_profile_skips_without_desktop_lookup(env, tmp_path, profile): +def test_explicit_opt_out_and_test_profile_skip_before_reading_secrets(env, tmp_path, profile): repo = tmp_path / "repo" (repo / "host_vars").mkdir(parents=True) (repo / "host_vars/phone.yml").write_text(profile) - (repo / "units").symlink_to(Path(__file__).resolve().parent.parent / "units") - bindir = Path(env["PATH"].split(":")[0]) - write_stub(bindir, "yq", 'exit 1\n') - r = run_unit("kube", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="phone", - DOTFILES_IS_DESKTOP="") - assert r.returncode == 0 and "SKIPPED" in r.stdout - assert calls(env) == "" + (repo / "units").symlink_to(REPO / "units") + result = run_unit("kube", env, DOTFILES_DIR=str(repo), DOTFILES_MACHINE="phone") + assert result.returncode == 0 and "SKIPPED" in result.stdout + assert calls(env) == "" and not Path(env["HOME"], ".kube").exists() -def test_failed_note_read_preserves_existing_config_even_with_stdout(env): - cfg = Path(env["HOME"], ".kube/config") - cfg.parent.mkdir() - cfg.write_text("previous\n") - bindir = Path(env["PATH"].split(":")[0]) - write_stub(bindir, "bw", UNLOCKED_BW % ''' -if [ "$1 $2 $3" = "get notes kubeconfig" ]; then - echo partial-private-config - exit 1 -fi -''') - result = run_unit("kube", env) - assert result.returncode == 0 - assert cfg.read_text() == "previous\n" - assert "set-cluster" not in calls(env) - assert "partial-private-config" not in result.stdout + result.stderr +def test_missing_k3s_note_preserves_existing_default_and_hides_partial_stdout(env): + path = existing_config(env) + original = path.read_bytes() + result = run_unit("kube", env, NO_K3S_NOTE="1") + assert result.returncode != 0 and "no cached config exists" in result.stdout + assert path.read_bytes() == original + assert "test-private-partial-config" not in result.stdout + result.stderr -def test_failed_cluster_listing_reports_failure_without_pinning_partial_output(env): - bindir = Path(env["PATH"].split(":")[0]) - write_stub(bindir, "kubectl", ''' -echo "kubectl $*" >> "$HOME/calls" -if [ "$2" = get-clusters ]; then printf 'NAME\\nhome\\n'; exit 1; fi -exit 0 -''') +def test_failed_refresh_uses_last_successful_k3s_cache(env): + assert run_unit("kube", env).returncode == 0 + result = run_unit("kube", env, NO_K3S_NOTE="1") + assert result.returncode == 0, result.stdout + result.stderr + assert config(env)["current-context"] == "karnataka-k3s" + assert "test-private-partial-config" not in result.stdout + result.stderr + + +@pytest.mark.parametrize("failure", ["FAIL_MERGE", "FAIL_SELECT"]) +def test_failed_merge_or_context_selection_is_atomic_and_private(env, failure): + path = existing_config(env) + original = path.read_bytes() + result = run_unit("kube", env, **{failure: "1"}) + assert result.returncode != 0 + assert path.read_bytes() == original + assert "test-private" not in result.stdout + result.stderr + assert not Path(env["HOME"], ".kube/config-karnataka-k3s").exists() + + +def test_invalid_k3s_note_preserves_existing_default(env): + path = existing_config(env) + original = path.read_bytes() + Path(env["HOME"], "k3s-note").write_text(json.dumps(kubeconfig("wrong-context"))) result = run_unit("kube", env) + assert result.returncode != 0 and "lacks a usable" in result.stdout + assert path.read_bytes() == original + assert "test-private" not in result.stdout + result.stderr + + +def test_failed_cluster_listing_does_not_install_partial_recovery_cache(env): + result = run_unit("kube", env, FAIL_LIST="1") assert result.returncode != 0 and "cannot list home kubeconfig clusters" in result.stdout + assert not Path(env["HOME"], ".kube/config-home-talos").exists() + assert config(env)["current-context"] == "karnataka-k3s" assert "set-cluster" not in calls(env) - assert Path(env["HOME"], ".kube/config-aws-migration").read_text().strip() == "aws-kube" -def test_missing_kubectl_reports_unpinned_config(env, tmp_path): +def test_symlinked_default_is_not_replaced(env): + home = Path(env["HOME"]) + target = home / "untouched" + target.write_text("retained-config") + (home / ".kube").mkdir() + (home / ".kube/config").symlink_to(target) + result = run_unit("kube", env) + assert result.returncode != 0 and "refusing non-regular" in result.stdout + assert target.read_text() == "retained-config" + assert (home / ".kube/config").is_symlink() + + +def test_missing_kubectl_reports_package_prerequisite_without_changing_files(env, tmp_path): bash_env = tmp_path / "bash-env" bash_env.write_text(''' command() { @@ -145,5 +307,5 @@ def test_missing_kubectl_reports_unpinned_config(env, tmp_path): } ''') result = run_unit("kube", env, BASH_ENV=str(bash_env)) - assert result.returncode != 0 and "kubectl missing; tls-server-name not pinned" in result.stdout - assert Path(env["HOME"], ".kube/config-aws-migration").read_text().strip() == "aws-kube" + assert result.returncode != 0 and "apply packages before kube" in result.stdout + assert not Path(env["HOME"], ".kube").exists() diff --git a/units/apk-packages.conf b/units/apk-packages.conf index 51ce56cb..8ce4f3da 100644 --- a/units/apk-packages.conf +++ b/units/apk-packages.conf @@ -4,6 +4,6 @@ APK_BOOTSTRAP=(chezmoi yq-go python3 py3-yaml just) APK_PACKAGES=( bash fish zsh git git-lfs delta vim neovim tmux openssh-client curl wget jq ripgrep fd fzf starship bat exa direnv zoxide atuin-sync shellcheck shfmt - nodejs npm busybox-extras podman podman-compose crun iptables + nodejs npm busybox-extras podman podman-compose crun iptables kubectl ) APK_OPTIONAL=(mise) diff --git a/units/kube.sh b/units/kube.sh index 112ff703..157d56af 100755 --- a/units/kube.sh +++ b/units/kube.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# kube unit: kubeconfig + talosconfig for the home and AWS clusters from Bitwarden notes. -# Seed the home notes with scripts/bw-seed-kube.sh. Desktops only. +# K3s is the fleet default; historical Talos configs remain recovery material. set -uo pipefail # shellcheck source=units/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" @@ -8,29 +7,14 @@ set -uo pipefail . "$DOTFILES_DIR/units/kube.conf" if skipped_by_profile skip_kube; then - note "SKIPPED: not a desktop (or skip_kube / vm-test)" + note "SKIPPED: skip_kube / vm-test" exit 0 fi - -desktop="${DOTFILES_IS_DESKTOP:-$(host_var is_desktop)}" -case "$desktop" in - 1|true|0|false) ;; - '') - if ! command -v yq >/dev/null 2>&1 || - ! kind="$(yq -r '.all.children.desktop.hosts | tag' "$DOTFILES_DIR/inventory.yml" 2>/dev/null)" || - [ "$kind" != '!!map' ] || - ! desktop="$(MACHINE="$MACHINE" yq -r '.all.children.desktop.hosts | has(strenv(MACHINE))' "$DOTFILES_DIR/inventory.yml" 2>/dev/null)"; then - note "FAIL cannot inspect desktop inventory (yq and valid inventory required)" - exit 1 - fi - case "$desktop" in true|false) ;; *) note "FAIL invalid desktop inventory result"; exit 1 ;; esac ;; - *) note "FAIL invalid desktop classification"; exit 1 ;; +case "$(uname -s)" in + Linux|Darwin) ;; + *) note "SKIPPED: unsupported kubectl platform"; exit 0 ;; esac -if [ "$desktop" = 0 ] || [ "$desktop" = false ]; then - note "SKIPPED: not a desktop (or skip_kube / vm-test)" - exit 0 -fi if [ "${UNITS_CHECK:-0}" = 1 ]; then if [ "${UNITS_NO_SECRETS:-0}" = 1 ]; then note "SKIPPED: secrets disabled; existing kube/talos configs left untouched" @@ -39,7 +23,7 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then for dir in "$HOME/.kube" "$HOME/.talos"; do if [ ! -e "$dir" ]; then note "PARTIAL: $dir absent; apply would create it if the vault is unlocked" - elif [ ! -d "$dir" ] || ! mode="$(stat -Lc '%a' "$dir")"; then + elif [ ! -d "$dir" ] || ! mode="$(stat -Lc '%a' "$dir" 2>/dev/null || stat -Lf '%Lp' "$dir" 2>/dev/null)"; then note "FAIL cannot inspect directory $dir" elif [ "$mode" != 700 ]; then note "PARTIAL: $dir mode differs from 0700; apply would change it if the vault is unlocked" @@ -47,60 +31,125 @@ if [ "${UNITS_CHECK:-0}" = 1 ]; then note "ok $dir mode 0700" fi done - for path in "$HOME/.kube/config" "$HOME/.talos/config" "$HOME/.kube/config-aws-migration" "$HOME/.talos/config-aws-migration"; do + for path in "$HOME/.kube/config" "$HOME/.kube/config-karnataka-k3s" "$HOME/.kube/config-home-talos" "$HOME/.talos/config" "$HOME/.kube/config-aws-migration" "$HOME/.talos/config-aws-migration"; do if [ -L "$path" ] || [ ! -e "$path" ]; then - note "PARTIAL: $path absent or symlinked; apply would replace it if the note is available" - elif [ ! -f "$path" ] || ! mode="$(stat -c '%a' "$path")"; then + note "PARTIAL: $path absent or symlinked; apply requires a regular config file" + elif [ ! -f "$path" ] || ! mode="$(stat -c '%a' "$path" 2>/dev/null || stat -f '%Lp' "$path" 2>/dev/null)"; then note "FAIL cannot inspect $path" elif [ "$mode" != 600 ]; then - note "PARTIAL: $path mode differs from 0600; apply would replace it if the note is available" + note "PARTIAL: $path mode differs from 0600" else note "ok $path mode 0600" fi done - note "FAIL cannot compare kube/talos notes or TLS pin safely: Bitwarden item reads may persist audit events" + note "FAIL cannot compare kube/talos notes or default context safely: Bitwarden item reads may persist audit events" exit 1 fi -s="$(bw_session)" -if [ -z "$s" ]; then - note "SKIPPED: vault locked or --no-secrets; existing kube/talos configs left untouched" - exit 0 +s="" +if [ "${KUBE_CACHED_ONLY:-0}" != 1 ]; then + s="$(bw_session)" + if [ -z "$s" ]; then + note "SKIPPED: vault locked or --no-secrets; existing kube/talos configs left untouched" + exit 0 + fi fi +if ! command -v kubectl >/dev/null 2>&1; then + note "FAIL kubectl missing; apply packages before kube" + exit 1 +fi +step "kube dir" install -d -m 0700 "$HOME/.kube" +if [ "${KUBE_CACHED_ONLY:-0}" != 1 ]; then + step "talos dir" install -d -m 0700 "$HOME/.talos" +fi +[ "$UNIT_FAILED" = 0 ] || finish +work="$(umask 077; mktemp -d "$HOME/.kube/.fleet.XXXXXX")" || exit 1 +trap 'rm -rf "$work"' EXIT -step "kube dirs" install -d -m 0700 "$HOME/.kube" "$HOME/.talos" - -# fetch_note ITEM DEST LABEL: write the note to DEST when Bitwarden has it; prints a warning otherwise. +# Never replace a symlink, special file, or an existing recovery cache. +regular_or_absent() { + if [ -L "$1" ] || { [ -e "$1" ] && [ ! -f "$1" ]; }; then + note "FAIL refusing non-regular config $1" + return 1 + fi +} fetch_note() { - local item="$1" dest="$2" label="$3" val - val="$(BW_SESSION="$s" bw get notes "$item" 2>/dev/null)" || val="" - if [ -z "$val" ]; then - note "warn: $label missing in BW (item $item)" - return 0 + local item="$1" dest="$2" val + if ! val="$(BW_SESSION="$s" bw get notes "$item" 2>/dev/null)" || [ -z "$val" ]; then + note "warn: $item missing in BW; existing config left untouched" + return 2 fi - printf '%s\n' "$val" | write_atomic "$dest" || return 1 - [ "$dest" != "$HOME/.kube/config" ] || home_written=1 + printf '%s\n' "$val" | write_atomic "$dest" } - -# The home kubeconfig is rewritten from Bitwarden each run, so the TLS pin is re-applied each time. -pin_tls() { - command -v kubectl >/dev/null 2>&1 || { note "FAIL kubectl missing; tls-server-name not pinned"; return 1; } - local c clusters header=1 rc=0 - if ! clusters="$(KUBECONFIG="$HOME/.kube/config" kubectl config get-clusters 2>/dev/null)"; then - note "FAIL cannot list home kubeconfig clusters; TLS pin not applied" +seed_recovery() { + local item="$1" dest="$2" + regular_or_absent "$dest" || return 1 + if [ -f "$dest" ]; then chmod 600 "$dest"; return; fi + # Historical notes are optional; write failures still fail the unit. + local rc=0 + fetch_note "$item" "$dest" || rc=$? + [ "$rc" = 0 ] || [ "$rc" = 2 ] +} +pin_home_recovery() { + local c clusters header=1 + if ! clusters="$(KUBECONFIG="$work/home" kubectl config get-clusters 2>/dev/null)"; then + note "FAIL cannot list home kubeconfig clusters; recovery cache not installed" return 1 fi while IFS= read -r c; do if [ "$header" = 1 ]; then header=0; continue; fi [ -n "$c" ] || continue - KUBECONFIG="$HOME/.kube/config" kubectl config set-cluster "$c" --tls-server-name="$KUBE_TLS_SERVER_NAME" >/dev/null || rc=1 + KUBECONFIG="$work/home" kubectl config set-cluster "$c" --tls-server-name="$KUBE_TLS_SERVER_NAME" >/dev/null 2>&1 || return 1 done <<< "$clusters" - return $rc +} +seed_home() { + regular_or_absent "$HOME/.kube/config-home-talos" || return 1 + if [ -f "$HOME/.kube/config-home-talos" ]; then chmod 600 "$HOME/.kube/config-home-talos"; return; fi + local rc=0 + fetch_note kubeconfig "$work/home" || rc=$? + [ "$rc" != 2 ] || return 0 + [ "$rc" = 0 ] || return 1 + pin_home_recovery && write_atomic "$HOME/.kube/config-home-talos" < "$work/home" +} +configure_k3s() { + local dest="$HOME/.kube/config" cache="$HOME/.kube/config-karnataka-k3s" inputs backup="$HOME/.kube/config-pre-karnataka-k3s" + regular_or_absent "$dest" && regular_or_absent "$cache" && regular_or_absent "$backup" || return 1 + # A failed refresh may use the last successful K3s cache, never partial stdout. + if [ "${KUBE_CACHED_ONLY:-0}" = 1 ] || ! fetch_note kubeconfig-karnataka-k3s "$work/k3s"; then + if [ ! -s "$cache" ]; then + note "FAIL Karnataka K3s note unavailable and no cached config exists" + return 1 + fi + cat "$cache" > "$work/k3s" || return 1 + fi + # Validate and select the reserved context before touching any deployed config. + if ! KUBECONFIG="$work/k3s" kubectl config use-context karnataka-k3s >/dev/null 2>&1; then + note "FAIL Karnataka K3s config lacks a usable karnataka-k3s context" + return 1 + fi + inputs="$work/k3s" + # Existing contexts take precedence over recovery notes; only K3s is managed. + [ ! -s "$dest" ] || inputs="$inputs:$dest" + [ ! -s "$HOME/.kube/config-home-talos" ] || inputs="$inputs:$HOME/.kube/config-home-talos" + if ! KUBECONFIG="$inputs" kubectl config view --merge --flatten --raw > "$work/merged" 2>/dev/null || [ ! -s "$work/merged" ]; then + note "FAIL cannot merge kubeconfigs; existing default left untouched" + return 1 + fi + if ! KUBECONFIG="$work/merged" kubectl config use-context karnataka-k3s >/dev/null 2>&1; then + note "FAIL cannot select Karnataka K3s; existing default left untouched" + return 1 + fi + if [ -f "$dest" ] && [ ! -e "$backup" ] && + [ -z "$(KUBECONFIG="$dest" kubectl config get-contexts karnataka-k3s -o name 2>/dev/null)" ]; then + write_atomic "$backup" < "$dest" || return 1 + fi + write_atomic "$cache" < "$work/k3s" && write_atomic "$dest" < "$work/merged" } -home_written=0 -step "home kubeconfig" fetch_note kubeconfig "$HOME/.kube/config" "kubeconfig (seed with scripts/bw-seed-kube.sh)" -step "home talosconfig" fetch_note talosconfig "$HOME/.talos/config" "talosconfig (seed with scripts/bw-seed-kube.sh)" -[ "$home_written" = 0 ] || step "pin tls-server-name" pin_tls -step "aws kubeconfig" fetch_note kubeconfig-aws-migration "$HOME/.kube/config-aws-migration" "AWS kubeconfig" -step "aws talosconfig" fetch_note talosconfig-aws-migration "$HOME/.talos/config-aws-migration" "AWS talosconfig" +if [ "${KUBE_CACHED_ONLY:-0}" != 1 ]; then + step "home recovery kubeconfig" seed_home + step "home talosconfig" seed_recovery talosconfig "$HOME/.talos/config" + step "aws kubeconfig" seed_recovery kubeconfig-aws-migration "$HOME/.kube/config-aws-migration" + step "aws talosconfig" seed_recovery talosconfig-aws-migration "$HOME/.talos/config-aws-migration" +fi +step "Karnataka K3s default" configure_k3s finish diff --git a/units/termux-packages.conf b/units/termux-packages.conf index f5e71b6e..ac8048a4 100644 --- a/units/termux-packages.conf +++ b/units/termux-packages.conf @@ -1,6 +1,6 @@ # The former role defaults, plus tools needed before native units can run. TERMUX_BOOTSTRAP=(python yq chezmoi just) -TERMUX_PACKAGES=(openssh git nodejs fish vim nano termux-api curl unzip file e2fsprogs) +TERMUX_PACKAGES=(openssh git nodejs fish vim nano termux-api curl unzip file e2fsprogs kubectl) TERMUX_OPTIONAL=(mise) -TERMUX_NPM_GLOBAL=('@earendil-works/pi-coding-agent') +TERMUX_NPM_GLOBAL=('@earendil-works/pi-coding-agent' '@bitwarden/cli') TERMUX_KEEPALIVE_PERIOD_MS=1800000 From 22da85b10a28adf8bb79e3c8013d6761e26f30de Mon Sep 17 00:00:00 2001 From: James Reilly <5840441+hanthor@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:37:35 +0000 Subject: [PATCH 60/62] Retire AWS resources and remove obsolete deployment paths Record owner-approved deletion of historical backups without a new archive, preserve live Karnataka/Spindle services, and remove stale infrastructure recreation and CI entrypoints. Co-authored-by: Claude Signed-off-by: James Reilly <5840441+hanthor@users.noreply.github.com> --- .github/CODEOWNERS | 1 - .github/workflows/agent-automerge.yml | 2 +- .github/workflows/ci.yml | 19 -- CLAUDE.md | 16 +- CONTRIBUTING.md | 3 +- Justfile | 40 --- aws/.terraform.lock.hcl | 39 --- aws/README.md | 25 +- aws/account.tf | 12 - aws/backups.tf | 150 ---------- aws/budgets.tf | 107 ------- aws/cluster.tf | 264 ------------------ aws/iam.tf | 54 ---- aws/imports.tf | 148 ---------- aws/kubevirt.tf | 160 ----------- aws/punjab.tf | 154 ---------- aws/terraform.tfvars.example | 11 - aws/tests/aws.tftest.hcl | 223 --------------- aws/tests/run.sh | 20 -- aws/variables.tf | 34 --- aws/versions.tf | 34 --- docs/matrix-cutover-runbook.md | 10 + docs/scripts.md | 35 +-- docs/src/architecture.md | 2 +- docs/src/automation.md | 11 +- docs/src/scripts.md | 2 +- docs/src/servers/aws-k8s/cluster.md | 29 +- docs/src/servers/aws/README.md | 218 ++++----------- docs/src/servers/home-k8s-images.md | 2 +- docs/src/servers/punjab/README.md | 46 +-- docs/src/servers/spindle-vps/README.md | 20 +- .../src/servers/talos-k8s/karnataka/README.md | 7 +- docs/src/talos-k8s.md | 4 +- renovate.json | 5 - scripts/kubevirt-node | 41 --- talos-k8s/backup/README.md | 20 +- talos-k8s/backup/postgres-backup.yaml | 6 +- talos-k8s/kubevirt-aws/README.md | 60 ++-- talos-k8s/kubevirt-aws/idle-stop.yaml | 7 +- talos-k8s/kubevirt-aws/talos-patch.yaml | 5 +- 40 files changed, 219 insertions(+), 1827 deletions(-) delete mode 100644 aws/.terraform.lock.hcl delete mode 100644 aws/account.tf delete mode 100644 aws/backups.tf delete mode 100644 aws/budgets.tf delete mode 100644 aws/cluster.tf delete mode 100644 aws/iam.tf delete mode 100644 aws/imports.tf delete mode 100644 aws/kubevirt.tf delete mode 100644 aws/punjab.tf delete mode 100644 aws/terraform.tfvars.example delete mode 100644 aws/tests/aws.tftest.hcl delete mode 100755 aws/tests/run.sh delete mode 100644 aws/variables.tf delete mode 100644 aws/versions.tf delete mode 100755 scripts/kubevirt-node diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 43c3c1bc..a87b70d2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -18,7 +18,6 @@ /roles/pi/files/package.json /tools/secret-broker/go.mod /tools/secret-broker/go.sum -/aws/.terraform.lock.hcl # ── Always owned, even inside carve-outs ──────────────────────────────── /.github/ @hanthor diff --git a/.github/workflows/agent-automerge.yml b/.github/workflows/agent-automerge.yml index a9fab45a..91235d74 100644 --- a/.github/workflows/agent-automerge.yml +++ b/.github/workflows/agent-automerge.yml @@ -38,7 +38,7 @@ jobs: owned=$({ printf '%s\n' "$files" \ | grep -vE '^(docs/|tests/|talos-k8s/|roles/pi/files/skills/)|\.md$' \ - | grep -vxE 'roles/pi/files/package\.json|tools/secret-broker/go\.(mod|sum)|aws/\.terraform\.lock\.hcl' + | grep -vxE 'roles/pi/files/package\.json|tools/secret-broker/go\.(mod|sum)' printf '%s\n' "$files" | grep '^\.github/' } | sort -u | sed '/^$/d') if [ -n "$owned" ]; then diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69599e2d..6809fb77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,25 +151,6 @@ jobs: - name: Validate Justfile run: just --justfile Justfile --list - tofu: - runs-on: ubuntu-latest - defaults: - run: - working-directory: aws - steps: - - uses: actions/checkout@v7 - - - uses: opentofu/setup-opentofu@v2 - - - name: fmt - run: tofu fmt -check -recursive - - - name: validate - run: tofu init -backend=false -input=false && tofu validate - - - name: test (mock provider, no AWS access) - run: tests/run.sh - broker: runs-on: ubuntu-latest defaults: diff --git a/CLAUDE.md b/CLAUDE.md index 7859bf28..5363361c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,7 @@ Three distinct concerns live here: ### Cluster recovery and current hosts - **Home:** Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` as its default and a single-node K3s `v1.36.5+k3s1` server bound to its tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are installed; `corral-vms/punjab` is now the active Punjab replacement, reached as `ubuntu@100.103.187.46` (tailnet name `punjab`). Reboot persistence, broker access and native no-secrets apply passed at cutover. Bihar's OS and services are unverified. Retain the former home cluster's `~/.kube/config`, `~/.talos/config`, Bitwarden notes and `talos-k8s/` manifests as recovery material; don't operate historical recovery commands on these machines. -- **AWS** (`eu-north-1`, built 2026-08-27) — **terminated 2026-10-25** after credits were exhausted. It previously ran Matrix, the Hives and the CFP dashboard. The S3 bucket `hanthor-fleet-backups-*` and the himachal-1 mirror (`~/backups/tunaos-aws-2026-10-25`) were retained; verify individual AWS resources rather than assuming they are gone. OpenTofu still describes the old cluster and can propose recreating resources. +- **AWS** — The former Talos cluster was terminated 2026-10-25. With explicit owner approval on 2026-10-29, the old Punjab EC2 instance, root volume, five snapshots, all three S3 buckets/versions, obsolete fleet IAM identities, DLM policies, budgets and migration network were deleted. No new archive of the old Hive/Spindle backups was retained. Prior off-AWS copies were not touched, but do not assume they replace every deleted cloud version. The executable AWS deployment module/recipes were removed; historical source is at Git commit `dcbbfbc`. Current inventory and billing caveats: [`docs/src/servers/aws/README.md`](docs/src/servers/aws/README.md). Configs retained as recovery material: `~/.kube/config-aws-migration`, `~/.talos/config-aws-migration`. Handbook: [`docs/src/servers/aws-k8s/cluster.md`](docs/src/servers/aws-k8s/cluster.md). - **Spindle VPS** — Matrix, standalone LiveKit, School, Reef and Hub run on an Oracle Always Free A1 VM in `eu-frankfurt-1`, 2 OCPU / 12 GB, joined to the tailnet and locked down. The kit and runbook are in [`docs/src/servers/spindle-vps/`](docs/src/servers/spindle-vps/README.md); Oracle API key/config are in Bitwarden. Cloudflare DNS failover is manual, not automatic. @@ -62,14 +62,14 @@ The active K3s note is **`kubeconfig-karnataka-k3s`**: it seeds `~/.kube/config- The former Bihar control plane and Karnataka worker ran Talos `v1.13.2`, Kubernetes `v1.36.1`, Lemonade and KubeVirt. This is historical architecture, not current workload status. The [cluster handbook](docs/src/servers/talos-k8s/cluster.md) preserves recovery instructions and warns against applying them to repurposed nodes. -## TunaOS AWS account (`aws/`) +## Retired TunaOS AWS account -- **punjab** is now a Karnataka-hosted VM. Its former EC2 `t3.large` in `us-east-1` was stopped on 2026-10-28 and its EIP released; its root volume and recovery snapshots remain. The former Talos cluster in `eu-north-1` is terminated. The separate `runs-on` CloudFormation stack in `us-east-2` reached `DELETE_COMPLETE`; its two S3 buckets and object versions were retained. -- **Do not blindly apply `aws/`:** its configuration still describes the terminated cluster and may propose recreating paid infrastructure. Inspect live resources and review plans before changes; credits are exhausted. -- State is in S3 (`hanthor-fleet-backups-*/tofu/aws/`) and contains Talos PKI. `aws/terraform.tfvars` is gitignored and lives in Bitwarden note `aws-tofu-tfvars` (`just aws-tfvars` / `just aws-seed-tfvars`). -- Do not restart EC2 Punjab or recreate retired runners without an explicit recovery decision. Preserve the retained root volume, snapshots and S3 backups; storage still incurs charges. The root-only cutover receipt on the replacement is `/srv/punjab-migration/aws-cutover-receipt.json`; the successful shutdown service journal records EC2 stopped and EIP released. -- The former cluster uploaded Postgres dumps to `s3://hanthor-fleet-backups-*/postgres/`; the `postgres-backup-writer` key is in Bitwarden, never in tofu state. -- `prevent_destroy` in configuration does not prevent creation of resources missing from the account. Never put Talos `user_data` in the config. +- **punjab** is the active Karnataka-hosted VM, not EC2. The old AWS instance is terminated, its root volume and five recovery snapshots are deleted, and the EIP was previously released. The `runs-on` stack and its retained S3 buckets are also gone. +- **Do not recreate AWS infrastructure.** The tracked `aws/` OpenTofu root module, deployment recipes and legacy AWS node helper were removed. Git commit `dcbbfbc` retains historical source; restoring it is an explicit paid-infrastructure decision, not drift repair. +- The fleet-backups bucket and every object version, including Postgres exports and Talos-PKI-bearing state, were deleted after the owner confirmed the applications were migrated and those backups unnecessary. The new archival transfer was canceled and its temporary copies removed. Do not claim a new complete off-AWS archive exists. +- Existing private local artifacts and Bitwarden historical notes were not swept or published. Old IAM access keys for the three deleted fleet users no longer work. Do not print credentials, state or raw kubeconfigs. +- The current deletion receipt is `~/backups/aws-final-retirement-20261029T015927Z/completion.json` on Punjab. The older root-only cutover receipt records the October 28 stop/EIP release, not current AWS resource state. +- AWS account defaults/service-linked roles remain, and canceled Kiro subscriptions expire at October month-end. Delayed/final charges can still appear; do not promise a zero-dollar invoice. See the [AWS handbook](docs/src/servers/aws/README.md). ## Automation (bots + Hive) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8bc49ba8..fc8b542f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,11 +22,10 @@ yamllint -d "{extends: default, rules: {line-length: disable, document-start: di find . -name "*.sh" -not -path "./.git/*" -type f -print0 | xargs -0 shellcheck -x -S error uvx --with pyyaml pytest -q tests just --justfile Justfile --list -(cd aws && tofu fmt -check -recursive) ``` CI also checks JSON validity, compiles tracked Python, runs native osquery smoke jobs, -`tofu validate` + `aws/tests/run.sh`, and Go checks in `tools/secret-broker/`. +and Go checks in `tools/secret-broker/`. See `.github/workflows/ci.yml` for the current jobs. These instructions do not assert that a particular checkout has passed them. diff --git a/Justfile b/Justfile index ead1b4ff..b8d17889 100644 --- a/Justfile +++ b/Justfile @@ -597,43 +597,3 @@ server-images tag="testing": # Build UEFI live media with bootc-installer-tui and this node's offline payload. server-installer node target_ref output tag="testing": "{{justfile_directory()}}/server-images/build-installer.sh" {{quote(node)}} {{quote(target_ref)}} {{quote(output)}} {{quote(tag)}} - -# ── AWS account IaC (aws/, OpenTofu) ───────────────────────────────────── -# Docs: docs/src/servers/aws/README.md. Needs AWS creds (`aws login`) and -# aws/terraform.tfvars, which holds personal IPs + alert address and so lives in -# the Bitwarden note `aws-tofu-tfvars`, not in git. - -# Fetch aws/terraform.tfvars from Bitwarden (no-op if it already exists) -aws-tfvars: - #!/usr/bin/env bash - set -euo pipefail - cd {{ dotfiles_dir }}/aws - [ -f terraform.tfvars ] && exit 0 - eval "$({{ dotfiles_dir }}/scripts/bw-resolve.sh local)" - (umask 077 && bw get notes aws-tofu-tfvars > terraform.tfvars) - echo "✓ aws/terraform.tfvars fetched from Bitwarden" - -# Push aws/terraform.tfvars to the Bitwarden note `aws-tofu-tfvars` (create or update) -aws-seed-tfvars: - #!/usr/bin/env bash - set -euo pipefail - cd {{ dotfiles_dir }}/aws - eval "$({{ dotfiles_dir }}/scripts/bw-resolve.sh local)" - bw sync >/dev/null - id=$(bw get item aws-tofu-tfvars 2>/dev/null | jq -r .id || true) - if [ -n "$id" ]; then - bw get item "$id" | jq --rawfile n terraform.tfvars '.notes=$n' | bw encode | bw edit item "$id" >/dev/null - else - bw get template item | jq --rawfile n terraform.tfvars \ - '.type=2 | .secureNote={type:0} | .name="aws-tofu-tfvars" | .notes=$n' \ - | bw encode | bw create item >/dev/null - fi - echo "✓ aws-tofu-tfvars saved to Bitwarden" - -# Show what OpenTofu would change in the AWS account (read-only) -aws-plan *args: aws-tfvars - cd {{ dotfiles_dir }}/aws && tofu init -input=false >/dev/null && tofu plan {{ args }} - -# Apply the AWS IaC (interactive confirmation; prevent_destroy guards the nodes) -aws-apply *args: aws-tfvars - cd {{ dotfiles_dir }}/aws && tofu init -input=false >/dev/null && tofu apply {{ args }} diff --git a/aws/.terraform.lock.hcl b/aws/.terraform.lock.hcl deleted file mode 100644 index ad3f4965..00000000 --- a/aws/.terraform.lock.hcl +++ /dev/null @@ -1,39 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/hashicorp/aws" { - version = "6.70.0" - constraints = "~> 6.0" - hashes = [ - "h1:0ppSZxnkr2mXy+OT2fE6FDY8B84U5l77bpAIDyIzxq0=", - "h1:8B3+vezC72MHmr3u5F5UWjCG4OrpjfKMLnPDyxjvrSo=", - "h1:GMjGoPo8GZ5iafPgiBK2TVZ6uiwaDrPdIb8hrNGceus=", - "h1:GOgheaNYt+Q3RnZEl0aIQjp/fYLEZHlWkXvmLk+12EU=", - "h1:JtwMUJT134sVjOaDFY9FJh1Mpdd5rgTEfeP3+PvezxA=", - "h1:K2u8xN6vPcBA24fwoyh4glebpIXprMzf9Q+omP0ditA=", - "h1:Oua4xmevWH4nSpBpLNo3SYHFSMorAZtYYI0cndQPHDA=", - "h1:RCdEQN76x2oY6wsOWxMuD1/aL++bSEWnVbHAB+vAZa4=", - "h1:WPSxowy+7ffZgsKexf9VNGXL/SfoAEiB8oSfQ/BWr+s=", - "h1:YwOjwe/o7JFnVz9H/xyAEEgHMVCYFtuX0/97iYAZmsc=", - "h1:e3kiK7jos0/EN/Dc6U+vbKBU53knKU6ILErBv0zdGec=", - "h1:f/fWBs1fWquPmk03fimTUDc4d871DeMA29TTa8dyjXA=", - "h1:iSpEapMJUOzrewKXUa0FJICuC2cKmy6H1PNSHH4L1Hc=", - "h1:kMw0FvF0bvj1L7fCSPlg6BApfzkBbTITDBtplJ1Vtek=", - "h1:lPMlFm5WfsjVFaDOzMYxnfwcDqfRuHeesiIXUca9oJQ=", - "zh:0e28c73ffe0c518db0f1dbcf6260f95e4a423267ec1888d08bc0f7cb7c02fec1", - "zh:2b4c26d4570d03cc2ebbe43a74034286ee66c85f0bcf5cf52a4c8652f3bccaa2", - "zh:3957610bbc3e6271f37260ba4bea151a4e3e0b7884a9f0fc448ee277a3ab730b", - "zh:4160ed56e45fabf4c44d44dc8c66d90fa53d657920e2385fcbe46e8c9d43964d", - "zh:43feb0d865e14ecfbe2066a3ec49becd5673f9102efdf4f2dcee08ae7effaf72", - "zh:495fb8da6f917d9e860ca945a0daa63bed01c3df4edf2c86bfbd32a90c3a5b05", - "zh:49b0eae54f54f39f38cc06661d13e4be94c49f03fc8d5fbb559f7a131d09a3ab", - "zh:57acc6485c95e28fc485b28ee198bf88e8bf5128fab8dbca7078fc04704fe50a", - "zh:7155dcc1ebf0b91741d28c3d42702b2cf3caafd61301ff8f8cc7a7d3b928d0a3", - "zh:7df06c80fd4fc811a931bff684d880eb52c740703c59850a4ca41050bfd922f7", - "zh:9fc7ba5238666273af242908b95ac4524b65a6f7e6f1f8f8348db0a0d9d20d49", - "zh:a79639c84900d0b8475218a5fa777f7f2fbb3a2def192ab01a041387dd76a074", - "zh:b5b81330ceb68f36e20ba0a0a73aa123e47689eaddf35cebba2c5796d8d83808", - "zh:f1efc1ef4d5e483833cb8aded5109aa93b6b9d95e098d8479037304f92e776a9", - "zh:f225a0a1a7dfb770ea5bb75c6a81a0efca175f8e461f16424f5acff841b3ddaa", - ] -} diff --git a/aws/README.md b/aws/README.md index 8bc9a736..aaaf297d 100644 --- a/aws/README.md +++ b/aws/README.md @@ -1,14 +1,17 @@ -# aws/ — AWS account IaC (OpenTofu) +# AWS deployment archive -Codifies everything in the AWS account except the `runs-on` CloudFormation -stack: the eu-north-1 Talos cluster's infrastructure, punjab, backups, IAM -and budgets. +The executable AWS OpenTofu root module, its tests, manual `just` recipes and +AWS KubeVirt node startup helper have been retired from this checkout. +Historical deployment material is preserved at +[`dcbbfbc`](https://github.com/hanthor/dotfiles/tree/dcbbfbc/aws), not as an +active deployment path. Restoring and applying it can recreate paid resources. -```bash -just aws-plan # should always say "No changes" -just aws-apply -``` +Final deletion on **2026-10-29** removed the AWS S3 backups (including the +OpenTofu state backend), Punjab's root volume and all owned snapshots. +Existing off-AWS migration data and vault notes were not deleted; no new +complete archive of the deleted S3 history is claimed. Exact scope and evidence +are documented in the [AWS handbook](../docs/src/servers/aws/README.md). -Full handbook: [docs/src/servers/aws/README.md](../docs/src/servers/aws/README.md). -State is in S3 and `terraform.tfvars` is in Bitwarden (`aws-tofu-tfvars`). -Neither is in git. +The ignore rules remain to protect local credentials, variables, state and +plans left from the former deployment. Those local files are not an active +root module and must not be committed. diff --git a/aws/account.tf b/aws/account.tf deleted file mode 100644 index 4833a6b4..00000000 --- a/aws/account.tf +++ /dev/null @@ -1,12 +0,0 @@ -# Account-level guardrails. - -# Every new EBS volume/snapshot copy is encrypted (AWS-managed key). Existing -# volumes are unaffected — re-encrypting them means snapshot → copy → swap. -resource "aws_ebs_encryption_by_default" "eu_north_1" { - enabled = true -} - -resource "aws_ebs_encryption_by_default" "us_east_1" { - provider = aws.us_east_1 - enabled = true -} diff --git a/aws/backups.tf b/aws/backups.tf deleted file mode 100644 index 76262254..00000000 --- a/aws/backups.tf +++ /dev/null @@ -1,150 +0,0 @@ -# Off-node backups: nightly EBS snapshots of both nodes' root volumes (DLM), -# plus an S3 bucket for Postgres dumps and this config's tofu state. - -resource "aws_iam_role" "dlm" { - name = "AWSDataLifecycleManagerDefaultRole" - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [{ - Effect = "Allow" - Action = "sts:AssumeRole" - Principal = { Service = "dlm.amazonaws.com" } - }] - }) -} - -resource "aws_iam_role_policy_attachment" "dlm" { - role = aws_iam_role.dlm.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSDataLifecycleManagerServiceRole" -} - -# Targets every volume tagged Backup=fleet-daily (both root volumes). -# 03:30 UTC daily ×7, 03:45 UTC Sunday ×4. -resource "aws_dlm_lifecycle_policy" "fleet_daily" { - description = "Fleet node root volumes daily and weekly snapshots" - execution_role_arn = aws_iam_role.dlm.arn - state = "DISABLED" # Retired cluster: freeze the retained recovery set. - - policy_details { - resource_types = ["VOLUME"] - target_tags = { - Backup = "fleet-daily" - } - - schedule { - name = "daily-7d" - copy_tags = true - tags_to_add = { dlm-schedule = "daily-7d" } - create_rule { - cron_expression = "cron(30 3 * * ? *)" - } - retain_rule { - count = 7 - } - } - - schedule { - name = "weekly-4w" - copy_tags = true - tags_to_add = { dlm-schedule = "weekly-4w" } - create_rule { - cron_expression = "cron(45 3 ? * SUN *)" - } - retain_rule { - count = 4 - } - } - } -} - -resource "aws_s3_bucket" "fleet_backups" { - bucket = "hanthor-fleet-backups-${data.aws_caller_identity.current.account_id}" - lifecycle { - prevent_destroy = true - } -} - -resource "aws_s3_bucket_versioning" "fleet_backups" { - bucket = aws_s3_bucket.fleet_backups.id - versioning_configuration { - status = "Enabled" - } -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "fleet_backups" { - bucket = aws_s3_bucket.fleet_backups.id - rule { - bucket_key_enabled = true - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } -} - -resource "aws_s3_bucket_public_access_block" "fleet_backups" { - bucket = aws_s3_bucket.fleet_backups.id - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -# postgres/ dumps: IA at 30d, Glacier IR at 90d, gone at 1y. Everything else -# (incl. tofu/ state) is kept; only stale multipart uploads are cleaned. -resource "aws_s3_bucket_lifecycle_configuration" "fleet_backups" { - bucket = aws_s3_bucket.fleet_backups.id - - rule { - id = "tier-and-expire" - status = "Enabled" - filter { - prefix = "postgres/" - } - transition { - days = 30 - storage_class = "STANDARD_IA" - } - transition { - days = 90 - storage_class = "GLACIER_IR" - } - expiration { - days = 365 - } - noncurrent_version_expiration { - noncurrent_days = 30 - } - } - - rule { - id = "abort-incomplete" - status = "Enabled" - filter {} - abort_incomplete_multipart_upload { - days_after_initiation = 7 - } - } -} - -# Writer for the in-cluster Postgres dump job (talos-k8s/backup/). Put + Get -# (for size verification) under postgres/ only — no delete, so with versioning -# a leaked key can't destroy existing dumps. Its access key is created out of -# band (never in state) and lives in the k8s Secret postgres/postgres-backup-s3 -# and the Bitwarden note `postgres-backup-s3`. -resource "aws_iam_user" "postgres_backup" { - name = "postgres-backup-writer" -} - -resource "aws_iam_user_policy" "postgres_backup" { - name = "fleet-backups-postgres-write" - user = aws_iam_user.postgres_backup.name - policy = jsonencode({ - Version = "2012-10-17" - Statement = [{ - Sid = "PostgresDumps" - Effect = "Allow" - Action = ["s3:PutObject", "s3:GetObject"] - Resource = "${aws_s3_bucket.fleet_backups.arn}/postgres/*" - }] - }) -} diff --git a/aws/budgets.tf b/aws/budgets.tf deleted file mode 100644 index b80d67e6..00000000 --- a/aws/budgets.tf +++ /dev/null @@ -1,107 +0,0 @@ -# Spend alarms. The account runs on AWS credits; `credits-exhausted-alarm` -# (credits *included*, so net spend) is the one that means real money. -# `runs-on-app-daily-budget` belongs to the runs-on CloudFormation stack and -# is not managed here. - -locals { - # include_credit/include_refund = false → gross usage, before credits. - gross_cost = { - include_credit = false - include_refund = false - } - - budgets = { - migration = { - name = "migration-monthly-spend" - limit = "1400.0" - start = "2026-08-01_00:00" - gross = true - service = null - actual = [500, 800, 1400] - forecasted = [1400] - percentages = false - } - bedrock = { - name = "bedrock-monthly-spend" - limit = "200.0" - start = "2026-09-01_00:00" - gross = true - service = "Amazon Bedrock" - actual = [150, 200] - forecasted = [200] - percentages = false - } - kiro = { - name = "kiro-monthly-spend" - limit = "500.0" - start = "2026-09-01_00:00" - gross = true - service = "Kiro" - actual = [375, 500] - forecasted = [500] - percentages = false - } - credits_exhausted = { - name = "credits-exhausted-alarm" - limit = "5.0" - start = "2026-08-01_00:00" - gross = false - service = null - actual = [1] - forecasted = [] - percentages = false - } - account_total = { - name = "tuna-os-account-total" - limit = "120.0" - start = "2026-07-01_00:00" - gross = false - service = null - actual = [50, 80, 90] - forecasted = [] - percentages = true - } - } -} - -resource "aws_budgets_budget" "this" { - for_each = local.budgets - - account_id = data.aws_caller_identity.current.account_id - name = each.value.name - budget_type = "COST" - limit_amount = each.value.limit - limit_unit = "USD" - time_unit = "MONTHLY" - time_period_start = each.value.start - - dynamic "cost_filter" { - for_each = each.value.service == null ? [] : [each.value.service] - content { - name = "Service" - values = [cost_filter.value] - } - } - - dynamic "cost_types" { - for_each = each.value.gross ? [local.gross_cost] : [] - content { - include_credit = cost_types.value.include_credit - include_refund = cost_types.value.include_refund - } - } - - dynamic "notification" { - for_each = concat( - [for t in each.value.actual : { type = "ACTUAL", threshold = t }], - [for t in each.value.forecasted : { type = "FORECASTED", threshold = t }], - ) - content { - comparison_operator = "GREATER_THAN" - notification_type = notification.value.type - threshold = notification.value.threshold - threshold_type = each.value.percentages ? "PERCENTAGE" : "ABSOLUTE_VALUE" - subscriber_email_addresses = [var.alert_email] - } - } -} diff --git a/aws/cluster.tf b/aws/cluster.tf deleted file mode 100644 index 69a963e9..00000000 --- a/aws/cluster.tf +++ /dev/null @@ -1,264 +0,0 @@ -# AWS Talos cluster (eu-north-1). Handbook: -# docs/src/servers/aws-k8s/cluster.md -# -# Nodes are configured by talosctl, not here: their user_data is the Talos -# machine config (cluster PKI) and is deliberately NOT in this repo — -# ignore_changes keeps tofu from ever diffing or rewriting it. - -# ── Network ─────────────────────────────────────────────────────────── -resource "aws_vpc" "migration" { - cidr_block = "10.20.0.0/16" - enable_dns_hostnames = true - enable_dns_support = true - tags = { - Name = "migration-vpc" - } -} - -resource "aws_internet_gateway" "migration" { - vpc_id = aws_vpc.migration.id - tags = { - Name = "migration-igw" - } -} - -# Single AZ, public only — no NAT gateway (unneeded, ~$32/mo). -resource "aws_subnet" "public_a" { - vpc_id = aws_vpc.migration.id - cidr_block = "10.20.1.0/24" - availability_zone = "eu-north-1a" - map_public_ip_on_launch = true - tags = { - Name = "migration-public-a" - } -} - -resource "aws_route_table" "public" { - vpc_id = aws_vpc.migration.id - - route { - cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.migration.id - } - - tags = { - Name = "migration-public-rt" - } -} - -resource "aws_route_table_association" "public_a" { - subnet_id = aws_subnet.public_a.id - route_table_id = aws_route_table.public.id -} - -# ── Security groups ─────────────────────────────────────────────────── -resource "aws_security_group" "intracluster" { - name = "migration-intracluster" - description = "Intra-cluster Talos/k8s traffic" - vpc_id = aws_vpc.migration.id - - ingress { - from_port = 0 - to_port = 0 - protocol = "-1" - self = true - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -# 80/443 → Traefik hostPorts; 8448 federation; 30001/30002 + 32700-32767 are -# the NodePorts the ESS chart's MatrixRTC SFU exposes. -resource "aws_security_group" "matrix_public" { - name = "migration-matrix-public" - description = "Public Matrix federation/web ports" - vpc_id = aws_vpc.migration.id - - dynamic "ingress" { - for_each = [ - { port = 80, to = 80, proto = "tcp", desc = "" }, - { port = 443, to = 443, proto = "tcp", desc = "" }, - { port = 8448, to = 8448, proto = "tcp", desc = "" }, - { port = 30001, to = 30001, proto = "tcp", desc = "matrix-rtc-sfu-tcp" }, - { port = 30002, to = 30002, proto = "udp", desc = "matrix-rtc-sfu-udp" }, - { port = 32700, to = 32767, proto = "udp", desc = "matrix-rtc-sfu-udp-range" }, - ] - content { - description = ingress.value.desc - from_port = ingress.value.port - to_port = ingress.value.to - protocol = ingress.value.proto - cidr_blocks = ["0.0.0.0/0"] - } - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -# Talos API + k8s API from admin IPs only (see var.cluster_admin_ingress). -# Only the control-plane carries this group. -resource "aws_security_group" "admin_bootstrap" { - name = "migration-admin-bootstrap" - description = "Temporary admin access for Talos/k8s API during bootstrap - restricted to admin IP" - vpc_id = aws_vpc.migration.id - - dynamic "ingress" { - for_each = flatten([ - for rule in var.cluster_admin_ingress : [ - for port in [6443, 50000] : merge(rule, { port = port }) - ] - ]) - content { - description = ingress.value.description - from_port = ingress.value.port - to_port = ingress.value.port - protocol = "tcp" - cidr_blocks = ingress.value.cidrs - } - } - - - dynamic "ingress" { - for_each = length(var.cluster_admin_pmtu_cidrs) > 0 ? [1] : [] - content { - description = "" - from_port = 3 - to_port = 4 - protocol = "icmp" - cidr_blocks = var.cluster_admin_pmtu_cidrs - } - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -# ── Nodes ───────────────────────────────────────────────────────────── -locals { - talos_ami = "ami-082ff045afa7ed0b3" # Talos Image Factory AMI the nodes were built from -} - -# Control plane also runs the Hives (their PVCs are node-bound local-path), so -# it must be sized for them: m6i.large OOM-killed the scheduler (2026-08-27), -# m6i.xlarge ran at load ~85 on 4 vCPU (2026-09-24). Now m6i.2xlarge. -resource "aws_instance" "controlplane" { - ami = local.talos_ami - instance_type = "m6i.2xlarge" - subnet_id = aws_subnet.public_a.id - private_ip = "10.20.1.10" - vpc_security_group_ids = [ - aws_security_group.intracluster.id, - aws_security_group.admin_bootstrap.id, - aws_security_group.matrix_public.id, - ] - - metadata_options { - http_tokens = "required" - http_put_response_hop_limit = 2 - } - - root_block_device { - volume_type = "gp3" - volume_size = 200 # grown live 2026-10-01 (DiskPressure); was 100 - tags = { - Backup = "fleet-daily" # picked up by aws_dlm_lifecycle_policy.fleet_daily - } - } - - tags = { - Name = "aws-migration-controlplane-hive" - } - - lifecycle { - prevent_destroy = true - ignore_changes = [ami, user_data, user_data_base64, user_data_replace_on_change] - } -} - -resource "aws_instance" "worker" { - ami = local.talos_ami - instance_type = "m6i.xlarge" - subnet_id = aws_subnet.public_a.id - private_ip = "10.20.1.11" - vpc_security_group_ids = [ - aws_security_group.intracluster.id, - aws_security_group.matrix_public.id, - ] - - metadata_options { - http_tokens = "required" - http_put_response_hop_limit = 2 - } - - root_block_device { - volume_type = "gp3" - volume_size = 200 # grown 2026-10-03: contributor build caches + Postgres left <20 GiB free - tags = { - Backup = "fleet-daily" - } - } - - tags = { - Name = "aws-migration-worker-matrix" - } - - lifecycle { - prevent_destroy = true - # pgdata is attached via aws_volume_attachment, not inline. - ignore_changes = [ami, user_data, user_data_base64, user_data_replace_on_change, ebs_block_device] - } -} - -resource "aws_eip" "controlplane" { - domain = "vpc" - instance = aws_instance.controlplane.id - tags = { - Name = "migration-controlplane" - } -} - -# DNS for matrix/auth/call/hive points here — changing it is an outage. -resource "aws_eip" "worker" { - domain = "vpc" - instance = aws_instance.worker.id - tags = { - Name = "migration-worker-matrix" - } - lifecycle { - prevent_destroy = true - } -} - -# Attached at the EC2 level but NOT yet mounted by Talos — Postgres still -# runs on local-path on the root volume. Unfinished work; see handbook. -resource "aws_ebs_volume" "pgdata" { - availability_zone = "eu-north-1a" - type = "gp3" - size = 50 - tags = { - Name = "migration-pgdata" - } - lifecycle { - prevent_destroy = true - } -} - -resource "aws_volume_attachment" "pgdata" { - device_name = "/dev/xvdb" - volume_id = aws_ebs_volume.pgdata.id - instance_id = aws_instance.worker.id -} diff --git a/aws/iam.tf b/aws/iam.tf deleted file mode 100644 index a330cc00..00000000 --- a/aws/iam.tf +++ /dev/null @@ -1,54 +0,0 @@ -# james-admin: the scoped IAM user used day-to-day instead of root. -# Its access key lives in Bitwarden, never here. - -resource "aws_iam_user" "james_admin" { - name = "james-admin" -} - -resource "aws_iam_policy" "james_admin_migration_scope" { - name = "james-admin-migration-scope" - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { Sid = "Compute", Effect = "Allow", Action = ["ec2:*"], Resource = "*" }, - { Sid = "StorageBackups", Effect = "Allow", Action = ["s3:*"], Resource = "*" }, - { Sid = "CostControl", Effect = "Allow", Action = ["budgets:*", "ce:Get*", "ce:Describe*"], Resource = "*" }, - { - Sid = "PassRoleToEC2" - Effect = "Allow" - Action = ["iam:PassRole"] - Resource = "*" - Condition = { StringEquals = { "iam:PassedToService" = "ec2.amazonaws.com" } } - }, - { Sid = "SelfIdentity", Effect = "Allow", Action = ["sts:GetCallerIdentity", "iam:GetUser", "iam:ListAccessKeys"], Resource = "*" }, - # Read-only IAM so `just aws-plan` works as james-admin; IAM *changes* - # still need root. - { Sid = "IamRead", Effect = "Allow", Action = ["iam:Get*", "iam:List*"], Resource = "*" }, - # Session Manager break-glass into punjab. - { Sid = "SsmSessions", Effect = "Allow", Action = ["ssm:StartSession", "ssm:TerminateSession", "ssm:ResumeSession", "ssm:DescribeSessions", "ssm:DescribeInstanceInformation", "ssm:GetConnectionStatus"], Resource = "*" }, - ] - }) -} - -resource "aws_iam_user_policy_attachment" "james_admin_migration_scope" { - user = aws_iam_user.james_admin.name - policy_arn = aws_iam_policy.james_admin_migration_scope.arn -} - -resource "aws_iam_user_policy" "james_admin_dlm" { - name = "james-admin-dlm" - user = aws_iam_user.james_admin.name - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { Sid = "ManageSnapshotLifecycle", Effect = "Allow", Action = "dlm:*", Resource = "*" }, - { - Sid = "PassOnlyTheDlmRole" - Effect = "Allow" - Action = "iam:PassRole" - Resource = aws_iam_role.dlm.arn - Condition = { StringEquals = { "iam:PassedToService" = "dlm.amazonaws.com" } } - }, - ] - }) -} diff --git a/aws/imports.tf b/aws/imports.tf deleted file mode 100644 index ef476efb..00000000 --- a/aws/imports.tf +++ /dev/null @@ -1,148 +0,0 @@ -# One-time adoption of resources that were created by hand (2026-07 → 09). -# Once imported these blocks are no-ops; they stay as a record of what was -# adopted and so a fresh state can be rebuilt with `tofu apply`. -# (Import IDs can't use data sources, hence the literal account ID below.) - -# punjab -import { - provider = aws.us_east_1 - to = aws_instance.punjab - id = "i-085690e02ca98c95a" -} -import { - provider = aws.us_east_1 - to = aws_security_group.punjab_ssh - id = "sg-0670c5a3a5e2115e9" -} -import { - provider = aws.us_east_1 - to = aws_key_pair.punjab - id = "punjab" -} -# cluster network -import { - to = aws_vpc.migration - id = "vpc-0c8fdba0b442b9af1" -} -import { - to = aws_subnet.public_a - id = "subnet-0213792ff023a3737" -} -import { - to = aws_internet_gateway.migration - id = "igw-0e2ffcdc182f80672" -} -import { - to = aws_route_table.public - id = "rtb-02a8a143552755b18" -} -import { - to = aws_route_table_association.public_a - id = "subnet-0213792ff023a3737/rtb-02a8a143552755b18" -} -import { - to = aws_security_group.intracluster - id = "sg-0421d172cf0a202d8" -} -import { - to = aws_security_group.matrix_public - id = "sg-09803bdbcc7b9273a" -} -import { - to = aws_security_group.admin_bootstrap - id = "sg-052a6292d2dcf728e" -} -import { - to = aws_instance.controlplane - id = "i-00a3e4532cdbfc87e" -} -import { - to = aws_instance.worker - id = "i-0bdfafdebc799d49d" -} -import { - to = aws_eip.controlplane - id = "eipalloc-0d1d4a449bb5eb751" -} -import { - to = aws_eip.worker - id = "eipalloc-0c8535b0d86005345" -} -import { - to = aws_ebs_volume.pgdata - id = "vol-01ff00a316340f0ad" -} -import { - to = aws_volume_attachment.pgdata - id = "/dev/xvdb:vol-01ff00a316340f0ad:i-0bdfafdebc799d49d" -} -import { - to = aws_dlm_lifecycle_policy.fleet_daily - id = "policy-0f074c7d13f94e355" -} -import { - to = aws_s3_bucket.fleet_backups - id = "hanthor-fleet-backups-181185361136" -} -import { - to = aws_s3_bucket_versioning.fleet_backups - id = "hanthor-fleet-backups-181185361136" -} -import { - to = aws_s3_bucket_lifecycle_configuration.fleet_backups - id = "hanthor-fleet-backups-181185361136" -} -import { - to = aws_s3_bucket_public_access_block.fleet_backups - id = "hanthor-fleet-backups-181185361136" -} -import { - to = aws_s3_bucket_server_side_encryption_configuration.fleet_backups - id = "hanthor-fleet-backups-181185361136" -} -# IAM -import { - to = aws_iam_user.james_admin - id = "james-admin" -} -import { - to = aws_iam_policy.james_admin_migration_scope - id = "arn:aws:iam::181185361136:policy/james-admin-migration-scope" -} -import { - to = aws_iam_user_policy_attachment.james_admin_migration_scope - id = "james-admin/arn:aws:iam::181185361136:policy/james-admin-migration-scope" -} -import { - to = aws_iam_user_policy.james_admin_dlm - id = "james-admin:james-admin-dlm" -} -import { - to = aws_iam_role.dlm - id = "AWSDataLifecycleManagerDefaultRole" -} -import { - to = aws_iam_role_policy_attachment.dlm - id = "AWSDataLifecycleManagerDefaultRole/arn:aws:iam::aws:policy/service-role/AWSDataLifecycleManagerServiceRole" -} -# budgets -import { - to = aws_budgets_budget.this["migration"] - id = "181185361136:migration-monthly-spend" -} -import { - to = aws_budgets_budget.this["bedrock"] - id = "181185361136:bedrock-monthly-spend" -} -import { - to = aws_budgets_budget.this["kiro"] - id = "181185361136:kiro-monthly-spend" -} -import { - to = aws_budgets_budget.this["credits_exhausted"] - id = "181185361136:credits-exhausted-alarm" -} -import { - to = aws_budgets_budget.this["account_total"] - id = "181185361136:tuna-os-account-total" -} diff --git a/aws/kubevirt.tf b/aws/kubevirt.tf deleted file mode 100644 index 7c4950a1..00000000 --- a/aws/kubevirt.tf +++ /dev/null @@ -1,160 +0,0 @@ -# ── KubeVirt node ───────────────────────────────────────────────────── -# A third Talos worker that only runs KubeVirt VMs (corral's AWS backend). -# -# Why m7i: KubeVirt needs /dev/kvm. EC2 exposes it on virtual instances only -# with nested virtualization, which the m6i nodes don't support (checked -# 2026-09-24: `aws ec2 describe-instance-types` lists `nested-virtualization` -# for m7i.2xlarge in eu-north-1, nothing for m6i.xlarge). -# -# Cost: on-demand $0.4284/h in eu-north-1, so ~$313/mo if it never stopped. -# It is stopped when idle (talos-k8s/kubevirt-aws/idle-stop.yaml) and started -# on demand (scripts/kubevirt-node up). The migration budget's action below -# stops it automatically if gross spend nears the $1400/mo credits. -# -# Joining: no user_data, so Talos boots into maintenance mode and no cluster -# secrets ever reach OpenTofu state; the machine config is applied from -# punjab with `talosctl apply-config --insecure` (admin_bootstrap allows -# punjab on :50000). See docs/src/servers/aws-k8s/kubevirt.md. -resource "aws_instance" "kubevirt" { - ami = local.talos_ami - instance_type = "m7i.2xlarge" - subnet_id = aws_subnet.public_a.id - private_ip = "10.20.1.12" - vpc_security_group_ids = [ - aws_security_group.intracluster.id, - aws_security_group.admin_bootstrap.id, - ] - - cpu_options { - nested_virtualization = "enabled" - } - - metadata_options { - http_tokens = "required" - http_put_response_hop_limit = 2 - } - - root_block_device { - # Talos system + VM disks (local-path). No Backup tag: VM disks are - # scratch; anything worth keeping lives in a VM snapshot or elsewhere. - volume_type = "gp3" - volume_size = 150 - } - - tags = { - Name = "aws-migration-kubevirt" - Role = "kubevirt" - # Opt-in for corral's aws-power plugin (the corral-web power button); - # the value is the name corral shows. - "corral:host-power" = "KubeVirt (AWS)" - "corral:node" = "ip-10-20-1-12" - } - - lifecycle { - ignore_changes = [ami, user_data, user_data_base64] - } -} - -# ── Spend kill switch ───────────────────────────────────────────────── -# If gross spend this month passes $1100 (79% of the $1400/mo credits), -# AWS Budgets stops the KubeVirt node by itself. No human in the loop: -# the one resource that can double the bill turns itself off first. -# Billing data lags ~a day, so the worst-case overshoot is about a day of -# node time (~$10). -resource "aws_iam_role" "budget_action" { - name = "budget-action-stop-kubevirt" - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [{ - Effect = "Allow" - Principal = { Service = "budgets.amazonaws.com" } - Action = "sts:AssumeRole" - Condition = { StringEquals = { "aws:SourceAccount" = data.aws_caller_identity.current.account_id } } - }] - }) -} - -resource "aws_iam_role_policy" "budget_action" { - name = "stop-kubevirt-node" - role = aws_iam_role.budget_action.id - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = ["ec2:StopInstances"] - Resource = aws_instance.kubevirt.arn - }, - { - Effect = "Allow" - Action = ["ec2:DescribeInstances", "ec2:DescribeInstanceStatus", "ssm:StartAutomationExecution", "ssm:GetAutomationExecution"] - Resource = "*" - }, - ] - }) -} - -resource "aws_budgets_budget_action" "stop_kubevirt" { - budget_name = aws_budgets_budget.this["migration"].name - action_type = "RUN_SSM_DOCUMENTS" - approval_model = "AUTOMATIC" - notification_type = "ACTUAL" - execution_role_arn = aws_iam_role.budget_action.arn - - action_threshold { - action_threshold_type = "ABSOLUTE_VALUE" - action_threshold_value = 1100 - } - - definition { - ssm_action_definition { - action_sub_type = "STOP_EC2_INSTANCES" - region = "eu-north-1" - instance_ids = [aws_instance.kubevirt.id] - } - } - - subscriber { - address = var.alert_email - subscription_type = "EMAIL" - } -} - -# ── Idle stop / on-demand start ─────────────────────────────────────── -# Credentials for the in-cluster idle-stop CronJob and for starting the node -# from the tailnet. Scoped to this one instance: start, stop, describe. -# The access key is created by hand (never in state or git) and stored in -# the cluster as secret kubevirt-aws/node-power (idle-stop CronJob) and -# tailvm/node-power (corral-web's aws-power plugin). -resource "aws_iam_user" "kubevirt_power" { - name = "kubevirt-node-power" -} - -resource "aws_iam_user_policy" "kubevirt_power" { - name = "start-stop-kubevirt-node" - user = aws_iam_user.kubevirt_power.name - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = ["ec2:StartInstances", "ec2:StopInstances"] - Resource = aws_instance.kubevirt.arn - }, - { - Effect = "Allow" - Action = ["ec2:DescribeInstances"] - Resource = "*" - }, - ] - }) -} - -output "kubevirt_instance_id" { - value = aws_instance.kubevirt.id -} - -output "kubevirt_public_ip" { - description = "Only used to apply the Talos config from punjab; day-to-day access is over the tailnet." - value = aws_instance.kubevirt.public_ip -} diff --git a/aws/punjab.tf b/aws/punjab.tf deleted file mode 100644 index 56c4f8cf..00000000 --- a/aws/punjab.tf +++ /dev/null @@ -1,154 +0,0 @@ -# Retained EC2 Punjab recovery host. The active fleet host is now the -# Karnataka VM at 100.103.187.46. The desired recovery state is stopped, -# retaining its original volume and SSM role but no reserved public IP. - -resource "aws_key_pair" "punjab" { - provider = aws.us_east_1 - key_name = "punjab" - public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDiAhiG3nENI7eON08aLsMKH1xeBYNCtAjS3yQF4fvXszgDD13fmJ10lDjp6YMgqk+ytBUmGIuioEdxk2u8ulxM+PnkpQffbw6pJKmM4XIneJp0Htn5ioCLyZi15wMHBd0YrDo1BQ9l2Zm6sK3kmLfKObuJvHcoh9n5B6jd/P/fp+m8hX/LppDeNL2Wt7N5jcaucuTkYx0O0Lbr/ev7uq1e7UvgJ2eHLIOPtNGFVUkwvhH0OsmJsqxjpW7LlRikH+UynCD02SqB5xYpZl4Wm6NgPoBxonECkOLKUfDAbsfFg9kolctT++v8um2FSE5nlO4DSg6Yz8r6xed4urG9xRF1 punjab" - - lifecycle { - # EC2 doesn't return public_key on import; without this tofu would - # replace the key pair on every plan. - ignore_changes = [public_key] - } -} - -# In the us-east-1 default VPC. -resource "aws_security_group" "punjab_ssh" { - provider = aws.us_east_1 - name = "punjab-ssh" - description = "SSH box punjab: SSH from admin IP only" - - # Attribute syntax (not dynamic blocks) so an empty list really means "no - # inbound" — omitted ingress blocks would leave existing rules unmanaged. - ingress = [for rule in var.punjab_ssh_ingress : { - description = rule.description - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = rule.cidrs - ipv6_cidr_blocks = [] - prefix_list_ids = [] - security_groups = [] - self = false - }] - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -# Only what Session Manager needs. Anything running on punjab can read these -# creds from IMDS, so keep this role minimal — admin work uses real users. -resource "aws_iam_role" "punjab" { - name = "punjab-ssm" - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [{ - Effect = "Allow" - Action = "sts:AssumeRole" - Principal = { Service = "ec2.amazonaws.com" } - }] - }) -} - -resource "aws_iam_role_policy_attachment" "punjab_ssm" { - role = aws_iam_role.punjab.name - policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" -} - -resource "aws_iam_instance_profile" "punjab" { - name = "punjab-ssm" - role = aws_iam_role.punjab.name -} - -resource "aws_instance" "punjab" { - provider = aws.us_east_1 - ami = "ami-0045d7fc2ad003464" # ubuntu-noble-24.04-amd64-server-20260923 - instance_type = "t3.large" - subnet_id = "subnet-0598105df43192616" # default VPC, us-east-1d - key_name = aws_key_pair.punjab.key_name - vpc_security_group_ids = [aws_security_group.punjab_ssh.id] - iam_instance_profile = aws_iam_instance_profile.punjab.name - - disable_api_termination = true - - credit_specification { - cpu_credits = "unlimited" - } - - metadata_options { - http_tokens = "required" - # 1 = only the host itself can reach IMDS (no containers hopping to the - # instance role's creds). - http_put_response_hop_limit = 1 - } - - root_block_device { - volume_type = "gp3" - volume_size = 250 - tags = { - Backup = "fleet-daily" # aws_dlm_lifecycle_policy.punjab - } - } - - tags = { - Name = "punjab" - } - - lifecycle { - prevent_destroy = true - # The box is configured by Ansible, not by re-imaging. - ignore_changes = [ami, user_data, user_data_base64] - } -} - -resource "aws_ec2_instance_state" "punjab_retired" { - provider = aws.us_east_1 - instance_id = aws_instance.punjab.id - state = "stopped" -} - -# DLM is regional, so punjab (us-east-1) needs its own policy. Same schedule -# as the cluster's: daily ×7, weekly ×4. -resource "aws_dlm_lifecycle_policy" "punjab" { - provider = aws.us_east_1 - description = "punjab root volume daily and weekly snapshots" - execution_role_arn = aws_iam_role.dlm.arn - state = "DISABLED" # Migration: retain snapshots without scheduled churn. - - policy_details { - resource_types = ["VOLUME"] - target_tags = { - Backup = "fleet-daily" - } - - schedule { - name = "daily-7d" - copy_tags = true - tags_to_add = { dlm-schedule = "daily-7d" } - create_rule { - cron_expression = "cron(30 3 * * ? *)" - } - retain_rule { - count = 7 - } - } - - schedule { - name = "weekly-4w" - copy_tags = true - tags_to_add = { dlm-schedule = "weekly-4w" } - create_rule { - cron_expression = "cron(45 3 ? * SUN *)" - } - retain_rule { - count = 4 - } - } - } -} diff --git a/aws/terraform.tfvars.example b/aws/terraform.tfvars.example deleted file mode 100644 index c66edca7..00000000 --- a/aws/terraform.tfvars.example +++ /dev/null @@ -1,11 +0,0 @@ -alert_email = "you@example.com" - -cluster_admin_ingress = [ - { description = "workstation 2026-09-23", cidrs = ["203.0.113.10/32"] }, -] - -cluster_admin_pmtu_cidrs = [] - -punjab_ssh_ingress = [ - { description = "admin 2026-09-23", cidrs = ["203.0.113.10/32"] }, -] diff --git a/aws/tests/aws.tftest.hcl b/aws/tests/aws.tftest.hcl deleted file mode 100644 index e774af4b..00000000 --- a/aws/tests/aws.tftest.hcl +++ /dev/null @@ -1,223 +0,0 @@ -# Plan-only against a mocked AWS provider — never touches the real account. -# Run via tests/run.sh (see there for why not plain `tofu test`). Every -# variable is set here so the gitignored terraform.tfvars is irrelevant. - -# Mock values the provider validates at plan time (random strings fail). -mock_provider "aws" { - mock_data "aws_caller_identity" { - defaults = { account_id = "123456789012" } - } - mock_resource "aws_iam_role" { - defaults = { arn = "arn:aws:iam::123456789012:role/mock" } - } - mock_resource "aws_iam_policy" { - defaults = { arn = "arn:aws:iam::123456789012:policy/mock" } - } - mock_resource "aws_eip" { - defaults = { public_ip = "192.0.2.10" } - } -} - -mock_provider "aws" { - alias = "us_east_1" - - mock_data "aws_caller_identity" { - defaults = { account_id = "123456789012" } - } - mock_resource "aws_iam_role" { - defaults = { arn = "arn:aws:iam::123456789012:role/mock" } - } - mock_resource "aws_iam_policy" { - defaults = { arn = "arn:aws:iam::123456789012:policy/mock" } - } -} - -variables { - alert_email = "alerts@example.com" - cluster_admin_ingress = [ - { description = "home", cidrs = ["198.51.100.1/32"] }, - { description = "travel", cidrs = ["203.0.113.7/32", "203.0.113.8/32"] }, - { description = "office", cidrs = ["192.0.2.0/24"] }, - ] - cluster_admin_pmtu_cidrs = [] - punjab_ssh_ingress = [ - { description = "admin", cidrs = ["198.51.100.1/32"] }, - { description = "backup", cidrs = ["203.0.113.9/32"] }, - ] -} - -run "admin_ingress_without_pmtu" { - command = plan - - assert { - condition = length([for r in aws_security_group.admin_bootstrap.ingress : r if r.protocol == "icmp"]) == 0 - error_message = "no icmp rule when cluster_admin_pmtu_cidrs is empty" - } - - assert { - condition = alltrue([ - for port in [6443, 50000] : length([ - for r in aws_security_group.admin_bootstrap.ingress : r - if r.from_port == port && r.to_port == port && r.protocol == "tcp" - ]) == 3 - ]) - error_message = "each port must have one rule per cluster_admin_ingress entry" - } - - assert { - condition = alltrue([ - for e in var.cluster_admin_ingress : length([ - for r in aws_security_group.admin_bootstrap.ingress : r - if r.description == e.description && toset(r.cidr_blocks) == toset(e.cidrs) - ]) == 2 - ]) - error_message = "rule descriptions/cidrs must mirror cluster_admin_ingress" - } - - assert { - condition = alltrue([ - for r in aws_security_group.admin_bootstrap.ingress : !contains(r.cidr_blocks, "0.0.0.0/0") - ]) - error_message = "admin_bootstrap must never be open to the world" - } -} - -run "admin_ingress_with_pmtu" { - command = plan - - variables { - cluster_admin_pmtu_cidrs = ["198.51.100.0/24"] - } - - assert { - condition = length([ - for r in aws_security_group.admin_bootstrap.ingress : r - if r.protocol == "icmp" && r.from_port == 3 && r.to_port == 4 - && toset(r.cidr_blocks) == toset(["198.51.100.0/24"]) - ]) == 1 - error_message = "icmp type 3 code 4 rule must carry cluster_admin_pmtu_cidrs" - } -} - -run "punjab_ssh_mirrors_variable" { - command = plan - - assert { - condition = length(aws_security_group.punjab_ssh.ingress) == length(var.punjab_ssh_ingress) - error_message = "one ssh rule per punjab_ssh_ingress entry" - } - - assert { - condition = alltrue([ - for e in var.punjab_ssh_ingress : length([ - for r in aws_security_group.punjab_ssh.ingress : r - if r.description == e.description && toset(r.cidr_blocks) == toset(e.cidrs) - && r.from_port == 22 && r.to_port == 22 && r.protocol == "tcp" - ]) == 1 - ]) - error_message = "punjab ssh rules must mirror punjab_ssh_ingress on tcp/22" - } -} - -run "punjab_ssh_closed_when_empty" { - command = plan - - variables { - punjab_ssh_ingress = [] - } - - assert { - condition = length(aws_security_group.punjab_ssh.ingress) == 0 - error_message = "no punjab_ssh_ingress entries must mean no public ssh" - } -} - -run "budgets" { - command = plan - - assert { - condition = toset(keys(aws_budgets_budget.this)) == toset([ - "account_total", "bedrock", "credits_exhausted", "kiro", "migration", - ]) - error_message = "expected exactly the 5 managed budgets" - } - - assert { - condition = { - for k, b in aws_budgets_budget.this : k => length(b.notification) - } == { - migration = 4 - bedrock = 3 - kiro = 3 - credits_exhausted = 1 - account_total = 3 - } - error_message = "notification counts (actual + forecasted) drifted" - } - - assert { - condition = alltrue(flatten([ - for b in aws_budgets_budget.this : [ - for n in b.notification : n.subscriber_email_addresses == toset(["alerts@example.com"]) - ] - ])) - error_message = "every notification must go to alert_email" - } - - assert { - condition = alltrue([ - for n in aws_budgets_budget.this["account_total"].notification : n.threshold_type == "PERCENTAGE" - ]) - error_message = "account_total thresholds are percentages" - } - - assert { - condition = length(aws_budgets_budget.this["bedrock"].cost_filter) == 1 && length(aws_budgets_budget.this["migration"].cost_filter) == 0 - error_message = "service budgets filter by Service; account-wide ones don't" - } - - assert { - condition = length(aws_budgets_budget.this["kiro"].cost_types) == 1 && length(aws_budgets_budget.this["credits_exhausted"].cost_types) == 0 - error_message = "gross budgets exclude credits; net budgets use the default cost_types" - } -} - -# punjab is internet-facing; these are the guardrails from securing it. -run "punjab_hardening" { - command = plan - - assert { - condition = aws_instance.punjab.metadata_options[0].http_tokens == "required" && aws_instance.punjab.metadata_options[0].http_put_response_hop_limit == 1 - error_message = "punjab must require IMDSv2 with hop limit 1" - } - - assert { - condition = aws_instance.punjab.disable_api_termination == true - error_message = "punjab must have termination protection" - } - - assert { - condition = aws_instance.punjab.iam_instance_profile == aws_iam_instance_profile.punjab.name - error_message = "punjab needs the SSM instance profile (break-glass path)" - } - - assert { - condition = aws_instance.punjab.root_block_device[0].tags["Backup"] == "fleet-daily" - error_message = "punjab's root volume must be in the snapshot policy" - } - - assert { - condition = aws_ebs_encryption_by_default.us_east_1.enabled && aws_ebs_encryption_by_default.eu_north_1.enabled - error_message = "EBS encryption by default must stay on in both regions" - } -} - -# The backup writer must never gain delete (it's what makes a leaked key harmless). -run "postgres_backup_writer_is_write_only" { - command = plan - - assert { - condition = !strcontains(aws_iam_user_policy.postgres_backup.policy, "Delete") - error_message = "postgres-backup-writer must not be able to delete objects" - } -} diff --git a/aws/tests/run.sh b/aws/tests/run.sh deleted file mode 100755 index f71aab2a..00000000 --- a/aws/tests/run.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Run the mock-provider tests (aws/tests/*.tftest.hcl) without touching AWS. -# -# `tofu test` can't run in aws/ directly: mock providers panic on import -# blocks (imports.tf), and a local .terraform/ initialised against the S3 -# backend makes init try to reach it. So test a scratch copy of the module -# minus imports.tf, with no backend and no terraform.tfvars. -set -euo pipefail - -src="$(cd "$(dirname "$0")/.." && pwd)" -work="$(mktemp -d)" -trap 'rm -rf "$work"' EXIT - -cp "$src"/*.tf "$src/.terraform.lock.hcl" "$work"/ -rm "$work/imports.tf" -cp -r "$src/tests" "$work"/ - -cd "$work" -tofu init -backend=false -input=false /dev/null -tofu test "$@" **Nonoperational historical record (2026-08-27).** Present-tense claims and +> commands below describe that cutover, not current AWS resources or tasks. +> The AWS Talos cluster was terminated on 2026-10-25; final deletion on +> **2026-10-29** removed AWS volumes, owned snapshots and all three S3 buckets, +> including backup history and OpenTofu state. Existing off-AWS migration +> data and vault notes were not deleted. See the +> [final AWS deletion evidence](src/servers/aws/README.md) and the +> [deployment module archived at `dcbbfbc`](https://github.com/hanthor/dotfiles/tree/dcbbfbc/aws). +> Do not run these historical commands or restart the retired Hetzner hosts. + Status 2026-08-27: **CUTOVER COMPLETE.** Downtime 08:47:27Z → 09:05:55Z (~18.5 min). Federation identity preserved (`ed25519:a_qMqD`), Synapse `v1.156.0`, 1371 users, 4.6GB media. diff --git a/docs/scripts.md b/docs/scripts.md index a9c272b0..cea04e9f 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -352,31 +352,14 @@ bw list items --session "$BW_SESSION" ### `kubevirt-node` -Power the AWS KubeVirt node up or down from the tailnet. +The AWS node startup/shutdown helper has been removed. Its implementation and +historical usage are preserved at +[`dcbbfbc`](https://github.com/hanthor/dotfiles/blob/dcbbfbc/scripts/kubevirt-node); +these are archived deployment instructions, not commands available in this +checkout. Restoring them can restart paid EC2 infrastructure. -**Purpose**: Manage the AWS EC2 instance running the KubeVirt migration cluster, which is automatically scaled down after 30 idle minutes or when compute credits are low. - -**Usage**: -```bash -./scripts/kubevirt-node up # start and wait for Ready -./scripts/kubevirt-node down # stop gracefully -./scripts/kubevirt-node status # current state -``` - -**Prerequisites**: -- AWS CLI configured with credentials -- `kubectl` with access to the AWS migration cluster -- `KUBECONFIG` pointing to `~/.kube/config-aws-migration` (or set explicitly) - -**What it does**: -- **up**: Starts the EC2 instance and waits for the Kubernetes node to report Ready -- **down**: Stops the EC2 instance gracefully (VMs on it are shut down first by the cluster) -- **status**: Prints the current EC2 state (running, stopped, pending, etc.) - -**Notes**: -- The cluster runs an idle-stop CronJob that automatically stops after 30 idle minutes -- A migration budget controller stops it if credits approach limits -- Daily workflow: just run `up` in the morning; `down` is automatic +The independent local KubeVirt host is documented in the +[Karnataka K3s handbook](src/servers/talos-k8s/karnataka/README.md). --- @@ -606,7 +589,3 @@ done **"Bitwarden vault is unauthenticated"** - Run `bw login` first, or set `BW_SESSION` if you have an active session -**"no aws-migration-kubevirt instance"** -- Check AWS region: `kubevirt-node` defaults to `eu-north-1` -- Set `AWS_DEFAULT_REGION` if your instance is elsewhere - diff --git a/docs/src/architecture.md b/docs/src/architecture.md index 8f834cc1..bf600951 100644 --- a/docs/src/architecture.md +++ b/docs/src/architecture.md @@ -7,7 +7,7 @@ - `units/*.sh` and their helpers/assets configure the host. `home/` is the chezmoi source for shell, Git, Neovim and Atuin configuration. - `inventory.yml`, `group_vars/` and `host_vars/` hold fleet membership and settings. Some connection keys retain their historical `ansible_` names and are read by native tooling. - `scripts/` holds operational and Bitwarden helpers. `roles/pi/files/skills/` remains a source of omp skills; not every directory named `roles` is an active Ansible role. -- `aws/` contains OpenTofu for the TunaOS AWS account; `talos-k8s/` contains home-cluster manifests; `docs/` is the handbook. +- AWS deployment IaC is archived at [`dcbbfbc`](https://github.com/hanthor/dotfiles/tree/dcbbfbc/aws), not executable in the current checkout; `talos-k8s/` contains historical cluster manifests; `docs/` is the handbook. ## Applying configuration diff --git a/docs/src/automation.md b/docs/src/automation.md index 2544a801..39172946 100644 --- a/docs/src/automation.md +++ b/docs/src/automation.md @@ -8,7 +8,7 @@ those machines unattended. Executable fleet changes require owner review. | Actor | What it does | Merges by itself? | |---|---|---| -| **Renovate** | minor/patch/digest bumps (pi npm deps, Go, GitHub Actions, OpenTofu providers), weekly lockfile maintenance | yes, once CI is green | +| **Renovate** | minor/patch/digest bumps (pi npm deps, Go, GitHub Actions), weekly lockfile maintenance | yes, once CI is green | | **Renovate** (cluster) | image/chart bumps in `talos-k8s/`, grouped weekly, label `deploy-needed` | **no** — manifests are applied by hand, so a merge would only make git lie about the cluster | | **Hive** ([hive.reilly.asia](https://hive.reilly.asia), `hanthor-hive-agent`, ACMM L5) | its agents open PRs labelled `hold`; the Hive itself only merges PRs labelled `lgtm` | only if the PR stays inside the carve-outs below | | **You** | review onboarding and other owned changes | admin bypass is available, but onboarding uses a PR | @@ -17,9 +17,12 @@ those machines unattended. Executable fleet changes require owner review. Three pieces, and none of them trusts the PR itself: -1. **Branch protection on `master`**: required checks `lint`, `python`, - `tests`, `tofu`, `justfile`, `broker`, `deps-guard`, plus **code-owner - review**. Admins can bypass, so your own pushes still work. +1. **Branch protection on `master`**: verified required checks are `lint`, + `python`, `tests`, `justfile` and `broker`, plus **code-owner review**. + Admins can bypass, so your own pushes still work. On 2026-10-29, only the + obsolete AWS `tofu` requirement was removed; the other check/application + bindings were preserved. Its configuration is archived at + [`dcbbfbc`](https://github.com/hanthor/dotfiles/tree/dcbbfbc/aws). 2. **[`.github/CODEOWNERS`](https://github.com/hanthor/dotfiles/blob/master/.github/CODEOWNERS)** is deny-by-default (`* @hanthor`). The paths carved out are the ones that never run on a fleet machine: `docs/`, `*.md`, `tests/`, `talos-k8s/`, pi diff --git a/docs/src/scripts.md b/docs/src/scripts.md index 752ae3d6..5c7cb7c8 100644 --- a/docs/src/scripts.md +++ b/docs/src/scripts.md @@ -29,6 +29,6 @@ The shared unit library skips retired `matrix` and `telengana` before unit work. ## Other helpers - `scripts/sync-brews.sh` compares Homebrew's dumped formulae with `core_brews` and `desktop_brews` in `group_vars/all.yml` and adds missing formulae to `desktop_brews`; review its changes before committing. -- `scripts/kubevirt-node up|down|status` historically managed an AWS KubeVirt instance and VMs, using `~/.kube/config-aws-migration` by default. The AWS Talos cluster was terminated; do not treat these commands as a home-cluster runbook or use them without checking the current AWS state. The home Talos cluster is separately offline in storage, not decommissioned. +- The retired AWS `scripts/kubevirt-node` helper is preserved only at [`dcbbfbc`](https://github.com/hanthor/dotfiles/blob/dcbbfbc/scripts/kubevirt-node); its startup/shutdown commands are no longer available. Use the [current Karnataka K3s handbook](servers/talos-k8s/karnataka/README.md) for the independent local KubeVirt host. - `scripts/ess-render-values.py` reads sensitive ESS Helm values on stdin and writes transformed values to stdout for the former AWS cluster. It does not retrieve the vault item itself or configure the standalone OCI deployment. Do not write secret-bearing output to a tracked file. - `scripts/osquery-fleet.py` collects private fleet monitoring reports; see [fleet facts and osquery](roles/fleet_facts.md). diff --git a/docs/src/servers/aws-k8s/cluster.md b/docs/src/servers/aws-k8s/cluster.md index c6e7b50c..1244fb04 100644 --- a/docs/src/servers/aws-k8s/cluster.md +++ b/docs/src/servers/aws-k8s/cluster.md @@ -1,16 +1,20 @@ # AWS Talos Cluster (archive) -> **Terminated 2026-10-25.** This is a historical record, not an operating -> runbook. The fleet-backups S3 bucket and a mirror on himachal-1 -> (`~/backups/tunaos-aws-2026-10-25`) were retained; the current status of -> individual volumes, snapshots, IPs and other AWS resources is not established -> here. Matrix and standalone LiveKit are deployed on the -> [OCI spindle VPS](../spindle-vps/README.md). +> **Terminated 2026-10-25; final AWS deletion 2026-10-29.** This is a +> historical record, not an operating runbook. The fleet-backups S3 bucket +> and himachal-1 mirror (`~/backups/tunaos-aws-2026-10-25`) were retained +> at the October 25 shutdown. On October 29 all three AWS S3 buckets, +> including backups and OpenTofu state, were deleted; Punjab was terminated, +> its root volume was absent and all owned snapshots were deleted. +> Existing off-AWS migration data was not deleted, but no complete new +> archive of the deleted cloud history is claimed. Matrix and standalone +> LiveKit are deployed on the [OCI spindle VPS](../spindle-vps/README.md). > -> **OpenTofu still describes removed cluster resources.** `just aws-plan` may -> propose recreating them, and `just aws-apply` can recreate infrastructure and -> incur costs. Review live state and every proposed action before applying; see -> [AWS account and IaC](../aws/README.md). +> **AWS deployment paths are retired.** The former OpenTofu root module and +> manual deployment commands are archived at +> [`dcbbfbc`](https://github.com/hanthor/dotfiles/tree/dcbbfbc/aws), not available +> in this checkout. Restoring them can recreate paid infrastructure. See the +> [AWS account handbook](../aws/README.md) for current account status. The following describes the former TunaOS two-node Talos Kubernetes cluster in AWS `eu-north-1`, built 2026-08-27 to consolidate the two Hetzner VPSes @@ -154,8 +158,9 @@ See [AWS account costs](../aws/README.md) for the current caution. - The 50GB Postgres volume was unmounted. - Nightly Postgres dumps were copied to S3 and size-verified; DLM snapshot - schedules covered the root volumes. Retention of individual snapshots is not - established by this historical record. + schedules covered the root volumes. This describes historical protection, + not retained backups: final AWS deletion on 2026-10-29 removed the S3 + copies and all owned snapshots. - API-server OIDC and finer-grained RBAC were deferred; access used an admin client certificate. diff --git a/docs/src/servers/aws/README.md b/docs/src/servers/aws/README.md index 6761009b..bbcfa0a9 100644 --- a/docs/src/servers/aws/README.md +++ b/docs/src/servers/aws/README.md @@ -1,163 +1,61 @@ -# TunaOS AWS Account - -> **Cluster terminated 2026-10-25; IaC is not reconciled.** The -> [AWS Talos cluster](../aws-k8s/cluster.md) is historical. Matrix and -> standalone LiveKit are deployed on the [OCI spindle VPS](../spindle-vps/README.md). -> S3 backups and a mirror on himachal-1 were retained; do not assume other -> AWS resources have been removed or retained without checking the account. -> -> **Do not blindly run `just aws-plan` or `just aws-apply`.** `aws/` still -> describes removed cluster resources. A plan can propose recreating them; -> applying it can recreate infrastructure and incur costs. Review live state, -> configuration and every proposed action before any apply. - -This TunaOS account's OpenTofu configuration is in -[`aws/`](https://github.com/hanthor/dotfiles/tree/master/aws); `runs-on` was -managed separately by CloudFormation. The table describes the configuration -and prior deployment, not a verified live resource inventory. - -| Region | Configured or historical resource | Configuration | -|---|---|---| -| `eu-north-1` | Terminated Talos cluster — VPC, nodes, Elastic IPs, security groups, pgdata volume | `aws/cluster.tf` | -| `eu-north-1` | Backup configuration — DLM policy and retained fleet-backups bucket | `aws/backups.tf` | -| `us-east-1` | Punjab EC2 stopped; EIP released; root volume and five recovery snapshots retained | `aws/punjab.tf` | -| `us-east-2` | `runs-on` stack deleted 2026-10-28; cache/log buckets retained | Former CloudFormation stack `runs-on` | -| global | IAM, backup, account and budget configuration | `aws/iam.tf`, `aws/backups.tf`, `aws/account.tf`, `aws/budgets.tf` | - -## Verified retirement inventory (2026-10-28) - -An administrator inventory covered all 17 enabled regions. Punjab was the only -remaining EC2 instance; no NAT gateways, load balancers or RDS instances were -found. The `runs-on` Fargate manager and six Lambda functions were retired with -its CloudFormation stack; deletion reached `DELETE_COMPLETE`, with no ECS -clusters or Lambda functions remaining in `us-east-2`. - -All three S3 buckets remain. The runner cache and logging buckets retained all -observed object versions; their lifecycle expiration was disabled. Private -runner recovery material is root-only on the replacement under -`/srv/punjab-migration/runs-on-retirement/`. The fleet-backups bucket was not -modified. Both DLM policies are disabled to freeze the recovery snapshot set; -the five completed Punjab snapshots remain. Retained storage still incurs -charges; this is not a zero-bill claim. - -## Shutdown re-verification (2026-10-29) - -Authenticated live inventory covered all 17 enabled regions in this account. -The only remaining EC2 instance was Punjab, already stopped. No running -compute, ECS/EKS clusters, Lambda functions, RDS databases, load balancers, -NAT gateways or allocated Elastic IPs were found. Auto Scaling groups, open -Spot requests, Spot Fleets and EC2 Fleets were also empty in every region. -App Runner and Lightsail endpoint failures were checked against AWS's public -SSM region-availability registry; those service/region pairs are unsupported. -No additional stop/delete operation was necessary. - -The migrated VM passed authenticated SSH using its recorded host key and a -fresh omp request explicitly using `anthropic/claude-opus-5-5`, independent of -AWS/Kiro. The old 250 GiB gp3 root volume, five snapshots and three S3 buckets -remain intact; both DLM policies remain disabled. Retained storage is billable. - -Kiro's live subscription API reports both Enterprise Power and Pro Plus as -`CANCELLED`, expiring **2026-10-31 23:59:59.999 UTC**. Historical cancellation -requests used end-of-period removal on October 25. Cost Explorer still shows -Power charges of about $6.45/day for October 27–29; these do not establish an -active renewal. [Kiro cancellation documentation](https://kiro.dev/docs/enterprise/subscription-management/) -explains month-end cancellation and separate SSO-assignment cleanup. Do not -delete identities or application assignments as a billing workaround. - -## OpenTofu layout +# TunaOS AWS Account — Retired +**Retired workload and recovery-storage resources were deleted on 2026-10-29 with the owner's explicit approval.** Punjab runs on Karnataka; Matrix, LiveKit and the Hives run on the OCI Spindle VPS. Do not recreate the former AWS deployment. + +## Verified final inventory + +Authenticated verification covered all **17 enabled regions**. All 102 checks for nonterminated EC2 instances, EBS volumes, owned snapshots, Elastic IP allocations, nondefault VPCs and DLM policies returned empty. + +| Retired resource | Final state | +|---|---| +| Punjab EC2 `i-085690e02ca98c95a`, `us-east-1` | Terminated | +| Root volume `vol-09fdb248c28e850ca`, 250 GiB gp3 | Absent after termination | +| Five Punjab recovery snapshots | Deleted | +| `hanthor-fleet-backups-181185361136` | Deleted, including 212 versions and 22 delete markers | +| `runs-on-s3bucketcache-qygxr14wsudh` | Deleted, including 71 versions and 26 delete markers | +| `runs-on-s3bucketlogging-dqtjtzk02gph` | Deleted, including 1,147 versions and 365 delete markers | +| Stockholm migration VPC, subnet, route table, IGW and custom security groups | Deleted | +| Punjab SSH security group | Deleted | +| Both DLM policies and five account budgets | Deleted; each original budget name returns `NotFoundException` | +| Three fleet IAM users, their credentials, three custom roles/instance profiles and custom admin policy | Deleted | + +The earlier `runs-on` CloudFormation stack had already reached `DELETE_COMPLETE`. Earlier regional inventories also found no remaining ECS/EKS clusters, Lambda functions, RDS databases, load balancers, NAT gateways, Auto Scaling groups, Spot requests or fleets. CloudWatch logs/alarms/dashboards, EFS, AWS Backup, SSM parameters and scheduled EventBridge rules were empty. There were no customer-managed KMS keys. + +The AWS account remains open. Default account networks, AWS-managed KMS keys, service-linked roles and Identity Center metadata were not removed; they are not the retired workload or backup storage. This is not an account-closure claim. + +## Backup deletion decision + +The owner confirmed the Hives and Spindle were already migrated and that the historical AWS backups were unnecessary. The proposed 182 GB fleet-bucket archive was canceled. Newly created transfer/staging directories on Punjab and Karnataka were removed; **no completed archive of those S3 objects is claimed**. + +All three buckets were emptied of every object version and delete marker before deletion. This includes historical Postgres dumps, Hive/media/rollback exports, encrypted cluster-state archives and OpenTofu state containing Talos PKI. Do not describe those cloud copies as retained or recoverable. + +Existing off-AWS migration data, previously existing local backups, Bitwarden recovery notes, Atuin archives and the live Spindle/Punjab data were not deleted. Their retention does not establish a complete replacement for the deleted S3 history or EBS snapshots. + +Private action logs and the final API-verification receipt are on Punjab at: + +```text +~/backups/aws-final-retirement-20261029T015927Z/deletion-actions.jsonl +~/backups/aws-final-retirement-20261029T015927Z/completion.json ``` -aws/ -├── versions.tf # provider (eu-north-1 default, us_east_1 alias) + S3 backend -├── cluster.tf # VPC/subnet/IGW/routes, 3 SGs, 2 Talos nodes, EIPs, pgdata -├── backups.tf # DLM role + policy, fleet-backups bucket (+ versioning/SSE/lifecycle) -├── punjab.tf # retained stopped instance, key pair, SG, SSM role, disabled DLM -├── account.tf # EBS encryption by default (both regions) -├── iam.tf # scoped admin user + its policies -├── budgets.tf # 5 budgets, one for_each -├── imports.tf # one-time adoption record (no-op once imported) -├── variables.tf -└── terraform.tfvars.example -``` -- **State**: in the fleet-backups bucket under `tofu/aws/` - (versioned, SSE-S3, lockfile). It contains the Talos nodes' `user_data` — - i.e. cluster PKI — so it never goes in git. -- **Variables**: `aws/terraform.tfvars` holds the alert email and the admin - ingress allowlists. The repo is public, so it is gitignored and stored in the - Bitwarden secure note **`aws-tofu-tfvars`**. -- **Talos user_data is not in the config.** The node resources ignore `user_data` - and `ami`; `prevent_destroy` is configured on the nodes, punjab, worker EIP, - pgdata volume and backup bucket. It does not prevent creation of resources - missing from the account. - -## Usage and drift - -The former expectation of a no-change plan no longer applies after termination. -Do not treat drift as a reason to restore resources. Before using `just aws-plan`, -inspect the account and configuration; review the resulting plan carefully and -reconcile the intended resource lifecycle before any `just aws-apply`. Planning -does not itself create resources, but applying proposed creates can restore -infrastructure and incur charges. `aws/terraform.tfvars` remains sensitive; -`just aws-seed-tfvars` stores it in Bitwarden after an intentional edit. - -### Punjab replacement transition (2026-10-28) - -The fleet inventory's logical `punjab` host now connects to the replacement -guest on Karnataka at `100.103.187.46`, using the verified `ubuntu` account. -The replacement now owns the Tailscale name **`punjab`**; the old node is -`punjab-aws-retired`. Authenticated SSH, broker access, the native no-secrets -apply and monitoring were exercised on the replacement, including reboot -persistence. Atuin's later key/history recovery restored sync on Punjab, -Karnataka and Kerala under the canonical vault key, with original data retained -in private recovery archives; see [Atuin](../../roles/shell_atuin.md#historykey-recovery-2026-10-28). - -`aws/punjab.tf` retains the original EC2 instance, volume protection, security -group, key pair and SSM role/profile for recovery. Its desired instance state -is explicitly stopped, the EIP declaration and former cluster allowlist -dependency are removed, and DLM is disabled. It does **not** manage the -replacement guest. The replacement retains AWS administration tools and its -SSH-hardening and no-proxy policies. - -The replacement's shutdown service completed successfully at 19:27:15 UTC, -observing EC2 stopped and releasing its EIP before logging out of AWS. Its -root-only receipt is `/srv/punjab-migration/aws-cutover-receipt.json`, with -systemd journal provenance retained rather than treating a config edit as proof. -The former cluster's broader declarations still require separate state/config -reconciliation: deleted resources declared here can be proposed for recreation. -`prevent_destroy` does not protect against recreation. Retain S3 backups, -recovery storage and state; do not use a blanket destroy to resolve that drift. - -## IAM & access - -- **Root** historically had MFA and no access keys. Verify current access - configuration before relying on it. -- **A scoped admin user** was the day-to-day identity, with EC2, S3, budgets, - DLM, SSM and read-only IAM access. The historical key item is `aws-james-admin`; - verify its current permissions and status before use. -- **A backup user** was configured for put/get under `postgres/` in the backup - bucket, with no delete. Its key was stored in Bitwarden as `postgres-backup-s3` - and in the former cluster's `postgres/postgres-backup-s3` Secret, not tofu state. -- **punjab's configured instance role** has `AmazonSSMManagedInstanceCore`. -- DNS was managed on Cloudflare. Verify current AWS security and DNS settings - before relying on this historical inventory. - -## Backups - -The S3 fleet-backups bucket was retained, as was a mirror on himachal-1 at -`~/backups/tunaos-aws-2026-10-25`. Protect these backups and the OpenTofu state; -do not assume their presence means every former cluster volume or snapshot -survived. The former cluster uploaded verified Postgres dumps to -`postgres//`; configured S3 lifecycle rules moved them to IA after 30 -days, Glacier IR after 90, and expired them after a year. Check live bucket -settings before relying on that schedule. - -The configuration also describes DLM schedules for the former Talos root -volumes and punjab. Those schedules and the availability of individual snapshots -are not a current backup inventory. - -## Cost - -The AWS credits were exhausted. Remaining resources can incur charges, and -recreating cluster infrastructure would add costs. `aws/budgets.tf` describes -budget alerts; verify current budgets and actual charges in the AWS account. +These are deletion/audit metadata, not application-data archives. The older root-only `/srv/punjab-migration/aws-cutover-receipt.json` records the October 28 stop/EIP release; it is historical evidence, not the final inventory. + +## Active replacements + +- [Karnataka K3s](../talos-k8s/karnataka/README.md): `https://100.66.158.127:6443`, context `karnataka-k3s`. +- [Punjab VM](../punjab/README.md): `ubuntu@100.103.187.46`, KubeVirt `corral-vms/punjab`. +- [Spindle VPS](../spindle-vps/README.md): OCI-hosted services, not AWS. + +After AWS deletion, Karnataka was `Ready` and the Punjab VMI was `Running`. A fresh omp request explicitly using Anthropic had already passed on the migrated VM, independently of AWS/Kiro. The K3s fleet rollout and its unreachable-client limits are recorded in the Karnataka handbook. + +## Deployment retirement + +The tracked AWS OpenTofu root module, variable/plan/apply recipes, legacy AWS KubeVirt startup helper, provider-update rule and AWS-only CI job were removed. Historical source is available at [commit `dcbbfbc`](https://github.com/hanthor/dotfiles/tree/dcbbfbc/aws). Restoring that source can recreate paid infrastructure; do not treat it as the desired current state. + +Untracked private files such as `aws/terraform.tfvars`, cached state/plans and provider caches were not swept or published. Historical Kubernetes manifests remain recovery reference, not instructions to apply to Karnataka. General AWS CLI tools remain available for billing administration. + +## Remaining billing + +Kiro's live subscription API reported both Enterprise Power and Pro Plus as `CANCELLED`, expiring **2026-10-31 23:59:59.999 UTC**. Their end-of-period cancellation was requested on October 25. Existing October charges can continue through that period; [Kiro's cancellation documentation](https://kiro.dev/docs/enterprise/subscription-management/) explains the timing and separate SSO-assignment cleanup. + +Final, prorated, request, transfer or minimum-storage-duration charges can arrive after resource deletion, and Cost Explorer is delayed. No zero-dollar final-invoice claim is made. The deleted compute, volumes, snapshots and buckets are no longer retained resources to accrue ongoing storage/compute charges. diff --git a/docs/src/servers/home-k8s-images.md b/docs/src/servers/home-k8s-images.md index 1c39ff25..a6278556 100644 --- a/docs/src/servers/home-k8s-images.md +++ b/docs/src/servers/home-k8s-images.md @@ -1,6 +1,6 @@ # Home Kubernetes Server Images -These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to tailnet address `100.66.158.127`; KubeVirt `v1.9.1`, CDI and Corral are installed. The `corral-vms/punjab` VM is now the active Punjab replacement at `100.103.187.46`; its former EC2 host is stopped with recovery storage retained. Bihar's current OS and services are unverified. Verify image provenance and physical recovery before any future installation or switch. +These are proposed bootable K3s images, not the current OS or workloads of either machine. Karnataka runs `ghcr.io/projectbluefin/utah:testing` with `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server bound to tailnet address `100.66.158.127`; KubeVirt `v1.9.1`, CDI and Corral are installed. The `corral-vms/punjab` VM is now the active Punjab replacement at `100.103.187.46`. Its former EC2 host was terminated on 2026-10-29, and its root volume, AWS snapshots and S3 backups were deleted; see the [final AWS deletion evidence](aws/README.md). Bihar's current OS and services are unverified. Verify image provenance and physical recovery before any future installation or switch. The Utah-based images built in CI; the starter ISO was about 5.94 GiB. Acceptance allows media up to 8 GiB, but the publication workflow still diff --git a/docs/src/servers/punjab/README.md b/docs/src/servers/punjab/README.md index 90421699..1c6adbdf 100644 --- a/docs/src/servers/punjab/README.md +++ b/docs/src/servers/punjab/README.md @@ -77,23 +77,29 @@ also copied into the active checkout. Other old temporary worktrees remain in the system recovery copy; their original absolute Git paths may need repair before reuse. -## AWS recovery and shutdown handoff - -The former EC2 instance `i-085690e02ca98c95a` in `us-east-1` reached **stopped** -at the cutover, not terminated. The 250 GiB root volume -`vol-09fdb248c28e850ca` and five completed snapshots are retained. Snapshot -`snap-0bc709516bb003e9a` was taken while EC2 was live: it is crash-consistent, -not proof of an application-consistent restore. - -The replacement's `punjab-aws-cutover.service` completed successfully at -2026-10-28 19:27:15 UTC, after observing EC2 stopped and releasing EIP allocation -`eipalloc-00f40f76478a3ea02`. It then logged out of the AWS CLI. The root-only -receipt at `/srv/punjab-migration/aws-cutover-receipt.json` preserves that -result with journal provenance; a later check-only authentication failure is -retained separately. Consult the receipt and AWS for current resource state. - -The `runs-on` stack reached `DELETE_COMPLETE`; its two S3 buckets and all -observed versions were retained. The fleet-backups S3 bucket remains intact. -Both DLM policies are disabled. Storage still incurs charges. The former -cluster's broader IaC remains historical: follow the warnings in the -[AWS handbook](../aws/README.md), not a blanket `tofu apply` or `destroy`. +## Final AWS retirement (2026-10-29) + +The owner approved deletion after confirming the applications had migrated. +Former EC2 `i-085690e02ca98c95a` is now **terminated**; root volume +`vol-09fdb248c28e850ca`, five snapshots and all three AWS S3 buckets/versions +were deleted. The proposed archive of old Hive/Spindle backups was canceled +and its new temporary copies removed. Existing local migration data above, +Atuin archives and live VM data were not deleted. + +Final API verification found no nonterminated EC2, EBS volumes, owned snapshots, +Elastic IPs, nondefault VPCs or DLM policies in any of the 17 enabled regions. +Karnataka remained `Ready` and `corral-vms/punjab` remained `Running`. +Private deletion evidence is at +`~/backups/aws-final-retirement-20261029T015927Z/completion.json`. +The [AWS handbook](../aws/README.md) records exact resource scope, remaining +account defaults and final-billing caveats. Deployment recipes/module are +removed; do not restore archived AWS source as routine configuration drift. + +### Earlier cutover history + +At the October 28 cutover, EC2 was initially stopped and recovery storage was +retained. `punjab-aws-cutover.service` succeeded at 19:27:15 UTC and released +EIP `eipalloc-00f40f76478a3ea02` before logging out of AWS. The root-only +`/srv/punjab-migration/aws-cutover-receipt.json` retains that journal-backed +history; a later check-only authentication failure is separate. Those records +must not be read as evidence that the now-deleted instance/storage still exist. diff --git a/docs/src/servers/spindle-vps/README.md b/docs/src/servers/spindle-vps/README.md index bed01118..f91e43e1 100644 --- a/docs/src/servers/spindle-vps/README.md +++ b/docs/src/servers/spindle-vps/README.md @@ -100,12 +100,20 @@ to avoid public-IP hairpin routing; `reilly.asia` remains on the portfolio Worke ## Caveats -- The MAS dump is from 15:22 UTC. The Spindle store's last write was 15:25 UTC and its archive was - taken at 15:33 UTC; the old cluster continued until 17:16 UTC. Writes after the store archive are - only in the encrypted cluster-state backup and are not in this VPS store. A real login and joined - room read passed against the VPS, but that does not prove the later writes were restored. -- The AWS cluster is terminated. The S3 and himachal-1 backups are untouched, but stopping the VPS - stack is an outage, not a rollback; recovery requires provisioning and restoring from backup. +- At the October 25 migration, the MAS dump was from 15:22 UTC. The Spindle + store's last write was 15:25 UTC and its archive was taken at 15:33 UTC; the + old cluster continued until 17:16 UTC. Writes after the store archive were + only in the encrypted cluster-state backup, not this VPS store. A real login + and joined-room read passed against the VPS, but did not prove those later + writes were restored. The cloud copy of that backup was deleted on October 29; + availability of those writes in any existing off-AWS copy is not established here. +- The AWS cluster was terminated on October 25. The S3 backups retained at that + shutdown were deleted in the **2026-10-29 final AWS retirement**, along with + AWS volumes and owned snapshots; see the [deletion evidence](../aws/README.md). + Existing himachal-1 migration data was not deleted, but it is not proof of a + complete replacement for the deleted cloud history. Stopping the VPS stack + is an outage, not a rollback; recovery requires an independently verified + off-AWS backup and a provisioned target. - When `tuna-os/spindle` publishes an arm64 release containing #723, switch the image in `compose.yaml` and delete the `ops/arm64-perf-723` branch. - LiveKit is capped at 2 GiB and 1 OCPU to leave room for Spindle and the Hives on this 2-OCPU host; diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index d14940c8..3fd4e4f7 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -6,8 +6,11 @@ available over Tailscale at `100.66.158.127`. A single-node K3s cluster runs KubeVirt, CDI, and Corral. The `corral-vms/punjab` VM now runs the migrated Punjab environment at `100.103.187.46` (tailnet name `punjab`). This is the [active replacement](../../punjab/README.md), not a staging VM or an EC2 host. -For the former EC2 host's retirement and retained recovery artifacts, consult -the [AWS account handbook](../../aws/README.md). +For the former EC2 host's final **2026-10-29** termination and deletion of its +root volume, AWS snapshots and S3 backups (including OpenTofu state), consult +the [AWS account handbook and deletion evidence](../../aws/README.md). +Existing off-AWS migration data and Bitwarden notes were not deleted; they +are not evidence that the deleted cloud backups remain recoverable. NetworkManager, Tailscale and SSH are enabled at boot. Ethernet is preferred; a locally stored Wi-Fi profile autoconnects with a higher route metric. Wi-Fi diff --git a/docs/src/talos-k8s.md b/docs/src/talos-k8s.md index 6be3b2ae..31fa930e 100644 --- a/docs/src/talos-k8s.md +++ b/docs/src/talos-k8s.md @@ -5,7 +5,9 @@ > `multi-user.target` and a single-node K3s `v1.36.5+k3s1` server on its > tailnet address `100.66.158.127`. KubeVirt `v1.9.1`, CDI and Corral are > installed; `corral-vms/punjab` is now the active Punjab replacement at -> `100.103.187.46`. The former EC2 host is stopped with recovery storage retained. +> `100.103.187.46`. The former EC2 host was terminated on 2026-10-29; +> its root volume, AWS snapshots and S3 backups were deleted. See the +> [final AWS deletion evidence](servers/aws/README.md). > Retain `talos-k8s/`, cluster configs and Bitwarden notes as recovery material, > not proof either machine still runs Talos. > diff --git a/renovate.json b/renovate.json index 01a1db16..7f0b52e4 100644 --- a/renovate.json +++ b/renovate.json @@ -17,11 +17,6 @@ "matchUpdateTypes": ["major"], "automerge": false }, - { - "description": "OpenTofu providers (aws/): lockfile bumps are validated + tofu-tested in CI; applying stays manual", - "matchManagers": ["terraform"], - "groupName": "opentofu providers" - }, { "description": "Cluster images/charts: manifests are applied by hand, so a merge would only drift git from the cluster. Open weekly grouped PRs as a to-do, never auto-merge.", "matchFileNames": ["talos-k8s/**"], diff --git a/scripts/kubevirt-node b/scripts/kubevirt-node deleted file mode 100755 index 4b86e835..00000000 --- a/scripts/kubevirt-node +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# Power the AWS KubeVirt node (aws/kubevirt.tf) up or down from the tailnet. -# -# scripts/kubevirt-node up # start it and wait until the k8s node is Ready -# scripts/kubevirt-node down # stop it (VMs on it are shut down first) -# scripts/kubevirt-node status -# -# The in-cluster idle-stop CronJob stops it after 30 idle minutes, and the -# migration budget action stops it if spend nears the credits, so `up` is -# the only command needed day to day. -set -euo pipefail -export AWS_DEFAULT_REGION=eu-north-1 -export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config-aws-migration}" -ID=$(aws ec2 describe-instances --filters Name=tag:Name,Values=aws-migration-kubevirt \ - Name=instance-state-name,Values=pending,running,stopping,stopped \ - --query 'Reservations[0].Instances[0].InstanceId' --output text) -[ "$ID" != None ] || { echo "no aws-migration-kubevirt instance" >&2; exit 1; } -state() { aws ec2 describe-instances --instance-ids "$ID" --query 'Reservations[0].Instances[0].State.Name' --output text; } - -case "${1:-status}" in - up) - aws ec2 start-instances --instance-ids "$ID" >/dev/null - aws ec2 wait instance-running --instance-ids "$ID" - node=$(kubectl get nodes -l tunaos.org/kubevirt=true -o jsonpath='{.items[0].metadata.name}') - kubectl annotate node "$node" tunaos.org/idle-since- >/dev/null 2>&1 || true - kubectl wait --for=condition=Ready "node/$node" --timeout=10m - echo "up: $node ($ID). Stops by itself after 30 idle minutes." - ;; - down) - for vm in $(kubectl get vmi -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}'); do - virtctl stop -n "${vm%/*}" "${vm#*/}" || kubectl -n "${vm%/*}" patch vm "${vm#*/}" --type merge -p '{"spec":{"runStrategy":"Halted"}}' - done - aws ec2 stop-instances --instance-ids "$ID" >/dev/null - echo "stopping $ID" - ;; - status) - echo "$ID: $(state)" - kubectl get vmi -A 2>/dev/null || true - ;; - *) echo "usage: $0 up|down|status" >&2; exit 2 ;; -esac diff --git a/talos-k8s/backup/README.md b/talos-k8s/backup/README.md index 98750c11..a0eac927 100644 --- a/talos-k8s/backup/README.md +++ b/talos-k8s/backup/README.md @@ -1,6 +1,20 @@ -# Backups — AWS Talos cluster - -Three layers, because they fail differently and cover different disasters. +# Historical backups — AWS Talos cluster + +> **Nonoperational historical reference.** This describes the backup design +> before the AWS cluster's 2026-10-25 shutdown, not current protection. +> Final deletion on **2026-10-29** removed all three AWS S3 buckets and their +> versions (including dumps and OpenTofu state), AWS volumes and all owned +> snapshots; DLM policies were deleted too. The commands below are preserved +> as historical reference and cannot recover those deleted cloud copies. +> Existing off-AWS migration data and vault notes were not deleted, but no +> new complete S3 archive is claimed. See the +> [final AWS deletion evidence](../../docs/src/servers/aws/README.md). +> The former deployment module is archived at +> [`dcbbfbc`](https://github.com/hanthor/dotfiles/tree/dcbbfbc/aws); +> do not restore it as an active deployment path. + +The historical design had three layers, because they failed differently and +covered different disasters. | Layer | What | Covers | Does not cover | |---|---|---|---| diff --git a/talos-k8s/backup/postgres-backup.yaml b/talos-k8s/backup/postgres-backup.yaml index f7839c35..f1a12f36 100644 --- a/talos-k8s/backup/postgres-backup.yaml +++ b/talos-k8s/backup/postgres-backup.yaml @@ -139,7 +139,7 @@ spec: # Off-cluster copy. The local PVC shares a node with the database, # so without this a lost node loses the backups too. The writer # key can put/get under postgres/ only (no delete); the bucket - # tiers and expires them (aws/backups.tf). + # formerly tiered/expired them (aws/backups.tf at Git dcbbfbc). for DB in synapse mas; do OUT="${DEST}/${DB}.dump" KEY="postgres/${STAMP}/${DB}.dump" @@ -169,8 +169,8 @@ spec: secretKeyRef: name: postgres-superuser key: POSTGRES_PASSWORD - # IAM user postgres-backup-writer (aws/backups.tf); key also - # in Bitwarden note `postgres-backup-s3`. + # Historical IAM user postgres-backup-writer (aws/backups.tf at + # Git dcbbfbc); key also in Bitwarden note `postgres-backup-s3`. - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: diff --git a/talos-k8s/kubevirt-aws/README.md b/talos-k8s/kubevirt-aws/README.md index 78198547..be243dcd 100644 --- a/talos-k8s/kubevirt-aws/README.md +++ b/talos-k8s/kubevirt-aws/README.md @@ -1,5 +1,15 @@ # KubeVirt on the AWS cluster (tailnet-only corral) +> **Historical AWS deployment only.** The AWS root module and node startup helper +> have been retired. Their source and original deployment instructions are +> archived at [`dcbbfbc`](https://github.com/hanthor/dotfiles/tree/dcbbfbc/talos-k8s/kubevirt-aws). +> Restoring them can recreate paid resources. These retained Kubernetes manifests +> are recovery reference, not the active Karnataka K3s/KubeVirt configuration. +> Final AWS deletion on **2026-10-29** removed AWS volumes, owned snapshots and +> all three S3 buckets, including backups and OpenTofu state. The disk-persistence +> caveat below describes historical stop/start behavior, not surviving storage. +> See the [final deletion evidence](../../docs/src/servers/aws/README.md). + A Proxmox-style VM host on the AWS Talos cluster: KubeVirt VMs on a dedicated node, managed through corral-web at `https://corral-aws..ts.net`. There is no public ingress; the node has no public services, and corral-web is @@ -7,10 +17,10 @@ published only through the Tailscale operator. | Piece | Where | |---|---| -| Node: m7i.2xlarge with nested virtualization, tainted for VMs only | `aws/kubevirt.tf` | -| Spend kill switch: AWS Budgets stops the node at $1100 gross/month | `aws/kubevirt.tf` | +| Node: m7i.2xlarge with nested virtualization, tainted for VMs only | Archived `aws/kubevirt.tf` at `dcbbfbc` | +| Spend kill switch: AWS Budgets stops the node at $1100 gross/month | Archived `aws/kubevirt.tf` at `dcbbfbc` | | Idle stop after 30 minutes with no VM pods | `idle-stop.yaml` | -| Start on demand | `scripts/kubevirt-node up` | +| Start on demand | Archived `scripts/kubevirt-node` at `dcbbfbc` | | KubeVirt/CDI placement (infra on always-on nodes, VMs on the node) | `kubevirt-cr.yaml` | | corral-web as `corral-aws` on the tailnet | `kustomization.yaml` | @@ -21,45 +31,11 @@ credits. ## Bring-up (once) -```bash -export KUBECONFIG=~/.kube/config-aws-migration TALOSCONFIG=~/.talos/config-aws-migration - -# 1. Node (boots Talos in maintenance mode: no user_data, no secrets in state) -cd aws && tofu plan -out tfplan && tofu apply tfplan && cd .. -IP=$(cd aws && tofu output -raw kubevirt_public_ip) - -# 2. Join it: the existing worker's config + the kubevirt patch. /tmp only, never commit. -talosctl -n 10.20.1.11 get machineconfig -o jsonpath='{.spec}' > /tmp/worker.yaml -talosctl machineconfig patch /tmp/worker.yaml --patch @talos-k8s/kubevirt-aws/talos-patch.yaml -o /tmp/kubevirt.yaml -talosctl apply-config --insecure -e "$IP" -n "$IP" -f /tmp/kubevirt.yaml -shred -u /tmp/worker.yaml /tmp/kubevirt.yaml -kubectl wait --for=condition=Ready node -l tunaos.org/kubevirt=true --timeout=15m -talosctl -n 10.20.1.12 ls /dev | grep kvm # must list kvm - -# 3. KubeVirt + CDI operators, then our placement/config -kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/v1.8.2/kubevirt-operator.yaml -kubectl apply -f https://github.com/kubevirt/containerized-data-importer/releases/download/v1.66.1/cdi-operator.yaml -kubectl apply -f talos-k8s/kubevirt-aws/kubevirt-cr.yaml -kubectl -n kubevirt wait kv kubevirt --for condition=Available --timeout=15m - -# 4. Idle-stop credentials (scoped IAM user from aws/kubevirt.tf), then corral-web + CronJob. -# The key already exists: Bitwarden secure note `kubevirt-node-power` -# (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / INSTANCE_ID / AWS_DEFAULT_REGION). -# Only if it is lost: aws iam create-access-key --user-name kubevirt-node-power -for ns in kubevirt-aws tailvm; do kubectl create namespace $ns --dry-run=client -o yaml | kubectl apply -f - -kubectl -n $ns create secret generic node-power \ - --from-literal=AWS_ACCESS_KEY_ID=... --from-literal=AWS_SECRET_ACCESS_KEY=... \ - --from-literal=INSTANCE_ID="$(cd aws && tofu output -raw kubevirt_instance_id)"; done -kubectl kustomize --load-restrictor LoadRestrictionsNone talos-k8s/kubevirt-aws | kubectl apply -f - -``` - -## Day to day - -```bash -scripts/kubevirt-node up # ~2 min; the node stops itself after 30 idle minutes -corral --context aws-migration list # or https://corral-aws..ts.net -scripts/kubevirt-node down # stop now -``` +The original bring-up and daily power-management commands are preserved in the +[runbook at `dcbbfbc`](https://github.com/hanthor/dotfiles/blob/dcbbfbc/talos-k8s/kubevirt-aws/README.md). +They depend on the retired AWS root module and startup helper and are not active +commands in this checkout. Do not apply these historical instructions to the +[current Karnataka K3s host](../../docs/src/servers/talos-k8s/karnataka/README.md). ## Caveats diff --git a/talos-k8s/kubevirt-aws/idle-stop.yaml b/talos-k8s/kubevirt-aws/idle-stop.yaml index 148be4ab..f39d7efd 100644 --- a/talos-k8s/kubevirt-aws/idle-stop.yaml +++ b/talos-k8s/kubevirt-aws/idle-stop.yaml @@ -1,6 +1,7 @@ -# Stops the AWS KubeVirt node (aws/kubevirt.tf) after IDLE_MINUTES with no -# workload pods on it (VM launchers, CDI importers; DaemonSets don't -# count). Start it again with `scripts/kubevirt-node up` from the tailnet. +# Historical AWS KubeVirt idle-stop manifest; not the active local K3s host. +# The node's aws/kubevirt.tf and startup helper are archived at Git commit +# dcbbfbc. This job only stops an existing instance after IDLE_MINUTES with no +# workload pods (VM launchers, CDI importers; DaemonSets do not count). # # Credentials: secret kubevirt-aws/node-power holds an access key for IAM # user kubevirt-node-power, which can only start/stop/describe this one diff --git a/talos-k8s/kubevirt-aws/talos-patch.yaml b/talos-k8s/kubevirt-aws/talos-patch.yaml index 03f862cd..c1d456fa 100644 --- a/talos-k8s/kubevirt-aws/talos-patch.yaml +++ b/talos-k8s/kubevirt-aws/talos-patch.yaml @@ -1,5 +1,6 @@ -# Talos machine-config patch for the AWS KubeVirt node (aws/kubevirt.tf). -# Applied on top of the existing AWS worker's config: +# Historical Talos patch for the AWS KubeVirt node; not for current K3s hosts. +# Its aws/kubevirt.tf deployment is archived at Git commit dcbbfbc. +# Applied on top of the former AWS worker's config: # # talosctl -n 10.20.1.11 get machineconfig -o jsonpath='{.spec}' > /tmp/worker.yaml # never commit # talosctl machineconfig patch /tmp/worker.yaml --patch @talos-patch.yaml -o /tmp/kubevirt.yaml From c49c599b9a2282c345a1c996a094cc188fe9b8b2 Mon Sep 17 00:00:00 2001 From: James Reilly Date: Fri, 30 Oct 2026 19:18:09 +0000 Subject: [PATCH 61/62] feat(beeper): install Beeper from FlatPark and sign it in on desktops - flatpak: new flatpark_flatpaks list, installed system-wide from the flatpark remote; system_flatpaks stays Flathub-only. - beeper-login unit: Beeper Desktop only supports emailed one-time codes, so it starts sign-in via the local Desktop API, reads the code over IMAP from the Bitwarden item 'beeper', then unlocks E2EE with the Recovery Code. Skipped with --no-secrets, a locked vault, non-desktops or skip_beeper. --- docs/src/bitwarden.md | 1 + docs/src/roles/flatpak.md | 4 + mise.toml | 3 + scripts/run-units.sh | 9 +- units/beeper-login.sh | 67 +++++++++++++++ units/beeper-login/beeper_login.py | 131 +++++++++++++++++++++++++++++ units/flatpak-packages.yml | 5 ++ units/flatpak.sh | 14 +-- 8 files changed, 225 insertions(+), 9 deletions(-) create mode 100755 units/beeper-login.sh create mode 100755 units/beeper-login/beeper_login.py diff --git a/docs/src/bitwarden.md b/docs/src/bitwarden.md index d8697dbb..af01bb52 100644 --- a/docs/src/bitwarden.md +++ b/docs/src/bitwarden.md @@ -36,6 +36,7 @@ Common items consumed by units and scripts (the unit sources and their configura | `kubeconfig`, `talosconfig` | Secure notes | `kube` unit — retained home Talos recovery credentials; never repurpose for K3s | | `kubeconfig-aws-migration`, `talosconfig-aws-migration` | Secure notes | `kube` unit — retained credentials for the terminated AWS cluster | | `accounts.firefox.com` | Login + TOTP | `browser-fxa` unit — Firefox Account credentials | +| `beeper` | Login + custom fields `imap_host`, `recovery_key`, optional `imap_user` | `beeper-login` unit — username is the Beeper email, password its IMAP app password; `recovery_key` is the Beeper Recovery Code | | `github-token` | Login (password) | `github` unit — PAT for `gh` | | `atuin.sh` | Login + custom field `key` | `atuin` unit — history sync | | `deepseek-api-key`, `tavily-api-key` | Login (password) | `shell-ai` unit — shell environment | diff --git a/docs/src/roles/flatpak.md b/docs/src/roles/flatpak.md index 66a5853a..2953dd79 100644 --- a/docs/src/roles/flatpak.md +++ b/docs/src/roles/flatpak.md @@ -7,6 +7,10 @@ The unit requires Flatpak and `yq`. It reads system remotes and `system_flatpaks` from `units/flatpak-packages.yml`, adds remotes, removes system Firefox and Thunderbird Flatpaks, and installs missing applications from Flathub. A failed batch is retried individually. A dangling `/var/lib/flatpak` symlink is preserved: restore its target before retrying system operations. User operations and wrapper deployment remain independent. `DOTFILES_UPGRADE=1` or `true` requests a best-effort system update. System operations require root or non-interactive sudo. +`flatpark_flatpaks` in the same file lists system applications installed from the `flatpark` remote instead of Flathub (currently `com.automattic.Beeper`); the key may be omitted. The remote must also appear in `flatpak_remotes`. + +The `beeper-login` unit then signs Beeper in on desktops where `com.automattic.Beeper` is installed. Beeper Desktop only supports emailed one-time codes (no password sign-in), so the unit starts sign-in through the local Desktop API (`127.0.0.1:23373`), reads the code from an IMAP mailbox, and unlocks encryption with the Recovery Code. It reads the Bitwarden login `beeper` (see [Bitwarden](../bitwarden.md)), is skipped with `--no-secrets` or a locked vault, never prompts, and does nothing once Beeper reports `ready`. It launches Beeper only on a fresh install (no `~/.var/app/com.automattic.Beeper/config/BeeperTexts`) or with `BEEPER_LOGIN_FORCE=1`; otherwise it queries a running app. Opt out per host with `skip_beeper: true`. Chat-network accounts (WhatsApp, Telegram and so on) are not signed in and still need linking in the app. + Host-specific `user_flatpak_remotes` and `user_flatpaks` in `host_vars/.yml` are applied with `--user`, separately from the system store. For `himachal`, these configure FlatPark and user Flathub, then install `org.tunaos.blueshell`, `org.tunaos.tavern`, and `org.tunaos.compass` from FlatPark. An already installed user app with a different origin is preserved rather than switched. This installation does not configure or unlock Compass's keyring or establish an application login; those depend on the user's session and app setup. The unit also writes `chromium`, `chromium-browser`, `google-chrome`, and `google-chrome-stable` wrappers in `~/.local/bin`, invoking `org.chromium.Chromium` via Flatpak. Existing custom files and symlinks are preserved; matching managed wrappers have their mode corrected. Wrappers are attempted even if Flatpak is unavailable. diff --git a/mise.toml b/mise.toml index b330c29c..a0e9beff 100644 --- a/mise.toml +++ b/mise.toml @@ -73,6 +73,9 @@ file = "units/zen-browser.sh" [tasks."unit:browser-fxa"] file = "units/browser-fxa.sh" +[tasks."unit:beeper-login"] +file = "units/beeper-login.sh" + [tasks."unit:pipewire-audio"] file = "units/pipewire-audio.sh" diff --git a/scripts/run-units.sh b/scripts/run-units.sh index b9097fee..3c55c6a2 100755 --- a/scripts/run-units.sh +++ b/scripts/run-units.sh @@ -3,7 +3,7 @@ # Usage: scripts/run-units.sh [--no-secrets] [--tags a,b] set -uo pipefail -ALL_UNITS="sudo sshd apk-packages termux-packages homebrew flatpak chezmoi fonts omp atuin shell-ai ssh-keys ssh-mesh github kube forgejo-registry tailscale bluefin gnome zen-browser browser-fxa pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services proxy tailscale-cert bst-dashboard fleet-facts" +ALL_UNITS="sudo sshd apk-packages termux-packages homebrew flatpak chezmoi fonts omp atuin shell-ai ssh-keys ssh-mesh github kube forgejo-registry tailscale bluefin gnome zen-browser browser-fxa beeper-login pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services proxy tailscale-cert bst-dashboard fleet-facts" tasks_for_tag() { case "$1" in system) echo "sudo sshd apk-packages termux-packages retired-services server-hardening" ;; sudo) echo sudo ;; @@ -20,7 +20,7 @@ tasks_for_tag() { case "$1" in git|neovim|editor) echo chezmoi ;; atuin) echo "chezmoi atuin" ;; ai) echo shell-ai ;; - secrets) echo "chezmoi atuin shell-ai ssh-keys github kube forgejo-registry tailscale browser-fxa" ;; + secrets) echo "chezmoi atuin shell-ai ssh-keys github kube forgejo-registry tailscale browser-fxa beeper-login" ;; ssh|ssh_keys) echo "ssh-keys ssh-mesh" ;; mesh_check) echo ssh-mesh ;; github) echo github ;; @@ -28,10 +28,11 @@ tasks_for_tag() { case "$1" in forgejo|registry) echo forgejo-registry ;; tailscale) echo tailscale ;; omp) echo omp ;; - desktop) echo "flatpak bluefin gnome zen-browser browser-fxa pipewire-audio" ;; + desktop) echo "flatpak bluefin gnome zen-browser browser-fxa beeper-login pipewire-audio" ;; bluefin) echo bluefin ;; gnome) echo gnome ;; browser|zen_browser) echo "zen-browser browser-fxa" ;; + beeper) echo "flatpak beeper-login" ;; audio|pipewire) echo pipewire-audio ;; services) echo "syncthing laptop-services system-services osquery proxy tailscale-cert bst-dashboard" ;; syncthing) echo syncthing ;; @@ -132,7 +133,7 @@ for unit in chezmoi bluefin gnome; do done zen_ok=1; run_stage zen-browser || { zen_ok=0; rc=1; } -later=(fonts omp atuin shell-ai kube forgejo-registry pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services bst-dashboard) +later=(fonts omp atuin shell-ai kube forgejo-registry beeper-login pipewire-audio syncthing hive-ops laptop-services system-services osquery server-hardening retired-services bst-dashboard) if [ "$zen_ok" = 1 ]; then later+=(browser-fxa) elif selected browser-fxa; then diff --git a/units/beeper-login.sh b/units/beeper-login.sh new file mode 100755 index 00000000..50a314ab --- /dev/null +++ b/units/beeper-login.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Sign Beeper Desktop (com.automattic.Beeper) in and unlock encryption on desktops. +# Beeper only supports emailed one-time codes, so the unit reads the code from the +# IMAP mailbox described by the Bitwarden login item `beeper`: +# username / password the Beeper email address / its IMAP app password +# field imap_host IMAP server (required) +# field imap_user IMAP login if it differs from username (optional) +# field recovery_key Beeper Recovery Code (required) +set -uo pipefail +# shellcheck source=units/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +APP=com.automattic.Beeper +API="${BEEPER_API:-http://127.0.0.1:23373}" +helper="$DOTFILES_DIR/units/beeper-login/beeper_login.py" + +if skipped_by_profile skip_beeper; then note 'SKIPPED: Beeper disabled for this host/profile'; exit 0; fi +if ! is_desktop; then note 'SKIPPED: Beeper login is only configured on desktops'; exit 0; fi +if [ "${UNITS_NO_SECRETS:-0}" = 1 ]; then note 'SKIPPED: Beeper login needs Bitwarden secrets'; exit 0; fi +if ! flatpak info --system "$APP" >/dev/null 2>&1 && ! flatpak info --user "$APP" >/dev/null 2>&1; then + note "SKIPPED: $APP is not installed"; exit 0 +fi +if [ "${UNITS_CHECK:-0}" = 1 ]; then + note 'PLANNED: sign Beeper in via the local Desktop API if it is logged out; current state not inspected' + exit 0 +fi +for tool in curl jq python3 bw; do + command -v "$tool" >/dev/null 2>&1 || { note "FAIL $tool is required"; exit 1; } +done + +state() { curl -fsS -m 5 "$API/v1/app/setup" 2>/dev/null | jq -r '.state // empty'; } + +# Launching Beeper opens a window, so only do it when the app has never run here +# (fresh install) or BEEPER_LOGIN_FORCE=1; an already-running app is just queried. +launch_app() { + [ -n "${WAYLAND_DISPLAY:-}${DISPLAY:-}" ] || { note 'SKIPPED: no graphical session to launch Beeper in'; return 2; } + setsid nohup flatpak run "$APP" >/dev/null 2>&1 < /dev/null & + return 0 +} + +cur="$(state)" +if [ -z "$cur" ]; then + if [ -d "$HOME/.var/app/$APP/config/BeeperTexts" ] && [ "${BEEPER_LOGIN_FORCE:-0}" != 1 ]; then + note 'SKIPPED: Beeper is not running; start it (or set BEEPER_LOGIN_FORCE=1) to check its login' + exit 0 + fi + launch_app; rc=$? + [ "$rc" = 0 ] || exit 0 +fi +if [ "$cur" = ready ]; then note 'ok Beeper already signed in'; exit 0; fi + +session="$(bw_session)" +if [ -z "$session" ]; then note 'PARTIAL: Bitwarden is locked; Beeper login not attempted'; exit 0; fi +item="$(BW_SESSION="$session" bw get item beeper 2>/dev/null)" || { note "FAIL Bitwarden item 'beeper' not found"; exit 1; } +field() { jq -r --arg n "$1" '.fields[]? | select(.name == $n) | .value' <<<"$item"; } +BEEPER_EMAIL="$(jq -r '.login.username // empty' <<<"$item")" +BEEPER_IMAP_PASSWORD="$(jq -r '.login.password // empty' <<<"$item")" +BEEPER_IMAP_HOST="$(field imap_host)" +BEEPER_IMAP_USER="$(field imap_user)"; BEEPER_IMAP_USER="${BEEPER_IMAP_USER:-$BEEPER_EMAIL}" +BEEPER_RECOVERY_KEY="$(field recovery_key)" +for v in BEEPER_EMAIL BEEPER_IMAP_PASSWORD BEEPER_IMAP_HOST BEEPER_RECOVERY_KEY; do + [ -n "${!v}" ] || { note "FAIL Bitwarden item 'beeper' is missing ${v#BEEPER_}"; exit 1; } +done +export BEEPER_EMAIL BEEPER_IMAP_PASSWORD BEEPER_IMAP_HOST BEEPER_IMAP_USER BEEPER_RECOVERY_KEY BEEPER_API="$API" + +step 'Beeper sign-in and recovery-key unlock' python3 "$helper" +finish diff --git a/units/beeper-login/beeper_login.py b/units/beeper-login/beeper_login.py new file mode 100755 index 00000000..0db9acae --- /dev/null +++ b/units/beeper-login/beeper_login.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Sign Beeper Desktop in from the local Desktop API and unlock encryption. + +Beeper Desktop only offers email-code sign-in (`signInMethods: ["email"]`), so +the one-time code is read from an IMAP mailbox. Credentials come from the +environment (never argv): BEEPER_EMAIL, BEEPER_IMAP_HOST, BEEPER_IMAP_USER, +BEEPER_IMAP_PASSWORD, BEEPER_RECOVERY_KEY. Exit 0 = ready, 3 = already signed in +elsewhere state we cannot repair, 1 = failure. Secrets are never printed. +""" +import email +import imaplib +import json +import os +import re +import sys +import time +import urllib.error +import urllib.request + +BASE = os.environ.get("BEEPER_API", "http://127.0.0.1:23373") +# Pre-login calls take a placeholder bearer, as Beeper's own CLI does. +PUBLIC_TOKEN = "setup-login-public-client" +CODE_RE = re.compile(r"\b(\d{6})\b") + + +def api(method, path, body=None, token=PUBLIC_TOKEN): + req = urllib.request.Request( + BASE + path, + method=method, + data=None if body is None else json.dumps(body).encode(), + headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + except urllib.error.HTTPError as e: + detail = e.read().decode(errors="replace")[:200] + raise SystemExit(f"{method} {path} failed: HTTP {e.code} {detail}") + + +def wait_for_api(timeout): + deadline = time.time() + timeout + while time.time() < deadline: + try: + return api("GET", "/v1/app/setup") + except SystemExit: + raise + except Exception: + time.sleep(2) + raise SystemExit("Beeper Desktop API did not come up") + + +def mailbox(): + m = imaplib.IMAP4_SSL(os.environ["BEEPER_IMAP_HOST"]) + m.login(os.environ["BEEPER_IMAP_USER"], os.environ["BEEPER_IMAP_PASSWORD"]) + m.select("INBOX", readonly=True) + return m + + +def newest_uid(m): + _, data = m.uid("search", None, "FROM", '"beeper"') + uids = data[0].split() + return int(uids[-1]) if uids else 0 + + +def extract_code(raw): + msg = email.message_from_bytes(raw) + parts = [msg.get("Subject", "")] + for p in msg.walk(): + if p.get_content_type() in ("text/plain", "text/html"): + payload = p.get_payload(decode=True) + if payload: + parts.append(payload.decode(p.get_content_charset() or "utf-8", "replace")) + m = CODE_RE.search("\n".join(parts)) + return m.group(1) if m else None + + +def wait_for_code(m, after_uid, timeout): + """Newest Beeper message with a UID above after_uid that holds a 6-digit code.""" + deadline = time.time() + timeout + while time.time() < deadline: + m.noop() + _, data = m.uid("search", None, "FROM", '"beeper"') + for uid in reversed([int(u) for u in data[0].split()]): + if uid <= after_uid: + break + _, msg = m.uid("fetch", str(uid), "(BODY.PEEK[])") + code = extract_code(msg[0][1]) + if code: + return code + time.sleep(3) + raise SystemExit("no Beeper sign-in code arrived in the mailbox") + + +def main(): + state = wait_for_api(int(os.environ.get("BEEPER_API_WAIT", "90")))["state"] + if state == "ready": + print("ready: already signed in") + return 0 + if state != "needs-login": + # Sign-in returns the session token exactly once; without it the + # recovery-key call cannot be authorised. Sign out in the app to retry. + print(f"state {state}: signed in but not unlocked; sign out in Beeper and re-run", file=sys.stderr) + return 3 + + m = mailbox() + baseline = newest_uid(m) + start = api("POST", "/v1/app/setup/start") + if "email" not in start.get("signInMethods", []): + raise SystemExit(f"email sign-in not offered: {start.get('signInMethods')}") + rid = start["setupRequestID"] + api("POST", "/v1/app/setup/email", {"setupRequestID": rid, "email": os.environ["BEEPER_EMAIL"]}) + code = wait_for_code(m, baseline, int(os.environ.get("BEEPER_CODE_WAIT", "120"))) + m.logout() + + done = api("POST", "/v1/app/setup/response", {"setupRequestID": rid, "response": code}) + token = (done.get("matrix") or {}).get("accessToken") + if not token: + raise SystemExit("sign-in did not return a token (account may not exist yet)") + + session = done["session"] + if session["state"] != "ready": + session = api("POST", "/v1/app/setup/verification/recovery-key", + {"recoveryKey": os.environ["BEEPER_RECOVERY_KEY"]}, token=token)["session"] + print(f"state: {session['state']}") + # needs-first-sync is the app catching up on chats, not a failure. + return 0 if session["state"] in ("ready", "needs-first-sync") else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/units/flatpak-packages.yml b/units/flatpak-packages.yml index e8d0c033..16fa13ad 100644 --- a/units/flatpak-packages.yml +++ b/units/flatpak-packages.yml @@ -2,6 +2,8 @@ flatpak_remotes: - name: flathub url: https://dl.flathub.org/repo/flathub.flatpakrepo + - name: flatpark + url: https://dl.flatpark.org/flatpark.flatpakrepo system_flatpaks: - io.typora.Typora @@ -49,3 +51,6 @@ system_flatpaks: - com.belmoussaoui.Obfuscate - org.nickvision.money +# Installed system-wide from the flatpark remote (extra-data apps not on Flathub). +flatpark_flatpaks: + - com.automattic.Beeper diff --git a/units/flatpak.sh b/units/flatpak.sh index 29a120a2..1befa557 100755 --- a/units/flatpak.sh +++ b/units/flatpak.sh @@ -30,6 +30,8 @@ load_config() { local kind package_list system_flatpaks "$flatpak_vars" || return 1 apps=("${PACKAGE_LIST[@]}") + package_list flatpark_flatpaks "$flatpak_vars" || return 1 + flatpark_apps=("${PACKAGE_LIST[@]}") kind="$(yq -r '.flatpak_remotes | tag' "$flatpak_vars")" || return 1 [ "$kind" = '!!seq' ] || { echo "$flatpak_vars: flatpak_remotes must be a list" >&2; return 1; } remotes="$(yq -r '.flatpak_remotes[] | .name + "\t" + .url' "$flatpak_vars")" || return 1 @@ -73,14 +75,15 @@ remove_unwanted() { [ "${#remove[@]}" = 0 ] || package_as_root flatpak uninstall --system -y "${remove[@]}" } install_missing() { - local app missing=() rc=0 - for app in "${apps[@]}"; do + local remote="$1" app missing=() rc=0 + shift + for app in "$@"; do if ! printf '%s\n' "$installed" | grep -Fxq "$app"; then missing+=("$app"); fi done [ "${#missing[@]}" -gt 0 ] || return 0 - if package_as_root flatpak install --system -y flathub "${missing[@]}"; then return 0; fi + if package_as_root flatpak install --system -y "$remote" "${missing[@]}"; then return 0; fi for app in "${missing[@]}"; do - package_as_root flatpak install --system -y flathub "$app" || rc=1 + package_as_root flatpak install --system -y "$remote" "$app" || rc=1 done return "$rc" } @@ -308,7 +311,8 @@ if command -v flatpak >/dev/null 2>&1; then step 'Flatpak remotes' add_remotes if get_installed; then step 'unwanted system Flatpaks' remove_unwanted - step 'system Flatpak applications' install_missing + step 'system Flatpak applications' install_missing flathub "${apps[@]}" + step 'FlatPark system applications' install_missing flatpark ${flatpark_apps[@]+"${flatpark_apps[@]}"} else step 'list system Flatpaks' false fi From b7fef39b736a8573c849787de10e6bbb551c026c Mon Sep 17 00:00:00 2001 From: James Reilly Date: Mon, 16 Nov 2026 12:24:42 +0000 Subject: [PATCH 62/62] omp: hold no usage reserve on Claude Max retry.usageReservePct 10 -> 0 and omp-route's Claude candidate reserve 10 -> 0 (OMP_ROUTE_CLAUDE_RESERVE_PCT), so sessions stay on Claude until its window is actually spent. Other omp-route candidates keep 10%. Co-Authored-By: Claude Opus 5.5 Signed-off-by: James Reilly --- units/omp/omp-route | 7 +++++-- units/omp/routing.yml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/units/omp/omp-route b/units/omp/omp-route index a4c79098..f761120e 100755 --- a/units/omp/omp-route +++ b/units/omp/omp-route @@ -12,6 +12,8 @@ set -uo pipefail RESERVE_PCT="${OMP_ROUTE_RESERVE_PCT:-10}" +# Claude Max runs to its limit: no reserve held back on it. +CLAUDE_RESERVE_PCT="${OMP_ROUTE_CLAUDE_RESERVE_PCT:-0}" CACHE="${XDG_CACHE_HOME:-$HOME/.cache}/omp-route/probe.json" CACHE_TTL="${OMP_ROUTE_CACHE_TTL:-300}" LAST_RESORT="meta/muse-spark-1.3-contributor" @@ -44,8 +46,9 @@ probe() { # Prints "ok", "skip", or "unknown" for one candidate. verdict() { - local json="$1" provider="$2" scope="$3" - jq -r --arg p "$provider" --arg s "$scope" --argjson r "$RESERVE_PCT" ' + local json="$1" provider="$2" scope="$3" reserve="$RESERVE_PCT" + [[ "$provider" == claude ]] && reserve="$CLAUDE_RESERVE_PCT" + jq -r --arg p "$provider" --arg s "$scope" --argjson r "$reserve" ' [.readings[] | select(.provider == $p)] as $rs | if ($rs | length) == 0 then "unknown" else $rs[0] as $x diff --git a/units/omp/routing.yml b/units/omp/routing.yml index 4338163a..d1b8915b 100644 --- a/units/omp/routing.yml +++ b/units/omp/routing.yml @@ -12,7 +12,7 @@ retry: fallbackOn: any fallbackRevertPolicy: cooldown-expiry # drift back to Claude once its window resets usageAwareFallback: true - usageReservePct: 10 + usageReservePct: 0 # no reserve: stay on Claude Max until its window is actually spent usageReservePolicy: auto # switch at the reserve without prompting fallbackChains: default: &chain