From d92574a591dd5b08848bb83bc39334dc4833188d Mon Sep 17 00:00:00 2001 From: architect Date: Sat, 17 Oct 2026 22:21:15 -0400 Subject: [PATCH] refactor: extract fleet-specific systemd services Signed-off-by: architect --- docs/src/SUMMARY.md | 1 + docs/src/architecture.md | 1 + docs/src/index.md | 2 +- docs/src/roles/systemd.md | 9 +-- docs/src/roles/systemd_hive_services.md | 33 ++++++++ roles/systemd/tasks/main.yml | 72 ----------------- .../files/system-mitigator.py | 0 roles/systemd_hive_services/handlers/main.yml | 5 ++ roles/systemd_hive_services/tasks/main.yml | 79 +++++++++++++++++++ .../templates/phone-apply.service.j2 | 0 .../templates/phone-apply.timer.j2 | 0 .../templates/system-mitigator.service.j2 | 0 site.yml | 3 + 13 files changed, 126 insertions(+), 79 deletions(-) create mode 100644 docs/src/roles/systemd_hive_services.md rename roles/{systemd => systemd_hive_services}/files/system-mitigator.py (100%) create mode 100644 roles/systemd_hive_services/handlers/main.yml create mode 100644 roles/systemd_hive_services/tasks/main.yml rename roles/{systemd => systemd_hive_services}/templates/phone-apply.service.j2 (100%) rename roles/{systemd => systemd_hive_services}/templates/phone-apply.timer.j2 (100%) rename roles/{systemd => systemd_hive_services}/templates/system-mitigator.service.j2 (100%) diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index e3f57f1b..7a157eb3 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -89,6 +89,7 @@ - [syncthing](roles/syncthing.md) - [systemd](roles/systemd.md) +- [systemd_hive_services](roles/systemd_hive_services.md) - [proxy (Caddy)](roles/proxy.md) - [tailscale_cert](roles/tailscale_cert.md) - [bst_dashboard](roles/bst_dashboard.md) diff --git a/docs/src/architecture.md b/docs/src/architecture.md index 940ef1ee..ee5892b1 100644 --- a/docs/src/architecture.md +++ b/docs/src/architecture.md @@ -83,6 +83,7 @@ Every role in `site.yml` carries one or more tags. Tags fall into two kinds: **p | Desktop apps | `pipewire_audio` | `desktop`, `audio`, `pipewire` | | Services | `syncthing` | `services`, `syncthing` | | Services | `systemd` | `services`, `systemd` | +| Services | `systemd_hive_services` | `services`, `systemd`, `systemd_hive_services` | | Services | `bst_dashboard` | `services`, `bst_dashboard` | | Services | `kirocrew` | `services`, `kirocrew` | | Services | `proxy` | `services`, `proxy`, `caddy` | diff --git a/docs/src/index.md b/docs/src/index.md index 20969ce2..f061205d 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -68,7 +68,7 @@ The `llm` group is dormant (commented out in `inventory.yml`). | 1 — System + packages + dotfiles | sudo, sshd, apk_packages, homebrew, termux_packages, bitwarden, shell_fonts, shell_dotfiles, omp, hive_ops, git, neovim | `system`, `packages`, `dotfiles` | | 2 — Secrets + auth | shell_atuin, shell_ai, ssh_keys, ssh_mesh, github, tailscale, kube, forgejo_registry | `secrets` | | 3 — Desktop apps | flatpak, bluefin_common, gnome, zen_browser (+ browser_fxa), pipewire_audio | `desktop` | -| 4 — Services | syncthing, systemd, bst_dashboard, kirocrew, proxy, tailscale_cert, server_hardening | `services` | +| 4 — Services | syncthing, systemd, systemd_hive_services, bst_dashboard, kirocrew, proxy, tailscale_cert, server_hardening | `services` | Only the `secrets`-tagged work needs Bitwarden. With the vault locked (or `--skip-tags secrets`), phases 1, 3 and 4 run normally and BW-touching tasks diff --git a/docs/src/roles/systemd.md b/docs/src/roles/systemd.md index 6afc4f16..eac5d290 100644 --- a/docs/src/roles/systemd.md +++ b/docs/src/roles/systemd.md @@ -27,12 +27,9 @@ If podman is installed: - Configures the podman socket for group access (mode 0660) - Enables and starts `podman.socket` -### system-mitigator (himachal only) - -A Python daemon that monitors for Lenovo EC firmware bugs: -- Detects false `BD_PROCHOT` signals that lock CPU cores to 400 MHz -- Watches for `systemd-journald` watchdog loops -- Sends desktop notifications when issues are detected +The host-specific fleet services run in `systemd_hive_services`, keeping +machine-specific monitoring and phone orchestration separate from the generic +systemd user services and conditional Podman socket setup. ## Notes diff --git a/docs/src/roles/systemd_hive_services.md b/docs/src/roles/systemd_hive_services.md new file mode 100644 index 00000000..4b428bec --- /dev/null +++ b/docs/src/roles/systemd_hive_services.md @@ -0,0 +1,33 @@ +# systemd_hive_services + +**Tags:** `services`, `systemd`, `systemd_hive_services`
+**Secrets needed:** No
+**Runs on:** All machines; service tasks target `himachal` + +Owns systemd user units that orchestrate fleet-specific work rather than the +common systemd services installed on every host. + +## Services Deployed + +### system-mitigator (`himachal` only) + +Deploys and enables the Lenovo EC and journald health monitor. The daemon: + +- Detects CPU cores stuck at their minimum frequency +- Watches for `systemd-journald` restart loops +- Sends desktop notifications when either condition needs attention + +### phone-apply timer (`himachal` only) + +Deploys a two-hourly timer that pulls the dotfiles checkout and applies the +no-secrets playbook to `termux` and `mumbai`. Those hosts are often asleep or +off the tailnet, so each remote apply is allowed to fail without failing the +local timer service. + +## Relationship to `systemd` + +The `systemd` role owns generic user services, the conditional Podman socket, +and automatic dotfiles updates. Both roles use the `systemd` tag and the +`skip_systemd` flag, so existing tagged and skipped applies keep the same +selection semantics while the fleet-specific units have an independent role +boundary. diff --git a/roles/systemd/tasks/main.yml b/roles/systemd/tasks/main.yml index 5157a560..7912d33e 100644 --- a/roles/systemd/tasks/main.yml +++ b/roles/systemd/tasks/main.yml @@ -132,75 +132,3 @@ scope: user enabled: true state: started - -- name: Deploy system-mitigator script (Lenovo 400MHz/Journald health) - ansible.builtin.copy: - src: system-mitigator.py - dest: "{{ ansible_facts['env']['HOME'] }}/.local/bin/system-mitigator.py" - mode: "0755" - when: inventory_hostname == 'himachal' - -- name: Deploy system-mitigator service template - ansible.builtin.template: - src: system-mitigator.service.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/system-mitigator.service" - mode: "0644" - when: inventory_hostname == 'himachal' - notify: Reload systemd user daemon - -- name: Enable and start system-mitigator daemon - ansible.builtin.systemd: - name: system-mitigator.service - scope: user - enabled: true - state: started - when: inventory_hostname == 'himachal' - -# Phone hosts (termux, mumbai) can't reliably self-apply on a schedule — -# Termux can't run Ansible at all (no Rust target for the cryptography -# dependency), and neither host is always reachable (asleep, app killed, -# off tailnet). himachal periodically pushes to them instead, tolerating -# unreachable as the expected common case rather than an error. -- name: Deploy phone-apply service template - ansible.builtin.template: - src: phone-apply.service.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/phone-apply.service" - mode: "0644" - when: inventory_hostname == 'himachal' - notify: Reload systemd user daemon - -- name: Deploy phone-apply timer template - ansible.builtin.template: - src: phone-apply.timer.j2 - dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/phone-apply.timer" - mode: "0644" - when: inventory_hostname == 'himachal' - notify: Reload systemd user daemon - -- name: Enable and start phone-apply timer - ansible.builtin.systemd: - name: phone-apply.timer - scope: user - enabled: true - state: started - daemon_reload: true - when: inventory_hostname == 'himachal' and phone_apply_enabled | default(true) | bool - -- name: Stop and disable phone-apply timer when automatic pushes are off - ansible.builtin.systemd: - name: phone-apply.timer - scope: user - enabled: false - state: stopped - when: inventory_hostname == 'himachal' and not (phone_apply_enabled | default(true) | bool) - failed_when: false - -- name: Remove disabled phone-apply units - ansible.builtin.file: - path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/{{ item }}" - state: absent - loop: - - phone-apply.service - - phone-apply.timer - when: inventory_hostname == 'himachal' and not (phone_apply_enabled | default(true) | bool) - notify: Reload systemd user daemon diff --git a/roles/systemd/files/system-mitigator.py b/roles/systemd_hive_services/files/system-mitigator.py similarity index 100% rename from roles/systemd/files/system-mitigator.py rename to roles/systemd_hive_services/files/system-mitigator.py diff --git a/roles/systemd_hive_services/handlers/main.yml b/roles/systemd_hive_services/handlers/main.yml new file mode 100644 index 00000000..17ca37a3 --- /dev/null +++ b/roles/systemd_hive_services/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload systemd hive user daemon + ansible.builtin.systemd: + daemon_reload: true + scope: user diff --git a/roles/systemd_hive_services/tasks/main.yml b/roles/systemd_hive_services/tasks/main.yml new file mode 100644 index 00000000..09667a8d --- /dev/null +++ b/roles/systemd_hive_services/tasks/main.yml @@ -0,0 +1,79 @@ +--- +# Host-specific fleet services that do not belong to the generic systemd role. +- name: Ensure systemd user directory exists + ansible.builtin.file: + path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user" + state: directory + mode: "0755" + +- name: Deploy system-mitigator script (Lenovo 400MHz/Journald health) + ansible.builtin.copy: + src: system-mitigator.py + dest: "{{ ansible_facts['env']['HOME'] }}/.local/bin/system-mitigator.py" + mode: "0755" + when: inventory_hostname == 'himachal' + +- name: Deploy system-mitigator service template + ansible.builtin.template: + src: system-mitigator.service.j2 + dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/system-mitigator.service" + mode: "0644" + when: inventory_hostname == 'himachal' + notify: Reload systemd hive user daemon + +- name: Enable and start system-mitigator daemon + ansible.builtin.systemd: + name: system-mitigator.service + scope: user + enabled: true + state: started + when: inventory_hostname == 'himachal' + +# Phone hosts (termux, mumbai) can't reliably self-apply on a schedule — +# Termux can't run Ansible at all (no Rust target for the cryptography +# dependency), and neither host is always reachable (asleep, app killed, +# off tailnet). himachal periodically pushes to them instead, tolerating +# unreachable as the expected common case rather than an error. +- name: Deploy phone-apply service template + ansible.builtin.template: + src: phone-apply.service.j2 + dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/phone-apply.service" + mode: "0644" + when: inventory_hostname == 'himachal' + notify: Reload systemd hive user daemon + +- name: Deploy phone-apply timer template + ansible.builtin.template: + src: phone-apply.timer.j2 + dest: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/phone-apply.timer" + mode: "0644" + when: inventory_hostname == 'himachal' + notify: Reload systemd hive user daemon + +- name: Enable and start phone-apply timer + ansible.builtin.systemd: + name: phone-apply.timer + scope: user + enabled: true + state: started + daemon_reload: true + when: inventory_hostname == 'himachal' and phone_apply_enabled | default(true) | bool + +- name: Stop and disable phone-apply timer when automatic pushes are off + ansible.builtin.systemd: + name: phone-apply.timer + scope: user + enabled: false + state: stopped + when: inventory_hostname == 'himachal' and not (phone_apply_enabled | default(true) | bool) + failed_when: false + +- name: Remove disabled phone-apply units + ansible.builtin.file: + path: "{{ ansible_facts['env']['HOME'] }}/.config/systemd/user/{{ item }}" + state: absent + loop: + - phone-apply.service + - phone-apply.timer + when: inventory_hostname == 'himachal' and not (phone_apply_enabled | default(true) | bool) + notify: Reload systemd hive user daemon diff --git a/roles/systemd/templates/phone-apply.service.j2 b/roles/systemd_hive_services/templates/phone-apply.service.j2 similarity index 100% rename from roles/systemd/templates/phone-apply.service.j2 rename to roles/systemd_hive_services/templates/phone-apply.service.j2 diff --git a/roles/systemd/templates/phone-apply.timer.j2 b/roles/systemd_hive_services/templates/phone-apply.timer.j2 similarity index 100% rename from roles/systemd/templates/phone-apply.timer.j2 rename to roles/systemd_hive_services/templates/phone-apply.timer.j2 diff --git a/roles/systemd/templates/system-mitigator.service.j2 b/roles/systemd_hive_services/templates/system-mitigator.service.j2 similarity index 100% rename from roles/systemd/templates/system-mitigator.service.j2 rename to roles/systemd_hive_services/templates/system-mitigator.service.j2 diff --git a/site.yml b/site.yml index 53ac15ca..703bbe1b 100644 --- a/site.yml +++ b/site.yml @@ -130,6 +130,9 @@ - role: systemd tags: [services, systemd] when: not skip_systemd | default(false) + - role: systemd_hive_services + tags: [services, systemd, systemd_hive_services] + when: not skip_systemd | default(false) - role: bst_dashboard tags: [services, bst_dashboard] - role: kirocrew