diff --git a/Justfile b/Justfile index 08c48444..f8079e04 100644 --- a/Justfile +++ b/Justfile @@ -537,6 +537,18 @@ seed-kube: seed-omp-broker: @{{ dotfiles_dir }}/scripts/bw-seed-omp-broker.sh +# Tailnet hygiene (scripts/tailnet.py): audit is read-only; the others dry-run unless `--apply` +tailnet-audit: + @{{ dotfiles_dir }}/scripts/tailnet.py audit + +# gc + rename + inventory alignment, e.g. `just tailnet-sync --apply` +tailnet-sync *args: + @{{ dotfiles_dir }}/scripts/tailnet.py sync {{ args }} + +# Store the Tailscale OAuth client secret (device gc/rename + join key) in Bitwarden +seed-tailnet-oauth: + @{{ dotfiles_dir }}/scripts/bw-seed-tailnet.sh + # Show all online fleet machines' doctor status in parallel doctor-fleet: #!/usr/bin/env bash diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index a7ea7562..89a5e9a7 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -6,6 +6,7 @@ - [Architecture](architecture.md) - [Units](units.md) +- [Tailnet hygiene](tailnet.md) - [Inventory & Groups](inventory.md) - [Secrets with Bitwarden](bitwarden.md) - [Onboarding a New Machine](onboarding.md) diff --git a/docs/src/roles/tailscale.md b/docs/src/roles/tailscale.md index e205f8fa..cdcc8133 100644 --- a/docs/src/roles/tailscale.md +++ b/docs/src/roles/tailscale.md @@ -11,7 +11,7 @@ Installs Tailscale, fetches a reusable auth key from BW, joins the network, conf 1. Configures `systemd-resolved` so MagicDNS works (link `/etc/resolv.conf` to the stub resolver, drop in `DNSStubListenerExtra=::1` for container DNS). 2. Sets `net.ipv4.conf.{all,default}.rp_filter=2` and IP forwarding on, so exit-node and subnet-router traffic actually routes. 3. If `tailscale status` reports connected → updates hostname, prefs, operator, systray, then exits the role. -4. If not connected and the vault is unlocked → fetches `tailscale-apikey` from BW, deletes any stale device with this hostname via the Tailscale API, then `tailscale up --authkey …`. +4. If not connected and the vault is unlocked → fetches `tailscale-apikey` (an OAuth client secret) from BW, runs `scripts/tailnet.py release ` to delete stale devices that still hold the hostname (see [Tailnet hygiene](../tailnet.md)), then `tailscale up --authkey …`. 5. If not connected and the vault is **locked** → emits a one-line warning and ends the role. Joining without a key would either fail noisily or, worse, succeed with an empty key. ## Failure modes @@ -20,7 +20,7 @@ Installs Tailscale, fetches a reusable auth key from BW, joins the network, conf |---|---|---| | Role aborts with "vault locked" warning, host is not on tailnet | First-time apply on this host without BW unlocked | Unlock BW (`bw unlock --raw | tee /tmp/bw_session > /dev/null`), then `just apply-tags tailscale` | | Tailscale connected but MagicDNS names don't resolve | `systemd-resolved` link to `/etc/resolv.conf` was overwritten by NetworkManager / DHCP client | The role re-applies it; just re-run `dots-apply`. To make it sticky, check `network-manager` settings (Bluefin does this correctly out of the box) | -| Duplicate device entries in admin console after rebuild | API-key device cleanup didn't run (vault locked at the time) | Manually delete in [tailscale admin console](https://login.tailscale.com/admin/machines) or unlock vault and re-apply | +| Duplicate device entries / `-1` names after rebuild | The BW credential is dead or lacks scopes, so `tailnet.py release` could not delete the old device | `just seed-tailnet-oauth`, then `just tailnet-sync --apply` | | Joins but no routes advertised | `tailscale_advertise_routes` not set in `host_vars` | Add it: `tailscale_advertise_routes: ""` | ## How to verify diff --git a/docs/src/tailnet.md b/docs/src/tailnet.md new file mode 100644 index 00000000..52345d1f --- /dev/null +++ b/docs/src/tailnet.md @@ -0,0 +1,31 @@ +# Tailnet hygiene + +`scripts/tailnet.py` keeps the tailnet and `inventory.yml` in agreement. Everything is a dry run unless `--apply` is given. + +| Command | What it does | +|---|---| +| `just tailnet-audit` | Read-only report; exit 1 when anything needs doing | +| `just tailnet-sync [--apply]` | GC, then rename, then inventory alignment, planned as one sequence | +| `scripts/tailnet.py gc [--apply] [--days N] [--k8s-days N]` | Delete duplicate devices and stale test VMs | +| `scripts/tailnet.py rename [--apply]` | Rename live devices to their intended name | +| `scripts/tailnet.py fix-inventory [--apply]` | Make `ansible_host` IPs and `tailnet_name` overrides follow the live devices | +| `scripts/tailnet.py release NAME` | Delete offline devices called `NAME` or `NAME-`; the `tailscale` role runs it before joining | + +## Rules + +- A device's tailnet name is the first MagicDNS label. An inventory host's **intended** name is its `tailnet_name` var, else the inventory name. A `tailnet_name` of the form `-` is the auto-suffix Tailscale adds when a stale device still holds ``, so it means "wants ``" and is removed once the rename has happened. +- **Duplicates:** several devices named `` / `-`: the live (else most recently seen) one is kept; older offline ones are deleted. +- **Test VMs:** offline longer than `--days` (default 14), named `gate-*`, `repro-*`, `tuna-v1-preflight-*` or `*-vm`, and not an inventory host. +- **k8s proxies** (`tag:k8s*`) are only deleted with `--k8s-days`. The home cluster is powered down and expected back, so its proxies are deliberately kept by default. +- Never touched: devices that are online, this machine, devices shared in from other tailnets, and the retired `vps` group. +- `fix-inventory` run alone reflects the live state, so it will not drop a `tailnet_name` override before the rename has actually happened. + +## Credential + +Reads fall back to the local `tailscale status --json`; writes need the API. The credential is a Tailscale **OAuth client** (scopes `devices:core` write and `auth_keys` write, tag `tag:fleet`) created at . Store it with `just seed-tailnet-oauth`: the secret is verified against Tailscale before it is written to Bitwarden item `tailscale-apikey`. `TS_OAUTH_CLIENT_SECRET` overrides the Bitwarden lookup. + +The same secret is the join auth key used by the [`tailscale`](roles/tailscale.md) role. The previous cleanup in that role treated it as a classic API key, got a 401 that `failed_when: false` hid, and never deleted anything; that is why re-joined machines came back as `himachal-1` and `kerala-1`. + +## Caveat when renaming + +After `rename --apply`, `himachal-1` stops resolving; use the new name. Do not rename while something is using the old MagicDNS name (an rsync or ssh session, for example). diff --git a/roles/tailscale/tasks/connect.yml b/roles/tailscale/tasks/connect.yml index ea38bd8a..d211b19c 100644 --- a/roles/tailscale/tasks/connect.yml +++ b/roles/tailscale/tasks/connect.yml @@ -23,39 +23,18 @@ PATH: "{{ ansible_env_path }}" BW_SESSION: "{{ bw_session }}" -- name: Remove old tailscale device with same hostname - block: - - name: Fetch tailscale device list - ansible.builtin.uri: - url: https://api.tailscale.com/api/v2/tailnet/-/devices - method: GET - user: "{{ tailscale_apikey.stdout }}" - password: "" - force_basic_auth: true - status_code: 200 - headers: - Content-Type: application/json - register: ts_devices - failed_when: false - no_log: true - - - name: Find old device matching our hostname - ansible.builtin.set_fact: - ts_old_device_id: >- - {{ (ts_devices.json.devices | default([]) - | selectattr('name', 'equalto', inventory_hostname + '.' + tailscale_domain) - | map(attribute='id') | first) | default('') }} - - - name: Delete old tailscale device - ansible.builtin.uri: - url: https://api.tailscale.com/api/v2/device/{{ ts_old_device_id }} - method: DELETE - user: "{{ tailscale_apikey.stdout }}" - password: "" - force_basic_auth: true - status_code: 200 - when: ts_old_device_id | length > 0 - no_log: true +# A stale device that still holds this hostname makes Tailscale join us as +# -1. scripts/tailnet.py exchanges the OAuth client secret for an API +# token (the old inline cleanup used it as a classic API key, got a silent 401, +# and never deleted anything). +- name: Remove stale tailscale devices with this hostname + ansible.builtin.command: python3 {{ playbook_dir }}/scripts/tailnet.py release {{ inventory_hostname }} + environment: + TS_OAUTH_CLIENT_SECRET: "{{ tailscale_apikey.stdout }}" + register: ts_release + changed_when: "'deleted' in ts_release.stdout" + failed_when: false + no_log: true when: tailscale_apikey.stdout | default('') | length > 0 - name: Join tailscale network diff --git a/scripts/bw-seed-tailnet.sh b/scripts/bw-seed-tailnet.sh new file mode 100755 index 00000000..a79661d3 --- /dev/null +++ b/scripts/bw-seed-tailnet.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Store the Tailscale OAuth client secret in Bitwarden item `tailscale-apikey`. +# Used by the tailscale role (as the join auth key) and scripts/tailnet.py (device gc/rename). +# +# Create the client first: https://login.tailscale.com/admin/settings/oauth +# scopes: devices:core (write), auth_keys (write); tag: tag:fleet +# Usage: ./scripts/bw-seed-tailnet.sh (prompts for the secret, no echo) +# printf %s "$SECRET" | ./scripts/bw-seed-tailnet.sh +# The secret is checked against Tailscale before anything is written. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/bw-item.sh" + +if [ -z "${BW_SESSION:-}" ]; then + BW_SESSION=$("$SCRIPT_DIR/bw-unlock.sh") + export BW_SESSION +fi + +if [ -t 0 ]; then + read -rs -p "Tailscale OAuth client secret (tskey-client-...): " secret; echo >&2 +else + secret=$(cat) +fi +secret="${secret//[[:space:]]/}" +case "$secret" in + tskey-client-*) ;; + *) echo "that is not an OAuth client secret (expected tskey-client-...)" >&2; exit 1 ;; +esac +cid=$(cut -d- -f3 <<<"$secret") + +code=$(curl -s -o /dev/null -w '%{http_code}' -X POST https://api.tailscale.com/api/v2/oauth/token \ + --data-urlencode "client_id=$cid" --data-urlencode "client_secret=$secret") +if [ "$code" != 200 ]; then + echo "Tailscale rejected the secret (HTTP $code); nothing written." >&2 + exit 1 +fi + +payload=$(bw get template item \ + | jq --arg tok "$secret" --arg cid "$cid" ' + . + { + name: "tailscale-apikey", + type: 1, + notes: "Tailscale OAuth client (devices:core, auth_keys; tag:fleet). Used as the join auth key by the tailscale role and by scripts/tailnet.py. Managed by scripts/bw-seed-tailnet.sh.", + login: {username: $cid, password: $tok} + }') + +bw_upsert_item "tailscale-apikey" "$payload" 1 +bw sync >/dev/null +echo "Done. Check the tailnet with: just tailnet-audit" diff --git a/scripts/inventory_parser.py b/scripts/inventory_parser.py index 0b7047fa..1a33fd05 100644 --- a/scripts/inventory_parser.py +++ b/scripts/inventory_parser.py @@ -47,6 +47,66 @@ def parse_inventory(text: str): return all_hosts, vps +def parse_host_vars(text: str): + """{host: {var: value}} for the 6-space `key: value` lines under each all.hosts entry. + + Values are stripped of inline comments and quotes; no PyYAML (see parse_inventory). + """ + hosts, section, host = {}, None, None + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + indent = len(line) - len(line.lstrip()) + if indent == 2: + section = line.strip().rstrip(":") + elif indent == 4 and section == "hosts": + host = line.strip().rstrip(":") + hosts[host] = {} + elif indent == 6 and section == "hosts" and host is not None: + m = re.match(r"([A-Za-z0-9_]+):\s*(.*?)\s*(?:#.*)?$", line.strip()) + if m and m.group(2): + hosts[host][m.group(1)] = m.group(2).strip("'\"") + return hosts + + +def set_host_var(text: str, host: str, key: str, value): + """Return `text` with `host`'s `key` set to `value` (or removed when value is None). + + Edits only the host's own block under all.hosts, keeping its inline comment. + A missing key is appended after the block's last variable line. Unknown hosts + leave the text unchanged. + """ + lines = text.split("\n") + start = None + for i, line in enumerate(lines): + if line.startswith(" children:"): + break + if re.fullmatch(rf" {re.escape(host)}:\s*(#.*)?", line): + start = i + break + if start is None: + return text + end = start + 1 + while end < len(lines) and (lines[end].startswith(" ") or not lines[end].strip() or lines[end].lstrip().startswith("#")): + if lines[end].startswith(" ") and not lines[end].startswith(" "): + break + end += 1 + last_var = start + for i in range(start + 1, end): + if lines[i].startswith(" ") and not lines[i].lstrip().startswith("#"): + last_var = i + m = re.match(rf"( {re.escape(key)}:\s*)([^#\n]*?)(\s*(?:#.*)?)$", lines[i]) + if m: + if value is None: + del lines[i] + else: + lines[i] = f"{m.group(1)}{value}{m.group(3) if m.group(3).strip() else ''}" + return "\n".join(lines) + if value is not None: + lines.insert(last_var + 1, f" {key}: {value}") + return "\n".join(lines) + + def register_host(name: str, mtype: str = "desktop", inv_path: str = "inventory.yml") -> bool: """Add `name` to all.hosts and to the `mtype` group. diff --git a/scripts/tailnet.py b/scripts/tailnet.py new file mode 100755 index 00000000..3d095faa --- /dev/null +++ b/scripts/tailnet.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""Tailnet hygiene: garbage-collect stale devices, rename them to the inventory, align inventory.yml. + + tailnet.py audit what is wrong (exit 1 when anything is) + tailnet.py gc [--apply] delete duplicates and stale test VMs + tailnet.py rename [--apply] rename live devices to their inventory (tailnet) name + tailnet.py fix-inventory [--apply] ansible_host IPs / tailnet_name overrides follow the live devices + tailnet.py sync [--apply] gc, then rename, then fix-inventory (planned as one sequence) + tailnet.py release NAME delete offline devices called NAME or NAME- (run before joining) + +Everything is a dry run unless --apply. Reads come from the Tailscale API when a credential is +available (TS_OAUTH_CLIENT_SECRET, or Bitwarden item `tailscale-apikey`), otherwise from the local +`tailscale status --json` (read-only: enough for audit and for planning). Writes need the API. + +Names: a device's tailnet name is its first MagicDNS label. An inventory host's *intended* name is +its `tailnet_name` var, else the inventory name; a `tailnet_name` of the form - is the +auto-suffix Tailscale adds when a stale device still holds , so it is treated as "wants ". +Retired hosts (the `vps` group) and devices shared in from other tailnets are never touched. +""" +import argparse +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timedelta, timezone + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from inventory_parser import parse_host_vars, parse_inventory, set_host_var # noqa: E402 + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +INVENTORY = os.environ.get("DOTFILES_INVENTORY", os.path.join(REPO, "inventory.yml")) +API = "https://api.tailscale.com/api/v2" +BW_ITEM = "tailscale-apikey" +TEST_VM = re.compile(r"(^gate-|^repro-|^tuna-v1-preflight-|.-vm$)") +IP100 = re.compile(r"^100\.\d+\.\d+\.\d+$") + + +class AuthError(Exception): + pass + + +# ── devices ──────────────────────────────────────────────────────────────── + +def _ts(value): + if not value or value.startswith("0001-"): + return None + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def devices_from_status(status, now): + """Normalise `tailscale status --json` (peers + self).""" + suffix = (status.get("MagicDNSSuffix") or (status.get("CurrentTailnet") or {}).get("MagicDNSSuffix") or "").strip(".") + out = [] + entries = [(p, False) for p in (status.get("Peer") or {}).values()] + if status.get("Self"): + entries.append((status["Self"], True)) + for p, is_self in entries: + dns = (p.get("DNSName") or "").rstrip(".") + name = (dns.split(".")[0] if dns else p.get("HostName", "")).lower() + online = bool(p.get("Online")) + seen = now if online else _ts(p.get("LastSeen")) + out.append({ + "id": p.get("ID") or "", + "name": name, + "ip": (p.get("TailscaleIPs") or [""])[0], + "tags": list(p.get("Tags") or []), + "online": online, + "seen": seen, + "self": is_self, + "foreign": bool(suffix and dns and not dns.endswith("." + suffix)), + }) + return out + + +def devices_from_api(payload, now, self_ids=()): + out = [] + for d in payload.get("devices", []): + addrs = d.get("addresses") or [""] + out.append({ + "id": d.get("nodeId") or d.get("id") or "", + "name": (d.get("name") or "").split(".")[0].lower(), + "ip": next((a for a in addrs if IP100.match(a)), addrs[0]), + "tags": list(d.get("tags") or []), + "online": bool(d.get("connectedToControl")), + "seen": now if d.get("connectedToControl") else _ts(d.get("lastSeen")), + "self": (d.get("nodeId") or d.get("id")) in self_ids, + "foreign": bool(d.get("isExternal")), + }) + return out + + +# ── API client ───────────────────────────────────────────────────────────── + +def _secret(): + secret = os.environ.get("TS_OAUTH_CLIENT_SECRET", "") + if secret: + return secret + session = os.environ.get("BW_SESSION", "") + if not session and os.path.exists("/tmp/bw_session"): + session = open("/tmp/bw_session").read().strip() + if not session: + return "" + env = {**os.environ, "BW_SESSION": session} + try: + r = subprocess.run(["bw", "get", "password", BW_ITEM], capture_output=True, text=True, env=env, timeout=60) + except (OSError, subprocess.TimeoutExpired): + return "" + return r.stdout.strip() if r.returncode == 0 else "" + + +class Api: + def __init__(self, secret): + self.secret = secret + self.auth = None + + def _request(self, method, path, data=None, form=None, auth=None): + headers = {"Authorization": auth} if auth else {} + body = None + if form is not None: + body = urllib.parse.urlencode(form).encode() + elif data is not None: + body = json.dumps(data).encode() + headers["Content-Type"] = "application/json" + req = urllib.request.Request(API + path, data=body, method=method, headers=headers) + try: + with urllib.request.urlopen(req, timeout=30) as r: + raw = r.read() + except urllib.error.HTTPError as e: + if e.code == 401: + raise AuthError(f"Tailscale rejected the credential (HTTP 401): {e.read().decode()[:120]}") from None + raise + return json.loads(raw) if raw else {} + + def _bearer(self): + if self.auth: + return self.auth + if self.secret.startswith("tskey-client-"): + cid = self.secret.split("-")[2] + tok = self._request("POST", "/oauth/token", form={"client_id": cid, "client_secret": self.secret}) + self.auth = "Bearer " + tok["access_token"] + else: # classic API key: HTTP basic with an empty password + import base64 + self.auth = "Basic " + base64.b64encode((self.secret + ":").encode()).decode() + return self.auth + + def list_devices(self): + return self._request("GET", "/tailnet/-/devices?fields=all", auth=self._bearer()) + + def delete(self, device_id): + self._request("DELETE", f"/device/{device_id}", auth=self._bearer()) + + def rename(self, device_id, name): + self._request("POST", f"/device/{device_id}/name", data={"name": name}, auth=self._bearer()) + + +AUTH_HELP = ( + "Create an OAuth client at https://login.tailscale.com/admin/settings/oauth (scopes: devices:core write, " + "auth_keys write; tag tag:fleet), then `just seed-tailnet-oauth` to store its secret in Bitwarden item " + f"`{BW_ITEM}` (or export TS_OAUTH_CLIENT_SECRET)." +) + + +def load_devices(source, now): + """(devices, api_or_None, source_label). Falls back from the API to local status for reads.""" + if source in ("auto", "api"): + secret = _secret() + if secret: + api = Api(secret) + try: + me = local_status().get("Self", {}).get("ID") + return devices_from_api(api.list_devices(), now, {me} if me else ()), api, "api" + except AuthError as e: + if source == "api": + raise + print(f"note: {e}\n {AUTH_HELP}\n falling back to local `tailscale status` (read-only)", file=sys.stderr) + elif source == "api": + raise AuthError("no Tailscale credential found. " + AUTH_HELP) + return devices_from_status(local_status(), now), None, "local" + + +def local_status(): + try: + return json.loads(subprocess.run(["tailscale", "status", "--json"], capture_output=True, text=True, timeout=10).stdout) + except (OSError, ValueError, subprocess.TimeoutExpired): + return {} + + +# ── planning (pure) ──────────────────────────────────────────────────────── + +def load_inventory(text): + all_hosts, vps = parse_inventory(text) + hv = parse_host_vars(text) + hosts = {} + for h in sorted(all_hosts - vps): + v = hv.get(h, {}) + tn = v.get("tailnet_name", "").lower() + intended = h.lower() if (not tn or re.fullmatch(rf"{re.escape(h.lower())}-\d+", tn)) else tn + hosts[h] = {"intended": intended, "override": tn, "ansible_host": v.get("ansible_host", "")} + return hosts + + +def _family(devs, name): + return [d for d in devs if not d["foreign"] and (d["name"] == name or re.fullmatch(rf"{re.escape(name)}-\d+", d["name"]))] + + +def _primary(family): + epoch = datetime.min.replace(tzinfo=timezone.utc) + return max(family, key=lambda d: (d["online"], d["seen"] or epoch)) + + +def plan_gc(devs, hosts, now, days=14, k8s_days=None): + """Delete actions: duplicates of an inventory host, stale test VMs, optionally stale k8s proxies.""" + epoch = datetime.min.replace(tzinfo=timezone.utc) + actions, doomed = [], set() + + def add(d, reason): + if d["id"] not in doomed and not d["self"] and not d["online"] and not d["foreign"]: + doomed.add(d["id"]) + actions.append({"op": "delete", "id": d["id"], "name": d["name"], "ip": d["ip"], "reason": reason}) + + inventory_names = {h["intended"] for h in hosts.values()} + for h in hosts.values(): + fam = _family(devs, h["intended"]) + if len(fam) > 1: + primary = _primary(fam) + for d in fam: + if d is not primary and (d["seen"] or epoch) < (primary["seen"] or epoch): + add(d, f"duplicate of {primary['name']}") + for d in devs: + old = d["seen"] is None or now - d["seen"] > timedelta(days=days) + if TEST_VM.search(d["name"]) and d["name"] not in inventory_names and old: + add(d, f"stale test VM (offline > {days}d)") + if k8s_days is not None and any(t.startswith("tag:k8s") for t in d["tags"]): + if d["seen"] is None or now - d["seen"] > timedelta(days=k8s_days): + add(d, f"stale k8s proxy (offline > {k8s_days}d)") + return actions + + +def apply_to_devices(devs, actions): + """The device list as it will be after `actions` (gc deletes, renames).""" + out = [dict(d) for d in devs] + for a in actions: + if a["op"] == "delete": + out = [d for d in out if d["id"] != a["id"]] + elif a["op"] == "rename": + for d in out: + if d["id"] == a["id"]: + d["name"] = a["to"] + return out + + +def plan_rename(devs, hosts): + actions, conflicts = [], [] + for host, h in hosts.items(): + fam = _family(devs, h["intended"]) + if not fam: + continue + primary = _primary(fam) + if primary["name"] == h["intended"]: + continue + holder = [d for d in devs if d["name"] == h["intended"] and not d["foreign"]] + if holder: + conflicts.append(f"{host}: wants {h['intended']} but it is held by {holder[0]['ip']} ({'online' if holder[0]['online'] else 'offline'})") + continue + actions.append({"op": "rename", "id": primary["id"], "from": primary["name"], "to": h["intended"], "host": host}) + return actions, conflicts + + +def plan_inventory(devs, hosts): + """Inventory edits that make inventory.yml agree with the devices as they are in `devs`.""" + actions, notes = [], [] + for host, h in hosts.items(): + fam = _family(devs, h["intended"]) + if not fam: + if h["ansible_host"] not in ("localhost", "127.0.0.1"): + notes.append(f"{host}: no tailnet device named {h['intended']}") + continue + primary = _primary(fam) + if primary["name"] != h["intended"]: + continue # not renamed yet: leave the override and IP alone + if IP100.match(h["ansible_host"]) and primary["ip"] and h["ansible_host"] != primary["ip"]: + actions.append({"op": "inventory", "host": host, "key": "ansible_host", "from": h["ansible_host"], "to": primary["ip"]}) + if h["override"] and h["intended"] == host.lower(): + actions.append({"op": "inventory", "host": host, "key": "tailnet_name", "from": h["override"], "to": None}) + return actions, notes + + +def build_plan(devs, hosts, now, days=14, k8s_days=None): + gc = plan_gc(devs, hosts, now, days, k8s_days) + after_gc = apply_to_devices(devs, gc) + renames, conflicts = plan_rename(after_gc, hosts) + after_rename = apply_to_devices(after_gc, renames) + inv, notes = plan_inventory(after_rename, hosts) + return {"gc": gc, "rename": renames, "inventory": inv, "conflicts": conflicts, "notes": notes} + + +def apply_inventory(text, actions): + for a in actions: + text = set_host_var(text, a["host"], a["key"], a["to"]) + return text + + +# ── CLI ──────────────────────────────────────────────────────────────────── + +def show(plan, kinds): + n = 0 + for a in plan["gc"] if "gc" in kinds else []: + print(f"delete {a['name']:<40} {a['ip']:<16} {a['reason']}") + n += 1 + for a in plan["rename"] if "rename" in kinds else []: + print(f"rename {a['from']} -> {a['to']} (inventory host {a['host']})") + n += 1 + for a in plan["inventory"] if "inventory" in kinds else []: + to = "(remove)" if a["to"] is None else a["to"] + print(f"inventory {a['host']}.{a['key']}: {a['from']} -> {to}") + n += 1 + for c in plan["conflicts"]: + print(f"conflict {c}") + for c in plan["notes"]: + print(f"note {c}") + return n + + +def run_writes(api, plan, kinds, apply_inv): + failed = 0 + for a in plan["gc"] if "gc" in kinds else []: + try: + api.delete(a["id"]) + print(f"deleted {a['name']}") + except (urllib.error.URLError, AuthError) as e: + print(f"FAILED delete {a['name']}: {e}") + failed += 1 + for a in plan["rename"] if "rename" in kinds else []: + try: + api.rename(a["id"], a["to"]) + print(f"renamed {a['from']} -> {a['to']}") + except (urllib.error.URLError, AuthError) as e: + print(f"FAILED rename {a['from']}: {e}") + failed += 1 + if "inventory" in kinds and plan["inventory"] and failed == 0: + apply_inv() + return failed + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("cmd", choices=["audit", "gc", "rename", "fix-inventory", "sync", "release"]) + ap.add_argument("name", nargs="?", help="device name (release)") + ap.add_argument("--apply", action="store_true", help="perform the changes (default: dry run)") + ap.add_argument("--days", type=int, default=14, help="test VMs offline longer than this are stale (default 14)") + ap.add_argument("--k8s-days", type=int, default=None, help="also delete tag:k8s proxies offline longer than this (off by default)") + ap.add_argument("--source", choices=["auto", "api", "local"], default="auto") + args = ap.parse_args(argv) + + now = datetime.now(timezone.utc) + try: + with open(INVENTORY) as f: + text = f.read() + hosts = load_inventory(text) + devs, api, label = load_devices("api" if (args.apply or args.cmd == "release") else args.source, now) + except AuthError as e: + print(f"error: {e}\n {AUTH_HELP}", file=sys.stderr) + return 2 + except urllib.error.URLError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + if args.cmd == "release": + if not args.name: + ap.error("release needs a device name") + name = args.name.lower() + victims = [d for d in _family(devs, name) if not d["self"] and not d["online"]] + for d in victims: + api.delete(d["id"]) + print(f"deleted {d['name']} ({d['ip']})") + return 0 + + plan = build_plan(devs, hosts, now, args.days, args.k8s_days) + kinds = {"audit": {"gc", "rename", "inventory"}, "sync": {"gc", "rename", "inventory"}, "gc": {"gc"}, + "rename": {"rename"}, "fix-inventory": {"inventory"}}[args.cmd] + if args.cmd == "gc": # gc alone: later steps are irrelevant + plan["conflicts"], plan["notes"] = [], [] + if args.cmd == "fix-inventory": # alone it must reflect the live state, not a simulated one + inv, notes = plan_inventory(devs, hosts) + plan = {"gc": [], "rename": [], "inventory": inv, "conflicts": [], "notes": notes} + if args.cmd == "rename": + renames, conflicts = plan_rename(devs, hosts) + plan = {"gc": [], "rename": renames, "inventory": [], "conflicts": conflicts, "notes": []} + + print(f"# {len(devs)} devices from {label}, {len(hosts)} inventory hosts", file=sys.stderr) + count = show(plan, kinds) + if not count and not plan["conflicts"] and not plan["notes"]: + print("tailnet and inventory agree") + if not args.apply or args.cmd == "audit": + return 1 if count or plan["conflicts"] or plan["notes"] else 0 + + def write_inventory(): + with open(INVENTORY, "w") as f: + f.write(apply_inventory(text, plan["inventory"])) + print(f"updated {INVENTORY}") + + if args.cmd == "fix-inventory": + write_inventory() + return 0 + if args.cmd == "sync" and plan["inventory"] and (plan["gc"] or plan["rename"]): + # Inventory edits assume the renames happened; re-plan from the live state afterwards. + failed = run_writes(api, plan, {"gc", "rename"}, lambda: None) + if failed: + return 1 + devs, _, _ = load_devices("api", datetime.now(timezone.utc)) + inv, _ = plan_inventory(devs, hosts) + plan["inventory"] = inv + if inv: + write_inventory() + return 0 + return 1 if run_writes(api, plan, kinds, write_inventory) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_tailnet.py b/tests/test_tailnet.py new file mode 100644 index 00000000..5cd2b61a --- /dev/null +++ b/tests/test_tailnet.py @@ -0,0 +1,127 @@ +import sys +from datetime import datetime, timedelta, timezone + +import pytest + +from conftest import REPO + +sys.path.insert(0, str(REPO / "scripts")) +import tailnet # noqa: E402 +from inventory_parser import parse_host_vars, set_host_var # noqa: E402 + +NOW = datetime(2026, 10, 25, 12, 0, tzinfo=timezone.utc) + +INVENTORY = """\ +--- +all: + hosts: + himachal: + ansible_host: 100.73.3.51 + ansible_connection: ssh + kerala: + ansible_host: 100.96.124.92 # keep this comment + ansible_user: james + tailnet_name: kerala-1 + goa: + ansible_host: localhost + ansible_connection: local + matrix: + ansible_host: 100.73.19.81 + tailnet_name: reilly-asia-matrix + children: + desktop: + hosts: + himachal: + kerala: + vps: + hosts: + matrix: +""" + + +def dev(name, ip, online=False, days_ago=1, tags=(), self_=False, foreign=False): + return {"id": f"id-{name}-{ip}", "name": name, "ip": ip, "tags": list(tags), "online": online, + "seen": NOW if online else NOW - timedelta(days=days_ago), "self": self_, "foreign": foreign} + + +def plan(devs, **kw): + return tailnet.build_plan(devs, tailnet.load_inventory(INVENTORY), NOW, **kw) + + +def test_stale_duplicate_is_deleted_and_live_device_takes_the_name(): + p = plan([dev("himachal", "100.73.3.51", days_ago=24), dev("himachal-1", "100.79.182.70", online=True)]) + assert [a["name"] for a in p["gc"]] == ["himachal"] + assert [(a["from"], a["to"]) for a in p["rename"]] == [("himachal-1", "himachal")] + # inventory follows the live device's address + assert {"op": "inventory", "host": "himachal", "key": "ansible_host", "from": "100.73.3.51", "to": "100.79.182.70"} in p["inventory"] + + +def test_auto_suffix_override_means_the_base_name_and_is_dropped_once_renamed(): + devs = [dev("kerala", "100.67.142.116", days_ago=57), dev("kerala-1", "100.96.124.92", days_ago=1)] + p = plan(devs) + assert [a["name"] for a in p["gc"]] == ["kerala"] + assert [(a["from"], a["to"]) for a in p["rename"]] == [("kerala-1", "kerala")] + assert {"op": "inventory", "host": "kerala", "key": "tailnet_name", "from": "kerala-1", "to": None} in p["inventory"] + # standalone inventory fix must not drop the override before the rename really happened + inv, _ = tailnet.plan_inventory(devs, tailnet.load_inventory(INVENTORY)) + assert inv == [] + + +def test_the_newer_device_is_never_the_one_deleted(): + # the suffixed device is older than the plain-named one: keep the newer, delete the older + p = plan([dev("kerala", "100.1.1.1", days_ago=1), dev("kerala-1", "100.2.2.2", days_ago=30)]) + assert [a["name"] for a in p["gc"]] == ["kerala-1"] + assert p["rename"] == [] + + +def test_online_self_and_foreign_devices_are_never_deleted(): + devs = [dev("himachal", "100.1.1.1", online=True), dev("himachal-1", "100.1.1.2", online=True), + dev("gate-x-vm", "100.1.1.3", days_ago=90, self_=True), dev("gate-y-vm", "100.1.1.4", days_ago=90, foreign=True)] + assert plan(devs)["gc"] == [] + + +def test_test_vms_are_deleted_only_when_stale_and_k8s_proxies_only_on_request(): + devs = [dev("gate-old-vm", "100.1.1.1", days_ago=60, tags=["tag:k8s"]), + dev("gate-new-vm", "100.1.1.2", days_ago=1, tags=["tag:k8s"]), + dev("grafana", "100.1.1.3", days_ago=100, tags=["tag:k8s"])] + assert [a["name"] for a in plan(devs)["gc"]] == ["gate-old-vm"] + assert {a["name"] for a in plan(devs, k8s_days=90)["gc"]} == {"gate-old-vm", "grafana"} + + +def test_two_live_devices_for_one_host_are_left_alone(): + p = plan([dev("kerala", "100.1.1.1", online=True), dev("kerala-1", "100.1.1.2", online=True)]) + assert p["gc"] == [] and p["rename"] == [] + + +def test_retired_vps_hosts_are_ignored(): + p = plan([dev("reilly-asia-matrix", "100.73.19.81", days_ago=50)]) + assert p["gc"] == [] and p["rename"] == [] and p["inventory"] == [] + assert not any("matrix" in n for n in p["notes"]) + + +def test_inventory_edits_keep_comments_and_other_hosts(): + out = tailnet.apply_inventory(INVENTORY, [ + {"host": "kerala", "key": "tailnet_name", "to": None}, + {"host": "kerala", "key": "ansible_host", "to": "100.9.9.9"}, + ]) + assert "tailnet_name: kerala-1" not in out + assert "ansible_host: 100.9.9.9 # keep this comment" in out + assert parse_host_vars(out)["himachal"] == parse_host_vars(INVENTORY)["himachal"] + assert "reilly-asia-matrix" in out + + +def test_set_host_var_appends_a_missing_key_inside_the_block(): + out = set_host_var(INVENTORY, "goa", "tailnet_name", "goa-x") + assert parse_host_vars(out)["goa"]["tailnet_name"] == "goa-x" + assert parse_host_vars(out)["kerala"] == parse_host_vars(INVENTORY)["kerala"] + assert set_host_var(INVENTORY, "nope", "k", "v") == INVENTORY + + +def test_status_normalisation_marks_shared_in_devices_foreign(): + status = {"MagicDNSSuffix": "tn.ts.net", "Self": {"ID": "n1", "DNSName": "me.tn.ts.net.", "TailscaleIPs": ["100.1.1.1"], "Online": True}, + "Peer": {"a": {"ID": "n2", "DNSName": "chuppvm.other.ts.net.", "TailscaleIPs": ["100.1.1.2"], "Online": False, "LastSeen": "2026-04-24T00:00:00Z"}, + "b": {"ID": "n3", "DNSName": "dilli.tn.ts.net.", "TailscaleIPs": ["100.1.1.3"], "Online": False, "LastSeen": "0001-01-01T00:00:00Z", "Tags": ["tag:fleet"]}}} + d = {x["name"]: x for x in tailnet.devices_from_status(status, NOW)} + assert d["me"]["self"] and d["me"]["online"] + assert d["chuppvm"]["foreign"] and not d["dilli"]["foreign"] + assert d["dilli"]["seen"] is None and d["dilli"]["tags"] == ["tag:fleet"]