From 196f3787a98752c26640947af01efb5707a3b501 Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Fri, 18 Sep 2026 21:17:11 +0000 Subject: [PATCH 1/2] feat(countme): implement weekly countme client reporting to countme.projectbluefin.io Implement weekly client reporting to countme.projectbluefin.io following Project Bluefin ADR 0006. - Add /usr/libexec/utah-countme script to compute weekly cohort bucket, extract image metadata (repo, tag, flavor, arch), and send empty GET ping. - Add systemd units utah-countme.service and utah-countme.timer with DynamicUser=yes and StateDirectory=utah-countme. - Enable utah-countme.timer in 85-utah-desktop.preset and configure-services.sh. - Add utah-countme.timer to contracts/bluefin-desktop.toml. - Support opt-out via /etc/projectbluefin/countme/disabled and timer masking. - Add comprehensive unittest suite in tests/test_countme.py. - Document telemetry query params, logging policy, resilience, and opt-out in docs/skills/desktop-contract.md and regenerate skill catalog. Closes projectbluefin/utah#49 Assisted-by: Gemini 3.8 Flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mrbobbytables --- Justfile | 4 + contracts/bluefin-desktop.toml | 1 + docs/skills/desktop-contract.md | 42 ++- docs/skills/index.json | 14 +- docs/skills/index.md | 4 +- scripts/configure-services.sh | 1 + .../system-preset/85-utah-desktop.preset | 1 + .../lib/systemd/system/utah-countme.service | 16 + .../usr/lib/systemd/system/utah-countme.timer | 16 + system_files/shared/usr/libexec/utah-countme | 134 +++++++++ tests/test_countme.py | 275 ++++++++++++++++++ 11 files changed, 490 insertions(+), 18 deletions(-) create mode 100644 system_files/shared/usr/lib/systemd/system/utah-countme.service create mode 100644 system_files/shared/usr/lib/systemd/system/utah-countme.timer create mode 100755 system_files/shared/usr/libexec/utah-countme create mode 100644 tests/test_countme.py diff --git a/Justfile b/Justfile index 28eda936..08a7b52e 100644 --- a/Justfile +++ b/Justfile @@ -34,6 +34,10 @@ check: test -f system_files/shared/usr/lib/systemd/system/bootc-unified-storage.service.d/10-utah-local-test.conf grep -q 'enable gdm.service' system_files/shared/usr/lib/systemd/system-preset/85-utah-desktop.preset grep -q 'enable ublue-system-setup.service' system_files/shared/usr/lib/systemd/system-preset/85-utah-desktop.preset + grep -q 'enable utah-countme.timer' system_files/shared/usr/lib/systemd/system-preset/85-utah-desktop.preset + test -f system_files/shared/usr/lib/systemd/system/utah-countme.service + test -f system_files/shared/usr/lib/systemd/system/utah-countme.timer + test -f system_files/shared/usr/libexec/utah-countme test -f scripts/configure-services.sh test -f scripts/configure-branding.sh test -f scripts/verify-desktop-contract.py diff --git a/contracts/bluefin-desktop.toml b/contracts/bluefin-desktop.toml index 2c326821..627c4b36 100644 --- a/contracts/bluefin-desktop.toml +++ b/contracts/bluefin-desktop.toml @@ -137,5 +137,6 @@ enabled = [ "flatpak-preinstall.service", "gdm.service", "ublue-system-setup.service", + "utah-countme.timer", "uupd.timer", ] diff --git a/docs/skills/desktop-contract.md b/docs/skills/desktop-contract.md index a27f7166..0361fb4c 100644 --- a/docs/skills/desktop-contract.md +++ b/docs/skills/desktop-contract.md @@ -1,20 +1,20 @@ --- name: desktop-contract -version: "1.0" -last_updated: "2026-09-16" +version: "1.1" +last_updated: "2026-09-18" id: desktop-contract -one_line_purpose: Maintain Utah identity, Bluefin desktop defaults, and first-boot Flatpak policy. +one_line_purpose: Maintain Utah identity, Bluefin desktop defaults, telemetry, and first-boot Flatpak policy. entry_point: docs/skills/desktop-contract.md category: contracts mcp_compliance_level: partial optimization_status: draft status: active dependencies: [] -tags: [desktop, branding, gnome, flatpak] +tags: [desktop, branding, gnome, flatpak, telemetry, countme] description: >- The runtime desktop contract in contracts/bluefin-desktop.toml and its in-image verifiers. Use when changing branding, os-release, service - presets, GNOME extensions, or first-boot Flatpak behavior. + presets, GNOME extensions, countme telemetry, or first-boot Flatpak behavior. metadata: type: policy --- @@ -57,7 +57,7 @@ The TOML's sections are the contract's table of contents: - **`[services]`** — systemd units the preset must enable: `gdm.service`, `ublue-system-setup.service`, `flatpak-preinstall.service`, `flatpak-nuke-fedora.service`, `brew-setup.service`, `dconf-update.service`, - `bootc-unified-storage.service`, `uupd.timer`. + `bootc-unified-storage.service`, `utah-countme.timer`, `uupd.timer`. ## GNOME extensions are pinned submodules @@ -94,16 +94,38 @@ on `GjsPrivate.DBusImplementation`, gracefully degrading to an inert portal on G Hummingbird defaults to a server preset and disables unlisted services, so the desktop policy is applied explicitly. `scripts/configure-services.sh` mirrors bluefin-lts's `40-services.sh`: it applies the desktop presets, -enables GDM, firmware updates, Tailscale, uupd, user setup and resolved, -configures authselect, and removes the extension build toolchain before -cleanup (Containerfile RUN comment; originated in `docs/building.md`'s former -design section and now lives in this skill). +enables GDM, firmware updates, Tailscale, uupd, user setup, resolved, and +the weekly `utah-countme.timer` client reporting, configures authselect, and +removes the extension build toolchain before cleanup (Containerfile RUN comment; +originated in `docs/building.md`'s former design section and now lives in this +skill). Hummingbird's base does not include `systemd-resolved` by default; it is listed under `[services]` in `packages/utah.toml` and configured in `scripts/configure-services.sh`, which also disables `PrivateTmp` on `systemd-resolved.service` for bootc early-boot DNS resolution. +## Countme telemetry + +Utah reports weekly anonymous telemetry to `countme.projectbluefin.io` matching +Project Bluefin ADR 0006: +- **Client implementation:** `/usr/libexec/utah-countme` executed via + `utah-countme.service` and scheduled by `utah-countme.timer`. +- **Systemd isolation:** Runs as `DynamicUser=yes` with + `StateDirectory=utah-countme` storing local installation `epoch` and + `lastrun` timestamps. +- **Reporting parameters & query string:** An empty HTTP GET request is sent to + `https://countme.projectbluefin.io/metalink?repo=${REPO}&tag=${IMAGE_TAG}&flavor=${IMAGE_FLAVOR}&arch=${ARCH}&countme=${BUCKET}` + - `repo`: image name (`utah` or custom variant) + - `tag`: OS image version / release tag + - `flavor`: desktop flavor (`main`, `nvidia`, `gaming`, etc.) + - `arch`: system architecture (`x86_64`) + - `countme`: installation age cohort bucket (1: first week, 2: 2–4 weeks, 3: 5–24 weeks, 4: >24 weeks) +- **Data & logging policy:** No personal information or hardware identifiers are transmitted; requests contain no HTTP body. As with all HTTP requests, standard server connection metadata (including client IP address) is received by the endpoint for aggregate deduplication and cohort counting. +- **Network & failure resilience:** Requests use an explicit `--max-time 10` timeout. If the server is unreachable or offline, the script exits cleanly without updating `lastrun`, allowing the ping to be retried on the next schedule without skipping a weekly reporting window. +- **Persistent state integrity:** The install epoch is stored in `/var/lib/utah-countme/epoch`. If this file exists but cannot be read or is corrupted, execution degrades to a no-op (no ping sent and no epoch reset) to avoid resetting systems into bucket 1. +- **Opt-out:** Honored whenever `/etc/projectbluefin/countme/disabled` exists, or by masking the timer (`systemctl mask utah-countme.timer`). + ## The verifiers run twice The same verifier runs in the Containerfile and on demand, so a local image diff --git a/docs/skills/index.json b/docs/skills/index.json index 02838e2d..f8923c39 100644 --- a/docs/skills/index.json +++ b/docs/skills/index.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-16", + "generated_at": "2026-09-18", "schema_version": "1.0", "skills": [ { @@ -41,7 +41,7 @@ { "id": "desktop-contract", "name": "desktop-contract", - "one_line_purpose": "Maintain Utah identity, Bluefin desktop defaults, and first-boot Flatpak policy.", + "one_line_purpose": "Maintain Utah identity, Bluefin desktop defaults, telemetry, and first-boot Flatpak policy.", "entry_point": "docs/skills/desktop-contract.md", "category": "contracts", "status": "active", @@ -49,11 +49,13 @@ "desktop", "branding", "gnome", - "flatpak" + "flatpak", + "telemetry", + "countme" ], - "description": "The runtime desktop contract in contracts/bluefin-desktop.toml and its in-image verifiers. Use when changing branding, os-release, service presets, GNOME extensions, or first-boot Flatpak behavior.", - "version": "1.0", - "last_updated": "2026-09-16", + "description": "The runtime desktop contract in contracts/bluefin-desktop.toml and its in-image verifiers. Use when changing branding, os-release, service presets, GNOME extensions, countme telemetry, or first-boot Flatpak behavior.", + "version": "1.1", + "last_updated": "2026-09-18", "doc_type": "policy" }, { diff --git a/docs/skills/index.md b/docs/skills/index.md index 688ac521..a77c69af 100644 --- a/docs/skills/index.md +++ b/docs/skills/index.md @@ -3,13 +3,13 @@ This file is a human-readable mirror of `index.json`. Both are generated by `scripts/generate_skill_index.py` — do not hand-edit either file. -Generated: 2026-09-16 · schema 1.0 · 8 skills +Generated: 2026-09-18 · schema 1.0 · 8 skills | id | category | status | one-line purpose | |---|---|---|---| | [ci-workflows](ci-workflows.md) | ci-ops | active | Navigate Utah's build, promote, and sync workflow topology. | | [containerfile](containerfile.md) | image-build | active | Edit the Containerfile without regressing layer count or cache hits. | -| [desktop-contract](desktop-contract.md) | contracts | active | Maintain Utah identity, Bluefin desktop defaults, and first-boot Flatpak policy. | +| [desktop-contract](desktop-contract.md) | contracts | active | Maintain Utah identity, Bluefin desktop defaults, telemetry, and first-boot Flatpak policy. | | [flavors](flavors.md) | image-build | active | Add, remove, or retire an image flavor safely. | | [kernel-cache](kernel-cache.md) | image-build | active | Understand and rebuild the OGC kernel and NVIDIA module cache image. | | [local-testing](local-testing.md) | testing | active | Build, install, and boot Utah locally in a VM or live ISO. | diff --git a/scripts/configure-services.sh b/scripts/configure-services.sh index 1eb2fb71..a27f63d3 100644 --- a/scripts/configure-services.sh +++ b/scripts/configure-services.sh @@ -65,6 +65,7 @@ enable_unit fwupd.service enable_unit fwupd-refresh.timer enable_unit dconf-update.service enable_unit tailscaled.service +enable_unit utah-countme.timer enable_unit uupd.timer enable_unit ublue-system-setup.service enable_unit systemd-resolved.service diff --git a/system_files/shared/usr/lib/systemd/system-preset/85-utah-desktop.preset b/system_files/shared/usr/lib/systemd/system-preset/85-utah-desktop.preset index ad4db080..dccec874 100644 --- a/system_files/shared/usr/lib/systemd/system-preset/85-utah-desktop.preset +++ b/system_files/shared/usr/lib/systemd/system-preset/85-utah-desktop.preset @@ -5,6 +5,7 @@ enable firewalld.service enable fwupd.service enable fwupd-refresh.timer enable tailscaled.service +enable utah-countme.timer enable uupd.timer enable ublue-system-setup.service enable systemd-resolved.service diff --git a/system_files/shared/usr/lib/systemd/system/utah-countme.service b/system_files/shared/usr/lib/systemd/system/utah-countme.service new file mode 100644 index 00000000..be9db5f3 --- /dev/null +++ b/system_files/shared/usr/lib/systemd/system/utah-countme.service @@ -0,0 +1,16 @@ +[Unit] +Description=Weekly Utah Count Me telemetry reporting +Documentation=https://docs.projectbluefin.io +Wants=network-online.target +After=network-online.target +ConditionPathExists=/run/ostree-booted +ConditionPathExists=/usr/share/ublue-os/image-info.json +ConditionPathExists=!/etc/projectbluefin/countme/disabled + +[Service] +Type=oneshot +DynamicUser=yes +StateDirectory=utah-countme +ExecStart=/usr/libexec/utah-countme +StandardOutput=journal +StandardError=journal diff --git a/system_files/shared/usr/lib/systemd/system/utah-countme.timer b/system_files/shared/usr/lib/systemd/system/utah-countme.timer new file mode 100644 index 00000000..660dd7bf --- /dev/null +++ b/system_files/shared/usr/lib/systemd/system/utah-countme.timer @@ -0,0 +1,16 @@ +[Unit] +Description=Weekly Utah Count Me telemetry timer +Documentation=https://docs.projectbluefin.io +ConditionPathExists=/run/ostree-booted +ConditionPathExists=/usr/share/ublue-os/image-info.json +ConditionPathExists=!/etc/projectbluefin/countme/disabled + +[Timer] +OnBootSec=10m +OnCalendar=weekly +Persistent=true +AccuracySec=1h +RandomizedDelaySec=12h + +[Install] +WantedBy=timers.target diff --git a/system_files/shared/usr/libexec/utah-countme b/system_files/shared/usr/libexec/utah-countme new file mode 100755 index 00000000..945153f3 --- /dev/null +++ b/system_files/shared/usr/libexec/utah-countme @@ -0,0 +1,134 @@ +#!/usr/bin/bash +# /usr/libexec/utah-countme +# Weekly user-counting client for Utah (bootc) systems +set -euo pipefail + +DISABLED_FILE="${DISABLED_FILE:-/etc/projectbluefin/countme/disabled}" +IMAGE_INFO="${IMAGE_INFO:-/usr/share/ublue-os/image-info.json}" +STATE_DIR="${STATE_DIRECTORY:-/var/lib/utah-countme}" +OS_RELEASE_FILE="${OS_RELEASE_FILE:-/etc/os-release}" +WEEK=604800 + +# Allow users to opt out without disabling the timer. +[ -e "$DISABLED_FILE" ] && exit 0 + +if ! command -v jq >/dev/null 2>&1; then + echo "utah-countme: jq is unavailable; skipping telemetry" >&2 + exit 0 +fi + +if ! command -v curl >/dev/null 2>&1; then + echo "utah-countme: curl is unavailable; skipping telemetry" >&2 + exit 0 +fi + +NOW=$(date +%s) + +# Ensure state directory exists +if [ ! -d "$STATE_DIR" ]; then + mkdir -p "$STATE_DIR" +fi + +EPOCH_FILE="${STATE_DIR}/epoch" +LASTRUN_FILE="${STATE_DIR}/lastrun" + +# 1. Read or initialize epoch file +if [ ! -f "$EPOCH_FILE" ]; then + echo "$NOW" > "$EPOCH_FILE" || { + echo "utah-countme: cannot initialize epoch file; skipping telemetry" >&2 + exit 0 + } +fi + +if ! EPOCH=$(cat "$EPOCH_FILE" 2>/dev/null) || [ -z "$EPOCH" ] || ! [[ "$EPOCH" =~ ^[0-9]+$ ]]; then + echo "utah-countme: cannot read epoch file; skipping telemetry" >&2 + exit 0 +fi + +# 2. Throttling: only run once per 7-day window +if [ -f "$LASTRUN_FILE" ]; then + LASTRUN=$(cat "$LASTRUN_FILE" 2>/dev/null || echo 0) + if [[ "$LASTRUN" =~ ^[0-9]+$ ]] && (( NOW - LASTRUN < WEEK )); then + exit 0 + fi +fi + +# 3. Compute Fedora-style week bucket +# 1: first week, 2: 2–4w, 3: 5–24w, 4: >24w +if (( NOW < EPOCH )); then + WEEKS=1 +else + WEEKS=$(( (NOW - EPOCH) / WEEK + 1 )) +fi + +if (( WEEKS <= 1 )); then + BUCKET=1 +elif (( WEEKS <= 4 )); then + BUCKET=2 +elif (( WEEKS <= 24 )); then + BUCKET=3 +else + BUCKET=4 +fi + +# 4. Extract image metadata +REPO="utah" +IMAGE_TAG="unknown" +IMAGE_FLAVOR="unknown" + +if [ -f "$IMAGE_INFO" ]; then + INFO_NAME=$(jq -r '."image-name" // empty' "$IMAGE_INFO" 2>/dev/null || true) + if [ -n "$INFO_NAME" ]; then + REPO="$INFO_NAME" + fi + INFO_TAG=$(jq -r '."image-tag" // empty' "$IMAGE_INFO" 2>/dev/null || true) + if [ -n "$INFO_TAG" ]; then + IMAGE_TAG="$INFO_TAG" + fi + INFO_FLAVOR=$(jq -r '."image-flavor" // empty' "$IMAGE_INFO" 2>/dev/null || true) + if [ -n "$INFO_FLAVOR" ]; then + IMAGE_FLAVOR="$INFO_FLAVOR" + fi +fi + +# Fallback to os-release if tag or flavor are still unknown +if [ -f "$OS_RELEASE_FILE" ]; then + if [ "$IMAGE_TAG" = "unknown" ]; then + OS_VERSION=$(grep -E '^OSTREE_VERSION=' "$OS_RELEASE_FILE" 2>/dev/null | head -n1 | cut -d= -f2 | tr -d '"' || true) + if [ -z "$OS_VERSION" ]; then + OS_VERSION=$(grep -E '^IMAGE_VERSION=' "$OS_RELEASE_FILE" 2>/dev/null | head -n1 | cut -d= -f2 | tr -d '"' || true) + fi + if [ -z "$OS_VERSION" ]; then + OS_VERSION=$(grep -E '^VERSION_ID=' "$OS_RELEASE_FILE" 2>/dev/null | head -n1 | cut -d= -f2 | tr -d '"' || true) + fi + if [ -n "$OS_VERSION" ]; then + IMAGE_TAG="$OS_VERSION" + fi + fi + if [ "$IMAGE_FLAVOR" = "unknown" ]; then + OS_FLAVOR=$(grep -E '^IMAGE_FLAVOR=' "$OS_RELEASE_FILE" 2>/dev/null | cut -d= -f2 | tr -d '"' || true) + if [ -n "$OS_FLAVOR" ]; then + IMAGE_FLAVOR="$OS_FLAVOR" + fi + fi +fi + +# Default flavor to "main" if still unspecified +if [ "$IMAGE_FLAVOR" = "unknown" ]; then + IMAGE_FLAVOR="main" +fi + +ARCH=$(uname -m) +OS_VERSION_ID="unknown" +if [ -f "$OS_RELEASE_FILE" ]; then + OS_VERSION_ID=$(grep -E '^VERSION_ID=' "$OS_RELEASE_FILE" 2>/dev/null | cut -d= -f2 | tr -d '"' || echo "unknown") +fi + +# 5. Formulate request +USER_AGENT="utah-countme (${REPO} ${OS_VERSION_ID}; ${IMAGE_FLAVOR}; Linux.${ARCH})" +URL="https://countme.projectbluefin.io/metalink?repo=${REPO}&tag=${IMAGE_TAG}&flavor=${IMAGE_FLAVOR}&arch=${ARCH}&countme=${BUCKET}" + +# 6. Send ping +if curl -sf --retry 3 --max-time 10 -H "User-Agent: ${USER_AGENT}" "$URL" >/dev/null; then + echo "$NOW" > "$LASTRUN_FILE" +fi diff --git a/tests/test_countme.py b/tests/test_countme.py new file mode 100644 index 00000000..3bea5e4c --- /dev/null +++ b/tests/test_countme.py @@ -0,0 +1,275 @@ +"""Unit tests for the utah-countme client script and systemd units.""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import time +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = ROOT / "system_files/shared/usr/libexec/utah-countme" +SERVICE_PATH = ROOT / "system_files/shared/usr/lib/systemd/system/utah-countme.service" +TIMER_PATH = ROOT / "system_files/shared/usr/lib/systemd/system/utah-countme.timer" +PRESET_PATH = ROOT / "system_files/shared/usr/lib/systemd/system-preset/85-utah-desktop.preset" +CONTRACT_PATH = ROOT / "contracts/bluefin-desktop.toml" +SERVICES_SCRIPT = ROOT / "scripts/configure-services.sh" + + +class CountmeUnitTest(unittest.TestCase): + def test_units_exist_and_match_contract(self): + self.assertTrue(SCRIPT_PATH.is_file()) + self.assertTrue(os.access(SCRIPT_PATH, os.X_OK)) + self.assertTrue(SERVICE_PATH.is_file()) + self.assertTrue(TIMER_PATH.is_file()) + + service_text = SERVICE_PATH.read_text() + self.assertIn("ConditionPathExists=!/etc/projectbluefin/countme/disabled", service_text) + self.assertIn("ConditionPathExists=/run/ostree-booted", service_text) + self.assertIn("ConditionPathExists=/usr/share/ublue-os/image-info.json", service_text) + self.assertIn("DynamicUser=yes", service_text) + self.assertIn("StateDirectory=utah-countme", service_text) + self.assertIn("ExecStart=/usr/libexec/utah-countme", service_text) + + timer_text = TIMER_PATH.read_text() + self.assertIn("ConditionPathExists=!/etc/projectbluefin/countme/disabled", timer_text) + self.assertIn("ConditionPathExists=/run/ostree-booted", timer_text) + self.assertIn("ConditionPathExists=/usr/share/ublue-os/image-info.json", timer_text) + self.assertIn("OnCalendar=weekly", timer_text) + self.assertIn("Persistent=true", timer_text) + + preset_text = PRESET_PATH.read_text() + self.assertIn("enable utah-countme.timer", preset_text) + + contract_text = CONTRACT_PATH.read_text() + self.assertIn('"utah-countme.timer"', contract_text) + + services_script_text = SERVICES_SCRIPT.read_text() + self.assertIn("enable_unit utah-countme.timer", services_script_text) + + def test_opt_out_file_skips_execution(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + disabled_file = tmp_path / "disabled" + disabled_file.touch() + + mock_bin = tmp_path / "bin" + mock_bin.mkdir() + curl_mock = mock_bin / "curl" + curl_mock.write_text("#!/bin/sh\nexit 1\n") + curl_mock.chmod(0o755) + + state_dir = tmp_path / "state" + + env = os.environ.copy() + env["PATH"] = f"{mock_bin}:{env['PATH']}" + env["DISABLED_FILE"] = str(disabled_file) + env["STATE_DIRECTORY"] = str(state_dir) + + res = subprocess.run([str(SCRIPT_PATH)], env=env, capture_output=True, text=True) + self.assertEqual(res.returncode, 0) + self.assertFalse((state_dir / "lastrun").exists()) + + def test_first_run_creates_epoch_and_pings(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + state_dir = tmp_path / "state" + mock_bin = tmp_path / "bin" + mock_bin.mkdir() + + curl_args_log = tmp_path / "curl_args.txt" + curl_mock = mock_bin / "curl" + curl_mock.write_text(f'#!/bin/sh\necho "$@" > "{curl_args_log}"\nexit 0\n') + curl_mock.chmod(0o755) + + image_info = tmp_path / "image-info.json" + image_info.write_text( + json.dumps( + { + "image-name": "utah", + "image-flavor": "nvidia", + "image-tag": "testing-20260911", + } + ) + ) + + os_release = tmp_path / "os-release" + os_release.write_text('VERSION_ID="43"\n') + + env = os.environ.copy() + env["PATH"] = f"{mock_bin}:{env['PATH']}" + env["STATE_DIRECTORY"] = str(state_dir) + env["IMAGE_INFO"] = str(image_info) + env["OS_RELEASE_FILE"] = str(os_release) + env["DISABLED_FILE"] = str(tmp_path / "nonexistent-disabled") + + res = subprocess.run([str(SCRIPT_PATH)], env=env, capture_output=True, text=True) + self.assertEqual(res.returncode, 0) + self.assertTrue((state_dir / "epoch").exists()) + self.assertTrue((state_dir / "lastrun").exists()) + + args = curl_args_log.read_text() + self.assertIn("https://countme.projectbluefin.io/metalink", args) + self.assertIn("repo=utah", args) + self.assertIn("tag=testing-20260911", args) + self.assertIn("flavor=nvidia", args) + self.assertIn("countme=1", args) + self.assertIn("utah-countme", args) + + def test_throttling_skips_within_7_days(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + state_dir = tmp_path / "state" + state_dir.mkdir() + mock_bin = tmp_path / "bin" + mock_bin.mkdir() + + curl_invoked = tmp_path / "invoked.txt" + curl_mock = mock_bin / "curl" + curl_mock.write_text(f'#!/bin/sh\ntouch "{curl_invoked}"\nexit 0\n') + curl_mock.chmod(0o755) + + now = int(time.time()) + (state_dir / "epoch").write_text(str(now - 100)) + (state_dir / "lastrun").write_text(str(now - 86400)) + + env = os.environ.copy() + env["PATH"] = f"{mock_bin}:{env['PATH']}" + env["STATE_DIRECTORY"] = str(state_dir) + env["DISABLED_FILE"] = str(tmp_path / "nonexistent-disabled") + + res = subprocess.run([str(SCRIPT_PATH)], env=env, capture_output=True, text=True) + self.assertEqual(res.returncode, 0) + self.assertFalse(curl_invoked.exists()) + + def test_age_cohort_buckets(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + mock_bin = tmp_path / "bin" + mock_bin.mkdir() + + curl_args_log = tmp_path / "curl_args.txt" + curl_mock = mock_bin / "curl" + curl_mock.write_text(f'#!/bin/sh\necho "$@" > "{curl_args_log}"\nexit 0\n') + curl_mock.chmod(0o755) + + image_info = tmp_path / "image-info.json" + image_info.write_text(json.dumps({"image-name": "utah", "image-flavor": "main", "image-tag": "latest"})) + + week_secs = 604800 + now = int(time.time()) + + test_cases = [ + (0, 1), # 0 weeks -> bucket 1 + (week_secs * 2, 2), # 2 weeks -> bucket 2 (2-4w) + (week_secs * 6, 3), # 6 weeks -> bucket 3 (5-24w) + (week_secs * 30, 4), # 30 weeks -> bucket 4 (>24w) + ] + + for offset_secs, expected_bucket in test_cases: + with self.subTest(bucket=expected_bucket): + state_dir = tmp_path / f"state_{expected_bucket}" + state_dir.mkdir() + (state_dir / "epoch").write_text(str(now - offset_secs)) + + env = os.environ.copy() + env["PATH"] = f"{mock_bin}:{env['PATH']}" + env["STATE_DIRECTORY"] = str(state_dir) + env["IMAGE_INFO"] = str(image_info) + env["DISABLED_FILE"] = str(tmp_path / "nonexistent-disabled") + + res = subprocess.run([str(SCRIPT_PATH)], env=env, capture_output=True, text=True) + self.assertEqual(res.returncode, 0) + args = curl_args_log.read_text() + self.assertIn(f"countme={expected_bucket}", args) + + def test_corrupt_epoch_skips_telemetry_without_resetting(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + state_dir = tmp_path / "state" + state_dir.mkdir() + (state_dir / "epoch").write_text("not-a-number") + + mock_bin = tmp_path / "bin" + mock_bin.mkdir() + curl_invoked = tmp_path / "invoked.txt" + curl_mock = mock_bin / "curl" + curl_mock.write_text(f'#!/bin/sh\ntouch "{curl_invoked}"\nexit 0\n') + curl_mock.chmod(0o755) + + env = os.environ.copy() + env["PATH"] = f"{mock_bin}:{env['PATH']}" + env["STATE_DIRECTORY"] = str(state_dir) + env["DISABLED_FILE"] = str(tmp_path / "nonexistent-disabled") + + res = subprocess.run([str(SCRIPT_PATH)], env=env, capture_output=True, text=True) + self.assertEqual(res.returncode, 0) + self.assertFalse(curl_invoked.exists()) + self.assertEqual((state_dir / "epoch").read_text(), "not-a-number") + self.assertFalse((state_dir / "lastrun").exists()) + + def test_curl_failure_does_not_update_lastrun(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + state_dir = tmp_path / "state" + mock_bin = tmp_path / "bin" + mock_bin.mkdir() + + curl_mock = mock_bin / "curl" + curl_mock.write_text("#!/bin/sh\nexit 1\n") + curl_mock.chmod(0o755) + + image_info = tmp_path / "image-info.json" + image_info.write_text(json.dumps({"image-name": "utah", "image-flavor": "main", "image-tag": "latest"})) + + env = os.environ.copy() + env["PATH"] = f"{mock_bin}:{env['PATH']}" + env["STATE_DIRECTORY"] = str(state_dir) + env["IMAGE_INFO"] = str(image_info) + env["DISABLED_FILE"] = str(tmp_path / "nonexistent-disabled") + + res = subprocess.run([str(SCRIPT_PATH)], env=env, capture_output=True, text=True) + self.assertEqual(res.returncode, 0) + self.assertTrue((state_dir / "epoch").exists()) + self.assertFalse((state_dir / "lastrun").exists()) + + def test_os_release_fallback_when_image_info_missing(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + state_dir = tmp_path / "state" + mock_bin = tmp_path / "bin" + mock_bin.mkdir() + + curl_args_log = tmp_path / "curl_args.txt" + curl_mock = mock_bin / "curl" + curl_mock.write_text(f'#!/bin/sh\necho "$@" > "{curl_args_log}"\nexit 0\n') + curl_mock.chmod(0o755) + + os_release = tmp_path / "os-release" + os_release.write_text( + 'VERSION_ID="43"\n' + 'IMAGE_VERSION="testing-osrelease"\n' + 'IMAGE_FLAVOR="gaming"\n' + ) + + env = os.environ.copy() + env["PATH"] = f"{mock_bin}:{env['PATH']}" + env["STATE_DIRECTORY"] = str(state_dir) + env["IMAGE_INFO"] = str(tmp_path / "nonexistent-image-info") + env["OS_RELEASE_FILE"] = str(os_release) + env["DISABLED_FILE"] = str(tmp_path / "nonexistent-disabled") + + res = subprocess.run([str(SCRIPT_PATH)], env=env, capture_output=True, text=True) + self.assertEqual(res.returncode, 0) + args = curl_args_log.read_text() + self.assertIn("repo=utah", args) + self.assertIn("tag=testing-osrelease", args) + self.assertIn("flavor=gaming", args) + self.assertIn("countme=1", args) + + +if __name__ == "__main__": + unittest.main() From 3c2dff94f8d5ba66432ad78efc1ef3744037889a Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Sat, 19 Sep 2026 15:52:53 +0000 Subject: [PATCH 2/2] feat(countme): drop jq dependency, advance throttle unconditionally, and document opt-out Address review feedback from @hanthor: - In utah-countme, replace jq invocation with inline python3 json parsing, removing the uncontracted runtime dependency on jq. - Advance the throttle by writing lastrun unconditionally before the ping to avoid retry storms across reboots during outages. - Document epoch semantics and add countme opt-out instructions to README.md. - Update test suite to verify lastrun is written regardless of curl exit status. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 4 +++ docs/skills/desktop-contract.md | 2 +- system_files/shared/usr/libexec/utah-countme | 26 +++++++++++--------- tests/test_countme.py | 4 +-- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index e94fdd29..b2115c96 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,10 @@ This is the honest list, and it is why the label above says pre-alpha. repository is consumed and the local QEMU image reaches GDM and GNOME Shell. - **CUDA is deliberately excluded** — 7.68 GB installed. Use the NVIDIA container toolkit, which is included, and run CUDA in a container. +- **Telemetry (countme).** Utah includes a weekly countme client reporting + to `countme.projectbluefin.io` following Bluefin ADR 0006. Users can opt out + by running `sudo touch /etc/projectbluefin/countme/disabled` or by masking + the timer: `sudo systemctl mask utah-countme.timer`. See the [open issues](https://github.com/projectbluefin/utah/issues) for where things stand. diff --git a/docs/skills/desktop-contract.md b/docs/skills/desktop-contract.md index 2ef33221..398a598d 100644 --- a/docs/skills/desktop-contract.md +++ b/docs/skills/desktop-contract.md @@ -128,7 +128,7 @@ Project Bluefin ADR 0006: - `arch`: system architecture (`x86_64`) - `countme`: installation age cohort bucket (1: first week, 2: 2–4 weeks, 3: 5–24 weeks, 4: >24 weeks) - **Data & logging policy:** No personal information or hardware identifiers are transmitted; requests contain no HTTP body. As with all HTTP requests, standard server connection metadata (including client IP address) is received by the endpoint for aggregate deduplication and cohort counting. -- **Network & failure resilience:** Requests use an explicit `--max-time 10` timeout. If the server is unreachable or offline, the script exits cleanly without updating `lastrun`, allowing the ping to be retried on the next schedule without skipping a weekly reporting window. +- **Network & failure resilience:** Requests use an explicit `--max-time 10` timeout. `lastrun` is updated unconditionally before the request so network or endpoint outages do not trigger daily retry storms on reboot; failed reports are simply dropped. - **Persistent state integrity:** The install epoch is stored in `/var/lib/utah-countme/epoch`. If this file exists but cannot be read or is corrupted, execution degrades to a no-op (no ping sent and no epoch reset) to avoid resetting systems into bucket 1. - **Opt-out:** Honored whenever `/etc/projectbluefin/countme/disabled` exists, or by masking the timer (`systemctl mask utah-countme.timer`). diff --git a/system_files/shared/usr/libexec/utah-countme b/system_files/shared/usr/libexec/utah-countme index 945153f3..a4856bf3 100755 --- a/system_files/shared/usr/libexec/utah-countme +++ b/system_files/shared/usr/libexec/utah-countme @@ -12,11 +12,6 @@ WEEK=604800 # Allow users to opt out without disabling the timer. [ -e "$DISABLED_FILE" ] && exit 0 -if ! command -v jq >/dev/null 2>&1; then - echo "utah-countme: jq is unavailable; skipping telemetry" >&2 - exit 0 -fi - if ! command -v curl >/dev/null 2>&1; then echo "utah-countme: curl is unavailable; skipping telemetry" >&2 exit 0 @@ -33,6 +28,8 @@ EPOCH_FILE="${STATE_DIR}/epoch" LASTRUN_FILE="${STATE_DIR}/lastrun" # 1. Read or initialize epoch file +# Note: The epoch is recorded the first time this script runs (typically ~10m +# after first boot via OnBootSec=10m) and acts as the baseline for cohort age. if [ ! -f "$EPOCH_FILE" ]; then echo "$NOW" > "$EPOCH_FILE" || { echo "utah-countme: cannot initialize epoch file; skipping telemetry" >&2 @@ -77,15 +74,21 @@ IMAGE_TAG="unknown" IMAGE_FLAVOR="unknown" if [ -f "$IMAGE_INFO" ]; then - INFO_NAME=$(jq -r '."image-name" // empty' "$IMAGE_INFO" 2>/dev/null || true) + read -r INFO_NAME INFO_TAG INFO_FLAVOR < <(python3 -c ' +import json, sys +try: + with open(sys.argv[1]) as f: + d = json.load(f) + print(d.get("image-name", "") or "", d.get("image-tag", "") or "", d.get("image-flavor", "") or "") +except Exception: + print("", "", "") +' "$IMAGE_INFO") if [ -n "$INFO_NAME" ]; then REPO="$INFO_NAME" fi - INFO_TAG=$(jq -r '."image-tag" // empty' "$IMAGE_INFO" 2>/dev/null || true) if [ -n "$INFO_TAG" ]; then IMAGE_TAG="$INFO_TAG" fi - INFO_FLAVOR=$(jq -r '."image-flavor" // empty' "$IMAGE_INFO" 2>/dev/null || true) if [ -n "$INFO_FLAVOR" ]; then IMAGE_FLAVOR="$INFO_FLAVOR" fi @@ -128,7 +131,6 @@ fi USER_AGENT="utah-countme (${REPO} ${OS_VERSION_ID}; ${IMAGE_FLAVOR}; Linux.${ARCH})" URL="https://countme.projectbluefin.io/metalink?repo=${REPO}&tag=${IMAGE_TAG}&flavor=${IMAGE_FLAVOR}&arch=${ARCH}&countme=${BUCKET}" -# 6. Send ping -if curl -sf --retry 3 --max-time 10 -H "User-Agent: ${USER_AGENT}" "$URL" >/dev/null; then - echo "$NOW" > "$LASTRUN_FILE" -fi +# 6. Send ping (update lastrun unconditionally to prevent outage retry storms) +echo "$NOW" > "$LASTRUN_FILE" +curl -sf --retry 3 --max-time 10 -H "User-Agent: ${USER_AGENT}" "$URL" >/dev/null || true diff --git a/tests/test_countme.py b/tests/test_countme.py index 3bea5e4c..d9e60ef7 100644 --- a/tests/test_countme.py +++ b/tests/test_countme.py @@ -211,7 +211,7 @@ def test_corrupt_epoch_skips_telemetry_without_resetting(self): self.assertEqual((state_dir / "epoch").read_text(), "not-a-number") self.assertFalse((state_dir / "lastrun").exists()) - def test_curl_failure_does_not_update_lastrun(self): + def test_curl_failure_still_updates_lastrun_to_prevent_retry_storm(self): with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) state_dir = tmp_path / "state" @@ -234,7 +234,7 @@ def test_curl_failure_does_not_update_lastrun(self): res = subprocess.run([str(SCRIPT_PATH)], env=env, capture_output=True, text=True) self.assertEqual(res.returncode, 0) self.assertTrue((state_dir / "epoch").exists()) - self.assertFalse((state_dir / "lastrun").exists()) + self.assertTrue((state_dir / "lastrun").exists()) def test_os_release_fallback_when_image_info_missing(self): with tempfile.TemporaryDirectory() as tmp: