diff --git a/docs/skills/ci-ops/e2e-workflow/references/startup-service-inventory.md b/docs/skills/ci-ops/e2e-workflow/references/startup-service-inventory.md index 4221c47dd..a50f521e7 100644 --- a/docs/skills/ci-ops/e2e-workflow/references/startup-service-inventory.md +++ b/docs/skills/ci-ops/e2e-workflow/references/startup-service-inventory.md @@ -63,6 +63,12 @@ that touch the lane's own files. It is not a release gate. - `startup_lane.py evaluate` judges that evidence. The workflow polls every 20s until nothing is pending; each unit fails once its `deadline_s` (seconds after QEMU start) passes. Exit 0 pass, 1 fail, 3 still pending. +- Every other non-deferred `startup_contract` unit (one the profile does not + list) is collected too and reported as `observed`. It never changes the + verdict; the detail says whether the image ships it, what a required-unit + judgment would be, and whether its `produces` paths exist. That is the + evidence for adding a profile to the unit's inventory entry. Lane masks never + include these units, so they run as shipped. - The `broken` matrix job adds `ExecStartPre=/bin/false` to `NetworkManager-wait-online.service` (required on every profile) and passes only if the run goes red for that unit (`--expect-failure`). @@ -130,8 +136,10 @@ entry uses it. profile. - The `dakota` profile requires only what the dakota build is known to enable: `flatpak-preinstall.service` and `NetworkManager-wait-online.service`. - Add `brew-setup.service` or the `podman-auto-update` units to it once a run - shows dakota ships them. + The lane now reports `brew-setup.service`, `firewalld.service`, + `bazzite-hardware-setup.service` and the `podman-auto-update` user units as + `observed` on dakota. Add `dakota` to a unit's `profiles` only when an + observed row shows it shipped and would pass, and cite that run as evidence. - Greenboot (`deferred`): automatic rollback needs a boot-counting bootloader path, which direct-kernel boot lacks (#706). The lane keeps both units masked and lists them as `not qualified`. diff --git a/scripts/startup_lane.py b/scripts/startup_lane.py index 1def75acc..8fdb50f37 100755 --- a/scripts/startup_lane.py +++ b/scripts/startup_lane.py @@ -10,13 +10,16 @@ Print ``KEY=value`` lines for ``$GITHUB_ENV``: the kernel ``systemd.mask=`` arguments (vm_inapplicable, ci_noise and deferred units only), the units to observe, the paths they must produce, the extra checks, and the overall - deadline. + deadline. The units to observe also include every other non-deferred + ``startup_contract`` unit, so a run records whether the image ships it. ``evaluate`` Parse the output of ``scripts/startup_lane_collect.sh`` and judge each required unit and, once flatpak-preinstall has finished, the system Flatpak state the image itself declares (preinstall.d, remotes.d and - tmpfiles.d permission overrides). Exit 0 when everything passed, 1 when + tmpfiles.d permission overrides). Units the profile does not require are + reported as ``observed`` and never change the verdict. Exit 0 when + everything passed, 1 when something failed, 3 while something is still pending inside its deadline. ``--final`` turns anything still pending into a failure. ``--expect-failure UNIT`` inverts the verdict for the negative control: @@ -42,6 +45,9 @@ INVENTORY = REPO / "startup_service_inventory.yaml" PASS, FAIL, PENDING, DEFERRED = "pass", "fail", "pending", "not qualified" +# Informational: a startup unit this profile does not require. Never affects +# the verdict; it is evidence for deciding whether the profile should require it. +OBSERVED = "observed" EXIT = {PASS: 0, FAIL: 1, PENDING: 3} # Broken on purpose by the negative-control job: every profile requires it and @@ -123,6 +129,17 @@ def required_units(inventory: dict, profile: str) -> dict[str, dict]: } +def observed_units(inventory: dict, profile: str) -> dict[str, dict]: + """startup_contract units another profile requires: recorded, never judged.""" + return { + unit: lane_entry(entry) + for unit, entry in inventory.items() + if entry["category"] == "startup_contract" + and not applies(entry, profile) + and not lane_entry(entry).get("deferred") + } + + def deferred_units(inventory: dict, profile: str) -> dict[str, str]: return { unit: lane_entry(entry)["deferred"] @@ -185,15 +202,19 @@ def plan(inventory: dict, profile: str) -> dict[str, str]: if profile not in known_profiles(inventory): raise SystemExit(f"unknown profile {profile!r}; inventory knows {sorted(known_profiles(inventory))}") required = required_units(inventory, profile) - by_scope = { - scope: sorted(u for u, lane in required.items() if lane.get("scope", "system") == scope) - for scope in SCOPES - } + observed = observed_units(inventory, profile) + + def scoped(units: dict[str, dict], scope: str) -> list[str]: + return sorted(u for u, lane in units.items() if lane.get("scope", "system") == scope) + + # Required units first, then observed ones; the collector reads both. return { "KERNEL_MASKS": " ".join(f"systemd.mask={u}" for u in lane_masks(inventory)), - "SYSTEM_UNITS": " ".join(by_scope["system"]), - "USER_UNITS": " ".join(by_scope["user"]), - "PRODUCES": " ".join(p for lane in required.values() for p in lane.get("produces", [])), + "SYSTEM_UNITS": " ".join(scoped(required, "system") + scoped(observed, "system")), + "USER_UNITS": " ".join(scoped(required, "user") + scoped(observed, "user")), + "PRODUCES": " ".join( + p for lane in [*required.values(), *observed.values()] for p in lane.get("produces", []) + ), "CHECKS": " ".join(sorted({c for lane in required.values() for c in lane.get("checks", [])})), "STARTUP_DEADLINE_S": str(max(lane["deadline_s"] for lane in required.values())), "NEGATIVE_CONTROL_UNIT": NEGATIVE_CONTROL_UNIT, @@ -436,9 +457,23 @@ def evaluate(inventory: dict, profile: str, evidence: str, elapsed: int, final: ) for unit, why in sorted(deferred_units(inventory, profile).items()): report.add(unit, DEFERRED, why) + for unit, lane in sorted(observed_units(inventory, profile).items()): + report.add(unit, OBSERVED, describe_observed(unit, lane, units.get(unit), paths, elapsed)) return report +def describe_observed(unit: str, lane: dict, props: dict[str, str] | None, paths: dict[str, str], elapsed: int) -> str: + """Say what an unrequired unit did, as evidence for adding it to a profile.""" + if props is None: + return "not required by this profile; no state collected" + if props.get("LoadState") == "not-found": + return "not required by this profile; not shipped on this image" + status, detail = judge_unit(unit, lane, props, elapsed) + produced = [f"{p} {'present' if paths.get(p) == 'yes' else 'missing'}" for p in lane.get("produces", [])] + text = f"not required by this profile; shipped, and as a required unit would be {status}: {detail}" + return text + (f"; {', '.join(produced)}" if produced else "") + + # --- CLI --------------------------------------------------------------------- diff --git a/startup_service_inventory.yaml b/startup_service_inventory.yaml index 2aca8ee3a..018cf8c1d 100644 --- a/startup_service_inventory.yaml +++ b/startup_service_inventory.yaml @@ -45,6 +45,10 @@ # deferred: why the lane cannot qualify it yet. The lane keeps the unit # masked and lists it as not qualified; it never passes it. # vm_inapplicable and ci_noise units stay masked in the startup lane too. +# On a profile that does not list it, a non-deferred startup_contract unit is +# left unmasked and reported as `observed`: the lane records whether the image +# ships it and how it ran, without changing the verdict. Add the profile here +# only when an observed run shows the image enables it. units: # --- startup_contract ------------------------------------------------------- diff --git a/tests/unit/test_startup_lane.py b/tests/unit/test_startup_lane.py index 50d638325..3a8ab932a 100644 --- a/tests/unit/test_startup_lane.py +++ b/tests/unit/test_startup_lane.py @@ -119,8 +119,9 @@ def test_plan_never_masks_a_required_startup_unit(inventory): masked = set(re.findall(r"systemd\.mask=(\S+)", env["KERNEL_MASKS"])) required = set(lane.required_units(inventory, profile)) assert not masked & required, profile - observed = set(env["SYSTEM_UNITS"].split()) | set(env["USER_UNITS"].split()) - assert observed == required, profile + assert not masked & set(lane.observed_units(inventory, profile)), profile + collected = set(env["SYSTEM_UNITS"].split()) | set(env["USER_UNITS"].split()) + assert collected == required | set(lane.observed_units(inventory, profile)), profile def test_plan_masks_only_inapplicable_noise_and_deferred_units(inventory): @@ -139,6 +140,8 @@ def test_plan_for_bluefin(inventory): "NetworkManager-wait-online.service", "brew-setup.service", "flatpak-preinstall.service", + "bazzite-hardware-setup.service", + "firewalld.service", ] assert env["USER_UNITS"].split() == ["podman-auto-update.service", "podman-auto-update.timer"] assert env["CHECKS"] == "flatpak" @@ -149,12 +152,30 @@ def test_plan_for_bluefin(inventory): def test_plan_for_dakota(inventory): env = lane.plan(inventory, "dakota") + assert set(lane.required_units(inventory, "dakota")) == { + "NetworkManager-wait-online.service", + "flatpak-preinstall.service", + } + # Required units first, then the ones dakota is not yet known to ship. assert env["SYSTEM_UNITS"].split() == [ "NetworkManager-wait-online.service", "flatpak-preinstall.service", + "bazzite-hardware-setup.service", + "brew-setup.service", + "firewalld.service", ] - assert env["USER_UNITS"].split() == [] + assert env["USER_UNITS"].split() == ["podman-auto-update.service", "podman-auto-update.timer"] + assert "/home/linuxbrew/.linuxbrew/bin/brew" in env["PRODUCES"].split() assert env["CHECKS"] == "flatpak" + assert env["STARTUP_DEADLINE_S"] == "1800" + + +def test_observed_units_exclude_required_and_deferred(inventory): + for profile in ("dakota", "bluefin", "bluefin-lts", "aurora", "bazzite"): + observed = set(lane.observed_units(inventory, profile)) + assert not observed & set(lane.required_units(inventory, profile)), profile + assert not observed & set(lane.deferred_units(inventory, profile)), profile + assert "greenboot-healthcheck.service" not in lane.observed_units(inventory, "dakota") def test_inventory_expects_the_image_to_ship_its_remote(inventory): @@ -184,6 +205,49 @@ def test_bazzite_reports_greenboot_as_not_qualified(inventory): assert report.verdict == lane.PASS +DAKOTA_SYSTEM = [ + unit("NetworkManager-wait-online.service"), + unit("flatpak-preinstall.service"), +] + + +def test_dakota_records_shipped_brew_setup_without_requiring_it(inventory): + system = DAKOTA_SYSTEM + [ + unit("brew-setup.service", active="inactive", sub="dead"), + unit("firewalld.service", load="not-found", active="inactive", sub="dead"), + ] + report = lane.evaluate(inventory, "dakota", evidence(system=system, user=[]), elapsed=600) + got = {f.subject: f for f in report.findings} + assert report.verdict == lane.PASS, report.markdown("dakota") + assert got["brew-setup.service"].status == lane.OBSERVED + assert "would be pass" in got["brew-setup.service"].detail + assert "/home/linuxbrew/.linuxbrew/bin/brew present" in got["brew-setup.service"].detail + assert "not shipped on this image" in got["firewalld.service"].detail + assert "no state collected" in got["podman-auto-update.timer"].detail + # Observed paths are evidence only, never their own pass/fail rows. + assert "/home/linuxbrew/.linuxbrew/bin/brew" not in got + + +def test_failed_observed_unit_does_not_change_the_verdict(inventory): + system = DAKOTA_SYSTEM + [ + unit("brew-setup.service", active="failed", sub="failed", result="exit-code", status="1"), + ] + paths = ["no /home/linuxbrew/.linuxbrew/bin/brew", "no /etc/.linuxbrew"] + report = lane.evaluate(inventory, "dakota", evidence(system=system, user=[], paths=paths), elapsed=600, final=True) + got = {f.subject: f for f in report.findings} + assert report.verdict == lane.PASS, report.markdown("dakota") + assert got["brew-setup.service"].status == lane.OBSERVED + assert "would be fail" in got["brew-setup.service"].detail + assert "/etc/.linuxbrew missing" in got["brew-setup.service"].detail + + +def test_required_unit_is_never_reported_as_observed(inventory): + report = lane.evaluate(inventory, "bluefin", evidence(), elapsed=600) + observed = {f.subject for f in report.findings if f.status == lane.OBSERVED} + assert observed == set(lane.observed_units(inventory, "bluefin")) + assert "brew-setup.service" not in observed + + # --- unit judging ------------------------------------------------------------ COMPLETED = {"expect": "completed", "deadline_s": 300}