From a089caf92457302f5f7888a79bf4cd7a767b205c Mon Sep 17 00:00:00 2001 From: Babissimo Date: Wed, 23 Sep 2026 18:42:22 +0100 Subject: [PATCH] Add just verify-deploy, which asks an environment whether its deploy worked Checking a deploy was a procedure spread across a CLAUDE.md paragraph and six memory notes, each recording a trap: Cloudflare answers 1010 to a script without a browser User-Agent, localhost curls on the droplets answer 400, /api/health reads degraded hourly and after every restart, the server answers 5xx and publishes an empty feed while it restarts and then takes minutes to refill the map, a status code cannot tell a bundle from the SPA fallback, and a lazily loaded page's change is not in the index chunk. Worktree sessions load none of that memory, so each trap was found again. deploy/verify-deploy.py encodes them. It reads the environment's hostnames, and the flags and RETINA_ENV it should report, from that environment's compose overlay, and the origin marker from deploy/origin-marker.sh, so none can drift from its source. It polls health, the public map's feed, the public node list and, on the fleet's box, the unfiltered feed together for up to --wait minutes, failing only what is still wrong at the end; reads a degraded health as a warning; checks the marker on every /api/ answer; walks every script and stylesheet the page reaches; and with --expect-string finds a string in any of them. It only reads, and only public hostnames. Its verdict rests on public reads because /api/test/* answers only an administrator or the radar key since #608: - The map is judged on aircraft-live.json, the real-node feed the public map draws on every deployed host, not the unfiltered aircraft.json. An empty public map with real nodes connected warns, since real nodes can solve nothing for hours; a feed that errors, stops or has the wrong shape fails, and so does a box with no real node connected. - On the fleet's box the unfiltered feed must carry aircraft a synthetic node saw, so real nodes' aircraft cannot stand in for the fleet. - /api/health's synthetic_fleet and polled_radar_polling, read against the overlay, tell prod, staging and test apart without a key. - RETINA_ENV and task health need the dashboard, so they run only when the radar key is in VERIFY_DEPLOY_RADAR_KEY. Otherwise they are reported as skipped, and the summary says they were not passed. The key goes in one header to that one route, is never followed through a redirect, and is redacted from an answer's body before anything quotes it. No public endpoint reports the deployed commit, so it cannot say what shipped beyond --expect-string; ClickUp 123zgec4n48 carries exposing one. The connected real nodes are listed for comparison with a run before the deploy, and the in-browser look at the map stays a human check. ClickUp: 123zgec4mxu Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 20 +- ONBOARDING.md | 3 +- backend/tests/test_verify_deploy.py | 789 +++++++++++++++++++++++++++ deploy/origin-marker.sh | 5 +- deploy/page-asset.sh | 7 +- deploy/verify-deploy.py | 816 ++++++++++++++++++++++++++++ justfile | 8 + 7 files changed, 1635 insertions(+), 13 deletions(-) create mode 100644 backend/tests/test_verify_deploy.py create mode 100644 deploy/verify-deploy.py diff --git a/CLAUDE.md b/CLAUDE.md index 7391f360d..9516207ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,11 +31,15 @@ FastAPI backend and React front-ends for the RETINA passive-radar network. accounts in anything committed here. - **Configuration lives in `backend/.env`**, which is gitignored. Add new keys to `backend/.env.example` so the list stays current. -- **Verify a deploy from the environment's API.** Green tests do not cover the - compose/env/frontend seams, so confirm the change against the environment - itself before calling a deploy done: `aircraft_on_map` in - `/api/test/dashboard`, the node set in `/api/radar/analytics`, and - `/api/radar/data/aircraft.json` for the data path. `/api/test/*` reads answer - only an administrator or the radar key, which stays on the droplet: read them - there with the runbook's `tst`. Scripted requests need a browser User-Agent or - Cloudflare answers `403 1010`. +- **Verify a deploy with `just verify-deploy `.** Green tests + do not cover the compose/env/frontend seams, so ask the environment itself + before calling a deploy done. Run it before the deploy too, and compare the + real nodes the two runs list. For a front-end change add + `--expect-string `, which the earlier run must not find. + Its verdict rests on public reads; it runs its keyed checks (RETINA_ENV, task + health) only when `VERIFY_DEPLOY_RADAR_KEY` holds the radar key, and reports + them skipped otherwise. `/api/test/*` reads answer only an administrator or + the radar key, which stays on the droplet: read them there with the + runbook's `tst`. Scripted requests need a browser User-Agent or Cloudflare + answers `403 1010`. `deploy/verify-deploy.py` says what it checks and what it + cannot see. diff --git a/ONBOARDING.md b/ONBOARDING.md index a87e44819..93d4ae7f2 100644 --- a/ONBOARDING.md +++ b/ONBOARDING.md @@ -357,7 +357,8 @@ CI runs on every PR, on push to `main`, and on demand through they fail after it; the runbook's Environments section has the shape. So merging to `main` deploys to production automatically. Work on a feature -branch, open a PR, get it green, then merge. +branch, open a PR, get it green, then merge. Once the deploy has run, +`just verify-deploy ` asks that environment whether it worked. ## Things that will bite you diff --git a/backend/tests/test_verify_deploy.py b/backend/tests/test_verify_deploy.py new file mode 100644 index 000000000..90b542c1b --- /dev/null +++ b/backend/tests/test_verify_deploy.py @@ -0,0 +1,789 @@ +"""deploy/verify-deploy.py's verdicts, against a fake environment. + +Each case is a way a deploy has been misjudged here: a degraded health that was +only the baseline, an empty map read before it had time to fill, an asset the SPA +fallback answered with a 200, a change searched for in the wrong chunk. The +checks must fail on the real faults and pass the baseline, so both directions are +asserted. +""" + +from __future__ import annotations + +import email.utils +import http.client +import importlib.util +import io +import json +import re +import sys +import time +import urllib.error +import urllib.parse +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parents[2] / "deploy" / "verify-deploy.py" +_spec = importlib.util.spec_from_file_location("verify_deploy", _SCRIPT) +vd = importlib.util.module_from_spec(_spec) +sys.modules["verify_deploy"] = vd +_spec.loader.exec_module(vd) + +API = "api.example.test" +APP = "app.example.test" +NOW = 1_800_000_000.0 +MARKER = ("x-retina-origin", "retina-server") +KEY = "rk-test-9f3c2a7e5b1d4c60" +HEALTH = f"{API}/api/health" +FEED = f"{APP}/api/radar/data/aircraft-live.json" +ALL = f"{APP}/api/radar/data/aircraft.json" +NODES = f"{API}/api/radar/nodes" +DASHBOARD = f"{API}/api/test/dashboard" +REAL = ("ndeaaa", "ndebbb", "ndeccc") + +INDEX_HTML = ( + '' + '' + '' + '' +) +INDEX_JS = 'const a=()=>import("./Lazy-def.js");__vite__mapDeps(["assets/Other-ghi.js","assets/Lazy-def.css"]);' + + +def _resp(body, *, status=200, ctype="application/json", marked=True, date=NOW) -> vd.Response: + headers = {"content-type": ctype, "date": email.utils.formatdate(date, usegmt=True)} + if marked: + headers[MARKER[0]] = MARKER[1] + raw = body if isinstance(body, bytes) else (body if isinstance(body, str) else json.dumps(body)).encode() + return vd.Response(status, headers, raw) + + +def _feed(aircraft=2, age=1, by="ndeaaa", also=()): + """A feed of `aircraft` aircraft, each seen by node `by` and contributed to by `also`.""" + seen = [{"hex": f"a{i}", "node_ref": by, "contributing_node_refs": list(also)} for i in range(aircraft)] + return _resp({"now": NOW - age, "aircraft": seen}) + + +def _health(fleet=False, polling=False, status="ok"): + return _resp({"status": status, "synthetic_fleet": fleet, "polled_radar_polling": polling}) + + +def _nodes(real=REAL, synthetic=("synth-1",), gone=("ndegone",)): + listed = {} + for refs, status, is_synthetic in ( + (real, "active", False), + (synthetic, "active", True), + (gone, "disconnected", False), + ): + for ref in refs: + listed[f"k-{ref}"] = {"node_ref": ref, "status": status, "is_synthetic": is_synthetic} + return _resp({"nodes": listed}) + + +def _dashboard(env="test", stale=(), subsystems=None): + return { + "environment": env, + "subsystem_health": subsystems or {"tcp_server": "ok"}, + "task_health": {"stale_tasks": list(stale)}, + } + + +def _site(**overrides) -> dict[str, object]: + """URL -> response (or a list of responses, served in turn) for a healthy box without the fleet.""" + site = { + f"https://{HEALTH}": _health(), + f"https://{FEED}": _feed(), + f"https://{ALL}": _feed(3, by="synth-1"), + f"https://{NODES}": _nodes(), + f"https://{DASHBOARD}": _resp(_dashboard()), + f"https://{APP}/": _resp(INDEX_HTML, ctype="text/html", marked=False), + f"https://{APP}/assets/index-abc.js": _resp(INDEX_JS, ctype="text/javascript", marked=False), + f"https://{APP}/assets/Lazy-def.js": _resp('export const t="Shipped today";', ctype="text/javascript"), + f"https://{APP}/assets/Other-ghi.js": _resp("export const u=1;", ctype="text/javascript"), + f"https://{APP}/assets/index-xyz.css": _resp(".retina-banner{color:teal}", ctype="text/css", marked=False), + f"https://{APP}/assets/Lazy-def.css": _resp(".map-surface{inset:0}", ctype="text/css", marked=False), + f"https://{APP}/theme-boot.js": _resp('document.documentElement.dataset.theme="dark"', ctype="text/javascript"), + } + site.update({(k if k.startswith("https://") else f"https://{k}"): v for k, v in overrides.items()}) + return site + + +def _fleet_site(**overrides) -> dict[str, object]: + """The fleet's box, healthy: the flag on and the synthetic fleet connected and seeing aircraft.""" + return _site(**{HEALTH: _health(fleet=True), **overrides}) + + +class _Clock: + def __init__(self): + self.t = NOW + self.sleeps = 0 + self.calls = 0 + + def __call__(self): + self.calls += 1 + return self.t + + def sleep(self, seconds): + self.sleeps += 1 + self.t += seconds + + +class _Asked(list): + """The URLs asked for, in order, and the headers each was last asked with.""" + + def __init__(self): + super().__init__() + self.headers: dict[str, dict | None] = {} + + +def _serve(site, url): + answer = site[url] + if isinstance(answer, list): + answer = answer.pop(0) if len(answer) > 1 else answer[0] + if isinstance(answer, Exception): + raise answer + return answer + + +def _verify(site, expect=(), *, absent=(), wait_s=60, retina_env="test", fleet=False, polling=False, key=""): + clock = _Clock() + asked = _Asked() + + def get(url, headers=None): + parts = urllib.parse.urlsplit(url) + path = f"{parts.scheme}://{parts.netloc}{parts.path}" + asked.append(path) + asked.headers[path] = headers + return _serve(site, path) + + env = vd.Environment(API, APP, retina_env, fleet, polling) + verifier = vd.Verifier( + env, get=get, clock=clock, sleep=clock.sleep, log=lambda _: None, wait_s=wait_s, marker=MARKER, key=key + ) + report = verifier.run(list(expect), list(absent)) + return report, clock, asked + + +def _rows(report, level): + return {name: detail for lvl, name, detail in report.rows if lvl == level} + + +def test_a_healthy_environment_passes(): + report, clock, asked = _verify(_site()) + assert _rows(report, "FAIL") == {} + assert set(_rows(report, "PASS")) == {"health", "aircraft feed", "map", "box", "origin", "bundle"} + assert clock.sleeps == 0 + # The unfiltered feed is the fleet's box's business alone. + assert f"https://{ALL}" not in asked + + +def test_degraded_health_is_the_baseline_and_only_warns(): + report, _, _ = _verify(_site(**{HEALTH: _health(status="degraded")})) + assert "health" in _rows(report, "WARN") + assert _rows(report, "FAIL") == {} + + +def test_a_cloudflare_1010_refusal_is_named(): + refused = _resp("error code: 1010", status=403, ctype="text/plain", marked=False) + report, _, _ = _verify(_site(**{HEALTH: refused})) + assert "1010" in _rows(report, "FAIL")["health"] + assert "User-Agent" in _rows(report, "FAIL")["health"] + + +def test_an_answer_without_the_origin_marker_fails(): + other = _resp({"now": NOW, "aircraft": []}, marked=False) + report, _, _ = _verify(_site(**{FEED: other})) + assert f"https://{FEED}" in _rows(report, "FAIL")["origin"] + + +def test_a_feed_the_flush_stopped_writing_fails(): + report, _, _ = _verify(_site(**{FEED: _feed(1, age=300)})) + assert "stopped" in _rows(report, "FAIL")["aircraft feed"] + + +def test_an_empty_map_is_polled_until_it_fills(): + report, clock, _ = _verify(_site(**{FEED: [_feed(0), _feed(0), _feed(4)]}), wait_s=600) + assert "4 aircraft on the public map" in _rows(report, "PASS")["map"] + assert clock.sleeps == 2 + + +def test_the_map_is_judged_on_the_feed_it_draws_not_the_unfiltered_one(): + report, _, asked = _verify(_site(**{FEED: _feed(0), ALL: _feed(9)}), wait_s=60) + assert "no aircraft on the public map" in _rows(report, "WARN")["map"] + assert f"https://{FEED}" in asked + + +# Real-node boxes: an empty public map with real nodes connected warns, and the +# control beside it passes. + + +def test_an_empty_public_map_over_real_nodes_only_warns(): + report, _, _ = _verify(_site(**{FEED: _feed(0)}), wait_s=60) + assert "quiet sky" in _rows(report, "WARN")["map"] + assert _rows(report, "FAIL") == {} + + +def test_the_same_box_with_aircraft_on_the_public_map_passes(): + report, _, _ = _verify(_site(**{FEED: _feed(5)}), wait_s=60) + assert _rows(report, "PASS")["map"] == "5 aircraft on the public map; 3 real node(s) connected" + + +# A feed that errors, times out or is not a feed fails; the healthy control is +# test_a_healthy_environment_passes. + + +def test_a_feed_that_errors_fails(): + broken = _resp("bad gateway", status=502, ctype="text/html", marked=False) + report, _, _ = _verify(_site(**{FEED: broken}), wait_s=60) + assert "502" in _rows(report, "FAIL")["aircraft feed"] + + +def test_a_feed_that_times_out_fails(): + report, _, _ = _verify(_site(**{FEED: vd.Unreachable(f"https://{FEED}: timed out")}), wait_s=60) + assert "timed out" in _rows(report, "FAIL")["aircraft feed"] + + +@pytest.mark.parametrize( + "body", + [{"now": NOW, "planes": []}, {"now": "soon", "aircraft": []}, {"now": NOW, "aircraft": {}}], + ids=["no aircraft list", "now not a number", "aircraft not a list"], +) +def test_a_feed_of_the_wrong_shape_fails(body): + report, _, _ = _verify(_site(**{FEED: _resp(body)}), wait_s=60) + assert "is not an aircraft feed" in _rows(report, "FAIL")["aircraft feed"] + + +def test_a_node_list_of_the_wrong_shape_fails(): + report, _, _ = _verify(_site(**{NODES: _resp({"nodes": []})}), wait_s=60) + assert "carries no node list" in _rows(report, "FAIL")["map"] + + +# The fleet's box: the fleet must be connected and must see aircraft. + + +def test_the_fleet_s_box_passes_with_the_fleet_seeing_aircraft(): + report, _, _ = _verify(_fleet_site(), fleet=True) + assert _rows(report, "PASS")["fleet"] == "3 aircraft seen by the 1 connected synthetic node(s)" + assert _rows(report, "FAIL") == {} + + +def test_the_fleet_connected_with_no_aircraft_after_the_window_fails(): + # The same box as the test above, with the fleet's aircraft taken away. + report, clock, _ = _verify(_fleet_site(**{ALL: _feed(0)}), wait_s=60, fleet=True) + assert "was seen by the 1 connected synthetic" in _rows(report, "FAIL")["fleet"] + assert "post-restart window" in _rows(report, "FAIL")["fleet"] + assert clock.t - NOW >= 60 + + +def test_on_the_fleet_s_box_real_nodes_aircraft_do_not_stand_in_for_the_fleet(): + report, clock, _ = _verify(_fleet_site(**{ALL: _feed(9, by="ndeaaa")}), wait_s=60, fleet=True) + assert "was seen by the 1 connected synthetic" in _rows(report, "FAIL")["fleet"] + assert clock.sleeps > 0 + + +def test_an_aircraft_the_fleet_contributed_to_counts_for_it(): + report, _, _ = _verify(_fleet_site(**{ALL: _feed(2, by="ndeaaa", also=("synth-1",))}), fleet=True) + assert "2 aircraft seen by" in _rows(report, "PASS")["fleet"] + + +def test_on_the_fleet_s_box_a_fleet_not_connected_fails(): + report, clock, _ = _verify(_fleet_site(**{NODES: _nodes(synthetic=())}), wait_s=60, fleet=True) + assert "synthetic fleet is not connected" in _rows(report, "FAIL")["fleet"] + assert clock.sleeps > 0 + + +def test_an_unfiltered_feed_of_the_wrong_shape_fails_the_fleet(): + report, _, _ = _verify(_fleet_site(**{ALL: _resp({"now": NOW})}), wait_s=60, fleet=True) + assert "is not an aircraft feed" in _rows(report, "FAIL")["fleet"] + + +def test_the_fleet_s_box_with_no_real_node_says_the_public_map_is_empty(): + report, _, _ = _verify(_fleet_site(**{NODES: _nodes(real=()), FEED: _feed(0)}), fleet=True) + assert "shows real nodes only" in _rows(report, "INFO")["map"] + assert _rows(report, "FAIL") == {} + + +def test_a_stale_feed_fails_even_under_a_quiet_sky(): + # Every frame rewrites the feed, aircraft or none, so a quiet sky leaves it fresh. + unpublished = _resp({"now": 0, "aircraft": []}) + report, _, _ = _verify(_site(**{FEED: unpublished}), wait_s=60) + assert "stopped" in _rows(report, "FAIL")["aircraft feed"] + + +def test_no_node_connected_fails_even_on_a_real_node_environment(): + report, _, _ = _verify(_site(**{FEED: _feed(0), NODES: _nodes(real=(), synthetic=())}), wait_s=60) + assert "no real node on the public list is connected" in _rows(report, "FAIL")["map"] + + +def test_synthetic_nodes_do_not_stand_in_for_real_ones_off_the_fleet_s_box(): + report, _, _ = _verify(_site(**{NODES: _nodes(real=())}), wait_s=60) + assert "no real node on the public list is connected" in _rows(report, "FAIL")["map"] + + +def test_the_real_nodes_are_listed_for_comparison(): + report, _, _ = _verify(_site()) + assert _rows(report, "INFO")["real nodes"] == "3 connected: ndeaaa ndebbb ndeccc" + + +def test_an_empty_node_list_is_read_again(): + report, clock, _ = _verify(_site(**{NODES: [_resp({"nodes": {}}), _nodes(real=("ndeaaa",))]})) + assert _rows(report, "INFO")["real nodes"].startswith("1 connected") + assert clock.sleeps == 1 + + +# Which box answered, from /api/health's flags. + + +def test_a_hostname_reaching_a_box_with_the_other_fleet_setting_fails(): + report, _, _ = _verify(_site(**{HEALTH: _health(fleet=True)})) + assert "synthetic_fleet=True" in _rows(report, "FAIL")["box"] + + +def test_prod_answering_for_staging_fails(): + report, _, _ = _verify(_site(**{HEALTH: _health(polling=True)}), retina_env="test", polling=False) + assert "polled_radar_polling=True where the overlay leaves off" in _rows(report, "FAIL")["box"] + report, _, _ = _verify(_site(**{HEALTH: _health(polling=True)}), retina_env="production", polling=True) + assert "polled_radar_polling=True" in _rows(report, "PASS")["box"] + + +def test_a_health_that_does_not_say_which_box_warns(): + report, _, _ = _verify(_site(**{HEALTH: _resp({"status": "ok"})})) + assert "unproven" in _rows(report, "WARN")["box"] + + +# The keyed checks. + + +def test_without_the_key_the_keyed_checks_are_skipped_and_not_asked_for(): + report, _, asked = _verify(_site()) + assert vd.KEY_VARIABLE in _rows(report, "SKIP")["keyed checks"] + assert f"https://{DASHBOARD}" not in asked + assert "environment" not in _rows(report, "PASS") + + +def test_the_key_goes_to_the_dashboard_alone_and_in_a_header(): + report, _, asked = _verify(_site(), key=KEY) + assert asked.headers[f"https://{DASHBOARD}"] == {"X-API-Key": KEY} + assert all(not headers for url, headers in asked.headers.items() if url != f"https://{DASHBOARD}") + assert not any(KEY in url for url in asked) + assert "RETINA_ENV=test" in _rows(report, "PASS")["environment"] + assert "keyed checks" not in _rows(report, "SKIP") + + +def test_with_the_key_a_hostname_reaching_the_wrong_box_fails(): + report, _, _ = _verify(_site(), retina_env="production", key=KEY) + assert "RETINA_ENV='test'" in _rows(report, "FAIL")["environment"] + + +def test_with_the_key_stale_tasks_and_waiting_subsystems_warn(): + dash = _dashboard(stale=["solver"], subsystems={"tcp_server": "ok", "mqtt": "starting"}) + report, _, _ = _verify(_site(**{DASHBOARD: _resp(dash)}), key=KEY) + assert "solver" in _rows(report, "WARN")["tasks"] + assert _rows(report, "WARN")["subsystems"] == "mqtt=starting" + assert _rows(report, "FAIL") == {} + + +def test_a_key_the_environment_refuses_fails_the_keyed_checks(): + refused = _resp({"detail": "Not authenticated"}, status=401, marked=False) + report, _, _ = _verify(_site(**{DASHBOARD: refused}), key=KEY) + assert "401 or 403 means the key is not this environment's" in _rows(report, "FAIL")["keyed checks"] + + +@pytest.mark.parametrize( + ("key", "echo"), + [ + (KEY, "x" * 190 + KEY + "y" * 40), # straddles the 200-character cut + ("rk'quoted\\key", "key rk'quoted\\key is not valid here"), # repr would escape it + ], + ids=["cut", "escaped"], +) +def test_a_key_echoed_back_is_redacted_before_the_body_is_quoted(key, echo): + refused = _resp(echo, status=403, ctype="text/plain", marked=False) + report, _, _ = _verify(_site(**{DASHBOARD: refused}), key=key) + shown = report.render() + assert "keyed checks" in _rows(report, "FAIL") + assert "[radar key" in shown + assert not any(key[i : i + 6] in shown for i in range(len(key) - 5)) + + +def test_the_report_redacts_the_key_whatever_brings_it(): + # A second line of defence, for a detail that reaches the report other than through an answer's body. + report = vd.Report(secret=KEY) + report.add("FAIL", "keyed checks", f"https://{DASHBOARD}: {KEY!r} refused") + assert KEY not in report.render() + assert "[radar key]" in report.render() + + +def _opener(site, asked): + """verify-deploy's _open, answering from `site` by path, as fetch() would see it.""" + by_path = {urllib.parse.urlsplit(url).path: answer for url, answer in site.items()} + + class Answer: + def __init__(self, resp): + self.status, self.headers, self._body = resp.status, resp.headers, resp.body + + def read(self): + return self._body + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def _open(request, *, follow): + asked.append((request.full_url, dict(request.header_items()), follow)) + resp = _serve(by_path, urllib.parse.urlsplit(request.full_url).path) + if resp.status != 200: + raise urllib.error.HTTPError(request.full_url, resp.status, "", resp.headers, io.BytesIO(resp.body)) + return Answer(resp) + + return _open + + +def _main(monkeypatch, capsys, site, key=None): + asked = [] + monkeypatch.setattr(vd, "_open", _opener(site, asked)) + if key is None: + monkeypatch.delenv(vd.KEY_VARIABLE, raising=False) + else: + monkeypatch.setenv(vd.KEY_VARIABLE, key) + status = vd.main(["test", "--wait", "0"]) + out = capsys.readouterr() + return status, out.out, out.err, asked + + +@pytest.mark.parametrize( + ("dashboard", "status"), + [ + (_resp(_dashboard()), 0), + # A server that echoed the key back in an error. + (_resp(f"key {KEY} is not valid here", status=403, ctype="text/plain", marked=False), 1), + ], +) +def test_the_key_never_appears_in_any_output(monkeypatch, capsys, dashboard, status): + code, out, err, asked = _main(monkeypatch, capsys, _fleet_site(**{DASHBOARD: dashboard}), key=KEY) + assert code == status + assert KEY not in out + err + sent = [(headers, follow) for url, headers, follow in asked if url.endswith("/api/test/dashboard")] + assert sent and all(headers.get("X-api-key") == KEY and not follow for headers, follow in sent) + assert all(follow and "X-api-key" not in headers for url, headers, follow in asked if "/api/test/" not in url) + assert not any(KEY in url for url, _, _ in asked) + + +def test_a_key_no_header_can_carry_is_refused_without_being_shown(monkeypatch, capsys): + with pytest.raises(SystemExit) as exited: + _main(monkeypatch, capsys, _fleet_site(), key="rkNEWLINEsecret\nrest") + assert exited.value.code == 2 + err = capsys.readouterr().err + assert vd.KEY_VARIABLE in err + assert "rkNEWLINEsecret" not in err + + +def test_a_request_with_the_key_is_not_redirected(monkeypatch): + built = [] + monkeypatch.setattr(vd.urllib.request, "build_opener", lambda *handlers: built.append(handlers) or None) + with pytest.raises(AttributeError): + vd._open(vd.urllib.request.Request("https://api.example.test/"), follow=False) + assert any(isinstance(h, vd._Stay) for h in built[-1]) + with pytest.raises(AttributeError): + vd._open(vd.urllib.request.Request("https://api.example.test/"), follow=True) + assert not any(isinstance(h, vd._Stay) for h in built[-1]) + assert vd._Stay().redirect_request(None, None, 302, "Found", {}, "https://elsewhere.example.test/") is None + + +def test_keyed_checks_skipped_are_said_so_and_not_counted_as_passed(monkeypatch, capsys): + code, out, _, asked = _main(monkeypatch, capsys, _fleet_site()) + assert code == 0 + assert re.search(r"^ SKIP +keyed checks", out, re.MULTILINE) + assert out.rstrip().splitlines()[-1].startswith("PASS with keyed checks skipped, not passed.") + assert not any(url.endswith("/api/test/dashboard") for url, _, _ in asked) + + +def test_the_spa_fallback_answering_the_bundle_fails(): + fallback = _resp(INDEX_HTML, ctype="text/html", marked=False) + report, _, _ = _verify(_site(**{f"{APP}/assets/index-abc.js": fallback})) + assert "SPA" in _rows(report, "FAIL")["bundle"] + + +def test_a_module_script_whose_src_comes_first_is_found(): + page = INDEX_HTML.replace( + 'type="module" crossorigin src="/assets/index-abc.js"', 'src="/assets/index-abc.js" type="module"' + ) + report, _, _ = _verify(_site(**{f"{APP}/": _resp(page, ctype="text/html", marked=False)})) + assert "index-abc.js" in _rows(report, "PASS")["bundle"] + + +def test_the_bundle_is_named_by_the_module_script_and_other_origins_are_not_read(): + report, _, asked = _verify(_site()) + assert "index-abc.js" in _rows(report, "PASS")["bundle"] + assert f"https://{APP}/theme-boot.js" in asked + assert not any("unpkg.example.test" in url for url in asked) + assert "https://unpkg.example.test/leaflet.css" in _rows(report, "INFO")["other hosts"] + + +def test_an_expected_string_is_found_in_a_lazy_chunk(): + report, _, _ = _verify(_site(), expect=["Shipped today"]) + assert "Lazy-def.js" in _rows(report, "PASS")["expect"] + + +def test_every_asset_the_page_reaches_is_checked_without_being_asked(): + report, _, asked = _verify(_site()) + assert {f"https://{APP}/assets/Lazy-def.js", f"https://{APP}/assets/index-xyz.css"} <= set(asked) + assert "5 other asset(s)" in _rows(report, "PASS")["bundle"] + + +def test_a_missing_expected_string_fails_after_searching_every_chunk(): + report, _, asked = _verify(_site(), expect=["never shipped"]) + assert "7 file(s)" in _rows(report, "FAIL")["expect"] + assert {f"https://{APP}/assets/Lazy-def.js", f"https://{APP}/assets/Other-ghi.js"} <= set(asked) + + +def test_an_unreachable_host_fails_rather_than_raising(): + report, _, _ = _verify(_site(**{HEALTH: vd.Unreachable("api: timed out")})) + assert "timed out" in _rows(report, "FAIL")["health"] + + +@pytest.mark.parametrize( + ("env", "expected"), + [ + ("prod", ("api.retina.fm", "app.retina.fm", "production", False, True)), + ("staging", ("staging-api.retina.fm", "staging-app.retina.fm", "test", False, False)), + ("test", ("test-api.retina.fm", "test-app.retina.fm", "test", True, False)), + ], +) +def test_each_environment_is_read_from_its_overlay(env, expected): + assert vd.environment(env) == vd.Environment(*expected) + + +def test_the_marker_is_the_one_the_smoke_suites_assert(): + assert vd.origin_marker() == MARKER + + +def test_an_unknown_environment_is_a_usage_error(): + with pytest.raises(SystemExit) as exited: + vd.main(["nowhere"]) + assert exited.value.code == 2 + + +def test_a_feed_not_yet_published_after_a_restart_is_waited_for(): + unpublished = _resp({"now": 0, "aircraft": []}) + report, clock, _ = _verify(_site(**{FEED: [unpublished, _feed(1)]}), wait_s=600) + assert _rows(report, "FAIL") == {} + assert clock.sleeps == 1 + + +def test_a_feed_left_over_from_before_a_restart_is_waited_for(): + report, clock, _ = _verify(_site(**{FEED: [_feed(3, age=400), _feed(3)]}), wait_s=600) + assert _rows(report, "FAIL") == {} + assert clock.sleeps == 1 + + +def test_a_server_answering_5xx_while_it_restarts_is_waited_for(): + warming = _resp("bad gateway", status=502, ctype="text/html", marked=False) + report, clock, _ = _verify(_site(**{NODES: [warming, _nodes()]}), wait_s=600) + assert _rows(report, "FAIL") == {} + assert clock.sleeps == 1 + + +def test_the_fleet_s_unfiltered_feed_is_waited_for(): + report, clock, _ = _verify(_fleet_site(**{ALL: [_feed(0), _feed(2, by="synth-1")]}), wait_s=600, fleet=True) + assert _rows(report, "FAIL") == {} + assert clock.sleeps == 1 + + +def test_an_error_from_the_edge_is_not_blamed_on_the_origin_rule(): + refused = _resp("error code: 1010", status=403, ctype="text/plain", marked=False) + report, _, _ = _verify(_site(**{HEALTH: refused})) + assert "origin" not in _rows(report, "FAIL") + + +def test_a_lazy_chunk_answered_by_the_spa_fallback_is_not_searched(): + fallback = _resp(INDEX_HTML + "Shipped today", ctype="text/html", marked=False) + report, _, _ = _verify(_site(**{f"{APP}/assets/Lazy-def.js": fallback}), expect=["Shipped today"]) + assert "expect" in _rows(report, "FAIL") + + +def test_a_connection_dropped_mid_response_is_unreachable_not_a_crash(monkeypatch): + def drop(*args, **kwargs): + raise http.client.IncompleteRead(b"partial") + + monkeypatch.setattr(vd, "_open", drop) + monkeypatch.setattr(vd.time, "sleep", lambda s: None) + with pytest.raises(vd.Unreachable): + vd.fetch("https://api.example.test/api/health") + + +def test_a_lazy_chunk_the_deploy_lacks_fails_the_bundle(): + missing = _resp("404", status=404, ctype="text/html", marked=False) + report, _, _ = _verify(_site(**{f"{APP}/assets/Other-ghi.js": missing})) + assert "the deploy does not carry it" in _rows(report, "FAIL")["bundle"] + + +def test_an_asset_answering_5xx_once_is_asked_again(): + flaky = [_resp("bad gateway", status=502, ctype="text/html", marked=False), _resp("x", ctype="text/javascript")] + report, _, _ = _verify(_site(**{f"{APP}/assets/Other-ghi.js": flaky})) + assert "bundle" in _rows(report, "PASS") + + +def test_a_string_only_a_stylesheet_carries_is_found(): + report, _, _ = _verify(_site(), expect=["retina-banner"]) + assert "index-xyz.css" in _rows(report, "PASS")["expect"] + + +def test_a_string_the_change_removes_must_be_gone(): + report, _, _ = _verify(_site(), absent=["Shipped today"]) + assert "still in Lazy-def.js" in _rows(report, "FAIL")["expect"] + report, _, _ = _verify(_site(), absent=["retired wording"]) + assert "expect" in _rows(report, "PASS") + + +def test_a_4xx_ends_the_wait_at_once(): + gone = _resp("not found", status=404, ctype="text/html", marked=False) + report, clock, _ = _verify(_site(**{NODES: gone}), wait_s=600) + assert "404" in _rows(report, "FAIL")["map"] + assert clock.sleeps == 0 + + +def test_a_4xx_on_the_unfiltered_feed_ends_the_wait_at_once(): + gone = _resp("not found", status=404, ctype="text/html", marked=False) + report, clock, _ = _verify(_fleet_site(**{ALL: gone}), wait_s=600, fleet=True) + assert "404" in _rows(report, "FAIL")["fleet"] + assert clock.sleeps == 0 + + +def test_a_lazy_page_s_stylesheet_is_read(): + report, _, _ = _verify(_site(), expect=["map-surface"]) + assert "Lazy-def.css" in _rows(report, "PASS")["expect"] + missing = _resp("404", status=404, ctype="text/html", marked=False) + report, _, _ = _verify(_site(**{f"{APP}/assets/Lazy-def.css": missing})) + assert "Lazy-def.css" in _rows(report, "FAIL")["bundle"] + + +def test_a_script_the_page_names_outside_assets_is_read(): + report, _, _ = _verify(_site(), expect=["dataset.theme"]) + assert "theme-boot.js" in _rows(report, "PASS")["expect"] + + +def test_one_failed_read_on_the_last_poll_after_good_ones_is_a_blip(): + blip = vd.Unreachable("api.example.test: timed out") + health = [_health(), _health(), _health(), blip] + report, _, _ = _verify(_site(**{FEED: _feed(0), HEALTH: health}), wait_s=60) + assert "health" not in _rows(report, "FAIL") + assert "quiet sky" in _rows(report, "WARN")["map"] + + +def test_a_blip_is_forgiven_only_when_the_poll_before_it_was_good(): + down = vd.Unreachable("api.example.test: timed out") + health = [_health(), down, down, down] + report, _, _ = _verify(_site(**{FEED: _feed(0), HEALTH: health}), wait_s=60) + assert "timed out" in _rows(report, "FAIL")["health"] + + +def test_a_node_list_blip_on_the_last_poll_keeps_the_earlier_list(): + blip = vd.Unreachable("api.example.test: timed out") + nodes = [_nodes(), _nodes(), _nodes(), blip] + report, _, _ = _verify(_site(**{FEED: _feed(0), NODES: nodes}), wait_s=60) + assert "quiet sky" in _rows(report, "WARN")["map"] + assert _rows(report, "INFO")["real nodes"].startswith("3 connected") + + +def test_an_unfiltered_feed_blip_on_the_last_poll_keeps_the_earlier_feed(): + blip = vd.Unreachable("app.example.test: timed out") + fleet = [_feed(0), _feed(0), _feed(0), blip] + report, _, _ = _verify(_fleet_site(**{ALL: fleet}), wait_s=60, fleet=True) + assert "was seen by the 1 connected synthetic" in _rows(report, "FAIL")["fleet"] + + +def test_a_rate_limit_is_waited_out(): + limited = _resp("too many requests", status=429, ctype="text/plain", marked=False) + report, clock, _ = _verify(_site(**{NODES: [limited, _nodes()]}), wait_s=600) + assert _rows(report, "FAIL") == {} + assert clock.sleeps == 1 + + +def test_the_page_is_asked_twice_on_a_5xx(): + page = [_resp("bad gateway", status=502, ctype="text/html", marked=False), _resp(INDEX_HTML, ctype="text/html")] + report, _, _ = _verify(_site(**{f"{APP}/": page})) + assert "bundle" in _rows(report, "PASS") + + +def test_a_walk_at_its_bound_still_reads_what_fits(monkeypatch): + monkeypatch.setattr(vd, "MAX_ASSETS", 4) + report, _, _ = _verify(_site(), expect=["Shipped today"]) + assert "Lazy-def.js" in _rows(report, "PASS")["expect"] + assert "2 more unread" in _rows(report, "WARN")["bundle"] + + +def test_a_fault_waiting_cannot_clear_ends_the_wait_at_once(): + bad_cert = vd.Unreachable("api.example.test: certificate verification failed", final=True) + report, clock, _ = _verify(_fleet_site(**{HEALTH: bad_cert, ALL: _feed(0)}), wait_s=600, fleet=True) + assert "certificate" in _rows(report, "FAIL")["health"] + assert clock.sleeps == 0 + # One poll was taken, so no row may claim the fleet stayed idle through a wait. + assert "was seen by the 1 connected synthetic" in _rows(report, "FAIL")["fleet"] + assert "of polling" not in report.render() + assert "post-restart window" not in report.render() + + +def test_a_walk_cut_short_says_so_and_cannot_prove_an_absence(monkeypatch): + monkeypatch.setattr(vd, "MAX_ASSETS", 3) + report, _, _ = _verify(_site(), absent=["retired wording"]) + assert "unread" in _rows(report, "WARN")["bundle"] + assert "not read" in _rows(report, "FAIL")["expect"] + + +def test_the_script_content_types_match_the_smoke_suite_s(): + shell = (Path(__file__).resolve().parents[2] / "deploy" / "page-asset.sh").read_text() + accepted = set(re.findall(r"200:\*(\w+)\*", shell)) + assert accepted == set(vd.JAVASCRIPT.pattern.split("|")) + + +def test_the_page_itself_is_searched(): + report, _, _ = _verify(_site(), expect=['