Installer report, deeper non-invasive monitoring, no games in emails

README.md

@@ -89,6 +89,34 @@ whoever owns the eero, or to AT&T.
 - **From a terminal:** `.venv\Scripts\python.exe -m hogwatch send-report` sends one now;
   add `--preview` to save it as `data\report_preview.html` instead.
 
+## Report for your installer
+
+**Report for your installer** on the dashboard (or `python -m hogwatch installer-report`, which also
+saves a PDF) builds a technical report for whoever installed or services the eeros:
+
+- **Summary of findings with numbers.** Which link is dropping and how often, eero reconnects,
+  whether the Ethernet run between eeros ever links, the band and channel width on each hop
+  (and whether it overlaps radar-shared DFS channels), mesh route changes, double NAT behind the
+  ISP gateway, and whether the internet line itself is healthy.
+- **Requested work**, generated from those findings.
+- **Evidence:** the eero layout, ping delay, jitter and loss to every hop, dropouts by day and time
+  of day, the longest dropouts, eero reconnects, Ethernet port link checks, radio and channel history,
+  speed tests, and an appendix listing every dropout.
+
+The report reads only measurements of the network itself. It never includes device names, people,
+programs or games, so it's safe to send to an outside company. Add your own notes in the report
+before printing it.
+
+### What's monitored (all non-invasive)
+
+- One small ping per second to this PC's eero, the main eero, every other eero, the ISP gateway
+  and two internet servers, with delay, loss and jitter stored per 10 seconds.
+- Each eero's status every 2 minutes, as eero's app reports it: backhaul (cable or wireless band,
+  and which eero it relays through), signal bars, channel, width, how busy each radio is, connected
+  device counts, firmware, and every Ethernet port's link state and speed.
+- eero reconnects, eero speed tests, and this PC's own network-card link going down or up.
+- Nothing about the content of anyone's traffic is captured.
+
 ## Reading the results
 
 - **"Internet crawled"**: the line was full. Look at "Biggest user". An **upload**
@@ -123,5 +151,5 @@ Settings › eero Labs › **Optimize for Conferencing and Gaming**.
 - The dashboard has no external scripts, so it keeps working while the internet is down.
 - The dashboard only answers requests addressed to `127.0.0.1` or `localhost` (blocks DNS
   rebinding), and every change needs a custom header that other websites can't send.
-- Command line: `.venv\Scripts\python.exe -m hogwatch run | eero-login | eero-check | selftest | send-report`
+- Command line: `.venv\Scripts\python.exe -m hogwatch run | eero-login | eero-check | selftest | send-report | installer-report`
 - Tests: `.venv\Scripts\python.exe -m unittest discover -s tests`

hogwatch/__main__.py

@@ -5,6 +5,7 @@
     python -m hogwatch eero-check           show what the eero reports right now
     python -m hogwatch selftest             15-second check that everything works
     python -m hogwatch send-report          email the last 24 hours now (--preview: save it as HTML instead)
+    python -m hogwatch installer-report     technical report for the eero installer (HTML + PDF in data\\)
 """
 
 from __future__ import annotations
@@ -223,6 +224,24 @@ def cmd_send_report(args) -> int:
     return 1
 
 
+def cmd_installer_report(args) -> int:
+    """Write the installer report as HTML and (with Edge or Chrome installed) PDF in data\\."""
+    from .db import DB
+    from .installer import build_installer_report, save_pdf
+
+    db = DB(DATA_DIR / "hogwatch.db")
+    page = build_installer_report(db, days=args.days, note=args.note or "")
+    html_path = DATA_DIR / "installer_report.html"
+    html_path.write_text(page, encoding="utf-8")
+    print(f"HTML report: {html_path}")
+    pdf_path = DATA_DIR / "installer_report.pdf"
+    if save_pdf(html_path, pdf_path, DATA_DIR / "pdf-browser-profile"):
+        print(f"PDF report:  {pdf_path}")
+    else:
+        print("No Edge or Chrome found for the PDF: open the HTML report and use Print > Save as PDF.")
+    return 0
+
+
 def main() -> int:
     """Parse the command and run it."""
     p = argparse.ArgumentParser(prog="hogwatch", description="Find out who is slowing the internet down.")
@@ -236,12 +255,16 @@ def main() -> int:
     sr = sub.add_parser("send-report", help="email the recent slowdowns and dropouts now")
     sr.add_argument("--hours", type=float, default=24, help="how far back to cover (default 24)")
     sr.add_argument("--preview", action="store_true", help="save the report as HTML instead of sending")
+    ir = sub.add_parser("installer-report", help="technical report for whoever services the eeros (HTML + PDF)")
+    ir.add_argument("--days", type=float, default=7, help="how far back to cover (1-14, default 7)")
+    ir.add_argument("--note", help="notes from the homeowner to include")
     args = p.parse_args()
     if args.cmd is None:
         args = p.parse_args(["run"])
     DATA_DIR.mkdir(parents=True, exist_ok=True)
     return {"run": cmd_run, "eero-login": cmd_eero_login, "eero-check": cmd_eero_check,
-            "selftest": cmd_selftest, "send-report": cmd_send_report}[args.cmd](args)
+            "selftest": cmd_selftest, "send-report": cmd_send_report,
+            "installer-report": cmd_installer_report}[args.cmd](args)
 
 
 if __name__ == "__main__":

hogwatch/collector.py

@@ -78,11 +78,11 @@ def __init__(self, cfg: dict, db: DB):
     def start(self) -> None:
         """Discover the network path, start per-program tracing, launch the threads."""
         self._close_stale_incidents()
-        lan = self.cfg.get("lan_target") or ping.hop(1)
-        isp = ping.hop(2, toward=self.cfg["internet_targets"][0])
-        self.targets = {"lan": lan, "isp": isp, "internet": self.cfg["internet_targets"]}
-        self.db.set_meta("targets", json.dumps(self.targets))
-        log.info("targets: %s", self.targets)
+        self.targets = {"lan": None, "isp": None, "internet": self.cfg["internet_targets"]}
+        self._discover_targets()  # retried from the ping loop if the network isn't up yet
+        if not self.db.get_meta("link_watch_since"):
+            # Reports may only claim "the PC's cable never dropped" for the time it was watched.
+            self.db.set_meta("link_watch_since", str(int(time.time())))
         self.pc = PCMonitor()
         for fn in (self._ping_loop, self._pc_loop, self._eero_loop, self._housekeeping_loop):
             threading.Thread(target=self._guard(fn), name=fn.__name__.strip("_"), daemon=True).start()
@@ -122,35 +122,61 @@ def _ping_loop(self) -> None:
         interval = float(self.cfg["ping_interval_s"])
         timeout = int(self.cfg["ping_timeout_ms"])
         inet = list(self.cfg["internet_targets"])
-        pool = ThreadPoolExecutor(max_workers=len(inet) + 3, thread_name_prefix="icmp")
+        pool = ThreadPoolExecutor(max_workers=12, thread_name_prefix="icmp")
         buckets: dict[str, dict] = {}
+        last_rtt: dict[str, float | None] = {}  # previous ping per role, for jitter
         bucket_ts = int(time.time()) // BUCKET_S * BUCKET_S
+        last_round = time.time()
+        last_discovery = 0.0
         while not self.stopping.is_set():
             t0 = time.time()
+            if t0 - last_round > 30:
+                # Monitoring paused (the PC slept or hung): close anything open at the last
+                # moment we saw it, so the gap isn't counted as a 10-hour dropout.
+                self._close_after_gap(last_round)
+                last_rtt.clear()
+            last_round = t0
+            if (not self.targets.get("lan") or not self.targets.get("isp")) and t0 - last_discovery > 30:
+                # At boot the network may not be up when HogWatch starts; keep looking.
+                last_discovery = t0
+                self._discover_targets()
             lan = self.targets.get("lan")
             node = self.targets.get("node")  # the eero this PC plugs into, once eero tells us
+            units = dict(self.targets.get("units") or {})  # the other eeros: one record per link
             futs = {("inet", n): pool.submit(ping.ping, ip, timeout) for n, ip in enumerate(inet)}
             if lan:
                 futs["lan"] = pool.submit(ping.ping, lan, timeout)
             if node:
                 futs["node"] = pool.submit(ping.ping, node, timeout)
+            for name, ip in units.items():
+                futs[("unit", name)] = pool.submit(ping.ping, ip, timeout)
             futs["isp"] = pool.submit(ping.ping, inet[0], timeout, 2)
             res = {k: f.result()[0] for k, f in futs.items()}
-            internet = min((v for k, v in res.items() if isinstance(k, tuple) and v is not None), default=None)
+            internet = min((v for k, v in res.items() if k[0] == "inet" and v is not None), default=None)
             lan_ms, isp_ms, node_ms = res.get("lan"), res.get("isp"), res.get("node")
 
-            for role, v, probed in (("internet", internet, True), ("lan", lan_ms, bool(lan)),
-                                    ("isp", isp_ms, True), ("node", node_ms, bool(node))):
+            rows = [("internet", internet, True), ("lan", lan_ms, bool(lan)),
+                    ("isp", isp_ms, True), ("node", node_ms, bool(node))]
+            rows += [(f"unit:{name}", res.get(("unit", name)), True) for name in units]
+            for role, v, probed in rows:
                 if not probed:
                     continue
-                b = buckets.setdefault(role, {"sent": 0, "lost": 0, "sum": 0.0, "n": 0, "max": None})
+                b = buckets.setdefault(role, {"sent": 0, "lost": 0, "sum": 0.0, "n": 0, "max": None,
+                                              "jsum": 0.0, "jn": 0})
                 b["sent"] += 1
                 if v is None:
                     b["lost"] += 1
                 else:
                     b["sum"] += v
                     b["n"] += 1
                     b["max"] = v if b["max"] is None else max(b["max"], v)
+                    # Jitter = how much each ping differs from the one before. Games and voice
+                    # suffer from uneven delay even when the average looks fine.
+                    prev = last_rtt.get(role)
+                    if prev is not None:
+                        b["jsum"] += abs(v - prev)
+                        b["jn"] += 1
+                last_rtt[role] = v
 
             with self.lock:
                 self.rounds.append((t0, internet, lan_ms, isp_ms, node_ms))
@@ -169,13 +195,46 @@ def _ping_loop(self) -> None:
 
             if t0 >= bucket_ts + BUCKET_S:
                 self.db.executemany(
-                    "INSERT INTO latency(ts, role, sent, lost, avg_ms, max_ms) VALUES(?,?,?,?,?,?)",
-                    [(bucket_ts, role, b["sent"], b["lost"], (b["sum"] / b["n"]) if b["n"] else None, b["max"])
+                    "INSERT INTO latency(ts, role, sent, lost, avg_ms, max_ms, jitter_ms) VALUES(?,?,?,?,?,?,?)",
+                    [(bucket_ts, role, b["sent"], b["lost"], (b["sum"] / b["n"]) if b["n"] else None, b["max"],
+                      (b["jsum"] / b["jn"]) if b["jn"] else None)
                      for role, b in buckets.items()])
                 buckets = {}
                 bucket_ts = int(t0) // BUCKET_S * BUCKET_S
             self.stopping.wait(max(0.0, interval - (time.time() - t0)))
 
+    def _discover_targets(self) -> None:
+        """Find the main eero (hop 1) and the ISP gateway (hop 2); fall back to the eero's own data."""
+        lan = self.cfg.get("lan_target") or ping.hop(1)
+        isp = ping.hop(2, toward=self.cfg["internet_targets"][0])
+        with self.lock:
+            if lan:
+                self.targets["lan"] = lan
+            elif not self.targets.get("lan"):
+                gw = next((u for u in self.path.get("units") or [] if u.get("gateway") and u.get("ip")), None)
+                if gw:
+                    self.targets["lan"] = gw["ip"]
+            if isp:
+                self.targets["isp"] = isp
+            targets = dict(self.targets)
+        self.db.set_meta("targets", json.dumps(targets))
+        if isp:
+            self.db.set_meta("isp_seen", isp)  # remembered even if a later discovery fails
+        if lan or isp:
+            log.info("targets: %s", targets)
+
+    def _close_after_gap(self, last_seen: float) -> None:
+        """Close an open dropout or slowdown at `last_seen` after a pause in monitoring."""
+        hiccup = self.hiccups.flush()
+        if hiccup:
+            self._record_hiccup(hiccup)
+        with self.lock:
+            if self.incident:
+                self._incident_end(last_seen)
+            self.detector.active = False
+            self.detector.recent.clear()
+        log.info("monitoring resumed after a %.0f s pause", time.time() - last_seen)
+
     # ------------------------------------------------------------------ lag spikes
 
     def _record_hiccup(self, hic: dict) -> None:
@@ -216,16 +275,29 @@ def _refresh_units(self, connected: list[dict], now: float) -> None:
                 "gateway": gateway["name"] if gateway else None, "chain": [c["name"] for c in chain],
                 "radio": node["radio"] if node else None, "wired": node["wired"] if node else None,
                 "on_link_units": on_link,
-                "units": [{k: u[k] for k in ("name", "gateway", "wired", "upstream", "radio", "model")} for u in units]}
+                "units": [{k: u[k] for k in ("name", "gateway", "wired", "upstream", "radio", "model", "ip")}
+                          for u in units]}
         with self.lock:
             self.path = path
             if node and node["ip"]:
                 self.targets["node"] = node["ip"]
             else:
                 self.targets.pop("node", None)
+            if not self.targets.get("lan") and gateway and gateway["ip"]:
+                self.targets["lan"] = gateway["ip"]  # hop discovery failed; eero told us anyway
+            # The other eeros (not ours, not the main one, which are pinged already) get pinged
+            # too, so every wireless link between eeros has its own record.
+            skip = {node["name"] if node else None, gateway["name"] if gateway else None}
+            self.targets["units"] = {u["name"]: u["ip"] for u in units if u["ip"] and u["name"] not in skip}
             targets = dict(self.targets)
         self.db.set_meta("eero_path", json.dumps(path))
         self.db.set_meta("targets", json.dumps(targets))
+        self.db.executemany(
+            "INSERT INTO unit_sample(ts, unit, gateway, wired, upstream, radio, bars, status, firmware, "
+            "wired_clients, wireless_clients, bands, ports) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            [(int(now), u["name"], int(u["gateway"]), int(u["wired"]), u["upstream"], u["radio"], u["bars"],
+              u["status"], u["firmware"], u["wired_clients"], u["wireless_clients"], json.dumps(u["bands"]),
+              json.dumps(u["ports"])) for u in units])
         for u in units:
             ts = u["reconnected_ts"]
             if not ts:
@@ -297,12 +369,23 @@ def _incident_end(self, ts: float) -> None:
     # ------------------------------------------------------------------ this PC
 
     def _pc_loop(self) -> None:
-        """Every 10s: record card totals and per-program usage."""
+        """Every 10s: record card totals and per-program usage, and note the PC's cable link changing."""
         last_update = 0.0
+        last_link = None
         while not self.stopping.wait(BUCKET_S):
             s = self.pc.sample()
             ts = int(time.time())
             secs = s["secs"]
+            # This PC's own Ethernet link: a drop or a fall to 100 Mbps would point at the
+            # PC's cable or port rather than the eeros.
+            stats = psutil.net_if_stats().get(self.pc.nic) if self.pc.nic else None
+            link = (bool(stats.isup), int(stats.speed or 0)) if stats else None
+            if link and last_link and link != last_link:
+                what = "up" if link[0] else "down"
+                self.db.execute("INSERT INTO link_event(ts, what, detail) VALUES(?,?,?)",
+                                (ts, what, json.dumps({"speed_mbps": link[1], "before_mbps": last_link[1]})))
+                log.warning("this PC's network link went %s (%s Mbps)", what, link[1])
+            last_link = link or last_link
             self.db.execute("INSERT INTO pc_total(ts, secs, rx_bytes, tx_bytes) VALUES(?,?,?,?)",
                             (ts, secs, s["rx"], s["tx"]))
             self.db.executemany("INSERT INTO pc_app(ts, app, rx_bytes, tx_bytes) VALUES(?,?,?,?)",
@@ -379,14 +462,16 @@ def _eero_loop(self) -> None:
                                       "usage_reported": bool(with_usage), "network": self.eero.network_name}
                     if self.incident:
                         self.incident.add_eero(connected)
-                if now - last_units > 300:
+                if now - last_units > 120:
                     last_units = now
                     self._refresh_units(connected, now)
                 if now - last_speed > 1800:
                     last_speed = now
                     speed = speed_from_network(self.eero.network())
                     if speed:
                         self.db.set_meta("eero_speed", json.dumps(speed))
+                        self.db.execute("INSERT OR IGNORE INTO speedtest(date, ts, down, up) VALUES(?,?,?,?)",
+                                        (speed.get("date") or str(int(now)), int(now), speed.get("down"), speed.get("up")))
             except LoginNeeded as e:
                 with self.lock:
                     self.eero_live = {"status": "login_needed", "message": str(e), "devices": [], "ts": None}