diff --git a/server-side/README.md b/server-side/README.md index 1ff0fff..4fd4d91 100644 --- a/server-side/README.md +++ b/server-side/README.md @@ -446,6 +446,22 @@ block rather than a warning. Only enable it once renewal is dependable: single a record, `certbot renew --dry-run` passing, `certbot.timer` active. Shorten `max-age` to something like `300` while testing if you want an easy way back. +### Views of pages this app does not serve + +The DBMF dashboard at `dbmf.bluemagicai.com` is a static page on GitHub Pages: no +server, no hook, no access log. Its views reach the same `visitor_logs` table and the +same /admin charts through a one-pixel beacon, `GET /seen.gif?p=dbmf&r=`, +which the publish step in the `dbmf-tracker` repo (`deploy/publish_pages.py`) appends to +the page. The app answers with a 1x1 GIF and writes the same two rows one of its own +document loads writes: a `landing` carrying the referrer and a `pageview` the admin +page counts, under path `/dbmf`. The `p` value is matched against an allowlist in +`routing.EXTERNAL_PAGES` (add a key there to admit another external page) and `r` is +kept only when it looks like a web address. Nothing to configure on the box: the route +is part of the app and rides the same nginx `proxy_pass`, so `X-Forwarded-For` reaches +it like any other request. Two limits worth knowing: an ad blocker that drops the +pixel drops the view, and a client that fetches HTML without loading images never +appears, which for that page is the bot filter. + ## 9. GoatCounter analytics (optional) Client-side analytics beside the app's own visit log, not replacing it: the built-in diff --git a/src/app_cot.py b/src/app_cot.py index b715d27..182ead8 100644 --- a/src/app_cot.py +++ b/src/app_cot.py @@ -448,12 +448,20 @@ def _visit_worker(): utils.cot_logger.error(f"visit worker failed on {row.get('path')}: {e}") -def _enqueue_visit(kind, path): +_REFERRER_FROM_REQUEST = object() + + +def _enqueue_visit(kind, path, referrer=_REFERRER_FROM_REQUEST): """Build a visit row from the CURRENT flask request and hand it to the worker. Must run inside a request context (a before_request hook or a callback, both qualify). The worker is started lazily rather than at import so importing this module (tests, tooling) starts no thread and writes nothing. + + `referrer` defaults to the request's own: kept on a landing, dropped on a + pageview (see the row below). The beacon passes one explicitly, None + included, because on that request the Referer header names the external + page itself, not where its visitor came from. """ global _visit_worker_started ip_addr = visitors.client_ip(request.headers.get('X-Forwarded-For'), @@ -469,7 +477,8 @@ def _enqueue_visit(kind, path): 'is_bot': bot, # The referrer on a callback POST is just our own page url; only the # document load carries where the visitor actually came from. - 'referrer': request.referrer if kind == 'landing' else None, + 'referrer': ((request.referrer if kind == 'landing' else None) + if referrer is _REFERRER_FROM_REQUEST else referrer), }) with _visit_worker_lock: if not _visit_worker_started: @@ -493,13 +502,42 @@ def record_visit(): '/_dash-component-suites/', '/assets/', '/favicon.ico', - '/_favicon.ico' + '/_favicon.ico', + # The external-page beacon logs its OWN rows (below); the request that + # carries it is not a visit to this site. + routing.BEACON_PATH, ] if not any(request.path.startswith(path) for path in ignored_paths): _enqueue_visit('landing', request.path) +@app.server.route(routing.BEACON_PATH) +def external_page_beacon(): + """Log a view of a page served from another origin; see routing.BEACON_PATH. + + The DBMF dashboard is a static page on GitHub Pages: no server, no hook, no + access log. It loads this pixel once per document load with `p` naming the + page and `r` carrying its `document.referrer`, and the result is exactly the + pair of rows one of this app's own document loads produces: a 'landing' + carrying the referrer and a 'pageview' that the admin page counts (the two + kinds are never summed; see the visit-capture note above). Everything after + the enqueue is shared: the forwarded client address, the daily visitor hash, + the geolocation cache, the datacenter flag. A visitor who opens both sites + on one day is therefore one visitor id across them. + + `p` is matched against an allowlist and `r` is cleaned, because both are + query values anyone can write. An unlisted `p` still gets the pixel: the + answer is for the browser, and there is nothing to tell a stranger. + """ + path = routing.external_page_path(request.args.get('p')) + if path: + _enqueue_visit('landing', path, + referrer=visitors.clean_referrer(request.args.get('r'))) + _enqueue_visit('pageview', path) + return routing.PIXEL_GIF, 200, {'Content-Type': 'image/gif'} + + # The dropdown pages, as data, so the items and the active-state wiring below # can never disagree about what is in the menus. OI Alignment and About are # absent on purpose: both were promoted to top-level links. diff --git a/src/routing.py b/src/routing.py index 7c1791b..65dce5b 100644 --- a/src/routing.py +++ b/src/routing.py @@ -25,6 +25,16 @@ '/static/', ) +#: The view beacon: a one-pixel image a page served from ANOTHER origin loads so +#: its views land in the same visitor log as this app's pages. The first such +#: page is the DBMF dashboard, a static GitHub Pages site under the same domain +#: (dbmf.bluemagicai.com) with no server of its own, so nothing on that side can +#: run `record_visit`. The name is deliberately bland: the common blocklists +#: match /beacon, /track, /pixel.gif and /collect, and a request they drop is a +#: view never counted. Named in EXTRA_PATHS below so the unknown-path guard +#: admits it. +BEACON_PATH = '/seen.gif' + #: Real paths belonging to no page: the browser's automatic favicon request, Dash's #: own copy of it, the email-link and crawler endpoints (plain Flask routes in #: app_cot, which `app.routes` does not list because it only knows Dash's own), @@ -33,7 +43,28 @@ EXTRA_PATHS = frozenset({'/favicon.ico', '/_favicon.ico', '/confirm', '/unsubscribe', '/graphs', '/robots.txt', '/sitemap.xml', - '/manifest.webmanifest', '/sw.js'}) + '/manifest.webmanifest', '/sw.js', + BEACON_PATH}) + +#: Which external pages the beacon will log, keyed by the `p` query value the +#: page sends, valued by the path recorded for it. An ALLOWLIST, not a +#: passthrough: the query string is anyone's to write, and the visitor log must +#: never carry a string a stranger chose. An unlisted key is answered with the +#: pixel and logged as nothing. +EXTERNAL_PAGES = {'dbmf': '/dbmf'} + +#: A 1x1 transparent GIF, 43 bytes. Answering with a real image rather than a +#: 204 keeps a plain beacon from logging a broken-image error in the +#: visitor's console; pinned by test_visit_capture through PIL. +PIXEL_GIF = (b'GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff' + b'!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00' + b'\x00\x02\x02D\x01\x00;') + + +def external_page_path(key): + """The path the visitor log records for an external page's beacon, or None.""" + return EXTERNAL_PAGES.get(key or '') + #: Operator surfaces and internal viewers: out of the sitemap, disallowed in #: robots.txt. A search result should never land a visitor on the admin login @@ -221,6 +252,12 @@ def cache_policy(path, fingerprinted, content_type): # worker script's HTTP cache at a day, but explicit beats capped). if path in ('/manifest.webmanifest', '/sw.js'): return 'no-cache' + # The view beacon counts document loads, so a cached pixel is a view that + # never reached the log. `no-store`, not `no-cache`: a revalidation is a + # request too, but one the browser may answer from its own copy on a + # back/forward navigation without asking. + if path == BEACON_PATH: + return 'no-store' # A DATED weekly report barely changes once its week has passed (a store # revision or a copy tweak, both rare), and its cold render is the most # expensive page this app serves, so a day of public caching is cheap diff --git a/src/visitors.py b/src/visitors.py index e034340..72e7cfd 100644 --- a/src/visitors.py +++ b/src/visitors.py @@ -49,6 +49,23 @@ def is_bot(user_agent): return any(marker in ua for marker in BOT_MARKERS) +def clean_referrer(value, limit=512): + """A referrer a client REPORTED (the beacon's `r` query value), or None. + + A browser-set Referer header is bounded by the browser; a query value is + bounded by whoever wrote the URL. Keep it only when it looks like a web + address, and never longer than `limit`, so the referrer column cannot be + used to park arbitrary text in the admin table. Empty means a direct visit, + which is None rather than '' to match what the Referer header gives. + """ + if not value: + return None + value = value.strip() + if not value.lower().startswith(('http://', 'https://')): + return None + return value[:limit] + + def visitor_id(ip, user_agent, day=None): """A daily-rotating pseudonymous id: sha256 of day, ip and user agent, 16 hex chars. diff --git a/tests/test_routing.py b/tests/test_routing.py index e6e6b02..c9d2941 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -11,7 +11,14 @@ """ import pytest -from routing import is_known_path, is_vendor_sourcemap, not_found_page +from routing import ( + BEACON_PATH, + cache_policy, + external_page_path, + is_known_path, + is_vendor_sourcemap, + not_found_page, +) # What the live app carries, as of the page registry at the time of writing. The nested # and the underscored ones are the interesting entries: a naive "one path segment" @@ -74,7 +81,8 @@ def test_the_browsers_automatic_favicon_request_is_not_a_probe(path): @pytest.mark.parametrize("path", ['/confirm', '/unsubscribe', '/robots.txt', '/sitemap.xml', - '/weekly', '/weekly/2026-08-25']) + '/weekly', '/weekly/2026-08-25', + '/seen.gif']) def test_the_flask_route_endpoints_are_served(path): """Plain Flask routes on app.server (see app_cot), which `app.routes` does not list; without this membership every link in a subscription email 404s @@ -152,3 +160,18 @@ def test_the_404_body_is_small_and_navigable(): assert len(body) < 2000 assert 'href="/"' in body assert '404' in body + + +# ── the external-page beacon ────────────────────────────────────────────────── + +def test_the_beacon_logs_only_allowlisted_pages(): + """`p` is a query value anyone can write. The log records the path the + allowlist names for it, never the value itself.""" + assert external_page_path('dbmf') == '/dbmf' + for key in ['', None, 'DBMF', '/dbmf', 'shell.php', '