Count views of external pages through a one-pixel beacon (/seen.gif)

server-side/README.md

@@ -446,6 +446,22 @@ block rather than a warning. Only enable it once renewal is dependable: single a
 record, `certbot renew --dry-run` passing, `certbot.timer` active. Shorten `max-age` to
 something like `300` while testing if you want an easy way back.
 
+### Views of pages this app does not serve
+
+The DBMF dashboard at `dbmf.bluemagicai.com` is a static page on GitHub Pages: no
+server, no hook, no access log. Its views reach the same `visitor_logs` table and the
+same /admin charts through a one-pixel beacon, `GET /seen.gif?p=dbmf&r=<referrer>`,
+which the publish step in the `dbmf-tracker` repo (`deploy/publish_pages.py`) appends to
+the page. The app answers with a 1x1 GIF and writes the same two rows one of its own
+document loads writes: a `landing` carrying the referrer and a `pageview` the admin
+page counts, under path `/dbmf`. The `p` value is matched against an allowlist in
+`routing.EXTERNAL_PAGES` (add a key there to admit another external page) and `r` is
+kept only when it looks like a web address. Nothing to configure on the box: the route
+is part of the app and rides the same nginx `proxy_pass`, so `X-Forwarded-For` reaches
+it like any other request. Two limits worth knowing: an ad blocker that drops the
+pixel drops the view, and a client that fetches HTML without loading images never
+appears, which for that page is the bot filter.
+
 ## 9. GoatCounter analytics (optional)
 
 Client-side analytics beside the app's own visit log, not replacing it: the built-in

src/app_cot.py

@@ -448,12 +448,20 @@ def _visit_worker():
             utils.cot_logger.error(f"visit worker failed on {row.get('path')}: {e}")
 
 
-def _enqueue_visit(kind, path):
+_REFERRER_FROM_REQUEST = object()
+
+
+def _enqueue_visit(kind, path, referrer=_REFERRER_FROM_REQUEST):
     """Build a visit row from the CURRENT flask request and hand it to the worker.
 
     Must run inside a request context (a before_request hook or a callback, both
     qualify). The worker is started lazily rather than at import so importing this
     module (tests, tooling) starts no thread and writes nothing.
+
+    `referrer` defaults to the request's own: kept on a landing, dropped on a
+    pageview (see the row below). The beacon passes one explicitly, None
+    included, because on that request the Referer header names the external
+    page itself, not where its visitor came from.
     """
     global _visit_worker_started
     ip_addr = visitors.client_ip(request.headers.get('X-Forwarded-For'),
@@ -469,7 +477,8 @@ def _enqueue_visit(kind, path):
         'is_bot': bot,
         # The referrer on a callback POST is just our own page url; only the
         # document load carries where the visitor actually came from.
-        'referrer': request.referrer if kind == 'landing' else None,
+        'referrer': ((request.referrer if kind == 'landing' else None)
+                     if referrer is _REFERRER_FROM_REQUEST else referrer),
     })
     with _visit_worker_lock:
         if not _visit_worker_started:
@@ -493,13 +502,42 @@ def record_visit():
         '/_dash-component-suites/',
         '/assets/',
         '/favicon.ico',
-        '/_favicon.ico'
+        '/_favicon.ico',
+        # The external-page beacon logs its OWN rows (below); the request that
+        # carries it is not a visit to this site.
+        routing.BEACON_PATH,
     ]
 
     if not any(request.path.startswith(path) for path in ignored_paths):
         _enqueue_visit('landing', request.path)
 
 
+@app.server.route(routing.BEACON_PATH)
+def external_page_beacon():
+    """Log a view of a page served from another origin; see routing.BEACON_PATH.
+
+    The DBMF dashboard is a static page on GitHub Pages: no server, no hook, no
+    access log. It loads this pixel once per document load with `p` naming the
+    page and `r` carrying its `document.referrer`, and the result is exactly the
+    pair of rows one of this app's own document loads produces: a 'landing'
+    carrying the referrer and a 'pageview' that the admin page counts (the two
+    kinds are never summed; see the visit-capture note above). Everything after
+    the enqueue is shared: the forwarded client address, the daily visitor hash,
+    the geolocation cache, the datacenter flag. A visitor who opens both sites
+    on one day is therefore one visitor id across them.
+
+    `p` is matched against an allowlist and `r` is cleaned, because both are
+    query values anyone can write. An unlisted `p` still gets the pixel: the
+    answer is for the browser, and there is nothing to tell a stranger.
+    """
+    path = routing.external_page_path(request.args.get('p'))
+    if path:
+        _enqueue_visit('landing', path,
+                       referrer=visitors.clean_referrer(request.args.get('r')))
+        _enqueue_visit('pageview', path)
+    return routing.PIXEL_GIF, 200, {'Content-Type': 'image/gif'}
+
+
 # The dropdown pages, as data, so the items and the active-state wiring below
 # can never disagree about what is in the menus. OI Alignment and About are
 # absent on purpose: both were promoted to top-level links.

src/routing.py

@@ -25,6 +25,16 @@
     '/static/',
 )
 
+#: The view beacon: a one-pixel image a page served from ANOTHER origin loads so
+#: its views land in the same visitor log as this app's pages. The first such
+#: page is the DBMF dashboard, a static GitHub Pages site under the same domain
+#: (dbmf.bluemagicai.com) with no server of its own, so nothing on that side can
+#: run `record_visit`. The name is deliberately bland: the common blocklists
+#: match /beacon, /track, /pixel.gif and /collect, and a request they drop is a
+#: view never counted. Named in EXTRA_PATHS below so the unknown-path guard
+#: admits it.
+BEACON_PATH = '/seen.gif'
+
 #: Real paths belonging to no page: the browser's automatic favicon request, Dash's
 #: own copy of it, the email-link and crawler endpoints (plain Flask routes in
 #: app_cot, which `app.routes` does not list because it only knows Dash's own),
@@ -33,7 +43,28 @@
 EXTRA_PATHS = frozenset({'/favicon.ico', '/_favicon.ico',
                          '/confirm', '/unsubscribe', '/graphs',
                          '/robots.txt', '/sitemap.xml',
-                         '/manifest.webmanifest', '/sw.js'})
+                         '/manifest.webmanifest', '/sw.js',
+                         BEACON_PATH})
+
+#: Which external pages the beacon will log, keyed by the `p` query value the
+#: page sends, valued by the path recorded for it. An ALLOWLIST, not a
+#: passthrough: the query string is anyone's to write, and the visitor log must
+#: never carry a string a stranger chose. An unlisted key is answered with the
+#: pixel and logged as nothing.
+EXTERNAL_PAGES = {'dbmf': '/dbmf'}
+
+#: A 1x1 transparent GIF, 43 bytes. Answering with a real image rather than a
+#: 204 keeps a plain <img> beacon from logging a broken-image error in the
+#: visitor's console; pinned by test_visit_capture through PIL.
+PIXEL_GIF = (b'GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff'
+             b'!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00'
+             b'\x00\x02\x02D\x01\x00;')
+
+
+def external_page_path(key):
+    """The path the visitor log records for an external page's beacon, or None."""
+    return EXTERNAL_PAGES.get(key or '')
+
 
 #: Operator surfaces and internal viewers: out of the sitemap, disallowed in
 #: robots.txt. A search result should never land a visitor on the admin login
@@ -221,6 +252,12 @@ def cache_policy(path, fingerprinted, content_type):
     # worker script's HTTP cache at a day, but explicit beats capped).
     if path in ('/manifest.webmanifest', '/sw.js'):
         return 'no-cache'
+    # The view beacon counts document loads, so a cached pixel is a view that
+    # never reached the log. `no-store`, not `no-cache`: a revalidation is a
+    # request too, but one the browser may answer from its own copy on a
+    # back/forward navigation without asking.
+    if path == BEACON_PATH:
+        return 'no-store'
     # A DATED weekly report barely changes once its week has passed (a store
     # revision or a copy tweak, both rare), and its cold render is the most
     # expensive page this app serves, so a day of public caching is cheap

src/visitors.py

@@ -49,6 +49,23 @@ def is_bot(user_agent):
     return any(marker in ua for marker in BOT_MARKERS)
 
 
+def clean_referrer(value, limit=512):
+    """A referrer a client REPORTED (the beacon's `r` query value), or None.
+
+    A browser-set Referer header is bounded by the browser; a query value is
+    bounded by whoever wrote the URL. Keep it only when it looks like a web
+    address, and never longer than `limit`, so the referrer column cannot be
+    used to park arbitrary text in the admin table. Empty means a direct visit,
+    which is None rather than '' to match what the Referer header gives.
+    """
+    if not value:
+        return None
+    value = value.strip()
+    if not value.lower().startswith(('http://', 'https://')):
+        return None
+    return value[:limit]
+
+
 def visitor_id(ip, user_agent, day=None):
     """A daily-rotating pseudonymous id: sha256 of day, ip and user agent, 16 hex chars.
 

tests/test_routing.py

@@ -11,7 +11,14 @@
 """
 import pytest
 
-from routing import is_known_path, is_vendor_sourcemap, not_found_page
+from routing import (
+    BEACON_PATH,
+    cache_policy,
+    external_page_path,
+    is_known_path,
+    is_vendor_sourcemap,
+    not_found_page,
+)
 
 # What the live app carries, as of the page registry at the time of writing. The nested
 # and the underscored ones are the interesting entries: a naive "one path segment"
@@ -74,7 +81,8 @@ def test_the_browsers_automatic_favicon_request_is_not_a_probe(path):
 
 @pytest.mark.parametrize("path", ['/confirm', '/unsubscribe',
                                   '/robots.txt', '/sitemap.xml',
-                                  '/weekly', '/weekly/2026-08-25'])
+                                  '/weekly', '/weekly/2026-08-25',
+                                  '/seen.gif'])
 def test_the_flask_route_endpoints_are_served(path):
     """Plain Flask routes on app.server (see app_cot), which `app.routes` does
     not list; without this membership every link in a subscription email 404s
@@ -152,3 +160,18 @@ def test_the_404_body_is_small_and_navigable():
     assert len(body) < 2000
     assert 'href="/"' in body
     assert '404' in body
+
+
+# ── the external-page beacon ──────────────────────────────────────────────────
+
+def test_the_beacon_logs_only_allowlisted_pages():
+    """`p` is a query value anyone can write. The log records the path the
+    allowlist names for it, never the value itself."""
+    assert external_page_path('dbmf') == '/dbmf'
+    for key in ['', None, 'DBMF', '/dbmf', 'shell.php', '<script>']:
+        assert external_page_path(key) is None, key
+
+
+def test_the_beacon_is_never_cached():
+    """A cached pixel is a document load that never reached the log."""
+    assert cache_policy(BEACON_PATH, False, 'image/gif') == 'no-store'

tests/test_visit_capture.py

@@ -53,6 +53,73 @@ def test_a_null_pathname_logs_nothing(captured):
     assert captured == []
 
 
+# ── what the external-page beacon logs ───────────────────────────────────────
+#
+# These go through the test client rather than calling the view, because the
+# interesting half is what the OTHER hooks do with the request: the unknown-path
+# guard must admit it and `record_visit` must not log it as a landing on top of
+# the rows the beacon writes itself.
+
+@pytest.fixture
+def captured_rows(monkeypatch):
+    rows = []
+    monkeypatch.setattr(app_cot, '_enqueue_visit',
+                        lambda kind, path, **kw: rows.append((kind, path, kw)))
+    return rows
+
+
+def test_a_dbmf_page_load_is_a_landing_plus_a_pageview(captured_rows):
+    """The same pair one of this app's own document loads produces: the landing
+    carries the external page's referrer, the pageview is what the admin page
+    counts. No third row for the pixel request itself."""
+    client = app_cot.app.server.test_client()
+    resp = client.get('/seen.gif?p=dbmf&r=https%3A%2F%2Fexample.com%2Fpost')
+    assert resp.status_code == 200
+    assert resp.content_type == 'image/gif'
+    assert resp.headers['Cache-Control'] == 'no-store'
+    assert captured_rows == [
+        ('landing', '/dbmf', {'referrer': 'https://example.com/post'}),
+        ('pageview', '/dbmf', {}),
+    ]
+
+
+def test_a_direct_dbmf_visit_has_no_referrer(captured_rows):
+    """`document.referrer` is '' on a direct visit; the row must say None, as the
+    Referer header would, not the dbmf page's own address from this request."""
+    client = app_cot.app.server.test_client()
+    client.get('/seen.gif?p=dbmf&r=',
+               headers={'Referer': 'https://dbmf.bluemagicai.com/'})
+    assert captured_rows[0] == ('landing', '/dbmf', {'referrer': None})
+
+
+def test_an_unlisted_page_gets_the_pixel_and_no_row(captured_rows):
+    client = app_cot.app.server.test_client()
+    for query in ['', '?p=', '?p=shell.php', '?p=DBMF']:
+        resp = client.get('/seen.gif' + query)
+        assert resp.status_code == 200, query
+    assert captured_rows == []
+
+
+def test_the_pixel_is_a_real_one_by_one_gif():
+    import io
+
+    import routing
+    assert routing.PIXEL_GIF[:6] == b'GIF89a' and routing.PIXEL_GIF[-1:] == b';'
+    Image = pytest.importorskip('PIL.Image')
+    assert Image.open(io.BytesIO(routing.PIXEL_GIF)).size == (1, 1)
+
+
+def test_an_explicit_referrer_wins_over_the_request_header(monkeypatch):
+    monkeypatch.setattr(app_cot, '_visit_worker_started', True)
+    with app_cot.app.server.test_request_context(
+            '/seen.gif', headers={'User-Agent': 'Mozilla/5.0',
+                                  'Referer': 'https://dbmf.bluemagicai.com/'}):
+        app_cot._enqueue_visit('landing', '/dbmf', referrer=None)
+        app_cot._enqueue_visit('landing', '/dbmf', referrer='https://t.co/x')
+    assert app_cot._visit_queue.get_nowait()['referrer'] is None
+    assert app_cot._visit_queue.get_nowait()['referrer'] == 'https://t.co/x'
+
+
 # ── what reaches the queue ────────────────────────────────────────────────────
 
 def test_the_enqueued_row_carries_identity_and_referrer(monkeypatch):

tests/test_visitors.py

@@ -1,5 +1,5 @@
 """The pure half of visitor tracking: ip parsing, bot classification, visitor ids."""
-from visitors import client_ip, is_bot, visitor_id
+from visitors import clean_referrer, client_ip, is_bot, visitor_id
 
 # ── client_ip ─────────────────────────────────────────────────────────────────
 
@@ -76,3 +76,29 @@ def test_the_id_is_short_hex():
 
 def test_today_is_the_default_day():
     assert visitor_id('1.2.3.4', 'ua') == visitor_id('1.2.3.4', 'ua')
+
+
+# ── clean_referrer ────────────────────────────────────────────────────────────
+
+def test_a_web_address_is_kept_as_reported():
+    assert clean_referrer('https://example.com/post?x=1') == 'https://example.com/post?x=1'
+    assert clean_referrer('http://example.com/') == 'http://example.com/'
+
+
+def test_an_empty_referrer_is_a_direct_visit():
+    """None, not '', so a direct visit reads the same whether it arrived via the
+    Referer header or the beacon's query value."""
+    assert clean_referrer('') is None
+    assert clean_referrer(None) is None
+    assert clean_referrer('   ') is None
+
+
+def test_anything_but_a_web_address_is_dropped():
+    """The query value is bounded by whoever wrote the URL, not by a browser."""
+    for value in ['javascript:alert(1)', '<img src=x>', 'example.com', 'ftp://x/y',
+                  'drop table visitor_logs']:
+        assert clean_referrer(value) is None, value
+
+
+def test_a_long_referrer_is_truncated():
+    assert len(clean_referrer('https://x/' + 'a' * 5000)) == 512