From 6363e185f4b1fa9027c4bf112a3107d0295ae987 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Fri, 25 Sep 2026 15:53:20 -0400 Subject: [PATCH] Send the weekly email before warming the page caches The store poller started the three page warmers the moment the rebuild finished and only then built and sent the email. The warmers are CPU-bound Python in the same process, so the email a subscriber is waiting for queued behind GIL contention with cache fills for pages nobody had opened yet. Now the send goes first and the warmers start after it, outside the try so a failed send still warms the pages. The rebuild itself was the larger cost and is fixed in cotmetrics 0.14.2 (per-row index loops made rolling, ~100s -> ~12s locally). Co-Authored-By: Claude Opus 5.5 --- src/main.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main.py b/src/main.py index 47eb22b..da8a97b 100644 --- a/src/main.py +++ b/src/main.py @@ -106,7 +106,7 @@ def store_poll_loop(): The navbar callback polls the same way and is the primary trigger, but dcc.Interval is client-side: it only ticks while a browser tab is open. Without this loop the - first visitor after a release pays the ~90 second rebuild. With it, an unattended + first visitor after a release pays the rebuild. With it, an unattended app is current before anyone arrives. Every 5 minutes, always, rather than a window around the Friday release. The store @@ -137,17 +137,11 @@ def store_poll_loop(): utils.cot_logger.info(weekly_email_trigger.startup_message()) while True: time.sleep(STORE_POLL_SECONDS) + new_week = False try: - if get_indexer().refresh_if_stale(): + new_week = get_indexer().refresh_if_stale() + if new_week: utils.cot_logger.info("Store poller: picked up a new COT week.") - # The new week keys every entry in the crowd board's and the - # heatmap's join caches, so the refresh just invalidated them - # all; re-warm in the background so the first visitor of the new - # week does not pay the cold renders. A thread rather than - # inline, to keep this loop's tick short. - threading.Thread( - target=warm_page_caches, name="page-warmer", daemon=True - ).start() # After the refresh, never before: refresh_if_stale blocks until the index # matches the store, so by here the matrix the email builds is the new @@ -166,6 +160,19 @@ def store_poll_loop(): # watching. Log and wait for the next tick. utils.cot_logger.error(f"Store poller failed, will retry: {e}") + # The new week keys every entry in the crowd board's and the heatmap's join + # caches, so the refresh just invalidated them all; re-warm in the background + # so the first visitor of the new week does not pay the cold renders. A thread, + # to keep this loop's tick short. AFTER the email, deliberately: the warmers + # are CPU-bound Python in this process, and started first they contended for + # the GIL with the email's matrix build, so the email a subscriber is waiting + # for queued behind cache fills for pages nobody had opened yet. Outside the + # try, so a failed send still warms the pages. + if new_week: + threading.Thread( + target=warm_page_caches, name="page-warmer", daemon=True + ).start() + def daily_options_update_scheduler(): import datetime