From f3b9a5455701892d41f7db5d72f32fb438f44514 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 5 Mar 2026 14:10:38 -0800 Subject: [PATCH 01/33] create a place to temporarily store alerts for display The FIFOQueueMixin class turns the Model's db table into a FIFO queue of the most recent alerts. The Alert model is just a generic data structure for an alert from a stream. Register new model with admin pages --- tom_alertstreams/admin.py | 9 +- tom_alertstreams/migrations/0001_initial.py | 33 +++++++ tom_alertstreams/models.py | 95 ++++++++++++++++++++- 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 tom_alertstreams/migrations/0001_initial.py diff --git a/tom_alertstreams/admin.py b/tom_alertstreams/admin.py index 8c38f3f..88256b3 100644 --- a/tom_alertstreams/admin.py +++ b/tom_alertstreams/admin.py @@ -1,3 +1,10 @@ from django.contrib import admin -# Register your models here. +from tom_alertstreams.models import Alert + + +@admin.register(Alert) +class AlertAdmin(admin.ModelAdmin): + list_display = ('stream_name', 'alert_id', 'timestamp', 'object_id', 'magnitude') + list_filter = ('stream_name',) + search_fields = ('alert_id', 'object_id') diff --git a/tom_alertstreams/migrations/0001_initial.py b/tom_alertstreams/migrations/0001_initial.py new file mode 100644 index 0000000..b3ddc5c --- /dev/null +++ b/tom_alertstreams/migrations/0001_initial.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.29 on 2026-03-05 17:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Alert', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('stream_name', models.CharField(db_index=True, max_length=100)), + ('topic', models.CharField(max_length=200)), + ('timestamp', models.DateTimeField(db_index=True)), + ('alert_id', models.CharField(max_length=200)), + ('object_id', models.CharField(blank=True, max_length=200, null=True)), + ('ra', models.FloatField(null=True)), + ('dec', models.FloatField(null=True)), + ('magnitude', models.FloatField(null=True)), + ('raw_payload', models.JSONField(default=dict)), + ], + options={ + 'ordering': ['-timestamp'], + 'indexes': [models.Index(fields=['stream_name', 'timestamp'], name='tom_alertst_stream__a592d5_idx')], + }, + ), + ] diff --git a/tom_alertstreams/models.py b/tom_alertstreams/models.py index 71a8362..aaf01ca 100644 --- a/tom_alertstreams/models.py +++ b/tom_alertstreams/models.py @@ -1,3 +1,96 @@ +from __future__ import annotations + +from typing import Any, ClassVar + +from django.conf import settings from django.db import models -# Create your models here. + +class FIFOQueueMixin(models.Model): + """Mixin that enforces a per-partition maximum row count. + + Basically, this limits the size that a Model's table can reach by + turning it into a First-In-First-Out (FIFO) queue. + + This is implemented by extending the `save()` method to save a model + instance (as per normal), then check the table size and delete records + (oldest first) over the FIFO_MAX limit. + + The wrinkle is the per-partion part: The FIFO_PARTIION_FIELD is a field + in the model that divides (i.e. "partitions") the table according to the + value of the field. What that means is that there can be FIFO_MAX records + that have a common value in the FIFO_PARTIION_FIELD. So, for example, if the + FIFO_PARTITION_FIELD is `stream_name`, then there can be FIFO_MAX records + with `stream_name` "alerce" and FIFO_MAX records with `stream_name` "fink", + etc. So, there can be FIFO_MAX records for each distinct value of the + FIFO_PARTIION_FIELD. The FIFO_PARTION_FIELD value of the instance being + saved specifies the partition whose size is checked, post-save(). + + To summarize, after every save(), rows beyond FIFO_MAX are deleted + (oldest first) within the same partition as the newly-saved instance. + 'Partition' means all rows sharing the same value of FIFO_PARTITION_FIELD + — e.g., all alerts from the same stream. If FIFO_PARTITION_FIELD is None, + then the FIFO_MAX limit applies to the entire table. + + Subclasses set FIFO_MAX and FIFO_PARTITION_FIELD as class variables. + """ + FIFO_MAX: ClassVar[int] = 10 + FIFO_PARTITION_FIELD: ClassVar[str | None] = None + + class Meta: + abstract = True + + def save(self, *args: Any, **kwargs: Any) -> None: + super().save(*args, **kwargs) # save, then trim: Ensures the newly-saved + self._enforce_fifo_limit() # row is counted against the FIFO_MAX limit. + + def _enforce_fifo_limit(self) -> None: + """Delete rows beyond FIFO_MAX, oldest first, within this instance's partition. + + Uses list() to materialise PKs before the DELETE to avoid a SQLite restriction + that forbids DELETE from a table referenced in the same statement's subquery. + """ + qs = self.__class__.objects.all() + if self.FIFO_PARTITION_FIELD is not None: + # Scope the FIFO_MAX limit to rows in the same partition as this instance. + partition_value = getattr(self, self.FIFO_PARTITION_FIELD) + qs = qs.filter(**{self.FIFO_PARTITION_FIELD: partition_value}) + # Materialise PKs to avoid a subquery-in-DELETE issue on SQLite. + excess_pks = list( + qs.order_by('-timestamp').values_list('pk', flat=True)[self.FIFO_MAX:] + ) + if excess_pks: + self.__class__.objects.filter(pk__in=excess_pks).delete() + + +class Alert(FIFOQueueMixin): + """A normalized alert received from an alert stream, stored for recent display. + + This class is designed specifically for a Recent Alerts demonstration page. + + Because of the FIFOQueueMixin, rows are automatically pruned to + ALERTSTREAMS_RECENT_COUNT per stream_name. The raw_payload JSONField preserves + the full original alert for handlers or views that need stream-specific + fields not captured here. + """ + # set the mixin class variables + FIFO_MAX: ClassVar[int] = getattr(settings, 'ALERTSTREAMS_RECENT_COUNT', 10) + FIFO_PARTITION_FIELD: ClassVar[str | None] = 'stream_name' + + stream_name = models.CharField(max_length=100, db_index=True) + topic = models.CharField(max_length=200) + timestamp = models.DateTimeField(db_index=True) + alert_id = models.CharField(max_length=200) + object_id = models.CharField(max_length=200, blank=True, null=True) + ra = models.FloatField(null=True) + dec = models.FloatField(null=True) + magnitude = models.FloatField(null=True) + raw_payload = models.JSONField(default=dict) + + class Meta(FIFOQueueMixin.Meta): # this is the way you subclass the internal Meta class + abstract = False # override for the concrete model (abstract is True in the super) + ordering = ['-timestamp'] + indexes = [models.Index(fields=['stream_name', 'timestamp'])] + + def __str__(self) -> str: + return f'Alert {self.alert_id} from {self.stream_name} at {self.timestamp}' From 3a180529b68c4db0f84bc50be67a72aee3fc3a8f Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 5 Mar 2026 16:36:27 -0800 Subject: [PATCH 02/33] add (optional) Recent Alerts page Uses integration points to add URL patterns and navbar item. Defines a basic HTMX table View for the Alerts model. --- tom_alertstreams/apps.py | 27 +++++ tom_alertstreams/tables.py | 104 ++++++++++++++++++ .../partials/navbar_link.html | 3 + .../tom_alertstreams/recent_alerts.html | 40 +++++++ tom_alertstreams/urls.py | 11 ++ tom_alertstreams/views.py | 73 +++++++++++- 6 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 tom_alertstreams/tables.py create mode 100644 tom_alertstreams/templates/tom_alertstreams/partials/navbar_link.html create mode 100644 tom_alertstreams/templates/tom_alertstreams/recent_alerts.html create mode 100644 tom_alertstreams/urls.py diff --git a/tom_alertstreams/apps.py b/tom_alertstreams/apps.py index 824575a..04296dc 100644 --- a/tom_alertstreams/apps.py +++ b/tom_alertstreams/apps.py @@ -1,6 +1,33 @@ from django.apps import AppConfig +from django.conf import settings +from django.urls import include, path class TomAlertstreamsConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'tom_alertstreams' + + def include_url_paths(self) -> list: + """Register tom_alertstreams URLs with the TOM Common URL configuration. + + Returns URL patterns that mount the Recent Alerts page at /alertstreams/, + but only when `SHOW_RECENT_ALERTS = True` is set in settings. If the setting + is absent or False, an empty list is returned and the page is not accessible. + """ + # only return urlpatterns if they opt-in + if not getattr(settings, 'SHOW_RECENT_ALERTS', False): + return [] + return [ + path('alertstreams/', include('tom_alertstreams.urls', namespace='alertstreams')), + ] + + def nav_items(self) -> list: + """Add the 'Recent Alerts' link to the TOM navbar. + + Returns a list of navbar item dicts auto-discovered by the navbar_app_addons + template tag, but only when SHOW_RECENT_ALERTS = True is set in settings. + """ + # only show the navbar item if they opt-in + if not getattr(settings, 'SHOW_RECENT_ALERTS', False): + return [] + return [{'partial': 'tom_alertstreams/partials/navbar_link.html'}] diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py new file mode 100644 index 0000000..c160107 --- /dev/null +++ b/tom_alertstreams/tables.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import logging +from typing import Any + +import django_filters +import django_tables2 as tables +from django import forms +from django.conf import settings +from django.utils.html import format_html +from django.utils.module_loading import import_string + +from tom_alertstreams.models import Alert +from tom_common.htmx_table import HTMXTable, HTMXTableFilterSet + +logger = logging.getLogger(__name__) + + +class AlertTable(HTMXTable): + """HTMX-driven table of recent alerts from all configured alert streams. + + Receives url_map at construction time so that render_alert_id() can construct + archive links on the fly, without storing URLs in the model. Streams with no + ARCHIVE_URL_TEMPLATE (GCN, Hopskotch, stubs) display alert_id as plain text. + """ + + def __init__(self, *args: Any, url_map: dict[str, str | None] | None = None, **kwargs: Any) -> None: + # Store url_map before calling super() so render_alert_id() can access it during rendering. + self.url_map = url_map or {} + super().__init__(*args, **kwargs) + + # + # Custom field renderers + # + + stream_name = tables.Column(verbose_name='Stream') # sets the column header value + + # render_FIELDNAME() methods are called automatically when present + def render_timestamp(self, value: Any) -> str: + """Render timestamp in unambiguous UTC 24-hour format. + + The result looks like this: 2026-03-05 18:51:30 UTC + """ + return value.strftime('%Y-%m-%d %H:%M:%S UTC') + + def render_alert_id(self, record: Alert, value: str) -> str: + """Render alert_id as a hyperlink to the stream's archive if a URL template exists.""" + template = self.url_map.get(record.stream_name) + if template: + url = template.format(alert_id=record.alert_id, object_id=record.object_id or '') + return format_html('{}', url, value) + return value + + class Meta(HTMXTable.Meta): + model = Alert + fields = ['selection', 'alert_id', 'stream_name', 'topic', 'timestamp', 'object_id', 'ra', 'dec', 'magnitude'] + + +def _get_stream_name_choices() -> list[tuple[str, str]]: + """Build dropdown choices from the active streams in settings.ALERT_STREAMS. + + Called at form-render time (not import time) so changes to settings take + effect without restarting the process. Returns a list of (value, label) + tuples using each stream's STREAM_NAME. Streams that fail to import are + silently skipped so a misconfigured entry doesn't break the filter form. + """ + choices = [] + for stream_config in getattr(settings, 'ALERT_STREAMS', []): + if not stream_config.get('ACTIVE', True): + continue + try: + klass = import_string(stream_config['NAME']) + name = klass.STREAM_NAME + choices.append((name, name)) + except (ImportError, AttributeError, KeyError) as exc: + logger.warning('_get_stream_name_choices: skipping stream %s: %s', stream_config.get('NAME'), exc) + return choices + + +class AlertFilterSet(HTMXTableFilterSet): + """FilterSet for the Recent Alerts table. + + Provides a 'query' full-text search (inherited from HTMXTableFilterSet) plus + the fields defined here, which appear in the Advanced> expansion of the form. + """ + # these are the fields that appear in the Advanced> expansion + stream_name = django_filters.ChoiceFilter( + field_name='stream_name', + label='Stream', + empty_label='All streams', + choices=_get_stream_name_choices, + widget=forms.Select(attrs={ + 'hx-get': '', # empty string: GET goes to the current page URL + 'hx-trigger': 'change', + 'hx-target': 'div.table-container', + 'hx-swap': 'innerHTML', + 'hx-indicator': '.progress', + 'hx-include': 'closest form', + }), + ) + + class Meta: + model = Alert + fields = ['stream_name'] diff --git a/tom_alertstreams/templates/tom_alertstreams/partials/navbar_link.html b/tom_alertstreams/templates/tom_alertstreams/partials/navbar_link.html new file mode 100644 index 0000000..d4d348f --- /dev/null +++ b/tom_alertstreams/templates/tom_alertstreams/partials/navbar_link.html @@ -0,0 +1,3 @@ +
  • + Recent Alerts +
    • diff --git a/tom_alertstreams/templates/tom_alertstreams/recent_alerts.html b/tom_alertstreams/templates/tom_alertstreams/recent_alerts.html new file mode 100644 index 0000000..8a67e5b --- /dev/null +++ b/tom_alertstreams/templates/tom_alertstreams/recent_alerts.html @@ -0,0 +1,40 @@ +{% extends 'tom_common/base.html' %} +{% load crispy_forms_tags %} + +{% block title %}Recent Alerts{% endblock %} + +{% block content %} +
    +
    +

    Recent Alerts

    +

    + The {{ record_count }} most recent alert{{ record_count|pluralize }} received from configured alert streams. + Alert IDs link to the stream's public archive where available. +

    +
    + + {# Filter form -- id="filter-form" must match hx-include in AlertTable.Meta.attrs #} +
    + {% crispy filter.form %} +
    + + {# Progress indicator (CSS provided by TOM Toolkit base template) #} +
    +
    +
    + + {# Table container -- HTMX swaps this outerHTML on filter changes and auto-polls every 5 s #} +
    + {% include table.get_partial_template_name %} +
    +
    +
    +{% endblock %} diff --git a/tom_alertstreams/urls.py b/tom_alertstreams/urls.py new file mode 100644 index 0000000..0578af6 --- /dev/null +++ b/tom_alertstreams/urls.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from django.urls import path + +from tom_alertstreams.views import RecentAlertsView + +app_name = 'tom_alertstreams' + +urlpatterns = [ + path('recent/', RecentAlertsView.as_view(), name='recent-alerts'), +] diff --git a/tom_alertstreams/views.py b/tom_alertstreams/views.py index 91ea44a..ebe4860 100644 --- a/tom_alertstreams/views.py +++ b/tom_alertstreams/views.py @@ -1,3 +1,72 @@ -from django.shortcuts import render +from __future__ import annotations -# Create your views here. +import logging +from typing import Any + +from django.conf import settings +from django.utils.module_loading import import_string +from django_filters.views import FilterView + +from tom_alertstreams.models import Alert +from tom_alertstreams.tables import AlertFilterSet, AlertTable +from tom_common.htmx_table import HTMXTableViewMixin + +logger = logging.getLogger(__name__) + + +def _build_archive_url_map() -> dict[str, str | None]: + """Build {stream_name → archive_url_template} from ALERT_STREAMS settings. + + Reads ALERT_STREAMS and imports each active stream class by dotted path to + access its STREAM_NAME and ARCHIVE_URL_TEMPLATE class variables. Does NOT + instantiate the streams — no network connections are made. Returns an empty + dict if ALERT_STREAMS is not configured. + """ + url_map: dict[str, str | None] = {} + for stream_config in getattr(settings, 'ALERT_STREAMS', []): + if not stream_config.get('ACTIVE', True): + continue + try: + klass = import_string(stream_config['NAME']) + url_map[klass.STREAM_NAME] = klass.ARCHIVE_URL_TEMPLATE + except (ImportError, AttributeError, KeyError) as exc: + logger.warning(f'_build_archive_url_map: could not read stream class {stream_config.get("NAME")}: {exc}') + return url_map + + +class RecentAlertsView(HTMXTableViewMixin, FilterView): + """Display the most recent alerts from all configured alert streams. + + No login is required — the Recent Alerts page is intentionally public so that + demo visitors and potential TOM developers can browse it without an account. + + Archive URL links (e.g. to ANTARES, ALeRCE) are built on the fly from each + stream's ARCHIVE_URL_TEMPLATE class variable, so no URL needs to be stored + in the database. + """ + template_name = 'tom_alertstreams/recent_alerts.html' + model = Alert + table_class = AlertTable + filterset_class = AlertFilterSet + paginate_by = 20 + + def get_table_kwargs(self) -> dict[str, Any]: + """Inject the archive url_map into the AlertTable constructor. + + The "archive url map" is used to create links that appear in the + Recent Alerts table. The link is to the alert at the alert brokers site. + + The map is built above using the class property set on the AlertStream + subclasses. + + This method is implemented in django-tables2.SingleTableMixin, + which HTMXTableViewMixin inherits from. It's called like this: + + get_context_data() # SingleTableMixin (django-tables2) + └── get_table(**self.get_table_kwargs()) + ├── get_table_kwargs() # returns {} by default; we override to add url_map + └── get_table(url_map=…) # instantiates AlertTable(data=…, url_map=…) + """ + kwargs = super().get_table_kwargs() + kwargs['url_map'] = _build_archive_url_map() + return kwargs From 2d106e70c2e4d138eaa6aa967c3111c13a49b890 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 5 Mar 2026 17:29:14 -0800 Subject: [PATCH 03/33] refactor AlertStream to use Pydantic for config validation Adds pydanic dependency to pyproject.toml. Defines a normalization mechanism for alerts. Adds a handler that works for any properly implemented and configured AlertStream subclass, demonstrating normalization mechanism. (`save_alert_to_database` adds alerts to the models.Alert(FIFOQueueMixin) db table. --- pyproject.toml | 3 +- tom_alertstreams/alertstreams/alertstream.py | 380 +++++++++++++++---- 2 files changed, 310 insertions(+), 73 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98e9cd7..4950008 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,8 @@ dynamic = ["version"] requires-python = ">=3.9.0,<=3.13" dependencies = [ "django <6", - "django-extensions >=3.2.3,<4" + "django-extensions >=3.2.3,<4", + "pydantic >=2.0,<3" ] [project.optional-dependencies] diff --git a/tom_alertstreams/alertstreams/alertstream.py b/tom_alertstreams/alertstreams/alertstream.py index 11b812d..64705a6 100644 --- a/tom_alertstreams/alertstreams/alertstream.py +++ b/tom_alertstreams/alertstreams/alertstream.py @@ -1,108 +1,344 @@ +from __future__ import annotations + import abc import logging +from datetime import datetime +from typing import Any, Callable, ClassVar from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.utils.module_loading import import_string +from pydantic import BaseModel, ValidationError + +from tom_alertstreams.models import Alert + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -def get_default_alert_streams(): - """Return the AlertStreams configured in settings.py - """ - try: - alert_streams = get_alert_streams(settings.ALERT_STREAMS) - except AttributeError as err: - raise ImproperlyConfigured(err) +# --------------------------------------------------------------------------- +# Typed alert intermediate +# --------------------------------------------------------------------------- - return alert_streams +class NormalizedAlert(BaseModel): + """Typed intermediate produced by AlertStream.normalize_alert(). + Every AlertStream subclass's normalize_alert() method returns a NormalizedAlert. + Handlers that need to persist alerts (e.g. save_alert_to_database) rely on this + type so that one handler function works across all streams. -def get_alert_streams(alert_stream_configs: list): - """Return the AlertStreams configured in the given alert_stream_configs - (a list of configuration dictionaries ) + All fields except stream_name, alert_id, and timestamp are optional because + not every stream provides the same metadata. The raw_payload preserves the full + original alert object (serialized to a dict) for handlers that need + stream-specific data not captured in the normalised fields. - Use get_default_alert_streams() if you want the AlertStreams configured in settings.py. + Fields: + stream_name: Short canonical name of the stream (from AlertStream.STREAM_NAME). + topic: Kafka topic the alert arrived on. Empty string if not available. + timestamp: UTC datetime of the alert. Defaults to an empty string if unknown. + alert_id: Stream-specific identifier for this alert. + object_id: Astronomical object identifier (e.g. ZTF object name), if available. + ra: Right ascension in decimal degrees, if available. + dec: Declination in decimal degrees, if available. + magnitude: Apparent magnitude, if available. + raw_payload: The full original alert as a plain dict for downstream use. """ - alert_streams = [] # build and return this list of AlertStream subclass instances - for alert_stream_config in alert_stream_configs: - if not alert_stream_config.get('ACTIVE', True): - logger.debug(f'get_alert_streams - ignoring inactive stream: {alert_stream_config["NAME"]}') - continue # skip configs that are not active; default to ACTIVE - try: - klass = import_string(alert_stream_config['NAME']) - except ImportError: - msg = ( - f'The module (the value of the NAME key): {alert_stream_config["NAME"]} could not be imported. ' - f'Check your ALERT_STREAMS setting.' - ) - raise ImproperlyConfigured(msg) + stream_name: str + alert_id: str + timestamp: datetime + topic: str = '' + object_id: str | None = None + ra: float | None = None + dec: float | None = None + magnitude: float | None = None + raw_payload: dict = {} - alert_stream: AlertStream = klass(**alert_stream_config.get("OPTIONS", {})) - alert_streams.append(alert_stream) - return alert_streams +# --------------------------------------------------------------------------- +# Pydantic configuration models +# --------------------------------------------------------------------------- +class AlertStreamConfig(BaseModel): + """Pydantic base configuration for all AlertStream subclasses. + + Inheriting from pydantic's BaseModel means that Pydantic validates required + fields, coerces types, and raises descriptive field-level ValidationError + messages when configuration is missing or incorrect — replacing the previous + manual required_keys / allowed_keys validation approach. + + Every stream must declare its topic-to-handler mapping. Subclass configs + inherit this and add stream-specific authentication and connection fields. + Handler values are dotted-path strings; they are imported at AlertStream + instantiation time by _process_topic_handlers(). + + Example subclass: + class MyStreamConfig(AlertStreamConfig): + USERNAME: str + PASSWORD: str + START_POSITION: str = 'LATEST' + """ + # Maps topic names to dotted-path strings of callable alert handler functions. + # Example: {'my.topic': 'myapp.handlers.save_alert_to_database'} + TOPIC_HANDLERS: dict[str, str] + + +# --------------------------------------------------------------------------- +# AlertStream abstract base class +# --------------------------------------------------------------------------- class AlertStream(abc.ABC): - """Base class for specific alert streams like Hopskotch, GCNClassic, etc. - - * kwargs to __init__ is the OPTIONS dictionary defined in ALERT_STREAMS configuration - dictionary (for example, see settings.py). - * allowed_keys and required_keys should be defined as class properties in subclasses. - * The allowed_keys are turned into instance properties in __init__. - * Missing required_keys result in an ImproperlyConfigured Django exception. - - - To implmement subclass: - 1. define allowed_key, required_keys as class variables - - 2. implement listen() - this method probably doesn't return - 3. write your alert_handlers. which proably take and alert do something. - The HopskotchAlertStream.listen() method defines an 'alert_handlers' dictionary keyed by - alert topic with callable values (i.e call this method with alerts from this topic). - The GCNClassicAlertStream.listen() is another example. + """Abstract base class for Kafka alert stream implementations. + + To implement a new AlertStream subclass: + + 1. Define a Pydantic config model (subclass of AlertStreamConfig) that declares + all required and optional configuration fields. Pydantic handles validation + and produces error messages for misconfigured streams. + + 2. Set class variables: + configuration_class = MyStreamConfig + STREAM_NAME = 'mystream' # short canonical name, written to Alert.stream_name + ARCHIVE_URL_TEMPLATE = 'https://...' # used to create links to alerts + + 3. Override normalize_alert(raw_alert, topic='') -> NormalizedAlert to extract + stream-specific fields (ra, dec, magnitude, object_id, etc.). + + 4. Implement listen() -> None. This method is not expected to return. It should: + a. Connect to the Kafka stream using credentials from self.config + b. Subscribe to the topics in self.config.TOPIC_HANDLERS + c. Dispatch each incoming alert to its handler. Alert handlers shoud + use the following function signiture: + self.alert_handler[topic](raw_alert, alert_stream=self, topic=topic) + Include any stream-specific extras as named keyword args (e.g. metadata=metadata). + Handlers use **kwargs to absorb extras they do not need. + + The alert_stream=self argument provides dependency injection, allowing the + same generic handler function (e.g. save_alert_to_database) to serve all + streams because it receives the stream instance and can call the AlertStream's + get_normalization_function() to obtain stream-specific parsing logic without + knowing which stream it is working with. """ + # alertstream.AlertStreamConfig is a Pydantic BaseModel subclass + # the settings.ALERT_STREAMS configuration dictionary will validated according + # to the AlertStreamConfig subclass specified here. + configuration_class: ClassVar[type[AlertStreamConfig]] - def __init__(self, *args, **kwargs) -> None: - super().__init__() + # Short canonical name written to Alert.stream_name. Must be unique across + # all configured streams. Used by the Recent Alerts view to build archive URL maps. + STREAM_NAME: ClassVar[str] - # filter the kwargs by allowed keys and add them as properties to AlertStream instance - self.__dict__.update((k.lower(), v) for k, v in kwargs.items() if k in self.allowed_keys) + # this should be a URL that can be used to create a link to an alert at a broker + ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = None - missing_keys = set(self.required_keys) - set(kwargs.keys()) - if missing_keys: - msg = ( - f'The following required keys are missing from the configuration OPTIONS of ' - f'{self._get_stream_classname()}: {list(missing_keys)} ; ' - f'These keys were found: {list(kwargs.keys())} ; ' - f'Check your ALERT_STREAMS setting.' - ) - raise ImproperlyConfigured(msg) + def __init__(self, **kwargs: Any) -> None: + # read and validate the alertstream configuration + self.config: AlertStreamConfig = self.configuration_class(**kwargs) - self.alert_handler = self._process_topic_handlers() + # Convert TOPIC_HANDLERS dotted-path strings to callable functions. + self.alert_handler: dict[str, Callable] = self._process_topic_handlers() def _get_stream_classname(self) -> str: + """Return the qualified class name of this AlertStream subclass. + + This is just a way to get the name of the subclass. + """ return type(self).__qualname__ - def _process_topic_handlers(self): - """ convert the TOPIC_HANDLERS values in to callable functions in - the returned message_handler dictionary. (keyed by topic, value is callable) + def _process_topic_handlers(self) -> dict[str, Callable]: + """Import and return handler callables from the TOPIC_HANDLERS configuration. + + In settings.py, the configuration dictionary TOPIC_HANDLER dictionary + for each stream maps a topic to a dotted-path string specifying the alert + handler for that topic's alerts. This method converts the dotted-path string + to a Callable. + + Returns: + A dict mapping topic name strings to callable handler functions. + + Raises: + ImproperlyConfigured: if any handler dotted-path cannot be imported. """ alert_handler = {} - for topic, callable_string in self.topic_handlers.items(): - # convert string from TOPIC_HANDLERS in to a callable function in - # the message_handler dictionary (both keyed by topic string) - alert_handler[topic] = import_string(callable_string) + for topic, callable_string in self.config.TOPIC_HANDLERS.items(): + try: + alert_handler[topic] = import_string(callable_string) + except ImportError as err: + msg = ( + f'Could not import handler "{callable_string}" for topic "{topic}" ' + f'in {self._get_stream_classname()}. Check your TOPIC_HANDLERS setting. ' + f'Error: {err}' + ) + raise ImproperlyConfigured(msg) return alert_handler @abc.abstractmethod - def listen(self): - """Listen at the steam and dispatch alerts to handlers. Subclass extentions of - this method are not expected to return. See hopskotch.py and gcn.py for example - implementations. + def normalize_alert(self, raw_alert: Any, topic: str = '') -> NormalizedAlert: + """Convert a raw stream-specific alert object to a NormalizedAlert. + + Every AlertStream subclass must implement this method. The returned + NormalizedAlert is a Pydantic BaseModel subclass. As such, it is typed + and validated (vs a dictionary of unvalidated key and values). + + While the stream is known by virtue of the AlertSteam subclass implementing + this method, the topic argument is provided by the listen() loop and should + be passed through to the NormalizedAlert.topic field. + + Args: + raw_alert: The stream-specific alert object received from listen(). + topic: The Kafka topic this alert arrived on. + + Returns: + A NormalizedAlert with as many fields populated as the stream supports. """ - pass + pass # implement me + + def get_normalization_function(self) -> Callable: + """Return the normalization callable for this stream. + + By default returns self.normalize_alert. Override this method to substitute + a completely different normalization implementation without subclassing, for + example to use a function defined outside this class/subclass (e.g. in your + `custom_code` or other INSTALLED_APP).) + + Returns: + A callable with signature (raw_alert, topic='') -> NormalizedAlert. + """ + return self.normalize_alert + + @abc.abstractmethod + def listen(self) -> None: + """Consume alerts from the stream indefinitely. + + This method is not expected to return. Implementations should: + 1. Connect to the Kafka stream using credentials from self.config + 2. Subscribe to the topics in self.config.TOPIC_HANDLERS (the topic + keys are also available via self.alert_handler.keys()) + 3. For each incoming alert, dispatch to the handler using the unified + calling convention: + + self.alert_handler[topic](raw_alert, alert_stream=self, topic=topic) + + The alert_stream=self argument injects this AlertStream instance so that + handlers can call get_normalization_function() without knowing the specific + stream type (dependency injection via keyword argument). + + Pass any additional stream-specific context as named keyword arguments: + + self.alert_handler[topic](raw_alert, alert_stream=self, topic=topic, + metadata=metadata) # Hopskotch example + + Handlers absorb extras they do not need via **kwargs. + """ + pass # implement me + + +# --------------------------------------------------------------------------- +# Module-level helper functions +# --------------------------------------------------------------------------- + +def get_default_alert_streams() -> list[AlertStream]: + """Return the AlertStream instances configured in settings.ALERT_STREAMS. + + Raises: + ImproperlyConfigured: if ALERT_STREAMS is not defined in settings, or if + any stream's configuration is invalid. + """ + try: + return get_alert_streams(settings.ALERT_STREAMS) + except AttributeError as err: + raise ImproperlyConfigured( + f'ALERT_STREAMS is not configured in settings.py: {err}' + ) + + +def get_alert_streams(alert_stream_configs: list) -> list[AlertStream]: + """Instantiate and return AlertStream objects from a list of config dicts. + + Use this fuction if your alert streams are configured somewhere other + than settings.ALERT_STREAMS. + + Each config dict must have: + NAME (str): dotted-path to an AlertStream subclass + OPTIONS (dict): keyword arguments passed to the subclass constructor + ACTIVE (bool, optional): if False, skip this stream (defaults to True) + + Args: + alert_stream_configs: List of configuration dictionaries from ALERT_STREAMS. + + Returns: + A list of instantiated AlertStream subclass objects for active streams. + + Raises: + ImproperlyConfigured: if a NAME cannot be imported, or if Pydantic + validation of OPTIONS fails for any stream. + """ + alert_streams = [] + for alert_stream_config in alert_stream_configs: + if not alert_stream_config.get('ACTIVE', True): + logger.debug( + f'get_alert_streams: skipping inactive stream: {alert_stream_config["NAME"]}' + ) + continue + + # Dynamically import the AlertStream subclass by dotted-path name. + try: + klass = import_string(alert_stream_config['NAME']) + except ImportError as err: + raise ImproperlyConfigured( + f'Could not import AlertStream class "{alert_stream_config["NAME"]}". ' + f'Check the NAME key in your ALERT_STREAMS setting. Error: {err}' + ) + + # Pydantic validates required fields in OPTIONS and raises ValidationError + # with field-level detail if anything is missing or mistyped. + try: + alert_stream: AlertStream = klass(**alert_stream_config.get('OPTIONS', {})) + except ValidationError as err: + raise ImproperlyConfigured( + f'Configuration for {alert_stream_config["NAME"]} is invalid:\n{err}' + ) + + alert_streams.append(alert_stream) + + return alert_streams + + +# --------------------------------------------------------------------------- +# Here's a handler that puts an alert (after normalization) into the the Alerts table +# --------------------------------------------------------------------------- + +def save_alert_to_database(raw_alert: Any, alert_stream: AlertStream, **kwargs: Any) -> Alert | None: + """Persist a raw alert to the database after normalization. + + This could be used as example code for an alert handler that might + handle alerts from multiple streams with different formats. The normalization + step (and injected alert stream) make this possible. + + `alert_stream` is injected by AlertStream.listen() so that this one generic + handler can serve all streams — each stream's normalize_alert() provides + stream-specific field extraction without this function needing to know + which stream it is working with (dependency injection via keyword argument). + + Args: + raw_alert: The stream-specific alert object received from listen(). + alert_stream: The AlertStream instance; provides the normalization function. + **kwargs: Stream-specific extras (e.g., topic, metadata from Hopskotch); + the 'topic' kwarg, if present, is forwarded to normalize_alert() so the + NormalizedAlert.topic field is populated correctly. + + Returns: + The created Alert instance, or None if normalization or save fails. + """ + # get the AlertStream subclass-specific normaization function + normalize = alert_stream.get_normalization_function() + topic: str = kwargs.get('topic', '') + try: + normalized_alert: NormalizedAlert = normalize(raw_alert, topic=topic) + # NormalizedAlert is a Pydantic BaseModel subclass with a model_dump method + alert = Alert.objects.create(**normalized_alert.model_dump()) + return alert + except Exception as ex: + logger.error(f'save_alert_to_database: failed to save alert: {ex}' + f'raw_alert: {raw_alert}') + return None From de95b4565fd862c7a3958e9652ae32f7cb312621 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 6 Mar 2026 10:19:38 -0800 Subject: [PATCH 04/33] add stub for ALeRCE stream --- tom_alertstreams/alertstreams/alerce.py | 91 +++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tom_alertstreams/alertstreams/alerce.py diff --git a/tom_alertstreams/alertstreams/alerce.py b/tom_alertstreams/alertstreams/alerce.py new file mode 100644 index 0000000..f43e4c6 --- /dev/null +++ b/tom_alertstreams/alertstreams/alerce.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import logging +import random +import time +from datetime import datetime, timezone +from typing import Any, ClassVar + +from tom_alertstreams.alertstreams.alertstream import AlertStream, AlertStreamConfig, NormalizedAlert + +logger = logging.getLogger(__name__) + +# TODO: remove when stubs are replaced +# Mock data constants — chosen to be unmistakably non-astronomical: +# (0, 0) is not a real survey pointing; 99.0 is the astronomical sentinel for "no data". +_MOCK_RA = 0.0 +_MOCK_DEC = 0.0 +_MOCK_MAGNITUDE = 99.0 + + +class AlerceConfig(AlertStreamConfig): + """Pydantic configuration model for AlerceAlertStream (stub). + + Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + """ + # TODO: replace this stub with actual implementation + pass + + +class AlerceAlertStream(AlertStream): + """Stub ALeRCE AlertStream that generates obviously-fake mock alerts. + """ + configuration_class = AlerceConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'alerce' + ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = 'https://alerce.online/object/{object_id}' + + def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: + """Map a mock ALeRCE alert dict to a NormalizedAlert. + + Args: + raw_alert: Dict produced by listen(); contains mock field values. + topic: Kafka topic the alert was consumed from. + + Returns: + NormalizedAlert populated from the mock dict fields. + """ + # TODO: replace this stub with actual implementation + # super().normalized_alert is @abs.abstractmethod, so the stub needs an implementation + normalized_alert = NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic or raw_alert.get('topic', ''), + timestamp=datetime.fromisoformat(raw_alert['timestamp']), + alert_id=raw_alert['alert_id'], + object_id=raw_alert.get('object_id'), + ra=raw_alert.get('ra'), + dec=raw_alert.get('dec'), + magnitude=raw_alert.get('magnitude'), + raw_payload=raw_alert, + ) + return normalized_alert + + def listen(self) -> None: + """Generate mock ALeRCE alerts and dispatch to configured topic handlers. + + Loops indefinitely, emitting one mock alert per iteration with a random + 5–30 second delay. Topics are round-robined if multiple are configured. + """ + # TODO: replace this stub with actual implementation + counter = 0 + topics = list(self.config.TOPIC_HANDLERS.keys()) + + # for this stub, generate mock alerts (rather than listen to the stream) endlessly + while True: + counter += 1 + topic = topics[counter % len(topics)] + timestamp = datetime.now(timezone.utc) + object_id = f'MOCK-{self.STREAM_NAME.upper()}-{counter:04d}' + alert_id = f'MOCK-{timestamp.strftime("%Y%m%d%H%M%S")}' + mock_alert: dict[str, Any] = { + 'alert_id': alert_id, + 'object_id': object_id, + 'topic': topic, + 'timestamp': timestamp.isoformat(), + 'ra': _MOCK_RA, + 'dec': _MOCK_DEC, + 'magnitude': _MOCK_MAGNITUDE, + 'mock': True, + } + logger.debug(f'AlerceAlertStream: mock alert {object_id}') + self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) + time.sleep(random.uniform(5.0, 30.0)) From ee34db91f99b3434a75a3e46610b86069ae843bb Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 6 Mar 2026 10:21:29 -0800 Subject: [PATCH 05/33] add stub for AMPEL stream --- tom_alertstreams/alertstreams/ampel.py | 91 ++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tom_alertstreams/alertstreams/ampel.py diff --git a/tom_alertstreams/alertstreams/ampel.py b/tom_alertstreams/alertstreams/ampel.py new file mode 100644 index 0000000..9a7b731 --- /dev/null +++ b/tom_alertstreams/alertstreams/ampel.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import logging +import random +import time +from datetime import datetime, timezone +from typing import Any, ClassVar + +from tom_alertstreams.alertstreams.alertstream import AlertStream, AlertStreamConfig, NormalizedAlert + +logger = logging.getLogger(__name__) + +# TODO: remove when stubs are replaced +# Mock data constants — chosen to be unmistakably non-astronomical: +# (0, 0) is not a real survey pointing; 99.0 is the astronomical sentinel for "no data". +_MOCK_RA = 0.0 +_MOCK_DEC = 0.0 +_MOCK_MAGNITUDE = 99.0 + + +class AmpelConfig(AlertStreamConfig): + """Pydantic configuration model for AmpelAlertStream (stub). + + Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + """ + # TODO: replace this stub with actual implementation + pass + + +class AmpelAlertStream(AlertStream): + """Stub AMPEL AlertStream that generates obviously-fake mock alerts. + """ + configuration_class = AmpelConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'ampel' + ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = None + + def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: + """Map a mock AMPEL alert dict to a NormalizedAlert. + + Args: + raw_alert: Dict produced by listen(); contains mock field values. + topic: Kafka topic the alert was consumed from. + + Returns: + NormalizedAlert populated from the mock dict fields. + """ + # TODO: replace this stub with actual implementation + # super().normalized_alert is @abs.abstractmethod, so the stub needs an implementation + normalized_alert = NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic or raw_alert.get('topic', ''), + timestamp=datetime.fromisoformat(raw_alert['timestamp']), + alert_id=raw_alert['alert_id'], + object_id=raw_alert.get('object_id'), + ra=raw_alert.get('ra'), + dec=raw_alert.get('dec'), + magnitude=raw_alert.get('magnitude'), + raw_payload=raw_alert, + ) + return normalized_alert + + def listen(self) -> None: + """Generate mock AMPEL alerts and dispatch to configured topic handlers. + + Loops indefinitely, emitting one mock alert per iteration with a random + 5–30 second delay. Topics are round-robined if multiple are configured. + """ + # TODO: replace this stub with actual implementation + counter = 0 + topics = list(self.config.TOPIC_HANDLERS.keys()) + + # for this stub, generate mock alerts (rather than listen to the stream) endlessly + while True: + counter += 1 + topic = topics[counter % len(topics)] # round-robin + timestamp = datetime.now(timezone.utc) + object_id = f'MOCK-{self.STREAM_NAME.upper()}-{counter:04d}' + alert_id = f'MOCK-{timestamp.strftime("%Y%m%d%H%M%S")}' + mock_alert: dict[str, Any] = { + 'alert_id': alert_id, + 'object_id': object_id, + 'topic': topic, + 'timestamp': timestamp.isoformat(), + 'ra': _MOCK_RA, + 'dec': _MOCK_DEC, + 'magnitude': _MOCK_MAGNITUDE, + 'mock': True, + } + logger.debug(f'AmpelAlertStream: mock alert {object_id}') + self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) + time.sleep(random.uniform(5.0, 30.0)) From 8a313ea8df238e3ea90e53e480ab1903e9ab3af0 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 6 Mar 2026 10:21:45 -0800 Subject: [PATCH 06/33] add stub for Babamul stream --- tom_alertstreams/alertstreams/babamul.py | 91 ++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tom_alertstreams/alertstreams/babamul.py diff --git a/tom_alertstreams/alertstreams/babamul.py b/tom_alertstreams/alertstreams/babamul.py new file mode 100644 index 0000000..82b6f05 --- /dev/null +++ b/tom_alertstreams/alertstreams/babamul.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import logging +import random +import time +from datetime import datetime, timezone +from typing import Any, ClassVar + +from tom_alertstreams.alertstreams.alertstream import AlertStream, AlertStreamConfig, NormalizedAlert + +logger = logging.getLogger(__name__) + +# TODO: remove when stubs are replaced +# Mock data constants — chosen to be unmistakably non-astronomical: +# (0, 0) is not a real survey pointing; 99.0 is the astronomical sentinel for "no data". +_MOCK_RA = 0.0 +_MOCK_DEC = 0.0 +_MOCK_MAGNITUDE = 99.0 + + +class BabamulConfig(AlertStreamConfig): + """Pydantic configuration model for BabamulAlertStream (stub). + + Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + """ + # TODO: replace this stub with actual implementation + pass + + +class BabamulAlertStream(AlertStream): + """Stub Babamul AlertStream that generates obviously-fake mock alerts. + """ + configuration_class = BabamulConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'babamul' + ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = None + + def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: + """Map a mock Babamul alert dict to a NormalizedAlert. + + Args: + raw_alert: Dict produced by listen(); contains mock field values. + topic: Kafka topic the alert was consumed from. + + Returns: + NormalizedAlert populated from the mock dict fields. + """ + # TODO: replace this stub with actual implementation + # super().normalized_alert is @abs.abstractmethod, so the stub needs an implementation + normalized_alert = NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic or raw_alert.get('topic', ''), + timestamp=datetime.fromisoformat(raw_alert['timestamp']), + alert_id=raw_alert['alert_id'], + object_id=raw_alert.get('object_id'), + ra=raw_alert.get('ra'), + dec=raw_alert.get('dec'), + magnitude=raw_alert.get('magnitude'), + raw_payload=raw_alert, + ) + return normalized_alert + + def listen(self) -> None: + """Generate mock Babamul alerts and dispatch to configured topic handlers. + + Loops indefinitely, emitting one mock alert per iteration with a random + 5–30 second delay. Topics are round-robined if multiple are configured. + """ + # TODO: replace this stub with actual implementation + counter = 0 + topics = list(self.config.TOPIC_HANDLERS.keys()) + + # for this stub, generate mock alerts (rather than listen to the stream) endlessly + while True: + counter += 1 + topic = topics[counter % len(topics)] + timestamp = datetime.now(timezone.utc) + object_id = f'MOCK-{self.STREAM_NAME.upper()}-{counter:04d}' + alert_id = f'MOCK-{timestamp.strftime("%Y%m%d%H%M%S")}' + mock_alert: dict[str, Any] = { + 'alert_id': alert_id, + 'object_id': object_id, + 'topic': topic, + 'timestamp': timestamp.isoformat(), + 'ra': _MOCK_RA, + 'dec': _MOCK_DEC, + 'magnitude': _MOCK_MAGNITUDE, + 'mock': True, + } + logger.debug(f'BabamulAlertStream: mock alert {object_id}') + self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) + time.sleep(random.uniform(5.0, 30.0)) From fd47bc0719c0322b8befaf4ddb78d443f0a9904f Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 6 Mar 2026 10:22:03 -0800 Subject: [PATCH 07/33] add stub for Fink stream listener --- tom_alertstreams/alertstreams/fink.py | 91 +++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tom_alertstreams/alertstreams/fink.py diff --git a/tom_alertstreams/alertstreams/fink.py b/tom_alertstreams/alertstreams/fink.py new file mode 100644 index 0000000..987a02d --- /dev/null +++ b/tom_alertstreams/alertstreams/fink.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import logging +import random +import time +from datetime import datetime, timezone +from typing import Any, ClassVar + +from tom_alertstreams.alertstreams.alertstream import AlertStream, AlertStreamConfig, NormalizedAlert + +logger = logging.getLogger(__name__) + +# TODO: remove when stubs are replaced +# Mock data constants — chosen to be unmistakably non-astronomical: +# (0, 0) is not a real survey pointing; 99.0 is the astronomical sentinel for "no data". +_MOCK_RA = 0.0 +_MOCK_DEC = 0.0 +_MOCK_MAGNITUDE = 99.0 + + +class FinkConfig(AlertStreamConfig): + """Pydantic configuration model for FinkAlertStream (stub). + + Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + """ + # TODO: replace this stub with actual implementation + pass + + +class FinkAlertStream(AlertStream): + """Stub Fink AlertStream that generates obviously-fake mock alerts. + """ + configuration_class = FinkConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'fink' + ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = 'https://fink-portal.org/{object_id}' + + def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: + """Map a mock Fink alert dict to a NormalizedAlert. + + Args: + raw_alert: Dict produced by listen(); contains mock field values. + topic: Kafka topic the alert was consumed from. + + Returns: + NormalizedAlert populated from the mock dict fields. + """ + # TODO: replace this stub with actual implementation + # super().normalized_alert is @abs.abstractmethod, so the stub needs an implementation + normalized_alert = NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic or raw_alert.get('topic', ''), + timestamp=datetime.fromisoformat(raw_alert['timestamp']), + alert_id=raw_alert['alert_id'], + object_id=raw_alert.get('object_id'), + ra=raw_alert.get('ra'), + dec=raw_alert.get('dec'), + magnitude=raw_alert.get('magnitude'), + raw_payload=raw_alert, + ) + return normalized_alert + + def listen(self) -> None: + """Generate mock Fink alerts and dispatch to configured topic handlers. + + Loops indefinitely, emitting one mock alert per iteration with a random + 5–30 second delay. Topics are round-robined if multiple are configured. + """ + # TODO: replace this stub with actual implementation + counter = 0 + topics = list(self.config.TOPIC_HANDLERS.keys()) + + # for this stub, generate mock alerts (rather than listen to the stream) endlessly + while True: + counter += 1 + topic = topics[counter % len(topics)] + timestamp = datetime.now(timezone.utc) + object_id = f'MOCK-{self.STREAM_NAME.upper()}-{counter:04d}' + alert_id = f'MOCK-{timestamp.strftime("%Y%m%d%H%M%S")}' + mock_alert: dict[str, Any] = { + 'alert_id': alert_id, + 'object_id': object_id, + 'topic': topic, + 'timestamp': timestamp.isoformat(), + 'ra': _MOCK_RA, + 'dec': _MOCK_DEC, + 'magnitude': _MOCK_MAGNITUDE, + 'mock': True, + } + logger.debug(f'FinkAlertStream: mock alert {object_id}') + self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) + time.sleep(random.uniform(5.0, 30.0)) From 17947f98e3dbb0fcf18a4ae5720bd9440cd1c853 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 6 Mar 2026 10:22:21 -0800 Subject: [PATCH 08/33] add stub for Lasair stream listener --- tom_alertstreams/alertstreams/lasair.py | 93 +++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tom_alertstreams/alertstreams/lasair.py diff --git a/tom_alertstreams/alertstreams/lasair.py b/tom_alertstreams/alertstreams/lasair.py new file mode 100644 index 0000000..3a5654c --- /dev/null +++ b/tom_alertstreams/alertstreams/lasair.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import logging +import random +import time +from datetime import datetime, timezone +from typing import Any, ClassVar + +from tom_alertstreams.alertstreams.alertstream import AlertStream, AlertStreamConfig, NormalizedAlert + +logger = logging.getLogger(__name__) + +# TODO: remove when stubs are replaced +# Mock data constants — chosen to be unmistakably non-astronomical: +# (0, 0) is not a real survey pointing; 99.0 is the astronomical sentinel for "no data". +_MOCK_RA = 0.0 +_MOCK_DEC = 0.0 +_MOCK_MAGNITUDE = 99.0 + + +class LasairConfig(AlertStreamConfig): + """Pydantic configuration model for LasairAlertStream (stub). + + Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + """ + # TODO: replace this stub with actual implementation + pass + + +class LasairAlertStream(AlertStream): + """Stub Lasair AlertStream that generates obviously-fake mock alerts. + + See https://lasair-ztf.lsst.ac.uk + """ + configuration_class = LasairConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'lasair' + ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = 'https://lasair-ztf.lsst.ac.uk/objects/{object_id}/' + + def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: + """Map a mock Lasair alert dict to a NormalizedAlert. + + Args: + raw_alert: Dict produced by listen(); contains mock field values. + topic: Kafka topic the alert was consumed from. + + Returns: + NormalizedAlert populated from the mock dict fields. + """ + # TODO: replace this stub with actual implementation + # super().normalized_alert is @abs.abstractmethod, so the stub needs an implementation + normalized_alert = NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic or raw_alert.get('topic', ''), + timestamp=datetime.fromisoformat(raw_alert['timestamp']), + alert_id=raw_alert['alert_id'], + object_id=raw_alert.get('object_id'), + ra=raw_alert.get('ra'), + dec=raw_alert.get('dec'), + magnitude=raw_alert.get('magnitude'), + raw_payload=raw_alert, + ) + return normalized_alert + + def listen(self) -> None: + """Generate mock Lasair alerts and dispatch to configured topic handlers. + + Loops indefinitely, emitting one mock alert per iteration with a random + 5–30 second delay. Topics are round-robined if multiple are configured. + """ + # TODO: replace this stub with actual implementation + counter = 0 + topics = list(self.config.TOPIC_HANDLERS.keys()) + + # for this stub, generate mock alerts (rather than listen to the stream) endlessly + while True: + counter += 1 + topic = topics[counter % len(topics)] + timestamp = datetime.now(timezone.utc) + object_id = f'MOCK-{self.STREAM_NAME.upper()}-{counter:04d}' + alert_id = f'MOCK-{timestamp.strftime("%Y%m%d%H%M%S")}' + mock_alert: dict[str, Any] = { + 'alert_id': alert_id, + 'object_id': object_id, + 'topic': topic, + 'timestamp': timestamp.isoformat(), + 'ra': _MOCK_RA, + 'dec': _MOCK_DEC, + 'magnitude': _MOCK_MAGNITUDE, + 'mock': True, + } + logger.debug(f'LasairAlertStream: mock alert {object_id}') + self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) + time.sleep(random.uniform(5.0, 30.0)) From 25523095f599b80ef34c5fcacca9b77a90b02ceb Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 6 Mar 2026 10:22:59 -0800 Subject: [PATCH 09/33] add stub for Pitt-Google stream listener --- tom_alertstreams/alertstreams/pittgoogle.py | 93 +++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tom_alertstreams/alertstreams/pittgoogle.py diff --git a/tom_alertstreams/alertstreams/pittgoogle.py b/tom_alertstreams/alertstreams/pittgoogle.py new file mode 100644 index 0000000..690574d --- /dev/null +++ b/tom_alertstreams/alertstreams/pittgoogle.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import logging +import random +import time +from datetime import datetime, timezone +from typing import Any, ClassVar + +from tom_alertstreams.alertstreams.alertstream import AlertStream, AlertStreamConfig, NormalizedAlert + +logger = logging.getLogger(__name__) + +# TODO: remove when stubs are replaced +# Mock data constants — chosen to be unmistakably non-astronomical: +# (0, 0) is not a real survey pointing; 99.0 is the astronomical sentinel for "no data". +_MOCK_RA = 0.0 +_MOCK_DEC = 0.0 +_MOCK_MAGNITUDE = 99.0 + + +class PittGoogleConfig(AlertStreamConfig): + """Pydantic configuration model for PittGoogleAlertStream (stub). + + Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + """ + # TODO: replace this stub with actual implementation + pass + + +class PittGoogleAlertStream(AlertStream): + """Stub Pitt-Google AlertStream that generates obviously-fake mock alerts. + + See https://pittgooglebroker.readthedocs.io for the real broker. + """ + configuration_class = PittGoogleConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'pittgoogle' + ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = None + + def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: + """Map a mock Pitt-Google alert dict to a NormalizedAlert. + + Args: + raw_alert: Dict produced by listen(); contains mock field values. + topic: Kafka topic the alert was consumed from. + + Returns: + NormalizedAlert populated from the mock dict fields. + """ + # TODO: replace this stub with actual implementation + # super().normalized_alert is @abs.abstractmethod, so the stub needs an implementation + normalized_alert = NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic or raw_alert.get('topic', ''), + timestamp=datetime.fromisoformat(raw_alert['timestamp']), + alert_id=raw_alert['alert_id'], + object_id=raw_alert.get('object_id'), + ra=raw_alert.get('ra'), + dec=raw_alert.get('dec'), + magnitude=raw_alert.get('magnitude'), + raw_payload=raw_alert, + ) + return normalized_alert + + def listen(self) -> None: + """Generate mock Pitt-Google alerts and dispatch to configured topic handlers. + + Loops indefinitely, emitting one mock alert per iteration with a random + 5–30 second delay. Topics are round-robined if multiple are configured. + """ + # TODO: replace this stub with actual implementation + counter = 0 + topics = list(self.config.TOPIC_HANDLERS.keys()) + + # for this stub, generate mock alerts (rather than listen to the stream) endlessly + while True: + counter += 1 + topic = topics[counter % len(topics)] + timestamp = datetime.now(timezone.utc) + object_id = f'MOCK-{self.STREAM_NAME.upper()}-{counter:04d}' + alert_id = f'MOCK-{timestamp.strftime("%Y%m%d%H%M%S")}' + mock_alert: dict[str, Any] = { + 'alert_id': alert_id, + 'object_id': object_id, + 'topic': topic, + 'timestamp': timestamp.isoformat(), + 'ra': _MOCK_RA, + 'dec': _MOCK_DEC, + 'magnitude': _MOCK_MAGNITUDE, + 'mock': True, + } + logger.debug(f'PittGoogleAlertStream: mock alert {object_id}') + self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) + time.sleep(random.uniform(5.0, 30.0)) From 2cc12c7d4497871de9323e5d5a48c75406cd120c Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Mon, 9 Mar 2026 16:50:17 -0700 Subject: [PATCH 10/33] set up "Time since last alert" dashboard --- .../partials/alert_table_partial.html | 34 +++++++++++++++++++ .../partials/stream_dashboard.html | 20 +++++++++++ .../tom_alertstreams/recent_alerts.html | 7 +++- 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tom_alertstreams/templates/tom_alertstreams/partials/alert_table_partial.html create mode 100644 tom_alertstreams/templates/tom_alertstreams/partials/stream_dashboard.html diff --git a/tom_alertstreams/templates/tom_alertstreams/partials/alert_table_partial.html b/tom_alertstreams/templates/tom_alertstreams/partials/alert_table_partial.html new file mode 100644 index 0000000..a2f52ec --- /dev/null +++ b/tom_alertstreams/templates/tom_alertstreams/partials/alert_table_partial.html @@ -0,0 +1,34 @@ +{# Custom partial for AlertTable HTMX responses. #} +{# Renders the table, then appends an OOB swap to update the stream status #} +{# dashboard that lives outside the table container on the full page. #} +{% load render_table from django_tables2 %} + +{% render_table table %} + +{% if not table.data %} +
    + {% if empty_database %} + No records in the database. + {% else %} + No records match those filters. + {% endif %} +
    +{% endif %} + +{# OOB swap: update the stream dashboard outside the table container. #} +{# Only included on HTMX requests — on full page loads the dashboard is #} +{# already rendered by recent_alerts.html and this partial is just inlined. #} +{% if request.htmx %} +
    + {% include "tom_alertstreams/partials/stream_dashboard.html" %} +
    +{% endif %} + + diff --git a/tom_alertstreams/templates/tom_alertstreams/partials/stream_dashboard.html b/tom_alertstreams/templates/tom_alertstreams/partials/stream_dashboard.html new file mode 100644 index 0000000..0e4bd7e --- /dev/null +++ b/tom_alertstreams/templates/tom_alertstreams/partials/stream_dashboard.html @@ -0,0 +1,20 @@ +{# Stream status dashboard: shows time-since-last-alert for each configured stream. #} +{# Included from recent_alerts.html (full page) and alert_table_partial.html (OOB). #} + +{% if stream_status %} +

    Time since last alert received:

    +
    + {% for s in stream_status %} +
    + {{ s.stream_name }} + + {% if s.latest_timestamp %} + {{ s.latest_timestamp|timesince:s.now }} ago + {% else %} + No alerts + {% endif %} + +
    + {% endfor %} +
    +{% endif %} diff --git a/tom_alertstreams/templates/tom_alertstreams/recent_alerts.html b/tom_alertstreams/templates/tom_alertstreams/recent_alerts.html index 8a67e5b..9680e69 100644 --- a/tom_alertstreams/templates/tom_alertstreams/recent_alerts.html +++ b/tom_alertstreams/templates/tom_alertstreams/recent_alerts.html @@ -11,6 +11,11 @@

    Recent Alerts

    The {{ record_count }} most recent alert{{ record_count|pluralize }} received from configured alert streams. Alert IDs link to the stream's public archive where available.

    + + {# Stream status dashboard -- auto-refreshes via OOB swap in the table partial #} +
    + {% include "tom_alertstreams/partials/stream_dashboard.html" %} +
    {# Filter form -- id="filter-form" must match hx-include in AlertTable.Meta.attrs #} @@ -33,7 +38,7 @@

    Recent Alerts

    hx-trigger="every 5s" hx-swap="innerHTML" hx-include="#filter-form"> - {% include table.get_partial_template_name %} + {% include table.get_partial_template_name %} {# render the table via it's partial #} From 5ceedbd728dfa3428d83ab58b494dcbfa50873c1 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Mon, 9 Mar 2026 16:52:21 -0700 Subject: [PATCH 11/33] refactor how we read the configuration; add last alert context --- tom_alertstreams/views.py | 97 ++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 33 deletions(-) diff --git a/tom_alertstreams/views.py b/tom_alertstreams/views.py index ebe4860..55246a3 100644 --- a/tom_alertstreams/views.py +++ b/tom_alertstreams/views.py @@ -1,37 +1,59 @@ from __future__ import annotations -import logging from typing import Any -from django.conf import settings -from django.utils.module_loading import import_string +from django.db.models import Max +from django.utils import timezone from django_filters.views import FilterView +from tom_alertstreams.alertstreams.alertstream import get_alert_stream_classes from tom_alertstreams.models import Alert -from tom_alertstreams.tables import AlertFilterSet, AlertTable +from tom_alertstreams.tables import ( + AlertFilterSet, AlertStreamPresenter, AlertTable, STREAM_PRESENTERS, +) from tom_common.htmx_table import HTMXTableViewMixin -logger = logging.getLogger(__name__) +def _build_presenter_map() -> dict[str, AlertStreamPresenter]: + """Build a presenter instance for each configured active alert stream. -def _build_archive_url_map() -> dict[str, str | None]: - """Build {stream_name → archive_url_template} from ALERT_STREAMS settings. + Looks up each stream's STREAM_NAME in the STREAM_PRESENTERS registry. + Streams not in the registry get the default AlertStreamPresenter (no URLs). + """ + return { + klass.STREAM_NAME: STREAM_PRESENTERS.get(klass.STREAM_NAME, AlertStreamPresenter)() + for klass in get_alert_stream_classes() + } + + +def _build_stream_status() -> list[dict[str, Any]]: + """Build per-stream "last seen" status for the dashboard. + + Returns a list of dicts with keys: stream_name, latest_timestamp, now. + Includes all configured active streams — streams with no alerts in the + database appear with latest_timestamp=None. - Reads ALERT_STREAMS and imports each active stream class by dotted path to - access its STREAM_NAME and ARCHIVE_URL_TEMPLATE class variables. Does NOT - instantiate the streams — no network connections are made. Returns an empty - dict if ALERT_STREAMS is not configured. + One aggregate DB query (covered by the (stream_name, timestamp) index). """ - url_map: dict[str, str | None] = {} - for stream_config in getattr(settings, 'ALERT_STREAMS', []): - if not stream_config.get('ACTIVE', True): - continue - try: - klass = import_string(stream_config['NAME']) - url_map[klass.STREAM_NAME] = klass.ARCHIVE_URL_TEMPLATE - except (ImportError, AttributeError, KeyError) as exc: - logger.warning(f'_build_archive_url_map: could not read stream class {stream_config.get("NAME")}: {exc}') - return url_map + # Latest alert timestamp per stream, in one query + latest_by_stream: dict[str, Any] = { + row['stream_name']: row['latest'] + for row in Alert.objects.values('stream_name').annotate(latest=Max('timestamp')) + } + + # Ordered list of configured stream names + configured_streams = [klass.STREAM_NAME for klass in get_alert_stream_classes()] + + # Single now value so timesince is consistent across all badges + now = timezone.now() + return [ + { + 'stream_name': name, + 'latest_timestamp': latest_by_stream.get(name), + 'now': now, + } + for name in configured_streams + ] class RecentAlertsView(HTMXTableViewMixin, FilterView): @@ -40,9 +62,9 @@ class RecentAlertsView(HTMXTableViewMixin, FilterView): No login is required — the Recent Alerts page is intentionally public so that demo visitors and potential TOM developers can browse it without an account. - Archive URL links (e.g. to ANTARES, ALeRCE) are built on the fly from each - stream's ARCHIVE_URL_TEMPLATE class variable, so no URL needs to be stored - in the database. + Alert and object links (e.g. to ANTARES loci, ALeRCE objects) are built on the + fly by AlertStreamPresenter subclasses (registered in tables.STREAM_PRESENTERS), + so no URLs need to be stored in the database. """ template_name = 'tom_alertstreams/recent_alerts.html' model = Alert @@ -50,23 +72,32 @@ class RecentAlertsView(HTMXTableViewMixin, FilterView): filterset_class = AlertFilterSet paginate_by = 20 - def get_table_kwargs(self) -> dict[str, Any]: - """Inject the archive url_map into the AlertTable constructor. + def get_context_data(self, **kwargs: Any) -> dict[str, Any]: + """Add stream status data for the dashboard. + + Runs on every request (both full page and HTMX partial) so the OOB + swap in the custom partial template can refresh the dashboard badges. + """ + context = super().get_context_data(**kwargs) + context['stream_status'] = _build_stream_status() + return context - The "archive url map" is used to create links that appear in the - Recent Alerts table. The link is to the alert at the alert brokers site. + def get_table_kwargs(self) -> dict[str, Any]: + """Inject the presenter map into the AlertTable constructor. - The map is built above using the class property set on the AlertStream - subclasses. + Each configured AlertStream is paired with an AlertStreamPresenter + (looked up by STREAM_NAME in the STREAM_PRESENTERS registry). The + presenter handles URL construction — the table just calls + presenter.alert_url() / presenter.object_url() and renders the result. This method is implemented in django-tables2.SingleTableMixin, which HTMXTableViewMixin inherits from. It's called like this: get_context_data() # SingleTableMixin (django-tables2) └── get_table(**self.get_table_kwargs()) - ├── get_table_kwargs() # returns {} by default; we override to add url_map - └── get_table(url_map=…) # instantiates AlertTable(data=…, url_map=…) + ├── get_table_kwargs() # returns {} by default; we override + └── get_table(presenter_map=…) """ kwargs = super().get_table_kwargs() - kwargs['url_map'] = _build_archive_url_map() + kwargs['presenter_map'] = _build_presenter_map() return kwargs From 1cb5250fc20ff4f7b34a8411fae023f60a20343e Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Mon, 9 Mar 2026 16:53:27 -0700 Subject: [PATCH 12/33] implement Babamul stream listener --- pyproject.toml | 2 + tom_alertstreams/alertstreams/babamul.py | 129 +++++++++++++---------- 2 files changed, 73 insertions(+), 58 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4950008..518a153 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,12 +44,14 @@ dependencies = [ gcn = ["gcn-kafka >=0.3,<1.0"] hopskotch = ["hop-client >=0.10,<1.0"] antares = ["antares-client"] +babamul = ["babamul >=0.1,<1"] fink = ["fink-client >=8.8,<9"] all-streams = [ "gcn-kafka >=0.3,<1.0", "hop-client >=0.10,<1.0", "antares-client", + "babamul >=0.1,<1", "fink-client >=8.8,<9", ] diff --git a/tom_alertstreams/alertstreams/babamul.py b/tom_alertstreams/alertstreams/babamul.py index 82b6f05..1ae3114 100644 --- a/tom_alertstreams/alertstreams/babamul.py +++ b/tom_alertstreams/alertstreams/babamul.py @@ -1,91 +1,104 @@ from __future__ import annotations import logging -import random -import time -from datetime import datetime, timezone -from typing import Any, ClassVar +from typing import ClassVar, Literal + +from babamul import AlertConsumer, LsstAlert, ZtfAlert +from pydantic import Field from tom_alertstreams.alertstreams.alertstream import AlertStream, AlertStreamConfig, NormalizedAlert logger = logging.getLogger(__name__) -# TODO: remove when stubs are replaced -# Mock data constants — chosen to be unmistakably non-astronomical: -# (0, 0) is not a real survey pointing; 99.0 is the astronomical sentinel for "no data". -_MOCK_RA = 0.0 -_MOCK_DEC = 0.0 -_MOCK_MAGNITUDE = 99.0 - class BabamulConfig(AlertStreamConfig): - """Pydantic configuration model for BabamulAlertStream (stub). + """Pydantic configuration model for BabamulAlertStream. Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + + Fields: + BABAMUL_KAFKA_USERNAME: Kafka username for the Babamul broker (required). + BABAMUL_KAFKA_PASSWORD: Kafka password for the Babamul broker (required). + BABAMUL_GROUP_ID: Kafka consumer group ID. Alerts are partitioned across + consumers sharing the same group_id, so each TOM instance should use + a unique group_id to receive all alerts. + BABAMUL_AUTO_COMMIT: Whether to auto-commit Kafka offsets after consuming. + False (default) means offsets are not committed, so restarting the + consumer replays from the configured BABAMUL_OFFSET position. + BABAMUL_OFFSET: Where to start reading when no committed offset exists. + 'EARLIEST' replays all available alerts; 'LATEST' starts from new ones. """ - # TODO: replace this stub with actual implementation - pass + BABAMUL_KAFKA_USERNAME: str = Field(min_length=1) # don't accept an empty string + BABAMUL_KAFKA_PASSWORD: str = Field(min_length=1) + BABAMUL_GROUP_ID: str = Field(min_length=1) + BABAMUL_AUTO_COMMIT: bool = False + BABAMUL_OFFSET: Literal['earliest', 'latest'] = 'latest' class BabamulAlertStream(AlertStream): - """Stub Babamul AlertStream that generates obviously-fake mock alerts. + """AlertStream implementation for Babamul (https://github.com/boom-astro/babamul). """ configuration_class = BabamulConfig # type: ignore[assignment] STREAM_NAME: ClassVar[str] = 'babamul' - ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = None - def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: - """Map a mock Babamul alert dict to a NormalizedAlert. + def normalize_alert(self, raw_alert: ZtfAlert | LsstAlert, topic: str = '') -> NormalizedAlert: + """Convert a babamul alert to a NormalizedAlert. + + ZtfAlert and LsstAlert are Pydantic BaseModel subclasses provided by the + ``babamul`` package. Because they are Pydantic models we can: + - access fields via typed attributes (raw_alert.objectId, candidate.ra, etc.) + - serialize to JSON-safe dicts with model_dump(mode='json'), which handles + datetimes, enums (e.g. Band), and nested models automatically + - skip defensive getattr() / try-except — field access is guaranteed by the schema Args: - raw_alert: Dict produced by listen(); contains mock field values. - topic: Kafka topic the alert was consumed from. + raw_alert: A babamul ZtfAlert or LsstAlert from AlertConsumer. + topic: The Kafka topic the alert arrived on. Falls back to + raw_alert.topic if not provided. Returns: - NormalizedAlert populated from the mock dict fields. + NormalizedAlert with fields extracted from the babamul alert's candidate. """ - # TODO: replace this stub with actual implementation - # super().normalized_alert is @abs.abstractmethod, so the stub needs an implementation - normalized_alert = NormalizedAlert( + candidate = raw_alert.candidate + + return NormalizedAlert( stream_name=self.STREAM_NAME, - topic=topic or raw_alert.get('topic', ''), - timestamp=datetime.fromisoformat(raw_alert['timestamp']), - alert_id=raw_alert['alert_id'], - object_id=raw_alert.get('object_id'), - ra=raw_alert.get('ra'), - dec=raw_alert.get('dec'), - magnitude=raw_alert.get('magnitude'), - raw_payload=raw_alert, + topic=topic or raw_alert.topic or '', + timestamp=candidate.datetime, + alert_id=str(raw_alert.candid), + object_id=raw_alert.objectId, + ra=candidate.ra, + dec=candidate.dec, + magnitude=candidate.magpsf, + raw_payload=raw_alert.model_dump(mode='json'), ) - return normalized_alert def listen(self) -> None: - """Generate mock Babamul alerts and dispatch to configured topic handlers. + """Consume Babamul alerts and dispatch to configured topic handlers. - Loops indefinitely, emitting one mock alert per iteration with a random - 5–30 second delay. Topics are round-robined if multiple are configured. + Opens a babamul AlertConsumer as a context manager and iterates over + incoming alerts indefinitely. Each alert is dispatched to the handler + configured for its topic. """ - # TODO: replace this stub with actual implementation - counter = 0 topics = list(self.config.TOPIC_HANDLERS.keys()) - # for this stub, generate mock alerts (rather than listen to the stream) endlessly - while True: - counter += 1 - topic = topics[counter % len(topics)] - timestamp = datetime.now(timezone.utc) - object_id = f'MOCK-{self.STREAM_NAME.upper()}-{counter:04d}' - alert_id = f'MOCK-{timestamp.strftime("%Y%m%d%H%M%S")}' - mock_alert: dict[str, Any] = { - 'alert_id': alert_id, - 'object_id': object_id, - 'topic': topic, - 'timestamp': timestamp.isoformat(), - 'ra': _MOCK_RA, - 'dec': _MOCK_DEC, - 'magnitude': _MOCK_MAGNITUDE, - 'mock': True, - } - logger.debug(f'BabamulAlertStream: mock alert {object_id}') - self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) - time.sleep(random.uniform(5.0, 30.0)) + with AlertConsumer( + topics=topics, + username=self.config.BABAMUL_KAFKA_USERNAME, + password=self.config.BABAMUL_KAFKA_PASSWORD, + group_id=self.config.BABAMUL_GROUP_ID, + offset=self.config.BABAMUL_OFFSET, + auto_commit=self.config.BABAMUL_AUTO_COMMIT, + ) as consumer: + alert: ZtfAlert | LsstAlert + for alert in consumer: # yields ZtfAlert | LsstAlert (Pydantic models) + topic = alert.topic or '' + if topic not in self.alert_handler: + logger.warning( + f'BabamulAlertStream: alert from topic "{topic}" has no handler. ' + f'Configured topics: {list(self.alert_handler.keys())}' + ) + continue + + logger.debug(f'BabamulAlertStream: alert {alert.objectId} (candid={alert.candid}) on {topic}') + self.alert_handler[topic](alert, alert_stream=self, topic=topic) From 1c891f395217739ba53f19d26d319aa6458c7050 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Mon, 9 Mar 2026 16:54:28 -0700 Subject: [PATCH 13/33] separate alert ingestion from presentation this introduces the AlertStreamPresenter class and it's stream-specific subclasses. This allows the AlertTable to create links to s and s as appropriate for each alert stream. (And the AlertStream class itself doesn't have to know about any of that). --- tom_alertstreams/alertstreams/alertstream.py | 44 ++++- tom_alertstreams/tables.py | 189 ++++++++++++++++--- 2 files changed, 200 insertions(+), 33 deletions(-) diff --git a/tom_alertstreams/alertstreams/alertstream.py b/tom_alertstreams/alertstreams/alertstream.py index 64705a6..07804b7 100644 --- a/tom_alertstreams/alertstreams/alertstream.py +++ b/tom_alertstreams/alertstreams/alertstream.py @@ -99,10 +99,11 @@ class AlertStream(abc.ABC): 2. Set class variables: configuration_class = MyStreamConfig STREAM_NAME = 'mystream' # short canonical name, written to Alert.stream_name - ARCHIVE_URL_TEMPLATE = 'https://...' # used to create links to alerts 3. Override normalize_alert(raw_alert, topic='') -> NormalizedAlert to extract - stream-specific fields (ra, dec, magnitude, object_id, etc.). + stream-specific fields (ra, dec, magnitude, object_id, etc.). This returns + a basic (Pydantic BaseModel subclass) NormaizedAlert that can be consistently + used reguardless of which stream the alert came from. 4. Implement listen() -> None. This method is not expected to return. It should: a. Connect to the Kafka stream using credentials from self.config @@ -125,12 +126,10 @@ class AlertStream(abc.ABC): configuration_class: ClassVar[type[AlertStreamConfig]] # Short canonical name written to Alert.stream_name. Must be unique across - # all configured streams. Used by the Recent Alerts view to build archive URL maps. + # all configured streams. Used by the presenter registry in tables.py to + # look up the appropriate AlertStreamPresenter for URL construction. STREAM_NAME: ClassVar[str] - # this should be a URL that can be used to create a link to an alert at a broker - ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = None - def __init__(self, **kwargs: Any) -> None: # read and validate the alertstream configuration self.config: AlertStreamConfig = self.configuration_class(**kwargs) @@ -148,6 +147,8 @@ def _get_stream_classname(self) -> str: def _process_topic_handlers(self) -> dict[str, Callable]: """Import and return handler callables from the TOPIC_HANDLERS configuration. + This is a step in the AlertStream instanciation: + In settings.py, the configuration dictionary TOPIC_HANDLER dictionary for each stream maps a topic to a dotted-path string specifying the alert handler for that topic's alerts. This method converts the dotted-path string @@ -237,9 +238,38 @@ def listen(self) -> None: # Module-level helper functions # --------------------------------------------------------------------------- +def get_alert_stream_classes() -> list[type[AlertStream]]: + """Return the imported class for each configured alert stream. + + Imports each active stream class from settings.ALERT_STREAMS by its dotted + NAME path. Does NOT instantiate the classes — this is the lightweight + alternative to get_alert_streams() for when you only need access to class + attributes (e.g. STREAM_NAME). + + Streams that are inactive (ACTIVE=False) are skipped. Streams that fail to + import are logged and skipped so a single misconfigured entry doesn't break + the caller. + """ + classes: list[type[AlertStream]] = [] + for stream_config in getattr(settings, 'ALERT_STREAMS', []): + if not stream_config.get('ACTIVE', True): + continue + try: + classes.append(import_string(stream_config['NAME'])) + except (ImportError, AttributeError, KeyError) as exc: + logger.warning( + 'get_alert_stream_classes: could not import %s: %s', + stream_config.get('NAME'), exc, + ) + return classes + + def get_default_alert_streams() -> list[AlertStream]: """Return the AlertStream instances configured in settings.ALERT_STREAMS. + `get_alert_streams()` is the general function. Here, we call that function + and pass in the configuration dictionary from settings.ALERT_STREAMS. + Raises: ImproperlyConfigured: if ALERT_STREAMS is not defined in settings, or if any stream's configuration is invalid. @@ -255,7 +285,7 @@ def get_default_alert_streams() -> list[AlertStream]: def get_alert_streams(alert_stream_configs: list) -> list[AlertStream]: """Instantiate and return AlertStream objects from a list of config dicts. - Use this fuction if your alert streams are configured somewhere other + Use this function if your alert streams are configured somewhere other than settings.ALERT_STREAMS. Each config dict must have: diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py index c160107..1f55c6a 100644 --- a/tom_alertstreams/tables.py +++ b/tom_alertstreams/tables.py @@ -1,32 +1,42 @@ from __future__ import annotations import logging -from typing import Any +from typing import Any, ClassVar import django_filters import django_tables2 as tables from django import forms -from django.conf import settings from django.utils.html import format_html -from django.utils.module_loading import import_string +from tom_alertstreams.alertstreams.alertstream import get_alert_stream_classes from tom_alertstreams.models import Alert from tom_common.htmx_table import HTMXTable, HTMXTableFilterSet logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# AlertTable +# --------------------------------------------------------------------------- + class AlertTable(HTMXTable): """HTMX-driven table of recent alerts from all configured alert streams. - Receives url_map at construction time so that render_alert_id() can construct - archive links on the fly, without storing URLs in the model. Streams with no - ARCHIVE_URL_TEMPLATE (GCN, Hopskotch, stubs) display alert_id as plain text. + Receives a presenter_map at construction time — a dict mapping stream_name + to an AlertStreamPresenter instance. render_alert_id() and render_object_id() + delegate URL construction to the presenter, keeping this table fully generic + with zero stream-specific logic. """ - - def __init__(self, *args: Any, url_map: dict[str, str | None] | None = None, **kwargs: Any) -> None: - # Store url_map before calling super() so render_alert_id() can access it during rendering. - self.url_map = url_map or {} + # Custom partial that includes an OOB swap to update the stream status dashboard + partial_template_name = 'tom_alertstreams/partials/alert_table_partial.html' + + def __init__( + self, + *args: Any, + presenter_map: dict[str, AlertStreamPresenter] | None = None, + **kwargs: Any, + ) -> None: + self.presenter_map = presenter_map or {} super().__init__(*args, **kwargs) # @@ -44,13 +54,37 @@ def render_timestamp(self, value: Any) -> str: return value.strftime('%Y-%m-%d %H:%M:%S UTC') def render_alert_id(self, record: Alert, value: str) -> str: - """Render alert_id as a hyperlink to the stream's archive if a URL template exists.""" - template = self.url_map.get(record.stream_name) - if template: - url = template.format(alert_id=record.alert_id, object_id=record.object_id or '') - return format_html('{}', url, value) + """Render alert_id as a hyperlink if the stream's presenter provides a URL.""" + presenter = self.presenter_map.get(record.stream_name) + if presenter: + url = presenter.alert_url(record) + if url: + return format_html('{}', url, value) + return value + + def render_object_id(self, record: Alert, value: str) -> str: + """Render object_id as a hyperlink if the stream's presenter provides a URL.""" + if not value: + return value + presenter = self.presenter_map.get(record.stream_name) + if presenter: + url = presenter.object_url(record) + if url: + return format_html('{}', url, value) return value + def render_ra(self, value: Any) -> str: + """Render RA to 5 decimal places (~0.04 arcsec, matching LSST precision).""" + return f'{value:.5f}' if value is not None else '' + + def render_dec(self, value: Any) -> str: + """Render Dec to 5 decimal places (~0.04 arcsec, matching LSST precision).""" + return f'{value:.5f}' if value is not None else '' + + def render_magnitude(self, value: Any) -> str: + """Render magnitude to 3 decimal places (~1 mmag, matching survey photometric precision).""" + return f'{value:.3f}' if value is not None else '' + class Meta(HTMXTable.Meta): model = Alert fields = ['selection', 'alert_id', 'stream_name', 'topic', 'timestamp', 'object_id', 'ra', 'dec', 'magnitude'] @@ -64,17 +98,7 @@ def _get_stream_name_choices() -> list[tuple[str, str]]: tuples using each stream's STREAM_NAME. Streams that fail to import are silently skipped so a misconfigured entry doesn't break the filter form. """ - choices = [] - for stream_config in getattr(settings, 'ALERT_STREAMS', []): - if not stream_config.get('ACTIVE', True): - continue - try: - klass = import_string(stream_config['NAME']) - name = klass.STREAM_NAME - choices.append((name, name)) - except (ImportError, AttributeError, KeyError) as exc: - logger.warning('_get_stream_name_choices: skipping stream %s: %s', stream_config.get('NAME'), exc) - return choices + return [(klass.STREAM_NAME, klass.STREAM_NAME) for klass in get_alert_stream_classes()] class AlertFilterSet(HTMXTableFilterSet): @@ -102,3 +126,116 @@ class AlertFilterSet(HTMXTableFilterSet): class Meta: model = Alert fields = ['stream_name'] + + +# --------------------------------------------------------------------------- +# AlertStreamPresenter — display adapter for URL construction +# --------------------------------------------------------------------------- + +class AlertStreamPresenter: + """Presentation adapter: constructs display URLs from an Alert record. + + Each presenter knows how to build URLs for a specific alert stream's web + portal. The base implementation returns None for all URLs — streams with + no web portal (AMPEL, Hopskotch, Pitt-Google) use this default. + + For streams with a web portal, create a subclass that: + 1. Sets BASE_URL to the portal's root URL + 2. Overrides alert_url() and/or object_url() to construct the full URL + by combining BASE_URL with the stream-specific path structure using + an f-string (e.g., f'{self.BASE_URL}/object/{alert.object_id}') + + Register custom presenters in the STREAM_PRESENTERS dict at the bottom + of this section. Streams not in the registry use this base class. + + Follows the same structural pattern as Django's ModelAdmin: domain objects + (AlertStream) are unaware of their presenter. Registration is in the + presentation layer (this module). + """ + BASE_URL: ClassVar[str | None] = None + + def alert_url(self, alert: Alert) -> str | None: + """Return the URL for an alert detail page, or None.""" + return None + + def object_url(self, alert: Alert) -> str | None: + """Return the URL for an object/source page, or None.""" + return None + + +class AlercePresenter(AlertStreamPresenter): + """ALeRCE object pages: https://alerce.online/object/{object_id}""" + BASE_URL = 'https://alerce.online' + + def object_url(self, alert: Alert) -> str | None: + if not alert.object_id: + return None + return f'{self.BASE_URL}/object/{alert.object_id}' + + +class AntaresPresenter(AlertStreamPresenter): + """ANTARES locus pages: https://antares.noirlab.edu/loci/{alert_id}""" + BASE_URL = 'https://antares.noirlab.edu' + + def alert_url(self, alert: Alert) -> str | None: + return f'{self.BASE_URL}/loci/{alert.alert_id}' + + +class BabamulPresenter(AlertStreamPresenter): + """Babamul object pages: https://babamul.caltech.edu/objects/{survey}/{object_id} + + Survey is inferred from the object ID prefix. ZTF IDs start with 'ZTF', + LSST IDs start with 'LSST'. Unrecognized prefixes get no link. + """ + BASE_URL = 'https://babamul.caltech.edu' + + def object_url(self, alert: Alert) -> str | None: + if not alert.object_id: + return None + if alert.object_id.startswith('ZTF'): + survey = 'ZTF' + elif alert.object_id.startswith('LSST'): + survey = 'LSST' + else: + logger.warning('BabamulPresenter: unrecognized object_id prefix: %s', alert.object_id) + return None + return f'{self.BASE_URL}/objects/{survey}/{alert.object_id}' + + +class FinkPresenter(AlertStreamPresenter): + """Fink object pages: https://fink-portal.org/{object_id}""" + BASE_URL = 'https://fink-portal.org' + + def object_url(self, alert: Alert) -> str | None: + if not alert.object_id: + return None + return f'{self.BASE_URL}/{alert.object_id}' + + +class GCNPresenter(AlertStreamPresenter): + """GCN circular pages: https://gcn.nasa.gov/circulars/{alert_id}""" + BASE_URL = 'https://gcn.nasa.gov' + + def alert_url(self, alert: Alert) -> str | None: + return f'{self.BASE_URL}/circulars/{alert.alert_id}' + + +class LasairPresenter(AlertStreamPresenter): + """Lasair object pages: https://lasair-ztf.lsst.ac.uk/objects/{object_id}/""" + BASE_URL = 'https://lasair-ztf.lsst.ac.uk' + + def object_url(self, alert: Alert) -> str | None: + if not alert.object_id: + return None + return f'{self.BASE_URL}/objects/{alert.object_id}/' + + +# Streams not listed here use the default AlertStreamPresenter (no URLs). +STREAM_PRESENTERS: dict[str, type[AlertStreamPresenter]] = { + 'alerce': AlercePresenter, + 'antares': AntaresPresenter, + 'babamul': BabamulPresenter, + 'fink': FinkPresenter, + 'gcn': GCNPresenter, + 'lasair': LasairPresenter, +} From d68fd119e3d960dfb72a6fc380ced6349537f50c Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Mon, 9 Mar 2026 16:58:25 -0700 Subject: [PATCH 14/33] remove ARCHIVE_URL_TEMPLATE class var This had to do with creating links to display in an alert table that is separate from reading the stream, which is what these classes do. (Single Responsibilty Principle ftw). --- tom_alertstreams/alertstreams/alerce.py | 1 - tom_alertstreams/alertstreams/ampel.py | 1 - tom_alertstreams/alertstreams/fink.py | 1 - tom_alertstreams/alertstreams/lasair.py | 1 - tom_alertstreams/alertstreams/pittgoogle.py | 1 - 5 files changed, 5 deletions(-) diff --git a/tom_alertstreams/alertstreams/alerce.py b/tom_alertstreams/alertstreams/alerce.py index f43e4c6..1bfedb1 100644 --- a/tom_alertstreams/alertstreams/alerce.py +++ b/tom_alertstreams/alertstreams/alerce.py @@ -32,7 +32,6 @@ class AlerceAlertStream(AlertStream): """ configuration_class = AlerceConfig # type: ignore[assignment] STREAM_NAME: ClassVar[str] = 'alerce' - ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = 'https://alerce.online/object/{object_id}' def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: """Map a mock ALeRCE alert dict to a NormalizedAlert. diff --git a/tom_alertstreams/alertstreams/ampel.py b/tom_alertstreams/alertstreams/ampel.py index 9a7b731..e7d5cfd 100644 --- a/tom_alertstreams/alertstreams/ampel.py +++ b/tom_alertstreams/alertstreams/ampel.py @@ -32,7 +32,6 @@ class AmpelAlertStream(AlertStream): """ configuration_class = AmpelConfig # type: ignore[assignment] STREAM_NAME: ClassVar[str] = 'ampel' - ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = None def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: """Map a mock AMPEL alert dict to a NormalizedAlert. diff --git a/tom_alertstreams/alertstreams/fink.py b/tom_alertstreams/alertstreams/fink.py index 987a02d..400b194 100644 --- a/tom_alertstreams/alertstreams/fink.py +++ b/tom_alertstreams/alertstreams/fink.py @@ -32,7 +32,6 @@ class FinkAlertStream(AlertStream): """ configuration_class = FinkConfig # type: ignore[assignment] STREAM_NAME: ClassVar[str] = 'fink' - ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = 'https://fink-portal.org/{object_id}' def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: """Map a mock Fink alert dict to a NormalizedAlert. diff --git a/tom_alertstreams/alertstreams/lasair.py b/tom_alertstreams/alertstreams/lasair.py index 3a5654c..4e1dd51 100644 --- a/tom_alertstreams/alertstreams/lasair.py +++ b/tom_alertstreams/alertstreams/lasair.py @@ -34,7 +34,6 @@ class LasairAlertStream(AlertStream): """ configuration_class = LasairConfig # type: ignore[assignment] STREAM_NAME: ClassVar[str] = 'lasair' - ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = 'https://lasair-ztf.lsst.ac.uk/objects/{object_id}/' def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: """Map a mock Lasair alert dict to a NormalizedAlert. diff --git a/tom_alertstreams/alertstreams/pittgoogle.py b/tom_alertstreams/alertstreams/pittgoogle.py index 690574d..584736f 100644 --- a/tom_alertstreams/alertstreams/pittgoogle.py +++ b/tom_alertstreams/alertstreams/pittgoogle.py @@ -34,7 +34,6 @@ class PittGoogleAlertStream(AlertStream): """ configuration_class = PittGoogleConfig # type: ignore[assignment] STREAM_NAME: ClassVar[str] = 'pittgoogle' - ARCHIVE_URL_TEMPLATE: ClassVar[str | None] = None def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: """Map a mock Pitt-Google alert dict to a NormalizedAlert. From 0cf257098e45a041f786b4c825a05c761b34a7ff Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Tue, 10 Mar 2026 10:32:45 -0700 Subject: [PATCH 15/33] reduce the number of mock alerts being generated --- tom_alertstreams/alertstreams/alerce.py | 5 +++-- tom_alertstreams/alertstreams/ampel.py | 4 +++- tom_alertstreams/alertstreams/fink.py | 4 +++- tom_alertstreams/alertstreams/lasair.py | 4 +++- tom_alertstreams/alertstreams/pittgoogle.py | 4 +++- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tom_alertstreams/alertstreams/alerce.py b/tom_alertstreams/alertstreams/alerce.py index 1bfedb1..b05de76 100644 --- a/tom_alertstreams/alertstreams/alerce.py +++ b/tom_alertstreams/alertstreams/alerce.py @@ -16,7 +16,8 @@ _MOCK_RA = 0.0 _MOCK_DEC = 0.0 _MOCK_MAGNITUDE = 99.0 - +INTER_ALERT_SLEEP_MIN = 360 # six minutes +INTER_ALERT_SLEEP_MAX = 420 # seven minutes class AlerceConfig(AlertStreamConfig): """Pydantic configuration model for AlerceAlertStream (stub). @@ -87,4 +88,4 @@ def listen(self) -> None: } logger.debug(f'AlerceAlertStream: mock alert {object_id}') self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) - time.sleep(random.uniform(5.0, 30.0)) + time.sleep(random.uniform(INTER_ALERT_SLEEP_MIN, INTER_ALERT_SLEEP_MAX)) diff --git a/tom_alertstreams/alertstreams/ampel.py b/tom_alertstreams/alertstreams/ampel.py index e7d5cfd..4e3f5d6 100644 --- a/tom_alertstreams/alertstreams/ampel.py +++ b/tom_alertstreams/alertstreams/ampel.py @@ -16,6 +16,8 @@ _MOCK_RA = 0.0 _MOCK_DEC = 0.0 _MOCK_MAGNITUDE = 99.0 +INTER_ALERT_SLEEP_MIN = 360 # six minutes +INTER_ALERT_SLEEP_MAX = 420 # seven minutes class AmpelConfig(AlertStreamConfig): @@ -87,4 +89,4 @@ def listen(self) -> None: } logger.debug(f'AmpelAlertStream: mock alert {object_id}') self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) - time.sleep(random.uniform(5.0, 30.0)) + time.sleep(random.uniform(INTER_ALERT_SLEEP_MIN, INTER_ALERT_SLEEP_MAX)) diff --git a/tom_alertstreams/alertstreams/fink.py b/tom_alertstreams/alertstreams/fink.py index 400b194..31525c4 100644 --- a/tom_alertstreams/alertstreams/fink.py +++ b/tom_alertstreams/alertstreams/fink.py @@ -16,6 +16,8 @@ _MOCK_RA = 0.0 _MOCK_DEC = 0.0 _MOCK_MAGNITUDE = 99.0 +INTER_ALERT_SLEEP_MIN = 360 # six minutes +INTER_ALERT_SLEEP_MAX = 420 # seven minutes class FinkConfig(AlertStreamConfig): @@ -87,4 +89,4 @@ def listen(self) -> None: } logger.debug(f'FinkAlertStream: mock alert {object_id}') self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) - time.sleep(random.uniform(5.0, 30.0)) + time.sleep(random.uniform(INTER_ALERT_SLEEP_MIN, INTER_ALERT_SLEEP_MAX)) diff --git a/tom_alertstreams/alertstreams/lasair.py b/tom_alertstreams/alertstreams/lasair.py index 4e1dd51..383e2bb 100644 --- a/tom_alertstreams/alertstreams/lasair.py +++ b/tom_alertstreams/alertstreams/lasair.py @@ -16,6 +16,8 @@ _MOCK_RA = 0.0 _MOCK_DEC = 0.0 _MOCK_MAGNITUDE = 99.0 +INTER_ALERT_SLEEP_MIN = 360 # six minutes +INTER_ALERT_SLEEP_MAX = 420 # seven minutes class LasairConfig(AlertStreamConfig): @@ -89,4 +91,4 @@ def listen(self) -> None: } logger.debug(f'LasairAlertStream: mock alert {object_id}') self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) - time.sleep(random.uniform(5.0, 30.0)) + time.sleep(random.uniform(INTER_ALERT_SLEEP_MIN, INTER_ALERT_SLEEP_MAX)) diff --git a/tom_alertstreams/alertstreams/pittgoogle.py b/tom_alertstreams/alertstreams/pittgoogle.py index 584736f..2037cf9 100644 --- a/tom_alertstreams/alertstreams/pittgoogle.py +++ b/tom_alertstreams/alertstreams/pittgoogle.py @@ -16,6 +16,8 @@ _MOCK_RA = 0.0 _MOCK_DEC = 0.0 _MOCK_MAGNITUDE = 99.0 +INTER_ALERT_SLEEP_MIN = 360 # six minutes +INTER_ALERT_SLEEP_MAX = 420 # seven minutes class PittGoogleConfig(AlertStreamConfig): @@ -89,4 +91,4 @@ def listen(self) -> None: } logger.debug(f'PittGoogleAlertStream: mock alert {object_id}') self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) - time.sleep(random.uniform(5.0, 30.0)) + time.sleep(random.uniform(INTER_ALERT_SLEEP_MIN, INTER_ALERT_SLEEP_MAX)) From b8489db1b221b1d95c5edfa8a00a220b7df8e84c Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Tue, 10 Mar 2026 17:31:15 -0700 Subject: [PATCH 16/33] add flux field: LSST alerts have psfFlux; ZTF alerts have magnitude --- tom_alertstreams/admin.py | 2 +- tom_alertstreams/migrations/0002_alert_flux.py | 18 ++++++++++++++++++ tom_alertstreams/models.py | 1 + tom_alertstreams/tables.py | 9 ++++++++- 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 tom_alertstreams/migrations/0002_alert_flux.py diff --git a/tom_alertstreams/admin.py b/tom_alertstreams/admin.py index 88256b3..56b6f6f 100644 --- a/tom_alertstreams/admin.py +++ b/tom_alertstreams/admin.py @@ -5,6 +5,6 @@ @admin.register(Alert) class AlertAdmin(admin.ModelAdmin): - list_display = ('stream_name', 'alert_id', 'timestamp', 'object_id', 'magnitude') + list_display = ('stream_name', 'alert_id', 'timestamp', 'object_id', 'magnitude', 'flux') list_filter = ('stream_name',) search_fields = ('alert_id', 'object_id') diff --git a/tom_alertstreams/migrations/0002_alert_flux.py b/tom_alertstreams/migrations/0002_alert_flux.py new file mode 100644 index 0000000..34d9f99 --- /dev/null +++ b/tom_alertstreams/migrations/0002_alert_flux.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.29 on 2026-03-11 00:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_alertstreams', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='alert', + name='flux', + field=models.FloatField(null=True), + ), + ] diff --git a/tom_alertstreams/models.py b/tom_alertstreams/models.py index aaf01ca..9e253cf 100644 --- a/tom_alertstreams/models.py +++ b/tom_alertstreams/models.py @@ -85,6 +85,7 @@ class Alert(FIFOQueueMixin): ra = models.FloatField(null=True) dec = models.FloatField(null=True) magnitude = models.FloatField(null=True) + flux = models.FloatField(null=True) raw_payload = models.JSONField(default=dict) class Meta(FIFOQueueMixin.Meta): # this is the way you subclass the internal Meta class diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py index 1f55c6a..b082e2b 100644 --- a/tom_alertstreams/tables.py +++ b/tom_alertstreams/tables.py @@ -85,9 +85,16 @@ def render_magnitude(self, value: Any) -> str: """Render magnitude to 3 decimal places (~1 mmag, matching survey photometric precision).""" return f'{value:.3f}' if value is not None else '' + def render_flux(self, value: Any) -> str: + """Render flux in nanojansky to 3 decimal places.""" + return f'{value:.3f}' if value is not None else '' + class Meta(HTMXTable.Meta): model = Alert - fields = ['selection', 'alert_id', 'stream_name', 'topic', 'timestamp', 'object_id', 'ra', 'dec', 'magnitude'] + fields = [ + 'selection', 'alert_id', 'stream_name', 'topic', 'timestamp', + 'object_id', 'ra', 'dec', 'magnitude', 'flux', + ] def _get_stream_name_choices() -> list[tuple[str, str]]: From d23b19174cf49820b72e01ff4c264a7dc47b1893 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Tue, 10 Mar 2026 17:35:47 -0700 Subject: [PATCH 17/33] add AMPEL client libraries --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 518a153..571863b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,9 @@ hopskotch = ["hop-client >=0.10,<1.0"] antares = ["antares-client"] babamul = ["babamul >=0.1,<1"] fink = ["fink-client >=8.8,<9"] +ampel-ztf = ["ampel-ztf"] +ampel-lsst = ["ampel-lsst"] +ampel = ["ampel-ztf", "ampel-lsst"] all-streams = [ "gcn-kafka >=0.3,<1.0", @@ -53,6 +56,8 @@ all-streams = [ "antares-client", "babamul >=0.1,<1", "fink-client >=8.8,<9", + "ampel-ztf", + "ampel-lsst", ] test = [ From be659502cddde04ac720e47e85c06169019f8488 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 12 Mar 2026 15:48:59 -0700 Subject: [PATCH 18/33] add flux field to NormalizedAlert Pydantic model LSST streams need this field to distinguish nanojansky fluxesfrom ZTF magnitudes --- tom_alertstreams/alertstreams/alertstream.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_alertstreams/alertstreams/alertstream.py b/tom_alertstreams/alertstreams/alertstream.py index 07804b7..c508109 100644 --- a/tom_alertstreams/alertstreams/alertstream.py +++ b/tom_alertstreams/alertstreams/alertstream.py @@ -41,7 +41,8 @@ class NormalizedAlert(BaseModel): object_id: Astronomical object identifier (e.g. ZTF object name), if available. ra: Right ascension in decimal degrees, if available. dec: Declination in decimal degrees, if available. - magnitude: Apparent magnitude, if available. + magnitude: Apparent magnitude, if available. ZTF streams populate this field. + flux: Flux in nanojansky, if available. LSST streams populate this field. raw_payload: The full original alert as a plain dict for downstream use. """ stream_name: str @@ -52,6 +53,7 @@ class NormalizedAlert(BaseModel): ra: float | None = None dec: float | None = None magnitude: float | None = None + flux: float | None = None raw_payload: dict = {} From f5357f9cc25a5de93177f0a3e85d87262e162456 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 12 Mar 2026 16:00:17 -0700 Subject: [PATCH 19/33] restructure ANTARES into mock + abstract base + ZTF/LSST subclasses - AntaresAlertStream: abstract base implements shared listen() with StreamingClient using context manager- AntaresZtfAlertStream: extracts newest_alert_magnitude, ztf_object_id, newest_alert_observation_time (MJD)- AntaresLsstAlertStream: forward-compatible stub for LSST properties (survey.lsst.dia_object_id); inactive until LSST topics are confirmed - AntaresMockAlertStream: generates fake alerts for demos without credentials (This is just for Recent Alerts table testing). Fixes normalize_alert to use correct property keys discovered from a live ZTF locus (newest_alert_magnitude, not the nonexistent ztf_magpsf). --- tom_alertstreams/alertstreams/antares.py | 367 +++++++++++++++++++++-- tom_alertstreams/tables.py | 2 + 2 files changed, 350 insertions(+), 19 deletions(-) diff --git a/tom_alertstreams/alertstreams/antares.py b/tom_alertstreams/alertstreams/antares.py index bd17fa4..a171d25 100644 --- a/tom_alertstreams/alertstreams/antares.py +++ b/tom_alertstreams/alertstreams/antares.py @@ -1,30 +1,359 @@ +from __future__ import annotations + import logging -from .alertstream import AlertStream +import random +import time +from datetime import datetime, timezone +from typing import Any, ClassVar + from antares_client.stream import StreamingClient +from tom_alertstreams.alertstreams.alertstream import AlertStream, AlertStreamConfig, NormalizedAlert + logger = logging.getLogger(__name__) +# Mock data constants — chosen to be unmistakably non-astronomical: +# (0, 0) is not a real survey pointing; 99.0 is the astronomical sentinel for "no data". +_MOCK_RA = 0.0 +_MOCK_DEC = 0.0 +_MOCK_MAGNITUDE = 99.0 +INTER_ALERT_SLEEP_MIN = 360 # six minutes +INTER_ALERT_SLEEP_MAX = 420 # seven minutes + + +# --------------------------------------------------------------------------- +# MJD helper +# --------------------------------------------------------------------------- + +def _mjd_to_datetime(mjd: float) -> datetime: + """Convert Modified Julian Date to a timezone-aware UTC datetime. + + MJD = JD - 2400000.5. JD 2440587.5 = Unix epoch (1970-01-01 00:00:00 UTC). + So: unix_seconds = (MJD + 2400000.5 - 2440587.5) * 86400 + = (MJD - 40587.0) * 86400 + """ + unix_seconds = (mjd - 40587.0) * 86400.0 + return datetime.fromtimestamp(unix_seconds, tz=timezone.utc) + + +# --------------------------------------------------------------------------- +# Pydantic configuration models +# --------------------------------------------------------------------------- + +class AntaresConfig(AlertStreamConfig): + """Pydantic configuration model for real ANTARES alert streams (ZTF and LSST). + + Inherits from AlertStreamConfig (a Pydantic BaseModel), so Pydantic validates + that API_KEY and API_SECRET are present and raises descriptive errors if not. + Both AntaresZtfAlertStream and AntaresLsstAlertStream share this config because + they use the same StreamingClient, same Kafka auth, and same Locus model — the + only differences are topic names and normalize_alert() field extraction. + + Fields: + API_KEY: ANTARES API key (required). Obtain at https://antares.noirlab.edu. + API_SECRET: ANTARES API secret (required). Obtain at https://antares.noirlab.edu. + TOPIC_HANDLERS: Inherited from AlertStreamConfig. Maps topic names to handler + dotted-paths. Known ZTF topics: 'extragalactic_staging', + 'nuclear_transient_staging'. + GROUP: Kafka consumer group ID. Distinct group IDs let multiple TOM instances + consume the same stream independently. + SSL_CA_LOCATION: Path to a TLS Certificate Authority (CA) certificate file. + When StreamingClient connects to the ANTARES Kafka broker, it uses TLS + (encrypted connection). TLS requires a CA cert — a file that tells the + client "trust connections signed by this authority." The antares_client + package bundles a default CA cert (certificates/kafka-ca.pem) that works + for the current ZTF broker. If ANTARES uses a different Kafka cluster or + TLS chain for LSST alerts, the default cert might not be trusted, and + you'd need to point SSL_CA_LOCATION at the correct CA cert file. + When None (the default), the bundled cert is used. + TODO: When researching LSST topic access, find out whether LSST topics + require a different CA cert and document how to obtain it. + ENABLE_AUTO_COMMIT: Whether Kafka should auto-commit offsets. Set to False + for at-least-once processing with manual offset management. + """ + API_KEY: str + API_SECRET: str + GROUP: str = 'tom-alertstreams' + SSL_CA_LOCATION: str | None = None + ENABLE_AUTO_COMMIT: bool = True + + +class AntaresMockConfig(AlertStreamConfig): + """Pydantic configuration model for AntaresMockAlertStream. + + Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + No additional fields needed for mock data generation. + """ + pass + + +# --------------------------------------------------------------------------- +# Mock ANTARES stream (for demo use without API credentials) +# --------------------------------------------------------------------------- + +class AntaresMockAlertStream(AlertStream): + """Mock ANTARES AlertStream that generates obviously-fake alerts. + + Provides a demo fallback when ANTARES API credentials are not available. + Uses the same mock pattern as the other stub streams (sentinel coordinates, + sentinel magnitude, 6–7 minute sleep between alerts). + + The mock stream uses STREAM_NAME='antares' so it occupies the same dashboard + slot as the real ANTARES streams would if they were the only ones configured. + """ + configuration_class = AntaresMockConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'antares' + + def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: + """Map a mock ANTARES alert dict to a NormalizedAlert. + + Args: + raw_alert: Dict produced by listen(); contains mock field values. + topic: Topic the alert was generated for. + + Returns: + NormalizedAlert populated from the mock dict fields. + """ + return NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic or raw_alert.get('topic', ''), + timestamp=datetime.fromisoformat(raw_alert['timestamp']), + alert_id=raw_alert['alert_id'], + object_id=raw_alert.get('object_id'), + ra=raw_alert.get('ra'), + dec=raw_alert.get('dec'), + magnitude=raw_alert.get('magnitude'), + raw_payload=raw_alert, + ) + + def listen(self) -> None: + """Generate mock ANTARES alerts and dispatch to configured topic handlers. + + Loops indefinitely, emitting one mock alert per iteration with a random + 6–7 minute delay. Topics are round-robined if multiple are configured. + """ + counter = 0 + topics = list(self.config.TOPIC_HANDLERS.keys()) + + while True: + counter += 1 + topic = topics[counter % len(topics)] + timestamp = datetime.now(timezone.utc) + object_id = f'MOCK-{self.STREAM_NAME.upper()}-{counter:04d}' + alert_id = f'MOCK-{timestamp.strftime("%Y%m%d%H%M%S")}' + mock_alert: dict[str, Any] = { + 'alert_id': alert_id, + 'object_id': object_id, + 'topic': topic, + 'timestamp': timestamp.isoformat(), + 'ra': _MOCK_RA, + 'dec': _MOCK_DEC, + 'magnitude': _MOCK_MAGNITUDE, + 'mock': True, + } + logger.debug(f'AntaresMockAlertStream: mock alert {object_id}') + self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) + time.sleep(random.uniform(INTER_ALERT_SLEEP_MIN, INTER_ALERT_SLEEP_MAX)) + + +# --------------------------------------------------------------------------- +# Real ANTARES streams — abstract base with ZTF and LSST subclasses +# --------------------------------------------------------------------------- class AntaresAlertStream(AlertStream): + """Abstract base class for real ANTARES alert streams. + + Handles StreamingClient setup and the listen() loop. Not configured directly — + users configure AntaresZtfAlertStream or AntaresLsstAlertStream, which override + normalize_alert() for survey-specific field extraction. + + ANTARES is unique among the LSST brokers: it presents a unified Kafka interface + for both ZTF and LSST alerts via the same StreamingClient and Locus model. The + only differences between surveys are topic names, properties dict keys, and + photometry units. This base class captures the shared listen() logic while + subclasses handle the divergent normalization. + + The StreamingClient is created inside listen() using a context manager (not in + __init__) so the Kafka consumer is properly closed on errors or shutdown. """ - Wrapper for the ANTARES broker streaming client. See https://nsf-noirlab.gitlab.io/csdc/antares/client/. + configuration_class = AntaresConfig # type: ignore[assignment] + + def listen(self) -> None: + """Consume ANTARES loci and dispatch to configured topic handlers. + + Creates a StreamingClient as a context manager so the underlying Kafka + consumer is properly closed on errors, KeyboardInterrupt, or normal exit. + + ANTARES delivers (topic, locus) pairs via StreamingClient.iter(). The topic + includes the stream prefix (e.g. 'client.extragalactic_staging'); we strip + the prefix to match the base topic names used in TOPIC_HANDLERS. + """ + # Build optional kwargs from config — only pass non-default values so the + # StreamingClient uses its own defaults for unset fields. + optional_kwargs: dict[str, Any] = {} + if self.config.SSL_CA_LOCATION is not None: + optional_kwargs['ssl_ca_location'] = self.config.SSL_CA_LOCATION + if not self.config.ENABLE_AUTO_COMMIT: + optional_kwargs['enable_auto_commit'] = self.config.ENABLE_AUTO_COMMIT + + with StreamingClient( + topics=list(self.config.TOPIC_HANDLERS.keys()), + api_key=self.config.API_KEY, + api_secret=self.config.API_SECRET, + group=self.config.GROUP, + **optional_kwargs, + ) as antares_streaming_client: + for topic, locus in antares_streaming_client.iter(): + base_topic = topic.removeprefix(antares_streaming_client._TOPIC_PREFIX) + logger.info(f'{self.STREAM_NAME} received {locus.locus_id} on {base_topic}') + self.alert_handler[base_topic](locus, alert_stream=self, topic=base_topic) + + +class AntaresZtfAlertStream(AntaresAlertStream): + """ANTARES alert stream for ZTF transient alerts. + + Connects to ANTARES Kafka topics that carry ZTF-originated alerts. Each alert + is an antares_client Locus object enriched with ZTF-specific properties. + + Known ZTF topics: 'extragalactic_staging', 'nuclear_transient_staging'. + Contact the ANTARES team for the full topic list. + + Configuration example (settings.py ALERT_STREAMS entry): + { + 'ACTIVE': True, + 'NAME': 'tom_alertstreams.alertstreams.antares.AntaresZtfAlertStream', + 'OPTIONS': { + 'API_KEY': os.environ.get('ANTARES_API_KEY', ''), + 'API_SECRET': os.environ.get('ANTARES_API_SECRET', ''), + 'TOPIC_HANDLERS': { + 'extragalactic_staging': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database', + 'nuclear_transient_staging': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database', + }, + }, + } """ - required_keys = ['API_KEY', 'API_SECRET', 'TOPIC_HANDLERS'] - allowed_keys = ['API_KEY', 'API_SECRET', 'TOPIC_HANDLERS', 'GROUP', 'SSL_CA_LOCATION', 'ENABLE_AUTO_COMMIT'] - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - logger.debug(f'AntaresAlertStream.__init__() kwargs: {kwargs}') - optional_keys = set(self.allowed_keys) - set(self.required_keys) - options = {key.lower(): kwargs[key] for key in optional_keys if key in kwargs} - self.stream = StreamingClient( - topics=self.topic_handlers.keys(), - api_key=self.api_key, - api_secret=self.api_secret, - **options + STREAM_NAME: ClassVar[str] = 'antares-ztf' + + def normalize_alert(self, raw_alert: Any, topic: str = '') -> NormalizedAlert: + """Extract common fields from an ANTARES Locus object carrying ZTF data. + + Field mappings (discovered via ex_antares.py introspection of a live ZTF locus): + - timestamp: locus.properties['newest_alert_observation_time'] (MJD float) + - magnitude: locus.properties['newest_alert_magnitude'] + - object_id: locus.properties['ztf_object_id'] (distinct from locus_id) + - alert_id: locus.locus_id (used by AntaresPresenter for locus page URL) + + The full Locus object is not JSON-serializable, so raw_payload is left empty. + Handlers that need Locus-specific data should access raw_alert directly. + + Args: + raw_alert: An antares_client.models.Locus object. + topic: The ANTARES topic (e.g. 'extragalactic_staging'). + + Returns: + NormalizedAlert with ZTF-specific fields populated. + """ + props = raw_alert.properties or {} + + # Timestamp from ANTARES-enriched locus property (MJD float). + # This avoids lazy-loading locus.alerts, which triggers an HTTP API call. + mjd = props.get('newest_alert_observation_time') + timestamp = _mjd_to_datetime(mjd) if mjd is not None else datetime.now(timezone.utc) + + # ZTF magnitude from ANTARES-enriched properties. + magnitude = props.get('newest_alert_magnitude') + + # ZTF object ID if available; fallback to locus_id. + object_id = props.get('ztf_object_id', raw_alert.locus_id) + + return NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic, + timestamp=timestamp, + alert_id=str(raw_alert.locus_id), + object_id=str(object_id), + ra=float(raw_alert.ra) if raw_alert.ra is not None else None, + dec=float(raw_alert.dec) if raw_alert.dec is not None else None, + magnitude=float(magnitude) if magnitude is not None else None, + flux=None, + raw_payload={}, ) - def listen(self): - for topic, locus in self.stream.iter(): - base_topic = topic.removeprefix(self.stream._TOPIC_PREFIX) - logger.info(f"received {locus.locus_id} on {base_topic}") - self.alert_handler[base_topic](locus) + +class AntaresLsstAlertStream(AntaresAlertStream): + """ANTARES alert stream for LSST transient alerts. + + Connects to ANTARES Kafka topics that carry LSST-originated alerts. ANTARES + is structurally ready for LSST — the Locus properties dict already contains + a 'survey.lsst' namespace with dia_object_id and ss_object_id arrays — but + LSST topic names and photometry property keys are not yet confirmed. + + This class is provided for forward-compatibility. Activate it in settings once + LSST topics are available and property key names are confirmed via introspection. + + Configuration example (settings.py ALERT_STREAMS entry): + { + 'ACTIVE': False, # activate once LSST topics are confirmed + 'NAME': 'tom_alertstreams.alertstreams.antares.AntaresLsstAlertStream', + 'OPTIONS': { + 'API_KEY': os.environ.get('ANTARES_API_KEY', ''), + 'API_SECRET': os.environ.get('ANTARES_API_SECRET', ''), + 'TOPIC_HANDLERS': { + 'lsst_placeholder': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database', + }, + }, + } + """ + STREAM_NAME: ClassVar[str] = 'antares-lsst' + + def normalize_alert(self, raw_alert: Any, topic: str = '') -> NormalizedAlert: + """Extract common fields from an ANTARES Locus object carrying LSST data. + + LSST-specific field mappings are provisional — based on the Locus properties + structure observed via ex_antares.py and the antares_client.search module: + - object_id: properties['survey']['lsst']['dia_object_id'][0] (nested dict) + - flux: property key TBD (need to inspect a real LSST locus) + - timestamp: properties['newest_alert_observation_time'] (same as ZTF) + + TODO: Verify all LSST property keys by introspecting a real LSST locus once + LSST topics are available. Update this method accordingly. + + Args: + raw_alert: An antares_client.models.Locus object. + topic: The ANTARES topic for LSST alerts. + + Returns: + NormalizedAlert with LSST-specific fields populated where known. + """ + props = raw_alert.properties or {} + + # Timestamp — same MJD property as ZTF (ANTARES-enriched). + mjd = props.get('newest_alert_observation_time') + timestamp = _mjd_to_datetime(mjd) if mjd is not None else datetime.now(timezone.utc) + + # LSST object ID from the nested survey dict structure. + # antares_client.search uses 'properties.survey.lsst.dia_object_id' as a + # flat key path, but the actual properties dict is nested: + # properties['survey']['lsst']['dia_object_id'] → list of IDs + object_id = raw_alert.locus_id # default fallback + survey = props.get('survey', {}) + lsst_survey = survey.get('lsst', {}) if isinstance(survey, dict) else {} + dia_object_ids = lsst_survey.get('dia_object_id', []) + if dia_object_ids: + object_id = dia_object_ids[0] + + # LSST flux — property key TBD. Need to inspect a real LSST locus. + flux = None + + return NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic, + timestamp=timestamp, + alert_id=str(raw_alert.locus_id), + object_id=str(object_id), + ra=float(raw_alert.ra) if raw_alert.ra is not None else None, + dec=float(raw_alert.dec) if raw_alert.dec is not None else None, + magnitude=None, + flux=flux, + raw_payload={}, + ) diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py index b082e2b..a26ae65 100644 --- a/tom_alertstreams/tables.py +++ b/tom_alertstreams/tables.py @@ -241,6 +241,8 @@ def object_url(self, alert: Alert) -> str | None: STREAM_PRESENTERS: dict[str, type[AlertStreamPresenter]] = { 'alerce': AlercePresenter, 'antares': AntaresPresenter, + 'antares-ztf': AntaresPresenter, + 'antares-lsst': AntaresPresenter, 'babamul': BabamulPresenter, 'fink': FinkPresenter, 'gcn': GCNPresenter, From a99b840abb86bbfead2d90cd43919271ef76c408 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 12 Mar 2026 16:10:40 -0700 Subject: [PATCH 20/33] WIP: restructure AMPEL into mock + real ZTF/LSST subclasses Replaces the AmpelAlertStream stub with a three-class hierarchy: - AmpelMockAlertStream: renamed from AmpelAlertStream, generates fake alerts while real AMPEL packages cannot be installed (pymongo bson conflict with antares_client) - AmpelZtfAlertStream: consumes ZTF alerts from UW Kafka broker via ampel-ztf ZiAlertSupplier/UWAlertLoader stack - AmpelLsstAlertStream: consumes LSST alerts via ampel-lsst LSSTAlertSupplier/KafkaAlertLoader with SASL_SSL auth Real AMPEL imports are commented out due to the bson conflict -- the classes are structurally complete but untested against live data. Also adds LsstPresenter to tables.py for RSP deep-link URLs on ampel-lsst alerts. --- tom_alertstreams/alertstreams/ampel.py | 336 +++++++++++++++++++++++-- tom_alertstreams/tables.py | 23 ++ 2 files changed, 342 insertions(+), 17 deletions(-) diff --git a/tom_alertstreams/alertstreams/ampel.py b/tom_alertstreams/alertstreams/ampel.py index 4e3f5d6..a3199a6 100644 --- a/tom_alertstreams/alertstreams/ampel.py +++ b/tom_alertstreams/alertstreams/ampel.py @@ -4,13 +4,24 @@ import random import time from datetime import datetime, timezone -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal + +# AMPEL imports are commented out because ampel-ztf / ampel-lsst pull in pymongo, +# whose bundled bson package shadows the standalone python-bson required by +# antares_client. Until that conflict is resolved, the real AMPEL classes cannot +# coexist with ANTARES in the same environment. +# +# from ampel.alert.AmpelAlert import AmpelAlert +# from ampel.base.AuxUnitRegister import AuxUnitRegister +# from ampel.lsst.alert.load.KafkaAlertLoader import KafkaAlertLoader +# from ampel.lsst.alert.LSSTAlertSupplier import LSSTAlertSupplier +# from ampel.ztf.alert.ZiAlertSupplier import ZiAlertSupplier +# from ampel.ztf.t0.load.UWAlertLoader import UWAlertLoader from tom_alertstreams.alertstreams.alertstream import AlertStream, AlertStreamConfig, NormalizedAlert logger = logging.getLogger(__name__) -# TODO: remove when stubs are replaced # Mock data constants — chosen to be unmistakably non-astronomical: # (0, 0) is not a real survey pointing; 99.0 is the astronomical sentinel for "no data". _MOCK_RA = 0.0 @@ -20,19 +31,48 @@ INTER_ALERT_SLEEP_MAX = 420 # seven minutes -class AmpelConfig(AlertStreamConfig): - """Pydantic configuration model for AmpelAlertStream (stub). +# --------------------------------------------------------------------------- +# Julian Date helpers +# --------------------------------------------------------------------------- + +def _jd_to_datetime(jd: float) -> datetime: + """Convert Julian Date to a timezone-aware UTC datetime. + + Uses the standard epoch offset: JD 2440587.5 = Unix epoch (1970-01-01 00:00:00 UTC). + """ + unix_seconds = (jd - 2440587.5) * 86400.0 + return datetime.fromtimestamp(unix_seconds, tz=timezone.utc) + + +def _mjd_to_datetime(mjd: float) -> datetime: + """Convert Modified Julian Date to a timezone-aware UTC datetime. + + MJD = JD - 2400000.5, so we convert back to JD and delegate. + """ + return _jd_to_datetime(mjd + 2400000.5) + + +# --------------------------------------------------------------------------- +# Mock AMPEL stream (for demo use while real AMPEL imports are unavailable) +# --------------------------------------------------------------------------- + +class AmpelMockConfig(AlertStreamConfig): + """Pydantic configuration model for AmpelMockAlertStream. Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + No additional fields needed for mock data generation. """ - # TODO: replace this stub with actual implementation pass -class AmpelAlertStream(AlertStream): - """Stub AMPEL AlertStream that generates obviously-fake mock alerts. +class AmpelMockAlertStream(AlertStream): + """Mock AMPEL AlertStream that generates obviously-fake alerts. + + Stands in for the real AMPEL streams while the ampel-ztf / ampel-lsst + packages cannot be installed alongside antares_client (pymongo bson + conflict). Uses the same mock pattern as the other stub streams. """ - configuration_class = AmpelConfig # type: ignore[assignment] + configuration_class = AmpelMockConfig # type: ignore[assignment] STREAM_NAME: ClassVar[str] = 'ampel' def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: @@ -45,9 +85,7 @@ def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: Returns: NormalizedAlert populated from the mock dict fields. """ - # TODO: replace this stub with actual implementation - # super().normalized_alert is @abs.abstractmethod, so the stub needs an implementation - normalized_alert = NormalizedAlert( + return NormalizedAlert( stream_name=self.STREAM_NAME, topic=topic or raw_alert.get('topic', ''), timestamp=datetime.fromisoformat(raw_alert['timestamp']), @@ -58,22 +96,19 @@ def normalize_alert(self, raw_alert: dict, topic: str = '') -> NormalizedAlert: magnitude=raw_alert.get('magnitude'), raw_payload=raw_alert, ) - return normalized_alert def listen(self) -> None: """Generate mock AMPEL alerts and dispatch to configured topic handlers. Loops indefinitely, emitting one mock alert per iteration with a random - 5–30 second delay. Topics are round-robined if multiple are configured. + 6–7 minute delay. Topics are round-robined if multiple are configured. """ - # TODO: replace this stub with actual implementation counter = 0 topics = list(self.config.TOPIC_HANDLERS.keys()) - # for this stub, generate mock alerts (rather than listen to the stream) endlessly while True: counter += 1 - topic = topics[counter % len(topics)] # round-robin + topic = topics[counter % len(topics)] timestamp = datetime.now(timezone.utc) object_id = f'MOCK-{self.STREAM_NAME.upper()}-{counter:04d}' alert_id = f'MOCK-{timestamp.strftime("%Y%m%d%H%M%S")}' @@ -87,6 +122,273 @@ def listen(self) -> None: 'magnitude': _MOCK_MAGNITUDE, 'mock': True, } - logger.debug(f'AmpelAlertStream: mock alert {object_id}') + logger.debug(f'AmpelMockAlertStream: mock alert {object_id}') self.alert_handler[topic](mock_alert, alert_stream=self, topic=topic) time.sleep(random.uniform(INTER_ALERT_SLEEP_MIN, INTER_ALERT_SLEEP_MAX)) + + +# --------------------------------------------------------------------------- +# ZTF via AMPEL (requires: pip install ampel-ztf) +# --------------------------------------------------------------------------- + +class AmpelZtfConfig(AlertStreamConfig): + """Pydantic configuration for AmpelZtfAlertStream. + + Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + Fields map to UWAlertLoader parameters. + + Config values: + BOOTSTRAP: Kafka broker address. The UW public broker is the default. + STREAM: 'ztf_uw_public' (all programid=1 alerts) or 'ztf_uw_private' + (adds programid=2). Must match one of the UWAlertLoader literals. + GROUP_NAME: Kafka consumer group prefix. UWAlertLoader appends '-{stream}'. + TIMEOUT: Seconds to wait for messages before the supplier stops iterating. + """ + BOOTSTRAP: str = 'partnership.alerts.ztf.uw.edu:9092' + STREAM: Literal['ztf_uw_private', 'ztf_uw_public'] = 'ztf_uw_public' + GROUP_NAME: str = 'tom-alertstreams' + TIMEOUT: int = 3600 + + +class AmpelZtfAlertStream(AlertStream): + """Consume ZTF alerts from the UW Kafka broker via AMPEL's ZiAlertSupplier. + + Uses ampel-ztf's supplier/loader stack to connect to the University of + Washington's ZTF alert archive. The supplier handles Avro deserialization + and shapes each alert into an AmpelAlert with structured datapoints. + + Alerts are dispatched to the handler registered for the single configured + topic in TOPIC_HANDLERS. ZTF topic metadata is not propagated through the + AMPEL iteration chain, so all alerts use the first (and expected only) + TOPIC_HANDLERS key as the topic. + + Requires: pip install tom-alertstreams[ampel-ztf] + """ + configuration_class = AmpelZtfConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'ampel-ztf' + + def normalize_alert(self, alert: Any, topic: str = '') -> NormalizedAlert: + """Extract common fields from an AMPEL-shaped ZTF alert. + + The AmpelAlert.datapoints[0] is the current candidate (a ReadOnlyDict) + containing jd, ra, dec, magpsf, etc. The original ZTF objectId string + is preserved in AmpelAlert.extra['name'] (alert.stock is an encoded + AMPEL-internal integer, not the human-readable ZTF name). + + Args: + alert: AmpelAlert yielded by ZiAlertSupplier. + topic: Kafka topic name (passed from listen()). + + Returns: + NormalizedAlert with magnitude populated; flux is None. + """ + candidate = alert.datapoints[0] + return NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic, + timestamp=_jd_to_datetime(candidate['jd']), + alert_id=str(alert.id), + object_id=alert.extra.get('name') if alert.extra else None, + ra=candidate.get('ra'), + dec=candidate.get('dec'), + magnitude=candidate.get('magpsf'), + flux=None, + raw_payload=alert.dict(), + ) + + def listen(self) -> None: + """Connect to the UW Kafka broker and iterate ZTF alerts indefinitely. + + Registers UWAlertLoader with AMPEL's AuxUnitRegister (required for the + supplier to resolve the loader by name), then creates a ZiAlertSupplier + that wraps the loader. UWAlertLoader subscribes to topics in __init__ + (via AllConsumingConsumer), so no context manager is needed. + + Each alert is dispatched to the handler for the single configured topic. + """ + # Deferred import: ampel-ztf's bson (via pymongo) conflicts with + # antares_client's standalone bson. Only import when actually used. + from ampel.alert.AmpelAlert import AmpelAlert # noqa: F811 + from ampel.base.AuxUnitRegister import AuxUnitRegister + from ampel.ztf.alert.ZiAlertSupplier import ZiAlertSupplier + from ampel.ztf.t0.load.UWAlertLoader import UWAlertLoader + + # Register the loader class so AuxUnitRegister can resolve it by name + # when the supplier's UnitModel reference is instantiated. + AuxUnitRegister._dyn['UWAlertLoader'] = UWAlertLoader + + supplier = ZiAlertSupplier( + deserialize='avro', + loader={ + 'unit': 'UWAlertLoader', + 'config': { + 'bootstrap': self.config.BOOTSTRAP, + 'stream': self.config.STREAM, + 'group_name': self.config.GROUP_NAME, + 'timeout': self.config.TIMEOUT, + }, + }, + ) + + # ZTF topics are not propagated through the supplier iteration chain + # (UWAlertLoader.alerts() reads message.topic() for stats but yields + # only the raw bytes). Use the single configured topic key for dispatch. + topic = next(iter(self.config.TOPIC_HANDLERS)) + logger.info( + '%s: listening on %s (stream=%s, group=%s)', + self.STREAM_NAME, self.config.BOOTSTRAP, self.config.STREAM, self.config.GROUP_NAME, + ) + + for alert in supplier: + self.alert_handler[topic](alert, alert_stream=self, topic=topic) + + +# --------------------------------------------------------------------------- +# LSST via AMPEL (requires: pip install ampel-lsst) +# --------------------------------------------------------------------------- + +class AmpelLsstConfig(AlertStreamConfig): + """Pydantic configuration for AmpelLsstAlertStream. + + Inherits TOPIC_HANDLERS from AlertStreamConfig (a Pydantic BaseModel). + Fields map to KafkaAlertLoader / KafkaConsumerBase parameters. + + Config values: + BOOTSTRAP: Kafka broker address (e.g. 'alert-stream-int.lsst.cloud:9094'). + TOPICS: Explicit list of Kafka topics to subscribe to. + GROUP_NAME: Kafka consumer group. LSST brokers require this to be prefixed + with the assigned username (e.g. 'ampel-idfint-tom-alertstreams'). + TIMEOUT: Seconds to wait for messages before the supplier stops iterating. + AVRO_SCHEMA: Schema registry URL for Avro deserialization + (e.g. 'https://alert-schemas-int.lsst.cloud'). None to skip. + SASL_USERNAME / SASL_PASSWORD: SASL/SCRAM credentials assigned by the + Rubin alert stream operators. See DMTN-210 for details. + SASL_MECHANISM: SCRAM variant. Rubin uses SCRAM-SHA-512. + SECURITY_PROTOCOL: Kafka security protocol. Rubin uses SASL_SSL. + KAFKA_CONSUMER_PROPERTIES: Extra confluent_kafka consumer config passed + directly to the underlying DeserializingConsumer. + """ + BOOTSTRAP: str + TOPICS: list[str] + GROUP_NAME: str = 'tom-alertstreams' + TIMEOUT: int = 300 + AVRO_SCHEMA: str | None = None + SASL_USERNAME: str | None = None + SASL_PASSWORD: str | None = None + SASL_MECHANISM: str = 'SCRAM-SHA-512' + SECURITY_PROTOCOL: str = 'SASL_SSL' + KAFKA_CONSUMER_PROPERTIES: dict[str, Any] = {} + + +class AmpelLsstAlertStream(AlertStream): + """Consume LSST alerts from a Kafka broker via AMPEL's LSSTAlertSupplier. + + Uses ampel-lsst's supplier/loader stack to connect to a Rubin-compatible + Kafka broker. The loader handles Avro deserialization (optionally via a + schema registry) and attaches Kafka metadata (__kafka dict) to each alert. + The supplier shapes alerts into AmpelAlert objects with field-name upgrades + (e.g. psFlux → psfFlux, midPointTai → midpointMjdTai, decl → dec). + + Authentication uses SASL_SSL + SCRAM-SHA-512 by default, matching the + Rubin alert distribution system (DMTN-210). + + Requires: pip install tom-alertstreams[ampel-lsst] + """ + configuration_class = AmpelLsstConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'ampel-lsst' + + def normalize_alert(self, alert: Any, topic: str = '') -> NormalizedAlert: + """Extract common fields from an AMPEL-shaped LSST alert. + + AmpelAlert.datapoints[0] is the triggering diaSource (a ReadOnlyDict) + with field-upgraded names: midpointMjdTai, psfFlux, ra, dec, band, etc. + AmpelAlert.stock is the diaObjectId. Kafka topic metadata is available + in alert.extra['kafka']['topic'] (preserved by KafkaAlertLoader). + + Args: + alert: AmpelAlert yielded by LSSTAlertSupplier. + topic: Kafka topic name (passed from listen()). + + Returns: + NormalizedAlert with flux populated (nanojansky); magnitude is None. + """ + dia_source = alert.datapoints[0] + # Topic from Kafka metadata, falling back to the caller-provided value. + kafka_topic = '' + if alert.extra and 'kafka' in alert.extra: + kafka_topic = alert.extra['kafka'].get('topic', '') + return NormalizedAlert( + stream_name=self.STREAM_NAME, + topic=topic or kafka_topic, + timestamp=_mjd_to_datetime(dia_source['midpointMjdTai']), + alert_id=str(alert.id), + object_id=str(alert.stock), + ra=dia_source.get('ra'), + dec=dia_source.get('dec'), + magnitude=None, + flux=dia_source.get('psfFlux'), + raw_payload=alert.dict(), + ) + + def listen(self) -> None: + """Connect to a Kafka broker and iterate LSST alerts indefinitely. + + Registers KafkaAlertLoader with AMPEL's AuxUnitRegister, then creates + an LSSTAlertSupplier. Unlike ZTF, the LSST loader subscribes to topics + in __enter__() (a context manager), so we wrap the iteration in a `with` + block on the loader. + + SASL credentials are passed to the loader via kafka_consumer_properties + rather than AMPEL's NamedSecret-based SASLAuthentication, which requires + AMPEL's secret store infrastructure. Passing them as plain consumer + properties is simpler and sufficient for our use case. + """ + # Deferred import: ampel-lsst's bson (via pymongo) conflicts with + # antares_client's standalone bson. Only import when actually used. + from ampel.base.AuxUnitRegister import AuxUnitRegister + from ampel.lsst.alert.load.KafkaAlertLoader import KafkaAlertLoader + from ampel.lsst.alert.LSSTAlertSupplier import LSSTAlertSupplier + + AuxUnitRegister._dyn['KafkaAlertLoader'] = KafkaAlertLoader + + # Build loader config, merging SASL auth into kafka_consumer_properties. + kafka_props = dict(self.config.KAFKA_CONSUMER_PROPERTIES) + if self.config.SASL_USERNAME: + kafka_props.update({ + 'security.protocol': self.config.SECURITY_PROTOCOL, + 'sasl.mechanism': self.config.SASL_MECHANISM, + 'sasl.username': self.config.SASL_USERNAME, + 'sasl.password': self.config.SASL_PASSWORD, + }) + + loader_config: dict[str, Any] = { + 'bootstrap': self.config.BOOTSTRAP, + 'topics': self.config.TOPICS, + 'group_name': self.config.GROUP_NAME, + 'timeout': self.config.TIMEOUT, + 'kafka_consumer_properties': kafka_props, + } + if self.config.AVRO_SCHEMA: + loader_config['avro_schema'] = self.config.AVRO_SCHEMA + + supplier = LSSTAlertSupplier( + deserialize=None, # KafkaAlertLoader handles deserialization + loader={'unit': 'KafkaAlertLoader', 'config': loader_config}, + ) + + logger.info( + '%s: listening on %s (topics=%s, group=%s)', + self.STREAM_NAME, self.config.BOOTSTRAP, self.config.TOPICS, self.config.GROUP_NAME, + ) + + # KafkaAlertLoader subscribes in __enter__() — must use context manager. + with supplier.alert_loader: + fallback_topic = next(iter(self.config.TOPIC_HANDLERS)) + for alert in supplier: + # Extract real topic from Kafka metadata preserved by KafkaAlertLoader. + topic = '' + if alert.extra and 'kafka' in alert.extra: + topic = alert.extra['kafka'].get('topic', '') + if not topic: + topic = fallback_topic + self.alert_handler[topic](alert, alert_stream=self, topic=topic) diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py index a26ae65..fa19594 100644 --- a/tom_alertstreams/tables.py +++ b/tom_alertstreams/tables.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import urllib.parse from typing import Any, ClassVar import django_filters @@ -237,9 +238,31 @@ def object_url(self, alert: Alert) -> str | None: return f'{self.BASE_URL}/objects/{alert.object_id}/' +class LsstPresenter(AlertStreamPresenter): + """Rubin Science Platform deep links via ADQL query API. + + Constructs a URL that opens the RSP Portal with a pre-filled ADQL query + for the diaObject. Requires RSP login (CILogon) — unauthenticated users + are redirected to the login page, then to the query result. + + The schema prefix (e.g. 'dp1') changes per Rubin data release. + """ + BASE_URL: ClassVar[str] = 'https://data.lsst.cloud' + TAP_URL: ClassVar[str] = 'https://data.lsst.cloud/api/tap' + SCHEMA_PREFIX: ClassVar[str] = 'dp1' + + def object_url(self, alert: Alert) -> str | None: + if not alert.object_id: + return None + adql = f'SELECT * FROM {self.SCHEMA_PREFIX}.DiaObject WHERE diaObjectId={alert.object_id}' + encoded_adql = urllib.parse.quote(adql) + return f'{self.BASE_URL}/portal/app/?api=tap&service={self.TAP_URL}&adql={encoded_adql}&execute=true' + + # Streams not listed here use the default AlertStreamPresenter (no URLs). STREAM_PRESENTERS: dict[str, type[AlertStreamPresenter]] = { 'alerce': AlercePresenter, + 'ampel-lsst': LsstPresenter, 'antares': AntaresPresenter, 'antares-ztf': AntaresPresenter, 'antares-lsst': AntaresPresenter, From 1045b15f7401b8598ac158f58badd845a75b489f Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 12 Mar 2026 16:15:46 -0700 Subject: [PATCH 21/33] add tests for alert model, normalization, persistence handler, and views Test coverage: - FIFOQueueMixin partition truncation and global limit - NormalizedAlert Pydantic validation (required fields, optional fields) - save_alert_to_database handler (success and error paths) - RecentAlertsView GET (200, stream dashboard context, empty state) - URL auto-registration via AppConfig.include_url_paths - AlertStreamConfig and AlertStream base class validation --- tom_alertstreams/tests/tests.py | 188 +++++++++++++++++++++++++++++++- 1 file changed, 187 insertions(+), 1 deletion(-) diff --git a/tom_alertstreams/tests/tests.py b/tom_alertstreams/tests/tests.py index 5eef763..638bf60 100644 --- a/tom_alertstreams/tests/tests.py +++ b/tom_alertstreams/tests/tests.py @@ -1,4 +1,16 @@ -from django.test import tag, TestCase +from __future__ import annotations + +from datetime import datetime, timezone +from unittest.mock import MagicMock + +from django.apps import apps +from django.test import TestCase, tag, override_settings +from django.urls import reverse +from pydantic import ValidationError + +from tom_alertstreams.alertstreams.alertstream import NormalizedAlert +from tom_alertstreams.alertstreams.alertstream import save_alert_to_database +from tom_alertstreams.models import Alert class TestDummy(TestCase): @@ -18,3 +30,177 @@ class TestDummyCanary(TestCase): def test_dummy_canary(self): assert True + + +class FIFOQueueMixinTest(TestCase): + """Tests for the FIFOQueueMixin via the Alert model.""" + + def setUp(self) -> None: + # Override FIFO_MAX to a small value so tests run quickly without needing 100 rows. + self._original_fifo_max = Alert.FIFO_MAX + Alert.FIFO_MAX = 3 + + def tearDown(self) -> None: + Alert.FIFO_MAX = self._original_fifo_max + + def _make_alert(self, stream_name: str, alert_id: str, day: int) -> Alert: + """Create and return an Alert with a specific timestamp day.""" + return Alert.objects.create( + stream_name=stream_name, + topic='test.topic', + timestamp=datetime(2024, 1, day, tzinfo=timezone.utc), + alert_id=alert_id, + raw_payload={}, + ) + + def test_oldest_rows_deleted_when_limit_exceeded(self) -> None: + """Inserting beyond FIFO_MAX removes the oldest rows for that partition.""" + stream = 'test_fifo' + for i in range(5): + self._make_alert(stream, f'alert-{i}', day=i + 1) + + remaining = Alert.objects.filter(stream_name=stream) + self.assertEqual(remaining.count(), 3) + # The two oldest (day 1 and day 2) must be gone; the newest must remain. + self.assertFalse(Alert.objects.filter(alert_id='alert-0').exists()) + self.assertFalse(Alert.objects.filter(alert_id='alert-1').exists()) + self.assertTrue(Alert.objects.filter(alert_id='alert-4').exists()) + + def test_fifo_limit_is_per_partition(self) -> None: + """The row limit applies per stream_name, not across the whole table.""" + for i in range(5): + self._make_alert('stream_a', f'a-{i}', day=i + 1) + self._make_alert('stream_b', f'b-{i}', day=i + 1) + + # Each partition should be trimmed independently. + self.assertEqual(Alert.objects.filter(stream_name='stream_a').count(), 3) + self.assertEqual(Alert.objects.filter(stream_name='stream_b').count(), 3) + + def test_rows_not_deleted_within_limit(self) -> None: + """Rows are not deleted when the row count is at or below FIFO_MAX.""" + stream = 'stream_under_limit' + for i in range(3): + self._make_alert(stream, f'x-{i}', day=i + 1) + + self.assertEqual(Alert.objects.filter(stream_name=stream).count(), 3) + self.assertTrue(Alert.objects.filter(alert_id='x-0').exists()) + + +class NormalizedAlertTest(TestCase): + """Tests for the NormalizedAlert Pydantic model.""" + + def test_required_fields_raise_validation_error_when_missing(self) -> None: + """NormalizedAlert raises ValidationError when required fields are absent.""" + with self.assertRaises(ValidationError): + NormalizedAlert(stream_name='test') # missing alert_id and timestamp + + def test_optional_fields_default_to_none(self) -> None: + """All optional NormalizedAlert fields default correctly when not supplied.""" + alert = NormalizedAlert( + stream_name='test', + alert_id='001', + timestamp=datetime.now(timezone.utc), + ) + self.assertIsNone(alert.ra) + self.assertIsNone(alert.dec) + self.assertIsNone(alert.magnitude) + self.assertIsNone(alert.object_id) + self.assertEqual(alert.topic, '') + self.assertEqual(alert.raw_payload, {}) + + def test_full_alert_round_trips_to_dict(self) -> None: + """model_dump() on a fully-populated NormalizedAlert produces the correct keys.""" + alert = NormalizedAlert( + stream_name='test', + alert_id='abc-123', + timestamp=datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc), + topic='test.topic', + object_id='ZTF24aaaaaaaa', + ra=180.5, + dec=-30.2, + magnitude=18.7, + raw_payload={'foo': 'bar'}, + ) + d = alert.model_dump() + self.assertEqual(d['stream_name'], 'test') + self.assertEqual(d['alert_id'], 'abc-123') + self.assertEqual(d['ra'], 180.5) + self.assertEqual(d['raw_payload'], {'foo': 'bar'}) + + +class SaveAlertToDatabaseTest(TestCase): + """Tests for the save_alert_to_database handler.""" + + def _make_mock_stream(self, stream_name: str = 'test', alert_id: str = 'test-001') -> MagicMock: + """Return a mock AlertStream whose normalize_alert produces a fixed NormalizedAlert.""" + mock_stream = MagicMock() + + def normalize(raw_alert: object, topic: str = '') -> NormalizedAlert: + return NormalizedAlert( + stream_name=stream_name, + alert_id=alert_id, + timestamp=datetime.now(timezone.utc), + topic=topic, + raw_payload={}, + ) + + mock_stream.get_normalization_function.return_value = normalize + return mock_stream + + def test_save_creates_alert_in_database(self) -> None: + """save_alert_to_database persists a NormalizedAlert as an Alert row.""" + mock_stream = self._make_mock_stream(stream_name='antares', alert_id='ant-007') + result = save_alert_to_database({'data': 'mock'}, alert_stream=mock_stream, topic='test.topic') + + self.assertIsInstance(result, Alert) + self.assertEqual(result.stream_name, 'antares') + self.assertEqual(result.alert_id, 'ant-007') + self.assertEqual(result.topic, 'test.topic') + self.assertTrue(Alert.objects.filter(alert_id='ant-007').exists()) + + def test_save_returns_none_on_normalization_error(self) -> None: + """save_alert_to_database returns None and does not crash if normalization fails.""" + mock_stream = MagicMock() + mock_stream.get_normalization_function.return_value = MagicMock(side_effect=RuntimeError('fail')) + + result = save_alert_to_database({'data': 'bad'}, alert_stream=mock_stream) + self.assertIsNone(result) + self.assertEqual(Alert.objects.count(), 0) + + +class RecentAlertsViewTest(TestCase): + """Tests for the RecentAlertsView.""" + + @override_settings(ALERT_STREAMS=[]) + def test_view_returns_200(self) -> None: + """GET /alertstreams/recent/ returns 200 with no streams configured.""" + url = reverse('alertstreams:recent-alerts') + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + + @override_settings(ALERT_STREAMS=[]) + def test_view_contains_recent_alerts_heading(self) -> None: + """The Recent Alerts page includes the page heading.""" + url = reverse('alertstreams:recent-alerts') + response = self.client.get(url) + self.assertContains(response, 'Recent Alerts') + + +class AppConfigIntegrationTest(TestCase): + """Tests for the TomAlertstreamsConfig AppConfig integration points.""" + + def test_include_url_paths_returns_alertstreams_pattern(self) -> None: + """include_url_paths() registers a URL pattern under 'alertstreams/'.""" + app_config = apps.get_app_config('tom_alertstreams') + url_patterns = app_config.include_url_paths() + self.assertGreater(len(url_patterns), 0) + # The first pattern should cover the 'alertstreams/' prefix. + self.assertIn('alertstreams', str(url_patterns[0].pattern)) + + def test_nav_items_returns_navbar_link_partial(self) -> None: + """nav_items() returns the correct navbar partial path for Recent Alerts.""" + app_config = apps.get_app_config('tom_alertstreams') + items = app_config.nav_items() + self.assertEqual(len(items), 1) + self.assertIn('partial', items[0]) + self.assertIn('navbar_link', items[0]['partial']) From 7d6ffaab5649d24911ccfc5a4553d15fc5aaa587 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Mon, 16 Mar 2026 10:25:02 -0700 Subject: [PATCH 22/33] For ANTARES, use the same AlertStream subclass for ZTF and LSST There are differences (e.g. ZTF has magnitude, while LSST has flux), but those can be handled in normalize_alerts() when we examine an LSST alert. (This commit as code in there to do the examination that will be removed later). --- tom_alertstreams/alertstreams/antares.py | 214 ++++++++--------------- tom_alertstreams/tables.py | 2 - tom_alertstreams/tests/tests.py | 98 +++++++++++ 3 files changed, 172 insertions(+), 142 deletions(-) diff --git a/tom_alertstreams/alertstreams/antares.py b/tom_alertstreams/alertstreams/antares.py index a171d25..3188a47 100644 --- a/tom_alertstreams/alertstreams/antares.py +++ b/tom_alertstreams/alertstreams/antares.py @@ -41,33 +41,25 @@ def _mjd_to_datetime(mjd: float) -> datetime: # --------------------------------------------------------------------------- class AntaresConfig(AlertStreamConfig): - """Pydantic configuration model for real ANTARES alert streams (ZTF and LSST). + """Pydantic configuration model for AntaresAlertStream. Inherits from AlertStreamConfig (a Pydantic BaseModel), so Pydantic validates that API_KEY and API_SECRET are present and raises descriptive errors if not. - Both AntaresZtfAlertStream and AntaresLsstAlertStream share this config because - they use the same StreamingClient, same Kafka auth, and same Locus model — the - only differences are topic names and normalize_alert() field extraction. Fields: API_KEY: ANTARES API key (required). Obtain at https://antares.noirlab.edu. API_SECRET: ANTARES API secret (required). Obtain at https://antares.noirlab.edu. - TOPIC_HANDLERS: Inherited from AlertStreamConfig. Maps topic names to handler - dotted-paths. Known ZTF topics: 'extragalactic_staging', - 'nuclear_transient_staging'. + TOPIC_HANDLERS: Inherited from AlertStreamConfig. Maps ANTARES filter topic + names to handler dotted-paths. ANTARES topics are filter outputs, not + survey-specific — a single topic can carry both ZTF and LSST loci. + Known topics: 'extragalactic_staging', 'nuclear_transient_staging', + 'in_shadow_virgo'. See the ANTARES tags page for the full list. GROUP: Kafka consumer group ID. Distinct group IDs let multiple TOM instances consume the same stream independently. SSL_CA_LOCATION: Path to a TLS Certificate Authority (CA) certificate file. - When StreamingClient connects to the ANTARES Kafka broker, it uses TLS - (encrypted connection). TLS requires a CA cert — a file that tells the - client "trust connections signed by this authority." The antares_client - package bundles a default CA cert (certificates/kafka-ca.pem) that works - for the current ZTF broker. If ANTARES uses a different Kafka cluster or - TLS chain for LSST alerts, the default cert might not be trusted, and - you'd need to point SSL_CA_LOCATION at the correct CA cert file. + The antares_client package bundles a default CA cert that works for the + current broker. Set this only if a different CA cert is needed. When None (the default), the bundled cert is used. - TODO: When researching LSST topic access, find out whether LSST topics - require a different CA cert and document how to obtain it. ENABLE_AUTO_COMMIT: Whether Kafka should auto-commit offsets. Set to False for at-least-once processing with manual offset management. """ @@ -157,26 +149,46 @@ def listen(self) -> None: # --------------------------------------------------------------------------- -# Real ANTARES streams — abstract base with ZTF and LSST subclasses +# Real ANTARES stream — unified for all surveys (ZTF, LSST, etc.) # --------------------------------------------------------------------------- class AntaresAlertStream(AlertStream): - """Abstract base class for real ANTARES alert streams. + """ANTARES alert stream for ZTF, LSST, and future survey alerts. - Handles StreamingClient setup and the listen() loop. Not configured directly — - users configure AntaresZtfAlertStream or AntaresLsstAlertStream, which override - normalize_alert() for survey-specific field extraction. - - ANTARES is unique among the LSST brokers: it presents a unified Kafka interface - for both ZTF and LSST alerts via the same StreamingClient and Locus model. The - only differences between surveys are topic names, properties dict keys, and - photometry units. This base class captures the shared listen() logic while - subclasses handle the divergent normalization. + ANTARES Kafka topics are filter outputs, not survey-specific — a single topic + like 'extragalactic_staging' can carry loci with ZTF data, LSST data, or both. + This class handles all surveys through a single normalize_alert() that extracts + whatever survey data is present on each locus. The StreamingClient is created inside listen() using a context manager (not in __init__) so the Kafka consumer is properly closed on errors or shutdown. + + Known topics (from the ANTARES tags page at https://antares.noirlab.edu): + ZTF: 'extragalactic_staging', 'nuclear_transient_staging' + Mixed/LSST: 'in_shadow_virgo' + Topic names generally follow the pattern '{tag_name}_staging', though some + (like 'in_shadow_virgo') omit the suffix. + + Configuration example (settings.py ALERT_STREAMS entry):: + + { + 'ACTIVE': True, + 'NAME': 'tom_alertstreams.alertstreams.antares.AntaresAlertStream', + 'OPTIONS': { + 'API_KEY': os.environ.get('ANTARES_API_KEY', ''), + 'API_SECRET': os.environ.get('ANTARES_API_SECRET', ''), + 'TOPIC_HANDLERS': { + # ZTF topics + 'extragalactic_staging': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database', + 'nuclear_transient_staging': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database', + # LSST topics + 'in_shadow_virgo': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database', + }, + }, + } """ configuration_class = AntaresConfig # type: ignore[assignment] + STREAM_NAME: ClassVar[str] = 'antares' def listen(self) -> None: """Consume ANTARES loci and dispatch to configured topic handlers. @@ -208,39 +220,16 @@ def listen(self) -> None: logger.info(f'{self.STREAM_NAME} received {locus.locus_id} on {base_topic}') self.alert_handler[base_topic](locus, alert_stream=self, topic=base_topic) - -class AntaresZtfAlertStream(AntaresAlertStream): - """ANTARES alert stream for ZTF transient alerts. - - Connects to ANTARES Kafka topics that carry ZTF-originated alerts. Each alert - is an antares_client Locus object enriched with ZTF-specific properties. - - Known ZTF topics: 'extragalactic_staging', 'nuclear_transient_staging'. - Contact the ANTARES team for the full topic list. - - Configuration example (settings.py ALERT_STREAMS entry): - { - 'ACTIVE': True, - 'NAME': 'tom_alertstreams.alertstreams.antares.AntaresZtfAlertStream', - 'OPTIONS': { - 'API_KEY': os.environ.get('ANTARES_API_KEY', ''), - 'API_SECRET': os.environ.get('ANTARES_API_SECRET', ''), - 'TOPIC_HANDLERS': { - 'extragalactic_staging': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database', - 'nuclear_transient_staging': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database', - }, - }, - } - """ - STREAM_NAME: ClassVar[str] = 'antares-ztf' - def normalize_alert(self, raw_alert: Any, topic: str = '') -> NormalizedAlert: - """Extract common fields from an ANTARES Locus object carrying ZTF data. + """Extract common fields from an ANTARES Locus object. - Field mappings (discovered via ex_antares.py introspection of a live ZTF locus): - - timestamp: locus.properties['newest_alert_observation_time'] (MJD float) - - magnitude: locus.properties['newest_alert_magnitude'] - - object_id: locus.properties['ztf_object_id'] (distinct from locus_id) + Handles both ZTF and LSST data by extracting whatever survey-specific + properties are present on the locus. Field mappings discovered via + ex_antares_ztf.py and ex_antares_lsst.py introspection: + + - timestamp: properties['newest_alert_observation_time'] (MJD float) + - magnitude: properties['newest_alert_magnitude'] (ANTARES-enriched) + - object_id: LSST dia_object_id (nested) → ZTF ztf_object_id (flat) → locus_id - alert_id: locus.locus_id (used by AntaresPresenter for locus page URL) The full Locus object is not JSON-serializable, so raw_payload is left empty. @@ -251,101 +240,45 @@ def normalize_alert(self, raw_alert: Any, topic: str = '') -> NormalizedAlert: topic: The ANTARES topic (e.g. 'extragalactic_staging'). Returns: - NormalizedAlert with ZTF-specific fields populated. + NormalizedAlert with survey-appropriate fields populated. """ - props = raw_alert.properties or {} + alert_properties = raw_alert.properties or {} + + # Extract properties from the raw_alert for transfer to NormalizedAlert. # Timestamp from ANTARES-enriched locus property (MJD float). # This avoids lazy-loading locus.alerts, which triggers an HTTP API call. - mjd = props.get('newest_alert_observation_time') + mjd = alert_properties.get('newest_alert_observation_time') timestamp = _mjd_to_datetime(mjd) if mjd is not None else datetime.now(timezone.utc) - # ZTF magnitude from ANTARES-enriched properties. - magnitude = props.get('newest_alert_magnitude') - - # ZTF object ID if available; fallback to locus_id. - object_id = props.get('ztf_object_id', raw_alert.locus_id) - - return NormalizedAlert( - stream_name=self.STREAM_NAME, - topic=topic, - timestamp=timestamp, - alert_id=str(raw_alert.locus_id), - object_id=str(object_id), - ra=float(raw_alert.ra) if raw_alert.ra is not None else None, - dec=float(raw_alert.dec) if raw_alert.dec is not None else None, - magnitude=float(magnitude) if magnitude is not None else None, - flux=None, - raw_payload={}, - ) - + # Magnitude — ANTARES-enriched, populated for ZTF loci. + magnitude = alert_properties.get('newest_alert_magnitude') -class AntaresLsstAlertStream(AntaresAlertStream): - """ANTARES alert stream for LSST transient alerts. - - Connects to ANTARES Kafka topics that carry LSST-originated alerts. ANTARES - is structurally ready for LSST — the Locus properties dict already contains - a 'survey.lsst' namespace with dia_object_id and ss_object_id arrays — but - LSST topic names and photometry property keys are not yet confirmed. - - This class is provided for forward-compatibility. Activate it in settings once - LSST topics are available and property key names are confirmed via introspection. - - Configuration example (settings.py ALERT_STREAMS entry): - { - 'ACTIVE': False, # activate once LSST topics are confirmed - 'NAME': 'tom_alertstreams.alertstreams.antares.AntaresLsstAlertStream', - 'OPTIONS': { - 'API_KEY': os.environ.get('ANTARES_API_KEY', ''), - 'API_SECRET': os.environ.get('ANTARES_API_SECRET', ''), - 'TOPIC_HANDLERS': { - 'lsst_placeholder': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database', - }, - }, - } - """ - STREAM_NAME: ClassVar[str] = 'antares-lsst' - - def normalize_alert(self, raw_alert: Any, topic: str = '') -> NormalizedAlert: - """Extract common fields from an ANTARES Locus object carrying LSST data. - - LSST-specific field mappings are provisional — based on the Locus properties - structure observed via ex_antares.py and the antares_client.search module: - - object_id: properties['survey']['lsst']['dia_object_id'][0] (nested dict) - - flux: property key TBD (need to inspect a real LSST locus) - - timestamp: properties['newest_alert_observation_time'] (same as ZTF) - - TODO: Verify all LSST property keys by introspecting a real LSST locus once - LSST topics are available. Update this method accordingly. - - Args: - raw_alert: An antares_client.models.Locus object. - topic: The ANTARES topic for LSST alerts. - - Returns: - NormalizedAlert with LSST-specific fields populated where known. - """ - props = raw_alert.properties or {} - - # Timestamp — same MJD property as ZTF (ANTARES-enriched). - mjd = props.get('newest_alert_observation_time') - timestamp = _mjd_to_datetime(mjd) if mjd is not None else datetime.now(timezone.utc) - - # LSST object ID from the nested survey dict structure. - # antares_client.search uses 'properties.survey.lsst.dia_object_id' as a - # flat key path, but the actual properties dict is nested: - # properties['survey']['lsst']['dia_object_id'] → list of IDs - object_id = raw_alert.locus_id # default fallback - survey = props.get('survey', {}) + # Object ID — try LSST nested structure first, then ZTF flat property, then locus_id. + # ANTARES stores LSST IDs in a nested dict: properties['survey']['lsst']['dia_object_id'] → list + # ZTF IDs are a flat property: properties['ztf_object_id'] → str + object_id = raw_alert.locus_id # fallback + survey = alert_properties.get('survey', {}) lsst_survey = survey.get('lsst', {}) if isinstance(survey, dict) else {} dia_object_ids = lsst_survey.get('dia_object_id', []) if dia_object_ids: object_id = dia_object_ids[0] + elif alert_properties.get('ztf_object_id'): + object_id = alert_properties['ztf_object_id'] - # LSST flux — property key TBD. Need to inspect a real LSST locus. + # Flux — LSST uses flux (nanojansky) instead of magnitude. + # Property key TBD: no LSST-only locus observed from the stream yet. + # TODO: once the flux property key is known, extract it here. flux = None - return NormalizedAlert( + # Log LSST locus properties so we can discover the flux key from production data. + # Remove this logging once we've confirmed the flux property key and updated + # the extraction above. + if dia_object_ids: + logger.info(f'LSST locus detected: {raw_alert.locus_id} — ' + f'full properties: {alert_properties}') + + normalized_alert = NormalizedAlert( stream_name=self.STREAM_NAME, topic=topic, timestamp=timestamp, @@ -353,7 +286,8 @@ def normalize_alert(self, raw_alert: Any, topic: str = '') -> NormalizedAlert: object_id=str(object_id), ra=float(raw_alert.ra) if raw_alert.ra is not None else None, dec=float(raw_alert.dec) if raw_alert.dec is not None else None, - magnitude=None, + magnitude=float(magnitude) if magnitude is not None else None, flux=flux, raw_payload={}, ) + return normalized_alert diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py index fa19594..2ed2afb 100644 --- a/tom_alertstreams/tables.py +++ b/tom_alertstreams/tables.py @@ -264,8 +264,6 @@ def object_url(self, alert: Alert) -> str | None: 'alerce': AlercePresenter, 'ampel-lsst': LsstPresenter, 'antares': AntaresPresenter, - 'antares-ztf': AntaresPresenter, - 'antares-lsst': AntaresPresenter, 'babamul': BabamulPresenter, 'fink': FinkPresenter, 'gcn': GCNPresenter, diff --git a/tom_alertstreams/tests/tests.py b/tom_alertstreams/tests/tests.py index 638bf60..8e74f4a 100644 --- a/tom_alertstreams/tests/tests.py +++ b/tom_alertstreams/tests/tests.py @@ -128,6 +128,104 @@ def test_full_alert_round_trips_to_dict(self) -> None: self.assertEqual(d['raw_payload'], {'foo': 'bar'}) +class AntaresNormalizeAlertTest(TestCase): + """Tests for the unified AntaresAlertStream.normalize_alert() method. + + Uses mock Locus objects to verify that normalize_alert() correctly handles + ZTF-only loci, LSST-only loci, cross-matched loci (both surveys), and bare + loci (neither survey ID present). + """ + + def setUp(self) -> None: + """Create an AntaresAlertStream instance with a minimal config.""" + from tom_alertstreams.alertstreams.antares import AntaresAlertStream + + self.stream = AntaresAlertStream( + API_KEY='fake-key', + API_SECRET='fake-secret', + TOPIC_HANDLERS={'test_topic': 'tom_alertstreams.alertstreams.alertstream.save_alert_to_database'}, + ) + + def _make_mock_locus( + self, + locus_id: str = 'ANT2026test', + ra: float = 180.0, + dec: float = -30.0, + properties: dict | None = None, + ) -> MagicMock: + """Return a mock antares_client Locus object.""" + locus = MagicMock() + locus.locus_id = locus_id + locus.ra = ra + locus.dec = dec + locus.properties = properties or {} + return locus + + def test_ztf_locus_extracts_ztf_object_id(self) -> None: + """ZTF locus: object_id comes from properties['ztf_object_id'].""" + locus = self._make_mock_locus(properties={ + 'newest_alert_observation_time': 60400.5, + 'newest_alert_magnitude': 18.5, + 'ztf_object_id': 'ZTF24aatest', + 'survey': {'ztf': {'id': ['ZTF24aatest']}, 'lsst': {'dia_object_id': [], 'ss_object_id': []}}, + }) + result = self.stream.normalize_alert(locus, topic='extragalactic_staging') + + self.assertEqual(result.object_id, 'ZTF24aatest') + self.assertEqual(result.magnitude, 18.5) + self.assertEqual(result.stream_name, 'antares') + self.assertEqual(result.topic, 'extragalactic_staging') + + def test_lsst_locus_extracts_dia_object_id(self) -> None: + """LSST locus: object_id comes from survey.lsst.dia_object_id[0].""" + locus = self._make_mock_locus(properties={ + 'newest_alert_observation_time': 60400.5, + 'newest_alert_magnitude': None, + 'survey': {'ztf': {'id': []}, 'lsst': {'dia_object_id': ['170028527925067818'], 'ss_object_id': []}}, + }) + result = self.stream.normalize_alert(locus, topic='in_shadow_virgo') + + self.assertEqual(result.object_id, '170028527925067818') + + def test_cross_matched_locus_prefers_lsst_object_id(self) -> None: + """Cross-matched locus: LSST dia_object_id takes priority over ZTF ztf_object_id.""" + locus = self._make_mock_locus(properties={ + 'newest_alert_observation_time': 60400.5, + 'newest_alert_magnitude': 19.0, + 'ztf_object_id': 'ZTF24aatest', + 'survey': { + 'ztf': {'id': ['ZTF24aatest']}, + 'lsst': {'dia_object_id': ['170028527925067818'], 'ss_object_id': []}, + }, + }) + result = self.stream.normalize_alert(locus, topic='extragalactic_staging') + + self.assertEqual(result.object_id, '170028527925067818') + # Magnitude should still be extracted even though LSST object_id was preferred + self.assertEqual(result.magnitude, 19.0) + + def test_bare_locus_falls_back_to_locus_id(self) -> None: + """Bare locus (no ZTF or LSST object ID): falls back to locus_id.""" + locus = self._make_mock_locus(locus_id='ANT2026bare', properties={ + 'newest_alert_observation_time': 60400.5, + 'survey': {'ztf': {'id': []}, 'lsst': {'dia_object_id': [], 'ss_object_id': []}}, + }) + result = self.stream.normalize_alert(locus, topic='test_topic') + + self.assertEqual(result.object_id, 'ANT2026bare') + self.assertEqual(result.alert_id, 'ANT2026bare') + + def test_missing_timestamp_defaults_to_now(self) -> None: + """Missing newest_alert_observation_time defaults to current UTC time.""" + locus = self._make_mock_locus(properties={}) + result = self.stream.normalize_alert(locus) + + self.assertIsNotNone(result.timestamp) + # Should be within the last few seconds + delta = (datetime.now(timezone.utc) - result.timestamp).total_seconds() + self.assertLess(abs(delta), 5) + + class SaveAlertToDatabaseTest(TestCase): """Tests for the save_alert_to_database handler.""" From b7e793bd251be95c8ac6232c24af3668e324a0aa Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 11 Jun 2026 14:52:22 -0700 Subject: [PATCH 23/33] add partioning to FiFOQueueMixin; use it (stream+topic) in Alert combinations of specified fields define what contributes to the FIFO_MAX limit. So, with (stream+topic), for a given Alert stream, the number of alerts within a topic count toward FIFO_MAX (and each topic gets it's own budget). --- tom_alertstreams/models.py | 91 +++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/tom_alertstreams/models.py b/tom_alertstreams/models.py index 9e253cf..d26a78f 100644 --- a/tom_alertstreams/models.py +++ b/tom_alertstreams/models.py @@ -9,73 +9,82 @@ class FIFOQueueMixin(models.Model): """Mixin that enforces a per-partition maximum row count. - Basically, this limits the size that a Model's table can reach by - turning it into a First-In-First-Out (FIFO) queue. - - This is implemented by extending the `save()` method to save a model - instance (as per normal), then check the table size and delete records - (oldest first) over the FIFO_MAX limit. - - The wrinkle is the per-partion part: The FIFO_PARTIION_FIELD is a field - in the model that divides (i.e. "partitions") the table according to the - value of the field. What that means is that there can be FIFO_MAX records - that have a common value in the FIFO_PARTIION_FIELD. So, for example, if the - FIFO_PARTITION_FIELD is `stream_name`, then there can be FIFO_MAX records - with `stream_name` "alerce" and FIFO_MAX records with `stream_name` "fink", - etc. So, there can be FIFO_MAX records for each distinct value of the - FIFO_PARTIION_FIELD. The FIFO_PARTION_FIELD value of the instance being - saved specifies the partition whose size is checked, post-save(). - - To summarize, after every save(), rows beyond FIFO_MAX are deleted - (oldest first) within the same partition as the newly-saved instance. - 'Partition' means all rows sharing the same value of FIFO_PARTITION_FIELD - — e.g., all alerts from the same stream. If FIFO_PARTITION_FIELD is None, - then the FIFO_MAX limit applies to the entire table. - - Subclasses set FIFO_MAX and FIFO_PARTITION_FIELD as class variables. + Turns a model's table into a First-In-First-Out (FIFO) queue by extending + save() to delete the oldest rows beyond FIFO_MAX after every insert. + + Subclasses set FIFO_MAX and FIFO_PARTITION_FIELDS as class variables. + + Partitioning: FIFO_PARTITION_FIELDS names one or more model fields whose + values together define a partition. Each unique combination of partition + field values gets its own independent FIFO budget of FIFO_MAX rows. + + With a single partition field (e.g. ``('stream_name',)``), the table holds + at most ``FIFO_MAX × number_of_distinct_stream_names`` rows. With compound + fields (e.g. ``('stream_name', 'topic')``), each unique combination gets + its own budget, so the table can hold up to + ``FIFO_MAX × number_of_distinct_(stream_name, topic)_pairs`` rows. + For example: 8 streams × 19 topics/stream × 1000 max = 152,000 rows + (though in practice most streams have far fewer topics). + + If FIFO_PARTITION_FIELDS is None, the limit applies to the entire table. """ FIFO_MAX: ClassVar[int] = 10 - FIFO_PARTITION_FIELD: ClassVar[str | None] = None + FIFO_PARTITION_FIELDS: ClassVar[tuple[str, ...] | None] = None class Meta: abstract = True def save(self, *args: Any, **kwargs: Any) -> None: super().save(*args, **kwargs) # save, then trim: Ensures the newly-saved + # this is the extention to the method: self._enforce_fifo_limit() # row is counted against the FIFO_MAX limit. def _enforce_fifo_limit(self) -> None: - """Delete rows beyond FIFO_MAX, oldest first, within this instance's partition. + """Delete rows beyond FIFO_MAX, oldest-received first, within this partition. + + The partition is defined by FIFO_PARTITION_FIELDS — all rows sharing the + same values across those fields form one partition. Each partition is + independently capped at FIFO_MAX rows. + + Eviction is by `created` (insertion/received time) — true FIFO order — NOT by + the alert's observation `timestamp`. Ordering a FIFO by observation time would + instantly evict a freshly-received alert whose observation time is old (e.g. a + broker streaming last night's alerts now, after the survey went quiet), so it + would never appear in the table. By `created`, the most-recently-received rows + are always kept. (The concrete model must provide a `created` insertion-time + field; Alert does, via auto_now_add.) Uses list() to materialise PKs before the DELETE to avoid a SQLite restriction that forbids DELETE from a table referenced in the same statement's subquery. """ - qs = self.__class__.objects.all() - if self.FIFO_PARTITION_FIELD is not None: - # Scope the FIFO_MAX limit to rows in the same partition as this instance. - partition_value = getattr(self, self.FIFO_PARTITION_FIELD) - qs = qs.filter(**{self.FIFO_PARTITION_FIELD: partition_value}) - # Materialise PKs to avoid a subquery-in-DELETE issue on SQLite. + qs = self.__class__.objects.all() # query set + if self.FIFO_PARTITION_FIELDS is not None: + # Scope to rows sharing this instance's values for all partition fields + partition_filter = { + field: getattr(self, field) for field in self.FIFO_PARTITION_FIELDS + } + qs = qs.filter(**partition_filter) + # Keep the FIFO_MAX most-recently-received rows; materialise PKs first to avoid + # a SQLite subquery-in-DELETE restriction. excess_pks = list( - qs.order_by('-timestamp').values_list('pk', flat=True)[self.FIFO_MAX:] + # the list of the pks beyond FIFO_MAX + qs.order_by('-created').values_list('pk', flat=True)[self.FIFO_MAX:] ) if excess_pks: - self.__class__.objects.filter(pk__in=excess_pks).delete() + self.__class__.objects.filter(pk__in=excess_pks).delete() # delete overflow class Alert(FIFOQueueMixin): """A normalized alert received from an alert stream, stored for recent display. This class is designed specifically for a Recent Alerts demonstration page. - - Because of the FIFOQueueMixin, rows are automatically pruned to - ALERTSTREAMS_RECENT_COUNT per stream_name. The raw_payload JSONField preserves - the full original alert for handlers or views that need stream-specific - fields not captured here. + It's the model shown in the table of recent alerts and size is limited by + the FIFOQueueMixin, FIFO_MAX, and FIFO_PARTITION_FIELDS. """ - # set the mixin class variables + # Set the mixin class variables — partition by (stream_name, topic) so each + # stream+topic combination gets its own independent FIFO budget. FIFO_MAX: ClassVar[int] = getattr(settings, 'ALERTSTREAMS_RECENT_COUNT', 10) - FIFO_PARTITION_FIELD: ClassVar[str | None] = 'stream_name' + FIFO_PARTITION_FIELDS: ClassVar[tuple[str, ...] | None] = ('stream_name', 'topic') stream_name = models.CharField(max_length=100, db_index=True) topic = models.CharField(max_length=200) From 93ea1b4a14d564086afd0457994cb2e254ab419c Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 11 Jun 2026 14:59:11 -0700 Subject: [PATCH 24/33] clarify what timestamps mean what - `observation_time` is the time of the observation that the alert is about. - `published_time` - is when the broker published the alert. - `created` is when the alert was received and saved by the consumer. --- tom_alertstreams/models.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tom_alertstreams/models.py b/tom_alertstreams/models.py index d26a78f..704c0c5 100644 --- a/tom_alertstreams/models.py +++ b/tom_alertstreams/models.py @@ -88,15 +88,33 @@ class Alert(FIFOQueueMixin): stream_name = models.CharField(max_length=100, db_index=True) topic = models.CharField(max_length=200) - timestamp = models.DateTimeField(db_index=True) + + # When the telescope observed the source the alert is about (a detection time). + # Nullable: some alerts (e.g. GCN Circulars) are not about a single observation. + observation_time = models.DateTimeField(null=True, db_index=True) + + # When the broker/survey issued/published the alert (e.g. GCN Circular createdOn, + # Fink brokerEndProcessTimestamp). Nullable: not every stream exposes it. The gap + # between observation_time and published_time is the broker's processing latency. + published_time = models.DateTimeField(null=True) + alert_id = models.CharField(max_length=200) object_id = models.CharField(max_length=200, blank=True, null=True) ra = models.FloatField(null=True) dec = models.FloatField(null=True) magnitude = models.FloatField(null=True) flux = models.FloatField(null=True) + raw_payload = models.JSONField(default=dict) + # When the row was received and saved — our receipt clock, distinct from + # `observation_time` (when observed) and `published_time` (when the broker issued + # it). Observation/publish times can be stale or backlogged, so `created` is what + # answers "is this stream still ingesting?" — and it drives ordering + the FIFO. + # Nullable so the column could be added while a running readstreams (old model) was + # still inserting; every new row sets it via auto_now_add. + created = models.DateTimeField(auto_now_add=True, null=True, db_index=True) + class Meta(FIFOQueueMixin.Meta): # this is the way you subclass the internal Meta class abstract = False # override for the concrete model (abstract is True in the super) ordering = ['-timestamp'] From 5babfcf502a669b7dcf64db06857cb7198d86852 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 11 Jun 2026 15:02:49 -0700 Subject: [PATCH 25/33] formatting, comments and more informative __str__ --- tom_alertstreams/models.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tom_alertstreams/models.py b/tom_alertstreams/models.py index 704c0c5..b7ff301 100644 --- a/tom_alertstreams/models.py +++ b/tom_alertstreams/models.py @@ -116,9 +116,15 @@ class Alert(FIFOQueueMixin): created = models.DateTimeField(auto_now_add=True, null=True, db_index=True) class Meta(FIFOQueueMixin.Meta): # this is the way you subclass the internal Meta class - abstract = False # override for the concrete model (abstract is True in the super) - ordering = ['-timestamp'] - indexes = [models.Index(fields=['stream_name', 'timestamp'])] + + # in the FIFOQueueMixin.Meta, abstract is True, + abstract = False # here, in the concrete model, we're not abstract. + + ordering = ['-created'] # most recently received + + # speed up the HTMX filtering + indexes = [models.Index(fields=['stream_name', 'topic', 'observation_time'])] def __str__(self) -> str: - return f'Alert {self.alert_id} from {self.stream_name} at {self.timestamp}' + return (f'Alert {self.alert_id} received from {self.stream_name} on topic {self.topic} ' + f'at {self.created}') From 8b4434f20a6e8dd5d1d8c4e922cdd6e735a63313 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 11 Jun 2026 15:06:30 -0700 Subject: [PATCH 26/33] add newly clarified timestamps --- tom_alertstreams/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_alertstreams/admin.py b/tom_alertstreams/admin.py index 56b6f6f..78a7564 100644 --- a/tom_alertstreams/admin.py +++ b/tom_alertstreams/admin.py @@ -5,6 +5,6 @@ @admin.register(Alert) class AlertAdmin(admin.ModelAdmin): - list_display = ('stream_name', 'alert_id', 'timestamp', 'object_id', 'magnitude', 'flux') + list_display = ('stream_name', 'alert_id', 'published_time', 'observation_time', 'object_id', 'magnitude', 'flux') list_filter = ('stream_name',) search_fields = ('alert_id', 'object_id') From 9a66b9aa8a9b67368b256e77b56fe2098e3c567b Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 11 Jun 2026 15:14:19 -0700 Subject: [PATCH 27/33] update AlertTable with new, more specific timestamp columns --- tom_alertstreams/tables.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py index 2ed2afb..34bca4f 100644 --- a/tom_alertstreams/tables.py +++ b/tom_alertstreams/tables.py @@ -45,14 +45,22 @@ def __init__( # stream_name = tables.Column(verbose_name='Stream') # sets the column header value + # Two-word headers so these time columns wrap rather than stretch the table wide. + observation_time = tables.Column(verbose_name='Observation Time') + published_time = tables.Column(verbose_name='Published Time') # render_FIELDNAME() methods are called automatically when present - def render_timestamp(self, value: Any) -> str: - """Render timestamp in unambiguous UTC 24-hour format. + def render_observation_time(self, value: Any) -> str: + """Render the observation time in UTC 24-hour format (blank if absent). - The result looks like this: 2026-03-05 18:51:30 UTC + The result looks like this: 2026-03-05 18:51:30 UTC. Blank for alerts with no + observation (e.g. GCN Circulars). """ - return value.strftime('%Y-%m-%d %H:%M:%S UTC') + return value.strftime('%Y-%m-%d %H:%M:%S UTC') if value else '' + + def render_published_time(self, value: Any) -> str: + """Render the broker/survey publish time in UTC 24-hour format (blank if absent).""" + return value.strftime('%Y-%m-%d %H:%M:%S UTC') if value else '' def render_alert_id(self, record: Alert, value: str) -> str: """Render alert_id as a hyperlink if the stream's presenter provides a URL.""" @@ -93,8 +101,8 @@ def render_flux(self, value: Any) -> str: class Meta(HTMXTable.Meta): model = Alert fields = [ - 'selection', 'alert_id', 'stream_name', 'topic', 'timestamp', - 'object_id', 'ra', 'dec', 'magnitude', 'flux', + 'selection', 'alert_id', 'stream_name', 'topic', 'published_time', + 'observation_time', 'object_id', 'ra', 'dec', 'magnitude', 'flux', ] From 592dcd5cc7436d3e6d9ef89e4d1392acf22f3f8d Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 11 Jun 2026 15:27:23 -0700 Subject: [PATCH 28/33] add presenters for Fink, Babamul, and GCN --- tom_alertstreams/tables.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py index 34bca4f..e631ef3 100644 --- a/tom_alertstreams/tables.py +++ b/tom_alertstreams/tables.py @@ -219,26 +219,49 @@ def object_url(self, alert: Alert) -> str | None: class FinkPresenter(AlertStreamPresenter): - """Fink object pages: https://fink-portal.org/{object_id}""" - BASE_URL = 'https://fink-portal.org' + """Fink object pages, survey-aware by topic suffix. + + Fink runs separate web portals per survey on per-survey subdomains, and + topics follow the '_' convention, so the topic suffix selects + the host: + ZTF: https://ztf.fink-portal.org/{object_id} + LSST: https://lsst.fink-portal.org/{object_id} + + Links the object (object_id), matching the other portal presenters + (Alerce/Babamul/Lasair). An unrecognized topic suffix yields no link. + """ + ZTF_BASE_URL: ClassVar[str] = 'https://ztf.fink-portal.org' + LSST_BASE_URL: ClassVar[str] = 'https://lsst.fink-portal.org' def object_url(self, alert: Alert) -> str | None: if not alert.object_id: return None - return f'{self.BASE_URL}/{alert.object_id}' + if alert.topic.endswith('_ztf'): + return f'{self.ZTF_BASE_URL}/{alert.object_id}' + if alert.topic.endswith('_lsst'): + return f'{self.LSST_BASE_URL}/{alert.object_id}' + logger.warning('FinkPresenter: unrecognized topic suffix: %s', alert.topic) + return None class GCNPresenter(AlertStreamPresenter): - """GCN circular pages: https://gcn.nasa.gov/circulars/{alert_id}""" + """GCN circular pages: https://gcn.nasa.gov/circulars/{circularId} + + Only the gcn.circulars topic maps to a circulars page (where alert_id is the + circularId). Other GCN topics (heartbeat, notices) have no such URL, so they get + no link rather than a broken /circulars/ one. + """ BASE_URL = 'https://gcn.nasa.gov' def alert_url(self, alert: Alert) -> str | None: - return f'{self.BASE_URL}/circulars/{alert.alert_id}' + if alert.topic == 'gcn.circulars': + return f'{self.BASE_URL}/circulars/{alert.alert_id}' + return None class LasairPresenter(AlertStreamPresenter): - """Lasair object pages: https://lasair-ztf.lsst.ac.uk/objects/{object_id}/""" - BASE_URL = 'https://lasair-ztf.lsst.ac.uk' + """Lasair object pages: https://lasair.lsst.ac.uk/objects/{object_id}/""" + BASE_URL = 'https://lasair.lsst.ac.uk' def object_url(self, alert: Alert) -> str | None: if not alert.object_id: From c38f87406e5498086cad18047e033da46c275dcc Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 11 Jun 2026 15:28:05 -0700 Subject: [PATCH 29/33] de-duplicate Google Pub/Sub-based brokers in table Google Pub/Sub streams are configured slightly differently than kafka streams in that a subscription binds one topic only. So, there's one config dict for each topic. This confuses the way we instrospect the streams for the AlertFilterSet choices: we see a Pub/Sub broker one for each of it's topics, but we only want to list it once (it's one broker). This removes the duplicate choices. --- tom_alertstreams/tables.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py index e631ef3..779ee92 100644 --- a/tom_alertstreams/tables.py +++ b/tom_alertstreams/tables.py @@ -113,8 +113,19 @@ def _get_stream_name_choices() -> list[tuple[str, str]]: effect without restarting the process. Returns a list of (value, label) tuples using each stream's STREAM_NAME. Streams that fail to import are silently skipped so a misconfigured entry doesn't break the filter form. + + Deduped by STREAM_NAME: a broker may be configured as several ALERT_STREAMS + entries that share a name (Pitt-Google runs one entry per Pub/Sub topic, all + 'pittgoogle'), and the dropdown should show one choice per stream. """ - return [(klass.STREAM_NAME, klass.STREAM_NAME) for klass in get_alert_stream_classes()] + seen: set[str] = set() + choices: list[tuple[str, str]] = [] + for klass in get_alert_stream_classes(): + if klass.STREAM_NAME in seen: + continue + seen.add(klass.STREAM_NAME) + choices.append((klass.STREAM_NAME, klass.STREAM_NAME)) + return choices class AlertFilterSet(HTMXTableFilterSet): From 51881c32ef907351da6c21ba7e35dd0a2aa6bb61 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 11 Jun 2026 15:36:15 -0700 Subject: [PATCH 30/33] implement AlertFilterSet with cascading ChoiceFields in addition to General search, populate a topics drop-down according to the stream that is selected in the stream drop-down. --- tom_alertstreams/tables.py | 58 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/tom_alertstreams/tables.py b/tom_alertstreams/tables.py index 779ee92..cf0d529 100644 --- a/tom_alertstreams/tables.py +++ b/tom_alertstreams/tables.py @@ -4,9 +4,11 @@ import urllib.parse from typing import Any, ClassVar +from crispy_forms.layout import Column, Layout, Row import django_filters import django_tables2 as tables from django import forms +from django.db.models import Q from django.utils.html import format_html from tom_alertstreams.alertstreams.alertstream import get_alert_stream_classes @@ -133,8 +135,13 @@ class AlertFilterSet(HTMXTableFilterSet): Provides a 'query' full-text search (inherited from HTMXTableFilterSet) plus the fields defined here, which appear in the Advanced> expansion of the form. + + Implements the HTMX "cascading select" pattern: the topic dropdown choices + are scoped to the currently selected stream. See __init__ for the dynamic + choice logic and recent_alerts.html for the HTMX trigger that refreshes + the topic + innerHTML. When no stream is selected, returns topics from all streams. + """ + stream_name = request.GET.get('stream_name', '') + qs = Alert.objects.all() + if stream_name: + qs = qs.filter(stream_name=stream_name) + topics = qs.values_list('topic', flat=True).distinct().order_by('topic') + + # Build '] + options.extend(f'' for topic in topics) + return HttpResponse('\n'.join(options)) diff --git a/typings/babamul/__init__.pyi b/typings/babamul/__init__.pyi new file mode 100644 index 0000000..6b6ea3b --- /dev/null +++ b/typings/babamul/__init__.pyi @@ -0,0 +1,9 @@ +"""Type stubs for babamul. + +Babamul does not ship a py.typed marker, so Pylance/Pyright cannot resolve +its types. These minimal stubs cover only what tom_alertstreams imports. +Remove this directory when babamul adds py.typed to its distribution. +""" + +from .consumer import AlertConsumer as AlertConsumer +from .models import LsstAlert as LsstAlert, ZtfAlert as ZtfAlert diff --git a/typings/babamul/consumer.pyi b/typings/babamul/consumer.pyi new file mode 100644 index 0000000..500ce9a --- /dev/null +++ b/typings/babamul/consumer.pyi @@ -0,0 +1,28 @@ +"""Type stub for babamul.consumer.""" + +from collections.abc import Iterator +from typing import Any + +from .models import LsstAlert, ZtfAlert + +class AlertConsumer: + def __init__( + self, + topics: str | list[str] = ..., + username: str | None = ..., + password: str | None = ..., + server: str | None = ..., + group_id: str | None = ..., + offset: str = ..., + timeout: float | None = ..., + auto_commit: bool = ..., + as_raw: bool = ..., + ) -> None: ... + def __iter__(self) -> Iterator[ZtfAlert | LsstAlert | dict[str, Any]]: ... + def __enter__(self) -> AlertConsumer: ... + def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: object) -> None: ... + def close(self) -> None: ... + @property + def topics(self) -> list[str]: ... + @property + def group_id(self) -> str: ... diff --git a/typings/babamul/models.pyi b/typings/babamul/models.pyi new file mode 100644 index 0000000..8af70e3 --- /dev/null +++ b/typings/babamul/models.pyi @@ -0,0 +1,36 @@ +"""Type stub for babamul.models — covers only what tom_alertstreams uses.""" + +from datetime import datetime + +from pydantic import BaseModel + +class ZtfCandidate(BaseModel): + jd: float + ra: float + dec: float + magpsf: float + sigmapsf: float + @property + def datetime(self) -> datetime: ... + +class LsstCandidate(BaseModel): + ra: float + dec: float + magpsf: float + sigmapsf: float + jd: float + objectId: str + @property + def datetime(self) -> datetime: ... + +class ZtfAlert(BaseModel): + candid: int + objectId: str + candidate: ZtfCandidate + topic: str | None + +class LsstAlert(BaseModel): + candid: int + objectId: str + candidate: LsstCandidate + topic: str | None