From cef0ea9f5b1fff0729de202320ac4acd68d164a9 Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 13:51:34 -0400 Subject: [PATCH 01/16] add a flag for sources which are monitored as a proxy for forced photometry. this allows for None flux sources to be included in forced phot (if minimum flux is none) --- .../a1b2c3d4e5f6_add_monitored_field.py | 35 ++++++++++ socat/client/core.py | 12 +++- socat/client/db.py | 10 ++- socat/client/mock.py | 33 ++++++--- socat/database/sources.py | 10 +++ socat/database/statements.py | 19 +++-- tests/test_db_client.py | 29 +++++++- tests/test_mock.py | 69 ++++++++++++------- 8 files changed, 170 insertions(+), 47 deletions(-) create mode 100644 socat/alembic/versions/a1b2c3d4e5f6_add_monitored_field.py diff --git a/socat/alembic/versions/a1b2c3d4e5f6_add_monitored_field.py b/socat/alembic/versions/a1b2c3d4e5f6_add_monitored_field.py new file mode 100644 index 0000000..5f25e43 --- /dev/null +++ b/socat/alembic/versions/a1b2c3d4e5f6_add_monitored_field.py @@ -0,0 +1,35 @@ +"""Add monitored field to fixed_sources and solarsystem_objects + +Revision ID: a1b2c3d4e5f6 +Revises: 35a6a33e0a34 +Create Date: 2026-06-01 00:00:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "a1b2c3d4e5f6" +down_revision: str | None = "35a6a33e0a34" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column( + "fixed_sources", + sa.Column("monitored", sa.Boolean, nullable=True), + ) + op.add_column( + "solarsystem_objects", + sa.Column("monitored", sa.Boolean, nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("fixed_sources", "monitored") + op.drop_column("solarsystem_objects", "monitored") diff --git a/socat/client/core.py b/socat/client/core.py index cfed88e..2782586 100644 --- a/socat/client/core.py +++ b/socat/client/core.py @@ -22,7 +22,12 @@ class ClientBase(ABC): @abstractmethod def create_source( - self, *, position: ICRS, name: str | None = None, flux: Quantity | None = None + self, + *, + position: ICRS, + name: str | None = None, + flux: Quantity | None = None, + monitored: bool = False, ) -> RegisteredFixedSource: """ Create a new source in the catlaog. @@ -59,10 +64,11 @@ def get_source(self, *, source_id: int) -> RegisteredFixedSource | None: @abstractmethod def get_forced_photometry_sources( - self, *, minimum_flux: Quantity + self, *, minimum_flux: Quantity | None = None ) -> list[RegisteredFixedSource]: """ - Get all sources that are used for forced photometry based on a minimum flux. + Get all sources that are monitored for forced photometry, optionally + filtered to those above a minimum flux. """ return [] # pragma: no cover diff --git a/socat/client/db.py b/socat/client/db.py index 06fdfc4..f112e81 100644 --- a/socat/client/db.py +++ b/socat/client/db.py @@ -83,7 +83,12 @@ def ephem(self) -> EphemClientBase: return self._ephem def create_source( - self, *, position: ICRS, name: str | None = None, flux: Quantity | None = None + self, + *, + position: ICRS, + name: str | None = None, + flux: Quantity | None = None, + monitored: bool = False, ) -> RegisteredFixedSource: if flux is not None: flux = flux.to_value("mJy") @@ -93,6 +98,7 @@ def create_source( dec_deg=position.dec.to_value("deg"), name=name, flux_mJy=flux, + monitored=monitored, ) with self._get_session() as session: session.add(source) @@ -129,7 +135,7 @@ def get_source(self, *, source_id: int) -> RegisteredFixedSource | None: return source.to_model() def get_forced_photometry_sources( - self, *, minimum_flux: Quantity + self, *, minimum_flux: Quantity | None = None ) -> list[RegisteredFixedSource]: with self._get_session() as session: sources = session.execute( diff --git a/socat/client/mock.py b/socat/client/mock.py index f4962f4..29fbdae 100644 --- a/socat/client/mock.py +++ b/socat/client/mock.py @@ -82,7 +82,12 @@ def ephem(self) -> EphemClientBase: return self._ephem def create_source( - self, *, position: ICRS, name: str | None = None, flux: Quantity | None = None + self, + *, + position: ICRS, + name: str | None = None, + flux: Quantity | None = None, + monitored: bool = False, ) -> RegisteredFixedSource: """ Create a new source and add it to the catalog. @@ -95,6 +100,8 @@ def create_source( Flux of source. name : str | None, Default: None Name of source + monitored : bool, Default: False + Whether this source is monitored by forced_photometry Returns ------- @@ -108,6 +115,7 @@ def create_source( position=position, flux=flux, name=name, + monitored=monitored, ) self.catalog[self.n] = source self.n += 1 @@ -228,25 +236,28 @@ def get_source(self, *, source_id: int) -> RegisteredFixedSource | None: return self.catalog.get(source_id, None) def get_forced_photometry_sources( - self, *, minimum_flux: Quantity + self, *, minimum_flux: Quantity | None = None ) -> list[RegisteredFixedSource]: """ - Get all sources that are used for forced photometry based on a minimum flux. + Get all monitored sources, optionally filtered to those above a minimum flux. Parameters ---------- - minimum_flux : Quantity - Minimum flux for source to be included + minimum_flux : Quantity | None + If provided, additionally filter to sources with flux >= minimum_flux. Returns ------- - filter : iterable[RegisteredFixedSource] - List of sources with flux greater than minimum_flux + list[RegisteredFixedSource] + List of monitored sources, filtered by flux if minimum_flux is given. """ - return filter( - lambda x: x.flux is not None and x.flux >= minimum_flux, - self.catalog.values(), - ) + sources = filter(lambda x: x.monitored, self.catalog.values()) + if minimum_flux is not None: + sources = filter( + lambda x: x.flux is not None and x.flux >= minimum_flux, + sources, + ) + return list(sources) def update_source( self, diff --git a/socat/database/sources.py b/socat/database/sources.py index 443866d..37306d3 100644 --- a/socat/database/sources.py +++ b/socat/database/sources.py @@ -34,10 +34,13 @@ class RegisteredFixedSource(RegisteredSource): Unique source identifier. Internal to SO name : str | None Name of source. Optional + monitored : bool + Whether this source is monitored by forced_photometry """ source_id: int | None = None name: str | None # Not a foreign key + monitored: bool = False class RegisteredMovingSource(RegisteredSource): @@ -84,11 +87,14 @@ class SolarSystemObject(BaseModel): Minor Planet Center ID of ephem. name : str Name of source + monitored : bool + Whether this source is monitored by forced_photometry """ sso_id: int MPC_id: int | None name: str + monitored: bool = False class RegisteredFixedSourceTable(SQLModel, table=True): @@ -110,6 +116,7 @@ class RegisteredFixedSourceTable(SQLModel, table=True): dec_deg: float = Field(nullable=False) flux_mJy: float | None = Field(nullable=True) name: str = Field(index=True, nullable=True) + monitored: bool = Field(default=False, nullable=True) def to_model(self) -> RegisteredFixedSource: """ @@ -129,6 +136,7 @@ def to_model(self) -> RegisteredFixedSource: position=ICRS(ra=self.ra_deg * u.deg, dec=self.dec_deg * u.deg), flux=flux, name=self.name, + monitored=self.monitored, ) @@ -144,6 +152,7 @@ class SolarSystemObjectTable(SolarSystemObject, SQLModel, table=True): sso_id: int = Field(primary_key=True) MPC_id: int | None = Field(index=True, nullable=True, unique=True) name: str = Field(index=True, nullable=False, unique=True) + monitored: bool = Field(default=False, nullable=True) def to_model(self) -> SolarSystemObject: """ @@ -158,6 +167,7 @@ def to_model(self) -> SolarSystemObject: sso_id=self.sso_id, MPC_id=self.MPC_id, name=self.name, + monitored=self.monitored, ) diff --git a/socat/database/statements.py b/socat/database/statements.py index 5dfbe95..3b5567c 100644 --- a/socat/database/statements.py +++ b/socat/database/statements.py @@ -145,24 +145,29 @@ def get_box_sso( ) -def get_forced_photometry_sources(minimum_flux: Quantity) -> select: +def get_forced_photometry_sources(minimum_flux: Quantity | None = None) -> select: """ - Get sources for which to perform forced photometry, i.e. sources with flux - above a certain threshold. + Get sources for which to perform forced photometry, i.e. sources with + monitored=True, optionally filtered to those above a minimum flux. Parameters ---------- - minimum_flux : Quantity - Minimum flux of sources to return + minimum_flux : Quantity | None + If provided, additionally filter to sources with flux >= minimum_flux. Returns ------- select: Database statement. """ - return select(RegisteredFixedSourceTable).where( - RegisteredFixedSourceTable.flux_mJy >= minimum_flux.to_value("mJy") + stmt = select(RegisteredFixedSourceTable).where( + RegisteredFixedSourceTable.monitored == True # noqa: E712 ) + if minimum_flux is not None: + stmt = stmt.where( + RegisteredFixedSourceTable.flux_mJy >= minimum_flux.to_value("mJy") + ) + return stmt def update_source( diff --git a/tests/test_db_client.py b/tests/test_db_client.py index 5276c4a..38db29d 100644 --- a/tests/test_db_client.py +++ b/tests/test_db_client.py @@ -39,10 +39,37 @@ def test_fixed_source_crud_and_queries(db_client): assert source_1.source_id in partial_ids assert source_2.source_id not in partial_ids + # monitored, flux above threshold + source_mon_hi = client.create_source( + position=ICRS(5.0 * u.deg, 5.0 * u.deg), + name="db-mon-hi", + flux=21.0 * u.mJy, + monitored=True, + ) + # monitored, flux below threshold + source_mon_lo = client.create_source( + position=ICRS(6.0 * u.deg, 6.0 * u.deg), + name="db-mon-lo", + flux=1.0 * u.mJy, + monitored=True, + ) + # not monitored, flux above threshold (source_2 already covers this) + + # Without minimum_flux: all monitored sources returned + all_monitored = client.get_forced_photometry_sources() + all_monitored_ids = {src.source_id for src in all_monitored} + assert source_mon_hi.source_id in all_monitored_ids + assert source_mon_lo.source_id in all_monitored_ids + assert source_1.source_id not in all_monitored_ids + assert source_2.source_id not in all_monitored_ids + + # With minimum_flux: only monitored sources above threshold forced = client.get_forced_photometry_sources(minimum_flux=10.0 * u.mJy) forced_ids = {src.source_id for src in forced} + assert source_mon_hi.source_id in forced_ids + assert source_mon_lo.source_id not in forced_ids assert source_1.source_id not in forced_ids - assert source_2.source_id in forced_ids + assert source_2.source_id not in forced_ids updated = client.update_source( source_id=source_1.source_id, diff --git a/tests/test_mock.py b/tests/test_mock.py index b870f0c..38e95a8 100644 --- a/tests/test_mock.py +++ b/tests/test_mock.py @@ -92,30 +92,53 @@ def test_box(mock_client): def test_photometry(mock_client): - position1 = ICRS(1.0 * u.deg, 1.0 * u.deg) - flux1 = 1.0 * u.mJy - source1 = mock_client.create_source(position=position1, name="mySrc", flux=flux1) - id1 = source1.source_id - position2 = ICRS(2.0 * u.deg, 2.0 * u.deg) - flux2 = 21.0 * u.mJy - source2 = mock_client.create_source(position=position2, name="mySrc2", flux=flux2) - id2 = source2.source_id - position3 = ICRS(3.0 * u.deg, 3.0 * u.deg) - flux3 = None - source3 = mock_client.create_source(position=position3, name="mySrc3", flux=flux3) - id3 = source3.source_id - - sources = mock_client.get_forced_photometry_sources(minimum_flux=10.0 * u.mJy) - - id_list = [source.source_id for source in sources] - - assert id1 not in id_list - assert id2 in id_list - assert id3 not in id_list + # monitored, flux above threshold + s_mon_hi = mock_client.create_source( + position=ICRS(1.0 * u.deg, 1.0 * u.deg), + name="mon-hi", + flux=21.0 * u.mJy, + monitored=True, + ) + # monitored, flux below threshold + s_mon_lo = mock_client.create_source( + position=ICRS(2.0 * u.deg, 2.0 * u.deg), + name="mon-lo", + flux=1.0 * u.mJy, + monitored=True, + ) + # not monitored, flux above threshold + s_unmon_hi = mock_client.create_source( + position=ICRS(3.0 * u.deg, 3.0 * u.deg), + name="unmon-hi", + flux=21.0 * u.mJy, + monitored=False, + ) + # not monitored, no flux + s_unmon_none = mock_client.create_source( + position=ICRS(4.0 * u.deg, 4.0 * u.deg), + name="unmon-none", + flux=None, + monitored=False, + ) - mock_client.delete_source(source_id=id1) - mock_client.delete_source(source_id=id2) - mock_client.delete_source(source_id=id3) + # Without minimum_flux: all monitored sources returned + all_monitored = mock_client.get_forced_photometry_sources() + all_ids = [s.source_id for s in all_monitored] + assert s_mon_hi.source_id in all_ids + assert s_mon_lo.source_id in all_ids + assert s_unmon_hi.source_id not in all_ids + assert s_unmon_none.source_id not in all_ids + + # With minimum_flux: only monitored sources above threshold + flux_filtered = mock_client.get_forced_photometry_sources(minimum_flux=10.0 * u.mJy) + filtered_ids = [s.source_id for s in flux_filtered] + assert s_mon_hi.source_id in filtered_ids + assert s_mon_lo.source_id not in filtered_ids + assert s_unmon_hi.source_id not in filtered_ids + assert s_unmon_none.source_id not in filtered_ids + + for s in [s_mon_hi, s_mon_lo, s_unmon_hi, s_unmon_none]: + mock_client.delete_source(source_id=s.source_id) def test_add_and_remove_astroquery(mock_client): From df4a1dab1d48fa0873a515696b76d793d951c20b Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 14:12:27 -0400 Subject: [PATCH 02/16] monitor sources above some flux on ingestion, monitor all ssos --- socat/ingest/actfits.py | 15 ++++++++++++++- socat/ingest/jplparquet.py | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/socat/ingest/actfits.py b/socat/ingest/actfits.py index 1f95640..aaa7dbb 100644 --- a/socat/ingest/actfits.py +++ b/socat/ingest/actfits.py @@ -17,6 +17,7 @@ def ingest_fits_file( client: ClientBase, filename: Path, hdu: int = 1, + flux_threshold: u.Quantity = 100.0 * u.mJy, ) -> int: """ Ingest a FITS file into the provided SOCat client. @@ -48,6 +49,7 @@ def ingest_fits_file( ), flux=row["fluxJy"] * u.Jy, name=row["name"], + monitored=row["fluxJy"] * u.Jy >= flux_threshold, ) number_of_sources += 1 @@ -70,6 +72,13 @@ def main(): # pragma: no cover required=True, ) + parser.add_argument( + "--monitored-flux-threshold-mJy", + type=float, + help="Flux threshold above which sources are considered monitored", + default=100.0, + ) + parser.add_argument( "-o", "--output", @@ -95,7 +104,11 @@ def main(): # pragma: no cover else: output_path = None - number_of_sources = ingest_fits_file(client=client, filename=args.file) + number_of_sources = ingest_fits_file( + client=client, + filename=args.file, + flux_threshold=args.monitored_flux_threshold_mJy * u.mJy, + ) print(f"Ingested {number_of_sources} sources") diff --git a/socat/ingest/jplparquet.py b/socat/ingest/jplparquet.py index 879f9d5..608905d 100644 --- a/socat/ingest/jplparquet.py +++ b/socat/ingest/jplparquet.py @@ -98,6 +98,7 @@ def ingest_jpl_parquet_file( dec=float(row["dec_deg"]) * u.deg, ), flux=flux, + monitored=True, ## all ssos monitored ) number_of_ephems += 1 From 6a25e46c7ea1c5149054cdd8dd2075a920a67fc2 Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 14:16:15 -0400 Subject: [PATCH 03/16] fix downsample iterator --- socat/ingest/jplparquet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/socat/ingest/jplparquet.py b/socat/ingest/jplparquet.py index 608905d..5199bfc 100644 --- a/socat/ingest/jplparquet.py +++ b/socat/ingest/jplparquet.py @@ -78,7 +78,7 @@ def ingest_jpl_parquet_file( sso = client.sso.get_sso_MPC_id(MPC_id=mpc_id)[0] for _, row in tqdm( - rows.iterrows()[::downsample], + rows[::downsample].iterrows(), desc=f"Ingesting {designation}", total=len(rows[::downsample]), ): From 6c579035bc503df2c99e6cbade7ef44a997e945a Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 14:19:20 -0400 Subject: [PATCH 04/16] sso object has monitored, not ephem --- socat/ingest/jplparquet.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/socat/ingest/jplparquet.py b/socat/ingest/jplparquet.py index 5199bfc..2dd2839 100644 --- a/socat/ingest/jplparquet.py +++ b/socat/ingest/jplparquet.py @@ -73,7 +73,9 @@ def ingest_jpl_parquet_file( for designation, rows in data.groupby("designation", sort=False): mpc_id, name = _parse_designation(designation) - client.sso.create_sso(name=name, MPC_id=mpc_id) + client.sso.create_sso( + name=name, MPC_id=mpc_id, monitored=True + ) ## ASSUME ALL JPL OBJECTS ARE MONITORED FOR NOW number_of_ssos += 1 sso = client.sso.get_sso_MPC_id(MPC_id=mpc_id)[0] @@ -98,7 +100,6 @@ def ingest_jpl_parquet_file( dec=float(row["dec_deg"]) * u.deg, ), flux=flux, - monitored=True, ## all ssos monitored ) number_of_ephems += 1 From 0c9ed2f32730562c5ba3096b135687cae42bddea Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 15:06:18 -0400 Subject: [PATCH 05/16] add monitored to create_sso method --- socat/client/core.py | 4 +++- socat/client/db.py | 6 ++++-- socat/client/mock.py | 10 ++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/socat/client/core.py b/socat/client/core.py index 2782586..acb6eb4 100644 --- a/socat/client/core.py +++ b/socat/client/core.py @@ -225,7 +225,9 @@ def delete_ephem(self, *, ephem_id: int) -> None: class SolarSystemClientBase(ABC): @abstractmethod - def create_sso(self, *, name: str, MPC_id: int | None) -> SolarSystemObject: + def create_sso( + self, *, name: str, MPC_id: int | None, monitored: bool = False + ) -> SolarSystemObject: """ Create a new solar system source in the catalog. """ diff --git a/socat/client/db.py b/socat/client/db.py index f112e81..54eede3 100644 --- a/socat/client/db.py +++ b/socat/client/db.py @@ -410,8 +410,10 @@ def __init__( session_factory=session_factory, ) - def create_sso(self, *, name: str, MPC_id: int | None) -> SolarSystemObject: - source = SolarSystemObjectTable(name=name, MPC_id=MPC_id) + def create_sso( + self, *, name: str, MPC_id: int | None, monitored: bool = False + ) -> SolarSystemObject: + source = SolarSystemObjectTable(name=name, MPC_id=MPC_id, monitored=monitored) with self._get_session() as session: session.add(source) diff --git a/socat/client/mock.py b/socat/client/mock.py index 29fbdae..06f11dd 100644 --- a/socat/client/mock.py +++ b/socat/client/mock.py @@ -698,7 +698,9 @@ def __init__(self): self.catalog = {} self.n = 0 - def create_sso(self, *, name: str, MPC_id: int | None) -> SolarSystemObject: + def create_sso( + self, *, name: str, MPC_id: int | None, monitored: bool = False + ) -> SolarSystemObject: """ Create a new solar system source. @@ -708,13 +710,17 @@ def create_sso(self, *, name: str, MPC_id: int | None) -> SolarSystemObject: Name of source MPC_id : int Minor Planet Center ID of source + monitored : bool, Default: False + Whether this source is monitored by forced_photometry Returns ------- solar_source : SolarSystemObject Solar system source that was added. """ - solar_source = SolarSystemObject(sso_id=self.n, name=name, MPC_id=MPC_id) + solar_source = SolarSystemObject( + sso_id=self.n, name=name, MPC_id=MPC_id, monitored=monitored + ) self.catalog[self.n] = solar_source self.n += 1 From 491882277ef6afc1d2a11c296413f29f2531f01b Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 15:32:54 -0400 Subject: [PATCH 06/16] allow monitored flag to be input in create_source and updated with update_source --- socat/client/core.py | 1 + socat/client/db.py | 2 ++ socat/client/mock.py | 4 ++++ socat/core/fixed_sources.py | 8 ++++++++ socat/database/statements.py | 4 ++++ 5 files changed, 19 insertions(+) diff --git a/socat/client/core.py b/socat/client/core.py index acb6eb4..1a25571 100644 --- a/socat/client/core.py +++ b/socat/client/core.py @@ -80,6 +80,7 @@ def update_source( position: ICRS | None = None, name: str | None = None, flux: Quantity | None = None, + monitored: bool | None = None, ) -> RegisteredFixedSource | None: """ Update a source. If the source is updated, return its new value. Else, return None. diff --git a/socat/client/db.py b/socat/client/db.py index 54eede3..36a068e 100644 --- a/socat/client/db.py +++ b/socat/client/db.py @@ -150,6 +150,7 @@ def update_source( position: ICRS | None = None, name: str | None = None, flux: Quantity | None = None, + monitored: bool | None = None, ) -> RegisteredFixedSource | None: with self._get_session() as session: session.execute( @@ -158,6 +159,7 @@ def update_source( position=position, name=name, flux=flux, + monitored=monitored, ) ) source = session.get(RegisteredFixedSourceTable, source_id) diff --git a/socat/client/mock.py b/socat/client/mock.py index 06f11dd..71f97df 100644 --- a/socat/client/mock.py +++ b/socat/client/mock.py @@ -266,6 +266,7 @@ def update_source( position: ICRS | None = None, name: str | None = None, flux: Quantity | None = None, + monitored: bool | None = None, ) -> RegisteredFixedSource | None: """ Update a source by id @@ -278,6 +279,8 @@ def update_source( Name of source flux : Quantity | None, Default: None Flux of source + monitored : bool | None, Default: None + Whether this source is monitored by forced_photometry Returns ------- @@ -294,6 +297,7 @@ def update_source( position=current.position if position is None else position, name=current.name if name is None else name, flux=current.flux if flux is None else flux, + monitored=current.monitored if monitored is None else monitored, ) self.catalog[source_id] = new diff --git a/socat/core/fixed_sources.py b/socat/core/fixed_sources.py index c93d591..ab8d7ce 100644 --- a/socat/core/fixed_sources.py +++ b/socat/core/fixed_sources.py @@ -14,6 +14,7 @@ async def create_source( session: AsyncSession, name: str | None = None, flux: Quantity | None = None, + monitored: bool = False, ) -> RegisteredFixedSource: """ Create a new source in the database. @@ -26,6 +27,8 @@ async def create_source( Flux of source. Optional. name : str | None Name of source. Optional. + monitored : bool + Whether this source is monitored by forced_photometry. Default False. session : AsyncSession Asynchronous session to use @@ -42,6 +45,7 @@ async def create_source( dec_deg=position.dec.to_value("deg"), name=name, flux_mJy=flux, + monitored=monitored, ) async with session.begin(): @@ -118,6 +122,7 @@ async def update_source( session: AsyncSession, flux: Quantity | None = None, name: str | None = None, + monitored: bool | None = None, ) -> RegisteredFixedSource: """ Update a source in the database. @@ -132,6 +137,8 @@ async def update_source( Asynchronous session to use name : str | None Name of source + monitored : bool | None + Whether this source is monitored by forced_photometry. Optional. Returns ------- @@ -151,6 +158,7 @@ async def update_source( position=position, flux=flux, name=name, + monitored=monitored, ) ) diff --git a/socat/database/statements.py b/socat/database/statements.py index 3b5567c..feba558 100644 --- a/socat/database/statements.py +++ b/socat/database/statements.py @@ -175,6 +175,7 @@ def update_source( position: ICRS | None = None, flux: Quantity | None = None, name: str | None = None, + monitored: bool | None = None, ) -> update: """ Generate an update statement for a source. @@ -189,6 +190,8 @@ def update_source( Flux of source. Optional. name : str | None Name of source. Optional. + monitored : bool | None + Whether this source is monitored by forced_photometry. Optional. Returns ------- @@ -211,6 +214,7 @@ def update_source( "dec_deg": position.dec.to_value("deg") if position is not None else None, "flux_mJy": flux.to_value("mJy") if flux is not None else None, "name": name, + "monitored": monitored, }.items() if v is not None } From a772e11c8abf74b7a010b20203e2b91f1257e0d3 Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 16:26:37 -0400 Subject: [PATCH 07/16] fix db SourceGenerator in case that flux is none --- socat/client/db.py | 49 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/socat/client/db.py b/socat/client/db.py index 36a068e..0a87872 100644 --- a/socat/client/db.py +++ b/socat/client/db.py @@ -584,11 +584,19 @@ def init_interp(self, *, ephem_cat: EphemClient) -> None: if type(self.source) is RegisteredFixedSource: self.ra_unit = self.source.position.ra.unit self.dec_unit = self.source.position.dec.unit - self.flux_unit = self.source.flux.unit + self.do_flux = self.source.flux is not None + self.flux_unit = self.source.flux.unit if self.do_flux else None self.interp = lambda _: ( - self.source.position.ra.value, - self.source.position.dec.value, - self.source.flux.value, + ( + self.source.position.ra.value, + self.source.position.dec.value, + self.source.flux.value, + ) + if self.do_flux + else ( + self.source.position.ra.value, + self.source.position.dec.value, + ) ) elif type(self.source) is SolarSystemObject: @@ -596,18 +604,33 @@ def init_interp(self, *, ephem_cat: EphemClient) -> None: sso_id=self.source.sso_id, t_min=self.t_min, t_max=self.t_max ) x = np.zeros(len(ephem_points)) - y = np.zeros((len(ephem_points), 3)) + + self.do_flux = True + for ephem in ephem_points: + if ephem.flux is None: + self.do_flux = False + break + + if self.do_flux: + y = np.zeros((len(ephem_points), 3)) + else: + y = np.zeros((len(ephem_points), 2)) + for i, ephem in enumerate(ephem_points): x[i] = ephem.time.unix y[i] = ( - ephem.position.ra.value, - ephem.position.dec.value, - ephem.flux.value, + ( + ephem.position.ra.value, + ephem.position.dec.value, + ephem.flux.value, + ) + if self.do_flux + else (ephem.position.ra.value, ephem.position.dec.value) ) self.ra_unit = ephem.position.ra.unit self.dec_unit = ephem.position.dec.unit - self.flux_unit = ephem.flux.unit + self.flux_unit = ephem.flux.unit if self.do_flux else None self.interp = make_interp_spline(x, y, k=1) def at_time(self, *, t: Time) -> tuple[ICRS, Quantity]: @@ -641,9 +664,13 @@ def at_time(self, *, t: Time) -> tuple[ICRS, Quantity]: if t < self.t_min or t > self.t_max: raise ValueError("Time out of range for source generator") - ra_deg, dec_deg, flux_mJy = self.interp(t.unix) + if self.do_flux: + ra_deg, dec_deg, flux_mJy = self.interp(t.unix) + flux = flux_mJy * self.flux_unit + else: + ra_deg, dec_deg = self.interp(t.unix) + flux = None position = ICRS(ra_deg * self.ra_unit, dec_deg * self.dec_unit) - flux = flux_mJy * self.flux_unit return (position, flux) From 870d23aeae3a15d7d1b70a8cfd24acae691d0fc8 Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 16:56:57 -0400 Subject: [PATCH 08/16] ruff --- socat/alembic/versions/35a6a33e0a34_initial_database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/socat/alembic/versions/35a6a33e0a34_initial_database.py b/socat/alembic/versions/35a6a33e0a34_initial_database.py index 28d8c7a..c62fb65 100644 --- a/socat/alembic/versions/35a6a33e0a34_initial_database.py +++ b/socat/alembic/versions/35a6a33e0a34_initial_database.py @@ -9,6 +9,7 @@ from collections.abc import Sequence import sqlalchemy as sa + from alembic import op # revision identifiers, used by Alembic. From c3ed87b9738d88e949c604ff3f962b88b6a3c80a Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 17:05:46 -0400 Subject: [PATCH 09/16] ruff fix --- socat/alembic/env.py | 2 +- tests/conftest.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/socat/alembic/env.py b/socat/alembic/env.py index 8d7000e..a12e8fc 100644 --- a/socat/alembic/env.py +++ b/socat/alembic/env.py @@ -1,8 +1,8 @@ from logging.config import fileConfig -from alembic import context from sqlalchemy import engine_from_config, pool +from alembic import context from socat.settings import settings # this is the Alembic Config object, which provides diff --git a/tests/conftest.py b/tests/conftest.py index 9456b88..bacc118 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,9 +11,10 @@ def run_migration(database_path: str): """ Run the migration on the database. """ - from alembic import command from alembic.config import Config + from alembic import command + alembic_cfg = Config("socat/alembic.ini") database_url = f"sqlite:///{database_path}" alembic_cfg.set_main_option("sqlalchemy.url", database_url) From 438e4eef6eedfb00c9368457dcfedcef584414a9 Mon Sep 17 00:00:00 2001 From: Allen Foster Date: Mon, 1 Jun 2026 17:12:33 -0400 Subject: [PATCH 10/16] fix typo Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 320206b..d8b3412 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Once created, you can add sources via the following example ingest scripts: `socat-jpl-parquet` was written to read in a parquet file containing JPL Horizons ephemerides for solar system objects. - These scripts store either FixedRegisteredSources, or SolarSystemObjects. + These scripts store either RegisteredFixedSources, or SolarSystemObjects. USING SOCAT From 42c66c08aeb0f1454769217494e0cd18ee94a71d Mon Sep 17 00:00:00 2001 From: Allen Foster Date: Mon, 1 Jun 2026 17:13:12 -0400 Subject: [PATCH 11/16] typo Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d8b3412..786632f 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ USING SOCAT Once sources are ingested into your db, you can use the socat to query sources within a box on the sky within some time range. -Make sure you environment variables point to the correct db, as above. +Make sure your environment variables point to the correct db, as above. ``` from socat.client.settings import SOCatClientSettings From 8b4c5140dcf438341e743ac2f4cc7967c166c9d7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Jun 2026 21:13:37 +0000 Subject: [PATCH 12/16] Fix monitored column: use server_default and backfill existing rows --- .../alembic/versions/a1b2c3d4e5f6_add_monitored_field.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/socat/alembic/versions/a1b2c3d4e5f6_add_monitored_field.py b/socat/alembic/versions/a1b2c3d4e5f6_add_monitored_field.py index 5f25e43..29e08e0 100644 --- a/socat/alembic/versions/a1b2c3d4e5f6_add_monitored_field.py +++ b/socat/alembic/versions/a1b2c3d4e5f6_add_monitored_field.py @@ -22,11 +22,15 @@ def upgrade() -> None: op.add_column( "fixed_sources", - sa.Column("monitored", sa.Boolean, nullable=True), + sa.Column("monitored", sa.Boolean, nullable=False, server_default=sa.false()), ) + op.execute("UPDATE fixed_sources SET monitored = false WHERE monitored IS NULL") op.add_column( "solarsystem_objects", - sa.Column("monitored", sa.Boolean, nullable=True), + sa.Column("monitored", sa.Boolean, nullable=False, server_default=sa.false()), + ) + op.execute( + "UPDATE solarsystem_objects SET monitored = false WHERE monitored IS NULL" ) From 297d1c256bbad420a5e6f72f2828ea10cf38001f Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 17:26:49 -0400 Subject: [PATCH 13/16] make sure nullable is false since its a boolean --- socat/database/sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/socat/database/sources.py b/socat/database/sources.py index 37306d3..1ffe2de 100644 --- a/socat/database/sources.py +++ b/socat/database/sources.py @@ -116,7 +116,7 @@ class RegisteredFixedSourceTable(SQLModel, table=True): dec_deg: float = Field(nullable=False) flux_mJy: float | None = Field(nullable=True) name: str = Field(index=True, nullable=True) - monitored: bool = Field(default=False, nullable=True) + monitored: bool = Field(default=False, nullable=False) def to_model(self) -> RegisteredFixedSource: """ @@ -152,7 +152,7 @@ class SolarSystemObjectTable(SolarSystemObject, SQLModel, table=True): sso_id: int = Field(primary_key=True) MPC_id: int | None = Field(index=True, nullable=True, unique=True) name: str = Field(index=True, nullable=False, unique=True) - monitored: bool = Field(default=False, nullable=True) + monitored: bool = Field(default=False, nullable=False) def to_model(self) -> SolarSystemObject: """ From 2dd6ae744ff21fcef161e81e6562fd072ca19d4c Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 17:32:57 -0400 Subject: [PATCH 14/16] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 786632f..a69b233 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ SOCAT The Source CATalog. -Contains information about sources that are being monitored by an obseratory, +Contains information about sources that are being monitored by an observatory, or that are being used to crossmatch detected sources with. From 2c6fb483b2344e3d003114befc9c9411f8013411 Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Mon, 1 Jun 2026 17:54:16 -0400 Subject: [PATCH 15/16] update doc for act ingest --- socat/ingest/actfits.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/socat/ingest/actfits.py b/socat/ingest/actfits.py index aaa7dbb..1857612 100644 --- a/socat/ingest/actfits.py +++ b/socat/ingest/actfits.py @@ -30,6 +30,8 @@ def ingest_fits_file( Path to the ACT-compatible FITS point source file to load. hdu: int = 1 The HDU in the file that corresponds to the sources table. + flux_threshold: u.Quantity = 100.0 * u.mJy + The flux threshold above which sources are considered for monitoring. Returns ------- From 1a36cafdc5557389869fce7613a326a27c298bdc Mon Sep 17 00:00:00 2001 From: "Allen M. Foster" Date: Tue, 2 Jun 2026 11:57:47 -0400 Subject: [PATCH 16/16] remove redundant test --- tests/test_generator.py | 73 ----------------------------------------- 1 file changed, 73 deletions(-) diff --git a/tests/test_generator.py b/tests/test_generator.py index 1ef7373..a84063a 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -80,79 +80,6 @@ async def test_gen(database_async_sessionmaker): await core.delete_source(source.source_id, session=session) -@pytest.mark.asyncio -async def test_gen_no_flux(database_async_sessionmaker): - t_min = Time("2025-02-01T00:00:00.00") - t_max = t_min + 1100 * u.s - async with database_async_sessionmaker() as session: - sso = await core.create_sso(name="Davida", MPC_id=511, session=session) - - position = ICRS(1 * u.deg, 1 * u.deg) - source = await core.create_source( - position, - session=session, - name="mySrc", - flux=None, - ) - - for i in range(10): - position = ICRS(i * u.deg, 1.5 * i * u.deg) - flux = None - time = t_min + (100 * i) * u.s - await core.create_ephem( - sso_id=sso.sso_id, - MPC_id=511, - name="Davida", - time=time, - position=position, - flux=flux, - session=session, - ) - - ephem_list = await core.get_ephem_points(sso, t_min, t_max, session=session) - - for ephem in ephem_list: - assert t_min <= ephem.time - assert ephem.time <= t_max - assert len(ephem_list) == 10 - - async with database_async_sessionmaker() as session: - gen = generator.SourceGenerator( - source, Time("2025-01-01T00:00:00.00"), Time("2026-01-01T00:00:00.00") - ) - await gen.init_interp(session=session) - - position, flux = gen.at_time(t=Time("2025-06-01T00:00:00.00")) - - assert position.ra.value == 1 - assert position.dec.value == 1 - assert flux is None - - async with database_async_sessionmaker() as session: - gen = generator.SourceGenerator(sso, t_min=t_min, t_max=t_max) - await gen.init_interp(session=session) - - position, flux = gen.at_time(Time("2025-02-01 00:04:10.000000")) - - assert position.ra.value == 2.5 - assert position.dec.value == 3.75 - assert flux is None - - # Check asking out of bounds doesn't work - with pytest.raises(ValueError): - gen.at_time(t_max + 100 * u.s) - - # Check not initializing interp raises error - with pytest.raises(RuntimeError): - async with database_async_sessionmaker() as session: - gen = generator.SourceGenerator(sso, t_min=t_min, t_max=t_max) - gen.at_time(Time("2025-02-01 00:04:10.000000")) - - async with database_async_sessionmaker() as session: - await core.delete_sso(sso.sso_id, session=session) - await core.delete_source(source.source_id, session=session) - - @pytest.mark.asyncio async def test_get_box(database_async_sessionmaker): t_min = Time("2025-02-01T00:00:00.00")