From e400cdf5e2725cc7a47236ec5d98b84d27e9bc6f Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 16:43:37 +1100 Subject: [PATCH] fix(appmgmt): send dump units to the source's topic dfe-fetcher lands a dump unit on `-`. The engine creates only the topic and table ``. Dump unit rows went to the default table. The compiled config for a keyed fetcher family (rest, db, file) sets `topic_per_dump_unit` to false. Every unit goes to the source's topic. A source's fetcher config cannot set the key. --- src/dfe_engine/appmgmt/routing.py | 3 +++ src/dfe_engine/source/models.py | 5 +++-- tests/unit/test_appmgmt/test_routing.py | 5 ++++- tests/unit/test_source/test_models.py | 7 ++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/dfe_engine/appmgmt/routing.py b/src/dfe_engine/appmgmt/routing.py index 4bdf8a03..e6456d58 100644 --- a/src/dfe_engine/appmgmt/routing.py +++ b/src/dfe_engine/appmgmt/routing.py @@ -364,6 +364,9 @@ def _fetcher( "topic": fetcher.landing_label(source.source), } stanza.update(fetcher.config) + if fetcher.source_type in app.keyed_source_types: + # One source is one topic and one table, and the engine creates only the landing topic and table. + stanza["topic_per_dump_unit"] = False # Checked on the flat stanza, the same shape the save path checks. require_one_type(source.source, {fetcher.source_type: stanza}, settings) polls: dict[str, Any] = {fetcher.source_type: stanza} diff --git a/src/dfe_engine/source/models.py b/src/dfe_engine/source/models.py index ce0bec67..98a1df92 100644 --- a/src/dfe_engine/source/models.py +++ b/src/dfe_engine/source/models.py @@ -1013,7 +1013,7 @@ def manifest_replication_factor() -> int: # shadow every rule after it. # Keys the engine sets on the compiled fetcher stanza. A source may not carry them. -_FETCHER_ENGINE_KEYS = frozenset({"enabled", "topic"}) +_FETCHER_ENGINE_KEYS = frozenset({"enabled", "topic", "topic_per_dump_unit"}) # Contact details the logger masks are personal data, not secrets, so a source may commit them. _PERSONAL_WORDS = frozenset({"email", "phone"}) @@ -1105,7 +1105,8 @@ class SourceFetcher(BaseModel): under the source's name for a family the app manifest declares keyed. The engine owns ``enabled`` and ``topic`` on that stanza: ``topic`` selects whether records land on the source's own topic (and table) or on the shared - ``main`` landing. + ``main`` landing. It also sets ``topic_per_dump_unit`` false on a keyed family's + stanza, so a dump unit lands on that topic rather than on ``-``. """ source_type: str = Field( diff --git a/tests/unit/test_appmgmt/test_routing.py b/tests/unit/test_appmgmt/test_routing.py index e61e51f1..14a5dcc9 100644 --- a/tests/unit/test_appmgmt/test_routing.py +++ b/tests/unit/test_appmgmt/test_routing.py @@ -889,6 +889,7 @@ def test_a_rest_source_nests_its_stanza_under_its_name(self, settings): "runzero-assets": { "enabled": True, "topic": "runzero-assets", + "topic_per_dump_unit": False, "profile": "runzero", } } @@ -904,7 +905,9 @@ def test_the_other_keyed_families_nest_the_same_way(self, settings, family): catalogue.descriptor(FETCHER), registry, settings, instance="exports" ) - assert compiled["sources"] == {family: {"exports": {"enabled": True, "topic": "exports"}}} + assert compiled["sources"] == { + family: {"exports": {"enabled": True, "topic": "exports", "topic_per_dump_unit": False}} + } @pytest.mark.parametrize("family", FLAT_FETCHER_FAMILIES) def test_every_flat_family_composes_onto_the_family_itself(self, settings, family): diff --git a/tests/unit/test_source/test_models.py b/tests/unit/test_source/test_models.py index 82f210ed..14ab2a7a 100644 --- a/tests/unit/test_source/test_models.py +++ b/tests/unit/test_source/test_models.py @@ -429,9 +429,10 @@ def test_unknown_source_type_is_refused(self): with pytest.raises(ValueError, match="Unknown fetcher source_type"): SourceFetcher(source_type="http_json") - def test_engine_owned_keys_are_refused(self): - with pytest.raises(ValueError, match="may not set"): - SourceFetcher(source_type="okta", config={"enabled": True, "topic": "x"}) + @pytest.mark.parametrize("key", ["enabled", "topic", "topic_per_dump_unit"]) + def test_engine_owned_keys_are_refused(self, key): + with pytest.raises(ValueError, match=f"may not set {key}"): + SourceFetcher(source_type="okta", config={key: True}) @pytest.mark.parametrize( "config",