Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/dfe_engine/appmgmt/routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,9 @@ def _fetcher(
"topic": fetcher.landing_label(source.source),
}
stanza.update(fetcher.config)
if fetcher.source_type in app.keyed_source_types:
# One source is one topic and one table, and the engine creates only the landing topic and table.
stanza["topic_per_dump_unit"] = False
# Checked on the flat stanza, the same shape the save path checks.
require_one_type(source.source, {fetcher.source_type: stanza}, settings)
polls: dict[str, Any] = {fetcher.source_type: stanza}
Expand Down
5 changes: 3 additions & 2 deletions src/dfe_engine/source/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -1013,7 +1013,7 @@ def manifest_replication_factor() -> int:
# shadow every rule after it.

# Keys the engine sets on the compiled fetcher stanza. A source may not carry them.
_FETCHER_ENGINE_KEYS = frozenset({"enabled", "topic"})
_FETCHER_ENGINE_KEYS = frozenset({"enabled", "topic", "topic_per_dump_unit"})

# Contact details the logger masks are personal data, not secrets, so a source may commit them.
_PERSONAL_WORDS = frozenset({"email", "phone"})
Expand Down Expand Up @@ -1105,7 +1105,8 @@ class SourceFetcher(BaseModel):
under the source's name for a family the app manifest declares keyed. The
engine owns ``enabled`` and ``topic`` on that stanza: ``topic`` selects
whether records land on the source's own topic (and table) or on the shared
``main`` landing.
``main`` landing. It also sets ``topic_per_dump_unit`` false on a keyed family's
stanza, so a dump unit lands on that topic rather than on ``<topic>-<unit>``.
"""

source_type: str = Field(
Expand Down
5 changes: 4 additions & 1 deletion tests/unit/test_appmgmt/test_routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -889,6 +889,7 @@ def test_a_rest_source_nests_its_stanza_under_its_name(self, settings):
"runzero-assets": {
"enabled": True,
"topic": "runzero-assets",
"topic_per_dump_unit": False,
"profile": "runzero",
}
}
Expand All @@ -904,7 +905,9 @@ def test_the_other_keyed_families_nest_the_same_way(self, settings, family):
catalogue.descriptor(FETCHER), registry, settings, instance="exports"
)

assert compiled["sources"] == {family: {"exports": {"enabled": True, "topic": "exports"}}}
assert compiled["sources"] == {
family: {"exports": {"enabled": True, "topic": "exports", "topic_per_dump_unit": False}}
}

@pytest.mark.parametrize("family", FLAT_FETCHER_FAMILIES)
def test_every_flat_family_composes_onto_the_family_itself(self, settings, family):
Expand Down
7 changes: 4 additions & 3 deletions tests/unit/test_source/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -429,9 +429,10 @@ def test_unknown_source_type_is_refused(self):
with pytest.raises(ValueError, match="Unknown fetcher source_type"):
SourceFetcher(source_type="http_json")

def test_engine_owned_keys_are_refused(self):
with pytest.raises(ValueError, match="may not set"):
SourceFetcher(source_type="okta", config={"enabled": True, "topic": "x"})
@pytest.mark.parametrize("key", ["enabled", "topic", "topic_per_dump_unit"])
def test_engine_owned_keys_are_refused(self, key):
with pytest.raises(ValueError, match=f"may not set {key}"):
SourceFetcher(source_type="okta", config={key: True})

@pytest.mark.parametrize(
"config",
Expand Down
Loading