From adb8d878b5aec1185369a53bd4ef34f0fcedf69e Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 15:34:27 +1100 Subject: [PATCH 1/6] fix(hunts): add autodisable settings, counter and API fields The runner stops hunts that keep hitting a ClickHouse resource limit. The API shows the auto-disable and lets an operator clear it. - hunts.autodisable settings: enabled (on), failures (3) and window_seconds (3600), each settable through DFE_ENGINE_HUNTS_AUTODISABLE_* environment variables - hunt_autodisabled_total counter, labelled by hunt_id and reason - RunStatus reads auto_disabled_at and auto_disabled_reason from hunt_state. - clear_auto_disable writes a row to replace the disabled row. - Hunt create, update, list and detail have enabled. - The list adds auto_disabled_at and auto_disabled_reason. - Saving a hunt with enabled true tries to clear the auto-disable. The save succeeds if ClickHouse is unreachable. - The hunt list reads enabled from the stored YAML --- openapi/openapi.json | 42 ++++++ src/dfe_engine/api/v1/hunts.py | 53 +++++++- src/dfe_engine/hunts/hunt_config_registry.py | 1 + src/dfe_engine/hunts/run_status.py | 45 ++++++- src/dfe_engine/hunts/runner_metrics.py | 13 ++ src/dfe_engine/settings.py | 37 ++++++ .../test_api/test_hunts_run_visibility.py | 123 ++++++++++++++++++ tests/unit/test_hunts/test_metrics.py | 15 ++- tests/unit/test_hunts/test_run_status.py | 94 ++++++++++++- tests/unit/test_settings.py | 13 ++ 10 files changed, 429 insertions(+), 7 deletions(-) diff --git a/openapi/openapi.json b/openapi/openapi.json index 05308c5d..3ecb4194 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -41369,6 +41369,12 @@ ], "title": "Explain Queries" }, + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Whether the runner fires this hunt. False keeps its YAML, state and watermark and fires nothing.", + "default": true + }, "rules": { "items": { "type": "string" @@ -41499,6 +41505,12 @@ ], "title": "Explain Queries" }, + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Whether the runner fires this hunt. False keeps its YAML, state and watermark and fires nothing.", + "default": true + }, "name": { "type": "string", "title": "Name", @@ -41758,6 +41770,30 @@ ], "title": "Next Due", "description": "Epoch seconds of the next scheduled fire; null = no rate schedule" + }, + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Whether the runner fires this hunt. False keeps its YAML, state and watermark and fires nothing.", + "default": true + }, + "auto_disabled_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Auto Disabled At", + "description": "When the runner stopped the hunt after repeated resource-limit failures, epoch seconds" + }, + "auto_disabled_reason": { + "type": "string", + "title": "Auto Disabled Reason", + "description": "The ClickHouse error category and code that tripped the auto-disable", + "default": "" } }, "type": "object", @@ -41879,6 +41915,12 @@ ], "title": "Explain Queries" }, + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Whether the runner fires this hunt. False keeps its YAML, state and watermark and fires nothing.", + "default": true + }, "rules": { "items": { "type": "string" diff --git a/src/dfe_engine/api/v1/hunts.py b/src/dfe_engine/api/v1/hunts.py index 91800407..d873be6c 100644 --- a/src/dfe_engine/api/v1/hunts.py +++ b/src/dfe_engine/api/v1/hunts.py @@ -50,7 +50,12 @@ default_display_name, resolve_display_name, ) -from dfe_engine.hunts.run_status import RunStatus, live_runner_count, read_run_status +from dfe_engine.hunts.run_status import ( + RunStatus, + clear_auto_disable, + live_runner_count, + read_run_status, +) from dfe_engine.hunts.schedule import next_due, read_schedule from dfe_engine.hunts.spec_loader import interval_seconds_for from dfe_engine.hunts.validator import HuntValidator @@ -185,6 +190,23 @@ class HuntSummary(BaseModel): default=None, description="Epoch seconds of the next scheduled fire; null = no rate schedule", ) + enabled: bool = Field( + default=True, + description=( + "Whether the runner fires this hunt. False keeps its YAML, state and watermark " + "and fires nothing." + ), + ) + auto_disabled_at: int | None = Field( + default=None, + description=( + "When the runner stopped the hunt after repeated resource-limit failures, epoch seconds" + ), + ) + auto_disabled_reason: str = Field( + default="", + description="The ClickHouse error category and code that tripped the auto-disable", + ) class HuntRuleEntry(BaseModel): @@ -221,6 +243,13 @@ class _HuntConfigFields(BaseModel): scheduling_mode: str | None = None min_interval_seconds: int | None = None explain_queries: bool | None = None + enabled: bool = Field( + default=True, + description=( + "Whether the runner fires this hunt. False keeps its YAML, state and watermark " + "and fires nothing." + ), + ) class HuntWriteRequest(_HuntConfigFields): @@ -383,6 +412,24 @@ def _run_status_for(hunt_names: list[str], now: int) -> dict[str, RunStatus]: return {} +def _clear_auto_disable_if_enabled(name: str, *, enabled: bool) -> None: + """Re-enable a hunt the runner auto-disabled when the save turns it on. + + Best-effort like the run state: the YAML is already saved, so an unreachable + ClickHouse leaves the stop in place for the next save rather than failing this one. + """ + if not enabled: + return + try: + run = _run_status_for([name], int(time.time())).get(name) + if run is None or run.auto_disabled_at is None: + return + client, db = _data_database_client() + clear_auto_disable(client, db, name) + except Exception as exc: + logger.warning(f"could not clear the auto-disable on hunt '{name}': {exc}") + + def _learned_offsets(enabled: bool) -> dict[str, tuple[int, int]]: """The offsets ``hunt_schedule`` holds, read once per listing when smoothing is on. @@ -441,6 +488,9 @@ def _hunt_row_to_summary( too_aggressive=run.too_aggressive, run_requested=run.run_requested, next_due=_next_due(row, now, fraction, learned), + enabled=bool(row.get("enabled", True)), + auto_disabled_at=run.auto_disabled_at, + auto_disabled_reason=run.auto_disabled_reason, ) @@ -611,6 +661,7 @@ def update_hunt( _validate_hunt_config(config, settings, rules) _resolve_customers(request, config) outcome = registry.save(name, config, created_by=user.user_id) + _clear_auto_disable_if_enabled(name, enabled=body.enabled) apply_review_headers(response, outcome) audit_resource_change(user.user_id, "hunt", name, "updated", review_audit_detail(outcome)) return HuntDetailResponse.from_stored_config(name, config) diff --git a/src/dfe_engine/hunts/hunt_config_registry.py b/src/dfe_engine/hunts/hunt_config_registry.py index f6ce92ae..b3cc71b6 100644 --- a/src/dfe_engine/hunts/hunt_config_registry.py +++ b/src/dfe_engine/hunts/hunt_config_registry.py @@ -394,6 +394,7 @@ def list_hunts(self) -> list[dict[str, Any]]: "rules": rule_names, "source_table": config.get("global_source_table_name", ""), "target_table": config.get("global_target_table_name", ""), + "enabled": config.get("enabled", True), } ) return results diff --git a/src/dfe_engine/hunts/run_status.py b/src/dfe_engine/hunts/run_status.py index 4278a507..1baab51a 100644 --- a/src/dfe_engine/hunts/run_status.py +++ b/src/dfe_engine/hunts/run_status.py @@ -48,6 +48,10 @@ class RunStatus: overrun_count: int = 0 run_requested: bool = False """An operator asked for a run the runner has not served yet.""" + auto_disabled_at: int | None = None + """Epoch seconds the runner stopped the hunt after repeated resource-limit failures.""" + auto_disabled_reason: str = "" + """The ClickHouse error category and code that tripped the auto-disable.""" def _query(database: str) -> str: @@ -56,14 +60,17 @@ def _query(database: str) -> str: return ( "SELECT h.hunt_id, coalesce(w.wm, 0), coalesce(l.lu, 0), " # noqa: S608 - quoted configured database, ids bound "coalesce(s.overruns, 0), coalesce(s.aggressive, 0), " - "coalesce(r.last_fire, 0), coalesce(r.rows_written, 0), coalesce(q.pending, 0) " + "coalesce(r.last_fire, 0), coalesce(r.rows_written, 0), coalesce(q.pending, 0), " + "toUnixTimestamp(s.disabled_at), coalesce(s.disabled_reason, '') " "FROM (SELECT arrayJoin({ids:Array(String)}) AS hunt_id) h " "LEFT JOIN (SELECT hunt_id, argMax(watermark, updated) AS wm " f"FROM {db}.hunt_watermark GROUP BY hunt_id) w USING (hunt_id) " "LEFT JOIN (SELECT hunt_id, argMax(lease_until, claimed) AS lu " f"FROM {db}.hunt_lease GROUP BY hunt_id) l USING (hunt_id) " "LEFT JOIN (SELECT hunt_id, argMax(overrun_count, updated) AS overruns, " - "argMax(too_aggressive, updated) AS aggressive " + "argMax(too_aggressive, updated) AS aggressive, " + "argMax(auto_disabled_at, updated) AS disabled_at, " + "argMax(auto_disabled_reason, updated) AS disabled_reason " f"FROM {db}.hunt_state GROUP BY hunt_id) s USING (hunt_id) " "LEFT JOIN (SELECT hunt_id, max(fire) AS last_fire, " "argMax(rows_written, fire) AS rows_written FROM (" @@ -124,5 +131,39 @@ def read_run_status(ch: Any, database: str, hunt_ids: list[str], now: int) -> di too_aggressive=bool(row[4]), overrun_count=int(row[3]), run_requested=bool(row[7]), + auto_disabled_at=int(row[8]) if row[8] else None, + auto_disabled_reason=str(row[9] or ""), ) return status + + +def clear_auto_disable(ch: Any, database: str, hunt_id: str) -> None: + """Insert the hunt_state row that replaces a disabled one. Only the API calls it. + + The overrun count and too-aggressive flag are read back and carried over, so + clearing the auto-disable leaves the UI's overrun signal as it was. + + Args: + ch: a ClickHouse client exposing ``query`` and ``insert``. + database: the data database the coordination tables live in. + hunt_id: the hunt to re-enable. + """ + rows = ch.query( + "SELECT argMax(overrun_count, updated), argMax(too_aggressive, updated) " # noqa: S608 - quoted configured database, id bound + f"FROM {quote_identifier(database)}.hunt_state WHERE hunt_id = {{h:String}}", + parameters={"h": hunt_id}, + ).result_rows + overrun_count = int(rows[0][0]) if rows else 0 + too_aggressive = int(rows[0][1]) if rows else 0 + ch.insert( + "hunt_state", + [[hunt_id, overrun_count, too_aggressive, None, ""]], + column_names=[ + "hunt_id", + "overrun_count", + "too_aggressive", + "auto_disabled_at", + "auto_disabled_reason", + ], + database=database, + ) diff --git a/src/dfe_engine/hunts/runner_metrics.py b/src/dfe_engine/hunts/runner_metrics.py index 38547053..44cdc5f2 100644 --- a/src/dfe_engine/hunts/runner_metrics.py +++ b/src/dfe_engine/hunts/runner_metrics.py @@ -61,6 +61,7 @@ PHASE_SHIFTS = "hunt_phase_shift_total" OBSERVED_OVERLAP_PEAK = "hunt_observed_overlap_peak" OBSERVED_MEM_PEAK = "hunt_observed_mem_peak_bytes" +AUTODISABLED = "hunt_autodisabled_total" class HuntRunnerMetrics: @@ -199,6 +200,11 @@ def __init__(self, manager: Any | None = None) -> None: "Largest summed peak memory of hunt queries finishing in one bucket of the " "trailing hour, read from query_log_archive", ) + self._autodisabled = manager.counter( + AUTODISABLED, + "Hunts the runner stopped after repeated resource-limit failures, by reason", + ["hunt_id", "reason"], + ) @property def enabled(self) -> bool: @@ -386,6 +392,12 @@ def observed(self, *, overlap_peak: int, mem_peak_bytes: float) -> None: self._observed_overlap.set(overlap_peak) self._observed_mem.set(mem_peak_bytes) + def auto_disabled(self, hunt_id: str, *, reason: str) -> None: + """Record a hunt the runner stopped after repeated resource-limit failures.""" + if self._manager is None: + return + self._autodisabled.labels(hunt_id=hunt_id, reason=reason).inc() + def create(app_name: str = SERVICE_NAME) -> HuntRunnerMetrics: """Build the instrument set on scalo's metrics backend. @@ -425,6 +437,7 @@ def _register_unprefixed_groups(manager: Any) -> None: __all__ = [ + "AUTODISABLED", "BACKLOG", "CLAIMS", "CLICKHOUSE_UNAVAILABLE", diff --git a/src/dfe_engine/settings.py b/src/dfe_engine/settings.py index 301604dc..4030f4b1 100644 --- a/src/dfe_engine/settings.py +++ b/src/dfe_engine/settings.py @@ -275,6 +275,36 @@ def effective_data_database(self) -> str: _CLICKHOUSE_CATALOGUES = frozenset({"system", "information_schema"}) +class HuntAutodisableSettings(BaseModel): + """A hunt that keeps tripping a ClickHouse resource limit is stopped by the runner. + + On by default, because a query that busts the memory limit fails its neighbours too. + """ + + enabled: bool = Field( + default=True, + description=( + "Stop a hunt after repeated resource-limit failures. " + "DFE_ENGINE_HUNTS_AUTODISABLE_ENABLED." + ), + ) + failures: int = Field( + default=3, + gt=0, + description=( + "Resource-limit failures within the window that disable the hunt. " + "DFE_ENGINE_HUNTS_AUTODISABLE_FAILURES." + ), + ) + window_seconds: int = Field( + default=3600, + gt=0, + description=( + "The window the failures are counted over. DFE_ENGINE_HUNTS_AUTODISABLE_WINDOW_SECONDS." + ), + ) + + class HuntSmoothingSettings(BaseModel): """The adaptive phase-drift loop that flattens ClickHouse load from rate hunts. @@ -603,6 +633,10 @@ class HuntsSettings(BaseModel): default_factory=HuntSmoothingSettings, description="Adaptive phase drift for rate hunts", ) + autodisable: HuntAutodisableSettings = Field( + default_factory=HuntAutodisableSettings, + description="Stop a hunt that keeps tripping a ClickHouse resource limit", + ) failure_backoff_initial_seconds: float = Field( default=FAILURE_BACKOFF_INITIAL_SECONDS, gt=0, @@ -2544,6 +2578,8 @@ def _strip_lower(value: str) -> str: "DFE_ENGINE_HUNTS_SMOOTHING_COOLDOWN_CYCLES": "hunts.smoothing.cooldown_cycles", "DFE_ENGINE_HUNTS_SMOOTHING_MEMORY_CAP_BYTES": "hunts.smoothing.memory_cap_bytes", "DFE_ENGINE_HUNTS_SMOOTHING_HORIZON_MAX_SECONDS": "hunts.smoothing.horizon_max_seconds", + "DFE_ENGINE_HUNTS_AUTODISABLE_FAILURES": "hunts.autodisable.failures", + "DFE_ENGINE_HUNTS_AUTODISABLE_WINDOW_SECONDS": "hunts.autodisable.window_seconds", "DFE_ENGINE_TRANSFORM_DRY_RUN_MAX_EVENTS": "transform_validation.dry_run_max_events", "DFE_ENGINE_TRANSFORM_DRY_RUN_MAX_OUTPUT_BYTES": "transform_validation.dry_run_max_output_bytes", "DFE_ENGINE_DEPLOYMENT_MAX_REPLICAS_CEILING": "deployment.max_replicas_ceiling", @@ -2636,6 +2672,7 @@ def _strip_lower(value: str) -> str: "DFE_ENGINE_HUNTS_SAFE_HORIZON": "hunts.safe_horizon", "DFE_ENGINE_DETECTION_GUARD_ENABLED": "hunts.detection_guard.enabled", "DFE_ENGINE_HUNTS_SMOOTHING_ENABLED": "hunts.smoothing.enabled", + "DFE_ENGINE_HUNTS_AUTODISABLE_ENABLED": "hunts.autodisable.enabled", "DFE_ENGINE_KAFKA_ENSURE_TOPICS": "kafka.ensure_topics", "DFE_ENGINE_KAFKA_BOOTSTRAP_TOPICS": "kafka.bootstrap_topics", "DFE_ENGINE_KAFKA_TIERED_STORAGE": "kafka.tiered_storage", diff --git a/tests/unit/test_api/test_hunts_run_visibility.py b/tests/unit/test_api/test_hunts_run_visibility.py index 0c62269f..a6119786 100644 --- a/tests/unit/test_api/test_hunts_run_visibility.py +++ b/tests/unit/test_api/test_hunts_run_visibility.py @@ -139,3 +139,126 @@ def test_a_programming_fault_is_a_500_not_an_outage_claim( # ClickHouse answered the socket. Saying it is unreachable would be a lie. assert resp.status_code == 500 assert resp.json()["code"] == "queue_failed" + + +class _CoordinationClient: + """A ClickHouse client whose run-status row says whether the hunt is auto-disabled.""" + + def __init__(self, hunt_id: str, disabled_at: int | None, reason: str = "") -> None: + self._row = [hunt_id, 0, 0, 3, 1, 0, 0, 0, disabled_at, reason] + self.inserts: list[tuple] = [] + + def query(self, sql: str, parameters: dict | None = None): + if sql.startswith("SELECT h.hunt_id"): + return type("Result", (), {"result_rows": [self._row]})() + return type("Result", (), {"result_rows": [[3, 1]]})() + + def insert(self, table: str, data: list[list], column_names: list[str], database: str): + self.inserts.append((table, data, column_names, database)) + + +@pytest.fixture +def coordination(monkeypatch, created_hunt): + """Install a coordination client for the created hunt, disabled or not.""" + from dfe_engine.api.v1 import hunts + + def _install(disabled_at: int | None, reason: str = "") -> _CoordinationClient: + client = _CoordinationClient(created_hunt, disabled_at, reason) + monkeypatch.setattr(hunts, "_data_database_client", lambda: (client, "dfe")) + return client + + return _install + + +def _update_body(**changes) -> dict: + body = {k: v for k, v in _HUNT.items() if k != "name"} + return {**body, **changes} + + +def test_a_hunt_row_carries_enabled_and_the_auto_disable( + client, admin_headers, created_hunt, coordination +): + coordination(1700000000, "resource_limit:241") + row = next( + item + for item in client.get("/api/v1/hunts", headers=admin_headers).json()["items"] + if item["name"] == created_hunt + ) + assert row["enabled"] is True + assert row["auto_disabled_at"] == 1700000000 + assert row["auto_disabled_reason"] == "resource_limit:241" + + +def test_a_hunt_saved_disabled_lists_as_disabled_and_keeps_the_field_in_its_detail( + client, admin_headers, created_hunt +): + resp = client.put( + f"/api/v1/hunts/{created_hunt}", json=_update_body(enabled=False), headers=admin_headers + ) + assert resp.status_code == 200 + assert resp.json()["enabled"] is False + row = next( + item + for item in client.get("/api/v1/hunts", headers=admin_headers).json()["items"] + if item["name"] == created_hunt + ) + assert row["enabled"] is False + assert ( + client.get(f"/api/v1/hunts/{created_hunt}", headers=admin_headers).json()["enabled"] + is False + ) + + +def test_enabling_an_auto_disabled_hunt_clears_the_auto_disable( + client, admin_headers, created_hunt, coordination +): + ch = coordination(1700000000, "resource_limit:241") + resp = client.put( + f"/api/v1/hunts/{created_hunt}", json=_update_body(enabled=True), headers=admin_headers + ) + assert resp.status_code == 200 + assert ch.inserts == [ + ( + "hunt_state", + [[created_hunt, 3, 1, None, ""]], + [ + "hunt_id", + "overrun_count", + "too_aggressive", + "auto_disabled_at", + "auto_disabled_reason", + ], + "dfe", + ) + ] + + +@pytest.mark.parametrize( + ("enabled", "disabled_at"), + [(False, 1700000000), (True, None)], + ids=["saved-disabled", "never-auto-disabled"], +) +def test_a_save_that_does_not_re_enable_an_auto_disabled_hunt_writes_no_state( + client, admin_headers, created_hunt, coordination, enabled, disabled_at +): + ch = coordination(disabled_at, "resource_limit:241" if disabled_at else "") + resp = client.put( + f"/api/v1/hunts/{created_hunt}", json=_update_body(enabled=enabled), headers=admin_headers + ) + assert resp.status_code == 200 + assert ch.inserts == [] + + +def test_a_save_still_succeeds_when_the_auto_disable_cannot_be_cleared( + client, admin_headers, created_hunt, coordination +): + ch = coordination(1700000000, "resource_limit:241") + + def _refuse(*args, **kwargs): + raise ConnectionError("[Errno 111] Connection refused") + + ch.insert = _refuse + resp = client.put( + f"/api/v1/hunts/{created_hunt}", json=_update_body(enabled=True), headers=admin_headers + ) + assert resp.status_code == 200 diff --git a/tests/unit/test_hunts/test_metrics.py b/tests/unit/test_hunts/test_metrics.py index 21edad72..8a1c86e6 100644 --- a/tests/unit/test_hunts/test_metrics.py +++ b/tests/unit/test_hunts/test_metrics.py @@ -57,6 +57,7 @@ def test_the_whole_instrument_set_is_registered_up_front(manager): ("counter", "hunt_phase_shift_total"), ("gauge", "hunt_observed_overlap_peak"), ("gauge", "hunt_observed_mem_peak_bytes"), + ("counter", "hunt_autodisabled_total"), ] @@ -95,6 +96,7 @@ def test_no_backend_registers_nothing_and_records_nothing(): metrics.hunt_cost("h", mem_bytes=1.0, duration_ms=1.0) metrics.phase_offset("h", 1) metrics.phase_shift("h", 1) + metrics.auto_disabled("h", reason="resource_limit:241") def test_a_completed_fire_counts_once_and_records_its_duration(manager): @@ -217,8 +219,19 @@ def test_a_capped_rule_counts_once_and_adds_what_it_dropped(manager): Observation("hunt_phase_offset", {"hunt_id": "h"}, "set", 120), ], ), + ( + lambda m: m.auto_disabled("h", reason="resource_limit:241"), + [ + Observation( + "hunt_autodisabled_total", + {"hunt_id": "h", "reason": "resource_limit:241"}, + "inc", + 1, + ) + ], + ), ], - ids=["enabled-on", "enabled-off", "cycle", "cost", "offset", "shift"], + ids=["enabled-on", "enabled-off", "cycle", "cost", "offset", "shift", "auto-disabled"], ) def test_the_smoothing_record_calls_produce_their_observations(manager, call, expected): call(HuntRunnerMetrics(manager)) diff --git a/tests/unit/test_hunts/test_run_status.py b/tests/unit/test_hunts/test_run_status.py index 9d65670f..23f31f0c 100644 --- a/tests/unit/test_hunts/test_run_status.py +++ b/tests/unit/test_hunts/test_run_status.py @@ -16,7 +16,7 @@ from __future__ import annotations -from dfe_engine.hunts.run_status import live_runner_count, read_run_status +from dfe_engine.hunts.run_status import clear_auto_disable, live_runner_count, read_run_status class _Rows: @@ -32,8 +32,41 @@ def query(self, sql: str, parameters: dict | None = None): return type("Result", (), {"result_rows": self._rows})() -def _row(hunt_id, wm=0, lease=0, overruns=0, aggressive=0, last_fire=0, rows=0, pending=0): - return [hunt_id, wm, lease, overruns, aggressive, last_fire, rows, pending] +class _Recording(_Rows): + """A client that also records what is inserted.""" + + def __init__(self, rows: list[list]) -> None: + super().__init__(rows) + self.inserts: list[tuple] = [] + + def insert(self, table: str, data: list[list], column_names: list[str], database: str): + self.inserts.append((table, data, column_names, database)) + + +def _row( + hunt_id, + wm=0, + lease=0, + overruns=0, + aggressive=0, + last_fire=0, + rows=0, + pending=0, + disabled_at=None, + disabled_reason="", +): + return [ + hunt_id, + wm, + lease, + overruns, + aggressive, + last_fire, + rows, + pending, + disabled_at, + disabled_reason, + ] def test_a_hunt_that_has_never_run_reports_nothing_rather_than_zero(): @@ -81,6 +114,61 @@ def test_a_queued_run_is_reported_as_requested(): assert status.run_requested is True +def test_a_hunt_that_was_never_auto_disabled_reports_neither_field(): + status = read_run_status(_Rows([_row("h")]), "dfe", ["h"], now=1000)["h"] + assert status.auto_disabled_at is None + assert status.auto_disabled_reason == "" + + +def test_an_auto_disabled_hunt_reports_when_and_why(): + status = read_run_status( + _Rows([_row("h", disabled_at=950, disabled_reason="resource_limit:241")]), + "dfe", + ["h"], + now=1000, + )["h"] + assert status.auto_disabled_at == 950 + assert status.auto_disabled_reason == "resource_limit:241" + + +def test_a_null_reason_reads_as_empty(): + status = read_run_status(_Rows([_row("h", disabled_reason=None)]), "dfe", ["h"], now=1000)["h"] + assert status.auto_disabled_reason == "" + + +def test_the_query_selects_the_auto_disable_columns_from_hunt_state(): + ch = _Rows([]) + read_run_status(ch, "dfe", ["h"], now=1000) + assert "argMax(auto_disabled_at, updated) AS disabled_at" in ch.sql + assert "argMax(auto_disabled_reason, updated) AS disabled_reason" in ch.sql + + +def test_clearing_an_auto_disable_inserts_a_row_that_keeps_the_overrun_signal(): + ch = _Recording([[3, 1]]) + clear_auto_disable(ch, "dfe", "h") + assert ch.parameters == {"h": "h"} + assert ch.inserts == [ + ( + "hunt_state", + [["h", 3, 1, None, ""]], + [ + "hunt_id", + "overrun_count", + "too_aggressive", + "auto_disabled_at", + "auto_disabled_reason", + ], + "dfe", + ) + ] + + +def test_clearing_a_hunt_with_no_state_row_writes_zeros(): + ch = _Recording([]) + clear_auto_disable(ch, "dfe", "fresh") + assert ch.inserts[0][1] == [["fresh", 0, 0, None, ""]] + + def test_no_hunt_ids_asks_clickhouse_nothing(): ch = _Rows([]) assert read_run_status(ch, "dfe", [], now=1000) == {} diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index 8a72554e..c76994e9 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -641,6 +641,15 @@ def test_hunt_detection_cap_refuses_unbounded(self, field, value): ("path", "env", "default", "raw", "expected"), [ ("hunts.phase_spread_fraction", "HUNTS_PHASE_SPREAD_FRACTION", 0.8, "0.5", 0.5), + ("hunts.autodisable.enabled", "HUNTS_AUTODISABLE_ENABLED", True, "false", False), + ("hunts.autodisable.failures", "HUNTS_AUTODISABLE_FAILURES", 3, "5", 5), + ( + "hunts.autodisable.window_seconds", + "HUNTS_AUTODISABLE_WINDOW_SECONDS", + 3600, + "900", + 900, + ), ("hunts.smoothing.enabled", "HUNTS_SMOOTHING_ENABLED", False, "true", True), ("hunts.smoothing.cycle_seconds", "HUNTS_SMOOTHING_CYCLE_SECONDS", 300.0, "60", 60.0), ("hunts.smoothing.bucket_seconds", "HUNTS_SMOOTHING_BUCKET_SECONDS", 30, "10", 10), @@ -754,6 +763,8 @@ def read() -> Any: [ {"phase_spread_fraction": 0}, {"phase_spread_fraction": 1.5}, + {"autodisable": {"failures": 0}}, + {"autodisable": {"window_seconds": 0}}, {"smoothing": {"hysteresis": 1.0}}, {"smoothing": {"max_step_fraction": 0}}, {"smoothing": {"weight_memory": 1.5}}, @@ -1364,6 +1375,8 @@ class TestEverySettingIsRead: # Read inside settings.py by e2e_routes_enabled, which this search skips. KNOWN_UNREAD = { ("DFESettings", "e2e_server"), + # Read by the runner's auto-disable, built beside this. + ("HuntsSettings", "autodisable"), } # Names a ClickHouse SERVER setting also carries, where SQL naming the From 3544e0b8cb983e3148e07fe650f5de1b6c6a8e8e Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 15:42:25 +1100 Subject: [PATCH 2/6] fix(hunts): enabled flag, auto-disable on limits Hunt YAML has `enabled`, default true. Disabled hunts stay out of the runner's specs and fire nothing. KEDA stops counting them. Their YAML, state and watermark stay. The validator and loader take only YAML booleans. A quoted "false" would otherwise read as on. The runner stops a hunt that trips a ClickHouse resource limit (241 memory, 158 rows, 201 quota) `hunts.autodisable.failures` times inside `hunts.autodisable.window_seconds`. It records when and why in hunt_state and increments `hunt_autodisabled_total`. It alerts through the hunt's destinations with the clear command. It skips the hunt every tick after. A PUT with `enabled: true` clears the auto-disable. A cron edit leaves it disabled. The API shows `enabled`, `auto_disabled_at` and `auto_disabled_reason` on the hunt list and detail. hunt_state 1.1.0 (dfe-schemas #128) has the two columns. --- src/dfe_engine/api/v1/hunts.py | 10 +- src/dfe_engine/clickhouse/errors.py | 1 + src/dfe_engine/hunts/alert.py | 4 + src/dfe_engine/hunts/ch_coordinator.py | 89 ++++++++++- src/dfe_engine/hunts/delivery.py | 37 +++++ src/dfe_engine/hunts/hunt_spec.py | 3 + src/dfe_engine/hunts/runner.py | 84 +++++++++++ src/dfe_engine/hunts/spec_loader.py | 10 ++ src/dfe_engine/hunts/validator.py | 5 + src/dfe_engine/hunts/worker.py | 5 + tests/integration/test_hunt_autodisable.py | 112 ++++++++++++++ .../test_api/test_hunts_run_visibility.py | 14 +- tests/unit/test_clickhouse/test_errors.py | 2 + tests/unit/test_hunts/test_match_delivery.py | 31 ++++ tests/unit/test_hunts/test_runner.py | 140 ++++++++++++++++++ tests/unit/test_hunts/test_spec_loader.py | 16 ++ tests/unit/test_hunts/test_validator.py | 19 +++ tests/unit/test_settings.py | 2 - 18 files changed, 567 insertions(+), 17 deletions(-) create mode 100644 tests/integration/test_hunt_autodisable.py diff --git a/src/dfe_engine/api/v1/hunts.py b/src/dfe_engine/api/v1/hunts.py index d873be6c..bdf51ccc 100644 --- a/src/dfe_engine/api/v1/hunts.py +++ b/src/dfe_engine/api/v1/hunts.py @@ -415,8 +415,10 @@ def _run_status_for(hunt_names: list[str], now: int) -> dict[str, RunStatus]: def _clear_auto_disable_if_enabled(name: str, *, enabled: bool) -> None: """Re-enable a hunt the runner auto-disabled when the save turns it on. - Best-effort like the run state: the YAML is already saved, so an unreachable - ClickHouse leaves the stop in place for the next save rather than failing this one. + Only a save that names ``enabled: true`` clears the stop, so an edit of the cron + or the rules leaves a hunt the runner stopped where it is. Best-effort like the + run state: the YAML is already saved, so an unreachable ClickHouse leaves the + stop in place for the next save rather than failing this one. """ if not enabled: return @@ -661,7 +663,9 @@ def update_hunt( _validate_hunt_config(config, settings, rules) _resolve_customers(request, config) outcome = registry.save(name, config, created_by=user.user_id) - _clear_auto_disable_if_enabled(name, enabled=body.enabled) + _clear_auto_disable_if_enabled( + name, enabled="enabled" in body.model_fields_set and body.enabled + ) apply_review_headers(response, outcome) audit_resource_change(user.user_id, "hunt", name, "updated", review_audit_detail(outcome)) return HuntDetailResponse.from_stored_config(name, config) diff --git a/src/dfe_engine/clickhouse/errors.py b/src/dfe_engine/clickhouse/errors.py index e198c40a..f628d32d 100644 --- a/src/dfe_engine/clickhouse/errors.py +++ b/src/dfe_engine/clickhouse/errors.py @@ -99,6 +99,7 @@ class _Meta: 164: _Meta(ErrorCategory.ACCESS, user_safe=True), # READONLY 192: _Meta(ErrorCategory.ACCESS, user_safe=True), # UNKNOWN_USER 193: _Meta(ErrorCategory.ACCESS, user_safe=True), # WRONG_PASSWORD + 201: _Meta(ErrorCategory.RESOURCE_LIMIT, user_safe=True), # QUOTA_EXCEEDED 202: _Meta(ErrorCategory.RATE_LIMITED, user_safe=True), # TOO_MANY_SIMULTANEOUS_QUERIES 209: _Meta(ErrorCategory.CONNECTION), # SOCKET_TIMEOUT 210: _Meta(ErrorCategory.CONNECTION), # NETWORK_ERROR diff --git a/src/dfe_engine/hunts/alert.py b/src/dfe_engine/hunts/alert.py index 8fcb65f7..bb9d9ee6 100644 --- a/src/dfe_engine/hunts/alert.py +++ b/src/dfe_engine/hunts/alert.py @@ -495,6 +495,10 @@ def _should_fire(self, result_count: int, results: list[dict[str, Any]] | None) return False + def send(self, title: str, body: str) -> bool: + """Send one notification to every destination, with no trigger evaluated.""" + return self._send(title, body) + def _send(self, title: str, body: str) -> bool: """Send notification via Apprise. Returns True on success.""" try: diff --git a/src/dfe_engine/hunts/ch_coordinator.py b/src/dfe_engine/hunts/ch_coordinator.py index f270f1d0..cce2a381 100644 --- a/src/dfe_engine/hunts/ch_coordinator.py +++ b/src/dfe_engine/hunts/ch_coordinator.py @@ -63,6 +63,7 @@ from collections.abc import Callable, Generator from contextlib import contextmanager from dataclasses import dataclass +from datetime import UTC, datetime from typing import Any from scalo.logger import logger @@ -111,11 +112,30 @@ class Lease: @dataclass(frozen=True, slots=True) class HuntStateRow: - """Persisted UI signal for a hunt (overrun / too-aggressive).""" + """Persisted UI signal for a hunt: overrun count, too-aggressive, and an auto-disable. + + Attributes: + hunt_id: The hunt. + overrun_count: Fires deferred because the previous one still ran. + too_aggressive: Whether the schedule is tighter than the hunt keeps up with. + auto_disabled_at: Epoch seconds the runner stopped the hunt, None while it runs. + auto_disabled_reason: The error category and code that tripped it, empty otherwise. + """ hunt_id: str overrun_count: int too_aggressive: bool + auto_disabled_at: int | None = None + auto_disabled_reason: str = "" + + +_STATE_COLUMNS = [ + "hunt_id", + "overrun_count", + "too_aggressive", + "auto_disabled_at", + "auto_disabled_reason", +] class ChCoordinator: @@ -507,16 +527,68 @@ def set_safe_horizon_mark(self, hunt_id: str, mark: str, value: int) -> None: def get_state(self, hunt_id: str) -> HuntStateRow | None: """The hunt's persisted overrun / too-aggressive signal, if any.""" rows = self._ch.query( - f"SELECT overrun_count, too_aggressive FROM {self._qdb}.hunt_state " # noqa: S608 - quoted configured database, values bound + "SELECT overrun_count, too_aggressive, toUnixTimestamp(auto_disabled_at), " # noqa: S608 - quoted configured database, values bound + f"auto_disabled_reason FROM {self._qdb}.hunt_state " "WHERE hunt_id = {h:String} ORDER BY updated DESC LIMIT 1", parameters={"h": hunt_id}, ).result_rows if not rows: return None + disabled_at = rows[0][2] return HuntStateRow( hunt_id=hunt_id, overrun_count=int(rows[0][0]), too_aggressive=bool(rows[0][1]), + auto_disabled_at=int(disabled_at) if disabled_at is not None else None, + auto_disabled_reason=str(rows[0][3] or ""), + ) + + def auto_disabled_hunts(self) -> dict[str, str]: + """Every hunt the runner has stopped, as ``hunt_id -> reason``, read fresh each tick. + + The API clears an auto-disable by writing a row with a NULL time, so a + re-enabled hunt leaves this map on the next read and fires again. + """ + rows = self._ch.query( + "SELECT hunt_id, reason FROM (SELECT hunt_id, " # noqa: S608 - quoted configured database + "argMax(auto_disabled_at, updated) AS at, argMax(auto_disabled_reason, updated) AS reason " + f"FROM {self._qdb}.hunt_state GROUP BY hunt_id) WHERE at IS NOT NULL" + ).result_rows + return {str(r[0]): str(r[1]) for r in rows} + + def _write_state(self, row: HuntStateRow) -> None: + """Write the hunt's whole state row, since a ReplacingMergeTree row replaces all columns.""" + disabled_at = ( + datetime.fromtimestamp(row.auto_disabled_at, tz=UTC) + if row.auto_disabled_at is not None + else None + ) + self._ch.insert( + "hunt_state", + [ + [ + row.hunt_id, + row.overrun_count, + 1 if row.too_aggressive else 0, + disabled_at, + row.auto_disabled_reason, + ] + ], + column_names=_STATE_COLUMNS, + database=self._db, + ) + + def record_auto_disable(self, hunt_id: str, reason: str, now: int) -> None: + """Stop the hunt: stamp when and why, keeping its overrun counters.""" + current = self.get_state(hunt_id) + self._write_state( + HuntStateRow( + hunt_id=hunt_id, + overrun_count=current.overrun_count if current else 0, + too_aggressive=current.too_aggressive if current else False, + auto_disabled_at=now, + auto_disabled_reason=reason, + ) ) # ---- runs (row count + operator run-now) -------------------------- @@ -582,9 +654,12 @@ def record_overrun(self, hunt_id: str) -> None: """ current = self.get_state(hunt_id) count = (current.overrun_count if current else 0) + 1 - self._ch.insert( - "hunt_state", - [[hunt_id, count, 1]], - column_names=["hunt_id", "overrun_count", "too_aggressive"], - database=self._db, + self._write_state( + HuntStateRow( + hunt_id=hunt_id, + overrun_count=count, + too_aggressive=True, + auto_disabled_at=current.auto_disabled_at if current else None, + auto_disabled_reason=current.auto_disabled_reason if current else "", + ) ) diff --git a/src/dfe_engine/hunts/delivery.py b/src/dfe_engine/hunts/delivery.py index b0bb7a0a..380aea63 100644 --- a/src/dfe_engine/hunts/delivery.py +++ b/src/dfe_engine/hunts/delivery.py @@ -87,6 +87,43 @@ def notify(self, spec: HuntSpec, rows_written: int) -> bool: logger.error("hunt alert delivery failed", hunt_id=spec.hunt_id, error=str(exc)) return False + def notify_auto_disabled(self, spec: HuntSpec, reason: str) -> bool: + """Tell the hunt's destinations the runner stopped it, and how to bring it back. + + Unconditional, unlike a match notification: a detection that no longer runs + is the thing alerting exists to surface. The hunt's own destinations and the + global channels receive it. + + Args: + spec: The hunt the runner stopped. + reason: The error category and code that tripped it. + + Returns: + True when the alert was sent. + """ + if not (spec.alerts or self._global_channels): + logger.warning("hunt auto-disabled with no alert destination", hunt_id=spec.hunt_id) + return False + try: + config = build_alert_config( + {"alerts": spec.alerts}, + global_channels=self._global_channels, + destination_registry=self._registry, + ) + if config is None: + logger.warning("hunt alerts resolve to no destination", hunt_id=spec.hunt_id) + return False + title = f"DFE hunt {spec.hunt_id} auto-disabled" + body = ( + f"The hunt runner stopped {spec.hunt_id} after repeated ClickHouse resource-limit " + f"failures ({reason}). It fires nothing until re-enabled: " + f"PUT /api/v1/hunts/{spec.hunt_id} with enabled true." + ) + return self._dispatcher_factory(config).send(title, body) + except Exception as exc: + logger.error("auto-disable alert delivery failed", hunt_id=spec.hunt_id, error=str(exc)) + return False + def _rule_ids(spec: HuntSpec) -> list[str]: """The distinct rules the hunt's statements were compiled from, in order.""" diff --git a/src/dfe_engine/hunts/hunt_spec.py b/src/dfe_engine/hunts/hunt_spec.py index 1a49822f..85d3fdc8 100644 --- a/src/dfe_engine/hunts/hunt_spec.py +++ b/src/dfe_engine/hunts/hunt_spec.py @@ -69,6 +69,9 @@ class HuntSpec(BaseModel): alerts: dict[str, Any] = {} # FIXED watermark field - the always-present common-header column. timestamp_field: str = TIMESTAMP_FIELD + # False keeps the hunt out of the runner's specs, so it fires nothing and KEDA + # stops counting it, while its YAML, state and watermark stay. + enabled: bool = True @field_validator("queries", mode="before") @classmethod diff --git a/src/dfe_engine/hunts/runner.py b/src/dfe_engine/hunts/runner.py index 37b0b121..2fea9653 100644 --- a/src/dfe_engine/hunts/runner.py +++ b/src/dfe_engine/hunts/runner.py @@ -41,6 +41,7 @@ """ import time +from collections import deque from collections.abc import Callable, Iterable from dataclasses import dataclass @@ -114,6 +115,10 @@ class HuntFailures: reminder_seconds: How often a failure that has not changed is logged again (``hunts.failure_reminder_seconds``). jitter: Spreads each wait, so hunts that failed together retry apart. + autodisable_failures: Resource-limit failures within the window that stop the + hunt (``hunts.autodisable.failures``), 0 for never. + autodisable_window: Seconds the failures are counted over + (``hunts.autodisable.window_seconds``). """ def __init__( @@ -123,23 +128,32 @@ def __init__( max_seconds: float = FAILURE_BACKOFF_MAX_SECONDS, reminder_seconds: float = FAILURE_REMINDER_SECONDS, jitter: Callable[[float], float] = jittered, + autodisable_failures: int = 0, + autodisable_window: float = 3600.0, ) -> None: self._initial = initial_seconds self._max = max_seconds self._reminder = reminder_seconds self._jitter = jitter self._failing: dict[str, _Failing] = {} + self._autodisable_failures = autodisable_failures + self._autodisable_window = autodisable_window + # Resource-limit failure times per hunt, inside the window. + self._limit_hits: dict[str, deque[int]] = {} @classmethod def from_settings( cls, hunts: HuntsSettings, *, jitter: Callable[[float], float] = jittered ) -> HuntFailures: """The tracker a deployment's ``hunts.failure_*`` settings describe.""" + autodisable = hunts.autodisable return cls( initial_seconds=hunts.failure_backoff_initial_seconds, max_seconds=hunts.failure_backoff_max_seconds, reminder_seconds=hunts.failure_reminder_seconds, jitter=jitter, + autodisable_failures=autodisable.failures if autodisable.enabled else 0, + autodisable_window=autodisable.window_seconds, ) def retry_at(self, hunt_id: str) -> float | None: @@ -173,6 +187,26 @@ def retain(self, hunt_ids: Iterable[str]) -> None: keep = set(hunt_ids) for hunt_id in [h for h in self._failing if h not in keep]: del self._failing[hunt_id] + for hunt_id in [h for h in self._limit_hits if h not in keep]: + del self._limit_hits[hunt_id] + + def limit_tripped(self, hunt_id: str, now: int, exc: BaseException) -> str | None: + """Count a resource-limit failure and say whether it is the one that stops the hunt. + + Only ``ErrorCategory.RESOURCE_LIMIT`` counts, never a syntax, access or + connection failure. Returns the reason to record (``resource_limit:``) + when the window now holds ``autodisable_failures`` hits, else None. + """ + if self._autodisable_failures <= 0 or classify(exc) is not ErrorCategory.RESOURCE_LIMIT: + return None + hits = self._limit_hits.setdefault(hunt_id, deque()) + hits.append(now) + while hits and now - hits[0] > self._autodisable_window: + hits.popleft() + if len(hits) < self._autodisable_failures: + return None + hits.clear() + return f"{ErrorCategory.RESOURCE_LIMIT.value}:{parse_code(exc)}" def failed( self, @@ -251,6 +285,8 @@ def __init__( smoother: PhaseSmoother | None = None, ) -> None: self._coord = coordinator + # Hunts this runner stopped, kept so a failed hunt_state read still skips them. + self._stopped: dict[str, str] = {} # hunts.phase_spread_fraction: the same value publish_schedule writes for KEDA. self._phase_fraction = phase_fraction # With smoothing on, every tick reads the offsets KEDA scales on and the smoother @@ -311,6 +347,48 @@ def _learned_offsets(self) -> dict[str, int]: ) return {} + def _auto_disabled(self) -> dict[str, str]: + """Hunts to skip this tick: every auto-disable in hunt_state, else the ones this runner stopped. + + The table is the truth, since the API clears an auto-disable there. A failed + read falls back to this runner's own record rather than firing a stopped hunt. + """ + try: + self._stopped = self._coord.auto_disabled_hunts() + except Exception as exc: + logger.warning( + "auto-disabled hunts unreadable; using this runner's record", error=str(exc) + ) + return self._stopped + + def _auto_disable(self, spec: HuntSpec, reason: str, now: int) -> None: + """Stop a hunt that kept tripping a resource limit: record, count, alert. + + A query that busts the memory limit on its own fails at 3am too, and each + retry fails whatever else is allocating, so the hunt stops until an operator + re-enables it through the API. + """ + self._stopped[spec.hunt_id] = reason + logger.error( + "hunt auto-disabled after repeated resource-limit failures", + hunt_id=spec.hunt_id, + reason=reason, + clear=f"PUT /api/v1/hunts/{spec.hunt_id} with enabled true", + ) + try: + self._coord.record_auto_disable(spec.hunt_id, reason, now) + except Exception as exc: + logger.error( + "auto-disable not recorded; this runner skips the hunt anyway", + hunt_id=spec.hunt_id, + error=str(exc), + ) + self._metrics.auto_disabled(spec.hunt_id, reason=reason) + try: + self._worker.alert_auto_disabled(spec, reason) + except Exception as exc: + logger.warning("auto-disable alert not sent", hunt_id=spec.hunt_id, error=str(exc)) + def _smooth(self, now: int) -> None: """Run the smoother's cycle if one is due. A failed cycle costs nothing but the cycle.""" if self._smoother is None: @@ -357,6 +435,7 @@ def tick(self, now: int) -> int: # Read every outstanding run-now once per tick, not once per hunt. requested = self._coord.pending_runs(now) offsets = self._learned_offsets() + disabled = self._auto_disabled() executed = 0 started = 0 specs = list(self._specs.values()) @@ -365,6 +444,8 @@ def tick(self, now: int) -> int: if running + started >= self._cap: self._start = first + reached break # global cap protects ClickHouse, lateness surfaces as overload + if spec.hunt_id in disabled: + continue fire = self._fire_for(spec, now, requested, offsets) if fire is None: continue @@ -411,6 +492,9 @@ def tick(self, now: int) -> int: self._failures.failed( spec.hunt_id, fire, now, exc, requested=requested.get(spec.hunt_id) ) + reason = self._failures.limit_tripped(spec.hunt_id, now, exc) + if reason is not None: + self._auto_disable(spec, reason, now) self._smooth(now) self._metrics.tick_completed(time.monotonic() - tick_began) return executed diff --git a/src/dfe_engine/hunts/spec_loader.py b/src/dfe_engine/hunts/spec_loader.py index a4a276c5..fa64bdf8 100644 --- a/src/dfe_engine/hunts/spec_loader.py +++ b/src/dfe_engine/hunts/spec_loader.py @@ -122,6 +122,11 @@ def _build_spec( Skips a direct ``query`` that calls a function reading outside the row, or that is not one SELECT or one INSERT INTO a table from a SELECT. """ + enabled = definition.get("enabled", True) + if not isinstance(enabled, bool): + # bool("false") is True, so a string here would fire a hunt its author stopped. + raise ValueError(f"'enabled' must be true or false, got {enabled!r}") + value = _resolve_schedule_value(definition, stem) if value is None: return None # anchored (already logged) or no schedule at all @@ -167,6 +172,7 @@ def _build_spec( target_table=str(definition.get("global_target_table_name", definition.get("target", ""))), timestamp_field=_timestamp_field(definition), alerts=alerts_block(definition), + enabled=enabled, ) @@ -230,6 +236,10 @@ def load_specs( max_detections, lookup_databases, ) + if spec is not None and not spec.enabled: + # Out of the runner's specs: it fires nothing and KEDA stops counting it. + logger.info("skipping hunt: disabled", hunt_id=stem) + continue if spec is not None: specs[stem] = spec except Exception as exc: # one bad file must never break the whole load diff --git a/src/dfe_engine/hunts/validator.py b/src/dfe_engine/hunts/validator.py index 7aa9b2cf..f495b226 100644 --- a/src/dfe_engine/hunts/validator.py +++ b/src/dfe_engine/hunts/validator.py @@ -116,6 +116,11 @@ def _validate_required_fields(hunt_data: dict, checkpoint_timestamp_field: str | "Invalid 'checkpoint_timestamp_field'. It should be a non-empty string." ) + # A string such as "false" would read as enabled, so only a YAML bool is taken. + enabled = hunt_data.get("enabled", True) + if not isinstance(enabled, bool): + raise ValueError(f"Invalid 'enabled' value: {enabled!r}. It should be true or false.") + HuntValidator._validate_query(hunt_data.get("query")) @staticmethod diff --git a/src/dfe_engine/hunts/worker.py b/src/dfe_engine/hunts/worker.py index c6a3309f..28820afe 100644 --- a/src/dfe_engine/hunts/worker.py +++ b/src/dfe_engine/hunts/worker.py @@ -453,6 +453,11 @@ def run(self, spec: HuntSpec, scheduled_start: int) -> int: self._notifier.notify(spec, written) return end + def alert_auto_disabled(self, spec: HuntSpec, reason: str) -> None: + """Send the hunt's auto-disable alert through its notifier, if one is wired.""" + if self._notifier is not None: + self._notifier.notify_auto_disabled(spec, reason) + def _summarise_cap( self, spec: HuntSpec, diff --git a/tests/integration/test_hunt_autodisable.py b/tests/integration/test_hunt_autodisable.py new file mode 100644 index 00000000..0d7f919c --- /dev/null +++ b/tests/integration/test_hunt_autodisable.py @@ -0,0 +1,112 @@ +# Project: dfe-engine +# File: tests/integration/test_hunt_autodisable.py +# Purpose: A hunt that keeps busting a ClickHouse limit is stopped, and only the API restarts it +# Language: Python +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED +"""Hunt disable and auto-disable on real ClickHouse, no mocks. + +The hunt's own SQL trips ``max_rows_to_read`` (code 158), a resource limit, so +the runner stops it after the configured failures and writes why into hunt_state. +The API's clear is the one way back. A hunt whose YAML says ``enabled: false`` is +dropped from the specs and tombstoned in ``hunt_schedule``, so KEDA stops counting it. +""" + +from pathlib import Path + +import pytest + +from dfe_engine.hunts.ch_coordinator import ChCoordinator +from dfe_engine.hunts.hunt_spec import HuntSpec +from dfe_engine.hunts.run_status import clear_auto_disable, read_run_status +from dfe_engine.hunts.runner import HuntFailures, HuntRunner +from dfe_engine.hunts.schedule import current_fire, publish_schedule, read_schedule +from dfe_engine.hunts.spec_loader import load_specs +from dfe_engine.hunts.worker import HuntWorker + +LIMIT_SQL = ( + "SELECT count() FROM (SELECT number AS timestamp_load FROM numbers(1000)) " + "WHERE {window} SETTINGS max_rows_to_read = 10" +) + + +class _CountingClient: + """The real client, counting the statements the worker sends.""" + + def __init__(self, ch) -> None: + self._ch = ch + self.commands = 0 + + def command(self, sql: str, settings: dict | None = None): + self.commands += 1 + return self._ch.command(sql, settings=settings) + + def query(self, *args, **kwargs): + return self._ch.query(*args, **kwargs) + + def insert(self, *args, **kwargs): + return self._ch.insert(*args, **kwargs) + + +@pytest.fixture +def state_db(ch_client, dfe_db): + """The coordination database once hunt_state carries the auto-disable columns.""" + columns = { + r[0] + for r in ch_client.query( + "SELECT name FROM system.columns WHERE database = {d:String} AND table = 'hunt_state'", + parameters={"d": dfe_db}, + ).result_rows + } + if "auto_disabled_at" not in columns: + pytest.skip("hunt_state 1.1.0 (auto_disabled_at) is not in the pinned dfe-schemas release") + return dfe_db + + +def test_k_limit_failures_stop_the_hunt_until_the_api_clears_it(ch_client, state_db): + ch = _CountingClient(ch_client) + coord = ChCoordinator(ch, database=state_db, sleep=lambda _s: None, monotonic=lambda: 0.0) + coord.ensure_schema() + spec = HuntSpec(hunt_id="h", interval_seconds=600, queries=[LIMIT_SQL]) + failures = HuntFailures( + initial_seconds=0, max_seconds=0, jitter=lambda w: 0.0, autodisable_failures=2 + ) + runner = HuntRunner(coord, HuntWorker(ch, coord), {"h": spec}, cap=4, failures=failures) + now = current_fire("h", 600, 5000) + 1 + + runner.tick(now) + assert coord.auto_disabled_hunts() == {} + runner.tick(now + 1) + assert coord.auto_disabled_hunts() == {"h": "resource_limit:158"} + status = read_run_status(ch_client, state_db, ["h"], now + 1)["h"] + assert (status.auto_disabled_at, status.auto_disabled_reason) == (now + 1, "resource_limit:158") + # Stopped: a tick with the back-off at zero sends ClickHouse nothing. + sent = ch.commands + runner.tick(now + 2) + assert ch.commands == sent + + clear_auto_disable(ch_client, state_db, "h") + assert coord.auto_disabled_hunts() == {} + assert read_run_status(ch_client, state_db, ["h"], now + 3)["h"].auto_disabled_at is None + runner.tick(now + 3) + assert ch.commands == sent + 1 + + +def test_a_hunt_disabled_in_yaml_is_tombstoned_in_the_schedule(ch_client, dfe_db, tmp_path: Path): + hunt_dir = tmp_path / "hunts" + hunt_dir.mkdir() + (hunt_dir / "keep.yaml").write_text('cron: "*/10 * * * *"\nquery: "SELECT 1 WHERE {window}"\n') + (hunt_dir / "stop.yaml").write_text('cron: "*/10 * * * *"\nquery: "SELECT 1 WHERE {window}"\n') + publish_schedule(ch_client, dfe_db, load_specs(hunt_dir)) + assert set(read_schedule(ch_client, dfe_db)) == {"keep", "stop"} + + (hunt_dir / "stop.yaml").write_text( + 'cron: "*/10 * * * *"\nquery: "SELECT 1 WHERE {window}"\nenabled: false\n' + ) + publish_schedule(ch_client, dfe_db, load_specs(hunt_dir)) + assert set(read_schedule(ch_client, dfe_db)) == {"keep"} + + (hunt_dir / "stop.yaml").write_text('cron: "*/10 * * * *"\nquery: "SELECT 1 WHERE {window}"\n') + publish_schedule(ch_client, dfe_db, load_specs(hunt_dir)) + assert set(read_schedule(ch_client, dfe_db)) == {"keep", "stop"} diff --git a/tests/unit/test_api/test_hunts_run_visibility.py b/tests/unit/test_api/test_hunts_run_visibility.py index a6119786..2ab698d0 100644 --- a/tests/unit/test_api/test_hunts_run_visibility.py +++ b/tests/unit/test_api/test_hunts_run_visibility.py @@ -234,16 +234,20 @@ def test_enabling_an_auto_disabled_hunt_clears_the_auto_disable( @pytest.mark.parametrize( - ("enabled", "disabled_at"), - [(False, 1700000000), (True, None)], - ids=["saved-disabled", "never-auto-disabled"], + ("changes", "disabled_at"), + [ + ({"enabled": False}, 1700000000), + ({"enabled": True}, None), + ({"cron": "*/7 * * * *"}, 1700000000), + ], + ids=["saved-disabled", "never-auto-disabled", "an-edit-that-omits-enabled"], ) def test_a_save_that_does_not_re_enable_an_auto_disabled_hunt_writes_no_state( - client, admin_headers, created_hunt, coordination, enabled, disabled_at + client, admin_headers, created_hunt, coordination, changes, disabled_at ): ch = coordination(disabled_at, "resource_limit:241" if disabled_at else "") resp = client.put( - f"/api/v1/hunts/{created_hunt}", json=_update_body(enabled=enabled), headers=admin_headers + f"/api/v1/hunts/{created_hunt}", json=_update_body(**changes), headers=admin_headers ) assert resp.status_code == 200 assert ch.inserts == [] diff --git a/tests/unit/test_clickhouse/test_errors.py b/tests/unit/test_clickhouse/test_errors.py index af82d3d3..7bd041cd 100644 --- a/tests/unit/test_clickhouse/test_errors.py +++ b/tests/unit/test_clickhouse/test_errors.py @@ -52,6 +52,7 @@ def test_query_logic_codes(self): assert classify(_exc(62)) is ErrorCategory.USER_ERROR # SYNTAX_ERROR assert classify(_exc(60)) is ErrorCategory.NOT_FOUND # UNKNOWN_TABLE assert classify(_exc(241)) is ErrorCategory.RESOURCE_LIMIT # MEMORY_LIMIT_EXCEEDED + assert classify(_exc(201)) is ErrorCategory.RESOURCE_LIMIT # QUOTA_EXCEEDED assert classify(_exc(159)) is ErrorCategory.TIMEOUT # TIMEOUT_EXCEEDED def test_python_builtin_connection_errors(self): @@ -82,6 +83,7 @@ def test_syntax_not_retryable(self): def test_memory_limit_not_retryable(self): assert is_retryable_error(_exc(241)) is False + assert is_retryable_error(_exc(201)) is False def test_exec_timeout_not_retryable(self): # 159 TIMEOUT_EXCEEDED is a query-exec timeout, NOT a transport timeout. diff --git a/tests/unit/test_hunts/test_match_delivery.py b/tests/unit/test_hunts/test_match_delivery.py index aa15cd9e..9bd5ee90 100644 --- a/tests/unit/test_hunts/test_match_delivery.py +++ b/tests/unit/test_hunts/test_match_delivery.py @@ -110,6 +110,9 @@ def active_count(self, now: int) -> int: def pending_runs(self, now: int) -> dict[str, int]: return {} + def auto_disabled_hunts(self) -> dict[str, str]: + return {} + def current_lease(self, hunt_id: str) -> None: return None @@ -245,3 +248,31 @@ def _send(self, title: str, body: str) -> bool: assert len(sent) == 1 assert "h" in sent[0][0] assert "4" in sent[0][1] + + +@pytest.mark.parametrize( + ("alerts", "channels", "want_sent"), + [ + pytest.param({"channels": ["json://example.invalid/x"]}, [], True, id="hunt_destinations"), + pytest.param({}, ["json://example.invalid/x"], True, id="global_channels"), + pytest.param({}, [], False, id="nowhere_to_send"), + ], +) +def test_an_auto_disable_alert_names_the_hunt_the_reason_and_the_clear(alerts, channels, want_sent): + sent: list[tuple[str, str]] = [] + + class _Dispatcher(AlertDispatcher): + def _send(self, title: str, body: str) -> bool: + sent.append((title, body)) + return True + + spec = HuntSpec(hunt_id="h", interval_seconds=60, alerts=alerts) + notifier = MatchNotifier(global_channels=channels, dispatcher_factory=_Dispatcher) + + assert notifier.notify_auto_disabled(spec, "resource_limit:241") is want_sent + assert len(sent) == (1 if want_sent else 0) + if want_sent: + title, body = sent[0] + assert "h" in title + assert "resource_limit:241" in body + assert "PUT /api/v1/hunts/h" in body diff --git a/tests/unit/test_hunts/test_runner.py b/tests/unit/test_hunts/test_runner.py index c3224250..84eadc06 100644 --- a/tests/unit/test_hunts/test_runner.py +++ b/tests/unit/test_hunts/test_runner.py @@ -120,6 +120,9 @@ def pending_runs(self, now: int) -> dict[str, int]: self.calls.append("pending_runs") return {} + def auto_disabled_hunts(self) -> dict[str, str]: + return {} + def get_watermark(self, hunt_id: str) -> int | None: return None @@ -768,3 +771,140 @@ def test_hunts_past_the_cap_take_their_turn_on_the_next_ticks(fails): assert len(worker.ran) == tick + 1, worker.ran assert sorted(worker.ran) == ids + + +def _code(code: int) -> Exception: + """A driver-style error carrying a ClickHouse ``Code: NNN``.""" + return Exception(f"Code: {code}. DB::Exception: boom") + + +@pytest.mark.parametrize( + ("failures", "window", "hits", "want"), + [ + pytest.param( + 3, 3600, [(0, 241), (10, 241), (20, 241)], "resource_limit:241", id="k_hits_in_window" + ), + pytest.param(3, 3600, [(0, 241), (10, 241)], None, id="under_k"), + pytest.param( + 3, 60, [(0, 241), (10, 241), (120, 241)], None, id="old_hits_fall_out_of_the_window" + ), + pytest.param( + 3, + 3600, + [(0, 241), (10, 201), (20, 158)], + "resource_limit:158", + id="quota_and_too_many_rows_count", + ), + pytest.param(3, 3600, [(0, 62), (10, 62), (20, 62)], None, id="syntax_errors_never_count"), + pytest.param( + 3, 3600, [(0, 516), (10, 241), (20, 241)], None, id="an_auth_failure_is_not_a_hit" + ), + pytest.param(0, 3600, [(0, 241), (10, 241), (20, 241)], None, id="autodisable_off"), + ], +) +def test_limit_tripped_counts_only_resource_limits_inside_the_window(failures, window, hits, want): + tracker = HuntFailures(autodisable_failures=failures, autodisable_window=window) + reasons = [tracker.limit_tripped("h", now, _code(code)) for now, code in hits] + assert reasons[-1] == want + assert all(r is None for r in reasons[:-1]) + + +def test_a_tripped_hunt_starts_its_count_again(): + tracker = HuntFailures(autodisable_failures=2, autodisable_window=3600) + assert tracker.limit_tripped("h", 0, _code(241)) is None + assert tracker.limit_tripped("h", 1, _code(241)) == "resource_limit:241" + assert tracker.limit_tripped("h", 2, _code(241)) is None + tracker.retain([]) + assert tracker.limit_tripped("h", 3, _code(241)) is None + + +class _StoppingCoordinator(_RecordingCoordinator): + """A coordinator whose hunt_state lists auto-disabled hunts and records new ones.""" + + def __init__(self, disabled: dict[str, str] | None = None, *, raises: bool = False) -> None: + super().__init__() + self.disabled = dict(disabled or {}) + self.recorded: list[tuple[str, str, int]] = [] + self._raises = raises + + def auto_disabled_hunts(self) -> dict[str, str]: + if self._raises: + raise ConnectionError("refused") + return dict(self.disabled) + + def record_auto_disable(self, hunt_id: str, reason: str, now: int) -> None: + self.recorded.append((hunt_id, reason, now)) + self.disabled[hunt_id] = reason + + +class _LimitWorker(_Worker): + """A worker whose every fire trips the ClickHouse memory limit.""" + + def __init__(self) -> None: + self.runs = 0 + self.alerts: list[tuple[str, str]] = [] + + def run(self, spec, fire: int) -> None: + self.runs += 1 + raise Exception("Code: 241. DB::Exception: Memory limit exceeded") + + def alert_auto_disabled(self, spec, reason: str) -> None: + self.alerts.append((spec.hunt_id, reason)) + + +def test_an_auto_disabled_hunt_in_hunt_state_is_never_fired(): + from dfe_engine.hunts.hunt_spec import HuntSpec + + coord = _StoppingCoordinator({"h": "resource_limit:241"}) + worker = _CountingWorker() + spec = HuntSpec(hunt_id="h", interval_seconds=600, queries=["SELECT 1"]) + runner = HuntRunner(coord, worker, {"h": spec}, cap=8) # ty: ignore[invalid-argument-type] # stand-ins carrying only the calls this path makes + runner.tick(600_000 + phase_offset("h", 600)) + assert worker.runs == 0 + + +def test_k_resource_limit_failures_stop_the_hunt_record_it_and_alert(): + from dfe_engine.hunts.hunt_spec import HuntSpec + + coord = _StoppingCoordinator() + worker = _LimitWorker() + failures = HuntFailures( + initial_seconds=0, + max_seconds=0, + jitter=lambda w: 0.0, + autodisable_failures=2, + autodisable_window=3600, + ) + spec = HuntSpec(hunt_id="h", interval_seconds=600, queries=["SELECT 1"]) + runner = HuntRunner(coord, worker, {"h": spec}, cap=8, failures=failures) # ty: ignore[invalid-argument-type] # stand-ins carrying only the calls this path makes + fire = 600_000 + phase_offset("h", 600) + runner.tick(fire) + runner.tick(fire + 1) + assert worker.runs == 2 + assert coord.recorded == [("h", "resource_limit:241", fire + 1)] + assert worker.alerts == [("h", "resource_limit:241")] + # Stopped: the next tick fires nothing, even with the back-off at zero. + runner.tick(fire + 2) + assert worker.runs == 2 + + +def test_an_unreadable_hunt_state_keeps_this_runners_own_stops(): + from dfe_engine.hunts.hunt_spec import HuntSpec + + coord = _StoppingCoordinator() + worker = _LimitWorker() + failures = HuntFailures( + initial_seconds=0, + max_seconds=0, + jitter=lambda w: 0.0, + autodisable_failures=1, + autodisable_window=3600, + ) + spec = HuntSpec(hunt_id="h", interval_seconds=600, queries=["SELECT 1"]) + runner = HuntRunner(coord, worker, {"h": spec}, cap=8, failures=failures) # ty: ignore[invalid-argument-type] # stand-ins carrying only the calls this path makes + fire = 600_000 + phase_offset("h", 600) + runner.tick(fire) + assert coord.recorded[0][0] == "h" + coord._raises = True + runner.tick(fire + 1) + assert worker.runs == 1 diff --git a/tests/unit/test_hunts/test_spec_loader.py b/tests/unit/test_hunts/test_spec_loader.py index 7addece5..f24b4c03 100644 --- a/tests/unit/test_hunts/test_spec_loader.py +++ b/tests/unit/test_hunts/test_spec_loader.py @@ -284,3 +284,19 @@ def test_the_runner_runs_only_the_hunts_the_registry_reads(tmp_path: Path): assert run == read == {"ok_hunt"} assert [h.hunt for h in registry.tenant_id_holders("newcorp", ["t1"])] == ["ok_hunt"] + + +def test_a_disabled_hunt_is_skipped_and_its_sibling_loads(tmp_path: Path): + """``enabled: false`` keeps a hunt out of the runner's specs, and the default is on.""" + _write(tmp_path, "off", 'schedule:\n mode: rate\n interval: "5m"\nenabled: false\n') + _write(tmp_path, "on", 'schedule:\n mode: rate\n interval: "5m"\n') + specs = load_specs(tmp_path) + assert set(specs) == {"on"} + assert specs["on"].enabled is True + + +def test_a_hunt_whose_enabled_is_not_a_bool_is_skipped_not_fired(tmp_path: Path): + """A quoted "false" must never load as an enabled hunt.""" + _write(tmp_path, "quoted", 'schedule:\n mode: rate\n interval: "5m"\nenabled: "false"\n') + _write(tmp_path, "on", 'schedule:\n mode: rate\n interval: "5m"\n') + assert set(load_specs(tmp_path)) == {"on"} diff --git a/tests/unit/test_hunts/test_validator.py b/tests/unit/test_hunts/test_validator.py index 32cfe5b7..4f2c7f5a 100644 --- a/tests/unit/test_hunts/test_validator.py +++ b/tests/unit/test_hunts/test_validator.py @@ -113,6 +113,25 @@ def test_no_scheduling_fields_ok(self, rule_dir): data = _base_hunt_data() HuntValidator.validate_hunt_configuration(data, rule_dir, "timestamp_load") + @pytest.mark.parametrize( + ("enabled", "accepted"), + [ + pytest.param(True, True, id="true"), + pytest.param(False, True, id="false"), + pytest.param("false", False, id="string-false-is-refused"), + pytest.param(0, False, id="int-is-refused"), + pytest.param(None, False, id="null-is-refused"), + ], + ) + def test_enabled_is_a_bool_or_refused(self, rule_dir, enabled, accepted): + data = _base_hunt_data() + data["enabled"] = enabled + if accepted: + HuntValidator.validate_hunt_configuration(data, rule_dir, "timestamp_load") + else: + with pytest.raises(ValueError, match="'enabled'"): + HuntValidator.validate_hunt_configuration(data, rule_dir, "timestamp_load") + class TestValidatorRuleFile: """Each rule is checked as the ``{name}.yaml`` the hunt runner reads.""" diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index c76994e9..cc5bef9b 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -1375,8 +1375,6 @@ class TestEverySettingIsRead: # Read inside settings.py by e2e_routes_enabled, which this search skips. KNOWN_UNREAD = { ("DFESettings", "e2e_server"), - # Read by the runner's auto-disable, built beside this. - ("HuntsSettings", "autodisable"), } # Names a ClickHouse SERVER setting also carries, where SQL naming the From 7022082881b2ce8576c19a123d9392d662701557 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 16:21:57 +1100 Subject: [PATCH 3/6] fix(hunts): read the latest hunt_state row as a tuple argMax skips a NULL argument. `argMax(auto_disabled_at, updated)` kept an older stop time after the API cleared it with NULL, until the merge collapsed the rows. The runner's auto-disabled read and the run-status join take both columns from the latest row as a tuple. The tuple is never NULL. The minimum dfe-schemas version is 0.2.30, the release with hunt_state 1.1.0. The auto-disable integration test runs against 0.2.30: - Two resource-limit failures stop the hunt. - The stopped tick fires nothing. - Clearing the stop through the API restarts the hunt. --- pyproject.toml | 4 ++-- src/dfe_engine/hunts/ch_coordinator.py | 9 ++++---- src/dfe_engine/hunts/run_status.py | 6 ++--- tests/integration/test_hunt_autodisable.py | 26 +++++++++++++--------- tests/unit/test_hunts/test_run_status.py | 5 +++-- uv.lock | 8 +++---- 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c790c82a..c3104a77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,8 +11,8 @@ readme = "README.md" dependencies = [ # [kafka] supplies genson to scalo.kafka.providers. Without [metrics], /metrics 404s. "scalo[api,cli,expression,http,kafka,metrics,secrets-kubernetes]>=2.32.7,<3", - # Floor: the first release with catalogue/. - "dfe-schemas>=0.2.29,<1", + # Floor: hunt_state 1.1.0, the auto-disable columns the runner writes. + "dfe-schemas>=0.2.30,<1", "pandas>=3.0.3", "numpy>=2.5.4", "clickhouse-connect>=1.10.0", diff --git a/src/dfe_engine/hunts/ch_coordinator.py b/src/dfe_engine/hunts/ch_coordinator.py index cce2a381..e8a5db52 100644 --- a/src/dfe_engine/hunts/ch_coordinator.py +++ b/src/dfe_engine/hunts/ch_coordinator.py @@ -547,12 +547,13 @@ def auto_disabled_hunts(self) -> dict[str, str]: """Every hunt the runner has stopped, as ``hunt_id -> reason``, read fresh each tick. The API clears an auto-disable by writing a row with a NULL time, so a - re-enabled hunt leaves this map on the next read and fires again. + re-enabled hunt leaves this map on the next read and fires again. argMax skips + a NULL argument, so the latest row is taken as a tuple, which is never NULL. """ rows = self._ch.query( - "SELECT hunt_id, reason FROM (SELECT hunt_id, " # noqa: S608 - quoted configured database - "argMax(auto_disabled_at, updated) AS at, argMax(auto_disabled_reason, updated) AS reason " - f"FROM {self._qdb}.hunt_state GROUP BY hunt_id) WHERE at IS NOT NULL" + "SELECT hunt_id, tupleElement(s, 2) FROM (SELECT hunt_id, " # noqa: S608 - quoted configured database + "argMax((auto_disabled_at, auto_disabled_reason), updated) AS s " + f"FROM {self._qdb}.hunt_state GROUP BY hunt_id) WHERE tupleElement(s, 1) IS NOT NULL" ).result_rows return {str(r[0]): str(r[1]) for r in rows} diff --git a/src/dfe_engine/hunts/run_status.py b/src/dfe_engine/hunts/run_status.py index 1baab51a..f4e9b4bb 100644 --- a/src/dfe_engine/hunts/run_status.py +++ b/src/dfe_engine/hunts/run_status.py @@ -61,7 +61,7 @@ def _query(database: str) -> str: "SELECT h.hunt_id, coalesce(w.wm, 0), coalesce(l.lu, 0), " # noqa: S608 - quoted configured database, ids bound "coalesce(s.overruns, 0), coalesce(s.aggressive, 0), " "coalesce(r.last_fire, 0), coalesce(r.rows_written, 0), coalesce(q.pending, 0), " - "toUnixTimestamp(s.disabled_at), coalesce(s.disabled_reason, '') " + "toUnixTimestamp(tupleElement(s.disabled, 1)), coalesce(tupleElement(s.disabled, 2), '') " "FROM (SELECT arrayJoin({ids:Array(String)}) AS hunt_id) h " "LEFT JOIN (SELECT hunt_id, argMax(watermark, updated) AS wm " f"FROM {db}.hunt_watermark GROUP BY hunt_id) w USING (hunt_id) " @@ -69,8 +69,8 @@ def _query(database: str) -> str: f"FROM {db}.hunt_lease GROUP BY hunt_id) l USING (hunt_id) " "LEFT JOIN (SELECT hunt_id, argMax(overrun_count, updated) AS overruns, " "argMax(too_aggressive, updated) AS aggressive, " - "argMax(auto_disabled_at, updated) AS disabled_at, " - "argMax(auto_disabled_reason, updated) AS disabled_reason " + # As a tuple, because argMax skips a NULL argument and a cleared auto-disable is one. + "argMax((auto_disabled_at, auto_disabled_reason), updated) AS disabled " f"FROM {db}.hunt_state GROUP BY hunt_id) s USING (hunt_id) " "LEFT JOIN (SELECT hunt_id, max(fire) AS last_fire, " "argMax(rows_written, fire) AS rows_written FROM (" diff --git a/tests/integration/test_hunt_autodisable.py b/tests/integration/test_hunt_autodisable.py index 0d7f919c..6a3d5a01 100644 --- a/tests/integration/test_hunt_autodisable.py +++ b/tests/integration/test_hunt_autodisable.py @@ -26,21 +26,22 @@ from dfe_engine.hunts.worker import HuntWorker LIMIT_SQL = ( - "SELECT count() FROM (SELECT number AS timestamp_load FROM numbers(1000)) " + "SELECT count() FROM (SELECT toDateTime64(number, 3) AS _timestamp_load FROM numbers(1000)) " "WHERE {window} SETTINGS max_rows_to_read = 10" ) class _CountingClient: - """The real client, counting the statements the worker sends.""" + """The real client, counting the fires of the hunt's own SQL among every statement.""" def __init__(self, ch) -> None: self._ch = ch - self.commands = 0 + self.fires = 0 - def command(self, sql: str, settings: dict | None = None): - self.commands += 1 - return self._ch.command(sql, settings=settings) + def command(self, sql: str, *args, **kwargs): + if "max_rows_to_read" in sql: + self.fires += 1 + return self._ch.command(sql, *args, **kwargs) def query(self, *args, **kwargs): return self._ch.query(*args, **kwargs) @@ -74,23 +75,28 @@ def test_k_limit_failures_stop_the_hunt_until_the_api_clears_it(ch_client, state ) runner = HuntRunner(coord, HuntWorker(ch, coord), {"h": spec}, cap=4, failures=failures) now = current_fire("h", 600, 5000) + 1 + # The fixture SQL trips the row limit, a resource limit, and nothing else. + with pytest.raises(Exception, match="Code: 158"): + HuntWorker(ch, coord).run(spec, scheduled_start=now) + assert ch.fires == 1 runner.tick(now) + assert ch.fires == 2 assert coord.auto_disabled_hunts() == {} runner.tick(now + 1) + assert ch.fires == 3, (coord.get_state("h"), coord.current_lease("h")) assert coord.auto_disabled_hunts() == {"h": "resource_limit:158"} status = read_run_status(ch_client, state_db, ["h"], now + 1)["h"] assert (status.auto_disabled_at, status.auto_disabled_reason) == (now + 1, "resource_limit:158") - # Stopped: a tick with the back-off at zero sends ClickHouse nothing. - sent = ch.commands + # Stopped: a tick with the back-off at zero fires nothing. runner.tick(now + 2) - assert ch.commands == sent + assert ch.fires == 3 clear_auto_disable(ch_client, state_db, "h") assert coord.auto_disabled_hunts() == {} assert read_run_status(ch_client, state_db, ["h"], now + 3)["h"].auto_disabled_at is None runner.tick(now + 3) - assert ch.commands == sent + 1 + assert ch.fires == 4 def test_a_hunt_disabled_in_yaml_is_tombstoned_in_the_schedule(ch_client, dfe_db, tmp_path: Path): diff --git a/tests/unit/test_hunts/test_run_status.py b/tests/unit/test_hunts/test_run_status.py index 23f31f0c..4d2518b3 100644 --- a/tests/unit/test_hunts/test_run_status.py +++ b/tests/unit/test_hunts/test_run_status.py @@ -139,8 +139,9 @@ def test_a_null_reason_reads_as_empty(): def test_the_query_selects_the_auto_disable_columns_from_hunt_state(): ch = _Rows([]) read_run_status(ch, "dfe", ["h"], now=1000) - assert "argMax(auto_disabled_at, updated) AS disabled_at" in ch.sql - assert "argMax(auto_disabled_reason, updated) AS disabled_reason" in ch.sql + # One argMax over the tuple: argMax skips a NULL argument, and a cleared stop is one. + assert "argMax((auto_disabled_at, auto_disabled_reason), updated) AS disabled" in ch.sql + assert "toUnixTimestamp(tupleElement(s.disabled, 1))" in ch.sql def test_clearing_an_auto_disable_inserts_a_row_that_keeps_the_overrun_signal(): diff --git a/uv.lock b/uv.lock index acdaef8c..0e6d2083 100644 --- a/uv.lock +++ b/uv.lock @@ -714,7 +714,7 @@ requires-dist = [ { name = "confluent-kafka", specifier = ">=2.16.0" }, { name = "croniter", specifier = ">=6.2.2" }, { name = "cryptography", specifier = ">=50.0.0" }, - { name = "dfe-schemas", specifier = ">=0.2.29,<1" }, + { name = "dfe-schemas", specifier = ">=0.2.30,<1" }, { name = "dulwich", specifier = ">=1.2.7" }, { name = "faker", specifier = ">=40.43.0" }, { name = "fastapi", specifier = ">=0.143.0" }, @@ -769,14 +769,14 @@ dev = [ [[package]] name = "dfe-schemas" -version = "0.2.29" +version = "0.2.30" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ruamel-yaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/71/42553557973204e0c574485304e401f54a32f39f6094439680d63e889824/dfe_schemas-0.2.29.tar.gz", hash = "sha256:00b9387d9dcc2cd6e6151612672e5c335276e99b02b521215b0f6eae79d48d8c", size = 1304657, upload-time = "2026-11-11T07:16:15.082Z" } +sdist = { url = "https://files.pythonhosted.org/packages/08/ab/8772d67d0a51d86ff5fcbd4ecf0e5ec5bffe96ed684a43b75df55219df42/dfe_schemas-0.2.30.tar.gz", hash = "sha256:847d5f4e163cd5a4a50a21304613d406025019545107cbb672a951e2ae155e5c", size = 1292946, upload-time = "2026-11-16T05:01:41.465Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/cf/66a4e20949f5322e0fd299127dd1a12a3ef5fec213b4a7d5f73c4b370b84/dfe_schemas-0.2.29-py3-none-any.whl", hash = "sha256:bca1b3308c06c4aa7050455838f4bef3ba05afa7e113854705ea1679fec2e23b", size = 2837471, upload-time = "2026-11-11T07:16:13.324Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ee/0fb7e59844d9074d4175c15cd52e4a0732655ea0d917b06b30bb59594f8b/dfe_schemas-0.2.30-py3-none-any.whl", hash = "sha256:f1b7ca6c2f3cfa6d713b6d19772fc0eaa8e591c4ccd909dad975d08ac00e1070", size = 2818125, upload-time = "2026-11-16T05:01:40.022Z" }, ] [[package]] From db9cc6b37f343c2b217b3c9b902cc2dfab99212e Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 17:16:20 +1100 Subject: [PATCH 4/6] fix(hunts): keep hunts stopped across edits and failed clears An update that omits `enabled` keeps the stored flag. The write model's default of true wrote `enabled: true` on every PUT, so a cron edit re-enabled a hunt an operator had stopped. The field is optional on writes. A new hunt is enabled. The runner stores the stop in a separate hunt_state row, keyed `#stopped`. The overrun counter's read-then-write cannot overwrite the separate stop row. The clear writes the marker without reading it. Saving `enabled: true` clears the stop only if the write succeeds on the branch the runner reads. A failed clear returns 503 `auto_disable_not_cleared` instead of 200. The YAML is saved, but the hunt is still stopped. The smoothing page documents the shipped auto-disable settings and counter. The scaling page says a YAML `enabled: false` hunt is tombstoned like a deleted one. --- docs/data-plane/hunt-runner-scaling.md | 2 +- docs/data-plane/hunt-schedule-smoothing.md | 9 ++- openapi/openapi.json | 44 +++++++++-- src/dfe_engine/api/v1/hunts.py | 69 ++++++++++++++---- src/dfe_engine/hunts/ch_coordinator.py | 73 ++++++------------- src/dfe_engine/hunts/run_status.py | 37 +++++----- .../test_api/test_hunts_run_visibility.py | 48 +++++++++++- tests/unit/test_hunts/test_run_status.py | 20 ++--- tests/unit/test_hunts/test_runner.py | 17 +++++ 9 files changed, 207 insertions(+), 112 deletions(-) diff --git a/docs/data-plane/hunt-runner-scaling.md b/docs/data-plane/hunt-runner-scaling.md index a5aa23d6..d29acd8f 100644 --- a/docs/data-plane/hunt-runner-scaling.md +++ b/docs/data-plane/hunt-runner-scaling.md @@ -25,7 +25,7 @@ hunt_schedule(hunt_id, interval_seconds, phase_offset, enabled, updated) ReplacingMergeTree(updated) ORDER BY hunt_id ``` -`enabled` is a soft tombstone: a deleted hunt is written `enabled=0` so it stops counting as due, without a DELETE. The other coordination tables (`src/dfe_engine/hunts/ch_coordinator.py`) are `hunt_lease`, `hunt_watermark`, `hunt_state`, `hunt_run` and `hunt_runner_heartbeat`. `hunt_schedule` is the only one written outside the worker loop. +`enabled` is a soft tombstone: a deleted hunt is written `enabled=0` so it stops counting as due, without a DELETE. A hunt whose YAML says `enabled: false` is tombstoned the same way, since the loader keeps it out of the specs the materialiser publishes. The other coordination tables (`src/dfe_engine/hunts/ch_coordinator.py`) are `hunt_lease`, `hunt_watermark`, `hunt_state`, `hunt_run` and `hunt_runner_heartbeat`. `hunt_schedule` is the only one written outside the worker loop. The shim's `/keda/hunt-backlog` endpoint runs `schedule.due_query(db)`, which counts hunts that are due and unclaimed. It uses the same arithmetic the worker uses (`schedule.latest_fire`): diff --git a/docs/data-plane/hunt-schedule-smoothing.md b/docs/data-plane/hunt-schedule-smoothing.md index 4f5ee7f6..562c96b7 100644 --- a/docs/data-plane/hunt-schedule-smoothing.md +++ b/docs/data-plane/hunt-schedule-smoothing.md @@ -14,7 +14,7 @@ Without the flag the schedule is the fixed offset: each rate hunt fires at an of Rate hunts (`schedule.mode: rate`, the default) fire on fixed intervals. Left alone, the heavy ones stack into ClickHouse CPU and memory peaks. Smoothing learns what each hunt costs and drifts WHEN it fires, never how often, so the sum flattens over time. Anchored hunts and one-off monsters are meant to spike, so the loop leaves them alone. -Still design only, not in the 2.2.3 beta: the ClickHouse `WORKLOAD 'hunts'` backstop, promoting a settled schedule into git, and auto-disable. +Still design only, not in the 2.2.3 beta: the ClickHouse `WORKLOAD 'hunts'` backstop and promoting a settled schedule into git. Auto-disable ships in 2.2.3, outside this flag, and is described below. ## The picture @@ -43,7 +43,7 @@ A move writes one row in `hunt_schedule` and touches nothing a running hunt hold With the flag off, nothing reads the archive, nothing leads, and every offset is the hash. Turning the flag off after a run rewrites `hunt_schedule` back to the hash offsets on the runner's next start. -Rescheduling fixes contention and nothing else. A query that busts the memory limit on its own busts it at 3am too, and moving it changes nothing. The ladder for that is still design only: smooth, then back off the interval of a hunt that runs into itself, then auto-disable a hunt that trips code 241 K times in a window, written as a governed-ops `enabled: false` with the reason, alerted, and never re-enabled without a human. +Rescheduling fixes contention and nothing else. A query that busts the memory limit on its own busts it at 3am too, and moving it changes nothing. So the runner stops that hunt itself, smoothing flag or not: `hunts.autodisable.failures` resource-limit failures (codes 241, 158, 201) inside `hunts.autodisable.window_seconds` stop it, 3 in an hour by default. When and why land in `hunt_state` and on the hunt list, `hunt_autodisabled_total{hunt_id,reason}` counts it, and the hunt's alert destinations get the reason. Only a human brings it back, with `PUT /api/v1/hunts/{id}` naming `enabled: true` or Re-enable in the console, and an edit that leaves `enabled` alone leaves the stop alone. Backing off the interval of a hunt that runs into itself is still design only. ## What a hunt costs @@ -81,7 +81,10 @@ On ClickHouse Cloud `system.query_log` is per node, and a server upgrade can ren | `hunts.smoothing.weight_memory` | 0.5 | w_m, with w_c = 1 - w_m | | `hunts.smoothing.memory_cap_bytes` | 0 | M_cap for the hard constraint, 0 reads `max_server_memory_usage` | | `hunts.smoothing.horizon_max_seconds` | 86400 | cap on the hyperperiod H, the least common multiple of the intervals | -| `smoothing.settling_period`, `autodisable.*` | - | design only, see the top of this page | +| `hunts.autodisable.enabled` | true | the runner stops a hunt that keeps tripping a resource limit (outside the smoothing flag) | +| `hunts.autodisable.failures` | 3 | resource-limit failures inside the window that stop it | +| `hunts.autodisable.window_seconds` | 3600 | the window those failures are counted over | +| `smoothing.settling_period` | - | design only, see the top of this page | Environment form: `DFE_ENGINE_HUNTS_SMOOTHING_`. Fixed in code, not settings: the tie-break blurs the curve 3, 7 and 15 buckets wide, and the two observed gauges look back one hour. diff --git a/openapi/openapi.json b/openapi/openapi.json index 3ecb4194..ccb48b75 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -15020,7 +15020,7 @@ "hunts" ], "summary": "Get Hunt", - "description": "Get full hunt configuration by file name.", + "description": "Get full hunt configuration by file name, with the runner's stop on it if any.", "operationId": "hunts.get", "parameters": [ { @@ -41370,10 +41370,16 @@ "title": "Explain Queries" }, "enabled": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Enabled", - "description": "Whether the runner fires this hunt. False keeps its YAML, state and watermark and fires nothing.", - "default": true + "description": "Whether the runner fires this hunt. Omitted on an update keeps the stored value. True also clears a stop the runner put on the hunt." }, "rules": { "items": { @@ -41534,6 +41540,24 @@ "type": "object", "title": "Customer Tenant Ids", "description": "Each customer's tenant ids, resolved when the hunt was saved: the _org_id values the runner reads. A customer missing here stops every run." + }, + "auto_disabled_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Auto Disabled At", + "description": "When the runner stopped the hunt after repeated resource-limit failures, epoch seconds" + }, + "auto_disabled_reason": { + "type": "string", + "title": "Auto Disabled Reason", + "description": "The ClickHouse error category and code that tripped the stop", + "default": "" } }, "type": "object", @@ -41916,10 +41940,16 @@ "title": "Explain Queries" }, "enabled": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Enabled", - "description": "Whether the runner fires this hunt. False keeps its YAML, state and watermark and fires nothing.", - "default": true + "description": "Whether the runner fires this hunt. Omitted on an update keeps the stored value. True also clears a stop the runner put on the hunt." }, "rules": { "items": { diff --git a/src/dfe_engine/api/v1/hunts.py b/src/dfe_engine/api/v1/hunts.py index bdf51ccc..4ed31461 100644 --- a/src/dfe_engine/api/v1/hunts.py +++ b/src/dfe_engine/api/v1/hunts.py @@ -43,6 +43,7 @@ from dfe_engine.api.write_turn import WRITE_TURN from dfe_engine.auth.audit import audit_resource_change from dfe_engine.auth.rbac_scopes import scopes_dict +from dfe_engine.gitcrud.routing import WriteOutcome from dfe_engine.hunts.alert_hunt_link import delete_destinations_owned_by_hunt from dfe_engine.hunts.customers import CUSTOMER_TENANT_IDS, hunt_customers from dfe_engine.hunts.hunt_config_registry import ( @@ -255,6 +256,15 @@ class _HuntConfigFields(BaseModel): class HuntWriteRequest(_HuntConfigFields): """Hunt scheduler payload for create/update (without hunt file ``name``).""" + # None keeps the stored flag on an update, so an edit of the cron never re-enables + # a hunt an operator stopped. A new hunt is on. + enabled: bool | None = Field( # ty: ignore[incompatible-attribute-override] # the write model widens the field to mean "unchanged" + default=None, + description=( + "Whether the runner fires this hunt. Omitted on an update keeps the stored " + "value. True also clears a stop the runner put on the hunt." + ), + ) rules: list[str] = Field( min_length=1, description="Rule names, each the ``{name}.yaml`` rule file the hunt runner reads", @@ -309,6 +319,15 @@ class HuntDetailResponse(_HuntConfigFields): "values the runner reads. A customer missing here stops every run." ), ) + # The runner's stop rides on GET alone: a save answers with what it stored, and a + # save naming enabled true has just cleared the stop. + auto_disabled_at: int | None = Field( + default=None, + description="When the runner stopped the hunt after repeated resource-limit failures, epoch seconds", + ) + auto_disabled_reason: str = Field( + default="", description="The ClickHouse error category and code that tripped the stop" + ) @classmethod def from_stored_config(cls, name: str, config: dict[str, Any]) -> HuntDetailResponse: @@ -412,24 +431,37 @@ def _run_status_for(hunt_names: list[str], now: int) -> dict[str, RunStatus]: return {} -def _clear_auto_disable_if_enabled(name: str, *, enabled: bool) -> None: - """Re-enable a hunt the runner auto-disabled when the save turns it on. +def _clear_auto_disable(name: str, *, enabled: bool | None, outcome: WriteOutcome | None) -> None: + """Lift the runner's stop on a hunt when a direct save names ``enabled: true``. - Only a save that names ``enabled: true`` clears the stop, so an edit of the cron - or the rules leaves a hunt the runner stopped where it is. Best-effort like the - run state: the YAML is already saved, so an unreachable ClickHouse leaves the - stop in place for the next save rather than failing this one. + An edit that omits ``enabled`` leaves the stop where it is, and so does a save + routed to a review branch, since nothing has changed on the branch the runner + reads. The YAML is saved by now, so a failure is told to the caller as 503 + rather than answering 200 over a hunt that is still stopped. + + Raises: + ProblemError: 503 ``auto_disable_not_cleared`` when ClickHouse did not take it. """ - if not enabled: + if enabled is not True or (outcome is not None and outcome.review_required): return try: - run = _run_status_for([name], int(time.time())).get(name) + client, db = _data_database_client() + run = read_run_status(client, db, [name], int(time.time())).get(name) if run is None or run.auto_disabled_at is None: return - client, db = _data_database_client() clear_auto_disable(client, db, name) except Exception as exc: - logger.warning(f"could not clear the auto-disable on hunt '{name}': {exc}") + raise ProblemError( + 503, + f"Hunt '{name}' was saved, but the runner's stop on it could not be cleared: " + f"{exc}. Repeat the request once ClickHouse answers.", + code="auto_disable_not_cleared", + ) from exc + + +def _enabled_flag(value: Any) -> bool: + """A stored ``enabled`` as the flag the runner reads: absent is on, anything but True is off.""" + return value is None or value is True def _learned_offsets(enabled: bool) -> dict[str, tuple[int, int]]: @@ -490,7 +522,7 @@ def _hunt_row_to_summary( too_aggressive=run.too_aggressive, run_requested=run.run_requested, next_due=_next_due(row, now, fraction, learned), - enabled=bool(row.get("enabled", True)), + enabled=_enabled_flag(row.get("enabled")), auto_disabled_at=run.auto_disabled_at, auto_disabled_reason=run.auto_disabled_reason, ) @@ -624,12 +656,17 @@ def get_hunt( user: CurrentUser, registry: HuntConfigReg, ) -> HuntDetailResponse: - """Get full hunt configuration by file name.""" + """Get full hunt configuration by file name, with the runner's stop on it if any.""" try: config = registry.get(name) except HuntConfigNotFoundError: raise ProblemError(404, f"Hunt '{name}' not found", code="not_found") from None - return HuntDetailResponse.from_stored_config(name, config) + detail = HuntDetailResponse.from_stored_config(name, config) + run = _run_status_for([name], int(time.time())).get(name) + if run is not None: + detail.auto_disabled_at = run.auto_disabled_at + detail.auto_disabled_reason = run.auto_disabled_reason + return detail @router.put( @@ -660,12 +697,12 @@ def update_hunt( _refuse_new_rule_names(body.rules, stored) config = body.to_config_dict(hunt_name=name) + if body.enabled is None and "enabled" in stored: + config["enabled"] = stored["enabled"] _validate_hunt_config(config, settings, rules) _resolve_customers(request, config) outcome = registry.save(name, config, created_by=user.user_id) - _clear_auto_disable_if_enabled( - name, enabled="enabled" in body.model_fields_set and body.enabled - ) + _clear_auto_disable(name, enabled=body.enabled, outcome=outcome) apply_review_headers(response, outcome) audit_resource_change(user.user_id, "hunt", name, "updated", review_audit_detail(outcome)) return HuntDetailResponse.from_stored_config(name, config) diff --git a/src/dfe_engine/hunts/ch_coordinator.py b/src/dfe_engine/hunts/ch_coordinator.py index e8a5db52..61b7aac5 100644 --- a/src/dfe_engine/hunts/ch_coordinator.py +++ b/src/dfe_engine/hunts/ch_coordinator.py @@ -112,21 +112,17 @@ class Lease: @dataclass(frozen=True, slots=True) class HuntStateRow: - """Persisted UI signal for a hunt: overrun count, too-aggressive, and an auto-disable. + """Persisted UI signal for a hunt: overrun count and too-aggressive. Attributes: hunt_id: The hunt. overrun_count: Fires deferred because the previous one still ran. too_aggressive: Whether the schedule is tighter than the hunt keeps up with. - auto_disabled_at: Epoch seconds the runner stopped the hunt, None while it runs. - auto_disabled_reason: The error category and code that tripped it, empty otherwise. """ hunt_id: str overrun_count: int too_aggressive: bool - auto_disabled_at: int | None = None - auto_disabled_reason: str = "" _STATE_COLUMNS = [ @@ -137,6 +133,10 @@ class HuntStateRow: "auto_disabled_reason", ] +# The runner's stop on a hunt is its own hunt_state row, keyed ``#stopped``, so +# the counter row's read-then-write never carries it and no overrun can overwrite it. +STOP_MARK = "#stopped" + class ChCoordinator: """Hunt coordination in ClickHouse: lease (claim), watermark, and state. @@ -527,69 +527,48 @@ def set_safe_horizon_mark(self, hunt_id: str, mark: str, value: int) -> None: def get_state(self, hunt_id: str) -> HuntStateRow | None: """The hunt's persisted overrun / too-aggressive signal, if any.""" rows = self._ch.query( - "SELECT overrun_count, too_aggressive, toUnixTimestamp(auto_disabled_at), " # noqa: S608 - quoted configured database, values bound - f"auto_disabled_reason FROM {self._qdb}.hunt_state " + f"SELECT overrun_count, too_aggressive FROM {self._qdb}.hunt_state " # noqa: S608 - quoted configured database, values bound "WHERE hunt_id = {h:String} ORDER BY updated DESC LIMIT 1", parameters={"h": hunt_id}, ).result_rows if not rows: return None - disabled_at = rows[0][2] return HuntStateRow( - hunt_id=hunt_id, - overrun_count=int(rows[0][0]), - too_aggressive=bool(rows[0][1]), - auto_disabled_at=int(disabled_at) if disabled_at is not None else None, - auto_disabled_reason=str(rows[0][3] or ""), + hunt_id=hunt_id, overrun_count=int(rows[0][0]), too_aggressive=bool(rows[0][1]) ) def auto_disabled_hunts(self) -> dict[str, str]: """Every hunt the runner has stopped, as ``hunt_id -> reason``, read fresh each tick. - The API clears an auto-disable by writing a row with a NULL time, so a + The API clears a stop by writing its marker row with a NULL time, so a re-enabled hunt leaves this map on the next read and fires again. argMax skips a NULL argument, so the latest row is taken as a tuple, which is never NULL. """ rows = self._ch.query( - "SELECT hunt_id, tupleElement(s, 2) FROM (SELECT hunt_id, " # noqa: S608 - quoted configured database + "SELECT k, tupleElement(s, 2) FROM (SELECT hunt_id AS k, " # noqa: S608 - quoted configured database, values bound "argMax((auto_disabled_at, auto_disabled_reason), updated) AS s " - f"FROM {self._qdb}.hunt_state GROUP BY hunt_id) WHERE tupleElement(s, 1) IS NOT NULL" + f"FROM {self._qdb}.hunt_state WHERE endsWith(hunt_id, {{mark:String}}) " + "GROUP BY hunt_id) WHERE tupleElement(s, 1) IS NOT NULL", + parameters={"mark": STOP_MARK}, ).result_rows - return {str(r[0]): str(r[1]) for r in rows} + return {str(r[0])[: -len(STOP_MARK)]: str(r[1]) for r in rows} def _write_state(self, row: HuntStateRow) -> None: - """Write the hunt's whole state row, since a ReplacingMergeTree row replaces all columns.""" - disabled_at = ( - datetime.fromtimestamp(row.auto_disabled_at, tz=UTC) - if row.auto_disabled_at is not None - else None - ) + """Write the hunt's counter row. The stop lives in its own row, so this never carries it.""" self._ch.insert( "hunt_state", - [ - [ - row.hunt_id, - row.overrun_count, - 1 if row.too_aggressive else 0, - disabled_at, - row.auto_disabled_reason, - ] - ], + [[row.hunt_id, row.overrun_count, 1 if row.too_aggressive else 0, None, ""]], column_names=_STATE_COLUMNS, database=self._db, ) def record_auto_disable(self, hunt_id: str, reason: str, now: int) -> None: - """Stop the hunt: stamp when and why, keeping its overrun counters.""" - current = self.get_state(hunt_id) - self._write_state( - HuntStateRow( - hunt_id=hunt_id, - overrun_count=current.overrun_count if current else 0, - too_aggressive=current.too_aggressive if current else False, - auto_disabled_at=now, - auto_disabled_reason=reason, - ) + """Stop the hunt: write its marker row with when and why. No read, so nothing to race.""" + self._ch.insert( + "hunt_state", + [[f"{hunt_id}{STOP_MARK}", 0, 0, datetime.fromtimestamp(now, tz=UTC), reason]], + column_names=_STATE_COLUMNS, + database=self._db, ) # ---- runs (row count + operator run-now) -------------------------- @@ -655,12 +634,4 @@ def record_overrun(self, hunt_id: str) -> None: """ current = self.get_state(hunt_id) count = (current.overrun_count if current else 0) + 1 - self._write_state( - HuntStateRow( - hunt_id=hunt_id, - overrun_count=count, - too_aggressive=True, - auto_disabled_at=current.auto_disabled_at if current else None, - auto_disabled_reason=current.auto_disabled_reason if current else "", - ) - ) + self._write_state(HuntStateRow(hunt_id=hunt_id, overrun_count=count, too_aggressive=True)) diff --git a/src/dfe_engine/hunts/run_status.py b/src/dfe_engine/hunts/run_status.py index f4e9b4bb..1619f7f0 100644 --- a/src/dfe_engine/hunts/run_status.py +++ b/src/dfe_engine/hunts/run_status.py @@ -25,12 +25,11 @@ from a lease that only exists while a hunt is mid-execution. """ -from __future__ import annotations - from dataclasses import dataclass from typing import Any from dfe_engine.clickhouse.quoting import quote_identifier +from dfe_engine.hunts.ch_coordinator import STOP_MARK @dataclass(frozen=True, slots=True) @@ -61,17 +60,22 @@ def _query(database: str) -> str: "SELECT h.hunt_id, coalesce(w.wm, 0), coalesce(l.lu, 0), " # noqa: S608 - quoted configured database, ids bound "coalesce(s.overruns, 0), coalesce(s.aggressive, 0), " "coalesce(r.last_fire, 0), coalesce(r.rows_written, 0), coalesce(q.pending, 0), " - "toUnixTimestamp(tupleElement(s.disabled, 1)), coalesce(tupleElement(s.disabled, 2), '') " + "toUnixTimestamp(tupleElement(d.disabled, 1)), coalesce(tupleElement(d.disabled, 2), '') " "FROM (SELECT arrayJoin({ids:Array(String)}) AS hunt_id) h " "LEFT JOIN (SELECT hunt_id, argMax(watermark, updated) AS wm " f"FROM {db}.hunt_watermark GROUP BY hunt_id) w USING (hunt_id) " "LEFT JOIN (SELECT hunt_id, argMax(lease_until, claimed) AS lu " f"FROM {db}.hunt_lease GROUP BY hunt_id) l USING (hunt_id) " "LEFT JOIN (SELECT hunt_id, argMax(overrun_count, updated) AS overruns, " - "argMax(too_aggressive, updated) AS aggressive, " - # As a tuple, because argMax skips a NULL argument and a cleared auto-disable is one. - "argMax((auto_disabled_at, auto_disabled_reason), updated) AS disabled " + "argMax(too_aggressive, updated) AS aggressive " f"FROM {db}.hunt_state GROUP BY hunt_id) s USING (hunt_id) " + # The stop is its own marker row, read as one tuple: argMax skips a NULL + # argument, and a cleared stop is one. + f"LEFT JOIN (SELECT substring(k, 1, length(k) - {len(STOP_MARK)}) AS hunt_id, " + "d AS disabled FROM (SELECT hunt_id AS k, " + "argMax((auto_disabled_at, auto_disabled_reason), updated) AS d " + f"FROM {db}.hunt_state WHERE endsWith(hunt_id, {{mark:String}}) GROUP BY hunt_id)) " + "d USING (hunt_id) " "LEFT JOIN (SELECT hunt_id, max(fire) AS last_fire, " "argMax(rows_written, fire) AS rows_written FROM (" "SELECT hunt_id, fire, argMax(status, updated) AS status, " @@ -117,7 +121,9 @@ def read_run_status(ch: Any, database: str, hunt_ids: list[str], now: int) -> di """ if not hunt_ids: return {} - rows = ch.query(_query(database), parameters={"ids": list(hunt_ids)}).result_rows + rows = ch.query( + _query(database), parameters={"ids": list(hunt_ids), "mark": STOP_MARK} + ).result_rows status: dict[str, RunStatus] = {} for row in rows: hunt_id = str(row[0]) @@ -138,26 +144,19 @@ def read_run_status(ch: Any, database: str, hunt_ids: list[str], now: int) -> di def clear_auto_disable(ch: Any, database: str, hunt_id: str) -> None: - """Insert the hunt_state row that replaces a disabled one. Only the API calls it. + """Write the hunt's stop marker row with a NULL time, which lifts the runner's stop. - The overrun count and too-aggressive flag are read back and carried over, so - clearing the auto-disable leaves the UI's overrun signal as it was. + The stop is its own row, keyed ``#stopped``, so the overrun counters are + untouched and there is no read to race the runner. Args: - ch: a ClickHouse client exposing ``query`` and ``insert``. + ch: a ClickHouse client exposing ``insert``. database: the data database the coordination tables live in. hunt_id: the hunt to re-enable. """ - rows = ch.query( - "SELECT argMax(overrun_count, updated), argMax(too_aggressive, updated) " # noqa: S608 - quoted configured database, id bound - f"FROM {quote_identifier(database)}.hunt_state WHERE hunt_id = {{h:String}}", - parameters={"h": hunt_id}, - ).result_rows - overrun_count = int(rows[0][0]) if rows else 0 - too_aggressive = int(rows[0][1]) if rows else 0 ch.insert( "hunt_state", - [[hunt_id, overrun_count, too_aggressive, None, ""]], + [[f"{hunt_id}{STOP_MARK}", 0, 0, None, ""]], column_names=[ "hunt_id", "overrun_count", diff --git a/tests/unit/test_api/test_hunts_run_visibility.py b/tests/unit/test_api/test_hunts_run_visibility.py index 2ab698d0..32437d3d 100644 --- a/tests/unit/test_api/test_hunts_run_visibility.py +++ b/tests/unit/test_api/test_hunts_run_visibility.py @@ -220,7 +220,7 @@ def test_enabling_an_auto_disabled_hunt_clears_the_auto_disable( assert ch.inserts == [ ( "hunt_state", - [[created_hunt, 3, 1, None, ""]], + [[f"{created_hunt}#stopped", 0, 0, None, ""]], [ "hunt_id", "overrun_count", @@ -253,7 +253,7 @@ def test_a_save_that_does_not_re_enable_an_auto_disabled_hunt_writes_no_state( assert ch.inserts == [] -def test_a_save_still_succeeds_when_the_auto_disable_cannot_be_cleared( +def test_a_clear_clickhouse_did_not_take_is_a_503_not_a_quiet_200( client, admin_headers, created_hunt, coordination ): ch = coordination(1700000000, "resource_limit:241") @@ -265,4 +265,46 @@ def _refuse(*args, **kwargs): resp = client.put( f"/api/v1/hunts/{created_hunt}", json=_update_body(enabled=True), headers=admin_headers ) - assert resp.status_code == 200 + assert resp.status_code == 503 + assert resp.json()["code"] == "auto_disable_not_cleared" + + +def test_a_save_routed_to_a_review_branch_lifts_no_stop(created_hunt, coordination): + from dfe_engine.api.v1.hunts import _clear_auto_disable + from dfe_engine.gitcrud.routing import WriteOutcome + + ch = coordination(1700000000, "resource_limit:241") + routed = WriteOutcome(changed=True, review_required=True, branch="dfe/hunt/x/1") + _clear_auto_disable(created_hunt, enabled=True, outcome=routed) + assert ch.inserts == [] + _clear_auto_disable(created_hunt, enabled=True, outcome=WriteOutcome(changed=True)) + assert len(ch.inserts) == 1 + + +def test_an_edit_that_omits_enabled_keeps_a_hunt_disabled(client, admin_headers, created_hunt): + off = client.put( + f"/api/v1/hunts/{created_hunt}", json=_update_body(enabled=False), headers=admin_headers + ) + assert off.status_code == 200 + assert off.json()["enabled"] is False + edit = client.put( + f"/api/v1/hunts/{created_hunt}", + json=_update_body(cron="*/7 * * * *"), + headers=admin_headers, + ) + assert edit.status_code == 200 + assert edit.json()["enabled"] is False + assert ( + client.get(f"/api/v1/hunts/{created_hunt}", headers=admin_headers).json()["enabled"] + is False + ) + + +def test_the_detail_carries_the_runners_stop_so_a_direct_link_shows_it( + client, admin_headers, created_hunt, coordination +): + coordination(1700000000, "resource_limit:241") + detail = client.get(f"/api/v1/hunts/{created_hunt}", headers=admin_headers).json() + assert detail["enabled"] is True + assert detail["auto_disabled_at"] == 1700000000 + assert detail["auto_disabled_reason"] == "resource_limit:241" diff --git a/tests/unit/test_hunts/test_run_status.py b/tests/unit/test_hunts/test_run_status.py index 4d2518b3..25319247 100644 --- a/tests/unit/test_hunts/test_run_status.py +++ b/tests/unit/test_hunts/test_run_status.py @@ -140,18 +140,20 @@ def test_the_query_selects_the_auto_disable_columns_from_hunt_state(): ch = _Rows([]) read_run_status(ch, "dfe", ["h"], now=1000) # One argMax over the tuple: argMax skips a NULL argument, and a cleared stop is one. - assert "argMax((auto_disabled_at, auto_disabled_reason), updated) AS disabled" in ch.sql - assert "toUnixTimestamp(tupleElement(s.disabled, 1))" in ch.sql + assert "argMax((auto_disabled_at, auto_disabled_reason), updated) AS d" in ch.sql + assert "WHERE endsWith(hunt_id, {mark:String})" in ch.sql + assert "toUnixTimestamp(tupleElement(d.disabled, 1))" in ch.sql -def test_clearing_an_auto_disable_inserts_a_row_that_keeps_the_overrun_signal(): +def test_clearing_an_auto_disable_writes_the_stop_marker_with_no_time_and_reads_nothing(): ch = _Recording([[3, 1]]) clear_auto_disable(ch, "dfe", "h") - assert ch.parameters == {"h": "h"} + # No read: the counters live in the hunt's own row and are not touched. + assert ch.parameters is None assert ch.inserts == [ ( "hunt_state", - [["h", 3, 1, None, ""]], + [["h#stopped", 0, 0, None, ""]], [ "hunt_id", "overrun_count", @@ -164,12 +166,6 @@ def test_clearing_an_auto_disable_inserts_a_row_that_keeps_the_overrun_signal(): ] -def test_clearing_a_hunt_with_no_state_row_writes_zeros(): - ch = _Recording([]) - clear_auto_disable(ch, "dfe", "fresh") - assert ch.inserts[0][1] == [["fresh", 0, 0, None, ""]] - - def test_no_hunt_ids_asks_clickhouse_nothing(): ch = _Rows([]) assert read_run_status(ch, "dfe", [], now=1000) == {} @@ -181,7 +177,7 @@ def test_the_hunt_ids_are_passed_as_a_bound_parameter(): # than interpolated into the SQL. ch = _Rows([]) read_run_status(ch, "dfe", ["a", "b"], now=1000) - assert ch.parameters == {"ids": ["a", "b"]} + assert ch.parameters == {"ids": ["a", "b"], "mark": "#stopped"} assert "{ids:Array(String)}" in ch.sql diff --git a/tests/unit/test_hunts/test_runner.py b/tests/unit/test_hunts/test_runner.py index 84eadc06..6a6c6ebc 100644 --- a/tests/unit/test_hunts/test_runner.py +++ b/tests/unit/test_hunts/test_runner.py @@ -616,6 +616,19 @@ def test_the_back_off_doubles_from_its_first_step_up_to_its_cap(): assert set(gaps[3:]) == {120} +def test_autodisable_off_in_settings_counts_nothing(): + from dfe_engine.settings import HuntAutodisableSettings, HuntsSettings + + off = HuntFailures.from_settings( + HuntsSettings(autodisable=HuntAutodisableSettings(enabled=False, failures=1)) + ) + assert off.limit_tripped("h", 0, _code(241)) is None + on = HuntFailures.from_settings( + HuntsSettings(autodisable=HuntAutodisableSettings(enabled=True, failures=1)) + ) + assert on.limit_tripped("h", 0, _code(241)) == "resource_limit:241" + + def test_the_back_off_takes_its_start_and_cap_from_the_hunts_settings(): from dfe_engine.settings import HuntsSettings @@ -800,6 +813,10 @@ def _code(code: int) -> Exception: 3, 3600, [(0, 516), (10, 241), (20, 241)], None, id="an_auth_failure_is_not_a_hit" ), pytest.param(0, 3600, [(0, 241), (10, 241), (20, 241)], None, id="autodisable_off"), + pytest.param( + 2, 60, [(0, 241), (60, 241)], "resource_limit:241", id="a_hit_on_the_window_edge_counts" + ), + pytest.param(2, 60, [(0, 241), (61, 241)], None, id="one_past_the_edge_does_not"), ], ) def test_limit_tripped_counts_only_resource_limits_inside_the_window(failures, window, hits, want): From 0a4085bbc44bdbf0c87958b9db90973f5e2cb8f4 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 17:42:34 +1100 Subject: [PATCH 5/6] fix(hunts): declare enabled on the write model Both models declare `enabled`. The detail defaults to true. A write defaults to None so an omitted flag keeps the stored value. Neither model overrides a parent field, so no type-check suppression is needed. CI and local ty results match. --- openapi/openapi.json | 10 +++++----- src/dfe_engine/api/v1/hunts.py | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/openapi/openapi.json b/openapi/openapi.json index ccb48b75..eb5f93bd 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -41511,17 +41511,17 @@ ], "title": "Explain Queries" }, + "name": { + "type": "string", + "title": "Name", + "description": "Hunt file name (YAML stem)" + }, "enabled": { "type": "boolean", "title": "Enabled", "description": "Whether the runner fires this hunt. False keeps its YAML, state and watermark and fires nothing.", "default": true }, - "name": { - "type": "string", - "title": "Name", - "description": "Hunt file name (YAML stem)" - }, "rules": { "items": { "$ref": "#/components/schemas/HuntRuleEntry" diff --git a/src/dfe_engine/api/v1/hunts.py b/src/dfe_engine/api/v1/hunts.py index 4ed31461..3bba7411 100644 --- a/src/dfe_engine/api/v1/hunts.py +++ b/src/dfe_engine/api/v1/hunts.py @@ -244,21 +244,14 @@ class _HuntConfigFields(BaseModel): scheduling_mode: str | None = None min_interval_seconds: int | None = None explain_queries: bool | None = None - enabled: bool = Field( - default=True, - description=( - "Whether the runner fires this hunt. False keeps its YAML, state and watermark " - "and fires nothing." - ), - ) class HuntWriteRequest(_HuntConfigFields): """Hunt scheduler payload for create/update (without hunt file ``name``).""" - # None keeps the stored flag on an update, so an edit of the cron never re-enables - # a hunt an operator stopped. A new hunt is on. - enabled: bool | None = Field( # ty: ignore[incompatible-attribute-override] # the write model widens the field to mean "unchanged" + # The read model carries its own enabled flag: here None keeps the stored flag on + # an update, so an edit of the cron never re-enables a hunt an operator stopped. + enabled: bool | None = Field( default=None, description=( "Whether the runner fires this hunt. Omitted on an update keeps the stored " @@ -307,6 +300,13 @@ class HuntDetailResponse(_HuntConfigFields): name: str = Field(description="Hunt file name (YAML stem)") display_name: str = Field(description="Resolved human-readable label") # ty: ignore[invalid-mutable-override] # the response model narrows the field, never writes it + enabled: bool = Field( + default=True, + description=( + "Whether the runner fires this hunt. False keeps its YAML, state and watermark " + "and fires nothing." + ), + ) rules: list[HuntRuleEntry] = Field(min_length=1) # A stored hunt naming none is shown, and runs nothing. customers: list[str] = Field( From fa308f8e39d187c893d51683b7e17d0a65d30d50 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 18:11:58 +1100 Subject: [PATCH 6/6] fix(tests): match dfe-schemas LowCardinality rules dfe-schemas 0.2.30 limits LowCardinality to String, FixedString and UUID. ClickHouse rejects LowCardinality over a number without allow_suspicious_low_cardinality_types (code 455). DFE never sets it. Type registry tests check rejection of a number, float, date or address with `lowcardinality`. The override case wraps a FixedString. --- tests/unit/test_source/test_type_registry.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_source/test_type_registry.py b/tests/unit/test_source/test_type_registry.py index 510ae1e1..29b9d07a 100644 --- a/tests/unit/test_source/test_type_registry.py +++ b/tests/unit/test_source/test_type_registry.py @@ -150,9 +150,11 @@ def test_not_null_override(self, registry: TypeRegistry): r = registry.resolve("string", attributes=["not_null"]) assert r.ch_type == "String" - def test_lowcardinality_integer(self, registry: TypeRegistry): - r = registry.resolve("integer", attributes=["lowcardinality"]) - assert r.ch_type == "LowCardinality(Nullable(Int64))" + def test_lowcardinality_integer_is_refused(self, registry: TypeRegistry): + # ClickHouse refuses LowCardinality over a number without + # allow_suspicious_low_cardinality_types (code 455), which DFE never sets. + with pytest.raises(InvalidAttributeError): + registry.resolve("integer", attributes=["lowcardinality"]) # --------------------------------------------------------------------------- @@ -167,8 +169,8 @@ def test_override_verbatim(self, registry: TypeRegistry): assert r.codec is None def test_override_with_lowcardinality(self, registry: TypeRegistry): - r = registry.resolve("integer", attributes=["lowcardinality"], ch_override="UInt16") - assert r.ch_type == "LowCardinality(UInt16)" + r = registry.resolve("string", attributes=["lowcardinality"], ch_override="FixedString(16)") + assert r.ch_type == "LowCardinality(FixedString(16))" assert r.codec is None def test_override_parameterised(self, registry: TypeRegistry): @@ -277,11 +279,14 @@ def test_unknown_use_case(self, registry: TypeRegistry): class TestValidateAttribute: - @pytest.mark.parametrize("primitive", ["string", "text", "integer", "float", "date", "ip"]) + @pytest.mark.parametrize("primitive", ["string", "text"]) def test_lowcardinality_valid(self, registry: TypeRegistry, primitive: str): registry.validate_attribute(primitive, "lowcardinality") - @pytest.mark.parametrize("primitive", ["json", "geo_point", "boolean", "uuid", "enum"]) + @pytest.mark.parametrize( + "primitive", + ["json", "geo_point", "boolean", "uuid", "enum", "integer", "float", "date", "ip"], + ) def test_lowcardinality_invalid(self, registry: TypeRegistry, primitive: str): with pytest.raises(InvalidAttributeError): registry.validate_attribute(primitive, "lowcardinality")