diff --git a/CHANGELOG.md b/CHANGELOG.md index 4806cff..ca8c22d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,16 @@ codes documented in `AGENT.md` are the public API. ### Fixed +- **Webhook filters work.** Both halves of `[webhook.filters]` were dead. + `chats` was checked against a list nothing ever filled, so setting it + dropped every event that had a chat; `@name` entries are now resolved + through the event's account and retried once a minute if that fails. + The other keys (`chat_type`, `contains`, ...) were evaluated against the + raw TL update, which the filters cannot read, so they raised or never + matched. They now see the same Telethon event a gateway job does. Event + types Telethon has no event for are not delivered while such a filter is + set, and the daemon says so once per type. + - **Gateway jobs work again, and incoming updates reach the daemon at all.** Four faults stacked up, and each one alone was enough to stop every job: diff --git a/tests/test_webhook.py b/tests/test_webhook.py index 30f99cf..010fe36 100644 --- a/tests/test_webhook.py +++ b/tests/test_webhook.py @@ -227,6 +227,53 @@ async def test_the_daemon_subscribes_the_webhook_to_the_bus( await daemon.shutdown(drain=0.1) assert json.loads(receiver["received"][0]["body"])["event"]["payload"]["id"] == 1 + async def test_a_filtered_webhook_delivers_an_incoming_update( + self, receiver, tlgr_home: Path, stub_account, world + ): + """End to end: Telethon update → bus → filters on a built event → POST.""" + from fake_telethon import fake_client_factory + from telethon.tl import types + + from tlgr.core.config import WebhookFilterConfig, save_webhook_config + from tlgr.daemon.app import Daemon + from tlgr.daemon.session import SessionState + + save_webhook_config( + WebhookConfig( + enabled=True, + url=receiver["url"], + secret="k", + events=["message_new"], + filters=WebhookFilterConfig(raw={"chat_type": "private"}), + ), + tlgr_home, + ) + daemon = Daemon(tlgr_home, client_factory=fake_client_factory(world)) + await daemon.start_services() + try: + session = await daemon.sessions.ensure(stub_account) + for _ in range(200): + if session.state == SessionState.ONLINE: + break + await asyncio.sleep(0.01) + for peer, text in ((types.PeerChannel(channel_id=9), "post"), (None, "dm")): + message = types.Message( + id=len(text), + peer_id=peer or types.PeerUser(user_id=4242), + from_id=types.PeerUser(user_id=4242), + date=None, + message=text, + ) + await session.client.feed( + types.UpdateNewMessage(message=message, pts=1, pts_count=1) + ) + await _drain(daemon.webhook, receiver["received"]) + await asyncio.sleep(0.1) + finally: + await daemon.shutdown(drain=0.1) + bodies = [json.loads(r["body"])["event"] for r in receiver["received"]] + assert [b["payload"]["text"] for b in bodies] == ["dm"] + async def test_an_enabled_webhook_disables_idle_stop(self, tlgr_home: Path, stub_account): """COR-08: a daemon that exits has silently unsubscribed.""" from tlgr.core.config import save_webhook_config diff --git a/tests/test_webhook_filters.py b/tests/test_webhook_filters.py new file mode 100644 index 0000000..0080bcb --- /dev/null +++ b/tests/test_webhook_filters.py @@ -0,0 +1,167 @@ +"""`[webhook.filters]` against the updates the daemon's bus really carries. + +Both halves were dead. `filters.chats` was compared against a set nothing +filled, so configuring it dropped every event with a chat. Every other key +was evaluated against the raw TL update, while the filters read a +high-level Telethon event, so they raised (the bus logged "event handler +failed") or never matched. These tests use real TL updates and Telethon +builders, the way the bus delivers them. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from fake_telethon import FakeTelegramClient, World +from telethon.tl import types + +from tlgr.core.config import WebhookConfig, WebhookFilterConfig +from tlgr.daemon.webhook import WebhookPusher +from tlgr.models.event import EventEnvelope + +SOURCE = 1234 +SOURCE_MARKED = -1000000001234 +ALICE = 4242 + + +def _channel_post(text: str = "breaking news", *, channel: int = SOURCE): + message = types.Message( + id=5, peer_id=types.PeerChannel(channel_id=channel), date=None, message=text + ) + return types.UpdateNewChannelMessage(message=message, pts=1, pts_count=1) + + +def _private(text: str = "hi", *, out: bool = False): + message = types.Message( + id=6, + peer_id=types.PeerUser(user_id=ALICE), + from_id=None if out else types.PeerUser(user_id=ALICE), + date=None, + message=text, + out=out, + ) + return types.UpdateNewMessage(message=message, pts=1, pts_count=1) + + +def _envelope(update, event_type: str = "message_new") -> EventEnvelope: + peer = update.message.peer_id if hasattr(update, "message") else None + if isinstance(peer, types.PeerChannel): + chat_id = int(f"-100{peer.channel_id:010d}") + elif isinstance(peer, types.PeerUser): + chat_id = peer.user_id + else: + chat_id = None + return EventEnvelope( + seq=1, ts="2026-10-03T12:00:00Z", account="Neo", type=event_type, chat_id=chat_id + ) + + +@pytest.fixture +async def account(): + return SimpleNamespace( + client=FakeTelegramClient(World()), + resolve_chat=AsyncMock(return_value=SOURCE_MARKED), + ) + + +def _pusher(account, *, chats=(), events=("message_new",), **filters) -> tuple[WebhookPusher, list]: + config = WebhookConfig( + enabled=True, + url="http://127.0.0.1:1/hook", + events=list(events), + filters=WebhookFilterConfig(chats=list(chats), raw=filters), + ) + pusher = WebhookPusher(config, accounts=lambda alias: account if alias == "Neo" else None) + sent: list = [] + pusher.enqueue = sent.append # type: ignore[method-assign] + return pusher, sent + + +async def _push(pusher: WebhookPusher, update, event_type: str = "message_new") -> None: + await pusher.on_event(_envelope(update, event_type), update) + + +class TestFilterKeys: + async def test_chat_type_lets_a_private_message_through(self, account): + pusher, sent = _pusher(account, chat_type="private") + await _push(pusher, _private()) + assert len(sent) == 1 + + async def test_chat_type_stops_a_channel_post(self, account): + pusher, sent = _pusher(account, chat_type="private") + await _push(pusher, _channel_post()) + assert sent == [] + + async def test_contains_reads_the_message_text(self, account): + pusher, sent = _pusher(account, contains=["breaking"]) + await _push(pusher, _channel_post("breaking news")) + await _push(pusher, _channel_post("weather")) + assert len(sent) == 1 + + async def test_the_accounts_own_messages_are_still_reported(self, account): + """Unlike a job, a webhook reports both directions; `is_incoming` narrows it.""" + pusher, sent = _pusher(account, chat_type="private") + await _push(pusher, _private(out=True)) + assert len(sent) == 1 + + narrowed, narrowed_sent = _pusher(account, chat_type="private", is_incoming=True) + await _push(narrowed, _private(out=True)) + assert narrowed_sent == [] + + async def test_a_type_telethon_does_not_model_is_held_back_and_logged_once( + self, account, caplog + ): + pusher, sent = _pusher(account, events=("typing",), chat_type="private") + update = types.UpdateUserTyping(user_id=ALICE, action=types.SendMessageTypingAction()) + for _ in range(2): + await pusher.on_event( + EventEnvelope(seq=1, ts="t", account="Neo", type="typing", chat_id=ALICE), update + ) + assert sent == [] + assert caplog.text.count("cannot be applied to typing events") == 1 + + async def test_the_daemons_own_sends_skip_the_filter_keys(self, account): + pusher, sent = _pusher(account, chat_type="private") + await pusher.on_event(_envelope(_channel_post()), None) + assert len(sent) == 1 + + async def test_an_account_that_is_not_connected_delivers_nothing_filtered(self, account): + pusher, sent = _pusher(account, chat_type="private") + envelope = _envelope(_private()) + envelope.account = "offline" + await pusher.on_event(envelope, _private()) + assert sent == [] + + +class TestChats: + async def test_a_named_chat_is_resolved_and_matched(self, account): + pusher, sent = _pusher(account, chats=["@source"]) + await _push(pusher, _channel_post()) + await _push(pusher, _channel_post(channel=777)) + await _push(pusher, _channel_post()) + assert len(sent) == 2 + account.resolve_chat.assert_awaited_once_with("@source") + + async def test_a_numeric_chat_needs_no_account(self, account): + pusher, sent = _pusher(account, chats=[str(SOURCE_MARKED)]) + envelope = _envelope(_channel_post()) + envelope.account = "offline" + await pusher.on_event(envelope, _channel_post()) + assert len(sent) == 1 + + async def test_an_unresolvable_chat_is_logged_and_retried_later(self, account, caplog): + account.resolve_chat = AsyncMock(side_effect=[ValueError("offline"), SOURCE_MARKED]) + pusher, sent = _pusher(account, chats=["@source"]) + + await _push(pusher, _channel_post()) + assert sent == [] + assert "webhook cannot resolve chat @source" in caplog.text + + await _push(pusher, _channel_post()) # still inside the retry interval + assert account.resolve_chat.await_count == 1 + + pusher._next_resolve = 0.0 + await _push(pusher, _channel_post()) + assert len(sent) == 1 diff --git a/tlgr/daemon/app.py b/tlgr/daemon/app.py index bb6e68c..ffeb0eb 100644 --- a/tlgr/daemon/app.py +++ b/tlgr/daemon/app.py @@ -100,7 +100,7 @@ def __init__( **factory_kwargs, ) self.webhook_config = load_webhook_config(self.paths.base) - self.webhook = WebhookPusher(self.webhook_config, self.paths.base) + self.webhook = WebhookPusher(self.webhook_config, self.paths.base, accounts=self.get_client) self._job_runner = JobRunner() # Long file transfers: the ones `--background` hands over, and the # per-DC budgets that stop one 2 GB download from starving five diff --git a/tlgr/daemon/webhook.py b/tlgr/daemon/webhook.py index d47165d..2ee3b3e 100644 --- a/tlgr/daemon/webhook.py +++ b/tlgr/daemon/webhook.py @@ -30,6 +30,7 @@ import random import time import uuid +from collections.abc import Callable from pathlib import Path from typing import Any @@ -51,15 +52,37 @@ _DEAD_LETTER_MAX_BYTES = 16 * 1024 * 1024 _DEAD_LETTER_BACKUPS = 3 +#: How long before retrying a `filters.chats` ref that did not resolve, +#: typically because the account was offline when the first event came in. +_RESOLVE_RETRY_SECONDS = 60.0 + class WebhookPusher: """Bounded queue + worker pool + HMAC + dead letter.""" - def __init__(self, config: WebhookConfig, base: Path | None = None) -> None: + def __init__( + self, + config: WebhookConfig, + base: Path | None = None, + *, + accounts: Callable[[str], Any] | None = None, + ) -> None: self.config = config self.base = Path(base) if base is not None else CONFIG_DIR self._session: Any = None - self._resolved_chat_ids: set[int] = set() + #: alias → that account's job client (`.client`, `.resolve_chat`), or + #: None while it is not connected. The filters need both. + self._accounts = accounts + self._chat_ids: set[int] = set() + self._pending_refs: list[str] = [] + self._next_resolve = 0.0 + self._logged: set[str] = set() + self._builders: dict[str, Any] = {} + for ref in config.filters.chats: + try: + self._chat_ids.add(int(ref)) + except (TypeError, ValueError): + self._pending_refs.append(str(ref)) self._dead_letter_path = self.base / "dead_letter.jsonl" self._queue: asyncio.Queue[tuple[bytes, dict[str, str]]] | None = None self._workers: list[asyncio.Task[None]] = [] @@ -113,36 +136,96 @@ async def stop(self, *, drain: float = 10.0) -> None: self._session = None # -- filtering --------------------------------------------------------- + # + # Both halves used to be dead. `filters.chats` was compared against a set + # nothing ever filled, so configuring it dropped every event that had a + # chat. The other `[webhook.filters]` keys were evaluated against the raw + # TL update the bus carries, while every filter reads a high-level + # Telethon event, so they raised or never matched. + + async def _chat_allowed(self, event: EventEnvelope) -> bool: + if not self.config.filters.chats or event.chat_id is None: + return True + if self._pending_refs and time.monotonic() >= self._next_resolve: + await self._resolve_refs(event.account) + return event.chat_id in self._chat_ids + + async def _resolve_refs(self, account: str) -> None: + """Resolve `@name` chats through *account*; retry failures in a minute.""" + self._next_resolve = time.monotonic() + _RESOLVE_RETRY_SECONDS + job = self._accounts(account) if self._accounts is not None else None + if job is None: + return + still: list[str] = [] + for ref in self._pending_refs: + try: + self._chat_ids.add(int(await job.resolve_chat(ref))) + except Exception as exc: + self._log_once(f"chat:{ref}", "webhook cannot resolve chat %s: %s", ref, exc) + still.append(ref) + self._pending_refs = still - def set_resolved_chats(self, chat_ids: set[int]) -> None: - self._resolved_chat_ids = chat_ids + async def _filters_pass(self, event: EventEnvelope, raw: Any) -> bool: + """Evaluate `[webhook.filters]` against the event Telethon would build. - def should_push( - self, event_type: str, chat_id: int | None = None, tg_event: Any = None - ) -> bool: - if not self.config.enabled: + An event type with no Telethon builder (typing, presence of a story, + ...) cannot be described to the filters at all, so it is not + delivered while filters are set; that is logged once per type. + """ + from tlgr.filters.compose import evaluate + from tlgr.gateway.event import Event + from tlgr.gateway.tlevents import build_event, builder_for_bus_type + + builder = self._builders.get(event.type) + if builder is None: + builder = builder_for_bus_type(event.type) + if builder is None: + self._log_once( + f"type:{event.type}", + "webhook filters cannot be applied to %s events; not delivering them", + event.type, + ) + return False + self._builders[event.type] = builder + job = self._accounts(event.account) if self._accounts is not None else None + client = getattr(job, "client", None) + if client is None: return False - if event_type not in self.config.events: + built = await build_event([(event.type, builder)], raw, client) + if built is None: return False - if self.config.filters.chats and chat_id is not None: - if chat_id not in self._resolved_chat_ids: - return False - if self._filter_node is not None and tg_event is not None: - from tlgr.filters.compose import evaluate - from tlgr.gateway.event import Event + tg_event, _label = built + envelope = Event( + source="telegram", raw=tg_event, account=event.account, event_type=event.type + ) + try: + ok, _reason = evaluate(self._filter_node, envelope) + except Exception as exc: + log.debug("webhook filter raised on %s: %s", event.type, exc) + return False + return ok - envelope = Event(source="telegram", raw=tg_event, event_type=event_type) - ok, _ = evaluate(self._filter_node, envelope) - if not ok: - return False - return True + def _log_once(self, key: str, message: str, *args: Any) -> None: + level = logging.DEBUG if key in self._logged else logging.WARNING + self._logged.add(key) + log.log(level, message, *args) # -- the bus handler --------------------------------------------------- async def on_event(self, event: EventEnvelope, raw: Any = None) -> None: - """The bus handler. Encodes, signs and enqueues; never sends inline.""" - if not self.should_push(event.type, event.chat_id, raw): + """The bus handler. Filters, encodes, signs and enqueues; never sends inline. + + A self-origin event (the daemon's own send) has no raw update, so + `[webhook.filters]` cannot be evaluated and does not apply to it; + `events` and `chats` still do. + """ + if not self.config.enabled or event.type not in self.config.events: return + if not await self._chat_allowed(event): + return + if self._filter_node is not None and raw is not None: + if not await self._filters_pass(event, raw): + return self.enqueue(event) def enqueue(self, event: EventEnvelope) -> None: diff --git a/tlgr/gateway/engine.py b/tlgr/gateway/engine.py index b6447f2..e6f7a7a 100644 --- a/tlgr/gateway/engine.py +++ b/tlgr/gateway/engine.py @@ -13,10 +13,10 @@ The pipeline itself is unchanged, and filters still read a high-level Telethon event (`NewMessage.Event` and friends). The bus carries the raw TL update beside the normalised envelope, so the job builds that event itself -with the same builder a Telethon handler would have used, including -`NewMessage(incoming=True)`, so a job never answers the account's own -messages. The full move to model-based filters belongs to the updates group -(PR-4). +(`gateway/tlevents.py`) with the builder a Telethon handler would have used, +including `NewMessage(incoming=True)`, so a job never answers the account's +own messages. The full move to model-based filters belongs to the updates +group (PR-4). Without a bus — a unit test, or a daemon that has not started one — the job falls back to registering Telethon handlers exactly as v1 did. @@ -25,17 +25,15 @@ from __future__ import annotations import asyncio -import inspect import logging import time from typing import Any -from telethon import events - from tlgr.actions import get_action from tlgr.filters.compose import FilterNode, Op, evaluate from tlgr.gateway.config import ActionConfig, GatewayConfig from tlgr.gateway.event import Event +from tlgr.gateway.tlevents import build_event, builder_for_job_event from tlgr.jobs.base import BaseJob from tlgr.jobs.client import JobClient @@ -73,31 +71,11 @@ def _bus_types(names: list[str]) -> set[str]: return wanted -_EVENT_TYPE_MAP = { - "new_message": (events.NewMessage, {}), - "message_edited": (events.MessageEdited, {}), - "message_deleted": (events.MessageDeleted, {}), - "chat_action": (events.ChatAction, {}), - "user_joined": (events.UserUpdate, {}), - "message_read": (events.MessageRead, {}), -} - - -def _builder_for(event_type_name: str) -> Any | None: - mapping = _EVENT_TYPE_MAP.get(event_type_name) - if not mapping: - return None - event_cls, kwargs = mapping - if event_type_name == "new_message": - kwargs = {"incoming": True} - return event_cls(**kwargs) - - def _builders(names: list[str]) -> list[tuple[str, Any]]: """One Telethon event builder per v1 event name, in the job's order.""" out: list[tuple[str, Any]] = [] for name in names: - builder = _builder_for(name) + builder = builder_for_job_event(name) if builder is not None: out.append((name, builder)) return out @@ -207,7 +185,7 @@ async def on_event(envelope, raw) -> None: # A self-origin echo or a synthesised event: the filters read # the raw Telethon object, so there is nothing to evaluate. return - built = await self._build(builders, raw) + built = await build_event(builders, raw, self.client.client) if built is None: return tg_event, event_type = built @@ -220,35 +198,9 @@ async def on_event(envelope, raw) -> None: except asyncio.CancelledError: raise - async def _build(self, builders: list[tuple[str, Any]], update: Any) -> tuple[Any, str] | None: - """The high-level event Telethon would have handed a handler, or None. - - This mirrors `EventBuilderDict` in Telethon's `_dispatch_update`: build, - attach the update's entities and the client, then apply the builder's - own filter, which is where `incoming=True` drops outgoing messages. - """ - client = self.client.client - self_id = getattr(client, "_self_id", None) - for event_type, builder in builders: - event = builder.build(update, None, self_id) - if not event: - continue - if isinstance(event, events.common.EventCommon): - event.original_update = update - event._entities = getattr(update, "_entities", None) or {} - event._set_client(client) - if not builder.resolved: - await builder.resolve(client) - passed = builder.filter(event) - if inspect.isawaitable(passed): - passed = await passed - if passed: - return event, event_type - return None - async def _run_on_client(self) -> None: for event_type_name in self._gw.events: - builder = _builder_for(event_type_name) + builder = builder_for_job_event(event_type_name) if builder is None: log.warning("[%s] unknown event type: %s", self.name, event_type_name) continue diff --git a/tlgr/gateway/tlevents.py b/tlgr/gateway/tlevents.py new file mode 100644 index 0000000..c3678a4 --- /dev/null +++ b/tlgr/gateway/tlevents.py @@ -0,0 +1,98 @@ +"""Raw TL updates → the high-level Telethon events filters read. + +The daemon's bus carries each raw TL `Update*` beside the normalised +envelope, because one `events.Raw()` handler sees every constructor while +Telethon's high-level builders drop service messages, topic ids and the +updates they do not model. Every filter in `tlgr/filters/` was written +against those high-level events, though: `event.chat_id`, `is_private`, +`event.message.sender_id`, `reply()`. Handing them the raw update made jobs +and webhook filters fail or never match. + +`build_event` closes that gap the way Telethon's own `_dispatch_update` +does (its `EventBuilderDict`): build with the builder, attach the update's +entities and the client, then apply the builder's filter. Gateway jobs and +the webhook pusher both go through it, so a filter behaves the same in a +job, in a webhook, and in the client-handler fallback. +""" + +from __future__ import annotations + +import inspect +from typing import Any + +from telethon import events + +__all__ = ["build_event", "builder_for_bus_type", "builder_for_job_event"] + +#: The v1 job event names (`jobs.yaml` `events:`) and their builders. +#: `new_message` means incoming only, as it always has for jobs: an +#: auto-reply must never answer the account's own messages. +_JOB_BUILDERS: dict[str, tuple[type, dict[str, Any]]] = { + "new_message": (events.NewMessage, {"incoming": True}), + "message_edited": (events.MessageEdited, {}), + "message_deleted": (events.MessageDeleted, {}), + "chat_action": (events.ChatAction, {}), + "user_joined": (events.UserUpdate, {}), + "message_read": (events.MessageRead, {}), +} + +#: Bus event types (the v2 taxonomy) that have a high-level builder. A +#: webhook reports both directions, so `message_new` is not incoming-only +#: here; `is_incoming` / `sender_is_self` filter on direction. +_BUS_BUILDERS: dict[str, tuple[type, dict[str, Any]]] = { + "message_new": (events.NewMessage, {}), + "message_edited": (events.MessageEdited, {}), + "message_deleted": (events.MessageDeleted, {}), + "message_service": (events.ChatAction, {}), + "member_chat": (events.ChatAction, {}), + "member_channel": (events.ChatAction, {}), + "read_inbox": (events.MessageRead, {"inbox": True}), + "read_outbox": (events.MessageRead, {"inbox": False}), + "user_status": (events.UserUpdate, {}), +} + + +def builder_for_job_event(name: str) -> Any | None: + """A fresh builder for a `jobs.yaml` event name, or None if unknown.""" + entry = _JOB_BUILDERS.get(name) + if entry is None: + return None + cls, kwargs = entry + return cls(**kwargs) + + +def builder_for_bus_type(event_type: str) -> Any | None: + """A fresh builder for a bus event type, or None if Telethon models none.""" + entry = _BUS_BUILDERS.get(event_type) + if entry is None: + return None + cls, kwargs = entry + return cls(**kwargs) + + +async def build_event( + builders: list[tuple[str, Any]], update: Any, client: Any +) -> tuple[Any, str] | None: + """The first `(event, label)` a builder makes of *update* and accepts. + + *builders* pairs a label with a builder; the label comes back so the + caller knows which one matched. None means no builder modelled the + update, or its own filter (`incoming=True`, `inbox=`) declined it. + """ + self_id = getattr(client, "_self_id", None) + for label, builder in builders: + event = builder.build(update, None, self_id) + if not event: + continue + if isinstance(event, events.common.EventCommon): + event.original_update = update + event._entities = getattr(update, "_entities", None) or {} + event._set_client(client) + if not builder.resolved: + await builder.resolve(client) + passed = builder.filter(event) + if inspect.isawaitable(passed): + passed = await passed + if passed: + return event, label + return None diff --git a/webhook.example.toml b/webhook.example.toml index 7886e4e..61543a4 100644 --- a/webhook.example.toml +++ b/webhook.example.toml @@ -19,3 +19,8 @@ backoff_base = 2 # seconds between retries (exponential) [webhook.filters] chats = [] # empty = all chats, or ["@channel", "-100123456"] +# Any gateway job filter works here too (tlgr/filters/README.md), e.g. +# chat_type = "private" +# contains = ["invoice"] +# With one set, event types Telethon has no event for (typing, for one) are +# not delivered, and the daemon logs that once per type.