diff --git a/CLAUDE.md b/CLAUDE.md index f2329df7d..9c23af14e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,6 +73,7 @@ Each layer returns `{ handled: true }` to stop propagation, or `{ handled: false - `tools.ts` — 40+ tool definitions (the Claude API schema). - `tool-executor.ts` — dispatches tool name → handler function. - `tool-handlers/` — one file per domain: `events`, `reminders`, `sharing`, `secretary`, `proposals`, `slots`, `settings`, `meta`, `feedback`, `history`. +- Picture tools (`render_day_image`, `render_week_image`, `render_month_image`, `render_table` in `tool-handlers/render.ts`) never wait for the render: they enqueue through `ctx.imageDelivery` (`src/services/image/image-delivery.ts`) and return "queued, NOT sent" at once; the picture arrives later as its own message, a failure sends one note and reaches the alert queue (runbook "Asynchronous pictures"). Never put a render wait back into a tool call and never tell the model a picture "was sent". - `system-prompt.ts` — builds the system prompt as `SystemPromptParts` in prefix-cache order. Providers cache by leading bytes (z.ai implicit cache, Anthropic `cache_control`), so a per-request value must never be added ahead of static text; `test/services/ai/prompt-prefix-stability.test.ts` enforces it and `scripts/measure-ai-payload.ts` reports the shared-prefix tokens. - `stable` — identical for every request (role, Context, static Rules, Proactive, Shared Events, Stall). - `modeRules` — static text that depends only on the mode (group chat rules, voice/live-call instructions), never on user data. @@ -111,7 +112,7 @@ If you add a new `FeatureKey`, add it to `FEATURE_KEYS` in `src/database/reposit For periodic/scheduled tasks always use BullMQ repeating jobs — never `setInterval` or `setTimeout`. Repeating jobs survive restarts and are observable in the queue. BullMQ on Redis; the main queues: -- **image-render** — Playwright renders weekly/monthly calendar images. +- **image-render** — Playwright renders the calendar images. EVERY picture (AI tools, `/today` `/tomorrow` `/week` `/month`, agenda buttons, the conflict picture sent to an inviter) goes through `ImageDelivery` (`queuePicture` for non-AI callers): the handler sends its text, queues the picture and returns; the picture arrives later, bounded at 150 s, one note on failure, a timeout reaches the alert queue as `reportTimeout({kind: 'render'})`. Nothing in the bot awaits `RenderService.renderDirect`; do not add a render wait to a handler. - **call-reminders** — voice call reminders via the Python bridge; created only when the MTProto service tier is enabled. - **bot-tasks** — periodic jobs: secretary expiry (daily), sharing cleanup (10min), proposal expiry (hourly), per-user session keepalive (14 days), AI usage ledger cleanup (daily), birthday sync (daily, only when the service tier is enabled). diff --git a/docs/intents/response-quality.md b/docs/intents/response-quality.md index a7a069b98..04a1331ce 100644 --- a/docs/intents/response-quality.md +++ b/docs/intents/response-quality.md @@ -29,6 +29,18 @@ integration is never an empty result. **Proportion.** A short question does not need a long formal report. Necessary clarifications stay; internal identifiers are shown only when they help choose or confirm an object. +**Pictures are announced as coming, never as attached.** A day, week, month, table or schedule-comparison +picture is rendered in the background and arrives as its own message after the text answer, whether it +comes from the AI, a command (`/today`, `/tomorrow`, `/week`, `/month`), an agenda button or an +invitation notice (see the deploy runbook, "Asynchronous pictures"). An answer or workflow reply therefore +never says the picture "is attached", "is below" or "was sent": the tool result reads "is being prepared +and will arrive in the chat as a separate message" (ru: «готовится и придёт в чат отдельным сообщением»), +i.e. the picture is on the way. When the AI answers, the facts a picture shows belong in its text as +before; a command's text already carries them. When a picture cannot be made the user gets exactly one +short note saying so (it timed out, it failed, or the bot restarted), naming the picture by its date, +week, month or title and offering to try again; the note never contains an internal error and never claims +the text already says everything the picture showed. + ## Example Request: "Что завтра?" ("What is tomorrow?") diff --git a/docs/reference/deploy-runbook.md b/docs/reference/deploy-runbook.md index cad2bb991..6e1c7e5ed 100644 --- a/docs/reference/deploy-runbook.md +++ b/docs/reference/deploy-runbook.md @@ -166,6 +166,61 @@ GramIO hands every Telegram update to its handler without waiting for the previo - The event row and its reminders are written in the same transaction (also for `allow_duplicate` and every other `EventService.createEvent` caller); a failed reminder write leaves no event, so a retry starts clean. All-day dates are read in the zone each event was saved with. - The result of a repeat is `success: true`, `mutationState: not_applied`, `alreadyExists`. Intent answers: it is carried through `intent-executor` into `ToolAnswer`, and both the renderer and a closing `respond:` step (`basis.event.create`, `basis.event.create_invite`) say the event is already in the calendar instead of «created». The run-failure receipt wording («Completed: Create event») lives in `write-outcomes.ts` and is left to the receipt rewrite tracked in #929. +## Asynchronous pictures (AI picture tools, 2026-11-01) + +Why: on 2026-11-01 (prod `bb7a9d47`) `render_day_image` / `render_week_image` / `render_month_image` / `render_table` waited for the BullMQ render inside the agent turn. On the loaded odroid real renders took 73–80 s (`Job wait render timed out ... 45000ms`, `Day image render failed`); two waits plus model rounds spent the ~90 s turn budget, the answer validator timed out and the user got the aborted-run notice instead of the text the model had already written (tracked in [#799](https://github.com/alex-mextner/HyperCalendarBot/issues/799) and [#800](https://github.com/alex-mextner/HyperCalendarBot/issues/800)). + +Now the tool only enqueues (`src/services/image/image-delivery.ts`, `ImageDelivery`) and returns at once: `success`, an output that says the picture "is being prepared and will arrive in the chat as a separate message", and a hint to the model that it is queued, NOT sent, and must not be called "attached". The turn ends with the text. In the background the service renders, sends the picture to the same chat as its own message (photo, or document when oversized, then the best-effort pin) and logs exactly one line `Image delivery outcome` with `outcome` = `delivered` (with the Telegram `messageId`) / `failed` / `timed_out` / `superseded` / `interrupted`. The line carries the picture kind, job type, chat id and timings, never user text. + +Owner instruction of 2026-11-01 (Telegram message 26331): ALL pictures are asynchronous, and every timeout goes to the repair flow. So the same service now carries every other picture the bot sends; nothing in the bot awaits a render any more (`RenderService.renderDirect` is called only inside `ImageDelivery`): + +- `/today`, `/tomorrow`, `/week`, `/month` (`src/bot/commands/*.ts`): the text answer goes out first, then the picture is queued with `queuePicture` (`src/services/image/queue-picture.ts`) and arrives as its own message when ready (`/today` and `/month` pin it, as before). Without `imageDelivery` (no Redis, Playwright failed to start) the commands stay text-only. +- The agenda buttons (`imd:` daily, `imw:` weekly, `src/bot/handlers/callback/agenda-routes.ts`): the callback query is answered at once, the picture follows as a reply to the tapped message. A second tap on the same picture while it is pending is a duplicate and gives no second picture. +- The conflict picture the inviter gets when an invitee accepts (`invitation-notify.ts`): queued the same way, sent to the inviter's chat, with the note "schedule comparison" if it fails. Before this change a failed or timed-out render was only logged. +- The scheduled morning/evening digest has no picture path: the `image` format is still sent as text (`TODO(#863)` in `src/services/notification/scheduler.ts`), so it never waited for a render. When #863 adds a photo it MUST go through `queuePicture`. + +The delivery id is deterministic: chat id, the Telegram message id of the command (or the callback query id of the tap, or `invitation::` for the conflict picture: answering the invitation again after a failed picture asks for a new one, a redelivered tap does not), kind, slot and payload, so a duplicate update or a double tap is one render and one picture. A failed, timed-out or oversized picture gives the user exactly one note and a timed-out one also reaches the alert queue as `reportTimeout({ kind: 'render' })`, for the commands and buttons exactly as for the AI tools. The payload hash covers what the picture shows, not the clock: the current-time line and the "Today" badge of a picture of today (`currentTimeMinutes`, `relativeDay`) are left out, so the same picture asked again in another minute is the same delivery (`duplicate`) and only a change of the data replaces the older pending picture (same slot, newer payload) instead of stacking a second one. + +Limits, all constants in `image-delivery.ts`: + +- Hard bound: `IMAGE_DELIVERY_BOUND_MS` = 150 s from the tool call to the end of the render (twice the worst render seen on 2026-11-01, 73–80 s). The job carries it as `expiresAt`; the worker drops a job that only starts after it (stalled job re-queued by a restart, retry) with `UnrecoverableError`, so no render happens for nobody. The upload of a finished PNG has its own `IMAGE_SEND_BOUND_MS` = 30 s. +- Per chat at most `MAX_PENDING_PER_CHAT` = 2 pending pictures, at most `MAX_PENDING_TOTAL` = 12 on the bot (a delivery that already ended and is only telling the user holds no place; that note is bounded by the send bound); a further request is not queued and the tool says so (`imageBusy`). The schedule comparison of an accepted invitation (`kind: conflict`) has its own limits and does not count against the chat's 2: `MAX_PENDING_CONFLICT_PER_CHAT` = 4 per chat and `MAX_PENDING_CONFLICT_TOTAL` = 6 on the bot. Three guests accepting within one render would otherwise lose the third comparison for good (no command can ask for it again), and two pending comparisons would refuse the inviter's own `/today` and AI pictures; the bot-wide share keeps the other half of the 12 places for the chats' own pictures. The limits are bounded, not removed: two accounts accepting each other's invitations can still hold the 6 comparison places for up to 150 s (ticket #969). The identical picture requested twice (same slot and same payload) is one render and one delivery; a newer picture of the same slot (same day / week / month / calendar, changed data) replaces the older pending one, so the user gets only the newest. The older one is recorded `superseded` at the moment of replacement, not when its run unwinds, so a shutdown in between cannot leave its row `queued` for the next start to send after the newer picture. +- Staleness: a calendar write for the picture's data while it renders (domain events `myCalendar.newEvent`/`updatedEvent`/`deletedEvent` and `myGroup.newEvent`; a group picture is stale after ANY write by any user, it is not keyed by group) makes the finished picture be rebuilt once from the current data instead of showing the old state beside a text that already states the new one. A rebuild whose render data is identical is skipped (see below), so an unrelated write costs a comparison, not a render. Writes that emit no domain event do NOT trigger it: Google pull sync, group event update and delete (`EventService.updateEventForGroup`/`deleteEventForGroup`) and the recurrence writes (cancel or edit one occurrence, split, delete the future ones); a picture in flight then shows the state before that write. Known gap, ticket #964. +- The rebuild only runs with at least `REBUILD_MIN_BUDGET_MS` = 45 s of the bound left; with less, or when the rebuild fails, the picture already rendered is sent (an older state beside newer text beats no picture and a timeout alert). A rebuild cut by the render bound is still reported as `reportTimeout({ kind: 'render', recovered: true })`, so a slow host stays visible although the user got a picture. +- A write that leaves what the picture shows unchanged (the rebuilt render data is identical) is not rendered a second time. After a rebuild the picture on its way counts as the new data, so the same request again is `duplicate`, not a newer picture that replaces it. The slot of an AI picture ends with the user's language, and a group picture's slot also carries the requester (`group::viewer:`, for the AI tools and the day/week buttons alike: its holidays are the requester's), so two members of one group chat never replace each other's picture. +- The same request again (same Telegram message, same picture) after its delivery ended is `settled`: nothing is queued, the AI tool says it was already handled (`imageAlreadyHandled`), a command is silent. While the first one is still live it is `duplicate` ("being prepared"). A new message asking for the same picture is a new delivery. + +What the user gets when the picture does not arrive: exactly ONE short note in their language naming the picture (date, week, month or table title) and offering to ask again; never the internal error. The schedule comparison of an accepted invitation is the exception: no command or tool can request it again, so its notes (`imageNote*Final`) do not offer it, and a job that cannot even be built says so in its own words (`invitation_conflict_picture_failed`) instead of pointing at a text version that does not exist. A picture that is undeliverable by nature (over the 50 MB document limit, a raster over the allocation limit) gets the localized note `imageNoteUndeliverable` ("too large to send, ask for a shorter period or a smaller table"; `imageNoteUndeliverableFinal` for the schedule comparison), never the renderer's English message, and is not an incident. A supplement turn (silent optional text after an intent answer) queues no picture at all: its sender drops everything, so the picture could only block or supersede a real one. + +An upload that ends in anything but Telegram's own refusal (a 4xx answer) is an uncertain send, because the picture may have landed: a dropped connection, an abort or a 5xx answer is recorded `unknown`, nothing is sent again, and the owner's repair flow gets one `render-failure` alert (`errorName: UncertainSend`). The user gets one note that does not claim a failure either way ("may not have arrived: the upload was cut off, ask again if you do not see it", `imageNoteUncertain`): the answer of an AI turn may have been only this picture (a `[SKIP]` text beside a queued picture counts as answered, #508), and without the note a user whose upload died would be left with nothing. The note is guarded by the delivery's own end alone (an `unknown` row is never in the recovery pass). Only a 4xx refusal is `failed` with one note and a `render-failure` alert. A refusal that asking again cannot fix (403 bot blocked or removed, 404 or 400 "chat not found", 400 "not enough rights to send photos") tells the user in its own words that Telegram does not let the bot send pictures in this chat (`imageNoteRefused`, no offer to ask again), logs at warn and raises no alert, the same rule as the broadcast worker's permanent failures. An upload that hangs past its bound is `unknown` too, tells the user the same uncertain note and reports `reportTimeout`. + +What reaches the repair flow (alerts queue -> `hcb-alert-watcher` -> omp session, see "Alert watcher"): + +- A render that does not finish within the bound: `reportTimeout({ kind: 'render', elapsedMs, limitMs, recovered: false })`, one aggregated alert per kind per window with `source` `timeout`, as for every other timeout of the bot (`src/services/incident/timeout-reporter.ts`). A hanging upload (the note says only that the picture may not have arrived: it may still land) reports the same kind with the send bound as `limitMs`. +- A render or upload that fails for another reason: one aggregated alert per 10 minutes of its own kind, `source` `render-failure` (`src/services/image/render-failure-reporter.ts`), with the picture kind, job type, error class name and durations (never the message text, user text or ids), and how many were held back since the last one. Failures inside the window are held back and sent as ONE summary when the window ends (tried again after 60 s while the alert queue refuses it, and a first failure the queue refused is held and retried the same way, so a lone failure is not lost), or at a graceful shutdown; a process killed without one loses the held summary. + +Durable deliveries (migration `070_image_deliveries`, [#936](https://github.com/alex-mextner/HyperCalendarBot/issues/936)): every picture has one row in `image_deliveries` with a deterministic id (hash of chat id, the user's message id, kind, slot and payload; also the BullMQ job id, so a repeated add is a no-op and a restarted bot finds the same job). The row is written BEFORE each step: `queued` at enqueue, `rendered` when the PNG is in hand, `sending` before the Telegram call, then `delivered` (with the Telegram `message_id`), `failed`, `expired`, `unknown` or `superseded`. Terminal rows drop their payload and are pruned once 7 days old, at the next start (the start-up reconciliation is the only caller). A finished render stays in Redis for 3 minutes (`removeOnComplete`, just over the bound) so a picture finished while the bot was down can still be collected. + +Restart and shutdown: shutdown closes `imageDelivery` right before the image queue (see `shutdown.ts`). Local waits are cancelled with the reason `shutdown`, which keeps the queued BullMQ job; no note is sent and each delivery is logged `interrupted` with `resumable: true`, its row stays `queued`. A picture already being uploaded gets a 1.5 s grace: if it lands it is `delivered`, otherwise it is `unknown`. A completion that arrives after the close finds nobody to send to and is dropped, never reported as delivered. Only a delivery without a store (tests, dev) tells its user at shutdown. The same close sends the `render-failure` summary still held back for its 10-minute window (`RenderFailureReporter.flush()`), so a deploy right after a burst of failures does not lose it; a SIGKILL or a crash still does. + +Start-up reconciliation (`ImageDelivery.reconcile()`, called from `src/index.ts` once the bot exists, fire-and-forget; recovery sends through `recoverImageDeliveries` in `src/bootstrap/image-render.ts`): + +- `sending` (the upload started, its end is unknown): marked `unknown`, never retried because a retry could send a second picture, and the user is not told (known gap, ticket #969: `reconcile` holds `transport.notify`, but `markNoteSent` claims only `failed`/`expired` rows, so a note for an `unknown` row needs its own once-only claim; the same gap exists for an upload the shutdown grace gave up on). One aggregated `render-failure` alert (`errorName: UncertainSend`) so the owner's repair flow hears of it. Whether the picture reached the chat is only visible in the chat; the `messageId` is not known. +- A `sending` row whose last update is more than `RECOVERY_NOTE_MAX_AGE_MS` (10 minutes) old (a rollback, a long outage) ends `unknown` as well, with no alert: the cause was downtime. +- `queued` or `rendered`, still within `expires_at`: the same BullMQ job id is awaited again. A render that finished while the bot was down is delivered at once, one still waiting or running is waited for, one the queue lost (a finished render is kept 3 minutes, a flushed Redis loses it) is rendered anew from the stored payload. The picture is then sent exactly once and recorded `delivered`. A recovered picture is not rebuilt from fresh calendar data (no request context after a restart) and is not pinned. +- A resumed picture keeps the slot and the payload hash stored on its row (`image_deliveries.slot`, `payload_hash`), so it is deduplicated and superseded exactly like a live one: the same picture asked again after a restart is `duplicate` (no second render, no second picture), and a newer one of the same slot replaces it. +- A rebuild that already started (a calendar write while the picture rendered) is recorded on the row before its render (`rebuilds`, the rebuilt `job_json` and `payload_hash`): a restart in between resumes the rebuilt `-r` BullMQ job, never the stale one that had already finished. A rebuild is not started again after a restart. +- Past its bound, or without a payload (still `queued` or `rendered`): `expired`, ONE note to the user (`note_sent` guards a second one) saying the bot was restarting (`imageNoteInterrupted`), and NO timeout alert. A running process records `expired` itself when the bound passes, so a row still unfinished at start-up past its bound means the previous process stopped before the bound: downtime, not a slow render, and a deploy with a picture in flight must not open a render-timeout investigation. A row the previous process already ended as `expired` (it died before telling the user) still gets the "did not get ready in time" note. +- A delivery past its bound by more than 10 minutes ends `expired` silently: its note is claimed but not sent and no timeout alert is raised, because the user stopped waiting long ago and the cause was downtime, not a slow render. The same goes for a failed/expired row that was never told about days earlier. +- `failed` or `expired` with `note_sent = 0` (the process died between recording the outcome and telling the user): the one missing note is sent now. `note_sent` is claimed before sending, so a note is at most once. An `unknown` row never gets a note. +- An end the store refused to record (a locked database) is written again up to `RECORD_RETRIES` = 3 times, 1, 2 and 3 s apart, and once more, synchronously, by the graceful shutdown (`close()`; it stops at the first refusal, because SQLite has then already waited its 5 s busy timeout on that thread and the shutdown deadline is 8 s). Until it is written, the row still looks unfinished, so a restart in that window would expire or resume it a second time (a superseded older picture would be sent after the newer one); a note already sent meanwhile is claimed (`note_sent`) as soon as the write succeeds, and a refusal in the claim alone is tried again as the claim alone. What can still repeat the note or resume a stale picture: a process killed inside those seconds, or a store that refuses the retries and the shutdown write alike. A locked database is also reported as a `db_busy` timeout. + +The repository enforces the single-send rule: `advance(id, 'sending')` only moves a row forward from `queued`/`rendered` and returns whether it won, and only the winner uploads; `expired`/`superseded` never apply to a row whose upload was claimed. + +Known limits: a kill between the Telegram call returning and the `delivered` write leaves `sending`, so that picture is reported `unknown` even though it arrived (the price of never sending twice). A kill inside the tool call, before the `queued` row is written, loses the picture together with the turn that promised it; the user can ask again. A stale-data rebuild after a calendar write happens only in the live process; after a restart the stored payload is rendered as it was. + +Reading an incident: `docker logs hypercal-bot 2>&1 | grep 'Image delivery outcome'` on `root@odroidn2` shows every outcome; `Image delivery failed` (error level) carries the stack of a failed one; the BullMQ side is `Image render failed` from the worker. A burst of `timed_out` means the host is overloaded (compare with the memory-pressure notes in [#800](https://github.com/alex-mextner/HyperCalendarBot/issues/800)), not that the tool is broken: the text answers still went out. + ## Public ingress (Cloudflare tunnel) `hypercal.mextner.com` is a proxied CNAME in the `mextner.com` Cloudflare zone (account `invntrm@gmail.com`) to the tunnel `odroid-home` (`872e1f2e-3488-4e80-955e-8aba2fada806`). cloudflared on the odroid (`cloudflared.service`, `--token-file /etc/cloudflared/token`) runs it with a remotely managed configuration, so the hostname rules are edited in Cloudflare, not on the odroid. Bot traffic is the rule `hypercal.mextner.com` -> `http://localhost:3001`; every path reaches the bot, which serves only its own routes (`/health`, `/ready`, `/webhook/telegram`, `/webhooks/google-calendar`, `/oauth/google/callback`, `/admin/alerts*` behind the bearer token). While the container is being replaced the tunnel answers 502; Telegram and Google retry, but there is no proxy holding requests as the retired DO Caddy did (`lb_try_duration`). diff --git a/docs/reference/migrations/069_chat_history_turn_order.md b/docs/reference/migrations/069_chat_history_turn_order.md index 7bcd4672f..8e963bf24 100644 --- a/docs/reference/migrations/069_chat_history_turn_order.md +++ b/docs/reference/migrations/069_chat_history_turn_order.md @@ -31,7 +31,7 @@ as before. No existing column or row is read, changed or deleted. ## Order in the file Appended after `067_ai_usage_role_preset_cost`, the last entry. Nothing is moved or renumbered. The number -069 leaves 068 to the other open changes that claim it (the AI speed metrics and the image deliveries). +069 leaves 068 to the other open change that claims it (the AI speed metrics); the image deliveries took 070. ## Why it is safe to deploy automatically diff --git a/docs/reference/migrations/070_image_deliveries.md b/docs/reference/migrations/070_image_deliveries.md new file mode 100644 index 000000000..df57435cd --- /dev/null +++ b/docs/reference/migrations/070_image_deliveries.md @@ -0,0 +1,61 @@ +--- +migration: 070_image_deliveries +rollback-compatible: yes +data-deletion: no +--- + +# Migration 070: durable picture deliveries + +`src/database/migrations.ts`, migration `070_image_deliveries`. Asynchronous AI pictures, tracked in #799 and #936. + +## What it does + +```sql +CREATE TABLE image_deliveries ( + id TEXT PRIMARY KEY, -- deterministic delivery id (also the BullMQ job id suffix) + chat_id INTEGER NOT NULL, + language TEXT NOT NULL, + kind TEXT NOT NULL, -- day | week | month | table | conflict + label TEXT NOT NULL, -- what the user asked for; only ever shown back to that chat + job_type TEXT NOT NULL, + slot TEXT NOT NULL, -- what the picture shows (kind, target, whose calendar; an AI picture's slot also ends with the user's language): a later request for the same slot is deduplicated or supersedes it, also after a restart + payload_hash TEXT NOT NULL, -- hash of the render data of job_json: the same slot with the same hash is the same picture + rebuilds INTEGER NOT NULL DEFAULT 0, -- how many times the job was rebuilt after a calendar write; each rebuild is its own BullMQ job id + job_json TEXT, -- the current render job (replaced by a rebuild), kept until the delivery is terminal, then NULL + state TEXT NOT NULL CHECK (state IN ('queued','rendered','sending','delivered','failed','expired','unknown','superseded')), + message_id INTEGER, -- Telegram message id once delivered + note_sent INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, updated_at INTEGER NOT NULL +); +CREATE INDEX idx_image_deliveries_state ON image_deliveries(state); +``` + +A new table that only this feature reads and writes. Terminal rows (delivered, failed, expired, unknown, superseded) carry no job payload and are pruned once they are 7 days old, at the next start of the bot (the start-up reconciliation is the only caller; a process that stays up keeps them until then). + +## Order in the file + +Appended after `069_chat_history_turn_order`, the last entry at the time of writing. It was written as 068 on its branch and renumbered to 070 before it was ever released, because main took 069 meanwhile; nothing has recorded the old name. If another branch appends first, this one is renamed before it ships, never after. + +## Why it is safe to deploy automatically + +Additive DDL: one new table and one index. The previous image does not know the table and ignores it. + +## Rollback + +An image rollback needs no schema step. Pending rows are then not recovered by the old image (the pictures are lost, as they were before this feature); the next start of this feature ends each of them as `expired`: recent ones (within 10 minutes of their bound) with the one "the bot was restarting, the picture did not arrive" note and no alert (the cause was downtime, not a slow render), older ones silently (their note is claimed unsent) and without an alert, because the user stopped waiting long ago. A row stuck in `sending` ends `unknown` and raises its one `UncertainSend` alert only when its last update is less than 10 minutes old; an older one ends `unknown` silently. Keep the table and the `migrations` row. If the table ever has to go, it holds only delivery bookkeeping: + +```sql +BEGIN IMMEDIATE; +DROP TABLE image_deliveries; +DELETE FROM migrations WHERE name = '070_image_deliveries'; +COMMIT; +``` + +## Verifying after deploy + +```sql +SELECT name FROM migrations WHERE name = '070_image_deliveries'; +SELECT state, count(*) FROM image_deliveries GROUP BY state; +``` + +Expect one migration row; `queued`/`rendered`/`sending` rows older than the 150 s bound mean a delivery that was never reconciled. diff --git a/pyproject.toml b/pyproject.toml index 7a581f67d..82b39787d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hypercalendarbot" -version = "0.3.5" +version = "0.3.6" requires-python = ">=3.12" dependencies = [ "pyrofork>=2.3", diff --git a/scripts/intent-benchmark/world.ts b/scripts/intent-benchmark/world.ts index ea8be60f4..d1101b9fc 100644 --- a/scripts/intent-benchmark/world.ts +++ b/scripts/intent-benchmark/world.ts @@ -30,7 +30,7 @@ import type { AgentContext, SharingCapability, TelegramSender } from '../../src/ import { ConversationLogger } from '../../src/services/conversation-logger.ts'; import { EventService } from '../../src/services/event/event-service.ts'; import { HolidayService } from '../../src/services/holiday/holiday-service.ts'; -import type { ImageRenderer } from '../../src/services/image/render-service.ts'; +import { ImageDelivery } from '../../src/services/image/image-delivery.ts'; import { IntentExecutor } from '../../src/services/intent/intent-executor.ts'; import type { IntentMatcher } from '../../src/services/intent/intent-matcher.ts'; import { @@ -272,7 +272,7 @@ export function buildFixture( privacyService, editProposalRepo: new EditProposalRepository(db), }; - const renderService: ImageRenderer = { renderDirect: async () => pngStub() }; + const imageDelivery = new ImageDelivery({ renderer: { renderDirect: async () => pngStub() } }); const sender: TelegramSender = { sendMessage: async () => ({ message_id: 3 }), editMessageText: async () => {}, @@ -296,7 +296,7 @@ export function buildFixture( actionLogRepo: new ActionLogRepository(db), toolOrigin: 'intent_workflow', sharing: sharing, - renderService, + imageDelivery, sender, }); const callTool = async (name: string, input: unknown, origin?: { text: string; actorId: number; chatId: number }) => { diff --git a/scripts/verify-agenda-three-transports.ts b/scripts/verify-agenda-three-transports.ts index 754fb0275..f0520f252 100644 --- a/scripts/verify-agenda-three-transports.ts +++ b/scripts/verify-agenda-three-transports.ts @@ -16,6 +16,7 @@ import type { AgentContext } from '../src/services/ai/types.ts'; import { ConversationLogger } from '../src/services/conversation-logger.ts'; import { EventService } from '../src/services/event/event-service.ts'; import { HolidayService } from '../src/services/holiday/holiday-service.ts'; +import { ImageDelivery } from '../src/services/image/image-delivery.ts'; const bytes = Buffer.from(await Bun.file('logs/agenda-three/synthetic100.png').arrayBuffer()); const db = new Database(':memory:'); @@ -40,14 +41,15 @@ try { }; bot.api.sendMessage = async () => payload; bot.api.pinChatMessage = async () => true; - const renderer = { renderDirect: async () => bytes }; + const delivery = new ImageDelivery({ renderer: { renderDirect: async () => bytes } }); const command = Object.assign(new MessageContext({ bot, payload }), { dbUser: user, userTimezone: 'UTC', lang: 'en' as const, scene: { enter: async () => {} }, }); - await handleToday(command, events, undefined, renderer); + await handleToday(command, events, undefined, delivery); + await delivery.whenIdle(); assert.equal(documents, 1); const history = new ChatHistoryRepository(db); const ctx: AgentContext = { @@ -62,10 +64,11 @@ try { conversationLogger: new ConversationLogger(history), eventReminderRepo: new EventReminderRepository(db), sender: createTelegramSender(bot), - renderService: renderer, + imageDelivery: delivery, }; const result = await handleRenderDayImage(ctx, { date: new Date().toISOString().slice(0, 10) }); assert.equal(result.success, true); + await delivery.whenIdle(); assert.equal(documents, 2); console.log( JSON.stringify({ diff --git a/src/bootstrap/domain-events.ts b/src/bootstrap/domain-events.ts index 627b6786d..a5f1ea3e0 100644 --- a/src/bootstrap/domain-events.ts +++ b/src/bootstrap/domain-events.ts @@ -11,6 +11,7 @@ // - TriggerService subscribes later, in scheduled-ai.ts, once the AI queue exists. import { AgendaRepository } from '../database/repositories/agenda.repository.ts'; +import type { ImageDelivery } from '../services/image/image-delivery.ts'; import { DomainEventBus } from '../services/scheduled/domain-event-bus.ts'; import { InvitationCardRefresher } from '../services/sharing/invitation-cards.ts'; import type { WeatherService } from '../services/weather/weather-service.ts'; @@ -77,9 +78,24 @@ function subscribeInvitationCardRefresh( }); } +/** + * A picture still rendering when the calendar changes would show the old state next to a text that + * already states the new one: pending pictures of the changed data are rebuilt once. + */ +export function subscribeImageStaleness( + bus: DomainEventBus, + imageDelivery: Pick, +): void { + bus.on('myCalendar.newEvent', ({ userId }) => imageDelivery.eventsChanged(userId)); + bus.on('myCalendar.updatedEvent', ({ userId }) => imageDelivery.eventsChanged(userId)); + bus.on('myCalendar.deletedEvent', ({ userId }) => imageDelivery.eventsChanged(userId)); + bus.on('myGroup.newEvent', ({ userId }) => imageDelivery.eventsChanged(userId)); +} + export interface DomainEventInputs { participantPushScheduler: ParticipantPushScheduler | undefined; weatherService: WeatherService | undefined; + imageDelivery: ImageDelivery | undefined; } export function createDomainEventBus(ctx: StartupContext, inputs: DomainEventInputs): DomainEventBus { @@ -88,5 +104,6 @@ export function createDomainEventBus(ctx: StartupContext, inputs: DomainEventInp subscribeParticipantGooglePush(bus, inputs.participantPushScheduler, ctx.db.groupMembers); } subscribeInvitationCardRefresh(bus, ctx, inputs.weatherService); + if (inputs.imageDelivery) subscribeImageStaleness(bus, inputs.imageDelivery); return bus; } diff --git a/src/bootstrap/image-render.ts b/src/bootstrap/image-render.ts index ddaf7097b..52dacafed 100644 --- a/src/bootstrap/image-render.ts +++ b/src/bootstrap/image-render.ts @@ -4,18 +4,35 @@ // // Ordering invariants: // - Only with REDIS_URL. The Playwright pool starts first; if it cannot initialise, image -// rendering is disabled (renderService stays undefined) and startup continues. +// rendering is disabled (imageDelivery stays undefined: commands send text only) and startup continues. // - The queue/worker modules are imported lazily so a deployment without rendering never loads // Playwright. // - The pool is shut down last when the queue is closed (see shutdown.ts: imageQueue). +// - ImageDelivery (the background delivery of EVERY picture: AI tools, commands, buttons, invitation +// notices) sits on top of the render service and +// is closed BEFORE the queue (shutdown.ts: imageDelivery): its pending renders are cancelled but their +// BullMQ jobs and database rows are kept, so the next start delivers them (a delivery without a store +// tells its user instead), and the held render-failure summary is sent. It learns of calendar writes from the domain event +// bus (domain-events.ts), which is created later. +// - Deliveries are durable (image_deliveries, migration 070). Once the bot exists, index.ts attaches +// the Telegram transport and calls `reconcile()`: what the previous process left unfinished is +// delivered, expired with one note, or marked unknown (never re-sent). import type { Queue } from 'bullmq'; -import type { RenderService } from '../services/image/render-service.ts'; +import type { TelegramSender } from '../services/ai/types.ts'; +import { ImageDelivery } from '../services/image/image-delivery.ts'; +import { RenderFailureReporter } from '../services/image/render-failure-reporter.ts'; +import { sendAgendaImage } from '../utils/agenda-image.ts'; import { botLogger } from '../utils/logger.ts'; import type { ImageRenderJob } from '../worker/image-render.queue.ts'; import type { StartupContext } from './runtime.ts'; -export async function initImageRender(ctx: StartupContext): Promise { +export interface ImageRender { + /** Background delivery of every calendar picture; nothing in the bot awaits a render. */ + imageDelivery: ImageDelivery; +} + +export async function initImageRender(ctx: StartupContext): Promise { const { config } = ctx; if (!config.REDIS_URL) return undefined; @@ -23,13 +40,30 @@ export async function initImageRender(ctx: StartupContext): Promise, queueEvents); + const renderService = new RenderServiceClass(imageQueue as Queue, queueEvents); + const failures = new RenderFailureReporter((text, source) => ctx.db.alerts.push(text, source)); + const imageDelivery = new ImageDelivery({ + renderer: renderService, + store: ctx.db.imageDeliveries, + reportFailure: (report) => failures.report(report), + }); + imageRender = { imageDelivery }; + + ctx.resources.imageDelivery = { + close: async () => { + try { + await imageDelivery.close(); + } finally { + failures.flush(); // the summary held for the alert window must not die with the process + } + }, + }; ctx.resources.imageQueue = { close: async () => { @@ -44,5 +78,28 @@ export async function initImageRender(ctx: StartupContext): Promise sender.sendMessage(chatId, text), + sendImage: async (chatId, png, language) => { + const file = new File([png], 'picture.png', { type: 'image/png' }); + const sent = await sendAgendaImage(file, { + language, + sendPhoto: (photo) => sender.sendPhoto!(chatId, photo), + sendDocument: sender.sendDocument + ? (document, options) => sender.sendDocument!(chatId, document, options.caption) + : undefined, + }); + return sent.message_id; + }, + }); + imageDelivery.reconcile().catch((err: unknown) => botLogger.error({ err }, 'Image delivery reconciliation failed')); } diff --git a/src/bootstrap/runtime.ts b/src/bootstrap/runtime.ts index 1218ed34b..484c7f80f 100644 --- a/src/bootstrap/runtime.ts +++ b/src/bootstrap/runtime.ts @@ -38,6 +38,12 @@ export interface ShutdownResources { notificationQueue?: ClosableResource; botTasksQueue?: ClosableResource; syncQueue?: ClosableResource; + /** + * The picture deliveries still pending, closed before the image queue. With the store (always in production) no note + * is sent: the rows stay queued and the BullMQ jobs are kept, so the next start resumes them. Also sends the + * render-failure summary still held back for its alert window. + */ + imageDelivery?: ClosableResource; imageQueue?: ClosableResource; broadcastQueue?: ClosableResource; callQueue?: ClosableResource; diff --git a/src/bootstrap/shutdown.ts b/src/bootstrap/shutdown.ts index 11deed822..a5f27116c 100644 --- a/src/bootstrap/shutdown.ts +++ b/src/bootstrap/shutdown.ts @@ -9,7 +9,11 @@ // agent (AGENT_DRAIN_SETTLE_MS) so each running AI turn tells its user, queues its retry and // writes its debug log while the queues, Redis and the database are still open. // 3. Close the queues in a fixed order: ai-messages, event checker, notifications, bot tasks, Google -// sync, image render, broadcast, calls. +// sync, image delivery, image render, broadcast, calls. Image delivery goes first of the two image +// closes: pictures still rendering are logged as interrupted and their records stay queued so the next start +// delivers them (no note is sent), a picture mid-upload gets a 1.5 s grace and, if it does not land, is recorded +// unknown; then the render-failure summary still held back for its alert window is sent. Redis and Telegram +// still answer while this runs. // 4. A second, shorter drain (LATE_AGENT_DRAIN_SETTLE_MS): turns started by handlers that outlived // bot.stop() while the queues closed. Nothing may still be writing when Redis and SQLite close. // 5. Google Redis client, summarizer Redis, the web server. @@ -51,6 +55,7 @@ function closableQueues(resources: ShutdownResources) { resources.notificationQueue, resources.botTasksQueue, resources.syncQueue, + resources.imageDelivery, resources.imageQueue, resources.broadcastQueue, resources.callQueue, diff --git a/src/bot/commands/month.ts b/src/bot/commands/month.ts index c20d54066..63683b1ea 100644 --- a/src/bot/commands/month.ts +++ b/src/bot/commands/month.ts @@ -1,22 +1,19 @@ -import { enrichAgenda } from '../../services/event/agenda-enrichment.ts'; -import { agendaImageErrorMessage, sendAgendaImage } from '../../utils/agenda-image.ts'; // src/bot/commands/month.ts import { TZDate } from '@date-fns/tz'; import { endOfMonth, format, getDay, getDaysInMonth, startOfMonth } from 'date-fns'; -import { t } from '../../config/constants.ts'; import type { GroupChatRepository } from '../../database/repositories/group-chat.repository.ts'; import type { EventOccurrence, User } from '../../database/types.ts'; +import { enrichAgenda } from '../../services/event/agenda-enrichment.ts'; import type { EventService } from '../../services/event/event-service.ts'; -import { mapMonthlyCalendarData } from '../../services/image/data-mapper.ts'; -import type { ImageRenderer } from '../../services/image/render-service.ts'; -import { autoPin } from '../../utils/auto-pin.ts'; -import { imageLogger } from '../../utils/logger.ts'; -import { getTheme } from '../../worker/templates/themes.ts'; +import type { ImageDelivery } from '../../services/image/image-delivery.ts'; +import { queuePicture } from '../../services/image/queue-picture.ts'; +import { monthImageJob } from '../../services/image/render-month.ts'; import { getGroupId, isGroup } from '../group-context.ts'; import { monthNavKeyboard } from '../keyboards.ts'; import type { BotCallbackContext, BotCommandContext } from '../types.ts'; import { isCallbackContext } from '../types.ts'; +import { commandPictureTransport, pinPicture } from './picture-transport.ts'; /** Calendar month being shown, with the UTC bounds used for the single range query over it. */ interface MonthWindow { @@ -148,79 +145,39 @@ async function handleGroupMonth( await deliverMonthText(ctx, text, window.monthStart, yearMonth); } -function localDayIso(date: Date): string { - return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`; -} - interface MonthImageInput { - renderService: ImageRenderer; - occurrences: EventOccurrence[]; - monthStart: Date; + imageDelivery: ImageDelivery; + /** The month's agenda as it is now; called when the picture is queued and again if the calendar changes while it renders. */ + load(): EventOccurrence[]; + window: MonthWindow; user: User; lang: 'en' | 'ru'; groupRepo: GroupChatRepository | undefined; } -/** Render the month as an image, send and pin it; render failures are reported to the user. */ -async function sendMonthImage(ctx: BotCommandContext | BotCallbackContext, input: MonthImageInput): Promise { - const { renderService, occurrences, monthStart, user, lang, groupRepo } = input; - try { - const occurrencesByDay = new Map(); - for (const occ of occurrences) { - const dayIso = localDayIso(new TZDate(occ.occurrence_start, user.timezone)); - if (!occurrencesByDay.has(dayIso)) occurrencesByDay.set(dayIso, []); - occurrencesByDay.get(dayIso)!.push(occ); - } - - const todayIso = localDayIso(new TZDate(new Date(), user.timezone)); - - const data = mapMonthlyCalendarData({ - occurrencesByDay, - year: monthStart.getFullYear(), - month: monthStart.getMonth(), - timezone: user.timezone, - locale: lang, - theme: getTheme(), - todayIso, - }); - - const buffer = await renderService.renderDirect({ - type: 'monthly-calendar', - data, - userId: user.telegram_id, - }); - const file = new File([buffer], 'month.png', { type: 'image/png' }); - const sent = await sendAgendaImage(file, { - language: user.language, - sendPhoto: (photo) => ctx.sendPhoto(photo), - sendDocument: (document, options) => ctx.sendDocument(document, options), - }); - autoPin(user.telegram_id, sent.id, { - pinChatMessage: (chatId, messageId, options) => - ctx.bot.api.pinChatMessage({ - chat_id: chatId, - message_id: messageId, - disable_notification: options.disable_notification, - }), - sendMessage: async (chatId, text) => { - await ctx.bot.api.sendMessage({ chat_id: chatId, text }); - }, - isGroupChat: false, - groupChatRepo: groupRepo, - }).catch((err) => { - imageLogger.error({ err }, 'autoPin failed'); - }); - } catch (err) { - imageLogger.error({ err }, 'Month render failed'); - await ctx.send(agendaImageErrorMessage(err) ?? t(lang).callbackErrors.imageGenerationFailed); - } +/** Queue the month picture (it is sent and pinned when ready); the text answer is already out. */ +async function queueMonthImage(ctx: BotCommandContext | BotCallbackContext, input: MonthImageInput): Promise { + const { imageDelivery, load, window, user, lang, groupRepo } = input; + const chatId = user.telegram_id; + await queuePicture(imageDelivery, { + ...commandPictureTransport(ctx, chatId), + language: lang, + kind: 'month', + label: window.monthLabel, + slot: `month:user:${chatId}:${window.monthStart.getFullYear()}-${window.monthStart.getMonth()}`, + dataOwner: chatId, + filename: 'month.png', + buildJob: () => + monthImageJob(load(), window.monthStart.getFullYear(), window.monthStart.getMonth(), user.timezone, lang, chatId), + onSent: (messageId) => pinPicture(ctx, chatId, messageId, groupRepo, false), + }); } export async function handleMonth( ctx: BotCommandContext | BotCallbackContext, eventService: EventService, yearMonth?: string, - renderService?: ImageRenderer, + imageDelivery?: ImageDelivery, groupRepo?: GroupChatRepository, ): Promise { const user = ctx.dbUser; @@ -235,24 +192,19 @@ export async function handleMonth( const window = resolveMonthWindow(yearMonth, user.timezone, lang); // Count events per day — single range query for the whole month, then bucket - const allOccurrences = enrichAgenda( - eventService.getEventsInRange(user.telegram_id, window.monthStartUtc, window.monthEndUtc), - { userId: user.telegram_id, language: lang }, - eventService.agendaRepository, - ); + const load = () => + enrichAgenda( + eventService.getEventsInRange(user.telegram_id, window.monthStartUtc, window.monthEndUtc), + { userId: user.telegram_id, language: lang }, + eventService.agendaRepository, + ); + const allOccurrences = load(); const text = buildMonthText(window, countEventsPerDay(allOccurrences, user.timezone)); await deliverMonthText(ctx, text, window.monthStart, yearMonth); - // Render month image (only on initial /month command, not nav callbacks) - if (!yearMonth && renderService) { - await sendMonthImage(ctx, { - renderService, - occurrences: allOccurrences, - monthStart: window.monthStart, - user, - lang, - groupRepo, - }); + // Month picture (only on the initial /month command, not nav callbacks) + if (!yearMonth && imageDelivery) { + await queueMonthImage(ctx, { imageDelivery, load, window, user, lang, groupRepo }); } } diff --git a/src/bot/commands/picture-transport.ts b/src/bot/commands/picture-transport.ts new file mode 100644 index 000000000..fd4f6c5d7 --- /dev/null +++ b/src/bot/commands/picture-transport.ts @@ -0,0 +1,49 @@ +// How a command (or a button that behaves like one) sends and pins its picture. The sends go through +// the context so a reply lands where the command was typed; the pin is the best-effort autoPin that +// /today and /month have always done. +import type { GroupChatRepository } from '../../database/repositories/group-chat.repository.ts'; +import type { PictureRequest } from '../../services/image/queue-picture.ts'; +import { autoPin } from '../../utils/auto-pin.ts'; +import { imageLogger } from '../../utils/logger.ts'; +import type { BotCallbackContext, BotCommandContext } from '../types.ts'; + +type PictureContext = BotCommandContext | BotCallbackContext; + +/** The transport half of a PictureRequest for a private-chat command. */ +export function commandPictureTransport( + ctx: PictureContext, + chatId: number, +): Pick { + return { + chatId, + requestId: ctx.id, + sendPhoto: async (file) => (await ctx.sendPhoto(file)).id, + sendDocument: async (file, options) => (await ctx.sendDocument(file, options)).id, + notify: (text) => ctx.send(text), + }; +} + +/** Pins the delivered picture when the user's pin preference says so; never throws. */ +export function pinPicture( + ctx: PictureContext, + chatId: number, + messageId: number, + groupRepo: GroupChatRepository | undefined, + isGroupChat: boolean, +): void { + autoPin(chatId, messageId, { + pinChatMessage: (cid, pinnedId, options) => + ctx.bot.api.pinChatMessage({ + chat_id: cid, + message_id: pinnedId, + disable_notification: options.disable_notification, + }), + sendMessage: async (cid, text) => { + await ctx.bot.api.sendMessage({ chat_id: cid, text }); + }, + isGroupChat, + groupChatRepo: groupRepo, + }).catch((err) => { + imageLogger.error({ err }, 'autoPin failed'); + }); +} diff --git a/src/bot/commands/today.ts b/src/bot/commands/today.ts index 82b34e134..244be4d62 100644 --- a/src/bot/commands/today.ts +++ b/src/bot/commands/today.ts @@ -1,25 +1,23 @@ -import { enrichAgenda } from '../../services/event/agenda-enrichment.ts'; -import { agendaImageErrorMessage, sendAgendaImage } from '../../utils/agenda-image.ts'; // src/bot/commands/today.ts import { TZDate } from '@date-fns/tz'; -import { t } from '../../config/constants.ts'; import type { GoogleCalendarRepository } from '../../database/repositories/google-calendar.repository.ts'; import type { GroupChatRepository } from '../../database/repositories/group-chat.repository.ts'; import type { EventOccurrence, User } from '../../database/types.ts'; +import { enrichAgenda } from '../../services/event/agenda-enrichment.ts'; import type { EventService } from '../../services/event/event-service.ts'; import { formatDayAgenda } from '../../services/event/formatters.ts'; import { googleCalendarColorEmoji } from '../../services/google/calendar-colors.ts'; import type { HolidayEntry, HolidayService } from '../../services/holiday/holiday-service.ts'; -import { renderDayImage } from '../../services/image/render-day.ts'; -import type { ImageRenderer } from '../../services/image/render-service.ts'; +import type { ImageDelivery } from '../../services/image/image-delivery.ts'; +import { queuePicture } from '../../services/image/queue-picture.ts'; +import { dayImageJob } from '../../services/image/render-day.ts'; import type { DayWeather } from '../../services/weather/types.ts'; import type { WeatherService } from '../../services/weather/weather-service.ts'; -import { autoPin } from '../../utils/auto-pin.ts'; -import { imageLogger } from '../../utils/logger.ts'; import { getGroupId, isGroup } from '../group-context.ts'; import type { BotCommandContext } from '../types.ts'; import { sendAgendaText } from './agenda-text.ts'; +import { commandPictureTransport, pinPicture } from './picture-transport.ts'; async function fetchTodayWeather( weatherService: WeatherService | undefined, @@ -69,60 +67,40 @@ async function handleGroupToday( } interface TodayImageInput { - renderService: ImageRenderer; - occurrences: EventOccurrence[]; + imageDelivery: ImageDelivery; + /** The agenda as it is now; called when the picture is queued and again if the calendar changes while it renders. */ + load(): { occurrences: EventOccurrence[]; holidays: HolidayEntry[] }; dateIso: string; - holidays: HolidayEntry[]; user: User; groupRepo: GroupChatRepository | undefined; lang: 'en' | 'ru'; } -/** Render today's agenda as an image, send it and pin it; render failures are reported to the user. */ -async function sendTodayImage(ctx: BotCommandContext, input: TodayImageInput): Promise { - const { renderService, occurrences, dateIso, holidays, user, groupRepo, lang } = input; - try { - const buffer = await renderDayImage( - renderService, - occurrences, - dateIso, - user.timezone, - user.language as 'ru' | 'en', - user.telegram_id, - holidays, - ); - const file = new File([buffer], 'today.png', { type: 'image/png' }); - const sent = await sendAgendaImage(file, { - language: user.language, - sendPhoto: (photo) => ctx.sendPhoto(photo), - sendDocument: (document, options) => ctx.sendDocument(document, options), - }); - autoPin(user.telegram_id, sent.id, { - pinChatMessage: (chatId, messageId, options) => - ctx.bot.api.pinChatMessage({ - chat_id: chatId, - message_id: messageId, - disable_notification: options.disable_notification, - }), - sendMessage: async (chatId, text) => { - await ctx.bot.api.sendMessage({ chat_id: chatId, text }); - }, - isGroupChat: false, - groupChatRepo: groupRepo, - }).catch((err) => { - imageLogger.error({ err }, 'autoPin failed'); - }); - } catch (err) { - imageLogger.error({ err }, 'Render failed'); - await ctx.send(agendaImageErrorMessage(err) ?? t(lang).callbackErrors.imageGenerationFailed); - } +/** Queue today's agenda picture (it is sent and pinned when ready); the text answer is already out. */ +async function queueTodayImage(ctx: BotCommandContext, input: TodayImageInput): Promise { + const { imageDelivery, load, dateIso, user, groupRepo, lang } = input; + const chatId = user.telegram_id; + await queuePicture(imageDelivery, { + ...commandPictureTransport(ctx, chatId), + language: lang, + kind: 'day', + label: dateIso, + slot: `day:user:${chatId}:${dateIso}`, + dataOwner: chatId, + filename: 'today.png', + buildJob: () => { + const { occurrences, holidays } = load(); + return dayImageJob(occurrences, dateIso, user.timezone, lang, chatId, holidays); + }, + onSent: (messageId) => pinPicture(ctx, chatId, messageId, groupRepo, false), + }); } export async function handleToday( ctx: BotCommandContext, eventService: EventService, holidayService?: HolidayService, - renderService?: ImageRenderer, + imageDelivery?: ImageDelivery, groupRepo?: GroupChatRepository, googleCalendarRepo?: GoogleCalendarRepository, weatherService?: WeatherService, @@ -137,13 +115,16 @@ export async function handleToday( } const now = new Date(); - const occurrences = enrichAgenda( - eventService.getEventsForDay(user.telegram_id, now, user.timezone), - { userId: user.telegram_id, language: lang }, - eventService.agendaRepository, - ); const dateIso = new TZDate(now, user.timezone).toISOString().slice(0, 10); - const holidays = holidayService?.getHolidaysForDate(user.telegram_id, dateIso) ?? []; + const load = () => ({ + occurrences: enrichAgenda( + eventService.getEventsForDay(user.telegram_id, now, user.timezone), + { userId: user.telegram_id, language: lang }, + eventService.agendaRepository, + ), + holidays: holidayService?.getHolidaysForDate(user.telegram_id, dateIso) ?? [], + }); + const { occurrences, holidays } = load(); const calendarColors = buildCalendarColorMap(googleCalendarRepo, user.telegram_id); const dayWeather = await fetchTodayWeather(weatherService, user.timezone, lang); const text = formatDayAgenda( @@ -158,8 +139,8 @@ export async function handleToday( await sendAgendaText(ctx, text); - if (renderService) { - await sendTodayImage(ctx, { renderService, occurrences, dateIso, holidays, user, groupRepo, lang }); + if (imageDelivery) { + await queueTodayImage(ctx, { imageDelivery, load, dateIso, user, groupRepo, lang }); } } diff --git a/src/bot/commands/tomorrow.ts b/src/bot/commands/tomorrow.ts index 577a095c8..0daa792ec 100644 --- a/src/bot/commands/tomorrow.ts +++ b/src/bot/commands/tomorrow.ts @@ -1,25 +1,24 @@ -import { enrichAgenda } from '../../services/event/agenda-enrichment.ts'; -import { agendaImageErrorMessage, sendAgendaImage } from '../../utils/agenda-image.ts'; // src/bot/commands/tomorrow.ts import { TZDate } from '@date-fns/tz'; import { addDays } from 'date-fns'; -import { t } from '../../config/constants.ts'; import type { GoogleCalendarRepository } from '../../database/repositories/google-calendar.repository.ts'; import type { GroupChatRepository } from '../../database/repositories/group-chat.repository.ts'; import type { EventOccurrence, User } from '../../database/types.ts'; +import { enrichAgenda } from '../../services/event/agenda-enrichment.ts'; import type { EventService } from '../../services/event/event-service.ts'; import { formatDayAgenda } from '../../services/event/formatters.ts'; import { googleCalendarColorEmoji } from '../../services/google/calendar-colors.ts'; import type { HolidayEntry, HolidayService } from '../../services/holiday/holiday-service.ts'; -import { renderDayImage } from '../../services/image/render-day.ts'; -import type { ImageRenderer } from '../../services/image/render-service.ts'; +import type { ImageDelivery } from '../../services/image/image-delivery.ts'; +import { queuePicture } from '../../services/image/queue-picture.ts'; +import { dayImageJob } from '../../services/image/render-day.ts'; import type { DayWeather } from '../../services/weather/types.ts'; import type { WeatherService } from '../../services/weather/weather-service.ts'; -import { imageLogger } from '../../utils/logger.ts'; import { getGroupId, isGroup } from '../group-context.ts'; import type { BotCommandContext } from '../types.ts'; import { sendAgendaText } from './agenda-text.ts'; +import { commandPictureTransport } from './picture-transport.ts'; async function handleGroupTomorrow( ctx: BotCommandContext, @@ -63,44 +62,38 @@ async function handleGroupTomorrow( } interface TomorrowImageInput { - renderService: ImageRenderer; - occurrences: EventOccurrence[]; + imageDelivery: ImageDelivery; + /** The agenda as it is now; called when the picture is queued and again if the calendar changes while it renders. */ + load(): { occurrences: EventOccurrence[]; holidays: HolidayEntry[] }; dateIso: string; - holidays: HolidayEntry[]; user: User; lang: 'en' | 'ru'; } -/** Render tomorrow's agenda as an image and send it; render failures are reported to the user. */ -async function sendTomorrowImage(ctx: BotCommandContext, input: TomorrowImageInput): Promise { - const { renderService, occurrences, dateIso, holidays, user, lang } = input; - try { - const buffer = await renderDayImage( - renderService, - occurrences, - dateIso, - user.timezone, - user.language as 'ru' | 'en', - user.telegram_id, - holidays, - ); - const file = new File([buffer], 'tomorrow.png', { type: 'image/png' }); - await sendAgendaImage(file, { - language: user.language, - sendPhoto: (photo) => ctx.sendPhoto(photo), - sendDocument: (document, options) => ctx.sendDocument(document, options), - }); - } catch (err) { - imageLogger.error({ err }, 'Render failed'); - await ctx.send(agendaImageErrorMessage(err) ?? t(lang).callbackErrors.imageGenerationFailed); - } +/** Queue tomorrow's agenda picture (it is sent when ready); the text answer is already out. */ +async function queueTomorrowImage(ctx: BotCommandContext, input: TomorrowImageInput): Promise { + const { imageDelivery, load, dateIso, user, lang } = input; + const chatId = user.telegram_id; + await queuePicture(imageDelivery, { + ...commandPictureTransport(ctx, chatId), + language: lang, + kind: 'day', + label: dateIso, + slot: `day:user:${chatId}:${dateIso}`, + dataOwner: chatId, + filename: 'tomorrow.png', + buildJob: () => { + const { occurrences, holidays } = load(); + return dayImageJob(occurrences, dateIso, user.timezone, lang, chatId, holidays); + }, + }); } export async function handleTomorrow( ctx: BotCommandContext, eventService: EventService, holidayService?: HolidayService, - renderService?: ImageRenderer, + imageDelivery?: ImageDelivery, groupRepo?: GroupChatRepository, googleCalendarRepo?: GoogleCalendarRepository, weatherService?: WeatherService, @@ -115,13 +108,16 @@ export async function handleTomorrow( } const tomorrow = addDays(new Date(), 1); - const occurrences = enrichAgenda( - eventService.getEventsForDay(user.telegram_id, tomorrow, user.timezone), - { userId: user.telegram_id, language: lang }, - eventService.agendaRepository, - ); const dateIso = new TZDate(tomorrow, user.timezone).toISOString().slice(0, 10); - const holidays = holidayService?.getHolidaysForDate(user.telegram_id, dateIso) ?? []; + const load = () => ({ + occurrences: enrichAgenda( + eventService.getEventsForDay(user.telegram_id, tomorrow, user.timezone), + { userId: user.telegram_id, language: lang }, + eventService.agendaRepository, + ), + holidays: holidayService?.getHolidaysForDate(user.telegram_id, dateIso) ?? [], + }); + const { occurrences, holidays } = load(); const calendarColors = buildCalendarColorMap(googleCalendarRepo, user.telegram_id); const dayWeather = await fetchDayForecast(weatherService, user.timezone, dateIso, lang); const text = formatDayAgenda( @@ -136,8 +132,8 @@ export async function handleTomorrow( await sendAgendaText(ctx, text); - if (renderService) { - await sendTomorrowImage(ctx, { renderService, occurrences, dateIso, holidays, user, lang }); + if (imageDelivery) { + await queueTomorrowImage(ctx, { imageDelivery, load, dateIso, user, lang }); } } diff --git a/src/bot/commands/week.ts b/src/bot/commands/week.ts index 02fb4f183..0562f8ae0 100644 --- a/src/bot/commands/week.ts +++ b/src/bot/commands/week.ts @@ -1,24 +1,24 @@ -import { enrichAgenda } from '../../services/event/agenda-enrichment.ts'; -import { agendaImageErrorMessage, sendAgendaImage } from '../../utils/agenda-image.ts'; // src/bot/commands/week.ts import { TZDate } from '@date-fns/tz'; import { startOfWeek } from 'date-fns'; -import { t } from '../../config/constants.ts'; import type { GroupChatRepository } from '../../database/repositories/group-chat.repository.ts'; import type { EventOccurrence, User } from '../../database/types.ts'; +import { enrichAgenda } from '../../services/event/agenda-enrichment.ts'; import type { EventService } from '../../services/event/event-service.ts'; import { formatWeekAgenda } from '../../services/event/formatters.ts'; import type { HolidayEntry, HolidayService } from '../../services/holiday/holiday-service.ts'; -import type { ImageRenderer } from '../../services/image/render-service.ts'; -import { renderWeekImage } from '../../services/image/render-week.ts'; +import { formatWeekLabel } from '../../services/image/data-mapper.ts'; +import type { ImageDelivery } from '../../services/image/image-delivery.ts'; +import { queuePicture } from '../../services/image/queue-picture.ts'; +import { weekImageJob } from '../../services/image/render-week.ts'; import type { DayWeather } from '../../services/weather/types.ts'; import type { WeatherService } from '../../services/weather/weather-service.ts'; import { getWeekRangeUtc, localCalendarWeekDays } from '../../utils/date.ts'; -import { imageLogger } from '../../utils/logger.ts'; import { getGroupId, isGroup } from '../group-context.ts'; import type { BotCommandContext } from '../types.ts'; import { sendAgendaText } from './agenda-text.ts'; +import { commandPictureTransport } from './picture-transport.ts'; /** Holidays per local calendar day of the week starting at `start`; undefined when holidays are not configured. */ function collectWeekHolidays( @@ -73,45 +73,36 @@ async function handleGroupWeek( } interface WeekImageInput { - renderService: ImageRenderer; - occurrences: EventOccurrence[]; - now: Date; + imageDelivery: ImageDelivery; + /** The week's agenda as it is now; called when the picture is queued and again if the calendar changes while it renders. */ + load(): EventOccurrence[]; + weekStartIso: string; weatherByDate: { [date: string]: DayWeather } | undefined; user: User; lang: 'en' | 'ru'; } -/** Render the week agenda as an image and send it; render failures are reported to the user. */ -async function sendWeekImage(ctx: BotCommandContext, input: WeekImageInput): Promise { - const { renderService, occurrences, now, weatherByDate, user, lang } = input; - try { - const weekStartIso = startOfWeek(new TZDate(now, user.timezone), { weekStartsOn: 1 }).toISOString().slice(0, 10); - const buffer = await renderWeekImage( - renderService, - occurrences, - weekStartIso, - user.timezone, - user.language as 'ru' | 'en', - user.telegram_id, - weatherByDate, - ); - const file = new File([buffer], 'week.png', { type: 'image/png' }); - await sendAgendaImage(file, { - language: user.language, - sendPhoto: (photo) => ctx.sendPhoto(photo), - sendDocument: (document, options) => ctx.sendDocument(document, options), - }); - } catch (err) { - imageLogger.error({ err }, 'Render failed'); - await ctx.send(agendaImageErrorMessage(err) ?? t(lang).callbackErrors.imageGenerationFailed); - } +/** Queue the week agenda picture (it is sent when ready); the text answer is already out. */ +async function queueWeekImage(ctx: BotCommandContext, input: WeekImageInput): Promise { + const { imageDelivery, load, weekStartIso, weatherByDate, user, lang } = input; + const chatId = user.telegram_id; + await queuePicture(imageDelivery, { + ...commandPictureTransport(ctx, chatId), + language: lang, + kind: 'week', + label: formatWeekLabel(weekStartIso, lang), + slot: `week:user:${chatId}:${weekStartIso}`, + dataOwner: chatId, + filename: 'week.png', + buildJob: () => weekImageJob(load(), weekStartIso, user.timezone, lang, chatId, weatherByDate), + }); } export async function handleWeek( ctx: BotCommandContext, eventService: EventService, holidayService?: HolidayService, - renderService?: ImageRenderer, + imageDelivery?: ImageDelivery, groupRepo?: GroupChatRepository, weatherService?: WeatherService, ): Promise { @@ -126,19 +117,22 @@ export async function handleWeek( const now = new Date(); const { start, end } = getWeekRangeUtc(now, user.timezone); - const occurrences = enrichAgenda( - eventService.getEventsInRange(user.telegram_id, start, end), - { userId: user.telegram_id, language: lang }, - eventService.agendaRepository, - ); + const load = () => + enrichAgenda( + eventService.getEventsInRange(user.telegram_id, start, end), + { userId: user.telegram_id, language: lang }, + eventService.agendaRepository, + ); + const occurrences = load(); const holidaysByDate = collectWeekHolidays(holidayService, user.telegram_id, start, user.timezone); const weatherByDate = await fetchWeekWeatherByDate(weatherService, user.timezone, lang); const text = formatWeekAgenda(occurrences, start, end, user.timezone, user.language, holidaysByDate, weatherByDate); await sendAgendaText(ctx, text); - if (renderService) { - await sendWeekImage(ctx, { renderService, occurrences, now, weatherByDate, user, lang }); + if (imageDelivery) { + const weekStartIso = startOfWeek(new TZDate(now, user.timezone), { weekStartsOn: 1 }).toISOString().slice(0, 10); + await queueWeekImage(ctx, { imageDelivery, load, weekStartIso, weatherByDate, user, lang }); } } diff --git a/src/bot/handlers/callback/agenda-routes.ts b/src/bot/handlers/callback/agenda-routes.ts index 7412814dd..f685c4a40 100644 --- a/src/bot/handlers/callback/agenda-routes.ts +++ b/src/bot/handlers/callback/agenda-routes.ts @@ -7,13 +7,15 @@ import { CB, t } from '../../../config/constants.ts'; import type { User } from '../../../database/types.ts'; import { enrichAgenda, enrichAgendaEvents } from '../../../services/event/agenda-enrichment.ts'; import { formatDayAgenda, formatEventDetail, formatWeekAgenda } from '../../../services/event/formatters.ts'; -import { mapDailyAgendaData, mapWeeklyOverviewData } from '../../../services/image/data-mapper.ts'; -import { agendaImageErrorMessage, sendAgendaImage } from '../../../utils/agenda-image.ts'; -import { autoPin } from '../../../utils/auto-pin.ts'; +import { formatWeekLabel, mapDailyAgendaData, mapWeeklyOverviewData } from '../../../services/image/data-mapper.ts'; +import type { ImageDelivery } from '../../../services/image/image-delivery.ts'; +import type { PictureRequest } from '../../../services/image/queue-picture.ts'; +import { queuePicture } from '../../../services/image/queue-picture.ts'; import { getDayRangeUtc, getWeekRangeUtc, localCalendarWeekDays } from '../../../utils/date.ts'; -import { imageLogger } from '../../../utils/logger.ts'; +import type { ImageRenderJob } from '../../../worker/image-render.queue.ts'; import { getTheme } from '../../../worker/templates/themes.ts'; import { sendAgendaText } from '../../commands/agenda-text.ts'; +import { pinPicture } from '../../commands/picture-transport.ts'; import { getGroupId, isGroup } from '../../group-context.ts'; import type { BotCallbackContext } from '../../types.ts'; import type { CallbackDispatch, CallbackRouteDeps } from './types.ts'; @@ -23,9 +25,14 @@ interface AgendaView { groupId: number | undefined; } +/** The part of a picture's slot that says whose picture it is: a viewer inside a group chat, or the user in private. */ +function viewerKey(groupId: number | undefined, userId: number): string { + return groupId === undefined ? `user:${userId}` : `group:${groupId}:viewer:${userId}`; +} + export function registerAgendaRoutes(dispatch: CallbackDispatch, deps: CallbackRouteDeps): void { const { eventService, holidayService } = deps; - const { renderService, groupRepo } = deps.opts; + const { imageDelivery, groupRepo } = deps.opts; function agendaView(ctx: BotCallbackContext, user: User): AgendaView | null { const chat = ctx.message?.chat; @@ -48,56 +55,40 @@ export function registerAgendaRoutes(dispatch: CallbackDispatch, deps: CallbackR groupId: number | undefined; } - async function deliverAgendaImage( + /** + * Queue the picture and return: it is sent as a reply to the tapped message when ready. The button + * spinner was already answered by the caller, so the user is never kept waiting on a render. + */ + async function queueAgendaImage( ctx: BotCallbackContext, + imageDelivery: ImageDelivery, lang: Lang, - userId: number, - filename: string, - render: () => Promise, + user: User, + view: AgendaView, + picture: Pick, ): Promise { - try { - const buffer = await render(); - const file = new File([buffer], filename, { type: 'image/png' }); - if (!ctx.message) { - await ctx.answer({ text: t(lang).callbackErrors.couldNotSendImage }); - return; - } - const sent = await sendAgendaImage(file, { - language: lang, - sendPhoto: (photo) => ctx.message!.sendPhoto(photo), - sendDocument: (document, options) => ctx.message!.sendDocument(document, options), - }); - if (!sent) return; - pinSentAgenda(ctx, Number(ctx.chatId ?? userId), sent.id); - } catch (err) { - imageLogger.error({ err }, 'Render failed'); - if (ctx.message) { - await ctx.message.send(agendaImageErrorMessage(err) ?? t(lang).callbackErrors.imageGenerationFailed); - } else { - await ctx.answer({ text: t(lang).callbackErrors.renderFailed }); - } + const message = ctx.message; + if (!message) { + await ctx.answer({ text: t(lang).callbackErrors.couldNotSendImage }); + return; } - } - - function pinSentAgenda(ctx: BotCallbackContext, chatId: number, messageId: number): void { - autoPin(chatId, messageId, { - pinChatMessage: (cid, pinnedId, options) => - ctx.bot.api.pinChatMessage({ - chat_id: cid, - message_id: pinnedId, - disable_notification: options.disable_notification, - }), - sendMessage: (cid, text) => ctx.bot.api.sendMessage({ chat_id: cid, text }).then(() => {}), - isGroupChat: isGroup(ctx), - groupChatRepo: groupRepo, - }).catch((err) => { - imageLogger.error({ err }, 'autoPin failed'); + const chatId = Number(ctx.chatId ?? user.telegram_id); + await queuePicture(imageDelivery, { + ...picture, + chatId, + language: lang, + requestId: ctx.id, + dataOwner: view.groupId === undefined ? user.telegram_id : 'group', + sendPhoto: async (file) => (await message.sendPhoto(file)).id, + sendDocument: async (file, options) => (await message.sendDocument(file, options)).id, + notify: (text) => message.send(text), + onSent: (messageId) => pinPicture(ctx, chatId, messageId, groupRepo, isGroup(ctx)), }); } // Image: daily agenda dispatch.set(CB.IMG_DAILY, async (ctx, payload, _parts, user) => { - if (!renderService) { + if (!imageDelivery) { await ctx.answer(); return; } @@ -108,44 +99,54 @@ export function registerAgendaRoutes(dispatch: CallbackDispatch, deps: CallbackR const view = agendaView(ctx, user); if (!view) return; const { timezone, groupId } = view; - const now = new Date(); - const [year, month, day] = dateIso.split('-').map(Number); - const dayDate = new TZDate(year!, month! - 1, day!, 12, timezone); - const { start, end } = getDayRangeUtc(dayDate, timezone); - const occurrences = enrichAgenda( - groupId === undefined - ? eventService.getEventsForDay(user.telegram_id, dayDate, timezone) - : eventService.getEventsInRangeForGroup(groupId, start, end), - { userId: user.telegram_id, language: lang, groupId }, - eventService.agendaRepository, - ); - const userNow = new TZDate(now, timezone); - const todayIso = userNow.toISOString().slice(0, 10); - const isToday = dateIso === todayIso; - const currentTimeMinutes = isToday ? userNow.getHours() * 60 + userNow.getMinutes() : undefined; + // Read the agenda when the render starts, and again if the calendar changes while it renders. + const buildJob = (): ImageRenderJob => { + const now = new Date(); + const [year, month, day] = dateIso.split('-').map(Number); + const dayDate = new TZDate(year!, month! - 1, day!, 12, timezone); + const { start, end } = getDayRangeUtc(dayDate, timezone); + const occurrences = enrichAgenda( + groupId === undefined + ? eventService.getEventsForDay(user.telegram_id, dayDate, timezone) + : eventService.getEventsInRangeForGroup(groupId, start, end), + { userId: user.telegram_id, language: lang, groupId }, + eventService.agendaRepository, + ); - const holidays = holidayService.getHolidaysForDate(user.telegram_id, dateIso); + const userNow = new TZDate(now, timezone); + const todayIso = userNow.toISOString().slice(0, 10); + const isToday = dateIso === todayIso; + const currentTimeMinutes = isToday ? userNow.getHours() * 60 + userNow.getMinutes() : undefined; - const imgData = mapDailyAgendaData({ - occurrences, - dateIso, - timezone, - locale: user.language as 'ru' | 'en', - theme: getTheme(), - currentTimeMinutes, - isHoliday: holidays.length > 0, - holidayName: holidays[0]?.name, - }); + const holidays = holidayService.getHolidaysForDate(user.telegram_id, dateIso); - await deliverAgendaImage(ctx, lang, user.telegram_id, 'agenda.png', () => - renderService.renderDirect({ type: 'daily-agenda', data: imgData, userId: user.telegram_id }), - ); + const data = mapDailyAgendaData({ + occurrences, + dateIso, + timezone, + locale: user.language as 'ru' | 'en', + theme: getTheme(), + currentTimeMinutes, + isHoliday: holidays.length > 0, + holidayName: holidays[0]?.name, + }); + return { type: 'daily-agenda', data, userId: user.telegram_id }; + }; + + await queueAgendaImage(ctx, imageDelivery, lang, user, view, { + kind: 'day', + label: dateIso, + // The picture is localised per viewer (holidays, language), so two members of one group tapping the same day are two pictures. + slot: `day:${viewerKey(groupId, user.telegram_id)}:${dateIso}:${lang}`, + filename: 'agenda.png', + buildJob, + }); }); // Image: weekly overview dispatch.set(CB.IMG_WEEKLY, async (ctx, payload, _parts, user) => { - if (!renderService) { + if (!imageDelivery) { await ctx.answer(); return; } @@ -156,45 +157,54 @@ export function registerAgendaRoutes(dispatch: CallbackDispatch, deps: CallbackR const view = agendaView(ctx, user); if (!view) return; const { timezone, groupId } = view; - const now = new Date(); - const [year, month, day] = weekStartIso.split('-').map(Number); - const weekStartDate = new TZDate(year!, month! - 1, day!, 12, timezone); - const { start, end } = getWeekRangeUtc(weekStartDate, timezone); - const occurrences = enrichAgenda( - groupId === undefined - ? eventService.getEventsInRange(user.telegram_id, start, end) - : eventService.getEventsInRangeForGroup(groupId, start, end), - { userId: user.telegram_id, language: lang, groupId }, - eventService.agendaRepository, - ); - const occurrencesByDay = new Map(); - for (const dayKey of localCalendarWeekDays(start, timezone)) { - occurrencesByDay.set(dayKey, []); - } - for (const occ of occurrences) { - const occDate = new TZDate(new Date(occ.occurrence_start), timezone).toISOString().slice(0, 10); - const dayList = occurrencesByDay.get(occDate); - if (dayList) { - dayList.push(occ); + // Read the agenda when the render starts, and again if the calendar changes while it renders. + const buildJob = (): ImageRenderJob => { + const now = new Date(); + const [year, month, day] = weekStartIso.split('-').map(Number); + const weekStartDate = new TZDate(year!, month! - 1, day!, 12, timezone); + const { start, end } = getWeekRangeUtc(weekStartDate, timezone); + const occurrences = enrichAgenda( + groupId === undefined + ? eventService.getEventsInRange(user.telegram_id, start, end) + : eventService.getEventsInRangeForGroup(groupId, start, end), + { userId: user.telegram_id, language: lang, groupId }, + eventService.agendaRepository, + ); + + const occurrencesByDay = new Map(); + for (const dayKey of localCalendarWeekDays(start, timezone)) { + occurrencesByDay.set(dayKey, []); + } + for (const occ of occurrences) { + const occDate = new TZDate(new Date(occ.occurrence_start), timezone).toISOString().slice(0, 10); + const dayList = occurrencesByDay.get(occDate); + if (dayList) { + dayList.push(occ); + } } - } - const userNow = new TZDate(now, timezone); - const todayIso = userNow.toISOString().slice(0, 10); + const userNow = new TZDate(now, timezone); + const todayIso = userNow.toISOString().slice(0, 10); - const imgData = mapWeeklyOverviewData({ - occurrencesByDay, - weekStartIso, - timezone, - locale: user.language as 'ru' | 'en', - theme: getTheme(), - todayIso, - }); + const data = mapWeeklyOverviewData({ + occurrencesByDay, + weekStartIso, + timezone, + locale: user.language as 'ru' | 'en', + theme: getTheme(), + todayIso, + }); + return { type: 'weekly-overview', data, userId: user.telegram_id }; + }; - await deliverAgendaImage(ctx, lang, user.telegram_id, 'week.png', () => - renderService.renderDirect({ type: 'weekly-overview', data: imgData, userId: user.telegram_id }), - ); + await queueAgendaImage(ctx, imageDelivery, lang, user, view, { + kind: 'week', + label: formatWeekLabel(weekStartIso, lang), + slot: `week:${viewerKey(groupId, user.telegram_id)}:${weekStartIso}:${lang}`, + filename: 'week.png', + buildJob, + }); }); async function sendShareAgenda( diff --git a/src/bot/handlers/callback/invitation-actions.ts b/src/bot/handlers/callback/invitation-actions.ts index 49e4abe25..cc332584e 100644 --- a/src/bot/handlers/callback/invitation-actions.ts +++ b/src/bot/handlers/callback/invitation-actions.ts @@ -295,7 +295,8 @@ export async function answerInvitation(call: InvitationActionCall, subAction: In user, opts.invitationNotifyDeps, opts.eventRepo, - opts.renderService, + opts.imageDelivery, + ctx.id, ).catch(() => {}); } if (!user.onboarding_completed && opts.onboardingScene) { diff --git a/src/bot/handlers/callback/invitation-notify.ts b/src/bot/handlers/callback/invitation-notify.ts index 6cec3ac04..c0e71b54c 100644 --- a/src/bot/handlers/callback/invitation-notify.ts +++ b/src/bot/handlers/callback/invitation-notify.ts @@ -7,10 +7,10 @@ import { CB, t } from '../../../config/constants.ts'; import type { EventRepository } from '../../../database/repositories/event.repository.ts'; import type { UserRepository } from '../../../database/repositories/user.repository.ts'; import type { Invitation, User } from '../../../database/types.ts'; -import { renderConflictImage } from '../../../services/image/render-conflict.ts'; -import type { ImageRenderer } from '../../../services/image/render-service.ts'; +import type { ImageDelivery } from '../../../services/image/image-delivery.ts'; +import { queuePicture } from '../../../services/image/queue-picture.ts'; +import { conflictImageJob } from '../../../services/image/render-conflict.ts'; import { ConflictService } from '../../../services/invite/conflict-service.ts'; -import { sendAgendaImage } from '../../../utils/agenda-image.ts'; import { cmdLogger } from '../../../utils/logger.ts'; import type { ParseMode } from '../../../utils/telegram.ts'; import type { BotCallbackContext } from '../../types.ts'; @@ -50,72 +50,85 @@ export async function answerProposalClosed(ctx: BotCallbackContext, lang: Lang): } interface ConflictImageRequest { + imageDelivery: ImageDelivery; invitation: Invitation; + /** The callback query of this answer: a repeated answer is a new request, a redelivered one is the same. */ + answerId: string; respondent: User; inviter: User; window: { startAt: string; endAt: string }; deps: { userRepo: UserRepository; - sendPhoto: (chatId: number, photo: File) => Promise; - sendDocument?: (chatId: number, document: File, caption: string) => Promise; + sendMessage: (chatId: number, text: string, options: { parse_mode: ParseMode }) => Promise; + sendPhoto: (chatId: number, photo: File) => Promise<{ message_id: number }>; + sendDocument?: (chatId: number, document: File, caption: string) => Promise<{ message_id: number }>; }; eventRepo?: EventRepository; - renderService: ImageRenderer; } -/** Fire-and-forget: renders the organizer/invitee conflict picture for an accepted invite and DMs it to the inviter. */ -function sendConflictImage({ +/** Queues the organizer/invitee conflict picture for an accepted invite; it is DMed to the inviter when ready. */ +async function queueConflictImage({ + imageDelivery, invitation, + answerId, respondent, inviter, window, deps, eventRepo, - renderService, -}: ConflictImageRequest): void { +}: ConflictImageRequest): Promise { const twoHoursMs = 2 * 60 * 60 * 1000; const windowStart = new Date(new Date(window.startAt).getTime() - twoHoursMs).toISOString(); const windowEnd = new Date(new Date(window.endAt).getTime() + twoHoursMs).toISOString(); - const organizerRawEvents = eventRepo - ? eventRepo.findVisibleOverlapping(invitation.inviter_id, windowStart, windowEnd) - : []; - const inviteeConflicts = eventRepo - ? new ConflictService(eventRepo, deps.userRepo).checkConflicts( - invitation.inviter_id, - [respondent.telegram_id], - window.startAt, - window.endAt, - inviter.timezone ?? 'UTC', - ) - : []; const organizerLabel = inviter.first_name ?? inviter.username ?? `#${invitation.inviter_id}`; const locale = inviter.language === 'ru' ? 'ru' : 'en'; - renderConflictImage( - renderService, - invitation.inviter_id, - organizerLabel, - organizerRawEvents.map((e) => ({ - startAt: e.start_at, - endAt: e.end_at ?? new Date(new Date(e.start_at).getTime() + 30 * 60 * 1000).toISOString(), - title: e.title ?? '', - })), - inviteeConflicts, - window.startAt, - window.endAt, - locale, - ) - .then((buffer) => { - const photo = new File([buffer], 'conflict.png', { type: 'image/png' }); - return sendAgendaImage(photo, { - sendPhoto: (file) => deps.sendPhoto(invitation.inviter_id, file), - sendDocument: deps.sendDocument - ? (file, options) => deps.sendDocument!(invitation.inviter_id, file, options.caption) - : undefined, - }); - }) - .catch((err) => { - cmdLogger.error({ err }, 'Failed to render/send conflict image'); - }); + const chatId = invitation.inviter_id; + + // Read the schedules when the render starts, and again if the inviter's calendar changes while it renders. + const buildJob = () => { + const organizerRawEvents = eventRepo ? eventRepo.findVisibleOverlapping(chatId, windowStart, windowEnd) : []; + const inviteeConflicts = eventRepo + ? new ConflictService(eventRepo, deps.userRepo).checkConflicts( + chatId, + [respondent.telegram_id], + window.startAt, + window.endAt, + inviter.timezone ?? 'UTC', + ) + : []; + return conflictImageJob( + chatId, + organizerLabel, + organizerRawEvents.map((e) => ({ + startAt: e.start_at, + endAt: e.end_at ?? new Date(new Date(e.start_at).getTime() + 30 * 60 * 1000).toISOString(), + title: e.title ?? '', + })), + inviteeConflicts, + window.startAt, + window.endAt, + locale, + ); + }; + + await queuePicture(imageDelivery, { + chatId, + language: locale, + kind: 'conflict', + label: t(locale).invitation_conflict_picture_label, + busyNote: t(locale).invitation_conflict_picture_skipped, + failureNote: t(locale).invitation_conflict_picture_failed, + slot: `conflict:${invitation.id}`, + requestId: `invitation:${invitation.id}:${answerId}`, + dataOwner: chatId, + filename: 'conflict.png', + buildJob, + sendPhoto: async (file) => (await deps.sendPhoto(chatId, file)).message_id, + sendDocument: deps.sendDocument + ? async (file, options) => (await deps.sendDocument!(chatId, file, options.caption)).message_id + : undefined, + notify: (text) => deps.sendMessage(chatId, text, { parse_mode: 'HTML' }), + }); } export async function notifyInviter( @@ -125,11 +138,12 @@ export async function notifyInviter( deps: { userRepo: UserRepository; sendMessage: (chatId: number, text: string, options: { parse_mode: ParseMode }) => Promise; - sendPhoto?: (chatId: number, photo: File) => Promise; - sendDocument?: (chatId: number, document: File, caption: string) => Promise; + sendPhoto?: (chatId: number, photo: File) => Promise<{ message_id: number }>; + sendDocument?: (chatId: number, document: File, caption: string) => Promise<{ message_id: number }>; }, - eventRepo?: EventRepository, - renderService?: ImageRenderer, + eventRepo: EventRepository | undefined, + imageDelivery: ImageDelivery | undefined, + answerId: string, ): Promise { const inviter = deps.userRepo.findByTelegramId(invitation.inviter_id); if (!inviter) return; @@ -149,15 +163,16 @@ export async function notifyInviter( : msgs.invitation_response_maybe(respondentName, eventTitle); await deps.sendMessage(invitation.inviter_id, text, { parse_mode: 'HTML' }); - if (action === 'accept' && renderService && deps.sendPhoto && event?.start_at && event.end_at) { - sendConflictImage({ + if (action === 'accept' && imageDelivery && deps.sendPhoto && event?.start_at && event.end_at) { + await queueConflictImage({ + imageDelivery, + answerId, invitation, respondent, inviter, window: { startAt: event.start_at, endAt: event.end_at }, deps: { ...deps, sendPhoto: deps.sendPhoto }, eventRepo, - renderService, }); } } diff --git a/src/bot/handlers/callback/types.ts b/src/bot/handlers/callback/types.ts index c5c268c32..b8a0d09d8 100644 --- a/src/bot/handlers/callback/types.ts +++ b/src/bot/handlers/callback/types.ts @@ -22,7 +22,7 @@ import type { CreateEventData, UpdateEventData, User } from '../../../database/t import type { EventService } from '../../../services/event/event-service.ts'; import type { GoogleOAuthService } from '../../../services/google/oauth.ts'; import type { HolidayService } from '../../../services/holiday/holiday-service.ts'; -import type { ImageRenderer } from '../../../services/image/render-service.ts'; +import type { ImageDelivery } from '../../../services/image/image-delivery.ts'; import type { AdminEditSession } from '../../../services/intent/admin-edit-session.ts'; import type { NotificationPreferencesService } from '../../../services/notification/preferences.ts'; import type { ScenePauseService } from '../../../services/scene-pause.ts'; @@ -46,7 +46,8 @@ export interface CallbackHandlerOpts { calendarRepo?: GoogleCalendarRepository; disconnectDeps?: DisconnectDeps; onCalendarsDone?: (userId: number) => Promise; - renderService?: ImageRenderer; + /** Buttons and invitation notices queue their pictures here; no handler waits for a render. */ + imageDelivery?: ImageDelivery; invitationService?: InvitationService; eventRepo?: EventRepository; chatHistoryRepo?: ChatHistoryRepository; @@ -66,8 +67,9 @@ export interface CallbackHandlerOpts { options: { parse_mode: ParseMode; reply_markup?: InlineKeyboard }, ) => Promise; editMessage?: (chatId: number, messageId: number, text: string, markup?: InlineKeyboard) => Promise; - sendPhoto?: (chatId: number, photo: File) => Promise; - sendDocument?: (chatId: number, document: File, caption: string) => Promise; + /** Return the Telegram message id: a durable picture delivery records it. */ + sendPhoto?: (chatId: number, photo: File) => Promise<{ message_id: number }>; + sendDocument?: (chatId: number, document: File, caption: string) => Promise<{ message_id: number }>; }; onboardingScene?: AnyScene; editProposalDeps?: { diff --git a/src/bot/handlers/message/agent-context.ts b/src/bot/handlers/message/agent-context.ts index b30cc38d1..388b91ad8 100644 --- a/src/bot/handlers/message/agent-context.ts +++ b/src/bot/handlers/message/agent-context.ts @@ -170,7 +170,7 @@ export function buildAgentContextFactory(deps: MessageHandlerDeps) { eventReminderRepo: deps.eventReminderRepo, contactRepo: deps.contactRepo, participantRepo: deps.participantRepo, - renderService: deps.renderService, + imageDelivery: deps.imageDelivery, deepLinkService: deps.deepLinkService, botUsername: deps.botUsername, resolveUsername: deps.resolveUsername, diff --git a/src/bot/handlers/message/types.ts b/src/bot/handlers/message/types.ts index e83190000..d673714b8 100644 --- a/src/bot/handlers/message/types.ts +++ b/src/bot/handlers/message/types.ts @@ -32,7 +32,7 @@ import type { EventService } from '../../../services/event/event-service.ts'; import type { GroupSessionManager } from '../../../services/group/group-session.ts'; import type { GroupMemberService } from '../../../services/group/member-service.ts'; import type { HolidayService } from '../../../services/holiday/holiday-service.ts'; -import type { RenderService } from '../../../services/image/render-service.ts'; +import type { ImageDelivery } from '../../../services/image/image-delivery.ts'; import type { AdminEditSession } from '../../../services/intent/admin-edit-session.ts'; import type { EventMentionStore } from '../../../services/intent/event-mention-store.ts'; import type { IntentExecutor } from '../../../services/intent/intent-executor.ts'; @@ -75,7 +75,7 @@ export interface MessageHandlerDeps { sharingSettingsRepo?: SharingSettingsRepository; sharedEventRepo?: SharedEventRepository; privacyService?: PrivacyService; - renderService?: RenderService; + imageDelivery?: ImageDelivery; notificationPrefs?: { getPrefs(userId: number): NotificationPreferencesRow; update(userId: number, patch: NotificationPreferencesUpdate): void; diff --git a/src/bot/index.ts b/src/bot/index.ts index 25d0d1736..898e4accf 100644 --- a/src/bot/index.ts +++ b/src/bot/index.ts @@ -23,7 +23,7 @@ import type { GoogleOAuthService } from '../services/google/oauth.ts'; import { GroupSessionManager } from '../services/group/group-session.ts'; import { GroupMemberService } from '../services/group/member-service.ts'; import { HolidayService } from '../services/holiday/holiday-service.ts'; -import type { RenderService } from '../services/image/render-service.ts'; +import type { ImageDelivery } from '../services/image/image-delivery.ts'; import { reportIfDatabaseBusy } from '../services/incident/database-busy.ts'; import type { AdminEditSession } from '../services/intent/admin-edit-session.ts'; import { databaseDirectory } from '../services/intent/db-backup.ts'; @@ -141,7 +141,8 @@ export interface GoogleBotDeps { export interface CreateBotOpts { googleDeps?: GoogleBotDeps; - renderService?: RenderService; + /** Calendar pictures (AI tools, commands, buttons, invitation notices) are queued here; their renders run in the background. */ + imageDelivery?: ImageDelivery; /** Logs and queues a call; rejects when the queue write fails (the logged call is then marked failed). */ callQueue?: { enqueue(data: CallRequest): Promise }; transcriptionService?: TranscriptionService; @@ -167,7 +168,7 @@ export interface CreateBotOpts { export function createBot(token: string, db: DatabaseService, aiConfig: AgentConfig, opts: CreateBotOpts) { const { googleDeps, - renderService, + imageDelivery, callQueue, transcriptionService, serviceTier, @@ -341,7 +342,7 @@ export function createBot(token: string, db: DatabaseService, aiConfig: AgentCon sharingSettingsRepo: db.sharingSettings, sharedEventRepo: db.sharedEvents, privacyService, - renderService, + imageDelivery, callSettingsRepo: db.callSettings, callQueue: buildCallQueueAdapter(callQueue, db), notificationPrefs: { @@ -536,7 +537,7 @@ export function createBot(token: string, db: DatabaseService, aiConfig: AgentCon ctx, eventService, holidayService, - renderService, + imageDelivery, db.groupChats, googleDeps?.calendarRepo, weatherService, @@ -547,16 +548,16 @@ export function createBot(token: string, db: DatabaseService, aiConfig: AgentCon ctx, eventService, holidayService, - renderService, + imageDelivery, db.groupChats, googleDeps?.calendarRepo, weatherService, ), ) .command('week', (ctx) => - handleWeek(ctx, eventService, holidayService, renderService, db.groupChats, weatherService), + handleWeek(ctx, eventService, holidayService, imageDelivery, db.groupChats, weatherService), ) - .command('month', (ctx) => handleMonth(ctx, eventService, undefined, renderService, db.groupChats)) + .command('month', (ctx) => handleMonth(ctx, eventService, undefined, imageDelivery, db.groupChats)) .command('add', (ctx) => handleAdd(ctx, scenesSetup.scenes.addEventScene, db.groupChats)) .command('edit', (ctx) => handleEdit(ctx, eventService, db.groupChats)) .command('delete', (ctx) => handleDelete(ctx, eventService, db.groupChats)) @@ -643,7 +644,7 @@ export function createBot(token: string, db: DatabaseService, aiConfig: AgentCon disconnectDeps: googleDeps?.disconnectDeps, onCalendarsDone: googleDeps?.onCalendarsDone, triggerSync: googleDeps?.triggerSync, - renderService, + imageDelivery, invitationService, eventRepo: db.events, chatHistoryRepo: db.chatHistory, @@ -681,12 +682,9 @@ export function createBot(token: string, db: DatabaseService, aiConfig: AgentCon } as Parameters[0]) .catch(() => {}); }, - sendDocument: async (chatId: number, document: File, caption: string) => { - await bot.api.sendDocument({ chat_id: chatId, document, caption }); - }, - sendPhoto: async (chatId: number, photo: File) => { - await bot.api.sendPhoto({ chat_id: chatId, photo }); - }, + sendDocument: (chatId: number, document: File, caption: string) => + bot.api.sendDocument({ chat_id: chatId, document, caption }), + sendPhoto: (chatId: number, photo: File) => bot.api.sendPhoto({ chat_id: chatId, photo }), }, onboardingScene: scenesSetup.scenes.onboardingScene, // Call settings exist only while the bot can place calls; otherwise /settings hides them. @@ -1026,7 +1024,6 @@ export function createBot(token: string, db: DatabaseService, aiConfig: AgentCon inlineService, groupSessions, db, - renderService, agentContextBuilder: buildAgentContextFactory(msgDeps), agent, intentMatcher, diff --git a/src/config/constants/messages/en/ai-tools.ts b/src/config/constants/messages/en/ai-tools.ts index 11c0e1627..6b2602199 100644 --- a/src/config/constants/messages/en/ai-tools.ts +++ b/src/config/constants/messages/en/ai-tools.ts @@ -78,12 +78,19 @@ export const enAiTools = { deleteKeptPast: (count: number) => `Past events kept: ${count}`, deleteFailed: (titles: string) => `⚠️ Not deleted: ${titles}`, deleteCancelled: '❌ Cancelled, nothing was deleted.', - dayImageSent: (date: string) => `Day calendar image for ${date} has been sent to the chat.`, - weekImageSent: (week: string) => `Calendar for ${week} — the picture is in the chat.`, - monthImageSent: (month: string) => `Monthly calendar image for ${month} has been sent to the chat.`, + dayImageQueued: (date: string) => + `Day calendar image for ${date} is being prepared and will arrive in the chat as a separate message.`, + weekImageQueued: (week: string) => + `Weekly calendar image for ${week} is being prepared and will arrive in the chat as a separate message.`, + monthImageQueued: (month: string) => + `Monthly calendar image for ${month} is being prepared and will arrive in the chat as a separate message.`, + imageAlreadyQueued: 'This picture is already being prepared and will arrive in the chat as a separate message.', + imageAlreadyHandled: + 'This picture was already handled for this message: it was sent, or you were told it did not work out. Ask for it again in a new message.', + imageBusy: 'Too many pictures are already being prepared, so this one was not queued. Try again a little later.', dayImageFailed: (date: string) => `Failed to render or send the day image for ${date}.`, - dayImageSentInstead: (date: string, requested: string) => - `Day calendar image for ${date} has been sent to the chat instead of the past day ${requested}: ${date} is the nearest upcoming day with the changes just made.`, + dayImageQueuedInstead: (date: string, requested: string) => + `Day calendar image for ${date} is being prepared instead of the past day ${requested} (${date} is the nearest upcoming day with the changes just made) and will arrive in the chat as a separate message.`, weekImageFailed: (weekStart: string) => `Failed to render or send the weekly image starting ${weekStart}.`, monthImageFailed: (month: string) => `Failed to render or send the monthly image for ${month}.`, callQueued: 'Call queued. You will receive a voice call shortly.', @@ -98,9 +105,34 @@ export const enAiTools = { userPickerSent: 'User picker sent. Waiting for user to select participants.', sceneResumed: 'Continue the wizard from where you left off.', sceneCancelled: 'Wizard cancelled.', - tableSent: (title: string) => `Table "${title}" has been sent to the chat.`, + tableQueued: (title: string) => + `Table "${title}" is being prepared as a picture and will arrive in the chat as a separate message.`, tableFailed: (title: string) => `Failed to render or send the table "${title}".`, - tableRenderingVoice: 'Check the chat — the table is there.', + tableRenderingVoice: 'The table picture will arrive in the chat in a moment.', + imageNoteFailed: (label: string) => + `I could not prepare the picture "${label}". Ask me again if you still want it.`, + imageNoteTimedOut: (label: string) => + `The picture "${label}" did not get ready in time: the server is busy right now. Ask me again a little later if you still want it.`, + imageNoteInterrupted: (label: string) => + `The bot was restarting, so the picture "${label}" did not arrive. Ask me again if you still want it.`, + // The picture came out in a form Telegram cannot take (too tall or too big): asking again for the same thing fails again. + imageNoteUndeliverable: (label: string) => + `The picture "${label}" is too large to send. Ask me for a shorter period or a smaller table.`, + // A picture nobody can request again (the schedule comparison after an accepted invitation): the same + // notes without the offer to ask again. + imageNoteFailedFinal: (label: string) => `I could not prepare the picture "${label}".`, + imageNoteTimedOutFinal: (label: string) => + `The picture "${label}" did not get ready in time: the server is busy right now.`, + imageNoteInterruptedFinal: (label: string) => `The bot was restarting, so the picture "${label}" did not arrive.`, + imageNoteUndeliverableFinal: (label: string) => `The picture "${label}" is too large to send.`, + // The upload ended in a way that does not say whether Telegram stored it, so neither "it came" nor "it did not come" is true. + imageNoteUncertain: (label: string) => + `The picture "${label}" may not have arrived: the upload was cut off. If you do not see it, ask me again.`, + imageNoteUncertainFinal: (label: string) => + `The picture "${label}" may not have arrived: the upload was cut off.`, + // Telegram refuses pictures in this chat for good (blocked or removed bot, a group without photo rights): asking again cannot help. + imageNoteRefused: (label: string) => + `The picture "${label}" could not be sent: Telegram does not let me send pictures in this chat.`, telegramConnectedStatus: 'Telegram account connected', telegramNotConnectedStatus: 'Telegram account not connected. Connect via /connect_telegram', }, diff --git a/src/config/constants/messages/en/invitations.ts b/src/config/constants/messages/en/invitations.ts index 1123dec82..eab7e1665 100644 --- a/src/config/constants/messages/en/invitations.ts +++ b/src/config/constants/messages/en/invitations.ts @@ -40,6 +40,13 @@ export const enInvitations = { `📨 ${name} declined your invitation to "${title}" ❌`, invitation_response_maybe: (name: string, title: string) => `📨 ${name} responded "maybe" to your invitation to "${title}" 🤔`, + /** What the inviter's picture note calls the schedule-comparison picture sent after an acceptance. */ + invitation_conflict_picture_label: 'schedule comparison', + /** The inviter's note when the schedule-comparison picture was skipped because the bot has too many pictures pending. */ + invitation_conflict_picture_skipped: + 'The schedule comparison picture was skipped: too many pictures are already being prepared.', + /** The inviter's note when the schedule-comparison job could not even be built; there is no text version of it. */ + invitation_conflict_picture_failed: 'The schedule comparison picture could not be prepared.', // SAFETY: every name argument must already be HTML-escaped. invitationRoster: { header: '👥 Participants:', diff --git a/src/config/constants/messages/ru/ai-tools.ts b/src/config/constants/messages/ru/ai-tools.ts index beb78fc14..8712cfb8c 100644 --- a/src/config/constants/messages/ru/ai-tools.ts +++ b/src/config/constants/messages/ru/ai-tools.ts @@ -83,12 +83,18 @@ export const ruAiTools = { deleteKeptPast: (count: number) => `Прошедшие не тронуты: ${count}`, deleteFailed: (titles: string) => `⚠️ Не удалось удалить: ${titles}`, deleteCancelled: '❌ Отменено, ничего не удалено.', - dayImageSent: (date: string) => `Картинка календаря за ${date} отправлена в чат.`, - weekImageSent: (week: string) => `Календарь на ${week} — картинка в чате.`, - monthImageSent: (month: string) => `Картинка месячного календаря за ${month} отправлена в чат.`, + dayImageQueued: (date: string) => `Картинка календаря за ${date} готовится и придёт в чат отдельным сообщением.`, + weekImageQueued: (week: string) => `Недельная картинка за ${week} готовится и придёт в чат отдельным сообщением.`, + monthImageQueued: (month: string) => + `Месячная картинка за ${month} готовится и придёт в чат отдельным сообщением.`, + imageAlreadyQueued: 'Эта картинка уже готовится и придёт в чат отдельным сообщением.', + imageAlreadyHandled: + 'Эта картинка по этому сообщению уже обработана: её прислали или сообщили, что она не получилась. Попроси её ещё раз новым сообщением.', + imageBusy: + 'Сейчас уже готовится слишком много картинок, поэтому эта не поставлена в очередь. Попробуй чуть позже.', dayImageFailed: (date: string) => `Не удалось отрендерить или отправить картинку за ${date}.`, - dayImageSentInstead: (date: string, requested: string) => - `Картинка календаря за ${date} отправлена в чат вместо прошедшего дня ${requested}: ${date} — ближайший предстоящий день с только что внесёнными изменениями.`, + dayImageQueuedInstead: (date: string, requested: string) => + `Картинка календаря за ${date} готовится вместо прошедшего дня ${requested} (${date} — ближайший предстоящий день с только что внесёнными изменениями) и придёт в чат отдельным сообщением.`, weekImageFailed: (weekStart: string) => `Не удалось отрендерить или отправить недельную картинку с ${weekStart}.`, monthImageFailed: (month: string) => `Не удалось отрендерить или отправить месячную картинку за ${month}.`, callQueued: 'Звонок поставлен в очередь. Ты получишь голосовой звонок в ближайшее время.', @@ -103,9 +109,32 @@ export const ruAiTools = { userPickerSent: 'Форма выбора участников отправлена. Ожидаю ответа.', sceneResumed: 'Продолжай заполнение с того места, где остановился.', sceneCancelled: 'Заполнение отменено.', - tableSent: (title: string) => `Таблица «${title}» отправлена в чат.`, + tableQueued: (title: string) => `Таблица «${title}» готовится как картинка и придёт в чат отдельным сообщением.`, tableFailed: (title: string) => `Не удалось отрендерить или отправить таблицу «${title}».`, - tableRenderingVoice: 'Загляни в чат — там таблица.', + tableRenderingVoice: 'Картинка с таблицей скоро придёт в чат.', + imageNoteFailed: (label: string) => + `Не получилось подготовить картинку «${label}». Если она всё ещё нужна, попроси ещё раз.`, + imageNoteTimedOut: (label: string) => + `Картинка «${label}» не успела подготовиться: сервер сейчас занят. Если она всё ещё нужна, попроси ещё раз чуть позже.`, + imageNoteInterrupted: (label: string) => + `Бот перезапускался, поэтому картинка «${label}» не дошла. Если она всё ещё нужна, попроси ещё раз.`, + // Картинка получилась в виде, который Telegram не принимает (слишком высокая или тяжёлая): тот же запрос снова не пройдёт. + imageNoteUndeliverable: (label: string) => + `Картинка «${label}» слишком большая, отправить её не получилось. Попроси более короткий период или меньшую таблицу.`, + // Картинка, которую нельзя запросить снова (сравнение расписаний после принятого приглашения): те же + // сообщения без предложения попросить ещё раз. + imageNoteFailedFinal: (label: string) => `Не получилось подготовить картинку «${label}».`, + imageNoteTimedOutFinal: (label: string) => `Картинка «${label}» не успела подготовиться: сервер сейчас занят.`, + imageNoteInterruptedFinal: (label: string) => `Бот перезапускался, поэтому картинка «${label}» не дошла.`, + imageNoteUndeliverableFinal: (label: string) => + `Картинка «${label}» слишком большая, отправить её не получилось.`, + // Загрузка оборвалась так, что неизвестно, сохранил ли её Telegram: нельзя сказать ни «пришла», ни «не пришла». + imageNoteUncertain: (label: string) => + `Картинка «${label}» могла не дойти: загрузка оборвалась. Если её нет в чате, попроси ещё раз.`, + imageNoteUncertainFinal: (label: string) => `Картинка «${label}» могла не дойти: загрузка оборвалась.`, + // Telegram навсегда отказывает в картинках в этом чате (бот заблокирован или удалён, в группе нет права на фото): повторный запрос не поможет. + imageNoteRefused: (label: string) => + `Картинку «${label}» не получилось отправить: Telegram не разрешает мне присылать картинки в этот чат.`, telegramConnectedStatus: 'Telegram-аккаунт подключён', telegramNotConnectedStatus: 'Telegram-аккаунт не подключён. Подключить: /connect_telegram', }, diff --git a/src/config/constants/messages/ru/invitations.ts b/src/config/constants/messages/ru/invitations.ts index 4ac7d6a3d..17d4aed98 100644 --- a/src/config/constants/messages/ru/invitations.ts +++ b/src/config/constants/messages/ru/invitations.ts @@ -42,6 +42,13 @@ export const ruInvitations = { `📨 ${name} отклонил(а) приглашение на "${title}" ❌`, invitation_response_maybe: (name: string, title: string) => `📨 ${name} ответил(а) "возможно" на приглашение "${title}" 🤔`, + /** What the inviter's picture note calls the schedule-comparison picture sent after an acceptance. */ + invitation_conflict_picture_label: 'сравнение расписаний', + /** The inviter's note when the schedule-comparison picture was skipped because the bot has too many pictures pending. */ + invitation_conflict_picture_skipped: + 'Картинку со сравнением расписаний пропустили: сейчас уже готовится слишком много картинок.', + /** The inviter's note when the schedule-comparison job could not even be built; there is no text version of it. */ + invitation_conflict_picture_failed: 'Не получилось подготовить картинку со сравнением расписаний.', // SAFETY: see en.invitationRoster — every name argument must already be HTML-escaped. invitationRoster: { header: '👥 Участники:', diff --git a/src/database/index.ts b/src/database/index.ts index 76c8e1e2f..3fb219af9 100644 --- a/src/database/index.ts +++ b/src/database/index.ts @@ -25,6 +25,7 @@ import { GroupChatRepository } from './repositories/group-chat.repository.ts'; import { GroupMemberRepository } from './repositories/group-member.repository.ts'; import { GroupSessionRepository } from './repositories/group-session.repository.ts'; import { HolidayRepository } from './repositories/holiday.repository.ts'; +import { ImageDeliveryRepository } from './repositories/image-delivery.repository.ts'; import { InvitationRepository } from './repositories/invitation.repository.ts'; import { NotificationLogRepository } from './repositories/notification-log.repository.ts'; import { NotificationPreferencesRepository } from './repositories/notification-preferences.repository.ts'; @@ -70,6 +71,7 @@ export class DatabaseService { readonly actionLog: ActionLogRepository; readonly aiUsage: AiUsageRepository; readonly alerts: AlertRepository; + readonly imageDeliveries: ImageDeliveryRepository; readonly featureUsage: FeatureUsageRepository; readonly participantGoogleSync: ParticipantGoogleSyncRepository; readonly telegramSessions: TelegramSessionRepository; @@ -118,6 +120,7 @@ export class DatabaseService { this.actionLog = new ActionLogRepository(this.db); this.aiUsage = new AiUsageRepository(this.db); this.alerts = new AlertRepository(this.db); + this.imageDeliveries = new ImageDeliveryRepository(this.db); this.featureUsage = new FeatureUsageRepository(this.db); this.participantGoogleSync = new ParticipantGoogleSyncRepository(this.db); this.telegramSessions = new TelegramSessionRepository(this.db); diff --git a/src/database/migrations.ts b/src/database/migrations.ts index b857a2922..97846e53a 100644 --- a/src/database/migrations.ts +++ b/src/database/migrations.ts @@ -1168,4 +1168,32 @@ export const migrations: Migration[] = [ db.exec('ALTER TABLE chat_history ADD COLUMN turn_order REAL'); }, }, + { + name: '070_image_deliveries', + up(db) { + // Durable bookkeeping of asynchronous AI pictures: how far each got, so the next process can + // finish, expire or mark it unknown after a crash. See docs/reference/migrations/070_image_deliveries.md. + db.exec(` + CREATE TABLE image_deliveries ( + id TEXT PRIMARY KEY, + chat_id INTEGER NOT NULL, + language TEXT NOT NULL, + kind TEXT NOT NULL, + label TEXT NOT NULL, + job_type TEXT NOT NULL, + slot TEXT NOT NULL, + payload_hash TEXT NOT NULL, + rebuilds INTEGER NOT NULL DEFAULT 0, + job_json TEXT, + state TEXT NOT NULL CHECK (state IN ('queued','rendered','sending','delivered','failed','expired','unknown','superseded')), + message_id INTEGER, + note_sent INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `); + db.exec('CREATE INDEX idx_image_deliveries_state ON image_deliveries(state)'); + }, + }, ]; diff --git a/src/database/repositories/image-delivery.repository.ts b/src/database/repositories/image-delivery.repository.ts new file mode 100644 index 000000000..d3dc70a95 --- /dev/null +++ b/src/database/repositories/image-delivery.repository.ts @@ -0,0 +1,200 @@ +// Durable bookkeeping of asynchronous AI pictures (migration 070). ImageDelivery writes the state +// BEFORE it acts (a send is recorded `sending` before the Telegram call), so a process killed at any +// point leaves a row that says how far the picture got; reconciliation at the next start decides from it. +import type { Database } from 'bun:sqlite'; + +export type ImageDeliveryState = + | 'queued' + | 'rendered' + | 'sending' + | 'delivered' + | 'failed' + | 'expired' + | 'unknown' + | 'superseded'; + +/** The states `finish` can end a delivery in. */ +export type ImageDeliveryEndState = Extract< + ImageDeliveryState, + 'delivered' | 'failed' | 'expired' | 'unknown' | 'superseded' +>; + +export interface ImageDeliveryRow { + id: string; + chat_id: number; + language: string; + kind: string; + label: string; + job_type: string; + slot: string; + payload_hash: string; + rebuilds: number; + job_json: string | null; + state: ImageDeliveryState; + message_id: number | null; + note_sent: number; + created_at: number; + expires_at: number; + updated_at: number; +} + +export interface NewImageDelivery { + id: string; + chatId: number; + language: string; + kind: string; + label: string; + jobType: string; + /** What the picture shows; a later request for the same slot is deduplicated or supersedes this one, also after a restart. */ + slot: string; + /** Hash of the render data this row's `jobJson` was built from. */ + payloadHash: string; + jobJson: string; + createdAt: number; + expiresAt: number; +} + +const COLUMNS = + 'id, chat_id, language, kind, label, job_type, slot, payload_hash, rebuilds, job_json, state, message_id, note_sent, created_at, expires_at, updated_at'; +const TERMINAL = "('delivered','failed','expired','unknown','superseded')"; +/** Terminal rows are bookkeeping only; a week old ones are pruned at the next start (the only caller is the start-up reconciliation). */ +export const IMAGE_DELIVERY_RETENTION_MS = 7 * 86_400_000; + +export class ImageDeliveryRepository { + constructor(private readonly db: Database) {} + + /** + * False when a delivery with this id exists in ANY state: the id is deterministic (chat, Telegram + * request, kind, slot, payload), so a replay of the same request is a duplicate. A finished row is not + * replaced: an `unknown` one (the upload may have landed) or a `delivered` one must never allow a + * second send, and its recorded message id must survive. The one exception is `superseded`: that picture + * was never uploaded and its user was never told anything, so the same request again (the data went back to + * what it showed) takes its row's place. Rows are pruned once a week old, at the next start. + */ + insert(row: NewImageDelivery): boolean { + // IMMEDIATE takes the write lock at the start, so a second connection cannot slip in between the read and the insert. + return this.db + .transaction(() => { + const existing = this.db + .query<{ id: string }, [string]>('SELECT id FROM image_deliveries WHERE id = ?') + .get(row.id); + if ( + existing && + !this.db.run("DELETE FROM image_deliveries WHERE id = ? AND state = 'superseded'", [row.id]).changes + ) { + return false; + } + this.db.run( + `INSERT INTO image_deliveries + (id, chat_id, language, kind, label, job_type, slot, payload_hash, job_json, state, message_id, note_sent, created_at, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, 0, ?, ?, ?)`, + [ + row.id, + row.chatId, + row.language, + row.kind, + row.label, + row.jobType, + row.slot, + row.payloadHash, + row.jobJson, + row.createdAt, + row.expiresAt, + row.createdAt, + ], + ); + return true; + }) + .immediate(); + } + + get(id: string): ImageDeliveryRow | null { + return this.db.query(`SELECT ${COLUMNS} FROM image_deliveries WHERE id = ?`).get(id); + } + + /** + * Moves a delivery forward (`queued` -> `rendered` -> `sending`) and says whether it did. Never backward + * and never out of a terminal state, so a redelivered job cannot turn a `sending` row (the upload may + * have landed) back into `rendered`. `advance(id, 'sending')` is the claim on the send: only the caller + * that gets `true` may upload. + */ + advance(id: string, state: 'rendered' | 'sending', at: number): boolean { + const from = state === 'rendered' ? "('queued')" : "('queued','rendered')"; + return ( + this.db.run(`UPDATE image_deliveries SET state = ?, updated_at = ? WHERE id = ? AND state IN ${from}`, [ + state, + at, + id, + ]).changes === 1 + ); + } + + /** + * Records a rebuild BEFORE its render starts: the row's job, payload hash and job-id counter now describe + * the rebuilt picture, so a process that dies meanwhile resumes that job and not the stale one already + * rendered. Only a row whose upload was not claimed can be rebuilt; false when it was not in such a state. + */ + rebuilt(id: string, job: { jobJson: string; payloadHash: string; rebuilds: number }, at: number): boolean { + return ( + this.db.run( + `UPDATE image_deliveries SET job_json = ?, payload_hash = ?, rebuilds = ?, updated_at = ? + WHERE id = ? AND state IN ('queued','rendered')`, + [job.jobJson, job.payloadHash, job.rebuilds, at, id], + ).changes === 1 + ); + } + + /** + * Ends a delivery: the payload is dropped, the message id kept. Terminal rows are never changed again. + * `expired` and `superseded` say "never uploaded", so they only apply before the upload was claimed: a + * `sending` row (the upload may have landed) can still end `delivered`, `unknown` or `failed`. False when + * the row was not in a state that allows the move. + */ + finish(id: string, state: ImageDeliveryEndState, at: number, messageId?: number): boolean { + const from = + state === 'expired' || state === 'superseded' ? "('queued','rendered')" : "('queued','rendered','sending')"; + return ( + this.db.run( + `UPDATE image_deliveries SET state = ?, message_id = ?, job_json = NULL, updated_at = ? + WHERE id = ? AND state IN ${from}`, + [state, messageId ?? null, at, id], + ).changes === 1 + ); + } + + /** + * The claim on the one note of a failed or expired delivery, recorded BEFORE the note is sent: a note is + * at most once. Only the caller that gets `true` sends it. A delivered, uncertain or superseded row never + * yields one: the user must not be told a picture failed that arrived or may have. + */ + markNoteSent(id: string): boolean { + return ( + this.db.run( + "UPDATE image_deliveries SET note_sent = 1 WHERE id = ? AND note_sent = 0 AND state IN ('failed','expired')", + [id], + ).changes === 1 + ); + } + + /** Deliveries a previous process left unfinished. */ + unfinished(): ImageDeliveryRow[] { + return this.db + .query( + `SELECT ${COLUMNS} FROM image_deliveries WHERE state IN ('queued','rendered','sending') ORDER BY created_at`, + ) + .all(); + } + + /** Failed or expired deliveries whose user was never told (the process died between `finish` and the note). */ + unnotified(): ImageDeliveryRow[] { + return this.db + .query( + `SELECT ${COLUMNS} FROM image_deliveries WHERE state IN ('failed','expired') AND note_sent = 0 ORDER BY created_at`, + ) + .all(); + } + + pruneOlderThan(before: number): void { + this.db.run(`DELETE FROM image_deliveries WHERE state IN ${TERMINAL} AND updated_at < ?`, [before]); + } +} diff --git a/src/index.ts b/src/index.ts index fd3d73e52..2e0ec12ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -29,7 +29,7 @@ import { installCrashHandlers } from './bootstrap/crash-handlers.ts'; import { createDomainEventBus } from './bootstrap/domain-events.ts'; import { initEventMentionStore } from './bootstrap/event-mentions.ts'; import { initGoogleSync } from './bootstrap/google-sync.ts'; -import { initImageRender } from './bootstrap/image-render.ts'; +import { initImageRender, recoverImageDeliveries } from './bootstrap/image-render.ts'; import { launchBot } from './bootstrap/launch.ts'; import { initLocationVerification } from './bootstrap/location.ts'; import { initNotificationScheduler } from './bootstrap/notifications.ts'; @@ -39,6 +39,7 @@ import { initSpeechServices } from './bootstrap/speech.ts'; import { initVoiceCalls } from './bootstrap/voice-calls.ts'; import { initWeatherService } from './bootstrap/weather.ts'; import { createBot } from './bot/index.ts'; +import { createTelegramSender } from './services/ai/telegram-sender.ts'; import { createServiceTier } from './services/telegram-session/service-tier.ts'; import { botLogger } from './utils/logger.ts'; import { startWebServer } from './web/server.ts'; @@ -63,7 +64,7 @@ if (serviceTier.enabled) { const botHandle = createBotHandle<{ api: TelegramSendApi }>(); const googleSync = await initGoogleSync(ctx, botHandle); -const renderService = await initImageRender(ctx); +const imageRender = await initImageRender(ctx); // Broadcast queue — enqueuer created early so createBot() can wire it into tool handlers. // The worker starts later, after botRef is patched. @@ -78,6 +79,7 @@ const eventMentionStore = await initEventMentionStore(ctx); const domainEventBus = createDomainEventBus(ctx, { participantPushScheduler: googleSync?.participantPushScheduler, weatherService, + imageDelivery: imageRender?.imageDelivery, }); const location = await initLocationVerification(ctx, { weatherService, @@ -97,7 +99,7 @@ const { bot, agentContextBuilder, agent, intentMatcher, intentExecutor, schedule }, { googleDeps: googleSync?.googleDeps, - renderService, + imageDelivery: imageRender?.imageDelivery, callQueue, transcriptionService: speech.transcriptionService, serviceTier, @@ -128,6 +130,7 @@ botHandle.set(bot); patchBotRef(ctx.botRef, bot.api); // Created after botRef is patched so every send is the real implementation. broadcast.startWorker(ctx, bot.api); +if (imageRender) recoverImageDeliveries(imageRender.imageDelivery, createTelegramSender(bot)); await initScheduledAi(ctx, { domainEventBus, diff --git a/src/services/ai/system-prompt.ts b/src/services/ai/system-prompt.ts index 7a0614fd3..444953f66 100644 --- a/src/services/ai/system-prompt.ts +++ b/src/services/ai/system-prompt.ts @@ -313,7 +313,7 @@ function buildProactiveSection(): string { Be a proactive assistant, not a passive tool executor. After completing any action, scan for what logically comes next and surface it. The examples below are not exhaustive — use judgment. **After any event create, update, or delete:** -- Identify the affected date(s). If all events are on the same day → call \`render_day_image\` for that day. If they span multiple days or fall in a different week → call \`render_week_image\` for the relevant week. Always pair the image with a text summary. +- Affected date(s): all on one day → call \`render_day_image\` for it; several days or a different week → call \`render_week_image\` for that week. Always pair the image with a text summary; the picture arrives as a separate message after your text. - When the image covers a specific day, describe the free windows naturally: morning before the first event, gaps ≥ 30 min between events, evening after the last event. Example: "Свободное утро до 11:00, перерыв с 12:00 до 13:45, и вечер после 14:45." Skip gaps under 30 min — they're not actionable. **When creating events:** @@ -326,7 +326,7 @@ Be a proactive assistant, not a passive tool executor. After completing any acti - Don't comment if the day looks fine — silence is better than noise. **When showing events for a day, week, or month:** -- ALWAYS call \`render_day_image\` (for a day), \`render_week_image\` (for a week), or \`render_month_image\` (for a month) alongside the text. Users expect both. +- ALWAYS call \`render_day_image\` (day), \`render_week_image\` (week) or \`render_month_image\` (month) alongside the text. Users expect both; the picture comes later: never claim the picture is attached. - Check invitation status (get_invitation_status) for events with other people. Report issues: - ⏳ Who hasn't responded yet → "⏳ Лена — ждёт ответа" - ⚠️ Who is missing from invitations → "⚠️ Вова — не приглашён" diff --git a/src/services/ai/tool-handlers/render.ts b/src/services/ai/tool-handlers/render.ts index 8bc767741..1ab95222f 100644 --- a/src/services/ai/tool-handlers/render.ts +++ b/src/services/ai/tool-handlers/render.ts @@ -6,12 +6,14 @@ import { agendaImageErrorMessage, sendAgendaImage } from '../../../utils/agenda- import { autoPin } from '../../../utils/auto-pin.ts'; import { getDayRangeUtc, getWeekRangeUtc, localCalendarDate } from '../../../utils/date.ts'; import { logger } from '../../../utils/logger.ts'; +import type { ImageRenderJob } from '../../../worker/image-render.queue.ts'; import { getTheme } from '../../../worker/templates/themes.ts'; import { enrichAgenda } from '../../event/agenda-enrichment.ts'; import { formatWeekLabel } from '../../image/data-mapper.ts'; -import { renderDayImage } from '../../image/render-day.ts'; -import { renderMonthImage } from '../../image/render-month.ts'; -import { renderWeekImage } from '../../image/render-week.ts'; +import type { ImageDelivery, ImageEnqueueResult, ImageKind } from '../../image/image-delivery.ts'; +import { dayImageJob } from '../../image/render-day.ts'; +import { monthImageJob } from '../../image/render-month.ts'; +import { weekImageJob } from '../../image/render-week.ts'; import type { AgentContext, ToolHandlerMeta, ToolResult } from '../types.ts'; import { checkSecretaryAccess } from './secretary-access.ts'; import { resolveScope } from './shared.ts'; @@ -66,19 +68,19 @@ type Guard = { ok: true; value: T } | { ok: false; result: ToolResult }; /** What every agenda-image tool needs once its shared guards have passed. */ interface ImageRenderTarget { - renderService: NonNullable; + imageDelivery: ImageDelivery; sender: NonNullable; userId: number; scope: Scope; } -/** Guards shared by the day/week/month image tools: renderer + photo transport, secretary read access, group scope. */ +/** Guards shared by the day/week/month image tools: image delivery + photo transport, secretary read access, group scope. */ function resolveImageRenderTarget( ctx: AgentContext, input: { scope?: Scope; owner_id?: number }, ): Guard { - const { renderService, sender } = ctx; - if (!renderService || !sender?.sendPhoto) { + const { imageDelivery, sender } = ctx; + if (!imageDelivery || !sender?.sendPhoto) { return { ok: false, result: { success: false, error: 'Image rendering not available.' } }; } const access = checkSecretaryAccess( @@ -92,7 +94,7 @@ function resolveImageRenderTarget( if (scope === 'group' && ctx.groupChatId === undefined) { return { ok: false, result: { success: false, error: 'Group context required for group scope' } }; } - return { ok: true, value: { renderService, sender, userId: access.effectiveUserId, scope } }; + return { ok: true, value: { imageDelivery, sender, userId: access.effectiveUserId, scope } }; } function imageLanguage(ctx: AgentContext): 'ru' | 'en' { @@ -125,23 +127,96 @@ function agendaForImage( ); } -/** Send the rendered PNG (photo, or document when oversized) and schedule the best-effort pin. */ -async function deliverAgendaPng( - ctx: AgentContext, - target: ImageRenderTarget, - buffer: Uint8Array, - filename: string, -): Promise { - const { sender } = target; - const file = new File([buffer], filename, { type: 'image/png' }); - const sent = await sendAgendaImage(file, { - language: ctx.user.language, - sendPhoto: (photo) => sender.sendPhoto!(ctx.chatId, photo), - sendDocument: sender.sendDocument - ? (document, options) => sender.sendDocument!(ctx.chatId, document, options.caption) - : undefined, +/** What one picture is, as the delivery service needs to know it. */ +interface QueuedImage { + kind: ImageKind; + /** What the user asked for, for their own failure note. */ + label: string; + /** The same slot shows the same thing at different times (kind + target + whose calendar). */ + slot: string; + filename: string; + dataOwner?: number | 'group'; + buildJob(): ImageRenderJob; +} + +type QueueImageResult = ImageEnqueueResult | { status: 'supplement' }; + +/** + * Hand the picture to the background delivery and return at once: it goes to this chat as its own + * message (a photo, or a document when oversized) whenever the render ends, then the best-effort pin + * runs. The failure note and the timeout alert are the delivery service's. + */ +function queueImage(ctx: AgentContext, imageDelivery: ImageDelivery, image: QueuedImage): QueueImageResult { + // A supplement is silent: its sender drops everything, so its picture would reach nobody. Queued anyway it + // would still hold the chat's places, match (and so block) the same picture a real request asks for a + // moment later, or supersede and abort a real one that renders. + if (ctx.supplementMode) return { status: 'supplement' }; + const { sender } = ctx; + if (!sender?.sendPhoto) return { status: 'closed' }; + const sendPhoto = sender.sendPhoto.bind(sender); + return imageDelivery.enqueue({ + chatId: ctx.chatId, + language: imageLanguage(ctx), + kind: image.kind, + label: image.label, + slot: image.slot, + requestId: ctx.incomingMessageId, + dataOwner: image.dataOwner, + buildJob: image.buildJob, + send: async (png) => { + const file = new File([png], image.filename, { type: 'image/png' }); + const sent = await sendAgendaImage(file, { + language: ctx.user.language, + sendPhoto: (photo) => sendPhoto(ctx.chatId, photo), + sendDocument: sender.sendDocument + ? (document, options) => sender.sendDocument!(ctx.chatId, document, options.caption) + : undefined, + }); + schedulePinFireAndForget(ctx, sent.message_id); + return sent.message_id; + }, + notify: (text) => sender.sendMessage(ctx.chatId, text), }); - schedulePinFireAndForget(ctx, sent.message_id); +} + +/** + * The tool result for one enqueue. `output` is plain for the user (the intent workflows reply with it); + * the hint is for the model: the picture is only queued, so it must not claim it was sent or attached. + */ +function queuedResult( + status: QueueImageResult['status'], + tr: { imageAlreadyQueued: string; imageAlreadyHandled: string; imageBusy: string }, + queued: { output: string; tool: string; repeat: string }, +): ToolResult { + const notSent = 'do NOT say the picture was sent or attached, write your text answer now'; + switch (status) { + case 'queued': + return { + success: true, + output: queued.output, + agentHint: `The picture is only QUEUED, NOT sent yet: it arrives later as a separate message (or the user gets a short note if it fails). Do NOT call ${queued.tool} again ${queued.repeat} in this turn, and ${notSent}.`, + }; + case 'duplicate': + return { + success: true, + output: tr.imageAlreadyQueued, + agentHint: `This exact picture is already queued and NOT sent yet. Do NOT call ${queued.tool} again, and ${notSent}.`, + }; + case 'settled': + return { success: false, error: tr.imageAlreadyHandled }; + case 'busy': + return { + success: false, + error: tr.imageBusy, + }; + case 'closed': + return { success: false, error: 'Image rendering not available.' }; + case 'supplement': + return { + success: false, + error: 'A background follow-up sends no pictures: nothing was rendered. Do not call this tool again.', + }; + } } /** Typed agenda-image errors are the user-facing answer; anything else is logged and reported with `fallbackError`. */ @@ -197,6 +272,17 @@ function resolveWeekStartIso(weekStart: string, timezone: string): string | null return startOfWeek(date, { weekStartsOn: 1 }).toISOString().slice(0, 10); } +/** + * The owner key a picture's slot and staleness hang on: the group, or the user whose calendar it shows. A group + * picture is also per viewer (its holidays and language are the requester's), so two members never replace + * each other's picture. + */ +function pictureOwner(ctx: AgentContext, target: ImageRenderTarget): { key: string; dataOwner: number | 'group' } { + return target.scope === 'group' + ? { key: `group:${ctx.groupChatId}:viewer:${ctx.user.telegram_id}`, dataOwner: 'group' } + : { key: `user:${target.userId}`, dataOwner: target.userId }; +} + export async function handleRenderDayImage( ctx: AgentContext, input: { date: string; scope?: Scope; owner_id?: number }, @@ -210,34 +296,37 @@ export async function handleRenderDayImage( if (!day) return { success: false, error: 'Invalid calendar date.' }; const { date, dateObj, redirected } = day; - const occurrences = - scope === 'group' - ? (() => { - const { start, end } = getDayRangeUtc(dateObj, ctx.user.timezone); - return ctx.eventService.getEventsInRangeForGroup(ctx.groupChatId!, start, end); - })() - : ctx.eventService.getEventsForDay(userId, dateObj, ctx.user.timezone); - const holidays = ctx.holidayService?.getHolidaysForDate(userId, date) ?? []; const lang = imageLanguage(ctx); const tr = t(lang).aiTools.meta; + const owner = pictureOwner(ctx, target); + + // Read the agenda when the render starts, and again if the calendar changes while it renders. + const buildJob = (): ImageRenderJob => { + const occurrences = + scope === 'group' + ? (() => { + const { start, end } = getDayRangeUtc(dateObj, ctx.user.timezone); + return ctx.eventService.getEventsInRangeForGroup(ctx.groupChatId!, start, end); + })() + : ctx.eventService.getEventsForDay(userId, dateObj, ctx.user.timezone); + const holidays = ctx.holidayService?.getHolidaysForDate(userId, date) ?? []; + return dayImageJob(agendaForImage(ctx, target, occurrences, lang), date, ctx.user.timezone, lang, userId, holidays); + }; try { - const buffer = await renderDayImage( - target.renderService, - agendaForImage(ctx, target, occurrences, lang), - date, - ctx.user.timezone, - lang, - userId, - holidays, - ); - await deliverAgendaPng(ctx, target, buffer, 'day.png'); - return { - success: true, - output: redirected ? tr.dayImageSentInstead(date, input.date) : tr.dayImageSent(date), - agentHint: - 'The day image has already been delivered to the chat. Do NOT call render_day_image again for this date in this turn.', - }; + const status = queueImage(ctx, target.imageDelivery, { + kind: 'day', + label: date, + slot: `day:${owner.key}:${date}:${lang}`, + filename: 'day.png', + dataOwner: owner.dataOwner, + buildJob, + }).status; + return queuedResult(status, tr, { + output: redirected ? tr.dayImageQueuedInstead(date, input.date) : tr.dayImageQueued(date), + tool: 'render_day_image', + repeat: 'for this date', + }); } catch (err) { return imageRenderFailure(err, tr.dayImageFailed(date), { date, requested: input.date }, 'Day image render failed'); } @@ -248,47 +337,39 @@ export async function handleRenderTable( ctx: AgentContext, input: { title: string; markdown: string; caption?: string }, ): Promise { - if (!ctx.renderService || !ctx.sender?.sendPhoto) { + const { imageDelivery, sender } = ctx; + if (!imageDelivery || !sender?.sendPhoto) { return { success: false, error: 'Image rendering not available.' }; } - const lang = (ctx.user.language ?? 'en') as 'ru' | 'en'; + const lang = imageLanguage(ctx); const tr = t(lang).aiTools.meta; - const sender = ctx.sender; try { - const buffer = await ctx.renderService.renderDirect({ - type: 'md-table', - data: { - title: input.title, - markdown: input.markdown, - caption: input.caption, - theme: getTheme(), - }, - userId: ctx.user.telegram_id, - }); - const file = new File([buffer], 'table.png', { type: 'image/png' }); - const sent = await sendAgendaImage(file, { - language: ctx.user.language, - sendPhoto: (photo) => sender.sendPhoto!(ctx.chatId, photo), - sendDocument: sender.sendDocument - ? (document, options) => sender.sendDocument!(ctx.chatId, document, options.caption) - : undefined, - }); - schedulePinFireAndForget(ctx, sent.message_id); - + const status = queueImage(ctx, imageDelivery, { + kind: 'table', + label: input.title, + slot: `table:${Bun.hash(JSON.stringify([input.title, input.markdown, input.caption ?? null]))}`, + filename: 'table.png', + buildJob: () => ({ + type: 'md-table', + data: { + title: input.title, + markdown: input.markdown, + caption: input.caption, + theme: getTheme(), + }, + userId: ctx.user.telegram_id, + }), + }).status; const voiceNote = ctx.inputMode === 'live_call' ? ` ${tr.tableRenderingVoice}` : ''; - return { - success: true, - output: `${tr.tableSent(input.title)}${voiceNote}`, - agentHint: - 'The table has already been delivered to the chat. Do NOT call render_table again with identical arguments in this turn.', - }; + return queuedResult(status, tr, { + output: `${tr.tableQueued(input.title)}${voiceNote}`, + tool: 'render_table', + repeat: 'with identical arguments', + }); } catch (err) { - const imageError = agendaImageErrorMessage(err); - if (imageError) return { success: false, error: imageError }; - renderLogger.error({ err, title: input.title }, 'Table image render failed'); - return { success: false, error: tr.tableFailed(input.title) }; + return imageRenderFailure(err, tr.tableFailed(input.title), { title: input.title }, 'Table image render failed'); } } handleRenderTable.meta = { skipActionLog: true, delivers: true } satisfies ToolHandlerMeta; @@ -313,25 +394,29 @@ export async function handleRenderWeekImage( localCalendarDate(weekStartIso, ctx.user.timezone), ctx.user.timezone, ); - - const occurrences = rangeOccurrences(ctx, target, startUtc, endUtc); + const owner = pictureOwner(ctx, target); try { - const buffer = await renderWeekImage( - target.renderService, - agendaForImage(ctx, target, occurrences, lang), - weekStartIso, - ctx.user.timezone, - lang, - target.userId, - ); - await deliverAgendaPng(ctx, target, buffer, 'week.png'); - return { - success: true, - output: tr.weekImageSent(formatWeekLabel(weekStartIso, lang)), - agentHint: - 'The weekly image has already been delivered to the chat. Do NOT call render_week_image again with identical arguments in this turn.', - }; + const status = queueImage(ctx, target.imageDelivery, { + kind: 'week', + label: formatWeekLabel(weekStartIso, lang), + slot: `week:${owner.key}:${weekStartIso}:${lang}`, + filename: 'week.png', + dataOwner: owner.dataOwner, + buildJob: () => + weekImageJob( + agendaForImage(ctx, target, rangeOccurrences(ctx, target, startUtc, endUtc), lang), + weekStartIso, + ctx.user.timezone, + lang, + target.userId, + ), + }).status; + return queuedResult(status, tr, { + output: tr.weekImageQueued(formatWeekLabel(weekStartIso, lang)), + tool: 'render_week_image', + repeat: 'with identical arguments', + }); } catch (err) { return imageRenderFailure( err, @@ -359,28 +444,32 @@ export async function handleRenderMonthImage( const startUtc = new Date(Date.UTC(year, month, 1)).toISOString(); const endUtc = new Date(Date.UTC(year, month + 1, 0, 23, 59, 59, 999)).toISOString(); - const occurrences = rangeOccurrences(ctx, target, startUtc, endUtc); - const lang = imageLanguage(ctx); const tr = t(lang).aiTools.meta; + const owner = pictureOwner(ctx, target); try { - const buffer = await renderMonthImage( - target.renderService, - agendaForImage(ctx, target, occurrences, lang), - year, - month, - ctx.user.timezone, - lang, - target.userId, - ); - await deliverAgendaPng(ctx, target, buffer, 'month.png'); - return { - success: true, - output: tr.monthImageSent(input.month), - agentHint: - 'The monthly image has already been delivered to the chat. Do NOT call render_month_image again with identical arguments in this turn.', - }; + const status = queueImage(ctx, target.imageDelivery, { + kind: 'month', + label: input.month, + slot: `month:${owner.key}:${year}-${month}:${lang}`, + filename: 'month.png', + dataOwner: owner.dataOwner, + buildJob: () => + monthImageJob( + agendaForImage(ctx, target, rangeOccurrences(ctx, target, startUtc, endUtc), lang), + year, + month, + ctx.user.timezone, + lang, + target.userId, + ), + }).status; + return queuedResult(status, tr, { + output: tr.monthImageQueued(input.month), + tool: 'render_month_image', + repeat: 'with identical arguments', + }); } catch (err) { return imageRenderFailure( err, diff --git a/src/services/ai/tools.ts b/src/services/ai/tools.ts index ead6012f5..8b3fb6ce3 100644 --- a/src/services/ai/tools.ts +++ b/src/services/ai/tools.ts @@ -580,7 +580,7 @@ const toolDefinitions: ToolDefinition[] = [ { name: 'render_day_image', description: - 'Generate and send a visual calendar image for a specific date. Use when the user explicitly asks to see their schedule as an image, OR when showing a full-day overview with multiple events and a visual layout would be genuinely helpful. Do NOT use for single event operations, confirmations, or quick replies.', + 'Queue a visual calendar image for a date (arrives as a separate message). Use when the user explicitly asks to see their schedule as an image, OR for a full-day overview with many events where a visual layout helps. Do NOT use for single event operations, confirmations, or quick replies.', input_schema: { type: 'object' as const, properties: { @@ -594,7 +594,7 @@ const toolDefinitions: ToolDefinition[] = [ { name: 'render_week_image', description: - 'Generate and send a visual weekly calendar image, for an explicit request or a week overview with many events.', + 'Queue a visual weekly calendar image (arrives as a separate message), for an explicit request or a week overview with many events.', input_schema: { type: 'object' as const, properties: { @@ -607,7 +607,8 @@ const toolDefinitions: ToolDefinition[] = [ }, { name: 'render_month_image', - description: 'Generate and send a visual monthly calendar image, for requests like "show April" or "monthly view".', + description: + 'Queue a visual monthly calendar image (arrives as a separate message), for requests like "show April" or "monthly view".', input_schema: { type: 'object' as const, properties: { @@ -621,9 +622,9 @@ const toolDefinitions: ToolDefinition[] = [ { name: 'render_table', description: - 'Render a Markdown table as an image and send it to the chat. Call it whenever you have tabular data, ' + - 'in parallel with your text reply — and in that reply present the same data as a bullet list, never as raw ' + - 'Markdown table syntax. During a voice call the image still goes to the chat, so mention it out loud.', + 'Queue a Markdown table as an image (arrives as a separate message). Call it whenever you have tabular data, ' + + 'in parallel with your text reply — present the same data there as a bullet list, never as raw ' + + 'Markdown table syntax. In a voice call, say the image goes to the chat.', input_schema: { type: 'object' as const, properties: { diff --git a/src/services/ai/types.ts b/src/services/ai/types.ts index d721c220b..e965dfdb8 100644 --- a/src/services/ai/types.ts +++ b/src/services/ai/types.ts @@ -30,7 +30,7 @@ import type { ConversationLogger } from '../conversation-logger.ts'; import type { EventService } from '../event/event-service.ts'; import type { GroupMemberService } from '../group/member-service.ts'; import type { HolidayService } from '../holiday/holiday-service.ts'; -import type { ImageRenderer } from '../image/render-service.ts'; +import type { ImageDelivery } from '../image/image-delivery.ts'; import type { EventSummary } from '../intent/variable-resolver.ts'; import type { AddressCache } from '../location/address-cache.ts'; import type { EventPlace } from '../location/event-venue.ts'; @@ -200,7 +200,8 @@ export interface AgentContext { sender?: TelegramSender; /** Called after any successful tool call that references an event (by ID or creation). */ onEventMentioned?: (eventId: number) => void; - renderService?: ImageRenderer; + /** Queues pictures for background delivery as their own message; absent when rendering is off (no Redis, no browser). */ + imageDelivery?: ImageDelivery; deepLinkService?: DeepLinkService; botUsername?: string; /** Telegram file_id of the voice message that triggered this interaction. */ @@ -408,7 +409,7 @@ export interface ToolHandlerMeta { skipActionLog?: boolean; /** Tool always results in [SKIP] — no status message or tool label shown. */ silent?: boolean; - /** A successful call delivers the answer itself (e.g. a rendered image); no text reply is owed. */ + /** A successful call delivers the answer itself (a reaction) or has queued it for delivery as a separate message (a picture, whose failure sends its own note); no text reply is owed. */ delivers?: boolean; /** * Tool result must NOT be persisted to chat_history. diff --git a/src/services/image/image-delivery.ts b/src/services/image/image-delivery.ts new file mode 100644 index 000000000..06e3414a1 --- /dev/null +++ b/src/services/image/image-delivery.ts @@ -0,0 +1,926 @@ +// Asynchronous picture delivery for EVERY picture the bot sends: the AI tools (render_day_image, +// render_week_image, render_month_image, render_table) and, through queue-picture.ts, /today, +// /tomorrow, /week, /month, the agenda buttons and the conflict picture of an accepted invitation. +// +// Why: on 2026-11-01 (prod, bb7a9d47) the tool call waited for the render inside the agent turn. On the +// overloaded odroid real renders took 73-80 s, two waits plus model rounds spent the whole ~90 s turn +// budget, the answer validator timed out and the user got an aborted-run notice instead of the correct +// text the model had already written. Now the tool only ENQUEUES: it returns at once ("queued, not +// sent") and this service finishes the job in the background: +// +// enqueue ──► render (BullMQ, bounded) ──► send as its own message ──► log the outcome +// │ timeout → ONE honest note to the user + reportTimeout({kind:'render'}) +// │ failure → ONE honest note to the user + one aggregated alert of its own kind (none when Telegram's refusal is permanent) +// │ upload of unknown end → ONE note that the picture MAY not have arrived + the same alert +// │ shutdown → with the store: no note, the row stays queued and the next start resumes it; +// │ without the store: ONE note (best effort within the shutdown deadline); logged 'interrupted' +// +// A render never blocks or fails the agent turn. Invariants (each has a test): +// - Same chat, same picture twice (same slot and same payload) → one render, one delivery. +// - Same chat, same slot, different payload (the data changed) → the older one is superseded, so the +// user gets only the newest. +// - At most MAX_PENDING_PER_CHAT pending pictures per chat (the conflict pictures of accepted invitations +// count apart, MAX_PENDING_CONFLICT_PER_CHAT) and MAX_PENDING_TOTAL overall: the shared +// host is CPU-bound; beyond that the tool says so honestly and nothing is queued. +// - A calendar write for the picture's data while it renders makes it stale: it is rebuilt from the +// current data once instead of delivering a picture that contradicts the text the user already has. +// - Every pending delivery ends in exactly one logged outcome. With the store, a process killed without a +// graceful shutdown (SIGKILL, crash, power) is reconciled by the next start (resumed, expired with one +// "the bot was restarting" note and no timeout alert, or marked unknown and never re-sent); without the store +// (tests, dev) those pictures are lost silently. +import { TelegramError } from 'gramio'; +import { t } from '../../config/constants.ts'; +import type { + ImageDeliveryEndState, + ImageDeliveryRepository, + ImageDeliveryRow, +} from '../../database/repositories/image-delivery.repository.ts'; +import { IMAGE_DELIVERY_RETENTION_MS } from '../../database/repositories/image-delivery.repository.ts'; +import { agendaImageErrorMessage, agendaImageTooLarge } from '../../utils/agenda-image.ts'; +import { imageLogger } from '../../utils/logger.ts'; +import { isPermanentTelegramError } from '../../worker/broadcast-queue.ts'; +import type { ImageRenderJob } from '../../worker/image-render.queue.ts'; +import { reportIfDatabaseBusy } from '../incident/database-busy.ts'; +import { reportTimeout } from '../incident/timeout-reporter.ts'; +import { type ImageRenderer, RenderAbortedError, RenderTimeoutError } from './render-service.ts'; + +/** + * How long a picture may take from the tool call to the end of its render. Measured production renders + * took 6–15 s healthy and 73–80 s when the host was overloaded (2026-11-01); 150 s is about twice the + * worst of those. The jobs are CPU-bound on a box shared with other bots, so a longer bound would only + * pile more work on it; the user already has the text answer and can ask again. + */ +export const IMAGE_DELIVERY_BOUND_MS = 150_000; +/** The Telegram upload of a finished PNG; a hang here must not hold a chat's slot forever. */ +export const IMAGE_SEND_BOUND_MS = 30_000; +export const MAX_PENDING_PER_CHAT = 2; +/** Schedule comparisons of accepted invitations queued for one chat at a time; they have their own limit (see `enqueue`). */ +export const MAX_PENDING_CONFLICT_PER_CHAT = 4; +/** Schedule comparisons queued on the whole bot at a time: the chats' own pictures always keep the other half of MAX_PENDING_TOTAL. */ +export const MAX_PENDING_CONFLICT_TOTAL = 6; +export const MAX_PENDING_TOTAL = 12; +/** A render that has to be redone after a calendar write only starts with at least this much of the bound left (the slow host needs ~15 s, worst seen 80 s). */ +export const REBUILD_MIN_BUDGET_MS = 45_000; +/** What shutdown spends on telling users their pending picture will not come (the whole deadline is 8 s). */ +export const SHUTDOWN_NOTICE_BUDGET_MS = 1_500; +/** + * A recovered delivery whose bound ran out more than this long ago gets no "it did not arrive" note and no + * timeout alert: the cause was downtime (a rollback, a long outage), not a slow render, and the user stopped + * waiting long ago. Within it (a normal deploy restart) the user is told once. + */ +export const RECOVERY_NOTE_MAX_AGE_MS = 10 * 60_000; +/** An outcome the store refused is written again after this long times the attempt number, RECORD_RETRIES times at most. */ +export const RECORD_RETRY_MS = 1_000; +export const RECORD_RETRIES = 3; + +export type ImageKind = 'day' | 'week' | 'month' | 'table' | 'conflict'; + +export interface ImageDeliveryRequest { + chatId: number; + language: 'ru' | 'en'; + kind: ImageKind; + /** What the user asked for (a date, a week, a month, a table title): shown only in that user's own failure note, never logged. */ + label: string; + /** Pictures of the same slot show the same thing at different times; a newer one replaces an older pending one. */ + slot: string; + /** The Telegram message (or callback query) this picture answers; part of the deterministic delivery id. Defaults to the enqueue time. */ + requestId?: number | string; + /** Whose calendar the picture shows: a user id (that user's writes make it stale) or the group (any write does); absent for pictures of no calendar data (a table). */ + dataOwner?: number | 'group'; + /** The render job from the CURRENT calendar data; called again once when the data changed meanwhile. */ + buildJob(): ImageRenderJob; + /** Delivers the finished PNG to the chat as its own message and returns the Telegram message id. */ + send(png: Buffer): Promise; + /** Sends a short note to the same chat. */ + notify(text: string): Promise; +} + +export type ImageEnqueueResult = + | { status: 'queued' } + /** An identical picture is already pending for this chat. */ + | { status: 'duplicate' } + /** The same request (same message, same picture) already ran to its end: delivered, or the user was told it failed. Nothing was queued. */ + | { status: 'settled' } + /** The chat (or the bot) already has the maximum of pending pictures; nothing was queued. */ + | { status: 'busy' } + /** The bot is shutting down; nothing was queued. */ + | { status: 'closed' }; + +export type ImageDeliveryOutcome = + | 'delivered' + | 'failed' + | 'timed_out' + | 'superseded' + | 'interrupted' + | 'unknown' + /** Another attempt (a process that overlapped this one during a deploy) already claimed the send: nothing was uploaded here. */ + | 'claimed_elsewhere'; + +/** The facts an outcome log line carries beyond the delivery's identity. */ +export interface ImageOutcomeDetails { + phase?: 'send' | 'render' | 'recovery'; + delivery?: 'unknown' | 'not_started'; + messageId?: number; + errorName?: string; + expected?: boolean; + recovered?: boolean; + stale?: boolean; + resumable?: boolean; +} + +export interface RenderFailureReport { + kind: ImageKind; + jobType: ImageRenderJob['type']; + /** The class name of the error; never its message. */ + errorName: string; + elapsedMs: number; +} + +/** What a recovered delivery uses after a restart, when the request's own closures are gone. */ +export interface ImageTransport { + /** Sends the PNG as a photo (document when oversized) and returns the Telegram message id. */ + sendImage(chatId: number, png: Buffer, language: 'ru' | 'en'): Promise; + notify(chatId: number, text: string): Promise; +} + +export interface ImageDeliveryDeps { + renderer: ImageRenderer; + /** Durable record of each delivery; without it (tests, dev) nothing survives a restart. */ + store?: ImageDeliveryRepository; + /** A render that failed for a reason other than time: one aggregated alert of its own kind. */ + reportFailure?: (report: RenderFailureReport) => void; + now?: () => number; + boundMs?: number; + sendBoundMs?: number; + /** Runs `fn` after `delayMs` (default: an unref'd timer). Only used to try again to write an outcome the store refused. */ + schedule?: (fn: () => void, delayMs: number) => void; +} + +interface Entry { + /** The deterministic delivery id; the BullMQ job id derives from it. */ + readonly id: string; + /** How many times the job was rebuilt after a calendar write (each rebuild is its own BullMQ job). */ + rebuilds: number; + readonly request: ImageDeliveryRequest; + readonly startedAt: number; + readonly deadline: number; + /** Hash of the data the picture on its way shows; follows a rebuild, so the same request again is still recognised as this one. */ + payload: string; + readonly abort: AbortController; + job: ImageRenderJob; + /** The calendar-change counter when `job` was built. */ + builtAtSeq: number; + phase: 'rendering' | 'sending'; + /** Why the wait was cut from outside; the run then ends quietly. */ + cancelled?: 'superseded' | 'shutdown'; + settled: boolean; + /** The end of the delivery could not be written to the store: the row never reached failed/expired, so no note can be claimed on it. */ + recordFailed?: boolean; + /** The note went out while the end of the delivery was not recorded: it is claimed once the write finally succeeds, so a restart does not send it again. */ + noteUnclaimed?: boolean; + /** The end of the delivery as the store has to hear it; set while the write is refused, cleared once it succeeds. */ + unrecorded?: { state: ImageDeliveryEndState; at: number; messageId: number | undefined }; + done: Promise; +} + +class SendTimeoutError extends Error { + constructor(limitMs: number) { + super(`Image upload did not finish within ${limitMs} ms`); + this.name = 'SendTimeoutError'; + } +} + +/** The persisted state an outcome ends in: an upload of unknown end is `unknown`, a render cut by the bound `expired`. */ +function storedState(outcome: ImageDeliveryOutcome, details: ImageOutcomeDetails): ImageDeliveryEndState { + if (outcome === 'delivered') return 'delivered'; + if (outcome === 'unknown') return 'unknown'; + if (outcome === 'superseded') return 'superseded'; + if (outcome === 'timed_out') return details.phase === 'send' ? 'unknown' : 'expired'; + if (outcome === 'interrupted') return 'unknown'; + return 'failed'; +} + +/** + * Deterministic: the same chat, request, kind, slot and payload always give the same id, so a repeated + * enqueue is recognised and BullMQ (which ignores a job id it already has) never renders it twice. + */ +function deliveryId(request: ImageDeliveryRequest, payload: string, startedAt: number): string { + const key = `${request.chatId}|${request.requestId ?? startedAt}|${request.kind}|${request.slot}|${payload}`; + return `img-${Bun.hash(key).toString(16)}`; +} + +/** + * Identifies WHAT a picture shows, not WHEN it was built. The picture of today carries the clock (the + * "now" line and the "Today" badge), so hashing it whole changed every minute: the same picture asked + * again a minute later was a new delivery, and a calendar write could never be recognised as "nothing the + * picture shows changed" for a render that took longer than a minute. + */ +function payloadOf(job: ImageRenderJob): string { + const data = + job.type === 'daily-agenda' ? { ...job.data, currentTimeMinutes: undefined, relativeDay: undefined } : job.data; + return String(Bun.hash(JSON.stringify(data))); +} + +/** + * The note of a picture that did not arrive. Pictures the user can ask for again say so; the schedule + * comparison of an accepted invitation cannot be requested by any command or tool, so its notes do not offer it. + */ +function imageNote( + kind: ImageKind, + language: 'ru' | 'en', + which: 'failed' | 'timedOut' | 'interrupted' | 'undeliverable' | 'uncertain' | 'refused', + label: string, +): string { + const meta = t(language).aiTools.meta; + // Telegram refuses pictures in this chat for good: asking again cannot help, so no picture kind offers it. + if (which === 'refused') return meta.imageNoteRefused(label); + if (kind === 'conflict') { + if (which === 'failed') return meta.imageNoteFailedFinal(label); + if (which === 'undeliverable') return meta.imageNoteUndeliverableFinal(label); + if (which === 'uncertain') return meta.imageNoteUncertainFinal(label); + return which === 'timedOut' ? meta.imageNoteTimedOutFinal(label) : meta.imageNoteInterruptedFinal(label); + } + if (which === 'failed') return meta.imageNoteFailed(label); + if (which === 'undeliverable') return meta.imageNoteUndeliverable(label); + if (which === 'uncertain') return meta.imageNoteUncertain(label); + return which === 'timedOut' ? meta.imageNoteTimedOut(label) : meta.imageNoteInterrupted(label); +} + +/** + * Telegram's own answer says the upload was not accepted (a 4xx: bad request, blocked bot, flood wait), so the + * picture did not land and the user can be told. Anything else thrown while uploading (a dropped connection, + * an abort, a 5xx) says nothing about whether Telegram stored the message: that is an uncertain send. + */ +function uploadRefused(err: unknown): boolean { + return err instanceof TelegramError && err.code < 500; +} + +/** + * Telegram refused the upload for a reason that will not change by asking again: the bot was blocked or removed + * (403), the chat is gone (404, or 400 "chat not found"), or a group does not let the bot send photos. The user still + * gets a note that does not offer to ask again (a group without photo rights can read text), but this is not a + * defect of the bot, so it raises no alert, the same rule as the permanent failures of the broadcast worker. + */ +function permanentRefusal(err: unknown): boolean { + if (!(err instanceof TelegramError)) return false; + return ( + isPermanentTelegramError(err.code) || + (err.code === 400 && /not enough rights|no rights to send|chat not found/i.test(err.message)) + ); +} + +/** What a row in this state says about the same request arriving again. */ +function repeatStatus(state: ImageDeliveryRow['state'] | undefined): 'duplicate' | 'settled' { + return state === 'queued' || state === 'rendered' || state === 'sending' ? 'duplicate' : 'settled'; +} + +/** + * Whether one more picture of this request's kind would pass a limit; `replaceable` pending pictures would be + * replaced by it and so free their places. The schedule comparison of an accepted invitation is not something the + * inviter asked for, so it neither uses up the chat's own limit (three guests accepting within one render would lose + * the third picture for good) nor is refused by a limit its own pictures filled. It has its own per-chat and + * bot-wide limits, so one chat (an event with many invitees accepting at once) cannot take the whole budget. + */ +function overLimits(request: ImageDeliveryRequest, active: readonly Entry[], replaceable: number): boolean { + if (active.length - replaceable >= MAX_PENDING_TOTAL) return true; + const inChat = active.filter((e) => e.request.chatId === request.chatId); + if (request.kind === 'conflict') { + const comparisons = active.filter((e) => e.request.kind === 'conflict'); + return ( + comparisons.length - replaceable >= MAX_PENDING_CONFLICT_TOTAL || + inChat.filter((e) => e.request.kind === 'conflict').length - replaceable >= MAX_PENDING_CONFLICT_PER_CHAT + ); + } + return inChat.filter((e) => e.request.kind !== 'conflict').length - replaceable >= MAX_PENDING_PER_CHAT; +} + +/** Rejects with `onTimeout()` after `ms`, unless cleared. */ +function deadlineAfter(ms: number, onTimeout: () => Error): { promise: Promise; clear(): void } { + const { promise, reject } = Promise.withResolvers(); + promise.catch(() => {}); + const timer = setTimeout(() => reject(onTimeout()), Math.max(0, ms)); + return { promise, clear: () => clearTimeout(timer) }; +} + +export class ImageDelivery { + private readonly pending = new Set(); + private readonly now: () => number; + private readonly boundMs: number; + private readonly sendBoundMs: number; + private closed = false; + /** Deliveries whose end the store has refused so far; each is tried again, and once more at shutdown. */ + private readonly unrecordedEntries = new Set(); + private transport: ImageTransport | undefined; + /** Bumped by every calendar write; compared with the value a job was built at. */ + private seq = 0; + private lastWriteSeq = 0; + private readonly writeSeqByUser = new Map(); + + constructor(private readonly deps: ImageDeliveryDeps) { + this.now = deps.now ?? Date.now; + this.boundMs = deps.boundMs ?? IMAGE_DELIVERY_BOUND_MS; + this.sendBoundMs = deps.sendBoundMs ?? IMAGE_SEND_BOUND_MS; + } + + /** + * Queues the picture and returns at once; the render and the delivery happen in the background. + * Throws only when `buildJob` throws (the caller reports that as a failed tool call). + */ + enqueue(request: ImageDeliveryRequest): ImageEnqueueResult { + if (this.closed) return { status: 'closed' }; + const startedAt = this.now(); + const deadline = startedAt + this.boundMs; + const job: ImageRenderJob = { ...request.buildJob(), expiresAt: deadline }; + const payload = payloadOf(job); + const id = deliveryId(request, payload, startedAt); + + // Only a delivery still on its way counts: one that ended and is only telling the user (or waiting on a slow + // Telegram call) must not hold a place against the limits. + const active = [...this.pending].filter((e) => !e.cancelled && !e.settled); + const sameChat = active.filter((e) => e.request.chatId === request.chatId); + const sameSlot = sameChat.filter((e) => e.request.slot === request.slot); + if (sameSlot.some((e) => e.payload === payload)) return { status: 'duplicate' }; + // The same request again after its row exists (see `knownRequest`). + const known = this.knownRequest(id); + if (known) return { status: known }; + // A different payload for the same slot means the data moved on: the older picture is the stale one. + const replaceable = sameSlot.filter((e) => e.phase === 'rendering'); + if (overLimits(request, active, replaceable.length)) return { status: 'busy' }; + const raced = this.recordNew({ id, request, job, payload, startedAt, deadline }); + if (raced) return { status: raced }; + for (const older of replaceable) { + // Settled at the moment of replacement, not when its run unwinds: a shutdown in between must not leave the + // stale picture's row `queued` for the next start to send after the newer one. + older.cancelled = 'superseded'; + this.settle(older, 'superseded', {}); + older.abort.abort(); + } + + this.start({ + id, + rebuilds: 0, + request, + startedAt, + deadline, + payload, + abort: new AbortController(), + job, + builtAtSeq: this.seq, + phase: 'rendering', + settled: false, + done: Promise.resolve(), + }); + return { status: 'queued' }; + } + + /** + * The same request again after its row exists, asked before the limits so a redelivered update whose picture + * already ended is never told "too many pictures". A live row is a picture on its way (this or the previous + * process); a finished one means this very request was already answered. A superseded one is neither: that + * picture never uploaded and nobody was told, so the request goes on (the data went back to what it showed), + * unless the delivery that replaced it is still unwinding. + */ + private knownRequest(id: string): 'duplicate' | 'settled' | undefined { + const stored = this.deps.store?.get(id); + if (!stored) return undefined; + if (stored.state !== 'superseded') return repeatStatus(stored.state); + return [...this.pending].some((e) => e.id === id) ? 'duplicate' : undefined; + } + + /** Writes the new delivery's row. Not undefined when another connection recorded the same id between the lookup and the insert. */ + private recordNew(fresh: { + id: string; + request: ImageDeliveryRequest; + job: ImageRenderJob; + payload: string; + startedAt: number; + deadline: number; + }): 'duplicate' | 'settled' | undefined { + const { store } = this.deps; + if (!store) return undefined; + const { id, request, job } = fresh; + const recorded = store.insert({ + id, + chatId: request.chatId, + language: request.language, + kind: request.kind, + label: request.label, + jobType: job.type, + slot: request.slot, + payloadHash: fresh.payload, + jobJson: JSON.stringify(job), + createdAt: fresh.startedAt, + expiresAt: fresh.deadline, + }); + return recorded ? undefined : repeatStatus(store.get(id)?.state); + } + + private start(entry: Entry): void { + this.pending.add(entry); + entry.done = this.run(entry) + .catch((err: unknown) => imageLogger.error({ err, kind: entry.request.kind }, 'Image delivery run failed')) + .finally(() => this.pending.delete(entry)); + } + + /** Recovery after a restart sends through this transport: the original request's closures died with the process. */ + attachTransport(transport: ImageTransport): void { + this.transport = transport; + } + + /** + * Start-up reconciliation of what the previous process left in the store, each delivery exactly once: + * - `sending`: the upload was started and its end is unknown, so it may have landed. Marked `unknown`, + * NEVER retried (a retry could send a second picture) and the user is not told it failed. + * - past its bound (or without a payload): `expired`, ONE note to the user ("the bot was restarting"), no + * timeout alert: a row still `queued` after its bound means the previous process stopped before the bound + * (a running process records `expired` itself), so the cause is downtime, not a slow render. + * - otherwise (`queued`, `rendered`): the same BullMQ job id is awaited again, so a render that + * finished while the bot was down is picked up, one still waiting or running is waited for, and one + * the queue lost is rendered anew from the stored payload; it is then sent once and recorded. + * A recovered picture is not rebuilt from fresh calendar data and is not pinned. + */ + async reconcile(): Promise { + const { store, transport } = + this.deps.store && this.transport ? { store: this.deps.store, transport: this.transport } : {}; + if (!store || !transport) return; + const at = this.now(); + store.pruneOlderThan(at - IMAGE_DELIVERY_RETENTION_MS); + const restarted = new Set(); + for (const row of store.unfinished()) { + if (row.state === 'sending') this.recoverUpload(row, store, at); + else if (this.now() >= row.expires_at || !row.job_json) this.recoverExpired(row, store, at, restarted); + else this.resume(row, row.job_json, transport); + } + // Ended as failed/expired (just now, or by the previous process which died before telling the user). + for (const row of store.unnotified()) await this.recoveryNote(row, transport, restarted); + } + + /** A row left `sending`: the upload may have landed, so it is `unknown` and never retried. */ + private recoverUpload(row: ImageDeliveryRow, store: ImageDeliveryRepository, at: number): void { + store.finish(row.id, 'unknown', at); + const stale = at - row.updated_at > RECOVERY_NOTE_MAX_AGE_MS; + this.logOutcome('unknown', row, at, { delivery: 'unknown', recovered: true, stale }); + if (!stale) this.reportUncertain(row, at); // an upload cut off days ago is downtime's doing: no alert about it now + } + + /** A row past its bound (or without a payload): `expired`; its note goes out in the pass after every live row was resumed. */ + private recoverExpired( + row: ImageDeliveryRow, + store: ImageDeliveryRepository, + at: number, + restarted: Set, + ): void { + store.finish(row.id, 'expired', at); + const stale = at - row.expires_at > RECOVERY_NOTE_MAX_AGE_MS; + this.logOutcome('interrupted', row, at, { phase: 'recovery', recovered: true, stale }); + // A slow Telegram call must not make a finished picture miss its deadline, so no note is sent here. + if (stale) + store.markNoteSent(row.id); // claimed unsent: the unnotified pass must not tell a user days later + else restarted.add(row.id); + } + + /** The one note of a recovered failed/expired delivery; recorded before it is sent, so it is never repeated. */ + private async recoveryNote( + row: ImageDeliveryRow, + transport: ImageTransport, + restarted: ReadonlySet, + ): Promise { + if (row.note_sent || !this.deps.store?.markNoteSent(row.id)) return; + if (this.now() - row.updated_at > RECOVERY_NOTE_MAX_AGE_MS) return; // claimed unsent: the user stopped waiting long ago + const text = imageNote( + row.kind as ImageKind, + row.language === 'ru' ? 'ru' : 'en', + row.state === 'failed' ? 'failed' : restarted.has(row.id) ? 'interrupted' : 'timedOut', + row.label, + ); + await transport + .notify(row.chat_id, text) + .catch((err: unknown) => + imageLogger.warn({ err, kind: row.kind }, 'Image delivery: the recovery note could not be sent'), + ); + } + + private resume(row: ImageDeliveryRow, jobJson: string, transport: ImageTransport): void { + const language = row.language === 'ru' ? 'ru' : 'en'; + const kind = row.kind as ImageKind; + const job = JSON.parse(jobJson) as ImageRenderJob; + this.start({ + id: row.id, + rebuilds: row.rebuilds, + request: { + chatId: row.chat_id, + language, + kind, + label: row.label, + slot: row.slot, + buildJob: () => job, + send: (png) => transport.sendImage(row.chat_id, png, language), + notify: (text) => transport.notify(row.chat_id, text), + }, + startedAt: row.created_at, + deadline: row.expires_at, + payload: row.payload_hash, + abort: new AbortController(), + job, + builtAtSeq: this.seq, + phase: 'rendering', + settled: false, + done: Promise.resolve(), + }); + } + + private reportUncertain(row: Pick, at: number): void { + try { + this.deps.reportFailure?.({ + kind: row.kind as ImageKind, + jobType: row.job_type as ImageRenderJob['type'], + errorName: 'UncertainSend', + elapsedMs: at - row.created_at, + }); + } catch (err) { + imageLogger.error({ err }, 'Image delivery: the uncertain send could not be reported'); + } + } + + /** A calendar write happened for this user (domain events): pending pictures of their data are stale. */ + eventsChanged(userId: number): void { + this.seq++; + this.lastWriteSeq = this.seq; + this.writeSeqByUser.set(userId, this.seq); + } + + /** Resolves when no delivery is pending. */ + async whenIdle(): Promise { + while (this.pending.size > 0) await Promise.all([...this.pending].map((e) => e.done)); + } + + /** + * Shutdown: stop accepting and log each picture still rendering as interrupted. With the store (always in + * production) no note is sent: the row stays queued, the BullMQ job is kept and the next start resumes it. + * Only without a store is the user told once, within the shutdown budget (best effort: a stuck Telegram call + * only logs a warning). A picture already being uploaded gets a short grace; if it does not finish, its + * outcome is unknown and the user is not told it failed. + */ + async close(budgetMs = SHUTDOWN_NOTICE_BUDGET_MS): Promise { + this.closed = true; + const entries = [...this.pending]; + await Promise.all( + entries.map(async (entry) => { + if (entry.phase === 'sending') { + const grace = deadlineAfter(budgetMs, () => new Error('send grace over')); + await Promise.race([entry.done, grace.promise]).catch(() => {}); + grace.clear(); + // The upload may still land: the same alert a crash at this point raises at the next start. + if (!entry.settled && this.settle(entry, 'interrupted', { delivery: 'unknown' })) { + this.reportUncertain( + { kind: entry.request.kind, job_type: entry.job.type, created_at: entry.startedAt }, + this.now(), + ); + } + return; + } + // Already ended (superseded, or told its outcome): nothing to interrupt, and its recorded end must stay what it is. + if (entry.settled || entry.cancelled) return; + entry.cancelled = 'shutdown'; + // 'shutdown' tells the renderer to stop waiting but keep the queued BullMQ job: the next start picks it up. + entry.abort.abort('shutdown'); + if (this.deps.store) { + // The record stays `queued`/`rendered`: the next process finishes or expires it, so no note now. + this.settle(entry, 'interrupted', { delivery: 'not_started', resumable: true }, false); + return; + } + if (!this.settle(entry, 'interrupted', { delivery: 'not_started' })) return; + const note = imageNote(entry.request.kind, entry.request.language, 'interrupted', entry.request.label); + const budget = deadlineAfter(budgetMs, () => new Error('shutdown notice budget over')); + await Promise.race([entry.request.notify(note), budget.promise]).catch((err: unknown) => + imageLogger.warn({ err, kind: entry.request.kind }, 'Image delivery: shutdown note not sent'), + ); + budget.clear(); + }), + ); + // A write the store still refuses gets one last synchronous try: after this process is gone, a row that looks + // unfinished would be resumed or expired again (a superseded picture sent after the newer one). + // Stops at the first refusal: SQLite has then already waited its whole busy timeout (5 s) on this thread, and the + // shutdown deadline is 8 s. + for (const entry of [...this.unrecordedEntries]) if (!this.tryRecord(entry)) break; + } + + /** Writes a rebuild to the store before its render starts, so a restart resumes the rebuilt job. A failed write only costs that. */ + private recordRebuild(entry: Entry): void { + try { + this.deps.store?.rebuilt( + entry.id, + { jobJson: JSON.stringify(entry.job), payloadHash: entry.payload, rebuilds: entry.rebuilds }, + this.now(), + ); + } catch (err) { + imageLogger.warn( + { err, kind: entry.request.kind }, + 'Image delivery: the rebuild could not be recorded, a restart would resume the earlier render', + ); + } + } + + private isStale(entry: Entry): boolean { + const { dataOwner } = entry.request; + if (dataOwner === undefined) return false; + const changedAt = dataOwner === 'group' ? this.lastWriteSeq : (this.writeSeqByUser.get(dataOwner) ?? 0); + return changedAt > entry.builtAtSeq; + } + + private async renderOnce(entry: Entry): Promise { + const remaining = entry.deadline - this.now(); + const elapsed = () => this.now() - entry.startedAt; + const cut = deadlineAfter(remaining, () => new RenderTimeoutError(elapsed(), this.boundMs)); + const rendering = this.deps.renderer.renderDirect(entry.job, { + timeoutMs: remaining, + signal: entry.abort.signal, + jobId: entry.rebuilds === 0 ? entry.id : `${entry.id}-r${entry.rebuilds}`, + }); + rendering.catch(() => {}); + try { + return await Promise.race([rendering, cut.promise]); + } catch (err) { + // Our own bound won the race (or the renderer timed out): withdraw the job from the queue. + if (err instanceof RenderTimeoutError) entry.abort.abort(); + throw err; + } finally { + cut.clear(); + } + } + + private async renderFresh(entry: Entry): Promise { + const png = await this.renderOnce(entry); + if (entry.cancelled || !this.isStale(entry)) return png; + // The calendar changed while this picture rendered; the text the user got already says the new state. + // A finished picture is worth more than none: the rebuild only runs with a realistic budget left and + // any failure of it (not a cancel) falls back to the picture already rendered. + if (entry.deadline - this.now() < REBUILD_MIN_BUDGET_MS) { + imageLogger.info( + { kind: entry.request.kind }, + 'Image delivery: data changed but no time to rebuild, sending as rendered', + ); + return png; + } + imageLogger.info({ kind: entry.request.kind }, 'Image delivery: data changed while rendering, rebuilding once'); + const previous = { job: entry.job, payload: entry.payload }; + try { + const rebuilt = { ...entry.request.buildJob(), expiresAt: entry.deadline }; + entry.builtAtSeq = this.seq; + const rebuiltPayload = payloadOf(rebuilt); + if (rebuiltPayload === entry.payload) { + // The write touched nothing this picture shows: the finished render is already the current one. + imageLogger.info({ kind: entry.request.kind }, 'Image delivery: data unchanged by the write, no rebuild'); + return png; + } + entry.rebuilds++; + entry.job = rebuilt; + // From here the picture on its way shows the new data: the same request again is this delivery. + entry.payload = rebuiltPayload; + this.recordRebuild(entry); + return await this.renderOnce(entry); + } catch (err) { + entry.job = previous.job; + entry.payload = previous.payload; + if (entry.cancelled || err instanceof RenderAbortedError) throw err; + if (err instanceof RenderTimeoutError) { + // The fallback absorbs it (the user still gets the earlier picture), but the slow host is still the owner's to hear about. + reportTimeout({ + kind: 'render', + elapsedMs: this.now() - entry.startedAt, + limitMs: this.boundMs, + recovered: true, + }); + } + imageLogger.warn( + { err, kind: entry.request.kind }, + 'Image delivery: the rebuild failed, sending the picture as rendered', + ); + return png; + } + } + + private async run(entry: Entry): Promise { + const { request } = entry; + try { + const png = await this.renderFresh(entry); + if (entry.cancelled) throw new RenderAbortedError(); + this.deps.store?.advance(entry.id, 'rendered', this.now()); + // Recorded BEFORE the upload starts: a process killed from here on leaves `sending`, which the next + // start reads as "may have landed" and never retries. It is also the claim on the send: a row another + // attempt already claimed, or that recovery settled meanwhile, is not uploaded a second time. + if (this.deps.store && !this.deps.store.advance(entry.id, 'sending', this.now())) { + this.settle(entry, 'claimed_elsewhere', { delivery: 'not_started' }, false); + return; + } + entry.phase = 'sending'; + const sending = request.send(png); + sending.catch(() => {}); + const cut = deadlineAfter(this.sendBoundMs, () => new SendTimeoutError(this.sendBoundMs)); + let messageId: number; + try { + messageId = await Promise.race([sending, cut.promise]); + } finally { + cut.clear(); + } + this.settle(entry, 'delivered', { messageId }); + } catch (err) { + await this.fail(entry, err); + } + } + + private async fail(entry: Entry, err: unknown): Promise { + const elapsedMs = this.now() - entry.startedAt; + if (entry.cancelled === 'superseded') { + this.settle(entry, 'superseded', {}); + return; + } + if (entry.cancelled === 'shutdown') return; // close() logged and told the user + if (err instanceof SendTimeoutError || err instanceof RenderTimeoutError) { + await this.failTimedOut(entry, err, elapsedMs); + return; + } + // A typed agenda-image error is thrown before any upload starts, so the picture did not land. + const typed = agendaImageErrorMessage(err) !== undefined; + if (entry.phase === 'sending' && !typed && !uploadRefused(err)) { + await this.failUncertain(entry, err); + return; + } + await this.failForGood(entry, err, elapsedMs); + } + + /** A bound ran out: the render's (the picture did not come) or the upload's (it may still land). */ + private async failTimedOut( + entry: Entry, + err: SendTimeoutError | RenderTimeoutError, + elapsedMs: number, + ): Promise { + const { request } = entry; + if (err instanceof SendTimeoutError) { + // The upload may still land: the note does not say it did not, and the owner's repair flow hears of it. + if (!this.settle(entry, 'timed_out', { phase: 'send', delivery: 'unknown' })) return; + reportTimeout({ kind: 'render', elapsedMs, limitMs: this.sendBoundMs, recovered: false }); + await this.tell(entry, imageNote(request.kind, request.language, 'uncertain', request.label), false); + return; + } + if (!this.settle(entry, 'timed_out', { phase: 'render' })) return; + reportTimeout({ kind: 'render', elapsedMs, limitMs: this.boundMs, recovered: false }); + await this.tell(entry, imageNote(request.kind, request.language, 'timedOut', request.label)); + } + + /** + * The upload was started and ended in an error that does not say Telegram refused it: the picture may have + * landed. Never a note that says it did not, never a retry; the user is told it may not have arrived (the answer + * may have been only this picture) and the owner's repair flow hears of it. + */ + private async failUncertain(entry: Entry, err: unknown): Promise { + const { request } = entry; + const errorName = err instanceof Error ? err.name : typeof err; + if (!this.settle(entry, 'unknown', { delivery: 'unknown', errorName })) return; + this.reportUncertain({ kind: request.kind, job_type: entry.job.type, created_at: entry.startedAt }, this.now()); + await this.tell(entry, imageNote(request.kind, request.language, 'uncertain', request.label), false); + } + + /** The picture did not land: one note that fits the cause, and an alert only for what is the bot's own defect. */ + private async failForGood(entry: Entry, err: unknown, elapsedMs: number): Promise { + const { request } = entry; + const errorName = err instanceof Error ? err.name : typeof err; + const tooLarge = agendaImageTooLarge(err); + const permanent = permanentRefusal(err); + if (!this.settle(entry, 'failed', { errorName, expected: tooLarge || permanent })) return; + if (tooLarge) { + // The picture is over a size limit (too tall, too big): the user must change the request, no incident. + // Any other typed error (an invalid PNG, no document sender) is a defect and goes on to the alert below. + // The raw message is English and internal; the user gets the localized note. + await this.tell(entry, imageNote(request.kind, request.language, 'undeliverable', request.label)); + return; + } + if (permanent) { + // Asking again fails the same way: the note says so instead of offering it, the owner's repair flow gets nothing to investigate. + imageLogger.warn({ err, kind: request.kind, jobType: entry.job.type }, 'Image delivery refused for good'); + await this.tell(entry, imageNote(request.kind, request.language, 'refused', request.label)); + return; + } + imageLogger.error({ err, kind: request.kind, jobType: entry.job.type }, 'Image delivery failed'); + try { + this.deps.reportFailure?.({ kind: request.kind, jobType: entry.job.type, errorName, elapsedMs }); + } catch (reportErr) { + imageLogger.error({ err: reportErr }, 'Image delivery failure could not be reported'); + } + await this.tell(entry, imageNote(request.kind, request.language, 'failed', request.label)); + } + + /** + * The one note to the user, bounded like the upload: a Telegram call that hangs or sleeps through a flood wait + * must not hold the delivery. `claim` is the store's once-only claim on the note of a failed/expired row; an + * uncertain upload ends `unknown`, which recovery never notes, so its single note is guarded by `settle` alone. + */ + private async tell(entry: Entry, text: string, claim = true): Promise { + const bound = deadlineAfter(this.sendBoundMs, () => new SendTimeoutError(this.sendBoundMs)); + try { + if (this.deps.store && claim) { + if (entry.recordFailed) + entry.noteUnclaimed = true; // claimed when the end of the delivery is finally written + else if (!this.deps.store.markNoteSent(entry.id)) return; // another path already owns the note + } + const sending = entry.request.notify(text); + sending.catch(() => {}); + await Promise.race([sending, bound.promise]); + } catch (err) { + imageLogger.warn({ err, kind: entry.request.kind }, 'Image delivery: the failure note could not be sent'); + } finally { + bound.clear(); + } + } + + /** Logs the one outcome of the delivery (never user text) and records it. False when it was already settled. */ + private settle(entry: Entry, outcome: ImageDeliveryOutcome, details: ImageOutcomeDetails, persist = true): boolean { + if (entry.settled) return false; + entry.settled = true; + const at = this.now(); + this.logOutcome( + outcome, + { + chat_id: entry.request.chatId, + kind: entry.request.kind, + job_type: entry.job.type, + created_at: entry.startedAt, + }, + at, + details, + ); + if (persist && this.deps.store) { + const messageId = typeof details.messageId === 'number' ? details.messageId : undefined; + try { + this.deps.store.finish(entry.id, storedState(outcome, details), at, messageId); + } catch (err) { + // Bookkeeping must not stop the user from being told (or the picture from counting as delivered). + entry.recordFailed = true; + entry.unrecorded = { state: storedState(outcome, details), at, messageId }; + this.unrecordedEntries.add(entry); + reportIfDatabaseBusy(err); + imageLogger.error({ err, kind: entry.request.kind }, 'Image delivery: the outcome could not be recorded'); + this.scheduleRecord(entry, 1); + } + } + return true; + } + + /** + * Writes an outcome the store refused (a locked database) once more, so a restart does not find the row still + * `queued` and send, expire or alert the same picture a second time. A note already sent meanwhile is claimed as + * soon as the row can carry the claim. Each step is cleared only once it is done, so a refusal in the claim is + * tried again as the claim alone. False when the store still refuses. + */ + private tryRecord(entry: Entry): boolean { + try { + const { unrecorded } = entry; + if (unrecorded) { + this.deps.store?.finish(entry.id, unrecorded.state, unrecorded.at, unrecorded.messageId); + entry.unrecorded = undefined; + entry.recordFailed = false; + } + if (entry.noteUnclaimed) { + this.deps.store?.markNoteSent(entry.id); + entry.noteUnclaimed = false; + } + this.unrecordedEntries.delete(entry); + return true; + } catch (err) { + reportIfDatabaseBusy(err); + return false; + } + } + + private scheduleRecord(entry: Entry, attempt: number): void { + this.schedule(() => { + if (this.tryRecord(entry)) return; + if (attempt < RECORD_RETRIES) this.scheduleRecord(entry, attempt + 1); + else imageLogger.error({ kind: entry.request.kind }, 'Image delivery: the outcome was never recorded'); + }, RECORD_RETRY_MS * attempt); + } + + private schedule(fn: () => void, delayMs: number): void { + if (this.deps.schedule) this.deps.schedule(fn, delayMs); + else setTimeout(fn, delayMs).unref(); + } + + private logOutcome( + outcome: ImageDeliveryOutcome, + row: { chat_id: number; kind: string; job_type: string; created_at: number }, + at: number, + details: ImageOutcomeDetails, + ): void { + imageLogger.info( + { + outcome, + kind: row.kind, + jobType: row.job_type, + chatId: row.chat_id, + elapsedMs: at - row.created_at, + ...details, + }, + 'Image delivery outcome', + ); + } +} diff --git a/src/services/image/queue-picture.ts b/src/services/image/queue-picture.ts new file mode 100644 index 000000000..71305a425 --- /dev/null +++ b/src/services/image/queue-picture.ts @@ -0,0 +1,88 @@ +// The one way a slash command, a button or an invitation notice sends a calendar picture: it QUEUES the +// picture on ImageDelivery and returns, so the handler never waits for a render. The text answer has +// already been sent by the caller; the picture arrives later as its own message. +// +// Why: the render queue shares a CPU-bound host (renders took 73-80 s on 2026-11-01), so a handler that +// awaited the render held its update (and, for buttons, the user's spinner) for as long as the host was +// slow, and a timeout lost the picture with only a log line. ImageDelivery owns the rest: dedupe of a +// repeated command or tap, the durable record, the restart reconcile, ONE honest note on failure or +// expiry and reportTimeout({kind: 'render'}) for the owner's repair flow. +import { t } from '../../config/constants.ts'; +import { sendAgendaImage } from '../../utils/agenda-image.ts'; +import { imageLogger } from '../../utils/logger.ts'; +import type { ImageRenderJob } from '../../worker/image-render.queue.ts'; +import type { ImageDelivery, ImageEnqueueResult, ImageKind } from './image-delivery.ts'; + +export interface PictureRequest { + chatId: number; + language: 'ru' | 'en'; + kind: ImageKind; + /** What the user asked for (a date, a week, a month): shown only in that user's own failure note. */ + label: string; + /** The same slot shows the same thing at different times: kind + target + whose calendar. */ + slot: string; + /** The Telegram message id (command) or callback query id (button): half of the deterministic delivery id. */ + requestId: number | string; + /** Whose calendar the picture shows, so a write meanwhile rebuilds it from the current data. */ + dataOwner?: number | 'group'; + filename: string; + /** Reads the CURRENT data and builds the render job; called again once if the calendar changes while it renders. */ + buildJob(): ImageRenderJob; + /** Each returns the Telegram message id of what it sent. */ + sendPhoto(file: File): Promise; + sendDocument?(file: File, options: { caption: string }): Promise; + /** What the user is told when the chat already has too many pictures pending; default: try again a little later. */ + busyNote?: string; + /** What the user is told when the job cannot even be built; default points at the text answer above the picture, so a picture with no text version supplies its own. */ + failureNote?: string; + /** Sends a short note to the same chat. */ + notify(text: string): Promise; + /** Runs after the picture is delivered (the best-effort pin); must not throw. */ + onSent?(messageId: number): void; +} + +async function say(request: PictureRequest, text: string): Promise { + try { + await request.notify(text); + } catch (err) { + imageLogger.warn({ err, kind: request.kind }, 'Picture note could not be sent'); + } +} + +/** + * Queues the picture and returns once it is queued. `duplicate` (the same picture is already on its + * way) and `closed` (the bot is shutting down) are silent; `busy` and a job that cannot even be built + * get one honest note, because nothing is on its way. + */ +export async function queuePicture(delivery: ImageDelivery, request: PictureRequest): Promise { + let result: ImageEnqueueResult; + try { + result = delivery.enqueue({ + chatId: request.chatId, + language: request.language, + kind: request.kind, + label: request.label, + slot: request.slot, + requestId: request.requestId, + dataOwner: request.dataOwner, + buildJob: request.buildJob, + send: async (png) => { + const file = new File([png], request.filename, { type: 'image/png' }); + const messageId = await sendAgendaImage(file, { + language: request.language, + sendPhoto: request.sendPhoto, + sendDocument: request.sendDocument, + }); + request.onSent?.(messageId); + return messageId; + }, + notify: request.notify, + }); + } catch (err) { + imageLogger.error({ err, kind: request.kind }, 'Picture job could not be built'); + await say(request, request.failureNote ?? t(request.language).callbackErrors.imageGenerationFailed); + return { status: 'closed' }; + } + if (result.status === 'busy') await say(request, request.busyNote ?? t(request.language).aiTools.meta.imageBusy); + return result; +} diff --git a/src/services/image/render-conflict.ts b/src/services/image/render-conflict.ts index dec798bc6..0aac8c9bc 100644 --- a/src/services/image/render-conflict.ts +++ b/src/services/image/render-conflict.ts @@ -1,9 +1,9 @@ // src/services/image/render-conflict.ts +import type { ImageRenderJob } from '../../worker/image-render.queue.ts'; import type { ConflictRow, ConflictScheduleData } from '../../worker/templates/conflict-schedule.ts'; import { getTheme } from '../../worker/templates/themes.ts'; import type { ConflictResult } from '../invite/conflict-service.ts'; -import type { ImageRenderer } from './render-service.ts'; const TWO_HOURS_MS = 2 * 60 * 60 * 1000; @@ -11,8 +11,8 @@ function toMinutes(iso: string): number { return Math.round(new Date(iso).getTime() / 60_000); } -export async function renderConflictImage( - renderService: ImageRenderer, +/** The render job for the organizer/invitee conflict picture, built from the schedules alone (no worker involved). */ +export function conflictImageJob( organizerId: number, organizerLabel: string, organizerEvents: { startAt: string; endAt: string; title: string }[], @@ -20,7 +20,7 @@ export async function renderConflictImage( eventStart: string, eventEnd: string, locale: 'ru' | 'en', -): Promise { +): ImageRenderJob { const startMs = new Date(eventStart).getTime(); const endMs = new Date(eventEnd).getTime(); const windowStartIso = new Date(startMs - TWO_HOURS_MS).toISOString(); @@ -76,9 +76,5 @@ export async function renderConflictImage( locale, }; - return renderService.renderDirect({ - type: 'conflict-schedule', - data, - userId: organizerId, - }); + return { type: 'conflict-schedule', data, userId: organizerId }; } diff --git a/src/services/image/render-day.ts b/src/services/image/render-day.ts index 864076a50..d54d634a5 100644 --- a/src/services/image/render-day.ts +++ b/src/services/image/render-day.ts @@ -1,19 +1,19 @@ import { TZDate } from '@date-fns/tz'; import type { EventOccurrence } from '../../database/types.ts'; +import type { ImageRenderJob } from '../../worker/image-render.queue.ts'; import { getTheme } from '../../worker/templates/themes.ts'; import type { HolidayEntry } from '../holiday/holiday-service.ts'; import { mapDailyAgendaData } from './data-mapper.ts'; -import type { ImageRenderer } from './render-service.ts'; -export async function renderDayImage( - renderService: ImageRenderer, +/** The render job for the picture, built from the agenda alone (no worker involved). */ +export function dayImageJob( occurrences: EventOccurrence[], dateIso: string, timezone: string, locale: 'ru' | 'en', userId: number, holidays?: HolidayEntry[], -): Promise { +): ImageRenderJob { const userNow = new TZDate(new Date(), timezone); const todayIso = userNow.toISOString().slice(0, 10); const isToday = dateIso === todayIso; @@ -30,9 +30,5 @@ export async function renderDayImage( holidayName: holidays?.[0]?.name, }); - return renderService.renderDirect({ - type: 'daily-agenda', - data, - userId, - }); + return { type: 'daily-agenda', data, userId }; } diff --git a/src/services/image/render-failure-reporter.ts b/src/services/image/render-failure-reporter.ts new file mode 100644 index 000000000..ede877465 --- /dev/null +++ b/src/services/image/render-failure-reporter.ts @@ -0,0 +1,111 @@ +// A picture that could not be made or sent for a reason other than time (the worker threw, the upload +// was rejected) is reported to the owner's repair flow (alerts queue -> hcb-alert-watcher -> omp +// investigation), as its own alert kind. Timeouts take the other road, reportTimeout({kind:'render'}). +// +// Aggregated: at most one alert per window, carrying how many failures were held back since the last +// one. A broken browser fails every picture the same way and must not flood the queue. The text holds +// the picture kind, the job type, the error class name and durations: never the user's request, +// event titles, a table, an error message or any chat/user id. +import type { RenderFailureReport } from './image-delivery.ts'; + +/** The `source` of the alerts-table row; the watcher's brief keys on it. */ +export const RENDER_FAILURE_ALERT_SOURCE = 'render-failure'; +export const RENDER_FAILURE_WINDOW_MS = 10 * 60_000; +/** A held summary the alert queue refused (database locked) is tried again after this long. */ +export const RENDER_FAILURE_RETRY_MS = 60_000; + +/** Runs `fn` after `delayMs` without keeping the process alive. */ +function defaultSchedule(fn: () => void, delayMs: number): void { + setTimeout(fn, delayMs).unref(); +} + +export class RenderFailureReporter { + private lastPushedAt = -Infinity; + private held = 0; + /** The newest report held back; it describes the alert sent when the window ends. */ + private heldReport: RenderFailureReport | null = null; + private timerScheduled = false; + + constructor( + private readonly push: (text: string, source: string) => void, + private readonly now: () => number = Date.now, + private readonly schedule: (fn: () => void, delayMs: number) => void = defaultSchedule, + ) {} + + report(report: RenderFailureReport): void { + const waitMs = this.lastPushedAt + RENDER_FAILURE_WINDOW_MS - this.now(); + if (waitMs > 0) { + // A burst inside the window is not lost: one summary alert goes out when the window ends. + this.held++; + this.heldReport = report; + if (!this.timerScheduled) { + this.timerScheduled = true; + this.schedule(() => this.sendHeld(), waitMs); + } + return; + } + if (this.send(report)) return; + // The queue refused the first failure of the window (database locked): it is held like a burst member and + // tried again, so a lone failure is not lost and a later alert counts it. + this.held++; + this.heldReport = report; + if (!this.timerScheduled) { + this.timerScheduled = true; + this.schedule(() => this.sendHeld(), RENDER_FAILURE_RETRY_MS); + } + } + + /** + * A timer fired: send the summary. Never inside a window another alert opened meanwhile (a retry timer outlives + * the alert that a later report sent directly): the summary then waits for that window's end. A refused push + * keeps it held and tries again, so a locked database does not lose it. + */ + private sendHeld(): void { + const pending = this.heldReport; + if (!pending) { + this.timerScheduled = false; + return; + } + const waitMs = this.lastPushedAt + RENDER_FAILURE_WINDOW_MS - this.now(); + if (waitMs > 0) { + this.schedule(() => this.sendHeld(), waitMs); + return; + } + if (this.send(pending)) { + this.timerScheduled = false; + return; + } + this.schedule(() => this.sendHeld(), RENDER_FAILURE_RETRY_MS); + } + + /** Shutdown: sends the summary still held for the window's end, so a deploy right after a burst does not lose it. */ + flush(): void { + if (this.heldReport) this.send(this.heldReport); + } + + /** True when the alert reached the queue (the held summary, if any, is then spent). */ + private send(report: RenderFailureReport): boolean { + const uncertain = report.errorName === 'UncertainSend'; + const text = [ + uncertain ? 'Picture upload outcome unknown (render-failure)' : 'Picture delivery failed (render-failure)', + `kind: ${report.kind}`, + `jobType: ${report.jobType}`, + `error: ${report.errorName}`, + `elapsedMs: ${Math.round(report.elapsedMs)}`, + `held back since the last alert: ${this.held}`, + uncertain + ? 'The upload started and its end is unknown: the picture may have reached the chat. The user was NOT told it failed.' + : 'The user got one short note and the text answer. Look for "Image delivery failed" and "Image delivery outcome" (outcome=failed) in the container log on root@odroidn2.', + ].join('\n'); + try { + this.push(text, RENDER_FAILURE_ALERT_SOURCE); + this.lastPushedAt = this.now(); + this.held = 0; + this.heldReport = null; + return true; + } catch { + // The alert queue is a side channel: failing to reach it must never fail the delivery path. + return false; + } + } +} diff --git a/src/services/image/render-month.ts b/src/services/image/render-month.ts index df59641b3..ccc94e9ef 100644 --- a/src/services/image/render-month.ts +++ b/src/services/image/render-month.ts @@ -1,18 +1,18 @@ import { TZDate } from '@date-fns/tz'; import type { EventOccurrence } from '../../database/types.ts'; +import type { ImageRenderJob } from '../../worker/image-render.queue.ts'; import { getTheme } from '../../worker/templates/themes.ts'; import { mapMonthlyCalendarData } from './data-mapper.ts'; -import type { ImageRenderer } from './render-service.ts'; -export async function renderMonthImage( - renderService: ImageRenderer, +/** The render job for the picture, built from the agenda alone (no worker involved). */ +export function monthImageJob( occurrences: EventOccurrence[], year: number, month: number, // 0-based timezone: string, locale: 'ru' | 'en', userId: number, -): Promise { +): ImageRenderJob { const occurrencesByDay = new Map(); for (const occ of occurrences) { const dayKey = new TZDate(new Date(occ.occurrence_start), timezone).toISOString().slice(0, 10); @@ -34,9 +34,5 @@ export async function renderMonthImage( todayIso, }); - return renderService.renderDirect({ - type: 'monthly-calendar', - data, - userId, - }); + return { type: 'monthly-calendar', data, userId }; } diff --git a/src/services/image/render-service.ts b/src/services/image/render-service.ts index e743a8067..34a215feb 100644 --- a/src/services/image/render-service.ts +++ b/src/services/image/render-service.ts @@ -1,18 +1,46 @@ import type { Queue, QueueEvents } from 'bullmq'; import { imageLogger } from '../../utils/logger.ts'; import type { ImageRenderJob, ImageRenderResult } from '../../worker/image-render.queue.ts'; -import { reportTimeout } from '../incident/timeout-reporter.ts'; // Production (ODROID-N2+, since 2026-10-09) renders take 6–15 s because the idle Chromium and // Bun heap sit in swap between renders; 15 s dropped a finished 15.2 s render on 2026-10-12. -// 45 s is about 3× the slowest measured render. Like every tool, a render is not cut by the AI -// turn's 90 s deadline (checked between rounds), so a late render can end a turn past 90 s — -// chosen over discarding an image the worker already finished. +// 45 s is about 3× the slowest measured render. It is only the default for a caller that passes no +// timeout of its own: every picture the bot sends (the AI tools, the slash commands, the buttons) +// goes through ImageDelivery, which never blocks its caller and passes its own, longer bound +// (image-delivery.ts). const RENDER_TIMEOUT_MS = 45_000; +export interface RenderOptions { + /** How long to wait for the worker before giving up; default RENDER_TIMEOUT_MS. */ + timeoutMs?: number; + /** Stop waiting (and withdraw a job the worker has not started) when this aborts. */ + signal?: AbortSignal; + /** BullMQ job id: a deterministic id makes a repeated add a no-op and lets a restarted bot find the job again. */ + jobId?: string; +} + +/** The worker did not finish within the wait. The job is withdrawn when it has not started. */ +export class RenderTimeoutError extends Error { + constructor( + readonly waitedMs: number, + readonly limitMs: number, + ) { + super(`Image render timed out after ${waitedMs} ms (limit ${limitMs} ms)`); + this.name = 'RenderTimeoutError'; + } +} + +/** The caller stopped waiting (shutdown, a newer picture replaced this one). */ +export class RenderAbortedError extends Error { + constructor() { + super('Image render wait was aborted'); + this.name = 'RenderAbortedError'; + } +} + /** Structural interface for image rendering — use this in AgentContext and tests instead of RenderService class. */ export interface ImageRenderer { - renderDirect(job: ImageRenderJob): Promise; + renderDirect(job: ImageRenderJob, options?: RenderOptions): Promise; } /** BullMQ's `Job.waitUntilFinished` has no typed timeout error; its message is the only signal. */ @@ -20,36 +48,51 @@ function isWaitTimeout(err: unknown): boolean { return err instanceof Error && /timed out before finishing/.test(err.message); } +function abortion(signal: AbortSignal | undefined): { promise: Promise; stop(): void } | null { + if (!signal) return null; + const { promise, reject } = Promise.withResolvers(); + const onAbort = () => reject(new RenderAbortedError()); + if (signal.aborted) onAbort(); + else signal.addEventListener('abort', onAbort, { once: true }); + promise.catch(() => {}); + return { promise, stop: () => signal.removeEventListener('abort', onAbort) }; +} + export class RenderService implements ImageRenderer { constructor( private queue: Queue, private queueEvents: QueueEvents, ) {} - async renderDirect(job: ImageRenderJob): Promise { + async renderDirect(job: ImageRenderJob, options: RenderOptions = {}): Promise { + const limitMs = options.timeoutMs ?? RENDER_TIMEOUT_MS; const startedAt = Date.now(); imageLogger.info({ type: job.type, userId: job.userId }, 'Enqueuing render job'); - const added = await this.queue.add('render', job, { priority: 1 }); + const added = await this.queue.add('render', job, { + priority: 1, + ...(options.jobId ? { jobId: options.jobId } : {}), + }); + const finished = added.waitUntilFinished(this.queueEvents, limitMs) as Promise; + finished.catch(() => {}); // the losing side of an abort race must not become an unhandled rejection + const abort = abortion(options.signal); - let result: ImageRenderResult; try { - result = (await added.waitUntilFinished(this.queueEvents, RENDER_TIMEOUT_MS)) as ImageRenderResult; + const result = await (abort ? Promise.race([finished, abort.promise]) : finished); + imageLogger.info({ type: job.type, userId: job.userId, renderTimeMs: result.renderTimeMs }, 'Render complete'); + return Buffer.from(result.bufferBase64, 'base64'); } catch (err) { - // The error goes to the caller, who owns the user's reply; nothing here recovers a lost render. - if (isWaitTimeout(err)) { - reportTimeout({ - kind: 'render', - elapsedMs: Date.now() - startedAt, - limitMs: RENDER_TIMEOUT_MS, - recovered: false, - }); + const timedOut = isWaitTimeout(err); + if (timedOut || err instanceof RenderAbortedError) { + // Nobody waits for this picture any more: take it off the queue so the shared host does not + // render it for nobody. A job the worker already runs cannot be removed; its expiry stops a retry. + // Not on shutdown: the durable delivery resumes this very job at the next start. + if (options.signal?.reason !== 'shutdown') await added.remove().catch(() => {}); } + if (timedOut) throw new RenderTimeoutError(Date.now() - startedAt, limitMs); throw err; + } finally { + abort?.stop(); } - - imageLogger.info({ type: job.type, userId: job.userId, renderTimeMs: result.renderTimeMs }, 'Render complete'); - - return Buffer.from(result.bufferBase64, 'base64'); } } diff --git a/src/services/image/render-week.ts b/src/services/image/render-week.ts index c221566e8..9dfa6b687 100644 --- a/src/services/image/render-week.ts +++ b/src/services/image/render-week.ts @@ -1,19 +1,19 @@ import { TZDate } from '@date-fns/tz'; import type { EventOccurrence } from '../../database/types.ts'; +import type { ImageRenderJob } from '../../worker/image-render.queue.ts'; import { getTheme } from '../../worker/templates/themes.ts'; import type { DayWeather } from '../weather/types.ts'; import { mapWeeklyOverviewData } from './data-mapper.ts'; -import type { ImageRenderer } from './render-service.ts'; -export async function renderWeekImage( - renderService: ImageRenderer, +/** The render job for the picture, built from the agenda alone (no worker involved). */ +export function weekImageJob( occurrences: EventOccurrence[], weekStartIso: string, timezone: string, locale: 'ru' | 'en', userId: number, weatherByDate?: { [date: string]: DayWeather }, -): Promise { +): ImageRenderJob { const startD = new Date(`${weekStartIso}T12:00:00Z`); const occurrencesByDay = new Map(); for (let i = 0; i < 7; i++) { @@ -42,9 +42,5 @@ export async function renderWeekImage( weatherByDate, }); - return renderService.renderDirect({ - type: 'weekly-overview', - data, - userId, - }); + return { type: 'weekly-overview', data, userId }; } diff --git a/src/utils/agenda-image.ts b/src/utils/agenda-image.ts index 743f899fa..bf99d1a50 100644 --- a/src/utils/agenda-image.ts +++ b/src/utils/agenda-image.ts @@ -1,6 +1,14 @@ import { Blob as BufferBlob } from 'node:buffer'; // Shared lossless PNG delivery policy for command, callback, and AI render transports. -export class AgendaImageError extends Error {} +/** `tooLarge`: the picture is over a limit the user can stay under by asking for less; any other typed error is a defect of the renderer or the transport. */ +export class AgendaImageError extends Error { + constructor( + message: string, + readonly tooLarge = false, + ) { + super(message); + } +} export const AGENDA_ALLOCATION_ERROR = 'Agenda image is too large for the screenshot allocation limit. Choose a shorter date range.'; @@ -12,6 +20,12 @@ export function agendaImageErrorMessage(error: unknown): string | undefined { return undefined; } +/** True for a picture over a size limit (the raster allocation, the 50 MB document), whichever way the error reached us. */ +export function agendaImageTooLarge(error: unknown): boolean { + if (error instanceof AgendaImageError) return error.tooLarge; + return error instanceof Error && error.message === AGENDA_ALLOCATION_ERROR; +} + export async function sendAgendaImage( file: File, transport: { @@ -21,7 +35,7 @@ export async function sendAgendaImage( }, ): Promise { if (file.size > 50 * 1024 * 1024) - throw new AgendaImageError('Agenda PNG exceeds the 50 MB document limit. Choose a shorter date range.'); + throw new AgendaImageError('Agenda PNG exceeds the 50 MB document limit. Choose a shorter date range.', true); const header = Buffer.from(await BufferBlob.prototype.slice.call(file, 0, 24).arrayBuffer()); if ( header.length < 24 || @@ -61,7 +75,7 @@ export function planAgendaRaster(height: number, density: number) { density < 1 || width * height > maxPixels ) { - throw new AgendaImageError(AGENDA_ALLOCATION_ERROR); + throw new AgendaImageError(AGENDA_ALLOCATION_ERROR, true); } const scale = width * height * density * density > maxPixels ? 'css' : 'device'; const effectiveDensity = scale === 'css' ? 1 : density; diff --git a/src/worker/image-render.queue.ts b/src/worker/image-render.queue.ts index eaf5873f0..5516c33e3 100644 --- a/src/worker/image-render.queue.ts +++ b/src/worker/image-render.queue.ts @@ -2,6 +2,7 @@ import { Queue, QueueEvents, Worker } from 'bullmq'; import { planAgendaRaster } from '../utils/agenda-image.ts'; import { imageLogger } from '../utils/logger.ts'; import { parseRedisUrl } from '../utils/redis.ts'; +import { UnrecoverableError } from '../utils/unrecoverable-error.ts'; import { playwrightPool } from './playwright-pool.ts'; import type { ConflictScheduleData } from './templates/conflict-schedule.ts'; import { getTemplate } from './templates/index.ts'; @@ -15,7 +16,7 @@ import type { // --- Job types --- -export type ImageRenderJob = +type ImageRenderPayload = | { type: 'daily-agenda'; data: DailyAgendaData; userId: number } | { type: 'weekly-overview'; data: WeeklyOverviewData; userId: number } | { type: 'event-card'; data: EventCardData; userId: number } @@ -23,6 +24,13 @@ export type ImageRenderJob = | { type: 'conflict-schedule'; data: ConflictScheduleData; userId: number } | { type: 'md-table'; data: MdTableData; userId: number }; +/** + * `expiresAt` (epoch ms) is the moment nobody waits for the picture any more (the delivery bound of + * ImageDelivery). A job that only reaches the worker after it — a stalled job re-queued by a restart, + * or a retry — is dropped unrendered: on the shared host a render for nobody costs a real user's picture. + */ +export type ImageRenderJob = ImageRenderPayload & { expiresAt?: number }; + export interface ImageRenderResult { bufferBase64: string; // PNG as base64 (Buffer doesn't survive Redis JSON roundtrip) width: number; @@ -37,6 +45,9 @@ const QUEUE_NAME = 'image-render'; // --- Process function (exported for testing) --- export async function processRenderJob(job: ImageRenderJob): Promise { + if (job.expiresAt !== undefined && Date.now() > job.expiresAt) { + throw new UnrecoverableError('Image render job expired before it started'); + } const start = performance.now(); const template = getTemplate(job.type); @@ -83,7 +94,9 @@ export function createImageRenderQueue(redisUrl: string) { defaultJobOptions: { attempts: 2, backoff: { type: 'fixed', delay: 1000 }, - removeOnComplete: { age: 60, count: 100 }, + // A finished render stays a little over the 150 s delivery bound, so a bot restarted inside the bound can + // still collect a picture finished while it was down; after that its row is expired and nobody wants it. + removeOnComplete: { age: 180, count: 100 }, removeOnFail: { age: 3600 }, }, }); diff --git a/test/bootstrap/domain-events.test.ts b/test/bootstrap/domain-events.test.ts index 3da732585..362af9c6d 100644 --- a/test/bootstrap/domain-events.test.ts +++ b/test/bootstrap/domain-events.test.ts @@ -1,5 +1,5 @@ import { describe, expect, spyOn, test } from 'bun:test'; -import { subscribeParticipantGooglePush } from '../../src/bootstrap/domain-events.ts'; +import { subscribeImageStaleness, subscribeParticipantGooglePush } from '../../src/bootstrap/domain-events.ts'; import type { GroupMemberRepository } from '../../src/database/repositories/group-member.repository.ts'; import type { CalendarEvent } from '../../src/database/types.ts'; import { DomainEventBus } from '../../src/services/scheduled/domain-event-bus.ts'; @@ -120,3 +120,19 @@ describe('subscribeParticipantGooglePush', () => { } }); }); + +describe('subscribeImageStaleness', () => { + test('every calendar write tells the picture delivery whose data changed', () => { + const bus = new DomainEventBus(); + const changed: number[] = []; + subscribeImageStaleness(bus, { eventsChanged: (userId) => changed.push(userId) }); + + bus.emit('myCalendar.newEvent', { userId: 1, newEvent: eventWithId(1) }); + bus.emit('myCalendar.updatedEvent', { userId: 2, updatedEvent: eventWithId(1), oldEvent: eventWithId(1) }); + bus.emit('myCalendar.deletedEvent', { userId: 3, eventId: 1, title: 'e' }); + bus.emit('myGroup.newEvent', { userId: 4, groupChatId: -9, newEvent: eventWithId(2), createdBy: 4 }); + bus.emit('myCalendar.eventStarting', { userId: 5, event: eventWithId(1) }); + + expect(changed).toEqual([1, 2, 3, 4]); + }); +}); diff --git a/test/bootstrap/image-delivery-recovery.test.ts b/test/bootstrap/image-delivery-recovery.test.ts new file mode 100644 index 000000000..d890b8cb6 --- /dev/null +++ b/test/bootstrap/image-delivery-recovery.test.ts @@ -0,0 +1,142 @@ +// recoverImageDeliveries: after a restart the bot hands ImageDelivery its Telegram sender and the previous +// process's unfinished pictures are delivered once, expired with one note, or marked unknown and never +// re-sent. A real ImageDelivery over a real SQLite store; only Telegram and the render queue are fakes. +import { Database } from 'bun:sqlite'; +import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from 'bun:test'; +import { recoverImageDeliveries } from '../../src/bootstrap/image-render.ts'; +import { t } from '../../src/config/constants.ts'; +import { migrations } from '../../src/database/migrations.ts'; +import { + ImageDeliveryRepository, + type NewImageDelivery, +} from '../../src/database/repositories/image-delivery.repository.ts'; +import { runMigrations } from '../../src/database/schema.ts'; +import type { TelegramSender } from '../../src/services/ai/types.ts'; +import { ImageDelivery } from '../../src/services/image/image-delivery.ts'; +import { configureTimeoutReporter, TimeoutReporter } from '../../src/services/incident/timeout-reporter.ts'; +import { botLogger } from '../../src/utils/logger.ts'; +import type { ImageRenderJob } from '../../src/worker/image-render.queue.ts'; +import { png } from '../fixtures/png.ts'; + +const flush = () => new Promise((resolve) => setImmediate(resolve)); + +const job = { type: 'daily-agenda', data: { date: '2026-11-02', theme: {}, locale: 'en' }, userId: 1 }; +const stored = (overrides: Partial = {}): NewImageDelivery => ({ + id: 'img-1', + chatId: 7, + language: 'en', + kind: 'day', + label: '2026-11-02', + jobType: 'daily-agenda', + slot: 'day:user:1:2026-11-02:en', + payloadHash: 'h', + jobJson: JSON.stringify(job), + createdAt: Date.now(), + expiresAt: Date.now() + 150_000, + ...overrides, +}); + +let db: Database; +let store: ImageDeliveryRepository; +let alerts: string[]; +let errorSpy: ReturnType; + +function sender() { + const sendMessage = mock(async (_chatId: number, _text: string) => ({ message_id: 1 })); + const sendPhoto = mock(async (_chatId: number, _photo: File) => ({ message_id: 77 })); + return { sendMessage, sendPhoto, telegram: { sendMessage, sendPhoto } as unknown as TelegramSender }; +} + +/** A fresh process: a renderer that has the finished picture ready, over the shared SQLite. */ +function process2() { + const renderDirect = mock(async (_job: ImageRenderJob) => png()); + return { delivery: new ImageDelivery({ renderer: { renderDirect }, store }), renderDirect }; +} + +beforeEach(() => { + db = new Database(':memory:'); + runMigrations(db, migrations); + store = new ImageDeliveryRepository(db); + alerts = []; + configureTimeoutReporter(new TimeoutReporter((text) => alerts.push(text))); + errorSpy = spyOn(botLogger, 'error').mockImplementation(() => {}); +}); + +afterEach(() => { + configureTimeoutReporter(null); + errorSpy.mockRestore(); + db.close(); +}); + +describe('recoverImageDeliveries', () => { + test('a picture the previous process left rendering is delivered as a photo to its own chat, once', async () => { + store.insert(stored()); + const { delivery, renderDirect } = process2(); + const tg = sender(); + + recoverImageDeliveries(delivery, tg.telegram); + await delivery.whenIdle(); + await flush(); + + expect(renderDirect).toHaveBeenCalledTimes(1); + expect(tg.sendPhoto).toHaveBeenCalledTimes(1); + const [chatId, file] = tg.sendPhoto.mock.calls[0]!; + expect(chatId).toBe(7); + expect(file.type).toBe('image/png'); + expect(tg.sendMessage).not.toHaveBeenCalled(); + expect(store.get('img-1')).toMatchObject({ state: 'delivered', message_id: 77 }); + + // A second start finds nothing left to do. + const again = process2(); + const tg2 = sender(); + recoverImageDeliveries(again.delivery, tg2.telegram); + await again.delivery.whenIdle(); + await flush(); + expect(tg2.sendPhoto).not.toHaveBeenCalled(); + }); + + test('a delivery past its bound earns one "the bot was restarting" note in its own chat, no timeout alert and no picture', async () => { + store.insert(stored({ createdAt: Date.now() - 210_000, expiresAt: Date.now() - 60_000 })); // a deploy restart + const { delivery, renderDirect } = process2(); + const tg = sender(); + + recoverImageDeliveries(delivery, tg.telegram); + await delivery.whenIdle(); + await flush(); + + expect(renderDirect).not.toHaveBeenCalled(); + expect(tg.sendPhoto).not.toHaveBeenCalled(); + expect(tg.sendMessage.mock.calls).toEqual([[7, t('en').aiTools.meta.imageNoteInterrupted('2026-11-02')]]); + expect(store.get('img-1')).toMatchObject({ state: 'expired', note_sent: 1 }); + expect(alerts).toEqual([]); + }); + + test('an upload that may have landed is marked unknown and never sent again, with no note', async () => { + store.insert(stored()); + store.advance('img-1', 'sending', Date.now()); + const { delivery, renderDirect } = process2(); + const tg = sender(); + + recoverImageDeliveries(delivery, tg.telegram); + await delivery.whenIdle(); + await flush(); + + expect(renderDirect).not.toHaveBeenCalled(); + expect(tg.sendPhoto).not.toHaveBeenCalled(); + expect(tg.sendMessage).not.toHaveBeenCalled(); + expect(store.get('img-1')?.state).toBe('unknown'); + }); + + test('a reconciliation that fails is logged and never thrown out of start-up', async () => { + const { delivery } = process2(); + store.unfinished = () => { + throw new Error('database is locked'); + }; + + expect(() => recoverImageDeliveries(delivery, sender().telegram)).not.toThrow(); + await flush(); + + expect(errorSpy).toHaveBeenCalledTimes(1); + expect(errorSpy.mock.calls[0]?.[1]).toBe('Image delivery reconciliation failed'); + }); +}); diff --git a/test/bootstrap/shutdown.test.ts b/test/bootstrap/shutdown.test.ts index b0f914f06..0c05af513 100644 --- a/test/bootstrap/shutdown.test.ts +++ b/test/bootstrap/shutdown.test.ts @@ -36,6 +36,7 @@ function harness(overrides: Partial = {}) { notificationQueue: closable('notification'), botTasksQueue: closable('botTasks'), syncQueue: closable('sync'), + imageDelivery: closable('imageDelivery'), imageQueue: closable('image'), broadcastQueue: closable('broadcast'), callQueue: closable('call'), @@ -74,6 +75,7 @@ describe('createShutdown', () => { 'notification', 'botTasks', 'sync', + 'imageDelivery', 'image', 'broadcast', 'call', diff --git a/test/bot/commands/agenda-enrichment.test.ts b/test/bot/commands/agenda-enrichment.test.ts index b3514c386..2b93016e2 100644 --- a/test/bot/commands/agenda-enrichment.test.ts +++ b/test/bot/commands/agenda-enrichment.test.ts @@ -10,6 +10,7 @@ import { InvitationRepository } from '../../../src/database/repositories/invitat import { UserRepository } from '../../../src/database/repositories/user.repository.ts'; import { runMigrations } from '../../../src/database/schema.ts'; import { EventService } from '../../../src/services/event/event-service.ts'; +import { makeImageDelivery } from '../../fixtures/image-delivery.ts'; import { png } from '../../fixtures/png.ts'; let db: Database; @@ -110,17 +111,16 @@ for (const command of ['today', 'tomorrow', 'week', 'search', 'month'] as const) return message; }, }); - const renderer = { - renderDirect: async (job: ImageRenderJob) => { - jobs.push(job); - return png(2160, 9000); - }, - }; - if (command === 'today') await handleToday(ctx, service, undefined, renderer); - if (command === 'tomorrow') await handleTomorrow(ctx, service, undefined, renderer); - if (command === 'week') await handleWeek(ctx, service, undefined, renderer); + const { delivery } = makeImageDelivery(async (job) => { + jobs.push(job); + return png(2160, 9000); + }); + if (command === 'today') await handleToday(ctx, service, undefined, delivery); + if (command === 'tomorrow') await handleTomorrow(ctx, service, undefined, delivery); + if (command === 'week') await handleWeek(ctx, service, undefined, delivery); if (command === 'search') await handleSearch(ctx, service); - if (command === 'month') await handleMonth(ctx, service, undefined, renderer); + if (command === 'month') await handleMonth(ctx, service, undefined, delivery); + await delivery.whenIdle(); // the picture is delivered in the background, after the command returned if (command !== 'month') { expect(sent.join('\n')).toContain('Alice <friend>: ⏳ pending'); expect(sent.join('\n')).toContain('Text only notes'); diff --git a/test/bot/commands/month-buckets.test.ts b/test/bot/commands/month-buckets.test.ts index 059701440..6ceea147f 100644 --- a/test/bot/commands/month-buckets.test.ts +++ b/test/bot/commands/month-buckets.test.ts @@ -2,6 +2,7 @@ // which local day an occurrence belongs to, including occurrences that cross midnight in the user's zone. import { describe, expect, mock, setSystemTime, test } from 'bun:test'; import { handleMonth } from '../../../src/bot/commands/month.ts'; +import { makeImageDelivery } from '../../fixtures/image-delivery.ts'; import { png } from '../../fixtures/png.ts'; type MonthArgs = Parameters; @@ -71,10 +72,14 @@ describe('handleMonth bucketing', () => { test('navigation callbacks never render an image', async () => { const ctx = makeCtx(); const renderDirect = mock(() => Promise.resolve(png())); + const { delivery } = makeImageDelivery(renderDirect); - await handleMonth(ctx as unknown as MonthArgs[0], makeEventService() as unknown as MonthArgs[1], '2026-06', { - renderDirect, - } as unknown as MonthArgs[3]); + await handleMonth( + ctx as unknown as MonthArgs[0], + makeEventService() as unknown as MonthArgs[1], + '2026-06', + delivery, + ); expect(renderDirect).not.toHaveBeenCalled(); expect(ctx.sendPhoto).not.toHaveBeenCalled(); @@ -86,10 +91,15 @@ describe('handleMonth bucketing', () => { try { const ctx = makeCtx(); const renderDirect = mock((_job: unknown) => Promise.resolve(png())); + const { delivery } = makeImageDelivery(renderDirect); - await handleMonth(ctx as unknown as MonthArgs[0], makeEventService() as unknown as MonthArgs[1], undefined, { - renderDirect, - } as unknown as MonthArgs[3]); + await handleMonth( + ctx as unknown as MonthArgs[0], + makeEventService() as unknown as MonthArgs[1], + undefined, + delivery, + ); + await delivery.whenIdle(); const job = renderDirect.mock.calls[0]![0] as { data: { weeks: { dayNumber: number; isOtherMonth: boolean; isToday: boolean; eventCount: number }[][] }; diff --git a/test/bot/commands/month.test.ts b/test/bot/commands/month.test.ts index aa4fa9b55..bb490fda2 100644 --- a/test/bot/commands/month.test.ts +++ b/test/bot/commands/month.test.ts @@ -1,7 +1,7 @@ import { describe, expect, mock, test } from 'bun:test'; import { handleMonth } from '../../../src/bot/commands/month.ts'; -import type { ImageRenderer } from '../../../src/services/image/render-service.ts'; -import { png } from '../../fixtures/png.ts'; +import { asCommandContext, asEventService } from '../../fixtures/doubles.ts'; +import { describePictureContract, gatedDelivery } from '../../fixtures/picture-contract.ts'; const user = { telegram_id: 100, language: 'en' as const, timezone: 'UTC' }; const userRu = { telegram_id: 100, language: 'ru' as const, timezone: 'UTC' }; @@ -21,20 +21,6 @@ function makeEventService(occurrences: Array<{ occurrence_start: string; occurre }; } -type MonthArgs = Parameters; - -/** Runs /month on partial doubles; the test-only cast stays in this one factory (CLAUDE.md). */ -async function runMonthWithRender(renderService: ImageRenderer) { - const ctx = makeCtx(); - await handleMonth( - ctx as unknown as MonthArgs[0], - makeEventService() as unknown as MonthArgs[1], - undefined, - renderService, - ); - return ctx; -} - describe('handleMonth', () => { test('sends calendar for current month when no yearMonth', async () => { const { handleMonth } = await import('../../../src/bot/commands/month.ts'); @@ -117,24 +103,29 @@ describe('handleMonth', () => { expect(opts.reply_markup).toBeDefined(); }); - test('pins image after successful month render', async () => { + test('the month picture is queued after the text, delivered later and pinned', async () => { const { handleMonth } = await import('../../../src/bot/commands/month.ts'); const pinChatMessage = mock(() => Promise.resolve()); const ctx = { + id: 7, dbUser: user, send: mock(() => Promise.resolve()), editText: mock(() => Promise.resolve()), sendPhoto: mock(() => Promise.resolve({ id: 55 })), + sendDocument: mock(() => Promise.resolve({ id: 56 })), bot: { api: { pinChatMessage, sendMessage: mock(() => Promise.resolve()) } }, }; const svc = makeEventService(); - const renderService = { renderDirect: mock(() => Promise.resolve(png())) }; + const { delivery, release } = gatedDelivery(); - // yearMonth undefined -> triggers image render (only on initial /month command) - await handleMonth(ctx as never, svc as never, undefined, renderService as never); - await Promise.resolve(); // flush fire-and-forget autoPin + // yearMonth undefined -> queues the picture (only on the initial /month command) + await handleMonth(asCommandContext(ctx), asEventService(svc), undefined, delivery); + expect(ctx.sendPhoto).not.toHaveBeenCalled(); + release(); + await delivery.whenIdle(); + await new Promise((resolve) => setImmediate(resolve)); // flush the fire-and-forget autoPin - expect(ctx.sendPhoto).toHaveBeenCalled(); + expect(ctx.sendPhoto).toHaveBeenCalledTimes(1); expect(pinChatMessage).toHaveBeenCalledWith({ chat_id: user.telegram_id, message_id: 55, @@ -142,13 +133,28 @@ describe('handleMonth', () => { }); }); - test('a month render failure does not blame the date range', async () => { - const renderService = { renderDirect: mock(() => Promise.reject(new Error('render failed'))) }; + test('month navigation does not queue a picture', async () => { + const { handleMonth } = await import('../../../src/bot/commands/month.ts'); + const ctx = makeCtx({ id: 7, sendPhoto: mock(() => Promise.resolve({ id: 55 })) }); + const { delivery, renderer } = gatedDelivery(); + + await handleMonth(asCommandContext(ctx), asEventService(makeEventService()), '2026-02', delivery); + + expect(renderer.renderDirect).not.toHaveBeenCalled(); + }); - const ctx = await runMonthWithRender(renderService); + test('a month render failure is told once by the delivery and does not blame the date range', async () => { + const ctx = makeCtx({ id: 7 }); + const { delivery, fail } = gatedDelivery(); + await handleMonth(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); + + fail(new Error('render failed')); + await delivery.whenIdle(); expect(ctx.send).toHaveBeenCalledTimes(2); - expect(ctx.send).toHaveBeenLastCalledWith('⚠️ Image generation failed. Use text version above.'); + const note = (ctx.send.mock.calls[1] as unknown[])[0] as string; + expect(note).toContain('could not prepare the picture'); + expect(note).not.toContain('date range'); }); }); @@ -202,3 +208,26 @@ describe('handleMonth group context', () => { expect(eventService.getEventsInRangeForGroup).not.toHaveBeenCalled(); }); }); + +describePictureContract('handleMonth', () => { + const texts: string[] = []; + let photos = 0; + return { + run: (delivery, requestId) => { + const ctx = makeCtx({ + id: requestId, + send: mock((text: string) => { + texts.push(text); + return Promise.resolve(); + }), + sendPhoto: mock(() => { + photos++; + return Promise.resolve({ id: 55 }); + }), + }); + return handleMonth(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); + }, + photos: () => photos, + texts: () => [...texts], + }; +}); diff --git a/test/bot/commands/today.test.ts b/test/bot/commands/today.test.ts index 006b8b277..5b6864caf 100644 --- a/test/bot/commands/today.test.ts +++ b/test/bot/commands/today.test.ts @@ -1,16 +1,22 @@ -import { describe, expect, mock, test } from 'bun:test'; +import { afterEach, describe, expect, mock, test } from 'bun:test'; import { handleToday } from '../../../src/bot/commands/today.ts'; -import type { ImageRenderer } from '../../../src/services/image/render-service.ts'; -import { png } from '../../fixtures/png.ts'; +import { t } from '../../../src/config/constants.ts'; +import { configureTimeoutReporter } from '../../../src/services/incident/timeout-reporter.ts'; +import { AgendaImageError } from '../../../src/utils/agenda-image.ts'; +import { asCommandContext, asEventService } from '../../fixtures/doubles.ts'; +import { captureTimeoutAlerts, gatedDelivery } from '../../fixtures/picture-contract.ts'; const user = { telegram_id: 100, language: 'en' as const, timezone: 'UTC' }; const userRu = { telegram_id: 100, language: 'ru' as const, timezone: 'UTC' }; function makeCtx(overrides = {}) { return { + id: 7, dbUser: user, send: mock(() => Promise.resolve()), - sendPhoto: mock(() => Promise.resolve()), + sendPhoto: mock(() => Promise.resolve({ id: 42 })), + sendDocument: mock(() => Promise.resolve({ id: 43 })), + bot: { api: { pinChatMessage: mock(() => Promise.resolve()), sendMessage: mock(() => Promise.resolve()) } }, ...overrides, }; } @@ -21,19 +27,10 @@ function makeEventService(occurrences = []) { }; } -type TodayArgs = Parameters; - -/** Runs /today on partial doubles; the test-only cast stays in this one factory (CLAUDE.md). */ -async function runTodayWithRender(renderService: ImageRenderer, overrides = {}) { - const ctx = makeCtx(overrides); - await handleToday( - ctx as unknown as TodayArgs[0], - makeEventService() as unknown as TodayArgs[1], - undefined, - renderService, - ); - return ctx; -} +/** Lets every already-settled promise continuation run (a macrotask turn, no wall-clock wait). */ +const flush = () => new Promise((resolve) => setImmediate(resolve)); + +afterEach(() => configureTimeoutReporter(null)); describe('handleToday', () => { test('sends day agenda with no events', async () => { @@ -61,7 +58,7 @@ describe('handleToday', () => { expect(text).toContain('Нет событий'); }); - test('does not call sendPhoto when no renderService', async () => { + test('does not call sendPhoto when there is no image delivery (text-only)', async () => { const { handleToday } = await import('../../../src/bot/commands/today.ts'); const ctx = makeCtx(); const svc = makeEventService(); @@ -69,56 +66,102 @@ describe('handleToday', () => { await handleToday(ctx as never, svc as never); expect(ctx.sendPhoto).not.toHaveBeenCalled(); + expect(ctx.send).toHaveBeenCalledTimes(1); }); - test('a render failure that is not about size does not blame the date range', async () => { - // The exact error production returned on 2026-10-12 when the render outlived the wait. - const renderService = { - renderDirect: mock(() => - Promise.reject( - new Error( - 'Job wait render timed out before finishing, no finish notification arrived after 15000ms (id=539)', - ), - ), - ), - }; + test('returns with the text before a slow render ends; the late picture is delivered once and pinned', async () => { + const { delivery, renderer, release } = gatedDelivery(); + const ctx = makeCtx(); - const ctx = await runTodayWithRender(renderService); + await handleToday(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); - expect(ctx.send).toHaveBeenCalledTimes(2); - expect(ctx.send).toHaveBeenLastCalledWith('⚠️ Image generation failed. Use text version above.'); + // The handler is done while the render is still pending: only the text went out. + expect(renderer.renderDirect).toHaveBeenCalledTimes(1); + expect(ctx.send).toHaveBeenCalledTimes(1); expect(ctx.sendPhoto).not.toHaveBeenCalled(); + + release(); + await delivery.whenIdle(); + await flush(); + + expect(ctx.sendPhoto).toHaveBeenCalledTimes(1); + expect(ctx.send).toHaveBeenCalledTimes(1); // no note: the picture arrived + expect(ctx.bot.api.pinChatMessage).toHaveBeenCalledWith({ + chat_id: user.telegram_id, + message_id: 42, + disable_notification: true, + }); }); - test('the render failure notice follows the user language', async () => { - const renderService = { renderDirect: mock(() => Promise.reject(new Error('render failed'))) }; + test('a double command while the first picture renders delivers one picture', async () => { + const { delivery, releaseAll } = gatedDelivery(); + const ctx = makeCtx(); + const again = makeCtx({ id: 8 }); - const ctx = await runTodayWithRender(renderService, { dbUser: userRu }); + await handleToday(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); + await handleToday(asCommandContext(again), asEventService(makeEventService()), undefined, delivery); + releaseAll(); + await delivery.whenIdle(); - expect(ctx.send).toHaveBeenLastCalledWith('⚠️ Не удалось сгенерировать изображение. Смотри текстовую версию выше.'); + expect(ctx.sendPhoto.mock.calls.length + again.sendPhoto.mock.calls.length).toBe(1); }); - test('pins image after successful render', async () => { - const { handleToday } = await import('../../../src/bot/commands/today.ts'); - const pinChatMessage = mock(() => Promise.resolve()); - const ctx = { - dbUser: user, - send: mock(() => Promise.resolve()), - sendPhoto: mock(() => Promise.resolve({ id: 42 })), - bot: { api: { pinChatMessage, sendMessage: mock(() => Promise.resolve()) } }, - }; - const svc = makeEventService(); - const renderService = { renderDirect: mock(() => Promise.resolve(png())) }; + test('a render failure gives one note and one owner alert, not the date-range wording', async () => { + const { delivery, fail, failures } = gatedDelivery(); + const ctx = makeCtx(); - await handleToday(ctx as never, svc as never, undefined, renderService as never); - await Promise.resolve(); // flush fire-and-forget autoPin + await handleToday(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); + fail(new Error('chromium crashed')); + await delivery.whenIdle(); - expect(ctx.sendPhoto).toHaveBeenCalled(); - expect(pinChatMessage).toHaveBeenCalledWith({ - chat_id: user.telegram_id, - message_id: 42, - disable_notification: true, - }); + expect(ctx.sendPhoto).not.toHaveBeenCalled(); + expect(ctx.send).toHaveBeenCalledTimes(2); // the text, then ONE note + const note = (ctx.send.mock.calls[1] as unknown[])[0] as string; + expect(note).toContain('could not prepare the picture'); + expect(note).not.toContain('date range'); + expect(failures).toHaveLength(1); + }); + + test('a render that outlives its bound gives one note and one timeout alert for the repair flow', async () => { + const alerts = captureTimeoutAlerts(); + const { delivery, failures } = gatedDelivery({ boundMs: 20 }); + const ctx = makeCtx(); + + await handleToday(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); + await delivery.whenIdle(); + + expect(ctx.send).toHaveBeenCalledTimes(2); + expect((ctx.send.mock.calls[1] as unknown[])[0]).toContain('did not get ready in time'); + expect(alerts).toHaveLength(1); + expect(alerts[0]?.text).toContain('Timeout (render)'); + expect(failures).toHaveLength(0); + }); + + test('a picture too large to send explains it once and raises no alert', async () => { + const { delivery, fail, failures } = gatedDelivery(); + const ctx = makeCtx(); + + await handleToday(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); + fail(new AgendaImageError('Agenda PNG exceeds the 50 MB document limit. Choose a shorter date range.', true)); + await delivery.whenIdle(); + + expect(ctx.send).toHaveBeenCalledTimes(2); + expect((ctx.send.mock.calls[1] as unknown[])[0]).toContain('too large to send'); + expect((ctx.send.mock.calls[1] as unknown[])[0]).not.toContain('Agenda PNG'); + expect(failures).toHaveLength(0); + }); + + test('the failure note follows the user language', async () => { + const { delivery, fail } = gatedDelivery(); + const ctx = makeCtx({ dbUser: userRu }); + + await handleToday(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); + fail(new Error('render failed')); + await delivery.whenIdle(); + + expect((ctx.send.mock.calls[1] as unknown[])[0]).toContain( + t('ru').aiTools.meta.imageNoteFailed(new Date().toISOString().slice(0, 10)), + ); }); test('shows holiday entries when holidayService returns them', async () => { diff --git a/test/bot/commands/tomorrow.test.ts b/test/bot/commands/tomorrow.test.ts index 0c153577a..af579f416 100644 --- a/test/bot/commands/tomorrow.test.ts +++ b/test/bot/commands/tomorrow.test.ts @@ -1,14 +1,18 @@ import { describe, expect, mock, test } from 'bun:test'; import { handleTomorrow } from '../../../src/bot/commands/tomorrow.ts'; +import { asCommandContext, asEventService } from '../../fixtures/doubles.ts'; +import { describePictureContract, gatedDelivery } from '../../fixtures/picture-contract.ts'; const user = { telegram_id: 100, language: 'en' as const, timezone: 'UTC' }; const userRu = { telegram_id: 100, language: 'ru' as const, timezone: 'UTC' }; function makeCtx(overrides = {}) { return { + id: 7, dbUser: user, send: mock(() => Promise.resolve()), - sendPhoto: mock(() => Promise.resolve()), + sendPhoto: mock(() => Promise.resolve({ id: 42 })), + sendDocument: mock(() => Promise.resolve({ id: 43 })), ...overrides, }; } @@ -45,7 +49,7 @@ describe('handleTomorrow', () => { expect(text).toContain('Нет событий'); }); - test('does not call sendPhoto when no renderService', async () => { + test('does not call sendPhoto when there is no image delivery (text-only)', async () => { const { handleTomorrow } = await import('../../../src/bot/commands/tomorrow.ts'); const ctx = makeCtx(); const svc = makeEventService(); @@ -55,15 +59,18 @@ describe('handleTomorrow', () => { expect(ctx.sendPhoto).not.toHaveBeenCalled(); }); - test('calls renderService when provided and catches errors', async () => { + test('a render failure is told once by the delivery, not by the command', async () => { const { handleTomorrow } = await import('../../../src/bot/commands/tomorrow.ts'); const ctx = makeCtx(); const svc = makeEventService(); - const renderService = { renderDirect: mock(() => Promise.reject(new Error('render failed'))) }; + const { delivery, fail } = gatedDelivery(); + + await handleTomorrow(asCommandContext(ctx), asEventService(svc), undefined, delivery); + fail(new Error('render failed')); + await delivery.whenIdle(); - await handleTomorrow(ctx as never, svc as never, undefined, renderService as never); expect(ctx.send).toHaveBeenCalledTimes(2); - expect(ctx.send).toHaveBeenLastCalledWith('⚠️ Image generation failed. Use text version above.'); + expect((ctx.send.mock.calls[1] as unknown[])[0]).toContain('could not prepare the picture'); }); test('shows holiday entries when holidayService returns them', async () => { @@ -179,3 +186,26 @@ describe('handleTomorrow group context', () => { expect(eventService.getEventsInRangeForGroup).not.toHaveBeenCalled(); }); }); + +describePictureContract('handleTomorrow', () => { + const texts: string[] = []; + let photos = 0; + return { + run: (delivery, requestId) => { + const ctx = makeCtx({ + id: requestId, + send: mock((text: string) => { + texts.push(text); + return Promise.resolve(); + }), + sendPhoto: mock(() => { + photos++; + return Promise.resolve({ id: 42 }); + }), + }); + return handleTomorrow(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); + }, + photos: () => photos, + texts: () => [...texts], + }; +}); diff --git a/test/bot/commands/week.test.ts b/test/bot/commands/week.test.ts index 9744c7525..910eb8cd5 100644 --- a/test/bot/commands/week.test.ts +++ b/test/bot/commands/week.test.ts @@ -1,15 +1,19 @@ import { describe, expect, mock, test } from 'bun:test'; import { TZDate } from '@date-fns/tz'; import { handleWeek } from '../../../src/bot/commands/week.ts'; +import { asCommandContext, asEventService } from '../../fixtures/doubles.ts'; +import { describePictureContract, gatedDelivery } from '../../fixtures/picture-contract.ts'; const user = { telegram_id: 100, language: 'en' as const, timezone: 'UTC' }; const userRu = { telegram_id: 100, language: 'ru' as const, timezone: 'UTC' }; function makeCtx(overrides = {}) { return { + id: 7, dbUser: user, send: mock(() => Promise.resolve()), - sendPhoto: mock(() => Promise.resolve()), + sendPhoto: mock(() => Promise.resolve({ id: 42 })), + sendDocument: mock(() => Promise.resolve({ id: 43 })), ...overrides, }; } @@ -46,7 +50,7 @@ describe('handleWeek', () => { expect(text).toContain('нет событий'); }); - test('does not call sendPhoto when no renderService', async () => { + test('does not call sendPhoto when there is no image delivery (text-only)', async () => { const { handleWeek } = await import('../../../src/bot/commands/week.ts'); const ctx = makeCtx(); const svc = makeEventService(); @@ -56,16 +60,18 @@ describe('handleWeek', () => { expect(ctx.sendPhoto).not.toHaveBeenCalled(); }); - test('calls renderService when provided and catches errors', async () => { + test('a render failure is told once by the delivery, not by the command', async () => { const { handleWeek } = await import('../../../src/bot/commands/week.ts'); const ctx = makeCtx(); const svc = makeEventService(); - const renderService = { renderDirect: mock(() => Promise.reject(new Error('render failed'))) }; + const { delivery, fail } = gatedDelivery(); - await handleWeek(ctx as never, svc as never, undefined, renderService as never); + await handleWeek(asCommandContext(ctx), asEventService(svc), undefined, delivery); + fail(new Error('render failed')); + await delivery.whenIdle(); expect(ctx.send).toHaveBeenCalledTimes(2); - expect(ctx.send).toHaveBeenLastCalledWith('⚠️ Image generation failed. Use text version above.'); + expect((ctx.send.mock.calls[1] as unknown[])[0]).toContain('could not prepare the picture'); }); test('queries holidayService for each of 7 days', async () => { @@ -258,3 +264,26 @@ describe('handleWeek group context', () => { expect(text).toContain('-5..1°'); }); }); + +describePictureContract('handleWeek', () => { + const texts: string[] = []; + let photos = 0; + return { + run: (delivery, requestId) => { + const ctx = makeCtx({ + id: requestId, + send: mock((text: string) => { + texts.push(text); + return Promise.resolve(); + }), + sendPhoto: mock(() => { + photos++; + return Promise.resolve({ id: 42 }); + }), + }); + return handleWeek(asCommandContext(ctx), asEventService(makeEventService()), undefined, delivery); + }, + photos: () => photos, + texts: () => [...texts], + }; +}); diff --git a/test/bot/handlers/agenda269-callback.test.ts b/test/bot/handlers/agenda269-callback.test.ts index f0ce6c8af..0d97806f7 100644 --- a/test/bot/handlers/agenda269-callback.test.ts +++ b/test/bot/handlers/agenda269-callback.test.ts @@ -2,7 +2,7 @@ import { Database } from 'bun:sqlite'; import { afterEach, beforeEach, expect, setSystemTime, test } from 'bun:test'; import { Scene } from '@gramio/scenes'; -import { Bot, CallbackQueryContext } from 'gramio'; +import { Bot, CallbackQueryContext, TelegramError } from 'gramio'; import { createCallbackHandler } from '../../../src/bot/handlers/callback.handler.ts'; import { CB } from '../../../src/config/constants.ts'; import { migrations } from '../../../src/database/migrations.ts'; @@ -17,6 +17,7 @@ import { EventService } from '../../../src/services/event/event-service.ts'; import { HolidayService } from '../../../src/services/holiday/holiday-service.ts'; import { NotificationPreferencesService } from '../../../src/services/notification/preferences.ts'; import type { ImageRenderJob } from '../../../src/worker/image-render.queue.ts'; +import { makeImageDelivery } from '../../fixtures/image-delivery.ts'; import { png } from '../../fixtures/png.ts'; let db: Database; @@ -73,6 +74,10 @@ function setup(data: string, chatId = -10, chatType: 'private' | 'supergroup' = }), { dbUser: user, userTimezone: user.timezone, lang: 'en' as const, scene: { enter: async () => {} } }, ); + const { delivery } = makeImageDelivery(async (job) => { + jobs.push(job); + return image; + }); const handler = createCallbackHandler( service, new Scene('unused'), @@ -87,20 +92,16 @@ function setup(data: string, chatId = -10, chatType: 'private' | 'supergroup' = ), eventRepo: new EventRepository(db), groupRepo: groups, - renderService: { - renderDirect: async (job) => { - jobs.push(job); - return image; - }, - }, + imageDelivery: delivery, }, ); - return { handler, ctx, jobs, groups, service, bot, user }; + return { handler, ctx, jobs, groups, service, bot, user, delivery }; } for (const prefix of [CB.IMG_DAILY, CB.IMG_WEEKLY]) { test(`${prefix} selects group calendar and timezone before enrichment`, async () => { - const { handler, ctx, jobs } = setup(`${prefix}:2099-06-01`); + const { handler, ctx, jobs, delivery } = setup(`${prefix}:2099-06-01`); await handler(ctx); + await delivery.whenIdle(); expect(jobs).toHaveLength(1); const payload = JSON.stringify(jobs[0]); expect(payload).toContain('GROUP MEETING'); @@ -319,7 +320,7 @@ test('share delivery failure never reports sent below', async () => { for (const prefix of [CB.IMG_DAILY, CB.IMG_WEEKLY]) { test(`${prefix} actual callback sends oversized image as document and reports rejection`, async () => { - const { handler, ctx, bot } = setup(`${prefix}:2099-06-01`, -10, 'supergroup', png(2160, 9000)); + const { handler, ctx, bot, delivery } = setup(`${prefix}:2099-06-01`, -10, 'supergroup', png(2160, 9000)); let documents = 0; const errors: string[] = []; bot.api.sendPhoto = async () => { @@ -332,15 +333,21 @@ for (const prefix of [CB.IMG_DAILY, CB.IMG_WEEKLY]) { return { message_id: 42, date: 0, chat: { id: -10, type: 'supergroup' } }; }; await handler(ctx); + await delivery.whenIdle(); expect(documents).toBe(1); bot.api.sendDocument = async () => { - throw new Error('Rejected'); + // Telegram's own answer: the document was not accepted, so the user is told (a dropped connection would not be). + throw new TelegramError({ ok: false, error_code: 400, description: 'Bad Request: rejected' }, 'sendDocument', { + chat_id: 1, + document: 'document', + }); }; bot.api.sendMessage = async (params) => { errors.push(String(params.text)); return { message_id: 43, date: 0, chat: { id: -10, type: 'supergroup' } }; }; await handler(ctx); + await delivery.whenIdle(); expect(errors.length).toBeGreaterThan(0); }); } diff --git a/test/bot/handlers/callback-invitation.test.ts b/test/bot/handlers/callback-invitation.test.ts index ef72dd67e..03ee682bc 100644 --- a/test/bot/handlers/callback-invitation.test.ts +++ b/test/bot/handlers/callback-invitation.test.ts @@ -1,20 +1,29 @@ import { Database } from 'bun:sqlite'; import { describe, expect, mock, test } from 'bun:test'; +import type { AnyScene } from '@gramio/scenes'; import type { InlineKeyboard } from 'gramio'; +import type { CallbackHandlerOpts } from '../../../src/bot/handlers/callback/types.ts'; import { createCallbackHandler } from '../../../src/bot/handlers/callback.handler'; import { t } from '../../../src/config/constants.ts'; import { migrations } from '../../../src/database/migrations.ts'; import { EventRepository } from '../../../src/database/repositories/event.repository.ts'; +import { ImageDeliveryRepository } from '../../../src/database/repositories/image-delivery.repository.ts'; import { InvitationRepository } from '../../../src/database/repositories/invitation.repository.ts'; import { ParticipantRepository } from '../../../src/database/repositories/participant.repository.ts'; import { SharingSettingsRepository } from '../../../src/database/repositories/sharing-settings.repository.ts'; import { UserRepository } from '../../../src/database/repositories/user.repository.ts'; import { runMigrations } from '../../../src/database/schema.ts'; import { EventService } from '../../../src/services/event/event-service.ts'; +import type { HolidayService } from '../../../src/services/holiday/holiday-service.ts'; +import type { ImageDelivery } from '../../../src/services/image/image-delivery.ts'; +import type { NotificationPreferencesService } from '../../../src/services/notification/preferences.ts'; import { DomainEventBus } from '../../../src/services/scheduled/domain-event-bus.ts'; import { InvitationCardRefresher, type InvitationEditOptions } from '../../../src/services/sharing/invitation-cards.ts'; import { invitationRsvpKeyboard } from '../../../src/services/sharing/invitation-rsvp-keyboard.ts'; import { InvitationService } from '../../../src/services/sharing/invitation-service.ts'; +import { asCallbackContext, asDouble } from '../../fixtures/doubles.ts'; +import { makeImageDelivery } from '../../fixtures/image-delivery.ts'; +import { describePictureContract, gatedDelivery } from '../../fixtures/picture-contract.ts'; import { png } from '../../fixtures/png.ts'; import { makeCallbackHandler, makeCallbackTap } from '../../helpers/callback-handler.ts'; import { flushPromises } from '../../helpers/mock-context.ts'; @@ -599,99 +608,86 @@ describe('propose-time callbacks', () => { }); describe('conflict image on accept', () => { - function makeHandlerWithRender( - invSvc: unknown, - notifyDeps: { - userRepo: { findByTelegramId: ReturnType }; - sendMessage: ReturnType; - sendPhoto?: ReturnType; - }, - eventRepo: { - findById: ReturnType; - findVisibleOverlapping?: ReturnType; - isParticipant?: ReturnType; - }, - renderService: { renderDirect: ReturnType }, - ) { - return createCallbackHandler({} as never, {} as never, {} as never, {} as never, { - renderService: renderService as never, - invitationService: invSvc as never, - eventRepo: eventRepo as never, - invitationNotifyDeps: notifyDeps as never, + const inv = { id: 1, status: 'accepted', event_id: 5, inviter_id: 100, invitee_id: 200 }; + const event = { id: 5, title: 'Party', start_at: '2026-03-20T10:00:00Z', end_at: '2026-03-20T11:00:00Z' }; + const inviter = { telegram_id: 100, first_name: 'Alice', language: 'en', username: null, timezone: 'UTC' }; + + /** An accept tap with the inviter's Telegram transport recorded: texts to the inviter and pictures. */ + function acceptHarness(options: { delivery?: ImageDelivery; inviteeEvents?: unknown[] } = {}) { + const texts: string[] = []; + const photoChats: number[] = []; + const sendMessage = mock((_chatId: number, text: string) => { + texts.push(text); + return Promise.resolve(); + }); + const sendPhoto = mock((chatId: number) => { + photoChats.push(chatId); + return Promise.resolve({ message_id: 9 }); }); - } - - test('sends conflict image to inviter on accept when renderService provided', async () => { - const sendPhoto = mock(() => Promise.resolve()); - const renderDirect = mock(() => Promise.resolve(png())); - const sendMessage = mock(() => Promise.resolve()); - const inv = { id: 1, status: 'accepted', event_id: 5, inviter_id: 100, invitee_id: 200 }; - const event = { id: 5, title: 'Party', start_at: '2026-03-20T10:00:00Z', end_at: '2026-03-20T11:00:00Z' }; - const invSvc = { acceptInvitation: mock(() => ({ success: true, invitation: inv })) }; const userRepo = { - findByTelegramId: mock(() => ({ - telegram_id: 100, - first_name: 'Alice', - language: 'en', - username: null, - timezone: 'UTC', - })), + findByTelegramId: mock((id: number) => + id === 100 + ? inviter + : { telegram_id: 200, first_name: 'Bob', language: 'en', username: 'bob', timezone: 'UTC' }, + ), }; const eventRepo = { findById: mock(() => event), - findVisibleOverlapping: mock(() => []), + // organizer call returns [] (no own overlapping), invitee call returns the invitee's events + findVisibleOverlapping: mock((userId: number) => (userId === 200 ? (options.inviteeEvents ?? []) : [])), isParticipant: mock(() => false), }; - const renderService = { renderDirect }; + const invSvc = { acceptInvitation: mock(() => ({ success: true, invitation: inv })) }; const notifyDeps = { userRepo, sendMessage, sendPhoto }; - const ctx = { - data: 'inv:accept:1', - dbUser: { telegram_id: 200, language: 'en', timezone: 'UTC', first_name: 'Bob', username: null }, - answer: mock(() => Promise.resolve()), - editText: mock(() => Promise.resolve()), - }; - const handler = makeHandlerWithRender(invSvc, notifyDeps, eventRepo, renderService); - await handler(ctx as never); - await flushPromises(); - expect(renderDirect).toHaveBeenCalledTimes(1); - expect(sendPhoto).toHaveBeenCalledTimes(1); - const photoCall = sendPhoto.mock.calls[0] as unknown[]; - expect(photoCall[0]).toBe(100); + const handler = createCallbackHandler( + asDouble({}), + asDouble({}), + asDouble({}), + asDouble({}), + { + imageDelivery: options.delivery, + invitationService: asDouble(invSvc), + eventRepo: asDouble(eventRepo), + invitationNotifyDeps: asDouble(notifyDeps), + }, + ); + const tap = async (callbackId = 'cb-1') => { + const ctx = { + id: callbackId, + data: 'inv:accept:1', + dbUser: { telegram_id: 200, language: 'en', timezone: 'UTC', first_name: 'Bob', username: 'bob' }, + answer: mock(() => Promise.resolve()), + editText: mock(() => Promise.resolve()), + }; + await handler(asCallbackContext(ctx)); + await flushPromises(); // notifyInviter is fire-and-forget + return ctx; + }; + return { tap, texts, photoChats, sendPhoto }; + } + + test('queues the conflict picture on accept and sends it to the inviter when the render ends', async () => { + const { delivery, renderer, release } = gatedDelivery(); + const h = acceptHarness({ delivery }); + + await h.tap(); + + expect(renderer.renderDirect).toHaveBeenCalledTimes(1); + expect(h.sendPhoto).not.toHaveBeenCalled(); // the tap finished while the render is pending + release(); + await delivery.whenIdle(); + expect(h.photoChats).toEqual([100]); }); test('invitee event titles are null for non-shared events in conflict image', async () => { - const sendPhoto = mock(() => Promise.resolve()); - const renderDirect = mock(() => Promise.resolve(png())); - const sendMessage = mock(() => Promise.resolve()); - const inv = { id: 1, status: 'accepted', event_id: 5, inviter_id: 100, invitee_id: 200 }; - const event = { id: 5, title: 'Party', start_at: '2026-03-20T10:00:00Z', end_at: '2026-03-20T11:00:00Z' }; const inviteeEvent = { id: 99, title: 'Secret', start_at: '2026-03-20T09:00:00Z', end_at: '2026-03-20T10:30:00Z' }; - const invSvc = { acceptInvitation: mock(() => ({ success: true, invitation: inv })) }; - const userRepo = { - findByTelegramId: mock((id: number) => - id === 100 - ? { telegram_id: 100, first_name: 'Alice', language: 'en', username: null, timezone: 'UTC' } - : { telegram_id: 200, first_name: 'Bob', language: 'en', username: 'bob', timezone: 'UTC' }, - ), - }; + const renderDirect = mock(() => Promise.resolve(png())); + const { delivery } = makeImageDelivery(renderDirect); // isParticipant returns false → organizer is NOT participant of invitee's event → title must be null - const eventRepo = { - findById: mock(() => event), - // organizer call returns [] (no own overlapping), invitee call returns [inviteeEvent] - findVisibleOverlapping: mock((userId: number) => (userId === 200 ? [inviteeEvent] : [])), - isParticipant: mock(() => false), - }; - const renderService = { renderDirect }; - const notifyDeps = { userRepo, sendMessage, sendPhoto }; - const ctx = { - data: 'inv:accept:1', - dbUser: { telegram_id: 200, language: 'en', timezone: 'UTC', first_name: 'Bob', username: 'bob' }, - answer: mock(() => Promise.resolve()), - editText: mock(() => Promise.resolve()), - }; - const handler = makeHandlerWithRender(invSvc, notifyDeps, eventRepo, renderService); - await handler(ctx as never); - await flushPromises(); + const h = acceptHarness({ delivery, inviteeEvents: [inviteeEvent] }); + + await h.tap(); + await delivery.whenIdle(); expect(renderDirect).toHaveBeenCalledTimes(1); const renderCall = renderDirect.mock.calls[0] as unknown[]; @@ -704,41 +700,60 @@ describe('conflict image on accept', () => { } }); - test('does not render when renderService missing', async () => { - const sendPhoto = mock(() => Promise.resolve()); - const sendMessage = mock(() => Promise.resolve()); - const inv = { id: 1, status: 'accepted', event_id: 5, inviter_id: 100, invitee_id: 200 }; - const event = { id: 5, title: 'Party', start_at: '2026-03-20T10:00:00Z', end_at: '2026-03-20T11:00:00Z' }; - const invSvc = { acceptInvitation: mock(() => ({ success: true, invitation: inv })) }; - const userRepo = { - findByTelegramId: mock(() => ({ - telegram_id: 100, - first_name: 'Alice', - language: 'en', - username: null, - timezone: 'UTC', - })), - }; - const eventRepo = { - findById: mock(() => event), - findVisibleOverlapping: mock(() => []), - isParticipant: mock(() => false), - }; - const notifyDeps = { userRepo, sendMessage, sendPhoto }; - const ctx = { - data: 'inv:accept:1', - dbUser: { telegram_id: 200, language: 'en', timezone: 'UTC', first_name: 'Bob', username: null }, - answer: mock(() => Promise.resolve()), - editText: mock(() => Promise.resolve()), + test('does not render when image delivery is missing', async () => { + const h = acceptHarness(); + + await h.tap(); + + expect(h.sendPhoto).not.toHaveBeenCalled(); + }); + + test('a failed render tells the inviter once with the schedule comparison named', async () => { + const { delivery, fail, failures } = gatedDelivery(); + const h = acceptHarness({ delivery }); + + await h.tap(); + const answered = h.texts.length; + fail(new Error('chromium crashed')); + await delivery.whenIdle(); + + expect(h.texts).toHaveLength(answered + 1); + expect(h.texts.at(-1)).toContain('schedule comparison'); + expect(failures).toHaveLength(1); + }); + + test('answering again after a failed picture queues a new one; a redelivered tap does not', async () => { + const db = new Database(':memory:'); + runMigrations(db, migrations); + const { delivery, fail, release, failures } = gatedDelivery({ store: new ImageDeliveryRepository(db) }); + const h = acceptHarness({ delivery }); + + await h.tap('cb-first'); + fail(new Error('chromium crashed')); + await delivery.whenIdle(); + expect(failures).toHaveLength(1); + expect(h.photoChats).toEqual([]); + + await h.tap('cb-second'); // the invitee answers again: a new request, so a new picture + release(1); + await delivery.whenIdle(); + expect(h.photoChats).toEqual([100]); + + await h.tap('cb-second'); // Telegram redelivers that tap: the same request, already delivered + await delivery.whenIdle(); + expect(h.photoChats).toEqual([100]); + }); + + describePictureContract('accepted invitation conflict picture', () => { + let h: ReturnType | undefined; + return { + run: async (delivery, requestId) => { + h ??= acceptHarness({ delivery }); + await h.tap(`cb-${requestId}`); + }, + photos: () => h?.photoChats.length ?? 0, + texts: () => [...(h?.texts ?? [])], }; - const handler = createCallbackHandler({} as never, {} as never, {} as never, {} as never, { - invitationService: invSvc as never, - eventRepo: eventRepo as never, - invitationNotifyDeps: notifyDeps as never, - }); - await handler(ctx as never); - await flushPromises(); - expect(sendPhoto).not.toHaveBeenCalled(); }); }); diff --git a/test/bot/handlers/callback-routes.test.ts b/test/bot/handlers/callback-routes.test.ts index 7f8138ade..4b601c6bc 100644 --- a/test/bot/handlers/callback-routes.test.ts +++ b/test/bot/handlers/callback-routes.test.ts @@ -1,9 +1,12 @@ import { describe, expect, mock, test } from 'bun:test'; import { createCallbackHandler } from '../../../src/bot/handlers/callback.handler'; -import { png } from '../../fixtures/png.ts'; +import type { ImageDelivery } from '../../../src/services/image/image-delivery.ts'; +import { asCallbackContext } from '../../fixtures/doubles.ts'; +import { describePictureContract, gatedDelivery } from '../../fixtures/picture-contract.ts'; function makeCtx(data: string, language = 'en', extras: { chat?: unknown; message?: unknown } = {}) { return { + id: 'cb-1', data, dbUser: { telegram_id: 100, language, timezone: 'UTC' }, chat: extras.chat, @@ -11,6 +14,7 @@ function makeCtx(data: string, language = 'en', extras: { chat?: unknown; messag answer: mock(() => Promise.resolve()), editText: mock(() => Promise.resolve()), send: mock(() => Promise.resolve()), + bot: { api: { pinChatMessage: mock(() => Promise.resolve()), sendMessage: mock(() => Promise.resolve()) } }, }; } @@ -36,7 +40,7 @@ function makeHandler( calendarRepo?: unknown; disconnectDeps?: unknown; onCalendarsDone?: unknown; - renderService?: unknown; + imageDelivery?: ImageDelivery; invitationService?: unknown; eventRepo?: unknown; } = {}, @@ -50,7 +54,7 @@ function makeHandler( calendarRepo: overrides.calendarRepo as never, disconnectDeps: overrides.disconnectDeps as never, onCalendarsDone: overrides.onCalendarsDone as never, - renderService: overrides.renderService as never, + imageDelivery: overrides.imageDelivery, invitationService: overrides.invitationService as never, eventRepo: overrides.eventRepo as never, }, @@ -342,138 +346,179 @@ describe('GCAL callback', () => { // ── IMG_DAILY ── +/** A tapped message whose sends are recorded: photos, documents and texts (notes). */ +function makeTappedMessage() { + const texts: string[] = []; + let photos = 0; + return { + message: { + chat: { id: 100, type: 'private' }, + sendPhoto: mock(() => { + photos++; + return Promise.resolve({ id: 5 }); + }), + sendDocument: mock(() => { + photos++; + return Promise.resolve({ id: 6 }); + }), + send: mock((text: string) => { + texts.push(text); + return Promise.resolve(); + }), + }, + photos: () => photos, + texts: () => [...texts], + }; +} + describe('IMG_DAILY callback', () => { - test('renders daily image and sends photo', async () => { - const sendPhoto = mock(() => Promise.resolve()); - const eventService = makeEventService({ - getEventsForDay: mock(() => []), - }); - const holidayService = { - getHolidaysForDate: mock(() => []), - }; - const renderService = { - renderDirect: mock(() => Promise.resolve(png())), - }; - const ctx = makeCtx('imd:2026-03-15', 'en', { - message: { chat: { id: 100, type: 'private' }, sendPhoto, send: mock(() => Promise.resolve()) }, - }); - const handler = makeHandler({ eventService, holidayService, renderService }); + const holidayService = { getHolidaysForDate: mock(() => []) }; + + test('answers the tap at once, queues the picture and delivers it when the render ends', async () => { + const eventService = makeEventService({ getEventsForDay: mock(() => []) }); + const { delivery, renderer, release } = gatedDelivery(); + const tapped = makeTappedMessage(); + const ctx = makeCtx('imd:2026-03-15', 'en', { message: tapped.message }); + const handler = makeHandler({ eventService, holidayService, imageDelivery: delivery }); + await handler(ctx as never); - expect(ctx.answer).toHaveBeenCalled(); - expect(renderService.renderDirect).toHaveBeenCalled(); - expect(sendPhoto).toHaveBeenCalled(); + + expect(ctx.answer).toHaveBeenCalledTimes(1); + expect(renderer.renderDirect).toHaveBeenCalledTimes(1); + expect(tapped.photos()).toBe(0); // the handler returned while the render is pending + + release(); + await delivery.whenIdle(); + + expect(tapped.photos()).toBe(1); + expect(tapped.texts()).toEqual([]); }); - test('render failure sends error message', async () => { - const sendFn = mock(() => Promise.resolve()); - const eventService = makeEventService({ - getEventsForDay: mock(() => []), - }); - const holidayService = { - getHolidaysForDate: mock(() => []), - }; - const renderService = { - renderDirect: mock(() => Promise.reject(new Error('render boom'))), - }; - const ctx = makeCtx('imd:2026-03-15', 'en', { - message: { chat: { id: 100, type: 'private' }, sendPhoto: mock(() => Promise.resolve()), send: sendFn }, - }); - const handler = makeHandler({ eventService, holidayService, renderService }); + test('a render failure sends one note naming the day', async () => { + const eventService = makeEventService({ getEventsForDay: mock(() => []) }); + const { delivery, fail, failures } = gatedDelivery(); + const tapped = makeTappedMessage(); + const ctx = makeCtx('imd:2026-03-15', 'en', { message: tapped.message }); + const handler = makeHandler({ eventService, holidayService, imageDelivery: delivery }); + await handler(ctx as never); - expect(sendFn).toHaveBeenCalled(); - const msg = (sendFn.mock.calls[0] as unknown[])[0] as string; - expect(msg).toContain('failed'); + fail(new Error('render boom')); + await delivery.whenIdle(); + + expect(tapped.texts()).toHaveLength(1); + expect(tapped.texts()[0]).toContain('2026-03-15'); + expect(failures).toHaveLength(1); }); test('missing message context fails closed before event selection or rendering', async () => { - const eventService = makeEventService({ - getEventsForDay: mock(() => []), - }); - const holidayService = { - getHolidaysForDate: mock(() => []), - }; - const renderService = { - renderDirect: mock(() => Promise.reject(new Error('boom'))), - }; + const eventService = makeEventService({ getEventsForDay: mock(() => []) }); + const { delivery, renderer } = gatedDelivery(); const ctx = makeCtx('imd:2026-03-15', 'en'); - const handler = makeHandler({ eventService, holidayService, renderService }); + const handler = makeHandler({ eventService, holidayService, imageDelivery: delivery }); await handler(ctx as never); expect(ctx.answer).toHaveBeenCalledTimes(1); expect(eventService.getEventsForDay).not.toHaveBeenCalled(); expect(eventService.getEventsInRange).not.toHaveBeenCalled(); - expect(renderService.renderDirect).not.toHaveBeenCalled(); + expect(renderer.renderDirect).not.toHaveBeenCalled(); }); - test('no renderService skips IMG_DAILY', async () => { + test('no image delivery skips IMG_DAILY', async () => { const ctx = makeCtx('imd:2026-03-15'); const handler = makeHandler(); await handler(ctx as never); // Falls through to unknown action warning + answer expect(ctx.answer).toHaveBeenCalled(); }); + + describePictureContract('IMG_DAILY', () => { + const tapped = makeTappedMessage(); + return { + run: (delivery, requestId) => { + const ctx = makeCtx('imd:2026-03-15', 'en', { message: tapped.message }); + const handler = makeHandler({ + eventService: makeEventService({ getEventsForDay: mock(() => []) }), + holidayService, + imageDelivery: delivery, + }); + return handler(asCallbackContext({ ...ctx, id: `cb-${requestId}` })); + }, + photos: tapped.photos, + texts: tapped.texts, + }; + }); }); // ── IMG_WEEKLY ── describe('IMG_WEEKLY callback', () => { - test('renders weekly image and sends photo', async () => { - const sendPhoto = mock(() => Promise.resolve()); - const eventService = makeEventService({ - getEventsInRange: mock(() => []), - }); - const renderService = { - renderDirect: mock(() => Promise.resolve(png())), - }; - const ctx = makeCtx('imw:2026-03-09', 'en', { - message: { chat: { id: 100, type: 'private' }, sendPhoto, send: mock(() => Promise.resolve()) }, - }); - const handler = makeHandler({ eventService, renderService }); + test('answers the tap at once, queues the picture and delivers it when the render ends', async () => { + const eventService = makeEventService({ getEventsInRange: mock(() => []) }); + const { delivery, renderer, release } = gatedDelivery(); + const tapped = makeTappedMessage(); + const ctx = makeCtx('imw:2026-03-09', 'en', { message: tapped.message }); + const handler = makeHandler({ eventService, imageDelivery: delivery }); + await handler(ctx as never); - expect(ctx.answer).toHaveBeenCalled(); - expect(renderService.renderDirect).toHaveBeenCalled(); - expect(sendPhoto).toHaveBeenCalled(); + + expect(ctx.answer).toHaveBeenCalledTimes(1); + expect(renderer.renderDirect).toHaveBeenCalledTimes(1); + expect(tapped.photos()).toBe(0); + + release(); + await delivery.whenIdle(); + + expect(tapped.photos()).toBe(1); }); - test('render failure sends error message', async () => { - const sendFn = mock(() => Promise.resolve()); - const eventService = makeEventService({ - getEventsInRange: mock(() => []), - }); - const renderService = { - renderDirect: mock(() => Promise.reject(new Error('render boom'))), - }; - const ctx = makeCtx('imw:2026-03-09', 'en', { - message: { chat: { id: 100, type: 'private' }, sendPhoto: mock(() => Promise.resolve()), send: sendFn }, - }); - const handler = makeHandler({ eventService, renderService }); + test('a render failure sends one note', async () => { + const eventService = makeEventService({ getEventsInRange: mock(() => []) }); + const { delivery, fail } = gatedDelivery(); + const tapped = makeTappedMessage(); + const ctx = makeCtx('imw:2026-03-09', 'en', { message: tapped.message }); + const handler = makeHandler({ eventService, imageDelivery: delivery }); + await handler(ctx as never); - expect(sendFn).toHaveBeenCalled(); - const msg = (sendFn.mock.calls[0] as unknown[])[0] as string; - expect(msg).toContain('failed'); + fail(new Error('render boom')); + await delivery.whenIdle(); + + expect(tapped.texts()).toHaveLength(1); + expect(tapped.texts()[0]).toContain('could not prepare the picture'); }); test('missing message context fails closed before event selection or rendering', async () => { - const eventService = makeEventService({ - getEventsInRange: mock(() => []), - }); - const renderService = { - renderDirect: mock(() => Promise.reject(new Error('boom'))), - }; + const eventService = makeEventService({ getEventsInRange: mock(() => []) }); + const { delivery, renderer } = gatedDelivery(); const ctx = makeCtx('imw:2026-03-09', 'en'); - const handler = makeHandler({ eventService, renderService }); + const handler = makeHandler({ eventService, imageDelivery: delivery }); await handler(ctx as never); expect(ctx.answer).toHaveBeenCalledTimes(1); expect(eventService.getEventsForDay).not.toHaveBeenCalled(); expect(eventService.getEventsInRange).not.toHaveBeenCalled(); - expect(renderService.renderDirect).not.toHaveBeenCalled(); + expect(renderer.renderDirect).not.toHaveBeenCalled(); }); - test('no renderService skips IMG_WEEKLY', async () => { + test('no image delivery skips IMG_WEEKLY', async () => { const ctx = makeCtx('imw:2026-03-09'); const handler = makeHandler(); await handler(ctx as never); expect(ctx.answer).toHaveBeenCalled(); }); + + describePictureContract('IMG_WEEKLY', () => { + const tapped = makeTappedMessage(); + return { + run: (delivery, requestId) => { + const ctx = makeCtx('imw:2026-03-09', 'en', { message: tapped.message }); + const handler = makeHandler({ + eventService: makeEventService({ getEventsInRange: mock(() => []) }), + imageDelivery: delivery, + }); + return handler(asCallbackContext({ ...ctx, id: `cb-${requestId}` })); + }, + photos: tapped.photos, + texts: tapped.texts, + }; + }); }); // ── Unknown action ── diff --git a/test/database/migrations-070.test.ts b/test/database/migrations-070.test.ts new file mode 100644 index 000000000..7481cb7d5 --- /dev/null +++ b/test/database/migrations-070.test.ts @@ -0,0 +1,175 @@ +import { Database } from 'bun:sqlite'; +import { describe, expect, test } from 'bun:test'; +import { migrations } from '../../src/database/migrations.ts'; +import { + ImageDeliveryRepository, + type NewImageDelivery, +} from '../../src/database/repositories/image-delivery.repository.ts'; +import { runMigrations } from '../../src/database/schema.ts'; + +const base: NewImageDelivery = { + id: 'img-1', + chatId: 5, + language: 'en', + kind: 'day', + label: '2026-11-02', + jobType: 'daily-agenda', + slot: 'day:user:5:2026-11-02:en', + payloadHash: '123', + jobJson: '{}', + createdAt: 1_000, + expiresAt: 151_000, +}; + +function repo() { + const db = new Database(':memory:'); + runMigrations(db, migrations); + return { db, repo: new ImageDeliveryRepository(db) }; +} + +describe('070_image_deliveries', () => { + test('creates the table with its state constraint', () => { + const { db } = repo(); + expect(migrations.map((m) => m.name)).toContain('070_image_deliveries'); + db.run( + "INSERT INTO image_deliveries (id, chat_id, language, kind, label, job_type, slot, payload_hash, state, created_at, expires_at, updated_at) VALUES ('a', 1, 'en', 'day', 'x', 'daily-agenda', 's', 'h', 'queued', 1, 2, 1)", + ); + expect(() => db.run("UPDATE image_deliveries SET state = 'bogus'")).toThrow(); + }); +}); + +describe('ImageDeliveryRepository', () => { + test('keeps the slot and the payload hash, so a recovered delivery still dedupes against a later request', () => { + const { repo: r } = repo(); + r.insert(base); + expect(r.get('img-1')).toMatchObject({ slot: 'day:user:5:2026-11-02:en', payload_hash: '123' }); + expect(r.unfinished()[0]).toMatchObject({ slot: 'day:user:5:2026-11-02:en', payload_hash: '123' }); + r.finish('img-1', 'delivered', 2_000, 5); + expect(r.get('img-1')).toMatchObject({ slot: 'day:user:5:2026-11-02:en', payload_hash: '123' }); + }); + + test('a rebuild is recorded before its render: a restart resumes the rebuilt job, not the stale one', () => { + const { repo: r } = repo(); + r.insert(base); + expect(r.get('img-1')).toMatchObject({ rebuilds: 0, job_json: '{}' }); + expect(r.rebuilt('img-1', { jobJson: '{"v":2}', payloadHash: '456', rebuilds: 1 }, 2_000)).toBe(true); + expect(r.unfinished()[0]).toMatchObject({ job_json: '{"v":2}', payload_hash: '456', rebuilds: 1 }); + r.advance('img-1', 'rendered', 3_000); + expect(r.rebuilt('img-1', { jobJson: '{"v":3}', payloadHash: '789', rebuilds: 2 }, 3_500)).toBe(true); + expect(r.get('img-1')).toMatchObject({ state: 'rendered', rebuilds: 2 }); + }); + + test('a row whose upload was claimed, or that ended, is never rebuilt', () => { + const { repo: r } = repo(); + r.insert(base); + r.insert({ ...base, id: 'img-2' }); + r.advance('img-1', 'sending', 2_000); + r.finish('img-2', 'failed', 2_000); + expect(r.rebuilt('img-1', { jobJson: '{"v":2}', payloadHash: '456', rebuilds: 1 }, 3_000)).toBe(false); + expect(r.rebuilt('img-2', { jobJson: '{"v":2}', payloadHash: '456', rebuilds: 1 }, 3_000)).toBe(false); + expect(r.get('img-1')).toMatchObject({ job_json: '{}', payload_hash: '123', rebuilds: 0 }); + expect(r.get('img-2')).toMatchObject({ job_json: null, rebuilds: 0 }); + }); + + test('an id already recorded in any state is a duplicate, so a replay never sends twice', () => { + const { repo: r } = repo(); + expect(r.insert(base)).toBe(true); + expect(r.insert(base)).toBe(false); + r.advance('img-1', 'sending', 2_000); + expect(r.insert(base)).toBe(false); + r.finish('img-1', 'unknown', 3_000); // the upload may have landed + expect(r.insert(base)).toBe(false); + expect(r.get('img-1')?.state).toBe('unknown'); + }); + + test('a replay of a delivered request keeps the recorded Telegram message id', () => { + const { repo: r } = repo(); + r.insert(base); + r.finish('img-1', 'delivered', 3_000, 77); + expect(r.insert(base)).toBe(false); + expect(r.get('img-1')).toMatchObject({ state: 'delivered', message_id: 77 }); + }); + + test('a terminal row never changes again and drops its payload', () => { + const { repo: r } = repo(); + r.insert(base); + r.finish('img-1', 'delivered', 2_000, 77); + r.finish('img-1', 'failed', 3_000); + r.advance('img-1', 'sending', 4_000); + expect(r.get('img-1')).toMatchObject({ + state: 'delivered', + message_id: 77, + job_json: null, + }); + }); + + test('transitions only move forward and advance(sending) is a single claim', () => { + const { repo: r } = repo(); + r.insert(base); + expect(r.advance('img-1', 'rendered', 2_000)).toBe(true); + expect(r.advance('img-1', 'rendered', 2_100)).toBe(false); // a second attempt cannot re-record it + expect(r.advance('img-1', 'sending', 3_000)).toBe(true); + expect(r.advance('img-1', 'sending', 3_100)).toBe(false); // the second claimant must not upload + expect(r.advance('img-1', 'rendered', 3_200)).toBe(false); // a redelivered job cannot undo `sending` + expect(r.get('img-1')?.state).toBe('sending'); + }); + + test('a terminal row cannot be claimed, and finish says whether it took effect', () => { + const { repo: r } = repo(); + r.insert(base); + expect(r.finish('img-1', 'expired', 2_000)).toBe(true); + expect(r.advance('img-1', 'sending', 3_000)).toBe(false); + expect(r.finish('img-1', 'delivered', 4_000, 9)).toBe(false); + expect(r.get('img-1')).toMatchObject({ state: 'expired', message_id: null }); + }); + + test('unnotified lists failed and expired rows whose note was never recorded', () => { + const { repo: r } = repo(); + for (const id of ['a', 'b', 'c', 'd']) r.insert({ ...base, id }); + r.finish('a', 'failed', 2_000); + r.finish('b', 'expired', 2_000); + r.finish('c', 'unknown', 2_000); // an uncertain send: the user is never told it failed + r.markNoteSent('b'); + expect(r.unnotified().map((row) => row.id)).toEqual(['a']); + }); + + test('a row whose upload was claimed is never ended as expired or superseded', () => { + const { repo: r } = repo(); + r.insert(base); + r.advance('img-1', 'sending', 2_000); + expect(r.finish('img-1', 'expired', 3_000)).toBe(false); + expect(r.finish('img-1', 'superseded', 3_000)).toBe(false); + expect(r.get('img-1')?.state).toBe('sending'); + expect(r.finish('img-1', 'delivered', 4_000, 41)).toBe(true); // the upload that landed is still recorded + expect(r.get('img-1')).toMatchObject({ state: 'delivered', message_id: 41 }); + }); + + test('the note is claimed once: the second caller must not send it', () => { + const { repo: r } = repo(); + r.insert(base); + r.finish('img-1', 'failed', 2_000); + expect(r.markNoteSent('img-1')).toBe(true); + expect(r.markNoteSent('img-1')).toBe(false); + }); + + test('a delivered or uncertain row never yields a failure note', () => { + const { repo: r } = repo(); + r.insert(base); + r.insert({ ...base, id: 'img-2' }); + r.finish('img-1', 'delivered', 2_000, 5); + r.finish('img-2', 'unknown', 2_000); + expect(r.markNoteSent('img-1')).toBe(false); + expect(r.markNoteSent('img-2')).toBe(false); + }); + + test('unfinished lists only live rows; terminal rows are pruned by age', () => { + const { repo: r } = repo(); + r.insert(base); + r.insert({ ...base, id: 'img-2' }); + r.finish('img-2', 'failed', 2_000); + expect(r.unfinished().map((row) => row.id)).toEqual(['img-1']); + r.pruneOlderThan(10_000); + expect(r.get('img-2')).toBeNull(); + expect(r.get('img-1')).not.toBeNull(); + }); +}); diff --git a/test/fixtures/doubles.ts b/test/fixtures/doubles.ts new file mode 100644 index 000000000..93b36c0cf --- /dev/null +++ b/test/fixtures/doubles.ts @@ -0,0 +1,13 @@ +// Where a hand-built test double is told it is the production type it stands in for. +// A double implements only what the code under test touches, so the compiler cannot prove it +// matches the full interface; naming the cast here keeps it out of every call site. +import type { BotCallbackContext, BotCommandContext } from '../../src/bot/types.ts'; +import type { EventService } from '../../src/services/event/event-service.ts'; + +export const asCommandContext = (double: unknown): BotCommandContext => double as BotCommandContext; + +export const asCallbackContext = (double: unknown): BotCallbackContext => double as BotCallbackContext; + +export const asEventService = (double: unknown): EventService => double as EventService; + +export const asDouble = (double: unknown): T => double as T; diff --git a/test/fixtures/image-delivery.ts b/test/fixtures/image-delivery.ts new file mode 100644 index 000000000..a6447e7aa --- /dev/null +++ b/test/fixtures/image-delivery.ts @@ -0,0 +1,38 @@ +// A real ImageDelivery over a scripted renderer: the AI picture tools are tested through the +// production queue-and-deliver path, not a stub of it. `await delivery.whenIdle()` waits for the +// background delivery the tool call started. +import { mock } from 'bun:test'; +import { + ImageDelivery, + type ImageDeliveryDeps, + type ImageDeliveryOutcome, + type ImageOutcomeDetails, +} from '../../src/services/image/image-delivery.ts'; +import type { RenderOptions } from '../../src/services/image/render-service.ts'; +import type { ImageRenderJob } from '../../src/worker/image-render.queue.ts'; +import { png } from './png.ts'; + +type RenderDirect = (job: ImageRenderJob, options?: RenderOptions) => Promise; + +export function makeImageDelivery( + renderDirect: RenderDirect = () => Promise.resolve(png()), + deps: Partial = {}, +) { + const renderer = { renderDirect: mock(renderDirect) }; + const delivery = new ImageDelivery({ renderer, ...deps }); + return { delivery, renderer }; +} + +/** One 'Image delivery outcome' log line: the outcome, the delivery's identity and the details it carried. */ +export interface LoggedOutcome extends ImageOutcomeDetails { + outcome: ImageDeliveryOutcome; + kind: string; + jobType: string; + chatId: number; + elapsedMs: number; +} + +/** The fields object of a logger call, narrowed to an outcome line (the service only logs this message with that shape). */ +export function asLoggedOutcome(fields: unknown): LoggedOutcome { + return fields as LoggedOutcome; +} diff --git a/test/fixtures/picture-contract.ts b/test/fixtures/picture-contract.ts new file mode 100644 index 000000000..94e474558 --- /dev/null +++ b/test/fixtures/picture-contract.ts @@ -0,0 +1,138 @@ +// Shared doubles for the "every picture is asynchronous" contract of the commands, buttons and +// invitation notices: a REAL ImageDelivery over a renderer whose every render stays pending until the +// test releases or fails it, so a test can prove the handler returned before the render ended and then +// finish the render later. Only the render queue and Telegram are fake. +import { afterEach, describe, expect, type Mock, test } from 'bun:test'; +import type { ImageDelivery, ImageDeliveryDeps, RenderFailureReport } from '../../src/services/image/image-delivery.ts'; +import { RenderAbortedError, type RenderOptions } from '../../src/services/image/render-service.ts'; +import { configureTimeoutReporter, TimeoutReporter } from '../../src/services/incident/timeout-reporter.ts'; +import type { ImageRenderJob } from '../../src/worker/image-render.queue.ts'; +import { makeImageDelivery } from './image-delivery.ts'; +import { png } from './png.ts'; + +type RenderDirect = (job: ImageRenderJob, options?: RenderOptions) => Promise; + +export interface GatedDelivery { + delivery: ImageDelivery; + /** Every render the delivery started is a call here, in order. */ + renderer: { renderDirect: Mock }; + /** Reports the delivery raised for failures that are not timeouts (each is one owner alert). */ + failures: RenderFailureReport[]; + /** Finishes the n-th render (default the first) with a PNG. */ + release(index?: number): void; + /** Finishes every render started so far with a PNG. */ + releaseAll(): void; + /** Fails the n-th render (default the first) with `err`. */ + fail(err: unknown, index?: number): void; +} + +export function gatedDelivery(deps: Partial = {}): GatedDelivery { + const gates: PromiseWithResolvers[] = []; + const failures: RenderFailureReport[] = []; + const { delivery, renderer } = makeImageDelivery( + (_job, options) => { + const gate = Promise.withResolvers(); + gate.promise.catch(() => {}); + options?.signal?.addEventListener('abort', () => gate.reject(new RenderAbortedError()), { once: true }); + gates.push(gate); + return gate.promise; + }, + { reportFailure: (report) => failures.push(report), ...deps }, + ); + return { + delivery, + renderer, + failures, + fail: (err, index = 0) => gates[index]!.reject(err), + release: (index = 0) => gates[index]!.resolve(png()), + releaseAll: () => { + for (const gate of gates) gate.resolve(png()); + }, + }; +} + +/** Routes reportTimeout to a list for the test; call `configureTimeoutReporter(null)` in afterEach. */ +export function captureTimeoutAlerts(): { text: string; source: string }[] { + const alerts: { text: string; source: string }[] = []; + configureTimeoutReporter(new TimeoutReporter((text, source) => alerts.push({ text, source }))); + return alerts; +} + +/** What a handler under the picture contract looks like to the shared tests. */ +export interface PictureHarness { + /** Runs the handler once; `requestId` is the Telegram message (or callback) id of this call. */ + run(delivery: ImageDelivery, requestId: number): Promise; + /** Pictures (photos or documents) the chat received. */ + photos(): number; + /** Every text the chat received, in order: the handler's own answer first, then any note. */ + texts(): string[]; +} + +/** + * The behaviour every picture path owes the user (owner instruction 2026-11-01: every picture is + * asynchronous, every timeout reaches the repair flow). `makeHarness` builds fresh doubles per test. + */ +export function describePictureContract(name: string, makeHarness: () => PictureHarness): void { + describe(`${name}: picture contract`, () => { + afterEach(() => configureTimeoutReporter(null)); + + test('returns before a slow render ends and delivers the late picture exactly once', async () => { + const h = makeHarness(); + const g = gatedDelivery(); + + await h.run(g.delivery, 1); + + expect(g.renderer.renderDirect).toHaveBeenCalledTimes(1); + expect(h.photos()).toBe(0); + const answered = h.texts(); + + g.release(); + await g.delivery.whenIdle(); + + expect(h.photos()).toBe(1); + expect(h.texts()).toEqual(answered); // the picture arrived: no note + }); + + test('a double tap or repeated command delivers one picture', async () => { + const h = makeHarness(); + const g = gatedDelivery(); + + await h.run(g.delivery, 1); + await h.run(g.delivery, 2); + g.releaseAll(); + await g.delivery.whenIdle(); + + expect(h.photos()).toBe(1); + }); + + test('a failed render gives one note and one owner alert, and no picture', async () => { + const h = makeHarness(); + const g = gatedDelivery(); + + await h.run(g.delivery, 1); + const answered = h.texts(); + g.fail(new Error('chromium crashed')); + await g.delivery.whenIdle(); + + expect(h.photos()).toBe(0); + expect(h.texts()).toHaveLength(answered.length + 1); + expect(g.failures).toHaveLength(1); + }); + + test('a render that outlives its bound gives one note and one timeout alert for the repair flow', async () => { + const alerts = captureTimeoutAlerts(); + const h = makeHarness(); + const g = gatedDelivery({ boundMs: 20 }); + + await h.run(g.delivery, 1); + const answered = h.texts(); + await g.delivery.whenIdle(); + + expect(h.photos()).toBe(0); + expect(h.texts()).toHaveLength(answered.length + 1); + expect(alerts).toHaveLength(1); + expect(alerts[0]?.text).toContain('Timeout (render)'); + expect(g.failures).toHaveLength(0); + }); + }); +} diff --git a/test/regressions/canonical-basis-helpers.ts b/test/regressions/canonical-basis-helpers.ts index 998f85c48..eff174b3a 100644 --- a/test/regressions/canonical-basis-helpers.ts +++ b/test/regressions/canonical-basis-helpers.ts @@ -17,6 +17,7 @@ import type { AgentContext, ToolResult } from '../../src/services/ai/types.ts'; import { ConversationLogger } from '../../src/services/conversation-logger.ts'; import { EventService } from '../../src/services/event/event-service.ts'; import { HolidayService } from '../../src/services/holiday/holiday-service.ts'; +import { ImageDelivery } from '../../src/services/image/image-delivery.ts'; import type { ImageRenderer } from '../../src/services/image/render-service.ts'; import { IntentExecutor } from '../../src/services/intent/intent-executor.ts'; import { IntentMatcher } from '../../src/services/intent/intent-matcher.ts'; @@ -126,7 +127,7 @@ export function createFixture(db: Database, options: FixtureOptions = {}): Fixtu }, ...(options.renderer ? { - renderService: options.renderer, + imageDelivery: new ImageDelivery({ renderer: options.renderer }), sender: { sendPhoto: async () => ({ message_id: 2 }), sendMessage: async () => ({ message_id: 3 }), diff --git a/test/services/ai/delete-confirmation.test.ts b/test/services/ai/delete-confirmation.test.ts index 9e56ae942..c59488d7d 100644 --- a/test/services/ai/delete-confirmation.test.ts +++ b/test/services/ai/delete-confirmation.test.ts @@ -21,6 +21,7 @@ import { HolidayService } from '../../../src/services/holiday/holiday-service.ts import { NotificationPreferencesService } from '../../../src/services/notification/preferences.ts'; import { createVoiceSender } from '../../../src/services/voice/voice-sender.ts'; import type { ImageRenderJob } from '../../../src/worker/image-render.queue.ts'; +import { makeImageDelivery } from '../../fixtures/image-delivery.ts'; import { png } from '../../fixtures/png.ts'; // Replay of 2026-09-27 (synthetic ids): on Sunday night the user asked to cancel "all English on @@ -228,20 +229,16 @@ describe('bot-rendered delete confirmation', () => { const rendered: string[] = []; let output = ''; onContinuation = async (ctx) => { + const imageDelivery = makeImageDelivery(async (job: ImageRenderJob) => { + if (job.type === 'daily-agenda') rendered.push(job.data.date); + return png(); + }).delivery; const result = await executeTool( - { - ...ctx, - sender: { ...ctx.sender!, sendPhoto: async () => ({ message_id: 3 }) }, - renderService: { - renderDirect: async (job: ImageRenderJob) => { - if (job.type === 'daily-agenda') rendered.push(job.data.date); - return png(); - }, - }, - }, + { ...ctx, sender: { ...ctx.sender!, sendPhoto: async () => ({ message_id: 3 }) }, imageDelivery }, 'render_day_image', { date: '2026-09-01' }, ); + await imageDelivery.whenIdle(); output = String(result.output); }; const message = await askAll(); diff --git a/test/services/ai/render-async-turn.test.ts b/test/services/ai/render-async-turn.test.ts new file mode 100644 index 000000000..bdeea86f8 --- /dev/null +++ b/test/services/ai/render-async-turn.test.ts @@ -0,0 +1,282 @@ +// Production path of the 2026-11-01 incident (prod bb7a9d47): the model calls a picture tool and the +// render takes longer than the whole turn budget. The real CalendarBotAgent runs the real tool executor +// and the real ImageDelivery over a scripted renderer and a fake Telegram sender (synthetic data only). +// +// Required behaviour: the turn ends with the text answer at once, the picture follows as its own +// message when the worker is done, a job that never finishes yields one honest note and one alert, +// and no render can block or fail the turn. +import { Database } from 'bun:sqlite'; +import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'; +import type OpenAI from 'openai'; +import { t } from '../../../src/config/constants.ts'; +import { migrations } from '../../../src/database/migrations.ts'; +import { ChatHistoryRepository } from '../../../src/database/repositories/chat-history.repository.ts'; +import { EventRepository } from '../../../src/database/repositories/event.repository.ts'; +import { EventReminderRepository } from '../../../src/database/repositories/event-reminder.repository.ts'; +import { HolidayRepository } from '../../../src/database/repositories/holiday.repository.ts'; +import { UserRepository } from '../../../src/database/repositories/user.repository.ts'; +import { runMigrations } from '../../../src/database/schema.ts'; +import { aiFailureNotices, CalendarBotAgent } from '../../../src/services/ai/agent.ts'; +import type { StreamCallbacks, StreamRoundOptions, StreamRoundResult } from '../../../src/services/ai/streaming.ts'; +import { _resetToolThrottleForTest } from '../../../src/services/ai/tool-executor.ts'; +import type { AgentContext, TelegramSender } from '../../../src/services/ai/types.ts'; +import { ConversationLogger } from '../../../src/services/conversation-logger.ts'; +import { EventService } from '../../../src/services/event/event-service.ts'; +import { HolidayService } from '../../../src/services/holiday/holiday-service.ts'; +import { ImageDelivery } from '../../../src/services/image/image-delivery.ts'; +import type { RenderOptions } from '../../../src/services/image/render-service.ts'; +import { + configureTimeoutReporter, + TIMEOUT_ALERT_SOURCE, + TimeoutReporter, +} from '../../../src/services/incident/timeout-reporter.ts'; +import type { ImageRenderJob } from '../../../src/worker/image-render.queue.ts'; +import { png } from '../../fixtures/png.ts'; + +const USER_ID = 7110; +const ANSWER = 'Синтетический ответ: завтра два события, всё в порядке.'; + +type Round = { text: string } | { tools: { name: string; input: { [key: string]: unknown } }[] }; + +/** One scripted model round per call; the response validator is always approved. */ +function scripted(rounds: Round[]) { + let index = 0; + return async (opts: StreamRoundOptions, cbs: StreamCallbacks = {}): Promise => { + const system = opts.messages[0]; + if (typeof system?.content === 'string' && system.content.includes('strict QA validator')) { + return { + text: 'APPROVE', + toolCalls: [], + finishReason: 'stop', + assistantMessage: { role: 'assistant', content: 'APPROVE' }, + providerUsed: 'mock-validator', + }; + } + const round = rounds[index++]; + if (!round) throw new Error(`Scripted stream ran out of rounds (call ${index})`); + if ('text' in round) { + cbs.onTextDelta?.(round.text); + return { + text: round.text, + toolCalls: [], + finishReason: 'stop', + assistantMessage: { role: 'assistant', content: round.text }, + providerUsed: 'mock', + }; + } + const toolCalls = round.tools.map((tool, i) => ({ + id: `call-${index}-${i}`, + name: tool.name, + arguments: JSON.stringify(tool.input), + })); + const assistantMessage: OpenAI.ChatCompletionMessageParam = { + role: 'assistant', + content: null, + tool_calls: toolCalls.map((call) => ({ + id: call.id, + type: 'function', + function: { name: call.name, arguments: call.arguments }, + })), + }; + return { text: '', toolCalls, finishReason: 'tool_calls', assistantMessage, providerUsed: 'mock' }; + }; +} + +interface Pending { + job: ImageRenderJob; + resolve(buffer: Buffer): void; +} + +describe('picture tools never hold the agent turn', () => { + let db: Database; + let ctx: AgentContext; + let sender: TelegramSender; + let timeline: string[]; + let jobs: Pending[]; + let alerts: { text: string; source: string }[]; + let makeDelivery: (boundMs?: number) => ImageDelivery; + + beforeEach(() => { + _resetToolThrottleForTest(); + aiFailureNotices.reset(); + alerts = []; + configureTimeoutReporter(new TimeoutReporter((text, source) => alerts.push({ text, source }))); + db = new Database(':memory:'); + runMigrations(db, migrations); + const userRepo = new UserRepository(db); + const chatHistory = new ChatHistoryRepository(db); + userRepo.create({ telegram_id: USER_ID, timezone: 'Europe/Belgrade', language: 'ru' }); + timeline = []; + jobs = []; + sender = { + sendMessage: mock(async (_chatId: number, text: string) => { + timeline.push(`text:${text}`); + return { message_id: 42 }; + }), + editMessageText: mock(async (_chatId: number, _messageId: number, text: string) => { + timeline.push(`text:${text}`); + }), + deleteMessage: mock(async () => {}), + sendPhoto: mock(async () => { + timeline.push('photo'); + return { message_id: 90 }; + }), + setReaction: async () => {}, + sendButtons: async () => ({ message_id: 43 }), + }; + makeDelivery = (boundMs) => + new ImageDelivery({ + boundMs, + renderer: { + // A worker that is slower than the whole turn: it answers only when the test lets it. + renderDirect: (job: ImageRenderJob, _options?: RenderOptions) => + new Promise((resolve) => jobs.push({ job, resolve })), + }, + }); + ctx = { + user: userRepo.findByTelegramId(USER_ID)!, + chatId: USER_ID, + messageText: 'покажи завтра картинкой', + isGroup: false, + eventService: new EventService({ eventRepo: new EventRepository(db) }), + holidayService: new HolidayService(new HolidayRepository(db)), + chatHistory, + conversationLogger: new ConversationLogger(chatHistory), + userRepo, + eventReminderRepo: new EventReminderRepository(db), + incomingMessageId: 900, + sender, + imageDelivery: makeDelivery(), + }; + chatHistory.save(USER_ID, 'user', ctx.messageText); + }); + + afterEach(() => { + configureTimeoutReporter(null); + db.close(); + }); + + const tomorrow = () => new Date(Date.now() + 86_400_000).toISOString().slice(0, 10); + const run = (rounds: Round[]) => new CalendarBotAgent({}, sender, { streamImpl: scripted(rounds) }).run(ctx); + const dayImage = () => ({ name: 'render_day_image', input: { date: tomorrow() } }); + + test('the turn ends with the text while the render is still running; the late picture follows the text', async () => { + const result = await run([{ tools: [dayImage()] }, { text: ANSWER }]); + + // The turn is over, the user has the answer, and the worker has not finished: nothing waited for it. + expect(result.responseText).toContain('два события'); + expect(timeline.filter((line) => line.startsWith('text:') && line.includes('два события'))).not.toHaveLength(0); + expect(timeline).not.toContain('photo'); + expect(jobs).toHaveLength(1); + + // 80 s later (the overloaded odroid) the worker answers: the picture is its own message, after the text. + jobs[0]!.resolve(png()); + await ctx.imageDelivery!.whenIdle(); + + expect(timeline.at(-1)).toBe('photo'); + expect(timeline.filter((line) => line === 'photo')).toHaveLength(1); + expect(timeline.findIndex((line) => line.includes('два события'))).toBeLessThan(timeline.indexOf('photo')); + // Nothing the user saw claims the picture was already attached or sent. + expect(timeline.join('\n')).not.toMatch(/прикреп|отправлена в чат/i); + expect(alerts).toHaveLength(0); + }); + + test('the model is told honestly that the picture is only queued', async () => { + const seen: string[] = []; + const stream = scripted([{ tools: [dayImage()] }, { text: ANSWER }]); + await new CalendarBotAgent({}, sender, { + streamImpl: async (opts, cbs) => { + for (const message of opts.messages) { + if (message.role === 'tool') seen.push(String(message.content)); + } + return stream(opts, cbs); + }, + }).run(ctx); + + const toolMessage = seen.at(-1) ?? ''; + expect(toolMessage).toContain('готовится'); + expect(toolMessage).toContain('NOT sent yet'); + expect(toolMessage).toContain('do NOT say the picture was sent or attached'); + expect(toolMessage).not.toContain('отправлена в чат'); + jobs[0]!.resolve(png()); + await ctx.imageDelivery!.whenIdle(); + }); + + test('a job that never finishes: the turn is untouched, the user gets one honest note and the repair flow one alert', async () => { + ctx.imageDelivery = makeDelivery(40); // the real bound timer, shortened from 150 s; the renderer never answers + const result = await run([{ tools: [dayImage()] }, { text: ANSWER }]); + expect(result.responseText).toContain('два события'); + + await ctx.imageDelivery.whenIdle(); + + const notes = timeline.filter((line) => line.includes(t('ru').aiTools.meta.imageNoteTimedOut(tomorrow()))); + expect(notes).toHaveLength(1); + expect(timeline).not.toContain('photo'); + expect(timeline.findIndex((line) => line.includes('два события'))).toBeLessThan(timeline.indexOf(notes[0]!)); + expect(alerts).toHaveLength(1); + expect(alerts[0]!.source).toBe(TIMEOUT_ALERT_SOURCE); + expect(alerts[0]!.text).toContain('Timeout (render)'); + // The alert carries no user text. + expect(alerts[0]!.text).not.toContain(tomorrow()); + expect(alerts[0]!.text).not.toContain('картинкой'); + }); + + test('two different pictures in one turn are two deliveries; a third is refused honestly without failing the turn', async () => { + const week = { name: 'render_week_image', input: { week_start: tomorrow() } }; + const month = { name: 'render_month_image', input: { month: tomorrow().slice(0, 7) } }; + const toolMessages: string[] = []; + const stream = scripted([{ tools: [dayImage(), week, month] }, { text: ANSWER }]); + const result = await new CalendarBotAgent({}, sender, { + streamImpl: async (opts, cbs) => { + for (const message of opts.messages) { + if (message.role === 'tool') toolMessages.push(String(message.content)); + } + return stream(opts, cbs); + }, + }).run(ctx); + + expect(result.responseText).toContain('два события'); + expect(jobs).toHaveLength(2); + expect(toolMessages.filter((message) => message.includes(t('ru').aiTools.meta.imageBusy))).toHaveLength(1); + for (const pending of jobs) pending.resolve(png()); + await ctx.imageDelivery!.whenIdle(); + expect(timeline.filter((line) => line === 'photo')).toHaveLength(2); + }); + + test('the same picture requested twice in one chat is delivered once', async () => { + await run([{ tools: [dayImage(), dayImage()] }, { text: ANSWER }]); + expect(jobs).toHaveLength(1); + jobs[0]!.resolve(png()); + await ctx.imageDelivery!.whenIdle(); + expect(timeline.filter((line) => line === 'photo')).toHaveLength(1); + }); + + test('a new message before the picture arrives gets its own answer; the picture still arrives once', async () => { + await run([{ tools: [dayImage()] }, { text: ANSWER }]); + expect(timeline).not.toContain('photo'); + + ctx.chatHistory.save(USER_ID, 'user', 'спасибо'); + ctx.messageText = 'спасибо'; + ctx.incomingMessageId = 901; + _resetToolThrottleForTest(); + await run([{ text: 'Пожалуйста!' }]); + expect(timeline.some((line) => line.includes('Пожалуйста!'))).toBe(true); + expect(timeline).not.toContain('photo'); + + jobs[0]!.resolve(png()); + await ctx.imageDelivery!.whenIdle(); + expect(timeline.at(-1)).toBe('photo'); + expect(timeline.filter((line) => line === 'photo')).toHaveLength(1); + }); + + test('a render failure after the turn never turns into a failed run: the answer is the one the model wrote', async () => { + ctx.imageDelivery = new ImageDelivery({ + renderer: { renderDirect: () => Promise.reject(new Error('synthetic worker crash')) }, + }); + const result = await run([{ tools: [dayImage()] }, { text: ANSWER }]); + expect(result.responseText).toContain('два события'); + await ctx.imageDelivery.whenIdle(); + expect(timeline.filter((line) => line.includes(t('ru').aiTools.meta.imageNoteFailed(tomorrow())))).toHaveLength(1); + expect(timeline.join('\n')).not.toContain('synthetic worker crash'); + }); +}); diff --git a/test/services/ai/render-tool-contract.test.ts b/test/services/ai/render-tool-contract.test.ts new file mode 100644 index 000000000..a70529ea6 --- /dev/null +++ b/test/services/ai/render-tool-contract.test.ts @@ -0,0 +1,196 @@ +// Contract fixtures for the picture tools (AGENTS.md "Prompt/tool contracts"): the concrete calls the +// prompt and the tool schemas teach the model are executed through the real tool exposure (lazy and +// hybrid), the real schema validation and the real handlers, at UTC and IANA zones and on DST dates. +// A queued picture must carry the right local day / week / month, and the model must be told it is +// only queued. +import { Database } from 'bun:sqlite'; +import { beforeEach, describe, expect, test } from 'bun:test'; +import { TZDate } from '@date-fns/tz'; +import { migrations } from '../../../src/database/migrations.ts'; +import { ChatHistoryRepository } from '../../../src/database/repositories/chat-history.repository.ts'; +import { EventRepository } from '../../../src/database/repositories/event.repository.ts'; +import { EventReminderRepository } from '../../../src/database/repositories/event-reminder.repository.ts'; +import { HolidayRepository } from '../../../src/database/repositories/holiday.repository.ts'; +import { UserRepository } from '../../../src/database/repositories/user.repository.ts'; +import { runMigrations } from '../../../src/database/schema.ts'; +import { _resetToolThrottleForTest, executeTool } from '../../../src/services/ai/tool-executor.ts'; +import { createToolExposure } from '../../../src/services/ai/tool-exposure.ts'; +import { getToolDefinitions } from '../../../src/services/ai/tools.ts'; +import type { AgentContext } from '../../../src/services/ai/types.ts'; +import { ConversationLogger } from '../../../src/services/conversation-logger.ts'; +import { EventService } from '../../../src/services/event/event-service.ts'; +import { HolidayService } from '../../../src/services/holiday/holiday-service.ts'; +import type { ImageRenderJob } from '../../../src/worker/image-render.queue.ts'; +import { getTemplate } from '../../../src/worker/templates/index.ts'; +import { makeImageDelivery } from '../../fixtures/image-delivery.ts'; +import { png } from '../../fixtures/png.ts'; + +const USER_ID = 7120; + +/** Zones with a DST date each: the shift happens on that local day, so the day has 23 or 25 hours. */ +const ZONES: [timezone: string, date: string][] = [ + ['UTC', '2026-03-29'], + ['Europe/Belgrade', '2026-03-29'], // spring forward + ['Europe/Belgrade', '2026-10-25'], // fall back + ['Pacific/Auckland', '2026-09-27'], // spring forward, UTC+12 -> +13 + ['Pacific/Auckland', '2027-04-04'], // fall back + ['America/Los_Angeles', '2026-03-08'], + ['America/Los_Angeles', '2026-11-01'], +]; + +/** Monday of the ISO week of a calendar date, by plain UTC arithmetic: independent of the handler's code. */ +function mondayOf(date: string): string { + const day = new Date(`${date}T12:00:00Z`); + day.setUTCDate(day.getUTCDate() - ((day.getUTCDay() + 6) % 7)); + return day.toISOString().slice(0, 10); +} + +function setup(timezone: string) { + const db = new Database(':memory:'); + runMigrations(db, migrations); + const userRepo = new UserRepository(db); + userRepo.create({ telegram_id: USER_ID, timezone, language: 'en' }); + const chatHistory = new ChatHistoryRepository(db); + const jobs: ImageRenderJob[] = []; + const photos: number[] = []; + const eventService = new EventService({ eventRepo: new EventRepository(db) }); + const { delivery } = makeImageDelivery(async (job) => { + jobs.push(job); + return png(); + }); + const ctx = { + user: userRepo.findByTelegramId(USER_ID)!, + chatId: USER_ID, + messageText: '', + isGroup: false, + eventService, + holidayService: new HolidayService(new HolidayRepository(db)), + chatHistory, + conversationLogger: new ConversationLogger(chatHistory), + userRepo, + eventReminderRepo: new EventReminderRepository(db), + sender: { + sendMessage: async () => ({ message_id: 1 }), + sendPhoto: async () => { + photos.push(1); + return { message_id: 2 }; + }, + }, + imageDelivery: delivery, + } as unknown as AgentContext; + return { ctx, jobs, photos, eventService, delivery, db }; +} + +/** What the model does in each tool schema mode: a tool that is not loaded is discovered one round earlier. */ +function callThroughExposure(mode: 'lazy' | 'hybrid', name: string, input: { [key: string]: unknown }) { + const exposure = createToolExposure(getToolDefinitions('text'), mode); + let exposed = exposure.snapshot(); + const discovered = !exposed.has(name); + if (discovered) { + const discovery = exposure.intercept('discover_tools', { tools: [name] }, exposed); + expect(discovery?.success).toBe(true); + exposed = exposure.snapshot(); + } + expect(exposed.has(name)).toBe(true); + // With the schema loaded, the call is no longer intercepted. + expect(exposure.intercept(name, input, exposed)).toBeUndefined(); + return { discovered }; +} + +beforeEach(() => { + _resetToolThrottleForTest(); +}); + +describe.each(['lazy', 'hybrid'] as const)('picture tool calls the prompt teaches (%s mode)', (mode) => { + test.each( + ZONES, + )('render_day_image at %s on %s queues the local day and the model learns it is queued', async (timezone, date) => { + const { ctx, jobs, photos, eventService, delivery, db } = setup(timezone); + // An event late in the DST day: it must sit on that local day, whatever the UTC offset did. + eventService.createEvent({ + user_id: USER_ID, + title: 'Synthetic late lesson', + start_at: new Date( + new TZDate(Number(date.slice(0, 4)), Number(date.slice(5, 7)) - 1, Number(date.slice(8)), 23, 30, timezone), + ).toISOString(), + timezone, + }); + const input = { date }; + callThroughExposure(mode, 'render_day_image', input); + + const result = await executeTool(ctx, 'render_day_image', input); + expect(result.success).toBe(true); + expect(photos).toHaveLength(0); + expect(result.agentHint).toContain('NOT sent yet'); + await delivery.whenIdle(); + + const job = jobs[0]; + if (job?.type !== 'daily-agenda') throw new Error(`expected a daily-agenda job, got ${job?.type}`); + expect(job.data.date).toBe(date); + expect(JSON.stringify(job.data)).toContain('Synthetic late lesson'); + expect(photos).toHaveLength(1); + db.close(); + }); + + test.each( + ZONES, + )('render_week_image at %s on %s queues the Monday-based week of that date', async (timezone, date) => { + const { ctx, jobs, delivery, db } = setup(timezone); + const input = { week_start: date }; + callThroughExposure(mode, 'render_week_image', input); + + const result = await executeTool(ctx, 'render_week_image', input); + expect(result.success).toBe(true); + await delivery.whenIdle(); + + const job = jobs[0]; + if (job?.type !== 'weekly-overview') throw new Error(`expected a weekly-overview job, got ${job?.type}`); + expect(job.data.days).toHaveLength(7); + expect(job.data.days[0]?.dayNumber).toBe(Number(mondayOf(date).slice(8))); + db.close(); + }); + + test.each(ZONES)('render_month_image at %s for the month of %s queues that month', async (timezone, date) => { + const { ctx, jobs, delivery, db } = setup(timezone); + const input = { month: date.slice(0, 7) }; + callThroughExposure(mode, 'render_month_image', input); + + const result = await executeTool(ctx, 'render_month_image', input); + expect(result.success).toBe(true); + await delivery.whenIdle(); + + const job = jobs[0]; + if (job?.type !== 'monthly-calendar') throw new Error(`expected a monthly-calendar job, got ${job?.type}`); + expect(job.data.monthLabel).toContain(date.slice(0, 4)); + db.close(); + }); + + test('render_table: the markdown example in its own schema runs through the handler and the real template', async () => { + const definition = getToolDefinitions('text').find( + (tool) => tool.type === 'function' && tool.function.name === 'render_table', + ); + if (definition?.type !== 'function') throw new Error('render_table is not offered in text mode'); + const markdownSchema = (definition.function.parameters as { properties: { markdown: { description: string } } }) + .properties.markdown; + // The schema quotes the table the model should send: "| Plan | Price |\n|---|---|\n| Basic | $5 |". + const quoted = /e\.g\. "(.+)"/.exec(markdownSchema.description)?.[1]; + if (!quoted) throw new Error('the render_table schema no longer quotes a markdown example'); + const markdown = quoted.replaceAll('\\n', '\n'); + + const { ctx, jobs, photos, delivery, db } = setup('Europe/Belgrade'); + const input = { title: 'Plans', markdown }; + callThroughExposure(mode, 'render_table', input); + const result = await executeTool(ctx, 'render_table', input); + + expect(result.success).toBe(true); + expect(photos).toHaveLength(0); + await delivery.whenIdle(); + const job = jobs[0]; + if (job?.type !== 'md-table') throw new Error(`expected an md-table job, got ${job?.type}`); + // The worker side parses the same example: it renders to HTML with its rows. + const html = getTemplate('md-table').render(job.data); + expect(html).toContain('Basic'); + expect(html).toContain('$5'); + db.close(); + }); +}); diff --git a/test/services/ai/silent-final.test.ts b/test/services/ai/silent-final.test.ts index 4544eb0d5..3c03d84e9 100644 --- a/test/services/ai/silent-final.test.ts +++ b/test/services/ai/silent-final.test.ts @@ -26,6 +26,7 @@ import { HolidayService } from '../../../src/services/holiday/holiday-service.ts import { InvitationService } from '../../../src/services/sharing/invitation-service.ts'; import { PrivacyService } from '../../../src/services/sharing/privacy-service.ts'; import { SharingService } from '../../../src/services/sharing/sharing-service.ts'; +import { makeImageDelivery } from '../../fixtures/image-delivery.ts'; const USER_ID = 5081; const INVITEE_ID = 5082; @@ -233,7 +234,7 @@ describe('silent final guard (#508)', () => { expect(delivered.at(-1)).toContain(`${tr.operations.create_event.done}: событие «Синтетическая встреча»`); }); - /** Wires a renderer and records every photo actually sent to the chat. */ + /** Wires a renderer behind the real background delivery and records every photo actually sent to the chat. */ const stubRenderer = (renderDirect: () => Promise) => { const photos: File[] = []; sender.sendPhoto = async (_chatId, photo) => { @@ -241,9 +242,11 @@ describe('silent final guard (#508)', () => { return { message_id: 44 }; }; ctx.sender = sender; - ctx.renderService = { renderDirect }; + ctx.imageDelivery = makeImageDelivery(renderDirect).delivery; return photos; }; + /** The picture is delivered after the turn: wait for the background delivery before asserting on it. */ + const settle = () => ctx.imageDelivery!.whenIdle(); // Smallest header sendAgendaImage accepts: PNG signature + IHDR 1×1. const png = Buffer.concat([ Buffer.from('89504e470d0a1a0a0000000d49484452', 'hex'), @@ -254,9 +257,10 @@ describe('silent final guard (#508)', () => { test.each([ ['[SKIP]', '[SKIP]'], ['empty', ''], - ])('a rendered image already answered the request: %s final adds no notice and no bare "..."', async (_label, finalText) => { + ])('a queued image already answers the request: %s final adds no notice and no bare "..."', async (_label, finalText) => { const photos = stubRenderer(async () => png); const result = await run([renderTable, { text: finalText }]); + await settle(); expect(photos).toHaveLength(1); expect(result.responseText).toBe(''); @@ -265,35 +269,51 @@ describe('silent final guard (#508)', () => { expect(delivered.some((text) => text.endsWith('...'))).toBe(false); }); - test("a throttled repeat render sent no photo, so '[SKIP]' still gets the could-not-answer line", async () => { + test("a throttled repeat render queued nothing, so '[SKIP]' still gets the could-not-answer line", async () => { const photos = stubRenderer(async () => png); await run([renderTable, { text: '[SKIP]' }]); + await settle(); delivered.length = 0; deleted.length = 0; - // The identical call within the 5 s throttle window is skipped without sending anything. + // The identical call within the 5 s throttle window is skipped without queueing anything. await run([renderTable, { text: '[SKIP]' }]); + await settle(); expect(photos).toHaveLength(1); expect(deleted).toEqual([]); expect(delivered.at(-1)).toContain(t('ru').ai_unanswered); }); - test("a failed render sent no photo, so '[SKIP]' still gets the could-not-answer line", async () => { + test("a render that fails after it was queued: '[SKIP]' stays quiet and the user gets the delivery's one honest note", async () => { const photos = stubRenderer(async () => { throw new Error('synthetic renderer outage'); }); await run([renderTable, { text: '[SKIP]' }]); + await settle(); expect(photos).toHaveLength(0); - expect(deleted).toEqual([]); - expect(delivered.at(-1)).toContain(t('ru').ai_unanswered); + expect(delivered.filter((text) => text === t('ru').aiTools.meta.imageNoteFailed('Неделя'))).toHaveLength(1); + expect(delivered.join('\n')).not.toContain(t('ru').ai_unanswered); + }); + + test("an upload cut off after it was queued: '[SKIP]' stays quiet and the user gets one note that the picture may not have arrived", async () => { + const photos = stubRenderer(async () => png); + sender.sendPhoto = async () => { + throw new Error('socket hang up'); + }; + await run([renderTable, { text: '[SKIP]' }]); + await settle(); + + expect(photos).toHaveLength(0); + expect(delivered.filter((text) => text === t('ru').aiTools.meta.imageNoteUncertain('Неделя'))).toHaveLength(1); + expect(delivered.join('\n')).not.toContain(t('ru').ai_unanswered); }); test.each([ - ['a reaction then a failed render', 'reaction-first'], - ['a failed render then a reaction', 'render-first'], - ])("%s: a 👍 does not stand in for the picture that never came, so '[SKIP]' gets the could-not-answer line", async (_label, order) => { + ['a reaction then a failing render', 'reaction-first'], + ['a failing render then a reaction', 'render-first'], + ])('%s: still one note for the picture, and no could-not-answer line', async (_label, order) => { const photos = stubRenderer(async () => { throw new Error('synthetic renderer outage'); }); @@ -302,30 +322,34 @@ describe('silent final guard (#508)', () => { ...(order === 'reaction-first' ? [reaction, renderTable] : [renderTable, reaction]), { text: '[SKIP]' }, ]); + await settle(); expect(photos).toHaveLength(0); - expect(deleted).toEqual([]); - expect(delivered.at(-1)).toContain(t('ru').ai_unanswered); + expect(delivered.filter((text) => text === t('ru').aiTools.meta.imageNoteFailed('Неделя'))).toHaveLength(1); + expect(delivered.join('\n')).not.toContain(t('ru').ai_unanswered); }); - test("a reaction then a throttled repeat render (no photo sent) does not excuse '[SKIP]'", async () => { + test("a reaction then a throttled repeat render (nothing queued) does not excuse '[SKIP]'", async () => { const photos = stubRenderer(async () => png); await run([renderTable, { text: '[SKIP]' }]); + await settle(); delivered.length = 0; deleted.length = 0; await run([{ tool: 'set_reaction', input: () => ({ emoji: '👍' }) }, renderTable, { text: '[SKIP]' }]); + await settle(); expect(photos).toHaveLength(1); expect(delivered.at(-1)).toContain(t('ru').ai_unanswered); }); - test("a delivered picture stays the answer when a reaction after it fails: '[SKIP]' adds no notice", async () => { + test("a queued picture stays the answer when a reaction after it fails: '[SKIP]' adds no notice", async () => { const photos = stubRenderer(async () => png); sender.setReaction = async () => { throw new Error('synthetic REACTION_INVALID'); }; await run([renderTable, { tool: 'set_reaction', input: () => ({ emoji: '👍' }) }, { text: '[SKIP]' }]); + await settle(); expect(photos).toHaveLength(1); expect(delivered.join('\n')).not.toContain(t('ru').ai_unanswered); diff --git a/test/services/ai/tool-handlers/agenda269-render.test.ts b/test/services/ai/tool-handlers/agenda269-render.test.ts index 603511ee9..17fb923e6 100644 --- a/test/services/ai/tool-handlers/agenda269-render.test.ts +++ b/test/services/ai/tool-handlers/agenda269-render.test.ts @@ -17,11 +17,12 @@ import { handleRenderTable, handleRenderWeekImage, } from '../../../../src/services/ai/tool-handlers/render.ts'; -import type { AgentContext } from '../../../../src/services/ai/types.ts'; +import type { AgentContext, ToolResult } from '../../../../src/services/ai/types.ts'; import { ConversationLogger } from '../../../../src/services/conversation-logger.ts'; import { EventService } from '../../../../src/services/event/event-service.ts'; import { HolidayService } from '../../../../src/services/holiday/holiday-service.ts'; import type { ImageRenderJob } from '../../../../src/worker/image-render.queue.ts'; +import { makeImageDelivery } from '../../../fixtures/image-delivery.ts'; import { png } from '../../../fixtures/png.ts'; let db: Database; @@ -91,19 +92,26 @@ function setup(userId: number, group = false) { sendMessage: async () => ({ message_id: 1 }), sendPhoto: async () => ({ message_id: 2 }), }, - renderService: { - renderDirect: async (job) => { - jobs.push(job); - return png(); - }, - }, + imageDelivery: makeImageDelivery(async (job) => { + jobs.push(job); + return png(); + }).delivery, }; return { ctx, jobs }; } +/** The tool call returns at once; the picture's render and delivery finish in the background. */ +async function settled(ctx: AgentContext, call: Promise): Promise { + const result = await call; + await ctx.imageDelivery?.whenIdle(); + return result; +} const handlers = { - day: (ctx: AgentContext, owner_id?: number) => handleRenderDayImage(ctx, { date: '2099-06-01', owner_id }), - week: (ctx: AgentContext, owner_id?: number) => handleRenderWeekImage(ctx, { week_start: '2099-06-01', owner_id }), - month: (ctx: AgentContext, owner_id?: number) => handleRenderMonthImage(ctx, { month: '2099-06', owner_id }), + day: (ctx: AgentContext, owner_id?: number) => + settled(ctx, handleRenderDayImage(ctx, { date: '2099-06-01', owner_id })), + week: (ctx: AgentContext, owner_id?: number) => + settled(ctx, handleRenderWeekImage(ctx, { week_start: '2099-06-01', owner_id })), + month: (ctx: AgentContext, owner_id?: number) => + settled(ctx, handleRenderMonthImage(ctx, { month: '2099-06', owner_id })), }; for (const [name, handler] of Object.entries(handlers)) { for (const userId of [1, 2, 3]) { @@ -174,47 +182,73 @@ for (const timezone of ['Pacific/Kiritimati', 'Etc/GMT+12', 'America/Los_Angeles for (const [name, handler] of Object.entries({ ...handlers, - table: (ctx: AgentContext) => handleRenderTable(ctx, { title: 'Table', markdown: '| A |\n|---|\n| B |' }), + table: (ctx: AgentContext) => + settled(ctx, handleRenderTable(ctx, { title: 'Table', markdown: '| A |\n|---|\n| B |' })), })) { test(`${name} delivers oversized PNG via actual sender document API`, async () => { const { Bot } = await import('gramio'); const { createTelegramSender } = await import('../../../../src/services/ai/telegram-sender.ts'); - const { png } = await import('../../../fixtures/png.ts'); const { ctx } = setup(1); const bot = new Bot('123:test'); const calls: string[] = []; + const documents: unknown[] = []; bot.api.sendPhoto = async () => { throw new Error('Oversized photo'); }; bot.api.sendDocument = async (params) => { calls.push(String(params.caption ?? '')); - expect(params.document).toBeInstanceOf(File); + documents.push(params.document); // checked below: a throw in here would be swallowed by the background delivery return { message_id: 42, date: 0, chat: { id: 1, type: 'private' } }; }; bot.api.pinChatMessage = async () => true; ctx.sender = createTelegramSender(bot); - ctx.renderService = { renderDirect: async () => png(2160, 9000) }; + ctx.imageDelivery = makeImageDelivery(async () => png(2160, 9000)).delivery; expect((await handler(ctx)).success).toBe(true); + await ctx.imageDelivery!.whenIdle(); expect(calls).toHaveLength(1); expect(calls[0]).toContain('PNG'); + expect(documents[0]).toBeInstanceOf(File); }); - test(`${name} cannot claim delivery without document capability`, async () => { - const { png } = await import('../../../fixtures/png.ts'); + test(`${name} cannot claim delivery without document capability: the user gets the reason as the one note`, async () => { const { ctx } = setup(1); - ctx.renderService = { renderDirect: async () => png(2160, 9000) }; - expect((await handler(ctx)).success).toBe(false); + const notes: string[] = []; + ctx.sender = { + ...ctx.sender!, + sendPhoto: async () => { + throw new Error('a photo must not be attempted for an oversized PNG'); + }, + sendMessage: async (_chatId: number, text: string) => { + notes.push(text); + return { message_id: 1 }; + }, + }; + ctx.imageDelivery = makeImageDelivery(async () => png(2160, 9000)).delivery; + // The tool call itself only queues; the undeliverable picture is told, not claimed. + expect((await handler(ctx)).success).toBe(true); + // No document sender is a defect of the transport, not a request that is too big: the failure note, not "too large". + expect(notes).toHaveLength(1); + expect(notes[0]).toContain('could not prepare the picture'); + expect(notes[0]).not.toContain('document delivery'); }); } -test('AI preserves the screenshot allocation error after queue serialization', async () => { +test('the screenshot allocation error, which crosses the queue as a bare message, still ends as the localized note', async () => { const { ctx } = setup(1); const message = 'Agenda image is too large for the screenshot allocation limit. Choose a shorter date range.'; - ctx.renderService = { - renderDirect: async () => { - throw new Error(message); + const notes: string[] = []; + ctx.sender = { + ...ctx.sender!, + sendMessage: async (_chatId: number, text: string) => { + notes.push(text); + return { message_id: 1 }; }, }; + ctx.imageDelivery = makeImageDelivery(async () => { + throw new Error(message); + }).delivery; const result = await handlers.day(ctx); - expect(result.success).toBe(false); - expect(result.error).toBe(message); + expect(result.success).toBe(true); + expect(notes).toHaveLength(1); + expect(notes[0]).toContain('too large to send'); + expect(notes[0]).not.toContain('allocation'); }); diff --git a/test/services/ai/tool-handlers/day-image-after-change.test.ts b/test/services/ai/tool-handlers/day-image-after-change.test.ts index cc4488789..761bf8637 100644 --- a/test/services/ai/tool-handlers/day-image-after-change.test.ts +++ b/test/services/ai/tool-handlers/day-image-after-change.test.ts @@ -1,6 +1,7 @@ // A day picture sent after calendar changes must show a changed upcoming day, not a past one (#506). import { Database } from 'bun:sqlite'; import { afterEach, beforeEach, expect, setSystemTime, test } from 'bun:test'; +import { t } from '../../../../src/config/constants.ts'; import { migrations } from '../../../../src/database/migrations.ts'; import { ChatHistoryRepository } from '../../../../src/database/repositories/chat-history.repository.ts'; import { EventRepository } from '../../../../src/database/repositories/event.repository.ts'; @@ -22,6 +23,7 @@ import { ConversationLogger } from '../../../../src/services/conversation-logger import { EventService } from '../../../../src/services/event/event-service.ts'; import { HolidayService } from '../../../../src/services/holiday/holiday-service.ts'; import type { ImageRenderJob } from '../../../../src/worker/image-render.queue.ts'; +import { makeImageDelivery } from '../../../fixtures/image-delivery.ts'; import { png } from '../../../fixtures/png.ts'; const USER_ID = 1001; @@ -69,12 +71,10 @@ function setup(timezone = TIMEZONE, messageText = 'cancel all my English lessons sendMessage: async () => ({ message_id: 1 }), sendPhoto: async () => ({ message_id: 2 }), }, - renderService: { - renderDirect: async (job: ImageRenderJob) => { - if (job.type === 'daily-agenda') renderedDates.push(job.data.date); - return png(); - }, - }, + imageDelivery: makeImageDelivery(async (job: ImageRenderJob) => { + if (job.type === 'daily-agenda') renderedDates.push(job.data.date); + return png(); + }).delivery, }; ctx.dayReferences = resolveTurnDayReferences(messageText, chatHistory.getRecent(USER_ID, 30), new Date(), timezone); return { ctx, lesson, renderedDates }; @@ -94,6 +94,7 @@ test('after deleting a past and an upcoming lesson, a past-day picture shows the expect((await deleteApproved(ctx, nextTuesday.id)).success).toBe(true); const result = await handleRenderDayImage(ctx, { date: '2026-09-01' }); + await ctx.imageDelivery!.whenIdle(); expect(result.success).toBe(true); expect(renderedDates).toEqual(['2026-09-29']); @@ -109,6 +110,7 @@ test('create and update in the same run steer a past-day picture to the earliest expect((await handleUpdateEvent(ctx, { event_id: moved.id, start_at: '2026-09-30T16:00:00Z' })).success).toBe(true); const result = await handleRenderDayImage(ctx, { date: '2026-09-01' }); + await ctx.imageDelivery!.whenIdle(); expect(renderedDates).toEqual(['2026-09-30']); expect(result.output).toContain('2026-09-30'); @@ -120,6 +122,7 @@ test('a snoozed event counts as a change for the day picture', async () => { expect(handleSnoozeEvent(ctx, { event_id: lateLesson.id, minutes: 10 }).success).toBe(true); const result = await handleRenderDayImage(ctx, { date: '2026-09-01' }); + await ctx.imageDelivery!.whenIdle(); expect(renderedDates).toEqual(['2026-09-29']); expect(result.output).toContain('2026-09-29'); @@ -129,10 +132,11 @@ test('a past-day picture in a run without changes renders the requested day', as const { ctx, renderedDates } = setup(); const result = await handleRenderDayImage(ctx, { date: '2026-09-01' }); + await ctx.imageDelivery!.whenIdle(); expect(result.success).toBe(true); expect(renderedDates).toEqual(['2026-09-01']); - expect(result.output).toBe('Day calendar image for 2026-09-01 has been sent to the chat.'); + expect(result.output).toBe(t('en').aiTools.meta.dayImageQueued('2026-09-01')); }); test('a past-day picture after changes only on past days renders the requested day', async () => { @@ -143,9 +147,10 @@ test('a past-day picture after changes only on past days renders the requested d expect((await deleteApproved(ctx, earlierTuesday.id)).success).toBe(true); const result = await handleRenderDayImage(ctx, { date: '2026-09-01' }); + await ctx.imageDelivery!.whenIdle(); expect(renderedDates).toEqual(['2026-09-01']); - expect(result.output).toBe('Day calendar image for 2026-09-01 has been sent to the chat.'); + expect(result.output).toBe(t('en').aiTools.meta.dayImageQueued('2026-09-01')); }); test('an upcoming-day picture is never redirected by other changed days', async () => { @@ -154,6 +159,7 @@ test('an upcoming-day picture is never redirected by other changed days', async expect((await deleteApproved(ctx, nextTuesday.id)).success).toBe(true); await handleRenderDayImage(ctx, { date: '2026-10-05' }); + await ctx.imageDelivery!.whenIdle(); expect(renderedDates).toEqual(['2026-10-05']); }); @@ -165,6 +171,7 @@ test('an all-day change west of UTC counts for the day written, not the day befo expect(created.success).toBe(true); const result = await handleRenderDayImage(ctx, { date: '2026-09-01' }); + await ctx.imageDelivery!.whenIdle(); expect(renderedDates).toEqual(['2026-09-29']); expect(result.output).toContain('2026-09-29'); @@ -177,9 +184,10 @@ test('a past day the user named for the picture is shown as asked, not replaced expect((await deleteApproved(ctx, nextTuesday.id)).success).toBe(true); const result = await handleRenderDayImage(ctx, { date: '2026-09-01' }); + await ctx.imageDelivery!.whenIdle(); expect(renderedDates).toEqual(['2026-09-01']); - expect(result.output).toBe('Day calendar image for 2026-09-01 has been sent to the chat.'); + expect(result.output).toBe(t('en').aiTools.meta.dayImageQueued('2026-09-01')); }); test('a past day the user named as the one to change still yields to the upcoming changed day', async () => { @@ -188,6 +196,7 @@ test('a past day the user named as the one to change still yields to the upcomin expect((await handleUpdateEvent(ctx, { event_id: moved.id, start_at: '2026-09-29T16:00:00Z' })).success).toBe(true); const result = await handleRenderDayImage(ctx, { date: '2026-09-01' }); + await ctx.imageDelivery!.whenIdle(); expect(renderedDates).toEqual(['2026-09-29']); expect(result.output).toContain('instead of the past day 2026-09-01'); @@ -201,6 +210,7 @@ test('a clock time alone names no day for the picture: a past-day picture still expect(created.success).toBe(true); const result = await handleRenderDayImage(ctx, { date: '2026-09-01' }); + await ctx.imageDelivery!.whenIdle(); expect(renderedDates).toEqual(['2026-09-27']); expect(result.output).toContain('instead of the past day 2026-09-01'); diff --git a/test/services/ai/tool-handlers/meta.test.ts b/test/services/ai/tool-handlers/meta.test.ts index 4a5fc2524..f3160d48f 100644 --- a/test/services/ai/tool-handlers/meta.test.ts +++ b/test/services/ai/tool-handlers/meta.test.ts @@ -29,6 +29,7 @@ import type { AgentContext } from '../../../../src/services/ai/types.ts'; import { EventService } from '../../../../src/services/event/event-service.ts'; import { HolidayService } from '../../../../src/services/holiday/holiday-service.ts'; import type { ImageRenderJob } from '../../../../src/worker/image-render.queue.ts'; +import { makeImageDelivery } from '../../../fixtures/image-delivery.ts'; import { png } from '../../../fixtures/png.ts'; function createTestDb() { @@ -590,12 +591,10 @@ describe('meta tool handlers', () => { beforeEach(() => { renderCalls = []; photoCalls = []; - ctx.renderService = { - renderDirect(job) { - renderCalls.push(job as unknown as ImageRenderJob); - return Promise.resolve(png()); - }, - }; + ctx.imageDelivery = makeImageDelivery((job) => { + renderCalls.push(job); + return Promise.resolve(png()); + }).delivery; ctx.sender = { sendMessage: (() => Promise.resolve({ message_id: 1 })) as never, editMessageText: (() => Promise.resolve()) as never, @@ -606,8 +605,8 @@ describe('meta tool handlers', () => { }; }); - test('returns error when renderService not available', async () => { - ctx.renderService = undefined; + test('returns error when image delivery is not available', async () => { + ctx.imageDelivery = undefined; const result = await handleRenderDayImage(ctx, { date: '2026-03-15' }); expect(result.success).toBe(false); expect(result.error).toContain('not available'); @@ -624,6 +623,9 @@ describe('meta tool handlers', () => { const result = await handleRenderDayImage(ctx, { date: '2026-03-15' }); expect(result.success).toBe(true); expect(result.output).toContain('2026-03-15'); + await ctx.imageDelivery!.whenIdle(); + expect(renderCalls).toHaveLength(1); + expect(photoCalls).toEqual([{ chatId: USER_ID }]); }); test('fetches group events when scope=group', async () => { @@ -653,6 +655,9 @@ describe('meta tool handlers', () => { const result = await handleRenderDayImage(gCtx, { date: '2026-03-15', scope: 'group' }); expect(result.success).toBe(true); expect(result.output).toContain('2026-03-15'); + await gCtx.imageDelivery!.whenIdle(); + // The group picture goes to the group chat, not to the requesting user's private chat. + expect(photoCalls).toEqual([{ chatId: GROUP_CHAT_ID }]); }); test('scope defaults to group when isGroup=true', async () => { diff --git a/test/services/ai/tool-handlers/render-image-guards.test.ts b/test/services/ai/tool-handlers/render-image-guards.test.ts index ca28be1aa..2b4723abc 100644 --- a/test/services/ai/tool-handlers/render-image-guards.test.ts +++ b/test/services/ai/tool-handlers/render-image-guards.test.ts @@ -1,6 +1,7 @@ -// Characterization tests for the shared guards and failure mapping of the day/week/month image handlers: -// secretary access denial, invalid dates, group-scope requirements, typed agenda-image errors and -// unexpected render failures reported with the localized fallback text. +// Characterization tests for the shared guards of the day/week/month image handlers (secretary access +// denial, invalid dates, group-scope requirements) and for how a failed picture reaches the user: the +// tool call only queues, so a typed agenda-image error or an unexpected render failure arrives as one +// short note in the chat, never as a failed tool call. import { describe, expect, mock, test } from 'bun:test'; import { t } from '../../../../src/config/constants.ts'; import type { UserRepository } from '../../../../src/database/repositories/user.repository.ts'; @@ -11,12 +12,13 @@ import { } from '../../../../src/services/ai/tool-handlers/render.ts'; import type { AgentContext } from '../../../../src/services/ai/types.ts'; import { AgendaImageError } from '../../../../src/utils/agenda-image.ts'; +import { makeImageDelivery } from '../../../fixtures/image-delivery.ts'; import { png } from '../../../fixtures/png.ts'; import { partialOf } from '../partial-stubs.ts'; const meta = t('en').aiTools.meta; -function makeCtx(renderDirect: () => Promise = () => Promise.resolve(png()), overrides = {}): AgentContext { +function makeCtx(renderDirect: () => Promise = () => Promise.resolve(png()), overrides = {}): AgentContext { return { user: { telegram_id: 1, language: 'en', timezone: 'UTC' }, chatId: 1, @@ -25,7 +27,7 @@ function makeCtx(renderDirect: () => Promise = () => Promise.resolve sendPhoto: mock(() => Promise.resolve({ message_id: 42 })), sendMessage: mock(() => Promise.resolve({ message_id: 1 })), }, - renderService: { renderDirect: mock(renderDirect) }, + imageDelivery: makeImageDelivery(renderDirect).delivery, eventService: { getEventsForDay: mock(() => []), getEventsInRange: mock(() => []), @@ -45,7 +47,7 @@ const handlers = [ describe.each(handlers)('%s image handler guards', (_name, run) => { test('unavailable renderer or sender is reported first', async () => { - const noRenderer = await run(makeCtx(undefined, { renderService: undefined })); + const noRenderer = await run(makeCtx(undefined, { imageDelivery: undefined })); expect(noRenderer).toEqual({ success: false, error: 'Image rendering not available.' }); const noPhoto = await run(makeCtx(undefined, { sender: {} })); expect(noPhoto).toEqual({ success: false, error: 'Image rendering not available.' }); @@ -74,17 +76,42 @@ describe.each(handlers)('%s image handler guards', (_name, run) => { expect(result).toEqual({ success: false, error: 'Group context required for group scope' }); }); - test('typed agenda image errors are surfaced verbatim', async () => { - const ctx = makeCtx(() => Promise.reject(new AgendaImageError('too tall'))); - expect(await run(ctx)).toEqual({ success: false, error: 'too tall' }); + test('a typed agenda image error reaches the user as one localized note, never its internal English text', async () => { + const ctx = makeCtx(() => Promise.reject(new AgendaImageError('too tall', true))); + const result = await run(ctx); + expect(result.success).toBe(true); + await ctx.imageDelivery!.whenIdle(); + expect(ctx.sender!.sendPhoto).not.toHaveBeenCalled(); + expect(ctx.sender!.sendMessage).toHaveBeenCalledTimes(1); + const note = (ctx.sender!.sendMessage as ReturnType).mock.calls[0]![1] as string; + expect(note).toContain('too large to send'); + expect(note).not.toContain('too tall'); + }); + + test('a supplement turn queues no picture: it could only block or cancel a real one', async () => { + const supplement = makeCtx(undefined, { supplementMode: true }); + const result = await run(supplement); + expect(result.success).toBe(false); + await supplement.imageDelivery!.whenIdle(); + expect(supplement.sender!.sendPhoto).not.toHaveBeenCalled(); + expect(supplement.sender!.sendMessage).not.toHaveBeenCalled(); + + // The same picture asked for by a real turn of the same chat is not seen as a duplicate of a hidden one. + const real = makeCtx(undefined, { imageDelivery: supplement.imageDelivery }); + expect(await run(real)).toMatchObject({ success: true }); + await real.imageDelivery!.whenIdle(); + expect(real.sender!.sendPhoto).toHaveBeenCalledTimes(1); }); - test('unexpected render failures use the localized fallback text', async () => { + test('unexpected render failures send one localized note without the internal error text', async () => { const ctx = makeCtx(() => Promise.reject(new Error('playwright down'))); const result = await run(ctx); - expect(result.success).toBe(false); - expect(result.error).toBeTruthy(); - expect(result.error).not.toContain('playwright down'); + expect(result.success).toBe(true); + await ctx.imageDelivery!.whenIdle(); + expect(ctx.sender!.sendMessage).toHaveBeenCalledTimes(1); + const note = (ctx.sender!.sendMessage as ReturnType).mock.calls[0]![1] as string; + expect(note).toContain('could not prepare the picture'); + expect(note).not.toContain('playwright down'); }); }); @@ -108,30 +135,54 @@ describe('handleRenderDayImage specifics', () => { expect(ctx.eventService.getEventsForDay).not.toHaveBeenCalled(); }); - test('success output names the date and tells the model not to re-render', async () => { + test('two members of one group asking for the same group day are two pictures, not a duplicate or a replacement', async () => { + const first = makeCtx(undefined, { groupChatId: -100 }); + const second = makeCtx(undefined, { + groupChatId: -100, + user: { telegram_id: 2, language: 'en', timezone: 'UTC' }, + imageDelivery: first.imageDelivery, + }); + expect(await handleRenderDayImage(first, { date: '2026-04-07', scope: 'group' })).toMatchObject({ success: true }); + expect(await handleRenderDayImage(second, { date: '2026-04-07', scope: 'group' })).toMatchObject({ success: true }); + await first.imageDelivery!.whenIdle(); + expect(first.sender!.sendPhoto).toHaveBeenCalledTimes(1); + expect(second.sender!.sendPhoto).toHaveBeenCalledTimes(1); + }); + + test('the result says queued, not sent, and tells the model not to re-render', async () => { const result = await handleRenderDayImage(makeCtx(), { date: '2026-04-07' }); - expect(result.output).toBe(meta.dayImageSent('2026-04-07')); + expect(result.output).toBe(meta.dayImageQueued('2026-04-07')); + // The output is what a workflow reply shows the user: it says "being prepared", never "sent". + expect(result.output).not.toMatch(/\bsent\b/); expect(result.agentHint).toBe( - 'The day image has already been delivered to the chat. Do NOT call render_day_image again for this date in this turn.', + 'The picture is only QUEUED, NOT sent yet: it arrives later as a separate message (or the user gets a short note if it fails). Do NOT call render_day_image again for this date in this turn, and do NOT say the picture was sent or attached, write your text answer now.', ); }); - test('fallback failure text names the requested date', async () => { - const result = await handleRenderDayImage( - makeCtx(() => Promise.reject(new Error('boom'))), - { date: '2026-04-07' }, - ); - expect(result.error).toBe(meta.dayImageFailed('2026-04-07')); + test('a day that cannot be mapped into a render job fails the tool call with the localized text', async () => { + const ctx = makeCtx(undefined, { + eventService: { + getEventsForDay: () => { + throw new Error('boom'); + }, + }, + }); + const result = await handleRenderDayImage(ctx, { date: '2026-04-07' }); + expect(result).toEqual({ success: false, error: meta.dayImageFailed('2026-04-07') }); }); }); describe('handleRenderWeekImage specifics', () => { - test('fallback failure text names the requested week_start', async () => { - const result = await handleRenderWeekImage( - makeCtx(() => Promise.reject(new Error('boom'))), - { week_start: '2026-04-08' }, - ); - expect(result.error).toBe(meta.weekImageFailed('2026-04-08')); + test('a week that cannot be read fails the tool call with the localized text', async () => { + const ctx = makeCtx(undefined, { + eventService: { + getEventsInRange: () => { + throw new Error('boom'); + }, + }, + }); + const result = await handleRenderWeekImage(ctx, { week_start: '2026-04-08' }); + expect(result).toEqual({ success: false, error: meta.weekImageFailed('2026-04-08') }); }); test('malformed or overflowing week_start is rejected with the localized failure text', async () => { @@ -154,11 +205,15 @@ describe('handleRenderWeekImage specifics', () => { }); describe('handleRenderMonthImage specifics', () => { - test('fallback failure text names the requested month', async () => { - const result = await handleRenderMonthImage( - makeCtx(() => Promise.reject(new Error('boom'))), - { month: '2026-04' }, - ); - expect(result.error).toBe(meta.monthImageFailed('2026-04')); + test('a month that cannot be read fails the tool call with the localized text', async () => { + const ctx = makeCtx(undefined, { + eventService: { + getEventsInRange: () => { + throw new Error('boom'); + }, + }, + }); + const result = await handleRenderMonthImage(ctx, { month: '2026-04' }); + expect(result).toEqual({ success: false, error: meta.monthImageFailed('2026-04') }); }); }); diff --git a/test/services/ai/tool-handlers/render-month.test.ts b/test/services/ai/tool-handlers/render-month.test.ts index 615a84cfb..6ac3c4f96 100644 --- a/test/services/ai/tool-handlers/render-month.test.ts +++ b/test/services/ai/tool-handlers/render-month.test.ts @@ -1,6 +1,7 @@ import { describe, expect, mock, test } from 'bun:test'; import { handleRenderMonthImage } from '../../../../src/services/ai/tool-handlers/render.ts'; import type { AgentContext } from '../../../../src/services/ai/types.ts'; +import { makeImageDelivery } from '../../../fixtures/image-delivery.ts'; import { png } from '../../../fixtures/png.ts'; function makeCtx(overrides: Partial = {}): AgentContext { @@ -12,9 +13,7 @@ function makeCtx(overrides: Partial = {}): AgentContext { sendPhoto: mock(() => Promise.resolve({ message_id: 42 })), sendMessage: mock(() => Promise.resolve({ message_id: 1 })), }, - renderService: { - renderDirect: mock(() => Promise.resolve(png())), - }, + imageDelivery: makeImageDelivery(() => Promise.resolve(png())).delivery, eventService: { getEventsInRange: mock(() => []), getEventsInRangeForGroup: mock(() => []), @@ -26,21 +25,22 @@ function makeCtx(overrides: Partial = {}): AgentContext { } describe('handleRenderMonthImage', () => { - test('returns success after render + send photo', async () => { + test('returns success once the picture is queued', async () => { const ctx = makeCtx(); const result = await handleRenderMonthImage(ctx, { month: '2026-04' }); expect(result.success).toBe(true); expect(result.output).toBeTruthy(); }); - test('output contains month and past-tense marker', async () => { + test('output names the month and says it is being prepared, never that it was sent', async () => { const ctx = makeCtx(); const result = await handleRenderMonthImage(ctx, { month: '2026-04' }); expect(result.output).toContain('2026-04'); - expect(result.output).toContain('отправлена'); + expect(result.output).toContain('готовится'); + expect(result.output).not.toContain('отправлена'); }); - test('sendPhoto is awaited before handler returns', async () => { + test('the photo is sent by the background delivery, after the handler returned', async () => { const sendPhoto = mock(() => Promise.resolve({ message_id: 42 })); const ctx = makeCtx({ sender: { @@ -49,6 +49,8 @@ describe('handleRenderMonthImage', () => { } as never, }); await handleRenderMonthImage(ctx, { month: '2026-04' }); + expect(sendPhoto).not.toHaveBeenCalled(); + await ctx.imageDelivery!.whenIdle(); expect(sendPhoto).toHaveBeenCalledTimes(1); }); @@ -57,10 +59,11 @@ describe('handleRenderMonthImage', () => { const result = await handleRenderMonthImage(ctx, { month: '2026-04' }); expect(result.agentHint).toBeDefined(); expect(result.agentHint!).toContain('Do NOT call render_month_image again'); + expect(result.agentHint!).toContain('do NOT say the picture was sent or attached'); }); - test('fails when renderService missing', async () => { - const ctx = makeCtx({ renderService: undefined }); + test('fails when image delivery is missing', async () => { + const ctx = makeCtx({ imageDelivery: undefined }); const result = await handleRenderMonthImage(ctx, { month: '2026-04' }); expect(result.success).toBe(false); }); @@ -94,14 +97,14 @@ describe('handleRenderMonthImage', () => { expect(result.success).toBe(true); }); - test('render failure returns success:false with error message', async () => { + test('a render failure is not a failed tool call: the user gets one note', async () => { const ctx = makeCtx({ - renderService: { - renderDirect: mock(() => Promise.reject(new Error('playwright down'))), - }, + imageDelivery: makeImageDelivery(() => Promise.reject(new Error('playwright down'))).delivery, }); const result = await handleRenderMonthImage(ctx, { month: '2026-04' }); - expect(result.success).toBe(false); - expect(result.error).toBeTruthy(); + expect(result.success).toBe(true); + await ctx.imageDelivery!.whenIdle(); + expect(ctx.sender!.sendMessage).toHaveBeenCalledTimes(1); + expect(ctx.sender!.sendPhoto).not.toHaveBeenCalled(); }); }); diff --git a/test/services/ai/tool-handlers/render-table.test.ts b/test/services/ai/tool-handlers/render-table.test.ts index 73a0bbf0b..837c5208c 100644 --- a/test/services/ai/tool-handlers/render-table.test.ts +++ b/test/services/ai/tool-handlers/render-table.test.ts @@ -1,6 +1,10 @@ import { describe, expect, mock, test } from 'bun:test'; +import { TelegramError } from 'gramio'; +import { t } from '../../../../src/config/constants.ts'; import { handleRenderTable } from '../../../../src/services/ai/tool-handlers/render.ts'; -import type { AgentContext } from '../../../../src/services/ai/types.ts'; +import type { AgentContext, TelegramSender } from '../../../../src/services/ai/types.ts'; +import { asDouble } from '../../../fixtures/doubles.ts'; +import { makeImageDelivery } from '../../../fixtures/image-delivery.ts'; import { png } from '../../../fixtures/png.ts'; function makeCtx(overrides: Partial = {}): AgentContext { @@ -12,9 +16,7 @@ function makeCtx(overrides: Partial = {}): AgentContext { sendPhoto: mock(() => Promise.resolve({ message_id: 1 })), sendMessage: mock(() => Promise.resolve({ message_id: 1 })), }, - renderService: { - renderDirect: mock(() => Promise.resolve(png())), - }, + imageDelivery: makeImageDelivery(() => Promise.resolve(png())).delivery, eventService: {} as never, userRepo: {} as never, holidayService: {} as never, @@ -23,7 +25,7 @@ function makeCtx(overrides: Partial = {}): AgentContext { } describe('handleRenderTable', () => { - test('returns success after render + send photo', async () => { + test('returns success once the table is queued', async () => { const ctx = makeCtx(); const result = await handleRenderTable(ctx, { title: 'Тест', @@ -33,27 +35,29 @@ describe('handleRenderTable', () => { expect(result.output).toBeTruthy(); }); - test('output contains the title and past-tense marker', async () => { + test('output names the title and says the table is being prepared, never that it was sent', async () => { const ctx = makeCtx(); const result = await handleRenderTable(ctx, { title: 'МойЗаголовок', markdown: '| A |\n|---|\n| 1 |', }); expect(result.output).toContain('МойЗаголовок'); - // Past tense — avoids the AI-loop class of bugs where models - // retry because they think the render hasn't completed. - expect(result.output).toContain('отправлена'); + expect(result.output).toContain('готовится'); + expect(result.output).not.toContain('отправлена'); + expect(result.agentHint).toContain('NOT sent yet'); }); - test('sendPhoto is awaited before handler returns', async () => { + test('the photo is sent by the background delivery, after the handler returned', async () => { const sendPhoto = mock(() => Promise.resolve({ message_id: 42 })); const ctx = makeCtx({ - sender: { + sender: asDouble({ sendPhoto, sendMessage: mock(() => Promise.resolve({ message_id: 1 })), - } as never, + }), }); await handleRenderTable(ctx, { title: 'T', markdown: '| A |\n|---|\n| 1 |' }); + expect(sendPhoto).not.toHaveBeenCalled(); + await ctx.imageDelivery!.whenIdle(); expect(sendPhoto).toHaveBeenCalledTimes(1); }); @@ -62,10 +66,11 @@ describe('handleRenderTable', () => { const result = await handleRenderTable(ctx, { title: 'T', markdown: '| A |\n|---|\n| 1 |' }); expect(result.agentHint).toBeDefined(); expect(result.agentHint!).toContain('Do NOT call render_table again'); + expect(result.agentHint!).toContain('do NOT say the picture was sent or attached'); }); - test('fails when renderService missing', async () => { - const ctx = makeCtx({ renderService: undefined }); + test('fails when image delivery is missing', async () => { + const ctx = makeCtx({ imageDelivery: undefined }); const result = await handleRenderTable(ctx, { title: 'Тест', markdown: '| A |\n|---|\n| 1 |', @@ -82,35 +87,50 @@ describe('handleRenderTable', () => { expect(result.success).toBe(false); }); - test('voice call: output tells user to check chat', async () => { - const ctx = makeCtx({ inputMode: 'live_call' }); - const result = await handleRenderTable(ctx, { - title: 'Тест', - markdown: '| A |\n|---|\n| 1 |', - }); - expect(result.success).toBe(true); - expect(result.output).toMatch(/чат|chat/i); + test('voice call: output tells the caller the picture follows in the chat, and only a live call hears it', async () => { + const voice = t('ru').aiTools.meta.tableRenderingVoice; + const args = { title: 'Тест', markdown: '| A |\n|---|\n| 1 |' }; + const live = await handleRenderTable(makeCtx({ inputMode: 'live_call' }), args); + expect(live.success).toBe(true); + expect(live.output).toContain(voice); + const typed = await handleRenderTable(makeCtx(), args); + expect(typed.output).not.toContain(voice); }); - test('render failure returns success:false with error message', async () => { + test('a render failure is not a failed tool call: the user gets one note', async () => { const ctx = makeCtx({ - renderService: { - renderDirect: mock(() => Promise.reject(new Error('playwright down'))), - }, + imageDelivery: makeImageDelivery(() => Promise.reject(new Error('playwright down'))).delivery, }); const result = await handleRenderTable(ctx, { title: 'T', markdown: '| A |\n|---|\n| 1 |' }); - expect(result.success).toBe(false); - expect(result.error).toBeTruthy(); + expect(result.success).toBe(true); + await ctx.imageDelivery!.whenIdle(); + expect(ctx.sender!.sendMessage).toHaveBeenCalledTimes(1); + expect(ctx.sender!.sendPhoto).not.toHaveBeenCalled(); }); - test('sendPhoto failure returns success:false with error message', async () => { + test('an upload Telegram refused sends one note and is not reported as delivered', async () => { + const sendMessage = mock(() => Promise.resolve({ message_id: 1 })); + const refused = new TelegramError({ ok: false, error_code: 429, description: 'Too Many Requests' }, 'sendPhoto', { + chat_id: 1, + photo: 'photo', + }); const ctx = makeCtx({ - sender: { - sendPhoto: mock(() => Promise.reject(new Error('Telegram 429'))), - sendMessage: mock(() => Promise.resolve({ message_id: 1 })), - } as never, + sender: asDouble({ sendPhoto: mock(() => Promise.reject(refused)), sendMessage }), }); const result = await handleRenderTable(ctx, { title: 'T', markdown: '| A |\n|---|\n| 1 |' }); - expect(result.success).toBe(false); + expect(result.success).toBe(true); + await ctx.imageDelivery!.whenIdle(); + expect(sendMessage).toHaveBeenCalledTimes(1); + }); + + test('an upload that ended in a dropped connection may have landed: the note says it may not have arrived, not that it failed', async () => { + const sendMessage = mock((_chatId: number, text: string) => Promise.resolve({ message_id: 1, text })); + const sendPhoto = mock(() => Promise.reject(new Error('socket hang up'))); + const ctx = makeCtx({ sender: asDouble({ sendPhoto, sendMessage }) }); + await handleRenderTable(ctx, { title: 'T', markdown: '| A |\n|---|\n| 1 |' }); + await ctx.imageDelivery!.whenIdle(); + expect(sendPhoto).toHaveBeenCalledTimes(1); // the upload was attempted, then cut off + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage).toHaveBeenCalledWith(1, t('ru').aiTools.meta.imageNoteUncertain('T')); }); }); diff --git a/test/services/ai/tool-handlers/render-week.test.ts b/test/services/ai/tool-handlers/render-week.test.ts index 7ecfb3501..8a714d828 100644 --- a/test/services/ai/tool-handlers/render-week.test.ts +++ b/test/services/ai/tool-handlers/render-week.test.ts @@ -1,8 +1,12 @@ import { describe, expect, mock, test } from 'bun:test'; +import { t } from '../../../../src/config/constants.ts'; import { handleRenderWeekImage } from '../../../../src/services/ai/tool-handlers/render.ts'; import type { AgentContext } from '../../../../src/services/ai/types.ts'; +import { makeImageDelivery } from '../../../fixtures/image-delivery.ts'; import { png } from '../../../fixtures/png.ts'; +const meta = (lang: 'ru' | 'en') => t(lang).aiTools.meta; + function makeCtx( overrides: Omit, 'user'> & { user?: Partial } = {}, ): AgentContext { @@ -15,9 +19,7 @@ function makeCtx( sendPhoto: mock(() => Promise.resolve({ message_id: 42 })), sendMessage: mock(() => Promise.resolve({ message_id: 1 })), }, - renderService: { - renderDirect: mock(() => Promise.resolve(png())), - }, + imageDelivery: makeImageDelivery(() => Promise.resolve(png())).delivery, eventService: { getEventsInRange: mock(() => []), getEventsInRangeForGroup: mock(() => []), @@ -29,7 +31,7 @@ function makeCtx( } describe('handleRenderWeekImage', () => { - test('returns success after render + send photo', async () => { + test('returns success once the picture is queued', async () => { const ctx = makeCtx(); const result = await handleRenderWeekImage(ctx, { week_start: '2026-04-07' }); expect(result.success).toBe(true); @@ -40,13 +42,13 @@ describe('handleRenderWeekImage', () => { const ctx = makeCtx(); // 2026-04-06 is Monday const result = await handleRenderWeekImage(ctx, { week_start: '2026-04-06' }); - expect(result.output).toBe('Календарь на 6–12 апреля 2026 — картинка в чате.'); + expect(result.output).toBe(meta('ru').weekImageQueued('6–12 апреля 2026')); const english = makeCtx({ user: { telegram_id: 1, language: 'en', timezone: 'Europe/Moscow' } }); const inEnglish = await handleRenderWeekImage(english, { week_start: '2026-04-06' }); - expect(inEnglish.output).toBe('Calendar for April 6–12, 2026 — the picture is in the chat.'); + expect(inEnglish.output).toBe(meta('en').weekImageQueued('April 6–12, 2026')); }); - test('sendPhoto is awaited before handler returns', async () => { + test('the photo is sent by the background delivery, after the handler returned', async () => { const sendPhoto = mock(() => Promise.resolve({ message_id: 42 })); const ctx = makeCtx({ sender: { @@ -55,6 +57,8 @@ describe('handleRenderWeekImage', () => { } as never, }); await handleRenderWeekImage(ctx, { week_start: '2026-04-07' }); + expect(sendPhoto).not.toHaveBeenCalled(); + await ctx.imageDelivery!.whenIdle(); expect(sendPhoto).toHaveBeenCalledTimes(1); }); @@ -63,10 +67,11 @@ describe('handleRenderWeekImage', () => { const result = await handleRenderWeekImage(ctx, { week_start: '2026-04-07' }); expect(result.agentHint).toBeDefined(); expect(result.agentHint!).toContain('Do NOT call render_week_image again'); + expect(result.agentHint!).toContain('do NOT say the picture was sent or attached'); }); - test('fails when renderService missing', async () => { - const ctx = makeCtx({ renderService: undefined }); + test('fails when image delivery is missing', async () => { + const ctx = makeCtx({ imageDelivery: undefined }); const result = await handleRenderWeekImage(ctx, { week_start: '2026-04-07' }); expect(result.success).toBe(false); }); @@ -94,15 +99,15 @@ describe('handleRenderWeekImage', () => { expect(getEventsInRangeForGroup).toHaveBeenCalled(); }); - test('render failure returns success:false with error message', async () => { + test('a render failure is not a failed tool call: the user gets one note', async () => { const ctx = makeCtx({ - renderService: { - renderDirect: mock(() => Promise.reject(new Error('playwright down'))), - }, + imageDelivery: makeImageDelivery(() => Promise.reject(new Error('playwright down'))).delivery, }); const result = await handleRenderWeekImage(ctx, { week_start: '2026-04-07' }); - expect(result.success).toBe(false); - expect(result.error).toBeTruthy(); + expect(result.success).toBe(true); + await ctx.imageDelivery!.whenIdle(); + expect(ctx.sender!.sendMessage).toHaveBeenCalledTimes(1); + expect(ctx.sender!.sendPhoto).not.toHaveBeenCalled(); }); test('normalizes non-Monday week_start to Monday of same week', async () => { diff --git a/test/services/image/image-delivery-durable.test.ts b/test/services/image/image-delivery-durable.test.ts new file mode 100644 index 000000000..4269c3025 --- /dev/null +++ b/test/services/image/image-delivery-durable.test.ts @@ -0,0 +1,1091 @@ +// Pictures survive a process death: the delivery record is written before each step and the next start +// reconciles it (deliver once / expire with one note / mark unknown and never re-send). A fake BullMQ +// (a job-id keyed map shared by "both processes") and fake Telegram functions; the store is the real +// repository over SQLite with the real migration. +import { Database } from 'bun:sqlite'; +import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from 'bun:test'; +import { TelegramError } from 'gramio'; +import { t } from '../../../src/config/constants.ts'; +import { migrations } from '../../../src/database/migrations.ts'; +import { ImageDeliveryRepository } from '../../../src/database/repositories/image-delivery.repository.ts'; +import { runMigrations } from '../../../src/database/schema.ts'; +import { + ImageDelivery, + type ImageDeliveryDeps, + type ImageDeliveryRequest, + type RenderFailureReport, +} from '../../../src/services/image/image-delivery.ts'; +import { + RenderAbortedError, + type RenderOptions, + RenderTimeoutError, +} from '../../../src/services/image/render-service.ts'; +import { configureTimeoutReporter, TimeoutReporter } from '../../../src/services/incident/timeout-reporter.ts'; +import { imageLogger } from '../../../src/utils/logger.ts'; +import type { ImageRenderJob } from '../../../src/worker/image-render.queue.ts'; +import { asLoggedOutcome, type LoggedOutcome } from '../../fixtures/image-delivery.ts'; +import { png } from '../../fixtures/png.ts'; + +const flush = () => new Promise((resolve) => setImmediate(resolve)); +const tr = t('ru').aiTools.meta; + +/** What survives a restart besides SQLite: Redis. Adding a job id Redis already has returns that job. */ +class FakeQueue { + readonly jobs = new Map; adds: number }>(); + renderDirect = (job: ImageRenderJob, options?: RenderOptions): Promise => { + const jobId = options?.jobId ?? 'no-id'; + const known = this.jobs.get(jobId) ?? { job, result: Promise.withResolvers(), adds: 0 }; + known.adds++; + this.jobs.set(jobId, known); + const { promise, resolve, reject } = Promise.withResolvers(); + options?.signal?.addEventListener('abort', () => reject(new RenderAbortedError()), { once: true }); + known.result.promise.then(resolve, reject); + return promise; + }; +} + +/** A process that dies right after queueing: Redis has the job, nobody in this process waits for it. */ +const dying = (q: FakeQueue) => ({ + renderDirect: (job: ImageRenderJob, options?: RenderOptions) => { + void q.renderDirect(job, options).catch(() => {}); + return new Promise(() => {}); + }, +}); + +let db: Database; +let store: ImageDeliveryRepository; +let queue: FakeQueue; +let alerts: string[]; +let outcomes: LoggedOutcome[]; +let failures: RenderFailureReport[]; +let infoSpy: ReturnType; +let errorSpy: ReturnType; +/** Every delivery the test created: each one still pending holds a render or an upload timer. */ +let deliveries: ImageDelivery[]; + +beforeEach(() => { + db = new Database(':memory:'); + runMigrations(db, migrations); + store = new ImageDeliveryRepository(db); + queue = new FakeQueue(); + alerts = []; + outcomes = []; + failures = []; + deliveries = []; + configureTimeoutReporter(new TimeoutReporter((text) => alerts.push(text))); + infoSpy = spyOn(imageLogger, 'info').mockImplementation((obj: unknown, message?: string) => { + if (message === 'Image delivery outcome') outcomes.push(asLoggedOutcome(obj)); + }); + errorSpy = spyOn(imageLogger, 'error').mockImplementation(() => {}); +}); + +afterEach(async () => { + // Cancel what is still pending BEFORE the database closes: an abandoned delivery keeps its 30 s upload and + // 150 s render timers, which would otherwise fire inside whichever test file runs next (a stray outcome + // log line, a timeout alert) and write to a closed database. + await Promise.all(deliveries.map((delivery) => delivery.close(0))); + configureTimeoutReporter(null); + infoSpy.mockRestore(); + errorSpy.mockRestore(); + db.close(); +}); + +function track(delivery: ImageDelivery): ImageDelivery { + deliveries.push(delivery); + return delivery; +} + +/** One bot process: its own ImageDelivery over the shared SQLite and Redis. */ +function startProcess(now?: () => number, dies = false, extra: Partial = {}) { + const delivery = track( + new ImageDelivery({ + renderer: dies ? dying(queue) : queue, + store, + reportFailure: (report) => failures.push(report), + now, + ...extra, + }), + ); + const sentImages: { chatId: number; language: string }[] = []; + const notes: { chatId: number; text: string }[] = []; + delivery.attachTransport({ + sendImage: async (chatId, _png, language) => { + sentImages.push({ chatId, language }); + return 500 + sentImages.length; + }, + notify: async (chatId, text) => { + notes.push({ chatId, text }); + }, + }); + return { delivery, sentImages, notes }; +} + +function request(overrides: Partial = {}): ImageDeliveryRequest { + return { + chatId: 7, + language: 'ru', + kind: 'day', + label: '2026-11-02', + slot: 'day:user:1:2026-11-02', + requestId: 900, + dataOwner: 1, + buildJob: () => + ({ type: 'daily-agenda', data: { date: '2026-11-02', theme: {}, locale: 'ru' }, userId: 1 }) as ImageRenderJob, + send: async () => 1, + notify: async () => {}, + ...overrides, + }; +} + +const onlyRow = () => { + const rows = db + .query<{ id: string; state: string; message_id: number | null; job_json: string | null }, []>( + 'SELECT * FROM image_deliveries', + ) + .all(); + expect(rows).toHaveLength(1); + return rows[0]!; +}; + +describe('a delivery is recorded before it acts', () => { + test('queued at enqueue, then rendered and sending before the upload, delivered with the Telegram message id', async () => { + const seen: string[] = []; + const process1 = startProcess(); + process1.delivery.enqueue( + request({ + send: async () => { + seen.push(onlyRow().state); // what a kill at this instant would leave behind + return 4242; + }, + }), + ); + expect(onlyRow().state).toBe('queued'); + expect(onlyRow().job_json).not.toBeNull(); + [...queue.jobs.values()][0]!.result.resolve(png()); + await process1.delivery.whenIdle(); + + expect(seen).toEqual(['sending']); + expect(onlyRow()).toMatchObject({ state: 'delivered', message_id: 4242, job_json: null }); + }); + + test('the BullMQ job id is the deterministic delivery id, and the same request twice is one record', async () => { + const process1 = startProcess(); + expect(process1.delivery.enqueue(request())).toEqual({ status: 'queued' }); + expect(process1.delivery.enqueue(request())).toEqual({ status: 'duplicate' }); + expect([...queue.jobs.keys()]).toEqual([onlyRow().id]); + expect(onlyRow().id).toStartWith('img-'); + process1.delivery.attachTransport({ sendImage: async () => 1, notify: async () => {} }); + // Another process (a restart) enqueueing the identical live request is also recognised. + const process2 = startProcess(); + expect(process2.delivery.enqueue(request())).toEqual({ status: 'duplicate' }); + }); +}); + +describe('crash after the render, before the send', () => { + test('the next start delivers the finished picture exactly once', async () => { + const process1 = startProcess(undefined, true); + process1.delivery.enqueue(request()); + const [jobId] = [...queue.jobs.keys()]; + // process 1 dies here (never closed, its wait is abandoned) while the worker finishes the job. + queue.jobs.get(jobId!)!.result.resolve(png()); + + const process2 = startProcess(); + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + + expect(process2.sentImages).toEqual([{ chatId: 7, language: 'ru' }]); + expect(process2.notes).toHaveLength(0); + expect(onlyRow()).toMatchObject({ state: 'delivered', message_id: 501 }); + expect(outcomes.filter((o) => o.outcome === 'delivered')).toHaveLength(1); + // The same job id was awaited again, not a second render. + expect(queue.jobs.get(jobId!)!.adds).toBe(2); + expect(queue.jobs.size).toBe(1); + + // A further restart finds nothing to do. + const process3 = startProcess(); + await process3.delivery.reconcile(); + await process3.delivery.whenIdle(); + expect(process3.sentImages).toHaveLength(0); + expect(process3.notes).toHaveLength(0); + }); + + test('a render still running when the bot returns is waited for, not duplicated', async () => { + startProcess(undefined, true).delivery.enqueue(request()); + const process2 = startProcess(); + await process2.delivery.reconcile(); + expect(process2.sentImages).toHaveLength(0); + [...queue.jobs.values()][0]!.result.resolve(png()); + await process2.delivery.whenIdle(); + expect(process2.sentImages).toHaveLength(1); + expect(queue.jobs.size).toBe(1); + }); + + describe('killed between the end of the render and the claim on the send (state rendered)', () => { + /** What the store holds when the process died after `advance(rendered)` and before `advance(sending)`. */ + function killedAfterRender() { + startProcess(undefined, true).delivery.enqueue(request()); + db.run("UPDATE image_deliveries SET state = 'rendered'"); + expect(onlyRow().state).toBe('rendered'); + [...queue.jobs.values()][0]!.result.resolve(png()); // the worker finished the job while the bot was down + } + + test('within the bound the next start delivers the picture once, with no note', async () => { + killedAfterRender(); + const process2 = startProcess(); + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + + expect(process2.sentImages).toEqual([{ chatId: 7, language: 'ru' }]); + expect(process2.notes).toEqual([]); + expect(onlyRow()).toMatchObject({ state: 'delivered', message_id: 501, job_json: null }); + + const process3 = startProcess(); + await process3.delivery.reconcile(); + expect(process3.sentImages).toHaveLength(0); // never twice + }); + + test('past the bound it expires with one "the bot was restarting" note and no timeout alert, and no picture', async () => { + killedAfterRender(); + const late = () => Date.now() + 5 * 60_000; // past the 150 s bound, within the ten minutes a user still waits + const process2 = startProcess(late); + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + + expect(process2.sentImages).toEqual([]); + // A row still `queued` past its bound means the previous process stopped before the bound: downtime, not a slow render. + expect(process2.notes).toEqual([{ chatId: 7, text: tr.imageNoteInterrupted('2026-11-02') }]); + expect(alerts).toEqual([]); + expect(onlyRow()).toMatchObject({ state: 'expired', note_sent: 1, job_json: null }); + + const process3 = startProcess(late); + await process3.delivery.reconcile(); + expect(process3.notes).toEqual([]); + expect(alerts).toEqual([]); + }); + }); +}); + +describe('crash after the send started, before it was recorded', () => { + test('the picture may have landed: marked unknown, never re-sent, the user is not told it failed', async () => { + const process1 = startProcess(); + process1.delivery.enqueue(request({ send: () => new Promise(() => {}) })); // killed mid-upload + [...queue.jobs.values()][0]!.result.resolve(png()); + await flush(); + expect(onlyRow().state).toBe('sending'); + + const process2 = startProcess(); + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + + expect(onlyRow().state).toBe('unknown'); + expect(process2.sentImages).toHaveLength(0); + expect(process2.notes).toHaveLength(0); + // The repair flow hears of the uncertainty, once. + expect(failures).toEqual([expect.objectContaining({ kind: 'day', errorName: 'UncertainSend' })]); + expect(outcomes).toContainEqual( + expect.objectContaining({ outcome: 'unknown', delivery: 'unknown', recovered: true }), + ); + + const process3 = startProcess(); + await process3.delivery.reconcile(); + expect(process3.sentImages).toHaveLength(0); + expect(failures).toHaveLength(1); + }); +}); + +describe('an expired delivery', () => { + test('one honest note, no timeout alert, recorded as expired; a later restart says nothing more', async () => { + const created = Date.now(); + startProcess(undefined, true).delivery.enqueue(request()); // the bot dies right after queueing + const late = () => created + 10 * 60_000; // it comes back far past the 150 s bound + + const process2 = startProcess(late); + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + + expect(process2.notes).toEqual([{ chatId: 7, text: tr.imageNoteInterrupted('2026-11-02') }]); + expect(process2.sentImages).toHaveLength(0); + expect(alerts).toEqual([]); + expect(onlyRow()).toMatchObject({ state: 'expired', job_json: null }); + expect(db.query<{ note_sent: number }, []>('SELECT note_sent FROM image_deliveries').get()?.note_sent).toBe(1); + + const process3 = startProcess(late); + await process3.delivery.reconcile(); + expect(process3.notes).toHaveLength(0); + expect(alerts).toEqual([]); + }); + + test('an expiry the previous process recorded itself (it died before telling the user) is still a slow render', async () => { + startProcess(undefined, true).delivery.enqueue(request()); + db.run("UPDATE image_deliveries SET state = 'expired', job_json = NULL, note_sent = 0"); + const process2 = startProcess(); + await process2.delivery.reconcile(); + + expect(process2.notes).toEqual([{ chatId: 7, text: tr.imageNoteTimedOut('2026-11-02') }]); + }); + + test('a record without its payload cannot be rendered: expired, not guessed', async () => { + startProcess(undefined, true).delivery.enqueue(request()); + db.run('UPDATE image_deliveries SET job_json = NULL'); + const process2 = startProcess(); + await process2.delivery.reconcile(); + expect(onlyRow().state).toBe('expired'); + expect(process2.notes).toHaveLength(1); + }); + + test('the schedule comparison, which nobody can ask for again, is told without the offer to ask again', async () => { + startProcess(undefined, true).delivery.enqueue(request({ kind: 'conflict', label: 'сравнение расписаний' })); + const process2 = startProcess(() => Date.now() + 5 * 60_000); + await process2.delivery.reconcile(); + expect(process2.notes).toEqual([{ chatId: 7, text: tr.imageNoteInterruptedFinal('сравнение расписаний') }]); + expect(process2.notes[0]!.text).not.toContain('ещё раз'); + }); +}); + +describe('a delivery that ended without its user being told', () => { + test('the next start sends the one missing note, and no more after that', async () => { + startProcess(undefined, true).delivery.enqueue(request()); + // The previous process recorded the failure and died before the note went out. + db.run("UPDATE image_deliveries SET state = 'failed', job_json = NULL, note_sent = 0"); + + const process2 = startProcess(); + await process2.delivery.reconcile(); + expect(process2.notes).toEqual([{ chatId: 7, text: tr.imageNoteFailed('2026-11-02') }]); + + const process3 = startProcess(); + await process3.delivery.reconcile(); + expect(process3.notes).toHaveLength(0); + }); + + test('an uncertain send is never reported to the user as a failure', async () => { + startProcess(undefined, true).delivery.enqueue(request()); + db.run("UPDATE image_deliveries SET state = 'unknown', job_json = NULL, note_sent = 0"); + const process2 = startProcess(); + await process2.delivery.reconcile(); + expect(process2.notes).toHaveLength(0); + }); +}); + +describe('the claim on the send', () => { + test('a delivery another attempt already claimed is not uploaded again', async () => { + const send = mock(async () => 1); + const process1 = startProcess(); + process1.delivery.enqueue(request({ send })); + db.run("UPDATE image_deliveries SET state = 'sending'"); // a redelivered attempt got there first + [...queue.jobs.values()][0]!.result.resolve(png()); + await process1.delivery.whenIdle(); + expect(send).not.toHaveBeenCalled(); + expect(onlyRow().state).toBe('sending'); + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'claimed_elsewhere', delivery: 'not_started' })]); + }); +}); + +describe('graceful shutdown with a durable record', () => { + test('no note is sent: the queued job is kept and the next start delivers the picture', async () => { + const process1 = startProcess(); + const requestNotes: string[] = []; + // A live delivery notifies through its own request closure, so that is the channel to watch. + process1.delivery.enqueue(request({ notify: async (text) => void requestNotes.push(text) })); + const notes: string[] = []; + process1.delivery.attachTransport({ sendImage: async () => 1, notify: async (_c, text) => void notes.push(text) }); + await process1.delivery.close(); + await process1.delivery.whenIdle(); + + expect(requestNotes).toHaveLength(0); + expect(notes).toHaveLength(0); + expect(onlyRow().state).toBe('queued'); + expect(outcomes).toContainEqual(expect.objectContaining({ outcome: 'interrupted', resumable: true })); + + [...queue.jobs.values()][0]!.result.resolve(png()); + const process2 = startProcess(); + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + expect(process2.sentImages).toHaveLength(1); + expect(onlyRow().state).toBe('delivered'); + }); + + test('a shutdown right after a newer picture replaced the older one keeps the older record superseded: the restart delivers only the newest', async () => { + const dayJob = (date: string) => () => + ({ type: 'daily-agenda', data: { date, theme: {}, locale: 'ru' }, userId: 1 }) as ImageRenderJob; + const process1 = startProcess(); + process1.delivery.enqueue(request({ requestId: 1, buildJob: dayJob('2026-11-02') })); + process1.delivery.enqueue(request({ requestId: 2, buildJob: dayJob('2026-11-03') })); // same slot, newer data + await process1.delivery.close(0); // before the older run has unwound + await process1.delivery.whenIdle(); + + const states = db.query<{ state: string }, []>('SELECT state FROM image_deliveries').all(); + expect(states.map((row) => row.state).sort()).toEqual(['queued', 'superseded']); + + for (const job of queue.jobs.values()) job.result.resolve(png()); + const process2 = startProcess(); + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + expect(process2.sentImages).toHaveLength(1); + }); +}); + +describe('the same request after its delivery ended', () => { + // `superseded` is not here: that picture never uploaded and nobody was told, so the same request goes on + // (see 'a request replaced within its own turn and asked for again goes on'). + test.each([ + 'delivered', + 'failed', + 'expired', + 'unknown', + ])('a request that already ended %s is settled: nothing is queued and it is not called "being prepared"', (state) => { + const process1 = startProcess(); + expect(process1.delivery.enqueue(request())).toEqual({ status: 'queued' }); + db.run('UPDATE image_deliveries SET state = ?, job_json = NULL', [state]); + + const process2 = startProcess(); + expect(process2.delivery.enqueue(request())).toEqual({ status: 'settled' }); + expect([...queue.jobs.values()].map((job) => job.adds)).toEqual([1]); + expect(onlyRow().state).toBe(state); + }); + + test('a different message asking for the same picture is a new delivery', () => { + const process1 = startProcess(); + process1.delivery.enqueue(request({ requestId: 900 })); + db.run("UPDATE image_deliveries SET state = 'delivered', job_json = NULL"); + const process2 = startProcess(); + expect(process2.delivery.enqueue(request({ requestId: 901 }))).toEqual({ status: 'queued' }); + }); +}); + +describe('supersede and rebuild with a durable record', () => { + const dayJob = (date: string) => () => + ({ type: 'daily-agenda', data: { date, theme: {}, locale: 'ru' }, userId: 1 }) as ImageRenderJob; + + test('a newer picture of the slot ends the older record as superseded, so a restart never delivers both', async () => { + const process1 = startProcess(); + const sends: string[] = []; + const record = (who: string, id: number) => async () => { + sends.push(who); + return id; + }; + process1.delivery.enqueue(request({ requestId: 900, buildJob: dayJob('2026-11-02'), send: record('older', 1) })); + process1.delivery.enqueue(request({ requestId: 901, buildJob: dayJob('2026-11-03'), send: record('newer', 2) })); + await flush(); // the older wait ends on its abort and records `superseded` + const rows = db.query<{ state: string }, []>('SELECT state FROM image_deliveries').all(); + expect(rows.map((row) => row.state).sort()).toEqual(['queued', 'superseded']); + + for (const job of queue.jobs.values()) job.result.resolve(png()); + await process1.delivery.whenIdle(); + expect(sends).toEqual(['newer']); + expect(db.query<{ state: string }, []>('SELECT state FROM image_deliveries ORDER BY state').all()).toEqual([ + { state: 'delivered' }, + { state: 'superseded' }, + ]); + + const process2 = startProcess(); + await process2.delivery.reconcile(); + expect(process2.sentImages).toHaveLength(0); + expect(process2.notes).toHaveLength(0); // a superseded record never earns a failure note + }); + + test('a request replaced within its own turn and asked for again goes on: it is not told "already handled"', async () => { + const process1 = startProcess(); + const sends: string[] = []; + const record = (who: string) => async () => { + sends.push(who); + return 1; + }; + // One message: the picture, the data changes, the picture again, the data goes back, the picture a third time. + expect( + process1.delivery.enqueue(request({ requestId: 900, buildJob: dayJob('2026-11-02'), send: record('first') })), + ).toEqual({ status: 'queued' }); + expect( + process1.delivery.enqueue(request({ requestId: 900, buildJob: dayJob('2026-11-03'), send: record('second') })), + ).toEqual({ status: 'queued' }); + await flush(); // the first one ends `superseded` + expect( + process1.delivery.enqueue(request({ requestId: 900, buildJob: dayJob('2026-11-02'), send: record('third') })), + ).toEqual({ status: 'queued' }); + + for (const job of queue.jobs.values()) job.result.resolve(png()); + await process1.delivery.whenIdle(); + expect(sends).toEqual(['third']); // the newest one only + expect( + db + .query<{ state: string }, []>('SELECT state FROM image_deliveries ORDER BY state') + .all() + .map((row) => row.state), + ).toEqual(['delivered', 'superseded']); + }); + + test('a replaced delivery that is still unwinding makes the same request again a duplicate, not a second row', async () => { + const process1 = startProcess(); + process1.delivery.enqueue(request({ requestId: 900, buildJob: dayJob('2026-11-02') })); + process1.delivery.enqueue(request({ requestId: 900, buildJob: dayJob('2026-11-03') })); // supersedes, not yet unwound + expect(process1.delivery.enqueue(request({ requestId: 900, buildJob: dayJob('2026-11-02') }))).toEqual({ + status: 'duplicate', + }); + }); + + test('a redelivered update whose picture already ended is settled even when the chat is at its limit', async () => { + const process1 = startProcess(); + expect(process1.delivery.enqueue(request({ requestId: 900, slot: 'day:a' }))).toEqual({ status: 'queued' }); + [...queue.jobs.values()][0]!.result.resolve(png()); + await process1.delivery.whenIdle(); + // Two other pictures now fill the chat's places. + process1.delivery.enqueue(request({ requestId: 901, slot: 'day:b', buildJob: dayJob('2026-11-04') })); + process1.delivery.enqueue(request({ requestId: 902, slot: 'day:c', buildJob: dayJob('2026-11-05') })); + expect( + process1.delivery.enqueue(request({ requestId: 903, slot: 'day:d', buildJob: dayJob('2026-11-06') })), + ).toEqual({ status: 'busy' }); + + expect(process1.delivery.enqueue(request({ requestId: 900, slot: 'day:a' }))).toEqual({ status: 'settled' }); + }); + + test('a calendar write while it renders rebuilds under a NEW job id, so the stale finished job is not reused', async () => { + const process1 = startProcess(); + let date = '2026-11-02'; + const send = mock(async () => 7); + process1.delivery.enqueue(request({ buildJob: () => dayJob(date)(), send })); + const [firstId] = [...queue.jobs.keys()]; + + date = '2026-11-03'; // the next message moved the event + process1.delivery.eventsChanged(1); + queue.jobs.get(firstId!)!.result.resolve(png()); + await flush(); + + expect(send).not.toHaveBeenCalled(); + const rebuiltId = `${firstId}-r1`; + expect([...queue.jobs.keys()]).toEqual([firstId!, rebuiltId]); + expect(JSON.stringify(queue.jobs.get(rebuiltId)!.job.data)).toContain('2026-11-03'); + + queue.jobs.get(rebuiltId)!.result.resolve(png()); + await process1.delivery.whenIdle(); + expect(send).toHaveBeenCalledTimes(1); + expect(onlyRow()).toMatchObject({ state: 'delivered', message_id: 7 }); + }); + + test('a finished picture is still sent when the rebuild cannot finish', async () => { + const process1 = startProcess(); + const send = mock(async () => 8); + let date = '2026-11-02'; + const requestNotes: string[] = []; + process1.delivery.enqueue( + request({ + buildJob: () => dayJob(date)(), + send, + notify: async (text) => void requestNotes.push(text), + }), + ); + const [firstId] = [...queue.jobs.keys()]; + date = '2026-11-03'; + process1.delivery.eventsChanged(1); + queue.jobs.get(firstId!)!.result.resolve(png()); + await flush(); + queue.jobs.get(`${firstId}-r1`)!.result.reject(new Error('worker crashed')); // the rebuild fails + await process1.delivery.whenIdle(); + + expect(send).toHaveBeenCalledTimes(1); + expect(onlyRow().state).toBe('delivered'); + expect(requestNotes).toHaveLength(0); + }); + + test('a rebuild cut by the render bound still sends the earlier picture and is reported as a recovered timeout', async () => { + const process1 = startProcess(); + const send = mock(async () => 8); + let date = '2026-11-02'; + process1.delivery.enqueue(request({ buildJob: () => dayJob(date)(), send })); + const [firstId] = [...queue.jobs.keys()]; + date = '2026-11-03'; + process1.delivery.eventsChanged(1); + queue.jobs.get(firstId!)!.result.resolve(png()); + await flush(); + queue.jobs.get(`${firstId}-r1`)!.result.reject(new RenderTimeoutError(120_000, 150_000)); + await process1.delivery.whenIdle(); + + expect(send).toHaveBeenCalledTimes(1); + expect(onlyRow().state).toBe('delivered'); + expect(alerts).toHaveLength(1); + expect(alerts[0]).toContain('Timeout (render)'); + expect(alerts[0]).toContain('recovered=1'); + }); + + test('a write that lands with too little of the bound left sends the picture as rendered', async () => { + let nowMs = 1_000_000; + const process1 = startProcess(() => nowMs); + const send = mock(async () => 9); + process1.delivery.enqueue(request({ send })); + const [firstId] = [...queue.jobs.keys()]; + nowMs += 120_000; // the render took 120 s of the 150 s bound + process1.delivery.eventsChanged(1); + queue.jobs.get(firstId!)!.result.resolve(png()); + await process1.delivery.whenIdle(); + + expect([...queue.jobs.keys()]).toEqual([firstId!]); // no rebuild job + expect(send).toHaveBeenCalledTimes(1); + expect(onlyRow().state).toBe('delivered'); + }); +}); + +describe('every end of a delivery is recorded, told and reported exactly once', () => { + const refused = (code: number) => + new TelegramError({ ok: false, error_code: code, description: 'Bad Request' }, 'sendPhoto', { + chat_id: 1, + photo: 'photo', + }); + + function enqueueWith(overrides: Partial) { + const process1 = startProcess(); + const requestNotes: string[] = []; + process1.delivery.enqueue(request({ notify: async (text) => void requestNotes.push(text), ...overrides })); + return { process1, requestNotes }; + } + + test('a render cut by the bound: recorded expired, one note, one timeout alert', async () => { + const { process1, requestNotes } = enqueueWith({}); + [...queue.jobs.values()][0]!.result.reject(new RenderTimeoutError(150_000, 150_000)); + await process1.delivery.whenIdle(); + + expect(onlyRow()).toMatchObject({ state: 'expired', note_sent: 1, job_json: null }); + expect(requestNotes).toEqual([tr.imageNoteTimedOut('2026-11-02')]); + expect(alerts).toHaveLength(1); + expect(alerts[0]).toContain('Timeout (render)'); + + const process2 = startProcess(); + await process2.delivery.reconcile(); + expect(process2.notes).toHaveLength(0); // the user was told once, not again after a restart + }); + + test('a render that failed: recorded failed, one note, one failure alert of its own kind', async () => { + const { process1, requestNotes } = enqueueWith({}); + [...queue.jobs.values()][0]!.result.reject(new Error('worker crashed')); + await process1.delivery.whenIdle(); + + expect(onlyRow()).toMatchObject({ state: 'failed', note_sent: 1 }); + expect(requestNotes).toEqual([tr.imageNoteFailed('2026-11-02')]); + expect(failures).toEqual([expect.objectContaining({ kind: 'day', errorName: 'Error' })]); + expect(alerts).toEqual([]); + }); + + test('Telegram refusing the upload (a 4xx) is a failure the user is told about: the picture did not land', async () => { + const { process1, requestNotes } = enqueueWith({ + send: async () => { + throw refused(400); + }, + }); + [...queue.jobs.values()][0]!.result.resolve(png()); + await process1.delivery.whenIdle(); + + expect(onlyRow()).toMatchObject({ state: 'failed', note_sent: 1 }); + expect(requestNotes).toEqual([tr.imageNoteFailed('2026-11-02')]); + expect(failures).toHaveLength(1); + }); + + for (const [name, error] of [ + ['a dropped connection', new Error('socket hang up')], + ['a Telegram server error (5xx)', refused(502)], + ] as const) { + test(`${name} during the upload is an uncertain send: recorded unknown, one note that it may not have arrived, never re-sent`, async () => { + const { process1, requestNotes } = enqueueWith({ + send: async () => { + throw error; + }, + }); + [...queue.jobs.values()][0]!.result.resolve(png()); + await process1.delivery.whenIdle(); + + expect(onlyRow()).toMatchObject({ state: 'unknown', note_sent: 0, job_json: null }); + // Not "it failed" (the picture may be in the chat), but not silence either: the answer may have been only this picture. + expect(requestNotes).toEqual([tr.imageNoteUncertain('2026-11-02')]); + expect(failures).toEqual([expect.objectContaining({ kind: 'day', errorName: 'UncertainSend' })]); + expect(outcomes).toContainEqual(expect.objectContaining({ outcome: 'unknown', delivery: 'unknown' })); + + const process2 = startProcess(); + await process2.delivery.reconcile(); + expect(process2.sentImages).toHaveLength(0); + expect(process2.notes).toHaveLength(0); + }); + } + + for (const [name, error] of [ + ['a blocked bot (403)', refused(403)], + ['a chat that is gone (404)', refused(404)], + [ + 'a chat Telegram reports as not found (400)', + new TelegramError({ ok: false, error_code: 400, description: 'Bad Request: chat not found' }, 'sendPhoto', { + chat_id: 1, + photo: 'photo', + }), + ], + [ + 'a group that does not let the bot send photos', + new TelegramError( + { ok: false, error_code: 400, description: 'Bad Request: not enough rights to send photos to the chat' }, + 'sendPhoto', + { chat_id: 1, photo: 'photo' }, + ), + ], + ] as const) { + test(`${name} is a failure asking again cannot fix: the user is told so without an offer to ask again, no alert is raised`, async () => { + const { process1, requestNotes } = enqueueWith({ + send: async () => { + throw error; + }, + }); + [...queue.jobs.values()][0]!.result.resolve(png()); + await process1.delivery.whenIdle(); + + expect(onlyRow()).toMatchObject({ state: 'failed', note_sent: 1 }); + expect(requestNotes).toEqual([tr.imageNoteRefused('2026-11-02')]); + expect(requestNotes[0]).not.toContain('ещё раз'); + expect(failures).toEqual([]); + expect(alerts).toEqual([]); + expect(errorSpy).not.toHaveBeenCalled(); + }); + } + + test('an upload that hangs past its bound: recorded unknown, one note that it may not have arrived, one timeout alert', async () => { + const delivery = track( + new ImageDelivery({ + renderer: queue, + store, + reportFailure: (report) => failures.push(report), + sendBoundMs: 5, + }), + ); + const requestNotes: string[] = []; + delivery.enqueue( + request({ send: () => new Promise(() => {}), notify: async (text) => void requestNotes.push(text) }), + ); + [...queue.jobs.values()][0]!.result.resolve(png()); + await delivery.whenIdle(); + + expect(onlyRow()).toMatchObject({ state: 'unknown', note_sent: 0 }); + expect(requestNotes).toEqual([tr.imageNoteUncertain('2026-11-02')]); + expect(alerts).toHaveLength(1); + }); + + test('a delivery closed mid-render or mid-upload stays silent when its bounds pass later', async () => { + // Bounds far above any scheduling stall: a GC pause or a loaded host between `enqueue` and `close` must not expire them. + const limits = { store, reportFailure: (report: RenderFailureReport) => failures.push(report), sendBoundMs: 200 }; + const rendering = track(new ImageDelivery({ renderer: dying(queue), boundMs: 200, ...limits })); + rendering.enqueue(request({ requestId: 1 })); // never finishes: its render bound would expire it + const uploading = track(new ImageDelivery({ renderer: queue, ...limits })); + uploading.enqueue( + request({ requestId: 2, slot: 'day:user:1:2026-11-03', send: () => Promise.withResolvers().promise }), + ); + [...queue.jobs.values()].at(-1)!.result.resolve(png()); + await flush(); + + await Promise.all([rendering.close(0), uploading.close(0)]); + const loggedAtClose = outcomes.length; + const reportedAtClose = failures.length; + // Each run still waits for its own 200 ms bound; whenIdle resolves exactly when those timers have fired. + await Promise.all([rendering.whenIdle(), uploading.whenIdle()]); + + expect(outcomes).toHaveLength(loggedAtClose); + expect(failures).toHaveLength(reportedAtClose); + expect(alerts).toEqual([]); + }); +}); + +describe('a delivery resumed after a restart keeps its slot', () => { + const dayJob = (date: string) => () => + ({ type: 'daily-agenda', data: { date, theme: {}, locale: 'ru' }, userId: 1 }) as ImageRenderJob; + + test('the same picture asked again is the same delivery: one render, one picture', async () => { + startProcess(undefined, true).delivery.enqueue(request({ requestId: 900 })); // the bot dies right after queueing + const process2 = startProcess(); + await process2.delivery.reconcile(); + + expect(process2.delivery.enqueue(request({ requestId: 901 }))).toEqual({ status: 'duplicate' }); + expect(queue.jobs.size).toBe(1); + [...queue.jobs.values()][0]!.result.resolve(png()); + await process2.delivery.whenIdle(); + + expect(process2.sentImages).toHaveLength(1); + expect(db.query<{ n: number }, []>('SELECT count(*) AS n FROM image_deliveries').get()?.n).toBe(1); + }); + + test('the same slot with newer data supersedes the resumed picture: only the newest arrives', async () => { + startProcess(undefined, true).delivery.enqueue(request({ requestId: 900, buildJob: dayJob('2026-11-02') })); + const process2 = startProcess(); + await process2.delivery.reconcile(); + + const newest = mock(async () => 11); + expect( + process2.delivery.enqueue(request({ requestId: 901, buildJob: dayJob('2026-11-03'), send: newest })), + ).toEqual({ + status: 'queued', + }); + await flush(); // the resumed wait ends on its abort and records `superseded` + for (const job of queue.jobs.values()) job.result.resolve(png()); + await process2.delivery.whenIdle(); + + expect(process2.sentImages).toHaveLength(0); + expect(newest).toHaveBeenCalledTimes(1); + expect(db.query<{ state: string }, []>('SELECT state FROM image_deliveries ORDER BY state').all()).toEqual([ + { state: 'delivered' }, + { state: 'superseded' }, + ]); + }); +}); + +describe('what a picture shows follows its rebuild', () => { + const dayJob = (date: string) => () => + ({ type: 'daily-agenda', data: { date, theme: {}, locale: 'ru' }, userId: 1 }) as ImageRenderJob; + + test('after a rebuild the same request again is this picture, not a newer one that replaces it', async () => { + const process1 = startProcess(); + let date = '2026-11-02'; + process1.delivery.enqueue(request({ buildJob: () => dayJob(date)() })); + const [firstId] = [...queue.jobs.keys()]; + date = '2026-11-03'; // the write moved the event; the rebuild now shows the new date + process1.delivery.eventsChanged(1); + queue.jobs.get(firstId!)!.result.resolve(png()); + await flush(); + expect([...queue.jobs.keys()]).toEqual([firstId!, `${firstId}-r1`]); + + const again = process1.delivery.enqueue(request({ requestId: 901, buildJob: () => dayJob(date)() })); + queue.jobs.get(`${firstId}-r1`)!.result.resolve(png()); + await process1.delivery.whenIdle(); + + expect(again).toEqual({ status: 'duplicate' }); // not a second render that cancels the rebuild + expect([...queue.jobs.keys()]).toHaveLength(2); + expect(onlyRow().state).toBe('delivered'); + }); + + test('a write that changes nothing the picture shows does not render it a second time', async () => { + const process1 = startProcess(); + const send = mock(async () => 12); + process1.delivery.enqueue(request({ send })); + const [firstId] = [...queue.jobs.keys()]; + process1.delivery.eventsChanged(1); // another event, not on this day + queue.jobs.get(firstId!)!.result.resolve(png()); + await process1.delivery.whenIdle(); + + expect([...queue.jobs.keys()]).toEqual([firstId!]); + expect(send).toHaveBeenCalledTimes(1); + expect(onlyRow().state).toBe('delivered'); + }); +}); + +describe('a restart while a picture is being rebuilt', () => { + const dayJob = (date: string) => () => + ({ type: 'daily-agenda', data: { date, theme: {}, locale: 'ru' }, userId: 1 }) as ImageRenderJob; + + test('resumes the rebuilt job, so the picture the user gets shows the data the text already states', async () => { + const process1 = startProcess(); + let date = '2026-11-02'; + process1.delivery.enqueue(request({ buildJob: () => dayJob(date)() })); + const [firstId] = [...queue.jobs.keys()]; + date = '2026-11-03'; // the user moved the event while the first render ran + process1.delivery.eventsChanged(1); + queue.jobs.get(firstId!)!.result.resolve(png()); // the stale picture finished: the rebuild starts + await flush(); + const rebuiltId = `${firstId}-r1`; + expect(queue.jobs.has(rebuiltId)).toBe(true); + await process1.delivery.close(); // a deploy: the rebuild is still rendering + await process1.delivery.whenIdle(); + expect(onlyRow()).toMatchObject({ state: 'queued', rebuilds: 1 }); + expect(onlyRow().job_json).toContain('2026-11-03'); + + const process2 = startProcess(); + await process2.delivery.reconcile(); + expect(queue.jobs.get(firstId!)!.adds).toBe(1); // the stale job is not waited for again + expect(queue.jobs.get(rebuiltId)!.adds).toBe(2); + queue.jobs.get(rebuiltId)!.result.resolve(png()); + await process2.delivery.whenIdle(); + expect(process2.sentImages).toHaveLength(1); + expect(onlyRow().state).toBe('delivered'); + }); +}); + +describe('a delivery left behind by a long downtime', () => { + const later = (ms: number) => () => Date.now() + ms; + + test('past its bound by more than ten minutes it ends expired with no note and no timeout alert', async () => { + startProcess(undefined, true).delivery.enqueue(request()); // the bot dies right after queueing + const process2 = startProcess(later(3 * 86_400_000)); // it is back three days later + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + + expect(process2.notes).toEqual([]); + expect(alerts).toEqual([]); + expect(onlyRow()).toMatchObject({ state: 'expired', note_sent: 1, job_json: null }); + expect(outcomes).toContainEqual( + expect.objectContaining({ outcome: 'interrupted', phase: 'recovery', stale: true }), + ); + }); + + test('a failure the previous process never told about is not told days later either', async () => { + startProcess(undefined, true).delivery.enqueue(request()); + db.run("UPDATE image_deliveries SET state = 'failed', job_json = NULL, note_sent = 0"); + const process2 = startProcess(later(3 * 86_400_000)); + await process2.delivery.reconcile(); + + expect(process2.notes).toEqual([]); + expect(onlyRow()).toMatchObject({ state: 'failed', note_sent: 1 }); + }); + + test('a restart within minutes still tells the user once', async () => { + startProcess(undefined, true).delivery.enqueue(request()); + const process2 = startProcess(later(5 * 60_000)); + await process2.delivery.reconcile(); + + expect(process2.notes).toEqual([{ chatId: 7, text: tr.imageNoteInterrupted('2026-11-02') }]); + expect(alerts).toEqual([]); + }); + + test('an upload cut off days ago ends unknown without an UncertainSend alert', async () => { + const process1 = startProcess(); + process1.delivery.enqueue(request({ send: () => new Promise(() => {}) })); // killed mid-upload + [...queue.jobs.values()][0]!.result.resolve(png()); + await flush(); + expect(onlyRow().state).toBe('sending'); + + const process2 = startProcess(later(3 * 86_400_000)); + await process2.delivery.reconcile(); + expect(onlyRow().state).toBe('unknown'); + expect(failures).toEqual([]); + expect(process2.notes).toEqual([]); + expect(outcomes).toContainEqual(expect.objectContaining({ outcome: 'unknown', recovered: true, stale: true })); + }); +}); + +describe('an outcome that could not be recorded', () => { + /** A scheduler the test drives by hand: the retries of the refused write wait here until the test runs them. */ + function manualRetries() { + const queued: (() => void)[] = []; + return { + schedule: (fn: () => void) => void queued.push(fn), + runAll() { + while (queued.length > 0) queued.shift()?.(); + }, + }; + } + + test('the user is still told once: the note cannot be claimed on a row that never reached its end state', async () => { + const retries = manualRetries(); + const process1 = startProcess(undefined, false, { schedule: retries.schedule }); + const requestNotes: string[] = []; + process1.delivery.enqueue(request({ notify: async (text) => void requestNotes.push(text) })); + store.finish = () => { + throw new Error('database is locked'); + }; + [...queue.jobs.values()][0]!.result.reject(new Error('worker crashed')); + await process1.delivery.whenIdle(); + retries.runAll(); // every retry is refused as well + + expect(requestNotes).toEqual([tr.imageNoteFailed('2026-11-02')]); + expect(failures).toHaveLength(1); + expect(errorSpy).toHaveBeenCalledWith(expect.anything(), 'Image delivery: the outcome was never recorded'); + }); + + test('the write is tried again: a restart then finds the row ended and told, and says nothing more', async () => { + const retries = manualRetries(); + const process1 = startProcess(undefined, false, { schedule: retries.schedule }); + const requestNotes: string[] = []; + process1.delivery.enqueue(request({ notify: async (text) => void requestNotes.push(text) })); + const realFinish = store.finish.bind(store); + let refusals = 1; + store.finish = (...args: Parameters) => { + if (refusals-- > 0) throw new Error('database is locked'); + return realFinish(...args); + }; + [...queue.jobs.values()][0]!.result.reject(new Error('worker crashed')); + await process1.delivery.whenIdle(); + expect(onlyRow().state).toBe('queued'); // the end was refused once: the row has not caught up yet + retries.runAll(); + + expect(onlyRow()).toMatchObject({ state: 'failed', note_sent: 1 }); + expect(requestNotes).toEqual([tr.imageNoteFailed('2026-11-02')]); + + const process2 = startProcess(); + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + expect(process2.notes).toEqual([]); + expect(process2.sentImages).toEqual([]); + expect(failures).toHaveLength(1); + }); + + test('a graceful shutdown tries the refused write once more: a superseded older picture is not resumed by the next start', async () => { + const dayJob = (date: string) => () => + ({ type: 'daily-agenda', data: { date, theme: {}, locale: 'ru' }, userId: 1 }) as ImageRenderJob; + const retries = manualRetries(); // its timers never fire: the process is going down first + const process1 = startProcess(undefined, false, { schedule: retries.schedule }); + const realFinish = store.finish.bind(store); + let locked = true; + store.finish = (...args: Parameters) => { + if (locked) throw new Error('database is locked'); + return realFinish(...args); + }; + process1.delivery.enqueue(request({ requestId: 1, buildJob: dayJob('2026-11-02') })); + process1.delivery.enqueue(request({ requestId: 2, buildJob: dayJob('2026-11-03') })); // supersedes the first; its end is refused + expect( + db + .query<{ state: string }, []>('SELECT state FROM image_deliveries') + .all() + .map((r) => r.state), + ).toEqual(['queued', 'queued']); + + locked = false; // the database is back by the time the bot shuts down + await process1.delivery.close(0); + await process1.delivery.whenIdle(); + + expect( + db + .query<{ state: string }, []>('SELECT state FROM image_deliveries') + .all() + .map((r) => r.state) + .sort(), + ).toEqual(['queued', 'superseded']); + for (const job of queue.jobs.values()) job.result.resolve(png()); + const process2 = startProcess(); + await process2.delivery.reconcile(); + await process2.delivery.whenIdle(); + expect(process2.sentImages).toHaveLength(1); + }); + + test('a refusal in the note claim is tried again as the claim alone: a restart does not tell the user a second time', async () => { + const retries = manualRetries(); + const process1 = startProcess(undefined, false, { schedule: retries.schedule }); + const requestNotes: string[] = []; + process1.delivery.enqueue(request({ notify: async (text) => void requestNotes.push(text) })); + const realFinish = store.finish.bind(store); + const realClaim = store.markNoteSent.bind(store); + let finishRefusals = 1; + let claimRefusals = 1; + store.finish = (...args: Parameters) => { + if (finishRefusals-- > 0) throw new Error('database is locked'); + return realFinish(...args); + }; + store.markNoteSent = (id: string) => { + if (claimRefusals-- > 0) throw new Error('database is locked'); + return realClaim(id); + }; + [...queue.jobs.values()][0]!.result.reject(new Error('worker crashed')); + await process1.delivery.whenIdle(); + retries.runAll(); // the end is written, the claim is refused, then the claim alone goes through + + expect(onlyRow()).toMatchObject({ state: 'failed', note_sent: 1 }); + const process2 = startProcess(); + await process2.delivery.reconcile(); + expect(process2.notes).toEqual([]); + expect(requestNotes).toEqual([tr.imageNoteFailed('2026-11-02')]); + }); + + test('the shutdown write stops at the first refusal: SQLite already waited its whole busy timeout on that thread', async () => { + const retries = manualRetries(); + const process1 = startProcess(undefined, false, { schedule: retries.schedule }); + for (const [index, slot] of ['day:user:1:a', 'day:user:1:b'].entries()) { + process1.delivery.enqueue(request({ requestId: index, slot })); + } + let writes = 0; + store.finish = () => { + writes++; + throw new Error('database is locked'); + }; + for (const job of queue.jobs.values()) job.result.reject(new Error('worker crashed')); + await process1.delivery.whenIdle(); + expect(writes).toBe(2); // each end was refused once while it happened + + await process1.delivery.close(0); + expect(writes).toBe(3); // one shutdown attempt, not one per delivery + }); +}); diff --git a/test/services/image/image-delivery.test.ts b/test/services/image/image-delivery.test.ts new file mode 100644 index 000000000..82fc84b22 --- /dev/null +++ b/test/services/image/image-delivery.test.ts @@ -0,0 +1,768 @@ +// ImageDelivery: the background delivery behind the AI picture tools. A scripted renderer stands in for +// the BullMQ worker and fake Telegram functions for the API; everything else is the production code. +import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from 'bun:test'; +import { TelegramError } from 'gramio'; +import { t } from '../../../src/config/constants.ts'; +import { + ImageDelivery, + type ImageDeliveryRequest, + MAX_PENDING_CONFLICT_PER_CHAT, + MAX_PENDING_CONFLICT_TOTAL, + MAX_PENDING_PER_CHAT, + MAX_PENDING_TOTAL, + type RenderFailureReport, +} from '../../../src/services/image/image-delivery.ts'; +import { RenderAbortedError, type RenderOptions } from '../../../src/services/image/render-service.ts'; +import { + configureTimeoutReporter, + TIMEOUT_ALERT_SOURCE, + TimeoutReporter, +} from '../../../src/services/incident/timeout-reporter.ts'; +import { AgendaImageError } from '../../../src/utils/agenda-image.ts'; +import { imageLogger } from '../../../src/utils/logger.ts'; +import type { ImageRenderJob } from '../../../src/worker/image-render.queue.ts'; +import { THEME_LIGHT } from '../../../src/worker/templates/themes.ts'; +import { asLoggedOutcome, type LoggedOutcome } from '../../fixtures/image-delivery.ts'; +import { png } from '../../fixtures/png.ts'; + +const tr = t('en').aiTools.meta; + +/** Lets every already-settled promise continuation run (a macrotask turn, no wall-clock wait). */ +const flush = () => new Promise((resolve) => setImmediate(resolve)); + +interface Pending { + job: ImageRenderJob; + options: RenderOptions | undefined; + resolve(buffer: Buffer): void; + reject(err: unknown): void; +} + +/** + * A renderer whose every job stays pending until the test settles it (or its signal aborts). With + * `ignoreAbort` it keeps running after the delivery stopped waiting, like a worker that already runs the + * job: its late result is the thing the delivery must not send. + */ +function scriptedRenderer(ignoreAbort = false) { + const jobs: Pending[] = []; + return { + jobs, + renderDirect: mock((job: ImageRenderJob, options?: RenderOptions) => { + const { promise, resolve, reject } = Promise.withResolvers(); + if (!ignoreAbort) + options?.signal?.addEventListener('abort', () => reject(new RenderAbortedError()), { once: true }); + jobs.push({ job, options, resolve, reject }); + return promise; + }), + }; +} + +function job(date: string, extra: string = ''): ImageRenderJob { + return { + type: 'daily-agenda', + data: { + date, + dayOfWeek: 'Tue', + dateFormatted: date, + eventCount: 0, + allDayEvents: [], + timedEvents: [], + theme: THEME_LIGHT, + locale: 'en', + holidayName: extra, + }, + userId: 1, + } as ImageRenderJob; +} + +function harness(deps: { boundMs?: number; sendBoundMs?: number; ignoreAbort?: boolean } = {}) { + const { ignoreAbort, ...limits } = deps; + const renderer = scriptedRenderer(ignoreAbort); + const failures: RenderFailureReport[] = []; + const delivery = new ImageDelivery({ + renderer, + reportFailure: (report) => failures.push(report), + ...limits, + }); + deliveries.push(delivery); + const sent: { chatId: number; label: string; messageId: number }[] = []; + const notes: { chatId: number; text: string }[] = []; + const events: string[] = []; + let nextMessageId = 100; + /** A request for `chatId`; `payload` is what makes two requests of one slot identical or different. */ + const request = ( + chatId: number, + slot: string, + payload = 'v1', + overrides: Partial = {}, + ): ImageDeliveryRequest => ({ + chatId, + language: 'en', + kind: 'day', + label: slot, + slot, + dataOwner: 1, + buildJob: () => job('2026-04-07', payload), + send: async () => { + const messageId = nextMessageId++; + sent.push({ chatId, label: slot, messageId }); + events.push(`photo:${slot}`); + return messageId; + }, + notify: async (text) => { + notes.push({ chatId, text }); + events.push(`note:${slot}`); + }, + ...overrides, + }); + return { delivery, renderer, failures, sent, notes, events, request }; +} + +/** Every delivery of the running test: one still pending holds a render timer that must not outlive the test. */ +let deliveries: ImageDelivery[]; +let alerts: { text: string; source: string }[]; +let outcomes: LoggedOutcome[]; +/** Every line the service logged at any level: [fields, message]. */ +let logCalls: unknown[][]; +let infoSpy: ReturnType; +let warnSpy: ReturnType; +let errorSpy: ReturnType; + +beforeEach(() => { + alerts = []; + deliveries = []; + configureTimeoutReporter(new TimeoutReporter((text, source) => alerts.push({ text, source }))); + outcomes = []; + logCalls = []; + infoSpy = spyOn(imageLogger, 'info').mockImplementation((obj: unknown, message?: string) => { + logCalls.push([obj, message]); + if (message === 'Image delivery outcome') outcomes.push(asLoggedOutcome(obj)); + }); + warnSpy = spyOn(imageLogger, 'warn').mockImplementation((obj: unknown, message?: string) => { + logCalls.push([obj, message]); + }); + errorSpy = spyOn(imageLogger, 'error').mockImplementation((obj: unknown, message?: string) => { + logCalls.push([obj, message]); + }); +}); + +afterEach(async () => { + await Promise.all(deliveries.map((delivery) => delivery.close(0))); + configureTimeoutReporter(null); + infoSpy.mockRestore(); + warnSpy.mockRestore(); + errorSpy.mockRestore(); +}); + +describe('enqueue returns at once; the picture follows', () => { + test('a slow render does not hold the call, and a late finish is delivered after the text', async () => { + const h = harness(); + const text = Promise.withResolvers(); + + const status = h.delivery.enqueue(h.request(1, 'day:a')); + // The tool call has already returned: the model finishes its turn and the text goes out first. + expect(status).toEqual({ status: 'queued' }); + expect(h.sent).toHaveLength(0); + h.events.push('text'); + text.resolve(); + await text.promise; + + // 80 s later (the overloaded odroid, 2026-11-01) the worker finishes. + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + + expect(h.events).toEqual(['text', 'photo:day:a']); + expect(h.notes).toHaveLength(0); + expect(outcomes).toEqual([ + expect.objectContaining({ outcome: 'delivered', kind: 'day', chatId: 1, messageId: 100 }), + ]); + }); + + test('the job carries the hard delivery bound as its expiry', async () => { + const h = harness({ boundMs: 5_000 }); + const before = Date.now(); + h.delivery.enqueue(h.request(1, 'day:a')); + await Promise.resolve(); + const { job: queued, options } = h.renderer.jobs[0]!; + expect(queued.expiresAt).toBeGreaterThanOrEqual(before + 5_000); + expect(queued.expiresAt).toBeLessThanOrEqual(Date.now() + 5_000); + expect(options?.timeoutMs).toBeLessThanOrEqual(5_000); + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + }); + + describe('the outcome log lines hold no user text: not the label, not the table, not the note', () => { + const SECRET = 'SECRET-TITLE'; + const mdTable = (): ImageRenderJob => + ({ + type: 'md-table', + data: { title: SECRET, markdown: `| ${SECRET} |\n|---|\n| ${SECRET} |`, theme: { name: 'synthetic' } }, + userId: 1, + }) as ImageRenderJob; + // An Error's message and stack are not enumerable: spell them out, since pino's `err` serializer writes both. + const logged = () => + JSON.stringify(logCalls, (_key, value) => + value instanceof Error ? { name: value.name, message: value.message, stack: value.stack } : value, + ); + + test('a delivered picture', async () => { + const h = harness(); + h.delivery.enqueue(h.request(1, `day:${SECRET}`, 'v1', { label: SECRET })); + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + expect(outcomes).toHaveLength(1); + expect(logged()).not.toContain(SECRET); + }); + + test('a failed render', async () => { + const h = harness(); + h.delivery.enqueue(h.request(1, `day:${SECRET}`, 'v1', { label: SECRET })); + h.renderer.jobs[0]!.reject(new Error('worker crashed')); + await h.delivery.whenIdle(); + expect(h.notes[0]!.text).toContain(SECRET); // the user's own note says what failed... + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'failed' })]); + expect(logged()).not.toContain(SECRET); // ...the log never does + }); + + test('a timed-out render', async () => { + const h = harness({ boundMs: 20 }); + h.delivery.enqueue(h.request(1, `day:${SECRET}`, 'v1', { label: SECRET })); + await h.delivery.whenIdle(); + expect(h.notes[0]!.text).toContain(SECRET); + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'timed_out' })]); + expect(logged()).not.toContain(SECRET); + }); + + test.each(['delivered', 'failed', 'timed_out'] as const)('a table with secret markdown: %s', async (outcome) => { + const h = harness(outcome === 'timed_out' ? { boundMs: 20 } : {}); + h.delivery.enqueue(h.request(1, `table:${SECRET}`, 'v1', { kind: 'table', label: SECRET, buildJob: mdTable })); + if (outcome === 'delivered') h.renderer.jobs[0]!.resolve(png()); + if (outcome === 'failed') h.renderer.jobs[0]!.reject(new Error('worker crashed')); + await h.delivery.whenIdle(); + expect(outcomes).toEqual([expect.objectContaining({ outcome, kind: 'table', jobType: 'md-table' })]); + expect(logged()).not.toContain(SECRET); + }); + }); +}); + +describe('a job that never finishes', () => { + test('yields exactly one honest note, exactly one alert and no picture', async () => { + const h = harness({ boundMs: 30 }); + h.delivery.enqueue(h.request(1, 'day:a', 'v1', { label: '2026-04-07' })); + + await h.delivery.whenIdle(); + + expect(h.sent).toHaveLength(0); + expect(h.notes).toEqual([{ chatId: 1, text: tr.imageNoteTimedOut('2026-04-07') }]); + expect(alerts).toHaveLength(1); + expect(alerts[0]!.source).toBe(TIMEOUT_ALERT_SOURCE); + expect(alerts[0]!.text).toContain('Timeout (render)'); + expect(alerts[0]!.text).toContain('limitMs: 30'); + expect(alerts[0]!.text).toContain('unrecovered=1'); + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'timed_out', phase: 'render' })]); + // The abandoned job is withdrawn from the queue instead of rendering for nobody. + expect(h.renderer.jobs[0]!.options?.signal?.aborted).toBe(true); + expect(h.failures).toEqual([]); + }); + + test('a late finish after the bound never sends a picture on top of the note', async () => { + const h = harness({ boundMs: 30, ignoreAbort: true }); + h.delivery.enqueue(h.request(1, 'day:a')); + await h.delivery.whenIdle(); + expect(h.notes).toHaveLength(1); + h.renderer.jobs[0]!.resolve(png()); // the worker finishes the job anyway + await flush(); + expect(h.sent).toHaveLength(0); + expect(h.notes).toHaveLength(1); + }); + + test('several timeouts in one window raise one aggregated alert (the reporter holds the rest back)', async () => { + const h = harness({ boundMs: 20 }); + h.delivery.enqueue(h.request(1, 'day:a')); + h.delivery.enqueue(h.request(2, 'day:b')); + await h.delivery.whenIdle(); + expect(h.notes).toHaveLength(2); + expect(alerts).toHaveLength(1); + }); +}); + +describe('a render that fails for another reason', () => { + test('one note without the internal error text and one aggregated alert of its own kind', async () => { + const h = harness(); + h.delivery.enqueue(h.request(1, 'day:a', 'v1', { label: '2026-04-07' })); + h.renderer.jobs[0]!.reject(new Error('playwright: browser has been closed')); + await h.delivery.whenIdle(); + + expect(h.notes).toEqual([{ chatId: 1, text: tr.imageNoteFailed('2026-04-07') }]); + expect(h.notes[0]!.text).not.toContain('playwright'); + expect(h.failures).toEqual([expect.objectContaining({ kind: 'day', jobType: 'daily-agenda', errorName: 'Error' })]); + expect(alerts).toHaveLength(0); + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'failed' })]); + }); + + test('a picture that is undeliverable by nature (too big) gets the localized note, never the internal English message, and is not an incident', async () => { + const h = harness(); + h.delivery.enqueue( + h.request(1, 'day:a', 'v1', { + send: async () => { + throw new AgendaImageError('Agenda PNG exceeds the 50 MB document limit. Choose a shorter date range.', true); + }, + }), + ); + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + + expect(h.notes.map((n) => n.text)).toEqual([tr.imageNoteUndeliverable('day:a')]); + expect(h.notes[0]!.text).not.toContain('Agenda'); + expect(h.failures).toEqual([]); + expect(alerts).toHaveLength(0); + }); + + test.each([ + 'Agenda renderer returned an invalid PNG.', + 'Agenda PNG has invalid dimensions.', + 'Agenda requires PNG document delivery, but document delivery is unavailable.', + ])('a typed renderer or transport defect (%s) is not "too large": the user gets the failure note and the owner an alert', async (message) => { + const h = harness(); + h.delivery.enqueue( + h.request(1, 'day:a', 'v1', { + send: async () => { + throw new AgendaImageError(message); + }, + }), + ); + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + + expect(h.notes.map((n) => n.text)).toEqual([tr.imageNoteFailed('day:a')]); + expect(h.failures).toEqual([expect.objectContaining({ kind: 'day', errorName: 'Error' })]); + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'failed', expected: false })]); + }); + + describe('a picture nobody can ask for again (the schedule comparison) does not offer it', () => { + const conflictOverrides = { kind: 'conflict', label: 'schedule comparison' } as const; + + test('a failed render', async () => { + const h = harness(); + h.delivery.enqueue(h.request(1, 'conflict:5', 'v1', conflictOverrides)); + h.renderer.jobs[0]!.reject(new Error('worker crashed')); + await h.delivery.whenIdle(); + expect(h.notes).toEqual([{ chatId: 1, text: tr.imageNoteFailedFinal('schedule comparison') }]); + expect(h.notes[0]!.text).not.toContain('Ask me again'); + }); + + test('a timed-out render', async () => { + const h = harness({ boundMs: 20 }); + h.delivery.enqueue(h.request(1, 'conflict:5', 'v1', conflictOverrides)); + await h.delivery.whenIdle(); + expect(h.notes).toEqual([{ chatId: 1, text: tr.imageNoteTimedOutFinal('schedule comparison') }]); + expect(h.notes[0]!.text).not.toContain('Ask me again'); + }); + }); + + test('an upload Telegram refused is a failure: one note, never logged as delivered', async () => { + const h = harness(); + h.delivery.enqueue( + h.request(1, 'day:a', 'v1', { + send: async () => { + throw new TelegramError({ ok: false, error_code: 429, description: 'Too Many Requests' }, 'sendPhoto', { + chat_id: 1, + photo: 'photo', + }); + }, + }), + ); + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + + expect(h.notes).toHaveLength(1); + expect(h.failures).toHaveLength(1); + expect(outcomes.map((o) => o.outcome)).toEqual(['failed']); + }); + + test('a note that cannot be sent does not throw out of the delivery', async () => { + const h = harness(); + const warn = spyOn(imageLogger, 'warn').mockImplementation(() => {}); + try { + h.delivery.enqueue( + h.request(1, 'day:a', 'v1', { + notify: async () => { + throw new Error('chat not found'); + }, + }), + ); + h.renderer.jobs[0]!.reject(new Error('boom')); + await h.delivery.whenIdle(); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + } + }); + + test('an upload that hangs holds no slot forever: one alert, one note that does not claim it failed', async () => { + const h = harness({ sendBoundMs: 20 }); + h.delivery.enqueue(h.request(1, 'day:a', 'v1', { send: () => new Promise(() => {}) })); + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + + expect(h.notes).toEqual([{ chatId: 1, text: tr.imageNoteUncertain('day:a') }]); + expect(alerts).toHaveLength(1); + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'timed_out', phase: 'send', delivery: 'unknown' })]); + }); +}); + +describe('two pictures in one chat', () => { + test('the identical request twice is one render and one delivery', async () => { + const h = harness(); + expect(h.delivery.enqueue(h.request(1, 'day:a'))).toEqual({ status: 'queued' }); + expect(h.delivery.enqueue(h.request(1, 'day:a'))).toEqual({ status: 'duplicate' }); + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + + expect(h.renderer.renderDirect).toHaveBeenCalledTimes(1); + expect(h.sent).toHaveLength(1); + }); + + test('the same picture in another chat is its own delivery', async () => { + const h = harness(); + h.delivery.enqueue(h.request(1, 'day:a')); + expect(h.delivery.enqueue(h.request(2, 'day:a'))).toEqual({ status: 'queued' }); + for (const pending of h.renderer.jobs) pending.resolve(png()); + await h.delivery.whenIdle(); + expect(h.sent.map((s) => s.chatId).sort()).toEqual([1, 2]); + }); + + test('a newer picture of the same slot replaces the older pending one: only the newest arrives, no note', async () => { + const h = harness(); + h.delivery.enqueue(h.request(1, 'day:a', 'before-the-move')); + h.delivery.enqueue(h.request(1, 'day:a', 'after-the-move')); + await Promise.resolve(); + // The older one is withdrawn; the newer one finishes. + expect(h.renderer.jobs[0]!.options?.signal?.aborted).toBe(true); + h.renderer.jobs[1]!.resolve(png()); + await h.delivery.whenIdle(); + + expect(h.sent).toHaveLength(1); + expect(h.notes).toHaveLength(0); + expect(outcomes.map((o) => o.outcome).sort()).toEqual(['delivered', 'superseded']); + expect(alerts).toHaveLength(0); + }); + + test('a superseded render that finishes anyway is not delivered', async () => { + const h = harness({ ignoreAbort: true }); + const delivered: string[] = []; + const withSend = (payload: string) => + h.request(1, 'day:a', payload, { + send: async () => { + delivered.push(payload); + return 1; + }, + }); + h.delivery.enqueue(withSend('v1')); + h.delivery.enqueue(withSend('v2')); + await Promise.resolve(); + h.renderer.jobs[0]!.resolve(png()); // a renderer that ignores the abort + h.renderer.jobs[1]!.resolve(png()); + await h.delivery.whenIdle(); + expect(delivered).toEqual(['v2']); + }); + + test('concurrency per chat is bounded: the pending cap is honest, other chats are unaffected', async () => { + const h = harness(); + for (let i = 0; i < MAX_PENDING_PER_CHAT; i++) { + expect(h.delivery.enqueue(h.request(1, `day:${i}`, `p${i}`))).toEqual({ status: 'queued' }); + } + expect(h.delivery.enqueue(h.request(1, 'day:extra', 'extra'))).toEqual({ status: 'busy' }); + expect(h.delivery.enqueue(h.request(2, 'day:other'))).toEqual({ status: 'queued' }); + expect(h.renderer.renderDirect).toHaveBeenCalledTimes(MAX_PENDING_PER_CHAT + 1); + + // A finished picture frees its place. + h.renderer.jobs[0]!.resolve(png()); + await flush(); + expect(h.delivery.enqueue(h.request(1, 'day:extra', 'extra'))).toEqual({ status: 'queued' }); + for (const pending of h.renderer.jobs) pending.resolve(png()); + await h.delivery.whenIdle(); + }); + + test("the schedule comparisons of accepted invitations have their own per-chat limit, apart from the chat's own pictures", async () => { + const h = harness(); + const conflict = (id: number) => + h.request(1, `conflict:${id}`, `c${id}`, { kind: 'conflict', label: 'schedule comparison' }); + // More guests accept within one render than the chat's own limit: each comparison still has its place. + for (let i = 0; i < MAX_PENDING_CONFLICT_PER_CHAT; i++) { + expect(h.delivery.enqueue(conflict(i))).toEqual({ status: 'queued' }); + } + // Their own limit is bounded too, so one chat cannot take the whole bot-wide budget. + expect(h.delivery.enqueue(conflict(99))).toEqual({ status: 'busy' }); + // The inviter's own pictures keep their full place in the chat whatever the comparisons hold. + for (let i = 0; i < MAX_PENDING_PER_CHAT; i++) { + expect(h.delivery.enqueue(h.request(1, `day:${i}`, `p${i}`))).toEqual({ status: 'queued' }); + } + expect(h.delivery.enqueue(h.request(1, 'day:extra', 'extra'))).toEqual({ status: 'busy' }); + + for (const pending of h.renderer.jobs) pending.resolve(png()); + await h.delivery.whenIdle(); + expect(h.sent).toHaveLength(MAX_PENDING_CONFLICT_PER_CHAT + MAX_PENDING_PER_CHAT); + }); + + test("the schedule comparisons are bounded bot-wide as well: the chats' own pictures always keep the other half", async () => { + const h = harness(); + const conflict = (chat: number, id: number) => + h.request(chat, `conflict:${chat}:${id}`, `c${id}`, { kind: 'conflict', label: 'schedule comparison' }); + let queued = 0; + for (let chat = 1; queued < MAX_PENDING_CONFLICT_TOTAL; chat++) { + for (let i = 0; i < MAX_PENDING_CONFLICT_PER_CHAT && queued < MAX_PENDING_CONFLICT_TOTAL; i++, queued++) { + expect(h.delivery.enqueue(conflict(chat, i))).toEqual({ status: 'queued' }); + } + } + expect(h.delivery.enqueue(conflict(50, 0))).toEqual({ status: 'busy' }); + // Other chats' own pictures still fit in the rest of the bot-wide budget. + for (let chat = 100; chat < 100 + MAX_PENDING_TOTAL - MAX_PENDING_CONFLICT_TOTAL; chat++) { + expect(h.delivery.enqueue(h.request(chat, 'day:a'))).toEqual({ status: 'queued' }); + } + for (const pending of h.renderer.jobs) pending.resolve(png()); + await h.delivery.whenIdle(); + }); + + test('the shared host is bounded too: pending pictures across chats are capped', async () => { + const h = harness(); + for (let chat = 1; chat <= MAX_PENDING_TOTAL; chat++) { + expect(h.delivery.enqueue(h.request(chat, 'day:a'))).toEqual({ status: 'queued' }); + } + expect(h.delivery.enqueue(h.request(999, 'day:a'))).toEqual({ status: 'busy' }); + for (const pending of h.renderer.jobs) pending.resolve(png()); + await h.delivery.whenIdle(); + }); +}); + +describe('a failure note that hangs', () => { + test('is given up on after the send bound and does not count against the bot-wide limit meanwhile', async () => { + const h = harness({ sendBoundMs: 40 }); + // The first picture fails, and its note never returns (a hung or flood-waiting Telegram call). + h.delivery.enqueue(h.request(1, 'day:stuck', 'v1', { notify: () => new Promise(() => {}) })); + h.renderer.jobs[0]!.reject(new Error('worker crashed')); + await flush(); + + // Its delivery has ended: it holds no place, so MAX_PENDING_TOTAL more pictures still fit. + for (let chat = 2; chat < 2 + MAX_PENDING_TOTAL; chat++) { + expect(h.delivery.enqueue(h.request(chat, 'day:a'))).toEqual({ status: 'queued' }); + } + for (const pending of h.renderer.jobs.slice(1)) pending.resolve(png()); + + // And the hung note itself ends within the bound instead of holding the delivery forever. + await h.delivery.whenIdle(); + expect(logCalls.some(([, message]) => message === 'Image delivery: the failure note could not be sent')).toBe(true); + }); +}); + +describe('the user sends a new message before the picture arrives', () => { + test('a calendar write while the picture renders: it is rebuilt once from the current data', async () => { + const h = harness(); + let version = 'before'; + h.delivery.enqueue(h.request(1, 'day:a', 'ignored', { buildJob: () => job('2026-04-07', version) })); + await Promise.resolve(); + + version = 'after'; // the next message moved the event + h.delivery.eventsChanged(1); + h.renderer.jobs[0]!.resolve(png()); + await flush(); + + expect(h.sent).toHaveLength(0); + expect(h.renderer.jobs).toHaveLength(2); + expect(JSON.stringify(h.renderer.jobs[1]!.job)).toContain('after'); + h.renderer.jobs[1]!.resolve(png()); + await h.delivery.whenIdle(); + expect(h.sent).toHaveLength(1); + }); + + test('a rebuild happens once only: a second write does not loop the render', async () => { + const h = harness(); + let version = 'v1'; + h.delivery.enqueue(h.request(1, 'day:a', 'ignored', { buildJob: () => job('2026-04-07', version) })); + await Promise.resolve(); + version = 'v2'; + h.delivery.eventsChanged(1); + h.renderer.jobs[0]!.resolve(png()); + await flush(); + version = 'v3'; + h.delivery.eventsChanged(1); + h.renderer.jobs[1]!.resolve(png()); + await h.delivery.whenIdle(); + expect(h.renderer.renderDirect).toHaveBeenCalledTimes(2); + expect(h.sent).toHaveLength(1); + }); + + test("another user's write, or a write before the picture was queued, leaves it alone", async () => { + const h = harness(); + h.delivery.eventsChanged(1); // before enqueue + h.delivery.enqueue(h.request(1, 'day:a')); + await Promise.resolve(); + h.delivery.eventsChanged(2); + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + expect(h.renderer.renderDirect).toHaveBeenCalledTimes(1); + expect(h.sent).toHaveLength(1); + }); + + test('a group picture is stale after any write; a table never is', async () => { + const h = harness(); + let version = 'g1'; + h.delivery.enqueue( + h.request(1, 'week:group', 'ignored', { dataOwner: 'group', buildJob: () => job('2026-04-07', version) }), + ); + h.delivery.enqueue( + h.request(1, 'table:x', 't', { kind: 'table', dataOwner: undefined, buildJob: () => job('2026-04-08', version) }), + ); + await Promise.resolve(); + version = 'g2'; + h.delivery.eventsChanged(77); + h.renderer.jobs[0]!.resolve(png()); + h.renderer.jobs[1]!.resolve(png()); + await flush(); + // group: rebuilt (a third job); table: delivered as is. + expect(h.renderer.renderDirect).toHaveBeenCalledTimes(3); + h.renderer.jobs[2]!.resolve(png()); + await h.delivery.whenIdle(); + expect(h.sent.map((s) => s.label).sort()).toEqual(['table:x', 'week:group']); + }); + + describe('a picture of today carries the clock, which is not what it shows', () => { + const todayJob = + (minutes: number, holidayName = ''): (() => ImageRenderJob) => + () => { + const base = job('2026-04-07', holidayName); + return { ...base, data: { ...base.data, relativeDay: 'Today', currentTimeMinutes: minutes } } as ImageRenderJob; + }; + + test('the same picture asked again a minute later is the same delivery: one render, one picture', async () => { + const h = harness(); + expect(h.delivery.enqueue(h.request(1, 'day:a', 'x', { buildJob: todayJob(600) }))).toEqual({ status: 'queued' }); + expect(h.delivery.enqueue(h.request(1, 'day:a', 'x', { buildJob: todayJob(601) }))).toEqual({ + status: 'duplicate', + }); + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + + expect(h.renderer.renderDirect).toHaveBeenCalledTimes(1); + expect(h.sent).toHaveLength(1); + }); + + test('a write that changed nothing the picture shows does not render it again, though the minute moved on', async () => { + const h = harness(); + let minutes = 600; + h.delivery.enqueue(h.request(1, 'day:a', 'x', { buildJob: () => todayJob(minutes)() })); + await Promise.resolve(); + minutes = 601; // the render takes longer than a minute + h.delivery.eventsChanged(1); // an event on another day + h.renderer.jobs[0]!.resolve(png()); + await h.delivery.whenIdle(); + + expect(h.renderer.renderDirect).toHaveBeenCalledTimes(1); + expect(h.sent).toHaveLength(1); + }); + + test('a real change to the day is still a different picture: it supersedes the older one', async () => { + const h = harness(); + expect(h.delivery.enqueue(h.request(1, 'day:a', 'x', { buildJob: todayJob(600, 'before') }))).toEqual({ + status: 'queued', + }); + expect(h.delivery.enqueue(h.request(1, 'day:a', 'x', { buildJob: todayJob(601, 'after') }))).toEqual({ + status: 'queued', + }); + await Promise.resolve(); + expect(h.renderer.jobs[0]!.options?.signal?.aborted).toBe(true); + h.renderer.jobs[1]!.resolve(png()); + await h.delivery.whenIdle(); + expect(h.sent).toHaveLength(1); + }); + + test('a write that changed what the picture shows still rebuilds it, whatever the minute', async () => { + const h = harness(); + let holidayName = 'before'; + h.delivery.enqueue(h.request(1, 'day:a', 'x', { buildJob: () => todayJob(600, holidayName)() })); + await Promise.resolve(); + holidayName = 'after'; + h.delivery.eventsChanged(1); + h.renderer.jobs[0]!.resolve(png()); + await flush(); + + expect(h.renderer.renderDirect).toHaveBeenCalledTimes(2); + expect(JSON.stringify(h.renderer.jobs[1]!.job)).toContain('after'); + h.renderer.jobs[1]!.resolve(png()); + await h.delivery.whenIdle(); + expect(h.sent).toHaveLength(1); + }); + }); +}); + +describe('shutdown while pictures are pending', () => { + test('a pending render is cancelled, the user is told once, and it is logged as interrupted, never delivered', async () => { + const h = harness({ ignoreAbort: true }); + h.delivery.enqueue(h.request(1, 'day:a', 'v1', { label: '2026-04-07' })); + h.delivery.enqueue(h.request(2, 'day:b', 'v1', { label: 'Table' })); + await Promise.resolve(); + + await h.delivery.close(); + + expect(h.notes.map((n) => n.text).sort()).toEqual( + [tr.imageNoteInterrupted('2026-04-07'), tr.imageNoteInterrupted('Table')].sort(), + ); + expect(h.renderer.jobs.every((pending) => pending.options?.signal?.aborted)).toBe(true); + expect(outcomes.map((o) => o.outcome)).toEqual(['interrupted', 'interrupted']); + // A completion that arrives after the close finds nobody to send to. + for (const pending of h.renderer.jobs) pending.resolve(png()); + await flush(); + expect(h.sent).toHaveLength(0); + expect(h.notes).toHaveLength(2); + expect(alerts).toHaveLength(0); + }); + + test('nothing is queued after the close', async () => { + const h = harness(); + await h.delivery.close(); + expect(h.delivery.enqueue(h.request(1, 'day:a'))).toEqual({ status: 'closed' }); + expect(h.renderer.renderDirect).not.toHaveBeenCalled(); + }); + + test('the notes are bounded by the shutdown budget: a stuck Telegram call cannot hold the exit', async () => { + const h = harness(); + const warn = spyOn(imageLogger, 'warn').mockImplementation(() => {}); + try { + h.delivery.enqueue(h.request(1, 'day:a', 'v1', { notify: () => new Promise(() => {}) })); + await Promise.resolve(); + const started = Date.now(); + await h.delivery.close(40); + expect(Date.now() - started).toBeLessThan(500); + expect(warn).toHaveBeenCalledTimes(1); + expect(outcomes.map((o) => o.outcome)).toEqual(['interrupted']); + } finally { + warn.mockRestore(); + } + }); + + test('a picture already uploading gets a grace; if it lands it is delivered, with no false note', async () => { + const h = harness(); + const upload = Promise.withResolvers(); + h.delivery.enqueue(h.request(1, 'day:a', 'v1', { send: () => upload.promise })); + h.renderer.jobs[0]!.resolve(png()); + await flush(); + + const closing = h.delivery.close(200); + upload.resolve(555); + await closing; + + expect(h.notes).toHaveLength(0); + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'delivered', messageId: 555 })]); + }); + + test('an upload that does not finish within the grace is logged unknown, never as delivered, and not announced as failed', async () => { + const h = harness(); + h.delivery.enqueue(h.request(1, 'day:a', 'v1', { send: () => new Promise(() => {}) })); + h.renderer.jobs[0]!.resolve(png()); + await flush(); + + await h.delivery.close(30); + + expect(h.notes).toHaveLength(0); + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'interrupted', delivery: 'unknown' })]); + }); +}); diff --git a/test/services/image/queue-picture.test.ts b/test/services/image/queue-picture.test.ts new file mode 100644 index 000000000..973ea2312 --- /dev/null +++ b/test/services/image/queue-picture.test.ts @@ -0,0 +1,144 @@ +// queuePicture: the one door every command, button and invitation notice sends a picture through. +// A REAL ImageDelivery over a gated renderer; only the render queue and Telegram are fake. +import { afterEach, describe, expect, mock, spyOn, test } from 'bun:test'; +import { t } from '../../../src/config/constants.ts'; +import { MAX_PENDING_PER_CHAT } from '../../../src/services/image/image-delivery.ts'; +import { type PictureRequest, queuePicture } from '../../../src/services/image/queue-picture.ts'; +import { configureTimeoutReporter } from '../../../src/services/incident/timeout-reporter.ts'; +import { imageLogger } from '../../../src/utils/logger.ts'; +import type { ImageRenderJob } from '../../../src/worker/image-render.queue.ts'; +import { gatedDelivery } from '../../fixtures/picture-contract.ts'; + +afterEach(() => configureTimeoutReporter(null)); + +/** The test-only cast to a job shape lives in this one factory (CLAUDE.md). */ +function dayJob(date: string): ImageRenderJob { + return { type: 'daily-agenda', data: { date }, userId: 1 } as unknown as ImageRenderJob; +} + +function request(overrides: Partial = {}) { + const notes: string[] = []; + const sent: number[] = []; + const picture: PictureRequest = { + chatId: 1, + language: 'en', + kind: 'day', + label: '2026-11-01', + slot: 'day:user:1:2026-11-01', + requestId: 10, + filename: 'day.png', + buildJob: () => dayJob('2026-11-01'), + sendPhoto: async () => { + sent.push(1); + return 77; + }, + notify: async (text) => { + notes.push(text); + }, + ...overrides, + }; + return { picture, notes, sent }; +} + +describe('queuePicture', () => { + test('queues and returns while the render is pending, then sends once and runs onSent', async () => { + const g = gatedDelivery(); + const onSent = mock((_messageId: number) => {}); + const { picture, notes, sent } = request({ onSent }); + + expect(await queuePicture(g.delivery, picture)).toEqual({ status: 'queued' }); + expect(sent).toHaveLength(0); + + g.release(); + await g.delivery.whenIdle(); + + expect(sent).toHaveLength(1); + expect(onSent).toHaveBeenCalledWith(77); + expect(notes).toHaveLength(0); + }); + + test('the same picture asked for twice is silent the second time', async () => { + const g = gatedDelivery(); + const first = request(); + const second = request({ requestId: 11 }); + + await queuePicture(g.delivery, first.picture); + expect(await queuePicture(g.delivery, second.picture)).toEqual({ status: 'duplicate' }); + g.releaseAll(); + await g.delivery.whenIdle(); + + expect(first.sent).toHaveLength(1); + expect(second.sent).toHaveLength(0); + expect(second.notes).toHaveLength(0); + }); + + test('a chat that already has the maximum pending gets one honest note and nothing is queued', async () => { + const g = gatedDelivery(); + for (let i = 0; i < MAX_PENDING_PER_CHAT; i++) { + await queuePicture( + g.delivery, + request({ slot: `day:${i}`, buildJob: () => dayJob(`2026-12-0${i + 1}`) }).picture, + ); + } + const over = request({ slot: 'day:over', buildJob: () => dayJob('2027-01-01') }); + + expect(await queuePicture(g.delivery, over.picture)).toEqual({ status: 'busy' }); + + expect(over.notes).toEqual([t('en').aiTools.meta.imageBusy]); + expect(g.renderer.renderDirect).toHaveBeenCalledTimes(MAX_PENDING_PER_CHAT); + g.releaseAll(); + await g.delivery.whenIdle(); + }); + + test('a job that cannot be built gives one note and never reaches the renderer', async () => { + const g = gatedDelivery(); + const { picture, notes } = request({ + buildJob: () => { + throw new Error('synthetic mapper bug'); + }, + }); + + await queuePicture(g.delivery, picture); + + expect(notes).toEqual([t('en').callbackErrors.imageGenerationFailed]); + expect(g.renderer.renderDirect).not.toHaveBeenCalled(); + }); + + test('a picture with no text version names its own failure: the default points at a text above it', async () => { + const g = gatedDelivery(); + const { picture, notes } = request({ + failureNote: 'The comparison could not be prepared.', + buildJob: () => { + throw new Error('synthetic mapper bug'); + }, + }); + + await queuePicture(g.delivery, picture); + + expect(notes).toEqual(['The comparison could not be prepared.']); + }); + + test('a note that cannot be sent is logged once, not thrown, and the picture is still not queued', async () => { + const g = gatedDelivery(); + const warn = spyOn(imageLogger, 'warn').mockImplementation(() => {}); + const error = spyOn(imageLogger, 'error').mockImplementation(() => {}); + try { + const { picture } = request({ + buildJob: () => { + throw new Error('synthetic mapper bug'); + }, + notify: () => Promise.reject(new Error('chat blocked the bot')), + }); + + await expect(queuePicture(g.delivery, picture)).resolves.toEqual({ status: 'closed' }); + + expect(g.renderer.renderDirect).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledTimes(1); // the build failure + expect(warn).toHaveBeenCalledTimes(1); // the note that could not be sent + expect(warn.mock.calls[0]![1]).toBe('Picture note could not be sent'); + } finally { + warn.mockRestore(); + error.mockRestore(); + } + }); +}); diff --git a/test/services/image/render-failure-reporter.test.ts b/test/services/image/render-failure-reporter.test.ts new file mode 100644 index 000000000..640922483 --- /dev/null +++ b/test/services/image/render-failure-reporter.test.ts @@ -0,0 +1,211 @@ +import { describe, expect, test } from 'bun:test'; +import type { RenderFailureReport } from '../../../src/services/image/image-delivery.ts'; +import { + RENDER_FAILURE_ALERT_SOURCE, + RENDER_FAILURE_RETRY_MS, + RENDER_FAILURE_WINDOW_MS, + RenderFailureReporter, +} from '../../../src/services/image/render-failure-reporter.ts'; + +const report: RenderFailureReport = { kind: 'week', jobType: 'weekly-overview', errorName: 'Error', elapsedMs: 1234.6 }; + +/** A reporter on a hand-driven clock; `timers` are the window-end flushes it asked for. */ +function reporter(push: (text: string, source: string) => void = () => {}) { + const clock = { now: 1_000_000 }; + const timers: { fn: () => void; delayMs: number }[] = []; + return { + clock, + timers, + reporter: new RenderFailureReporter( + push, + () => clock.now, + (fn, delayMs) => timers.push({ fn, delayMs }), + ), + }; +} + +describe('RenderFailureReporter', () => { + test('the first failure raises an alert of its own kind, with identifiers only', () => { + const pushed: [string, string][] = []; + const { reporter: r } = reporter((text, source) => pushed.push([text, source])); + r.report(report); + + expect(pushed).toHaveLength(1); + expect(pushed[0]![1]).toBe(RENDER_FAILURE_ALERT_SOURCE); + expect(pushed[0]![0]).toContain('Picture delivery failed (render-failure)'); + expect(pushed[0]![0]).toContain('kind: week'); + expect(pushed[0]![0]).toContain('jobType: weekly-overview'); + expect(pushed[0]![0]).toContain('elapsedMs: 1235'); + expect(pushed[0]![0]).toContain('held back since the last alert: 0'); + }); + + test('a burst inside the window is one alert now and ONE summary when the window ends, with no later report needed', () => { + const pushed: string[] = []; + const { clock, timers, reporter: r } = reporter((text) => pushed.push(text)); + r.report(report); + r.report(report); + r.report(report); + expect(pushed).toHaveLength(1); + expect(timers).toHaveLength(1); // one flush for the whole burst + expect(timers[0]!.delayMs).toBe(RENDER_FAILURE_WINDOW_MS); + + clock.now += RENDER_FAILURE_WINDOW_MS; + timers[0]!.fn(); + expect(pushed).toHaveLength(2); + expect(pushed[1]).toContain('held back since the last alert: 2'); + }); + + test('a report after the window sends at once and the pending flush then sends nothing more', () => { + const pushed: string[] = []; + const { clock, timers, reporter: r } = reporter((text) => pushed.push(text)); + r.report(report); + r.report(report); + clock.now += RENDER_FAILURE_WINDOW_MS; + r.report(report); + expect(pushed).toHaveLength(2); + timers[0]!.fn(); + expect(pushed).toHaveLength(2); + }); + + test('an upload of unknown end is described as unknown, not as a failure the user was told about', () => { + const pushed: string[] = []; + const { reporter: r } = reporter((text) => pushed.push(text)); + r.report({ ...report, errorName: 'UncertainSend' }); + expect(pushed[0]).toContain('outcome unknown'); + expect(pushed[0]).toContain('The user was NOT told it failed'); + expect(pushed[0]).not.toContain('The user got one short note'); + }); + + test('an unreachable alert queue never fails the delivery path', () => { + const { reporter: r } = reporter(() => { + throw new Error('database is locked'); + }); + expect(() => r.report(report)).not.toThrow(); + expect(() => r.report(report)).not.toThrow(); + }); + + test('a first failure the alert queue refuses is held and sent by the retry, so a lone failure is not lost', () => { + const pushed: string[] = []; + let locked = true; + const { timers, reporter: r } = reporter((text) => { + if (locked) throw new Error('database is locked'); + pushed.push(text); + }); + r.report(report); + expect(pushed).toHaveLength(0); + expect(timers).toHaveLength(1); + expect(timers[0]!.delayMs).toBe(RENDER_FAILURE_RETRY_MS); + + locked = false; + timers[0]!.fn(); + expect(pushed).toHaveLength(1); + expect(pushed[0]).toContain('kind: week'); + expect(pushed[0]).toContain('held back since the last alert: 1'); // itself, counted + expect(timers).toHaveLength(1); // nothing more to retry + }); + + test('a retry timer left over from a refused first failure does not open a second alert inside the window another report started', () => { + const pushed: string[] = []; + let locked = true; + const { + clock, + timers, + reporter: r, + } = reporter((text) => { + if (locked) throw new Error('database is locked'); + pushed.push(text); + }); + const start = clock.now; + r.report(report); // refused: held, a retry timer set for RENDER_FAILURE_RETRY_MS + locked = false; + clock.now = start + 10_000; + r.report({ ...report, kind: 'day' }); // sent directly: this opens the window (alert 1) + clock.now = start + 30_000; + r.report({ ...report, kind: 'month' }); // inside that window: held, no new timer + expect(pushed).toHaveLength(1); + + clock.now = start + RENDER_FAILURE_RETRY_MS; + timers[0]!.fn(); // the old retry timer fires inside the window + expect(pushed).toHaveLength(1); // not a second alert 50 s after the first + expect(timers[1]!.delayMs).toBe(RENDER_FAILURE_WINDOW_MS - (RENDER_FAILURE_RETRY_MS - 10_000)); + + clock.now = start + 10_000 + RENDER_FAILURE_WINDOW_MS; + timers[1]!.fn(); + expect(pushed).toHaveLength(2); + expect(pushed[1]).toContain('kind: month'); + }); + + test('a summary the alert queue refuses at the window end stays held and is tried again, not lost', () => { + const pushed: string[] = []; + let locked = false; + const { + clock, + timers, + reporter: r, + } = reporter((text) => { + if (locked) throw new Error('database is locked'); + pushed.push(text); + }); + r.report(report); + r.report(report); + r.report({ ...report, kind: 'day' }); + expect(pushed).toHaveLength(1); + + clock.now += RENDER_FAILURE_WINDOW_MS; + locked = true; + timers[0]!.fn(); + expect(pushed).toHaveLength(1); + expect(timers).toHaveLength(2); + expect(timers[1]!.delayMs).toBe(RENDER_FAILURE_RETRY_MS); + + locked = false; + timers[1]!.fn(); + expect(pushed).toHaveLength(2); + expect(pushed[1]).toContain('kind: day'); + expect(pushed[1]).toContain('held back since the last alert: 2'); + expect(timers).toHaveLength(2); // nothing more to retry + }); + + describe('flush (shutdown)', () => { + test('sends the summary still held for the window at once, so a deploy after a burst does not lose it', () => { + const pushed: [string, string][] = []; + const { reporter: r } = reporter((text, source) => pushed.push([text, source])); + r.report(report); + r.report({ ...report, kind: 'month' }); + r.report({ ...report, kind: 'day' }); + expect(pushed).toHaveLength(1); // the first went out, two are held back + + r.flush(); + + expect(pushed).toHaveLength(2); + expect(pushed[1]![1]).toBe(RENDER_FAILURE_ALERT_SOURCE); + expect(pushed[1]![0]).toContain('kind: day'); // the newest held report describes the summary + expect(pushed[1]![0]).toContain('held back since the last alert: 2'); + }); + + test('with nothing held there is no alert, with or without an earlier alert', () => { + const pushed: string[] = []; + const { reporter: r } = reporter((text) => pushed.push(text)); + r.flush(); + expect(pushed).toEqual([]); + + r.report(report); // sent at once, nothing held + r.flush(); + expect(pushed).toHaveLength(1); + }); + + test('is idempotent, and the window-end timer that fires afterwards sends nothing more', () => { + const pushed: string[] = []; + const { clock, timers, reporter: r } = reporter((text) => pushed.push(text)); + r.report(report); + r.report(report); + r.flush(); + r.flush(); + expect(pushed).toHaveLength(2); + + clock.now += RENDER_FAILURE_WINDOW_MS; + timers[0]!.fn(); + expect(pushed).toHaveLength(2); + }); + }); +}); diff --git a/test/services/image/render-service.test.ts b/test/services/image/render-service.test.ts index f7eb1217e..14c970a97 100644 --- a/test/services/image/render-service.test.ts +++ b/test/services/image/render-service.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'; -import { RenderService } from '../../../src/services/image/render-service.ts'; +import { RenderAbortedError, RenderService, RenderTimeoutError } from '../../../src/services/image/render-service.ts'; import { configureTimeoutReporter, TimeoutReporter } from '../../../src/services/incident/timeout-reporter.ts'; import type { ImageRenderJob } from '../../../src/worker/image-render.queue.ts'; import { THEME_LIGHT } from '../../../src/worker/templates/themes.ts'; @@ -95,56 +95,115 @@ describe('RenderService', () => { // cannot hold a command for long. expect(waitUntilFinished.mock.calls[0]?.[1]).toBe(45_000); }); +}); - describe('a render that does not finish in time', () => { - const alerts: { text: string; source: string }[] = []; - const job: ImageRenderJob = { - type: 'daily-agenda', - data: { - date: '2026-10-12', - dayOfWeek: 'Mon', - dateFormatted: 'October 12', - eventCount: 0, - allDayEvents: [], - timedEvents: [], - theme: THEME_LIGHT, - locale: 'en', - }, - userId: 123, - }; +describe('RenderService with a caller-chosen bound (picture delivery)', () => { + const dailyJob: ImageRenderJob = { + type: 'daily-agenda', + data: { + date: '2026-11-01', + dayOfWeek: 'Sun', + dateFormatted: 'November 1', + eventCount: 0, + allDayEvents: [], + timedEvents: [], + theme: THEME_LIGHT, + locale: 'en', + }, + userId: 123, + }; - function serviceFailingWith(error: Error): RenderService { - const waitUntilFinished = mock(() => Promise.reject(error)); - return new RenderService( - { add: mock(() => Promise.resolve({ id: '7', waitUntilFinished })) } as unknown as ConstructorParameters< - typeof RenderService - >[0], - {} as unknown as ConstructorParameters[1], - ); - } + function serviceWith(waitUntilFinished: (events: unknown, ttl?: number) => Promise) { + const remove = mock(() => Promise.resolve()); + const service = new RenderService( + { + add: mock(() => Promise.resolve({ id: '9', waitUntilFinished: mock(waitUntilFinished), remove })), + } as unknown as ConstructorParameters[0], + {} as unknown as ConstructorParameters[1], + ); + return { service, remove }; + } + test('waits for the given timeout instead of the default 45 s', async () => { + const waitUntilFinished = mock((_events: unknown, _ttl?: number) => + Promise.resolve({ bufferBase64: Buffer.from('x').toString('base64'), width: 1, height: 1, renderTimeMs: 1 }), + ); + const { service } = serviceWith(waitUntilFinished); + await service.renderDirect(dailyJob, { timeoutMs: 150_000 }); + expect(waitUntilFinished.mock.calls[0]?.[1]).toBe(150_000); + }); + + test("the worker's wait timeout (prod 2026-11-01: 'Job wait render timed out ... 45000ms') becomes a typed RenderTimeoutError and withdraws the job", async () => { + const { service, remove } = serviceWith((_events, ttl) => + Promise.reject( + new Error(`Job wait render timed out before finishing, no finish notification arrived after ${ttl}ms (id=9)`), + ), + ); + const failure = await service.renderDirect(dailyJob, { timeoutMs: 1_000 }).catch((err: unknown) => err); + expect(failure).toBeInstanceOf(RenderTimeoutError); + expect(failure).toMatchObject({ limitMs: 1_000 }); + expect(remove).toHaveBeenCalledTimes(1); + }); + + describe('reporting', () => { + const alerts: string[] = []; beforeEach(() => { alerts.length = 0; - configureTimeoutReporter(new TimeoutReporter((text, source) => alerts.push({ text, source }))); + configureTimeoutReporter(new TimeoutReporter((text) => alerts.push(text))); }); afterEach(() => configureTimeoutReporter(null)); - test('reaches the repair flow as an unrecovered render timeout and still fails the caller', async () => { - const error = new Error( - 'Job wait render timed out before finishing, no finish notification arrived after 45000ms (id=7)', - ); - await expect(serviceFailingWith(error).renderDirect(job)).rejects.toBe(error); - expect(alerts).toHaveLength(1); - expect(alerts[0]?.text).toContain('Timeout (render)'); - expect(alerts[0]?.text).toContain('limitMs: 45000'); - expect(alerts[0]?.text).toContain('unrecovered=1 recovered=0'); - }); - - test('a render that fails for another reason is not reported as a timeout', async () => { - await expect(serviceFailingWith(new Error('Browser crashed')).renderDirect(job)).rejects.toThrow( - 'Browser crashed', + // ImageDelivery reports a picture's timeout once, for its own deadline and for this one alike; a + // second report here would count one slow render twice. + test('a wait timeout is thrown to the caller and not reported by the render call itself', async () => { + const { service } = serviceWith((_events, ttl) => + Promise.reject( + new Error(`Job wait render timed out before finishing, no finish notification arrived after ${ttl}ms (id=9)`), + ), ); + await expect(service.renderDirect(dailyJob, { timeoutMs: 1_000 })).rejects.toBeInstanceOf(RenderTimeoutError); expect(alerts).toEqual([]); }); }); + + test('an aborted wait stops at once and withdraws the job nobody waits for', async () => { + const { service, remove } = serviceWith(() => new Promise(() => {})); + const controller = new AbortController(); + const rendering = service.renderDirect(dailyJob, { signal: controller.signal }); + controller.abort(); + expect(await rendering.catch((err: unknown) => err)).toBeInstanceOf(RenderAbortedError); + expect(remove).toHaveBeenCalledTimes(1); + }); + + test('a render that fails in the worker is rethrown as it is and the job is left for BullMQ', async () => { + const { service, remove } = serviceWith(() => Promise.reject(new Error('page crashed'))); + await expect(service.renderDirect(dailyJob)).rejects.toThrow('page crashed'); + expect(remove).not.toHaveBeenCalled(); + }); + + test('a deterministic job id reaches BullMQ so a repeated add is a no-op', async () => { + const add = mock((_name: string, _job: unknown, _options: unknown) => + Promise.resolve({ + id: 'img-1', + remove: mock(() => Promise.resolve()), + waitUntilFinished: () => + Promise.resolve({ bufferBase64: Buffer.from('x').toString('base64'), width: 1, height: 1, renderTimeMs: 1 }), + }), + ); + const service = new RenderService( + { add } as unknown as ConstructorParameters[0], + {} as unknown as ConstructorParameters[1], + ); + await service.renderDirect(dailyJob, { jobId: 'img-1' }); + expect(add.mock.calls[0]?.[2]).toEqual({ priority: 1, jobId: 'img-1' }); + }); + + test('an abort for shutdown keeps the queued job: the next start resumes it', async () => { + const { service, remove } = serviceWith(() => new Promise(() => {})); + const controller = new AbortController(); + const rendering = service.renderDirect(dailyJob, { signal: controller.signal }); + controller.abort('shutdown'); + expect(await rendering.catch((err: unknown) => err)).toBeInstanceOf(RenderAbortedError); + expect(remove).not.toHaveBeenCalled(); + }); }); diff --git a/test/services/incident/timeout-sites.test.ts b/test/services/incident/timeout-sites.test.ts index c02b5a5c8..0ee467e60 100644 --- a/test/services/incident/timeout-sites.test.ts +++ b/test/services/incident/timeout-sites.test.ts @@ -77,7 +77,41 @@ const EXEMPT: { [site: string]: Exemption } = { reason: 'a 50 ms poll sleep inside the acquire loop; the loop deadline is reported by reportAcquireTimeout', reportedBy: 'src/worker/playwright-pool.ts#PlaywrightPool.reportAcquireTimeout', }, + 'src/services/image/render-service.ts#RenderService.renderDirect': { + reason: + "the worker wait of a picture's render; it throws a typed RenderTimeoutError and ImageDelivery.failTimedOut reports it once, with the user's note, so a second report here would count one slow render twice", + reportedBy: 'src/services/image/image-delivery.ts#ImageDelivery.failTimedOut', + }, + 'src/services/image/image-delivery.ts#deadlineAfter': { + reason: + 'the shared timer of the render and upload bounds of a picture (renderOnce and run; ImageDelivery.failTimedOut reports the one that fired, renderFresh the one that cut a rebuild) and of the two waits of close(), which are accounted for on that entry', + reportedBy: 'src/services/image/image-delivery.ts#ImageDelivery.failTimedOut', + }, + 'src/services/image/image-delivery.ts#ImageDelivery.renderOnce': { + reason: + 'the picture render bound; a render cut by it is reported by ImageDelivery.failTimedOut, and a rebuild cut by it (the earlier picture is sent instead) by ImageDelivery.renderFresh as recovered', + reportedBy: 'src/services/image/image-delivery.ts#ImageDelivery.failTimedOut', + }, + 'src/services/image/image-delivery.ts#ImageDelivery.tell': { + reason: + "the bound on the one failure note of a picture, so a Telegram call that hangs or sleeps through a flood wait does not hold the delivery. Deliberately NOT reported: the delivery's own end (failed, timed out) is already reported once, and a note that does not go out only logs 'the failure note could not be sent'", + }, + 'src/services/image/image-delivery.ts#ImageDelivery.run': { + reason: 'the picture upload bound; an upload cut by it is reported by ImageDelivery.failTimedOut', + reportedBy: 'src/services/image/image-delivery.ts#ImageDelivery.failTimedOut', + }, + 'src/services/image/image-delivery.ts#ImageDelivery.close': { + reason: + "two waits at shutdown. The upload grace: a picture whose upload is still running ends unknown and raises the UncertainSend alert (reportUncertain). The shutdown note budget: deliberately NOT reported, it is best effort at exit (a stuck Telegram call only logs 'shutdown note not sent'); the whole shutdown deadline is already bounded and reported in bootstrap/shutdown.ts", + }, // Timers that are not deadlines: nothing is cut off or abandoned when they fire. + 'src/services/image/render-failure-reporter.ts#defaultSchedule': { + reason: 'the end of the 10 min aggregation window of the render-failure alert; it sends what was held back', + }, + 'src/services/image/image-delivery.ts#ImageDelivery.schedule': { + reason: + 'the pause before the store is asked again to write a delivery outcome it refused (a locked database); nothing is cut off or abandoned when it fires', + }, 'src/services/ai/telegram-stream.ts#sleep': { reason: 'edit throttle and flood-wait sleep' }, 'src/services/voice/thinking-phrase-player.ts#ThinkingPhrasePlayer.start': { reason: 'UI timers for filler phrases', diff --git a/test/worker/image-render-expiry.test.ts b/test/worker/image-render-expiry.test.ts new file mode 100644 index 000000000..4b648b78d --- /dev/null +++ b/test/worker/image-render-expiry.test.ts @@ -0,0 +1,52 @@ +// A render job whose delivery bound has passed is dropped by the worker unrendered: on the shared host +// a render for nobody (a stalled job re-queued by a restart, a retry) costs a real user's picture. +import { describe, expect, spyOn, test } from 'bun:test'; +import { UnrecoverableError } from '../../src/utils/unrecoverable-error.ts'; +import { type ImageRenderJob, processRenderJob } from '../../src/worker/image-render.queue.ts'; +import { playwrightPool } from '../../src/worker/playwright-pool.ts'; +import { THEME_LIGHT } from '../../src/worker/templates/themes.ts'; + +const job = (expiresAt: number | undefined): ImageRenderJob => ({ + type: 'daily-agenda', + data: { + date: '2026-11-01', + dayOfWeek: 'Sun', + dateFormatted: 'November 1', + eventCount: 0, + allDayEvents: [], + timedEvents: [], + theme: THEME_LIGHT, + locale: 'en', + }, + userId: 1, + expiresAt, +}); + +describe('processRenderJob expiry', () => { + test('an expired job fails permanently (no retry, no worker alert) without touching the browser', async () => { + const acquire = spyOn(playwrightPool, 'acquire').mockRejectedValue(new Error('the browser must not be used')); + try { + const failure = await processRenderJob(job(Date.now() - 1)).catch((err: unknown) => err); + expect(failure).toBeInstanceOf(UnrecoverableError); // the repo's own class: bullmq's cannot be imported under Bun on Linux CI + expect(failure).toMatchObject({ + name: 'UnrecoverableError', + message: 'Image render job expired before it started', + }); + expect(acquire).not.toHaveBeenCalled(); + } finally { + acquire.mockRestore(); + } + }); + + test('a job within its bound, or without one, starts rendering', async () => { + const acquire = spyOn(playwrightPool, 'acquire').mockRejectedValue(new Error('browser reached')); + try { + for (const expiresAt of [Date.now() + 60_000, undefined]) { + await expect(processRenderJob(job(expiresAt))).rejects.toThrow('browser reached'); + } + expect(acquire).toHaveBeenCalledTimes(2); + } finally { + acquire.mockRestore(); + } + }); +}); diff --git a/uv.lock b/uv.lock index dbc015c96..5895ba5dc 100644 --- a/uv.lock +++ b/uv.lock @@ -335,7 +335,7 @@ wheels = [ [[package]] name = "hypercalendarbot" -version = "0.3.5" +version = "0.3.6" source = { virtual = "." } dependencies = [ { name = "numpy" },