fix(tasks): detect stalled critical scheduler lanes

docs/operations/monitoring.md

@@ -10,6 +10,15 @@ Check **Admin → System** for migrations, task failures, queue state and search
 | `/api/ready` | Database and scheduler readiness |
 | `/api/system/tick` | Task execution; HTTP 200 can still contain `ok: false` |
 
+Readiness fails when enabled `outbox.relay` or `queue.drain` tasks are stale
+(three of their own intervals since the last run), or have never run. Healthy
+daily tasks cannot mask a stalled critical lane. The age check also applies when
+a task is labelled failing after repeated errors. Disabled tasks and tasks holding
+a live execution lease do not trigger this condition. Other stale or failing
+tasks remain visible individually under **Admin → System**. This assessment is
+shared by web readiness and the standalone worker readiness command. A daily-only
+cron schedule does not keep one-minute delivery tasks ready between ticks.
+
 Read response bodies and task results. A working home page does not prove background delivery or backups work.
 
 ## Metrics

packages/i18n/src/catalogs/en.json

@@ -938,7 +938,7 @@
   "adminSystem.sample": "This board is running on in-memory sample data, so it has no scheduler and nothing to maintain.",
   "adminSystem.schedulerStopped": "Nothing is broken and nothing is lost — the tasks are written to catch up, so they will work through the backlog once it runs again. Check that whatever invokes the scheduled endpoint is still configured and still authorised.",
   "adminSystem.schedulerStoppedAfter": ", digests and notification emails are not sent, counters drift, uploads that failed to process are not retried, and queued mail sits in the queue.",
-  "adminSystem.schedulerStoppedBefore": "Every task is overdue, which means the tick is not firing at all. While this is true:",
+  "adminSystem.schedulerStoppedBefore": "Critical tasks are overdue or the scheduler has stopped. If ticks are not running:",
   "adminSystem.schedulerStoppedStrong": "bans do not expire",
   "adminSystem.searchIndexHint": "Make older posts searchable. New and edited posts are indexed automatically.",
   "adminSystem.searchIndexIndexed": "{count} indexed",

packages/tasks/src/health.test.ts

@@ -99,11 +99,75 @@ describe('assessScheduler', () => {
   const stale = (key: string) =>
     task({ key, lastRunAt: new Date(NOW.getTime() - 300_000 * STALE_INTERVALS) })
 
-  it('calls the scheduler stopped only when every enabled task is stale', () => {
+  it('calls the scheduler stopped when every enabled ordinary task is stale', () => {
     expect(assessScheduler([stale('a'), stale('b')], NOW).schedulerStopped).toBe(true)
     expect(assessScheduler([stale('a'), task({ key: 'b' })], NOW).schedulerStopped).toBe(false)
   })
 
+  it.each(['outbox.relay', 'queue.drain'])(
+    'fails readiness when %s is stale even while a daily task is healthy',
+    (key) => {
+      const critical = task({
+        key,
+        intervalSeconds: 60,
+        lastRunAt: new Date(NOW.getTime() - 180_000),
+      })
+      const daily = task({ key: 'marketplace.refresh', intervalSeconds: 86_400 })
+      expect(assessScheduler([critical, daily], NOW).schedulerStopped).toBe(true)
+      expect(assessScheduler([{ ...critical, enabled: false }, daily], NOW).schedulerStopped).toBe(
+        false,
+      )
+      expect(
+        assessScheduler(
+          [{ ...critical, lockedUntil: new Date(NOW.getTime() + 60_000) }, daily],
+          NOW,
+        ).schedulerStopped,
+      ).toBe(false)
+    },
+  )
+
+  it('does not wait for a daily task when a critical task has never run', () => {
+    expect(
+      assessScheduler(
+        [
+          task({ key: 'outbox.relay', lastRunAt: null }),
+          task({ key: 'daily', intervalSeconds: 86_400 }),
+        ],
+        NOW,
+      ).schedulerStopped,
+    ).toBe(true)
+  })
+
+  it('does not let a failing status mask a stopped critical task or a live lease', () => {
+    const failed = task({
+      key: 'queue.drain',
+      intervalSeconds: 60,
+      consecutiveFailures: FAILING_THRESHOLD,
+    })
+    expect(assessScheduler([{ ...failed, lastRunAt: NOW }], NOW).schedulerStopped).toBe(false)
+    const stalled = { ...failed, lastRunAt: new Date(NOW.getTime() - 180_000) }
+    expect(assessScheduler([stalled], NOW).schedulerStopped).toBe(true)
+    expect(
+      assessScheduler([{ ...stalled, lockedUntil: new Date(NOW.getTime() + 60_000) }], NOW)
+        .schedulerStopped,
+    ).toBe(false)
+  })
+
+  it('keeps readiness during a single delayed critical tick', () => {
+    expect(
+      assessScheduler(
+        [
+          task({
+            key: 'queue.drain',
+            intervalSeconds: 60,
+            lastRunAt: new Date(NOW.getTime() - 60_000),
+          }),
+        ],
+        NOW,
+      ).schedulerStopped,
+    ).toBe(false)
+  })
+
   it('ignores disabled tasks when deciding that', () => {
     const health = assessScheduler(
       [stale('a'), task({ key: 'off', enabled: false, lastRunAt: null })],

packages/tasks/src/health.ts

@@ -59,12 +59,18 @@ export interface SchedulerHealth {
 export function assessScheduler(tasks: readonly TaskHealthInput[], now: Date): SchedulerHealth {
   const assessed = tasks.map((task) => assessTask(task, now))
   const enabled = assessed.filter((task) => task.status !== 'disabled')
+  const overdue = enabled.filter(
+    (task) =>
+      (task.lockedUntil == null || task.lockedUntil.getTime() <= now.getTime()) &&
+      (task.lastRunAt === null || task.intervalsLate >= STALE_INTERVALS),
+  )
 
   return {
     tasks: assessed,
     schedulerStopped:
       enabled.length > 0 &&
-      enabled.every((task) => task.status === 'stale' || task.status === 'never-run'),
+      (overdue.length === enabled.length ||
+        overdue.some((task) => task.key === 'outbox.relay' || task.key === 'queue.drain')),
     stale: assessed.filter((task) => task.status === 'stale').length,
     failing: assessed.filter((task) => task.status === 'failing').length,
   }