diff --git a/services/platform/convex/auth.ts b/services/platform/convex/auth.ts index 5b85e4d572..d2a6437992 100644 --- a/services/platform/convex/auth.ts +++ b/services/platform/convex/auth.ts @@ -544,6 +544,22 @@ export const getAuthOptions = (ctx: GenericCtx) => { internal.members.mirror_sync.cascadeDeleteOrgMembersMirror, { organizationId: bodyOrgId }, ); + // Same posture for the org's automation triggers: a surviving + // schedule row would keep coming due (and crashing its runs) + // forever (#3022). Own catch so a failure here isn't logged + // under the mirror's label — the schedule scan retires + // orphaned rows as the backstop either way. + try { + await runCtx.runMutation( + internal.automations.triggers.cascadeDeleteOrgTriggers, + { organizationId: bodyOrgId }, + ); + } catch (err) { + console.warn( + '[automations] trigger cascade after organization delete failed; the schedule scan will retire the rows', + err instanceof Error ? err.message : err, + ); + } } } else { const returned = isRecord(mw.context.returned) diff --git a/services/platform/convex/automations/triggers.test.ts b/services/platform/convex/automations/triggers.test.ts index 936257b526..f604f2fcda 100644 --- a/services/platform/convex/automations/triggers.test.ts +++ b/services/platform/convex/automations/triggers.test.ts @@ -18,7 +18,7 @@ import { convexTest, type TestConvex } from 'convex-test'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { Automation } from '../../lib/engine/core/types'; -import { internal } from '../_generated/api'; +import { components, internal } from '../_generated/api'; import betterAuthSchema from '../betterAuth/schema'; import schema from '../schema'; import { cronMatches, dueOccurrence, parseCron } from './cron'; @@ -78,6 +78,28 @@ async function publish( }); } +/** + * A real betterAuth organization row. The scan retires a due schedule whose + * organization no longer resolves (#3022), so a schedule that should FIRE + * needs an org that exists — the fixture org names above deliberately do + * not. Returns the component-side document id. + */ +async function seedOrganization(t: T, slug: string): Promise { + return t.run(async (ctx) => { + const created = await ctx.runMutation( + components.betterAuth.adapter.create, + { + input: { + model: 'organization', + data: { name: slug, slug, createdAt: 0 }, + }, + }, + ); + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- adapter returns the created record as unknown + return (created as { _id: string })._id; + }); +} + /** * Move a trigger's creation into the past. A schedule is armed at creation — * the minute it was created in is not "missed" — so a test that wants a due @@ -185,7 +207,8 @@ describe('schedule triggers', () => { it('fires a due schedule once and stamps it', async () => { const t = newWorld(); - await publish(t, ORG, 'ops/nightly', { + const org = await seedOrganization(t, 'triggers-live'); + await publish(t, org, 'ops/nightly', { kind: 'schedule', cron: '* * * * *', timezone: 'UTC', @@ -197,7 +220,7 @@ describe('schedule triggers', () => { {}, ); expect(first).toEqual({ examined: 1, fired: 1 }); - expect(await runsOf(t, ORG)).toHaveLength(1); + expect(await runsOf(t, org)).toHaveLength(1); // The same minute does not fire again — `lastFiredAt` is the guard. const second = await t.mutation( @@ -205,21 +228,23 @@ describe('schedule triggers', () => { {}, ); expect(second.fired).toBe(0); - expect(await runsOf(t, ORG)).toHaveLength(1); + expect(await runsOf(t, org)).toHaveLength(1); }); it('skips disabled schedules and survives one that cannot be parsed', async () => { const t = newWorld(); - await publish(t, ORG, 'ops/broken-cron', { + const orgA = await seedOrganization(t, 'triggers-a'); + const orgB = await seedOrganization(t, 'triggers-b'); + await publish(t, orgA, 'ops/broken-cron', { kind: 'schedule', cron: 'not a cron', }); - await publish(t, OTHER_ORG, 'ops/nightly', { + await publish(t, orgB, 'ops/nightly', { kind: 'schedule', cron: '* * * * *', timezone: 'UTC', }); - await publish(t, ORG, 'ops/off', { + await publish(t, orgA, 'ops/off', { kind: 'schedule', cron: '* * * * *', enabled: false, @@ -235,8 +260,99 @@ describe('schedule triggers', () => { // The unusable expression is skipped; the healthy one in the OTHER org // still fires, and each run belongs to its own organization. expect(result.fired).toBe(1); + expect(await runsOf(t, orgA)).toHaveLength(0); + expect(await runsOf(t, orgB)).toHaveLength(1); + }); + + it('retires a due schedule whose organization is gone (#3022)', async () => { + const t = newWorld(); + // The fixture ORG never existed as a betterAuth organization row — the + // exact state a deleted org leaves behind when its triggers survived. + await publish(t, ORG, 'ops/orphan', { + kind: 'schedule', + cron: '* * * * *', + timezone: 'UTC', + }); + await backdate(t, 'ops/orphan', 5 * 60 * 1000); + + const first = await t.mutation( + internal.automations.triggers.scanScheduledTriggers, + {}, + ); + // Examined, not fired — and no run row: the trigger is retired instead. + expect(first).toEqual({ examined: 1, fired: 0 }); + expect(await runsOf(t, ORG)).toHaveLength(0); + const rows = await t.run( + async (ctx) => await ctx.db.query('automationTriggers').collect(), + ); + expect(rows).toHaveLength(1); + expect(rows[0].enabled).toBe(false); + + // Retired means gone from every later scan — nothing to examine, ever. + const second = await t.mutation( + internal.automations.triggers.scanScheduledTriggers, + {}, + ); + expect(second).toEqual({ examined: 0, fired: 0 }); + }); +}); + +describe('organization delete cascade (#3022)', () => { + it('deletes exactly the deleted organization triggers, idempotently', async () => { + const t = newWorld(); + const token = mintWebhookToken(); + await publish(t, ORG, 'ops/nightly', { + kind: 'schedule', + cron: '* * * * *', + timezone: 'UTC', + }); + await publish(t, ORG, 'ops/inbound', { + kind: 'webhook', + tokenHash: await hashWebhookToken(token), + }); + await publish(t, OTHER_ORG, 'ops/nightly', { + kind: 'schedule', + cron: '* * * * *', + timezone: 'UTC', + }); + + const result = await t.mutation( + internal.automations.triggers.cascadeDeleteOrgTriggers, + { organizationId: ORG }, + ); + expect(result).toEqual({ deleted: 2 }); + + const rows = await t.run( + async (ctx) => await ctx.db.query('automationTriggers').collect(), + ); + expect(rows).toHaveLength(1); + expect(rows[0].organizationId).toBe(OTHER_ORG); + + // A second cascade finds nothing — safe to re-run from the delete hook. + await expect( + t.mutation(internal.automations.triggers.cascadeDeleteOrgTriggers, { + organizationId: ORG, + }), + ).resolves.toEqual({ deleted: 0 }); + }); + + it('leaves the dead-org webhook token unusable', async () => { + const t = newWorld(); + const token = mintWebhookToken(); + await publish(t, ORG, 'ops/inbound', { + kind: 'webhook', + tokenHash: await hashWebhookToken(token), + }); + await t.mutation(internal.automations.triggers.cascadeDeleteOrgTriggers, { + organizationId: ORG, + }); + + const response = await t.fetch(`/api/automations/webhook/${token}`, { + method: 'POST', + body: '{}', + }); + expect(response.status).toBe(404); expect(await runsOf(t, ORG)).toHaveLength(0); - expect(await runsOf(t, OTHER_ORG)).toHaveLength(1); }); }); diff --git a/services/platform/convex/automations/triggers.ts b/services/platform/convex/automations/triggers.ts index 87f3e5f381..a15af01b40 100644 --- a/services/platform/convex/automations/triggers.ts +++ b/services/platform/convex/automations/triggers.ts @@ -39,6 +39,7 @@ import { internalQuery, type MutationCtx, } from '../_generated/server'; +import { orgSlugFromIdOrNull } from '../lib/helpers/org_slug'; import { dueOccurrence } from './cron'; import { LIVENESS_GRACE_MS, @@ -112,6 +113,22 @@ export const scanScheduledTriggers = internalMutation({ } if (due === null) continue; + // A trigger can outlive its organization: the org was deleted before + // `cascadeDeleteOrgTriggers` existed, or the delete hook failed + // mid-flight. Fired anyway, the run would only crash deep in the agent + // host's throwing slug resolution — one uncaught error per occurrence, + // forever. Retire the trigger the first time it comes due after the org + // is gone instead; a transient lookup failure still throws, so the scan + // simply retries next tick. (#3022) + const orgSlug = await orgSlugFromIdOrNull(ctx, trigger.organizationId); + if (orgSlug === null) { + await ctx.db.patch(trigger._id, { enabled: false, updatedAt: now }); + console.warn( + `[automations] trigger ${trigger.organizationId}/${trigger.name}: organization no longer resolves; disabling the trigger`, + ); + continue; + } + // Stamp BEFORE starting: a run that throws must not leave the schedule // re-firing the same minute on every tick. await ctx.db.patch(trigger._id, { lastFiredAt: due }); @@ -378,6 +395,37 @@ export const automationWebhookHandler = httpAction(async (ctx, request) => { }); }); +// --------------------------------------------------------------- lifecycle + +/** + * Delete every trigger of one organization. Runs from the + * `/organization/delete` after-hook (next to the member-mirror cascade), i.e. + * only after Better Auth actually deleted the organization row — a failed + * deletion keeps the org's schedules and webhook tokens intact. Deleting + * rather than disabling matches the platform's hard-delete posture for + * org-owned rows (memberMirror, personalization): the rows are unreachable + * from any UI once the org is gone, and a merely-disabled row would keep + * surfacing the dead org id to every scan. Idempotent; the schedule scan's + * retirement above is the backstop for rows this cascade never saw (orgs + * deleted before it existed, or a hook that failed mid-flight). (#3022) + */ +export const cascadeDeleteOrgTriggers = internalMutation({ + args: { organizationId: v.string() }, + returns: v.object({ deleted: v.number() }), + handler: async (ctx, args) => { + let deleted = 0; + for await (const row of ctx.db + .query('automationTriggers') + .withIndex('by_org', (q) => + q.eq('organizationId', args.organizationId), + )) { + await ctx.db.delete(row._id); + deleted += 1; + } + return { deleted }; + }, +}); + // -------------------------------------------------------------------- event /**