Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions services/platform/convex/auth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,22 @@ export const getAuthOptions = (ctx: GenericCtx<DataModel>) => {
internal.members.mirror_sync.cascadeDeleteOrgMembersMirror,
{ organizationId: bodyOrgId },
);
// Same posture for the org's automation triggers: a surviving
// schedule row would keep coming due (and crashing its runs)
// forever (#3022). Own catch so a failure here isn't logged
// under the mirror's label — the schedule scan retires
// orphaned rows as the backstop either way.
try {
await runCtx.runMutation(
internal.automations.triggers.cascadeDeleteOrgTriggers,
{ organizationId: bodyOrgId },
);
} catch (err) {
console.warn(
'[automations] trigger cascade after organization delete failed; the schedule scan will retire the rows',
err instanceof Error ? err.message : err,
);
}
}
} else {
const returned = isRecord(mw.context.returned)
Expand Down
132 changes: 124 additions & 8 deletions services/platform/convex/automations/triggers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { convexTest, type TestConvex } from 'convex-test';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';

import type { Automation } from '../../lib/engine/core/types';
import { internal } from '../_generated/api';
import { components, internal } from '../_generated/api';
import betterAuthSchema from '../betterAuth/schema';
import schema from '../schema';
import { cronMatches, dueOccurrence, parseCron } from './cron';
Expand Down Expand Up @@ -78,6 +78,28 @@ async function publish(
});
}

/**
* A real betterAuth organization row. The scan retires a due schedule whose
* organization no longer resolves (#3022), so a schedule that should FIRE
* needs an org that exists — the fixture org names above deliberately do
* not. Returns the component-side document id.
*/
async function seedOrganization(t: T, slug: string): Promise<string> {
return t.run(async (ctx) => {
const created = await ctx.runMutation(
components.betterAuth.adapter.create,
{
input: {
model: 'organization',
data: { name: slug, slug, createdAt: 0 },
},
},
);
// oxlint-disable-next-line typescript/no-unsafe-type-assertion -- adapter returns the created record as unknown
return (created as { _id: string })._id;
});
}

/**
* Move a trigger's creation into the past. A schedule is armed at creation —
* the minute it was created in is not "missed" — so a test that wants a due
Expand Down Expand Up @@ -185,7 +207,8 @@ describe('schedule triggers', () => {

it('fires a due schedule once and stamps it', async () => {
const t = newWorld();
await publish(t, ORG, 'ops/nightly', {
const org = await seedOrganization(t, 'triggers-live');
await publish(t, org, 'ops/nightly', {
kind: 'schedule',
cron: '* * * * *',
timezone: 'UTC',
Expand All @@ -197,29 +220,31 @@ describe('schedule triggers', () => {
{},
);
expect(first).toEqual({ examined: 1, fired: 1 });
expect(await runsOf(t, ORG)).toHaveLength(1);
expect(await runsOf(t, org)).toHaveLength(1);

// The same minute does not fire again — `lastFiredAt` is the guard.
const second = await t.mutation(
internal.automations.triggers.scanScheduledTriggers,
{},
);
expect(second.fired).toBe(0);
expect(await runsOf(t, ORG)).toHaveLength(1);
expect(await runsOf(t, org)).toHaveLength(1);
});

it('skips disabled schedules and survives one that cannot be parsed', async () => {
const t = newWorld();
await publish(t, ORG, 'ops/broken-cron', {
const orgA = await seedOrganization(t, 'triggers-a');
const orgB = await seedOrganization(t, 'triggers-b');
await publish(t, orgA, 'ops/broken-cron', {
kind: 'schedule',
cron: 'not a cron',
});
await publish(t, OTHER_ORG, 'ops/nightly', {
await publish(t, orgB, 'ops/nightly', {
kind: 'schedule',
cron: '* * * * *',
timezone: 'UTC',
});
await publish(t, ORG, 'ops/off', {
await publish(t, orgA, 'ops/off', {
kind: 'schedule',
cron: '* * * * *',
enabled: false,
Expand All @@ -235,8 +260,99 @@ describe('schedule triggers', () => {
// The unusable expression is skipped; the healthy one in the OTHER org
// still fires, and each run belongs to its own organization.
expect(result.fired).toBe(1);
expect(await runsOf(t, orgA)).toHaveLength(0);
expect(await runsOf(t, orgB)).toHaveLength(1);
});

it('retires a due schedule whose organization is gone (#3022)', async () => {
const t = newWorld();
// The fixture ORG never existed as a betterAuth organization row — the
// exact state a deleted org leaves behind when its triggers survived.
await publish(t, ORG, 'ops/orphan', {
kind: 'schedule',
cron: '* * * * *',
timezone: 'UTC',
});
await backdate(t, 'ops/orphan', 5 * 60 * 1000);

const first = await t.mutation(
internal.automations.triggers.scanScheduledTriggers,
{},
);
// Examined, not fired — and no run row: the trigger is retired instead.
expect(first).toEqual({ examined: 1, fired: 0 });
expect(await runsOf(t, ORG)).toHaveLength(0);
const rows = await t.run(
async (ctx) => await ctx.db.query('automationTriggers').collect(),
);
expect(rows).toHaveLength(1);
expect(rows[0].enabled).toBe(false);

// Retired means gone from every later scan — nothing to examine, ever.
const second = await t.mutation(
internal.automations.triggers.scanScheduledTriggers,
{},
);
expect(second).toEqual({ examined: 0, fired: 0 });
});
});

describe('organization delete cascade (#3022)', () => {
it('deletes exactly the deleted organization triggers, idempotently', async () => {
const t = newWorld();
const token = mintWebhookToken();
await publish(t, ORG, 'ops/nightly', {
kind: 'schedule',
cron: '* * * * *',
timezone: 'UTC',
});
await publish(t, ORG, 'ops/inbound', {
kind: 'webhook',
tokenHash: await hashWebhookToken(token),
});
await publish(t, OTHER_ORG, 'ops/nightly', {
kind: 'schedule',
cron: '* * * * *',
timezone: 'UTC',
});

const result = await t.mutation(
internal.automations.triggers.cascadeDeleteOrgTriggers,
{ organizationId: ORG },
);
expect(result).toEqual({ deleted: 2 });

const rows = await t.run(
async (ctx) => await ctx.db.query('automationTriggers').collect(),
);
expect(rows).toHaveLength(1);
expect(rows[0].organizationId).toBe(OTHER_ORG);

// A second cascade finds nothing — safe to re-run from the delete hook.
await expect(
t.mutation(internal.automations.triggers.cascadeDeleteOrgTriggers, {
organizationId: ORG,
}),
).resolves.toEqual({ deleted: 0 });
});

it('leaves the dead-org webhook token unusable', async () => {
const t = newWorld();
const token = mintWebhookToken();
await publish(t, ORG, 'ops/inbound', {
kind: 'webhook',
tokenHash: await hashWebhookToken(token),
});
await t.mutation(internal.automations.triggers.cascadeDeleteOrgTriggers, {
organizationId: ORG,
});

const response = await t.fetch(`/api/automations/webhook/${token}`, {
method: 'POST',
body: '{}',
});
expect(response.status).toBe(404);
expect(await runsOf(t, ORG)).toHaveLength(0);
expect(await runsOf(t, OTHER_ORG)).toHaveLength(1);
});
});

Expand Down
48 changes: 48 additions & 0 deletions services/platform/convex/automations/triggers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import {
internalQuery,
type MutationCtx,
} from '../_generated/server';
import { orgSlugFromIdOrNull } from '../lib/helpers/org_slug';
import { dueOccurrence } from './cron';
import {
LIVENESS_GRACE_MS,
Expand Down Expand Up @@ -112,6 +113,22 @@ export const scanScheduledTriggers = internalMutation({
}
if (due === null) continue;

// A trigger can outlive its organization: the org was deleted before
// `cascadeDeleteOrgTriggers` existed, or the delete hook failed
// mid-flight. Fired anyway, the run would only crash deep in the agent
// host's throwing slug resolution — one uncaught error per occurrence,
// forever. Retire the trigger the first time it comes due after the org
// is gone instead; a transient lookup failure still throws, so the scan
// simply retries next tick. (#3022)
const orgSlug = await orgSlugFromIdOrNull(ctx, trigger.organizationId);
if (orgSlug === null) {
await ctx.db.patch(trigger._id, { enabled: false, updatedAt: now });
console.warn(
`[automations] trigger ${trigger.organizationId}/${trigger.name}: organization no longer resolves; disabling the trigger`,
);
continue;
}

// Stamp BEFORE starting: a run that throws must not leave the schedule
// re-firing the same minute on every tick.
await ctx.db.patch(trigger._id, { lastFiredAt: due });
Expand Down Expand Up @@ -378,6 +395,37 @@ export const automationWebhookHandler = httpAction(async (ctx, request) => {
});
});

// --------------------------------------------------------------- lifecycle

/**
* Delete every trigger of one organization. Runs from the
* `/organization/delete` after-hook (next to the member-mirror cascade), i.e.
* only after Better Auth actually deleted the organization row — a failed
* deletion keeps the org's schedules and webhook tokens intact. Deleting
* rather than disabling matches the platform's hard-delete posture for
* org-owned rows (memberMirror, personalization): the rows are unreachable
* from any UI once the org is gone, and a merely-disabled row would keep
* surfacing the dead org id to every scan. Idempotent; the schedule scan's
* retirement above is the backstop for rows this cascade never saw (orgs
* deleted before it existed, or a hook that failed mid-flight). (#3022)
*/
export const cascadeDeleteOrgTriggers = internalMutation({
args: { organizationId: v.string() },
returns: v.object({ deleted: v.number() }),
handler: async (ctx, args) => {
let deleted = 0;
for await (const row of ctx.db
.query('automationTriggers')
.withIndex('by_org', (q) =>
q.eq('organizationId', args.organizationId),
)) {
await ctx.db.delete(row._id);
deleted += 1;
}
return { deleted };
},
});

// -------------------------------------------------------------------- event

/**
Expand Down
Loading