diff --git a/README.md b/README.md index 38b0a9c..ea5def8 100644 --- a/README.md +++ b/README.md @@ -132,16 +132,41 @@ export const clearField = migrations.define({ // is equivalent to `await ctx.db.patch(doc._id, { optionalField: undefined })` ``` +### Runtime migration arguments + +If you need to configure a migration at run time, define validated args and +consume them in `migrateOne`. + +```ts +export const deleteByTag = migrations.define({ + table: "events", + args: v.object({ tag: v.string() }), + migrateOne: async (ctx, doc, args) => { + if (doc.tags.includes(args.tag)) { + await ctx.db.delete(doc._id); + } + }, +}); +``` + +Then pass `args` when starting the migration: + +```sh +npx convex run migrations:run '{"fn": "migrations:deleteByTag", "args": {"tag": "important"}}' +``` + ### Migrating a subset of a table using an index If you only want to migrate a range of documents, you can avoid processing the whole table by specifying a `customRange`. You can use any existing index you -have on the table, or the built-in `by_creation_time` index. +have on the table, or the built-in `by_creation_time` index. The `customRange` +callback receives `(query, args)` so you can parameterize the range using +migration args passed from the CLI or runner. ```ts export const validateRequiredField = migrations.define({ table: "myTable", - customRange: (query) => + customRange: (query, _args) => query.withIndex("by_requiredField", (q) => q.eq("requiredField", "")), migrateOne: async (_ctx, doc) => { console.log("Needs fixup: " + doc._id); diff --git a/convex.json b/convex.json index 8cd4cfd..350a498 100644 --- a/convex.json +++ b/convex.json @@ -1,7 +1,7 @@ { - "$schema": "./node_modules/convex/schemas/convex.schema.json", "functions": "example/convex", "codegen": { "legacyComponentApi": false - } + }, + "$schema": "./node_modules/convex/schemas/convex.schema.json" } diff --git a/example/convex/example.test.ts b/example/convex/example.test.ts index 45da5c8..60aa375 100644 --- a/example/convex/example.test.ts +++ b/example/convex/example.test.ts @@ -1,4 +1,12 @@ -import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { + afterEach, + assertType, + beforeEach, + describe, + expect, + test, + vi, +} from "vitest"; import { initConvexTest } from "./setup.test"; import { components, internal } from "./_generated/api"; import { runToCompletion } from "@convex-dev/migrations"; @@ -73,4 +81,98 @@ describe("example", () => { expect(after.every((doc) => doc.optionalField !== undefined)).toBe(true); }); }); + test("test migration with runtime args", async () => { + const t = initConvexTest(); + await t.mutation(internal.example.seed, { count: 10 }); + await t.run(async (ctx) => { + await runToCompletion( + ctx, + components.migrations, + internal.example.setConfiguredValue, + { args: { value: "configured" } }, + ); + }); + await t.run(async (ctx) => { + const after = await ctx.db.query("myTable").collect(); + expect(after).toHaveLength(10); + expect(after.every((doc) => doc.optionalField === "configured")).toBe( + true, + ); + }); + }); + + test("same migration with different args runs independently", async () => { + const t = initConvexTest(); + await t.mutation(internal.example.seed, { count: 10 }); + // Run with first set of args + await t.run(async (ctx) => { + await runToCompletion( + ctx, + components.migrations, + internal.example.setConfiguredValue, + { args: { value: "first" } }, + ); + }); + await t.run(async (ctx) => { + const after = await ctx.db.query("myTable").collect(); + expect(after.every((doc) => doc.optionalField === "first")).toBe(true); + }); + // Run with second set of args — should NOT no-op + await t.run(async (ctx) => { + await runToCompletion( + ctx, + components.migrations, + internal.example.setConfiguredValue, + { args: { value: "second" } }, + ); + }); + await t.run(async (ctx) => { + const after = await ctx.db.query("myTable").collect(); + expect(after.every((doc) => doc.optionalField === "second")).toBe(true); + }); + }); + + test("runner with a series passes args to each migration", async () => { + const t = initConvexTest(); + await t.mutation(internal.example.seed, { count: 10 }); + // Run the series runner with args — setDefaultValue runs first, then + // setConfiguredValue should receive the args. + await t.mutation(internal.example.runSeriesWithArgs, { + args: { value: "from-series" }, + }); + // Process all scheduled batches until both migrations complete. + await t.finishAllScheduledFunctions(vi.runAllTimers); + await t.run(async (ctx) => { + const after = await ctx.db.query("myTable").collect(); + expect(after).toHaveLength(10); + // setDefaultValue should have run (fills undefined optionalField with "default") + // then setConfiguredValue should have overwritten all to "from-series" + expect(after.every((doc) => doc.optionalField === "from-series")).toBe( + true, + ); + }); + }); + + test("args type is inferred from the migration definition", () => { + // Type-level only test: verify that args for setConfiguredValue is inferred + // as { value: string }, not `any`. + // We use a function that is never called to avoid runtime errors. + function _typeCheck(ctx: any) { + // Correct args — should type-check fine + void runToCompletion( + ctx, + components.migrations, + internal.example.setConfiguredValue, + { args: { value: "test" } }, + ); + + void runToCompletion( + ctx, + components.migrations, + internal.example.setConfiguredValue, + // @ts-expect-error — wrong args: `notAField` is not in { value: string } + { args: { notAField: 123 } }, + ); + } + }); }); diff --git a/example/convex/example.ts b/example/convex/example.ts index 040b76b..14e05e1 100644 --- a/example/convex/example.ts +++ b/example/convex/example.ts @@ -23,6 +23,26 @@ export const setDefaultValue = migrations.define({ parallelize: true, }); +export const setConfiguredValue = migrations.define({ + table: "myTable", + args: { value: v.string() }, + migrateOne: async (_ctx, doc, args) => { + if (doc.optionalField !== args.value) { + return { optionalField: args.value }; + } + }, +}); + +export const setConfiguredValueWithHello = migrations.define({ + table: "myTable", + args: { value: v.string() }, + migrateOne: async (_ctx, doc, args) => { + if (doc.optionalField !== args.value) { + return { optionalField: args.value + "hello" }; + } + }, +}); + export const clearField = migrations.define({ table: "myTable", migrateOne: () => ({ optionalField: undefined }), @@ -32,8 +52,8 @@ export const validateRequiredField = migrations.define({ table: "myTable", // Specify a custom range to only include documents that need to change. // This is useful if you have a large dataset and only a small percentage of - // documents need to be migrated. - customRange: (query) => + // documents need to be migrated. The second argument is the migration args. + customRange: (query, _args) => query.withIndex("by_requiredField", (q) => q.eq("requiredField", "")), migrateOne: async (_ctx, doc) => { console.log("Needs fixup: " + doc._id); @@ -136,3 +156,9 @@ export const migrationsWithPrefix = new Migrations(components.migrations, { // Allows you to run `npx convex run example:runWithPrefix '{"fn":"setDefaultValue"}'` export const runWithPrefix = migrationsWithPrefix.runner(); + +// A runner for a series that includes a migration with args. +export const runSeriesWithArgs = migrations.runner([ + internal.example.setConfiguredValue, + internal.example.setConfiguredValueWithHello, +]); diff --git a/example/convex/tsconfig.json b/example/convex/tsconfig.json index 891d309..830b19b 100644 --- a/example/convex/tsconfig.json +++ b/example/convex/tsconfig.json @@ -1,5 +1,11 @@ { "extends": "../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@convex-dev/migrations": ["../../src/client/index.ts"] + } + }, "include": ["."], "exclude": ["_generated"] } diff --git a/src/client/index.ts b/src/client/index.ts index 3d1accc..db1d48c 100644 --- a/src/client/index.ts +++ b/src/client/index.ts @@ -25,14 +25,51 @@ import { } from "../shared.js"; export type { MigrationArgs, MigrationResult, MigrationStatus }; -import { ConvexError, type GenericId } from "convex/values"; +import { + ConvexError, + type GenericId, + type ObjectType, + type PropertyValidators, + v, +} from "convex/values"; import type { ComponentApi } from "../component/_generated/component.js"; +import { + migrationNameWithArgs, + type MigrationFunctionHandle, +} from "../component/lib.js"; import { logStatusAndInstructions } from "./log.js"; -import type { MigrationFunctionHandle } from "../component/lib.js"; // Note: this value is hard-coded in the docstring below. Please keep in sync. export const DEFAULT_BATCH_SIZE = 100; +/** + * Infer the TypeScript type from an args validator definition. + * - If a single Validator (e.g. `v.object({...})`), use `Infer`. + * - If PropertyValidators (e.g. `{ tag: v.string() }`), use `ObjectType`. + * - If void/undefined (no args defined), the args type is `undefined`. + */ +type InferMigrationArgs = T extends PropertyValidators + ? ObjectType + : undefined; + +/** + * MigrationArgs with the `args` field typed to a specific type. + */ +type MigrationArgsWithTypedArgs = Omit & { + args?: Args; +}; + +/** + * Extract the migration-specific args type from a function reference. + * Falls back to `any` for untyped migration references. + */ +type ExtractMigrationArgs = + Ref extends FunctionReference + ? A extends { args?: infer MArgs } + ? MArgs + : undefined + : any; + export class Migrations { /** * Makes the migration wrapper, with types for your own tables. @@ -136,6 +173,7 @@ export class Migrations { specificMigrationOrSeries.slice(1).map(async (fnRef) => ({ name: getFunctionName(fnRef), fnHandle: await createFunctionHandle(fnRef), + args: args.args, })), ), ] @@ -159,9 +197,12 @@ export class Migrations { ctx: MutationCtx | ActionCtx, args: MigrationArgs, fnRef?: MigrationFunctionReference, - next?: { name: string; fnHandle: string }[], + next?: { name: string; fnHandle: string; args?: unknown }[], ) { - const name = args.fn ? this.prefixedName(args.fn) : getFunctionName(fnRef!); + const baseName = args.fn + ? this.prefixedName(args.fn) + : getFunctionName(fnRef!); + const name = migrationNameWithArgs(baseName, args.args); async function makeFn(fn: string) { try { return await createFunctionHandle( @@ -197,6 +238,7 @@ export class Migrations { batchSize: args.batchSize, next, dryRun: args.dryRun ?? false, + args: args.args, }); } catch (e) { if ( @@ -260,26 +302,33 @@ export class Migrations { * @param parallelize - If true, each migration batch will be run in parallel. * @returns An internal mutation that runs the migration. */ - define>({ + define< + TableName extends TableNamesInDataModel, + ArgsValidator extends PropertyValidators | void = void, + >({ table, migrateOne, customRange, batchSize: functionDefaultBatchSize, parallelize, + args: defineArgs, }: { table: TableName; migrateOne: ( ctx: GenericMutationCtx, doc: DocumentByName & { _id: GenericId }, + args: InferMigrationArgs, ) => | void | Partial> | Promise> | void>; customRange?: ( q: QueryInitializer>, + args: InferMigrationArgs, ) => OrderedQuery>; batchSize?: number; parallelize?: boolean; + args?: ArgsValidator; }) { const defaultBatchSize = functionDefaultBatchSize ?? @@ -293,7 +342,13 @@ export class Migrations { "internal" >) ?? (internalMutationGeneric as MutationBuilder) )({ - args: migrationArgs, + args: { + ...migrationArgs, + args: + defineArgs == null || Object.keys(defineArgs).length === 0 + ? v.optional(v.object({})) + : v.object(defineArgs), + }, handler: async (ctx, args) => { if (args.fn) { // This is a one-off execution from the CLI or dashboard. @@ -337,7 +392,9 @@ export class Migrations { } const q = ctx.db.query(table); - const range = customRange ? customRange(q) : q; + const range = customRange + ? customRange(q, args.args as InferMigrationArgs) + : q; let continueCursor: string; let page: DocumentByName[]; let isDone: boolean; @@ -363,6 +420,7 @@ export class Migrations { const next = await migrateOne( ctx, doc as { _id: GenericId }, + args.args as any, ); if (next && Object.keys(next).length > 0) { await ctx.db.patch(doc._id as GenericId, next); @@ -428,6 +486,10 @@ export class Migrations { "internal", MigrationArgs, Promise + > as RegisteredMutation< + "internal", + MigrationArgsWithTypedArgs>, + Promise >; } @@ -464,21 +526,24 @@ export class Migrations { * @param opts.dryRun If true, it will run a batch and then throw an error. * It's helpful to see what it would do without committing the transaction. */ - async runOne( + async runOne( ctx: MutationCtx | ActionCtx, - fnRef: MigrationFunctionReference, + fnRef: Ref, opts?: { cursor?: string | null; batchSize?: number; dryRun?: boolean; + args?: ExtractMigrationArgs; }, ) { + const baseName = getFunctionName(fnRef); return ctx.runMutation(this.component.lib.migrate, { - name: getFunctionName(fnRef), + name: migrationNameWithArgs(baseName, opts?.args), fnHandle: await createFunctionHandle(fnRef), cursor: opts?.cursor, batchSize: opts?.batchSize, dryRun: opts?.dryRun ?? false, + args: opts?.args, }); } @@ -636,10 +701,12 @@ export type MigrationFunctionReference = FunctionReference< * @param opts Options to start the migration. * It's helpful to see what it would do without committing the transaction. */ -export async function runToCompletion( +export async function runToCompletion< + Ref extends MigrationFunctionReference | MigrationFunctionHandle, +>( ctx: ActionCtx, component: ComponentApi, - fnRef: MigrationFunctionReference | MigrationFunctionHandle, + fnRef: Ref, opts?: { /** * The name of the migration function, generated with getFunctionName. @@ -661,14 +728,13 @@ export async function runToCompletion( * It's helpful to see what it would do without committing the transaction. */ dryRun?: boolean; + args?: ExtractMigrationArgs; }, ): Promise { let cursor = opts?.cursor; - const { - name = getFunctionName(fnRef), - batchSize, - dryRun = false, - } = opts ?? {}; + const { name: nameOverride, batchSize, dryRun = false, args } = opts ?? {}; + const name = + nameOverride ?? migrationNameWithArgs(getFunctionName(fnRef), args); const address = getFunctionAddress(fnRef); const fnHandle = address.functionHandle ?? (await createFunctionHandle(fnRef)); @@ -679,6 +745,7 @@ export async function runToCompletion( cursor, batchSize, dryRun, + args, oneBatchOnly: true, }); if (status.isDone) { diff --git a/src/component/_generated/component.ts b/src/component/_generated/component.ts index 9d92acc..9630202 100644 --- a/src/component/_generated/component.ts +++ b/src/component/_generated/component.ts @@ -89,12 +89,13 @@ export type ComponentApi = "mutation", "internal", { + args?: any; batchSize?: number; cursor?: string | null; dryRun: boolean; fnHandle: string; name: string; - next?: Array<{ fnHandle: string; name: string }>; + next?: Array<{ args?: any; fnHandle: string; name: string }>; oneBatchOnly?: boolean; }, { diff --git a/src/component/lib.ts b/src/component/lib.ts index c987300..62c0286 100644 --- a/src/component/lib.ts +++ b/src/component/lib.ts @@ -33,10 +33,12 @@ const runMigrationArgs = { v.object({ name: v.string(), fnHandle: v.string(), + args: v.optional(v.any()), }), ), ), dryRun: v.boolean(), + args: v.optional(v.any()), }; export const migrate = mutation({ @@ -68,6 +70,7 @@ export const migrate = mutation({ isDone: false, processed: 0, latestStart: Date.now(), + args: args.args, }), ))!; @@ -116,6 +119,7 @@ export const migrate = mutation({ cursor: state.cursor, batchSize, dryRun, + args: args.args, }, ); updateState(result); @@ -140,14 +144,17 @@ export const migrate = mutation({ for (; i < next.length; i++) { const doc = await ctx.db .query("migrations") - .withIndex("name", (q) => q.eq("name", next[i]!.name)) + .withIndex("name", (q) => + q.eq("name", migrationNameWithArgs(next[i]!.name, next[i]!.args)), + ) .unique(); if (!doc || !doc.isDone) { const [nextFn, ...rest] = next.slice(i); if (nextFn) { await ctx.scheduler.runAfter(0, api.lib.migrate, { - name: nextFn.name, + name: migrationNameWithArgs(nextFn.name, nextFn.args), fnHandle: nextFn.fnHandle, + args: nextFn.args, next: rest, batchSize, dryRun, @@ -351,3 +358,16 @@ export const clearAll = mutation({ } }, }); + +/** + * When args are provided, append a deterministic serialization to the migration + * name so that each unique set of args is tracked as a separate migration run. + */ +export function migrationNameWithArgs( + baseName: string, + args?: unknown, +): string { + if (args === undefined || args === null) return baseName; + const serialized = JSON.stringify(args, Object.keys(args as any).sort()); + return `${baseName}(${serialized})`; +} diff --git a/src/component/schema.ts b/src/component/schema.ts index f3ec0eb..6d67728 100644 --- a/src/component/schema.ts +++ b/src/component/schema.ts @@ -12,6 +12,8 @@ export default defineSchema({ processed: v.number(), latestStart: v.number(), latestEnd: v.optional(v.number()), + // Runtime args passed to the migration. + args: v.optional(v.any()), }) .index("name", ["name"]) .index("isDone", ["isDone"]), diff --git a/src/shared.ts b/src/shared.ts index 75b8dd7..82f2f83 100644 --- a/src/shared.ts +++ b/src/shared.ts @@ -6,6 +6,7 @@ export const migrationArgs = { batchSize: v.optional(v.number()), dryRun: v.optional(v.boolean()), next: v.optional(v.array(v.string())), + args: v.optional(v.any()), }; export type MigrationArgs = ObjectType; diff --git a/tsconfig.test.json b/tsconfig.test.json index aff5230..326a56c 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -1,5 +1,11 @@ { "extends": "./tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@convex-dev/migrations": ["./src/client/index.ts"] + } + }, "include": [ "src/**/*.ts", "src/**/*.tsx", diff --git a/vitest.config.ts b/vitest.config.ts index e9408f8..8dd9ca1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,6 +1,18 @@ +import { resolve } from "path"; import { defineConfig } from "vitest/config"; +// This sucks but import.meta.url error in TypeScript without a way to fix it +// and __dirname is not available in ESM. This relies on where the tests are run from. +const root = resolve(process.cwd()); + export default defineConfig({ + resolve: { + alias: { + // More specific alias first so "@convex-dev/migrations/test" matches correctly + "@convex-dev/migrations/test": resolve(root, "src/test.ts"), + "@convex-dev/migrations": resolve(root, "src/client/index.ts"), + }, + }, test: { environment: "edge-runtime", typecheck: {