Merge pull request #275 from RhysSullivan/rs/storage-bulk-insert

fix(storage): require createMany on CustomAdapter; drizzle issues chunked bulk INSERT

Author: RhysSullivan

packages/core/storage-core/src/adapter.ts

@@ -207,6 +207,18 @@ export interface CustomAdapter {
     select?: string[] | undefined;
   }) => Effect.Effect<T, Error>;
 
+  /**
+   * Native bulk insert. Required because over a real network (e.g.
+   * Hyperdrive), per-row `create` inflates to N round-trips and can't
+   * finish inside a request budget for specs with ~1000+ rows. SQL
+   * backends issue a single multi-row INSERT; in-memory backends just
+   * push the rows.
+   */
+  createMany: <T extends Record<string, unknown>>(data: {
+    model: string;
+    data: ReadonlyArray<T>;
+  }) => Effect.Effect<T[], Error>;
+
   update: <T>(data: {
     model: string;
     where: CleanedWhere[];

packages/core/storage-core/src/factory.ts

@@ -637,14 +637,36 @@ export const createAdapter = (
       forceAllowId?: boolean | undefined;
     }) =>
       Effect.gen(function* () {
-        const out: R[] = [];
+        // Delegates straight to the backend's native bulk insert — no
+        // per-row fallback, because over a real network connection
+        // (Hyperdrive, etc.) N round-trips would blow the request
+        // budget for specs with thousands of operations. Transforms
+        // still run per-row so JSON / dates / booleans serialize the
+        // same as single `create`.
+        const inputs: Record<string, unknown>[] = [];
         for (const row of data.data) {
-          const created = yield* self.create<T, R>({
-            model: data.model,
-            data: row,
-            forceAllowId: data.forceAllowId,
-          });
-          out.push(created);
+          inputs.push(
+            yield* maybeTransformInput(
+              data.model,
+              row as Record<string, unknown>,
+              "create",
+              data.forceAllowId === true,
+            ),
+          );
+        }
+        const res = yield* inner.createMany({
+          model: getModelName(data.model),
+          data: inputs,
+        });
+        const out: R[] = [];
+        for (const row of res) {
+          out.push(
+            (yield* maybeTransformOutput(
+              data.model,
+              row as Record<string, unknown>,
+              undefined,
+            )) as unknown as R,
+          );
         }
         return out as unknown as readonly R[];
       }),

packages/core/storage-core/src/testing/memory.ts

@@ -168,6 +168,13 @@ export const makeMemoryAdapter = (
         return data;
       }),
 
+    createMany: ({ model, data }) =>
+      Effect.sync(() => {
+        const table = tableFor(model);
+        for (const row of data) table.push(row as Row);
+        return data.slice() as never;
+      }),
+
     findOne: ({ model, where, join }) =>
       Effect.sync(() => {
         const rows = filterWhere(tableFor(model), where);

packages/core/storage-drizzle/src/adapter.ts

@@ -312,6 +312,30 @@ export const drizzleAdapter = (options: DrizzleAdapterOptions): DBAdapter => {
         return row as never;
       }),
 
+    // Real multi-row INSERT in fixed-size chunks. One statement per
+    // chunk, not one per row — per-row loops blow the Hyperdrive
+    // request budget on specs with thousands of operations. Chunking
+    // (vs a single giant statement) also keeps payload size bounded:
+    // JSON columns like tool schemas / operation bindings can be a
+    // few KB each, so a 2700-row insert becomes a >10MB statement
+    // otherwise, which chokes both Hyperdrive ingress and WASM
+    // Postgres (PGlite) in the test harness.
+    createMany: ({ model, data }) =>
+      Effect.gen(function* () {
+        if (data.length === 0) return [] as never;
+        const table = getTable(model);
+        const CHUNK = 500;
+        const all: Record<string, unknown>[] = [];
+        for (let i = 0; i < data.length; i += CHUNK) {
+          const slice = data.slice(i, i + CHUNK) as Record<string, unknown>[];
+          const rows = (yield* runPromise("insert many returning", () =>
+            db.insert(table).values(slice).returning(),
+          )) as Record<string, unknown>[];
+          for (const row of rows) all.push(row);
+        }
+        return all as never;
+      }),
+
     findOne: ({ model, where, join }) =>
       Effect.gen(function* () {
         const table = getTable(model);