Experimental bundling of backend

scripts/build_utils/bundle_plugins.js

@@ -0,0 +1,92 @@
+/**
+ * Bundles each plugin's server-side dist/index.js with its pure-JS dependencies
+ * into a single file using esbuild. This is analogous to bundle_server.js but
+ * for plugins — it inlines packages like youtubei.js, mupdf-wasm helpers, etc.
+ * so that NFT only traces the shared/native externals.
+ *
+ * Run: node scripts/build_utils/bundle_plugins.js
+ * (called in the Dockerfile after all plugins are built, before extract_plugins)
+ */
+
+const esbuild = require("esbuild");
+const path = require("path");
+const fs = require("fs");
+
+const projectRoot = path.resolve(__dirname, "../..");
+const pluginsDir = path.join(projectRoot, "plugins");
+
+async function main() {
+  if (!fs.existsSync(pluginsDir)) {
+    console.log("bundle_plugins: plugins/ directory not found, skipping.");
+    return;
+  }
+
+  const pluginEntries = fs
+    .readdirSync(pluginsDir)
+    .filter((name) => {
+      const entry = path.join(pluginsDir, name, "dist/index.js");
+      return fs.existsSync(entry);
+    })
+    .map((name) => ({
+      name,
+      entry: path.join(pluginsDir, name, "dist/index.js"),
+      outfile: path.join(pluginsDir, name, "dist/plugin.bundle.js"),
+    }));
+
+  if (pluginEntries.length === 0) {
+    console.log("bundle_plugins: no plugin dist/index.js files found, skipping.");
+    return;
+  }
+
+  console.log(`bundle_plugins: bundling ${pluginEntries.length} plugin(s)...`);
+
+  for (const { name, entry, outfile } of pluginEntries) {
+    try {
+      await esbuild.build({
+        entryPoints: [entry],
+        bundle: true,
+        platform: "node",
+        target: "node22",
+        format: "cjs",
+        outfile,
+
+        // ── Packages that MUST remain external ──────────────────────────
+        //
+        // Shared with the server — plugins are loaded into the server
+        // process at runtime via aki-plugin-manager.  If a plugin bundles
+        // its own copy of yjs/valtio/etc., collaborative state breaks.
+        external: [
+          "@repo/base-plugin",
+          "yjs",
+          "lib0",
+          "y-protocols",
+          "valtio",
+          "valtio-yjs",
+          "@trpc/server",
+          "zod",
+          // mupdf uses top-level await (WASM loader) which esbuild cannot
+          // emit in CJS format.  Keep it external; NFT will trace it.
+          "mupdf",
+        ],
+
+        define: {
+          "process.env.NODE_ENV": '"production"',
+        },
+
+        allowOverwrite: true,
+        logLevel: "warning",
+      });
+      console.log(`  ✓ ${name}`);
+    } catch (err) {
+      console.error(`  ✗ ${name}: ${err.message}`);
+      process.exit(1);
+    }
+  }
+
+  console.log("bundle_plugins: done.");
+}
+
+main().catch((err) => {
+  console.error("bundle_plugins: FAILED\n", err);
+  process.exit(1);
+});

scripts/build_utils/bundle_server.js

@@ -0,0 +1,149 @@
+/**
+ * Bundles the server's compiled JS (from tsup) with all its pure-JS dependencies
+ * into a single file using esbuild. This dramatically reduces the number of files
+ * that need to be shipped with the Tauri desktop app (from ~100k+ node_modules
+ * files to just a handful of native addon packages).
+ *
+ * Run: node scripts/build_utils/bundle_server.js
+ * (called automatically by backend/server's build script after tsup)
+ */
+
+const esbuild = require("esbuild");
+const path = require("path");
+const fs = require("fs");
+
+const projectRoot = path.resolve(__dirname, "../..");
+
+async function main() {
+  const entryPoint = path.join(
+    projectRoot,
+    "backend/server/dist/index.js",
+  );
+
+  if (!fs.existsSync(entryPoint)) {
+    console.error(
+      `bundle_server: entry point not found: ${entryPoint}\n` +
+        `Run 'yarn server build' (tsup step) first.`,
+    );
+    process.exit(1);
+  }
+
+  console.log("bundle_server: bundling server with esbuild...");
+
+  const result = await esbuild.build({
+    entryPoints: [entryPoint],
+    bundle: true,
+    platform: "node",
+    // Target the Node.js version bundled inside the Tauri app.
+    // Adjust if the embedded Node version changes.
+    target: "node22",
+    format: "cjs",
+    outfile: path.join(projectRoot, "backend/server/dist/server.bundle.js"),
+
+    // ── Packages that MUST remain external ───────────────────────────────
+    //
+    // 1. Native / binary packages — esbuild cannot inline .node files or
+    //    OS binaries; they must exist on disk at runtime.
+    //
+    // 2. Modules shared with dynamically-loaded plugins — plugins are
+    //    loaded at runtime via aki-plugin-manager (aki.load). If both the
+    //    server bundle and a plugin bundle their own copy of e.g. `yjs`,
+    //    collaborative documents would break because the two Y.Doc classes
+    //    would be different objects.  Keeping these external ensures every
+    //    module in the process uses the single copy from node_modules.
+    //
+    // 3. graphile-worker — used by the server to *enqueue* jobs, but the
+    //    actual worker process is a separate process using the graphile-
+    //    worker CLI.  Bundling it could break its internal __dirname-based
+    //    file lookups for SQL migration files.
+    // ─────────────────────────────────────────────────────────────────────
+    external: [
+      // (1) Native / binary
+      "sharp",
+      "ffmpeg-static",
+      "embedded-postgres",
+      "@embedded-postgres/*",
+      // lightningcss ships platform-specific .node files and a WASM fallback;
+      // it must not be bundled.
+      "lightningcss",
+
+      // (2) Build tools / dev-server libs — see the plugin below which stubs
+      // them out entirely rather than externalising them, so they don't appear
+      // in the bundle at all and NFT won't trace them.
+      // (Listed here as a reminder; they are handled by the plugin, not by
+      // this external array.)
+      // "vite", "astro", "esbuild" → handled by productionStubPlugin below
+
+      // (2) Shared with plugins
+      "@repo/base-plugin", // Plugin interface — plugins extend these classes
+      "yjs", // Collaborative document state
+      "lib0", // yjs internal dependency
+      "y-protocols", // yjs protocol layer
+      "valtio", // Reactive state shared between server ↔ plugins
+      "valtio-yjs", // valtio ↔ yjs bridge
+      "@trpc/server", // Plugins register tRPC routers into the server
+      "zod", // Schema types used across plugin ↔ server tRPC boundaries
+
+      // (3) Complex internals
+      "graphile-worker",
+    ],
+
+    // Treat this as a production build.  This lets esbuild dead-code-
+    // eliminate dev-only branches (e.g. vite-express's dev server setup)
+    // so vite is NOT pulled into the bundle.
+    define: {
+      "process.env.NODE_ENV": '"production"',
+    },
+
+    // Stub out packages that are only imported inside dev-mode branches.
+    // Even with define:NODE_ENV=production, esbuild keeps function definitions
+    // around (it only eliminates the call sites), so dynamic imports inside
+    // functions like `installAstroDev` still appear in the bundle.
+    // Replacing those imports with empty objects here ensures they are fully
+    // absent from both the bundle and the NFT file-trace results.
+    plugins: [
+      {
+        name: "production-stubs",
+        setup(build) {
+          const devOnlyPackages = /^(astro|vite|esbuild)$/;
+          build.onResolve({ filter: devOnlyPackages }, (args) => ({
+            namespace: "production-stub",
+            path: args.path,
+          }));
+          build.onLoad(
+            { filter: /.*/, namespace: "production-stub" },
+            () => ({ contents: "module.exports = {};", loader: "js" }),
+          );
+        },
+      },
+    ],
+
+    allowOverwrite: true,
+    metafile: true,
+    logLevel: "info",
+  });
+
+  // Write the metafile so you can inspect what got bundled vs external:
+  //   npx esbuild-visualizer --metadata backend/server/dist/server.bundle.meta.json
+  if (result.metafile) {
+    fs.writeFileSync(
+      path.join(
+        projectRoot,
+        "backend/server/dist/server.bundle.meta.json",
+      ),
+      JSON.stringify(result.metafile),
+    );
+  }
+
+  const stat = fs.statSync(
+    path.join(projectRoot, "backend/server/dist/server.bundle.js"),
+  );
+  console.log(
+    `bundle_server: done — ${(stat.size / 1024 / 1024).toFixed(1)} MB`,
+  );
+}
+
+main().catch((err) => {
+  console.error("bundle_server: FAILED\n", err);
+  process.exit(1);
+});

scripts/build_utils/bundle_worker.js

@@ -0,0 +1,73 @@
+/**
+ * Bundles each graphile-worker task file (backend/worker/dist/tasks/*.js)
+ * with its pure-JS dependencies into self-contained files using esbuild.
+ *
+ * graphile-worker loads tasks by scanning the tasks/ directory and require()-ing
+ * each file by name, so output file names are preserved.  After bundling, the
+ * task files no longer need most of their node_modules entries.
+ *
+ * Run: node scripts/build_utils/bundle_worker.js
+ * (called automatically by backend/worker's build script after pkgroll)
+ */
+
+const esbuild = require("esbuild");
+const path = require("path");
+const fs = require("fs");
+
+const projectRoot = path.resolve(__dirname, "../..");
+
+async function main() {
+  const tasksDir = path.join(projectRoot, "backend/worker/dist/tasks");
+
+  if (!fs.existsSync(tasksDir)) {
+    console.log("bundle_worker: tasks/ directory not found, skipping.");
+    return;
+  }
+
+  const taskFiles = fs
+    .readdirSync(tasksDir)
+    .filter((f) => f.endsWith(".js") && !f.endsWith(".bundle.js"))
+    .map((f) => path.join(tasksDir, f));
+
+  if (taskFiles.length === 0) {
+    console.log("bundle_worker: no task files found, skipping.");
+    return;
+  }
+
+  console.log(`bundle_worker: bundling ${taskFiles.length} task(s)...`);
+
+  await esbuild.build({
+    entryPoints: taskFiles,
+    bundle: true,
+    platform: "node",
+    target: "node22",
+    format: "cjs",
+    // Output back to the same tasks/ directory with the same file names.
+    // graphile-worker discovers tasks by file name, so names must not change.
+    outdir: tasksDir,
+    entryNames: "[name]",
+
+    external: [
+      // Binary packages that ship OS executables
+      "ffmpeg-static",
+
+      // graphile-worker is the task RUNNER — tasks run inside it,
+      // so it must not be bundled into the tasks themselves.
+      "graphile-worker",
+    ],
+
+    define: {
+      "process.env.NODE_ENV": '"production"',
+    },
+
+    allowOverwrite: true,
+    logLevel: "info",
+  });
+
+  console.log("bundle_worker: done.");
+}
+
+main().catch((err) => {
+  console.error("bundle_worker: FAILED\n", err);
+  process.exit(1);
+});