feat(wizard-program): headless runProgram and runAgent harness

.github/workflows/checks.yml

@@ -49,3 +49,6 @@ jobs:
 
       - name: Test the PR evaluator
         run: pnpm test:evaluator
+
+      - name: Test the headless wizard harness
+        run: pnpm test:wizard-program

README.md

@@ -83,6 +83,7 @@ services/
 ├── wizard-ci/        # Automated wizard runs with PR creation
 ├── wizard-run/       # Interactive wizard runner
 ├── mcp-stub/         # Stub PostHog MCP server for warehouse e2e runs
+├── wizard-program/   # Headless runProgram / runAgent against a wizard checkout
 ├── wizard-commands.ts # Registry of wizard commands (integration, revenue, …)
 └── github/           # GitHub/git utilities
 ```
@@ -298,6 +299,36 @@ You can activate `wizard-ci.yml` in a few ways:
 2. **Schedule** - Runs on cron
 3. **Dispatch** - Webhook call via `repository_dispatch` with event type `wizard-ci-trigger`
 
+## Headless wizard runs
+
+`services/wizard-program/` runs one wizard program through `runProgram`, or one
+agent through `runAgent`, in this process with no TUI. It imports them from the
+wizard checkout in `WIZARD_REPO`.
+
+```bash
+# Resolve the wizard modules and exit. Reads no credentials, makes no request.
+WIZARD_REPO=~/development/wizard pnpm wizard-program --check
+WIZARD_REPO=~/development/wizard pnpm wizard-agent --check
+
+# One program on an app copy. PROGRAM picks it and defaults to posthog-integration.
+WIZARD_REPO=… APP_DIR=/tmp/app-copy PROJECT_ID=… POSTHOG_KEY_FILE=… \
+  pnpm wizard-program
+
+# One agent run on a local `quack` skill, in its own empty directory. PROGRAM
+# sets the program id the gateway token is minted under, default posthog-integration.
+WIZARD_REPO=… PROJECT_ID=… POSTHOG_KEY_FILE=… pnpm wizard-agent
+```
+
+- The scripts start tsx with `--tsconfig "$WIZARD_REPO/tsconfig.json"`, so the
+  wizard's path aliases (`@programs`, `@agent`, `@shared/*`) resolve against
+  that checkout. Set `WIZARD_REPO` in the shell: the scripts read it before tsx
+  starts, so `.env` cannot set it. It is separate from `WIZARD_PATH`.
+- Runs without `--check` are live and credentialed. `POSTHOG_PERSONAL_API_KEY`
+  works in place of `POSTHOG_KEY_FILE`. The wizard's runner mints its own
+  gateway token from that key.
+- Point `APP_DIR` at a copy, never at a fixture in `apps/`. The run edits it.
+- Set `E2E_RESULT_JSON` to a path to get the result as JSON.
+
 ---
 
 ## Running with a proxy

package.json

@@ -12,9 +12,12 @@
     "yara-scan": "tsx services/yara-scan/index.ts",
     "mcp-stub": "tsx services/mcp-stub/cli.ts",
     "mcp-stub:record": "tsx services/mcp-stub/record.ts",
+    "wizard-program": "tsx --tsconfig \"${WIZARD_REPO:?set WIZARD_REPO to a wizard checkout}/tsconfig.json\" services/wizard-program/run-program.ts",
+    "wizard-agent": "tsx --tsconfig \"${WIZARD_REPO:?set WIZARD_REPO to a wizard checkout}/tsconfig.json\" services/wizard-program/run-agent.ts",
     "test:evaluator": "tsx --test services/pr-evaluator/evaluator.test.ts",
     "test:mcp-stub": "tsx --test services/mcp-stub/mcp-stub.test.ts",
-    "test:warehouse-checks": "tsx --test services/wizard-ci/warehouse-checks.test.ts"
+    "test:warehouse-checks": "tsx --test services/wizard-ci/warehouse-checks.test.ts",
+    "test:wizard-program": "tsx --test services/wizard-program/harness.test.ts"
   },
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "0.2.73",

services/wizard-program/harness.test.ts

@@ -0,0 +1,56 @@
+/**
+ * Pins the env contract both headless wizard routes share, so a route never
+ * starts a live run with an input the other would reject.
+ *
+ *   pnpm test:wizard-program
+ */
+
+import assert from "node:assert/strict";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { it, mock } from "node:test";
+
+import { readE2eEnv, readPersonalApiKey } from "./harness.js";
+
+const appDir = mkdtempSync(join(tmpdir(), "wizard-program-"));
+const complete = {
+  APP_DIR: appDir,
+  POSTHOG_PERSONAL_API_KEY: "phx_inline",
+  PROJECT_ID: "228144",
+};
+
+it("prefers the inline key and falls back to the key file when it is blank", () => {
+  const readFile = mock.fn((_file: string) => " phx_file \n");
+  assert.equal(readPersonalApiKey(complete, readFile), "phx_inline");
+  assert.equal(
+    readPersonalApiKey({ POSTHOG_PERSONAL_API_KEY: "  ", POSTHOG_KEY_FILE: "/keys/phx" }, readFile),
+    "phx_file",
+  );
+  assert.deepEqual(
+    readFile.mock.calls.map((call) => call.arguments),
+    [["/keys/phx"]],
+  );
+});
+
+it("reads a complete env", () => {
+  assert.deepEqual(readE2eEnv(complete), {
+    appDir,
+    apiKey: "phx_inline",
+    projectId: 228144,
+  });
+});
+
+it("lets the agent route run without an app directory", () => {
+  assert.equal(readE2eEnv({ ...complete, APP_DIR: "" }, { needsAppDir: false }).appDir, "");
+});
+
+for (const [name, override] of [
+  ["APP_DIR", { APP_DIR: join(appDir, "missing") }],
+  ["POSTHOG_PERSONAL_API_KEY", { POSTHOG_PERSONAL_API_KEY: "" }],
+  ["PROJECT_ID", { PROJECT_ID: "0" }],
+] as const) {
+  it(`names a missing ${name}`, () => {
+    assert.throws(() => readE2eEnv({ ...complete, ...override }), new RegExp(name));
+  });
+}

services/wizard-program/harness.ts

@@ -0,0 +1,240 @@
+/**
+ * Shared inputs and outputs for the headless wizard routes, `run-program.ts`
+ * and `run-agent.ts`. Each runs one wizard surface in this process.
+ *
+ * The wizard is not a dependency of this repo. Its source comes from the
+ * checkout in `WIZARD_REPO`, through the wizard's own path aliases (`@programs`,
+ * `@agent`, `@shared/*`). The package scripts start tsx with
+ * `--tsconfig "$WIZARD_REPO/tsconfig.json"`, so tsx resolves those aliases
+ * against that checkout, here and inside every wizard file.
+ *
+ * Every route reads the same env: a PostHog personal key from
+ * `POSTHOG_PERSONAL_API_KEY` or `POSTHOG_KEY_FILE`, and a project from
+ * `PROJECT_ID`. The wizard's runner mints its own gateway token from that key.
+ * The program route also takes `APP_DIR`. The agent route makes its own empty
+ * directory. `--check` exits once the wizard modules load, before any of that
+ * env is read and before any request.
+ */
+
+import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { isAbsolute, join, relative, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/** `--check`: load the wizard modules a run needs, print where they resolved, exit. */
+export const CHECK = process.argv.includes("--check");
+
+export type E2eEnv = {
+  /** Empty when the route makes its own working directory. */
+  appDir: string;
+  apiKey: string;
+  projectId: number;
+};
+
+/** The progress events the wizard's `AgentProgress` carries, typed here by hand. */
+export type AgentProgress =
+  | { kind: "log"; message: string }
+  | { kind: "status"; message: string }
+  | { kind: "stage"; stage: string }
+  | { kind: "tasks"; tasks: { status: string; content: string }[] }
+  | { kind: "url"; which: string; url: string }
+  | { kind: "authError" }
+  | {
+      kind:
+        | "lifecycle"
+        | "spinner"
+        | "usage"
+        | "finalCost"
+        | "handoff"
+        | "completion"
+        | "activity";
+    };
+
+/** The wizard's `@shared` modules the credential step calls. */
+export type SharedModules = {
+  api: {
+    fetchProjectData(
+      apiKey: string,
+      projectId: number,
+      baseUrl: string,
+    ): Promise<{ api_token: string }>;
+    fetchUserData(apiKey: string, baseUrl: string): Promise<unknown>;
+  };
+  hosts: {
+    HostResolution: {
+      fromAccessToken(
+        apiKey: string,
+        options: { region: string },
+      ): Promise<{ appHost: string }>;
+    };
+  };
+};
+
+export type E2eCredentials = {
+  posthog: {
+    accessToken: string;
+    projectApiKey: string;
+    host: unknown;
+    projectId: number;
+  };
+  project: unknown;
+  apiUser: unknown;
+};
+
+/** Where each wizard module resolved, for `--check`. */
+const loaded: { specifier: string; file: string }[] = [];
+
+/** The wizard checkout the package script handed to tsx. */
+export function wizardRepo(): string {
+  const repo = process.env.WIZARD_REPO?.trim();
+  if (!repo || !existsSync(join(repo, "tsconfig.json")))
+    throw new Error("Set WIZARD_REPO to a wizard checkout with a tsconfig.json");
+  return realpathSync(resolve(repo));
+}
+
+/** `file` relative to the wizard checkout, or null when it sits outside it. */
+function inWizardRepo(file: string): string | null {
+  const inRepo = relative(wizardRepo(), realpathSync(file));
+  return inRepo.startsWith("..") || isAbsolute(inRepo) ? null : inRepo;
+}
+
+/**
+ * Import one wizard module through its alias. Fails unless the alias resolves
+ * inside `WIZARD_REPO` and the module exports every name in `names`.
+ */
+export async function importWizard<T>(
+  specifier: string,
+  names: readonly (keyof T & string)[],
+): Promise<T> {
+  let url: string;
+  try {
+    url = import.meta.resolve(specifier);
+  } catch {
+    throw new Error(
+      `${specifier} did not resolve. Start through the package script, which runs tsx with --tsconfig "$WIZARD_REPO/tsconfig.json".`,
+    );
+  }
+  const file = inWizardRepo(fileURLToPath(url));
+  if (!file) throw new Error(`${specifier} resolved to ${url}, outside WIZARD_REPO ${wizardRepo()}`);
+  loaded.push({ specifier, file });
+  const module = (await import(url)) as Record<string, unknown>;
+  const missing = names.filter((name) => module[name] === undefined);
+  if (missing.length > 0)
+    throw new Error(`${specifier} in ${wizardRepo()} exports no ${missing.join(", ")}`);
+  return module as T;
+}
+
+/** Load a package from the wizard's own dependencies, not the workbench's. */
+export function requireWizardDependency<T>(name: string): T {
+  const wizardRequire = createRequire(join(wizardRepo(), "package.json"));
+  const file = wizardRequire.resolve(name);
+  loaded.push({ specifier: name, file: inWizardRepo(file) ?? file });
+  return wizardRequire(name) as T;
+}
+
+/** Load the `@shared` modules `resolveE2eCredentials` calls. */
+export async function importSharedModules(): Promise<SharedModules> {
+  return {
+    api: await importWizard<SharedModules["api"]>("@shared/api", [
+      "fetchProjectData",
+      "fetchUserData",
+    ]),
+    hosts: await importWizard<SharedModules["hosts"]>(
+      "@shared/host-resolution",
+      ["HostResolution"],
+    ),
+  };
+}
+
+/** Print where every wizard module resolved and exit, before any env read or request. */
+export function exitAfterCheck(route: string): never {
+  console.log(`${route}: wizard modules resolve from ${wizardRepo()}`);
+  for (const { specifier, file } of loaded) console.log(`  ${specifier} -> ${file}`);
+  process.exit(0);
+}
+
+/** A blank variable counts as unset, so the key file is the fallback. */
+export function readPersonalApiKey(
+  env: NodeJS.ProcessEnv,
+  readFile: (file: string) => string = (file) => readFileSync(file, "utf8"),
+): string {
+  const inline = env.POSTHOG_PERSONAL_API_KEY?.trim();
+  if (inline) return inline;
+  const file = env.POSTHOG_KEY_FILE?.trim();
+  return file ? readFile(file).trim() : "";
+}
+
+/** Throws one message listing every missing input, before any run starts. */
+export function readE2eEnv(
+  env: NodeJS.ProcessEnv,
+  { needsAppDir = true }: { needsAppDir?: boolean } = {},
+): E2eEnv {
+  const missing: string[] = [];
+  const appDir = env.APP_DIR?.trim() ?? "";
+  if (needsAppDir && (!appDir || !existsSync(appDir)))
+    missing.push("APP_DIR: an existing app copy, never the fixture in apps/");
+  let apiKey = "";
+  try {
+    apiKey = readPersonalApiKey(env);
+  } catch {
+    // An unreadable key file is reported as a missing key below.
+  }
+  if (!apiKey) missing.push("POSTHOG_PERSONAL_API_KEY or a readable POSTHOG_KEY_FILE");
+  const projectId = Number(env.PROJECT_ID);
+  if (!Number.isInteger(projectId) || projectId <= 0)
+    missing.push("PROJECT_ID: a positive project id");
+  if (missing.length > 0) throw new Error(`Missing e2e inputs:\n- ${missing.join("\n- ")}`);
+  return { appDir, apiKey, projectId };
+}
+
+/**
+ * Resolve PostHog credentials from the personal key through the wizard's
+ * `@shared` modules only, so the agent route loads nothing from programs, the
+ * TUI or the CLI.
+ */
+export async function resolveE2eCredentials(
+  e2e: E2eEnv,
+  shared: SharedModules,
+): Promise<E2eCredentials> {
+  const host = await shared.hosts.HostResolution.fromAccessToken(e2e.apiKey, {
+    region: "us",
+  });
+  const project = await shared.api.fetchProjectData(e2e.apiKey, e2e.projectId, host.appHost);
+  const apiUser = await shared.api.fetchUserData(e2e.apiKey, host.appHost).catch(() => null);
+  return {
+    posthog: {
+      accessToken: e2e.apiKey,
+      projectApiKey: project.api_token,
+      host,
+      projectId: e2e.projectId,
+    },
+    project,
+    apiUser,
+  };
+}
+
+/** Write the route's result to `E2E_RESULT_JSON`, when set, for a caller to assert on. */
+export function writeE2eResult(result: Record<string, unknown>): void {
+  const file = process.env.E2E_RESULT_JSON;
+  if (file) writeFileSync(file, JSON.stringify(result, null, 2));
+}
+
+/** One line per progress event a person would want in a CI log. */
+export function formatProgress(event: AgentProgress): string | null {
+  switch (event.kind) {
+    case "log":
+      return event.message;
+    case "status":
+      return `status: ${event.message}`;
+    case "stage":
+      return `stage: ${event.stage}`;
+    case "tasks":
+      return `tasks: ${event.tasks.map((task) => `${task.status} ${task.content}`).join(", ")}`;
+    case "url":
+      return `${event.which}: ${event.url}`;
+    case "authError":
+      return "gateway rejected the inference token";
+    default:
+      return null;
+  }
+}