From 2b0ac02cde7b0bfd656e019d200ba05f597b0fb7 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 11:20:05 -0400 Subject: [PATCH 1/5] feat(wizard-program): headless runProgram and runAgent harness Run one wizard program through runProgram, or one agent through runAgent, against the wizard checkout in WIZARD_REPO. The package scripts start tsx with the checkout's tsconfig, so the wizard's path aliases resolve there. --check loads the modules a run needs and exits before any env read or request. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- .github/workflows/checks.yml | 3 + README.md | 30 +++ package.json | 5 +- services/wizard-program/harness.test.ts | 59 ++++++ services/wizard-program/harness.ts | 260 ++++++++++++++++++++++++ services/wizard-program/run-agent.ts | 172 ++++++++++++++++ services/wizard-program/run-program.ts | 132 ++++++++++++ 7 files changed, 660 insertions(+), 1 deletion(-) create mode 100644 services/wizard-program/harness.test.ts create mode 100644 services/wizard-program/harness.ts create mode 100644 services/wizard-program/run-agent.ts create mode 100644 services/wizard-program/run-program.ts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b42f8dc17..8f82efe04 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -49,3 +49,6 @@ jobs: - name: Test the PR evaluator run: pnpm test:evaluator + + - name: Test the headless wizard harness + run: pnpm test:wizard-program diff --git a/README.md b/README.md index eb5068590..fbbf30a47 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ services/ ├── wizard-ci/ # Automated wizard runs with PR creation ├── wizard-run/ # Interactive wizard runner ├── mcp-stub/ # Stub PostHog MCP server for warehouse e2e runs +├── wizard-program/ # Headless runProgram / runAgent against a wizard checkout ├── wizard-commands.ts # Registry of wizard commands (integration, revenue, …) └── github/ # GitHub/git utilities ``` @@ -298,6 +299,35 @@ You can activate `wizard-ci.yml` in a few ways: 2. **Schedule** - Runs on cron 3. **Dispatch** - Webhook call via `repository_dispatch` with event type `wizard-ci-trigger` +## Headless wizard runs + +`services/wizard-program/` runs one wizard program through `runProgram`, or one +agent through `runAgent`, in this process with no TUI. It imports them from the +wizard checkout in `WIZARD_REPO`. + +```bash +# Resolve the wizard modules and exit. Reads no credentials, makes no request. +WIZARD_REPO=~/development/wizard pnpm wizard-program --check +WIZARD_REPO=~/development/wizard pnpm wizard-agent --check + +# One program on an app copy. PROGRAM picks it and defaults to posthog-integration. +WIZARD_REPO=… APP_DIR=/tmp/app-copy PROJECT_ID=… POSTHOG_KEY_FILE=… \ + WIZARD_CI_GATEWAY_TOKEN_FILE=… pnpm wizard-program + +# One agent run on a local `quack` skill, in its own empty directory. +WIZARD_REPO=… PROJECT_ID=… POSTHOG_KEY_FILE=… \ + WIZARD_CI_GATEWAY_TOKEN_FILE=… pnpm wizard-agent +``` + +- The scripts start tsx with `--tsconfig "$WIZARD_REPO/tsconfig.json"`, so the + wizard's path aliases (`@programs`, `@agent`, `@shared/*`) resolve against + that checkout. Set `WIZARD_REPO` in the shell: the scripts read it before tsx + starts, so `.env` cannot set it. It is separate from `WIZARD_PATH`. +- Runs without `--check` are live and credentialed. `POSTHOG_PERSONAL_API_KEY` + works in place of `POSTHOG_KEY_FILE`. +- Point `APP_DIR` at a copy, never at a fixture in `apps/`. The run edits it. +- Set `E2E_RESULT_JSON` to a path to get the result as JSON. + --- ## Running with a proxy diff --git a/package.json b/package.json index 1d7d4a00c..121a884d4 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,12 @@ "yara-scan": "tsx services/yara-scan/index.ts", "mcp-stub": "tsx services/mcp-stub/cli.ts", "mcp-stub:record": "tsx services/mcp-stub/record.ts", + "wizard-program": "tsx --tsconfig \"${WIZARD_REPO:?set WIZARD_REPO to a wizard checkout}/tsconfig.json\" services/wizard-program/run-program.ts", + "wizard-agent": "tsx --tsconfig \"${WIZARD_REPO:?set WIZARD_REPO to a wizard checkout}/tsconfig.json\" services/wizard-program/run-agent.ts", "test:evaluator": "tsx --test services/pr-evaluator/evaluator.test.ts", "test:mcp-stub": "tsx --test services/mcp-stub/mcp-stub.test.ts", - "test:warehouse-checks": "tsx --test services/wizard-ci/warehouse-checks.test.ts" + "test:warehouse-checks": "tsx --test services/wizard-ci/warehouse-checks.test.ts", + "test:wizard-program": "tsx --test services/wizard-program/harness.test.ts" }, "dependencies": { "@anthropic-ai/claude-agent-sdk": "0.2.73", diff --git a/services/wizard-program/harness.test.ts b/services/wizard-program/harness.test.ts new file mode 100644 index 000000000..96e738d2f --- /dev/null +++ b/services/wizard-program/harness.test.ts @@ -0,0 +1,59 @@ +/** + * Pins the env contract both headless wizard routes share, so a route never + * starts a live run with an input the other would reject. + * + * pnpm test:wizard-program + */ + +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { it, mock } from "node:test"; + +import { readE2eEnv, readPersonalApiKey } from "./harness.js"; + +const appDir = mkdtempSync(join(tmpdir(), "wizard-program-")); +const complete = { + APP_DIR: appDir, + POSTHOG_PERSONAL_API_KEY: "phx_inline", + PROJECT_ID: "228144", + WIZARD_CI_GATEWAY_TOKEN_FILE: "/tokens/gateway", +}; + +it("prefers the inline key and falls back to the key file when it is blank", () => { + const readFile = mock.fn((_file: string) => " phx_file \n"); + assert.equal(readPersonalApiKey(complete, readFile), "phx_inline"); + assert.equal( + readPersonalApiKey({ POSTHOG_PERSONAL_API_KEY: " ", POSTHOG_KEY_FILE: "/keys/phx" }, readFile), + "phx_file", + ); + assert.deepEqual( + readFile.mock.calls.map((call) => call.arguments), + [["/keys/phx"]], + ); +}); + +it("reads a complete env", () => { + assert.deepEqual(readE2eEnv(complete), { + appDir, + apiKey: "phx_inline", + projectId: 228144, + gatewayTokenFile: "/tokens/gateway", + }); +}); + +it("lets the agent route run without an app directory", () => { + assert.equal(readE2eEnv({ ...complete, APP_DIR: "" }, { needsAppDir: false }).appDir, ""); +}); + +for (const [name, override] of [ + ["APP_DIR", { APP_DIR: join(appDir, "missing") }], + ["POSTHOG_PERSONAL_API_KEY", { POSTHOG_PERSONAL_API_KEY: "" }], + ["PROJECT_ID", { PROJECT_ID: "0" }], + ["WIZARD_CI_GATEWAY_TOKEN_FILE", { WIZARD_CI_GATEWAY_TOKEN_FILE: " " }], +] as const) { + it(`names a missing ${name}`, () => { + assert.throws(() => readE2eEnv({ ...complete, ...override }), new RegExp(name)); + }); +} diff --git a/services/wizard-program/harness.ts b/services/wizard-program/harness.ts new file mode 100644 index 000000000..b210129f8 --- /dev/null +++ b/services/wizard-program/harness.ts @@ -0,0 +1,260 @@ +/** + * Shared inputs and outputs for the headless wizard routes, `run-program.ts` + * and `run-agent.ts`. Each runs one wizard surface in this process. + * + * The wizard is not a dependency of this repo. Its source comes from the + * checkout in `WIZARD_REPO`, through the wizard's own path aliases (`@programs`, + * `@agent`, `@shared/*`). The package scripts start tsx with + * `--tsconfig "$WIZARD_REPO/tsconfig.json"`, so tsx resolves those aliases + * against that checkout, here and inside every wizard file. + * + * Every route reads the same env: a PostHog personal key from + * `POSTHOG_PERSONAL_API_KEY` or `POSTHOG_KEY_FILE`, a project from `PROJECT_ID`, + * and an already-issued gateway token through `WIZARD_CI_GATEWAY_TOKEN_FILE`. + * The program route also takes `APP_DIR`. The agent route makes its own empty + * directory. `--check` exits once the wizard modules load, before any of that + * env is read and before any request. + */ + +import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** `--check`: load the wizard modules a run needs, print where they resolved, exit. */ +export const CHECK = process.argv.includes("--check"); + +export type E2eEnv = { + /** Empty when the route makes its own working directory. */ + appDir: string; + apiKey: string; + projectId: number; + gatewayTokenFile: string; +}; + +/** The progress events the wizard's `AgentProgress` carries, typed here by hand. */ +export type AgentProgress = + | { kind: "log"; message: string } + | { kind: "status"; message: string } + | { kind: "stage"; stage: string } + | { kind: "tasks"; tasks: { status: string; content: string }[] } + | { kind: "url"; which: string; url: string } + | { kind: "authError" } + | { + kind: + | "lifecycle" + | "spinner" + | "usage" + | "finalCost" + | "handoff" + | "completion" + | "activity"; + }; + +/** The wizard's `@shared` modules the credential step calls. */ +export type SharedModules = { + api: { + fetchProjectData( + apiKey: string, + projectId: number, + baseUrl: string, + ): Promise<{ api_token: string }>; + fetchUserData(apiKey: string, baseUrl: string): Promise; + }; + gateway: { + createCiGatewayAuth(token: string, projectId: number, url: string): unknown; + }; + hosts: { + HostResolution: { + fromAccessToken( + apiKey: string, + options: { region: string }, + ): Promise<{ appHost: string }>; + }; + }; +}; + +export type E2eCredentials = { + posthog: { + accessToken: string; + projectApiKey: string; + host: unknown; + projectId: number; + }; + inferenceAuth: { resolve: () => Promise }; + project: unknown; + apiUser: unknown; +}; + +/** Where each wizard module resolved, for `--check`. */ +const loaded: { specifier: string; file: string }[] = []; + +/** The wizard checkout the package script handed to tsx. */ +export function wizardRepo(): string { + const repo = process.env.WIZARD_REPO?.trim(); + if (!repo || !existsSync(join(repo, "tsconfig.json"))) + throw new Error("Set WIZARD_REPO to a wizard checkout with a tsconfig.json"); + return realpathSync(resolve(repo)); +} + +/** `file` relative to the wizard checkout, or null when it sits outside it. */ +function inWizardRepo(file: string): string | null { + const inRepo = relative(wizardRepo(), realpathSync(file)); + return inRepo.startsWith("..") || isAbsolute(inRepo) ? null : inRepo; +} + +/** + * Import one wizard module through its alias. Fails unless the alias resolves + * inside `WIZARD_REPO` and the module exports every name in `names`. + */ +export async function importWizard( + specifier: string, + names: readonly (keyof T & string)[], +): Promise { + let url: string; + try { + url = import.meta.resolve(specifier); + } catch { + throw new Error( + `${specifier} did not resolve. Start through the package script, which runs tsx with --tsconfig "$WIZARD_REPO/tsconfig.json".`, + ); + } + const file = inWizardRepo(fileURLToPath(url)); + if (!file) throw new Error(`${specifier} resolved to ${url}, outside WIZARD_REPO ${wizardRepo()}`); + loaded.push({ specifier, file }); + const module = (await import(url)) as Record; + const missing = names.filter((name) => module[name] === undefined); + if (missing.length > 0) + throw new Error(`${specifier} in ${wizardRepo()} exports no ${missing.join(", ")}`); + return module as T; +} + +/** Load a package from the wizard's own dependencies, not the workbench's. */ +export function requireWizardDependency(name: string): T { + const wizardRequire = createRequire(join(wizardRepo(), "package.json")); + const file = wizardRequire.resolve(name); + loaded.push({ specifier: name, file: inWizardRepo(file) ?? file }); + return wizardRequire(name) as T; +} + +/** Load the `@shared` modules `resolveE2eCredentials` calls. */ +export async function importSharedModules(): Promise { + return { + api: await importWizard("@shared/api", [ + "fetchProjectData", + "fetchUserData", + ]), + gateway: await importWizard( + "@shared/ci-gateway-auth", + ["createCiGatewayAuth"], + ), + hosts: await importWizard( + "@shared/host-resolution", + ["HostResolution"], + ), + }; +} + +/** Print where every wizard module resolved and exit, before any env read or request. */ +export function exitAfterCheck(route: string): never { + console.log(`${route}: wizard modules resolve from ${wizardRepo()}`); + for (const { specifier, file } of loaded) console.log(` ${specifier} -> ${file}`); + process.exit(0); +} + +/** A blank variable counts as unset, so the key file is the fallback. */ +export function readPersonalApiKey( + env: NodeJS.ProcessEnv, + readFile: (file: string) => string = (file) => readFileSync(file, "utf8"), +): string { + const inline = env.POSTHOG_PERSONAL_API_KEY?.trim(); + if (inline) return inline; + const file = env.POSTHOG_KEY_FILE?.trim(); + return file ? readFile(file).trim() : ""; +} + +/** Throws one message listing every missing input, before any run starts. */ +export function readE2eEnv( + env: NodeJS.ProcessEnv, + { needsAppDir = true }: { needsAppDir?: boolean } = {}, +): E2eEnv { + const missing: string[] = []; + const appDir = env.APP_DIR?.trim() ?? ""; + if (needsAppDir && (!appDir || !existsSync(appDir))) + missing.push("APP_DIR: an existing app copy, never the fixture in apps/"); + let apiKey = ""; + try { + apiKey = readPersonalApiKey(env); + } catch { + // An unreadable key file is reported as a missing key below. + } + if (!apiKey) missing.push("POSTHOG_PERSONAL_API_KEY or a readable POSTHOG_KEY_FILE"); + const projectId = Number(env.PROJECT_ID); + if (!Number.isInteger(projectId) || projectId <= 0) + missing.push("PROJECT_ID: a positive project id"); + const gatewayTokenFile = env.WIZARD_CI_GATEWAY_TOKEN_FILE?.trim() ?? ""; + if (!gatewayTokenFile) + missing.push("WIZARD_CI_GATEWAY_TOKEN_FILE: an already-issued gateway token"); + if (missing.length > 0) throw new Error(`Missing e2e inputs:\n- ${missing.join("\n- ")}`); + return { appDir, apiKey, projectId, gatewayTokenFile }; +} + +/** + * Resolve PostHog credentials from the personal key through the wizard's + * `@shared` modules only, so the agent route loads nothing from programs, the + * TUI or the CLI. + */ +export async function resolveE2eCredentials( + e2e: E2eEnv, + shared: SharedModules, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const host = await shared.hosts.HostResolution.fromAccessToken(e2e.apiKey, { + region: "us", + }); + const project = await shared.api.fetchProjectData(e2e.apiKey, e2e.projectId, host.appHost); + const apiUser = await shared.api.fetchUserData(e2e.apiKey, host.appHost).catch(() => null); + const token = readFileSync(e2e.gatewayTokenFile, "utf8"); + const gateway = shared.gateway.createCiGatewayAuth( + token, + e2e.projectId, + env.WIZARD_CI_GATEWAY_URL || "https://ai-gateway.us.posthog.com", + ); + return { + posthog: { + accessToken: e2e.apiKey, + projectApiKey: project.api_token, + host, + projectId: e2e.projectId, + }, + inferenceAuth: { resolve: () => Promise.resolve(gateway) }, + project, + apiUser, + }; +} + +/** Write the route's result to `E2E_RESULT_JSON`, when set, for a caller to assert on. */ +export function writeE2eResult(result: Record): void { + const file = process.env.E2E_RESULT_JSON; + if (file) writeFileSync(file, JSON.stringify(result, null, 2)); +} + +/** One line per progress event a person would want in a CI log. */ +export function formatProgress(event: AgentProgress): string | null { + switch (event.kind) { + case "log": + return event.message; + case "status": + return `status: ${event.message}`; + case "stage": + return `stage: ${event.stage}`; + case "tasks": + return `tasks: ${event.tasks.map((task) => `${task.status} ${task.content}`).join(", ")}`; + case "url": + return `${event.which}: ${event.url}`; + case "authError": + return "gateway rejected the inference token"; + default: + return null; + } +} diff --git a/services/wizard-program/run-agent.ts b/services/wizard-program/run-agent.ts new file mode 100644 index 000000000..cb6225856 --- /dev/null +++ b/services/wizard-program/run-agent.ts @@ -0,0 +1,172 @@ +/** + * `pnpm wizard-agent` — one real agent run through the wizard's `runAgent` on + * a skill that has nothing to do with PostHog programs. A local skills server + * hands the agent a `quack` skill, the agent writes `quack/quack.txt` into an + * empty directory, and this script checks the file. No programs, TUI, store or + * context-mill are involved. + * + * WIZARD_REPO= PROJECT_ID=… POSTHOG_KEY_FILE=… \ + * WIZARD_CI_GATEWAY_TOKEN_FILE=… [E2E_RESULT_JSON=result.json] \ + * pnpm wizard-agent + * + * WIZARD_REPO= pnpm wizard-agent --check + */ + +import { existsSync, mkdtempSync, readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + CHECK, + exitAfterCheck, + formatProgress, + importSharedModules, + importWizard, + readE2eEnv, + requireWizardDependency, + resolveE2eCredentials, + writeE2eResult, + type AgentProgress, +} from "./harness.js"; + +/** The slice of `@agent` this route calls. */ +type Agent = { + runAgent( + config: Record, + input: Record, + options: { onProgress: (event: AgentProgress) => void }, + ): Promise<{ outcome: string; failure?: { message: string } }>; + RunOutcome: { Success: string }; + DEFAULT_AGENT_BINDING: unknown; +}; +type Fflate = { zipSync(files: Record): Uint8Array }; + +const SKILL_ID = "quack"; +const QUACK_FILE = join("quack", "quack.txt"); +const SKILL_MD = `--- +name: quack +description: Write the word quack into quack/quack.txt. +--- + +# Quack + +1. Create a directory named \`quack\` in the current working directory. +2. Write a file \`quack/quack.txt\` whose entire content is the word \`quack\`. +3. Change nothing else. Do not install packages or call any PostHog tool. +`; + +/** Serve a one-skill menu and its archive on a loopback port. */ +async function serveQuackSkill(fflate: Fflate): Promise<{ url: string; close: () => void }> { + const archive = Buffer.from(fflate.zipSync({ "SKILL.md": new TextEncoder().encode(SKILL_MD) })); + let base = ""; + const server = createServer((req, res) => { + if (req.url === "/skill-menu.json") { + res.setHeader("content-type", "application/json"); + res.end( + JSON.stringify({ + categories: { + e2e: [{ id: SKILL_ID, name: "Quack", downloadUrl: `${base}/quack.zip` }], + }, + }), + ); + } else if (req.url === "/quack.zip") { + res.setHeader("content-type", "application/zip"); + res.end(archive); + } else { + res.statusCode = 404; + res.end(); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + const address = server.address() as { port: number }; + base = `http://127.0.0.1:${address.port}`; + return { url: base, close: () => server.close() }; +} + +async function main(): Promise { + const { runAgent, RunOutcome, DEFAULT_AGENT_BINDING } = await importWizard("@agent", [ + "runAgent", + "RunOutcome", + "DEFAULT_AGENT_BINDING", + ]); + const shared = await importSharedModules(); + const fflate = requireWizardDependency("fflate"); + if (CHECK) exitAfterCheck("wizard-agent"); + + const e2e = readE2eEnv(process.env, { needsAppDir: false }); + const workDir = mkdtempSync(join(tmpdir(), "wizard-e2e-agent-")); + const credentials = await resolveE2eCredentials(e2e, shared); + const skills = await serveQuackSkill(fflate); + + const config = { + programId: "e2e-agent", + run: { + integrationLabel: SKILL_ID, + skillId: SKILL_ID, + spinnerMessage: "Quacking", + successMessage: "Quacked", + estimatedDurationMinutes: 1, + reportFile: "quack-report.md", + docsUrl: "https://posthog.com/docs", + }, + composed: false, + binding: DEFAULT_AGENT_BINDING, + skillsBaseUrl: skills.url, + wizardFlags: {}, + wizardFlagPayloads: {}, + wizardMetadata: {}, + }; + + try { + const result = await runAgent( + config, + { + installDir: workDir, + credentials: credentials.posthog, + inferenceAuth: credentials.inferenceAuth, + project: credentials.project, + apiUser: credentials.apiUser, + skillId: SKILL_ID, + flags: { + ci: true, + signup: false, + debug: false, + e2eAsk: false, + localMcp: false, + captureAio: false, + benchmark: false, + yaraReport: false, + }, + host: {}, + }, + { + onProgress: (event) => { + const line = formatProgress(event); + if (line) console.log(line); + }, + }, + ); + + const quackPath = join(workDir, QUACK_FILE); + const quack = existsSync(quackPath) ? readFileSync(quackPath, "utf8").trim() : null; + const passed = result.outcome === RunOutcome.Success && quack === "quack"; + writeE2eResult({ + route: "agent", + skillId: SKILL_ID, + workDir, + outcome: result.outcome, + failure: result.outcome === RunOutcome.Success ? null : (result.failure?.message ?? null), + quack, + passed, + }); + console.log(`${SKILL_ID}: ${result.outcome}, ${QUACK_FILE} = ${quack}`); + if (!passed) process.exitCode = 1; + } finally { + skills.close(); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/services/wizard-program/run-program.ts b/services/wizard-program/run-program.ts new file mode 100644 index 000000000..2a415ed40 --- /dev/null +++ b/services/wizard-program/run-program.ts @@ -0,0 +1,132 @@ +/** + * `pnpm wizard-program` — one real program run through the wizard's + * `runProgram`, with no TUI, no store and no session. This process is the + * host: it detects the framework and supplies the integration effects a CLI + * host would. + * + * WIZARD_REPO= APP_DIR= PROJECT_ID=… \ + * POSTHOG_KEY_FILE=… WIZARD_CI_GATEWAY_TOKEN_FILE=… \ + * [PROGRAM=posthog-integration] [E2E_RESULT_JSON=result.json] \ + * pnpm wizard-program + * + * WIZARD_REPO= pnpm wizard-program --check + */ + +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + CHECK, + exitAfterCheck, + formatProgress, + importSharedModules, + importWizard, + readE2eEnv, + resolveE2eCredentials, + writeE2eResult, + type AgentProgress, +} from "./harness.js"; + +/** The slice of `ProgramRunOutcome` this route reads. */ +type ProgramRunOutcome = { + outcome: string; + failure?: { message: string }; + settledRuns: unknown[]; + runResults: { snapshot: { tasks: unknown[] } }[]; +}; + +type ProgramProgress = { kind: "run"; event: AgentProgress } | { kind: "program" }; + +type Programs = { + runProgram( + programId: string, + input: Record, + options: { + integrationEffects: Record; + onProgress: (progress: ProgramProgress) => void; + }, + ): Promise; +}; +type Registry = { FRAMEWORK_REGISTRY: Record }; +type Detection = { detectFramework(installDir: string): Promise }; + +type PackageJson = { + dependencies?: Record; + devDependencies?: Record; +} | null; + +const effects = { + readPackageJson: (installDir: string) => { + const file = join(installDir, "package.json"); + return Promise.resolve(existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : null); + }, + hasDeclaredDependency: (name: string, packageJson: unknown) => { + const pkg = packageJson as PackageJson; + return Boolean(pkg?.dependencies?.[name] ?? pkg?.devDependencies?.[name]); + }, + warn: (message: string) => console.warn(message), + setTag: () => undefined, + capture: () => undefined, + // A synthetic run never writes to a hosting provider. + uploadEnvironmentVariables: () => Promise.resolve([]), + requestDeepLink: () => Promise.resolve(null), + openDashboardDeepLink: () => undefined, +}; + +async function main(): Promise { + const { runProgram } = await importWizard("@programs", ["runProgram"]); + const { FRAMEWORK_REGISTRY } = await importWizard("@programs/registry", [ + "FRAMEWORK_REGISTRY", + ]); + const { detectFramework } = await importWizard("@programs/detection/framework", [ + "detectFramework", + ]); + const shared = await importSharedModules(); + if (CHECK) exitAfterCheck("wizard-program"); + + const e2e = readE2eEnv(process.env); + const programId = process.env.PROGRAM || "posthog-integration"; + const credentials = await resolveE2eCredentials(e2e, shared); + + const integration = + programId === "posthog-integration" ? await detectFramework(e2e.appDir) : undefined; + if (programId === "posthog-integration" && !integration) + throw new Error(`No supported framework detected in ${e2e.appDir}`); + + const outcome = await runProgram( + programId, + { + installDir: e2e.appDir, + credentials, + integration: integration ?? null, + frameworkConfig: integration ? FRAMEWORK_REGISTRY[integration] : undefined, + frameworkContext: {}, + flags: { ci: true }, + }, + { + integrationEffects: effects, + onProgress: (progress) => { + // Program-data snapshots carry state, not a line to print. + if (progress.kind !== "run") return; + const line = formatProgress(progress.event); + if (line) console.log(line); + }, + }, + ); + + writeE2eResult({ + route: "programs", + programId, + integration: integration ?? null, + outcome: outcome.outcome, + failure: outcome.failure?.message ?? null, + settledRuns: outcome.settledRuns.length, + tasks: outcome.runResults.flatMap((run) => run.snapshot.tasks), + }); + console.log(`${programId}: ${outcome.outcome}`); + if (outcome.outcome !== "success") process.exitCode = 1; +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); From 83a68398c7f3f30297f1b3263ab8d0041a710217 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 12:39:44 -0400 Subject: [PATCH 2/5] fix(wizard-program): read task snapshots from settledRuns Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- services/wizard-program/run-program.ts | 42 +++++++++++++++++--------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/services/wizard-program/run-program.ts b/services/wizard-program/run-program.ts index 2a415ed40..518bf0718 100644 --- a/services/wizard-program/run-program.ts +++ b/services/wizard-program/run-program.ts @@ -30,11 +30,11 @@ import { type ProgramRunOutcome = { outcome: string; failure?: { message: string }; - settledRuns: unknown[]; - runResults: { snapshot: { tasks: unknown[] } }[]; + settledRuns: { result: { snapshot?: { tasks: unknown[] } } }[]; }; -type ProgramProgress = { kind: "run"; event: AgentProgress } | { kind: "program" }; +type ProgramProgress = + { kind: "run"; event: AgentProgress } | { kind: "program" }; type Programs = { runProgram( @@ -47,7 +47,9 @@ type Programs = { ): Promise; }; type Registry = { FRAMEWORK_REGISTRY: Record }; -type Detection = { detectFramework(installDir: string): Promise }; +type Detection = { + detectFramework(installDir: string): Promise; +}; type PackageJson = { dependencies?: Record; @@ -57,7 +59,9 @@ type PackageJson = { const effects = { readPackageJson: (installDir: string) => { const file = join(installDir, "package.json"); - return Promise.resolve(existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : null); + return Promise.resolve( + existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : null, + ); }, hasDeclaredDependency: (name: string, packageJson: unknown) => { const pkg = packageJson as PackageJson; @@ -73,13 +77,17 @@ const effects = { }; async function main(): Promise { - const { runProgram } = await importWizard("@programs", ["runProgram"]); - const { FRAMEWORK_REGISTRY } = await importWizard("@programs/registry", [ - "FRAMEWORK_REGISTRY", - ]); - const { detectFramework } = await importWizard("@programs/detection/framework", [ - "detectFramework", + const { runProgram } = await importWizard("@programs", [ + "runProgram", ]); + const { FRAMEWORK_REGISTRY } = await importWizard( + "@programs/registry", + ["FRAMEWORK_REGISTRY"], + ); + const { detectFramework } = await importWizard( + "@programs/detection/framework", + ["detectFramework"], + ); const shared = await importSharedModules(); if (CHECK) exitAfterCheck("wizard-program"); @@ -88,7 +96,9 @@ async function main(): Promise { const credentials = await resolveE2eCredentials(e2e, shared); const integration = - programId === "posthog-integration" ? await detectFramework(e2e.appDir) : undefined; + programId === "posthog-integration" + ? await detectFramework(e2e.appDir) + : undefined; if (programId === "posthog-integration" && !integration) throw new Error(`No supported framework detected in ${e2e.appDir}`); @@ -98,7 +108,9 @@ async function main(): Promise { installDir: e2e.appDir, credentials, integration: integration ?? null, - frameworkConfig: integration ? FRAMEWORK_REGISTRY[integration] : undefined, + frameworkConfig: integration + ? FRAMEWORK_REGISTRY[integration] + : undefined, frameworkContext: {}, flags: { ci: true }, }, @@ -120,7 +132,9 @@ async function main(): Promise { outcome: outcome.outcome, failure: outcome.failure?.message ?? null, settledRuns: outcome.settledRuns.length, - tasks: outcome.runResults.flatMap((run) => run.snapshot.tasks), + tasks: outcome.settledRuns.flatMap( + (run) => run.result.snapshot?.tasks ?? [], + ), }); console.log(`${programId}: ${outcome.outcome}`); if (outcome.outcome !== "success") process.exitCode = 1; From 23b43bbc5a9444cd71902d31bbb1671bff263c4f Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 16:04:03 -0400 Subject: [PATCH 3/5] fix(wizard-program): build the run and program settings for runProgram runProgram is now host-only: it takes the run definition in input.run and the program-level settings in input.program, and drops integration effects and framework config. The program route builds both from the program's ProgramConfig, as the wizard's legacy adapter does: it calls run(session) on a session from buildSession, after it installs the headless UI that --ci installs. posthog-integration still detects its framework here, now onto the session that run(session) reads. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- services/wizard-program/run-program.ts | 153 ++++++++++++++++--------- 1 file changed, 101 insertions(+), 52 deletions(-) diff --git a/services/wizard-program/run-program.ts b/services/wizard-program/run-program.ts index 518bf0718..a7a541822 100644 --- a/services/wizard-program/run-program.ts +++ b/services/wizard-program/run-program.ts @@ -1,8 +1,8 @@ /** * `pnpm wizard-program` — one real program run through the wizard's - * `runProgram`, with no TUI, no store and no session. This process is the - * host: it detects the framework and supplies the integration effects a CLI - * host would. + * `runProgram`, with no TUI. This process is the host: it builds the run + * definition and program settings from the program's `ProgramConfig`, the way + * the wizard's legacy adapter does, and supplies the credentials. * * WIZARD_REPO= APP_DIR= PROJECT_ID=… \ * POSTHOG_KEY_FILE=… WIZARD_CI_GATEWAY_TOKEN_FILE=… \ @@ -12,8 +12,6 @@ * WIZARD_REPO= pnpm wizard-program --check */ -import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; import { CHECK, exitAfterCheck, @@ -36,49 +34,69 @@ type ProgramRunOutcome = { type ProgramProgress = { kind: "run"; event: AgentProgress } | { kind: "program" }; +type FrameworkConfig = { metadata: { docsUrl: string } }; + +/** The slice of `WizardSession` this route reads and writes. */ +type Session = { + installDir: string; + integration: string | null; + frameworkConfig: FrameworkConfig | null; +}; + +/** The slice of `ProgramConfig` a run is built from. */ +type ProgramConfig = { + steps: unknown[]; + run?: object | ((session: Session) => Promise); + requiresAi?: boolean; + agentFlow?: string; + allowedTools?: readonly string[]; + disallowedTools?: readonly string[]; + excludedTaskTypes?: unknown; + auditLedgerFile?: string; + auditSeedChecks?: readonly unknown[]; + eventPlanFile?: string; +}; + type Programs = { runProgram( programId: string, input: Record, - options: { - integrationEffects: Record; - onProgress: (progress: ProgramProgress) => void; - }, + options: { onProgress: (progress: ProgramProgress) => void }, ): Promise; + getProgramConfig(programId: string): ProgramConfig | undefined; +}; +type ProgramSteps = { + postAuthGateSteps(steps: unknown[]): { id: string }[]; +}; +type Sessions = { + buildSession(args: { installDir: string; ci: boolean }): Session; }; -type Registry = { FRAMEWORK_REGISTRY: Record }; +type Ui = { setUI(ui: unknown): void }; +type HeadlessUi = { HeadlessUI: new (store: unknown) => unknown }; +type Store = { WizardStore: new (programId: string) => { session: Session } }; +type Registry = { FRAMEWORK_REGISTRY: Record }; type Detection = { detectFramework(installDir: string): Promise; }; -type PackageJson = { - dependencies?: Record; - devDependencies?: Record; -} | null; - -const effects = { - readPackageJson: (installDir: string) => { - const file = join(installDir, "package.json"); - return Promise.resolve( - existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : null, - ); - }, - hasDeclaredDependency: (name: string, packageJson: unknown) => { - const pkg = packageJson as PackageJson; - return Boolean(pkg?.dependencies?.[name] ?? pkg?.devDependencies?.[name]); - }, - warn: (message: string) => console.warn(message), - setTag: () => undefined, - capture: () => undefined, - // A synthetic run never writes to a hosting provider. - uploadEnvironmentVariables: () => Promise.resolve([]), - requestDeepLink: () => Promise.resolve(null), - openDashboardDeepLink: () => undefined, -}; - async function main(): Promise { - const { runProgram } = await importWizard("@programs", [ - "runProgram", + const { runProgram, getProgramConfig } = await importWizard( + "@programs", + ["runProgram", "getProgramConfig"], + ); + const { postAuthGateSteps } = await importWizard( + "@programs/program-step", + ["postAuthGateSteps"], + ); + const { buildSession } = await importWizard("@lib/wizard-session", [ + "buildSession", + ]); + const { setUI } = await importWizard("@ui", ["setUI"]); + const { HeadlessUI } = await importWizard("@ui/headless-ui", [ + "HeadlessUI", + ]); + const { WizardStore } = await importWizard("@ui/tui/store", [ + "WizardStore", ]); const { FRAMEWORK_REGISTRY } = await importWizard( "@programs/registry", @@ -93,29 +111,60 @@ async function main(): Promise { const e2e = readE2eEnv(process.env); const programId = process.env.PROGRAM || "posthog-integration"; - const credentials = await resolveE2eCredentials(e2e, shared); + const programConfig = getProgramConfig(programId); + if (!programConfig?.run) + throw new Error(`${programId} is not a registered program with a run`); + + // A program's `run(session)` can call `getUI()`. Install the headless UI + // the `--ci` runner installs, over a store that holds this session. + const session = buildSession({ installDir: e2e.appDir, ci: true }); + const store = new WizardStore(programId); + store.session = session; + setUI(new HeadlessUI(store)); - const integration = - programId === "posthog-integration" - ? await detectFramework(e2e.appDir) - : undefined; - if (programId === "posthog-integration" && !integration) - throw new Error(`No supported framework detected in ${e2e.appDir}`); + // posthog-integration's `run(session)` reads the framework off the session. + // A `--ci` run fills it in `ciPreRun`, which also logs in and can scan the + // repo with an agent, so this route detects the framework alone. + if (programId === "posthog-integration") { + const integration = await detectFramework(e2e.appDir); + if (!integration) + throw new Error(`No supported framework detected in ${e2e.appDir}`); + session.integration = integration; + session.frameworkConfig = FRAMEWORK_REGISTRY[integration]; + } + const credentials = await resolveE2eCredentials(e2e, shared); + const run = + typeof programConfig.run === "function" + ? await programConfig.run(session) + : programConfig.run; + + // No hooks or seed tasks: postRun uploads env vars to a hosting provider, + // the outro builders feed a screen, and a CI session seeds no tasks. const outcome = await runProgram( programId, { - installDir: e2e.appDir, + installDir: session.installDir, + run, + program: { + requiresAi: programConfig.requiresAi, + agentFlow: programConfig.agentFlow, + allowedTools: programConfig.allowedTools, + disallowedTools: programConfig.disallowedTools, + excludedTaskTypes: programConfig.excludedTaskTypes, + auditLedgerFile: programConfig.auditLedgerFile, + auditSeedChecks: programConfig.auditSeedChecks, + eventPlanFile: programConfig.eventPlanFile, + postAuthGates: postAuthGateSteps(programConfig.steps).map( + (step) => step.id, + ), + }, credentials, - integration: integration ?? null, - frameworkConfig: integration - ? FRAMEWORK_REGISTRY[integration] - : undefined, - frameworkContext: {}, + integration: session.integration, + frameworkDocsUrl: session.frameworkConfig?.metadata.docsUrl, flags: { ci: true }, }, { - integrationEffects: effects, onProgress: (progress) => { // Program-data snapshots carry state, not a line to print. if (progress.kind !== "run") return; @@ -128,7 +177,7 @@ async function main(): Promise { writeE2eResult({ route: "programs", programId, - integration: integration ?? null, + integration: session.integration, outcome: outcome.outcome, failure: outcome.failure?.message ?? null, settledRuns: outcome.settledRuns.length, From a01ba169eed02d12901e31c3de1a104c3d58b188 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 19:32:02 -0400 Subject: [PATCH 4/5] fix(wizard-program): match the host-capabilities runProgram and runAgent The program route builds a ProgramRunHost from the headless UI, the way the wizard's legacy adapter does, and passes it to the program's run. Its program settings drop the audit and event-plan fields runProgram no longer takes. The agent route binds its run the way agentic detection does, with a switchboard context, and passes no inference auth. The runner mints its own gateway token from the personal key, so the harness no longer reads WIZARD_CI_GATEWAY_TOKEN_FILE or loads @shared/ci-gateway-auth. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- README.md | 8 ++--- services/wizard-program/harness.test.ts | 3 -- services/wizard-program/harness.ts | 26 ++--------------- services/wizard-program/run-agent.ts | 28 +++++++++++------- services/wizard-program/run-program.ts | 39 ++++++++++++++++--------- 5 files changed, 51 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index fbbf30a47..24d2096dc 100644 --- a/README.md +++ b/README.md @@ -312,11 +312,10 @@ WIZARD_REPO=~/development/wizard pnpm wizard-agent --check # One program on an app copy. PROGRAM picks it and defaults to posthog-integration. WIZARD_REPO=… APP_DIR=/tmp/app-copy PROJECT_ID=… POSTHOG_KEY_FILE=… \ - WIZARD_CI_GATEWAY_TOKEN_FILE=… pnpm wizard-program + pnpm wizard-program # One agent run on a local `quack` skill, in its own empty directory. -WIZARD_REPO=… PROJECT_ID=… POSTHOG_KEY_FILE=… \ - WIZARD_CI_GATEWAY_TOKEN_FILE=… pnpm wizard-agent +WIZARD_REPO=… PROJECT_ID=… POSTHOG_KEY_FILE=… pnpm wizard-agent ``` - The scripts start tsx with `--tsconfig "$WIZARD_REPO/tsconfig.json"`, so the @@ -324,7 +323,8 @@ WIZARD_REPO=… PROJECT_ID=… POSTHOG_KEY_FILE=… \ that checkout. Set `WIZARD_REPO` in the shell: the scripts read it before tsx starts, so `.env` cannot set it. It is separate from `WIZARD_PATH`. - Runs without `--check` are live and credentialed. `POSTHOG_PERSONAL_API_KEY` - works in place of `POSTHOG_KEY_FILE`. + works in place of `POSTHOG_KEY_FILE`. The wizard's runner mints its own + gateway token from that key. - Point `APP_DIR` at a copy, never at a fixture in `apps/`. The run edits it. - Set `E2E_RESULT_JSON` to a path to get the result as JSON. diff --git a/services/wizard-program/harness.test.ts b/services/wizard-program/harness.test.ts index 96e738d2f..b2f4a74f5 100644 --- a/services/wizard-program/harness.test.ts +++ b/services/wizard-program/harness.test.ts @@ -18,7 +18,6 @@ const complete = { APP_DIR: appDir, POSTHOG_PERSONAL_API_KEY: "phx_inline", PROJECT_ID: "228144", - WIZARD_CI_GATEWAY_TOKEN_FILE: "/tokens/gateway", }; it("prefers the inline key and falls back to the key file when it is blank", () => { @@ -39,7 +38,6 @@ it("reads a complete env", () => { appDir, apiKey: "phx_inline", projectId: 228144, - gatewayTokenFile: "/tokens/gateway", }); }); @@ -51,7 +49,6 @@ for (const [name, override] of [ ["APP_DIR", { APP_DIR: join(appDir, "missing") }], ["POSTHOG_PERSONAL_API_KEY", { POSTHOG_PERSONAL_API_KEY: "" }], ["PROJECT_ID", { PROJECT_ID: "0" }], - ["WIZARD_CI_GATEWAY_TOKEN_FILE", { WIZARD_CI_GATEWAY_TOKEN_FILE: " " }], ] as const) { it(`names a missing ${name}`, () => { assert.throws(() => readE2eEnv({ ...complete, ...override }), new RegExp(name)); diff --git a/services/wizard-program/harness.ts b/services/wizard-program/harness.ts index b210129f8..1a3f19dc3 100644 --- a/services/wizard-program/harness.ts +++ b/services/wizard-program/harness.ts @@ -9,8 +9,8 @@ * against that checkout, here and inside every wizard file. * * Every route reads the same env: a PostHog personal key from - * `POSTHOG_PERSONAL_API_KEY` or `POSTHOG_KEY_FILE`, a project from `PROJECT_ID`, - * and an already-issued gateway token through `WIZARD_CI_GATEWAY_TOKEN_FILE`. + * `POSTHOG_PERSONAL_API_KEY` or `POSTHOG_KEY_FILE`, and a project from + * `PROJECT_ID`. The wizard's runner mints its own gateway token from that key. * The program route also takes `APP_DIR`. The agent route makes its own empty * directory. `--check` exits once the wizard modules load, before any of that * env is read and before any request. @@ -29,7 +29,6 @@ export type E2eEnv = { appDir: string; apiKey: string; projectId: number; - gatewayTokenFile: string; }; /** The progress events the wizard's `AgentProgress` carries, typed here by hand. */ @@ -61,9 +60,6 @@ export type SharedModules = { ): Promise<{ api_token: string }>; fetchUserData(apiKey: string, baseUrl: string): Promise; }; - gateway: { - createCiGatewayAuth(token: string, projectId: number, url: string): unknown; - }; hosts: { HostResolution: { fromAccessToken( @@ -81,7 +77,6 @@ export type E2eCredentials = { host: unknown; projectId: number; }; - inferenceAuth: { resolve: () => Promise }; project: unknown; apiUser: unknown; }; @@ -144,10 +139,6 @@ export async function importSharedModules(): Promise { "fetchProjectData", "fetchUserData", ]), - gateway: await importWizard( - "@shared/ci-gateway-auth", - ["createCiGatewayAuth"], - ), hosts: await importWizard( "@shared/host-resolution", ["HostResolution"], @@ -192,11 +183,8 @@ export function readE2eEnv( const projectId = Number(env.PROJECT_ID); if (!Number.isInteger(projectId) || projectId <= 0) missing.push("PROJECT_ID: a positive project id"); - const gatewayTokenFile = env.WIZARD_CI_GATEWAY_TOKEN_FILE?.trim() ?? ""; - if (!gatewayTokenFile) - missing.push("WIZARD_CI_GATEWAY_TOKEN_FILE: an already-issued gateway token"); if (missing.length > 0) throw new Error(`Missing e2e inputs:\n- ${missing.join("\n- ")}`); - return { appDir, apiKey, projectId, gatewayTokenFile }; + return { appDir, apiKey, projectId }; } /** @@ -207,19 +195,12 @@ export function readE2eEnv( export async function resolveE2eCredentials( e2e: E2eEnv, shared: SharedModules, - env: NodeJS.ProcessEnv = process.env, ): Promise { const host = await shared.hosts.HostResolution.fromAccessToken(e2e.apiKey, { region: "us", }); const project = await shared.api.fetchProjectData(e2e.apiKey, e2e.projectId, host.appHost); const apiUser = await shared.api.fetchUserData(e2e.apiKey, host.appHost).catch(() => null); - const token = readFileSync(e2e.gatewayTokenFile, "utf8"); - const gateway = shared.gateway.createCiGatewayAuth( - token, - e2e.projectId, - env.WIZARD_CI_GATEWAY_URL || "https://ai-gateway.us.posthog.com", - ); return { posthog: { accessToken: e2e.apiKey, @@ -227,7 +208,6 @@ export async function resolveE2eCredentials( host, projectId: e2e.projectId, }, - inferenceAuth: { resolve: () => Promise.resolve(gateway) }, project, apiUser, }; diff --git a/services/wizard-program/run-agent.ts b/services/wizard-program/run-agent.ts index cb6225856..9552528f3 100644 --- a/services/wizard-program/run-agent.ts +++ b/services/wizard-program/run-agent.ts @@ -6,8 +6,7 @@ * context-mill are involved. * * WIZARD_REPO= PROJECT_ID=… POSTHOG_KEY_FILE=… \ - * WIZARD_CI_GATEWAY_TOKEN_FILE=… [E2E_RESULT_JSON=result.json] \ - * pnpm wizard-agent + * [E2E_RESULT_JSON=result.json] pnpm wizard-agent * * WIZARD_REPO= pnpm wizard-agent --check */ @@ -37,10 +36,15 @@ type Agent = { options: { onProgress: (event: AgentProgress) => void }, ): Promise<{ outcome: string; failure?: { message: string } }>; RunOutcome: { Success: string }; - DEFAULT_AGENT_BINDING: unknown; +}; +type Constants = { + Sequence: { linear: string }; + Harness: { anthropic: string }; + HAIKU_MODEL: string; }; type Fflate = { zipSync(files: Record): Uint8Array }; +const PROGRAM_ID = "e2e-agent"; const SKILL_ID = "quack"; const QUACK_FILE = join("quack", "quack.txt"); const SKILL_MD = `--- @@ -84,10 +88,11 @@ async function serveQuackSkill(fflate: Fflate): Promise<{ url: string; close: () } async function main(): Promise { - const { runAgent, RunOutcome, DEFAULT_AGENT_BINDING } = await importWizard("@agent", [ - "runAgent", - "RunOutcome", - "DEFAULT_AGENT_BINDING", + const { runAgent, RunOutcome } = await importWizard("@agent", ["runAgent", "RunOutcome"]); + const { Sequence, Harness, HAIKU_MODEL } = await importWizard("@shared/constants", [ + "Sequence", + "Harness", + "HAIKU_MODEL", ]); const shared = await importSharedModules(); const fflate = requireWizardDependency("fflate"); @@ -99,7 +104,7 @@ async function main(): Promise { const skills = await serveQuackSkill(fflate); const config = { - programId: "e2e-agent", + programId: PROGRAM_ID, run: { integrationLabel: SKILL_ID, skillId: SKILL_ID, @@ -110,7 +115,11 @@ async function main(): Promise { docsUrl: "https://posthog.com/docs", }, composed: false, - binding: DEFAULT_AGENT_BINDING, + // Bound the way agentic detection binds its direct runAgent call: linear + // Haiku on the Anthropic harness. + binding: { sequence: Sequence.linear, harness: Harness.anthropic, model: HAIKU_MODEL }, + // Only the orchestrator reads it; this run is linear. + switchboard: { program: PROGRAM_ID, composed: false, flags: {}, flagPayloads: {} }, skillsBaseUrl: skills.url, wizardFlags: {}, wizardFlagPayloads: {}, @@ -123,7 +132,6 @@ async function main(): Promise { { installDir: workDir, credentials: credentials.posthog, - inferenceAuth: credentials.inferenceAuth, project: credentials.project, apiUser: credentials.apiUser, skillId: SKILL_ID, diff --git a/services/wizard-program/run-program.ts b/services/wizard-program/run-program.ts index a7a541822..84c08e4cb 100644 --- a/services/wizard-program/run-program.ts +++ b/services/wizard-program/run-program.ts @@ -5,9 +5,8 @@ * the wizard's legacy adapter does, and supplies the credentials. * * WIZARD_REPO= APP_DIR= PROJECT_ID=… \ - * POSTHOG_KEY_FILE=… WIZARD_CI_GATEWAY_TOKEN_FILE=… \ - * [PROGRAM=posthog-integration] [E2E_RESULT_JSON=result.json] \ - * pnpm wizard-program + * POSTHOG_KEY_FILE=… [PROGRAM=posthog-integration] \ + * [E2E_RESULT_JSON=result.json] pnpm wizard-program * * WIZARD_REPO= pnpm wizard-program --check */ @@ -43,18 +42,22 @@ type Session = { frameworkConfig: FrameworkConfig | null; }; +/** The UI effects a program's `run(session, host)` may call. */ +type ProgramRunHost = { + getFrameworkContext(key: string): unknown; + setFrameworkContext(key: string, value: unknown): void; + warn(message: string): void; +}; + /** The slice of `ProgramConfig` a run is built from. */ type ProgramConfig = { steps: unknown[]; - run?: object | ((session: Session) => Promise); + run?: object | ((session: Session, host: ProgramRunHost) => Promise); requiresAi?: boolean; agentFlow?: string; allowedTools?: readonly string[]; disallowedTools?: readonly string[]; excludedTaskTypes?: unknown; - auditLedgerFile?: string; - auditSeedChecks?: readonly unknown[]; - eventPlanFile?: string; }; type Programs = { @@ -71,7 +74,14 @@ type ProgramSteps = { type Sessions = { buildSession(args: { installDir: string; ci: boolean }): Session; }; -type Ui = { setUI(ui: unknown): void }; +type Ui = { + setUI(ui: unknown): void; + getUI(): { + getFrameworkContext(key: string): unknown; + setFrameworkContext(key: string, value: unknown): void; + log: { warn(message: string): void }; + }; +}; type HeadlessUi = { HeadlessUI: new (store: unknown) => unknown }; type Store = { WizardStore: new (programId: string) => { session: Session } }; type Registry = { FRAMEWORK_REGISTRY: Record }; @@ -91,7 +101,7 @@ async function main(): Promise { const { buildSession } = await importWizard("@lib/wizard-session", [ "buildSession", ]); - const { setUI } = await importWizard("@ui", ["setUI"]); + const { setUI, getUI } = await importWizard("@ui", ["setUI", "getUI"]); const { HeadlessUI } = await importWizard("@ui/headless-ui", [ "HeadlessUI", ]); @@ -134,9 +144,15 @@ async function main(): Promise { } const credentials = await resolveE2eCredentials(e2e, shared); + // The legacy adapter's run host: each effect reaches `getUI()` at call time. + const host: ProgramRunHost = { + getFrameworkContext: (key) => getUI().getFrameworkContext(key), + setFrameworkContext: (key, value) => getUI().setFrameworkContext(key, value), + warn: (message) => getUI().log.warn(message), + }; const run = typeof programConfig.run === "function" - ? await programConfig.run(session) + ? await programConfig.run(session, host) : programConfig.run; // No hooks or seed tasks: postRun uploads env vars to a hosting provider, @@ -152,9 +168,6 @@ async function main(): Promise { allowedTools: programConfig.allowedTools, disallowedTools: programConfig.disallowedTools, excludedTaskTypes: programConfig.excludedTaskTypes, - auditLedgerFile: programConfig.auditLedgerFile, - auditSeedChecks: programConfig.auditSeedChecks, - eventPlanFile: programConfig.eventPlanFile, postAuthGates: postAuthGateSteps(programConfig.steps).map( (step) => step.id, ), From 698bc06c7b9829ebff0285846f61edf8247cb86e Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 21:08:24 -0400 Subject: [PATCH 5/5] fix(wizard-program): mint the agent route's token under a real program id The agent route minted its gateway token under e2e-agent, an id the gateway may refuse. Agentic detection passes its caller's real program id, so the route now defaults to posthog-integration. PROGRAM overrides it, as it does for the program route. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- README.md | 3 ++- services/wizard-program/run-agent.ts | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 24d2096dc..d4527b568 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,8 @@ WIZARD_REPO=~/development/wizard pnpm wizard-agent --check WIZARD_REPO=… APP_DIR=/tmp/app-copy PROJECT_ID=… POSTHOG_KEY_FILE=… \ pnpm wizard-program -# One agent run on a local `quack` skill, in its own empty directory. +# One agent run on a local `quack` skill, in its own empty directory. PROGRAM +# sets the program id the gateway token is minted under, default posthog-integration. WIZARD_REPO=… PROJECT_ID=… POSTHOG_KEY_FILE=… pnpm wizard-agent ``` diff --git a/services/wizard-program/run-agent.ts b/services/wizard-program/run-agent.ts index 9552528f3..2d8506e85 100644 --- a/services/wizard-program/run-agent.ts +++ b/services/wizard-program/run-agent.ts @@ -6,7 +6,7 @@ * context-mill are involved. * * WIZARD_REPO= PROJECT_ID=… POSTHOG_KEY_FILE=… \ - * [E2E_RESULT_JSON=result.json] pnpm wizard-agent + * [PROGRAM=posthog-integration] [E2E_RESULT_JSON=result.json] pnpm wizard-agent * * WIZARD_REPO= pnpm wizard-agent --check */ @@ -44,7 +44,10 @@ type Constants = { }; type Fflate = { zipSync(files: Record): Uint8Array }; -const PROGRAM_ID = "e2e-agent"; +// The runner mints the gateway token under this id, and the gateway may refuse +// an id it does not know. Agentic detection passes its caller's real program +// id, so this defaults to a real one too. +const PROGRAM_ID = process.env.PROGRAM || "posthog-integration"; const SKILL_ID = "quack"; const QUACK_FILE = join("quack", "quack.txt"); const SKILL_MD = `---