From 7f8f045b3c9c68637e29814ad400f1a4a22b7144 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 10:48:52 -0400 Subject: [PATCH 01/10] refactor(shared): move cleanup, scan consent, run tags and ask policy into shared Four stateless leaves move into src/shared with no behavior change: the cleanup registry out of wizard-abort, the scan-consent helpers and DiscoveredFeature out of the session, buildRunTags out of the agent interface, and the ask policy and longer ask timeout out of the runner bootstrap and the ask bridge. Each old module re-exports what it gave up, so shouldDisableAsk and every other existing import keeps working. The run-tags test moves with its function. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- src/agent/agent-interface.ts | 25 +-------------- src/agent/runner/shared/bootstrap.ts | 24 ++------------ src/agent/wizard-ask-bridge.ts | 8 +---- src/lib/wizard-session.ts | 30 +++++------------- .../__tests__/run-tags.test.ts | 2 +- src/shared/ask-policy.ts | 31 +++++++++++++++++++ src/shared/run-tags.ts | 26 ++++++++++++++++ src/shared/scan-consent.ts | 26 ++++++++++++++++ src/shared/utils/cleanup-registry.ts | 22 +++++++++++++ src/shared/utils/wizard-abort.ts | 25 ++------------- 10 files changed, 120 insertions(+), 99 deletions(-) rename src/{agent => shared}/__tests__/run-tags.test.ts (97%) create mode 100644 src/shared/ask-policy.ts create mode 100644 src/shared/run-tags.ts create mode 100644 src/shared/scan-consent.ts create mode 100644 src/shared/utils/cleanup-registry.ts diff --git a/src/agent/agent-interface.ts b/src/agent/agent-interface.ts index 005315236..f19cd85f2 100644 --- a/src/agent/agent-interface.ts +++ b/src/agent/agent-interface.ts @@ -365,30 +365,7 @@ type AgentRunConfig = { const NO_PROGRESS: ProgressEmitter = () => undefined; -/** - * Global identifiers attached to every LLM gateway trace for a run. They ride on - * each `$ai_generation` the gateway emits (in the `X-PostHog-Properties` blob - * `buildAgentEnv` builds), so traces are filterable by program, framework, run, - * and build type for cost attribution and dashboards. `skill_id` is omitted when - * the run has none. - */ -export function buildRunTags(args: { - programId: string; - integration: string; - runId: string; - build: string; - skillId?: string; -}): Record { - return { - program_id: args.programId, - integration: args.integration, - run_id: args.runId, - build: args.build, - // Triage and detection spread these tags and override this one. - call_type: CallType.agent, - ...(args.skillId ? { skill_id: args.skillId } : {}), - }; -} +export { buildRunTags } from '@shared/run-tags'; /** * Whether Warlock/YARA scanning is disabled for this run. Off by default: diff --git a/src/agent/runner/shared/bootstrap.ts b/src/agent/runner/shared/bootstrap.ts index 1ed76bae0..4bb164dda 100644 --- a/src/agent/runner/shared/bootstrap.ts +++ b/src/agent/runner/shared/bootstrap.ts @@ -15,31 +15,11 @@ import { CallType, IS_DEV } from '@shared/constants'; import { VERSION } from '@shared/version'; import { mcpUrlFor } from '@shared/host-resolution'; import type { WizardRunOptions } from '@utils/types'; -import type { BootstrapResult, RunConfig, RunFlags, RunInput } from './types'; +import type { BootstrapResult, RunConfig, RunInput } from './types'; // ── Helpers ────────────────────────────────────────────────────────── -/** - * Decide whether the `wizard_ask` overlay should be wired for this run. - * Disabled in non-interactive modes (CI, signup) — there's no human to - * answer. Per-program disabling is done by adding WIZARD_ASK_TOOL_NAME to - * the program's `disallowedTools` so the SDK rejects calls outright. - * Extracted so the policy can be unit-tested directly. - * - * `e2eAsk` is the one escape hatch. The e2e harness runs a `ci` - * session, but it does have an answerer — the driver loop answers each - * `wizard_ask` batch from the program's e2e profile. Without the flag the - * agent-in-the-loop layer (the ask bridge in both sequence arms, and the - * orchestrator's seeded warehouse task) stays unreachable from a test. - * - * Only the e2e TUI host sets the flag, from the `E2E_ASK` env var. No CLI flag - * populates it, so plain `--ci` and `--signup` runs behave exactly as before. - */ -export function shouldDisableAsk( - flags: Pick, -): boolean { - return (flags.ci || flags.signup) && !flags.e2eAsk; -} +export { isAskDisabled as shouldDisableAsk } from '@shared/ask-policy'; /** The option bag the agent interface and the middleware read. */ export function runOptions(input: RunInput): WizardRunOptions { diff --git a/src/agent/wizard-ask-bridge.ts b/src/agent/wizard-ask-bridge.ts index 5c57a1b79..e03b5c91b 100644 --- a/src/agent/wizard-ask-bridge.ts +++ b/src/agent/wizard-ask-bridge.ts @@ -92,13 +92,7 @@ export const CANCELLED_SENTINEL = '__cancelled__'; /** Default per-question timeout (5 minutes). */ export const DEFAULT_ASK_TIMEOUT_MS = 5 * 60 * 1000; -/** - * The longer per-question timeout, for asks that send the user on an errand — - * open a database console, mint a restricted API key. The default above is - * sized for a question answerable from memory and expires long before an - * errand is done. - */ -export const LONGER_ASK_TIMEOUT_MS = 20 * 60 * 1000; +export { LONGER_ASK_TIMEOUT_MS } from '@shared/ask-policy'; function buildCancelledAnswers(questions: AskQuestion[]): AskAnswers { const out: AskAnswers = {}; diff --git a/src/lib/wizard-session.ts b/src/lib/wizard-session.ts index 4739c3aa6..2d9a0dca2 100644 --- a/src/lib/wizard-session.ts +++ b/src/lib/wizard-session.ts @@ -11,6 +11,7 @@ */ import { POSTHOG_LOCAL_URL, resolveLocalDev } from '@shared/local-dev'; +import { DiscoveredFeature } from '@shared/scan-consent'; import { AdditionalFeature, ADDITIONAL_FEATURE_LABELS, @@ -67,11 +68,7 @@ export enum RunPhase { Error = 'error', } -/** Features discovered by the feature-discovery subagent */ -export enum DiscoveredFeature { - Stripe = 'stripe', - LLM = 'llm', -} +export { DiscoveredFeature }; /** Consent to report what local detection found (see `scanConsent` below). */ export enum ScanConsent { @@ -458,21 +455,8 @@ export function buildSession(args: { }; } -/** One place to ask, so a new consent state does not need three edits. */ -export function mayReportScanResults(session: WizardSession): boolean { - return session.scanConsent === ScanConsent.Granted; -} - -/** Lives here so analytics infrastructure never learns what consent means. */ -export function reportableDiscoveredFeatures( - session: WizardSession, -): DiscoveredFeature[] | undefined { - return mayReportScanResults(session) ? session.discoveredFeatures : undefined; -} - -/** Also a scan result, so it travels under the same consent as the rest. */ -export function reportablePosthogSdkDetected( - session: WizardSession, -): boolean | undefined { - return mayReportScanResults(session) ? session.posthogSdkDetected : undefined; -} +export { + mayReportScanResults, + reportableDiscoveredFeatures, + reportablePosthogSdkDetected, +} from '@shared/scan-consent'; diff --git a/src/agent/__tests__/run-tags.test.ts b/src/shared/__tests__/run-tags.test.ts similarity index 97% rename from src/agent/__tests__/run-tags.test.ts rename to src/shared/__tests__/run-tags.test.ts index 8211b96e8..9a01156bb 100644 --- a/src/agent/__tests__/run-tags.test.ts +++ b/src/shared/__tests__/run-tags.test.ts @@ -1,4 +1,4 @@ -import { buildRunTags } from '@agent/agent-interface'; +import { buildRunTags } from '@shared/run-tags'; import { CallType } from '@shared/constants'; describe('buildRunTags', () => { diff --git a/src/shared/ask-policy.ts b/src/shared/ask-policy.ts new file mode 100644 index 000000000..5266e499c --- /dev/null +++ b/src/shared/ask-policy.ts @@ -0,0 +1,31 @@ +/** When a run may put a `wizard_ask` question to a person, and how long it waits. */ + +/** + * Whether the `wizard_ask` overlay stays unwired for this run. Non-interactive + * modes (CI, signup) have no human to answer. Per-program disabling adds + * WIZARD_ASK_TOOL_NAME to the program's `disallowedTools` instead, so the SDK + * rejects calls outright. + * + * `e2eAsk` is the one escape hatch. The e2e harness runs a `ci` session, but it + * does have an answerer: the driver loop answers each `wizard_ask` batch from + * the program's e2e profile. Without the flag the agent-in-the-loop layer (the + * ask bridge in both sequence arms, and the orchestrator's seeded warehouse + * task) stays unreachable from a test. + * + * Only the e2e TUI host sets the flag, from the `E2E_ASK` env var. No CLI flag + * populates it, so plain `--ci` and `--signup` runs keep the overlay unwired. + */ +export function isAskDisabled(flags: { + ci: boolean; + signup: boolean; + e2eAsk: boolean; +}): boolean { + return (flags.ci || flags.signup) && !flags.e2eAsk; +} + +/** + * The longer per-question timeout, for asks that send the user on an errand — + * open a database console, mint a restricted API key. The default is sized for + * a question answerable from memory and expires long before an errand is done. + */ +export const LONGER_ASK_TIMEOUT_MS = 20 * 60 * 1000; diff --git a/src/shared/run-tags.ts b/src/shared/run-tags.ts new file mode 100644 index 000000000..144d4b638 --- /dev/null +++ b/src/shared/run-tags.ts @@ -0,0 +1,26 @@ +import { CallType } from '@shared/constants'; + +/** + * Global identifiers attached to every LLM gateway trace for a run. They ride on + * each `$ai_generation` the gateway emits (in the `X-PostHog-Properties` blob + * `buildAgentEnv` builds), so traces are filterable by program, framework, run, + * and build type for cost attribution and dashboards. `skill_id` is omitted when + * the run has none. + */ +export function buildRunTags(args: { + programId: string; + integration: string; + runId: string; + build: string; + skillId?: string; +}): Record { + return { + program_id: args.programId, + integration: args.integration, + run_id: args.runId, + build: args.build, + // Triage and detection spread these tags and override this one. + call_type: CallType.agent, + ...(args.skillId ? { skill_id: args.skillId } : {}), + }; +} diff --git a/src/shared/scan-consent.ts b/src/shared/scan-consent.ts new file mode 100644 index 000000000..9188cac6c --- /dev/null +++ b/src/shared/scan-consent.ts @@ -0,0 +1,26 @@ +/** Features discovered by the feature-discovery subagent */ +export enum DiscoveredFeature { + Stripe = 'stripe', + LLM = 'llm', +} + +type ScanConsentState = { scanConsent: string }; + +/** One place to ask, so a new consent state does not need three edits. */ +export function mayReportScanResults(session: ScanConsentState): boolean { + return session.scanConsent === 'granted'; +} + +/** Lives here so analytics infrastructure never learns what consent means. */ +export function reportableDiscoveredFeatures( + session: ScanConsentState & { discoveredFeatures: DiscoveredFeature[] }, +): DiscoveredFeature[] | undefined { + return mayReportScanResults(session) ? session.discoveredFeatures : undefined; +} + +/** Also a scan result, so it travels under the same consent as the rest. */ +export function reportablePosthogSdkDetected( + session: ScanConsentState & { posthogSdkDetected: boolean }, +): boolean | undefined { + return mayReportScanResults(session) ? session.posthogSdkDetected : undefined; +} diff --git a/src/shared/utils/cleanup-registry.ts b/src/shared/utils/cleanup-registry.ts new file mode 100644 index 000000000..071955574 --- /dev/null +++ b/src/shared/utils/cleanup-registry.ts @@ -0,0 +1,22 @@ +/** Process-local cleanup callbacks shared by the CLI and agent. */ +const cleanupFns: Array<() => void> = []; + +export function registerCleanup(fn: () => void): void { + cleanupFns.push(fn); +} + +export function clearCleanup(): void { + cleanupFns.length = 0; +} + +/** Runs all registered cleanup functions and drains the array. */ +export function runCleanups(): void { + const fns = cleanupFns.splice(0); + for (const fn of fns) { + try { + fn(); + } catch { + /* cleanup should not prevent exit */ + } + } +} diff --git a/src/shared/utils/wizard-abort.ts b/src/shared/utils/wizard-abort.ts index 0a2b8ae30..ec5275ac8 100644 --- a/src/shared/utils/wizard-abort.ts +++ b/src/shared/utils/wizard-abort.ts @@ -17,6 +17,9 @@ import { emitWizardError, sanitizeErrorDetail, } from '@shared/errors'; +import { runCleanups } from './cleanup-registry'; + +export { registerCleanup, clearCleanup, runCleanups } from './cleanup-registry'; // Still importable from here; the class lives with the error codes. export { WizardError }; @@ -33,28 +36,6 @@ interface WizardAbortOptions { status?: 'error' | 'cancelled'; } -const cleanupFns: Array<() => void> = []; - -export function registerCleanup(fn: () => void): void { - cleanupFns.push(fn); -} - -export function clearCleanup(): void { - cleanupFns.length = 0; -} - -/** Runs all registered cleanup functions and drains the array. */ -export function runCleanups(): void { - const fns = cleanupFns.splice(0); - for (const fn of fns) { - try { - fn(); - } catch { - /* cleanup should not prevent exit */ - } - } -} - function resolveErrorCode( options: WizardAbortOptions, error: Error | WizardError | undefined, From 57f1a82f9448bfc37119832e29c15c46165c9d71 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 10:50:41 -0400 Subject: [PATCH 02/10] refactor(shared): move the OAuth token helpers into oauth-token OAuthTokenResponseSchema, getOAuthClientId and refreshAccessToken need no UI, so they move from utils/oauth.ts into utils/oauth-token.ts with no behavior change. oauth.ts imports what its login flow uses and re-exports the schema, its type and refreshAccessToken for existing callers. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- src/shared/utils/oauth-token.ts | 84 +++++++++++++++++++++++++++++++ src/shared/utils/oauth.ts | 87 +++++---------------------------- 2 files changed, 95 insertions(+), 76 deletions(-) create mode 100644 src/shared/utils/oauth-token.ts diff --git a/src/shared/utils/oauth-token.ts b/src/shared/utils/oauth-token.ts new file mode 100644 index 000000000..92443d5c5 --- /dev/null +++ b/src/shared/utils/oauth-token.ts @@ -0,0 +1,84 @@ +/** OAuth token-endpoint helpers that need no UI: the token response and the refresh grant. */ +import axios from 'axios'; +import { z } from 'zod'; +import { + POSTHOG_DEV_CLIENT_ID, + POSTHOG_PROXY_CLIENT_ID, + WIZARD_USER_AGENT, +} from '@shared/constants'; +import { logToFile } from './debug'; +import { getOAuthUrl, resolveBaseUrl } from './urls'; +import { oauthErrorFromTokenBody } from './oauth-errors'; + +export const OAuthTokenResponseSchema = z.object({ + access_token: z.string(), + expires_in: z.number(), + token_type: z.string(), + scope: z.string(), + refresh_token: z.string().optional(), + scoped_teams: z.array(z.number()).optional(), + scoped_organizations: z.array(z.string()).optional(), + // Sent by PostHog Cloud (and passed through the oauth.posthog.com proxy); absent on + // self-hosted. `.catch(undefined)` so an unrecognized value degrades to the probe + // fallback instead of failing the whole login. + posthog_region: z.enum(['us', 'eu']).optional().catch(undefined), + posthog_base_url: z.string().optional().catch(undefined), +}); + +export type OAuthTokenResponse = z.infer; + +/** + * OAuth client ID for the current target. A pinned base URL (`--base-url`, or + * IS_DEV's implicit localhost) means we're talking to a dev-seeded stack, which + * registers the dev client; prod uses the proxy client. + * + * TODO: this assumes any pinned base URL is a dev-seeded instance that + * registers POSTHOG_DEV_CLIENT_ID. If we ever point `--base-url` at a non-dev + * instance with its own OAuth app, make the client ID configurable (e.g. a + * `--oauth-client-id` flag) instead of always falling back to the dev client. + */ +export function getOAuthClientId(baseUrl?: string): string { + return resolveBaseUrl(baseUrl) + ? POSTHOG_DEV_CLIENT_ID + : POSTHOG_PROXY_CLIENT_ID; +} + +// Refresh-token grant (RFC 6749 §6); the server rotates, so callers must store the returned refresh_token. +export async function refreshAccessToken( + refreshToken: string, + baseUrl?: string, + clientId?: string, +): Promise { + const oauthUrl = getOAuthUrl(baseUrl); + logToFile(`[oauth] refreshing access token at ${oauthUrl}/oauth/token`); + try { + const response = await axios.post( + `${oauthUrl}/oauth/token`, + { + grant_type: 'refresh_token', + refresh_token: refreshToken, + // The grant only refreshes under its minting app — provisioning signups pass their regional client. + client_id: clientId ?? getOAuthClientId(baseUrl), + }, + { + headers: { + 'Content-Type': 'application/json', + 'User-Agent': WIZARD_USER_AGENT, + }, + timeout: 30_000, + }, + ); + const token = OAuthTokenResponseSchema.parse(response.data); + logToFile('[oauth] access token refreshed'); + return token; + } catch (e) { + logToFile( + '[oauth] token refresh failed:', + e instanceof Error ? e.message : e, + ); + const refreshError = axios.isAxiosError(e) + ? oauthErrorFromTokenBody(e.response?.data) + : null; + throw refreshError ?? e; + } +} diff --git a/src/shared/utils/oauth.ts b/src/shared/utils/oauth.ts index dda07db81..5ac3f5934 100644 --- a/src/shared/utils/oauth.ts +++ b/src/shared/utils/oauth.ts @@ -3,13 +3,10 @@ import * as http from 'node:http'; import { execSync } from 'node:child_process'; import axios from 'axios'; import { logToFile } from './debug'; -import { z } from 'zod'; import { getUI } from '@ui'; import { OAUTH_PORTS, OAUTH_TIMEOUT_MS, - POSTHOG_DEV_CLIENT_ID, - POSTHOG_PROXY_CLIENT_ID, WIZARD_USER_AGENT, } from '@shared/constants'; import { getOAuthUrl, resolveBaseUrl } from './urls'; @@ -23,6 +20,17 @@ import { oauthErrorFromCallbackParams, oauthErrorFromTokenBody, } from './oauth-errors'; +import { + getOAuthClientId, + OAuthTokenResponseSchema, + type OAuthTokenResponse, +} from './oauth-token'; + +export { + OAuthTokenResponseSchema, + refreshAccessToken, + type OAuthTokenResponse, +} from './oauth-token'; const OAUTH_CALLBACK_STYLES = ` `; -export const OAuthTokenResponseSchema = z.object({ - access_token: z.string(), - expires_in: z.number(), - token_type: z.string(), - scope: z.string(), - refresh_token: z.string().optional(), - scoped_teams: z.array(z.number()).optional(), - scoped_organizations: z.array(z.string()).optional(), - // Sent by PostHog Cloud (and passed through the oauth.posthog.com proxy); absent on - // self-hosted. `.catch(undefined)` so an unrecognized value degrades to the probe - // fallback instead of failing the whole login. - posthog_region: z.enum(['us', 'eu']).optional().catch(undefined), - posthog_base_url: z.string().optional().catch(undefined), -}); - -export type OAuthTokenResponse = z.infer; - export const WIZARD_COMPLETION_SCOPE = 'event_definition:write'; export function parseOAuthScopes(scope: string): string[] { @@ -117,22 +108,6 @@ interface OAuthConfig { baseUrl?: string; } -/** - * OAuth client ID for the current target. A pinned base URL (`--base-url`, or - * IS_DEV's implicit localhost) means we're talking to a dev-seeded stack, which - * registers the dev client; prod uses the proxy client. - * - * TODO: this assumes any pinned base URL is a dev-seeded instance that - * registers POSTHOG_DEV_CLIENT_ID. If we ever point `--base-url` at a non-dev - * instance with its own OAuth app, make the client ID configurable (e.g. a - * `--oauth-client-id` flag) instead of always falling back to the dev client. - */ -function getOAuthClientId(baseUrl?: string): string { - return resolveBaseUrl(baseUrl) - ? POSTHOG_DEV_CLIENT_ID - : POSTHOG_PROXY_CLIENT_ID; -} - function getLocalOAuthOrigin(port: number): string { return `http://localhost:${port}`; } @@ -467,46 +442,6 @@ async function exchangeCodeForToken( return token; } -// Refresh-token grant (RFC 6749 §6); the server rotates, so callers must store the returned refresh_token. -export async function refreshAccessToken( - refreshToken: string, - baseUrl?: string, - clientId?: string, -): Promise { - const oauthUrl = getOAuthUrl(baseUrl); - logToFile(`[oauth] refreshing access token at ${oauthUrl}/oauth/token`); - try { - const response = await axios.post( - `${oauthUrl}/oauth/token`, - { - grant_type: 'refresh_token', - refresh_token: refreshToken, - // The grant only refreshes under its minting app — provisioning signups pass their regional client. - client_id: clientId ?? getOAuthClientId(baseUrl), - }, - { - headers: { - 'Content-Type': 'application/json', - 'User-Agent': WIZARD_USER_AGENT, - }, - timeout: 30_000, - }, - ); - const token = OAuthTokenResponseSchema.parse(response.data); - logToFile('[oauth] access token refreshed'); - return token; - } catch (e) { - logToFile( - '[oauth] token refresh failed:', - e instanceof Error ? e.message : e, - ); - const refreshError = axios.isAxiosError(e) - ? oauthErrorFromTokenBody(e.response?.data) - : null; - throw refreshError ?? e; - } -} - /** * Warn — at login, while the user is still watching — when the grant came back * narrower than the request, and record the gap so narrowed runs are countable. From acecf6b74a3bd7ec0f4b9b165be895182cc43a14 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 10:52:26 -0400 Subject: [PATCH 03/10] refactor(shared): move the gateway bearer type and checks into gateway-auth GatewayAuth, isPastRefresh, isTrustedGatewayUrl and buildWizardPropertiesBlob hold no mint state, so they move verbatim from agent/gateway-session.ts into shared/gateway-auth.ts. The mint, its cache and the CI credential loading stay where they are. gateway-session.ts imports what the mint uses and re-exports all four, so no caller changes. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- src/agent/gateway-session.ts | 104 +++-------------------------------- src/shared/gateway-auth.ts | 99 +++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 97 deletions(-) create mode 100644 src/shared/gateway-auth.ts diff --git a/src/agent/gateway-session.ts b/src/agent/gateway-session.ts index e8a1ef1b2..c1bb23d59 100644 --- a/src/agent/gateway-session.ts +++ b/src/agent/gateway-session.ts @@ -15,21 +15,14 @@ import { checkLlmGatewayHealth } from '@shared/health-checks/endpoints'; import { ServiceHealthStatus } from '@shared/health-checks/types'; import { IS_PRODUCTION_BUILD, runtimeEnv } from '@env'; import type { CloudRegion } from '@utils/types'; +import { isTrustedGatewayUrl, type GatewayAuth } from '@shared/gateway-auth'; -export interface GatewayAuth { - /** Base URL for model calls (no `/v1`; transports append their route). */ - gatewayUrl: string; - /** Gateway bearer, minted normally or supplied directly by CI. */ - token: string; - /** Team verified by the mint, or explicitly supplied for CI attribution. */ - teamId?: number; - /** - * Instant past which a 401 on this bearer is age rather than a bad - * credential: the cache re-mints past it, and a session still holding the - * old bearer may re-mint once. Before it the mint has to be trusted. - */ - refreshAtMs: number; -} +export { + buildWizardPropertiesBlob, + isPastRefresh, + isTrustedGatewayUrl, + type GatewayAuth, +} from '@shared/gateway-auth'; interface CachedAuth { key: string; @@ -191,54 +184,6 @@ export function resetGatewaySession(): void { ciAuth = null; } -/** Whether a 401 on this bearer may be age (past its refresh instant) rather than a bad credential. */ -export function isPastRefresh(auth: GatewayAuth, now = Date.now()): boolean { - return now >= auth.refreshAtMs; -} - -/** - * Whether a server-supplied origin may receive a bearer and prompt content: - * https (loopback excepted), and either a current cloud gateway or the host the run - * authenticated against. - */ -export function isTrustedGatewayUrl(value: string, apiHost: string): boolean { - let url: URL; - try { - url = new URL(value); - } catch { - return false; - } - // Consumers append routes to this value, so anything beyond an origin - // (path, query, fragment, userinfo) would build a malformed endpoint. - if ( - url.pathname !== '/' || - url.search || - url.hash || - url.username || - url.password - ) { - return false; - } - const localhost = - url.hostname === 'localhost' || - url.hostname === '127.0.0.1' || - url.hostname === 'host.docker.internal'; - // Loopback is the dev gateway, and is the one case allowed over http. - if (localhost) return true; - if (url.protocol !== 'https:') return false; - if (url.hostname.endsWith('.posthog.com')) { - return ( - url.origin === 'https://ai-gateway.us.posthog.com' || - url.origin === 'https://ai-gateway.eu.posthog.com' - ); - } - try { - return url.hostname === new URL(apiHost).hostname; - } catch { - return false; - } -} - interface MintedToken { token: string; expiresAt: string; @@ -450,38 +395,3 @@ async function mintGatewayToken( ); } } - -/** - * The v2 run-metadata carrier: one JSON blob for the `X-PostHog-Properties` - * header. Plain keys only, since the gateway strips `$`-prefixed keys as reserved, - * so feature-flag variants land as `wizard_flag_` instead of the legacy - * `$feature/` (dashboards keying on `$feature/wizard-*` read the new key - * post-cutover). - */ -export function buildWizardPropertiesBlob( - wizardMetadata: Record, - wizardFlags: Record, - teamId?: number, -): string { - // The gateway pins `$ai_product` to `wizard:`, and rejects a legacy - // product override on a scoped token, so the unprefixed key every cost and - // error consumer reads is only present if this blob declares it. - const props: Record = { ai_product: 'wizard' }; - if (teamId !== undefined) props.team_id = teamId; - for (const [key, value] of Object.entries(wizardMetadata)) { - props[stripPropertyPrefix(key)] = value; - } - for (const [flagKey, variant] of Object.entries(wizardFlags)) { - if (!flagKey.toLowerCase().startsWith('wizard')) continue; - props[`wizard_flag_${flagKey.toLowerCase()}`] = variant; - } - return JSON.stringify(props); -} - -const LEGACY_PROPERTY_PREFIX = 'X-POSTHOG-PROPERTY-'; - -function stripPropertyPrefix(key: string): string { - return key.toUpperCase().startsWith(LEGACY_PROPERTY_PREFIX) - ? key.slice(LEGACY_PROPERTY_PREFIX.length).toLowerCase() - : key; -} diff --git a/src/shared/gateway-auth.ts b/src/shared/gateway-auth.ts new file mode 100644 index 000000000..f673c5eb2 --- /dev/null +++ b/src/shared/gateway-auth.ts @@ -0,0 +1,99 @@ +/** Gateway bearer types and checks shared by the agent and its hosts. */ + +export interface GatewayAuth { + /** Base URL for model calls (no `/v1`; transports append their route). */ + gatewayUrl: string; + /** Gateway bearer, minted normally or supplied directly by CI. */ + token: string; + /** Team verified by the mint, or explicitly supplied for CI attribution. */ + teamId?: number; + /** + * Instant past which a 401 on this bearer is age rather than a bad + * credential: the cache re-mints past it, and a session still holding the + * old bearer may re-mint once. Before it the mint has to be trusted. + */ + refreshAtMs: number; +} + +/** Whether a 401 on this bearer may be age (past its refresh instant) rather than a bad credential. */ +export function isPastRefresh(auth: GatewayAuth, now = Date.now()): boolean { + return now >= auth.refreshAtMs; +} + +/** + * Whether a server-supplied origin may receive a bearer and prompt content: + * https (loopback excepted), and either a current cloud gateway or the host the run + * authenticated against. + */ +export function isTrustedGatewayUrl(value: string, apiHost: string): boolean { + let url: URL; + try { + url = new URL(value); + } catch { + return false; + } + // Consumers append routes to this value, so anything beyond an origin + // (path, query, fragment, userinfo) would build a malformed endpoint. + if ( + url.pathname !== '/' || + url.search || + url.hash || + url.username || + url.password + ) { + return false; + } + const localhost = + url.hostname === 'localhost' || + url.hostname === '127.0.0.1' || + url.hostname === 'host.docker.internal'; + // Loopback is the dev gateway, and is the one case allowed over http. + if (localhost) return true; + if (url.protocol !== 'https:') return false; + if (url.hostname.endsWith('.posthog.com')) { + return ( + url.origin === 'https://ai-gateway.us.posthog.com' || + url.origin === 'https://ai-gateway.eu.posthog.com' + ); + } + try { + return url.hostname === new URL(apiHost).hostname; + } catch { + return false; + } +} + +/** + * The v2 run-metadata carrier: one JSON blob for the `X-PostHog-Properties` + * header. Plain keys only, since the gateway strips `$`-prefixed keys as reserved, + * so feature-flag variants land as `wizard_flag_` instead of the legacy + * `$feature/` (dashboards keying on `$feature/wizard-*` read the new key + * post-cutover). + */ +export function buildWizardPropertiesBlob( + wizardMetadata: Record, + wizardFlags: Record, + teamId?: number, +): string { + // The gateway pins `$ai_product` to `wizard:`, and rejects a legacy + // product override on a scoped token, so the unprefixed key every cost and + // error consumer reads is only present if this blob declares it. + const props: Record = { ai_product: 'wizard' }; + if (teamId !== undefined) props.team_id = teamId; + for (const [key, value] of Object.entries(wizardMetadata)) { + props[stripPropertyPrefix(key)] = value; + } + for (const [flagKey, variant] of Object.entries(wizardFlags)) { + if (!flagKey.toLowerCase().startsWith('wizard')) continue; + props[`wizard_flag_${flagKey.toLowerCase()}`] = variant; + } + return JSON.stringify(props); +} + +const LEGACY_PROPERTY_PREFIX = 'X-POSTHOG-PROPERTY-'; + +function stripPropertyPrefix(key: string): string { + return key.toUpperCase().startsWith(LEGACY_PROPERTY_PREFIX) + ? key.slice(LEGACY_PROPERTY_PREFIX.length).toLowerCase() + : key; +} From 1eb4fe5836625bf4c9d1b774408495ad09d190b1 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 10:53:54 -0400 Subject: [PATCH 04/10] refactor(shared): move the JSON file watcher into shared src/lib/file-watcher.ts moves unchanged to src/shared/file-watcher.ts. The audit ledger watcher, the event-plan watcher and the TUI hook import it from @shared, which drops two programs-to-legacy rows from the architecture allowlist. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- src/__tests__/architecture/known-violations.json | 2 -- src/programs/audit/ledger-watcher.ts | 2 +- src/programs/task-stream/event-plan-watcher.ts | 2 +- src/{lib => shared}/file-watcher.ts | 0 src/ui/tui/hooks/file-watcher.ts | 12 +++++++++--- 5 files changed, 11 insertions(+), 7 deletions(-) rename src/{lib => shared}/file-watcher.ts (100%) diff --git a/src/__tests__/architecture/known-violations.json b/src/__tests__/architecture/known-violations.json index 0d110b9d3..c91511369 100644 --- a/src/__tests__/architecture/known-violations.json +++ b/src/__tests__/architecture/known-violations.json @@ -42,7 +42,6 @@ "src/programs/ai-opt-in-gate.ts -> src/lib/wizard-session.ts", "src/programs/audit/index.ts -> src/lib/headless-mode.ts", "src/programs/audit/index.ts -> src/lib/wizard-session.ts", - "src/programs/audit/ledger-watcher.ts -> src/lib/file-watcher.ts", "src/programs/audit/ledger-watcher.ts -> src/ui/index.ts", "src/programs/audit/types.ts -> src/lib/wizard-session.ts", "src/programs/authenticate.ts -> src/lib/wizard-session.ts", @@ -111,7 +110,6 @@ "src/programs/shared/posthog-cli-preinstall.ts -> src/steps/install-cli-steering/index.ts", "src/programs/shared/posthog-cli-preinstall.ts -> src/ui/index.ts", "src/programs/task-stream/destinations/posthog.ts -> src/lib/wizard-session.ts", - "src/programs/task-stream/event-plan-watcher.ts -> src/lib/file-watcher.ts", "src/programs/task-stream/event-plan-watcher.ts -> src/ui/tui/store.ts", "src/programs/task-stream/task-stream-push.ts -> src/lib/wizard-session.ts", "src/programs/task-stream/task-stream-push.ts -> src/ui/tui/store.ts", diff --git a/src/programs/audit/ledger-watcher.ts b/src/programs/audit/ledger-watcher.ts index f7a312543..cf0c0f441 100644 --- a/src/programs/audit/ledger-watcher.ts +++ b/src/programs/audit/ledger-watcher.ts @@ -10,7 +10,7 @@ import { startFileWatcher, type FileWatcherHandle, type FileWatcherOptions, -} from '@lib/file-watcher'; +} from '@shared/file-watcher'; import { logToFile } from '@utils/debug'; import { AUDIT_CHECKS_KEY, coerceAuditChecks } from './types.js'; diff --git a/src/programs/task-stream/event-plan-watcher.ts b/src/programs/task-stream/event-plan-watcher.ts index d04cee77c..7fadb0118 100644 --- a/src/programs/task-stream/event-plan-watcher.ts +++ b/src/programs/task-stream/event-plan-watcher.ts @@ -3,7 +3,7 @@ import { startFileWatcher, type FileWatcherHandle, type FileWatcherOptions, -} from '@lib/file-watcher'; +} from '@shared/file-watcher'; const MAX_EVENT_PLAN_FILE_BYTES = 256 * 1024; const MAX_EVENT_COUNT = 50; diff --git a/src/lib/file-watcher.ts b/src/shared/file-watcher.ts similarity index 100% rename from src/lib/file-watcher.ts rename to src/shared/file-watcher.ts diff --git a/src/ui/tui/hooks/file-watcher.ts b/src/ui/tui/hooks/file-watcher.ts index 5dd272aa0..974466508 100644 --- a/src/ui/tui/hooks/file-watcher.ts +++ b/src/ui/tui/hooks/file-watcher.ts @@ -1,8 +1,14 @@ import { useEffect } from 'react'; -import { startFileWatcher, type FileWatcherOptions } from '@lib/file-watcher'; +import { + startFileWatcher, + type FileWatcherOptions, +} from '@shared/file-watcher'; -export { startFileWatcher } from '@lib/file-watcher'; -export type { FileWatcherHandle, FileWatcherOptions } from '@lib/file-watcher'; +export { startFileWatcher } from '@shared/file-watcher'; +export type { + FileWatcherHandle, + FileWatcherOptions, +} from '@shared/file-watcher'; /** React hook wrapping `startFileWatcher`. Starts on mount, stops on unmount * or when `path` changes. `onUpdate` and `options` are captured at mount From 0eb5f7381376b58356b990ce763f8bbf0968520e Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 10:55:11 -0400 Subject: [PATCH 05/10] refactor(shared): move the self-driving pricing copy into shared The pricing constants move unchanged from the self-driving deck into shared/self-driving-pricing.ts. The deck's pricing.ts re-exports them, so its importers keep their paths. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- src/shared/self-driving-pricing.ts | 30 +++++++++++++++++++ src/ui/tui/decks/self-driving/pricing.ts | 37 +++++------------------- 2 files changed, 37 insertions(+), 30 deletions(-) create mode 100644 src/shared/self-driving-pricing.ts diff --git a/src/shared/self-driving-pricing.ts b/src/shared/self-driving-pricing.ts new file mode 100644 index 000000000..001c0f20f --- /dev/null +++ b/src/shared/self-driving-pricing.ts @@ -0,0 +1,30 @@ +/** + * Self-driving pricing, in one place. It's stated at six points across the run (intro, tips, + * learn deck, outro), and a price quoted six ways is a price the reader stops trusting. + * + * Careful with the word "free". The tracking this wizard installs bills as ordinary PostHog + * usage, and scouts are heading for usage-based pricing too — so anything called free here is + * a promise with an expiry date. State what the PR costs and leave the rest to usage-based + * billing, which stays true either side of that change. + */ + +/** Flat USD charge per report that ships a pull request. */ +export const PRICE_PER_PR_USD = 15; + +/** One line, for a screen that only has room to say what it costs. */ +export const PRICING_SHORT = `Agents charge a flat $${PRICE_PER_PR_USD} per pull request they ship.`; + +/** The same fact at length, plus the bit people are surprised by later. */ +export const PRICING_LONG = + `A report that ships a pull request costs a flat $${PRICE_PER_PR_USD}. ` + + `Everything else follows your usual PostHog usage-based pricing.`; + +/** + * Nothing caps the spend unless you set it, which is worth saying out loud. Checked against + * billing: `custom_limits_map` starts empty and a product with no custom limit resolves to + * `usage_limit: None`, i.e. uncapped. A free plan is capped by its own allocation instead, + * hence the qualifier. + */ +export const NO_DEFAULT_LIMIT = + 'A paid plan has no monthly cap until you set one. Add a PR limit in your inbox sidebar, ' + + 'and agents pause when they hit it.'; diff --git a/src/ui/tui/decks/self-driving/pricing.ts b/src/ui/tui/decks/self-driving/pricing.ts index 001c0f20f..237c09860 100644 --- a/src/ui/tui/decks/self-driving/pricing.ts +++ b/src/ui/tui/decks/self-driving/pricing.ts @@ -1,30 +1,7 @@ -/** - * Self-driving pricing, in one place. It's stated at six points across the run (intro, tips, - * learn deck, outro), and a price quoted six ways is a price the reader stops trusting. - * - * Careful with the word "free". The tracking this wizard installs bills as ordinary PostHog - * usage, and scouts are heading for usage-based pricing too — so anything called free here is - * a promise with an expiry date. State what the PR costs and leave the rest to usage-based - * billing, which stays true either side of that change. - */ - -/** Flat USD charge per report that ships a pull request. */ -export const PRICE_PER_PR_USD = 15; - -/** One line, for a screen that only has room to say what it costs. */ -export const PRICING_SHORT = `Agents charge a flat $${PRICE_PER_PR_USD} per pull request they ship.`; - -/** The same fact at length, plus the bit people are surprised by later. */ -export const PRICING_LONG = - `A report that ships a pull request costs a flat $${PRICE_PER_PR_USD}. ` + - `Everything else follows your usual PostHog usage-based pricing.`; - -/** - * Nothing caps the spend unless you set it, which is worth saying out loud. Checked against - * billing: `custom_limits_map` starts empty and a product with no custom limit resolves to - * `usage_limit: None`, i.e. uncapped. A free plan is capped by its own allocation instead, - * hence the qualifier. - */ -export const NO_DEFAULT_LIMIT = - 'A paid plan has no monthly cap until you set one. Add a PR limit in your inbox sidebar, ' + - 'and agents pause when they hit it.'; +/** UI compatibility entry for the self-driving pricing copy. */ +export { + PRICE_PER_PR_USD, + PRICING_SHORT, + PRICING_LONG, + NO_DEFAULT_LIMIT, +} from '@shared/self-driving-pricing'; From aed54993cee3b802f31a74bd1fe78f041398c968 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 10:56:37 -0400 Subject: [PATCH 06/10] refactor(agent): move the wizard-tools names into tool-names SERVER_NAME and WIZARD_TOOL_NAMES move unchanged from tools/tools.ts into tools/tool-names.ts, a data-only module. tools.ts re-exports both, so every importer keeps its path. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- src/agent/tools/tool-names.ts | 24 ++++++++++++++++++++++++ src/agent/tools/tools.ts | 23 +---------------------- 2 files changed, 25 insertions(+), 22 deletions(-) create mode 100644 src/agent/tools/tool-names.ts diff --git a/src/agent/tools/tool-names.ts b/src/agent/tools/tool-names.ts new file mode 100644 index 000000000..40853267d --- /dev/null +++ b/src/agent/tools/tool-names.ts @@ -0,0 +1,24 @@ +/** Tool ids programs name in allowedTools and disallowedTools. Data only. */ + +export const SERVER_NAME = 'wizard-tools'; + +/** Tool names exposed by the wizard-tools server, keyed for selective use. */ +// SDK expects MCP tool names in allowedTools/disallowedTools to be the +// fully-qualified `mcp____` form (sdk.d.ts: "Fully-qualified +// MCP tool name, e.g. mcp__server__tool_name."). The colon form silently +// fails to match, which made every program's `disallowedTools` entry a no-op. +export const WIZARD_TOOL_NAMES = { + checkEnvKeys: `mcp__${SERVER_NAME}__check_env_keys`, + setEnvValues: `mcp__${SERVER_NAME}__set_env_values`, + detectPackageManager: `mcp__${SERVER_NAME}__detect_package_manager`, + loadSkillMenu: `mcp__${SERVER_NAME}__load_skill_menu`, + installSkill: `mcp__${SERVER_NAME}__install_skill`, + auditSeedChecks: `mcp__${SERVER_NAME}__audit_seed_checks`, + auditAddChecks: `mcp__${SERVER_NAME}__audit_add_checks`, + auditResolveChecks: `mcp__${SERVER_NAME}__audit_resolve_checks`, + wizardAsk: `mcp__${SERVER_NAME}__wizard_ask`, + publishHandoff: `mcp__${SERVER_NAME}__publish_handoff`, + enqueueTask: `mcp__${SERVER_NAME}__enqueue_task`, + completeTask: `mcp__${SERVER_NAME}__complete_task`, + readHandoffs: `mcp__${SERVER_NAME}__read_handoffs`, +} as const; diff --git a/src/agent/tools/tools.ts b/src/agent/tools/tools.ts index 6751232a7..0ab9a6283 100644 --- a/src/agent/tools/tools.ts +++ b/src/agent/tools/tools.ts @@ -1156,28 +1156,7 @@ export function appendAuditChecksToLedger( return { ok: true, added: additions.length }; } -export const SERVER_NAME = 'wizard-tools'; - -/** Tool names exposed by the wizard-tools server, keyed for selective use. */ -// SDK expects MCP tool names in allowedTools/disallowedTools to be the -// fully-qualified `mcp____` form (sdk.d.ts: "Fully-qualified -// MCP tool name, e.g. mcp__server__tool_name."). The colon form silently -// fails to match, which made every program's `disallowedTools` entry a no-op. -export const WIZARD_TOOL_NAMES = { - checkEnvKeys: `mcp__${SERVER_NAME}__check_env_keys`, - setEnvValues: `mcp__${SERVER_NAME}__set_env_values`, - detectPackageManager: `mcp__${SERVER_NAME}__detect_package_manager`, - loadSkillMenu: `mcp__${SERVER_NAME}__load_skill_menu`, - installSkill: `mcp__${SERVER_NAME}__install_skill`, - auditSeedChecks: `mcp__${SERVER_NAME}__audit_seed_checks`, - auditAddChecks: `mcp__${SERVER_NAME}__audit_add_checks`, - auditResolveChecks: `mcp__${SERVER_NAME}__audit_resolve_checks`, - wizardAsk: `mcp__${SERVER_NAME}__wizard_ask`, - publishHandoff: `mcp__${SERVER_NAME}__publish_handoff`, - enqueueTask: `mcp__${SERVER_NAME}__enqueue_task`, - completeTask: `mcp__${SERVER_NAME}__complete_task`, - readHandoffs: `mcp__${SERVER_NAME}__read_handoffs`, -} as const; +export { SERVER_NAME, WIZARD_TOOL_NAMES } from './tool-names'; // --------------------------------------------------------------------------- // Test-only exports From 32151373834132ec5b6bc973b6d7e311884893a5 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 11:00:54 -0400 Subject: [PATCH 07/10] refactor(programs): move program run data into run and abort-case modules Eight programs keep their run definitions, skill options and abort cases in their index or detect files. That data now lives in run.ts, abort-cases.ts and agent-skill/run-definition.ts, and each index imports it. The data is unchanged: every exported config, prompt and abort case resolves to the same values as before. The old modules re-export SkillProgramOptions and each abort-case list, so importers keep their paths. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- src/programs/agent-skill/index.ts | 45 ++------------ src/programs/agent-skill/run-definition.ts | 43 +++++++++++++ src/programs/ai-observability/index.ts | 38 +----------- src/programs/ai-observability/run.ts | 39 ++++++++++++ src/programs/mcp-analytics/index.ts | 62 +------------------ src/programs/mcp-analytics/run.ts | 62 +++++++++++++++++++ src/programs/metrics/index.ts | 45 +------------- src/programs/metrics/run.ts | 44 +++++++++++++ src/programs/migration/index.ts | 44 ++----------- src/programs/migration/run.ts | 40 ++++++++++++ src/programs/replay-vision/index.ts | 47 +------------- src/programs/replay-vision/run.ts | 47 ++++++++++++++ src/programs/revenue-analytics/abort-cases.ts | 25 ++++++++ src/programs/revenue-analytics/detect.ts | 25 +------- src/programs/revenue-analytics/index.ts | 14 +---- src/programs/revenue-analytics/run.ts | 14 +++++ .../web-analytics-doctor/abort-cases.ts | 34 ++++++++++ src/programs/web-analytics-doctor/detect.ts | 34 +--------- src/programs/web-analytics-doctor/index.ts | 27 +------- src/programs/web-analytics-doctor/run.ts | 27 ++++++++ 20 files changed, 403 insertions(+), 353 deletions(-) create mode 100644 src/programs/agent-skill/run-definition.ts create mode 100644 src/programs/ai-observability/run.ts create mode 100644 src/programs/mcp-analytics/run.ts create mode 100644 src/programs/metrics/run.ts create mode 100644 src/programs/migration/run.ts create mode 100644 src/programs/replay-vision/run.ts create mode 100644 src/programs/revenue-analytics/abort-cases.ts create mode 100644 src/programs/revenue-analytics/run.ts create mode 100644 src/programs/web-analytics-doctor/abort-cases.ts create mode 100644 src/programs/web-analytics-doctor/run.ts diff --git a/src/programs/agent-skill/index.ts b/src/programs/agent-skill/index.ts index 75752731e..7632d30cf 100644 --- a/src/programs/agent-skill/index.ts +++ b/src/programs/agent-skill/index.ts @@ -20,36 +20,14 @@ */ import type { ProgramConfig } from '@programs/program-step'; -import type { AbortCase } from '@agent/types'; -import type { ProgramRun } from '@programs/program-run'; import { AGENT_SKILL_STEPS } from './steps.js'; import { getContentBlocks } from '../../ui/tui/decks/agent-skill/index.js'; +import { + skillRunDefinition, + type SkillProgramOptions, +} from './run-definition.js'; -export interface SkillProgramOptions { - /** Context-mill skill ID to install */ - skillId: string; - /** CLI subcommand name */ - command: string; - /** Unique flow key — must match a Program enum entry */ - id: string; - /** CLI description shown in --help */ - description: string; - /** Analytics integration label */ - integrationLabel: string; - /** Custom prompt instruction. Appended after default project prompt. */ - customPrompt?: string; - successMessage: string; - reportFile: string; - docsUrl: string; - spinnerMessage: string; - estimatedDurationMinutes: number; - /** Other program ids that must be satisfied first */ - requires?: string[]; - /** Override the default outro. Receives the same args as ProgramRun.buildOutroData. */ - buildOutroData?: ProgramRun['buildOutroData']; - /** Known `[ABORT] ` cases the skill can emit. */ - abortCases?: AbortCase[]; -} +export type { SkillProgramOptions } from './run-definition.js'; export function createSkillProgram(opts: SkillProgramOptions): ProgramConfig { return { @@ -60,18 +38,7 @@ export function createSkillProgram(opts: SkillProgramOptions): ProgramConfig { steps: AGENT_SKILL_STEPS, reportFile: opts.reportFile, getContentBlocks, - run: { - skillId: opts.skillId, - integrationLabel: opts.integrationLabel, - customPrompt: opts.customPrompt ? () => opts.customPrompt! : undefined, - successMessage: opts.successMessage, - reportFile: opts.reportFile, - docsUrl: opts.docsUrl, - spinnerMessage: opts.spinnerMessage, - estimatedDurationMinutes: opts.estimatedDurationMinutes, - buildOutroData: opts.buildOutroData, - abortCases: opts.abortCases, - }, + run: skillRunDefinition(opts), requires: opts.requires, }; } diff --git a/src/programs/agent-skill/run-definition.ts b/src/programs/agent-skill/run-definition.ts new file mode 100644 index 000000000..6a13a9570 --- /dev/null +++ b/src/programs/agent-skill/run-definition.ts @@ -0,0 +1,43 @@ +import type { AbortCase } from '@agent/types'; +import type { ProgramRun } from '@programs/program-run'; + +export interface SkillProgramOptions { + /** Context-mill skill ID to install */ + skillId: string; + /** CLI subcommand name */ + command: string; + /** Unique flow key — must match a Program enum entry */ + id: string; + /** CLI description shown in --help */ + description: string; + /** Analytics integration label */ + integrationLabel: string; + /** Custom prompt instruction. Appended after default project prompt. */ + customPrompt?: string; + successMessage: string; + reportFile: string; + docsUrl: string; + spinnerMessage: string; + estimatedDurationMinutes: number; + /** Other program ids that must be satisfied first */ + requires?: string[]; + /** Override the default outro. Receives the same args as ProgramRun.buildOutroData. */ + buildOutroData?: ProgramRun['buildOutroData']; + /** Known `[ABORT] ` cases the skill can emit. */ + abortCases?: AbortCase[]; +} + +export function skillRunDefinition(opts: SkillProgramOptions): ProgramRun { + return { + skillId: opts.skillId, + integrationLabel: opts.integrationLabel, + customPrompt: opts.customPrompt ? () => opts.customPrompt! : undefined, + successMessage: opts.successMessage, + reportFile: opts.reportFile, + docsUrl: opts.docsUrl, + spinnerMessage: opts.spinnerMessage, + estimatedDurationMinutes: opts.estimatedDurationMinutes, + buildOutroData: opts.buildOutroData, + abortCases: opts.abortCases, + }; +} diff --git a/src/programs/ai-observability/index.ts b/src/programs/ai-observability/index.ts index b71697d30..68b08030c 100644 --- a/src/programs/ai-observability/index.ts +++ b/src/programs/ai-observability/index.ts @@ -2,13 +2,12 @@ import type { ProgramConfig, ProgramStep } from '@programs/program-step'; import { AGENT_SKILL_STEPS } from '@programs/agent-skill/index'; import { getContentBlocks } from '@ui/tui/decks/agent-skill/index'; import { headlessOption, regionOption } from '@lib/headless-mode'; +import { AI_OBSERVABILITY_REPORT_FILE, AI_OBSERVABILITY_RUN } from './run.js'; const AI_OBSERVABILITY_STEPS: ProgramStep[] = AGENT_SKILL_STEPS.map((step) => step.id === 'intro' ? { ...step, screenId: 'ai-observability-intro' } : step, ); -const AI_OBSERVABILITY_REPORT_FILE = 'posthog-ai-observability-report.md'; - /** * `wizard ai-observability` — wrap the project's LLM client calls so they emit * `$ai_generation` events into AI Observability. @@ -27,38 +26,5 @@ export const aiObservabilityConfig: ProgramConfig = { steps: AI_OBSERVABILITY_STEPS, reportFile: AI_OBSERVABILITY_REPORT_FILE, getContentBlocks, - run: { - integrationLabel: 'ai-observability', - // No `skillId`: linear.ts skips its pre-install step (see the gate on - // `linear.ts:47`), so the agent must load the menu and install the right - // variant itself. The prompt below tells it how. - customPrompt: - () => `Instrument this project's LLM calls with PostHog AI Observability. - -This flow has no pre-installed skill — you install the right one yourself: - -1. Call \`load_skill_menu\` with \`category: "ai-observability"\`. The menu is - the source of truth: one variant per (LLM provider × language), plus a - \`manual-capture\` variant for projects with no vendor SDK. - -2. Scan the project manifest (\`package.json\`, \`pyproject.toml\`, - \`requirements.txt\`) for a vendor LLM SDK and pick the variant that matches - it — the language follows the manifest (\`package.json\` → Node, Python - tooling → Python). Multiple SDKs (e.g. LangChain wrapping OpenAI) → prefer - the higher abstraction. No vendor SDK → the \`manual-capture\` variant. - Genuinely ambiguous → \`wizard_ask\` with a multi-choice picker. - -3. Call \`install_skill\` with the picked variant id. Then follow that skill's - \`SKILL.md\` and references end-to-end. The skill itself will install - packages, wire OTel, set env vars, and describe verification. - -Make only additive changes — do not touch existing PostHog init, identify -calls, event capture, or dashboards. Those belong to other skills. The -final report is written to ./${AI_OBSERVABILITY_REPORT_FILE}.`, - successMessage: `AI Observability configured! View the report at ./${AI_OBSERVABILITY_REPORT_FILE}`, - reportFile: AI_OBSERVABILITY_REPORT_FILE, - docsUrl: 'https://posthog.com/docs/ai-observability', - spinnerMessage: 'Setting up AI Observability...', - estimatedDurationMinutes: 5, - }, + run: AI_OBSERVABILITY_RUN, }; diff --git a/src/programs/ai-observability/run.ts b/src/programs/ai-observability/run.ts new file mode 100644 index 000000000..1e9510e42 --- /dev/null +++ b/src/programs/ai-observability/run.ts @@ -0,0 +1,39 @@ +import type { ProgramRun } from '@programs/program-run'; + +export const AI_OBSERVABILITY_REPORT_FILE = + 'posthog-ai-observability-report.md'; + +export const AI_OBSERVABILITY_RUN: ProgramRun = { + integrationLabel: 'ai-observability', + // No `skillId`: linear.ts skips its pre-install step (see the gate on + // `linear.ts:47`), so the agent must load the menu and install the right + // variant itself. The prompt below tells it how. + customPrompt: + () => `Instrument this project's LLM calls with PostHog AI Observability. + +This flow has no pre-installed skill — you install the right one yourself: + +1. Call \`load_skill_menu\` with \`category: "ai-observability"\`. The menu is + the source of truth: one variant per (LLM provider × language), plus a + \`manual-capture\` variant for projects with no vendor SDK. + +2. Scan the project manifest (\`package.json\`, \`pyproject.toml\`, + \`requirements.txt\`) for a vendor LLM SDK and pick the variant that matches + it — the language follows the manifest (\`package.json\` → Node, Python + tooling → Python). Multiple SDKs (e.g. LangChain wrapping OpenAI) → prefer + the higher abstraction. No vendor SDK → the \`manual-capture\` variant. + Genuinely ambiguous → \`wizard_ask\` with a multi-choice picker. + +3. Call \`install_skill\` with the picked variant id. Then follow that skill's + \`SKILL.md\` and references end-to-end. The skill itself will install + packages, wire OTel, set env vars, and describe verification. + +Make only additive changes — do not touch existing PostHog init, identify +calls, event capture, or dashboards. Those belong to other skills. The +final report is written to ./${AI_OBSERVABILITY_REPORT_FILE}.`, + successMessage: `AI Observability configured! View the report at ./${AI_OBSERVABILITY_REPORT_FILE}`, + reportFile: AI_OBSERVABILITY_REPORT_FILE, + docsUrl: 'https://posthog.com/docs/ai-observability', + spinnerMessage: 'Setting up AI Observability...', + estimatedDurationMinutes: 5, +}; diff --git a/src/programs/mcp-analytics/index.ts b/src/programs/mcp-analytics/index.ts index faa9afd1c..c8b4bba50 100644 --- a/src/programs/mcp-analytics/index.ts +++ b/src/programs/mcp-analytics/index.ts @@ -1,44 +1,7 @@ -import type { AbortCase } from '@agent/types'; -import { ErrorCodes } from '@shared/errors'; import { createSkillProgram } from '@programs/agent-skill/index'; +import { MCP_ANALYTICS_OPTIONS } from './run.js'; -const MCP_ANALYTICS_REPORT_FILE = 'posthog-mcp-analytics-report.md'; - -/** - * `[ABORT]` reasons the mcp-analytics skill emits when the project can't be - * instrumented. Kept in sync with the stop conditions in the skill's - * `description.md` (context-mill `context/skills/mcp-analytics`). - */ -export const MCP_ANALYTICS_ABORT_CASES: AbortCase[] = [ - { - match: /^unsupported language for mcp analytics$/i, - errorCode: ErrorCodes.DetectUnsupportedPlatform, - message: 'Unsupported language for MCP analytics', - body: - 'MCP analytics supports TypeScript/JavaScript (`@posthog/mcp`) and Python ' + - '(`posthog.mcp`, shipped inside the `posthog` package). This project ' + - "doesn't look like either, so there's nothing to instrument. " + - 'See https://posthog.com/docs/mcp-analytics for the supported setups.', - }, - { - match: /^no mcp server found$/i, - message: 'No MCP server found', - body: - 'This command instruments an existing MCP server with PostHog analytics, ' + - 'but no MCP server was found in this project. If you just want PostHog ' + - 'product analytics, run `npx @posthog/wizard` instead.', - }, - { - match: /^could not locate the server entry point$/i, - message: 'Could not locate the MCP server entry point', - body: - "This project has MCP signals, but the agent couldn't find where the " + - "server is constructed or requests are dispatched, so there's nowhere " + - 'safe to add instrumentation. See https://posthog.com/docs/mcp-analytics ' + - 'for the supported server styles, or point the wizard at the package ' + - "that defines the server if it's in a monorepo subdirectory.", - }, -]; +export { MCP_ANALYTICS_ABORT_CASES } from './run.js'; /** * `wizard mcp-analytics` — flat skill command. @@ -54,23 +17,4 @@ export const MCP_ANALYTICS_ABORT_CASES: AbortCase[] = [ * 'mcp-analytics'` from context-mill — a deliberate breaking change, done then, * not pre-emptively. */ -export const mcpAnalyticsConfig = createSkillProgram({ - skillId: 'mcp-analytics', - command: 'mcp-analytics', - id: 'mcp-analytics', - description: 'Add PostHog MCP Analytics to your MCP server', - integrationLabel: 'mcp-analytics', - customPrompt: - "Instrument this project's MCP server with PostHog MCP analytics. Run the " + - '`mcp-analytics` skill end-to-end: detect the server style, install ' + - '`@posthog/mcp` and `posthog-node`, wrap the server (or use `PostHogMCP` ' + - 'for a custom dispatcher), wire the project API key and host, and verify. ' + - 'Make only additive changes — do not alter tool behavior. The final report ' + - `is written to ./${MCP_ANALYTICS_REPORT_FILE}.`, - successMessage: `MCP analytics configured! View the report at ./${MCP_ANALYTICS_REPORT_FILE}`, - reportFile: MCP_ANALYTICS_REPORT_FILE, - docsUrl: 'https://posthog.com/docs/mcp-analytics', - spinnerMessage: 'Setting up MCP analytics...', - estimatedDurationMinutes: 5, - abortCases: MCP_ANALYTICS_ABORT_CASES, -}); +export const mcpAnalyticsConfig = createSkillProgram(MCP_ANALYTICS_OPTIONS); diff --git a/src/programs/mcp-analytics/run.ts b/src/programs/mcp-analytics/run.ts new file mode 100644 index 000000000..690a31291 --- /dev/null +++ b/src/programs/mcp-analytics/run.ts @@ -0,0 +1,62 @@ +import type { AbortCase } from '@agent/types'; +import { ErrorCodes } from '@shared/errors'; +import type { SkillProgramOptions } from '@programs/agent-skill/run-definition'; + +const MCP_ANALYTICS_REPORT_FILE = 'posthog-mcp-analytics-report.md'; + +/** + * `[ABORT]` reasons the mcp-analytics skill emits when the project can't be + * instrumented. Kept in sync with the stop conditions in the skill's + * `description.md` (context-mill `context/skills/mcp-analytics`). + */ +export const MCP_ANALYTICS_ABORT_CASES: AbortCase[] = [ + { + match: /^unsupported language for mcp analytics$/i, + errorCode: ErrorCodes.DetectUnsupportedPlatform, + message: 'Unsupported language for MCP analytics', + body: + 'MCP analytics supports TypeScript/JavaScript (`@posthog/mcp`) and Python ' + + '(`posthog.mcp`, shipped inside the `posthog` package). This project ' + + "doesn't look like either, so there's nothing to instrument. " + + 'See https://posthog.com/docs/mcp-analytics for the supported setups.', + }, + { + match: /^no mcp server found$/i, + message: 'No MCP server found', + body: + 'This command instruments an existing MCP server with PostHog analytics, ' + + 'but no MCP server was found in this project. If you just want PostHog ' + + 'product analytics, run `npx @posthog/wizard` instead.', + }, + { + match: /^could not locate the server entry point$/i, + message: 'Could not locate the MCP server entry point', + body: + "This project has MCP signals, but the agent couldn't find where the " + + "server is constructed or requests are dispatched, so there's nowhere " + + 'safe to add instrumentation. See https://posthog.com/docs/mcp-analytics ' + + 'for the supported server styles, or point the wizard at the package ' + + "that defines the server if it's in a monorepo subdirectory.", + }, +]; + +export const MCP_ANALYTICS_OPTIONS: SkillProgramOptions = { + skillId: 'mcp-analytics', + command: 'mcp-analytics', + id: 'mcp-analytics', + description: 'Add PostHog MCP Analytics to your MCP server', + integrationLabel: 'mcp-analytics', + customPrompt: + "Instrument this project's MCP server with PostHog MCP analytics. Run the " + + '`mcp-analytics` skill end-to-end: detect the server style, install ' + + '`@posthog/mcp` and `posthog-node`, wrap the server (or use `PostHogMCP` ' + + 'for a custom dispatcher), wire the project API key and host, and verify. ' + + 'Make only additive changes — do not alter tool behavior. The final report ' + + `is written to ./${MCP_ANALYTICS_REPORT_FILE}.`, + successMessage: `MCP analytics configured! View the report at ./${MCP_ANALYTICS_REPORT_FILE}`, + reportFile: MCP_ANALYTICS_REPORT_FILE, + docsUrl: 'https://posthog.com/docs/mcp-analytics', + spinnerMessage: 'Setting up MCP analytics...', + estimatedDurationMinutes: 5, + abortCases: MCP_ANALYTICS_ABORT_CASES, +}; diff --git a/src/programs/metrics/index.ts b/src/programs/metrics/index.ts index c96612ae3..3cce288e5 100644 --- a/src/programs/metrics/index.ts +++ b/src/programs/metrics/index.ts @@ -1,13 +1,12 @@ import type { ProgramConfig, ProgramStep } from '@programs/program-step'; import { AGENT_SKILL_STEPS } from '@programs/agent-skill/index'; import { getContentBlocks } from '@ui/tui/decks/agent-skill/index'; +import { METRICS_REPORT_FILE, METRICS_RUN } from './run.js'; const METRICS_STEPS: ProgramStep[] = AGENT_SKILL_STEPS.map((step) => step.id === 'intro' ? { ...step, screenId: 'metrics-intro' } : step, ); -const METRICS_REPORT_FILE = 'posthog-metrics-report.md'; - /** * `wizard metrics` — instrument the project with PostHog application metrics * (`posthog.metrics` counters, gauges, and histograms). @@ -30,45 +29,5 @@ export const metricsConfig: ProgramConfig = { steps: METRICS_STEPS, reportFile: METRICS_REPORT_FILE, getContentBlocks, - run: { - integrationLabel: 'metrics', - // No `skillId`: the agent must load the menu and install the right - // variant itself. The prompt below tells it how. - customPrompt: - () => `Instrument this project with PostHog application metrics. - -This flow has no pre-installed skill — you install the right one yourself: - -1. Call \`load_skill_menu\` with \`category: "metrics"\`. The menu is the - source of truth: one variant per platform. - -2. Pick the variant that matches the project: - - Python tooling (\`pyproject.toml\`, \`requirements.txt\`, \`Pipfile\`) → - \`metrics-python\` (needs \`posthog\` >= 7.23.0) - - \`package.json\` with server-side Node code → \`metrics-nodejs\` - (needs \`posthog-node\` >= 5.43.0) - - \`package.json\` that is browser-only → \`metrics-javascript\` - (needs \`posthog-js\` >= 1.399.0) - - Kubernetes manifests / Helm charts and the user wants cluster-level - scraping → \`metrics-kubernetes\` - - Any other language → \`metrics-other\` (plain OTLP exporter) - A full-stack app (e.g. Next.js) usually wants the server variant — metrics - measure service work, not user actions. Genuinely ambiguous → - \`wizard_ask\` with a multi-choice picker. - -3. Call \`install_skill\` with the picked variant id. Then follow that skill's - \`SKILL.md\` and references end-to-end — it covers where to place metrics - (middleware, background jobs, external calls, business commit sites) and - the low-cardinality attribute rules. - -Make only additive changes — reuse an existing PostHog client by adding the -\`metrics\` config to it rather than constructing a second client, and do not -touch existing identify calls, event capture, or dashboards. The final report -is written to ./${METRICS_REPORT_FILE}.`, - successMessage: `Application metrics configured! View the report at ./${METRICS_REPORT_FILE}`, - reportFile: METRICS_REPORT_FILE, - docsUrl: 'https://posthog.com/docs/metrics', - spinnerMessage: 'Setting up application metrics...', - estimatedDurationMinutes: 5, - }, + run: METRICS_RUN, }; diff --git a/src/programs/metrics/run.ts b/src/programs/metrics/run.ts new file mode 100644 index 000000000..295cc2455 --- /dev/null +++ b/src/programs/metrics/run.ts @@ -0,0 +1,44 @@ +import type { ProgramRun } from '@programs/program-run'; + +export const METRICS_REPORT_FILE = 'posthog-metrics-report.md'; + +export const METRICS_RUN: ProgramRun = { + integrationLabel: 'metrics', + // No `skillId`: the agent must load the menu and install the right + // variant itself. The prompt below tells it how. + customPrompt: () => `Instrument this project with PostHog application metrics. + +This flow has no pre-installed skill — you install the right one yourself: + +1. Call \`load_skill_menu\` with \`category: "metrics"\`. The menu is the + source of truth: one variant per platform. + +2. Pick the variant that matches the project: + - Python tooling (\`pyproject.toml\`, \`requirements.txt\`, \`Pipfile\`) → + \`metrics-python\` (needs \`posthog\` >= 7.23.0) + - \`package.json\` with server-side Node code → \`metrics-nodejs\` + (needs \`posthog-node\` >= 5.43.0) + - \`package.json\` that is browser-only → \`metrics-javascript\` + (needs \`posthog-js\` >= 1.399.0) + - Kubernetes manifests / Helm charts and the user wants cluster-level + scraping → \`metrics-kubernetes\` + - Any other language → \`metrics-other\` (plain OTLP exporter) + A full-stack app (e.g. Next.js) usually wants the server variant — metrics + measure service work, not user actions. Genuinely ambiguous → + \`wizard_ask\` with a multi-choice picker. + +3. Call \`install_skill\` with the picked variant id. Then follow that skill's + \`SKILL.md\` and references end-to-end — it covers where to place metrics + (middleware, background jobs, external calls, business commit sites) and + the low-cardinality attribute rules. + +Make only additive changes — reuse an existing PostHog client by adding the +\`metrics\` config to it rather than constructing a second client, and do not +touch existing identify calls, event capture, or dashboards. The final report +is written to ./${METRICS_REPORT_FILE}.`, + successMessage: `Application metrics configured! View the report at ./${METRICS_REPORT_FILE}`, + reportFile: METRICS_REPORT_FILE, + docsUrl: 'https://posthog.com/docs/metrics', + spinnerMessage: 'Setting up application metrics...', + estimatedDurationMinutes: 5, +}; diff --git a/src/programs/migration/index.ts b/src/programs/migration/index.ts index 907160f6d..81f0db084 100644 --- a/src/programs/migration/index.ts +++ b/src/programs/migration/index.ts @@ -1,28 +1,12 @@ import type { ProgramConfig } from '@programs/program-step'; -import type { AbortCase } from '@agent/types'; import { WIZARD_TOOL_NAMES } from '@agent'; import { MIGRATION_PROGRAM } from './steps.js'; import { getContentBlocks } from '../../ui/tui/decks/migration/index.js'; - -const MIGRATION_REPORT_FILE = 'migration-report.md'; - -const MIGRATION_ABORT_CASES: AbortCase[] = [ - { - match: /^no source-sdk calls found$/i, - message: 'No source-SDK calls found', - body: - 'The migration needs an existing third-party SDK to migrate from. No ' + - 'calls to the source SDK appear anywhere in this project. If you ' + - "haven't installed PostHog yet, you don't need this command — run " + - '`npx @posthog/wizard@latest` to add PostHog from scratch.', - }, -]; - -// Default skill id when nothing else picks one. The `wizard migrate ` -// subcommands override this via skillCommandFactory using each manifest -// entry's skillId, so this default only kicks in for legacy callers (e.g. -// programmatic uses of migrationConfig directly). -const DEFAULT_MIGRATE_SKILL_ID = 'migrate-statsig'; +import { + DEFAULT_MIGRATE_SKILL_ID, + MIGRATION_REPORT_FILE, + MIGRATION_RUN, +} from './run.js'; export const migrationConfig: ProgramConfig = { command: 'migrate', @@ -34,23 +18,7 @@ export const migrationConfig: ProgramConfig = { getContentBlocks, allowedTools: ['Agent'], disallowedTools: [WIZARD_TOOL_NAMES.wizardAsk], - run: { - skillId: DEFAULT_MIGRATE_SKILL_ID, - integrationLabel: 'migration', - customPrompt: () => - 'Migrate this project from its existing third-party analytics, ' + - 'feature-flag, and observability tools to PostHog. Run the `migrate` ' + - 'skill end-to-end: follow the step chain starting at ' + - 'references/1-presence.md. Only replace existing source-SDK call sites ' + - 'with PostHog equivalents — make zero unrelated changes and no ' + - `net-new instrumentation. The final report is written to ./${MIGRATION_REPORT_FILE}.`, - successMessage: `Migration complete! View the report at ./${MIGRATION_REPORT_FILE}`, - reportFile: MIGRATION_REPORT_FILE, - docsUrl: '', - spinnerMessage: 'Migrating to PostHog...', - estimatedDurationMinutes: 8, - abortCases: MIGRATION_ABORT_CASES, - }, + run: MIGRATION_RUN, requires: ['posthog-integration'], }; diff --git a/src/programs/migration/run.ts b/src/programs/migration/run.ts new file mode 100644 index 000000000..25a28f14d --- /dev/null +++ b/src/programs/migration/run.ts @@ -0,0 +1,40 @@ +import type { AbortCase } from '@agent/types'; +import type { ProgramRun } from '@programs/program-run'; + +export const MIGRATION_REPORT_FILE = 'migration-report.md'; + +const MIGRATION_ABORT_CASES: AbortCase[] = [ + { + match: /^no source-sdk calls found$/i, + message: 'No source-SDK calls found', + body: + 'The migration needs an existing third-party SDK to migrate from. No ' + + 'calls to the source SDK appear anywhere in this project. If you ' + + "haven't installed PostHog yet, you don't need this command — run " + + '`npx @posthog/wizard@latest` to add PostHog from scratch.', + }, +]; + +// Default skill id when nothing else picks one. The `wizard migrate ` +// subcommands override this via skillCommandFactory using each manifest +// entry's skillId, so this default only kicks in for legacy callers (e.g. +// programmatic uses of migrationConfig directly). +export const DEFAULT_MIGRATE_SKILL_ID = 'migrate-statsig'; + +export const MIGRATION_RUN: ProgramRun = { + skillId: DEFAULT_MIGRATE_SKILL_ID, + integrationLabel: 'migration', + customPrompt: () => + 'Migrate this project from its existing third-party analytics, ' + + 'feature-flag, and observability tools to PostHog. Run the `migrate` ' + + 'skill end-to-end: follow the step chain starting at ' + + 'references/1-presence.md. Only replace existing source-SDK call sites ' + + 'with PostHog equivalents — make zero unrelated changes and no ' + + `net-new instrumentation. The final report is written to ./${MIGRATION_REPORT_FILE}.`, + successMessage: `Migration complete! View the report at ./${MIGRATION_REPORT_FILE}`, + reportFile: MIGRATION_REPORT_FILE, + docsUrl: '', + spinnerMessage: 'Migrating to PostHog...', + estimatedDurationMinutes: 8, + abortCases: MIGRATION_ABORT_CASES, +}; diff --git a/src/programs/replay-vision/index.ts b/src/programs/replay-vision/index.ts index 314bfc9a4..2c6561d97 100644 --- a/src/programs/replay-vision/index.ts +++ b/src/programs/replay-vision/index.ts @@ -1,4 +1,3 @@ -import type { AbortCase } from '@agent/types'; import { Integration } from '@shared/constants'; import { detectFramework, @@ -18,8 +17,9 @@ import type { WizardSession } from '@lib/wizard-session'; import { analytics } from '@utils/analytics'; import { wizardAbort } from '@utils/wizard-abort'; import { ErrorCodes } from '@shared/errors'; +import { REPLAY_VISION_OPTIONS } from './run.js'; -const REPLAY_VISION_REPORT_FILE = 'posthog-replay-vision-report.md'; +export { REPLAY_VISION_ABORT_CASES } from './run.js'; /** * The platforms session replay can actually record on. Replay vision watches @@ -77,22 +77,6 @@ async function abortUnsupportedPlatform( }); } -/** - * `[ABORT]` reasons the replay-vision skill emits when the run can't proceed. - * Kept in sync with the stop conditions in the skill's `description.md` - * (context-mill `context/skills/replay-vision`). - */ -export const REPLAY_VISION_ABORT_CASES: AbortCase[] = [ - { - match: /^replay vision not available for this project$/i, - message: 'Replay vision is not available for this project', - body: - 'Every Replay vision scanner endpoint reported that the feature is not ' + - 'available here yet. Session replay setup done so far is kept. See ' + - 'https://posthog.com/docs/replay-vision for availability.', - }, -]; - /** * Framework detection ahead of the run, exactly like the default integration * program. The orchestrator requires it: `session.skillId` must hold the @@ -119,32 +103,7 @@ const DETECT_STEP: ProgramStep = { }, }; -const base = createSkillProgram({ - // The menu ids this skill `-`, and context-mill's - // `replay-vision/config.yaml` declares a single variant, `setup`. The bare - // `replay-vision` id does not exist — the orchestrator never installs this - // (it resolves per-task mini-skills instead), but the linear path does, and - // aborts `skill-not-found` on a miss. - skillId: 'replay-vision-setup', - command: 'replay-vision', - id: 'replay-vision', - description: 'Set up PostHog Replay Vision scanners for your product', - integrationLabel: 'replay-vision', - customPrompt: - 'Set up PostHog Replay vision. Run the `replay-vision` skill end-to-end: ' + - 'make sure session replay is recording (server-side enable plus a ' + - 'posthog-js init check), then create the vision scanners the skill ' + - "defines, scoped to this product's key flows read out of the repo. If " + - 'PostHog is not integrated yet, install and initialize the SDK first as ' + - 'the skill instructs — do not abort. The final report is written to ' + - `./${REPLAY_VISION_REPORT_FILE}.`, - successMessage: `Replay vision configured! View the report at ./${REPLAY_VISION_REPORT_FILE}`, - reportFile: REPLAY_VISION_REPORT_FILE, - docsUrl: 'https://posthog.com/docs/replay-vision', - spinnerMessage: 'Setting up Replay vision...', - estimatedDurationMinutes: 6, - abortCases: REPLAY_VISION_ABORT_CASES, -}); +const base = createSkillProgram(REPLAY_VISION_OPTIONS); /** * `wizard replay-vision` — flat skill command on the orchestrator sequence. diff --git a/src/programs/replay-vision/run.ts b/src/programs/replay-vision/run.ts new file mode 100644 index 000000000..b8eaca446 --- /dev/null +++ b/src/programs/replay-vision/run.ts @@ -0,0 +1,47 @@ +import type { AbortCase } from '@agent/types'; +import type { SkillProgramOptions } from '@programs/agent-skill/run-definition'; + +const REPLAY_VISION_REPORT_FILE = 'posthog-replay-vision-report.md'; + +/** + * `[ABORT]` reasons the replay-vision skill emits when the run can't proceed. + * Kept in sync with the stop conditions in the skill's `description.md` + * (context-mill `context/skills/replay-vision`). + */ +export const REPLAY_VISION_ABORT_CASES: AbortCase[] = [ + { + match: /^replay vision not available for this project$/i, + message: 'Replay vision is not available for this project', + body: + 'Every Replay vision scanner endpoint reported that the feature is not ' + + 'available here yet. Session replay setup done so far is kept. See ' + + 'https://posthog.com/docs/replay-vision for availability.', + }, +]; + +export const REPLAY_VISION_OPTIONS: SkillProgramOptions = { + // The menu ids this skill `-`, and context-mill's + // `replay-vision/config.yaml` declares a single variant, `setup`. The bare + // `replay-vision` id does not exist — the orchestrator never installs this + // (it resolves per-task mini-skills instead), but the linear path does, and + // aborts `skill-not-found` on a miss. + skillId: 'replay-vision-setup', + command: 'replay-vision', + id: 'replay-vision', + description: 'Set up PostHog Replay Vision scanners for your product', + integrationLabel: 'replay-vision', + customPrompt: + 'Set up PostHog Replay vision. Run the `replay-vision` skill end-to-end: ' + + 'make sure session replay is recording (server-side enable plus a ' + + 'posthog-js init check), then create the vision scanners the skill ' + + "defines, scoped to this product's key flows read out of the repo. If " + + 'PostHog is not integrated yet, install and initialize the SDK first as ' + + 'the skill instructs — do not abort. The final report is written to ' + + `./${REPLAY_VISION_REPORT_FILE}.`, + successMessage: `Replay vision configured! View the report at ./${REPLAY_VISION_REPORT_FILE}`, + reportFile: REPLAY_VISION_REPORT_FILE, + docsUrl: 'https://posthog.com/docs/replay-vision', + spinnerMessage: 'Setting up Replay vision...', + estimatedDurationMinutes: 6, + abortCases: REPLAY_VISION_ABORT_CASES, +}; diff --git a/src/programs/revenue-analytics/abort-cases.ts b/src/programs/revenue-analytics/abort-cases.ts new file mode 100644 index 000000000..abb917a32 --- /dev/null +++ b/src/programs/revenue-analytics/abort-cases.ts @@ -0,0 +1,25 @@ +import type { AbortCase } from '@agent/types'; + +/** `[ABORT] ` cases the revenue analytics skill can emit. */ +export const REVENUE_ABORT_CASES: AbortCase[] = [ + { + // Skill emits: [ABORT] Could not find a PostHog distinct_id + match: /^could not find a posthog distinct_id$/i, + message: 'Could not find a PostHog distinct_id', + body: + 'The agent could not find PostHog distinct_id usage in your codebase. ' + + 'Your users must be identified in PostHog before they can be tagged in Stripe. ' + + 'Please identify your users and try again.', + docsUrl: 'https://posthog.com/docs/product-analytics/identify', + }, + { + // Skill emits: [ABORT] Could not find a Stripe integration + match: /^could not find a stripe integration$/i, + message: 'Could not find a Stripe integration', + body: + 'The Wizard could not find an existing Stripe customer, charge, ' + + 'subscription, or other Stripe operations. Please run the Revenue ' + + 'Analytics Wizard on a project with an existing Stripe integration.', + docsUrl: 'https://posthog.com/docs/revenue-analytics', + }, +]; diff --git a/src/programs/revenue-analytics/detect.ts b/src/programs/revenue-analytics/detect.ts index b81090ffd..380a40628 100644 --- a/src/programs/revenue-analytics/detect.ts +++ b/src/programs/revenue-analytics/detect.ts @@ -7,7 +7,6 @@ import { existsSync, statSync } from 'fs'; import type { WizardSession } from '@lib/wizard-session'; -import type { AbortCase } from '@agent/types'; import { findPackageJsons } from '@programs/shared/package-scanning'; export { @@ -32,29 +31,7 @@ export type RevenueDetectError = | { kind: 'missing-posthog'; foundStripe: string[] } | { kind: 'missing-stripe'; foundPosthog: string[] }; -/** `[ABORT] ` cases the revenue analytics skill can emit. */ -export const REVENUE_ABORT_CASES: AbortCase[] = [ - { - // Skill emits: [ABORT] Could not find a PostHog distinct_id - match: /^could not find a posthog distinct_id$/i, - message: 'Could not find a PostHog distinct_id', - body: - 'The agent could not find PostHog distinct_id usage in your codebase. ' + - 'Your users must be identified in PostHog before they can be tagged in Stripe. ' + - 'Please identify your users and try again.', - docsUrl: 'https://posthog.com/docs/product-analytics/identify', - }, - { - // Skill emits: [ABORT] Could not find a Stripe integration - match: /^could not find a stripe integration$/i, - message: 'Could not find a Stripe integration', - body: - 'The Wizard could not find an existing Stripe customer, charge, ' + - 'subscription, or other Stripe operations. Please run the Revenue ' + - 'Analytics Wizard on a project with an existing Stripe integration.', - docsUrl: 'https://posthog.com/docs/revenue-analytics', - }, -]; +export { REVENUE_ABORT_CASES } from './abort-cases.js'; /** * Scan `session.installDir` for PostHog + Stripe SDKs. Writes detection diff --git a/src/programs/revenue-analytics/index.ts b/src/programs/revenue-analytics/index.ts index 4fae357ac..ac321880b 100644 --- a/src/programs/revenue-analytics/index.ts +++ b/src/programs/revenue-analytics/index.ts @@ -1,8 +1,8 @@ import type { ProgramConfig } from '@programs/program-step'; import { WIZARD_TOOL_NAMES } from '@agent'; import { REVENUE_ANALYTICS_PROGRAM } from './steps.js'; -import { REVENUE_ABORT_CASES } from './detect.js'; import { getContentBlocks } from '../../ui/tui/decks/revenue-analytics/index.js'; +import { REVENUE_ANALYTICS_RUN } from './run.js'; export const revenueAnalyticsConfig: ProgramConfig = { command: 'revenue-analytics', @@ -13,17 +13,7 @@ export const revenueAnalyticsConfig: ProgramConfig = { getContentBlocks, allowedTools: ['Agent'], disallowedTools: [WIZARD_TOOL_NAMES.wizardAsk], - run: { - skillId: 'revenue-analytics-setup', - integrationLabel: 'revenue-analytics-setup', - customPrompt: () => 'Set up revenue analytics for this project.', - successMessage: 'Revenue analytics configured!', - reportFile: 'posthog-revenue-report.md', - docsUrl: 'https://posthog.com/docs/revenue-analytics', - spinnerMessage: 'Setting up revenue analytics...', - estimatedDurationMinutes: 5, - abortCases: REVENUE_ABORT_CASES, - }, + run: REVENUE_ANALYTICS_RUN, requires: ['posthog-integration'], }; diff --git a/src/programs/revenue-analytics/run.ts b/src/programs/revenue-analytics/run.ts new file mode 100644 index 000000000..861c75992 --- /dev/null +++ b/src/programs/revenue-analytics/run.ts @@ -0,0 +1,14 @@ +import type { ProgramRun } from '@programs/program-run'; +import { REVENUE_ABORT_CASES } from './abort-cases.js'; + +export const REVENUE_ANALYTICS_RUN: ProgramRun = { + skillId: 'revenue-analytics-setup', + integrationLabel: 'revenue-analytics-setup', + customPrompt: () => 'Set up revenue analytics for this project.', + successMessage: 'Revenue analytics configured!', + reportFile: 'posthog-revenue-report.md', + docsUrl: 'https://posthog.com/docs/revenue-analytics', + spinnerMessage: 'Setting up revenue analytics...', + estimatedDurationMinutes: 5, + abortCases: REVENUE_ABORT_CASES, +}; diff --git a/src/programs/web-analytics-doctor/abort-cases.ts b/src/programs/web-analytics-doctor/abort-cases.ts new file mode 100644 index 000000000..7b5fbd129 --- /dev/null +++ b/src/programs/web-analytics-doctor/abort-cases.ts @@ -0,0 +1,34 @@ +import type { AbortCase } from '@agent/types'; +import { ErrorCodes } from '@shared/errors'; + +export const WEB_ANALYTICS_ABORT_CASES: AbortCase[] = [ + { + match: /^no web analytics events$/i, + message: 'No web analytics events', + body: + 'The doctor found no $pageview events in the last 30 days, so there is ' + + 'nothing to audit yet. Make sure PostHog is initialized and capturing ' + + 'pageviews, then run the doctor again.', + docsUrl: 'https://posthog.com/docs/web-analytics/getting-started', + }, + { + match: /^insufficient permissions$/i, + errorCode: ErrorCodes.AuthMissingScope, + message: 'Insufficient permissions', + body: + 'The doctor could not query your project — the authenticated token is ' + + 'missing query access. Re-run the wizard to sign in again, or use a key ' + + 'with read access to your events.', + docsUrl: 'https://posthog.com/docs/web-analytics', + }, + { + match: /^posthog sdk not installed$/i, + errorCode: ErrorCodes.DetectNoPosthogSdk, + message: 'PostHog SDK not installed', + body: + 'The doctor could not find a PostHog SDK in this project. Install and ' + + 'configure PostHog first (run `npx @posthog/wizard`), then run the ' + + 'doctor to check your web analytics setup.', + docsUrl: 'https://posthog.com/docs/libraries/js', + }, +]; diff --git a/src/programs/web-analytics-doctor/detect.ts b/src/programs/web-analytics-doctor/detect.ts index e7df63673..6340ff6e9 100644 --- a/src/programs/web-analytics-doctor/detect.ts +++ b/src/programs/web-analytics-doctor/detect.ts @@ -1,7 +1,5 @@ import { existsSync, statSync } from 'fs'; import type { WizardSession } from '@lib/wizard-session'; -import type { AbortCase } from '@agent/types'; -import { ErrorCodes } from '@shared/errors'; import { findPackageJsons } from '@programs/shared/package-scanning'; export type WebAnalyticsDetectError = @@ -13,37 +11,7 @@ export type WebAnalyticsDetectError = | { kind: 'no-package-json' } | { kind: 'no-posthog'; scannedCount: number }; -export const WEB_ANALYTICS_ABORT_CASES: AbortCase[] = [ - { - match: /^no web analytics events$/i, - message: 'No web analytics events', - body: - 'The doctor found no $pageview events in the last 30 days, so there is ' + - 'nothing to audit yet. Make sure PostHog is initialized and capturing ' + - 'pageviews, then run the doctor again.', - docsUrl: 'https://posthog.com/docs/web-analytics/getting-started', - }, - { - match: /^insufficient permissions$/i, - errorCode: ErrorCodes.AuthMissingScope, - message: 'Insufficient permissions', - body: - 'The doctor could not query your project — the authenticated token is ' + - 'missing query access. Re-run the wizard to sign in again, or use a key ' + - 'with read access to your events.', - docsUrl: 'https://posthog.com/docs/web-analytics', - }, - { - match: /^posthog sdk not installed$/i, - errorCode: ErrorCodes.DetectNoPosthogSdk, - message: 'PostHog SDK not installed', - body: - 'The doctor could not find a PostHog SDK in this project. Install and ' + - 'configure PostHog first (run `npx @posthog/wizard`), then run the ' + - 'doctor to check your web analytics setup.', - docsUrl: 'https://posthog.com/docs/libraries/js', - }, -]; +export { WEB_ANALYTICS_ABORT_CASES } from './abort-cases.js'; export function detectWebAnalyticsPrerequisites( session: WizardSession, diff --git a/src/programs/web-analytics-doctor/index.ts b/src/programs/web-analytics-doctor/index.ts index 4d11413c2..a7c2863ce 100644 --- a/src/programs/web-analytics-doctor/index.ts +++ b/src/programs/web-analytics-doctor/index.ts @@ -1,33 +1,10 @@ import type { ProgramConfig } from '@programs/program-step'; import { createSkillProgram } from '../agent-skill/index.js'; import { WEB_ANALYTICS_DOCTOR_PROGRAM } from './steps.js'; -import { WEB_ANALYTICS_ABORT_CASES } from './detect.js'; - -const REPORT_FILE = 'posthog-web-analytics-report.md'; -const DOCS_URL = 'https://posthog.com/docs/web-analytics'; +import { WEB_ANALYTICS_DOCTOR_OPTIONS } from './run.js'; export const webAnalyticsDoctorConfig: ProgramConfig = { - ...createSkillProgram({ - skillId: 'web-analytics-doctor', - command: 'web-analytics', - id: 'web-analytics-doctor', - description: 'Audit and fix your PostHog web analytics setup', - integrationLabel: 'web-analytics-doctor', - customPrompt: - "Run the web-analytics-doctor skill to check this project's PostHog web " + - 'analytics setup. Audit read-only first, then present the findings to the ' + - 'user with a single wizard_ask multi-select and apply only the fixes they ' + - 'choose — editing project code and/or PostHog project settings via the ' + - 'MCP — before writing the report.', - successMessage: - 'Web analytics check complete! You can view the report at ./posthog-web-analytics-report.md', - reportFile: REPORT_FILE, - docsUrl: DOCS_URL, - spinnerMessage: 'Checking your web analytics setup...', - estimatedDurationMinutes: 5, - requires: ['posthog-integration'], - abortCases: WEB_ANALYTICS_ABORT_CASES, - }), + ...createSkillProgram(WEB_ANALYTICS_DOCTOR_OPTIONS), steps: WEB_ANALYTICS_DOCTOR_PROGRAM, parentCommand: 'audit', }; diff --git a/src/programs/web-analytics-doctor/run.ts b/src/programs/web-analytics-doctor/run.ts new file mode 100644 index 000000000..cd6bf6b44 --- /dev/null +++ b/src/programs/web-analytics-doctor/run.ts @@ -0,0 +1,27 @@ +import type { SkillProgramOptions } from '@programs/agent-skill/run-definition'; +import { WEB_ANALYTICS_ABORT_CASES } from './abort-cases.js'; + +const REPORT_FILE = 'posthog-web-analytics-report.md'; +const DOCS_URL = 'https://posthog.com/docs/web-analytics'; + +export const WEB_ANALYTICS_DOCTOR_OPTIONS: SkillProgramOptions = { + skillId: 'web-analytics-doctor', + command: 'web-analytics', + id: 'web-analytics-doctor', + description: 'Audit and fix your PostHog web analytics setup', + integrationLabel: 'web-analytics-doctor', + customPrompt: + "Run the web-analytics-doctor skill to check this project's PostHog web " + + 'analytics setup. Audit read-only first, then present the findings to the ' + + 'user with a single wizard_ask multi-select and apply only the fixes they ' + + 'choose — editing project code and/or PostHog project settings via the ' + + 'MCP — before writing the report.', + successMessage: + 'Web analytics check complete! You can view the report at ./posthog-web-analytics-report.md', + reportFile: REPORT_FILE, + docsUrl: DOCS_URL, + spinnerMessage: 'Checking your web analytics setup...', + estimatedDurationMinutes: 5, + requires: ['posthog-integration'], + abortCases: WEB_ANALYTICS_ABORT_CASES, +}; From 793a34fd41927ddd7098c44a6eb03e4ac065a3dd Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 15:40:16 -0400 Subject: [PATCH 08/10] refactor(programs): own binding policy and experiment selection Move program routing out of the agent: - PROGRAM_BINDINGS and resolveProgramBinding go to src/programs/binding.ts. - The flag experiments move from src/agent/runner/switchboard/flags to src/programs/experiments, with their tests. - The program-axis commandments go to src/programs/commandments.ts, and the switchboard telemetry to src/programs/binding-telemetry.ts. - The switchboard and variant-gating tests move to src/programs. - DEFAULT_AGENT_BINDING, in src/agent/default-binding.ts, is the route standalone callers pass. RunConfig drops switchboard. The caller passes what it already resolved: programCommandments, stageOverrides, seededTasksEnabled and binding.roleBindings. The orchestrator reads the role routes and the stage and seeded-task policy from them. Pi assembles the supplied commandments on both sequences, and Anthropic on task runs. Anthropic's linear run looked guidance up by the run label, which names no program, so it still gets none. The agent's resolveBinding and resolveHarness take the base binding and the flag route as data. Behavior is unchanged. For every program, across 22 flag and CLI cases, in dev, production and cloud-surface builds, the resolved binding, trace, role routes, stage overrides, seeded-task gate and commandments match the previous code. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- .claude/skills/adding-skill-program/SKILL.md | 4 +- .claude/skills/wizard-development/SKILL.md | 6 +- .../references/ARCHITECTURE.md | 18 +-- AGENTS.md | 2 +- docs/benchmarking.md | 2 +- scripts/a3-fault-probe.no-jest.ts | 5 - .../architecture/known-violations.json | 3 - src/agent/README.md | 20 ++-- src/agent/__tests__/commandments.test.ts | 10 +- .../__tests__/run-agent-standalone.test.ts | 42 +------ src/agent/agent-interface.ts | 10 +- src/agent/default-binding.ts | 10 ++ src/agent/index.ts | 17 +-- src/agent/runner/README.md | 2 +- .../__tests__/pending-question.test.ts | 1 - src/agent/runner/harness/anthropic/index.ts | 1 + src/agent/runner/harness/pi/index.ts | 2 +- src/agent/runner/harness/pi/task.ts | 2 +- src/agent/runner/sequence/README.md | 5 +- .../orchestrator/orchestrator-runner.ts | 30 ++--- src/agent/runner/shared/types.ts | 19 +-- src/agent/runner/switchboard/commandments.ts | 29 +---- src/agent/runner/switchboard/harness.ts | 32 +++-- src/agent/runner/switchboard/index.ts | 98 +++------------- src/agent/runner/switchboard/sequence.ts | 25 ++-- src/agent/types.ts | 6 +- .../__tests__/switchboard.test.ts | 12 +- .../__tests__/variant-gating.test.ts | 2 +- src/programs/binding-telemetry.ts | 50 ++++++++ src/programs/binding.ts | 109 ++++++++++++++++++ src/programs/commandments.ts | 18 +++ .../experiments}/__tests__/binding-cases.ts | 9 +- .../experiments}/__tests__/flags.test.ts | 34 +++--- .../flags => programs/experiments}/index.ts | 0 .../experiments}/orchestrator.ts | 0 .../flags => programs/experiments}/schemes.ts | 2 +- .../experiments}/self-driving.ts | 0 src/programs/run-agent-legacy.ts | 73 +++--------- src/programs/types.ts | 1 + 39 files changed, 376 insertions(+), 335 deletions(-) create mode 100644 src/agent/default-binding.ts rename src/{agent/runner => programs}/__tests__/switchboard.test.ts (97%) rename src/{agent => programs}/__tests__/variant-gating.test.ts (89%) create mode 100644 src/programs/binding-telemetry.ts create mode 100644 src/programs/binding.ts create mode 100644 src/programs/commandments.ts rename src/{agent/runner/switchboard/flags => programs/experiments}/__tests__/binding-cases.ts (91%) rename src/{agent/runner/switchboard/flags => programs/experiments}/__tests__/flags.test.ts (93%) rename src/{agent/runner/switchboard/flags => programs/experiments}/index.ts (100%) rename src/{agent/runner/switchboard/flags => programs/experiments}/orchestrator.ts (100%) rename src/{agent/runner/switchboard/flags => programs/experiments}/schemes.ts (99%) rename src/{agent/runner/switchboard/flags => programs/experiments}/self-driving.ts (100%) diff --git a/.claude/skills/adding-skill-program/SKILL.md b/.claude/skills/adding-skill-program/SKILL.md index 3aedd8778..28e4ed022 100644 --- a/.claude/skills/adding-skill-program/SKILL.md +++ b/.claude/skills/adding-skill-program/SKILL.md @@ -62,8 +62,8 @@ changing execution behavior. 3. Register the config in [PROGRAM_REGISTRY](../../../src/programs/program-registry.ts) and add its Pi/orchestrator entry to - [PROGRAM_BINDINGS](../../../src/agent/runner/switchboard/index.ts). - [Existing binding checks](../../../src/agent/runner/__tests__/switchboard.test.ts) + [PROGRAM_BINDINGS](../../../src/programs/binding.ts). + [Existing binding checks](../../../src/programs/__tests__/switchboard.test.ts) enforce coverage; `ProgramId` currently widens to `string`. 4. For a standalone native command, create a command module with [nativeCommandFactory](../../../src/commands/factories/native-command-factory.ts) diff --git a/.claude/skills/wizard-development/SKILL.md b/.claude/skills/wizard-development/SKILL.md index 5fee5c7a5..c9819acd4 100644 --- a/.claude/skills/wizard-development/SKILL.md +++ b/.claude/skills/wizard-development/SKILL.md @@ -22,7 +22,7 @@ infrastructure should consume those boundaries. | Framework detection, context, env conventions | [FrameworkConfig](../../../src/programs/framework-config.ts) and [framework configs](../../../src/programs/frameworks/) | | Integration instructions and orchestrator flows/tasks | [context-mill](https://github.com/PostHog/context-mill) | | Programs, steps, prerequisites and outcomes | [programs](../../../src/programs/) | -| Sequence, harness, model and effort selection | [switchboard](../../../src/agent/runner/switchboard/) | +| Sequence, harness, model and effort selection | [program bindings](../../../src/programs/binding.ts) and [agent clamps](../../../src/agent/runner/switchboard/) | | Local tool permissions and scanner adapters | [agent-interface](../../../src/agent/agent-interface.ts), [YARA hooks](../../../src/agent/yara-hooks.ts), [Pi security](../../../src/agent/runner/harness/pi/security.ts) | | Scanner rules | [warlock](https://github.com/PostHog/warlock) | | Token admission and budgets | [PostHog mint endpoint](https://github.com/PostHog/posthog/blob/master/posthog/llm/wizard_gateway_token.py) and [ai-gateway](https://github.com/PostHog/ai-gateway) | @@ -43,8 +43,8 @@ infrastructure should consume those boundaries. new Anthropic models. Existing routing has not all migrated: -[DEFAULT_BINDING](../../../src/agent/runner/switchboard/index.ts) still -selects Anthropic + linear, with per-program and flag overrides. Set new +[DEFAULT_AGENT_BINDING](../../../src/agent/default-binding.ts) selects Pi + linear +for standalone runs; programs apply their own binding and flag overrides. Set new bindings explicitly. Migrating an existing program requires checking its flow, tasks, and lifecycle hooks; changing the default constant alone is insufficient. Both harnesses implement `run` and `runTask`. diff --git a/.claude/skills/wizard-development/references/ARCHITECTURE.md b/.claude/skills/wizard-development/references/ARCHITECTURE.md index 570d1e5e1..72a0228b4 100644 --- a/.claude/skills/wizard-development/references/ARCHITECTURE.md +++ b/.claude/skills/wizard-development/references/ARCHITECTURE.md @@ -60,10 +60,11 @@ example. Native command modules still need registration in ## Switchboard contract -`resolveBinding(ctx, role?)` is the routing seam. Read its exported types rather -than copying their fields into another document. It receives the program, flag -snapshot/payloads, composition state, and development overrides, returning the -binding and stamping a trace of the selected precedence rungs. +`resolveProgramBinding(ctx)` in [programs](../../../../src/programs/binding.ts) +is the routing seam. It receives the program, flag snapshot/payloads, +composition state and development overrides, returning a resolved binding and +stamping a trace of the selected precedence rungs. The agent receives this +binding and any pre-resolved task-role routes as data. - Harness/model: development CLI override, declared flag route, per-program binding, default. @@ -72,7 +73,8 @@ binding and stamping a trace of the selected precedence rungs. [Harness](../../../../src/agent/runner/switchboard/harness.ts) and [sequence](../../../../src/agent/runner/switchboard/sequence.ts) contain the -exact chains. Published builds omit CLI overrides. `RUN_SURFACE` can disable +generic precedence and clamp chains over caller-supplied policy. Published +builds omit CLI overrides. `RUN_SURFACE` can disable harness experiments; static bindings and harness capabilities also affect resolution. Composed sub-runs remain linear even when a CLI override requests orchestration. @@ -84,11 +86,11 @@ admitted by the minted token and gateway. Local routing cannot bypass that external policy; see the [model admission checklist](../SKILL.md#execution-policy-and-model-admission). -Flags belong in -[switchboard/flags](../../../../src/agent/runner/switchboard/flags/). +Program flags belong in +[experiments](../../../../src/programs/experiments/). Experiments declare their program scope; malformed payloads yield no experiment route. Reuse the -[switchboard tests](../../../../src/agent/runner/__tests__/switchboard.test.ts) +[switchboard tests](../../../../src/programs/__tests__/switchboard.test.ts) and experiment tests to check full bindings and isolation of unrelated programs. Do not add a second flag-reading path inside a harness or sequence. diff --git a/AGENTS.md b/AGENTS.md index 773faf9cf..17463d33e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,7 +76,7 @@ Agent SDK is a supported legacy fallback, deprecated as the default; retain it for major Pi vulnerabilities or gaps in support for new Anthropic models. This is the contribution policy, not a claim that every existing binding has -migrated: `DEFAULT_BINDING` is still Anthropic + linear. Set new bindings +migrated. The default, `DEFAULT_AGENT_BINDING`, is Pi + linear. Set new bindings explicitly and check sequence-specific hooks before migrating existing flows. See [execution policy and model admission](.claude/skills/wizard-development/SKILL.md#execution-policy-and-model-admission) diff --git a/docs/benchmarking.md b/docs/benchmarking.md index e5fadf4f7..8d4a15bd9 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -102,7 +102,7 @@ Everything below ships in this repo (`wizard/`) and its workbench orchestrator on pi, per-task models from context-mill frontmatter; off → the linear anthropic default). Per-stage variations ride `wizard-orchestrator-override` payloads (`{stage: {model?, effort?}}`, - variant keys in `wizard/src/agent/runner/switchboard/flags/schemes.ts`). + variant keys in `wizard/src/programs/experiments/schemes.ts`). The baseline is `{"wizard-orchestrator":"false"}` — never an empty override, or live remote flags leak into the baseline. diff --git a/scripts/a3-fault-probe.no-jest.ts b/scripts/a3-fault-probe.no-jest.ts index 91b83e5ac..58ed9d8bd 100644 --- a/scripts/a3-fault-probe.no-jest.ts +++ b/scripts/a3-fault-probe.no-jest.ts @@ -66,11 +66,6 @@ const config: RunConfig = { harness: harness as Harness, model: DEFAULT_AGENT_MODEL, }, - switchboard: { - program: 'fault-probe', - flags: {}, - cliHarness: harness as Harness, - }, skillsBaseUrl: 'http://127.0.0.1:1', wizardFlags: {}, wizardFlagPayloads: {}, diff --git a/src/__tests__/architecture/known-violations.json b/src/__tests__/architecture/known-violations.json index c91511369..e1bf710f5 100644 --- a/src/__tests__/architecture/known-violations.json +++ b/src/__tests__/architecture/known-violations.json @@ -1,8 +1,5 @@ { "violations": [ - "src/agent/runner/switchboard/flags/index.ts -> src/programs/types.ts", - "src/agent/runner/switchboard/flags/schemes.ts -> src/programs/types.ts", - "src/agent/runner/switchboard/index.ts -> src/programs/types.ts", "src/commands/ai-observability.ts -> src/programs/ai-observability/index.ts", "src/commands/audit.ts -> src/programs/audit/index.ts", "src/commands/basic-integration/ci-install.ts -> src/programs/posthog-integration/index.ts", diff --git a/src/agent/README.md b/src/agent/README.md index 4116171cd..a3c188057 100644 --- a/src/agent/README.md +++ b/src/agent/README.md @@ -20,10 +20,11 @@ runAgent(config: RunConfig, input: RunInput, options?: { }): Promise ``` -- `RunConfig`: the program id, its `AgentRunDefinition` (prompt, skill, tools, - copy), the resolved `binding` (sequence, harness, model), the switchboard - inputs, the skills origin, flag snapshot, trace tags, tool allow and deny - lists, seed tasks and bound completion `hooks`. +- `RunConfig`: the opaque program id, its `AgentRunDefinition` (prompt, skill, + tools, copy), the resolved `binding` (sequence, harness, model and task-role + routes), supplied program commandments and stage policy, the skills origin, + flag snapshot, trace tags, tool allow and deny lists, seed tasks and bound + completion `hooks`. - `RunInput`: install directory, resolved credentials, project and user payloads, skill id, detected integration, `flags` (`ci`, `signup`, `debug`, `e2eAsk`, `localMcp`, `captureAio`, `benchmark`, `yaraReport`) and the host @@ -54,11 +55,12 @@ runAgent(config: RunConfig, input: RunInput, options?: { `setup wizard finished` event. The host sends it from the outcome: `Success` is `success`, `Aborted` is `cancelled`, `Failed` and `Crashed` are `error`. -Other runtime exports: `resolveBinding`, `shouldDisableAsk`, `initializeAgent`, -`executeAgent`, `buildRunTags`, `AgentSignals`, -`configureGatewayFromCIEnvironment`, `downloadSkill`, `WIZARD_TOOL_NAMES`, -`LONGER_ASK_TIMEOUT_MS`, `flushScanReport`, and `runMcpPromptViaSdk`, which -loads the streaming module on first call. +Other runtime exports: `DEFAULT_AGENT_BINDING` for standalone callers, +`resolveHarness` and `resolveBinding`, which programs resolve a binding with, +`shouldDisableAsk`, `initializeAgent`, `executeAgent`, `buildRunTags`, +`AgentSignals`, `configureGatewayFromCIEnvironment`, `downloadSkill`, +`WIZARD_TOOL_NAMES`, `LONGER_ASK_TIMEOUT_MS`, `flushScanReport`, and +`runMcpPromptViaSdk`, which loads the streaming module on first call. Minimal invocation: diff --git a/src/agent/__tests__/commandments.test.ts b/src/agent/__tests__/commandments.test.ts index 2a694c0f2..d4a9722d5 100644 --- a/src/agent/__tests__/commandments.test.ts +++ b/src/agent/__tests__/commandments.test.ts @@ -1,5 +1,6 @@ import { WIZARD_COMMANDMENTS } from '@agent/commandments'; import { assembleCommandments } from '@agent/runner/switchboard/commandments'; +import { getProgramCommandments } from '@programs/commandments'; import { Harness, Sequence } from '@shared/constants'; const global = WIZARD_COMMANDMENTS.join('\n'); @@ -10,7 +11,13 @@ const prompt = ( harness: Harness, sequence: Sequence, program = 'posthog-integration', -) => assembleCommandments({ program, sequence, harness, caps: CAPS }); +) => + assembleCommandments({ + programCommandments: getProgramCommandments(program), + sequence, + harness, + caps: CAPS, + }); const COMBOS = [ ['anthropic', Harness.anthropic, Sequence.linear], @@ -148,7 +155,6 @@ describe('commandments by axis', () => { describe('runtime caps gate the pi runtime notes', () => { const withCaps = (caps: { bash: boolean; posthogMcp: boolean }) => assembleCommandments({ - program: 'warehouse-source', sequence: Sequence.linear, harness: Harness.pi, caps, diff --git a/src/agent/__tests__/run-agent-standalone.test.ts b/src/agent/__tests__/run-agent-standalone.test.ts index df73af8fe..19e75663a 100644 --- a/src/agent/__tests__/run-agent-standalone.test.ts +++ b/src/agent/__tests__/run-agent-standalone.test.ts @@ -175,6 +175,12 @@ vi.mock('@agent/runner/switchboard/harness', () => { harness: ctx.cliHarness ?? Harness.pi, model: DEFAULT_AGENT_MODEL, }), + resolveRoleHarness: (binding: RunConfig['binding'], role: string) => + binding.roleBindings?.[role] ?? { + harness: binding.harness, + model: binding.model, + thinkingLevel: binding.thinkingLevel, + }, }; }); @@ -237,7 +243,6 @@ const config = (over: Partial = {}): RunConfig => ({ }, composed: false, binding: { sequence: Sequence.linear, harness: Harness.pi, model: 'm' }, - switchboard: { program: 'test-program', flags: {} }, skillsBaseUrl: 'https://skills.test', wizardFlags: {}, wizardFlagPayloads: {}, @@ -316,11 +321,6 @@ describe('runAgent standalone', () => { const running = runAgent( config({ binding: { harness, sequence, model: DEFAULT_AGENT_MODEL }, - switchboard: { - program: 'test-program', - flags: {}, - cliHarness: harness, - }, }), input(), { interaction: { ask }, onProgress: (event) => events.push(event) }, @@ -387,11 +387,6 @@ describe('runAgent standalone', () => { const result = await runAgent( config({ binding: { harness, sequence, model: DEFAULT_AGENT_MODEL }, - switchboard: { - program: 'test-program', - flags: {}, - cliHarness: harness, - }, }), input(), ); @@ -427,11 +422,6 @@ describe('runAgent standalone', () => { sequence: Sequence.orchestrator, model: DEFAULT_AGENT_MODEL, }, - switchboard: { - program: 'test-program', - flags: {}, - cliHarness: Harness.anthropic, - }, }), input(), ); @@ -454,11 +444,6 @@ describe('runAgent standalone', () => { sequence: Sequence.orchestrator, model: DEFAULT_AGENT_MODEL, }, - switchboard: { - program: 'test-program', - flags: {}, - cliHarness: Harness.anthropic, - }, }), input(), ); @@ -482,11 +467,6 @@ describe('runAgent standalone', () => { sequence: Sequence.orchestrator, model: DEFAULT_AGENT_MODEL, }, - switchboard: { - program: 'test-program', - flags: {}, - cliHarness: Harness.pi, - }, }), input(), ); @@ -532,11 +512,6 @@ describe('runAgent standalone', () => { sequence: Sequence.orchestrator, model: DEFAULT_AGENT_MODEL, }, - switchboard: { - program: 'test-program', - flags: {}, - cliHarness: Harness.pi, - }, }), input(), { @@ -922,11 +897,6 @@ describe('runAgent standalone', () => { sequence, model: DEFAULT_AGENT_MODEL, }, - switchboard: { - program: 'test-program', - flags: {}, - cliHarness: Harness.pi, - }, }), input(), { diff --git a/src/agent/agent-interface.ts b/src/agent/agent-interface.ts index f19cd85f2..ff13db4d2 100644 --- a/src/agent/agent-interface.ts +++ b/src/agent/agent-interface.ts @@ -211,6 +211,8 @@ export type AgentConfig = { * another program's budget, so neither should depend on an optional string bag. */ programId: string; + /** Program-owned guidance supplied as data, never looked up here. */ + programCommandments?: readonly string[]; /** Program identifier — selects the model for that program. */ integrationLabel?: string; /** @@ -355,8 +357,8 @@ type AgentRunConfig = { * bearer. */ refreshGatewayAuth?: () => Promise; - /** Program id, for the program-axis commandments. */ - program?: string; + /** Program-owned guidance supplied as data. */ + programCommandments?: readonly string[]; /** Resolved sequence, for the sequence-axis commandments. */ sequence: Sequence; /** Where the run reports. A no-op when the caller passed none. */ @@ -646,7 +648,7 @@ export async function initializeAgent( triageProvider, gatewayAuth: auth, refreshGatewayAuth: currentGatewayAuth, - program: config.integrationLabel, + programCommandments: config.programCommandments, // A queue context is present only on a task run; that is the sequence. sequence: config.orchestrator ? Sequence.orchestrator : Sequence.linear, emit, @@ -1146,7 +1148,7 @@ export async function runAgent( // we keep default Claude Code behaviors. An orchestrator context is // present only on a task run — that is what picks the sequence. append: assembleCommandments({ - program: agentConfig.program, + programCommandments: agentConfig.programCommandments, sequence: agentConfig.sequence, harness: Harness.anthropic, }), diff --git a/src/agent/default-binding.ts b/src/agent/default-binding.ts new file mode 100644 index 000000000..7bad35666 --- /dev/null +++ b/src/agent/default-binding.ts @@ -0,0 +1,10 @@ +import { GPT5_6_SOL_MODEL, Harness, Sequence } from '@shared/constants'; +import type { ResolvedBinding } from './runner/shared/types'; + +/** Standalone callers can supply this resolved route without a program registry. */ +export const DEFAULT_AGENT_BINDING: ResolvedBinding = { + sequence: Sequence.linear, + harness: Harness.pi, + model: GPT5_6_SOL_MODEL, + thinkingLevel: 'medium', +}; diff --git a/src/agent/index.ts b/src/agent/index.ts index 605eeb0c8..577ffa776 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -10,21 +10,22 @@ /** * Stays. The agent's contract: the one way to run it, the marker strings - * program prompts embed, and the tool ids programs put in allowedTools and - * disallowedTools. + * program prompts embed, the tool ids programs put in allowedTools and + * disallowedTools, and what programs resolve a binding with: the default + * binding and the harness axis. */ export type * from './types'; export { runAgent, RunOutcome } from './runner'; export { AgentSignals } from './agent-interface'; export { WIZARD_TOOL_NAMES } from './tools'; +export { DEFAULT_AGENT_BINDING } from './default-binding'; +export { resolveHarness } from './runner/switchboard'; /** - * Leaves in B2 (B1 deferred the bindings). Bindings and program data move to - * programs: resolveBinding - * is keyed by PROGRAM_BINDINGS and the agent keeps only "run from an - * already-resolved binding"; shouldDisableAsk is a flags policy programs - * decide and pass in; LONGER_ASK_TIMEOUT_MS is a tuning number programs own - * as askTimeoutMs. + * Leaves in B2. resolveBinding applies the generic precedence to the base + * binding and routes programs select, and programs/binding.ts is its one + * caller; shouldDisableAsk is a flags policy programs decide and pass in; + * LONGER_ASK_TIMEOUT_MS is a tuning number programs own as askTimeoutMs. */ export { resolveBinding, shouldDisableAsk } from './runner'; export { LONGER_ASK_TIMEOUT_MS } from './wizard-ask-bridge'; diff --git a/src/agent/runner/README.md b/src/agent/runner/README.md index 480c1280b..611be4c79 100644 --- a/src/agent/runner/README.md +++ b/src/agent/runner/README.md @@ -23,7 +23,7 @@ retained for very simple tasks and legacy support. The Anthropic Agent SDK is a supported legacy fallback, deprecated as the default, retained for major Pi vulnerabilities or gaps in support for new Anthropic models. -Existing `DEFAULT_BINDING` remains Anthropic + linear; explicit program bindings +`DEFAULT_AGENT_BINDING`, the standalone default, is Pi + linear; explicit program bindings and flags determine actual behavior. Both harnesses implement `run` and `runTask`. Composed sub-runs are clamped to linear, and linear-only post-run/outro hooks do not automatically transfer to an orchestrated flow. diff --git a/src/agent/runner/harness/anthropic/__tests__/pending-question.test.ts b/src/agent/runner/harness/anthropic/__tests__/pending-question.test.ts index 86ff59c3f..bd7580a5f 100644 --- a/src/agent/runner/harness/anthropic/__tests__/pending-question.test.ts +++ b/src/agent/runner/harness/anthropic/__tests__/pending-question.test.ts @@ -44,7 +44,6 @@ async function initializeHarness( sequence: Sequence.linear, model: 'test', }, - switchboard: { program: 'test', flags: {} }, skillsBaseUrl: 'https://skills.test', wizardFlags: {}, wizardFlagPayloads: {}, diff --git a/src/agent/runner/harness/anthropic/index.ts b/src/agent/runner/harness/anthropic/index.ts index 8a75169bf..4d622f8ad 100644 --- a/src/agent/runner/harness/anthropic/index.ts +++ b/src/agent/runner/harness/anthropic/index.ts @@ -136,6 +136,7 @@ export const anthropicBackend: AgentHarness = { detectPackageManager: detectNodePackageManagers, skillsBaseUrl: boot.skillsBaseUrl, programId: boot.programId, + programCommandments: config.programCommandments, wizardFlags: boot.wizardFlags, wizardMetadata: boot.wizardMetadata, integrationLabel: config.programId, diff --git a/src/agent/runner/harness/pi/index.ts b/src/agent/runner/harness/pi/index.ts index 90ea3ac54..17e9b6660 100644 --- a/src/agent/runner/harness/pi/index.ts +++ b/src/agent/runner/harness/pi/index.ts @@ -394,7 +394,7 @@ export const piBackend: AgentHarness = { agentDir: getAgentDir(), systemPrompt: assembleCommandments({ - program: runConfig.programId, + programCommandments: runConfig.programCommandments, sequence: Sequence.linear, harness: Harness.pi, caps: { bash: true, posthogMcp }, diff --git a/src/agent/runner/harness/pi/task.ts b/src/agent/runner/harness/pi/task.ts index 290950f40..6caf8e62d 100644 --- a/src/agent/runner/harness/pi/task.ts +++ b/src/agent/runner/harness/pi/task.ts @@ -340,7 +340,7 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { cwd: input.installDir, agentDir: getAgentDir(), systemPrompt: assembleCommandments({ - program: config.programId, + programCommandments: config.programCommandments, sequence: Sequence.orchestrator, harness: Harness.pi, caps: { bash: codingTools.has('bash'), posthogMcp }, diff --git a/src/agent/runner/sequence/README.md b/src/agent/runner/sequence/README.md index ac4436f9a..ce795d52b 100644 --- a/src/agent/runner/sequence/README.md +++ b/src/agent/runner/sequence/README.md @@ -49,6 +49,7 @@ frontmatter contract rather than copying an old manifest example: task types after planning. It is not a universal automatic graph built from frontmatter. -Per-role `PROGRAM_BINDINGS[id].contextMillOverride` can adjust a task's harness, -model or effort. New task types usually belong in context-mill; new native +Per-role routes in the resolved binding can adjust a task's harness, model or +effort. Programs resolve `PROGRAM_BINDINGS[id].contextMillOverride` before the +agent starts. New task types usually belong in context-mill; new native behavior still requires the appropriate Wizard configuration or implementation. diff --git a/src/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/agent/runner/sequence/orchestrator/orchestrator-runner.ts index a5d42f8ab..850453cb3 100644 --- a/src/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -43,10 +43,8 @@ import type { import { createEmitSpinner } from '../../shared/progress-collector'; import { createAskBridge } from '../../shared/ask'; import { - areSeededTasksEnabled, getHarness, - resolveHarness, - resolveStageOverrides, + resolveRoleHarness, type HarnessPick, } from '../../switchboard'; import { isValidModel, requireKnownModel } from '../../switchboard/models'; @@ -608,11 +606,6 @@ async function executeOrchestrator( const { run } = config; const programId = config.programId; - // Switchboard context — reused for every per-role harness resolution below. - // The caller resolved the run-level binding from it; per-task roles overlay - // `binding.contextMillOverride[role]` on the same inputs. - const switchboardCtx = { ...config.switchboard, trace: undefined }; - // The WHAT (agent prompts) is served from context-mill. Fetch the registry // once up front: its types drive enqueue validation, and resolving a task to // its run config is then synchronous, with no mid-drain network latency. @@ -620,11 +613,7 @@ async function executeOrchestrator( const registry = await loadAgentRegistry(boot.skillsBaseUrl, flow, { exclude: effectiveExcludedTaskTypes(config, boot.wizardFlags), // Baked into the prompts at load, so enqueue, dispatch, and telemetry all read one effective spec. - overrides: resolveStageOverrides( - programId, - boot.wizardFlags, - boot.wizardFlagPayloads, - ), + overrides: config.stageOverrides, }); if (signal?.aborted) return cancelledRun(); const seedPrompt = registry.seed; @@ -638,7 +627,7 @@ async function executeOrchestrator( const taskModels = Object.fromEntries( ['seed', ...registry.types].map((type) => { const prompt = type === 'seed' ? seedPrompt : registry.get(type); - const pick = resolveHarness(switchboardCtx, type); + const pick = resolveRoleHarness(config.binding, type); const specModel = prompt && promptModelFor(prompt, pick.harness).model; return [type, isValidModel(specModel) ? specModel : pick.model]; }), @@ -663,7 +652,7 @@ async function executeOrchestrator( const store = new QueueStore(input.installDir, runId, { onTransition: (event, task) => { - const pick = resolveHarness(switchboardCtx, task.type); + const pick = resolveRoleHarness(config.binding, task.type); // Mirror dispatch's allow-list fallback so attribution names the model that runs. const specModel = taskModelSpec(registry, task, pick.harness).model; const base = { @@ -880,7 +869,7 @@ async function executeOrchestrator( // depend on them, and no prompt has to remember they are there. // Kill switch: off (or unset), the wizard queues nothing itself and the run // is byte-identical to a project with no detected sources. - const seedEntries = areSeededTasksEnabled(boot.wizardFlags) + const seedEntries = config.seededTasksEnabled ? config.seedTasks?.() ?? [] : []; const seededTypes: string[] = []; @@ -997,7 +986,7 @@ async function executeOrchestrator( // Prompt-frontmatter model wins over the switchboard pick (§3.6 of the // switchboard plan) — the switchboard's model is the fallback when the // prompt is silent. - const seedPick = resolveHarness(switchboardCtx, 'seed'); + const seedPick = resolveRoleHarness(config.binding, 'seed'); const seedHarness = requireTaskHarness(seedPick); const seedModel = promptModelFor(seedPrompt, seedPick.harness); const seedResult = await seedHarness.runTask({ @@ -1207,10 +1196,9 @@ async function executeOrchestrator( // panel shows progress); errors still surface — the harness stops the // spinner with its own error text. // - // Per-task role = task.type — the switchboard consults - // PROGRAM_BINDINGS[id].contextMillOverride?.[task.type] for wizard-side - // per-agent overrides. Prompt-frontmatter model still wins (§3.6). - const taskPick = resolveHarness(switchboardCtx, task.type); + // Per-task role = task.type. Programs resolved any role override before + // invocation; prompt-frontmatter model still wins (§3.6). + const taskPick = resolveRoleHarness(config.binding, task.type); const taskHarness = requireTaskHarness(taskPick); const taskModel = taskModelSpec(registry, task, taskPick.harness); let taskResult: AgentResult; diff --git a/src/agent/runner/shared/types.ts b/src/agent/runner/shared/types.ts index b82f7af7b..9f2efd792 100644 --- a/src/agent/runner/shared/types.ts +++ b/src/agent/runner/shared/types.ts @@ -21,7 +21,6 @@ import type { ErrorCode } from '@shared/errors'; import type { LLMProvider } from '@posthog/warlock'; import type { AgentInteraction, ProgressEmitter } from '@agent/progress'; import type { EffortLevel } from '../switchboard/models'; -import type { SwitchboardCtx } from '../switchboard'; export type { PromptContext, Credentials }; @@ -139,6 +138,11 @@ export interface ResolvedBinding { model: string; /** Reasoning-effort override. Absent → the model's table default. */ thinkingLevel?: EffortLevel; + /** Role-specific routes resolved by the caller before the agent starts. */ + roleBindings?: Record< + string, + { harness: Harness; model: string; thinkingLevel?: EffortLevel } + >; } /** @@ -147,7 +151,7 @@ export interface ResolvedBinding { * treats every label as opaque. */ export interface RunConfig { - /** Program id: gateway spend pin, analytics label, commandments axis. */ + /** Opaque program label for gateway spend pin and analytics. */ programId: string; /** The run definition. A program's session-taking hooks are the caller's, see `hooks`. */ run: AgentRunDefinition; @@ -155,11 +159,12 @@ export interface RunConfig { composed: boolean; /** Run-level sequence, harness and model. */ binding: ResolvedBinding; - /** - * The inputs the run-level binding was resolved from. The orchestrator - * re-resolves the harness per task role from these; nothing else reads them. - */ - switchboard: SwitchboardCtx; + /** Program text selected by the caller; the agent only assembles it. */ + programCommandments?: readonly string[]; + /** Validated stage policy selected by the caller; absent keeps flow frontmatter. */ + stageOverrides?: Record; + /** The caller's resolved seeded-task experiment. */ + seededTasksEnabled?: boolean; /** Primary skills origin (context-mill dev or GitHub Releases). */ skillsBaseUrl: string; /** Feature flag key → variant, evaluated before the run. */ diff --git a/src/agent/runner/switchboard/commandments.ts b/src/agent/runner/switchboard/commandments.ts index 5f709b1fb..a170b271a 100644 --- a/src/agent/runner/switchboard/commandments.ts +++ b/src/agent/runner/switchboard/commandments.ts @@ -1,11 +1,8 @@ /** * System-prompt commandments, keyed by the axes the switchboard resolves. * - * A run is a resolved (program, sequence, harness, model). Guidance belongs to - * whichever axis makes it true, declared here beside the tables that resolve - * them, and assembled once by `assembleCommandments`. A rule that is true for - * every run stays in `@lib/agent/commandments`; anything narrower lives here so - * the call sites never re-derive it. + * Programs select their guidance before the run; the agent assembles that + * supplied text with global, sequence, model and harness guidance. * * Leaf module by design — it imports the axis enums and the per-axis text, never * a runner or a harness backend, so the harnesses can call the assembler without @@ -40,20 +37,6 @@ const SEQUENCE_COMMANDMENTS: Record = { [Sequence.orchestrator]: [], }; -// ── Program axis ──────────────────────────────────────────────────────── - -const SELF_DRIVING = [ - 'ALWAYS surface a custom-scout proposal in step 6b: bring the user your one or two strongest candidate scouts even when the built-in troop looks sufficient. The proposal ask leads with a "None — keep the built-in troop" option, so declining costs the user one keystroke — but a proposal you silently skip is coverage they never got to see or judge. Where the skill says to skip the ask when the gap analysis finds no candidate, do NOT skip: pick your best candidates anyway and let the user decide.', - - 'Rank candidates at the discriminator level, not the category level. "Covered" only means an enabled scout would actually FIRE for that failure mode: a conversion-rate watcher does not catch entry volume collapsing; a Stripe-transaction watcher does not catch a lead form going silent. A surface whose failure mode has no firing condition among the enabled scouts is your strongest candidate.', - - 'Be honest in the option descriptions: if a candidate overlaps something an enabled scout partially watches, say so in its description rather than dropping the candidate. The user chooses with full information; you do not gatekeep on their behalf.', -]; - -const PROGRAM_COMMANDMENTS: Record = { - 'self-driving': SELF_DRIVING, -}; - // ── Harness axis ──────────────────────────────────────────────────────── /** @@ -75,8 +58,8 @@ const MODEL_COMMANDMENTS: Record = {}; // ── Assembly ──────────────────────────────────────────────────────────── export interface CommandmentAxes { - /** Program id, as resolved into `PROGRAM_BINDINGS`. */ - program?: string; + /** Selected by programs; the agent only assembles supplied text. */ + programCommandments?: readonly string[]; sequence: Sequence; harness: Harness; /** Gateway model id. */ @@ -87,14 +70,14 @@ export interface CommandmentAxes { /** Every commandment this run's axes call for, broad to narrow. */ export function assembleCommandments(axes: CommandmentAxes): string { - const { program, sequence, harness, model, caps } = axes; + const { programCommandments, sequence, harness, model, caps } = axes; const harnessNotes = HARNESS_NOTES[harness]?.( sequence, caps ?? { bash: true, posthogMcp: true }, ); return [ ...WIZARD_COMMANDMENTS, - ...(program ? PROGRAM_COMMANDMENTS[program] ?? [] : []), + ...(programCommandments ?? []), ...SEQUENCE_COMMANDMENTS[sequence], ...(model ? MODEL_COMMANDMENTS[model] ?? [] : []), // Blank line first: the notes open their own `## This runtime` section. diff --git a/src/agent/runner/switchboard/harness.ts b/src/agent/runner/switchboard/harness.ts index 0f5673ed2..edfac1b59 100644 --- a/src/agent/runner/switchboard/harness.ts +++ b/src/agent/runner/switchboard/harness.ts @@ -8,10 +8,8 @@ import { logToFile } from '@utils/debug'; import { anthropicBackend } from '../harness/anthropic'; import { piBackend } from '../harness/pi'; import type { AgentHarness } from '../harness/types'; -import { resolveFlagRoute } from './flags'; import { DEFAULT_BINDING, - PROGRAM_BINDINGS, runChain, type HarnessPick, type Middleware, @@ -32,12 +30,11 @@ export function getHarness(name: Harness): AgentHarness { } /** - * PostHog-flag routing to pi (see `./flags`). No valid route — flag off, no - * config, or an invalid payload — keeps the non-flagged binding default. + * A validated caller-supplied route overlays the base binding. */ const flagRunnerOverride: Middleware = (ctx, next) => { const pick = next(); - const route = resolveFlagRoute(ctx.program, ctx.flags, ctx.flagPayloads); + const route = ctx.flagRoute; if (!route) return pick; if (ctx.trace) { ctx.trace.harness = 'flag'; @@ -86,7 +83,7 @@ export function resolveHarness( const pick = runChain(HARNESS_MIDDLEWARE, ctx, () => { if (ctx.trace) Object.assign(ctx.trace, { harness: 'binding', model: 'binding' }); - const binding = PROGRAM_BINDINGS[ctx.program] ?? DEFAULT_BINDING; + const binding = ctx.baseBinding ?? DEFAULT_BINDING; return { harness: binding.harness, model: binding.model, @@ -95,7 +92,9 @@ export function resolveHarness( }; }); logToFile( - `[switchboard] resolved: program=${ctx.program} harness=${pick.harness}` + + `[switchboard] resolved: program=${ctx.program ?? '?'} harness=${ + pick.harness + }` + `${ctx.trace?.harness ? ` (${ctx.trace.harness})` : ''} model=${ pick.model }` + @@ -103,3 +102,22 @@ export function resolveHarness( ); return pick; } + +/** The agent resolves a task role only from data the caller already supplied. */ +export function resolveRoleHarness( + binding: { + harness: Harness; + model: string; + thinkingLevel?: HarnessPick['thinkingLevel']; + roleBindings?: Record; + }, + role: string, +): HarnessPick { + return ( + binding.roleBindings?.[role] ?? { + harness: binding.harness, + model: binding.model, + thinkingLevel: binding.thinkingLevel, + } + ); +} diff --git a/src/agent/runner/switchboard/index.ts b/src/agent/runner/switchboard/index.ts index b2d84add2..a1a668ee0 100644 --- a/src/agent/runner/switchboard/index.ts +++ b/src/agent/runner/switchboard/index.ts @@ -1,13 +1,7 @@ // Resolves routing; model additions also require mint allowlists and gateway prompt/transport support. -import { - DEFAULT_AGENT_MODEL, - GPT5_6_SOL_MODEL, - GPT5_6_TERRA_MODEL, - Harness, - Sequence, -} from '@shared/constants'; -import type { ProgramId } from '@programs/types'; +import { Harness, Sequence } from '@shared/constants'; +import { DEFAULT_AGENT_BINDING } from '@agent/default-binding'; import { resolveHarness } from './harness'; import type { EffortLevel } from './models'; import { resolveSequence } from './sequence'; @@ -29,12 +23,22 @@ export interface SwitchboardTrace { /** Everything a resolver middleware may branch on. Built once per run. */ export interface SwitchboardCtx { - program: ProgramId; + /** Opaque log label. Program lookup stays with the caller. */ + program?: string; + /** The caller's selected base binding, before route/CLI overlays. */ + baseBinding?: ProgramBinding; /** Composed sub-run (a dependency inside a parent program). Structurally linear — no override can orchestrate it. */ composed?: boolean; - flags: Record; - /** Flag payloads from the same snapshot (payload-carrying flags, e.g. self-driving pi). */ - flagPayloads?: Record; + /** Already validated experiment route; no flag parsing happens in the agent. */ + flagRoute?: { + harness?: Harness; + model?: string; + thinkingLevel?: EffortLevel; + sequence?: Sequence; + }; + flagSequence?: Sequence; + /** Raw boolean only for the existing capability-clamp log line. */ + orchestratorFlagOn?: boolean; /** CLI override (`--harness`). Wins over `flags`. */ cliHarness?: Harness; /** CLI override (`--sequence`). Wins over `flags`. */ @@ -102,68 +106,8 @@ export interface ProgramBinding { contextMillOverride?: Record>; } -// Legacy fallback; new programs should explicitly choose Pi and prefer orchestration. -export const DEFAULT_BINDING: ProgramBinding = { - sequence: Sequence.linear, - harness: Harness.pi, - model: GPT5_6_SOL_MODEL, - thinkingLevel: 'medium', -}; - -/** - * Per-program routing. Kept in lockstep with `PROGRAM_REGISTRY` by the - * switchboard test. Anything absent falls back to `DEFAULT_BINDING`. - */ -export const PROGRAM_BINDINGS: Partial> = { - 'posthog-integration': DEFAULT_BINDING, - 'revenue-analytics-setup': DEFAULT_BINDING, - 'warehouse-source': DEFAULT_BINDING, - 'error-tracking-upload-source-maps': { - sequence: Sequence.linear, - harness: Harness.pi, - model: GPT5_6_SOL_MODEL, - thinkingLevel: 'medium', - }, - audit: DEFAULT_BINDING, - 'events-audit': DEFAULT_BINDING, - 'posthog-doctor': DEFAULT_BINDING, - 'web-analytics-doctor': DEFAULT_BINDING, - migration: DEFAULT_BINDING, - 'self-driving': DEFAULT_BINDING, - 'agent-skill': DEFAULT_BINDING, - 'mcp-add': DEFAULT_BINDING, - 'mcp-remove': DEFAULT_BINDING, - 'mcp-tutorial': DEFAULT_BINDING, - 'mcp-analytics': DEFAULT_BINDING, - // Orchestrator on pi. The binding routes only; every stage's model and - // effort are pinned context-mill side in the flow's frontmatter - // (`model_pi`/`effort_pi`: terra seed, sol tasks, luna report). - metrics: { - sequence: Sequence.orchestrator, - harness: Harness.pi, - model: DEFAULT_AGENT_MODEL, - }, - 'replay-vision': { - sequence: Sequence.orchestrator, - harness: Harness.anthropic, - model: DEFAULT_AGENT_MODEL, - }, - // Orchestrator on pi, like metrics. The binding routes only; every stage's - // model and effort are pinned context-mill side in the flow's frontmatter - // (`model_pi`/`effort_pi`: terra seed, install and init, sol tasks, luna report). - 'error-tracking': { - sequence: Sequence.orchestrator, - harness: Harness.pi, - model: DEFAULT_AGENT_MODEL, - }, - 'ai-observability': { - sequence: Sequence.linear, - harness: Harness.pi, - model: GPT5_6_TERRA_MODEL, - thinkingLevel: 'high', - }, - slack: DEFAULT_BINDING, -}; +/** Legacy alias until the public runner export is removed in B2 integration. */ +export const DEFAULT_BINDING: ProgramBinding = DEFAULT_AGENT_BINDING; // ── Unified resolver ──────────────────────────────────────────────────── @@ -186,8 +130,4 @@ export { resolveSequence, type SequenceRunner, } from './sequence'; -export { - isOrchestratorEnabled, - areSeededTasksEnabled, - resolveStageOverrides, -} from './flags'; +export { resolveRoleHarness } from './harness'; diff --git a/src/agent/runner/switchboard/sequence.ts b/src/agent/runner/switchboard/sequence.ts index b6d241720..09af8843a 100644 --- a/src/agent/runner/switchboard/sequence.ts +++ b/src/agent/runner/switchboard/sequence.ts @@ -1,23 +1,16 @@ /** - * Sequence axis: gate helpers, registry, middleware, resolver. - * Percentage rollouts are PostHog-side — the gate just reads the resolved bool. + * Sequence axis: registry, clamps and generic resolution of caller-supplied policy. */ import { IS_PRODUCTION_BUILD } from '@env'; import { Sequence } from '@shared/constants'; import { logToFile } from '@utils/debug'; -import { - isOrchestratorEnabled, - resolveFlagRoute, - resolveFlagSequence, -} from './flags'; import { getHarness, resolveHarness } from './harness'; import type { SequenceResult, SequenceContext } from '../shared/types'; import { runLinearProgram } from '../sequence/linear'; import { runOrchestrator } from '../sequence/orchestrator/orchestrator-runner'; import { DEFAULT_BINDING, - PROGRAM_BINDINGS, runChain, type Middleware, type SwitchboardCtx, @@ -70,17 +63,17 @@ const cliSequenceMw: Middleware = (ctx, next) => { return ctx.cliSequence; }; -/** A program's own flag route may pin the sequence; wins over the global orchestrator flag. Traced as 'payload' to stay distinguishable from sequence experiments ('flag'). */ +/** A caller-supplied payload route may pin the sequence. */ const flagRouteSequenceMw: Middleware = (ctx, next) => { - const route = resolveFlagRoute(ctx.program, ctx.flags, ctx.flagPayloads); + const route = ctx.flagRoute; if (!route?.sequence) return next(); if (ctx.trace) ctx.trace.sequence = 'payload'; return route.sequence; }; -/** Sequence experiments (e.g. wizard-orchestrator), each inert outside its declared programs. */ +/** A caller-supplied sequence experiment, already scoped to its program. */ const sequenceExperimentMw: Middleware = (ctx, next) => { - const sequence = resolveFlagSequence(ctx.program, ctx.flags); + const sequence = ctx.flagSequence; if (!sequence) return next(); if (ctx.trace) ctx.trace.sequence = 'flag'; return sequence; @@ -96,7 +89,7 @@ const sequenceExperimentMw: Middleware = (ctx, next) => { const runTaskCapabilityClampMw: Middleware = (ctx, next) => { const pick = resolveHarness(ctx); if (getHarness(pick.harness).runTask) return next(); - if (isOrchestratorEnabled(ctx.flags)) { + if (ctx.orchestratorFlagOn) { logToFile( `[switchboard] wizard-orchestrator ignored: ${pick.harness} has no runTask, clamping to linear`, ); @@ -119,11 +112,13 @@ const SEQUENCE_MIDDLEWARE: Middleware[] = [ export function resolveSequence(ctx: SwitchboardCtx): Sequence { const sequence = runChain(SEQUENCE_MIDDLEWARE, ctx, () => { if (ctx.trace) ctx.trace.sequence = 'binding'; - const binding = PROGRAM_BINDINGS[ctx.program] ?? DEFAULT_BINDING; + const binding = ctx.baseBinding ?? DEFAULT_BINDING; return binding.sequence; }); logToFile( - `[switchboard] resolved: program=${ctx.program} sequence=${sequence}` + + `[switchboard] resolved: program=${ + ctx.program ?? '?' + } sequence=${sequence}` + `${ctx.trace?.sequence ? ` (${ctx.trace.sequence})` : ''}`, ); return sequence; diff --git a/src/agent/types.ts b/src/agent/types.ts index e3476645b..2b4bd4352 100644 --- a/src/agent/types.ts +++ b/src/agent/types.ts @@ -12,8 +12,11 @@ export type { AgentRunDefinition, PromptContext, RunConfig, + ResolvedBinding, + RunFlags, RunInput, RunResult, + SeedTaskEntry, } from './runner'; export type { AgentInteraction, @@ -28,8 +31,9 @@ export type { TokenUsageDelta, } from './progress'; -/** Leaves in B2 with the bindings table; B1 deferred PROGRAM_BINDINGS. */ +/** Input types of the exported resolveBinding and resolveHarness. */ export type { ProgramBinding, SwitchboardCtx } from './runner'; +export type { EffortLevel } from './runner/switchboard/models'; /** Leaves in B2 with downloadSkill. */ export type { InstallSkillResult } from './tools'; diff --git a/src/agent/runner/__tests__/switchboard.test.ts b/src/programs/__tests__/switchboard.test.ts similarity index 97% rename from src/agent/runner/__tests__/switchboard.test.ts rename to src/programs/__tests__/switchboard.test.ts index 0a16a2048..d6efea02d 100644 --- a/src/agent/runner/__tests__/switchboard.test.ts +++ b/src/programs/__tests__/switchboard.test.ts @@ -2,7 +2,7 @@ * Switchboard machinery tests: binding registry lockstep, precedence chains, * trace stamping, model capabilities, and structural clamps. Per-experiment * flag behavior and cross-program isolation live in one file per experiment - * under `switchboard/flags/__tests__/`. + * under `experiments/__tests__/`. * * Every resolution test is a BindingCase: (SwitchboardCtx in) → (full * four-axis resolveBinding out), optionally pinning the trace. @@ -24,10 +24,10 @@ import { } from '@shared/constants'; import { PROGRAM_BINDINGS, - DEFAULT_BINDING, - resolveBinding, - type SwitchboardCtx, -} from '@agent/runner/switchboard'; + resolveProgramBinding as resolveBinding, +} from '../binding'; +import type { ProgramSwitchboardCtx as SwitchboardCtx } from '@programs/types'; +import { DEFAULT_AGENT_BINDING as DEFAULT_BINDING } from '@agent'; import { modelCapabilities, MINT_ALLOWED_EFFORTS, @@ -36,7 +36,7 @@ import { TRIAGE_MODELS, VALID_MODELS, } from '@agent/runner/switchboard/models'; -import { runBindingCases } from '@agent/runner/switchboard/flags/__tests__/binding-cases'; +import { runBindingCases } from '@programs/experiments/__tests__/binding-cases'; const PROGRAM_IDS = PROGRAM_REGISTRY.map((c) => c.id); const DEFAULT_RESOLVED = { diff --git a/src/agent/__tests__/variant-gating.test.ts b/src/programs/__tests__/variant-gating.test.ts similarity index 89% rename from src/agent/__tests__/variant-gating.test.ts rename to src/programs/__tests__/variant-gating.test.ts index 7efd2ca49..b0d6ed00e 100644 --- a/src/agent/__tests__/variant-gating.test.ts +++ b/src/programs/__tests__/variant-gating.test.ts @@ -1,4 +1,4 @@ -import { isOrchestratorEnabled } from '@agent/runner/switchboard'; +import { isOrchestratorEnabled } from '@programs/experiments'; describe('isOrchestratorEnabled', () => { it('is true only when the wizard-orchestrator flag is true', () => { diff --git a/src/programs/binding-telemetry.ts b/src/programs/binding-telemetry.ts new file mode 100644 index 000000000..fa6a53bef --- /dev/null +++ b/src/programs/binding-telemetry.ts @@ -0,0 +1,50 @@ +import { + Sequence, + WIZARD_ORCHESTRATOR_FLAG_KEY, + WIZARD_SELF_DRIVING_USE_PI_HARNESS_FLAG_KEY, +} from '@shared/constants'; +import type { ResolvedBinding } from '@agent/types'; +import { analytics } from '@utils/analytics'; +import { logToFile } from '@utils/debug'; +import type { ProgramSwitchboardCtx } from './binding'; + +/** Record the selected route and its precedence sources once per program run. */ +export function captureSwitchboardDecision( + ctx: ProgramSwitchboardCtx, + binding: ResolvedBinding, +): void { + const trace = ctx.trace ?? {}; + const perTaskModel = + binding.sequence === Sequence.orchestrator && trace.model === 'binding'; + const model = perTaskModel ? 'chosen-per-task' : binding.model; + const modelSource = perTaskModel ? 'agent-prompts' : trace.model; + analytics.wizardCapture('switchboard resolved', { + program: ctx.program, + flag_self_driving_use_pi_harness: + ctx.flags[WIZARD_SELF_DRIVING_USE_PI_HARNESS_FLAG_KEY], + flag_self_driving_pi_payload: JSON.stringify( + ctx.flagPayloads?.[WIZARD_SELF_DRIVING_USE_PI_HARNESS_FLAG_KEY] ?? null, + ), + flag_orchestrator: ctx.flags[WIZARD_ORCHESTRATOR_FLAG_KEY], + cli_harness: ctx.cliHarness, + cli_sequence: ctx.cliSequence, + cli_model: ctx.cliModel, + harness_source: trace.harness, + model_source: modelSource, + sequence_source: trace.sequence, + harness: binding.harness, + model, + thinking_level: binding.thinkingLevel, + sequence: binding.sequence, + }); + logToFile( + `[switchboard] decision: program=${ctx.program}` + + ` in(orchestrator=${ctx.flags[WIZARD_ORCHESTRATOR_FLAG_KEY] ?? '-'},` + + ` cli=${ctx.cliHarness ?? '-'}/${ctx.cliSequence ?? '-'}/${ + ctx.cliModel ?? '-' + })` + + ` → harness=${binding.harness} (${trace.harness ?? '?'})` + + ` model=${model} (${modelSource ?? '?'})` + + ` sequence=${binding.sequence} (${trace.sequence ?? '?'})`, + ); +} diff --git a/src/programs/binding.ts b/src/programs/binding.ts new file mode 100644 index 000000000..007058cf2 --- /dev/null +++ b/src/programs/binding.ts @@ -0,0 +1,109 @@ +import { + DEFAULT_AGENT_MODEL, + GPT5_6_SOL_MODEL, + GPT5_6_TERRA_MODEL, + Harness, + Sequence, +} from '@shared/constants'; +import { DEFAULT_AGENT_BINDING, resolveBinding, resolveHarness } from '@agent'; +import type { + ProgramBinding, + ResolvedBinding, + SwitchboardCtx, +} from '@agent/types'; +import type { ProgramId } from './program-registry'; +import { + isOrchestratorEnabled, + resolveFlagRoute, + resolveFlagSequence, +} from './experiments'; + +export interface ProgramSwitchboardCtx { + program: ProgramId; + composed?: boolean; + flags: Record; + flagPayloads?: Record; + cliHarness?: Harness; + cliSequence?: Sequence; + cliModel?: string; + trace?: SwitchboardCtx['trace']; +} + +/** Program routes. The registry lockstep contract is tested at this boundary. */ +export const PROGRAM_BINDINGS: Partial> = { + 'posthog-integration': DEFAULT_AGENT_BINDING, + 'revenue-analytics-setup': DEFAULT_AGENT_BINDING, + 'warehouse-source': DEFAULT_AGENT_BINDING, + 'error-tracking-upload-source-maps': { + sequence: Sequence.linear, + harness: Harness.pi, + model: GPT5_6_SOL_MODEL, + thinkingLevel: 'medium', + }, + audit: DEFAULT_AGENT_BINDING, + 'events-audit': DEFAULT_AGENT_BINDING, + 'posthog-doctor': DEFAULT_AGENT_BINDING, + 'web-analytics-doctor': DEFAULT_AGENT_BINDING, + migration: DEFAULT_AGENT_BINDING, + 'self-driving': DEFAULT_AGENT_BINDING, + 'agent-skill': DEFAULT_AGENT_BINDING, + 'mcp-add': DEFAULT_AGENT_BINDING, + 'mcp-remove': DEFAULT_AGENT_BINDING, + 'mcp-tutorial': DEFAULT_AGENT_BINDING, + 'mcp-analytics': DEFAULT_AGENT_BINDING, + metrics: { + sequence: Sequence.orchestrator, + harness: Harness.pi, + model: DEFAULT_AGENT_MODEL, + }, + 'replay-vision': { + sequence: Sequence.orchestrator, + harness: Harness.anthropic, + model: DEFAULT_AGENT_MODEL, + }, + 'error-tracking': { + sequence: Sequence.orchestrator, + harness: Harness.pi, + model: DEFAULT_AGENT_MODEL, + }, + 'ai-observability': { + sequence: Sequence.linear, + harness: Harness.pi, + model: GPT5_6_TERRA_MODEL, + thinkingLevel: 'high', + }, + slack: DEFAULT_AGENT_BINDING, +}; + +/** Resolve product policy once; the agent receives only the resulting route. */ +export function resolveProgramBinding( + ctx: ProgramSwitchboardCtx, +): ResolvedBinding { + ctx.trace ??= {}; + const baseBinding: ProgramBinding = + PROGRAM_BINDINGS[ctx.program] ?? DEFAULT_AGENT_BINDING; + const resolution = { + program: ctx.program, + baseBinding, + composed: ctx.composed, + flagRoute: resolveFlagRoute(ctx.program, ctx.flags, ctx.flagPayloads), + flagSequence: resolveFlagSequence(ctx.program, ctx.flags), + orchestratorFlagOn: isOrchestratorEnabled(ctx.flags), + cliHarness: ctx.cliHarness, + cliSequence: ctx.cliSequence, + cliModel: ctx.cliModel, + trace: ctx.trace, + }; + const binding = resolveBinding(resolution); + const roles = Object.keys(baseBinding.contextMillOverride ?? {}); + if (roles.length === 0) return binding; + return { + ...binding, + roleBindings: Object.fromEntries( + roles.map((role) => [ + role, + resolveHarness({ ...resolution, trace: undefined }, role), + ]), + ), + }; +} diff --git a/src/programs/commandments.ts b/src/programs/commandments.ts new file mode 100644 index 000000000..d89d267de --- /dev/null +++ b/src/programs/commandments.ts @@ -0,0 +1,18 @@ +// ── Program axis ──────────────────────────────────────────────────────── + +const SELF_DRIVING = [ + 'ALWAYS surface a custom-scout proposal in step 6b: bring the user your one or two strongest candidate scouts even when the built-in troop looks sufficient. The proposal ask leads with a "None — keep the built-in troop" option, so declining costs the user one keystroke — but a proposal you silently skip is coverage they never got to see or judge. Where the skill says to skip the ask when the gap analysis finds no candidate, do NOT skip: pick your best candidates anyway and let the user decide.', + + 'Rank candidates at the discriminator level, not the category level. "Covered" only means an enabled scout would actually FIRE for that failure mode: a conversion-rate watcher does not catch entry volume collapsing; a Stripe-transaction watcher does not catch a lead form going silent. A surface whose failure mode has no firing condition among the enabled scouts is your strongest candidate.', + + 'Be honest in the option descriptions: if a candidate overlaps something an enabled scout partially watches, say so in its description rather than dropping the candidate. The user chooses with full information; you do not gatekeep on their behalf.', +]; + +const PROGRAM_COMMANDMENTS: Record = { + 'self-driving': SELF_DRIVING, +}; + +/** Select program-specific guidance before invoking the agent. */ +export function getProgramCommandments(program: string): readonly string[] { + return PROGRAM_COMMANDMENTS[program] ?? []; +} diff --git a/src/agent/runner/switchboard/flags/__tests__/binding-cases.ts b/src/programs/experiments/__tests__/binding-cases.ts similarity index 91% rename from src/agent/runner/switchboard/flags/__tests__/binding-cases.ts rename to src/programs/experiments/__tests__/binding-cases.ts index 341a5d9a8..da7a31439 100644 --- a/src/agent/runner/switchboard/flags/__tests__/binding-cases.ts +++ b/src/programs/experiments/__tests__/binding-cases.ts @@ -5,11 +5,8 @@ */ import { describe, it, expect } from 'vitest'; import { GPT5_6_SOL_MODEL, Harness, Sequence } from '@shared/constants'; -import { - resolveBinding, - type SwitchboardCtx, - type SwitchboardTrace, -} from '@agent/runner/switchboard'; +import { resolveProgramBinding as resolveBinding } from '../../binding'; +import type { ProgramSwitchboardCtx as SwitchboardCtx } from '@programs/types'; import type { EffortLevel } from '@agent/runner/switchboard/models'; /** The complete resolved binding — every axis stated, nothing implicit. */ @@ -27,7 +24,7 @@ export interface BindingCase { ctx: Omit; binding: ExpectedBinding; /** Also pin which precedence rung decided each axis. */ - trace?: SwitchboardTrace; + trace?: SwitchboardCtx['trace']; } export function runBindingCases( diff --git a/src/agent/runner/switchboard/flags/__tests__/flags.test.ts b/src/programs/experiments/__tests__/flags.test.ts similarity index 93% rename from src/agent/runner/switchboard/flags/__tests__/flags.test.ts rename to src/programs/experiments/__tests__/flags.test.ts index 8c04f0983..e34adf86c 100644 --- a/src/agent/runner/switchboard/flags/__tests__/flags.test.ts +++ b/src/programs/experiments/__tests__/flags.test.ts @@ -15,17 +15,14 @@ import { WIZARD_ORCHESTRATOR_OVERRIDE_FLAG_KEY, WIZARD_SELF_DRIVING_USE_PI_HARNESS_FLAG_KEY, } from '@shared/constants'; -import { - areSeededTasksEnabled, - resolveBinding, - resolveStageOverrides, - type SwitchboardCtx, -} from '@agent/runner/switchboard'; +import { resolveProgramBinding as resolveBinding } from '../../binding'; +import type { ProgramSwitchboardCtx as SwitchboardCtx } from '@programs/types'; +import { areSeededTasksEnabled, resolveStageOverrides } from '..'; import { ORCHESTRATOR_SEQUENCE_ROUTE, ORCHESTRATOR_HARNESS_ROUTE, -} from '@agent/runner/switchboard/flags/orchestrator'; -import { SELF_DRIVING_EXPERIMENT } from '@agent/runner/switchboard/flags/self-driving'; +} from '@programs/experiments/orchestrator'; +import { SELF_DRIVING_EXPERIMENT } from '@programs/experiments/self-driving'; import { runBindingCases } from './binding-cases'; const envState = vi.hoisted(() => ({ @@ -332,22 +329,17 @@ describe('isolation — everything on at once', () => { }); }); -describe('seam scan — routing reads live only in flags/', () => { - const switchboardDir = join( - dirname(fileURLToPath(import.meta.url)), - '..', - '..', - ); - // orchestrator-runner consumes a flags/ resolver; it may pass the snapshot through, never index it. +describe('seam scan — routing reads live only in experiments/', () => { + const programsDir = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); for (const file of [ - 'harness.ts', - 'sequence.ts', - 'models.ts', - 'index.ts', - '../sequence/orchestrator/orchestrator-runner.ts', + '../agent/runner/switchboard/harness.ts', + '../agent/runner/switchboard/sequence.ts', + '../agent/runner/switchboard/models.ts', + '../agent/runner/switchboard/index.ts', + '../agent/runner/sequence/orchestrator/orchestrator-runner.ts', ]) { it(`${file} contains no direct flag reads or flag-key imports`, () => { - const src = readFileSync(join(switchboardDir, file), 'utf8'); + const src = readFileSync(join(programsDir, file), 'utf8'); expect(src).not.toMatch(/ctx\.flags\[/); expect(src).not.toMatch(/flags\[['"`]/); expect(src).not.toMatch(/WIZARD_\w+_FLAG_KEY/); diff --git a/src/agent/runner/switchboard/flags/index.ts b/src/programs/experiments/index.ts similarity index 100% rename from src/agent/runner/switchboard/flags/index.ts rename to src/programs/experiments/index.ts diff --git a/src/agent/runner/switchboard/flags/orchestrator.ts b/src/programs/experiments/orchestrator.ts similarity index 100% rename from src/agent/runner/switchboard/flags/orchestrator.ts rename to src/programs/experiments/orchestrator.ts diff --git a/src/agent/runner/switchboard/flags/schemes.ts b/src/programs/experiments/schemes.ts similarity index 99% rename from src/agent/runner/switchboard/flags/schemes.ts rename to src/programs/experiments/schemes.ts index 99f9c4a97..1fd30ce34 100644 --- a/src/agent/runner/switchboard/flags/schemes.ts +++ b/src/programs/experiments/schemes.ts @@ -15,7 +15,7 @@ import { } from '@shared/constants'; import type { ProgramId } from '@programs/types'; import { logToFile } from '@utils/debug'; -import type { EffortLevel } from '../models'; +import type { EffortLevel } from '@agent/types'; // ── Shared vocabulary ───────────────────────────────────────────────────── diff --git a/src/agent/runner/switchboard/flags/self-driving.ts b/src/programs/experiments/self-driving.ts similarity index 100% rename from src/agent/runner/switchboard/flags/self-driving.ts rename to src/programs/experiments/self-driving.ts diff --git a/src/programs/run-agent-legacy.ts b/src/programs/run-agent-legacy.ts index b343cccf6..8e2174986 100644 --- a/src/programs/run-agent-legacy.ts +++ b/src/programs/run-agent-legacy.ts @@ -21,17 +21,15 @@ import { createUiReducer, uiInteraction } from '@ui/agent-progress'; import { buildRunTags, flushScanReport, - resolveBinding, runAgent, RunOutcome, TASK_OUTCOMES_KEY, } from '@agent'; -import type { - ProgramBinding, - RunConfig, - RunInput, - SwitchboardCtx, -} from '@agent/types'; +import type { RunConfig, RunInput } from '@agent/types'; +import { resolveProgramBinding, type ProgramSwitchboardCtx } from './binding'; +import { getProgramCommandments } from './commandments'; +import { captureSwitchboardDecision } from './binding-telemetry'; +import { areSeededTasksEnabled, resolveStageOverrides } from './experiments'; import type { ProgramRun } from './program-run'; import { backupAndFixClaudeSettings, @@ -53,8 +51,6 @@ import { isNonInteractiveEnvironment } from '@utils/environment'; import { getSkillsBaseUrl, Sequence, - WIZARD_ORCHESTRATOR_FLAG_KEY, - WIZARD_SELF_DRIVING_USE_PI_HARNESS_FLAG_KEY, type Integration, } from '@shared/constants'; import { FRAMEWORK_REGISTRY } from '@programs/registry'; @@ -181,7 +177,7 @@ async function runProgram( // Resolve which sequence and harness will run a program (CLI → PostHog flag → // per-program binding → default), tag both axes onto analytics, and hand the // binding to the agent for dispatch. - const switchboard: SwitchboardCtx = { + const switchboard: ProgramSwitchboardCtx = { program: programConfig.id, composed, flags: wizardFlags, @@ -190,7 +186,7 @@ async function runProgram( cliSequence: session.sequence, cliModel: session.model, }; - const binding = resolveBinding(switchboard); + const binding = resolveProgramBinding(switchboard); analytics.setTag('sequence', binding.sequence); analytics.setTag('harness', binding.harness); wizardMetadata.SEQUENCE = binding.sequence; @@ -221,7 +217,13 @@ async function runProgram( run, composed, binding, - switchboard, + programCommandments: getProgramCommandments(programConfig.id), + stageOverrides: resolveStageOverrides( + programConfig.id, + wizardFlags, + wizardFlagPayloads, + ), + seededTasksEnabled: areSeededTasksEnabled(wizardFlags), skillsBaseUrl: getSkillsBaseUrl(), wizardFlags, wizardFlagPayloads, @@ -440,50 +442,3 @@ async function runSettingsGate(session: WizardSession): Promise { logToFile('[agent-runner] settings override resolved'); } } - -// ── Switchboard telemetry ───────────────────────────────────────────── - -/** - * One event + one log line per run: what entered the switchboard, which - * precedence rung decided each axis, and the final pick. - */ -function captureSwitchboardDecision( - ctx: SwitchboardCtx, - binding: ProgramBinding, -): void { - const trace = ctx.trace ?? {}; - // Unpinned orchestrator runs choose a model per task from the context-mill agent prompts; the orchestrator logs that map once the prompts load. - const perTaskModel = - binding.sequence === Sequence.orchestrator && trace.model === 'binding'; - const model = perTaskModel ? 'chosen-per-task' : binding.model; - const modelSource = perTaskModel ? 'agent-prompts' : trace.model; - analytics.wizardCapture('switchboard resolved', { - program: ctx.program, - flag_self_driving_use_pi_harness: - ctx.flags[WIZARD_SELF_DRIVING_USE_PI_HARNESS_FLAG_KEY], - flag_self_driving_pi_payload: JSON.stringify( - ctx.flagPayloads?.[WIZARD_SELF_DRIVING_USE_PI_HARNESS_FLAG_KEY] ?? null, - ), - flag_orchestrator: ctx.flags[WIZARD_ORCHESTRATOR_FLAG_KEY], - cli_harness: ctx.cliHarness, - cli_sequence: ctx.cliSequence, - cli_model: ctx.cliModel, - harness_source: trace.harness, - model_source: modelSource, - sequence_source: trace.sequence, - harness: binding.harness, - model, - thinking_level: binding.thinkingLevel, - sequence: binding.sequence, - }); - logToFile( - `[switchboard] decision: program=${ctx.program}` + - ` in(orchestrator=${ctx.flags[WIZARD_ORCHESTRATOR_FLAG_KEY] ?? '-'},` + - ` cli=${ctx.cliHarness ?? '-'}/${ctx.cliSequence ?? '-'}/${ - ctx.cliModel ?? '-' - })` + - ` → harness=${binding.harness} (${trace.harness ?? '?'})` + - ` model=${model} (${modelSource ?? '?'})` + - ` sequence=${binding.sequence} (${trace.sequence ?? '?'})`, - ); -} diff --git a/src/programs/types.ts b/src/programs/types.ts index 620e94d18..5ac379afe 100644 --- a/src/programs/types.ts +++ b/src/programs/types.ts @@ -7,3 +7,4 @@ export type { StoreInitContext, } from './program-step'; export type { FrameworkConfig, SetupQuestion } from './framework-config'; +export type { ProgramSwitchboardCtx } from './binding'; From 09c681b9bd629cfa1e7edd0cb87fffd9750bf34b Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 15:43:04 -0400 Subject: [PATCH 09/10] refactor(agent): move the harness resolver into resolve-harness resolveHarness and resolveRoleHarness move from switchboard/harness.ts to switchboard/resolve-harness.ts, so harness.ts holds only the registry. The three-step middleware chain becomes straight-line code with the same precedence: binding, then the flag route, then the dev-build CLI model and harness overrides. The traces and log line are unchanged. HARNESS_RUNS_TASKS records which backends implement runTask, and the sequence capability clamp reads it through harnessRunsTasks instead of asking the live registry. Both backends implement runTask, so the clamp decides the same way. A registry test keeps the table in step with HARNESS_OPTIONS. The DEFAULT_BINDING alias goes, and the sequence resolver reads DEFAULT_AGENT_BINDING. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- .../__tests__/harness-capabilities.test.ts | 16 +++ .../__tests__/run-agent-standalone.test.ts | 10 -- src/agent/index.ts | 2 +- src/agent/runner/switchboard/harness.ts | 105 +----------------- src/agent/runner/switchboard/index.ts | 14 +-- .../runner/switchboard/resolve-harness.ts | 90 +++++++++++++++ src/agent/runner/switchboard/sequence.ts | 18 ++- 7 files changed, 123 insertions(+), 132 deletions(-) create mode 100644 src/agent/__tests__/harness-capabilities.test.ts create mode 100644 src/agent/runner/switchboard/resolve-harness.ts diff --git a/src/agent/__tests__/harness-capabilities.test.ts b/src/agent/__tests__/harness-capabilities.test.ts new file mode 100644 index 000000000..d491f90d0 --- /dev/null +++ b/src/agent/__tests__/harness-capabilities.test.ts @@ -0,0 +1,16 @@ +import { Harness } from '@shared/constants'; +import { HARNESS_OPTIONS } from '@agent/runner/switchboard/harness'; +import { HARNESS_RUNS_TASKS } from '@agent/runner/switchboard/resolve-harness'; + +describe('harness capabilities', () => { + it.each(Object.values(Harness))( + 'records whether the %s backend implements runTask', + (harness) => { + const backend = HARNESS_OPTIONS[harness]; + expect(backend).toBeDefined(); + expect(HARNESS_RUNS_TASKS[harness]).toBe( + typeof backend?.runTask === 'function', + ); + }, + ); +}); diff --git a/src/agent/__tests__/run-agent-standalone.test.ts b/src/agent/__tests__/run-agent-standalone.test.ts index 19e75663a..24c3f39a8 100644 --- a/src/agent/__tests__/run-agent-standalone.test.ts +++ b/src/agent/__tests__/run-agent-standalone.test.ts @@ -171,16 +171,6 @@ vi.mock('@agent/runner/switchboard/harness', () => { harnessState.selected.push(name); return { ...fake, name }; }, - resolveHarness: (ctx: { cliHarness?: Harness }) => ({ - harness: ctx.cliHarness ?? Harness.pi, - model: DEFAULT_AGENT_MODEL, - }), - resolveRoleHarness: (binding: RunConfig['binding'], role: string) => - binding.roleBindings?.[role] ?? { - harness: binding.harness, - model: binding.model, - thinkingLevel: binding.thinkingLevel, - }, }; }); diff --git a/src/agent/index.ts b/src/agent/index.ts index 577ffa776..7b2c47553 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -19,7 +19,7 @@ export { runAgent, RunOutcome } from './runner'; export { AgentSignals } from './agent-interface'; export { WIZARD_TOOL_NAMES } from './tools'; export { DEFAULT_AGENT_BINDING } from './default-binding'; -export { resolveHarness } from './runner/switchboard'; +export { resolveHarness } from './runner/switchboard/resolve-harness'; /** * Leaves in B2. resolveBinding applies the generic precedence to the base diff --git a/src/agent/runner/switchboard/harness.ts b/src/agent/runner/switchboard/harness.ts index edfac1b59..b9078ac34 100644 --- a/src/agent/runner/switchboard/harness.ts +++ b/src/agent/runner/switchboard/harness.ts @@ -1,20 +1,12 @@ /** - * Harness axis: registry, middleware, resolver. Mirrors `sequence.ts`. + * Harness axis: the registry. Mirrors `sequence.ts`; the resolver is + * `resolve-harness.ts`. */ -import { IS_PRODUCTION_BUILD } from '@env'; import { Harness } from '@shared/constants'; -import { logToFile } from '@utils/debug'; import { anthropicBackend } from '../harness/anthropic'; import { piBackend } from '../harness/pi'; import type { AgentHarness } from '../harness/types'; -import { - DEFAULT_BINDING, - runChain, - type HarnessPick, - type Middleware, - type SwitchboardCtx, -} from '.'; export const HARNESS_OPTIONS: Partial> = { [Harness.anthropic]: anthropicBackend, @@ -28,96 +20,3 @@ export function getHarness(name: Harness): AgentHarness { } return harness; } - -/** - * A validated caller-supplied route overlays the base binding. - */ -const flagRunnerOverride: Middleware = (ctx, next) => { - const pick = next(); - const route = ctx.flagRoute; - if (!route) return pick; - if (ctx.trace) { - ctx.trace.harness = 'flag'; - // Harness-only routes keep the binding's model — trace it truthfully so - // analytics never attributes the fallback model to the flag. - if (route.model) ctx.trace.model = 'flag'; - } - return { - harness: route.harness ?? Harness.pi, - model: route.model ?? pick.model, - thinkingLevel: route.thinkingLevel ?? pick.thinkingLevel, - }; -}; - -/** `--harness` override. Dev/test only — the option is gated out of published builds. */ -const cliHarnessOverride: Middleware = (ctx, next) => { - const pick = next(); - if (!ctx.cliHarness) return pick; - if (ctx.trace) ctx.trace.harness = 'cli'; - return { ...pick, harness: ctx.cliHarness }; -}; - -/** `--model` override. Dev/test only — the option is gated out of published builds. */ -const cliModelOverride: Middleware = (ctx, next) => { - const pick = next(); - if (!ctx.cliModel) return pick; - if (ctx.trace) ctx.trace.model = 'cli'; - return { ...pick, model: ctx.cliModel }; -}; - -// Order = precedence: CLI > flag > binding default. The prod spread collapses -// to [], dropping the CLI overrides from the chain. -const HARNESS_MIDDLEWARE: Middleware[] = [ - ...(IS_PRODUCTION_BUILD ? [] : [cliHarnessOverride, cliModelOverride]), - flagRunnerOverride, -]; - -/** - * Resolve the harness for a role. Linear callers omit `role`; orchestrator - * callers pass `'seed'` or `task.type`. `contextMillOverride[role]` overlays. - */ -export function resolveHarness( - ctx: SwitchboardCtx, - role = 'default', -): HarnessPick { - const pick = runChain(HARNESS_MIDDLEWARE, ctx, () => { - if (ctx.trace) - Object.assign(ctx.trace, { harness: 'binding', model: 'binding' }); - const binding = ctx.baseBinding ?? DEFAULT_BINDING; - return { - harness: binding.harness, - model: binding.model, - thinkingLevel: binding.thinkingLevel, - ...binding.contextMillOverride?.[role], - }; - }); - logToFile( - `[switchboard] resolved: program=${ctx.program ?? '?'} harness=${ - pick.harness - }` + - `${ctx.trace?.harness ? ` (${ctx.trace.harness})` : ''} model=${ - pick.model - }` + - `${ctx.trace?.model ? ` (${ctx.trace.model})` : ''}`, - ); - return pick; -} - -/** The agent resolves a task role only from data the caller already supplied. */ -export function resolveRoleHarness( - binding: { - harness: Harness; - model: string; - thinkingLevel?: HarnessPick['thinkingLevel']; - roleBindings?: Record; - }, - role: string, -): HarnessPick { - return ( - binding.roleBindings?.[role] ?? { - harness: binding.harness, - model: binding.model, - thinkingLevel: binding.thinkingLevel, - } - ); -} diff --git a/src/agent/runner/switchboard/index.ts b/src/agent/runner/switchboard/index.ts index a1a668ee0..aff3b7716 100644 --- a/src/agent/runner/switchboard/index.ts +++ b/src/agent/runner/switchboard/index.ts @@ -1,9 +1,8 @@ // Resolves routing; model additions also require mint allowlists and gateway prompt/transport support. import { Harness, Sequence } from '@shared/constants'; -import { DEFAULT_AGENT_BINDING } from '@agent/default-binding'; -import { resolveHarness } from './harness'; import type { EffortLevel } from './models'; +import { resolveHarness } from './resolve-harness'; import { resolveSequence } from './sequence'; // ── Shared machinery ──────────────────────────────────────────────────── @@ -106,9 +105,6 @@ export interface ProgramBinding { contextMillOverride?: Record>; } -/** Legacy alias until the public runner export is removed in B2 integration. */ -export const DEFAULT_BINDING: ProgramBinding = DEFAULT_AGENT_BINDING; - // ── Unified resolver ──────────────────────────────────────────────────── /** Compose both axes. Callers needing only one axis use the per-axis resolver. */ @@ -123,11 +119,15 @@ export function resolveBinding( } // ── Unified re-export surface ─────────────────────────────────────────── -export { HARNESS_OPTIONS, getHarness, resolveHarness } from './harness'; +export { HARNESS_OPTIONS, getHarness } from './harness'; +export { + harnessRunsTasks, + resolveHarness, + resolveRoleHarness, +} from './resolve-harness'; export { SEQUENCE_OPTIONS, getSequence, resolveSequence, type SequenceRunner, } from './sequence'; -export { resolveRoleHarness } from './harness'; diff --git a/src/agent/runner/switchboard/resolve-harness.ts b/src/agent/runner/switchboard/resolve-harness.ts new file mode 100644 index 000000000..af47f4fa5 --- /dev/null +++ b/src/agent/runner/switchboard/resolve-harness.ts @@ -0,0 +1,90 @@ +/** + * Harness axis: the resolver that picks a harness and model, and which + * harnesses the orchestrator can drive. Data and pure functions only, so the + * agent entry loads this at startup; the registry is `harness.ts`. + */ + +import { IS_PRODUCTION_BUILD } from '@env'; +import { Harness } from '@shared/constants'; +import { logToFile } from '@utils/debug'; +import { DEFAULT_AGENT_BINDING } from '@agent/default-binding'; +import type { ResolvedBinding } from '../shared/types'; +import type { HarnessPick, ProgramBinding, SwitchboardCtx } from '.'; + +/** Which backends implement `runTask`; a registry test keeps this in step with HARNESS_OPTIONS. */ +export const HARNESS_RUNS_TASKS: Record = { + [Harness.anthropic]: true, + [Harness.pi]: true, +}; + +/** Whether the orchestrator can drive this harness. */ +export function harnessRunsTasks(name: Harness): boolean { + return HARNESS_RUNS_TASKS[name] === true; +} + +/** + * Resolve the harness for a role. Linear callers omit `role`; orchestrator + * callers pass `'seed'` or `task.type`. `contextMillOverride[role]` overlays. + * Precedence: CLI > flag route > binding default; the CLI overrides are + * dev/test only and gated out of published builds. + */ +export function resolveHarness( + ctx: SwitchboardCtx, + role = 'default', +): HarnessPick { + if (ctx.trace) + Object.assign(ctx.trace, { harness: 'binding', model: 'binding' }); + const binding: ProgramBinding = ctx.baseBinding ?? DEFAULT_AGENT_BINDING; + let pick: HarnessPick = { + harness: binding.harness, + model: binding.model, + thinkingLevel: binding.thinkingLevel, + ...binding.contextMillOverride?.[role], + }; + const route = ctx.flagRoute; + if (route) { + if (ctx.trace) { + ctx.trace.harness = 'flag'; + // Harness-only routes keep the binding's model — trace it truthfully so + // analytics never attributes the fallback model to the flag. + if (route.model) ctx.trace.model = 'flag'; + } + pick = { + harness: route.harness ?? Harness.pi, + model: route.model ?? pick.model, + thinkingLevel: route.thinkingLevel ?? pick.thinkingLevel, + }; + } + if (!IS_PRODUCTION_BUILD && ctx.cliModel) { + if (ctx.trace) ctx.trace.model = 'cli'; + pick = { ...pick, model: ctx.cliModel }; + } + if (!IS_PRODUCTION_BUILD && ctx.cliHarness) { + if (ctx.trace) ctx.trace.harness = 'cli'; + pick = { ...pick, harness: ctx.cliHarness }; + } + logToFile( + `[switchboard] resolved: program=${ctx.program ?? '?'} harness=${ + pick.harness + }` + + `${ctx.trace?.harness ? ` (${ctx.trace.harness})` : ''} model=${ + pick.model + }` + + `${ctx.trace?.model ? ` (${ctx.trace.model})` : ''}`, + ); + return pick; +} + +/** The agent resolves a task role only from data the caller already supplied. */ +export function resolveRoleHarness( + binding: ResolvedBinding, + role: string, +): HarnessPick { + return ( + binding.roleBindings?.[role] ?? { + harness: binding.harness, + model: binding.model, + thinkingLevel: binding.thinkingLevel, + } + ); +} diff --git a/src/agent/runner/switchboard/sequence.ts b/src/agent/runner/switchboard/sequence.ts index 09af8843a..8b3eb7792 100644 --- a/src/agent/runner/switchboard/sequence.ts +++ b/src/agent/runner/switchboard/sequence.ts @@ -5,16 +5,12 @@ import { IS_PRODUCTION_BUILD } from '@env'; import { Sequence } from '@shared/constants'; import { logToFile } from '@utils/debug'; -import { getHarness, resolveHarness } from './harness'; +import { DEFAULT_AGENT_BINDING } from '@agent/default-binding'; +import { harnessRunsTasks, resolveHarness } from './resolve-harness'; import type { SequenceResult, SequenceContext } from '../shared/types'; import { runLinearProgram } from '../sequence/linear'; import { runOrchestrator } from '../sequence/orchestrator/orchestrator-runner'; -import { - DEFAULT_BINDING, - runChain, - type Middleware, - type SwitchboardCtx, -} from '.'; +import { runChain, type Middleware, type SwitchboardCtx } from '.'; // ── Registry ──────────────────────────────────────────────────────────── @@ -82,13 +78,13 @@ const sequenceExperimentMw: Middleware = (ctx, next) => { /** * The orchestrator drives harnesses through `runTask`; a harness that has not * implemented it clamps the run to linear. A capability check, not a harness - * identity check — a harness gains orchestrator support by implementing the - * method, with no switchboard change. Sits below the CLI override so + * identity check — `HARNESS_RUNS_TASKS` records which backends implement the + * method, and a registry test keeps it in step. Sits below the CLI override so * `--sequence orchestrator` still reproduces the hard error in dev builds. */ const runTaskCapabilityClampMw: Middleware = (ctx, next) => { const pick = resolveHarness(ctx); - if (getHarness(pick.harness).runTask) return next(); + if (harnessRunsTasks(pick.harness)) return next(); if (ctx.orchestratorFlagOn) { logToFile( `[switchboard] wizard-orchestrator ignored: ${pick.harness} has no runTask, clamping to linear`, @@ -112,7 +108,7 @@ const SEQUENCE_MIDDLEWARE: Middleware[] = [ export function resolveSequence(ctx: SwitchboardCtx): Sequence { const sequence = runChain(SEQUENCE_MIDDLEWARE, ctx, () => { if (ctx.trace) ctx.trace.sequence = 'binding'; - const binding = ctx.baseBinding ?? DEFAULT_BINDING; + const binding = ctx.baseBinding ?? DEFAULT_AGENT_BINDING; return binding.sequence; }); logToFile( From ef5ee439bf9186f7b8d0414c30fd84f1d6a2f9d2 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Thu, 24 Sep 2026 15:45:40 -0400 Subject: [PATCH 10/10] refactor(programs): move the sequence precedence into binding resolveProgramBinding now resolves the sequence itself, so programs own the whole binding precedence. The agent keeps the harness axis. The middleware chain becomes straight-line code with the same order: the composed clamp, the dev-build CLI override, the runTask capability clamp, the flag route, the sequence experiment, then the base binding. The traces and log lines are unchanged. The agent's switchboard keeps the registries, resolveHarness, resolveRoleHarness and harnessRunsTasks. It drops resolveBinding, resolveSequence, runChain, and the flagSequence and orchestratorFlagOn context fields, which only programs read. @agent exports harnessRunsTasks beside resolveHarness in place of resolveBinding. Generated-By: PostHog Desktop Task-Id: d14e92bb-6ee1-49b5-8502-39cb80079589 --- src/agent/README.md | 2 +- src/agent/index.ts | 15 ++-- src/agent/runner/README.md | 17 ++--- src/agent/runner/index.ts | 1 - src/agent/runner/switchboard/index.ts | 60 +--------------- src/agent/runner/switchboard/sequence.ts | 88 +----------------------- src/agent/types.ts | 2 +- src/programs/binding.ts | 57 ++++++++++++++- 8 files changed, 77 insertions(+), 165 deletions(-) diff --git a/src/agent/README.md b/src/agent/README.md index a3c188057..90352d0cf 100644 --- a/src/agent/README.md +++ b/src/agent/README.md @@ -56,7 +56,7 @@ runAgent(config: RunConfig, input: RunInput, options?: { is `success`, `Aborted` is `cancelled`, `Failed` and `Crashed` are `error`. Other runtime exports: `DEFAULT_AGENT_BINDING` for standalone callers, -`resolveHarness` and `resolveBinding`, which programs resolve a binding with, +`resolveHarness` and `harnessRunsTasks`, which programs resolve a binding with, `shouldDisableAsk`, `initializeAgent`, `executeAgent`, `buildRunTags`, `AgentSignals`, `configureGatewayFromCIEnvironment`, `downloadSkill`, `WIZARD_TOOL_NAMES`, `LONGER_ASK_TIMEOUT_MS`, `flushScanReport`, and diff --git a/src/agent/index.ts b/src/agent/index.ts index 7b2c47553..2878fa23c 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -12,22 +12,23 @@ * Stays. The agent's contract: the one way to run it, the marker strings * program prompts embed, the tool ids programs put in allowedTools and * disallowedTools, and what programs resolve a binding with: the default - * binding and the harness axis. + * binding, the harness axis and each harness's task capability. */ export type * from './types'; export { runAgent, RunOutcome } from './runner'; export { AgentSignals } from './agent-interface'; export { WIZARD_TOOL_NAMES } from './tools'; export { DEFAULT_AGENT_BINDING } from './default-binding'; -export { resolveHarness } from './runner/switchboard/resolve-harness'; +export { + harnessRunsTasks, + resolveHarness, +} from './runner/switchboard/resolve-harness'; /** - * Leaves in B2. resolveBinding applies the generic precedence to the base - * binding and routes programs select, and programs/binding.ts is its one - * caller; shouldDisableAsk is a flags policy programs decide and pass in; - * LONGER_ASK_TIMEOUT_MS is a tuning number programs own as askTimeoutMs. + * Leaves in B2. shouldDisableAsk is a flags policy programs decide and pass + * in; LONGER_ASK_TIMEOUT_MS is a tuning number programs own as askTimeoutMs. */ -export { resolveBinding, shouldDisableAsk } from './runner'; +export { shouldDisableAsk } from './runner'; export { LONGER_ASK_TIMEOUT_MS } from './wizard-ask-bridge'; /** diff --git a/src/agent/runner/README.md b/src/agent/runner/README.md index 611be4c79..afd87744d 100644 --- a/src/agent/runner/README.md +++ b/src/agent/runner/README.md @@ -52,14 +52,15 @@ The gates, OAuth, flags and binding lookup that used to run here live in targets, the gateway mint and the scan-triage classifier. Whether the run turns out to be linear or orchestrator, anthropic or pi, the setup is the same. -**The switchboard** (`switchboard/`) is the router. Given a program id + the -fetched flags + any CLI overrides, it returns a `ProgramBinding` — which query -shape (sequence), which agent SDK (harness), which model. Two independent -middleware chains, one per axis, apply precedence rules (CLI > flag > program -config > default). This is the only layer that makes routing decisions. - -**Sequences** (`sequence/`) are LLM query shapes. Once the switchboard has -picked one, that sequence takes over the run and owns _how the LLM's work is +**The switchboard** (`switchboard/`) holds the sequence and harness registries +and the harness axis. Programs resolve the binding (`resolveProgramBinding`): +the sequence precedence lives there, and the harness and model come from this +layer's `resolveHarness` (CLI > flag > program config > +default). `harnessRunsTasks` tells programs which harnesses the orchestrator can +drive. + +**Sequences** (`sequence/`) are LLM query shapes. Once the binding has picked +one, that sequence takes over the run and owns _how the LLM's work is shaped_. See `sequence/README.md`. - **linear** — one long conversation with the model, start to finish. diff --git a/src/agent/runner/index.ts b/src/agent/runner/index.ts index 91d35dedc..0544bffb3 100644 --- a/src/agent/runner/index.ts +++ b/src/agent/runner/index.ts @@ -62,7 +62,6 @@ export type { ProgressEmitter, } from '@agent/progress'; export { shouldDisableAsk } from './shared/bootstrap'; -export { resolveBinding } from './switchboard'; export type { ProgramBinding, SwitchboardCtx } from './switchboard'; export { TASK_OUTCOMES_KEY } from './sequence/orchestrator/queue'; export type { TaskOutcome } from './sequence/orchestrator/queue'; diff --git a/src/agent/runner/switchboard/index.ts b/src/agent/runner/switchboard/index.ts index aff3b7716..7b6203f81 100644 --- a/src/agent/runner/switchboard/index.ts +++ b/src/agent/runner/switchboard/index.ts @@ -2,12 +2,10 @@ import { Harness, Sequence } from '@shared/constants'; import type { EffortLevel } from './models'; -import { resolveHarness } from './resolve-harness'; -import { resolveSequence } from './sequence'; // ── Shared machinery ──────────────────────────────────────────────────── -/** Which precedence rung decided each axis. Stamped by middlewares as they assert. */ +/** Which precedence rung decided each axis. Stamped by the resolvers as they decide. */ export interface SwitchboardTrace { harness?: 'cli' | 'flag' | 'binding'; model?: 'cli' | 'flag' | 'binding'; @@ -20,7 +18,7 @@ export interface SwitchboardTrace { | 'binding'; } -/** Everything a resolver middleware may branch on. Built once per run. */ +/** Everything a resolver may branch on. Built once per run. */ export interface SwitchboardCtx { /** Opaque log label. Program lookup stays with the caller. */ program?: string; @@ -35,9 +33,6 @@ export interface SwitchboardCtx { thinkingLevel?: EffortLevel; sequence?: Sequence; }; - flagSequence?: Sequence; - /** Raw boolean only for the existing capability-clamp log line. */ - orchestratorFlagOn?: boolean; /** CLI override (`--harness`). Wins over `flags`. */ cliHarness?: Harness; /** CLI override (`--sequence`). Wins over `flags`. */ @@ -48,37 +43,6 @@ export interface SwitchboardCtx { trace?: SwitchboardTrace; } -/** A resolver middleware: defer via `next()`, or assert by returning a value. */ -export type Middleware = (ctx: SwitchboardCtx, next: () => D) => D; - -/** - * Run a middleware chain over `ctx`. Each middleware receives `next` (which - * runs the rest of the chain) and can either: - * - defer: call `next()` and optionally modify its result (overlay pattern) - * - short-circuit: return a value without calling `next()` (skip the rest) - * - * **Earlier in the array = higher precedence.** Index 0 runs first and can - * short-circuit the rest; index 1 only runs if index 0 deferred. So - * `[cliSequenceMw, orchestratorFeatureFlagMw]` means CLI takes precedence over the - * flag, not the other way around. - * - * `fallback` runs at the end — reached only when every middleware deferred. - * Typically the map read for the base value. - */ -export function runChain( - chain: Middleware[], - ctx: SwitchboardCtx, - fallback: () => D, -): D { - function step(index: number): D { - if (index >= chain.length) return fallback(); - const middleware = chain[index]; - const next = () => step(index + 1); - return middleware(ctx, next); - } - return step(0); -} - // ── Data model ────────────────────────────────────────────────────────── /** Harness + model for one leaf of agent work. */ @@ -105,19 +69,6 @@ export interface ProgramBinding { contextMillOverride?: Record>; } -// ── Unified resolver ──────────────────────────────────────────────────── - -/** Compose both axes. Callers needing only one axis use the per-axis resolver. */ -export function resolveBinding( - ctx: SwitchboardCtx, - role = 'default', -): ProgramBinding { - ctx.trace ??= {}; - const sequence = resolveSequence(ctx); - const { harness, model, thinkingLevel } = resolveHarness(ctx, role); - return { sequence, harness, model, thinkingLevel }; -} - // ── Unified re-export surface ─────────────────────────────────────────── export { HARNESS_OPTIONS, getHarness } from './harness'; export { @@ -125,9 +76,4 @@ export { resolveHarness, resolveRoleHarness, } from './resolve-harness'; -export { - SEQUENCE_OPTIONS, - getSequence, - resolveSequence, - type SequenceRunner, -} from './sequence'; +export { SEQUENCE_OPTIONS, getSequence, type SequenceRunner } from './sequence'; diff --git a/src/agent/runner/switchboard/sequence.ts b/src/agent/runner/switchboard/sequence.ts index 8b3eb7792..43881f268 100644 --- a/src/agent/runner/switchboard/sequence.ts +++ b/src/agent/runner/switchboard/sequence.ts @@ -1,16 +1,11 @@ /** - * Sequence axis: registry, clamps and generic resolution of caller-supplied policy. + * Sequence axis: the registry. Programs resolve which sequence a run uses. */ -import { IS_PRODUCTION_BUILD } from '@env'; import { Sequence } from '@shared/constants'; -import { logToFile } from '@utils/debug'; -import { DEFAULT_AGENT_BINDING } from '@agent/default-binding'; -import { harnessRunsTasks, resolveHarness } from './resolve-harness'; import type { SequenceResult, SequenceContext } from '../shared/types'; import { runLinearProgram } from '../sequence/linear'; import { runOrchestrator } from '../sequence/orchestrator/orchestrator-runner'; -import { runChain, type Middleware, type SwitchboardCtx } from '.'; // ── Registry ──────────────────────────────────────────────────────────── @@ -38,84 +33,3 @@ export function getSequence(name: Sequence): SequenceRunner { } return sequence; } - -// ── Middleware + resolver ─────────────────────────────────────────────── - -/** - * A composed sub-run (integration inside self-driving) is structurally - * linear: the orchestrator owns the full run lifecycle (queue, outro) and - * cannot nest. Sits above every override, including CLI. - */ -const composedClampMw: Middleware = (ctx, next) => { - if (!ctx.composed) return next(); - if (ctx.trace) ctx.trace.sequence = 'composed'; - return Sequence.linear; -}; - -/** `--sequence` override. Dev/test only — the option is gated out of published builds. */ -const cliSequenceMw: Middleware = (ctx, next) => { - if (!ctx.cliSequence) return next(); - if (ctx.trace) ctx.trace.sequence = 'cli'; - return ctx.cliSequence; -}; - -/** A caller-supplied payload route may pin the sequence. */ -const flagRouteSequenceMw: Middleware = (ctx, next) => { - const route = ctx.flagRoute; - if (!route?.sequence) return next(); - if (ctx.trace) ctx.trace.sequence = 'payload'; - return route.sequence; -}; - -/** A caller-supplied sequence experiment, already scoped to its program. */ -const sequenceExperimentMw: Middleware = (ctx, next) => { - const sequence = ctx.flagSequence; - if (!sequence) return next(); - if (ctx.trace) ctx.trace.sequence = 'flag'; - return sequence; -}; - -/** - * The orchestrator drives harnesses through `runTask`; a harness that has not - * implemented it clamps the run to linear. A capability check, not a harness - * identity check — `HARNESS_RUNS_TASKS` records which backends implement the - * method, and a registry test keeps it in step. Sits below the CLI override so - * `--sequence orchestrator` still reproduces the hard error in dev builds. - */ -const runTaskCapabilityClampMw: Middleware = (ctx, next) => { - const pick = resolveHarness(ctx); - if (harnessRunsTasks(pick.harness)) return next(); - if (ctx.orchestratorFlagOn) { - logToFile( - `[switchboard] wizard-orchestrator ignored: ${pick.harness} has no runTask, clamping to linear`, - ); - } - if (ctx.trace) ctx.trace.sequence = 'runtask-clamp'; - return Sequence.linear; -}; - -// Order = precedence: CLI > capability clamp > flag > binding default. The -// prod spread collapses to [], dropping cliSequenceMw from the chain. -const SEQUENCE_MIDDLEWARE: Middleware[] = [ - composedClampMw, - ...(IS_PRODUCTION_BUILD ? [] : [cliSequenceMw]), - runTaskCapabilityClampMw, - flagRouteSequenceMw, - sequenceExperimentMw, -]; - -/** CLI wins over `wizard-orchestrator` flag wins over binding default. */ -export function resolveSequence(ctx: SwitchboardCtx): Sequence { - const sequence = runChain(SEQUENCE_MIDDLEWARE, ctx, () => { - if (ctx.trace) ctx.trace.sequence = 'binding'; - const binding = ctx.baseBinding ?? DEFAULT_AGENT_BINDING; - return binding.sequence; - }); - logToFile( - `[switchboard] resolved: program=${ - ctx.program ?? '?' - } sequence=${sequence}` + - `${ctx.trace?.sequence ? ` (${ctx.trace.sequence})` : ''}`, - ); - return sequence; -} diff --git a/src/agent/types.ts b/src/agent/types.ts index 2b4bd4352..94bce6df4 100644 --- a/src/agent/types.ts +++ b/src/agent/types.ts @@ -31,7 +31,7 @@ export type { TokenUsageDelta, } from './progress'; -/** Input types of the exported resolveBinding and resolveHarness. */ +/** Input types of the exported resolveHarness. */ export type { ProgramBinding, SwitchboardCtx } from './runner'; export type { EffortLevel } from './runner/switchboard/models'; diff --git a/src/programs/binding.ts b/src/programs/binding.ts index 007058cf2..59c30b348 100644 --- a/src/programs/binding.ts +++ b/src/programs/binding.ts @@ -1,3 +1,4 @@ +import { IS_PRODUCTION_BUILD } from '@env'; import { DEFAULT_AGENT_MODEL, GPT5_6_SOL_MODEL, @@ -5,11 +6,16 @@ import { Harness, Sequence, } from '@shared/constants'; -import { DEFAULT_AGENT_BINDING, resolveBinding, resolveHarness } from '@agent'; +import { logToFile } from '@utils/debug'; +import { + DEFAULT_AGENT_BINDING, + harnessRunsTasks, + resolveHarness, +} from '@agent'; import type { ProgramBinding, ResolvedBinding, - SwitchboardCtx, + SwitchboardCtx as HarnessCtx, } from '@agent/types'; import type { ProgramId } from './program-registry'; import { @@ -18,6 +24,12 @@ import { resolveFlagSequence, } from './experiments'; +/** The agent's harness inputs plus the sequence inputs only programs read. */ +type SwitchboardCtx = HarnessCtx & { + flagSequence?: Sequence; + orchestratorFlagOn?: boolean; +}; + export interface ProgramSwitchboardCtx { program: ProgramId; composed?: boolean; @@ -94,7 +106,9 @@ export function resolveProgramBinding( cliModel: ctx.cliModel, trace: ctx.trace, }; - const binding = resolveBinding(resolution); + const sequence = resolveSequence(resolution); + const { harness, model, thinkingLevel } = resolveHarness(resolution); + const binding = { sequence, harness, model, thinkingLevel }; const roles = Object.keys(baseBinding.contextMillOverride ?? {}); if (roles.length === 0) return binding; return { @@ -107,3 +121,40 @@ export function resolveProgramBinding( ), }; } + +function resolveSequence(ctx: SwitchboardCtx): Sequence { + const [source, sequence] = pickSequence(ctx); + if (ctx.trace) ctx.trace.sequence = source; + logToFile( + `[switchboard] resolved: program=${ + ctx.program ?? '?' + } sequence=${sequence} (${source})`, + ); + return sequence; +} + +/** + * The first rung that decides wins: the composed clamp, the dev-build CLI + * override, the runTask capability clamp, the flag route, the experiment, then + * the base binding. CLI sits above the capability clamp, so `--sequence + * orchestrator` still reaches the orchestrator's hard error in dev builds. + */ +function pickSequence( + ctx: SwitchboardCtx, +): [Required>['sequence'], Sequence] { + // The orchestrator owns the whole run lifecycle and cannot nest. + if (ctx.composed) return ['composed', Sequence.linear]; + if (!IS_PRODUCTION_BUILD && ctx.cliSequence) return ['cli', ctx.cliSequence]; + const { harness } = resolveHarness(ctx); + if (!harnessRunsTasks(harness)) { + if (ctx.orchestratorFlagOn) { + logToFile( + `[switchboard] wizard-orchestrator ignored: ${harness} has no runTask, clamping to linear`, + ); + } + return ['runtask-clamp', Sequence.linear]; + } + if (ctx.flagRoute?.sequence) return ['payload', ctx.flagRoute.sequence]; + if (ctx.flagSequence) return ['flag', ctx.flagSequence]; + return ['binding', (ctx.baseBinding ?? DEFAULT_AGENT_BINDING).sequence]; +}