diff --git a/src/lib/programs/__tests__/self-driving-detect.test.ts b/src/lib/programs/__tests__/self-driving-detect.test.ts index 305ab9c70..27c4f6626 100644 --- a/src/lib/programs/__tests__/self-driving-detect.test.ts +++ b/src/lib/programs/__tests__/self-driving-detect.test.ts @@ -9,11 +9,11 @@ import { import { detectPostHogPresent, POSTHOG_MANIFESTS, + SELF_DRIVING_DETECTED_TOOLS_KEY, + SELF_DRIVING_TOOL_KINDS, } from '@lib/programs/self-driving/detect'; -import { - DETECTED_WAREHOUSE_SOURCES_KEY, - getDetectedWarehouseSources, -} from '@lib/programs/warehouse-source/detect'; +import { SOURCE_DETECTORS } from '@lib/warehouse-sources/registry'; +import type { DetectedSource } from '@lib/warehouse-sources/types'; import { toIntegrationReport } from '@lib/programs/self-driving/detect-agentic'; import { PROJECT_MANIFESTS, @@ -64,20 +64,33 @@ describe('detectSelfDrivingPrerequisites', () => { expect(ctx.detectError).toBeUndefined(); }); - it('stashes tools detected in the codebase for the connected-tools ask', () => { + /** Kinds stashed for the connected-tools ask after a scan of `tmpDir`. */ + const detectedKinds = (deps: Record): string[] => { fs.writeFileSync( path.join(tmpDir, 'package.json'), - JSON.stringify({ dependencies: { '@sentry/node': '^7.0.0' } }), + JSON.stringify({ dependencies: deps }), ); - const session = buildSession({ installDir: tmpDir }); - detectSelfDrivingPrerequisites(session, setCtx); + detectSelfDrivingPrerequisites( + buildSession({ installDir: tmpDir }), + setCtx, + ); + const tools = ctx[SELF_DRIVING_DETECTED_TOOLS_KEY] as + | DetectedSource[] + | undefined; + return (tools ?? []).map((s) => s.kind); + }; + + it('stashes tools detected in the codebase for the connected-tools ask', () => { + expect(detectedKinds({ '@sentry/node': '^7.0.0' })).toContain('Sentry'); + }); - // Mirror the value onto the session the way the store setter would, so the - // shared accessor the run closure uses reads it back. - session.frameworkContext[DETECTED_WAREHOUSE_SOURCES_KEY] = - ctx[DETECTED_WAREHOUSE_SOURCES_KEY]; - const detected = getDetectedWarehouseSources(session); - expect(detected.map((s) => s.kind)).toContain('Sentry'); + it('keeps only the tools the inbox can connect', () => { + // `pg` and `stripe` are warehouse sources, not connected tools — surfacing + // them would lead STEP 5 with a database and a payment processor. + expect(detectedKinds({ pg: '^8.0.0', stripe: '^14.0.0' })).toEqual([]); + expect(detectedKinds({ pg: '^8.0.0', '@sentry/node': '^7.0.0' })).toEqual([ + 'Sentry', + ]); }); it('writes nothing when the codebase has no detectable tools', () => { @@ -87,7 +100,18 @@ describe('detectSelfDrivingPrerequisites', () => { detectSelfDrivingPrerequisites(session, setCtx); expect(ctx.detectError).toBeUndefined(); - expect(ctx[DETECTED_WAREHOUSE_SOURCES_KEY]).toBeUndefined(); + expect(ctx[SELF_DRIVING_DETECTED_TOOLS_KEY]).toBeUndefined(); + }); +}); + +describe('SELF_DRIVING_TOOL_KINDS', () => { + it('names only kinds the source registry can actually detect', () => { + // The filter is a plain string set, so a registry rename would silently + // drop a tool from the ask. Fail here instead. + const known = new Set(SOURCE_DETECTORS.map((d) => d.kind)); + expect([...SELF_DRIVING_TOOL_KINDS].filter((k) => !known.has(k))).toEqual( + [], + ); }); }); diff --git a/src/lib/programs/self-driving/detect.ts b/src/lib/programs/self-driving/detect.ts index 6e21d2d33..93f2a5de8 100644 --- a/src/lib/programs/self-driving/detect.ts +++ b/src/lib/programs/self-driving/detect.ts @@ -31,7 +31,7 @@ import { analytics } from '@utils/analytics'; import type { WizardSession } from '@lib/wizard-session'; import type { AbortCase } from '@lib/agent/agent-runner'; import { detectWarehouseSources } from '@lib/warehouse-sources/detect'; -import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@lib/programs/warehouse-source/detect'; +import type { DetectedSource } from '@lib/warehouse-sources/types'; /** frameworkContext key holding the deterministic PostHog-presence result. */ export const POSTHOG_PRESENT_KEY = 'postHogPresent'; @@ -44,6 +44,31 @@ export const POSTHOG_PRESENT_KEY = 'postHogPresent'; */ export const SELF_DRIVING_INTEGRATE_PATH_KEY = 'selfDrivingIntegratePath'; +/** + * frameworkContext key holding the tools this codebase uses that are worth + * promoting in the connected-tools ask. Self-driving's own key, not the + * warehouse program's `DETECTED_WAREHOUSE_SOURCES_KEY`: the integration program + * reads that one to build its prompt, and the integrate-run phase inherits a + * copy of this frameworkContext — so writing there would silently rewrite the + * integration agent's prompt on every self-driving run that installs PostHog + * first. (Its outro is safe: a composed run returns before `buildOutroData`.) + */ +export const SELF_DRIVING_DETECTED_TOOLS_KEY = 'selfDrivingDetectedTools'; + +/** + * Read the detected tools out of frameworkContext. Single accessor shared by + * the detect step and the prompt builder so the key + cast live in one place. + */ +export function getSelfDrivingDetectedTools( + session: WizardSession, +): DetectedSource[] { + return ( + (session.frameworkContext[SELF_DRIVING_DETECTED_TOOLS_KEY] as + | DetectedSource[] + | undefined) ?? [] + ); +} + // Matches `posthog` at a dependency boundary (line start, or after "'/=:.@ or // whitespace): catches `com.posthog:posthog-android` and `@posthog/ai`, skips // substrings inside other words. @@ -299,12 +324,50 @@ export function detectSelfDrivingPrerequisites( } /** - * Scan the codebase for the tools it uses (Sentry, Linear, GitHub, Stripe, …) - * so STEP 5's connected-tools ask can surface detected tools first instead of - * dumping the full ~500-source catalog on the user. Same deterministic scanner - * the warehouse program uses, so the result lands under the shared - * `DETECTED_WAREHOUSE_SOURCES_KEY` and is read back with - * `getDetectedWarehouseSources`. + * Source kinds worth promoting in the connected-tools ask. The shared scanner + * matches the whole warehouse catalog (databases, payments, LLM vendors, ad + * platforms) and most of that has nothing to do with STEP 5: unfiltered, a + * routine repo leads the issue-tracker ask with Postgres (matched on + * `DATABASE_URL`), Stripe and OpenAI — the wall of irrelevant options this is + * supposed to remove. + * + * Exactly the kinds enumerated as inbox tools in #1022, which were sourced from + * the context-mill `self-driving` skill's connected-tools list. Deliberately no + * guesses beyond it: this list only decides what gets PROMOTED, so leaving a + * kind out costs a nudge (the skill still offers the tool), while putting a kind + * in that the inbox can't connect sends the agent after a source it can't + * create. When the skill's catalog grows, reconcile here. + * + * A shadow list of the registry, so it drifts in one direction the guard test + * can't catch: a new inbox-connectable kind added to `SOURCE_DETECTORS` has to + * be added here too or it never gets promoted. If that bites, the fix is a + * field on `SourceDetector` rather than a third copy of this list. + */ +export const SELF_DRIVING_TOOL_KINDS: ReadonlySet = new Set([ + // Issue trackers / code hosts + 'Github', + 'GitLab', + 'Gitea', + 'Linear', + 'Jira', + // Error trackers + 'Sentry', + 'Rollbar', + 'Bugsnag', + // Support desks + 'Zendesk', + 'Freshdesk', + 'Front', + 'Gorgias', + 'Intercom', +]); + +/** + * Scan the codebase for the tools it uses that the inbox can connect (Sentry, + * Linear, GitHub, Zendesk, …) so STEP 5's connected-tools ask can surface those + * first instead of dumping the full source catalog on the user. Stashed under + * self-driving's own `SELF_DRIVING_DETECTED_TOOLS_KEY` and read back with + * `getSelfDrivingDetectedTools`. * * Best-effort: the connected-tools ask degrades to the skill's default * ordering when nothing is detected, so a scan failure must never break the @@ -315,19 +378,29 @@ function detectConnectedTools( setFrameworkContext: (key: string, value: unknown) => void, ): void { try { - const sources = detectWarehouseSources(installDir); - if (sources.length === 0) return; + const tools = detectWarehouseSources(installDir).filter((s) => + SELF_DRIVING_TOOL_KINDS.has(s.kind), + ); - // Tag the run so the connected-tools funnel can slice on what the project - // had available. Deliberately NOT the `warehouse sources detected` event - // the integration flow emits — that metric's denominator is integration + // Tagged on every run that scans, including the empty case — without the + // zero rows there is no way to tell "the scan found nothing" from "this code + // path never ran", which is the first thing to check when the ask looks + // unprioritised. Deliberately NOT the `warehouse sources detected` event + // the integration flow emits: that metric's denominator is integration // runs, and firing it here would fold self-driving runs into it. + analytics.setTag('connected_tools_detected_count', tools.length); + if (tools.length === 0) return; + analytics.setTag( 'connected_tools_detected', - sources.map((s) => s.kind).join(','), + tools.map((s) => s.kind).join(','), ); - setFrameworkContext(DETECTED_WAREHOUSE_SOURCES_KEY, sources); + setFrameworkContext(SELF_DRIVING_DETECTED_TOOLS_KEY, tools); } catch (error) { + // -1 rather than nothing: an absent tag would be indistinguishable from a + // build that never ran this scan, which is what the count is here to rule + // out. The captured exception carries the why. + analytics.setTag('connected_tools_detected_count', -1); analytics.captureException( error instanceof Error ? error : new Error(String(error)), { step: 'detectConnectedTools' }, diff --git a/src/lib/programs/self-driving/index.ts b/src/lib/programs/self-driving/index.ts index 7474e58fc..b4b19fad1 100644 --- a/src/lib/programs/self-driving/index.ts +++ b/src/lib/programs/self-driving/index.ts @@ -4,9 +4,11 @@ import type { ProgramConfig } from '@lib/programs/program-step'; import type { ProgramRun } from '@lib/agent/agent-runner'; import { OutroKind, type WizardSession } from '@lib/wizard-session'; import { createSkillProgram } from '../agent-skill/index.js'; -import { getDetectedWarehouseSources } from '@lib/programs/warehouse-source/detect'; import { SELF_DRIVING_PROGRAM } from './steps.js'; -import { SELF_DRIVING_ABORT_CASES } from './detect.js'; +import { + SELF_DRIVING_ABORT_CASES, + getSelfDrivingDetectedTools, +} from './detect.js'; import { buildSelfDrivingPrompt } from './prompt.js'; import { getTips } from './content/tips.js'; import { getContentBlocks } from './content/index.js'; @@ -44,7 +46,7 @@ const buildRun = (session: WizardSession): Promise => skillId: SELF_DRIVING_SKILL_ID, integrationLabel: SELF_DRIVING_SKILL_ID, customPrompt: (ctx) => - buildSelfDrivingPrompt(ctx, getDetectedWarehouseSources(session)), + buildSelfDrivingPrompt(ctx, getSelfDrivingDetectedTools(session)), successMessage: SUCCESS_MESSAGE, reportFile: REPORT_FILE, docsUrl: DOCS_URL, diff --git a/src/lib/runners/run-wizard.ts b/src/lib/runners/run-wizard.ts index bf99bc9c8..436e7ce58 100644 --- a/src/lib/runners/run-wizard.ts +++ b/src/lib/runners/run-wizard.ts @@ -18,7 +18,14 @@ type Step = ProgramConfig['steps'][number]; /** The session a run step's agent runs in: scoped to the step's target dir * (e.g. a monorepo sub-app) with its own framework context, after any prep. - * A step without `targetDir` runs in the live session, unchanged. */ + * A step without `targetDir` runs in the live session, unchanged. + * + * The copy is shallow and unfiltered, so a composed sub-run inherits every + * frameworkContext key the host program wrote — including ones the sub-run's + * own program reads for its prompt. Name keys for the program that owns them + * (`selfDrivingDetectedTools`, not `detectedWarehouseSources`) so a host can't + * silently rewrite a spliced-in program's behaviour. If that collision shows up + * a second time, scope the inheritance here instead of renaming again. */ async function prepareRunSession( step: Step, live: WizardSession,