diff --git a/src/lib/programs/__tests__/self-driving-detect.test.ts b/src/lib/programs/__tests__/self-driving-detect.test.ts index ba4b992cf..209d3572d 100644 --- a/src/lib/programs/__tests__/self-driving-detect.test.ts +++ b/src/lib/programs/__tests__/self-driving-detect.test.ts @@ -9,7 +9,14 @@ import { import { detectPostHogPresent, POSTHOG_MANIFESTS, + SELF_DRIVING_DETECTED_TOOLS_KEY, + SELF_DRIVING_TOOL_KINDS, + getSelfDrivingDetectedTools, } from '@lib/programs/self-driving/detect'; +import { getDetectedWarehouseSources } from '@lib/programs/warehouse-source/detect'; +import { WizardStore } from '@ui/tui/store'; +import { SOURCE_DETECTORS } from '@lib/warehouse-sources/registry'; +import type { DetectedSource } from '@lib/warehouse-sources/types'; import { toIntegrationReport } from '@lib/programs/self-driving/detect-agentic'; import { PROJECT_MANIFESTS, @@ -59,6 +66,87 @@ describe('detectSelfDrivingPrerequisites', () => { expect(ctx.detectError).toBeUndefined(); }); + + /** Kinds stashed for the connected-tools ask after a scan of `tmpDir`. */ + const detectedKinds = (deps: Record): string[] => { + fs.writeFileSync( + path.join(tmpDir, 'package.json'), + JSON.stringify({ dependencies: deps }), + ); + detectSelfDrivingPrerequisites( + buildSession({ installDir: tmpDir }), + setCtx, + ); + const tools = ctx[SELF_DRIVING_DETECTED_TOOLS_KEY] as + | DetectedSource[] + | undefined; + return (tools ?? []).map((s) => s.kind); + }; + + it('stashes tools detected in the codebase for the connected-tools ask', () => { + expect(detectedKinds({ '@sentry/node': '^7.0.0' })).toContain('Sentry'); + }); + + it('keeps only the tools the inbox can connect', () => { + // `pg` and `stripe` are warehouse sources, not connected tools. + expect(detectedKinds({ pg: '^8.0.0', stripe: '^14.0.0' })).toEqual([]); + expect(detectedKinds({ pg: '^8.0.0', '@sentry/node': '^7.0.0' })).toEqual([ + 'Sentry', + ]); + }); + + it('writes nothing when the codebase has no detectable tools', () => { + // Bare dir: valid, but no tools to prioritise, so the key stays unset. + const session = buildSession({ installDir: tmpDir }); + detectSelfDrivingPrerequisites(session, setCtx); + + expect(ctx.detectError).toBeUndefined(); + expect(ctx[SELF_DRIVING_DETECTED_TOOLS_KEY]).toBeUndefined(); + }); +}); + +describe('SELF_DRIVING_TOOL_KINDS', () => { + it('names only kinds the source registry can actually detect', () => { + // A plain string set, so a registry rename would otherwise drop a tool silently. + const known = new Set(SOURCE_DETECTORS.map((d) => d.kind)); + expect([...SELF_DRIVING_TOOL_KINDS].filter((k) => !known.has(k))).toEqual( + [], + ); + }); +}); + +describe('the detect step does not leak into the composed integration run', () => { + // Through the real store — the leak lived in the plumbing, not in detectConnectedTools. + let tmpDir: string; + + beforeEach(() => { + tmpDir = makeTmpDir(); + fs.writeFileSync( + path.join(tmpDir, 'package.json'), + JSON.stringify({ + dependencies: { '@sentry/node': '^7.0.0', pg: '^8.0.0' }, + }), + ); + }); + afterEach(() => cleanup(tmpDir)); + + it('stashes under its own key and leaves the warehouse key untouched', async () => { + const store = new WizardStore('self-driving'); + store.session = buildSession({ installDir: tmpDir }); + await store.runReadyHooks(); + + // Self-driving sees its tools... + expect( + getSelfDrivingDetectedTools(store.session).map((s) => s.kind), + ).toContain('Sentry'); + // ...and the integration program, on the session it inherits, sees nothing. + expect(getDetectedWarehouseSources(store.session)).toEqual([]); + const inherited = { + ...store.session, + frameworkContext: { ...store.session.frameworkContext }, + }; + expect(getDetectedWarehouseSources(inherited)).toEqual([]); + }); }); describe('SELF_DRIVING_ABORT_CASES', () => { diff --git a/src/lib/programs/__tests__/self-driving-prompt.test.ts b/src/lib/programs/__tests__/self-driving-prompt.test.ts index ede157e25..52b7ed616 100644 --- a/src/lib/programs/__tests__/self-driving-prompt.test.ts +++ b/src/lib/programs/__tests__/self-driving-prompt.test.ts @@ -1,6 +1,7 @@ import { buildSelfDrivingPrompt } from '@lib/programs/self-driving/prompt'; import type { PromptContext } from '@lib/agent/agent-runner'; import { HostResolution } from '@lib/host-resolution'; +import type { DetectedSource } from '@lib/warehouse-sources/types'; const ctx: PromptContext = { projectId: 123, @@ -8,6 +9,19 @@ const ctx: PromptContext = { host: HostResolution.fromApiHost('https://us.posthog.com'), }; +const SENTRY: DetectedSource = { + kind: 'Sentry', + label: 'Sentry', + mode: 'in-cli', + matchedSignal: 'found `@sentry/node` in package.json', +}; +const LINEAR: DetectedSource = { + kind: 'Linear', + label: 'Linear', + mode: 'deep-link', + matchedSignal: 'found `LINEAR_API_KEY` in .env', +}; + describe('buildSelfDrivingPrompt', () => { it('covers only the Self-driving steps — integration is a separate phase', () => { const prompt = buildSelfDrivingPrompt(ctx); @@ -34,3 +48,45 @@ describe('buildSelfDrivingPrompt', () => { expect(prompt).toContain('STEP 7 — Write the report and hand off'); }); }); + +describe('detected-tools block', () => { + it('lists each detected tool with its source_type and matched signal', () => { + const prompt = buildSelfDrivingPrompt(ctx, [SENTRY, LINEAR]); + expect(prompt).toContain('Tools detected in this codebase'); + expect(prompt).toContain('Sentry (source_type: Sentry)'); + expect(prompt).toContain('found `@sentry/node` in package.json'); + expect(prompt).toContain('Linear (source_type: Linear)'); + }); + + it('appears before STEP 5 so the connected-tools ask can read it', () => { + const prompt = buildSelfDrivingPrompt(ctx, [SENTRY]); + expect(prompt.indexOf('Tools detected in this codebase')).toBeLessThan( + prompt.indexOf('STEP 5 — Offer issue-tracker integrations'), + ); + }); + + it('points STEP 5 at the detected list, basics, then others', () => { + const prompt = buildSelfDrivingPrompt(ctx, [SENTRY]); + // Scope to STEP 5's text (up to STEP 6) and ignore incidental wrapping. + const step5 = prompt + .slice(prompt.indexOf('STEP 5 —'), prompt.indexOf('STEP 6 —')) + .replace(/\s+/g, ' '); + expect(step5).toContain('Tools detected in this codebase'); + expect(step5).toContain('SaaS basics'); + expect(step5).toContain('others'); + }); + + it('states nothing was found when the scan is empty (no invented scan)', () => { + const prompt = buildSelfDrivingPrompt(ctx, []); + expect(prompt).toContain('none found by the dependency + env scan'); + expect(prompt).not.toContain('source_type:'); + }); + + it('defaults to an empty scan when no sources are passed', () => { + // The single-arg call site (older tests, and the type default) must not throw. + expect(() => buildSelfDrivingPrompt(ctx)).not.toThrow(); + expect(buildSelfDrivingPrompt(ctx)).toContain( + 'none found by the dependency + env scan', + ); + }); +}); diff --git a/src/lib/programs/self-driving/detect.ts b/src/lib/programs/self-driving/detect.ts index e2d6a811e..377e5a5fa 100644 --- a/src/lib/programs/self-driving/detect.ts +++ b/src/lib/programs/self-driving/detect.ts @@ -27,8 +27,11 @@ import { type Dirent, } from 'fs'; import { join } from 'path'; +import { analytics } from '@utils/analytics'; import type { WizardSession } from '@lib/wizard-session'; import type { AbortCase } from '@lib/agent/agent-runner'; +import { detectWarehouseSources } from '@lib/warehouse-sources/detect'; +import type { DetectedSource } from '@lib/warehouse-sources/types'; /** frameworkContext key holding the deterministic PostHog-presence result. */ export const POSTHOG_PRESENT_KEY = 'postHogPresent'; @@ -41,6 +44,20 @@ export const POSTHOG_PRESENT_KEY = 'postHogPresent'; */ export const SELF_DRIVING_INTEGRATE_PATH_KEY = 'selfDrivingIntegratePath'; +/** Self-driving's own detected-tools key — not the warehouse one, which the integration program reads. */ +export const SELF_DRIVING_DETECTED_TOOLS_KEY = 'selfDrivingDetectedTools'; + +/** Read the detected tools out of frameworkContext. */ +export function getSelfDrivingDetectedTools( + session: WizardSession, +): DetectedSource[] { + return ( + (session.frameworkContext[SELF_DRIVING_DETECTED_TOOLS_KEY] as + | DetectedSource[] + | undefined) ?? [] + ); +} + // Matches `posthog` at a dependency boundary (line start, or after "'/=:.@ or // whitespace): catches `com.posthog:posthog-android` and `@posthog/ai`, skips // substrings inside other words. @@ -291,4 +308,66 @@ export function detectSelfDrivingPrerequisites( // screen: found → skip straight to self-driving; not found → ask to set up // PostHog first. setFrameworkContext(POSTHOG_PRESENT_KEY, detectPostHogPresent(installDir)); + + detectConnectedTools(installDir, setFrameworkContext); +} + +/** Step 5's ask options intersected with `SOURCE_DETECTORS` — reconcile when the skill's catalog grows. */ +export const SELF_DRIVING_TOOL_KINDS: ReadonlySet = new Set([ + // Issue trackers / code hosts + 'Github', + 'GitLab', + 'Gitea', + 'Linear', + 'Jira', + 'Shortcut', + // Error tracking + 'Sentry', + 'Rollbar', + 'Bugsnag', + 'Honeybadger', + 'Raygun', + // Support desks + 'Zendesk', + 'Freshdesk', + 'Front', + 'Gorgias', + 'Kustomer', + 'Plain', + // Security scanners + 'Snyk', + // Product feedback + 'Canny', + 'Productboard', + // Search analytics + 'GoogleSearchConsole', +]); + +/** Scan for inbox-connectable tools so STEP 5 can surface them first. Best-effort — never blocks detection. */ +function detectConnectedTools( + installDir: string, + setFrameworkContext: (key: string, value: unknown) => void, +): void { + try { + const tools = detectWarehouseSources(installDir).filter((s) => + SELF_DRIVING_TOOL_KINDS.has(s.kind), + ); + + // Tagged even at zero, so "found nothing" is distinguishable from "never ran". + analytics.setTag('connected_tools_detected_count', tools.length); + if (tools.length === 0) return; + + analytics.setTag( + 'connected_tools_detected', + tools.map((s) => s.kind).join(','), + ); + setFrameworkContext(SELF_DRIVING_DETECTED_TOOLS_KEY, tools); + } catch (error) { + // -1, not absent, so a failed scan stays distinguishable from one that never ran. + analytics.setTag('connected_tools_detected_count', -1); + analytics.captureException( + error instanceof Error ? error : new Error(String(error)), + { step: 'detectConnectedTools' }, + ); + } } diff --git a/src/lib/programs/self-driving/index.ts b/src/lib/programs/self-driving/index.ts index 9ea3cbf6c..b4b19fad1 100644 --- a/src/lib/programs/self-driving/index.ts +++ b/src/lib/programs/self-driving/index.ts @@ -2,10 +2,13 @@ import { join } from 'path'; import { access, rm } from 'node:fs/promises'; import type { ProgramConfig } from '@lib/programs/program-step'; import type { ProgramRun } from '@lib/agent/agent-runner'; -import { OutroKind } from '@lib/wizard-session'; +import { OutroKind, type WizardSession } from '@lib/wizard-session'; import { createSkillProgram } from '../agent-skill/index.js'; import { SELF_DRIVING_PROGRAM } from './steps.js'; -import { SELF_DRIVING_ABORT_CASES } from './detect.js'; +import { + SELF_DRIVING_ABORT_CASES, + getSelfDrivingDetectedTools, +} from './detect.js'; import { buildSelfDrivingPrompt } from './prompt.js'; import { getTips } from './content/tips.js'; import { getContentBlocks } from './content/index.js'; @@ -35,64 +38,69 @@ async function removeInstalledSkill(installDir: string): Promise { await rm(skillDir, { recursive: true, force: true }).catch(() => undefined); } -const run: ProgramRun = { - skillId: SELF_DRIVING_SKILL_ID, - integrationLabel: SELF_DRIVING_SKILL_ID, - customPrompt: buildSelfDrivingPrompt, - successMessage: SUCCESS_MESSAGE, - reportFile: REPORT_FILE, - docsUrl: DOCS_URL, - spinnerMessage: 'Setting up PostHog Self-driving...', - estimatedDurationMinutes: 10, - abortCases: SELF_DRIVING_ABORT_CASES, - // The flow legitimately needs several interactions (GitHub connect + - // verify, issue-tracker picks, the scout-tailoring proposal), so raise - // the wizard_ask budget a little above the default 10. - maxQuestions: 13, - // This flow hands the user long OAuth/authorize URLs (Linear, GitHub - // fallback, Zendesk) in wizard_ask prompts. Render them as OSC 8 - // hyperlinks + clipboard copy so the overlay's line wrapping can't break - // the click target. Scoped to this program only. - richLinks: true, - // STEP 3 (GitHub App install) and STEP 5 (Linear OAuth) park on wizard_ask - // while the user does slow browser work; a first-time GitHub App install - // routinely exceeds the 5-min default, and a timeout is indistinguishable - // from a decline (both resolve to __cancelled__). Match upload-source-maps. - askTimeoutMs: 30 * 60 * 1000, +// A session closure (not a static object) so `customPrompt` can read the +// tools detected in the codebase — written to frameworkContext by the detect +// step — and hand them to the prompt for STEP 4/STEP 5 prioritisation. +const buildRun = (session: WizardSession): Promise => + Promise.resolve({ + skillId: SELF_DRIVING_SKILL_ID, + integrationLabel: SELF_DRIVING_SKILL_ID, + customPrompt: (ctx) => + buildSelfDrivingPrompt(ctx, getSelfDrivingDetectedTools(session)), + successMessage: SUCCESS_MESSAGE, + reportFile: REPORT_FILE, + docsUrl: DOCS_URL, + spinnerMessage: 'Setting up PostHog Self-driving...', + estimatedDurationMinutes: 10, + abortCases: SELF_DRIVING_ABORT_CASES, + // The flow legitimately needs several interactions (GitHub connect + + // verify, issue-tracker picks, the scout-tailoring proposal), so raise + // the wizard_ask budget a little above the default 10. + maxQuestions: 13, + // This flow hands the user long OAuth/authorize URLs (Linear, GitHub + // fallback, Zendesk) in wizard_ask prompts. Render them as OSC 8 + // hyperlinks + clipboard copy so the overlay's line wrapping can't break + // the click target. Scoped to this program only. + richLinks: true, + // STEP 3 (GitHub App install) and STEP 5 (Linear OAuth) park on wizard_ask + // while the user does slow browser work; a first-time GitHub App install + // routinely exceeds the 5-min default, and a timeout is indistinguishable + // from a decline (both resolve to __cancelled__). Match upload-source-maps. + askTimeoutMs: 30 * 60 * 1000, - // Emit a `wizard: step` analytics event on each agent task transition so we - // can build a step-level drop-off funnel (where a run stops — GitHub connect, - // scout enable, etc.), including silent steps with no wizard_ask. Opt-in, so - // only self-driving runs emit these; every other program is unchanged. - trackStepProgress: true, + // Emit a `wizard: step` analytics event on each agent task transition so we + // can build a step-level drop-off funnel (where a run stops — GitHub connect, + // scout enable, etc.), including silent steps with no wizard_ask. Opt-in, so + // only self-driving runs emit these; every other program is unchanged. + trackStepProgress: true, - postRun: async (session) => { - await removeInstalledSkill(session.installDir); - }, + postRun: async (session) => { + await removeInstalledSkill(session.installDir); + }, - buildOutroData: (_session, credentials) => { - const uiHost = credentials.host.appHost.replace(/\/$/, ''); - const inboxUrl = `${uiHost}/project/${credentials.projectId}/inbox`; - return { - kind: OutroKind.Success as const, - message: SUCCESS_MESSAGE, - primaryLink: { label: 'Your Self-driving inbox', url: inboxUrl }, - nextSteps: { - heading: 'In your inbox you can:', - items: [ - 'Investigate reports with the agent', - 'Tag teammates to loop them in', - 'Kick off a PR when you like the proposed fix ($15 flat)', - 'Or work from Slack (tag @PostHog) and MCP', - ], - }, - body: - 'Pricing: scouts, signals, and reports are free. You pay a flat ' + - '$15 only when a report ships a PR.', - reportFile: REPORT_FILE, - }; - }, -}; + buildOutroData: (_session, credentials) => { + const uiHost = credentials.host.appHost.replace(/\/$/, ''); + const inboxUrl = `${uiHost}/project/${credentials.projectId}/inbox`; + return { + kind: OutroKind.Success as const, + message: SUCCESS_MESSAGE, + primaryLink: { label: 'Your Self-driving inbox', url: inboxUrl }, + nextSteps: { + heading: 'In your inbox you can:', + items: [ + 'Investigate reports with the agent', + 'Tag teammates to loop them in', + 'Kick off a PR when you like the proposed fix ($15 flat)', + 'Or work from Slack (tag @PostHog) and MCP', + ], + }, + body: + 'Pricing: scouts, signals, and reports are free. You pay a flat ' + + '$15 only when a report ships a PR.', + reportFile: REPORT_FILE, + }; + }, + }); export const selfDrivingConfig: ProgramConfig = { ...createSkillProgram({ @@ -110,7 +118,7 @@ export const selfDrivingConfig: ProgramConfig = { abortCases: SELF_DRIVING_ABORT_CASES, }), steps: SELF_DRIVING_PROGRAM, - run, + run: buildRun, getTips, getContentBlocks, }; diff --git a/src/lib/programs/self-driving/prompt.ts b/src/lib/programs/self-driving/prompt.ts index f47857f4b..866d5fe1c 100644 --- a/src/lib/programs/self-driving/prompt.ts +++ b/src/lib/programs/self-driving/prompt.ts @@ -1,5 +1,33 @@ import { AgentSignals } from '@lib/agent/agent-interface'; import type { PromptContext } from '@lib/agent/agent-runner'; +import type { DetectedSource } from '@lib/warehouse-sources/types'; + +/** + * Render the deterministic codebase-tool scan for the prompt. STEP 4 and + * STEP 5 read this instead of the agent doing its own flaky "light scan": + * enable the matching signal sources, and surface detected tools first in the + * connected-tools ask rather than dumping the full ~500-source catalog. When + * nothing is detected we still say so explicitly, so the agent falls back to + * the skill's default ordering instead of inventing a scan. + */ +function renderDetectedTools(sources: DetectedSource[]): string { + if (sources.length === 0) { + return `Tools detected in this codebase: none found by the dependency + env scan. Use the skill's default ordering for STEP 5.`; + } + + const lines = sources.map( + (s) => `- ${s.label} (source_type: ${s.kind}) — ${s.matchedSignal}`, + ); + return [ + 'Tools detected in this codebase (deterministic dependency + env-key scan — this is evidence, not a guess):', + ...lines, + '', + 'Use this list in STEP 4 (enable the matching signal sources) and STEP 5 ' + + '(surface these detected tools first in the connected-tools ask, ahead ' + + 'of the SaaS basics and the "others" option), exactly as the skill ' + + 'describes. Never enable or pre-select a tool the user has not confirmed.', + ].join('\n'); +} /** * Build the self-driving run prompt. The installed @@ -8,11 +36,18 @@ import type { PromptContext } from '@lib/agent/agent-runner'; * to verify); this prompt carries the order, the wizard-specific * mechanics (wizard_ask, abort signals), and the project URLs. * + * `detectedSources` is the deterministic codebase scan (from the detect step); + * it drives STEP 4/STEP 5 tool prioritisation. Empty is fine — the block then + * tells the agent to use the skill's default ordering. + * * Integration (when the project has no PostHog yet) runs as a separate phase * before this — the real integration program, with its own screens and task * list — so this prompt only covers the Self-driving steps. */ -export function buildSelfDrivingPrompt(ctx: PromptContext): string { +export function buildSelfDrivingPrompt( + ctx: PromptContext, + detectedSources: DetectedSource[] = [], +): string { const uiHost = ctx.host.appHost.replace(/\/$/, ''); const projectBase = `${uiHost}/project/${ctx.projectId}`; const integrationsSettingsUrl = `${projectBase}/settings/environment-integrations`; @@ -39,6 +74,8 @@ snippet, so repo evidence may rule a product IN but never OUT): - Exception autocapture (error tracking): ${optIn(optIns?.exceptionAutocapture)} - Surveys: ${optIn(optIns?.surveys)} +${renderDetectedTools(detectedSources)} + The installed skill is the source of truth for the HOW of every step: which MCP tools to call, which sources and scouts apply to this product, and how to verify each change. The STEPS below give the order and the @@ -124,7 +161,10 @@ STEP 4 — Enable signal sources. (skill: "Enable sources") STEP 5 — Offer issue-tracker integrations. (skill: "Connected tools") One batched multi-select wizard_ask for the external tools the skill - lists. The run auto-connects the ones it can (GitHub Issues, and + lists. Order it per the skill: the tools from the "Tools detected in + this codebase" list above come first, then the SaaS basics, then an + "others" option for the long tail — never dump the whole catalog. + The run auto-connects the ones it can (GitHub Issues, and Linear via a one-click OAuth link), verifying each with a single silent check — never nudge. For GitHub Issues: when the GitHub integration has exactly one repository connected, use that repo by diff --git a/src/lib/runners/run-wizard.ts b/src/lib/runners/run-wizard.ts index bf99bc9c8..c9beb886f 100644 --- a/src/lib/runners/run-wizard.ts +++ b/src/lib/runners/run-wizard.ts @@ -18,7 +18,8 @@ type Step = ProgramConfig['steps'][number]; /** The session a run step's agent runs in: scoped to the step's target dir * (e.g. a monorepo sub-app) with its own framework context, after any prep. - * A step without `targetDir` runs in the live session, unchanged. */ + * A step without `targetDir` runs in the live session, unchanged. + * The frameworkContext copy is shallow and unfiltered — name keys per owning program. */ async function prepareRunSession( step: Step, live: WizardSession,