From fd50bec3474266ccbbabc4ee93c93da5446ec20e Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" <29069505+gewenyu99@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:08:32 -0400 Subject: [PATCH] fix(self-driving): reconcile promoted tools with the real ask, restore the leak test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups to #1028, found by running the flow and reading the skill the ask actually comes from. The promoted-tools list was hand-written from a PR description rather than computed, so it was wrong both ways. Recomputed as the intersection of the options step 5 offers and the kinds this repo can detect: adds Shortcut, Honeybadger, Raygun, Kustomer, Plain, Canny, Productboard, Snyk and GoogleSearchConsole, all offered and all detectable, so all previously dropped on the floor; removes Intercom, which the ask never lists, so promoting it pointed the agent at a source it cannot create. The regression test guarding the frameworkContext fix is also missing from this branch — #1028 was merged at its first commit, so the test pushed afterwards was orphaned. Restored verbatim, and re-verified it fails when the shared-key write is reintroduced. Generated-By: PostHog Code Task-Id: df870021-4bdf-465b-ba4f-d486c37b97d3 --- .../__tests__/self-driving-detect.test.ts | 45 +++++++++++++++++++ src/lib/programs/self-driving/detect.ts | 42 ++++++++++++----- 2 files changed, 75 insertions(+), 12 deletions(-) diff --git a/src/lib/programs/__tests__/self-driving-detect.test.ts b/src/lib/programs/__tests__/self-driving-detect.test.ts index 27c4f6626..cdc5132b9 100644 --- a/src/lib/programs/__tests__/self-driving-detect.test.ts +++ b/src/lib/programs/__tests__/self-driving-detect.test.ts @@ -11,7 +11,10 @@ import { POSTHOG_MANIFESTS, SELF_DRIVING_DETECTED_TOOLS_KEY, SELF_DRIVING_TOOL_KINDS, + getSelfDrivingDetectedTools, } from '@lib/programs/self-driving/detect'; +import { getDetectedWarehouseSources } from '@lib/programs/warehouse-source/detect'; +import { WizardStore } from '@ui/tui/store'; import { SOURCE_DETECTORS } from '@lib/warehouse-sources/registry'; import type { DetectedSource } from '@lib/warehouse-sources/types'; import { toIntegrationReport } from '@lib/programs/self-driving/detect-agentic'; @@ -115,6 +118,48 @@ describe('SELF_DRIVING_TOOL_KINDS', () => { }); }); +describe('the detect step does not leak into the composed integration run', () => { + // Driven through the REAL store, the way run-wizard does it + // (`await store.runReadyHooks()`), because the leak lived in the plumbing + // rather than in `detectConnectedTools`: writing the warehouse program's key + // here put the scan into the integration agent's prompt, since the + // integrate-run phase inherits a copy of this frameworkContext and + // `posthog-integration` reads that key to build its prompt. Asserting on the + // setter's argument alone would not have caught it. + let tmpDir: string; + + beforeEach(() => { + tmpDir = makeTmpDir(); + fs.writeFileSync( + path.join(tmpDir, 'package.json'), + JSON.stringify({ + dependencies: { '@sentry/node': '^7.0.0', pg: '^8.0.0' }, + }), + ); + }); + afterEach(() => cleanup(tmpDir)); + + it('stashes under its own key and leaves the warehouse key untouched', async () => { + const store = new WizardStore('self-driving'); + store.session = buildSession({ installDir: tmpDir }); + await store.runReadyHooks(); + + // Self-driving sees its tools... + expect( + getSelfDrivingDetectedTools(store.session).map((s) => s.kind), + ).toContain('Sentry'); + // ...and the integration program, reading its own key off the session it + // inherits, sees nothing — so its prompt is byte-identical to a run without + // self-driving in front of it. + expect(getDetectedWarehouseSources(store.session)).toEqual([]); + const inherited = { + ...store.session, + frameworkContext: { ...store.session.frameworkContext }, + }; + expect(getDetectedWarehouseSources(inherited)).toEqual([]); + }); +}); + describe('SELF_DRIVING_ABORT_CASES', () => { const reasons = [ 'self-driving is not available for this project', diff --git a/src/lib/programs/self-driving/detect.ts b/src/lib/programs/self-driving/detect.ts index 93f2a5de8..87ffa660e 100644 --- a/src/lib/programs/self-driving/detect.ts +++ b/src/lib/programs/self-driving/detect.ts @@ -331,17 +331,24 @@ export function detectSelfDrivingPrerequisites( * `DATABASE_URL`), Stripe and OpenAI — the wall of irrelevant options this is * supposed to remove. * - * Exactly the kinds enumerated as inbox tools in #1022, which were sourced from - * the context-mill `self-driving` skill's connected-tools list. Deliberately no - * guesses beyond it: this list only decides what gets PROMOTED, so leaving a - * kind out costs a nudge (the skill still offers the tool), while putting a kind - * in that the inbox can't connect sends the agent after a source it can't - * create. When the skill's catalog grows, reconcile here. + * The intersection of two lists, computed rather than guessed: the tools the + * connected-tools ask actually offers (the `options` array in context-mill's + * `self-driving/references/5-connected-tools.md`) and the kinds this repo can + * detect (`SOURCE_DETECTORS`). Promoting anything outside that intersection is + * wasted at best and misleading at worst — a tool the ask never lists can't be + * picked, so pointing the agent at it sends it after a source it can't create. * - * A shadow list of the registry, so it drifts in one direction the guard test - * can't catch: a new inbox-connectable kind added to `SOURCE_DETECTORS` has to - * be added here too or it never gets promoted. If that bites, the fix is a - * field on `SourceDetector` rather than a third copy of this list. + * The ask's remaining options are deliberately absent because no detector + * matches them (Freshservice, Dixa, pganalyze, SonarQube, Semgrep, Rapid7 + * InsightVM, Featurebase, Frill, Aha, UserVoice, AskNicely, Retently, + * Appfigures, AppFollow, Judge.me). They stay offered by the skill; they just + * never get promoted, which is the safe direction. + * + * A shadow list of both sources, and the guard test only covers one of them — + * it catches a kind that leaves `SOURCE_DETECTORS`, but nothing here can see the + * skill's catalog change in another repo. So when step 5's option list grows, + * reconcile against it. If that becomes a habit, the fix is a field on + * `SourceDetector` rather than a third copy of this list. */ export const SELF_DRIVING_TOOL_KINDS: ReadonlySet = new Set([ // Issue trackers / code hosts @@ -350,16 +357,27 @@ export const SELF_DRIVING_TOOL_KINDS: ReadonlySet = new Set([ 'Gitea', 'Linear', 'Jira', - // Error trackers + 'Shortcut', + // Error tracking 'Sentry', 'Rollbar', 'Bugsnag', + 'Honeybadger', + 'Raygun', // Support desks 'Zendesk', 'Freshdesk', 'Front', 'Gorgias', - 'Intercom', + 'Kustomer', + 'Plain', + // Security scanners + 'Snyk', + // Product feedback + 'Canny', + 'Productboard', + // Search analytics + 'GoogleSearchConsole', ]); /**