Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 5 additions & 16 deletions src/lib/programs/__tests__/self-driving-detect.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,17 +88,15 @@ describe('detectSelfDrivingPrerequisites', () => {
});

it('keeps only the tools the inbox can connect', () => {
// `pg` and `stripe` are warehouse sources, not connected tools — surfacing
// them would lead STEP 5 with a database and a payment processor.
// `pg` and `stripe` are warehouse sources, not connected tools.
expect(detectedKinds({ pg: '^8.0.0', stripe: '^14.0.0' })).toEqual([]);
expect(detectedKinds({ pg: '^8.0.0', '@sentry/node': '^7.0.0' })).toEqual([
'Sentry',
]);
});

it('writes nothing when the codebase has no detectable tools', () => {
// Bare dir: valid (no detectError) but no tools to prioritise, so the key
// stays unset and STEP 5 falls back to the skill default.
// Bare dir: valid, but no tools to prioritise, so the key stays unset.
const session = buildSession({ installDir: tmpDir });
detectSelfDrivingPrerequisites(session, setCtx);

Expand All @@ -109,8 +107,7 @@ describe('detectSelfDrivingPrerequisites', () => {

describe('SELF_DRIVING_TOOL_KINDS', () => {
it('names only kinds the source registry can actually detect', () => {
// The filter is a plain string set, so a registry rename would silently
// drop a tool from the ask. Fail here instead.
// A plain string set, so a registry rename would otherwise drop a tool silently.
const known = new Set(SOURCE_DETECTORS.map((d) => d.kind));
expect([...SELF_DRIVING_TOOL_KINDS].filter((k) => !known.has(k))).toEqual(
[],
Expand All @@ -119,13 +116,7 @@ describe('SELF_DRIVING_TOOL_KINDS', () => {
});

describe('the detect step does not leak into the composed integration run', () => {
// Driven through the REAL store, the way run-wizard does it
// (`await store.runReadyHooks()`), because the leak lived in the plumbing
// rather than in `detectConnectedTools`: writing the warehouse program's key
// here put the scan into the integration agent's prompt, since the
// integrate-run phase inherits a copy of this frameworkContext and
// `posthog-integration` reads that key to build its prompt. Asserting on the
// setter's argument alone would not have caught it.
// Through the real store — the leak lived in the plumbing, not in detectConnectedTools.
let tmpDir: string;

beforeEach(() => {
Expand All @@ -148,9 +139,7 @@ describe('the detect step does not leak into the composed integration run', () =
expect(
getSelfDrivingDetectedTools(store.session).map((s) => s.kind),
).toContain('Sentry');
// ...and the integration program, reading its own key off the session it
// inherits, sees nothing — so its prompt is byte-identical to a run without
// self-driving in front of it.
// ...and the integration program, on the session it inherits, sees nothing.
expect(getDetectedWarehouseSources(store.session)).toEqual([]);
const inherited = {
...store.session,
Expand Down
66 changes: 6 additions & 60 deletions src/lib/programs/self-driving/detect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,21 +44,10 @@ export const POSTHOG_PRESENT_KEY = 'postHogPresent';
*/
export const SELF_DRIVING_INTEGRATE_PATH_KEY = 'selfDrivingIntegratePath';

/**
* frameworkContext key holding the tools this codebase uses that are worth
* promoting in the connected-tools ask. Self-driving's own key, not the
* warehouse program's `DETECTED_WAREHOUSE_SOURCES_KEY`: the integration program
* reads that one to build its prompt, and the integrate-run phase inherits a
* copy of this frameworkContext — so writing there would silently rewrite the
* integration agent's prompt on every self-driving run that installs PostHog
* first. (Its outro is safe: a composed run returns before `buildOutroData`.)
*/
/** Self-driving's own detected-tools key — not the warehouse one, which the integration program reads. */
export const SELF_DRIVING_DETECTED_TOOLS_KEY = 'selfDrivingDetectedTools';

/**
* Read the detected tools out of frameworkContext. Single accessor shared by
* the detect step and the prompt builder so the key + cast live in one place.
*/
/** Read the detected tools out of frameworkContext. */
export function getSelfDrivingDetectedTools(
session: WizardSession,
): DetectedSource[] {
Expand Down Expand Up @@ -323,33 +312,7 @@ export function detectSelfDrivingPrerequisites(
detectConnectedTools(installDir, setFrameworkContext);
}

/**
* Source kinds worth promoting in the connected-tools ask. The shared scanner
* matches the whole warehouse catalog (databases, payments, LLM vendors, ad
* platforms) and most of that has nothing to do with STEP 5: unfiltered, a
* routine repo leads the issue-tracker ask with Postgres (matched on
* `DATABASE_URL`), Stripe and OpenAI — the wall of irrelevant options this is
* supposed to remove.
*
* The intersection of two lists, computed rather than guessed: the tools the
* connected-tools ask actually offers (the `options` array in context-mill's
* `self-driving/references/5-connected-tools.md`) and the kinds this repo can
* detect (`SOURCE_DETECTORS`). Promoting anything outside that intersection is
* wasted at best and misleading at worst — a tool the ask never lists can't be
* picked, so pointing the agent at it sends it after a source it can't create.
*
* The ask's remaining options are deliberately absent because no detector
* matches them (Freshservice, Dixa, pganalyze, SonarQube, Semgrep, Rapid7
* InsightVM, Featurebase, Frill, Aha, UserVoice, AskNicely, Retently,
* Appfigures, AppFollow, Judge.me). They stay offered by the skill; they just
* never get promoted, which is the safe direction.
*
* A shadow list of both sources, and the guard test only covers one of them —
* it catches a kind that leaves `SOURCE_DETECTORS`, but nothing here can see the
* skill's catalog change in another repo. So when step 5's option list grows,
* reconcile against it. If that becomes a habit, the fix is a field on
* `SourceDetector` rather than a third copy of this list.
*/
/** Step 5's ask options intersected with `SOURCE_DETECTORS` — reconcile when the skill's catalog grows. */
export const SELF_DRIVING_TOOL_KINDS: ReadonlySet<string> = new Set([
// Issue trackers / code hosts
'Github',
Expand Down Expand Up @@ -380,17 +343,7 @@ export const SELF_DRIVING_TOOL_KINDS: ReadonlySet<string> = new Set([
'GoogleSearchConsole',
]);

/**
* Scan the codebase for the tools it uses that the inbox can connect (Sentry,
* Linear, GitHub, Zendesk, …) so STEP 5's connected-tools ask can surface those
* first instead of dumping the full source catalog on the user. Stashed under
* self-driving's own `SELF_DRIVING_DETECTED_TOOLS_KEY` and read back with
* `getSelfDrivingDetectedTools`.
*
* Best-effort: the connected-tools ask degrades to the skill's default
* ordering when nothing is detected, so a scan failure must never break the
* surrounding prerequisite check.
*/
/** Scan for inbox-connectable tools so STEP 5 can surface them first. Best-effort — never blocks detection. */
function detectConnectedTools(
installDir: string,
setFrameworkContext: (key: string, value: unknown) => void,
Expand All @@ -400,12 +353,7 @@ function detectConnectedTools(
SELF_DRIVING_TOOL_KINDS.has(s.kind),
);

// Tagged on every run that scans, including the empty case — without the
// zero rows there is no way to tell "the scan found nothing" from "this code
// path never ran", which is the first thing to check when the ask looks
// unprioritised. Deliberately NOT the `warehouse sources detected` event
// the integration flow emits: that metric's denominator is integration
// runs, and firing it here would fold self-driving runs into it.
// Tagged even at zero, so "found nothing" is distinguishable from "never ran".
analytics.setTag('connected_tools_detected_count', tools.length);
if (tools.length === 0) return;

Expand All @@ -415,9 +363,7 @@ function detectConnectedTools(
);
setFrameworkContext(SELF_DRIVING_DETECTED_TOOLS_KEY, tools);
} catch (error) {
// -1 rather than nothing: an absent tag would be indistinguishable from a
// build that never ran this scan, which is what the count is here to rule
// out. The captured exception carries the why.
// -1, not absent, so a failed scan stays distinguishable from one that never ran.
analytics.setTag('connected_tools_detected_count', -1);
analytics.captureException(
error instanceof Error ? error : new Error(String(error)),
Expand Down
8 changes: 1 addition & 7 deletions src/lib/runners/run-wizard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,7 @@ type Step = ProgramConfig['steps'][number];
/** The session a run step's agent runs in: scoped to the step's target dir
* (e.g. a monorepo sub-app) with its own framework context, after any prep.
* A step without `targetDir` runs in the live session, unchanged.
*
* The copy is shallow and unfiltered, so a composed sub-run inherits every
* frameworkContext key the host program wrote — including ones the sub-run's
* own program reads for its prompt. Name keys for the program that owns them
* (`selfDrivingDetectedTools`, not `detectedWarehouseSources`) so a host can't
* silently rewrite a spliced-in program's behaviour. If that collision shows up
* a second time, scope the inheritance here instead of renaming again. */
* The frameworkContext copy is shallow and unfiltered — name keys per owning program. */
async function prepareRunSession(
step: Step,
live: WizardSession,
Expand Down
Loading