diff --git a/package.json b/package.json index 617175ae8..5bdcb34d1 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "@langchain/core": "^0.3.40", "@modelcontextprotocol/sdk": "^1.29.0", "@posthog/warlock": "0.2.4", + "@typesafe-ai/sdk": "^0.5.7", "axios": "1.7.4", "fast-glob": "^3.3.3", "fflate": "^0.8.3", @@ -141,7 +142,8 @@ "prepare": "husky", "screens:check": "tsx scripts/check-screens.tsx", "wizard-ci-explore": "tsx scripts/wizard-ci-explore.no-jest.ts", - "wizard-ci-replay": "tsx scripts/tui-replay.no-jest.ts" + "wizard-ci-replay": "tsx scripts/tui-replay.no-jest.ts", + "jev-detect": "tsx scripts/jev-detect.no-jest.ts" }, "lint-staged": { ".claude/settings.json": "sh -c 'printf \"\\n\\033[31mDo not commit .claude/settings.json — use .claude/settings.local.json (gitignored).\\nUnstage with: git restore --staged .claude/settings.json\\033[0m\\n\\n\" >&2 && exit 1'", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7c2d2ae30..a658b7931 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,9 @@ importers: '@posthog/warlock': specifier: 0.2.4 version: 0.2.4 + '@typesafe-ai/sdk': + specifier: ^0.5.7 + version: 0.5.7 axios: specifier: 1.7.4 version: 1.7.4 @@ -2337,6 +2340,10 @@ packages: '@types/yargs@17.0.33': resolution: {integrity: sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==} + '@typesafe-ai/sdk@0.5.7': + resolution: {integrity: sha512-OrbhDXyxDoX9l45/Mcts+TsNuvubCuZQzINQv0BMbi32ztcapzgrCnMmaW9gK5+JkgQAZcnJH1f47lAur+i1hw==} + engines: {node: '>=20'} + '@typescript-eslint/eslint-plugin@5.62.0': resolution: {integrity: sha512-TiZzBSJja/LbhNPvk6yc0JrX9XqhQ0hdh6M2svYfsHGejaKFIAGd9MQ+ERIMzLGlN/kZoYIgdxFV0PuljTKXag==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -7733,6 +7740,8 @@ snapshots: dependencies: '@types/yargs-parser': 21.0.3 + '@typesafe-ai/sdk@0.5.7': {} + '@typescript-eslint/eslint-plugin@5.62.0(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1)(typescript@5.7.3)': dependencies: '@eslint-community/regexpp': 4.12.1 diff --git a/scripts/jev-detect.no-jest.ts b/scripts/jev-detect.no-jest.ts new file mode 100644 index 000000000..a934fae78 --- /dev/null +++ b/scripts/jev-detect.no-jest.ts @@ -0,0 +1,520 @@ +/** + * Standalone Jev detection runner — the demo / benchmarking entry point. + * Classifies one or more project dirs with Jev and (optionally) the static + * detector, without running the wizard. + * + * Usage: + * TYPESAFE_API_KEY=... pnpm jev-detect [dir...] [--compare] [--json] [--state] [--md[=path]] + * + * --compare also run static detectFramework and show agreement + * --json machine-readable output (one JSON object per dir) + * --state print the assembled classifier state and exit (no API call) + * --no-descend skip monorepo descent (per-subdirectory parallel calls) + * --md write a detailed markdown report (distributions, signals, + * full report JSON, request state); default path is + * scratch/jev-report-.md, override with --md=path + * + * Multiple dirs render as a comparison table — point it at a glob of + * wizard-workbench apps, e.g. "jev-detect .../apps/basic-integration//" expanded by the shell. + */ + +import path from 'path'; +import fs from 'fs'; +import { + detectWithJev, + sourceHits, + type JevChoiceAnswer, + type JevDetectionReport, +} from '@lib/detection/jev/index'; +import { assembleProjectState } from '@lib/detection/jev/state'; +import { JEV_QUESTIONS } from '@lib/detection/jev/questions'; +import { detectFramework } from '@lib/detection/framework'; + +type Row = { + dir: string; + jev: JevDetectionReport | { error: string }; + staticResult?: string | null; + staticMs?: number; +}; + +function bar(p: number, width = 20): string { + return '█'.repeat(Math.max(0, Math.round(p * width))).padEnd(width); +} + +function topProbabilities( + probabilities: Record, + n: number, +): Array<[string, number]> { + return Object.entries(probabilities) + .sort(([, a], [, b]) => b - a) + .slice(0, n); +} + +function printSingle(row: Row): void { + console.log(`\nJev detection — ${row.dir}\n`); + if ('error' in row.jev) { + console.log(` jev failed: ${row.jev.error}`); + return; + } + const r = row.jev; + for (const [label, p] of topProbabilities(r.framework.probabilities, 3)) { + const marker = label === r.framework.choice ? '→' : ' '; + console.log( + ` ${marker} framework ${label.padEnd(18)} ${bar(p)} ${p.toFixed(2)}`, + ); + } + console.log(` confidence ${r.framework.confidence.toFixed(2)}\n`); + if (r.variant) { + console.log( + ` variant ${r.variant.choice.padEnd( + 18, + )} conf ${r.variant.confidence.toFixed(2)} (${r.variant.key})`, + ); + } + const dims: Array<[string, { choice: string; confidence: number }]> = [ + ['language', r.language], + ['package mgr', r.packageManager], + ['use case', r.useCase], + ['industry', r.industry], + ]; + for (const [name, answer] of dims) { + console.log( + ` ${name.padEnd(12)} ${answer.choice.padEnd( + 18, + )} conf ${answer.confidence.toFixed(2)}`, + ); + } + const s = r.signals; + console.log( + `\n signals monorepo ${s.isMonorepo.toFixed( + 2, + )} · ts ${s.typescript.toFixed(2)} · posthog ${s.hasPosthog.toFixed( + 2, + )} · stripe ${s.hasStripe.toFixed(2)} · llm ${s.usesLlm.toFixed( + 2, + )} · auth ${s.hasAuth.toFixed(2)} · web ${s.webFrontend.toFixed( + 2, + )} · mobile ${s.isMobile.toFixed(2)}`, + ); + console.log( + ` logs structured ${s.structuredLogs.toFixed( + 2, + )} · opentelemetry ${s.openTelemetry.toFixed(2)}`, + ); + const fmtHits = (sources: Record): string => { + const hits = sourceHits(sources); + return hits.length > 0 + ? hits.map(([kind, p]) => `${kind} ${p.toFixed(2)}`).join(' · ') + : 'none ≥ 0.50'; + }; + console.log(` frameworks ${fmtHits(r.frameworkPresence)}`); + console.log(` languages ${fmtHits(r.languagePresence)}`); + console.log(` warehouse ${fmtHits(r.warehouseSources)}`); + console.log(` ai/llm ${fmtHits(r.aiSources)}`); + if (r.subprojects && r.subprojects.length > 0) { + console.log( + `\n monorepo descent — ${r.subprojects.length} subproject(s):`, + ); + for (const sub of r.subprojects) { + const sr = sub.report; + console.log( + ` ${sub.path.padEnd(28)} ${sr.framework.choice.padEnd(16)} ${( + sr.variant?.choice ?? '—' + ).padEnd(13)} conf ${sr.framework.confidence.toFixed(2)} · ${ + sr.useCase.choice + }`, + ); + } + } + console.log( + `\n ${ + r.model + } · ${r.usage.inputTokens.toLocaleString()} tokens in · $${r.estCostUsd.toFixed( + 4, + )} · ${r.durationMs}ms · state ${(r.stateBytes / 1024).toFixed(0)}KB`, + ); + if (row.staticResult !== undefined) { + const agree = + !('error' in row.jev) && + row.jev.framework.choice === (row.staticResult ?? 'none'); + console.log( + ` static ${(row.staticResult ?? 'none').padEnd(18)} ${ + row.staticMs + }ms · agreement ${agree ? '✓' : '✗'}`, + ); + } +} + +function printTable(rows: Row[]): void { + const name = (d: string): string => path.basename(d); + const width = Math.max(24, ...rows.map((r) => name(r.dir).length + 2)); + const header = `${'app'.padEnd(width)} ${'static'.padEnd(16)} ${'jev'.padEnd( + 18, + )} ${'variant'.padEnd(13)} conf agree ms tokens cost`; + console.log(`\n${header}\n${'─'.repeat(header.length)}`); + for (const row of rows) { + const staticLabel = (row.staticResult ?? 'none').padEnd(16); + if ('error' in row.jev) { + console.log( + `${name(row.dir).padEnd(width)} ${staticLabel} error: ${row.jev.error}`, + ); + continue; + } + const r = row.jev; + const agree = r.framework.choice === (row.staticResult ?? 'none'); + console.log( + `${name(row.dir).padEnd( + width, + )} ${staticLabel} ${r.framework.choice.padEnd(18)} ${( + r.variant?.choice ?? '—' + ).padEnd(13)} ${r.framework.confidence.toFixed(2)} ${ + agree ? '✓' : '✗' + } ${String(r.durationMs).padEnd(6)} ${String(tokensOf(r)).padEnd( + 8, + )} $${costOf(r).toFixed(5)}`, + ); + } + const ok = rows.filter( + (r) => + !('error' in r.jev) && + r.jev.framework.choice === (r.staticResult ?? 'none'), + ).length; + const reports = rows + .map((r) => r.jev) + .filter((j): j is JevDetectionReport => !('error' in j)); + const totalTokens = reports.reduce((n, r) => n + tokensOf(r), 0); + const totalCost = reports.reduce((n, r) => n + costOf(r), 0); + console.log( + `\nagreement: ${ok}/${ + rows.length + } · ${totalTokens.toLocaleString()} input tokens · $${totalCost.toFixed( + 4, + )} total`, + ); +} + +// ── Markdown report ────────────────────────────────────────────────── + +function mdProbTable(probabilities: Record): string { + const rows = Object.entries(probabilities) + .sort(([, a], [, b]) => b - a) + .filter(([, p], i) => i < 2 || p >= 0.005) + .map( + ([label, p]) => `| \`${label}\` | ${p.toFixed(3)} | ${bar(p, 24).trim()}`, + ); + return ['| label | p | |', '| --- | ---: | :-- |', ...rows].join('\n'); +} + +function mdChoiceSection(name: string, answer: JevChoiceAnswer): string { + return [ + `#### ${name} → \`${ + answer.choice + }\` (confidence ${answer.confidence.toFixed(2)})`, + '', + mdProbTable(answer.probabilities), + '', + ].join('\n'); +} + +function tokensOf(r: JevDetectionReport): number { + return ( + r.usage.inputTokens + + (r.subprojects?.reduce((n, s) => n + s.report.usage.inputTokens, 0) ?? 0) + ); +} + +function costOf(r: JevDetectionReport): number { + return ( + r.estCostUsd + + (r.subprojects?.reduce((n, s) => n + s.report.estCostUsd, 0) ?? 0) + ); +} + +function mdDescentSection( + subprojects: NonNullable, +): string { + const hitList = (sources: Record): string => + sourceHits(sources) + .map(([kind, p]) => `${kind} ${p.toFixed(2)}`) + .join(', ') || '—'; + return [ + `### Monorepo descent (${subprojects.length} subprojects, classified in parallel)`, + '', + '| path | framework | variant | conf | use case | warehouse | ai/llm |', + '| --- | --- | --- | ---: | --- | --- | --- |', + ...subprojects.map(({ path: p, report: sr }) => { + return `| \`${p}\` | \`${sr.framework.choice}\` | ${ + sr.variant?.choice ?? '—' + } | ${sr.framework.confidence.toFixed(2)} | ${ + sr.useCase.choice + } | ${hitList(sr.warehouseSources)} | ${hitList(sr.aiSources)} |`; + }), + '', + '
Subproject reports JSON', + '', + '```json', + JSON.stringify(subprojects, null, 2), + '```', + '', + '
', + '', + ].join('\n'); +} + +function mdSourceSection( + title: string, + sources: Record, +): string { + const shown = Object.entries(sources) + .filter(([, p]) => p >= 0.05) + .sort(([, a], [, b]) => b - a); + const hidden = Object.keys(sources).length - shown.length; + return [ + `### ${title} (noul ≥ 0.05)`, + '', + ...(shown.length > 0 + ? [ + '| kind | p |', + '| --- | ---: |', + ...shown.map(([kind, p]) => `| ${kind} | ${p.toFixed(3)} |`), + ] + : ['_none above 0.05_']), + '', + `_${hidden} more kind(s) below 0.05_`, + '', + ].join('\n'); +} + +function mdApp(row: Row): string { + const app = path.basename(row.dir); + const lines: string[] = [`## ${app}`, '', `\`${row.dir}\``, '']; + if ('error' in row.jev) { + return [...lines, `**Jev call failed:** ${row.jev.error}`, ''].join('\n'); + } + const r = row.jev; + const agree = + row.staticResult === undefined + ? null + : r.framework.choice === (row.staticResult ?? 'none'); + lines.push( + `**framework** \`${ + r.framework.choice + }\` (confidence ${r.framework.confidence.toFixed(2)})` + + (r.variant + ? ` · **variant** \`${r.variant.choice}\` (${ + r.variant.key + }, ${r.variant.confidence.toFixed(2)})` + : '') + + (row.staticResult !== undefined + ? ` · **static** \`${row.staticResult ?? 'none'}\` ${ + agree ? '✓ agree' : '✗ disagree' + }` + : ''), + '', + '### Choice distributions', + '', + mdChoiceSection('framework', r.framework), + mdChoiceSection('language', r.language), + mdChoiceSection('package_manager', r.packageManager), + mdChoiceSection('use_case', r.useCase), + mdChoiceSection('industry', r.industry), + '### Variant answers (speculative — only the matching one is used)', + '', + ...Object.entries(r.variants).map(([key, answer]) => + mdChoiceSection(key, answer), + ), + '### Signals (noul probabilities)', + '', + '| signal | p |', + '| --- | ---: |', + ...Object.entries(r.signals).map( + ([key, p]) => `| ${key} | ${p.toFixed(3)} |`, + ), + '', + mdSourceSection( + 'Framework presence (multi-label, independent of the primary choice)', + r.frameworkPresence, + ), + mdSourceSection('Language presence', r.languagePresence), + mdSourceSection('Warehouse sources', r.warehouseSources), + mdSourceSection('AI / LLM sources', r.aiSources), + ...(r.subprojects && r.subprojects.length > 0 + ? [mdDescentSection(r.subprojects)] + : []), + '### Call', + '', + `model \`${ + r.model + }\` · ${r.usage.inputTokens.toLocaleString()} tokens in / ${r.usage.outputTokens.toLocaleString()} out · $${r.estCostUsd.toFixed( + 5, + )} · ${r.durationMs}ms · state ${(r.stateBytes / 1024).toFixed(1)}KB` + + (row.staticMs !== undefined ? ` · static ${row.staticMs}ms` : ''), + '', + '
Full report JSON (mapped API response)', + '', + '```json', + JSON.stringify(r, null, 2), + '```', + '', + '
', + '', + ); + try { + const state = assembleProjectState(row.dir); + lines.push( + '
Request state (what was sent to the API)', + '', + '```json', + JSON.stringify(state, null, 2), + '```', + '', + '
', + '', + ); + } catch { + lines.push('_Request state unavailable (re-assembly failed)._', ''); + } + return lines.join('\n'); +} + +function mdReport(rows: Row[]): string { + const name = (d: string): string => path.basename(d); + const summary = rows.map((row) => { + if ('error' in row.jev) { + return `| ${name(row.dir)} | ${ + row.staticResult ?? '' + } | _error_ | | | | | | |`; + } + const r = row.jev; + const agree = + row.staticResult === undefined + ? '' + : r.framework.choice === (row.staticResult ?? 'none') + ? '✓' + : '✗'; + return `| ${name(row.dir)} | \`${row.staticResult ?? 'none'}\` | \`${ + r.framework.choice + }\` | ${r.variant?.choice ?? '—'} | ${r.framework.confidence.toFixed( + 2, + )} | ${agree} | ${r.durationMs} | ${tokensOf( + r, + ).toLocaleString()} | $${costOf(r).toFixed(5)} |`; + }); + const ok = rows.filter( + (r) => + !('error' in r.jev) && + r.jev.framework.choice === (r.staticResult ?? 'none'), + ).length; + const reports = rows + .map((r) => r.jev) + .filter((j): j is JevDetectionReport => !('error' in j)); + const totalTokens = reports.reduce((n, r) => n + tokensOf(r), 0); + const totalCost = reports.reduce((n, r) => n + costOf(r), 0); + return [ + '# Jev detection report', + '', + `_generated ${new Date().toISOString()} · ${ + rows.length + } project(s) · static↔jev agreement ${ok}/${ + rows.length + } · ${totalTokens.toLocaleString()} input tokens · $${totalCost.toFixed( + 4, + )} total_`, + '', + '## Summary', + '', + '| app | static | jev | variant | conf | agree | ms | tokens | cost |', + '| --- | --- | --- | --- | ---: | :-: | ---: | ---: | ---: |', + ...summary, + '', + '## API input', + '', + 'Every call is one `POST /v1/systemone` with `model: "jev-latest"`, the question catalog below, and a per-project `state` (embedded per app under "Request state").', + '', + '
Request questions (sent with every call)', + '', + '```json', + JSON.stringify(JEV_QUESTIONS, null, 2), + '```', + '', + '
', + '', + ...rows.map(mdApp), + ].join('\n'); +} + +function resolveMdPath(mdArg: string): string { + const custom = mdArg.includes('=') ? mdArg.slice(mdArg.indexOf('=') + 1) : ''; + if (custom) return path.resolve(custom); + const stamp = new Date() + .toISOString() + .replace(/[:.]/g, '-') + .replace('T', '-') + .slice(0, 19); + return path.resolve('scratch', `jev-report-${stamp}.md`); +} + +async function main(): Promise { + const args = process.argv.slice(2); + const flags = new Set(args.filter((a) => a.startsWith('--'))); + const mdArg = args.find((a) => a === '--md' || a.startsWith('--md=')); + const dirs = args + .filter((a) => !a.startsWith('--')) + .map((d) => path.resolve(d)) + .filter((d) => { + if (!fs.existsSync(d) || !fs.statSync(d).isDirectory()) { + console.error(`skipping: not a directory: ${d}`); + return false; + } + return true; + }); + if (dirs.length === 0) { + console.error( + 'Usage: TYPESAFE_API_KEY=... pnpm jev-detect [dir...] [--compare] [--json] [--state] [--md[=path]]', + ); + process.exit(1); + } + + if (flags.has('--state')) { + for (const dir of dirs) { + console.log(JSON.stringify(assembleProjectState(dir), null, 2)); + } + return; + } + + const compare = flags.has('--compare') || dirs.length > 1; + const rows: Row[] = []; + for (const dir of dirs) { + const row: Row = { + dir, + jev: await detectWithJev(dir, { + descend: !flags.has('--no-descend'), + }).catch((e: unknown) => ({ + error: e instanceof Error ? e.message : String(e), + })), + }; + if (compare) { + const started = Date.now(); + row.staticResult = (await detectFramework(dir)) ?? null; + row.staticMs = Date.now() - started; + } + rows.push(row); + } + + if (flags.has('--json')) { + console.log(JSON.stringify(rows, null, 2)); + } else if (rows.length === 1) { + printSingle(rows[0]); + } else { + printTable(rows); + } + + if (mdArg) { + const mdPath = resolveMdPath(mdArg); + fs.mkdirSync(path.dirname(mdPath), { recursive: true }); + fs.writeFileSync(mdPath, mdReport(rows)); + console.log(`\nmarkdown report: ${mdPath}`); + } +} + +void main(); diff --git a/src/env.ts b/src/env.ts index 1c5424c14..59581c61a 100644 --- a/src/env.ts +++ b/src/env.ts @@ -71,6 +71,9 @@ type RuntimeEnvKey = | 'POSTHOG_HANDOFF_OUTPUT_PATH' // Local/CI escape hatch to disable Warlock scanning without the PostHog flag. | 'POSTHOG_WIZARD_WARLOCK_DISABLED' + // Jev classifier detection prototype, dev builds only (@lib/detection/jev). + | 'WIZARD_JEV_DETECTION' + | 'TYPESAFE_API_KEY' | 'DEBUG' // Agent / MCP | 'MCP_URL' diff --git a/src/lib/detection/__tests__/jev.test.ts b/src/lib/detection/__tests__/jev.test.ts new file mode 100644 index 000000000..361ecbba0 --- /dev/null +++ b/src/lib/detection/__tests__/jev.test.ts @@ -0,0 +1,260 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { Integration } from '@lib/constants'; +import { + AI_KINDS, + AI_NOUL_PREFIX, + FRAMEWORK_CRITERIA, + JEV_QUESTIONS, + NO_FRAMEWORK, + PRESENCE_LANGUAGES, + PRESENCE_QUESTIONS, + SOURCE_QUESTIONS, + VARIANT_BY_INTEGRATION, + WAREHOUSE_KINDS, + WAREHOUSE_NOUL_PREFIX, + integrationFromChoice, + variantKeyFor, +} from '@lib/detection/jev/questions'; +import { + AI_SOURCE_KINDS, + CORE_SOURCE_KINDS, +} from '@lib/warehouse-sources/registry'; +import { + routeIntegration, + JEV_ACCEPT_CONFIDENCE, + JEV_AGREE_CONFIDENCE, +} from '@lib/detection/jev/route'; +import { + assembleProjectState, + renderFileTree, + scanProject, +} from '@lib/detection/jev/state'; + +describe('framework criteria', () => { + it('covers every Integration plus the none label, nothing else', () => { + const expected = new Set([ + ...Object.values(Integration), + NO_FRAMEWORK, + ]); + expect(new Set(Object.keys(FRAMEWORK_CRITERIA))).toEqual(expected); + }); + + it('maps every variant to a real question and a real Integration', () => { + for (const [integration, key] of Object.entries(VARIANT_BY_INTEGRATION)) { + expect(Object.values(Integration)).toContain(integration); + expect(Object.keys(JEV_QUESTIONS)).toContain(key); + } + expect(variantKeyFor(Integration.nextjs)).toBe('nextjs_router'); + expect(variantKeyFor(Integration.tanstackStart)).toBe( + 'tanstack_router_mode', + ); + expect(variantKeyFor(Integration.django)).toBeUndefined(); + expect(variantKeyFor(null)).toBeUndefined(); + }); + + it('maps choice labels back to Integrations', () => { + expect(integrationFromChoice('nextjs')).toBe(Integration.nextjs); + expect(integrationFromChoice('react-router')).toBe(Integration.reactRouter); + expect(integrationFromChoice(NO_FRAMEWORK)).toBeNull(); + expect(integrationFromChoice('not-a-framework')).toBeNull(); + }); +}); + +describe('presence questions', () => { + it('covers every Integration and language with prefixed nouls', () => { + const keys = Object.keys(PRESENCE_QUESTIONS); + for (const integration of Object.values(Integration)) { + expect(keys).toContain(`fw_${integration}`); + } + for (const language of PRESENCE_LANGUAGES) { + expect(keys).toContain(`lang_${language}`); + } + expect(keys).toHaveLength( + Object.values(Integration).length + PRESENCE_LANGUAGES.length, + ); + }); +}); + +describe('generated source questions', () => { + it('covers every core and AI registry kind exactly once', () => { + expect(new Set(WAREHOUSE_KINDS)).toEqual(new Set(CORE_SOURCE_KINDS)); + expect(new Set(AI_KINDS)).toEqual(new Set(AI_SOURCE_KINDS)); + expect(Object.keys(SOURCE_QUESTIONS)).toHaveLength( + WAREHOUSE_KINDS.length + AI_KINDS.length, + ); + }); + + it('generates prefixed nouls with instructions', () => { + for (const [key, question] of Object.entries(SOURCE_QUESTIONS)) { + expect( + key.startsWith(WAREHOUSE_NOUL_PREFIX) || key.startsWith(AI_NOUL_PREFIX), + ).toBe(true); + expect(question.type).toBe('noul'); + expect(typeof question.instructions).toBe('string'); + } + }); +}); + +describe('routeIntegration', () => { + const jev = ( + integration: Integration | null, + confidence: number, + ): { integration: Integration | null; confidence: number } => ({ + integration, + confidence, + }); + + it('falls back to static when jev failed or abstained', () => { + expect(routeIntegration(null, Integration.nextjs)).toEqual({ + integration: Integration.nextjs, + source: 'static', + }); + expect(routeIntegration(jev(null, 0.99), Integration.django)).toEqual({ + integration: Integration.django, + source: 'static', + }); + }); + + it('accepts jev outright at the accept gate', () => { + expect( + routeIntegration( + jev(Integration.sveltekit, JEV_ACCEPT_CONFIDENCE), + Integration.javascriptNode, + ), + ).toEqual({ integration: Integration.sveltekit, source: 'jev' }); + }); + + it('needs static agreement (or a static miss) in the mid band', () => { + const mid = jev(Integration.nuxt, JEV_AGREE_CONFIDENCE + 0.1); + expect(routeIntegration(mid, Integration.nuxt)).toEqual({ + integration: Integration.nuxt, + source: 'jev', + }); + expect(routeIntegration(mid, undefined)).toEqual({ + integration: Integration.nuxt, + source: 'jev', + }); + expect(routeIntegration(mid, Integration.vue)).toEqual({ + integration: Integration.vue, + source: 'static', + }); + }); + + it('lets static decide below the agree gate', () => { + expect( + routeIntegration( + jev(Integration.rails, JEV_AGREE_CONFIDENCE - 0.1), + undefined, + ), + ).toEqual({ integration: undefined, source: 'static' }); + }); +}); + +describe('renderFileTree', () => { + it('renders nested paths as an indented tree', () => { + const tree = renderFileTree([ + 'package.json', + 'app/page.tsx', + 'app/layout.tsx', + ]); + expect(tree).toBe( + ['package.json', 'app/', ' layout.tsx', ' page.tsx'].join('\n'), + ); + }); + + it('caps entries per directory with a legible remainder', () => { + const paths = Array.from( + { length: 30 }, + (_, i) => `f${String(i).padStart(2, '0')}.ts`, + ); + const tree = renderFileTree(paths, { maxEntriesPerDir: 5 }); + expect(tree.split('\n')).toHaveLength(6); + expect(tree).toContain('… +25 more files'); + }); + + it('caps total lines and marks truncation', () => { + const paths = Array.from({ length: 100 }, (_, i) => `dir${i}/file.ts`); + const tree = renderFileTree(paths, { maxLines: 10 }); + expect(tree.split('\n').length).toBeLessThanOrEqual(11); + expect(tree).toContain('… (tree truncated)'); + }); +}); + +describe('assembleProjectState', () => { + const tmpDirs: string[] = []; + afterAll(() => { + for (const dir of tmpDirs) fs.rmSync(dir, { recursive: true, force: true }); + }); + + function project(files: Record): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'wizard-jev-')); + tmpDirs.push(dir); + for (const [rel, content] of Object.entries(files)) { + const abs = path.join(dir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, content); + } + return dir; + } + + it('collects tree, trimmed manifests, lockfiles, and readme head', () => { + const dir = project({ + 'package.json': JSON.stringify({ + name: 'demo', + dependencies: { next: '15.0.0' }, + private: true, + description: 'dropped by trimming', + }), + 'pnpm-lock.yaml': 'lockfileVersion: 9', + 'README.md': '# Demo app', + 'app/page.tsx': 'export default function Page() {}', + '.env': 'SECRET=nope', + }); + const state = assembleProjectState(dir); + expect(state.file_tree).toContain('app/'); + expect(state.file_tree).toContain('page.tsx'); + expect(state.lockfiles).toEqual(['pnpm-lock.yaml']); + expect(state.readme_head).toBe('# Demo app'); + const pkg = JSON.parse(state.manifests['package.json']) as Record< + string, + unknown + >; + expect(pkg.dependencies).toEqual({ next: '15.0.0' }); + expect(pkg).not.toHaveProperty('description'); + // .env contents must never enter the state. + expect(JSON.stringify(state)).not.toContain('SECRET=nope'); + }); + + it('enumerates manifest-bearing subdirectories for descent, root excluded', () => { + const dir = project({ + 'package.json': '{"name":"root","workspaces":["apps/*"]}', + 'pnpm-workspace.yaml': 'packages:\n - apps/*', + 'apps/web/package.json': '{"name":"web"}', + 'apps/api/package.json': '{"name":"api"}', + 'services/worker/go.mod': 'module worker', + 'apps/web/src/index.ts': '', + }); + const { subprojectDirs } = scanProject(dir); + expect(subprojectDirs).toEqual( + expect.arrayContaining(['apps/web', 'apps/api', 'services/worker']), + ); + expect(subprojectDirs).not.toContain('.'); + expect(subprojectDirs).not.toContain('apps/web/src'); + }); + + it('prefers shallow manifests when capping', () => { + const files: Record = { + 'package.json': '{"name":"root"}', + }; + for (let i = 0; i < 30; i++) { + files[ + `packages/p${String(i).padStart(2, '0')}/package.json` + ] = `{"name":"p${i}"}`; + } + const state = assembleProjectState(project(files)); + expect(Object.keys(state.manifests)[0]).toBe('package.json'); + expect(Object.keys(state.manifests).length).toBeLessThanOrEqual(20); + }); +}); diff --git a/src/lib/detection/index.ts b/src/lib/detection/index.ts index 408cc5b1c..707cb4c95 100644 --- a/src/lib/detection/index.ts +++ b/src/lib/detection/index.ts @@ -14,3 +14,14 @@ export { type AgenticDetectOptions, type DetectEvent, } from './agentic.js'; +export { + detectFrameworkRouted, + getJevMode, + type JevMode, + type RoutedDetection, +} from './jev/route.js'; +export { + detectWithJev, + summarizeJevReport, + type JevDetectionReport, +} from './jev/index.js'; diff --git a/src/lib/detection/jev/index.ts b/src/lib/detection/jev/index.ts new file mode 100644 index 000000000..b78e5e7a5 --- /dev/null +++ b/src/lib/detection/jev/index.ts @@ -0,0 +1,276 @@ +/** + * Jev classifier detection — one TypeSafe systemOne call classifies the + * project along every onboarding dimension. Prototype: dev builds only, + * driven by WIZARD_JEV_DETECTION (see route.ts). Sits next to the other + * detection tools (framework, agentic, features) so it's discoverable. + */ + +import path from 'path'; +import { TypeSafeClient } from '@typesafe-ai/sdk'; +import { runtimeEnv } from '@env'; +import { Integration } from '@lib/constants'; +import { scanProject } from './state.js'; +import { + AI_KINDS, + AI_NOUL_PREFIX, + FRAMEWORK_PRESENCE_PREFIX, + JEV_QUESTIONS, + LANGUAGE_PRESENCE_PREFIX, + PRESENCE_LANGUAGES, + PRESENCE_QUESTIONS, + SOURCE_QUESTIONS, + WAREHOUSE_KINDS, + WAREHOUSE_NOUL_PREFIX, + integrationFromChoice, + variantKeyFor, + type VariantKey, +} from './questions.js'; + +export type JevChoiceAnswer = { + choice: string; + confidence: number; + probabilities: Record; +}; + +export type JevDetectionReport = { + framework: JevChoiceAnswer & { integration: Integration | null }; + /** The sub-framework answer relevant to the framework choice (e.g. Next.js router), if any. */ + variant: (JevChoiceAnswer & { key: VariantKey }) | null; + /** Every speculative variant answer, for shadow comparison and benchmarking. */ + variants: Record; + language: JevChoiceAnswer; + packageManager: JevChoiceAnswer; + useCase: JevChoiceAnswer; + industry: JevChoiceAnswer; + /** Noul probabilities (0–1), keyed by signal. */ + signals: { + isMonorepo: number; + typescript: number; + hasPosthog: number; + hasStripe: number; + usesLlm: number; + hasAuth: number; + webFrontend: number; + isMobile: number; + structuredLogs: number; + openTelemetry: number; + }; + /** Warehouse-source noul probabilities keyed by registry kind (core tier). */ + warehouseSources: Record; + /** AI/LLM source noul probabilities keyed by registry kind. */ + aiSources: Record; + /** Multi-label presence: "is X anywhere in the repo", keyed by Integration value. */ + frameworkPresence: Record; + /** Multi-label presence per language. */ + languagePresence: Record; + model: string; + usage: { inputTokens: number; outputTokens: number }; + estCostUsd: number; + durationMs: number; + stateBytes: number; + /** Per-subdirectory classifications when monorepo descent ran (one Jev call each, in parallel). */ + subprojects?: Array<{ path: string; report: JevDetectionReport }>; +}; + +/** Per-attempt timeout; the SDK retries once on transient failures. */ +const JEV_TIMEOUT_MS = 8_000; +/** jev-1.13 input pricing; output is free. */ +const JEV_USD_PER_INPUT_TOKEN = 0.042 / 1_000_000; +/** Descend into subprojects when is_monorepo clears this. */ +export const JEV_MONOREPO_CONFIDENCE = 0.8; + +export function getTypesafeApiKey(): string | undefined { + return runtimeEnv('TYPESAFE_API_KEY') || undefined; +} + +/** + * Classify the project with one Jev systemOne call. Throws without an API + * key. With `descend` (default off), a high is_monorepo answer fans out one + * additional call per manifest-bearing subdirectory, all in parallel — the + * decision tree lives here in code; Jev answers each node. + */ +export async function detectWithJev( + installDir: string, + options: { descend?: boolean } = {}, +): Promise { + const apiKey = getTypesafeApiKey(); + if (!apiKey) { + throw new Error('TYPESAFE_API_KEY is not set — Jev detection needs it.'); + } + const { state, subprojectDirs } = scanProject(installDir); + const stateBytes = JSON.stringify(state).length; + const client = new TypeSafeClient({ + apiKey, + timeout: JEV_TIMEOUT_MS, + retry: { maxRetries: 1 }, + }); + + const started = Date.now(); + // Generated source nouls ride along in the same call; the cast keeps the + // core answers typed (runtime carries a superset of the declared keys). + const questions = { + ...JEV_QUESTIONS, + ...SOURCE_QUESTIONS, + ...PRESENCE_QUESTIONS, + } as typeof JEV_QUESTIONS; + const result = await client.systemOne({ state, questions }); + const durationMs = Date.now() - started; + + const { answers, usage, model } = result; + const toChoice = (answer: { + choice: string; + confidence: number; + probabilities: Readonly>; + }): JevChoiceAnswer => ({ + choice: answer.choice, + confidence: answer.confidence, + probabilities: { ...answer.probabilities }, + }); + + const rawAnswers = answers as unknown as Record< + string, + { noul?: number } | undefined + >; + const collectNouls = ( + prefix: string, + kinds: readonly string[], + ): Record => + Object.fromEntries( + kinds.map((kind) => [kind, rawAnswers[`${prefix}${kind}`]?.noul ?? 0]), + ); + + const integration = integrationFromChoice(answers.framework.choice); + const variants: Record = { + nextjs_router: toChoice(answers.nextjs_router), + react_native_flavor: toChoice(answers.react_native_flavor), + astro_rendering: toChoice(answers.astro_rendering), + tanstack_router_mode: toChoice(answers.tanstack_router_mode), + laravel_stack: toChoice(answers.laravel_stack), + }; + const variantKey = variantKeyFor(integration); + + const report: JevDetectionReport = { + framework: { + ...toChoice(answers.framework), + integration, + }, + variant: variantKey ? { key: variantKey, ...variants[variantKey] } : null, + variants, + language: toChoice(answers.language), + packageManager: toChoice(answers.package_manager), + useCase: toChoice(answers.use_case), + industry: toChoice(answers.industry), + signals: { + isMonorepo: answers.is_monorepo.noul, + typescript: answers.typescript.noul, + hasPosthog: answers.has_posthog.noul, + hasStripe: answers.has_stripe.noul, + usesLlm: answers.uses_llm.noul, + hasAuth: answers.has_auth.noul, + webFrontend: answers.web_frontend.noul, + isMobile: answers.is_mobile.noul, + structuredLogs: answers.logs_structured.noul, + openTelemetry: answers.logs_otel.noul, + }, + warehouseSources: collectNouls(WAREHOUSE_NOUL_PREFIX, WAREHOUSE_KINDS), + aiSources: collectNouls(AI_NOUL_PREFIX, AI_KINDS), + frameworkPresence: collectNouls( + FRAMEWORK_PRESENCE_PREFIX, + Object.values(Integration), + ), + languagePresence: collectNouls( + LANGUAGE_PRESENCE_PREFIX, + PRESENCE_LANGUAGES, + ), + model, + usage: { + inputTokens: usage.input_tokens, + outputTokens: usage.output_tokens, + }, + estCostUsd: usage.input_tokens * JEV_USD_PER_INPUT_TOKEN, + durationMs, + stateBytes, + }; + + if ( + options.descend === true && + report.signals.isMonorepo >= JEV_MONOREPO_CONFIDENCE && + subprojectDirs.length > 0 + ) { + const subprojects = await Promise.all( + subprojectDirs.map((dir) => + detectWithJev(path.join(installDir, dir), { descend: false }) + .then((subReport) => ({ path: dir, report: subReport })) + .catch(() => null), + ), + ); + report.subprojects = subprojects.filter( + (s): s is { path: string; report: JevDetectionReport } => s !== null, + ); + } + + return report; +} + +/** Kinds whose noul probability clears the threshold, strongest first. */ +export function sourceHits( + sources: Record, + threshold = 0.5, +): Array<[string, number]> { + return Object.entries(sources) + .filter(([, p]) => p >= threshold) + .sort(([, a], [, b]) => b - a); +} + +/** Compact summary for frameworkContext / logs — no probability maps. */ +export function summarizeJevReport( + report: JevDetectionReport, +): Record { + const round = (n: number): number => Math.round(n * 100) / 100; + return { + framework: report.framework.choice, + frameworkConfidence: round(report.framework.confidence), + ...(report.variant + ? { + variant: `${report.variant.key}=${report.variant.choice}`, + variantConfidence: round(report.variant.confidence), + } + : {}), + language: report.language.choice, + packageManager: report.packageManager.choice, + useCase: report.useCase.choice, + industry: report.industry.choice, + isMonorepo: round(report.signals.isMonorepo), + typescript: round(report.signals.typescript), + hasPosthog: round(report.signals.hasPosthog), + hasStripe: round(report.signals.hasStripe), + usesLlm: round(report.signals.usesLlm), + hasAuth: round(report.signals.hasAuth), + webFrontend: round(report.signals.webFrontend), + isMobile: round(report.signals.isMobile), + structuredLogs: round(report.signals.structuredLogs), + openTelemetry: round(report.signals.openTelemetry), + warehouseSources: sourceHits(report.warehouseSources) + .map(([kind, p]) => `${kind}:${round(p)}`) + .join(', '), + aiSources: sourceHits(report.aiSources) + .map(([kind, p]) => `${kind}:${round(p)}`) + .join(', '), + frameworksPresent: sourceHits(report.frameworkPresence) + .map(([kind, p]) => `${kind}:${round(p)}`) + .join(', '), + languagesPresent: sourceHits(report.languagePresence) + .map(([kind, p]) => `${kind}:${round(p)}`) + .join(', '), + ...(report.subprojects + ? { + subprojects: report.subprojects + .map((s) => `${s.path}=${s.report.framework.choice}`) + .join(', '), + } + : {}), + model: report.model, + durationMs: report.durationMs, + inputTokens: report.usage.inputTokens, + }; +} diff --git a/src/lib/detection/jev/questions.ts b/src/lib/detection/jev/questions.ts new file mode 100644 index 000000000..6ce5e4f19 --- /dev/null +++ b/src/lib/detection/jev/questions.ts @@ -0,0 +1,335 @@ +/** + * The Jev question catalog — one systemOne fan-out covering framework + * identity plus every onboarding dimension. Speculative questions are free: + * all questions in a call are evaluated in parallel with no added latency. + */ + +import { choice, noul, type NoulQuestion } from '@typesafe-ai/sdk'; +import { Integration } from '@lib/constants'; +import { + SOURCE_DETECTORS, + AI_SOURCE_KINDS, + CORE_SOURCE_KINDS, +} from '@lib/warehouse-sources/registry'; +import type { SourceDetector } from '@lib/warehouse-sources/types'; + +/** Choice label meaning "no supported framework or language detected". */ +export const NO_FRAMEWORK = 'none'; + +/** + * One discriminating description per Integration, keyed by enum value. + * Kept in sync with the Integration enum by a unit test. + */ +export const FRAMEWORK_CRITERIA: Record = { + [Integration.nextjs]: + 'Next.js — "next" dependency, next.config.*, app/ or pages/ router', + [Integration.nuxt]: 'Nuxt — "nuxt" dependency, nuxt.config.*', + [Integration.vue]: + 'Vue SPA — "vue" dependency without Nuxt or another meta-framework', + [Integration.reactRouter]: + 'React Router v7 framework mode (formerly Remix) — "react-router" with @react-router/* packages', + [Integration.tanstackStart]: + 'TanStack Start — "@tanstack/react-start" dependency', + [Integration.tanstackRouter]: + 'TanStack Router SPA — "@tanstack/react-router" without TanStack Start', + [Integration.reactNative]: + 'React Native or Expo mobile app — "react-native" or "expo" dependency', + [Integration.angular]: 'Angular — "@angular/core" dependency, angular.json', + [Integration.astro]: 'Astro — "astro" dependency, astro.config.*', + [Integration.django]: 'Django — manage.py, django in requirements/pyproject', + [Integration.flask]: 'Flask — flask in requirements/pyproject', + [Integration.fastapi]: 'FastAPI — fastapi in requirements/pyproject', + [Integration.laravel]: + 'Laravel — composer.json with laravel/framework, artisan file', + [Integration.sveltekit]: + 'SvelteKit — "@sveltejs/kit" dependency, svelte.config.js', + [Integration.flutter]: 'Flutter — pubspec.yaml with a flutter sdk entry', + [Integration.kmp]: + 'Kotlin Multiplatform — build.gradle.kts with the kotlin multiplatform plugin', + [Integration.swift]: + 'Native iOS/macOS Swift app — .xcodeproj, Package.swift, or Podfile', + [Integration.android]: + 'Native Android app — build.gradle with com.android.application', + [Integration.rails]: + 'Ruby on Rails — Gemfile with rails, config/application.rb', + [Integration.elixir]: 'Elixir (often Phoenix) — mix.exs', + [Integration.go]: 'Go service or app — go.mod', + [Integration.rust]: 'Rust — Cargo.toml', + [Integration.java]: + 'JVM app (Maven or Gradle) that is not Android or Kotlin Multiplatform', + [Integration.python]: + 'Python project with no recognized web framework (scripts, ML, CLI)', + [Integration.ruby]: 'Ruby project that is not Rails', + [Integration.javascript_web]: + 'Browser JavaScript/TypeScript frontend with no recognized framework', + [Integration.javascriptNode]: + 'Node.js backend, CLI, or library with no recognized framework', + [NO_FRAMEWORK]: 'No supported framework or language is identifiable', +}; + +const LANGUAGE_CRITERIA = { + typescript: null, + javascript: null, + python: null, + ruby: null, + php: null, + go: null, + rust: null, + java: null, + kotlin: null, + swift: null, + dart: null, + elixir: null, + csharp: null, + other: null, +} as const; + +const PACKAGE_MANAGER_CRITERIA = { + npm: null, + pnpm: null, + yarn: null, + bun: null, + pip: null, + poetry: null, + uv: null, + bundler: null, + composer: null, + cargo: null, + gomod: 'Go modules', + gradle: null, + maven: null, + spm: 'Swift Package Manager', + cocoapods: null, + mix: null, + other: null, +} as const; + +const USE_CASE_CRITERIA = { + saas: 'Software-as-a-service product with user accounts', + ecommerce: 'Online store or checkout-centric product', + marketing_site: 'Marketing or landing site', + docs_site: 'Documentation or content site', + internal_tool: 'Internal dashboard or back-office tool', + ai_app: 'AI-first product (chat, agents, generation)', + api_service: 'Headless API or backend service', + mobile_app: 'Consumer or business mobile application', + library_or_cli: 'Library, SDK, or developer CLI', + other: null, +} as const; + +const INDUSTRY_CRITERIA = { + developer_tools: null, + fintech: null, + healthcare: null, + commerce_retail: null, + media_entertainment: null, + education: null, + productivity: null, + social: null, + logistics: null, + other_unclear: null, +} as const; + +/** The full fan-out sent with every detection call. */ +export const JEV_QUESTIONS = { + framework: choice( + 'Which framework or platform should the PostHog wizard integrate FIRST? Judge from the manifests and file tree. If several apps exist, pick the primary user-facing application. Prefer the most specific framework; use a language fallback only when no framework matches.', + FRAMEWORK_CRITERIA, + ), + language: choice( + 'Primary implementation language of this project', + LANGUAGE_CRITERIA, + ), + package_manager: choice( + 'Package manager this project is installed with, judged from lockfiles and manifests', + PACKAGE_MANAGER_CRITERIA, + ), + use_case: choice('What kind of product is this codebase?', USE_CASE_CRITERIA), + industry: choice('What industry does this product serve?', INDUSTRY_CRITERIA), + is_monorepo: noul( + 'This repository contains multiple independently deployable apps or packages', + ), + typescript: noul('The project is written in TypeScript'), + has_posthog: noul('A PostHog SDK is already a dependency of this project'), + has_stripe: noul('Stripe or another billing SDK is a dependency'), + uses_llm: noul('The project calls LLM APIs (OpenAI, Anthropic, etc.)'), + has_auth: noul('The application has user authentication'), + web_frontend: noul('There is a browser-rendered web frontend'), + is_mobile: noul('The primary deliverable is a mobile app'), + logs_structured: noul( + 'The project uses a structured logging library (e.g. winston, pino, structlog, loguru, monolog, zap, slog, tracing)', + ), + logs_otel: noul( + 'The project uses OpenTelemetry or an OTLP exporter (@opentelemetry/*, opentelemetry-sdk, otel collector config)', + ), + + // Sub-framework variants — speculative (fan-out is free); code reads only + // the one matching the framework choice. Mirrors gatherContext/setup + // disambiguation in the framework configs. + nextjs_router: choice('If this is a Next.js project, which router?', { + app_router: 'app/ directory with layout.tsx and page.tsx files', + pages_router: 'pages/ directory with _app and per-page files', + mixed: 'Both app/ and pages/ routers are present', + }), + react_native_flavor: choice( + 'If this is a React Native project, which flavor?', + { + expo: 'An "expo" dependency or app.json with an expo section', + bare: 'react-native without Expo', + }, + ), + astro_rendering: choice( + 'If this is an Astro project, which rendering mode?', + { + static: 'Fully prerendered, no SSR adapter', + server: 'output "server" or an SSR adapter renders every route', + hybrid: 'Mostly static with some server-rendered routes', + }, + ), + tanstack_router_mode: choice( + 'If this project uses TanStack Router, how are routes defined?', + { + file_based: + 'A routes/ directory of route files and a generated routeTree.gen.ts appear in the file tree', + code_based: + 'NO routeTree.gen.ts and NO routes/ directory anywhere in the file tree — routes are built with createRoute() calls inside ordinary source files', + }, + ), + laravel_stack: choice('If this is a Laravel project, which frontend stack?', { + standard: 'Blade views only', + inertia: 'Inertia.js with a JS frontend framework', + livewire: 'Livewire components', + }), +}; + +/** The variant question relevant to each framework, keyed by Integration. */ +export const VARIANT_BY_INTEGRATION = { + [Integration.nextjs]: 'nextjs_router', + [Integration.reactNative]: 'react_native_flavor', + [Integration.astro]: 'astro_rendering', + [Integration.tanstackRouter]: 'tanstack_router_mode', + [Integration.tanstackStart]: 'tanstack_router_mode', + [Integration.laravel]: 'laravel_stack', +} as const satisfies Partial>; + +export type VariantKey = + (typeof VARIANT_BY_INTEGRATION)[keyof typeof VARIANT_BY_INTEGRATION]; + +/** The variant question key for a detected integration, if it has one. */ +export function variantKeyFor( + integration: Integration | null, +): VariantKey | undefined { + if (integration === null) return undefined; + return (VARIANT_BY_INTEGRATION as Partial>)[ + integration + ]; +} + +// ── Generated source questions (warehouse + AI observability) ──────── +// One noul per warehouse-source kind, derived from the registry's own +// dep/env footprints — product knowledge stays in the registry. + +export const WAREHOUSE_NOUL_PREFIX = 'wh_'; +export const AI_NOUL_PREFIX = 'ai_'; + +function sourceNoul(detector: SourceDetector): NoulQuestion { + const deps = [ + ...(detector.signals.npm ?? []), + ...(detector.signals.python ?? []), + ...(detector.signals.ruby ?? []), + ].slice(0, 5); + const envs = (detector.signals.envKeys ?? []) + .slice(0, 2) + .map((re) => re.source.replace(/[\^$\\]/g, '').replace(/\(.*\)/, '*')); + const hints = [ + deps.length > 0 ? `dependencies like ${deps.join(', ')}` : '', + envs.length > 0 ? `env keys like ${envs.join(', ')}` : '', + ] + .filter(Boolean) + .join('; '); + return noul( + `The project uses ${detector.label}${hints ? ` (${hints})` : ''}`, + ); +} + +function buildSourceQuestions(kinds: ReadonlySet, prefix: string) { + const questions: Record = {}; + const kindList: string[] = []; + for (const detector of SOURCE_DETECTORS) { + if (!kinds.has(detector.kind)) continue; + if (`${prefix}${detector.kind}` in questions) continue; + questions[`${prefix}${detector.kind}`] = sourceNoul(detector); + kindList.push(detector.kind); + } + return { questions, kindList }; +} + +const warehouse = buildSourceQuestions( + CORE_SOURCE_KINDS, + WAREHOUSE_NOUL_PREFIX, +); +const ai = buildSourceQuestions(AI_SOURCE_KINDS, AI_NOUL_PREFIX); + +/** Warehouse-source kinds asked about, in registry order. */ +export const WAREHOUSE_KINDS: readonly string[] = warehouse.kindList; +/** AI/LLM source kinds asked about, in registry order. */ +export const AI_KINDS: readonly string[] = ai.kindList; + +/** Generated noul questions, merged into the systemOne call alongside JEV_QUESTIONS. */ +export const SOURCE_QUESTIONS: Record = { + ...warehouse.questions, + ...ai.questions, +}; + +// ── Presence questions (multi-label, independent of the primary Choice) ── +// The Choice picks ONE primary target; these nouls answer "is X present +// anywhere in the repo" independently, so a monorepo is just several yeses +// and "no" is every question's natural default. + +export const FRAMEWORK_PRESENCE_PREFIX = 'fw_'; +export const LANGUAGE_PRESENCE_PREFIX = 'lang_'; + +/** Languages asked about for presence (the language Choice minus 'other'). */ +export const PRESENCE_LANGUAGES: readonly string[] = [ + 'typescript', + 'javascript', + 'python', + 'ruby', + 'php', + 'go', + 'rust', + 'java', + 'kotlin', + 'swift', + 'dart', + 'elixir', + 'csharp', +]; + +function buildPresenceQuestions(): Record { + const questions: Record = {}; + for (const integration of Object.values(Integration)) { + questions[`${FRAMEWORK_PRESENCE_PREFIX}${integration}`] = noul( + `Somewhere in this repository there is a project matching: ${FRAMEWORK_CRITERIA[integration]}. It does not need to be the primary app.`, + ); + } + for (const language of PRESENCE_LANGUAGES) { + questions[`${LANGUAGE_PRESENCE_PREFIX}${language}`] = noul( + `Somewhere in this repository there is ${language} source code or a ${language} project`, + ); + } + return questions; +} + +/** Presence nouls for every Integration and language, merged into the call. */ +export const PRESENCE_QUESTIONS: Record = + buildPresenceQuestions(); + +const INTEGRATION_BY_VALUE = new Map( + Object.values(Integration).map((value) => [value, value as Integration]), +); + +/** Map a framework choice label back to an Integration; null for "none" or anything unrecognized. */ +export function integrationFromChoice(label: string): Integration | null { + return INTEGRATION_BY_VALUE.get(label) ?? null; +} diff --git a/src/lib/detection/jev/route.ts b/src/lib/detection/jev/route.ts new file mode 100644 index 000000000..d30006015 --- /dev/null +++ b/src/lib/detection/jev/route.ts @@ -0,0 +1,98 @@ +/** + * Confidence-routed framework detection: Jev classifies, static registry + * order stays the fallback. Three modes via WIZARD_JEV_DETECTION: + * + * off (default) static detection only, untouched. + * shadow run both, log agreement; static still decides. + * primary Jev decides above the confidence gates, static below them. + * + * Dev builds only — published builds are pinned to 'off'. + */ + +import { IS_PRODUCTION_BUILD, runtimeEnv } from '@env'; +import type { Integration } from '@lib/constants'; +import { logToFile } from '@utils/debug'; +import { detectFramework } from '../framework.js'; +import { + detectWithJev, + summarizeJevReport, + type JevDetectionReport, +} from './index.js'; + +export type JevMode = 'off' | 'shadow' | 'primary'; + +/** Accept Jev's answer outright at or above this confidence. */ +export const JEV_ACCEPT_CONFIDENCE = 0.85; +/** Between the gates, Jev needs static agreement (or a static miss) to win. */ +export const JEV_AGREE_CONFIDENCE = 0.6; + +export function getJevMode(): JevMode { + if (IS_PRODUCTION_BUILD) return 'off'; + const raw = runtimeEnv('WIZARD_JEV_DETECTION')?.toLowerCase(); + return raw === 'shadow' || raw === 'primary' ? raw : 'off'; +} + +export type RoutedDetection = { + integration: Integration | undefined; + source: 'static' | 'jev'; + jevReport: JevDetectionReport | null; +}; + +/** Pure routing decision — exported for testing. */ +export function routeIntegration( + jev: { integration: Integration | null; confidence: number } | null, + staticResult: Integration | undefined, +): { integration: Integration | undefined; source: 'static' | 'jev' } { + if (!jev || jev.integration === null) { + return { integration: staticResult, source: 'static' }; + } + if (jev.confidence >= JEV_ACCEPT_CONFIDENCE) { + return { integration: jev.integration, source: 'jev' }; + } + if (jev.confidence >= JEV_AGREE_CONFIDENCE) { + if (staticResult === undefined || staticResult === jev.integration) { + return { integration: jev.integration, source: 'jev' }; + } + } + return { integration: staticResult, source: 'static' }; +} + +/** + * Drop-in replacement for detectFramework in the main detect step. In 'off' + * mode it IS detectFramework; otherwise both detectors run concurrently and + * the mode decides who wins. A failed Jev call never blocks the run. + */ +export async function detectFrameworkRouted( + installDir: string, +): Promise { + const mode = getJevMode(); + if (mode === 'off') { + const integration = await detectFramework(installDir); + return { integration, source: 'static', jevReport: null }; + } + + const [staticResult, jevReport] = await Promise.all([ + detectFramework(installDir), + detectWithJev(installDir).catch((error: unknown) => { + logToFile(`[jev] detection failed: ${String(error)}`); + return null; + }), + ]); + + if (jevReport) { + const agree = jevReport.framework.integration === (staticResult ?? null); + logToFile( + `[jev] mode=${mode} framework=${jevReport.framework.choice} ` + + `conf=${jevReport.framework.confidence.toFixed(2)} ` + + `static=${staticResult ?? 'none'} agree=${String(agree)} ` + + `ms=${jevReport.durationMs} tokens=${jevReport.usage.inputTokens}`, + ); + logToFile(`[jev] report: ${JSON.stringify(summarizeJevReport(jevReport))}`); + } + + if (mode === 'shadow' || !jevReport) { + return { integration: staticResult, source: 'static', jevReport }; + } + const routed = routeIntegration(jevReport.framework, staticResult); + return { ...routed, jevReport }; +} diff --git a/src/lib/detection/jev/state.ts b/src/lib/detection/jev/state.ts new file mode 100644 index 000000000..ca2034222 --- /dev/null +++ b/src/lib/detection/jev/state.ts @@ -0,0 +1,238 @@ +/** + * Assembles the Jev classifier state for a project: a truncated file tree, + * manifest heads, README head, and lockfile names. Everything flows through + * the bounded-fs primitives; budgets keep the worst case near ~70KB (~17k + * tokens), inside Jev's 32k-token state window. Never sends .env or source + * file bodies — tree paths and manifest/README heads only. + */ + +import path from 'path'; +import { + walkProjectFiles, + readFileHead, + readProjectFile, +} from '@utils/bounded-fs'; +import { PROJECT_MANIFESTS } from '../agentic.js'; + +export type JevProjectState = { + file_tree: string; + manifests: Record; + lockfiles: string[]; + readme_head?: string; +}; + +const MAX_TREE_DEPTH = 4; +const MAX_TREE_FILES = 4_000; +const MAX_ENTRIES_PER_DIR = 25; +const MAX_TREE_LINES = 500; +const MAX_MANIFEST_FILES = 20; +const MANIFEST_HEAD_BYTES = 3_000; +const MANIFEST_TOTAL_BYTES = 48_000; +const README_HEAD_BYTES = 1_500; + +const MANIFEST_BASENAMES = new Set( + PROJECT_MANIFESTS.filter((m) => !m.includes('/') && !m.startsWith('*')), +); + +const LOCKFILE_NAMES = new Set([ + 'pnpm-lock.yaml', + 'yarn.lock', + 'package-lock.json', + 'bun.lockb', + 'bun.lock', + 'poetry.lock', + 'uv.lock', + 'Pipfile.lock', + 'Gemfile.lock', + 'composer.lock', + 'Cargo.lock', + 'go.sum', + 'mix.lock', + 'Podfile.lock', + 'gradle.lockfile', +]); + +function isManifest(relPath: string, name: string): boolean { + return ( + MANIFEST_BASENAMES.has(name) || + name.endsWith('.csproj') || + relPath.endsWith('gradle/libs.versions.toml') + ); +} + +type TreeNode = { files: string[]; dirs: Map }; + +function insertPath(root: TreeNode, relPath: string): void { + const segments = relPath.split('/'); + let node = root; + for (const segment of segments.slice(0, -1)) { + let child = node.dirs.get(segment); + if (!child) { + child = { files: [], dirs: new Map() }; + node.dirs.set(segment, child); + } + node = child; + } + node.files.push(segments[segments.length - 1]); +} + +/** + * Render relative file paths as an indented tree with per-directory and + * total-line caps, truncating legibly. Pure — exported for testing. + */ +export function renderFileTree( + relPaths: readonly string[], + opts: { maxEntriesPerDir?: number; maxLines?: number } = {}, +): string { + const maxEntries = opts.maxEntriesPerDir ?? MAX_ENTRIES_PER_DIR; + const maxLines = opts.maxLines ?? MAX_TREE_LINES; + const root: TreeNode = { files: [], dirs: new Map() }; + for (const relPath of relPaths) insertPath(root, relPath); + + const lines: string[] = []; + let truncated = false; + const render = (node: TreeNode, indent: string): void => { + if (truncated) return; + const files = [...node.files].sort(); + for (const [i, file] of files.entries()) { + if (lines.length >= maxLines) { + truncated = true; + return; + } + if (i >= maxEntries) { + lines.push(`${indent}… +${files.length - maxEntries} more files`); + break; + } + lines.push(`${indent}${file}`); + } + const dirs = [...node.dirs.entries()].sort(([a], [b]) => + a.localeCompare(b), + ); + for (const [i, [name, child]] of dirs.entries()) { + if (lines.length >= maxLines) { + truncated = true; + return; + } + if (i >= maxEntries) { + lines.push(`${indent}… +${dirs.length - maxEntries} more directories`); + break; + } + lines.push(`${indent}${name}/`); + render(child, `${indent} `); + } + }; + render(root, ''); + if (truncated) lines.push('… (tree truncated)'); + return lines.join('\n'); +} + +/** Trim a package.json to its classification-relevant fields; head on parse failure. */ +function manifestContent(fullPath: string, name: string): string | null { + if (name === 'package.json') { + const raw = readProjectFile(fullPath); + if (raw) { + try { + const pkg = JSON.parse(raw) as Record; + return JSON.stringify({ + name: pkg.name, + workspaces: pkg.workspaces, + scripts: pkg.scripts, + dependencies: pkg.dependencies, + devDependencies: pkg.devDependencies, + }); + } catch { + // fall through to head + } + } + } + return readFileHead(fullPath, MANIFEST_HEAD_BYTES); +} + +/** Subdirectory candidates for monorepo descent, capped. */ +const MAX_SUBPROJECTS = 12; + +export type ScannedProject = { + state: JevProjectState; + /** Repo-relative dirs holding a manifest, shallowest first, root excluded. */ + subprojectDirs: string[]; +}; + +/** Walk the project once and assemble the classifier state. */ +export function assembleProjectState(installDir: string): JevProjectState { + return scanProject(installDir).state; +} + +/** Walk once, returning both the state and monorepo descent candidates. */ +export function scanProject(installDir: string): ScannedProject { + const relPaths: string[] = []; + const manifestPaths: string[] = []; + const lockfiles: string[] = []; + let readmePath: string | null = null; + + walkProjectFiles( + installDir, + (name, fullPath) => { + const relPath = path + .relative(installDir, fullPath) + .split(path.sep) + .join('/'); + // Only the rendered tree shares this cap — manifests, lockfiles, and + // the README keep collecting for the rest of the (bounded) walk, so + // huge repos don't lose descent candidates to tree truncation. + if (relPaths.length < MAX_TREE_FILES) relPaths.push(relPath); + if (isManifest(relPath, name)) manifestPaths.push(relPath); + if (LOCKFILE_NAMES.has(name)) lockfiles.push(relPath); + if (!readmePath && relPath.toLowerCase() === 'readme.md') { + readmePath = relPath; + } + }, + MAX_TREE_DEPTH, + ); + + // Shallowest manifests carry the most signal; deep ones drop first. + manifestPaths.sort( + (a, b) => a.split('/').length - b.split('/').length || a.localeCompare(b), + ); + + const manifests: Record = {}; + let manifestBytes = 0; + for (const relPath of manifestPaths.slice(0, MAX_MANIFEST_FILES)) { + const content = manifestContent( + path.join(installDir, relPath), + path.basename(relPath), + ); + if (!content) continue; + if (manifestBytes + content.length > MANIFEST_TOTAL_BYTES) break; + manifests[relPath] = content; + manifestBytes += content.length; + } + + const readmeHead = readmePath + ? readFileHead(path.join(installDir, readmePath), README_HEAD_BYTES) + : null; + + // A dir owning a manifest is a descent candidate (agentic.ts's project + // rule). Xcode wrappers and gradle catalogs resolve to their parent. + const subprojectDirs: string[] = []; + const seen = new Set(); + for (const relPath of manifestPaths) { + let dir = path.posix.dirname(relPath); + if (dir.endsWith('.xcodeproj') || path.posix.basename(dir) === 'gradle') { + dir = path.posix.dirname(dir); + } + if (dir === '.' || seen.has(dir)) continue; + seen.add(dir); + subprojectDirs.push(dir); + if (subprojectDirs.length >= MAX_SUBPROJECTS) break; + } + + return { + state: { + file_tree: renderFileTree(relPaths), + manifests, + lockfiles, + ...(readmeHead ? { readme_head: readmeHead } : {}), + }, + subprojectDirs, + }; +} diff --git a/src/lib/programs/posthog-integration/detect.ts b/src/lib/programs/posthog-integration/detect.ts index f27faa6a9..01df8d9da 100644 --- a/src/lib/programs/posthog-integration/detect.ts +++ b/src/lib/programs/posthog-integration/detect.ts @@ -18,10 +18,11 @@ import { } from '@lib/wizard-session'; import { FRAMEWORK_REGISTRY } from '@lib/registry'; import { - detectFramework, + detectFrameworkRouted, discoverFeatures, gatherFrameworkContext, checkFrameworkVersion, + summarizeJevReport, } from '@lib/detection/index'; import { analytics } from '@utils/analytics'; import { detectWarehouseSources } from '@lib/warehouse-sources/detect'; @@ -39,7 +40,12 @@ export async function detectPostHogIntegration( const session = ctx.session; const installDir = session.installDir; - const detectedIntegration = await detectFramework(installDir); + // Routed detection: static-only unless WIZARD_JEV_DETECTION says otherwise. + const { integration: detectedIntegration, jevReport } = + await detectFrameworkRouted(installDir); + if (jevReport) { + ctx.setFrameworkContext('jevDetection', summarizeJevReport(jevReport)); + } if (detectedIntegration) { const config = FRAMEWORK_REGISTRY[detectedIntegration]; diff --git a/src/lib/warehouse-sources/registry.ts b/src/lib/warehouse-sources/registry.ts index 0bd2a8674..7469c0a3b 100644 --- a/src/lib/warehouse-sources/registry.ts +++ b/src/lib/warehouse-sources/registry.ts @@ -2537,3 +2537,8 @@ export const SOURCE_DETECTORS: SourceDetector[] = [ export const AI_SOURCE_KINDS: ReadonlySet = new Set( LLM_SOURCE_DETECTORS.map((detector) => detector.kind), ); + +/** Kinds from the core section above — databases and primary SaaS sources. */ +export const CORE_SOURCE_KINDS: ReadonlySet = new Set( + CORE_SOURCE_DETECTORS.map((detector) => detector.kind), +);