diff --git a/.gitignore b/.gitignore index 92aa0ee8..a03616d2 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ dist/ # Eval results tests/eval-results/ +.next/ +.react-router/ diff --git a/skills/workos-authkit-nextjs/SKILL.md b/skills/workos-authkit-nextjs/SKILL.md index fb9ff394..215ce099 100644 --- a/skills/workos-authkit-nextjs/SKILL.md +++ b/skills/workos-authkit-nextjs/SKILL.md @@ -55,6 +55,45 @@ Next.js version? Middleware/proxy code: See README for `authkitMiddleware()` export pattern. +### Existing Middleware (IMPORTANT) + +If `middleware.ts` already exists with custom logic (rate limiting, logging, headers, etc.), use the **`authkit()` composable function** instead of `authkitMiddleware`. + +**Pattern for composing with existing middleware:** + +```typescript +import { NextRequest, NextResponse } from 'next/server'; +import { authkit, handleAuthkitHeaders } from '@workos-inc/authkit-nextjs'; + +export default async function middleware(request: NextRequest) { + // 1. Get auth session and headers from AuthKit + const { session, headers, authorizationUrl } = await authkit(request); + const { pathname } = request.nextUrl; + + // 2. === YOUR EXISTING MIDDLEWARE LOGIC === + // Rate limiting, logging, custom headers, etc. + const rateLimitResult = checkRateLimit(request); + if (!rateLimitResult.allowed) { + return new NextResponse('Too Many Requests', { status: 429 }); + } + + // 3. Protect routes - redirect to auth if needed + if (pathname.startsWith('/dashboard') && !session.user && authorizationUrl) { + return handleAuthkitHeaders(request, headers, { redirect: authorizationUrl }); + } + + // 4. Continue with AuthKit headers properly handled + return handleAuthkitHeaders(request, headers); +} +``` + +**Key functions:** +- `authkit(request)` - Returns `{ session, headers, authorizationUrl }` for composition +- `handleAuthkitHeaders(request, headers, options?)` - Ensures AuthKit headers pass through correctly +- For rewrites, use `partitionAuthkitHeaders()` and `applyResponseHeaders()` (see README) + +**Critical:** Always return via `handleAuthkitHeaders()` to ensure `withAuth()` works in pages. + ## Step 5: Create Callback Route Parse `NEXT_PUBLIC_WORKOS_REDIRECT_URI` to determine route path: @@ -78,33 +117,57 @@ export const GET = handleAuth(); Check README for exact usage. If build fails with "cookies outside request scope", the handler is likely missing async/await. -## Step 6: Provider Setup +## Step 6: Provider Setup (REQUIRED) + +**CRITICAL:** You MUST wrap the app in `AuthKitProvider` in `app/layout.tsx`. + +This is required for: +- Client-side auth state via `useAuth()` hook +- Consistent auth UX across client/server boundaries +- Proper migration from Auth0 (which uses client-side auth) + +```tsx +// app/layout.tsx +import { AuthKitProvider } from '@workos-inc/authkit-nextjs'; + +export default function RootLayout({ children }: { children: React.ReactNode }) { + return ( + + + {children} + + + ); +} +``` + +Check README for exact import path - it may be a subpath export like `@workos-inc/authkit-nextjs/components`. -Wrap app in `AuthKitProvider` in `app/layout.tsx`. See README for import path. +**Do NOT skip this step** even if using server-side auth patterns elsewhere. ## Step 7: UI Integration Add auth UI to `app/page.tsx` using SDK functions. See README for `getUser`, `getSignInUrl`, `signOut` usage. -## Verification Checklist +## Verification Checklist (ALL MUST PASS) -Run these commands to confirm integration: +Run these commands to confirm integration. **Do not mark complete until all pass:** ```bash -# Check middleware/proxy exists (one should match) +# 1. Check middleware/proxy exists (one should match) ls proxy.ts middleware.ts src/proxy.ts src/middleware.ts 2>/dev/null -# Check provider is wrapped -grep -l "AuthKitProvider" app/layout.tsx +# 2. CRITICAL: Check AuthKitProvider is in layout (REQUIRED) +grep "AuthKitProvider" app/layout.tsx || echo "FAIL: AuthKitProvider missing from layout" -# Check callback route exists +# 3. Check callback route exists find app -name "route.ts" -path "*/callback/*" -# Build succeeds +# 4. Build succeeds npm run build ``` -All checks must pass before marking complete. +**If check #2 fails:** Go back to Step 6 and add AuthKitProvider. This is not optional. ## Error Recovery diff --git a/skills/workos-authkit-tanstack-start/SKILL.md b/skills/workos-authkit-tanstack-start/SKILL.md index 5d0e531e..c83ed2d5 100644 --- a/skills/workos-authkit-tanstack-start/SKILL.md +++ b/skills/workos-authkit-tanstack-start/SKILL.md @@ -43,6 +43,7 @@ From README, extract: ## Directory Structure Detection **Modern TanStack Start (v1.132+)** uses `src/`: + ``` src/ ├── start.ts # Middleware config (CRITICAL) @@ -54,6 +55,7 @@ src/ ``` **Legacy (vinxi-based)** uses `app/`: + ``` app/ ├── start.ts or router.tsx @@ -62,6 +64,7 @@ app/ ``` **Detection:** + ```bash ls src/routes 2>/dev/null && echo "Modern (src/)" || echo "Legacy (app/)" ``` @@ -94,6 +97,7 @@ export default { ``` Alternative pattern with createStart: + ```typescript import { createStart } from '@tanstack/react-start'; import { authkitMiddleware } from '@workos/authkit-tanstack-react-start'; @@ -132,6 +136,7 @@ export const Route = createFileRoute('/api/auth/callback')({ ``` **Key points:** + - Use `handleCallbackRoute()` - do not write custom OAuth logic - Route path string must match the URI path exactly - This is a server-only route (no component needed) @@ -221,6 +226,7 @@ function Profile() { **Cause:** Route file path doesn't match WORKOS_REDIRECT_URI **Fix:** + - URI `/api/auth/callback` → file `src/routes/api.auth.callback.tsx` (flat) or `app/routes/api/auth/callback.tsx` (nested) - Route path string in `createFileRoute()` must match exactly @@ -242,6 +248,7 @@ function Profile() { ## SDK Exports Reference **Server (main export):** + - `authkitMiddleware()` - Request middleware - `handleCallbackRoute()` - OAuth callback handler - `getAuth()` - Get current session @@ -250,6 +257,7 @@ function Profile() { - `switchToOrganization()` - Change org context **Client (`/client` subpath):** + - `AuthKitProvider` - Context provider - `useAuth()` - Auth state hook - `useAccessToken()` - Token management diff --git a/tests/evals/README.md b/tests/evals/README.md index 71009ee8..292e29d5 100644 --- a/tests/evals/README.md +++ b/tests/evals/README.md @@ -1,6 +1,6 @@ # Installer Evaluations -Automated evaluation framework for testing WorkOS AuthKit installer skills against realistic project scenarios. +Automated evaluation framework for testing WorkOS AuthKit installer skills. ## Quick Start @@ -11,72 +11,137 @@ pnpm eval # Run specific framework pnpm eval --framework=nextjs -# Run specific scenario -pnpm eval --framework=react --state=example-auth0 +# Run with quality grading +pnpm eval --quality ``` +## Success Criteria + +The eval framework validates against these thresholds: + +| Metric | Threshold | +| ----------------------- | --------- | +| First-attempt pass rate | ≥90% | +| With-retry pass rate | ≥95% | + +Use `--no-fail` to run without exit code validation. + ## Test Matrix -The framework tests 10 scenarios (5 frameworks × 2 project states): +**Scenarios: 24 total (5 frameworks × 4-5 states)** -| State | Description | -| --------------- | ---------------------------------------------------- | -| `example` | Project with routes, components, custom config | -| `example-auth0` | Project with Auth0 authentication already integrated | +| State | Description | +| ------------------------ | --------------------------------- | +| `example` | Clean project, no existing auth | +| `example-auth0` | Project with Auth0 to migrate | +| `partial-install` | Half-completed AuthKit attempt | +| `typescript-strict` | Strict TypeScript configuration | +| `conflicting-middleware` | Existing middleware to merge | -| Framework | Skill | Key Checks | -| ---------------- | ----------------------------- | ---------------------------------------------- | -| `nextjs` | workos-authkit-nextjs | middleware.ts, callback route, AuthKitProvider | -| `react` | workos-authkit-react | AuthKitProvider, callback component, useAuth | -| `react-router` | workos-authkit-react-router | Auth loader, protected routes | -| `tanstack-start` | workos-authkit-tanstack-start | Server functions, callback route | -| `vanilla-js` | workos-authkit-vanilla-js | Auth script, callback page | +| Framework | Skill | Key Checks | +| ---------------- | ----------------------------- | ----------------------------------------------- | +| `nextjs` | workos-authkit-nextjs | middleware.ts, callback route, AuthKitProvider | +| `react` | workos-authkit-react | AuthKitProvider, callback component, useAuth | +| `react-router` | workos-authkit-react-router | Auth loader, protected routes | +| `tanstack-start` | workos-authkit-tanstack-start | Server functions, callback route | +| `vanilla-js` | workos-authkit-vanilla-js | Auth script, callback page | ## CLI Options ``` ---framework= Filter by framework +--framework= Filter by framework (nextjs, react, react-router, tanstack-start, vanilla-js) --state= Filter by project state ---verbose, -v Show agent tool calls and detailed output +--quality, -q Enable LLM-based quality grading +--verbose, -v Show agent output and tool calls --debug Extra verbose, preserve temp dirs on failure --keep-on-fail Don't cleanup temp directory when scenario fails ---retry= Number of retry attempts (default: 2) +--retry= Retry attempts (default: 2) --no-retry Disable retries ---json Output results as JSON +--no-fail Don't exit 1 on threshold failure +--sequential Run scenarios sequentially (disable parallelism) +--no-dashboard Disable live dashboard, use sequential logging +--json Output as JSON --help, -h Show help ``` -## Debugging Failures +## Quality Grading -### 1. Inspect the failure details +When enabled with `--quality`, passing scenarios are graded on: -```bash -pnpm eval --framework=react --state=example-auth0 --verbose -``` +| Dimension | Description | +| -------------- | ----------------------------------- | +| Code Style | Adherence to project conventions | +| Minimalism | Changes are focused, no extras | +| Error Handling | Proper error handling and messages | +| Idiomatic | Follows framework best practices | -### 2. Preserve the temp directory +Each dimension scored 1-5. See `quality-rubrics.ts` for detailed rubrics. -```bash -pnpm eval --framework=react --state=example-auth0 --keep-on-fail -# Output will show: "Temp directory preserved: /tmp/eval-react-xxxxx" -``` +## Latency Metrics -### 3. Manually inspect the project state +Every run tracks: -```bash -cd /tmp/eval-react-xxxxx -ls -la -cat middleware.ts -``` +- **TTFT**: Time to first token +- **Agent Thinking**: Time spent deliberating +- **Tool Execution**: Time in tool calls +- **Tokens/sec**: Output throughput -### 4. Compare with previous runs +## Comparing Runs ```bash # List recent runs pnpm eval:history +# Show more runs +pnpm eval:history --limit=20 + # Compare two runs -pnpm eval:compare 2024-01-15T10-30-00 2024-01-16T14-45-00 +pnpm eval:diff 2024-01-15T10-30-00 2024-01-16T14-45-00 + +# Use 'latest' as alias for most recent run +pnpm eval:diff latest 2024-01-15T10-30-00 +``` + +The diff command shows: + +- Pass rate changes (first-attempt and with-retry) +- Skill version changes (with correlation analysis) +- Scenario regressions/improvements +- Latency changes (p50, p95) +- Quality score changes + +### Correlation Analysis + +When skill files change AND scenarios regress, the diff command highlights likely causes: + +``` +Likely Causes: + ⚠ nextjs skill changed (03133745 → a1b2c3d4) and 2 scenario(s) regressed +``` + +## Results Storage + +Results saved to `tests/eval-results/`: + +- `{timestamp}.json` - Full results with metadata +- `latest.json` - Symlink to most recent + +Each result file includes: + +- Summary (pass rates, scenario counts) +- Per-scenario results with checks +- Latency metrics (TTFT, tool breakdown) +- Quality grades (if enabled) +- Metadata (skill versions, CLI version, model version) + +Prune old results: + +```bash +# Keep only 10 most recent (default) +pnpm eval:prune + +# Keep specific number +pnpm eval:prune --keep=5 ``` ## Adding a New Fixture @@ -135,16 +200,29 @@ checks.push(await this.buildGrader.checkBuild()); return { passed: checks.every((c) => c.passed), checks }; ``` -## Results Storage +## Troubleshooting -Results are saved to `tests/eval-results/`: +### "Build failed" but files look correct -- Each run creates `{timestamp}.json` -- `latest.json` symlinks to most recent -- Use `pnpm eval:history` to list runs -- Use `pnpm eval:compare` to diff runs +Use `--keep-on-fail` to preserve temp directory and inspect: -## Troubleshooting +```bash +pnpm eval --framework=nextjs --keep-on-fail +cd /tmp/eval-nextjs-xxxxx && pnpm build +``` + +### Flaky passes/failures + +Increase retries: `pnpm eval --retry=3` + +If consistently flaky, check if skill instructions are ambiguous. + +### Pass rate regression + +1. Run `pnpm eval:diff latest ` +2. Check "Likely Causes" section +3. Review skill file changes listed +4. If no skill changes, check for external factors (API changes, dependency updates) ### "pnpm install failed" @@ -155,21 +233,13 @@ cd tests/fixtures/{framework}/{state} pnpm install ``` -### "Build failed" but files look correct +### High latency -The agent may have created correct files but with syntax errors. Use `--keep-on-fail` to inspect: +Check the tool breakdown in the summary output to identify bottlenecks: -```bash -pnpm eval --framework=nextjs --keep-on-fail -# Then run build manually in temp dir to see full error ``` - -### Flaky passes/failures - -LLM responses vary. Use `--retry=3` for more attempts: - -```bash -pnpm eval --retry=3 +Tool Time Breakdown (total across all scenarios): + Bash: 206.5s (27 calls) + Read: 54.3s (14 calls) + ... ``` - -If a scenario is consistently flaky, check if the skill instructions are ambiguous. diff --git a/tests/evals/__tests__/latency-tracker.spec.ts b/tests/evals/__tests__/latency-tracker.spec.ts new file mode 100644 index 00000000..3e0a7412 --- /dev/null +++ b/tests/evals/__tests__/latency-tracker.spec.ts @@ -0,0 +1,192 @@ +import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; +import { LatencyTracker } from '../latency-tracker.js'; + +describe('LatencyTracker', () => { + let tracker: LatencyTracker; + let mockTime: number; + + beforeEach(() => { + tracker = new LatencyTracker(); + mockTime = 0; + vi.spyOn(performance, 'now').mockImplementation(() => mockTime); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('start()', () => { + it('resets all counters', () => { + // First run with some data + tracker.start(); + mockTime = 100; + tracker.recordFirstContent(); + tracker.startToolCall('Bash'); + mockTime = 200; + tracker.endToolCall(); + tracker.recordTokens(1000, 500); + + // Start a new tracking session + mockTime = 0; + tracker.start(); + mockTime = 50; + const metrics = tracker.finish(); + + // Should be fresh - no TTFT recorded, no tools + expect(metrics.ttftMs).toBeNull(); + expect(metrics.toolBreakdown).toHaveLength(0); + expect(metrics.tokenMetrics?.inputTokens).toBe(0); + expect(metrics.tokenMetrics?.outputTokens).toBe(0); + }); + }); + + describe('recordFirstContent()', () => { + it('only records first call', () => { + tracker.start(); + + mockTime = 100; + tracker.recordFirstContent(); + + mockTime = 200; + tracker.recordFirstContent(); // Should be ignored + + mockTime = 300; + const metrics = tracker.finish(); + + expect(metrics.ttftMs).toBe(100); + }); + + it('returns null if never called', () => { + tracker.start(); + mockTime = 100; + const metrics = tracker.finish(); + + expect(metrics.ttftMs).toBeNull(); + }); + }); + + describe('tool timing', () => { + it('aggregates by tool name', () => { + tracker.start(); + + // First Bash call: 100ms + mockTime = 0; + tracker.startToolCall('Bash'); + mockTime = 100; + tracker.endToolCall(); + + // Second Bash call: 50ms + mockTime = 150; + tracker.startToolCall('Bash'); + mockTime = 200; + tracker.endToolCall(); + + // Write call: 30ms + mockTime = 200; + tracker.startToolCall('Write'); + mockTime = 230; + tracker.endToolCall(); + + mockTime = 300; + const metrics = tracker.finish(); + + const bashBreakdown = metrics.toolBreakdown?.find((t) => t.tool === 'Bash'); + const writeBreakdown = metrics.toolBreakdown?.find((t) => t.tool === 'Write'); + + expect(bashBreakdown?.count).toBe(2); + expect(bashBreakdown?.durationMs).toBe(150); // 100 + 50 + expect(writeBreakdown?.count).toBe(1); + expect(writeBreakdown?.durationMs).toBe(30); + }); + + it('uses end time for unclosed tool calls', () => { + tracker.start(); + + mockTime = 0; + tracker.startToolCall('Bash'); + // Don't call endToolCall + + mockTime = 100; + const metrics = tracker.finish(); + + expect(metrics.toolBreakdown?.[0]?.durationMs).toBe(100); + }); + }); + + describe('finish()', () => { + it('calculates correct derived metrics', () => { + tracker.start(); + + // TTFT at 50ms + mockTime = 50; + tracker.recordFirstContent(); + + // Tool takes 200ms (100-300) + mockTime = 100; + tracker.startToolCall('Bash'); + mockTime = 300; + tracker.endToolCall(); + + // Record tokens + tracker.recordTokens(1000, 400); + + // End at 500ms + mockTime = 500; + const metrics = tracker.finish(); + + expect(metrics.ttftMs).toBe(50); + expect(metrics.totalDurationMs).toBe(500); + expect(metrics.toolExecutionMs).toBe(200); + expect(metrics.agentThinkingMs).toBe(300); // 500 - 200 + expect(metrics.tokenMetrics?.inputTokens).toBe(1000); + expect(metrics.tokenMetrics?.outputTokens).toBe(400); + // 400 tokens / 0.5 seconds = 800 tokens/sec + expect(metrics.tokenMetrics?.tokensPerSecond).toBe(800); + }); + + it('returns 0 tool execution time for empty tool list', () => { + tracker.start(); + mockTime = 100; + const metrics = tracker.finish(); + + expect(metrics.toolExecutionMs).toBe(0); + expect(metrics.toolBreakdown).toHaveLength(0); + }); + + it('handles edge case of zero duration', () => { + tracker.start(); + const metrics = tracker.finish(); + + expect(metrics.totalDurationMs).toBe(0); + expect(metrics.tokenMetrics?.tokensPerSecond).toBe(0); + }); + + it('clamps negative durations to 0', () => { + tracker.start(); + mockTime = 100; + tracker.startToolCall('Bash'); + // Simulate clock going backwards (edge case) + mockTime = 50; + tracker.endToolCall(); + + const metrics = tracker.finish(); + + // Duration should be clamped to 0, not negative + expect(metrics.toolExecutionMs).toBeGreaterThanOrEqual(0); + }); + }); + + describe('recordTokens()', () => { + it('records input and output tokens', () => { + tracker.start(); + tracker.recordTokens(5000, 2000); + + mockTime = 1000; // 1 second + const metrics = tracker.finish(); + + expect(metrics.tokenMetrics?.inputTokens).toBe(5000); + expect(metrics.tokenMetrics?.outputTokens).toBe(2000); + expect(metrics.tokenMetrics?.tokensPerSecond).toBe(2000); // 2000 / 1 + }); + }); +}); diff --git a/tests/evals/agent-executor.ts b/tests/evals/agent-executor.ts index 897d2e68..cfc64ed5 100644 --- a/tests/evals/agent-executor.ts +++ b/tests/evals/agent-executor.ts @@ -4,13 +4,15 @@ import { Integration } from '../../src/lib/constants.js'; import { loadCredentials } from './env-loader.js'; import { writeEnvLocal } from '../../src/lib/env-writer.js'; import { getConfig } from '../../src/lib/settings.js'; -import type { ToolCall } from './types.js'; +import { LatencyTracker } from './latency-tracker.js'; +import type { ToolCall, LatencyMetrics } from './types.js'; export interface AgentResult { success: boolean; output: string; toolCalls: ToolCall[]; error?: string; + latencyMetrics?: LatencyMetrics; } export interface AgentExecutorOptions { @@ -30,6 +32,7 @@ const SKILL_NAMES: Record = { export class AgentExecutor { private options: AgentExecutorOptions; private credentials: ReturnType; + private latencyTracker: LatencyTracker; constructor( private workDir: string, @@ -38,6 +41,7 @@ export class AgentExecutor { ) { this.options = options; this.credentials = loadCredentials(); + this.latencyTracker = new LatencyTracker(); } async run(): Promise { @@ -50,6 +54,9 @@ export class AgentExecutor { console.log(`${label} Initializing agent for ${integration}...`); } + // Start latency tracking + this.latencyTracker.start(); + // Write .env.local with credentials (agent configures redirect URI per framework) writeEnvLocal(this.workDir, { WORKOS_API_KEY: this.credentials.workosApiKey, @@ -104,16 +111,20 @@ export class AgentExecutor { this.handleMessage(message, toolCalls, collectedOutput, label); } + const latencyMetrics = this.latencyTracker.finish(); return { success: true, output: collectedOutput.join('\n'), toolCalls, + latencyMetrics, }; } catch (error) { + const latencyMetrics = this.latencyTracker.finish(); return { success: false, output: collectedOutput.join('\n'), toolCalls, + latencyMetrics, error: error instanceof Error ? error.message : String(error), }; } @@ -139,18 +150,23 @@ Begin by invoking the ${skillName} skill.`; private handleMessage(message: any, toolCalls: ToolCall[], collectedOutput: string[], label: string): void { if (message.type === 'assistant') { + // End any in-progress tool call when we get a new assistant message + this.latencyTracker.endToolCall(); + const content = message.message?.content; if (Array.isArray(content)) { for (const block of content) { - // Capture text output + // Capture text output and track TTFT if (block.type === 'text' && typeof block.text === 'string') { + this.latencyTracker.recordFirstContent(); collectedOutput.push(block.text); if (this.options.verbose) { console.log(`${label} Agent: ${block.text.slice(0, 100)}...`); } } - // Capture tool calls + // Capture tool calls and start timing if (block.type === 'tool_use') { + this.latencyTracker.startToolCall(block.name); const call: ToolCall = { tool: block.name, input: block.input as Record, @@ -165,6 +181,13 @@ Begin by invoking the ${skillName} skill.`; } if (message.type === 'result') { + // Capture token usage from result + if (message.usage) { + this.latencyTracker.recordTokens( + message.usage.input_tokens ?? 0, + message.usage.output_tokens ?? 0, + ); + } if (message.subtype !== 'success' && message.errors?.length > 0) { collectedOutput.push(`Error: ${message.errors.join(', ')}`); } diff --git a/tests/evals/cli.ts b/tests/evals/cli.ts index c2481449..b701c95c 100644 --- a/tests/evals/cli.ts +++ b/tests/evals/cli.ts @@ -11,9 +11,13 @@ export interface CliOptions { noRetry: boolean; sequential: boolean; noDashboard: boolean; - command?: 'run' | 'history' | 'compare' | 'logs' | 'show'; + noFail: boolean; + quality: boolean; + command?: 'run' | 'history' | 'compare' | 'diff' | 'prune' | 'logs' | 'show'; compareIds?: [string, string]; logFile?: string; + limit?: number; + pruneKeep?: number; } const FRAMEWORKS = ['nextjs', 'react', 'react-router', 'tanstack-start', 'vanilla-js']; @@ -31,20 +35,40 @@ export function parseArgs(args: string[]): CliOptions { noRetry: false, sequential: false, noDashboard: false, + noFail: false, + quality: false, }; // Check for subcommands if (args[0] === 'history') { options.command = 'history'; + // Parse --limit=N option + for (const arg of args.slice(1)) { + if (arg.startsWith('--limit=')) { + options.limit = parseInt(arg.split('=')[1], 10); + } + } return options; } - if (args[0] === 'compare' && args.length >= 3) { - options.command = 'compare'; + // Support both 'compare' (legacy) and 'diff' (new) + if ((args[0] === 'compare' || args[0] === 'diff') && args.length >= 3) { + options.command = 'diff'; options.compareIds = [args[1], args[2]]; return options; } + if (args[0] === 'prune') { + options.command = 'prune'; + // Parse --keep=N option + for (const arg of args.slice(1)) { + if (arg.startsWith('--keep=')) { + options.pruneKeep = parseInt(arg.split('=')[1], 10); + } + } + return options; + } + if (args[0] === 'logs') { options.command = 'logs'; return options; @@ -93,6 +117,10 @@ export function parseArgs(args: string[]): CliOptions { options.sequential = true; } else if (arg === '--no-dashboard') { options.noDashboard = true; + } else if (arg === '--no-fail') { + options.noFail = true; + } else if (arg === '--quality' || arg === '-q') { + options.quality = true; } } @@ -109,8 +137,9 @@ Usage: pnpm eval [command] [options] Commands: run (default) Run evaluations - history List recent eval runs - compare Compare two eval runs + history List recent eval runs (--limit=N) + diff Compare two eval runs with correlation analysis + prune Delete old results (--keep=N, default 10) logs List recent detailed log files show Display formatted log summary @@ -137,6 +166,10 @@ Options: --no-dashboard Disable live dashboard, use sequential logging + --no-fail Exit 0 even if success criteria thresholds not met + + --quality, -q Enable LLM-based quality grading (adds cost/time) + --json Output results as JSON (for scripting) --help, -h Show this help message @@ -150,6 +183,8 @@ Examples: pnpm eval --debug # Verbose output, keep failed dirs pnpm eval --retry=3 # More retry attempts pnpm eval:history # List recent runs - pnpm eval:compare # Compare two runs + pnpm eval:history --limit=20 # Show more runs + pnpm eval:diff # Compare two runs + pnpm eval:prune --keep=5 # Keep only 5 most recent runs `); } diff --git a/tests/evals/commands/diff.ts b/tests/evals/commands/diff.ts new file mode 100644 index 00000000..9be4143d --- /dev/null +++ b/tests/evals/commands/diff.ts @@ -0,0 +1,274 @@ +import chalk from 'chalk'; +import type { EvalRun } from '../history.js'; +import type { EvalResultMetadata, LatencyMetrics, QualityGrade } from '../types.js'; + +export interface DiffResult { + passRateDelta: { + firstAttempt: number; + withRetry: number; + }; + skillChanges: Array<{ + framework: string; + oldHash: string; + newHash: string; + }>; + scenarioChanges: { + regressions: string[]; + improvements: string[]; + unchanged: string[]; + }; + latencyChanges?: { + ttftP50Delta: number; + ttftP95Delta: number; + durationP50Delta: number; + durationP95Delta: number; + }; + qualityChanges?: { + overallDelta: number; + dimensionDeltas: Record; + }; + likelyCauses: string[]; +} + +export function diffRuns(run1: EvalRun, run2: EvalRun): DiffResult { + // Calculate pass rate deltas + const passRateDelta = { + firstAttempt: calculateFirstAttemptRate(run2) - calculateFirstAttemptRate(run1), + withRetry: run2.summary.passRate - run1.summary.passRate, + }; + + // Find skill version changes + const skillChanges = findSkillChanges(run1.metadata, run2.metadata); + + // Find scenario status changes + const scenarioChanges = findScenarioChanges(run1, run2); + + // Calculate latency changes (if available) + const latencyChanges = calculateLatencyChanges(run1, run2); + + // Calculate quality changes (if available) + const qualityChanges = calculateQualityChanges(run1, run2); + + // Determine likely causes + const likelyCauses = determineLikelyCauses(skillChanges, scenarioChanges, passRateDelta); + + return { + passRateDelta, + skillChanges, + scenarioChanges, + latencyChanges, + qualityChanges, + likelyCauses, + }; +} + +function calculateFirstAttemptRate(run: EvalRun): number { + const firstAttemptPassed = run.results.filter((r) => r.attempts === 1 && r.passed).length; + return run.results.length > 0 ? firstAttemptPassed / run.results.length : 0; +} + +function findSkillChanges( + meta1?: EvalResultMetadata, + meta2?: EvalResultMetadata, +): Array<{ framework: string; oldHash: string; newHash: string }> { + if (!meta1?.skillVersions || !meta2?.skillVersions) return []; + + const changes: Array<{ framework: string; oldHash: string; newHash: string }> = []; + + for (const [framework, newHash] of Object.entries(meta2.skillVersions)) { + const oldHash = meta1.skillVersions[framework] || 'unknown'; + if (oldHash !== newHash) { + changes.push({ framework, oldHash, newHash }); + } + } + + return changes; +} + +function findScenarioChanges( + run1: EvalRun, + run2: EvalRun, +): { regressions: string[]; improvements: string[]; unchanged: string[] } { + const results1 = new Map(run1.results.map((r) => [r.scenario, r.passed])); + const results2 = new Map(run2.results.map((r) => [r.scenario, r.passed])); + + const regressions: string[] = []; + const improvements: string[] = []; + const unchanged: string[] = []; + + for (const [scenario, passed2] of results2) { + const passed1 = results1.get(scenario); + if (passed1 === true && passed2 === false) { + regressions.push(scenario); + } else if (passed1 === false && passed2 === true) { + improvements.push(scenario); + } else { + unchanged.push(scenario); + } + } + + return { regressions, improvements, unchanged }; +} + +function calculateLatencyChanges( + run1: EvalRun, + run2: EvalRun, +): DiffResult['latencyChanges'] | undefined { + const latencies1 = run1.results.map((r) => r.latencyMetrics).filter(Boolean) as LatencyMetrics[]; + const latencies2 = run2.results.map((r) => r.latencyMetrics).filter(Boolean) as LatencyMetrics[]; + + if (latencies1.length === 0 || latencies2.length === 0) return undefined; + + const ttfts1 = latencies1.map((l) => l.ttftMs).filter((t): t is number => t !== null); + const ttfts2 = latencies2.map((l) => l.ttftMs).filter((t): t is number => t !== null); + const durations1 = latencies1.map((l) => l.totalDurationMs); + const durations2 = latencies2.map((l) => l.totalDurationMs); + + if (ttfts1.length === 0 || ttfts2.length === 0) return undefined; + + return { + ttftP50Delta: percentile(ttfts2, 50) - percentile(ttfts1, 50), + ttftP95Delta: percentile(ttfts2, 95) - percentile(ttfts1, 95), + durationP50Delta: percentile(durations2, 50) - percentile(durations1, 50), + durationP95Delta: percentile(durations2, 95) - percentile(durations1, 95), + }; +} + +function calculateQualityChanges( + run1: EvalRun, + run2: EvalRun, +): DiffResult['qualityChanges'] | undefined { + const grades1 = run1.results.map((r) => r.qualityGrade).filter(Boolean) as QualityGrade[]; + const grades2 = run2.results.map((r) => r.qualityGrade).filter(Boolean) as QualityGrade[]; + + if (grades1.length === 0 || grades2.length === 0) return undefined; + + const avgScore1 = grades1.reduce((s, g) => s + g.score, 0) / grades1.length; + const avgScore2 = grades2.reduce((s, g) => s + g.score, 0) / grades2.length; + + // Calculate dimension averages + const dims = ['codeStyle', 'minimalism', 'errorHandling', 'idiomatic'] as const; + const dimensionDeltas: Record = {}; + + for (const dim of dims) { + const avg1 = grades1.reduce((s, g) => s + g.dimensions[dim], 0) / grades1.length; + const avg2 = grades2.reduce((s, g) => s + g.dimensions[dim], 0) / grades2.length; + dimensionDeltas[dim] = avg2 - avg1; + } + + return { + overallDelta: avgScore2 - avgScore1, + dimensionDeltas, + }; +} + +function determineLikelyCauses( + skillChanges: Array<{ framework: string; oldHash: string; newHash: string }>, + scenarioChanges: { regressions: string[] }, + passRateDelta: { firstAttempt: number; withRetry: number }, +): string[] { + const causes: string[] = []; + + // If pass rate dropped AND skill changed, correlate + if (passRateDelta.withRetry < -0.05) { + // >5% drop + for (const change of skillChanges) { + const relatedRegressions = scenarioChanges.regressions.filter((s) => + s.startsWith(change.framework), + ); + if (relatedRegressions.length > 0) { + causes.push( + `${change.framework} skill changed (${change.oldHash.slice(0, 8)} → ${change.newHash.slice(0, 8)}) ` + + `and ${relatedRegressions.length} scenario(s) regressed`, + ); + } + } + } + + // No skill changes but regressions occurred + if (skillChanges.length === 0 && scenarioChanges.regressions.length > 0) { + causes.push('Regressions occurred without skill changes - possible flaky tests or external factors'); + } + + return causes; +} + +function percentile(values: number[], p: number): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +export function printDiff(diff: DiffResult, run1Id: string, run2Id: string): void { + console.log(chalk.bold(`\nComparing: ${run1Id} → ${run2Id}\n`)); + + // Pass rate changes + console.log(chalk.bold('Pass Rate Changes:')); + printDelta(' First-attempt', diff.passRateDelta.firstAttempt * 100, '%'); + printDelta(' With-retry', diff.passRateDelta.withRetry * 100, '%'); + + // Skill changes + if (diff.skillChanges.length > 0) { + console.log(chalk.bold('\nSkill Version Changes:')); + for (const change of diff.skillChanges) { + console.log( + ` ${change.framework}: ${change.oldHash.slice(0, 8)} → ${change.newHash.slice(0, 8)}`, + ); + } + } + + // Scenario changes + if (diff.scenarioChanges.regressions.length > 0) { + console.log(chalk.bold.red('\nRegressions (PASS → FAIL):')); + for (const s of diff.scenarioChanges.regressions) { + console.log(chalk.red(` ✗ ${s}`)); + } + } + + if (diff.scenarioChanges.improvements.length > 0) { + console.log(chalk.bold.green('\nImprovements (FAIL → PASS):')); + for (const s of diff.scenarioChanges.improvements) { + console.log(chalk.green(` ✓ ${s}`)); + } + } + + // Latency changes + if (diff.latencyChanges) { + console.log(chalk.bold('\nLatency Changes:')); + printDelta(' TTFT p50', diff.latencyChanges.ttftP50Delta, 'ms'); + printDelta(' TTFT p95', diff.latencyChanges.ttftP95Delta, 'ms'); + printDelta(' Duration p50', diff.latencyChanges.durationP50Delta / 1000, 's'); + printDelta(' Duration p95', diff.latencyChanges.durationP95Delta / 1000, 's'); + } + + // Quality changes + if (diff.qualityChanges) { + console.log(chalk.bold('\nQuality Changes:')); + printDelta(' Overall', diff.qualityChanges.overallDelta, '/5'); + for (const [dim, delta] of Object.entries(diff.qualityChanges.dimensionDeltas)) { + printDelta(` ${dim}`, delta, '/5'); + } + } + + // Likely causes + if (diff.likelyCauses.length > 0) { + console.log(chalk.bold.yellow('\nLikely Causes:')); + for (const cause of diff.likelyCauses) { + console.log(chalk.yellow(` ⚠ ${cause}`)); + } + } + + // Summary + const totalChanges = + diff.scenarioChanges.regressions.length + diff.scenarioChanges.improvements.length; + if (totalChanges === 0 && diff.skillChanges.length === 0) { + console.log(chalk.gray('\nNo significant changes between runs.')); + } +} + +function printDelta(label: string, delta: number, unit: string): void { + const sign = delta > 0 ? '+' : ''; + const color = delta > 0 ? chalk.green : delta < 0 ? chalk.red : chalk.gray; + console.log(`${label}: ${color(`${sign}${delta.toFixed(1)}${unit}`)}`); +} diff --git a/tests/evals/commands/history.ts b/tests/evals/commands/history.ts new file mode 100644 index 00000000..c42b6972 --- /dev/null +++ b/tests/evals/commands/history.ts @@ -0,0 +1,90 @@ +import { readdir, unlink, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import chalk from 'chalk'; +import type { EvalRun } from '../history.js'; + +const RESULTS_DIR = join(process.cwd(), 'tests/eval-results'); + +export async function listHistory(limit: number = 10): Promise { + let files: string[]; + try { + files = await readdir(RESULTS_DIR); + } catch { + console.log(chalk.yellow('No eval results found. Run `pnpm eval` first.')); + return; + } + + const runFiles = files + .filter((f) => f.endsWith('.json') && f !== 'latest.json' && !f.startsWith('eval-run-')) + .sort() + .reverse() + .slice(0, limit); + + if (runFiles.length === 0) { + console.log(chalk.yellow('No eval results found. Run `pnpm eval` first.')); + return; + } + + console.log(chalk.bold('\nRecent Eval Runs:\n')); + console.log(' ID Pass Rate Scenarios Avg Duration'); + console.log(' ' + '─'.repeat(68)); + + for (const file of runFiles) { + try { + const content = await readFile(join(RESULTS_DIR, file), 'utf-8'); + const run: EvalRun = JSON.parse(content); + + const passRate = (run.summary.passRate * 100).toFixed(0) + '%'; + const scenarios = `${run.summary.passed}/${run.summary.total}`; + const avgDuration = + run.results.length > 0 + ? Math.round(run.results.reduce((s, r) => s + r.duration, 0) / run.results.length / 1000) + + 's' + : 'N/A'; + + const color = run.summary.passRate >= 0.9 ? chalk.green : chalk.red; + const id = run.id.padEnd(32); + + console.log( + ` ${id} ${color(passRate.padEnd(10))} ${scenarios.padEnd(11)} ${avgDuration}`, + ); + } catch { + const id = file.replace('.json', '').padEnd(32); + console.log(` ${id} ${chalk.gray('(unable to read)')}`); + } + } + + const totalRuns = files.filter((f) => f.endsWith('.json') && f !== 'latest.json' && !f.startsWith('eval-run-')).length; + console.log(`\n Showing ${runFiles.length} of ${totalRuns} runs. Use --limit=N for more.`); +} + +export async function pruneHistory(keep: number = 10): Promise { + let files: string[]; + try { + files = await readdir(RESULTS_DIR); + } catch { + console.log('No results directory found.'); + return; + } + + const runFiles = files + .filter((f) => f.endsWith('.json') && f !== 'latest.json' && !f.startsWith('eval-run-')) + .sort() + .reverse(); + + const toDelete = runFiles.slice(keep); + + if (toDelete.length === 0) { + console.log(`No runs to prune. Keeping all ${runFiles.length} runs.`); + return; + } + + console.log(`Pruning ${toDelete.length} old runs, keeping ${keep} most recent...`); + + for (const file of toDelete) { + await unlink(join(RESULTS_DIR, file)); + console.log(chalk.gray(` Deleted: ${file}`)); + } + + console.log(chalk.green(`Done. ${keep} runs remaining.`)); +} diff --git a/tests/evals/dashboard/EvalDashboard.tsx b/tests/evals/dashboard/EvalDashboard.tsx index d5ad2e99..1847ae45 100644 --- a/tests/evals/dashboard/EvalDashboard.tsx +++ b/tests/evals/dashboard/EvalDashboard.tsx @@ -1,11 +1,6 @@ import React, { useState, useEffect } from 'react'; import { Box, Text, useApp } from 'ink'; -import { - evalEvents, - type ScenarioStartEvent, - type ScenarioCompleteEvent, - type RunProgressEvent, -} from '../events.js'; +import { evalEvents, type ScenarioStartEvent, type ScenarioCompleteEvent, type RunProgressEvent } from '../events.js'; import { Header } from './Header.js'; import { ScenarioRow } from './ScenarioRow.js'; diff --git a/tests/evals/fixture-manager.ts b/tests/evals/fixture-manager.ts index d98e3cc7..eae3fdc7 100644 --- a/tests/evals/fixture-manager.ts +++ b/tests/evals/fixture-manager.ts @@ -39,6 +39,11 @@ export class FixtureManager { throw new Error(`pnpm install failed: ${result.stderr}`); } + // Initialize git repo for diff capture (quality grading) + await execFileNoThrow('git', ['init'], { cwd: this.tempDir }); + await execFileNoThrow('git', ['add', '-A'], { cwd: this.tempDir }); + await execFileNoThrow('git', ['commit', '-m', 'initial', '--no-gpg-sign'], { cwd: this.tempDir }); + return this.tempDir; } diff --git a/tests/evals/graders/collect-key-files.ts b/tests/evals/graders/collect-key-files.ts new file mode 100644 index 00000000..0987767a --- /dev/null +++ b/tests/evals/graders/collect-key-files.ts @@ -0,0 +1,75 @@ +import { readFile } from 'node:fs/promises'; +import { join, relative } from 'node:path'; +import fg from 'fast-glob'; +import { QUALITY_KEY_FILES } from '../quality-key-files.js'; + +/** + * Collects the content of key integration files for quality grading. + * + * Uses glob patterns to find files, reads their content, and returns + * a map of relative paths to file contents. + * + * @param workDir - The working directory to search in + * @param framework - The framework name (e.g., 'nextjs', 'react') + * @returns Map of relative file paths to their contents + */ +export async function collectKeyFiles( + workDir: string, + framework: string, +): Promise> { + const patterns = QUALITY_KEY_FILES[framework]; + if (!patterns) { + return new Map(); + } + + const files = new Map(); + const foundPaths = new Set(); + + for (const pattern of patterns) { + // Find files matching the pattern + const matches = await fg(pattern, { + cwd: workDir, + absolute: true, + onlyFiles: true, + ignore: ['**/node_modules/**', '**/dist/**', '**/build/**', '**/.next/**'], + }); + + for (const absPath of matches) { + const relPath = relative(workDir, absPath); + + // Skip if we already have this file + if (foundPaths.has(relPath)) { + continue; + } + + try { + const content = await readFile(absPath, 'utf-8'); + files.set(relPath, content); + foundPaths.add(relPath); + } catch { + // Skip unreadable files + } + } + } + + return files; +} + +/** + * Formats key files for inclusion in the quality grading prompt. + * + * @param keyFiles - Map of file paths to contents + * @returns Formatted string with all files as markdown code blocks + */ +export function formatKeyFilesForPrompt(keyFiles: Map): string { + if (keyFiles.size === 0) { + return 'No key integration files found.'; + } + + return Array.from(keyFiles.entries()) + .map(([path, content]) => { + const ext = path.split('.').pop() || 'txt'; + return `### ${path}\n\`\`\`${ext}\n${content}\n\`\`\``; + }) + .join('\n\n'); +} diff --git a/tests/evals/graders/file-grader.ts b/tests/evals/graders/file-grader.ts index 0a680faf..1118ed95 100644 --- a/tests/evals/graders/file-grader.ts +++ b/tests/evals/graders/file-grader.ts @@ -54,9 +54,7 @@ export class FileGrader { for (const file of files) { try { const content = await readFile(file, 'utf-8'); - const allMatch = contentPatterns.every((p) => - typeof p === 'string' ? content.includes(p) : p.test(content), - ); + const allMatch = contentPatterns.every((p) => (typeof p === 'string' ? content.includes(p) : p.test(content))); if (allMatch) { const relativePath = file.replace(this.workDir + '/', ''); return { diff --git a/tests/evals/graders/nextjs.grader.ts b/tests/evals/graders/nextjs.grader.ts index c214479d..a9ec2109 100644 --- a/tests/evals/graders/nextjs.grader.ts +++ b/tests/evals/graders/nextjs.grader.ts @@ -25,16 +25,35 @@ export class NextjsGrader implements Grader { // Check middleware exists checks.push(await this.fileGrader.checkFileExists('middleware.ts')); - // Check middleware imports authkit - checks.push( - ...(await this.fileGrader.checkFileContains('middleware.ts', [ - '@workos-inc/authkit-nextjs', - 'authkitMiddleware', - ])), - ); + // Check middleware imports authkit SDK + const sdkImportChecks = await this.fileGrader.checkFileContains('middleware.ts', ['@workos-inc/authkit-nextjs']); + checks.push(...sdkImportChecks); + + // Check for authkit integration: authkitMiddleware OR (authkit + handleAuthkitHeaders) + const middlewareChecks = await this.fileGrader.checkFileContains('middleware.ts', ['authkitMiddleware']); + const composableChecks = await this.fileGrader.checkFileContains('middleware.ts', ['authkit(', 'handleAuthkitHeaders']); + + const usesAuthkitMiddleware = middlewareChecks.every((c) => c.passed); + const usesComposable = composableChecks.every((c) => c.passed); - // Check AuthKitProvider in layout - checks.push(...(await this.fileGrader.checkFileContains('app/layout.tsx', ['AuthKitProvider']))); + const authkitCheck: GradeCheck = { + name: 'AuthKit middleware integration', + passed: usesAuthkitMiddleware || usesComposable, + message: usesAuthkitMiddleware + ? 'Uses authkitMiddleware' + : usesComposable + ? 'Uses authkit() composable with handleAuthkitHeaders' + : 'Missing authkitMiddleware or authkit() composable integration', + }; + checks.push(authkitCheck); + + // Check AuthKitProvider in layout or extracted providers file + const authKitProviderCheck = await this.fileGrader.checkFileWithPattern( + 'app/**/*.tsx', + ['AuthKitProvider'], + 'AuthKitProvider in app', + ); + checks.push(authKitProviderCheck); // Check build succeeds checks.push(await this.buildGrader.checkBuild()); diff --git a/tests/evals/graders/quality-grader.ts b/tests/evals/graders/quality-grader.ts new file mode 100644 index 00000000..91165a42 --- /dev/null +++ b/tests/evals/graders/quality-grader.ts @@ -0,0 +1,130 @@ +import Anthropic from '@anthropic-ai/sdk'; +import { QUALITY_RUBRICS, QUALITY_DIMENSIONS } from '../quality-rubrics.js'; +import type { QualityGrade, QualityInput } from '../types.js'; +import { formatKeyFilesForPrompt } from './collect-key-files.js'; + +const QUALITY_MODEL = 'claude-3-5-haiku-20241022'; + +export class QualityGrader { + private client: Anthropic; + + constructor(apiKey: string) { + this.client = new Anthropic({ apiKey }); + } + + async grade(input: QualityInput): Promise { + if (input.keyFiles.size === 0) { + return null; + } + + const prompt = this.buildPrompt(input); + + try { + const response = await this.client.messages.create({ + model: QUALITY_MODEL, + max_tokens: 1024, + messages: [{ role: 'user', content: prompt }], + }); + + const content = response.content[0]; + if (content.type !== 'text') { + return null; + } + + return this.parseResponse(content.text); + } catch (error) { + console.warn('Quality grading failed:', error); + return null; + } + } + + private buildPrompt(input: QualityInput): string { + const rubricText = QUALITY_DIMENSIONS.map((dim) => { + const rubric = QUALITY_RUBRICS[dim]; + const scaleText = Object.entries(rubric.scale) + .map(([score, desc]) => ` ${score}: ${desc}`) + .join('\n'); + return `### ${rubric.name}\n${rubric.description}\n${scaleText}`; + }).join('\n\n'); + + const keyFilesText = formatKeyFilesForPrompt(input.keyFiles); + + // Chain-of-thought before scoring improves grading accuracy (Anthropic best practice) + return `You are evaluating code written by an AI agent installing WorkOS AuthKit into a ${input.framework} project. + +## Key Integration Files + +${keyFilesText} + +## Installation Metadata +- Files created: ${input.metadata.filesCreated.join(', ') || 'None'} +- Files modified: ${input.metadata.filesModified.join(', ') || 'None'} +- Tool activity: ${input.metadata.toolCallSummary} +- Checks passed: ${input.metadata.checksPassed.join(', ') || 'None'} + +## Grading Rubrics +${rubricText} + +## Instructions +First, analyze the code thoroughly in tags. For each dimension, examine the code and determine the appropriate score based on the rubric. Consider specific examples from the code. + +Then, output your final scores as JSON. + + +[Analyze each dimension here - what patterns do you see? What's done well? What could be better?] + + +{ + "codeStyle": <1-5>, + "minimalism": <1-5>, + "errorHandling": <1-5>, + "idiomatic": <1-5> +}`; + } + + private parseResponse(text: string): QualityGrade | null { + try { + // Extract chain-of-thought reasoning from tags + const thinkingMatch = text.match(/([\s\S]*?)<\/thinking>/); + const reasoning = thinkingMatch?.[1]?.trim() || 'No reasoning provided'; + + // Extract JSON scores (after thinking block) + const jsonMatch = text.match(/\{[\s\S]*\}/); + if (!jsonMatch) return null; + + const parsed = JSON.parse(jsonMatch[0]) as Record; + + // Handle both formats: direct scores or nested { score: n } + const getScore = (val: unknown): number => { + if (typeof val === 'number') return val; + if (typeof val === 'object' && val !== null && 'score' in val) { + return (val as { score: unknown }).score as number; + } + return 3; + }; + + const dimensions = { + codeStyle: this.clampScore(getScore(parsed.codeStyle)), + minimalism: this.clampScore(getScore(parsed.minimalism)), + errorHandling: this.clampScore(getScore(parsed.errorHandling)), + idiomatic: this.clampScore(getScore(parsed.idiomatic)), + }; + + const score = Object.values(dimensions).reduce((a, b) => a + b, 0) / 4; + + return { + score: Math.round(score * 10) / 10, + dimensions, + reasoning, + }; + } catch (error) { + console.warn('Failed to parse quality response:', error); + return null; + } + } + + private clampScore(score: unknown): number { + const num = typeof score === 'number' ? score : 3; + return Math.max(1, Math.min(5, Math.round(num))); + } +} diff --git a/tests/evals/graders/vanilla.grader.ts b/tests/evals/graders/vanilla.grader.ts index 9031f88c..36008d46 100644 --- a/tests/evals/graders/vanilla.grader.ts +++ b/tests/evals/graders/vanilla.grader.ts @@ -44,11 +44,7 @@ export class VanillaGrader implements Grader { // Check createClient usage (the core initialization pattern) checks.push( - await this.fileGrader.checkFileWithPattern( - '**/*.{js,ts}', - ['createClient'], - 'createClient initialization', - ), + await this.fileGrader.checkFileWithPattern('**/*.{js,ts}', ['createClient'], 'createClient initialization'), ); // Check for auth methods usage (signIn, signOut, or getUser) @@ -61,13 +57,7 @@ export class VanillaGrader implements Grader { ); // Check index.html exists and references auth script or module - checks.push( - await this.fileGrader.checkFileWithPattern( - '*.html', - [/ + + diff --git a/tests/fixtures/react/conflicting-auth/package.json b/tests/fixtures/react/conflicting-auth/package.json new file mode 100644 index 00000000..70785f7c --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/package.json @@ -0,0 +1,23 @@ +{ + "name": "react-conflicting-auth-fixture", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^6.28.0" + }, + "devDependencies": { + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.4", + "typescript": "~5.6.2", + "vite": "^6.0.5" + } +} diff --git a/tests/fixtures/react/conflicting-auth/src/App.tsx b/tests/fixtures/react/conflicting-auth/src/App.tsx new file mode 100644 index 00000000..57e4bde7 --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/src/App.tsx @@ -0,0 +1,31 @@ +import { Routes, Route, Link } from 'react-router-dom'; +import { useAuth } from './auth/AuthProvider'; +import { Home } from './pages/Home'; +import { About } from './pages/About'; +import { Dashboard } from './pages/Dashboard'; + +function App() { + const { isAuthenticated, user, logout } = useAuth(); + + return ( +
+ + + } /> + } /> + } /> + +
+ ); +} + +export default App; diff --git a/tests/fixtures/react/conflicting-auth/src/auth/AuthProvider.tsx b/tests/fixtures/react/conflicting-auth/src/auth/AuthProvider.tsx new file mode 100644 index 00000000..2de8e79f --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/src/auth/AuthProvider.tsx @@ -0,0 +1,123 @@ +import { createContext, useContext, useState, useCallback, useEffect, type ReactNode } from 'react'; + +interface User { + id: string; + email: string; + name: string; + role: 'admin' | 'user' | 'guest'; + preferences: { + theme: 'light' | 'dark'; + notifications: boolean; + }; +} + +interface Credentials { + email: string; + password: string; +} + +interface AuthContextValue { + user: User | null; + isLoading: boolean; + isAuthenticated: boolean; + login: (credentials: Credentials) => Promise; + logout: () => Promise; + updatePreferences: (prefs: Partial) => Promise; +} + +const AuthContext = createContext(null); + +interface AuthProviderProps { + children: ReactNode; +} + +export function AuthProvider({ children }: AuthProviderProps) { + const [user, setUser] = useState(null); + const [isLoading, setIsLoading] = useState(true); + + // Check for existing session on mount + useEffect(() => { + const checkSession = async () => { + try { + const stored = localStorage.getItem('auth_user'); + if (stored) { + setUser(JSON.parse(stored)); + } + } finally { + setIsLoading(false); + } + }; + checkSession(); + }, []); + + const login = useCallback(async (credentials: Credentials) => { + setIsLoading(true); + try { + // Simulated API call - in real app this would hit your backend + await new Promise((resolve) => setTimeout(resolve, 500)); + + const mockUser: User = { + id: '123', + email: credentials.email, + name: credentials.email.split('@')[0] ?? 'User', + role: 'user', + preferences: { + theme: 'light', + notifications: true, + }, + }; + + localStorage.setItem('auth_user', JSON.stringify(mockUser)); + setUser(mockUser); + } finally { + setIsLoading(false); + } + }, []); + + const logout = useCallback(async () => { + setIsLoading(true); + try { + localStorage.removeItem('auth_user'); + setUser(null); + } finally { + setIsLoading(false); + } + }, []); + + const updatePreferences = useCallback( + async (prefs: Partial) => { + if (!user) return; + + const updatedUser = { + ...user, + preferences: { ...user.preferences, ...prefs }, + }; + localStorage.setItem('auth_user', JSON.stringify(updatedUser)); + setUser(updatedUser); + }, + [user], + ); + + return ( + + {children} + + ); +} + +export function useAuth(): AuthContextValue { + const context = useContext(AuthContext); + if (!context) { + throw new Error('useAuth must be used within an AuthProvider'); + } + return context; +} diff --git a/tests/fixtures/react/conflicting-auth/src/main.tsx b/tests/fixtures/react/conflicting-auth/src/main.tsx new file mode 100644 index 00000000..a7541f0d --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/src/main.tsx @@ -0,0 +1,15 @@ +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; +import { BrowserRouter } from 'react-router-dom'; +import { AuthProvider } from './auth/AuthProvider'; +import App from './App.tsx'; + +createRoot(document.getElementById('root')!).render( + + + + + + + , +); diff --git a/tests/fixtures/react/conflicting-auth/src/pages/About.tsx b/tests/fixtures/react/conflicting-auth/src/pages/About.tsx new file mode 100644 index 00000000..20ddb997 --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/src/pages/About.tsx @@ -0,0 +1,8 @@ +export function About() { + return ( +
+

About

+

This is an existing React application with custom auth.

+
+ ); +} diff --git a/tests/fixtures/react/conflicting-auth/src/pages/Dashboard.tsx b/tests/fixtures/react/conflicting-auth/src/pages/Dashboard.tsx new file mode 100644 index 00000000..ad52c245 --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/src/pages/Dashboard.tsx @@ -0,0 +1,27 @@ +import { useAuth } from '../auth/AuthProvider'; + +export function Dashboard() { + const { isAuthenticated, user, isLoading } = useAuth(); + + if (isLoading) { + return
Loading...
; + } + + if (!isAuthenticated) { + return ( +
+

Dashboard

+

Please log in to view the dashboard.

+
+ ); + } + + return ( +
+

Dashboard

+

Welcome back, {user?.name}!

+

Role: {user?.role}

+

Theme: {user?.preferences.theme}

+
+ ); +} diff --git a/tests/fixtures/react/conflicting-auth/src/pages/Home.tsx b/tests/fixtures/react/conflicting-auth/src/pages/Home.tsx new file mode 100644 index 00000000..ed07ba17 --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/src/pages/Home.tsx @@ -0,0 +1,8 @@ +export function Home() { + return ( +
+

Home

+

Welcome to the home page.

+
+ ); +} diff --git a/tests/fixtures/react/conflicting-auth/src/vite-env.d.ts b/tests/fixtures/react/conflicting-auth/src/vite-env.d.ts new file mode 100644 index 00000000..11f02fe2 --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/tests/fixtures/react/conflicting-auth/tsconfig.json b/tests/fixtures/react/conflicting-auth/tsconfig.json new file mode 100644 index 00000000..d5419222 --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["src"] +} diff --git a/tests/fixtures/react/conflicting-auth/vite.config.ts b/tests/fixtures/react/conflicting-auth/vite.config.ts new file mode 100644 index 00000000..d192dba1 --- /dev/null +++ b/tests/fixtures/react/conflicting-auth/vite.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; +import { resolve } from 'path'; + +export default defineConfig({ + plugins: [react()], + resolve: { + alias: { + '@': resolve(__dirname, './src'), + }, + }, +}); diff --git a/tests/fixtures/react/partial-install/README.md b/tests/fixtures/react/partial-install/README.md new file mode 100644 index 00000000..94f8c125 --- /dev/null +++ b/tests/fixtures/react/partial-install/README.md @@ -0,0 +1,29 @@ +# React SPA - Partial Install Fixture + +## Edge Case Description + +This fixture represents a React SPA where AuthKit was partially installed - the package is in dependencies but integration was never completed. + +## Expected Agent Behavior + +- Detect that @workos-inc/authkit-react is already installed +- Complete the integration by: + - Adding AuthKitProvider to main.tsx + - Creating callback route +- Should NOT reinstall the package + +## Files of Interest + +- `package.json` - Already has @workos-inc/authkit-react dependency +- `src/main.tsx` - Has commented-out import as signal of abandoned attempt + +## Success Criteria + +- [ ] AuthKitProvider wraps the app in main.tsx +- [ ] Callback route is configured +- [ ] Build succeeds with no type errors +- [ ] Package is not reinstalled + +## Notes + +Common scenario when developers start integration but don't finish. diff --git a/tests/fixtures/react/partial-install/index.html b/tests/fixtures/react/partial-install/index.html new file mode 100644 index 00000000..5da656cb --- /dev/null +++ b/tests/fixtures/react/partial-install/index.html @@ -0,0 +1,12 @@ + + + + + + React App + + +
+ + + diff --git a/tests/fixtures/react/partial-install/package.json b/tests/fixtures/react/partial-install/package.json new file mode 100644 index 00000000..ed132ca0 --- /dev/null +++ b/tests/fixtures/react/partial-install/package.json @@ -0,0 +1,24 @@ +{ + "name": "react-partial-install-fixture", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview" + }, + "dependencies": { + "@workos-inc/authkit-react": "^0.5.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^6.28.0" + }, + "devDependencies": { + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.4", + "typescript": "~5.6.2", + "vite": "^6.0.5" + } +} diff --git a/tests/fixtures/react/partial-install/src/App.tsx b/tests/fixtures/react/partial-install/src/App.tsx new file mode 100644 index 00000000..97221e77 --- /dev/null +++ b/tests/fixtures/react/partial-install/src/App.tsx @@ -0,0 +1,21 @@ +import { Routes, Route, Link } from 'react-router-dom'; +import { Home } from './pages/Home'; +import { About } from './pages/About'; +import { Dashboard } from './pages/Dashboard'; + +function App() { + return ( +
+ + + } /> + } /> + } /> + +
+ ); +} + +export default App; diff --git a/tests/fixtures/react/partial-install/src/main.tsx b/tests/fixtures/react/partial-install/src/main.tsx new file mode 100644 index 00000000..ff2401bf --- /dev/null +++ b/tests/fixtures/react/partial-install/src/main.tsx @@ -0,0 +1,14 @@ +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; +import { BrowserRouter } from 'react-router-dom'; +// TODO: Add AuthKitProvider +// import { AuthKitProvider } from '@workos-inc/authkit-react'; +import App from './App.tsx'; + +createRoot(document.getElementById('root')!).render( + + + + + , +); diff --git a/tests/fixtures/react/partial-install/src/pages/About.tsx b/tests/fixtures/react/partial-install/src/pages/About.tsx new file mode 100644 index 00000000..9c57a60f --- /dev/null +++ b/tests/fixtures/react/partial-install/src/pages/About.tsx @@ -0,0 +1,8 @@ +export function About() { + return ( +
+

About

+

This is an existing React application.

+
+ ); +} diff --git a/tests/fixtures/react/partial-install/src/pages/Dashboard.tsx b/tests/fixtures/react/partial-install/src/pages/Dashboard.tsx new file mode 100644 index 00000000..42409fdc --- /dev/null +++ b/tests/fixtures/react/partial-install/src/pages/Dashboard.tsx @@ -0,0 +1,8 @@ +export function Dashboard() { + return ( +
+

Dashboard

+

Protected content would go here.

+
+ ); +} diff --git a/tests/fixtures/react/partial-install/src/pages/Home.tsx b/tests/fixtures/react/partial-install/src/pages/Home.tsx new file mode 100644 index 00000000..ed07ba17 --- /dev/null +++ b/tests/fixtures/react/partial-install/src/pages/Home.tsx @@ -0,0 +1,8 @@ +export function Home() { + return ( +
+

Home

+

Welcome to the home page.

+
+ ); +} diff --git a/tests/fixtures/react/partial-install/src/vite-env.d.ts b/tests/fixtures/react/partial-install/src/vite-env.d.ts new file mode 100644 index 00000000..11f02fe2 --- /dev/null +++ b/tests/fixtures/react/partial-install/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/tests/fixtures/react/partial-install/tsconfig.json b/tests/fixtures/react/partial-install/tsconfig.json new file mode 100644 index 00000000..d5419222 --- /dev/null +++ b/tests/fixtures/react/partial-install/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["src"] +} diff --git a/tests/fixtures/react/partial-install/vite.config.ts b/tests/fixtures/react/partial-install/vite.config.ts new file mode 100644 index 00000000..d192dba1 --- /dev/null +++ b/tests/fixtures/react/partial-install/vite.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; +import { resolve } from 'path'; + +export default defineConfig({ + plugins: [react()], + resolve: { + alias: { + '@': resolve(__dirname, './src'), + }, + }, +}); diff --git a/tests/fixtures/react/typescript-strict/README.md b/tests/fixtures/react/typescript-strict/README.md new file mode 100644 index 00000000..ca16ce54 --- /dev/null +++ b/tests/fixtures/react/typescript-strict/README.md @@ -0,0 +1,28 @@ +# React SPA - TypeScript Strict Fixture + +## Edge Case Description + +This fixture has the strictest TypeScript configuration. It tests whether the agent generates fully type-safe code. + +## Expected Agent Behavior + +- Generate code with explicit return types +- Use proper type annotations +- Handle null/undefined properly +- Not introduce unused variables + +## Files of Interest + +- `tsconfig.json` - Has all strict flags including exactOptionalPropertyTypes and noUncheckedIndexedAccess +- All `.tsx` files - Have explicit return types + +## Success Criteria + +- [ ] `pnpm build` passes with zero type errors +- [ ] Generated code has proper types +- [ ] No implicit any errors +- [ ] No unused variable errors + +## Notes + +Critical for enterprise React apps with strict TypeScript. diff --git a/tests/fixtures/react/typescript-strict/index.html b/tests/fixtures/react/typescript-strict/index.html new file mode 100644 index 00000000..5da656cb --- /dev/null +++ b/tests/fixtures/react/typescript-strict/index.html @@ -0,0 +1,12 @@ + + + + + + React App + + +
+ + + diff --git a/tests/fixtures/react/typescript-strict/package.json b/tests/fixtures/react/typescript-strict/package.json new file mode 100644 index 00000000..864f987e --- /dev/null +++ b/tests/fixtures/react/typescript-strict/package.json @@ -0,0 +1,23 @@ +{ + "name": "react-typescript-strict-fixture", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^6.28.0" + }, + "devDependencies": { + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.4", + "typescript": "~5.6.2", + "vite": "^6.0.5" + } +} diff --git a/tests/fixtures/react/typescript-strict/src/App.tsx b/tests/fixtures/react/typescript-strict/src/App.tsx new file mode 100644 index 00000000..a6fc1fc5 --- /dev/null +++ b/tests/fixtures/react/typescript-strict/src/App.tsx @@ -0,0 +1,22 @@ +import { Routes, Route, Link } from 'react-router-dom'; +import type { JSX } from 'react'; +import { Home } from './pages/Home'; +import { About } from './pages/About'; +import { Dashboard } from './pages/Dashboard'; + +function App(): JSX.Element { + return ( +
+ + + } /> + } /> + } /> + +
+ ); +} + +export default App; diff --git a/tests/fixtures/react/typescript-strict/src/main.tsx b/tests/fixtures/react/typescript-strict/src/main.tsx new file mode 100644 index 00000000..954c6f0c --- /dev/null +++ b/tests/fixtures/react/typescript-strict/src/main.tsx @@ -0,0 +1,17 @@ +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; +import { BrowserRouter } from 'react-router-dom'; +import App from './App.tsx'; + +const rootElement = document.getElementById('root'); +if (!rootElement) { + throw new Error('Root element not found'); +} + +createRoot(rootElement).render( + + + + + , +); diff --git a/tests/fixtures/react/typescript-strict/src/pages/About.tsx b/tests/fixtures/react/typescript-strict/src/pages/About.tsx new file mode 100644 index 00000000..bf07559f --- /dev/null +++ b/tests/fixtures/react/typescript-strict/src/pages/About.tsx @@ -0,0 +1,10 @@ +import type { JSX } from 'react'; + +export function About(): JSX.Element { + return ( +
+

About

+

This is an existing React application with strict TypeScript.

+
+ ); +} diff --git a/tests/fixtures/react/typescript-strict/src/pages/Dashboard.tsx b/tests/fixtures/react/typescript-strict/src/pages/Dashboard.tsx new file mode 100644 index 00000000..27ffcdb2 --- /dev/null +++ b/tests/fixtures/react/typescript-strict/src/pages/Dashboard.tsx @@ -0,0 +1,10 @@ +import type { JSX } from 'react'; + +export function Dashboard(): JSX.Element { + return ( +
+

Dashboard

+

Protected content would go here.

+
+ ); +} diff --git a/tests/fixtures/react/typescript-strict/src/pages/Home.tsx b/tests/fixtures/react/typescript-strict/src/pages/Home.tsx new file mode 100644 index 00000000..a1129ee5 --- /dev/null +++ b/tests/fixtures/react/typescript-strict/src/pages/Home.tsx @@ -0,0 +1,10 @@ +import type { JSX } from 'react'; + +export function Home(): JSX.Element { + return ( +
+

Home

+

Welcome to the home page.

+
+ ); +} diff --git a/tests/fixtures/react/typescript-strict/src/vite-env.d.ts b/tests/fixtures/react/typescript-strict/src/vite-env.d.ts new file mode 100644 index 00000000..11f02fe2 --- /dev/null +++ b/tests/fixtures/react/typescript-strict/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/tests/fixtures/react/typescript-strict/tsconfig.json b/tests/fixtures/react/typescript-strict/tsconfig.json new file mode 100644 index 00000000..9058ee71 --- /dev/null +++ b/tests/fixtures/react/typescript-strict/tsconfig.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "strictBindCallApply": true, + "strictPropertyInitialization": true, + "noImplicitThis": true, + "noImplicitReturns": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "exactOptionalPropertyTypes": true, + "noUncheckedIndexedAccess": true, + "noFallthroughCasesInSwitch": true, + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["src"] +} diff --git a/tests/fixtures/react/typescript-strict/vite.config.ts b/tests/fixtures/react/typescript-strict/vite.config.ts new file mode 100644 index 00000000..d192dba1 --- /dev/null +++ b/tests/fixtures/react/typescript-strict/vite.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; +import { resolve } from 'path'; + +export default defineConfig({ + plugins: [react()], + resolve: { + alias: { + '@': resolve(__dirname, './src'), + }, + }, +}); diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/README.md b/tests/fixtures/tanstack-start/conflicting-middleware/README.md new file mode 100644 index 00000000..aa5e5b54 --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/README.md @@ -0,0 +1,32 @@ +# TanStack Start - Conflicting Middleware Fixture + +## Edge Case Description + +This fixture has existing server middleware using TanStack Start's server functions for rate limiting, request logging, and security headers. The agent must integrate AuthKit while preserving this custom middleware. + +## Expected Agent Behavior + +- Detect existing middleware in `src/middleware.server.ts` +- Integrate AuthKit while PRESERVING: + - Rate limiting logic via server functions + - Request logging + - Security headers + - Existing route loaders +- Compose AuthKit with existing server functions + +## Files of Interest + +- `src/middleware.server.ts` - Custom server functions for middleware +- `src/routes/dashboard.tsx` - Uses middleware in loader + +## Success Criteria + +- [ ] AuthKit is integrated +- [ ] Rate limiting still works +- [ ] Security headers still added +- [ ] Request logging still works +- [ ] Build succeeds + +## Notes + +TanStack Start uses server functions for server-side logic. The agent should compose AuthKit session management with existing patterns. diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/package.json b/tests/fixtures/tanstack-start/conflicting-middleware/package.json new file mode 100644 index 00000000..fdd08c28 --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/package.json @@ -0,0 +1,25 @@ +{ + "name": "tanstack-start-conflicting-middleware-fixture", + "private": true, + "type": "module", + "scripts": { + "dev": "vite dev --port 3000", + "build": "vite build", + "start": "vite preview" + }, + "dependencies": { + "@tanstack/react-router": "latest", + "@tanstack/react-start": "latest", + "@tanstack/router-plugin": "latest", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "vite-tsconfig-paths": "^6.0.0" + }, + "devDependencies": { + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", + "@vitejs/plugin-react": "^5.0.0", + "typescript": "^5.7.0", + "vite": "^7.0.0" + } +} diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/src/middleware.server.ts b/tests/fixtures/tanstack-start/conflicting-middleware/src/middleware.server.ts new file mode 100644 index 00000000..7ef0893c --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/src/middleware.server.ts @@ -0,0 +1,75 @@ +// Custom server middleware for TanStack Start +// This file contains middleware logic that should be preserved when adding AuthKit + +import { createServerFn } from '@tanstack/react-start'; + +interface RateLimitRecord { + count: number; + resetTime: number; +} + +const rateLimitStore = new Map(); +const RATE_LIMIT = 100; +const WINDOW_MS = 60 * 1000; + +export function checkRateLimit(ip: string): { allowed: boolean; remaining: number } { + const now = Date.now(); + const record = rateLimitStore.get(ip); + + if (!record || now > record.resetTime) { + rateLimitStore.set(ip, { count: 1, resetTime: now + WINDOW_MS }); + return { allowed: true, remaining: RATE_LIMIT - 1 }; + } + + if (record.count >= RATE_LIMIT) { + return { allowed: false, remaining: 0 }; + } + + record.count++; + return { allowed: true, remaining: RATE_LIMIT - record.count }; +} + +export function logRequest(method: string, path: string, ip: string): void { + console.log(`[${new Date().toISOString()}] ${method} ${path} from ${ip}`); +} + +export interface SecurityContext { + ip: string; + userAgent: string; + timestamp: number; +} + +export const getSecurityContext = createServerFn({ method: 'GET' }).handler(async (): Promise => { + // In a real app, this would get request headers from the server context + return { + ip: 'server-rendered', + userAgent: 'server', + timestamp: Date.now(), + }; +}); + +export const validateRequest = createServerFn({ method: 'POST' }) + .validator((data: { ip: string; path: string }) => data) + .handler(async ({ data }): Promise<{ valid: boolean; error?: string }> => { + const { ip, path } = data; + + // Log the request + logRequest('POST', path, ip); + + // Check rate limit + const { allowed } = checkRateLimit(ip); + if (!allowed) { + return { valid: false, error: 'Rate limit exceeded' }; + } + + return { valid: true }; + }); + +export const getServerHeaders = (): Record => { + return { + 'X-App-Version': '1.0.0', + 'X-Content-Type-Options': 'nosniff', + 'X-Frame-Options': 'DENY', + 'X-XSS-Protection': '1; mode=block', + }; +}; diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/src/routeTree.gen.ts b/tests/fixtures/tanstack-start/conflicting-middleware/src/routeTree.gen.ts new file mode 100644 index 00000000..fba95d05 --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/src/routeTree.gen.ts @@ -0,0 +1,84 @@ +/* eslint-disable */ + +// @ts-nocheck + +// noinspection JSUnusedGlobalSymbols + +// This file was automatically generated by TanStack Router. +// You should NOT make any changes in this file as it will be overwritten. +// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. + +import { Route as rootRouteImport } from './routes/__root'; +import { Route as DashboardRouteImport } from './routes/dashboard'; +import { Route as IndexRouteImport } from './routes/index'; + +const DashboardRoute = DashboardRouteImport.update({ + id: '/dashboard', + path: '/dashboard', + getParentRoute: () => rootRouteImport, +} as any); +const IndexRoute = IndexRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => rootRouteImport, +} as any); + +export interface FileRoutesByFullPath { + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; +} +export interface FileRoutesByTo { + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; +} +export interface FileRoutesById { + __root__: typeof rootRouteImport; + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; +} +export interface FileRouteTypes { + fileRoutesByFullPath: FileRoutesByFullPath; + fullPaths: '/' | '/dashboard'; + fileRoutesByTo: FileRoutesByTo; + to: '/' | '/dashboard'; + id: '__root__' | '/' | '/dashboard'; + fileRoutesById: FileRoutesById; +} +export interface RootRouteChildren { + IndexRoute: typeof IndexRoute; + DashboardRoute: typeof DashboardRoute; +} + +declare module '@tanstack/react-router' { + interface FileRoutesByPath { + '/dashboard': { + id: '/dashboard'; + path: '/dashboard'; + fullPath: '/dashboard'; + preLoaderRoute: typeof DashboardRouteImport; + parentRoute: typeof rootRouteImport; + }; + '/': { + id: '/'; + path: '/'; + fullPath: '/'; + preLoaderRoute: typeof IndexRouteImport; + parentRoute: typeof rootRouteImport; + }; + } +} + +const rootRouteChildren: RootRouteChildren = { + IndexRoute: IndexRoute, + DashboardRoute: DashboardRoute, +}; +export const routeTree = rootRouteImport._addFileChildren(rootRouteChildren)._addFileTypes(); + +import type { getRouter } from './router.tsx'; +import type { createStart } from '@tanstack/react-start'; +declare module '@tanstack/react-start' { + interface Register { + ssr: true; + router: Awaited>; + } +} diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/src/router.tsx b/tests/fixtures/tanstack-start/conflicting-middleware/src/router.tsx new file mode 100644 index 00000000..4df65543 --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/src/router.tsx @@ -0,0 +1,13 @@ +import { createRouter } from '@tanstack/react-router'; +import { routeTree } from './routeTree.gen'; + +export const getRouter = () => { + const router = createRouter({ + routeTree, + context: {}, + scrollRestoration: true, + defaultPreloadStaleTime: 0, + }); + + return router; +}; diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/src/routes/__root.tsx b/tests/fixtures/tanstack-start/conflicting-middleware/src/routes/__root.tsx new file mode 100644 index 00000000..22319ff9 --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/src/routes/__root.tsx @@ -0,0 +1,28 @@ +import { HeadContent, Scripts, createRootRoute } from '@tanstack/react-router'; +import appCss from '../styles.css?url'; + +export const Route = createRootRoute({ + head: () => ({ + meta: [ + { charSet: 'utf-8' }, + { name: 'viewport', content: 'width=device-width, initial-scale=1' }, + { title: 'TanStack Start App' }, + ], + links: [{ rel: 'stylesheet', href: appCss }], + }), + shellComponent: RootDocument, +}); + +function RootDocument({ children }: { children: React.ReactNode }) { + return ( + + + + + + {children} + + + + ); +} diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/src/routes/dashboard.tsx b/tests/fixtures/tanstack-start/conflicting-middleware/src/routes/dashboard.tsx new file mode 100644 index 00000000..659e2008 --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/src/routes/dashboard.tsx @@ -0,0 +1,51 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { getSecurityContext, validateRequest } from '../middleware.server'; + +export const Route = createFileRoute('/dashboard')({ + loader: async () => { + // Get security context from server + const securityContext = await getSecurityContext(); + + // Validate the request through our middleware + const validation = await validateRequest({ + data: { + ip: securityContext.ip, + path: '/dashboard', + }, + }); + + if (!validation.valid) { + throw new Error(validation.error ?? 'Request validation failed'); + } + + return { + securityContext, + stats: { + users: 1234, + revenue: 12345, + }, + }; + }, + component: Dashboard, +}); + +function Dashboard() { + const { stats } = Route.useLoaderData(); + + return ( +
+

Dashboard

+

This is a protected dashboard page.

+
+
+

Users

+

{stats.users.toLocaleString()}

+
+
+

Revenue

+

${stats.revenue.toLocaleString()}

+
+
+
+ ); +} diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/src/routes/index.tsx b/tests/fixtures/tanstack-start/conflicting-middleware/src/routes/index.tsx new file mode 100644 index 00000000..6d4a097f --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/src/routes/index.tsx @@ -0,0 +1,17 @@ +import { createFileRoute, Link } from '@tanstack/react-router'; + +export const Route = createFileRoute('/')({ + component: Home, +}); + +function Home() { + return ( +
+

Welcome to My App

+

This is an existing TanStack Start application with custom middleware.

+ +
+ ); +} diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/src/styles.css b/tests/fixtures/tanstack-start/conflicting-middleware/src/styles.css new file mode 100644 index 00000000..3ac4e274 --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/src/styles.css @@ -0,0 +1,21 @@ +body { + margin: 0; + font-family: + -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Fira Sans', 'Droid Sans', + 'Helvetica Neue', sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.container { + max-width: 800px; + margin: 0 auto; + padding: 2rem; +} + +code { + font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', monospace; + background: #f4f4f4; + padding: 0.2em 0.4em; + border-radius: 3px; +} diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/tsconfig.json b/tests/fixtures/tanstack-start/conflicting-middleware/tsconfig.json new file mode 100644 index 00000000..2b33c015 --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["DOM", "DOM.Iterable", "ES2022"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "paths": { + "~/*": ["./src/*"] + } + }, + "include": ["src"] +} diff --git a/tests/fixtures/tanstack-start/conflicting-middleware/vite.config.ts b/tests/fixtures/tanstack-start/conflicting-middleware/vite.config.ts new file mode 100644 index 00000000..edd01ec9 --- /dev/null +++ b/tests/fixtures/tanstack-start/conflicting-middleware/vite.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vite'; +import { tanstackStart } from '@tanstack/react-start/plugin/vite'; +import viteReact from '@vitejs/plugin-react'; +import viteTsConfigPaths from 'vite-tsconfig-paths'; + +export default defineConfig({ + plugins: [ + viteTsConfigPaths({ + projects: ['./tsconfig.json'], + }), + tanstackStart(), + viteReact(), + ], +}); diff --git a/tests/fixtures/tanstack-start/example-auth0/src/routeTree.gen.ts b/tests/fixtures/tanstack-start/example-auth0/src/routeTree.gen.ts index 1acbf7ff..fba95d05 100644 --- a/tests/fixtures/tanstack-start/example-auth0/src/routeTree.gen.ts +++ b/tests/fixtures/tanstack-start/example-auth0/src/routeTree.gen.ts @@ -8,79 +8,77 @@ // You should NOT make any changes in this file as it will be overwritten. // Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. -import { Route as rootRouteImport } from './routes/__root' -import { Route as DashboardRouteImport } from './routes/dashboard' -import { Route as IndexRouteImport } from './routes/index' +import { Route as rootRouteImport } from './routes/__root'; +import { Route as DashboardRouteImport } from './routes/dashboard'; +import { Route as IndexRouteImport } from './routes/index'; const DashboardRoute = DashboardRouteImport.update({ id: '/dashboard', path: '/dashboard', getParentRoute: () => rootRouteImport, -} as any) +} as any); const IndexRoute = IndexRouteImport.update({ id: '/', path: '/', getParentRoute: () => rootRouteImport, -} as any) +} as any); export interface FileRoutesByFullPath { - '/': typeof IndexRoute - '/dashboard': typeof DashboardRoute + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; } export interface FileRoutesByTo { - '/': typeof IndexRoute - '/dashboard': typeof DashboardRoute + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; } export interface FileRoutesById { - __root__: typeof rootRouteImport - '/': typeof IndexRoute - '/dashboard': typeof DashboardRoute + __root__: typeof rootRouteImport; + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; } export interface FileRouteTypes { - fileRoutesByFullPath: FileRoutesByFullPath - fullPaths: '/' | '/dashboard' - fileRoutesByTo: FileRoutesByTo - to: '/' | '/dashboard' - id: '__root__' | '/' | '/dashboard' - fileRoutesById: FileRoutesById + fileRoutesByFullPath: FileRoutesByFullPath; + fullPaths: '/' | '/dashboard'; + fileRoutesByTo: FileRoutesByTo; + to: '/' | '/dashboard'; + id: '__root__' | '/' | '/dashboard'; + fileRoutesById: FileRoutesById; } export interface RootRouteChildren { - IndexRoute: typeof IndexRoute - DashboardRoute: typeof DashboardRoute + IndexRoute: typeof IndexRoute; + DashboardRoute: typeof DashboardRoute; } declare module '@tanstack/react-router' { interface FileRoutesByPath { '/dashboard': { - id: '/dashboard' - path: '/dashboard' - fullPath: '/dashboard' - preLoaderRoute: typeof DashboardRouteImport - parentRoute: typeof rootRouteImport - } + id: '/dashboard'; + path: '/dashboard'; + fullPath: '/dashboard'; + preLoaderRoute: typeof DashboardRouteImport; + parentRoute: typeof rootRouteImport; + }; '/': { - id: '/' - path: '/' - fullPath: '/' - preLoaderRoute: typeof IndexRouteImport - parentRoute: typeof rootRouteImport - } + id: '/'; + path: '/'; + fullPath: '/'; + preLoaderRoute: typeof IndexRouteImport; + parentRoute: typeof rootRouteImport; + }; } } const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, DashboardRoute: DashboardRoute, -} -export const routeTree = rootRouteImport - ._addFileChildren(rootRouteChildren) - ._addFileTypes() +}; +export const routeTree = rootRouteImport._addFileChildren(rootRouteChildren)._addFileTypes(); -import type { getRouter } from './router.tsx' -import type { createStart } from '@tanstack/react-start' +import type { getRouter } from './router.tsx'; +import type { createStart } from '@tanstack/react-start'; declare module '@tanstack/react-start' { interface Register { - ssr: true - router: Awaited> + ssr: true; + router: Awaited>; } } diff --git a/tests/fixtures/tanstack-start/example-auth0/src/router.tsx b/tests/fixtures/tanstack-start/example-auth0/src/router.tsx index 0c83bf0d..4df65543 100644 --- a/tests/fixtures/tanstack-start/example-auth0/src/router.tsx +++ b/tests/fixtures/tanstack-start/example-auth0/src/router.tsx @@ -1,5 +1,5 @@ -import { createRouter } from '@tanstack/react-router' -import { routeTree } from './routeTree.gen' +import { createRouter } from '@tanstack/react-router'; +import { routeTree } from './routeTree.gen'; export const getRouter = () => { const router = createRouter({ @@ -7,7 +7,7 @@ export const getRouter = () => { context: {}, scrollRestoration: true, defaultPreloadStaleTime: 0, - }) + }); - return router -} + return router; +}; diff --git a/tests/fixtures/tanstack-start/example-auth0/src/routes/dashboard.tsx b/tests/fixtures/tanstack-start/example-auth0/src/routes/dashboard.tsx index 3f372064..f820fe20 100644 --- a/tests/fixtures/tanstack-start/example-auth0/src/routes/dashboard.tsx +++ b/tests/fixtures/tanstack-start/example-auth0/src/routes/dashboard.tsx @@ -1,12 +1,12 @@ -import { createFileRoute } from '@tanstack/react-router' -import { useAuth0, withAuthenticationRequired } from '@auth0/auth0-react' +import { createFileRoute } from '@tanstack/react-router'; +import { useAuth0, withAuthenticationRequired } from '@auth0/auth0-react'; export const Route = createFileRoute('/dashboard')({ component: withAuthenticationRequired(Dashboard), -}) +}); function Dashboard() { - const { user } = useAuth0() + const { user } = useAuth0(); return (
@@ -23,5 +23,5 @@ function Dashboard() {
- ) + ); } diff --git a/tests/fixtures/tanstack-start/example-auth0/src/routes/index.tsx b/tests/fixtures/tanstack-start/example-auth0/src/routes/index.tsx index b3abc830..e12b17bf 100644 --- a/tests/fixtures/tanstack-start/example-auth0/src/routes/index.tsx +++ b/tests/fixtures/tanstack-start/example-auth0/src/routes/index.tsx @@ -1,12 +1,12 @@ -import { createFileRoute, Link } from '@tanstack/react-router' -import { useAuth0 } from '@auth0/auth0-react' +import { createFileRoute, Link } from '@tanstack/react-router'; +import { useAuth0 } from '@auth0/auth0-react'; export const Route = createFileRoute('/')({ component: Home, -}) +}); function Home() { - const { isAuthenticated, loginWithRedirect, logout, user } = useAuth0() + const { isAuthenticated, loginWithRedirect, logout, user } = useAuth0(); return (
@@ -25,5 +25,5 @@ function Home() { )}
- ) + ); } diff --git a/tests/fixtures/tanstack-start/example-auth0/src/styles.css b/tests/fixtures/tanstack-start/example-auth0/src/styles.css index c48d4c36..3ac4e274 100644 --- a/tests/fixtures/tanstack-start/example-auth0/src/styles.css +++ b/tests/fixtures/tanstack-start/example-auth0/src/styles.css @@ -1,7 +1,8 @@ body { margin: 0; - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, - Ubuntu, Cantarell, 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; + font-family: + -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Fira Sans', 'Droid Sans', + 'Helvetica Neue', sans-serif; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; } diff --git a/tests/fixtures/tanstack-start/example-auth0/vite.config.ts b/tests/fixtures/tanstack-start/example-auth0/vite.config.ts index a3594f99..edd01ec9 100644 --- a/tests/fixtures/tanstack-start/example-auth0/vite.config.ts +++ b/tests/fixtures/tanstack-start/example-auth0/vite.config.ts @@ -1,7 +1,7 @@ -import { defineConfig } from 'vite' -import { tanstackStart } from '@tanstack/react-start/plugin/vite' -import viteReact from '@vitejs/plugin-react' -import viteTsConfigPaths from 'vite-tsconfig-paths' +import { defineConfig } from 'vite'; +import { tanstackStart } from '@tanstack/react-start/plugin/vite'; +import viteReact from '@vitejs/plugin-react'; +import viteTsConfigPaths from 'vite-tsconfig-paths'; export default defineConfig({ plugins: [ @@ -11,4 +11,4 @@ export default defineConfig({ tanstackStart(), viteReact(), ], -}) +}); diff --git a/tests/fixtures/tanstack-start/example/src/routeTree.gen.ts b/tests/fixtures/tanstack-start/example/src/routeTree.gen.ts index 1acbf7ff..fba95d05 100644 --- a/tests/fixtures/tanstack-start/example/src/routeTree.gen.ts +++ b/tests/fixtures/tanstack-start/example/src/routeTree.gen.ts @@ -8,79 +8,77 @@ // You should NOT make any changes in this file as it will be overwritten. // Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. -import { Route as rootRouteImport } from './routes/__root' -import { Route as DashboardRouteImport } from './routes/dashboard' -import { Route as IndexRouteImport } from './routes/index' +import { Route as rootRouteImport } from './routes/__root'; +import { Route as DashboardRouteImport } from './routes/dashboard'; +import { Route as IndexRouteImport } from './routes/index'; const DashboardRoute = DashboardRouteImport.update({ id: '/dashboard', path: '/dashboard', getParentRoute: () => rootRouteImport, -} as any) +} as any); const IndexRoute = IndexRouteImport.update({ id: '/', path: '/', getParentRoute: () => rootRouteImport, -} as any) +} as any); export interface FileRoutesByFullPath { - '/': typeof IndexRoute - '/dashboard': typeof DashboardRoute + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; } export interface FileRoutesByTo { - '/': typeof IndexRoute - '/dashboard': typeof DashboardRoute + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; } export interface FileRoutesById { - __root__: typeof rootRouteImport - '/': typeof IndexRoute - '/dashboard': typeof DashboardRoute + __root__: typeof rootRouteImport; + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; } export interface FileRouteTypes { - fileRoutesByFullPath: FileRoutesByFullPath - fullPaths: '/' | '/dashboard' - fileRoutesByTo: FileRoutesByTo - to: '/' | '/dashboard' - id: '__root__' | '/' | '/dashboard' - fileRoutesById: FileRoutesById + fileRoutesByFullPath: FileRoutesByFullPath; + fullPaths: '/' | '/dashboard'; + fileRoutesByTo: FileRoutesByTo; + to: '/' | '/dashboard'; + id: '__root__' | '/' | '/dashboard'; + fileRoutesById: FileRoutesById; } export interface RootRouteChildren { - IndexRoute: typeof IndexRoute - DashboardRoute: typeof DashboardRoute + IndexRoute: typeof IndexRoute; + DashboardRoute: typeof DashboardRoute; } declare module '@tanstack/react-router' { interface FileRoutesByPath { '/dashboard': { - id: '/dashboard' - path: '/dashboard' - fullPath: '/dashboard' - preLoaderRoute: typeof DashboardRouteImport - parentRoute: typeof rootRouteImport - } + id: '/dashboard'; + path: '/dashboard'; + fullPath: '/dashboard'; + preLoaderRoute: typeof DashboardRouteImport; + parentRoute: typeof rootRouteImport; + }; '/': { - id: '/' - path: '/' - fullPath: '/' - preLoaderRoute: typeof IndexRouteImport - parentRoute: typeof rootRouteImport - } + id: '/'; + path: '/'; + fullPath: '/'; + preLoaderRoute: typeof IndexRouteImport; + parentRoute: typeof rootRouteImport; + }; } } const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, DashboardRoute: DashboardRoute, -} -export const routeTree = rootRouteImport - ._addFileChildren(rootRouteChildren) - ._addFileTypes() +}; +export const routeTree = rootRouteImport._addFileChildren(rootRouteChildren)._addFileTypes(); -import type { getRouter } from './router.tsx' -import type { createStart } from '@tanstack/react-start' +import type { getRouter } from './router.tsx'; +import type { createStart } from '@tanstack/react-start'; declare module '@tanstack/react-start' { interface Register { - ssr: true - router: Awaited> + ssr: true; + router: Awaited>; } } diff --git a/tests/fixtures/tanstack-start/example/src/router.tsx b/tests/fixtures/tanstack-start/example/src/router.tsx index 0c83bf0d..4df65543 100644 --- a/tests/fixtures/tanstack-start/example/src/router.tsx +++ b/tests/fixtures/tanstack-start/example/src/router.tsx @@ -1,5 +1,5 @@ -import { createRouter } from '@tanstack/react-router' -import { routeTree } from './routeTree.gen' +import { createRouter } from '@tanstack/react-router'; +import { routeTree } from './routeTree.gen'; export const getRouter = () => { const router = createRouter({ @@ -7,7 +7,7 @@ export const getRouter = () => { context: {}, scrollRestoration: true, defaultPreloadStaleTime: 0, - }) + }); - return router -} + return router; +}; diff --git a/tests/fixtures/tanstack-start/example/src/routes/__root.tsx b/tests/fixtures/tanstack-start/example/src/routes/__root.tsx index 4c191584..22319ff9 100644 --- a/tests/fixtures/tanstack-start/example/src/routes/__root.tsx +++ b/tests/fixtures/tanstack-start/example/src/routes/__root.tsx @@ -1,5 +1,5 @@ -import { HeadContent, Scripts, createRootRoute } from '@tanstack/react-router' -import appCss from '../styles.css?url' +import { HeadContent, Scripts, createRootRoute } from '@tanstack/react-router'; +import appCss from '../styles.css?url'; export const Route = createRootRoute({ head: () => ({ @@ -11,7 +11,7 @@ export const Route = createRootRoute({ links: [{ rel: 'stylesheet', href: appCss }], }), shellComponent: RootDocument, -}) +}); function RootDocument({ children }: { children: React.ReactNode }) { return ( @@ -24,5 +24,5 @@ function RootDocument({ children }: { children: React.ReactNode }) { - ) + ); } diff --git a/tests/fixtures/tanstack-start/example/src/routes/dashboard.tsx b/tests/fixtures/tanstack-start/example/src/routes/dashboard.tsx index 2d523ae6..34a7b445 100644 --- a/tests/fixtures/tanstack-start/example/src/routes/dashboard.tsx +++ b/tests/fixtures/tanstack-start/example/src/routes/dashboard.tsx @@ -1,8 +1,8 @@ -import { createFileRoute } from '@tanstack/react-router' +import { createFileRoute } from '@tanstack/react-router'; export const Route = createFileRoute('/dashboard')({ component: Dashboard, -}) +}); function Dashboard() { return ( @@ -20,5 +20,5 @@ function Dashboard() { - ) + ); } diff --git a/tests/fixtures/tanstack-start/example/src/routes/index.tsx b/tests/fixtures/tanstack-start/example/src/routes/index.tsx index aaec5a53..2edc996d 100644 --- a/tests/fixtures/tanstack-start/example/src/routes/index.tsx +++ b/tests/fixtures/tanstack-start/example/src/routes/index.tsx @@ -1,8 +1,8 @@ -import { createFileRoute, Link } from '@tanstack/react-router' +import { createFileRoute, Link } from '@tanstack/react-router'; export const Route = createFileRoute('/')({ component: Home, -}) +}); function Home() { return ( @@ -13,5 +13,5 @@ function Home() { Go to Dashboard - ) + ); } diff --git a/tests/fixtures/tanstack-start/example/src/styles.css b/tests/fixtures/tanstack-start/example/src/styles.css index c48d4c36..3ac4e274 100644 --- a/tests/fixtures/tanstack-start/example/src/styles.css +++ b/tests/fixtures/tanstack-start/example/src/styles.css @@ -1,7 +1,8 @@ body { margin: 0; - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, - Ubuntu, Cantarell, 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; + font-family: + -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Fira Sans', 'Droid Sans', + 'Helvetica Neue', sans-serif; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; } diff --git a/tests/fixtures/tanstack-start/example/vite.config.ts b/tests/fixtures/tanstack-start/example/vite.config.ts index a3594f99..edd01ec9 100644 --- a/tests/fixtures/tanstack-start/example/vite.config.ts +++ b/tests/fixtures/tanstack-start/example/vite.config.ts @@ -1,7 +1,7 @@ -import { defineConfig } from 'vite' -import { tanstackStart } from '@tanstack/react-start/plugin/vite' -import viteReact from '@vitejs/plugin-react' -import viteTsConfigPaths from 'vite-tsconfig-paths' +import { defineConfig } from 'vite'; +import { tanstackStart } from '@tanstack/react-start/plugin/vite'; +import viteReact from '@vitejs/plugin-react'; +import viteTsConfigPaths from 'vite-tsconfig-paths'; export default defineConfig({ plugins: [ @@ -11,4 +11,4 @@ export default defineConfig({ tanstackStart(), viteReact(), ], -}) +}); diff --git a/tests/fixtures/tanstack-start/partial-install/README.md b/tests/fixtures/tanstack-start/partial-install/README.md new file mode 100644 index 00000000..8c249c0e --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/README.md @@ -0,0 +1,30 @@ +# TanStack Start - Partial Install Fixture + +## Edge Case Description + +This fixture represents a TanStack Start project where AuthKit was partially installed - the package is in dependencies but integration was never completed. + +## Expected Agent Behavior + +- Detect that @workos-inc/authkit-react is already installed +- Complete the integration by: + - Adding AuthKitProvider + - Creating callback route + - Setting up server functions for auth +- Should NOT reinstall the package + +## Files of Interest + +- `package.json` - Already has @workos-inc/authkit-react dependency +- `src/router.tsx` - Has commented-out import + +## Success Criteria + +- [ ] AuthKitProvider wraps the app +- [ ] Callback route is created +- [ ] Build succeeds +- [ ] Package is not reinstalled + +## Notes + +Common scenario when developers start integration but don't finish. diff --git a/tests/fixtures/tanstack-start/partial-install/package.json b/tests/fixtures/tanstack-start/partial-install/package.json new file mode 100644 index 00000000..35ec5589 --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/package.json @@ -0,0 +1,26 @@ +{ + "name": "tanstack-start-partial-install-fixture", + "private": true, + "type": "module", + "scripts": { + "dev": "vite dev --port 3000", + "build": "vite build", + "start": "vite preview" + }, + "dependencies": { + "@tanstack/react-router": "latest", + "@tanstack/react-start": "latest", + "@tanstack/router-plugin": "latest", + "@workos-inc/authkit-react": "^0.5.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "vite-tsconfig-paths": "^6.0.0" + }, + "devDependencies": { + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", + "@vitejs/plugin-react": "^5.0.0", + "typescript": "^5.7.0", + "vite": "^7.0.0" + } +} diff --git a/tests/fixtures/tanstack-start/partial-install/src/routeTree.gen.ts b/tests/fixtures/tanstack-start/partial-install/src/routeTree.gen.ts new file mode 100644 index 00000000..fba95d05 --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/src/routeTree.gen.ts @@ -0,0 +1,84 @@ +/* eslint-disable */ + +// @ts-nocheck + +// noinspection JSUnusedGlobalSymbols + +// This file was automatically generated by TanStack Router. +// You should NOT make any changes in this file as it will be overwritten. +// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. + +import { Route as rootRouteImport } from './routes/__root'; +import { Route as DashboardRouteImport } from './routes/dashboard'; +import { Route as IndexRouteImport } from './routes/index'; + +const DashboardRoute = DashboardRouteImport.update({ + id: '/dashboard', + path: '/dashboard', + getParentRoute: () => rootRouteImport, +} as any); +const IndexRoute = IndexRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => rootRouteImport, +} as any); + +export interface FileRoutesByFullPath { + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; +} +export interface FileRoutesByTo { + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; +} +export interface FileRoutesById { + __root__: typeof rootRouteImport; + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; +} +export interface FileRouteTypes { + fileRoutesByFullPath: FileRoutesByFullPath; + fullPaths: '/' | '/dashboard'; + fileRoutesByTo: FileRoutesByTo; + to: '/' | '/dashboard'; + id: '__root__' | '/' | '/dashboard'; + fileRoutesById: FileRoutesById; +} +export interface RootRouteChildren { + IndexRoute: typeof IndexRoute; + DashboardRoute: typeof DashboardRoute; +} + +declare module '@tanstack/react-router' { + interface FileRoutesByPath { + '/dashboard': { + id: '/dashboard'; + path: '/dashboard'; + fullPath: '/dashboard'; + preLoaderRoute: typeof DashboardRouteImport; + parentRoute: typeof rootRouteImport; + }; + '/': { + id: '/'; + path: '/'; + fullPath: '/'; + preLoaderRoute: typeof IndexRouteImport; + parentRoute: typeof rootRouteImport; + }; + } +} + +const rootRouteChildren: RootRouteChildren = { + IndexRoute: IndexRoute, + DashboardRoute: DashboardRoute, +}; +export const routeTree = rootRouteImport._addFileChildren(rootRouteChildren)._addFileTypes(); + +import type { getRouter } from './router.tsx'; +import type { createStart } from '@tanstack/react-start'; +declare module '@tanstack/react-start' { + interface Register { + ssr: true; + router: Awaited>; + } +} diff --git a/tests/fixtures/tanstack-start/partial-install/src/router.tsx b/tests/fixtures/tanstack-start/partial-install/src/router.tsx new file mode 100644 index 00000000..865a4d49 --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/src/router.tsx @@ -0,0 +1,15 @@ +import { createRouter } from '@tanstack/react-router'; +import { routeTree } from './routeTree.gen'; +// TODO: Complete AuthKit integration +// import { AuthKitProvider } from '@workos-inc/authkit-react'; + +export const getRouter = () => { + const router = createRouter({ + routeTree, + context: {}, + scrollRestoration: true, + defaultPreloadStaleTime: 0, + }); + + return router; +}; diff --git a/tests/fixtures/tanstack-start/partial-install/src/routes/__root.tsx b/tests/fixtures/tanstack-start/partial-install/src/routes/__root.tsx new file mode 100644 index 00000000..22319ff9 --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/src/routes/__root.tsx @@ -0,0 +1,28 @@ +import { HeadContent, Scripts, createRootRoute } from '@tanstack/react-router'; +import appCss from '../styles.css?url'; + +export const Route = createRootRoute({ + head: () => ({ + meta: [ + { charSet: 'utf-8' }, + { name: 'viewport', content: 'width=device-width, initial-scale=1' }, + { title: 'TanStack Start App' }, + ], + links: [{ rel: 'stylesheet', href: appCss }], + }), + shellComponent: RootDocument, +}); + +function RootDocument({ children }: { children: React.ReactNode }) { + return ( + + + + + + {children} + + + + ); +} diff --git a/tests/fixtures/tanstack-start/partial-install/src/routes/dashboard.tsx b/tests/fixtures/tanstack-start/partial-install/src/routes/dashboard.tsx new file mode 100644 index 00000000..34a7b445 --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/src/routes/dashboard.tsx @@ -0,0 +1,24 @@ +import { createFileRoute } from '@tanstack/react-router'; + +export const Route = createFileRoute('/dashboard')({ + component: Dashboard, +}); + +function Dashboard() { + return ( +
+

Dashboard

+

This is a protected dashboard page.

+
+
+

Users

+

1,234

+
+
+

Revenue

+

$12,345

+
+
+
+ ); +} diff --git a/tests/fixtures/tanstack-start/partial-install/src/routes/index.tsx b/tests/fixtures/tanstack-start/partial-install/src/routes/index.tsx new file mode 100644 index 00000000..2edc996d --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/src/routes/index.tsx @@ -0,0 +1,17 @@ +import { createFileRoute, Link } from '@tanstack/react-router'; + +export const Route = createFileRoute('/')({ + component: Home, +}); + +function Home() { + return ( +
+

Welcome to My App

+

This is an existing TanStack Start application.

+ +
+ ); +} diff --git a/tests/fixtures/tanstack-start/partial-install/src/styles.css b/tests/fixtures/tanstack-start/partial-install/src/styles.css new file mode 100644 index 00000000..3ac4e274 --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/src/styles.css @@ -0,0 +1,21 @@ +body { + margin: 0; + font-family: + -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Fira Sans', 'Droid Sans', + 'Helvetica Neue', sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.container { + max-width: 800px; + margin: 0 auto; + padding: 2rem; +} + +code { + font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', monospace; + background: #f4f4f4; + padding: 0.2em 0.4em; + border-radius: 3px; +} diff --git a/tests/fixtures/tanstack-start/partial-install/tsconfig.json b/tests/fixtures/tanstack-start/partial-install/tsconfig.json new file mode 100644 index 00000000..2b33c015 --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["DOM", "DOM.Iterable", "ES2022"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "paths": { + "~/*": ["./src/*"] + } + }, + "include": ["src"] +} diff --git a/tests/fixtures/tanstack-start/partial-install/vite.config.ts b/tests/fixtures/tanstack-start/partial-install/vite.config.ts new file mode 100644 index 00000000..edd01ec9 --- /dev/null +++ b/tests/fixtures/tanstack-start/partial-install/vite.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vite'; +import { tanstackStart } from '@tanstack/react-start/plugin/vite'; +import viteReact from '@vitejs/plugin-react'; +import viteTsConfigPaths from 'vite-tsconfig-paths'; + +export default defineConfig({ + plugins: [ + viteTsConfigPaths({ + projects: ['./tsconfig.json'], + }), + tanstackStart(), + viteReact(), + ], +}); diff --git a/tests/fixtures/tanstack-start/typescript-strict/README.md b/tests/fixtures/tanstack-start/typescript-strict/README.md new file mode 100644 index 00000000..0228c15f --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/README.md @@ -0,0 +1,27 @@ +# TanStack Start - TypeScript Strict Fixture + +## Edge Case Description + +This fixture has the strictest TypeScript configuration. Tests whether agent generates fully type-safe code. + +## Expected Agent Behavior + +- Generate code with explicit return types +- Use proper type annotations +- Handle null/undefined properly +- Not introduce unused variables + +## Files of Interest + +- `tsconfig.json` - Has all strict flags +- All `.tsx` files - Have explicit return types + +## Success Criteria + +- [ ] `pnpm build` passes with zero type errors +- [ ] Generated code has proper types +- [ ] No implicit any errors + +## Notes + +Critical for enterprise TanStack Start apps with strict TypeScript. diff --git a/tests/fixtures/tanstack-start/typescript-strict/package.json b/tests/fixtures/tanstack-start/typescript-strict/package.json new file mode 100644 index 00000000..7c540afe --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/package.json @@ -0,0 +1,25 @@ +{ + "name": "tanstack-start-typescript-strict-fixture", + "private": true, + "type": "module", + "scripts": { + "dev": "vite dev --port 3000", + "build": "vite build", + "start": "vite preview" + }, + "dependencies": { + "@tanstack/react-router": "latest", + "@tanstack/react-start": "latest", + "@tanstack/router-plugin": "latest", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "vite-tsconfig-paths": "^6.0.0" + }, + "devDependencies": { + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", + "@vitejs/plugin-react": "^5.0.0", + "typescript": "^5.7.0", + "vite": "^7.0.0" + } +} diff --git a/tests/fixtures/tanstack-start/typescript-strict/src/routeTree.gen.ts b/tests/fixtures/tanstack-start/typescript-strict/src/routeTree.gen.ts new file mode 100644 index 00000000..fba95d05 --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/src/routeTree.gen.ts @@ -0,0 +1,84 @@ +/* eslint-disable */ + +// @ts-nocheck + +// noinspection JSUnusedGlobalSymbols + +// This file was automatically generated by TanStack Router. +// You should NOT make any changes in this file as it will be overwritten. +// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. + +import { Route as rootRouteImport } from './routes/__root'; +import { Route as DashboardRouteImport } from './routes/dashboard'; +import { Route as IndexRouteImport } from './routes/index'; + +const DashboardRoute = DashboardRouteImport.update({ + id: '/dashboard', + path: '/dashboard', + getParentRoute: () => rootRouteImport, +} as any); +const IndexRoute = IndexRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => rootRouteImport, +} as any); + +export interface FileRoutesByFullPath { + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; +} +export interface FileRoutesByTo { + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; +} +export interface FileRoutesById { + __root__: typeof rootRouteImport; + '/': typeof IndexRoute; + '/dashboard': typeof DashboardRoute; +} +export interface FileRouteTypes { + fileRoutesByFullPath: FileRoutesByFullPath; + fullPaths: '/' | '/dashboard'; + fileRoutesByTo: FileRoutesByTo; + to: '/' | '/dashboard'; + id: '__root__' | '/' | '/dashboard'; + fileRoutesById: FileRoutesById; +} +export interface RootRouteChildren { + IndexRoute: typeof IndexRoute; + DashboardRoute: typeof DashboardRoute; +} + +declare module '@tanstack/react-router' { + interface FileRoutesByPath { + '/dashboard': { + id: '/dashboard'; + path: '/dashboard'; + fullPath: '/dashboard'; + preLoaderRoute: typeof DashboardRouteImport; + parentRoute: typeof rootRouteImport; + }; + '/': { + id: '/'; + path: '/'; + fullPath: '/'; + preLoaderRoute: typeof IndexRouteImport; + parentRoute: typeof rootRouteImport; + }; + } +} + +const rootRouteChildren: RootRouteChildren = { + IndexRoute: IndexRoute, + DashboardRoute: DashboardRoute, +}; +export const routeTree = rootRouteImport._addFileChildren(rootRouteChildren)._addFileTypes(); + +import type { getRouter } from './router.tsx'; +import type { createStart } from '@tanstack/react-start'; +declare module '@tanstack/react-start' { + interface Register { + ssr: true; + router: Awaited>; + } +} diff --git a/tests/fixtures/tanstack-start/typescript-strict/src/router.tsx b/tests/fixtures/tanstack-start/typescript-strict/src/router.tsx new file mode 100644 index 00000000..85328dc0 --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/src/router.tsx @@ -0,0 +1,13 @@ +import { createRouter, type Router } from '@tanstack/react-router'; +import { routeTree } from './routeTree.gen'; + +export const getRouter = (): Router => { + const router = createRouter({ + routeTree, + context: {}, + scrollRestoration: true, + defaultPreloadStaleTime: 0, + }); + + return router; +}; diff --git a/tests/fixtures/tanstack-start/typescript-strict/src/routes/__root.tsx b/tests/fixtures/tanstack-start/typescript-strict/src/routes/__root.tsx new file mode 100644 index 00000000..0986bfcd --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/src/routes/__root.tsx @@ -0,0 +1,33 @@ +import { HeadContent, Scripts, createRootRoute } from '@tanstack/react-router'; +import type { ReactNode, JSX } from 'react'; +import appCss from '../styles.css?url'; + +interface RootDocumentProps { + children: ReactNode; +} + +export const Route = createRootRoute({ + head: () => ({ + meta: [ + { charSet: 'utf-8' }, + { name: 'viewport', content: 'width=device-width, initial-scale=1' }, + { title: 'TanStack Start App' }, + ], + links: [{ rel: 'stylesheet', href: appCss }], + }), + shellComponent: RootDocument, +}); + +function RootDocument({ children }: RootDocumentProps): JSX.Element { + return ( + + + + + + {children} + + + + ); +} diff --git a/tests/fixtures/tanstack-start/typescript-strict/src/routes/dashboard.tsx b/tests/fixtures/tanstack-start/typescript-strict/src/routes/dashboard.tsx new file mode 100644 index 00000000..9357587a --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/src/routes/dashboard.tsx @@ -0,0 +1,25 @@ +import { createFileRoute } from '@tanstack/react-router'; +import type { JSX } from 'react'; + +export const Route = createFileRoute('/dashboard')({ + component: Dashboard, +}); + +function Dashboard(): JSX.Element { + return ( +
+

Dashboard

+

This is a protected dashboard page.

+
+
+

Users

+

1,234

+
+
+

Revenue

+

$12,345

+
+
+
+ ); +} diff --git a/tests/fixtures/tanstack-start/typescript-strict/src/routes/index.tsx b/tests/fixtures/tanstack-start/typescript-strict/src/routes/index.tsx new file mode 100644 index 00000000..4a1f734a --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/src/routes/index.tsx @@ -0,0 +1,18 @@ +import { createFileRoute, Link } from '@tanstack/react-router'; +import type { JSX } from 'react'; + +export const Route = createFileRoute('/')({ + component: Home, +}); + +function Home(): JSX.Element { + return ( +
+

Welcome to My App

+

This is an existing TanStack Start application with strict TypeScript.

+ +
+ ); +} diff --git a/tests/fixtures/tanstack-start/typescript-strict/src/styles.css b/tests/fixtures/tanstack-start/typescript-strict/src/styles.css new file mode 100644 index 00000000..3ac4e274 --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/src/styles.css @@ -0,0 +1,21 @@ +body { + margin: 0; + font-family: + -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Fira Sans', 'Droid Sans', + 'Helvetica Neue', sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.container { + max-width: 800px; + margin: 0 auto; + padding: 2rem; +} + +code { + font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', monospace; + background: #f4f4f4; + padding: 0.2em 0.4em; + border-radius: 3px; +} diff --git a/tests/fixtures/tanstack-start/typescript-strict/tsconfig.json b/tests/fixtures/tanstack-start/typescript-strict/tsconfig.json new file mode 100644 index 00000000..6195aaa2 --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["DOM", "DOM.Iterable", "ES2022"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "strictBindCallApply": true, + "strictPropertyInitialization": true, + "noImplicitThis": true, + "noImplicitReturns": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "exactOptionalPropertyTypes": true, + "noUncheckedIndexedAccess": true, + "noFallthroughCasesInSwitch": true, + "paths": { + "~/*": ["./src/*"] + } + }, + "include": ["src"] +} diff --git a/tests/fixtures/tanstack-start/typescript-strict/vite.config.ts b/tests/fixtures/tanstack-start/typescript-strict/vite.config.ts new file mode 100644 index 00000000..edd01ec9 --- /dev/null +++ b/tests/fixtures/tanstack-start/typescript-strict/vite.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vite'; +import { tanstackStart } from '@tanstack/react-start/plugin/vite'; +import viteReact from '@vitejs/plugin-react'; +import viteTsConfigPaths from 'vite-tsconfig-paths'; + +export default defineConfig({ + plugins: [ + viteTsConfigPaths({ + projects: ['./tsconfig.json'], + }), + tanstackStart(), + viteReact(), + ], +}); diff --git a/tests/fixtures/vanilla-js/conflicting-auth/README.md b/tests/fixtures/vanilla-js/conflicting-auth/README.md new file mode 100644 index 00000000..8bbe8bf2 --- /dev/null +++ b/tests/fixtures/vanilla-js/conflicting-auth/README.md @@ -0,0 +1,38 @@ +# Vanilla JS - Conflicting Auth Fixture + +## Edge Case Description + +This fixture has an existing custom authentication module with localStorage-based session management, auth state listeners, and protected routes. The agent must integrate AuthKit while preserving or migrating this functionality. + +## Expected Agent Behavior + +- Detect existing auth implementation in `auth.js` +- Integrate AuthKit while handling: + - Existing `onAuthStateChange` listeners + - User preferences storage + - Protected route patterns (`requireAuth`) +- Should NOT simply delete existing auth code without migration + +## Files of Interest + +- `auth.js` - Full auth module with login, logout, session management +- `main.js` - Uses auth for login form and UI updates +- `dashboard.js` - Uses `requireAuth` for page protection +- All HTML files - Reference auth status in nav + +## Success Criteria + +- [ ] AuthKit is integrated +- [ ] Existing auth state listeners still work +- [ ] Dashboard protection still works +- [ ] User preferences are migrated or preserved +- [ ] Build succeeds + +## Notes + +This is a realistic scenario - many vanilla JS apps have custom auth before adopting a third-party solution. The agent should recognize this and propose a migration strategy. + +Ideal approaches: +1. Replace custom auth with AuthKit but preserve the listener pattern +2. Migrate user preferences to AuthKit user profile +3. Update requireAuth to use AuthKit session checking diff --git a/tests/fixtures/vanilla-js/conflicting-auth/about.html b/tests/fixtures/vanilla-js/conflicting-auth/about.html new file mode 100644 index 00000000..1c9cae91 --- /dev/null +++ b/tests/fixtures/vanilla-js/conflicting-auth/about.html @@ -0,0 +1,20 @@ + + + + + + About - Vanilla JS App + + + + +
+

About

+

This is an existing Vanilla JS application with custom auth.

+
+ + + diff --git a/tests/fixtures/vanilla-js/conflicting-auth/auth.js b/tests/fixtures/vanilla-js/conflicting-auth/auth.js new file mode 100644 index 00000000..2475e9c9 --- /dev/null +++ b/tests/fixtures/vanilla-js/conflicting-auth/auth.js @@ -0,0 +1,95 @@ +// Custom authentication module for Vanilla JS +// This module provides existing auth functionality that should be preserved + +const AUTH_STORAGE_KEY = 'app_auth_state'; + +// Simple event system for auth state changes +const authListeners = new Set(); + +export function onAuthStateChange(callback) { + authListeners.add(callback); + return () => authListeners.delete(callback); +} + +function notifyAuthStateChange(user) { + authListeners.forEach((callback) => callback(user)); +} + +// Get current auth state from localStorage +export function getCurrentUser() { + try { + const stored = localStorage.getItem(AUTH_STORAGE_KEY); + if (stored) { + const data = JSON.parse(stored); + // Check if session is expired + if (data.expiresAt && Date.now() > data.expiresAt) { + logout(); + return null; + } + return data.user; + } + } catch { + // Invalid stored data + localStorage.removeItem(AUTH_STORAGE_KEY); + } + return null; +} + +// Simple login function (mock implementation) +export async function login(credentials) { + // Simulate API call + await new Promise((resolve) => setTimeout(resolve, 500)); + + if (!credentials.email || !credentials.password) { + throw new Error('Email and password required'); + } + + const user = { + id: 'user-' + Math.random().toString(36).substr(2, 9), + email: credentials.email, + name: credentials.email.split('@')[0], + role: 'user', + preferences: { + theme: 'light', + notifications: true, + }, + }; + + const authState = { + user, + expiresAt: Date.now() + 24 * 60 * 60 * 1000, // 24 hours + }; + + localStorage.setItem(AUTH_STORAGE_KEY, JSON.stringify(authState)); + notifyAuthStateChange(user); + + return user; +} + +export function logout() { + localStorage.removeItem(AUTH_STORAGE_KEY); + notifyAuthStateChange(null); +} + +// Check if user is authenticated +export function isAuthenticated() { + return getCurrentUser() !== null; +} + +// Protect a page - redirect if not authenticated +export function requireAuth(redirectTo = '/') { + if (!isAuthenticated()) { + window.location.href = redirectTo; + return false; + } + return true; +} + +// Initialize auth state on page load +export function initAuth() { + const user = getCurrentUser(); + if (user) { + notifyAuthStateChange(user); + } + return user; +} diff --git a/tests/fixtures/vanilla-js/conflicting-auth/dashboard.html b/tests/fixtures/vanilla-js/conflicting-auth/dashboard.html new file mode 100644 index 00000000..eded59b6 --- /dev/null +++ b/tests/fixtures/vanilla-js/conflicting-auth/dashboard.html @@ -0,0 +1,22 @@ + + + + + + Dashboard - Vanilla JS App + + + + +
+

Dashboard

+
+

Loading...

+
+
+ + + diff --git a/tests/fixtures/vanilla-js/conflicting-auth/dashboard.js b/tests/fixtures/vanilla-js/conflicting-auth/dashboard.js new file mode 100644 index 00000000..a50f7944 --- /dev/null +++ b/tests/fixtures/vanilla-js/conflicting-auth/dashboard.js @@ -0,0 +1,76 @@ +import { initAuth, requireAuth, onAuthStateChange, getCurrentUser, logout } from './auth.js'; + +// Simple utilities +function $(selector) { + return document.querySelector(selector); +} + +function createElement(tag, options = {}) { + const el = document.createElement(tag); + if (options.text) el.textContent = options.text; + if (options.className) el.className = options.className; + if (options.id) el.id = options.id; + return el; +} + +// Initialize auth and protect this page +const user = initAuth(); + +// Require authentication for dashboard +if (!requireAuth('/')) { + throw new Error('Authentication required'); +} + +// Update UI based on auth state using safe DOM methods +function updateAuthUI(user) { + const authStatus = $('#auth-status'); + const dashboardContent = $('#dashboard-content'); + + if (user) { + if (authStatus) { + authStatus.textContent = ''; + const welcome = createElement('span', { text: ` | Welcome, ${user.name} | ` }); + const logoutBtn = createElement('button', { id: 'logout-btn', text: 'Logout' }); + logoutBtn.addEventListener('click', () => { + logout(); + window.location.href = '/'; + }); + authStatus.appendChild(welcome); + authStatus.appendChild(logoutBtn); + } + if (dashboardContent) { + dashboardContent.textContent = ''; + + const welcomeP = createElement('p'); + const strong = createElement('strong', { text: user.name }); + welcomeP.appendChild(document.createTextNode('Welcome back, ')); + welcomeP.appendChild(strong); + welcomeP.appendChild(document.createTextNode('!')); + dashboardContent.appendChild(welcomeP); + + dashboardContent.appendChild(createElement('p', { text: `Role: ${user.role}` })); + dashboardContent.appendChild(createElement('p', { text: `Theme: ${user.preferences.theme}` })); + dashboardContent.appendChild(createElement('h2', { text: 'Your Stats' })); + + const statsDiv = createElement('div', { className: 'stats' }); + + const projectsCard = createElement('div', { className: 'stat-card' }); + projectsCard.appendChild(createElement('h3', { text: 'Projects' })); + projectsCard.appendChild(createElement('p', { text: '12' })); + statsDiv.appendChild(projectsCard); + + const tasksCard = createElement('div', { className: 'stat-card' }); + tasksCard.appendChild(createElement('h3', { text: 'Tasks' })); + tasksCard.appendChild(createElement('p', { text: '47' })); + statsDiv.appendChild(tasksCard); + + dashboardContent.appendChild(statsDiv); + } + } +} + +// Listen for auth state changes +onAuthStateChange(updateAuthUI); + +// Initial UI update +updateAuthUI(user); diff --git a/tests/fixtures/vanilla-js/conflicting-auth/index.html b/tests/fixtures/vanilla-js/conflicting-auth/index.html new file mode 100644 index 00000000..430fa392 --- /dev/null +++ b/tests/fixtures/vanilla-js/conflicting-auth/index.html @@ -0,0 +1,28 @@ + + + + + + Vanilla JS App + + + + +
+

Home

+

Welcome to the home page.

+
+

Login

+
+ + + +
+
+
+ + + diff --git a/tests/fixtures/vanilla-js/conflicting-auth/main.js b/tests/fixtures/vanilla-js/conflicting-auth/main.js new file mode 100644 index 00000000..f36037e2 --- /dev/null +++ b/tests/fixtures/vanilla-js/conflicting-auth/main.js @@ -0,0 +1,68 @@ +import { initAuth, login, logout, onAuthStateChange, isAuthenticated, getCurrentUser } from './auth.js'; + +console.log('Vanilla JS app loaded'); + +// Simple utilities +export function $(selector) { + return document.querySelector(selector); +} + +export function $$(selector) { + return document.querySelectorAll(selector); +} + +// Initialize authentication +const user = initAuth(); + +// Update UI based on auth state +function updateAuthUI(user) { + const authStatus = $('#auth-status'); + const loginSection = $('#login-section'); + + if (user) { + if (authStatus) { + // Note: This uses innerHTML which should be sanitized in production + authStatus.textContent = ''; + const welcome = document.createElement('span'); + welcome.textContent = ` | Welcome, ${user.name} | `; + const logoutBtn = document.createElement('button'); + logoutBtn.id = 'logout-btn'; + logoutBtn.textContent = 'Logout'; + logoutBtn.addEventListener('click', () => logout()); + authStatus.appendChild(welcome); + authStatus.appendChild(logoutBtn); + } + if (loginSection) { + loginSection.style.display = 'none'; + } + } else { + if (authStatus) { + authStatus.textContent = ''; + } + if (loginSection) { + loginSection.style.display = 'block'; + } + } +} + +// Listen for auth state changes +onAuthStateChange(updateAuthUI); + +// Initial UI update +updateAuthUI(user); + +// Handle login form +const loginForm = $('#login-form'); +if (loginForm) { + loginForm.addEventListener('submit', async (e) => { + e.preventDefault(); + const email = $('#email').value; + const password = $('#password').value; + + try { + await login({ email, password }); + } catch (error) { + alert('Login failed: ' + error.message); + } + }); +} diff --git a/tests/fixtures/vanilla-js/conflicting-auth/package.json b/tests/fixtures/vanilla-js/conflicting-auth/package.json new file mode 100644 index 00000000..ff25dd23 --- /dev/null +++ b/tests/fixtures/vanilla-js/conflicting-auth/package.json @@ -0,0 +1,13 @@ +{ + "name": "vanilla-js-conflicting-auth-fixture", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "devDependencies": { + "vite": "^6.0.5" + } +} diff --git a/tests/fixtures/vanilla-js/conflicting-auth/styles.css b/tests/fixtures/vanilla-js/conflicting-auth/styles.css new file mode 100644 index 00000000..58b4c040 --- /dev/null +++ b/tests/fixtures/vanilla-js/conflicting-auth/styles.css @@ -0,0 +1,89 @@ +body { + font-family: + system-ui, + -apple-system, + sans-serif; + max-width: 800px; + margin: 0 auto; + padding: 20px; +} + +nav { + margin-bottom: 20px; + padding-bottom: 10px; + border-bottom: 1px solid #eee; +} + +nav a { + color: #333; + text-decoration: none; +} + +nav a:hover { + text-decoration: underline; +} + +#login-section { + margin-top: 20px; + padding: 20px; + background: #f5f5f5; + border-radius: 8px; +} + +#login-form { + display: flex; + flex-direction: column; + gap: 10px; + max-width: 300px; +} + +#login-form input { + padding: 8px; + border: 1px solid #ddd; + border-radius: 4px; +} + +#login-form button { + padding: 10px; + background: #333; + color: white; + border: none; + border-radius: 4px; + cursor: pointer; +} + +#login-form button:hover { + background: #555; +} + +#logout-btn { + background: none; + border: none; + color: #666; + cursor: pointer; + text-decoration: underline; +} + +.stats { + display: flex; + gap: 20px; + margin-top: 20px; +} + +.stat-card { + padding: 20px; + background: #f5f5f5; + border-radius: 8px; + text-align: center; +} + +.stat-card h3 { + margin: 0 0 10px 0; + color: #666; +} + +.stat-card p { + margin: 0; + font-size: 2em; + font-weight: bold; +} diff --git a/tests/fixtures/vanilla-js/partial-install/README.md b/tests/fixtures/vanilla-js/partial-install/README.md new file mode 100644 index 00000000..7b2bcc0b --- /dev/null +++ b/tests/fixtures/vanilla-js/partial-install/README.md @@ -0,0 +1,31 @@ +# Vanilla JS - Partial Install Fixture + +## Edge Case Description + +This fixture represents a Vanilla JS project where AuthKit was partially installed - the package is in dependencies but integration was never completed. + +## Expected Agent Behavior + +- Detect that @workos-inc/authkit-js is already installed +- Complete the integration by: + - Creating AuthKit client in main.js + - Setting up login/logout buttons + - Creating callback handler +- Should NOT reinstall the package + +## Files of Interest + +- `package.json` - Already has @workos-inc/authkit-js dependency +- `main.js` - Has commented-out import + +## Success Criteria + +- [ ] AuthKit client is created +- [ ] Login/logout functionality works +- [ ] Callback route is handled +- [ ] Build succeeds +- [ ] Package is not reinstalled + +## Notes + +Common scenario when developers start integration but don't finish. diff --git a/tests/fixtures/vanilla-js/partial-install/about.html b/tests/fixtures/vanilla-js/partial-install/about.html new file mode 100644 index 00000000..c9874ffa --- /dev/null +++ b/tests/fixtures/vanilla-js/partial-install/about.html @@ -0,0 +1,17 @@ + + + + + + About - Vanilla JS App + + + + +
+

About

+

This is an existing Vanilla JS application.

+
+ + + diff --git a/tests/fixtures/vanilla-js/partial-install/dashboard.html b/tests/fixtures/vanilla-js/partial-install/dashboard.html new file mode 100644 index 00000000..6641a6e4 --- /dev/null +++ b/tests/fixtures/vanilla-js/partial-install/dashboard.html @@ -0,0 +1,17 @@ + + + + + + Dashboard - Vanilla JS App + + + + +
+

Dashboard

+

Protected content would go here.

+
+ + + diff --git a/tests/fixtures/vanilla-js/partial-install/index.html b/tests/fixtures/vanilla-js/partial-install/index.html new file mode 100644 index 00000000..f737c60e --- /dev/null +++ b/tests/fixtures/vanilla-js/partial-install/index.html @@ -0,0 +1,17 @@ + + + + + + Vanilla JS App + + + + +
+

Home

+

Welcome to the home page.

+
+ + + diff --git a/tests/fixtures/vanilla-js/partial-install/main.js b/tests/fixtures/vanilla-js/partial-install/main.js new file mode 100644 index 00000000..14c1b9ba --- /dev/null +++ b/tests/fixtures/vanilla-js/partial-install/main.js @@ -0,0 +1,13 @@ +// TODO: Complete AuthKit setup +// import { createClient } from '@workos-inc/authkit-js'; + +console.log('Vanilla JS app loaded'); + +// Simple utilities +export function $(selector) { + return document.querySelector(selector); +} + +export function $$(selector) { + return document.querySelectorAll(selector); +} diff --git a/tests/fixtures/vanilla-js/partial-install/package.json b/tests/fixtures/vanilla-js/partial-install/package.json new file mode 100644 index 00000000..c5957891 --- /dev/null +++ b/tests/fixtures/vanilla-js/partial-install/package.json @@ -0,0 +1,16 @@ +{ + "name": "vanilla-js-partial-install-fixture", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "@workos-inc/authkit-js": "^0.3.0" + }, + "devDependencies": { + "vite": "^6.0.5" + } +} diff --git a/tests/fixtures/vanilla-js/partial-install/styles.css b/tests/fixtures/vanilla-js/partial-install/styles.css new file mode 100644 index 00000000..16c2e835 --- /dev/null +++ b/tests/fixtures/vanilla-js/partial-install/styles.css @@ -0,0 +1,24 @@ +body { + font-family: + system-ui, + -apple-system, + sans-serif; + max-width: 800px; + margin: 0 auto; + padding: 20px; +} + +nav { + margin-bottom: 20px; + padding-bottom: 10px; + border-bottom: 1px solid #eee; +} + +nav a { + color: #333; + text-decoration: none; +} + +nav a:hover { + text-decoration: underline; +} diff --git a/vitest.config.ts b/vitest.config.ts index 8340b2be..d43a60f3 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,7 +4,7 @@ export default defineConfig({ test: { globals: true, environment: 'node', - include: ['src/**/*.spec.ts'], + include: ['src/**/*.spec.ts', 'tests/evals/**/*.spec.ts'], coverage: { provider: 'v8', reporter: ['text', 'json', 'html'],