diff --git a/.gitignore b/.gitignore index af0ec82..c01c66d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,9 @@ release/ *.provisionprofile # Build cache -.electron-builder-cache/ \ No newline at end of file +.electron-builder-cache/ + +# Test artifacts +playwright-report/ +test-results/ +.playwright/ \ No newline at end of file diff --git a/E2E_TESTING.md b/E2E_TESTING.md new file mode 100644 index 0000000..8a9d709 --- /dev/null +++ b/E2E_TESTING.md @@ -0,0 +1,220 @@ +# E2E Testing Guide + +This document provides guidance for running and developing end-to-end (E2E) tests for Pinecone Explorer using Playwright. + +## Overview + +The E2E testing infrastructure uses **Playwright** for Electron app automation. + +> **Note**: E2E tests run locally only. Electron cannot run in CI sandbox environments like GitHub Actions. + +## Architecture + +### Test Strategy +Tests create database connection profiles programmatically via the `window.electronAPI` exposed by the preload script. This approach: +- Tests the actual IPC communication path users take +- Avoids modifying production code for test purposes +- Validates the full integration flow + +## Prerequisites + +- Node.js 22+ +- pnpm 9+ + +## Quick Start + +### Run Tests + +```bash +# Run all tests +pnpm run test:e2e + +# Run with UI mode (interactive) +pnpm run test:e2e:ui + +# Run with debugger +pnpm run test:e2e:debug +``` + +### View Test Results + +After tests complete: +```bash +pnpm exec playwright show-report +``` + +## Available NPM Scripts + +| Script | Description | +|--------|-------------| +| `test:build` | Build Electron app for testing | +| `test:e2e` | Build and run E2E tests | +| `test:e2e:ui` | Run tests in interactive UI mode | +| `test:e2e:debug` | Run tests with debugger | + +## Project Structure + +``` +pinecone-explorer/ +├── e2e/ +│ ├── electron.setup.ts # Electron launcher and profile utilities +│ └── example.spec.ts # Example test suite +└── playwright.config.ts # Playwright configuration +``` + +## Writing Tests + +### Basic Test Structure + +```typescript +import { test, expect } from '@playwright/test' +import { + launchElectronApp, + closeElectronApp, + cleanupTestProfiles, + createPineconeTestProfile, + connectToProfile, + type ElectronTestContext, +} from './electron.setup' + +let electronContext: ElectronTestContext + +test.beforeAll(async () => { + electronContext = await launchElectronApp() +}) + +test.afterAll(async () => { + await cleanupTestProfiles(electronContext.page) + await closeElectronApp(electronContext.app) +}) + +test('should create Pinecone profile', async () => { + const { page } = electronContext + + // Create profile programmatically + const profileId = await createPineconeTestProfile(page) + + // Connect to profile + await connectToProfile(page, profileId) + + // Use electronAPI to interact with the database + const indexes = await page.evaluate(async (id) => { + return await (window as any).electronAPI.pinecone.listIndexes(id) + }, profileId) + + expect(Array.isArray(indexes)).toBe(true) +}) +``` + +### Available Setup Utilities + +#### `launchElectronApp(): Promise` +Launches the Electron app with test environment variables. + +#### `createPineconeTestProfile(page, name?, apiKey?): Promise` +Creates a Pinecone profile and returns the profile ID. + +#### `connectToProfile(page, profileId): Promise` +Connects to a profile via IPC. + +#### `cleanupTestProfiles(page): Promise` +Deletes all test profiles (IDs starting with 'test-'). + +#### `closeElectronApp(app): Promise` +Gracefully closes the Electron app. + +### Test Isolation + +Each test should: +1. Create its own unique profile using timestamp-based IDs +2. Clean up indexes/data created during the test +3. Use the `afterAll` hook to delete test profiles + +## Environment Variables + +The following environment variables are automatically set during tests: + +```bash +NODE_ENV=test +DISABLE_ANALYTICS=true +``` + +## Troubleshooting + +### Electron won't launch + +**Problem**: Tests fail with "Could not find Electron app" + +**Solution**: +```bash +# Ensure the app is built +pnpm run test:build + +# Verify main.js exists +ls -la dist-electron/main.js +``` + +### Profiles not created + +**Problem**: Tests fail with "electronAPI is not defined" + +**Solution**: +- Verify the preload script is loaded +- Check that `window.electronAPI` is exposed +- Enable debug mode: + ```bash + DEBUG=pw:api pnpm run test:e2e:debug + ``` + +### Tests timeout + +**Problem**: Tests hang or timeout after 60 seconds + +**Solutions**: +- Check if the app is launching properly +- Increase timeout in `playwright.config.ts` if needed +- Run in headed mode to see what's happening: + ```bash + pnpm run test:e2e:debug + ``` + +## Pinecone Testing + +Since Pinecone requires a cloud connection, testing against real Pinecone requires: + +1. Set the `PINECONE_API_KEY` environment variable: + ```bash + PINECONE_API_KEY=your-key-here pnpm run test:e2e + ``` + +2. Use the free tier for testing (avoid costs) + +3. Consider skipping Pinecone API tests in local development: + ```typescript + test.skip(!process.env.PINECONE_API_KEY, 'should work with Pinecone', async () => { + // Pinecone test + }) + ``` + +## Best Practices + +1. **Always build before testing**: The `test:e2e` script does this automatically +2. **Use unique IDs**: All test profiles use timestamp-based IDs to avoid conflicts +3. **Clean up resources**: Delete test indexes and profiles in `afterAll` hooks +4. **Serial execution**: Tests run serially (workers: 1) to avoid conflicts +5. **Capture artifacts**: Screenshots, videos, and traces are captured on failure + +## Performance + +- **Build time**: ~10-15 seconds +- **Test execution**: ~5-10 seconds per test +- **Total workflow**: ~30 seconds - 1 minute + +## Future Improvements + +- [ ] Add visual regression testing +- [ ] Add performance benchmarks +- [ ] Test more complex workflows (multi-step operations) +- [ ] Add accessibility testing +- [ ] Test offline/error scenarios +- [ ] Investigate CI options for Electron testing (self-hosted runners, etc.) diff --git a/e2e/electron.setup.ts b/e2e/electron.setup.ts new file mode 100644 index 0000000..0c431ff --- /dev/null +++ b/e2e/electron.setup.ts @@ -0,0 +1,174 @@ +import { _electron as electron, ElectronApplication, Page } from '@playwright/test' +import * as path from 'path' +import * as os from 'os' +import * as fs from 'fs' +import { fileURLToPath } from 'url' + +export interface ElectronTestContext { + app: ElectronApplication + page: Page +} + +/** + * Get the path to the app's userData directory + * In test mode, uses E2E_USER_DATA_DIR if available to isolate test data + */ +function getAppDataPath(): string { + const explicit = process.env.E2E_USER_DATA_DIR + if (explicit) return explicit + const appName = 'Pinecone Explorer' + if (process.platform === 'darwin') { + return path.join(os.homedir(), 'Library', 'Application Support', appName) + } else if (process.platform === 'win32') { + return path.join(process.env.APPDATA || '', appName) + } else { + return path.join(os.homedir(), '.config', appName.toLowerCase().replace(/ /g, '-')) + } +} + +/** + * Clear encrypted store files to avoid encryption key mismatch issues in tests. + * The encryption key is derived from app path, which differs between normal and test runs. + * @param appDataPath - The path to the app's userData directory (must be test-specific) + */ +function clearEncryptedStores(appDataPath: string): void { + const filesToClear = [ + 'encryption-key.enc', + 'pinecone-connections.json', + 'pinecone-settings.json', + 'chroma-settings-v2.json', + ] + + for (const file of filesToClear) { + const filePath = path.join(appDataPath, file) + try { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath) + console.log(`[E2E Setup] Cleared ${file}`) + } + } catch (error) { + console.warn(`[E2E Setup] Failed to clear ${file}:`, error) + } + } +} + +/** + * Launch the Electron application with test environment variables + */ +export async function launchElectronApp(): Promise { + const __dirname = path.dirname(fileURLToPath(import.meta.url)) + const electronPath = path.join(__dirname, '../dist-electron/main.js') + + // Use a test-specific userData directory to isolate test data + const explicitUserDataDir = process.env.E2E_USER_DATA_DIR?.trim() + const testUserDataDir = + explicitUserDataDir && explicitUserDataDir.length > 0 + ? explicitUserDataDir + : path.join(os.tmpdir(), 'pinecone-explorer-e2e') + + // Clear encrypted stores to avoid encryption key mismatch + clearEncryptedStores(testUserDataDir) + + const env = { + ...process.env, + NODE_ENV: 'test', + DISABLE_ANALYTICS: 'true', + E2E_USER_DATA_DIR: testUserDataDir, + } + + // In CI environments, Electron needs to run without sandboxing + const args = [electronPath] + if (process.env.CI) { + args.push('--no-sandbox') + } + + const app = await electron.launch({ + args, + env, + }) + + const page = await app.firstWindow() + + // Wait for the app to be ready + await page.waitForLoadState('domcontentloaded') + + // Wait for setup window or main content to be visible + try { + await page.waitForSelector('[data-testid="setup-window"]', { timeout: 10000 }) + } catch { + // If setup window doesn't exist, the app might already be set up + // This is fine for subsequent tests + } + + return { app, page } +} + +/** + * Create a Pinecone test profile programmatically via electronAPI + * Note: Use dummy key for local testing or provide real API key for cloud testing + */ +export async function createPineconeTestProfile( + page: Page, + name?: string, + apiKey?: string +): Promise { + const profileId = `test-pinecone-${Date.now()}` + const profileName = name || `Test Pinecone ${Date.now()}` + const pineconeKey = apiKey || process.env.PINECONE_API_KEY || 'dummy-key-for-local-testing' + + await page.evaluate( + async ({ id, name, apiKey }) => { + const profile = { + id, + name, + provider: 'pinecone' as const, + apiKey, + } + await (window as any).electronAPI.profiles.save(profile) + return id + }, + { id: profileId, name: profileName, apiKey: pineconeKey } + ) + + return profileId +} + +/** + * Connect to a Pinecone profile via IPC + */ +export async function connectToProfile(page: Page, profileId: string): Promise { + await page.evaluate(async (id) => { + const profiles = await (window as any).electronAPI.profiles.getAll() + const profile = profiles.find((p: any) => p.id === id) + if (!profile) { + throw new Error(`Profile ${id} not found`) + } + await (window as any).electronAPI.pinecone.connect(id, profile) + }, profileId) +} + +/** + * Clean up test profiles - delete all profiles starting with 'test-' + */ +export async function cleanupTestProfiles(page: Page): Promise { + try { + await page.evaluate(async () => { + const profiles = await (window as any).electronAPI.profiles.getAll() + + for (const profile of profiles) { + if (profile.id.startsWith('test-')) { + await (window as any).electronAPI.profiles.delete(profile.id) + } + } + }) + } catch (error) { + console.error('Error cleaning up test profiles:', error) + } +} + +/** + * Close the Electron application gracefully + */ +export async function closeElectronApp(app: ElectronApplication): Promise { + await app.close() +} diff --git a/e2e/example.spec.ts b/e2e/example.spec.ts new file mode 100644 index 0000000..3fb8f15 --- /dev/null +++ b/e2e/example.spec.ts @@ -0,0 +1,31 @@ +import { test, expect } from '@playwright/test' +import { + launchElectronApp, + closeElectronApp, + cleanupTestProfiles, + type ElectronTestContext, +} from './electron.setup' + +let electronContext: ElectronTestContext + +test.beforeAll(async () => { + electronContext = await launchElectronApp() +}) + +test.afterAll(async () => { + if (electronContext?.page) { + await cleanupTestProfiles(electronContext.page) + } + if (electronContext?.app) { + await closeElectronApp(electronContext.app) + } +}) + +test.describe('Pinecone Explorer E2E', () => { + test('should launch app successfully', async () => { + const { page } = electronContext + // Verify that the app window is visible + expect(page).toBeTruthy() + expect(await page.title()).toBeTruthy() + }) +}) diff --git a/electron/main.ts b/electron/main.ts index 123d78d..9245c47 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1,6 +1,12 @@ import 'dotenv/config' import { app, BrowserWindow, ipcMain, Menu, MenuItemConstructorOptions, shell } from 'electron' +// Redirect userData to test directory if running in test mode +// This MUST happen before any store initialization +if (process.env.NODE_ENV === 'test' && process.env.E2E_USER_DATA_DIR) { + app.setPath('userData', process.env.E2E_USER_DATA_DIR) +} + // Set app name before anything else (affects menu bar, about dialog, etc.) app.name = 'Pinecone Explorer' import path from 'node:path' diff --git a/package.json b/package.json index 93d358b..d199d89 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,11 @@ "build": "vite build && electron-builder --dir", "build:release": "vite build && electron-builder", "preview": "vite preview", - "postinstall": "electron-builder install-app-deps" + "postinstall": "electron-builder install-app-deps", + "test:build": "vite build", + "test:e2e": "pnpm test:build && playwright test", + "test:e2e:ui": "pnpm test:build && playwright test --ui", + "test:e2e:debug": "pnpm test:build && playwright test --debug" }, "keywords": [ "electron", @@ -20,6 +24,7 @@ "author": "stepandel", "license": "MIT", "devDependencies": { + "@playwright/test": "^1.58.1", "@tailwindcss/postcss": "^4.1.18", "@types/node": "^25.0.3", "@types/react": "^19.2.7", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..bf14d5e --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,50 @@ +import { defineConfig, devices } from '@playwright/test' + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './e2e', + + /* Maximum time one test can run for */ + timeout: 60_000, + + /* Run tests in files in serial */ + fullyParallel: false, + + /* Fail the build on CI if you accidentally left test.only in the source code */ + forbidOnly: !!process.env.CI, + + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + + /* Single worker for serial execution to avoid conflicts */ + workers: 1, + + /* Reporter to use */ + reporter: [ + ['list'], + ['html'], + ['json', { outputFile: 'test-results/results.json' }] + ], + + /* Shared settings for all the projects below */ + use: { + /* Collect trace when retrying the failed test */ + trace: 'on-first-retry', + + /* Capture screenshot only on failure */ + screenshot: 'only-on-failure', + + /* Capture video only when retaining on failure */ + video: 'retain-on-failure', + }, + + /* Configure projects for major browsers - not needed for Electron but kept for future web testing */ + projects: [ + { + name: 'electron', + testMatch: /.*\.spec\.ts/, + }, + ], +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a1f41d5..31d005b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -85,6 +85,9 @@ dependencies: version: 6.0.6 devDependencies: + '@playwright/test': + specifier: ^1.58.1 + version: 1.58.1 '@tailwindcss/postcss': specifier: ^4.1.18 version: 4.1.18 @@ -884,6 +887,14 @@ packages: dev: true optional: true + /@playwright/test@1.58.1: + resolution: {integrity: sha512-6LdVIUERWxQMmUSSQi0I53GgCBYgM2RpGngCPY7hSeju+VrKjq3lvs7HpJoPbDiY5QM5EYRtRX5fvrinnMAz3w==} + engines: {node: '>=18'} + hasBin: true + dependencies: + playwright: 1.58.1 + dev: true + /@radix-ui/number@1.1.1: resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==} dev: false @@ -3034,6 +3045,14 @@ packages: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} dev: true + /fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + requiresBuild: true + dev: true + optional: true + /fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -3928,6 +3947,22 @@ packages: engines: {node: '>=12'} dev: true + /playwright-core@1.58.1: + resolution: {integrity: sha512-bcWzOaTxcW+VOOGBCQgnaKToLJ65d6AqfLVKEWvexyS3AS6rbXl+xdpYRMGSRBClPvyj44njOWoxjNdL/H9UNg==} + engines: {node: '>=18'} + hasBin: true + dev: true + + /playwright@1.58.1: + resolution: {integrity: sha512-+2uTZHxSCcxjvGc5C891LrS1/NlxglGxzrC4seZiVjcYVQfUa87wBL6rTDqzGjuoWNjnBzRqKmF6zRYGMvQUaQ==} + engines: {node: '>=18'} + hasBin: true + dependencies: + playwright-core: 1.58.1 + optionalDependencies: + fsevents: 2.3.2 + dev: true + /plist@3.1.0: resolution: {integrity: sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ==} engines: {node: '>=10.4.0'} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..c82c20d --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,13 @@ +packages: + - '.' + +ignoredBuiltDependencies: + - bufferutil + - electron-winstaller + - esbuild + - protobufjs + - sharp + - utf-8-validate + +onlyBuiltDependencies: + - electron diff --git a/tsconfig.json b/tsconfig.json index 878e84e..7b2ffd5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,6 @@ "@/*": ["./src/*"] } }, - "include": ["src", "electron"], - "exclude": ["node_modules", "dist", "dist-electron"] + "include": ["src", "electron", "e2e"], + "exclude": ["node_modules", "dist", "dist-electron", "playwright-report", "test-results"] }