diff --git a/e2e/README.md b/e2e/README.md index 0cc10daaa..2a0c41810 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -35,3 +35,39 @@ mac `OpenSwarm.app` variants. `.github/workflows/e2e.yml` runs this on a `windows-latest` + `macos-latest` matrix: it builds the unsigned app, then runs the suite. Tag-driven signed releases are covered separately by `release-windows.yml` / `release-macos.yml`. + +## Release characterization (`tests/v1.7-characterization/`) + +A regression net for the user-facing claims in the release notes +(`backend/apps/help/changelog.py`, versions 1.7.4-1.7.7). Each test names the +changelog line it pins, so a later refactor of the files that carry that +behaviour (the chat surface, the canvas and window state, the shell's dashboard +host, the layout slice) has to keep the claim true, not just compile. Same rules +as the rest of this suite: packaged app, no provider key, macOS + Windows. + +What it drives, and how, without an agent turn: + +- **canvas-fullscreen** - the composer yields to open windows; a wheel inside a + window never moves the canvas; the default wash is flat colour; nothing drags + while a window is fullscreen and Escape exits; exactly one fullscreen owner, + ever; dock tiles are images or glyphs, never letters. +- **chat-toolui** - sessions come from the app's own launch route (parked, + pre-warming); the messages an agent would stream (AskUI approval cards, ShowUI + links) are dispatched through the store's own reducer in the wire shape the + backend emits, and the renders and clicks are real. The `respond` call is + routed so both server verdicts are exercised (accepted -> receipt; gone -> + the honest "didn't reach the agent" notice). Provider-retry pill survives + status frames and never shows a scary card. +- **resilience** - a failed or unscoped sessions read never wipes the board; + then the packaged backend is really killed: a quick recovery is silent, a + sustained outage shows "Reconnecting to OpenSwarm..." and heals when it can + (each outage test boots its own app so the respawn budget starts clean). +- **workflows-and-settings** - off means off (Run Now on a switched-off workflow + is refused and the refusal shows in History; a deleted one cannot run at all); + a fact saved in Settings -> Memory is the store's own list; the dictation cue + volume defaults to an audible 70%. + +The renderer's store is read and dispatched through `window.__OPENSWARM_STORE__` +(exposed by the packaged renderer under `OPENSWARM_E2E=1`), and the backend is +called from inside the renderer with plain `fetch`, which the app already +bearer-patches, so no spec handles a token or a port by hand. diff --git a/e2e/tests/v1.7-characterization/canvas-fullscreen.spec.ts b/e2e/tests/v1.7-characterization/canvas-fullscreen.spec.ts new file mode 100644 index 000000000..ce0b138f2 --- /dev/null +++ b/e2e/tests/v1.7-characterization/canvas-fullscreen.spec.ts @@ -0,0 +1,137 @@ +import { test, expect, ElectronApplication, Page } from '@playwright/test'; +import { bootIntoDashboard, closeSettingsCard, dispatch, openSettingsCard, paintedCamera, select } from './support'; + +// Characterization of the canvas / window-state claims in the 1.7.4-1.7.7 release notes +// (backend/apps/help/changelog.py), against the PACKAGED app with no provider key. Each test names +// the changelog line it pins. These are regression nets for the files that carry the behaviour +// (dashboardLayoutSlice, the canvas cards, the shell's dashboard host), so a later refactor of any of +// them has to keep the user-visible claim true, not just compile. +test.describe.configure({ mode: 'serial' }); +test.describe('characterization: canvas, fullscreen, layout (1.7.4-1.7.7)', () => { + let app: ElectronApplication; + let win: Page; + let dashboardId: string; + + test.beforeAll(async () => { + ({ app, win, dashboardId } = await bootIntoDashboard()); + }); + test.afterAll(async () => { await app?.close().catch(() => {}); }); + + test('1.7.4 H2: the spawn composer steps aside when a window is open', async () => { + // Fresh dashboard, no sessions: the empty-state hero (with the composer) is the page. + const hero = win.getByText('What do you want done?').first(); + await expect(hero).toBeVisible({ timeout: 15_000 }); + await openSettingsCard(win); + await expect(hero).toBeHidden({ timeout: 10_000 }); + await closeSettingsCard(win); + await expect(hero).toBeVisible({ timeout: 10_000 }); + }); + + test('1.7.5 H3: a wheel inside a window stays in that window; over the canvas it drives the canvas', async () => { + await openSettingsCard(win); + const card = win.locator('[data-select-type="settings-card"]'); + await expect(card).toBeVisible(); + // Let the open-pan finish so the "before" camera is the resting one. + await win.waitForTimeout(800); + const before = await paintedCamera(win); + const box = (await card.boundingBox())!; + // Wheel over the body of the window (below its 42px header), a few notches each way. + await win.mouse.move(box.x + box.width / 2, box.y + box.height / 2); + await win.mouse.wheel(0, 240); + await win.mouse.wheel(0, -120); + await win.waitForTimeout(400); + expect(await paintedCamera(win), 'a wheel inside a window must not move the canvas').toBe(before); + // Same gesture over empty canvas: the camera moves (mouse notch = zoom, trackpad = pan; either way it changes). + const viewport = (await win.locator('[data-canvas-viewport]').boundingBox())!; + const emptyX = Math.min(viewport.x + 60, box.x - 30 > viewport.x ? box.x - 30 : viewport.x + 60); + await win.mouse.move(emptyX, viewport.y + viewport.height - 80); + await win.mouse.wheel(0, 240); + await expect.poll(() => paintedCamera(win), { timeout: 5000 }).not.toBe(before); + await closeSettingsCard(win); + }); + + test('1.7.5 F1: the default canvas paints a flat wash, not an image texture that can drop', async () => { + const wash = await win.evaluate(() => { + const viewport = document.querySelector('[data-canvas-viewport]') as HTMLElement; + const cs = getComputedStyle(viewport); + return { backgroundImage: cs.backgroundImage, hasImgChild: !!viewport.querySelector(':scope > img') }; + }); + expect(wash.backgroundImage.includes('url(')).toBe(false); + expect(wash.hasImgChild).toBe(false); + }); + + test('1.7.7 F11: nothing drags while a window is fullscreen, and canvas panning is locked', async () => { + await openSettingsCard(win); + await dispatch(win, { type: 'dashboardLayout/setTiledCard', payload: { cardId: 'settings', zone: 'fullscreen' } }); + const card = win.locator('[data-select-type="settings-card"]'); + await expect.poll(() => select(win, (s) => s.dashboardLayout.tiledCards.settings), { timeout: 5000 }).toBe('fullscreen'); + await win.waitForTimeout(900); + const before = (await card.boundingBox())!; + const cameraBefore = await paintedCamera(win); + // A header drag (the strip above the traffic lights): in fullscreen this must be a no-op. + await win.mouse.move(before.x + before.width / 2, before.y + 20); + await win.mouse.down(); + for (let i = 1; i <= 8; i++) await win.mouse.move(before.x + before.width / 2 + i * 30, before.y + 20 + i * 15); + await win.mouse.up(); + await win.waitForTimeout(500); + const after = (await card.boundingBox())!; + expect(Math.abs(after.x - before.x)).toBeLessThan(2); + expect(Math.abs(after.y - before.y)).toBeLessThan(2); + expect(await select(win, (s) => s.dashboardLayout.tiledCards.settings)).toBe('fullscreen'); + // A drag on the canvas itself is also inert while a window is fullscreen (drag joined the wheel gate). + await win.mouse.move(before.x + 4, before.y + before.height - 4); + await win.mouse.down(); + await win.mouse.move(before.x + 200, before.y + before.height - 60, { steps: 6 }); + await win.mouse.up(); + await win.waitForTimeout(400); + expect(await paintedCamera(win)).toBe(cameraBefore); + // Escape is one of the sanctioned exits. + await win.keyboard.press('Escape'); + await expect.poll(() => select(win, (s) => s.dashboardLayout.tiledCards.settings ?? null), { timeout: 5000 }).toBeNull(); + await closeSettingsCard(win); + }); + + test('1.7.7 F11: there is exactly one fullscreen owner, ever, and a removed owner leaves no stuck fullscreen', async () => { + // Two browser cards through the app's own reducer. + await dispatch(win, { type: 'dashboardLayout/addBrowserCard', payload: { url: 'about:blank' } }); + await dispatch(win, { type: 'dashboardLayout/addBrowserCard', payload: { url: 'about:blank' } }); + const ids = await select(win, (s) => Object.keys(s.dashboardLayout.browserCards)); + expect(ids.length).toBeGreaterThanOrEqual(2); + const [a, b] = ids.slice(-2); + const owners = () => select(win, (s) => Object.entries(s.dashboardLayout.tiledCards as Record).filter(([, z]) => z === 'fullscreen').map(([id]) => id)); + await dispatch(win, { type: 'dashboardLayout/setTiledCard', payload: { cardId: a, zone: 'fullscreen' } }); + expect(await owners()).toEqual([a]); + // Crowning a second owner dethrones the first: two entries would fight over who hides the chrome and who gets the Escape. + await dispatch(win, { type: 'dashboardLayout/setTiledCard', payload: { cardId: b, zone: 'fullscreen' } }); + expect(await owners()).toEqual([b]); + // Removing the owner must not strand a chromeless shell. + await dispatch(win, { type: 'dashboardLayout/removeBrowserCard', payload: b }); + await expect.poll(owners, { timeout: 5000 }).toEqual([]); + // A tile request for a card that no longer exists is refused outright (the rail defers its dispatch, so the card can be gone by then). + await dispatch(win, { type: 'dashboardLayout/setTiledCard', payload: { cardId: b, zone: 'fullscreen' } }); + expect(await owners()).toEqual([]); + // Leave the profile as we found it. + await dispatch(win, { type: 'dashboardLayout/removeBrowserCard', payload: a }); + await expect(win.locator('[data-onboarding="settings-close-button"]')).toHaveCount(0); + }); + + test('1.7.7 F14: dock tiles for apps and browsers are images or glyphs, never letters', async () => { + await dispatch(win, { type: 'dashboardLayout/addBrowserCard', payload: { url: 'about:blank' } }); + const id = (await select(win, (s) => Object.keys(s.dashboardLayout.browserCards))).at(-1)!; + const tiles = win.locator('[data-desktop-dock] [aria-label]'); + await expect.poll(() => tiles.count(), { timeout: 10_000 }).toBeGreaterThan(0); + const audit = await win.evaluate(() => + Array.from(document.querySelectorAll('[data-desktop-dock] [aria-label]')).map((tile) => ({ + label: tile.getAttribute('aria-label') || '', + pictorial: tile.querySelectorAll('img, svg').length > 0, + // Any bare glyph text would be a letter/number standing in for the mark. + looseText: (tile as HTMLElement).innerText.replace(/\s+/g, ''), + })), + ); + for (const tile of audit) { + expect(tile.pictorial, `${tile.label}: tile must carry an image or glyph`).toBe(true); + expect(tile.looseText, `${tile.label}: no letters or numbers as the mark`).toBe(''); + } + await dispatch(win, { type: 'dashboardLayout/removeBrowserCard', payload: id }); + }); +}); diff --git a/e2e/tests/v1.7-characterization/chat-toolui.spec.ts b/e2e/tests/v1.7-characterization/chat-toolui.spec.ts new file mode 100644 index 000000000..04dd0a1d1 --- /dev/null +++ b/e2e/tests/v1.7-characterization/chat-toolui.spec.ts @@ -0,0 +1,183 @@ +import { test, expect, ElectronApplication, Page } from '@playwright/test'; +import { bootIntoDashboard, closeSessionQuietly, dispatch, launchSessionWithCard, select } from './support'; + +// Characterization of the chat-surface claims in the 1.7.5/1.7.7 release notes: tool-ui questions, +// links inside widgets, and the mid-turn provider pill. No provider key: sessions come from the app's +// own launch route (parked, pre-warming), and the messages a live agent would stream are dispatched +// through the store's own reducer in the wire shape the backend emits. The renders and clicks are real. +test.describe.configure({ mode: 'serial' }); +test.describe('characterization: chat tool-ui and provider pill (1.7.5-1.7.7)', () => { + let app: ElectronApplication; + let win: Page; + let dashboardId: string; + let sessionId: string; + let branchId: string; + const created: string[] = []; + + const stamp = () => new Date().toISOString(); + const card = () => win.locator(`[data-select-type="agent-card"][data-select-id="${sessionId}"]`); + + test.beforeAll(async () => { + ({ app, win, dashboardId } = await bootIntoDashboard()); + sessionId = await launchSessionWithCard(win, dashboardId, 'Characterization chat'); + created.push(sessionId); + branchId = await select(win, (s) => s.agents.sessions[Object.keys(s.agents.sessions)[0]].active_branch_id) as string; + // The card opens expanded; the transcript is where the tool-ui cards paint. + await dispatch(win, { type: 'agents/expandSession', payload: sessionId }); + }); + test.afterAll(async () => { + for (const id of created) await closeSessionQuietly(win, id).catch(() => {}); + await app?.close().catch(() => {}); + }); + + // The pill is renderer-local: a wholesale session snapshot (fetchSession) legitimately replaces it, and + // a fresh launch runs a couple of those in its first seconds. Arm it in a quiet moment, so what the + // test measures is the claim under test — a status FRAME must not wipe it — not launch traffic. + const armProviderRetrying = async (attempt: number): Promise => { + for (let round = 0; round < 5; round++) { + await dispatch(win, { type: 'agents/setProviderRetrying', payload: { sessionId, attempt, delayMs: 60_000 } }); + await win.waitForTimeout(2500); + if (await select(win, (s) => s.agents.sessions[Object.keys(s.agents.sessions)[0]]?.provider_retrying != null)) return; + } + throw new Error('provider_retrying never held for 2.5s'); + }; + + test('1.7.7 F13: a provider retry shows a live status pill, and a status frame does not wipe it', async () => { + await armProviderRetrying(2); + const pill = card().getByText(/Provider busy, retrying \(attempt 2\)/); + await expect(pill).toBeVisible({ timeout: 10_000 }); + // A WS status frame carries no transient pill fields; the reducer must keep the renderer-local one. + const session = await select(win, (s) => s.agents.sessions[Object.keys(s.agents.sessions)[0]]); + const frame = { ...session, provider_retrying: undefined, rate_limited: undefined, status: 'running' }; + await dispatch(win, { type: 'agents/updateSession', payload: frame }); + expect(await select(win, (s) => s.agents.sessions[Object.keys(s.agents.sessions)[0]]?.provider_retrying?.attempt)).toBe(2); + await win.waitForTimeout(600); + await expect(pill).toBeVisible(); + await dispatch(win, { type: 'agents/clearProviderRetrying', payload: { sessionId } }); + await expect(pill).toBeHidden({ timeout: 5000 }); + }); + + test('1.7.7 F7/H4: an AskUI approval card answers once, its receipt lands, and a second press is inert', async () => { + // Route the respond POST so this run does not depend on an agent parked server-side: the first + // answer is accepted (200, not gone), and every call is counted. + let responds = 0; + await win.route('**/api/ui-requests/respond', async (route) => { + responds += 1; + await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ ok: true }) }); + }); + const callId = 'toolu_char_ask_1'; + await dispatch(win, { + type: 'agents/addMessage', + payload: { + sessionId, + message: { + id: 'char-ask-call-1', role: 'tool_call', timestamp: stamp(), branch_id: branchId, parent_id: null, + content: { + tool: 'AskUI', id: callId, + input: { component: 'approval-card', props: { id: 'char-approval-1', title: 'Deploy the change?', description: 'Characterization ask', confirmLabel: 'Ship it', cancelLabel: 'Hold' } }, + }, + }, + }, + }); + const approve = card().getByRole('button', { name: 'Ship it' }); + await expect(approve).toBeVisible({ timeout: 15_000 }); + await expect(card().getByRole('heading', { name: 'Deploy the change?' })).toBeVisible(); + await approve.click(); + // The choice flips to its receipt the instant the user answers, before any tool result lands. + await expect(approve).toBeHidden({ timeout: 10_000 }); + await expect.poll(() => responds, { timeout: 5000 }).toBe(1); + // A second press anywhere on that question is inert: one answer across every surface. + await win.waitForTimeout(500); + expect(responds).toBe(1); + await win.unroute('**/api/ui-requests/respond'); + }); + + test('1.7.7 F7: an answer nothing is waiting for is reported honestly, not swallowed', async () => { + // When nothing is parked server-side for an ask (agent gone, ask expired, transcript replayed) the + // bridge answers 200 + gone rather than a red 404. Play that verdict back to the renderer: the + // bubble must say so instead of quietly showing a receipt for an answer that never reached an agent. + await win.route('**/api/ui-requests/respond', async (route) => { + await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ ok: false, gone: true }) }); + }); + await dispatch(win, { + type: 'agents/addMessage', + payload: { + sessionId, + message: { + id: 'char-ask-call-2', role: 'tool_call', timestamp: stamp(), branch_id: branchId, parent_id: null, + content: { + tool: 'AskUI', id: 'toolu_char_ask_2', + input: { component: 'approval-card', props: { id: 'char-approval-2', title: 'Second question', confirmLabel: 'Yes please', cancelLabel: 'No thanks' } }, + }, + }, + }, + }); + const yes = card().getByRole('button', { name: 'Yes please' }); + await expect(yes).toBeVisible({ timeout: 15_000 }); + await yes.click(); + await expect(card().getByText(/This answer didn't reach the agent/)).toBeVisible({ timeout: 15_000 }); + // And it is answerable again (the registry released it), so the user can retry. + await expect(card().getByRole('button', { name: 'Yes please' })).toBeVisible({ timeout: 10_000 }); + await win.unroute('**/api/ui-requests/respond'); + }); + + test('1.7.7 F7: an older pending ask goes quiet once a newer one is live', async () => { + // Two unanswered asks: only the LATEST renders as a form; the older is a quiet row, not a second clickable copy. + for (const n of [3, 4]) { + await dispatch(win, { + type: 'agents/addMessage', + payload: { + sessionId, + message: { + id: `char-ask-call-${n}`, role: 'tool_call', timestamp: stamp(), branch_id: branchId, parent_id: null, + content: { + tool: 'AskUI', id: `toolu_char_ask_${n}`, + input: { component: 'approval-card', props: { id: `char-approval-${n}`, title: `Question ${n}`, confirmLabel: `Confirm ${n}`, cancelLabel: `Cancel ${n}` } }, + }, + }, + }, + }); + } + await expect(card().getByRole('button', { name: 'Confirm 4' })).toBeVisible({ timeout: 15_000 }); + await expect(card().getByRole('button', { name: 'Confirm 3' })).toHaveCount(0); + }); + + test('1.7.7 F8: a link inside a widget opens as a browser card, leaving fullscreen first', async () => { + const before = await select(win, (s) => Object.keys(s.dashboardLayout.browserCards).length); + await dispatch(win, { + type: 'agents/addMessage', + payload: { + sessionId, + message: { + id: 'char-show-call-1', role: 'tool_call', timestamp: stamp(), branch_id: branchId, parent_id: null, + content: { + tool: 'ShowUI', id: 'toolu_char_show_1', + input: { component: 'link-preview', props: { id: 'char-link-1', href: 'https://example.com/characterization', title: 'Characterization link', domain: 'example.com' } }, + }, + }, + }, + }); + const link = card().getByRole('link').filter({ hasText: 'Characterization link' }).first(); + await expect(link).toBeVisible({ timeout: 15_000 }); + // Put the chat fullscreen: the click must exit fullscreen so the new browser card is actually visible. + await dispatch(win, { type: 'dashboardLayout/setTiledCard', payload: { cardId: sessionId, zone: 'fullscreen' } }); + await expect.poll(() => select(win, (s) => s.dashboardLayout.tiledCards[Object.keys(s.agents.sessions)[0]] ?? null), { timeout: 5000 }).toBe('fullscreen'); + await win.waitForTimeout(700); + await link.click(); + await expect.poll(() => select(win, (s) => Object.keys(s.dashboardLayout.browserCards).length), { timeout: 10_000 }).toBe(before + 1); + expect(await select(win, (s) => Object.entries(s.dashboardLayout.tiledCards).filter(([, z]) => z === 'fullscreen').length)).toBe(0); + const opened = await select(win, (s) => Object.values(s.dashboardLayout.browserCards).map((b: any) => b.url)); + expect(opened.some((u: string) => u.startsWith('https://example.com/characterization'))).toBe(true); + // Leave the profile as we found it. + const ids = await select(win, (s) => Object.keys(s.dashboardLayout.browserCards)); + for (const id of ids) await dispatch(win, { type: 'dashboardLayout/removeBrowserCard', payload: id }); + }); + + test('1.7.5 F5: a self-healing provider hiccup never shows a scary card, only the muted pill', async () => { + // The pill from F13 is the whole surface for a mid-turn provider hiccup: no red card, no CTA. + await armProviderRetrying(1); + await expect(card().getByText(/Provider busy, retrying/)).toBeVisible({ timeout: 10_000 }); + await expect(card().getByRole('button', { name: /reconnect|retry now|try again/i })).toHaveCount(0); + await dispatch(win, { type: 'agents/clearProviderRetrying', payload: { sessionId } }); + }); +}); diff --git a/e2e/tests/v1.7-characterization/resilience.spec.ts b/e2e/tests/v1.7-characterization/resilience.spec.ts new file mode 100644 index 000000000..12531483f --- /dev/null +++ b/e2e/tests/v1.7-characterization/resilience.spec.ts @@ -0,0 +1,135 @@ +import { test, expect, ElectronApplication, Page } from '@playwright/test'; +import { execSync } from 'child_process'; +import { api, bootIntoDashboard, closeSessionQuietly, dispatch, launchSessionWithCard, select } from './support'; + +// Characterization of the resilience claims in the 1.7.7 release notes: the board never wipes on a +// momentary read error, and a lost local connection retries, heals, and only speaks up when it cannot. +// The outage tests kill the packaged backend for real (the app's own respawn brings it back). + +// The packaged backend is a child of the app: uvicorn on the port the renderer reports. +function backendPid(port: number): number | null { + try { + if (process.platform === 'win32') { + const out = execSync( + `powershell -NoProfile -Command "Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like '*uvicorn*backend.main:app*--port ${port}*' } | Select-Object -ExpandProperty ProcessId"`, + { encoding: 'utf8', timeout: 20_000 }, + ); + const pid = parseInt(out.trim().split(/\s+/)[0] || '', 10); + return Number.isFinite(pid) ? pid : null; + } + const out = execSync(`ps -axo pid=,command= | grep -F -- "uvicorn backend.main:app" | grep -F -- "--port ${port}" | grep -v grep`, { encoding: 'utf8' }); + const pid = parseInt(out.trim().split(/\s+/)[0] || '', 10); + return Number.isFinite(pid) ? pid : null; + } catch { + return null; + } +} + +test.describe.configure({ mode: 'serial' }); +test.describe('characterization: resilience (1.7.7)', () => { + let app: ElectronApplication; + let win: Page; + let dashboardId: string; + let sessionId: string; + const created: string[] = []; + + test.beforeAll(async () => { + ({ app, win, dashboardId } = await bootIntoDashboard()); + }); + test.afterAll(async () => { + for (const id of created) await closeSessionQuietly(win, id).catch(() => {}); + await app?.close().catch(() => {}); + }); + + const sessionIds = () => select(win, (s) => Object.keys(s.agents.sessions)); + const cardIds = () => select(win, (s) => Object.keys(s.dashboardLayout.cards)); + + test('1.7.7 F2: a failed or unscoped sessions read never wipes the board', async () => { + sessionId = await launchSessionWithCard(win, dashboardId, 'Characterization board'); + created.push(sessionId); + // Make the session strippable in principle: settled and no longer tracked as a live notification. + await dispatch(win, { type: 'agents/updateSessionStatus', payload: { sessionId, status: 'completed' } }); + await dispatch(win, { type: 'agents/dismissAgentNotification', payload: sessionId }); + expect(await sessionIds()).toContain(sessionId); + expect(await cardIds()).toContain(sessionId); + // A read error lands in .rejected, which strips nothing. + await dispatch(win, { type: 'agents/fetchSessions/rejected', error: { message: 'sessions fetch failed: 500' }, meta: { arg: { dashboardId }, requestId: 'char-r1', requestStatus: 'rejected' } } as any); + // An UNSCOPED empty answer has no authority to delete anything (memory-only after a respawn). + await dispatch(win, { type: 'agents/fetchSessions/fulfilled', payload: [], meta: { arg: {}, requestId: 'char-r2', requestStatus: 'fulfilled' } } as any); + // Another dashboard's list is only authoritative for its own sessions. + await dispatch(win, { type: 'agents/fetchSessions/fulfilled', payload: [], meta: { arg: { dashboardId: 'some-other-dashboard' }, requestId: 'char-r3', requestStatus: 'fulfilled' } } as any); + await win.waitForTimeout(1200); + expect(await sessionIds()).toContain(sessionId); + expect(await cardIds()).toContain(sessionId); + await expect(win.locator(`[data-select-type="agent-card"][data-select-id="${sessionId}"]`)).toBeVisible(); + // The backend's own list for this dashboard still carries the session: the read path is honest. + const { status, body } = await api<{ sessions: Array<{ id: string }> }>(win, `/agents/sessions?dashboard_id=${dashboardId}`); + expect(status).toBe(200); + expect(body.sessions.some((s) => s.id === sessionId)).toBe(true); + }); + +}); + +// Each outage test boots its own app: the respawn budget and the renderer's reachability state both +// start clean, exactly as they would on the boot a user actually experiences. +test.describe('characterization: resilience — lost local connection (1.7.7 F6)', () => { + let app: ElectronApplication; + let win: Page; + test.beforeEach(async () => { ({ app, win } = await bootIntoDashboard()); }); + test.afterEach(async () => { await app?.close().catch(() => {}); }); + + const health = () => win.evaluate(async () => { + const port = (window as any).openswarm.getBackendPort(); + const host = window.location.hostname || 'localhost'; + try { return (await fetch(`http://${host}:${port}/api/health/check`, { cache: 'no-store' })).status; } catch { return 0; } + }); + + test('1.7.7 F6: a lost local connection retries and heals itself, without a scary card', async () => { + const port: number = await win.evaluate(() => (window as any).openswarm.getBackendPort()); + const pid = backendPid(port); + test.skip(pid === null, 'could not identify the packaged backend process on this host'); + const pill = win.getByText('Reconnecting to OpenSwarm…'); + process.kill(pid!, 'SIGKILL'); + // The app respawns it (bounded, backoff); a quick recovery must be silent — no pill flashes. + // (A cold packaged-Python boot on a CI runner can take a while; the budget is generous.) + let pillSeen = false; + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + if (await pill.count()) pillSeen = true; + const status = await health(); + const livePid = backendPid(port); + if (status === 200 && livePid !== null && livePid !== pid) break; + await win.waitForTimeout(500); + } + expect(await health(), 'the backend must be back on the same port').toBe(200); + expect(pillSeen, 'a recovery inside the pill grace period must not flash the reconnecting pill').toBe(false); + }); + + test('1.7.7 F6: when it cannot heal quickly, it says so plainly, and heals when it can', async () => { + const port: number = await win.evaluate(() => (window as any).openswarm.getBackendPort()); + test.skip(backendPid(port) === null, 'could not identify the packaged backend process on this host'); + const pill = win.getByText('Reconnecting to OpenSwarm…'); + // The pill is deliberately slow to speak: a GET only counts as a failure after its whole retry + // ladder, two of those flip the connection to down, and the pill waits a further grace period. So + // keep the outage continuous well past that: kill each respawn as it appears (the budget is + // bounded, so at most four, leaving the fifth to heal) while the renderer keeps asking. + const killed = new Set(); + const stopAt = Date.now() + 45_000; + let live = backendPid(port); + const asking = setInterval(() => { void health().catch(() => 0); }, 400); + try { + while (Date.now() < stopAt && killed.size < 4) { + if (live !== null && !killed.has(live)) { try { process.kill(live, 'SIGKILL'); } catch { /* already gone */ } killed.add(live); } + if (await pill.count()) break; + await win.waitForTimeout(200); + live = backendPid(port); + } + await expect(pill).toBeVisible({ timeout: 40_000 }); + } finally { + clearInterval(asking); + } + // Now the app's next respawn is left alone: it heals, and the pill withdraws. + await expect.poll(health, { timeout: 90_000, intervals: [1000] }).toBe(200); + await expect(pill).toBeHidden({ timeout: 30_000 }); + }); +}); diff --git a/e2e/tests/v1.7-characterization/support.ts b/e2e/tests/v1.7-characterization/support.ts new file mode 100644 index 000000000..e5154cf1c --- /dev/null +++ b/e2e/tests/v1.7-characterization/support.ts @@ -0,0 +1,119 @@ +import { expect, ElectronApplication, Locator, Page } from '@playwright/test'; +import { launchApp, waitForMainWindow } from '../../helpers/launch'; + +// Shared plumbing for the 1.7.4-1.7.7 characterization specs. Everything drives the PACKAGED app +// through the same doors a user (or the app itself) uses: the Redux store the packaged renderer +// exposes under OPENSWARM_E2E=1, and the renderer's own fetch (already bearer-patched by +// shared/config.ts), so no spec ever handles a token or a port by hand. + +export interface AppHandle { + app: ElectronApplication; + win: Page; + dashboardId: string; +} + +// Launch, wait for the dashboard to be the LOADED one (entering a dashboard resets the per-dashboard +// layout and then fetches the saved one; anything dispatched inside that window is wiped), and clear +// the release card so its overlay can't sit over what a spec opens next. +export async function bootIntoDashboard(): Promise { + const app = await launchApp(); + const win = await waitForMainWindow(app); + await win.waitForFunction(() => { + const route = window.location.hash.match(/^#\/dashboard\/([^/?#]+)/); + const store = (window as any).__OPENSWARM_STORE__; + if (!route || !store) return false; + const layout = store.getState().dashboardLayout; + return layout.initialized === true && layout.loading === false; + }, null, { timeout: 120_000 }); + const gotIt = win.getByRole('button', { name: 'Got it' }); + if (await gotIt.count()) await gotIt.first().click({ timeout: 5000 }).catch(() => {}); + const dashboardId = await win.evaluate(() => window.location.hash.match(/^#\/dashboard\/([^/?#]+)/)![1]); + return { app, win, dashboardId }; +} + +export async function dispatch(win: Page, action: { type: string; payload?: unknown; meta?: unknown }): Promise { + await win.evaluate((a) => { (window as any).__OPENSWARM_STORE__.dispatch(a); }, action); +} + +// A store read; the selector runs inside the renderer, so pass a self-contained function. +export async function select(win: Page, selector: (state: any) => T): Promise { + return win.evaluate((src) => { + const fn = new Function('state', `return (${src})(state);`) as (state: unknown) => unknown; + return fn((window as any).__OPENSWARM_STORE__.getState()); + }, selector.toString()) as Promise; +} + +// The renderer's fetch against the packaged backend: same origin, same auth, same retry policy the app uses. +export async function api(win: Page, path: string, init?: { method?: string; body?: unknown }): Promise<{ status: number; body: T }> { + return win.evaluate(async ({ path, init }) => { + const port = (window as any).openswarm.getBackendPort(); + const res = await fetch(`http://127.0.0.1:${port}/api${path}`, { + method: init?.method ?? 'GET', + headers: init?.body !== undefined ? { 'Content-Type': 'application/json' } : undefined, + body: init?.body !== undefined ? JSON.stringify(init.body) : undefined, + }); + let body: unknown = null; + try { body = await res.json(); } catch { body = null; } + return { status: res.status, body }; + }, { path, init }) as Promise<{ status: number; body: T }>; +} + +// A live session with a card on the current dashboard, created through the app's own launch route and +// its own reducer (no provider key needed: the launch parks a session and pre-warms in the background). +export async function launchSessionWithCard(win: Page, dashboardId: string, name: string): Promise { + const { status, body } = await api<{ session: { id: string } }>(win, '/agents/launch', { method: 'POST', body: { name, dashboard_id: dashboardId } }); + expect(status, 'agents/launch').toBe(200); + await dispatch(win, { type: 'agents/launchAgent/fulfilled', payload: body.session }); + await expect(win.locator(`[data-select-type="agent-card"][data-select-id="${body.session.id}"]`)).toBeVisible({ timeout: 15_000 }); + return body.session.id; +} + +// Close through the app's own route + reducer, so the card leaves and the session lands in history. +export async function closeSessionQuietly(win: Page, sessionId: string): Promise { + await api(win, `/agents/sessions/${sessionId}/close`, { method: 'POST' }).catch(() => {}); + await dispatch(win, { type: 'agents/closeSession/fulfilled', payload: sessionId }); +} + +export async function openSettingsCard(win: Page, tab?: string): Promise { + await dispatch(win, { type: 'dashboardLayout/openSettingsCard', payload: tab ? { tab } : undefined }); + const close = win.locator('[data-onboarding="settings-close-button"]').first(); + await expect(close).toBeVisible({ timeout: 15_000 }); + return close; +} + +// The window lights are pointer-inert until their .osw-card is hovered (a card crossing can't hit-test +// through the dots) and Playwright hit-tests before it moves the mouse; opening also pans to the card. +// So: let it settle, cross the card the way a hand does, then click the dot. +export async function clickWindowLight(win: Page, light: Locator): Promise { + await settle(light); + await win.locator('.osw-card', { has: light }).last().hover({ timeout: 8000 }); + await light.click({ timeout: 8000 }); +} + +export async function closeSettingsCard(win: Page): Promise { + const close = win.locator('[data-onboarding="settings-close-button"]').first(); + if (!(await close.count())) return; + await clickWindowLight(win, close); + await expect(close).toBeHidden({ timeout: 8000 }); +} + +// Resolves once the element's box is unchanged across two polls 300ms apart (a pan/zoom has finished). +export async function settle(target: Locator, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + let previous = ''; + while (Date.now() < deadline) { + const box = JSON.stringify(await target.boundingBox()); + if (box !== 'null' && box === previous) return; + previous = box; + await target.page().waitForTimeout(300); + } +} + +// The painted camera: the canvas content layer's transform is written imperatively per gesture frame, +// so it is the truth about whether a wheel or a drag moved the world. +export async function paintedCamera(win: Page): Promise { + return win.evaluate(() => { + const content = document.querySelector('[data-canvas-content]') as HTMLElement | null; + return content ? content.style.transform || getComputedStyle(content).transform : 'missing'; + }); +} diff --git a/e2e/tests/v1.7-characterization/workflows-and-settings.spec.ts b/e2e/tests/v1.7-characterization/workflows-and-settings.spec.ts new file mode 100644 index 000000000..d9536ee3e --- /dev/null +++ b/e2e/tests/v1.7-characterization/workflows-and-settings.spec.ts @@ -0,0 +1,112 @@ +import { test, expect, ElectronApplication, Page } from '@playwright/test'; +import { api, bootIntoDashboard, closeSettingsCard, openSettingsCard } from './support'; + +// Characterization of the workflow and settings claims in the 1.7.4-1.7.7 release notes, against the +// packaged backend and the Settings window. No provider key: an off or deleted workflow is refused +// before anything would spawn, and memory / dictation defaults are plain state. +test.describe.configure({ mode: 'serial' }); +test.describe('characterization: workflows and settings (1.7.4-1.7.7)', () => { + let app: ElectronApplication; + let win: Page; + let dashboardId: string; + const workflowIds: string[] = []; + const factIds: string[] = []; + + test.beforeAll(async () => { + ({ app, win, dashboardId } = await bootIntoDashboard()); + }); + test.afterAll(async () => { + for (const id of workflowIds) { + await api(win, `/workflows/${id}`, { method: 'DELETE' }).catch(() => {}); + await api(win, `/workflows/${id}/purge`, { method: 'DELETE' }).catch(() => {}); + } + for (const id of factIds) await api(win, `/memory/${id}`, { method: 'DELETE' }).catch(() => {}); + await app?.close().catch(() => {}); + }); + + // A schedule that is fully configured (so "on" really means on) but will not fire during the test. + const schedule = (enabled: boolean) => ({ enabled, repeat_every: 12, repeat_unit: 'month', hour: 3, minute: 0, day_of_month: 1 }); + + // A workflow with one real step and that schedule. + async function createWorkflow(title: string, enabled: boolean): Promise { + const { status, body } = await api<{ id: string }>(win, '/workflows/create', { + method: 'POST', + body: { + title, auto_named: false, unsaved: false, dashboard_id: dashboardId, + steps: [{ text: 'Say hello and stop.', enabled: true }], + schedule: schedule(enabled), + }, + }); + expect(status, `workflows/create ${title}`).toBe(200); + workflowIds.push(body.id); + return body.id; + } + + test('1.7.5 H2: switching a workflow off stops everything — Run Now is refused and the refusal shows in History', async () => { + const id = await createWorkflow('Characterization: switched off', true); + const patched = await api<{ schedule: { enabled: boolean } }>(win, `/workflows/${id}`, { method: 'PATCH', body: { schedule: schedule(false) } }); + expect(patched.status).toBe(200); + expect(patched.body.schedule.enabled).toBe(false); + const run = await api<{ run_id: string; status: string | null; error: string | null }>(win, `/workflows/${id}/run`, { method: 'POST', body: {} }); + expect(run.status).toBe(200); + expect(run.body.status).toBe('skipped'); + expect(run.body.error).toMatch(/paused/i); + // Recorded, not just returned: a refusal the user cannot see in History reads as the run vanishing. + const { body: history } = await api<{ runs: Array<{ status: string; error: string | null; triggered_by: string }> }>(win, `/workflows/${id}/runs`); + expect(history.runs.some((r) => r.status === 'skipped' && /paused/i.test(r.error ?? '') && r.triggered_by === 'manual')).toBe(true); + // Turn it back on: the same route now admits the run (no key here, so it fails downstream, but it is no longer refused as paused). + const reenabled = await api<{ schedule: { enabled: boolean } }>(win, `/workflows/${id}`, { method: 'PATCH', body: { schedule: schedule(true) } }); + expect(reenabled.body.schedule.enabled).toBe(true); + const admitted = await api<{ status: string | null; error: string | null }>(win, `/workflows/${id}/run`, { method: 'POST', body: {} }); + expect(admitted.status).toBe(200); + expect(admitted.body.error ?? '').not.toMatch(/paused/i); + }); + + test('1.7.5 H1: deleting a scheduled workflow makes it stay deleted — it cannot run from Run Now', async () => { + const id = await createWorkflow('Characterization: deleted', true); + const del = await api(win, `/workflows/${id}`, { method: 'DELETE' }); + expect(del.status).toBe(200); + const run = await api<{ detail: string }>(win, `/workflows/${id}/run`, { method: 'POST', body: {} }); + expect(run.status).toBe(409); + expect(run.body.detail).toMatch(/Trash/i); + // It is in the trash, not resurrected into the live list. + const { body: deleted } = await api<{ workflows?: Array<{ id: string }> } | Array<{ id: string }>>(win, '/workflows/deleted'); + const deletedIds = Array.isArray(deleted) ? deleted.map((w) => w.id) : (deleted.workflows ?? []).map((w) => w.id); + expect(deletedIds).toContain(id); + const { body: live } = await api<{ workflows?: Array<{ id: string }> } | Array<{ id: string }>>(win, '/workflows/list'); + const liveIds = Array.isArray(live) ? live.map((w) => w.id) : (live.workflows ?? []).map((w) => w.id); + expect(liveIds).not.toContain(id); + }); + + test('1.7.7 H1: a fact you save in Settings → Memory is the whole memory, visible and editable', async () => { + const text = `Characterization fact ${Date.now()}`; + await openSettingsCard(win, 'memory'); + const input = win.getByPlaceholder('Add a fact agents should always know (Enter to save)'); + await expect(input).toBeVisible({ timeout: 15_000 }); + // The renderer serves identical GETs from a 1s cache; the panel's post-save refresh must not land + // inside the window of the panel's own mount fetch, so let that second pass before typing. + await win.waitForTimeout(1500); + // Typed the way a hand types it: the draft is React state, and Enter reads it on the next commit. + await input.click(); + await win.keyboard.type(text, { delay: 10 }); + await win.keyboard.press('Enter'); + // The window shows the saved fact (its own refresh runs the moment the save lands). + await expect(win.locator('[data-select-type="settings-card"]').getByText(text, { exact: true })).toBeVisible({ timeout: 15_000 }); + // And the list is the API's list: nothing hidden behind it. + const { body } = await api<{ facts: Array<{ id: string; text: string; source: string }> }>(win, '/memory'); + const fact = body.facts.find((f) => f.text === text); + expect(fact, 'the saved fact is in the memory store').toBeTruthy(); + expect(fact!.source).toBe('user'); + factIds.push(fact!.id); + await closeSettingsCard(win); + }); + + test('1.7.4 F2: dictation cue sounds default to an audible level (70%)', async () => { + await openSettingsCard(win, 'dictation'); + const row = win.locator('[data-select-id="dictation_sounds"]'); + await expect(row).toBeVisible({ timeout: 15_000 }); + const slider = row.getByRole('slider'); + await expect(slider).toHaveAttribute('aria-valuenow', '70'); + await closeSettingsCard(win); + }); +});