diff --git a/README.md b/README.md index 48c32c0..d21e078 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,8 @@ Embedded from [`docs/scenario-matrix.md`](docs/scenario-matrix.md), generated by | UI-DRAGDROP-002 | P1 | Drag box B over box A | ✅ | ✅ | — | | UI-DRAGDROP-003 | P1 | Drag boxes in both directions | ✅ | ✅ | — | | UI-DROPDOWN-001 | P0 | Dropdown options can be selected | ✅ | ✅ | ✅ | -| UI-DYNCONTENT-001 | P1 | Dynamic content changes on reload | ✅ | — | — | -| UI-DYNCONTENT-002 | P1 | Dynamic content partially changes on reload | ✅ | — | — | +| UI-DYNCONTENT-001 | P2 | Dynamic content changes on reload | ✅ | ✅ | — | +| UI-DYNCONTENT-002 | P2 | Dynamic content partially changes on reload | ✅ | ✅ | — | | UI-DYNCTRL-001 | P0 | Dynamic controls remove and add a checkbox | ✅ | ✅ | ✅ | | UI-DYNCTRL-002 | P0 | Dynamic controls checkbox toggles | ✅ | ✅ | ✅ | | UI-DYNCTRL-003 | P0 | Dynamic controls enable and disable a textbox | ✅ | ✅ | ✅ | @@ -93,6 +93,14 @@ Embedded from [`docs/scenario-matrix.md`](docs/scenario-matrix.md), generated by | UI-INFINITE-001 | P1 | Infinite scroll loads additional content on demand | — | ✅ | — | | UI-CHALLENGING-001 | P1 | Challenging DOM is asserted without dynamic identifiers | — | ✅ | — | | UI-SLIDER-001 | P1 | Horizontal slider moves by keyboard increments | — | ✅ | — | +| UI-DISAPPEAR-001 | P2 | Disappearing elements keep a stable core menu | — | ✅ | — | +| UI-ENTRYAD-001 | P2 | Entry ad modal can be observed and closed | — | ✅ | — | +| UI-EXITINTENT-001 | P2 | Exit intent modal opens on desktop mouse leave | — | ✅ | — | +| UI-TYPOS-001 | P2 | Typos page demonstrates nondeterministic copy | — | ✅ | — | +| UI-SHIFTING-001 | P2 | Shifting content demonstrates randomized layout | — | ✅ | — | +| UI-LARGEDOM-001 | P2 | Large and deep DOM supports targeted traversal | — | ✅ | — | +| UI-JQUERYUI-001 | P2 | JQuery UI menu exposes nested desktop options | — | ✅ | — | +| UI-ABTEST-001 | P2 | A/B testing page allows either experiment variant | — | ✅ | — | ## 6. Learning paths diff --git a/docs/flakiness-guide.md b/docs/flakiness-guide.md new file mode 100644 index 0000000..f40d7dd --- /dev/null +++ b/docs/flakiness-guide.md @@ -0,0 +1,36 @@ +# Flakiness guide + +The flake lab is a deliberately isolated TypeScript Playwright slice for studying unstable UI patterns without weakening normal gates. + +## Tags + +- `@flaky-demo` marks examples that are safe to study locally but excluded from PR gates and nightly by default. +- `@not-ci` marks examples that are unsuitable for scheduled automation and should stay local/manual only. +- `@desktop` keeps mouse-only behavior out of mobile-emulation projects. + +## Local commands + +```bash +cd stacks/ts-playwright +THE_INTERNET_BASE_URL=http://localhost:7080 npm run test:flake-lab +``` + +Run `@not-ci` examples only when you are deliberately investigating the interaction: + +```bash +cd stacks/ts-playwright +THE_INTERNET_BASE_URL=http://localhost:7080 npx playwright test --project=chromium --grep @not-ci +``` + +## What to study + +| Scenario | Lesson | +| --- | --- | +| Disappearing Elements | Optional navigation items make exact menu assertions brittle. | +| Dynamic Content | Randomized copy or assets require retry-aware assertions and clear test intent. | +| Entry Ad / Exit Intent | Modal timing and mouse-leave behavior should not be part of a fast PR gate. | +| Typos | Nondeterministic text demonstrates why exact copy checks need stable contracts. | +| Shifting Content | Layout movement belongs in targeted visual or geometry checks, not broad smoke suites. | +| Large and Deep DOM | Traversal-heavy pages are useful for locator performance experiments. | +| JQuery UI Menus | Deep hover chains are desktop-only and sensitive to pointer sequencing. | +| A/B Testing | Experiment variants require either fixed test data or variant-tolerant assertions. | diff --git a/docs/scenario-matrix.md b/docs/scenario-matrix.md index d850ba9..e8ca14f 100644 --- a/docs/scenario-matrix.md +++ b/docs/scenario-matrix.md @@ -14,8 +14,8 @@ Generated by `tools/check-scenarios.py --write-matrix`. | UI-DRAGDROP-002 | P1 | Drag box B over box A | ✅ | ✅ | — | | UI-DRAGDROP-003 | P1 | Drag boxes in both directions | ✅ | ✅ | — | | UI-DROPDOWN-001 | P0 | Dropdown options can be selected | ✅ | ✅ | ✅ | -| UI-DYNCONTENT-001 | P1 | Dynamic content changes on reload | ✅ | — | — | -| UI-DYNCONTENT-002 | P1 | Dynamic content partially changes on reload | ✅ | — | — | +| UI-DYNCONTENT-001 | P2 | Dynamic content changes on reload | ✅ | ✅ | — | +| UI-DYNCONTENT-002 | P2 | Dynamic content partially changes on reload | ✅ | ✅ | — | | UI-DYNCTRL-001 | P0 | Dynamic controls remove and add a checkbox | ✅ | ✅ | ✅ | | UI-DYNCTRL-002 | P0 | Dynamic controls checkbox toggles | ✅ | ✅ | ✅ | | UI-DYNCTRL-003 | P0 | Dynamic controls enable and disable a textbox | ✅ | ✅ | ✅ | @@ -44,3 +44,11 @@ Generated by `tools/check-scenarios.py --write-matrix`. | UI-INFINITE-001 | P1 | Infinite scroll loads additional content on demand | — | ✅ | — | | UI-CHALLENGING-001 | P1 | Challenging DOM is asserted without dynamic identifiers | — | ✅ | — | | UI-SLIDER-001 | P1 | Horizontal slider moves by keyboard increments | — | ✅ | — | +| UI-DISAPPEAR-001 | P2 | Disappearing elements keep a stable core menu | — | ✅ | — | +| UI-ENTRYAD-001 | P2 | Entry ad modal can be observed and closed | — | ✅ | — | +| UI-EXITINTENT-001 | P2 | Exit intent modal opens on desktop mouse leave | — | ✅ | — | +| UI-TYPOS-001 | P2 | Typos page demonstrates nondeterministic copy | — | ✅ | — | +| UI-SHIFTING-001 | P2 | Shifting content demonstrates randomized layout | — | ✅ | — | +| UI-LARGEDOM-001 | P2 | Large and deep DOM supports targeted traversal | — | ✅ | — | +| UI-JQUERYUI-001 | P2 | JQuery UI menu exposes nested desktop options | — | ✅ | — | +| UI-ABTEST-001 | P2 | A/B testing page allows either experiment variant | — | ✅ | — | diff --git a/scenarios/catalog.yml b/scenarios/catalog.yml index aff538b..94540c9 100644 --- a/scenarios/catalog.yml +++ b/scenarios/catalog.yml @@ -51,14 +51,14 @@ scenarios: coverage: {java-selenium-testng: true, ts-playwright: true, python-playwright: true} - id: UI-DYNCONTENT-001 title: Dynamic content changes on reload - priority: P1 - tags: [] - coverage: {java-selenium-testng: true, ts-playwright: false, python-playwright: false} + priority: P2 + tags: [flaky-demo] + coverage: {java-selenium-testng: true, ts-playwright: true, python-playwright: false} - id: UI-DYNCONTENT-002 title: Dynamic content partially changes on reload - priority: P1 - tags: [] - coverage: {java-selenium-testng: true, ts-playwright: false, python-playwright: false} + priority: P2 + tags: [flaky-demo] + coverage: {java-selenium-testng: true, ts-playwright: true, python-playwright: false} - id: UI-DYNCTRL-001 title: Dynamic controls remove and add a checkbox priority: P0 @@ -199,3 +199,43 @@ scenarios: priority: P1 tags: [] coverage: {java-selenium-testng: false, ts-playwright: true, python-playwright: false} + - id: UI-DISAPPEAR-001 + title: Disappearing elements keep a stable core menu + priority: P2 + tags: [flaky-demo] + coverage: {java-selenium-testng: false, ts-playwright: true, python-playwright: false} + - id: UI-ENTRYAD-001 + title: Entry ad modal can be observed and closed + priority: P2 + tags: [flaky-demo] + coverage: {java-selenium-testng: false, ts-playwright: true, python-playwright: false} + - id: UI-EXITINTENT-001 + title: Exit intent modal opens on desktop mouse leave + priority: P2 + tags: [desktop, not-ci] + coverage: {java-selenium-testng: false, ts-playwright: true, python-playwright: false} + - id: UI-TYPOS-001 + title: Typos page demonstrates nondeterministic copy + priority: P2 + tags: [flaky-demo] + coverage: {java-selenium-testng: false, ts-playwright: true, python-playwright: false} + - id: UI-SHIFTING-001 + title: Shifting content demonstrates randomized layout + priority: P2 + tags: [flaky-demo] + coverage: {java-selenium-testng: false, ts-playwright: true, python-playwright: false} + - id: UI-LARGEDOM-001 + title: Large and deep DOM supports targeted traversal + priority: P2 + tags: [not-ci] + coverage: {java-selenium-testng: false, ts-playwright: true, python-playwright: false} + - id: UI-JQUERYUI-001 + title: JQuery UI menu exposes nested desktop options + priority: P2 + tags: [desktop, not-ci] + coverage: {java-selenium-testng: false, ts-playwright: true, python-playwright: false} + - id: UI-ABTEST-001 + title: A/B testing page allows either experiment variant + priority: P2 + tags: [flaky-demo] + coverage: {java-selenium-testng: false, ts-playwright: true, python-playwright: false} diff --git a/stacks/ts-playwright/package.json b/stacks/ts-playwright/package.json index 303301f..8f631f7 100644 --- a/stacks/ts-playwright/package.json +++ b/stacks/ts-playwright/package.json @@ -14,7 +14,8 @@ "lint": "eslint .", "format:check": "prettier --check .", "format": "prettier --write .", - "test:chromium:smoke": "playwright test --project=chromium --grep @smoke" + "test:chromium:smoke": "playwright test --project=chromium --grep @smoke", + "test:flake-lab": "playwright test --project=chromium --grep @flaky-demo --grep-invert @not-ci" }, "devDependencies": { "@eslint/js": "10.0.1", diff --git a/stacks/ts-playwright/tests/ab-testing.spec.ts b/stacks/ts-playwright/tests/ab-testing.spec.ts new file mode 100644 index 0000000..36f430b --- /dev/null +++ b/stacks/ts-playwright/tests/ab-testing.spec.ts @@ -0,0 +1,10 @@ +import { expect, test } from './fixtures.js'; + +test.describe('A/B Testing', () => { + test('UI-ABTEST-001 @flaky-demo accepts either experiment variant', async ({ page }) => { + await page.goto('/abtest'); + + await expect(page.getByRole('heading')).toHaveText(/A\/B Test/); + await expect(page.locator('.example p')).toContainText('Also known as split testing'); + }); +}); diff --git a/stacks/ts-playwright/tests/disappearing-elements.spec.ts b/stacks/ts-playwright/tests/disappearing-elements.spec.ts new file mode 100644 index 0000000..fc3b62f --- /dev/null +++ b/stacks/ts-playwright/tests/disappearing-elements.spec.ts @@ -0,0 +1,15 @@ +import { expect, test } from './fixtures.js'; + +const stableMenuItems = ['Home', 'About', 'Contact Us', 'Portfolio'] as const; +const allowedMenuItems = new Set([...stableMenuItems, 'Gallery']); + +test.describe('Disappearing Elements', () => { + test('UI-DISAPPEAR-001 @flaky-demo tolerates the optional menu item', async ({ page }) => { + await page.goto('/disappearing_elements'); + + const menuItems = await page.locator('#content a').allTextContents(); + + expect(menuItems).toEqual(expect.arrayContaining([...stableMenuItems])); + expect(menuItems.every((item) => allowedMenuItems.has(item))).toBe(true); + }); +}); diff --git a/stacks/ts-playwright/tests/dynamic-content.spec.ts b/stacks/ts-playwright/tests/dynamic-content.spec.ts new file mode 100644 index 0000000..4f8e3a3 --- /dev/null +++ b/stacks/ts-playwright/tests/dynamic-content.spec.ts @@ -0,0 +1,42 @@ +import type { Page } from '@playwright/test'; + +import { expect, test } from './fixtures.js'; + +type DynamicContentSnapshot = readonly string[]; + +async function contentSnapshot(page: Page, staticImages: boolean): Promise { + await page.goto(`/dynamic_content${staticImages ? '?with_content=static' : ''}`); + + return page + .locator('.large-10.columns') + .evaluateAll((nodes) => + nodes.map((node) => node.textContent?.replace(/\s+/g, ' ').trim() ?? ''), + ); +} + +async function expectContentToChange(page: Page, staticImages: boolean): Promise { + const first = await contentSnapshot(page, staticImages); + let changed = false; + + for (let attempt = 0; attempt < 5; attempt += 1) { + const next = await contentSnapshot(page, staticImages); + if (JSON.stringify(next) !== JSON.stringify(first)) { + changed = true; + break; + } + } + + expect(changed).toBe(true); +} + +test.describe('Dynamic Content', () => { + test('UI-DYNCONTENT-001 @flaky-demo observes fully dynamic content changes', async ({ page }) => { + await expectContentToChange(page, false); + }); + + test('UI-DYNCONTENT-002 @flaky-demo observes text changes with static images', async ({ + page, + }) => { + await expectContentToChange(page, true); + }); +}); diff --git a/stacks/ts-playwright/tests/entry-ad.spec.ts b/stacks/ts-playwright/tests/entry-ad.spec.ts new file mode 100644 index 0000000..11f7169 --- /dev/null +++ b/stacks/ts-playwright/tests/entry-ad.spec.ts @@ -0,0 +1,14 @@ +import { expect, test } from './fixtures.js'; + +test.describe('Entry Ad', () => { + test('UI-ENTRYAD-001 @flaky-demo closes the timed entry modal', async ({ page }) => { + await page.goto('/entry_ad'); + + const modal = page.locator('.modal'); + await expect(modal).toBeVisible(); + await expect(modal).toContainText('This is a modal window'); + + await modal.getByText('Close').click(); + await expect(modal).toBeHidden(); + }); +}); diff --git a/stacks/ts-playwright/tests/exit-intent.spec.ts b/stacks/ts-playwright/tests/exit-intent.spec.ts new file mode 100644 index 0000000..4d65c0c --- /dev/null +++ b/stacks/ts-playwright/tests/exit-intent.spec.ts @@ -0,0 +1,16 @@ +import { expect, test } from './fixtures.js'; + +test.describe('Exit Intent', () => { + test('UI-EXITINTENT-001 @desktop @not-ci opens the exit modal on mouse leave', async ({ + page, + }) => { + await page.goto('/exit_intent'); + + await page.mouse.move(300, 300); + await page.mouse.move(300, -10); + + const modal = page.locator('.modal'); + await expect(modal).toBeVisible(); + await expect(modal).toContainText('This is a modal window'); + }); +}); diff --git a/stacks/ts-playwright/tests/jquery-ui-menu.spec.ts b/stacks/ts-playwright/tests/jquery-ui-menu.spec.ts new file mode 100644 index 0000000..198ff42 --- /dev/null +++ b/stacks/ts-playwright/tests/jquery-ui-menu.spec.ts @@ -0,0 +1,14 @@ +import { expect, test } from './fixtures.js'; + +test.describe('JQuery UI Menus', () => { + test('UI-JQUERYUI-001 @desktop @not-ci models nested download options', async ({ page }) => { + await page.goto('/jqueryui/menu'); + + const menu = page.locator('#menu'); + + await expect(menu.locator('a', { hasText: 'Enabled' })).toBeVisible(); + await expect(menu.locator('a[href$="menu.pdf"]')).toHaveText('PDF'); + await expect(menu.locator('a[href$="menu.csv"]')).toHaveText('CSV'); + await expect(menu.locator('a[href$="menu.xls"]')).toHaveText('Excel'); + }); +}); diff --git a/stacks/ts-playwright/tests/large-dom.spec.ts b/stacks/ts-playwright/tests/large-dom.spec.ts new file mode 100644 index 0000000..11336a8 --- /dev/null +++ b/stacks/ts-playwright/tests/large-dom.spec.ts @@ -0,0 +1,13 @@ +import { expect, test } from './fixtures.js'; + +test.describe('Large and Deep DOM', () => { + test('UI-LARGEDOM-001 @not-ci reaches deep nodes and large-table cells', async ({ page }) => { + await page.goto('/large'); + + await expect(page.locator('#sibling-50\\.3')).toHaveText('50.3'); + await expect(page.locator('#large-table tbody tr')).toHaveCount(50); + await expect(page.locator('#large-table tbody tr').nth(49).locator('td').nth(49)).toHaveText( + '50.50', + ); + }); +}); diff --git a/stacks/ts-playwright/tests/shifting-content.spec.ts b/stacks/ts-playwright/tests/shifting-content.spec.ts new file mode 100644 index 0000000..e72f2a7 --- /dev/null +++ b/stacks/ts-playwright/tests/shifting-content.spec.ts @@ -0,0 +1,13 @@ +import { expect, test } from './fixtures.js'; + +test.describe('Shifting Content', () => { + test('UI-SHIFTING-001 @flaky-demo documents randomized menu placement', async ({ page }) => { + await page.goto('/shifting_content/menu?mode=random&pixel_shift=100'); + + const gallery = page.getByRole('link', { name: 'Gallery' }); + await expect(gallery).toHaveAttribute('href', '/gallery/'); + + const className = await gallery.getAttribute('class'); + expect([null, 'shift']).toContain(className); + }); +}); diff --git a/stacks/ts-playwright/tests/typos.spec.ts b/stacks/ts-playwright/tests/typos.spec.ts new file mode 100644 index 0000000..aeea33e --- /dev/null +++ b/stacks/ts-playwright/tests/typos.spec.ts @@ -0,0 +1,13 @@ +import { expect, test } from './fixtures.js'; + +test.describe('Typos', () => { + test('UI-TYPOS-001 @flaky-demo documents nondeterministic copy', async ({ page }) => { + await page.goto('/typos'); + + const paragraphs = page.locator('.example p'); + await expect(paragraphs.first()).toContainText('This example demonstrates a typo'); + + const variant = (await paragraphs.nth(1).textContent())?.trim() ?? ''; + expect(variant).toMatch(/Sometimes you'll see a typo, other times you (won't|won,t)\./); + }); +});