From 3d30a5ffafdb8a57881202d80391f6b55fffab52 Mon Sep 17 00:00:00 2001
From: Leon Krix <98028536+leonkrix@users.noreply.github.com>
Date: Wed, 28 Oct 2026 23:41:24 +0100
Subject: [PATCH 1/2] feat: ignoreFrames, armWhen and a scan command
---
.changeset/frames-arm-when-and-scan.md | 9 +
README.md | 79 +++-
docs/recipes.md | 16 +
docs/survey.md | 2 +-
package.json | 3 +
src/bin.ts | 4 +
src/cli.test.ts | 214 +++++++++
src/cli.ts | 607 +++++++++++++++++++++++++
src/fixture.ts | 7 +
src/install.ts | 25 +
src/options.ts | 21 +-
src/page-script.ts | 58 ++-
src/types.ts | 18 +
test/cli.spec.ts | 168 +++++++
test/features.spec.ts | 81 ++++
test/fixture.spec.ts | 21 +
test/fixtures/app-after-load.html | 29 ++
17 files changed, 1338 insertions(+), 24 deletions(-)
create mode 100644 .changeset/frames-arm-when-and-scan.md
create mode 100644 src/bin.ts
create mode 100644 src/cli.test.ts
create mode 100644 src/cli.ts
create mode 100644 test/cli.spec.ts
create mode 100644 test/fixtures/app-after-load.html
diff --git a/.changeset/frames-arm-when-and-scan.md b/.changeset/frames-arm-when-and-scan.md
new file mode 100644
index 0000000..4de5b41
--- /dev/null
+++ b/.changeset/frames-arm-when-and-scan.md
@@ -0,0 +1,9 @@
+---
+'playwright-layout-guard': minor
+---
+
+Three additions that came out of the trial on public pages:
+
+- **`ignoreFrames`** leaves single iframes alone, by a text that their address contains (`*` for any characters), for widgets you do not control. The box of the iframe in your page is still watched.
+- **`armWhen`** is a CSS selector of something that is shown when an app that draws itself after the load event is ready: the guard reports nothing until it matches, takes the layout then, and reports from there on. `guard.armed()` says whether the guard is watching, and a test whose selector never matched fails instead of passing without having watched anything.
+- **A command line**: `npx playwright-layout-guard scan
` opens pages in a browser and reports every box that moved after the load, next to the layout shift score of the browser (Chromium). Addresses can come from a file (`--file`), the report can be text, JSON, Markdown or HTML (`--format`, `--out`), the options come from a JSON file (`--options`), `--immediate` and `--arm-when` set `start` and `armWhen`, `--screenshots` saves the picture with the moved boxes marked, and the exit code is 0, 1 (something moved) or 2 (wrong command, a page that could not be loaded, an `--arm-when` element that never showed). The script of the trial is the first version of it.
diff --git a/README.md b/README.md
index 31a9283..1e88959 100644
--- a/README.md
+++ b/README.md
@@ -91,20 +91,22 @@ test.use({ layoutGuard: false }); // off for this file, say why in a comment
All optional. Set them with `test.use({ layoutGuard: { ... } })`, or pass them to `installLayoutGuard(page, options)`.
-| Option | Default | What it does |
-| ------------------------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `ignore` | `[]` | CSS selectors of elements (and everything inside) to leave alone. |
-| `fluidAttribute` | `'data-layout-fluid'` | Attribute that marks a box whose inside may rearrange. |
-| `tolerance` | `1` | Pixels an element may be off before it counts as moved. |
-| `quietMs` | `600` | How long to wait after a resize, an opened `` or a `pushState` before looking again. |
-| `scope` | `'body'` | CSS selector of the part of the page to watch. |
-| `start` | `'load'` | `'load'` reports from the load event on. `'immediate'` also reports what moves while the page loads (noisier, see below). |
-| `ignoreInsideInteractive` | `true` | Leave out what is inside a link or a button. |
-| `pageHeight` | `'growth'` | `'growth'`: the page became higher than the window. `'any'`: also a page that became lower. |
-| `animations` | `'report'` | `'allow'`: while a CSS transition or animation of a layout property runs and ends (a panel that slides open), nothing is reported. See [Animations that are meant](#animations-that-are-meant). |
-| `baseline` | none | Path of a JSON file with known shifts: they do not fail the test, a new one does. See [Adopting it in a project that moves today](#adopting-it-in-a-project-that-moves-today). |
-| `screenshot` | `true` | Attach a picture with the moved boxes marked to a failed test. See [See what moved](#see-what-moved). |
-| `maxReports` | `50` | The most shifts reported in one test. |
+| Option | Default | What it does |
+| ------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `ignore` | `[]` | CSS selectors of elements (and everything inside) to leave alone. |
+| `fluidAttribute` | `'data-layout-fluid'` | Attribute that marks a box whose inside may rearrange. |
+| `tolerance` | `1` | Pixels an element may be off before it counts as moved. |
+| `quietMs` | `600` | How long to wait after a resize, an opened `` or a `pushState` before looking again. |
+| `scope` | `'body'` | CSS selector of the part of the page to watch. |
+| `start` | `'load'` | `'load'` reports from the load event on. `'immediate'` also reports what moves while the page loads (noisier, see below). |
+| `ignoreInsideInteractive` | `true` | Leave out what is inside a link or a button. |
+| `pageHeight` | `'growth'` | `'growth'`: the page became higher than the window. `'any'`: also a page that became lower. |
+| `animations` | `'report'` | `'allow'`: while a CSS transition or animation of a layout property runs and ends (a panel that slides open), nothing is reported. See [Animations that are meant](#animations-that-are-meant). |
+| `baseline` | none | Path of a JSON file with known shifts: they do not fail the test, a new one does. See [Adopting it in a project that moves today](#adopting-it-in-a-project-that-moves-today). |
+| `screenshot` | `true` | Attach a picture with the moved boxes marked to a failed test. See [See what moved](#see-what-moved). |
+| `ignoreFrames` | `[]` | Frames (iframes) to leave alone, as text that their address contains (`*` for any characters): `['widget.example']`. See [Frames you do not control](#frames-you-do-not-control). |
+| `armWhen` | none | CSS selector of something that is shown when an app that draws itself after the load is ready: the guard reports from then on. See [An app that renders after the load](docs/recipes.md#an-app-that-renders-after-the-load). |
+| `maxReports` | `50` | The most shifts reported in one test. |
### Set the options for the whole project
@@ -143,6 +145,16 @@ While a CSS transition or animation of a layout property (`height`, `width`, `to
The price: while such an animation runs, a real shift somewhere else on the page is let go as well, which is why it is off by default.
+### Frames you do not control
+
+An iframe with an embedded video, a payment form or a chat widget moves when its provider changes it, and the guard watches every frame on its own. Leave a frame out by its address:
+
+```ts
+test.use({ layoutGuard: { ignoreFrames: ['widget.example', 'https://chat.example/embed'] } });
+```
+
+Each entry is text that the address of the frame (origin and path, without the query) contains; `*` stands for any characters. A frame inside a frame is matched by its own address. The box of the iframe in your page is still watched, so a widget that changes its height and pushes your page is found; add its selector to `ignore` to leave that out too.
+
### Adopting it in a project that moves today
A project that has shifts already cannot switch the guard on without a red test run. Record what is there, and fail only on what is new:
@@ -162,6 +174,36 @@ npx playwright test # from now on only new sh
An entry is the kind of shift, the address of the page and the element (`main > div.banner`), not how far it moved, so a known shift that gets worse is still known. Fix a shift, delete its line, and the guard holds the line for good. The message of a failing test says how many known shifts it left out, and `layout-shifts.json` marks each shift as `known`. The file is sorted and locked while it is written, so the update also works with several workers; `--workers=1` is only simpler to follow.
+### Command line
+
+No tests needed: point the guard at addresses and read what moved. This is how the [trial on public pages](docs/survey.md) was made.
+
+```sh
+npx playwright-layout-guard scan https://example.com
+npx playwright-layout-guard scan https://staging.example.com/ https://staging.example.com/pricing --device "iPhone 13"
+npx playwright-layout-guard scan --file pages.txt --format markdown --out layout-report.md --screenshots pictures
+```
+
+Every address is opened in a fresh browser, left alone for four seconds and scrolled once, without clicks. The report lists every box that moved after the load, how far and when, and next to it the layout shift score (CLS) of the browser itself where it has one (Chromium). It needs Playwright and its browser (`npx playwright install chromium`).
+
+| Option | What it does |
+| ----------------------- | -------------------------------------------------------------------------------------------------- |
+| `--file ` | More addresses, one per line (`#` starts a comment) |
+| `--browser ` | `chromium` (default), `firefox` or `webkit` |
+| `--format ` | `text` (default), `json`, `markdown` or `html` |
+| `--out ` | Write the report to a file instead of the console |
+| `--fail-on ` | `shift` (default) or `never`: when the exit code is 1 |
+| `--options ` | A JSON file with the [options](#options), for example `{ "ignore": [".ad-slot"] }` |
+| `--immediate` | Also report what moves while the page loads (`start: 'immediate'`) |
+| `--arm-when ` | For an app that draws itself after the load: report from the moment this element is shown |
+| `--screenshots ` | Save the picture with the moved boxes marked for every page with a shift |
+| `--wait ` | How long to leave a page alone after the load (default 4) |
+| `--no-scroll` | Do not scroll through the page |
+| `--timeout ` | How long a page may take to load (default 45) |
+| `--device ` | A [device of Playwright](https://playwright.dev/docs/emulation#devices), for example `"iPhone 13"` |
+
+The exit code is 0 when nothing moved (or with `--fail-on never`), 1 when something did, and 2 when the command was wrong, a page could not be loaded or the element of `--arm-when` never showed (nothing was watched). A name in the options file that the guard does not know is refused, so a typo does not silently switch something off.
+
### Without the fixture
```ts
@@ -174,6 +216,7 @@ await page.goto('/');
// guard.screenshot() a PNG with the moved boxes marked
// guard.assertStable() throws an error that names every shift
// guard.forget() start over from the current layout (after a client-side app rendered)
+// guard.armed() whether the guard is watching on every page (false until armWhen matches)
// guard.pause() / guard.resume()
```
@@ -183,9 +226,9 @@ Every failed test also gets the shifts attached as `layout-shifts.json`.
- **Catches** anything that changes its place or size in the layout: content pushed by a late banner, an image or an ad without reserved space, a font swap that re-wraps text, a button that moves its neighbours, a page that grows at the bottom.
- **Does not count** `transform` and `opacity` animations (they do not change the layout), scrolling, sticky and fixed parts, and everything that is not visible. [How it works](docs/how-it-works.md) has the full list and the reasons.
-- **A page that renders after the load event** (a client-side app) looks like a page that is changing. Wait for the app, then call `await guard.forget()`.
+- **A page that renders after the load event** (a client-side app) looks like a page that is changing. Name an element that is shown when the app is ready with `armWhen: '#app'`, or wait for the app and call `await guard.forget()`.
- **Loading is not judged by default.** Pages change a lot while they arrive, and the guard reports from the load event on. That is the job of CLS and Lighthouse. `start: 'immediate'` watches from the first frame, but then it also reports a page that is still being delivered in pieces.
-- **Web components and frames:** open shadow trees are looked into, and so are iframes (each frame is watched on its own). Closed shadow trees cannot be seen from outside.
+- **Web components and frames:** open shadow trees are looked into, and so are iframes (each frame is watched on its own, and a widget you do not control can be left out with `ignoreFrames`). Closed shadow trees cannot be seen from outside.
- **It reads the layout, not the picture.** Something can change its look (colour, text) without moving: that is not a layout shift.
## How well does it work on real pages?
@@ -196,10 +239,8 @@ The package was tried on 23 public pages (documentation sites, news sites, frame
Not promises, and in no particular order. If one of them would help you, [open an issue](https://github.com/leonkrix/playwright-layout-guard/issues) and say what you test.
-- **Leave out a single frame** (`ignoreFrames`), for embedded widgets you do not control.
-- **Arm on a condition** (`armWhen: 'main'`) instead of calling `guard.forget()`, for client-side rendered apps.
- **A viewport-only mode**, to count only what a visitor could see, like CLS does.
-- **A command line tool** to point the guard at a URL (`npx`), which is what the survey script does by hand.
+- **The command line**: several addresses at once, and the addresses from a sitemap.
- **Other test runners** (Puppeteer, Cypress, WebdriverIO): the part that runs in the page does not depend on Playwright.
More recipes (cookie banners, carousels, client-side apps, several pages in one test) are in [docs/recipes.md](docs/recipes.md).
diff --git a/docs/recipes.md b/docs/recipes.md
index 4c6fae8..64c17aa 100644
--- a/docs/recipes.md
+++ b/docs/recipes.md
@@ -63,6 +63,14 @@ test.use({ layoutGuard: { ignore: ['.ad-slot', '.live-ticker', '[data-carousel]'
The selectors are CSS selectors; the element and everything inside it is left alone. Keep the list short and say why in a comment: every entry is a place where the guard is blind.
+An embedded widget in an iframe has a page of its own that the guard watches too. Leave it out by its address with `ignoreFrames`:
+
+```ts
+test.use({ layoutGuard: { ignoreFrames: ['widget.example'] } });
+```
+
+The iframe box in your page is still watched (it can push your content), the page inside it is not.
+
## An app that renders after the load
A client-side rendered app is an empty page at the load event and fills in afterwards, which looks like a page that moves. Wait until the app has rendered, then tell the guard to start from there:
@@ -76,6 +84,14 @@ test('the dashboard', async ({ page, guard }) => {
});
```
+Or let the guard wait for the app by itself: name something that is shown when it is ready.
+
+```ts
+test.use({ layoutGuard: { armWhen: '#app' } });
+```
+
+The guard reports nothing until the element is shown, takes the layout then, and reports from there on. If the element never shows (a wrong selector, an app that failed to start), the test fails and says so, because nothing was watched.
+
Server-rendered pages need none of this. A framework that renders on the server and then **hydrates** is what the guard is best at: the page must look the same before and after.
## Several pages in one test
diff --git a/docs/survey.md b/docs/survey.md
index a50e911..9fd6e46 100644
--- a/docs/survey.md
+++ b/docs/survey.md
@@ -18,7 +18,7 @@ What this says:
- **`immediate` finds most of what CLS finds** (8 of 9), but 30 of 46 loads report something and 16 hit the limit: a page that is still being delivered in pieces does move, and the guard says so. Use it for your own pages when the load itself is the thing to check, not as a default.
- **The default mode is quiet on most real pages** (38 of 46 loads without a report). What it reports was checked by hand:
- **Real layout changes after the load:** web.dev (a footer pushed 78 px down), nodejs.org (the navigation settles 4 px at a time), bbc.com (a few pixels when a script finishes) and the Vue guide (the whole app moves 72 px up when a bar above it goes away). Some pages show up in one run and not in the next (MDN on a phone: a sidebar appears at 1 s and pushes the whole article 392 px down, with a CLS of 0 because the article was below the window), because what a page does after the load depends on timing.
- - **Small frames:** react.dev embeds frames that are 15 px high and grow to 20 px. They are reported with the name of the frame (`[frame /path]`). There is no option yet to leave out a single frame (listed as an idea in the README).
+ - **Small frames:** react.dev embeds frames that are 15 px high and grow to 20 px. They are reported with the name of the frame (`[frame /path]`). Since 0.4 a frame can be left out with `ignoreFrames`.
## Pages with a report or a poor CLS
diff --git a/package.json b/package.json
index 9088286..b0e2d41 100644
--- a/package.json
+++ b/package.json
@@ -31,6 +31,9 @@
"sideEffects": false,
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
+ "bin": {
+ "playwright-layout-guard": "./dist/bin.js"
+ },
"exports": {
".": {
"types": "./dist/index.d.ts",
diff --git a/src/bin.ts b/src/bin.ts
new file mode 100644
index 0000000..25a1191
--- /dev/null
+++ b/src/bin.ts
@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { main } from './cli.js';
+
+process.exitCode = await main(process.argv.slice(2));
diff --git a/src/cli.test.ts b/src/cli.test.ts
new file mode 100644
index 0000000..310cf5c
--- /dev/null
+++ b/src/cli.test.ts
@@ -0,0 +1,214 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+ exitCodeOf,
+ fileNameOf,
+ mergeOptions,
+ normalizeUrl,
+ type PageResult,
+ parseCli,
+ parseUrlList,
+ renderReport,
+ renderText,
+} from './cli.js';
+import type { Shift } from './types.js';
+
+const shift = (element: string, extra: Partial = {}): Shift => ({
+ kind: 'moved',
+ element,
+ text: 'Hello',
+ before: { top: 100, left: 0, width: 600, height: 40 },
+ after: { top: 160, left: 0, width: 600, height: 40 },
+ affected: 2,
+ at: 1200,
+ url: '/',
+ ...extra,
+});
+
+const calm: PageResult = { url: 'https://a.example/', http: 200, cls: 0, armed: true, shifts: [] };
+const moving: PageResult = {
+ url: 'https://b.example/',
+ finalUrl: 'https://b.example/home',
+ http: 200,
+ cls: 0.1281,
+ armed: true,
+ shifts: [shift('main'), shift('footer', { affected: 0 })],
+ screenshot: 'pictures/02-b-example.png',
+};
+const broken: PageResult = {
+ url: 'https://c.example/',
+ error: 'page.goto: Timeout 45000ms exceeded.',
+};
+const waiting: PageResult = { url: 'https://d.example/', http: 200, armed: false, shifts: [] };
+
+describe('parseCli', () => {
+ it('reads addresses and options', () => {
+ const parsed = parseCli([
+ 'scan',
+ 'example.com',
+ 'http://localhost:3000/a',
+ '--browser',
+ 'webkit',
+ '--format=json',
+ '--fail-on',
+ 'never',
+ '--wait',
+ '1.5',
+ '--timeout',
+ '10',
+ '--no-scroll',
+ '--immediate',
+ '--arm-when',
+ '#app',
+ '--out',
+ 'report/layout.json',
+ '--options',
+ 'guard.json',
+ '--device',
+ 'iPhone 13',
+ '--file',
+ 'urls.txt',
+ '--screenshots',
+ 'pictures',
+ ]);
+ expect(parsed).toEqual({
+ kind: 'scan',
+ scan: {
+ urls: ['https://example.com/', 'http://localhost:3000/a'],
+ file: 'urls.txt',
+ browser: 'webkit',
+ format: 'json',
+ out: 'report/layout.json',
+ failOn: 'never',
+ wait: 1.5,
+ scroll: false,
+ timeout: 10_000,
+ immediate: true,
+ armWhen: '#app',
+ optionsFile: 'guard.json',
+ device: 'iPhone 13',
+ screenshots: 'pictures',
+ },
+ });
+ });
+
+ it('has defaults', () => {
+ expect(parseCli(['scan', 'https://example.com'])).toMatchObject({
+ kind: 'scan',
+ scan: { browser: 'chromium', format: 'text', failOn: 'shift', wait: 4, scroll: true },
+ });
+ });
+
+ it('answers help, version and no command', () => {
+ expect(parseCli(['--help'])).toEqual({ kind: 'help' });
+ expect(parseCli(['-h'])).toEqual({ kind: 'help' });
+ expect(parseCli([])).toEqual({ kind: 'help' });
+ expect(parseCli(['--version'])).toEqual({ kind: 'version' });
+ });
+
+ it('says what is wrong', () => {
+ const message = (args: string[]): string => {
+ const parsed = parseCli(args);
+ return parsed.kind === 'error' ? parsed.message : parsed.kind;
+ };
+ expect(message(['crawl', 'example.com'])).toContain('unknown command "crawl"');
+ expect(message(['scan', 'example.com', '--browser', 'edge'])).toContain(
+ '--browser is one of chromium, firefox, webkit',
+ );
+ expect(message(['scan', 'example.com', '--format', 'xml'])).toContain('--format');
+ expect(message(['scan', 'example.com', '--fail-on', 'error'])).toContain('--fail-on');
+ expect(message(['scan', 'example.com', '--wait', 'soon'])).toContain('--wait needs a number');
+ expect(message(['scan', 'ftp://example.com'])).toContain('not an http or https address');
+ expect(message(['scan', 'example.com', '--nope'])).toContain('--nope');
+ });
+});
+
+describe('addresses', () => {
+ it('adds https when there is no scheme', () => {
+ expect(normalizeUrl('example.com/a')).toBe('https://example.com/a');
+ expect(normalizeUrl('file:///etc/passwd')).toBeUndefined();
+ });
+
+ it('reads a list: one per line, comments and empty lines left out', () => {
+ expect(parseUrlList('# pages\nexample.com\n\nexample.org/a # second\r\n#last\n')).toEqual([
+ 'example.com',
+ 'example.org/a',
+ ]);
+ });
+
+ it('makes a file name from an address', () => {
+ expect(fileNameOf('https://www.example.com/docs/intro?a=1', 1)).toBe(
+ '02-www-example-com-docs-intro.png',
+ );
+ });
+});
+
+describe('options', () => {
+ it('puts the flags of the command line over the file', () => {
+ expect(
+ mergeOptions({ ignore: ['.ad'], start: 'load' }, { immediate: true, armWhen: '#app' }),
+ ).toEqual({ ignore: ['.ad'], start: 'immediate', armWhen: '#app' });
+ expect(mergeOptions({ start: 'immediate' }, { immediate: false, armWhen: undefined })).toEqual({
+ start: 'immediate',
+ });
+ });
+});
+
+describe('exit code', () => {
+ it('is 0 for calm pages and 1 for a shift', () => {
+ expect(exitCodeOf([calm], 'shift')).toBe(0);
+ expect(exitCodeOf([calm, moving], 'shift')).toBe(1);
+ expect(exitCodeOf([moving], 'never')).toBe(0);
+ });
+
+ it('is 2 when a page could not be loaded or the guard never started to watch', () => {
+ expect(exitCodeOf([broken], 'never')).toBe(2);
+ expect(exitCodeOf([moving, broken], 'shift')).toBe(2);
+ expect(exitCodeOf([waiting], 'shift')).toBe(2);
+ });
+});
+
+describe('the reports', () => {
+ it('writes text with a block per page and the score of the browser', () => {
+ const text = renderText([calm, moving, broken, waiting]);
+ expect(text).toContain('https://a.example/ (HTTP 200, CLS 0)\n nothing moved');
+ expect(text).toContain(
+ 'https://b.example/ (now https://b.example/home) (HTTP 200, CLS 0.1281)',
+ );
+ expect(text).toContain('2 shifts:');
+ expect(text).toContain('1. "Hello" moved 60 px down, with 2 elements inside it');
+ expect(text).toContain('picture: pictures/02-b-example.png');
+ expect(text).toContain('could not be loaded: page.goto: Timeout');
+ expect(text).toContain('never started to watch');
+ expect(text).toContain('4 pages: 1 page with shifts (2 shifts), 1 could not be loaded');
+ });
+
+ it('writes JSON that can be read back', () => {
+ const json = JSON.parse(renderReport([calm, moving, broken], 'json', 'webkit', 'then')) as {
+ version: number;
+ browser: string;
+ pages: PageResult[];
+ };
+ expect(json).toMatchObject({ version: 1, browser: 'webkit' });
+ expect(json.pages[1]?.shifts).toHaveLength(2);
+ expect(json.pages[2]?.error).toContain('Timeout');
+ });
+
+ it('writes a Markdown table with the biggest move of each page', () => {
+ const markdown = renderReport([calm, moving, broken], 'markdown', 'chromium');
+ expect(markdown).toContain('## Layout guard');
+ expect(markdown).toContain('| https://b.example/ | 2 | 0.1281 | ');
+ expect(markdown).toContain('| https://c.example/ | could not be loaded');
+ });
+
+ it('writes an HTML page that escapes what it shows', () => {
+ const html = renderReport(
+ [{ ...moving, shifts: [shift('div.
+