Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,7 @@ jobs:
run: >-
node scripts/run-browser-tests.mjs
src/content/__tests__/record-input-buttons.browser.test.ts
src/content/__tests__/record-editor-value.browser.test.ts
src/tools/__tests__/focus.browser.test.ts
src/tools/__tests__/scroll.browser.test.ts
src/tools/__tests__/select.browser.test.ts
Expand Down
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ determines which versions can connect.
rules to visible non-sensitive `<input>` and `<textarea>` values while preserving browser
UA `text-transform: none` defaults and byte-budget accounting (follow-up to
[#613](https://github.com/Tencent/BrowserSkill/pull/613)).
- Recording a contenteditable editor keeps the line breaks the person typed, so a
multi-line edit replays with its lines instead of collapsing into one
([#594](https://github.com/Tencent/BrowserSkill/issues/594)).

### Removed

Expand Down
2 changes: 2 additions & 0 deletions CHANGELOG.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ CLI 和扩展可以分别发布,能否连接由协议兼容性决定。更新
- VOM 树折叠规则(`shouldSkipRedundantRefChildren` 与 `renderTreeRows`)识别具名交互父节点下独立可交互的后代(如 Vue 3 + Element Plus 将二级浮层嵌在父 `menuitem` 内的多级 `<el-sub-menu>` 或自定义 ARIA `combobox` 弹出项),使每个已展开、可操作的子项在 `bsk sync`、`observe` 和 `snapshot` 中均输出独立行与专属 `@eN` 引用,同时保持被动内部包装节点、继承 `cursor: pointer` / 标题皮肤点击节点及原生 `<input>` / `<textarea>` / `<select>` 的折叠行为不变而不产生重复伪引用,并新增发版前必测用例 `EDGE-NESTED-MENU-REFS`。
- 结构化列表提取(`extract list`)支持通过 open Shadow DOM `<slot>` 投影分配的列表项及默认链接(`url`),保留插槽渲染顺序、闭合 `<details>` 隐藏行位置、嵌套子列表项与链接归属、可见性判断及采集限制([#588](https://github.com/Tencent/BrowserSkill/issues/588)、[#612](https://github.com/Tencent/BrowserSkill/pull/612)),并新增发版前必测用例 `EDGE-SLOTTED-LIST-EXTRACT`。
- 修复结构化提取(`extract table`、`extract list` 与 `extract discover`)未对可见非敏感 `<input>` 和 `<textarea>` 值应用计算后的 CSS `text-transform`(`uppercase` 与 `lowercase`)及渲染语言环境大小写规则的问题,同时保留浏览器默认样式表的 `text-transform: none` 行为与字节预算统计([#613](https://github.com/Tencent/BrowserSkill/pull/613) 后续完善)。
- 录制 contenteditable 编辑器时保留用户输入的换行,多行编辑按原样重放,不再被合并成一行
([#594](https://github.com/Tencent/BrowserSkill/issues/594))。

### 移除

Expand Down
89 changes: 89 additions & 0 deletions apps/extension/src/content/__tests__/record-capture.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1749,3 +1749,92 @@ describe("record-capture semantic", () => {
}
});
});

describe("record-capture editor values", () => {
let steps: RecordStepPayload[];

// Earlier cases replace window.getComputedStyle to fake hover styles; the
// white-space mode these cases depend on must come from the real cascade.
beforeEach(() => {
vi.restoreAllMocks();
});

/**
* Types into an editor that already holds `html`. Chrome stores the lines a
* person types as <div> blocks and <br> elements exactly as spelled here; the
* browser cases in `record-editor-value.browser.test.ts` confirm it.
*/
function recordEditor(html: string, edited?: string): RecordStepPayload[] {
steps = [];
document.body.innerHTML = `<div id="editor" contenteditable="true" aria-label="Editor">${html}</div>`;
const editor = document.querySelector<HTMLElement>("#editor")!;
// happy-dom does not derive editability from the contenteditable attribute.
Object.defineProperty(editor, "isContentEditable", { value: true });
const capture = startRecordCapture("rec-editor-value", (step) => steps.push(step), {
captureNavigation: false,
});

editor.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
if (edited !== undefined) {
editor.innerHTML = edited;
editor.dispatchEvent(new Event("input", { bubbles: true }));
}
capture.dispose();
return steps;
}

const value = () => steps.find((step) => step.op === "fill")?.value;

it.each([
["a typed second line", "Alpha<div>Beta</div>", "Alpha\nBeta"],
["a Shift+Enter line", "Alpha<br>Beta", "Alpha\nBeta"],
["an empty line between two lines", "Alpha<div><br></div><div>Beta</div>", "Alpha\n\nBeta"],
["a trailing empty line", "Alpha<div><br></div>", "Alpha\n"],
["a single typed line", "Alpha", "Alpha"],
["rich structure kept as rendered text", "<p>One</p><p>Two</p>", "One\nTwo"],
])("records the value with %s", (_name, html, expected) => {
recordEditor("", html);

expect(value()).toBe(expected);
});

it("keeps whitespace that a pre-wrap editor displays", () => {
document.body.innerHTML = `<div id="editor" contenteditable="true" aria-label="Editor" style="white-space: pre-wrap"></div>`;
const editor = document.querySelector<HTMLElement>("#editor")!;
Object.defineProperty(editor, "isContentEditable", { value: true });
steps = [];
const capture = startRecordCapture("rec-editor-pre", (step) => steps.push(step), {
captureNavigation: false,
});

editor.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
editor.innerHTML = " Alpha ";
editor.dispatchEvent(new Event("input", { bubbles: true }));
capture.dispose();

expect(value()).toBe(" Alpha ");
});

it("trims line edges and reads NBSP as a space where white-space collapses", () => {
document.body.innerHTML = `<div id="editor" contenteditable="true" aria-label="Editor" style="white-space: normal"></div>`;
const editor = document.querySelector<HTMLElement>("#editor")!;
Object.defineProperty(editor, "isContentEditable", { value: true });
steps = [];
const capture = startRecordCapture("rec-editor-collapse", (step) => steps.push(step), {
captureNavigation: false,
});

editor.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
editor.innerHTML = " Alpha\u00a0Beta ";
editor.dispatchEvent(new Event("input", { bubbles: true }));
capture.dispose();

expect(value()).toBe("Alpha Beta");
});

it("records nothing for an editor that only holds an empty line", () => {
recordEditor("<p><br></p>", "<p><br></p>");

expect(steps).toEqual([]);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
// @vitest-environment node
// Opt in with BSK_TEST_CHROME. Uses real input in its own browser/profile.

import { fileURLToPath } from "node:url";
import { build } from "esbuild";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import type { RecordStepPayload } from "@/lib/record-bridge";

type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;

let send: Send;
let bundle: string;
let close: (() => void) | undefined;
let browserRun: Promise<void> | undefined;

if (process.env.BSK_TEST_CHROME) {
beforeAll(async () => {
const built = await build({
entryPoints: [fileURLToPath(new URL("../record-capture.ts", import.meta.url))],
bundle: true,
write: false,
format: "iife",
globalName: "Recording",
platform: "browser",
});
bundle = built.outputFiles[0].text;
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
const ready = Promise.withResolvers<Send>();
const closed = Promise.withResolvers<void>();
close = () => closed.resolve();
browserRun = withChrome(
{ executable: process.env.BSK_TEST_CHROME, deviceScale: 1, zoom: 1 },
async (command: Send) => {
ready.resolve(command);
await closed.promise;
},
);
browserRun!.catch(ready.reject);
send = await ready.promise;
}, 30_000);
afterAll(async () => {
close?.();
await browserRun;
}, 30_000);
}

interface Helpers {
click: (selector: string) => Promise<void>;
insert: (text: string) => Promise<void>;
enter: (shift?: boolean) => Promise<void>;
}

/** Records one editing session on a fresh tab and returns its emitted steps. */
async function recordEditor(
markup: string,
edit: (helpers: Helpers) => Promise<void>,
): Promise<{ steps: RecordStepPayload[]; html: string; inner: string }> {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
try {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
const evaluate = async <T>(expression: string): Promise<T> => {
const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{ expression, returnByValue: true },
sessionId,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
const click = async (selector: string) => {
const point = await evaluate<{ x: number; y: number }>(`(() => {
const r = document.querySelector(${JSON.stringify(selector)}).getBoundingClientRect();
return {x:r.x+r.width/2,y:r.y+r.height/2};
})()`);
for (const type of ["mousePressed", "mouseReleased"]) {
await send(
"Input.dispatchMouseEvent",
{ type, ...point, button: "left", clickCount: 1 },
sessionId,
);
}
};
const insert = async (text: string) => {
await send("Input.insertText", { text }, sessionId);
};
const enter = async (shift = false) => {
const key = {
key: "Enter",
code: "Enter",
windowsVirtualKeyCode: 13,
nativeVirtualKeyCode: 13,
};
const modifiers = shift ? 8 : 0;
await send(
"Input.dispatchKeyEvent",
{ type: "keyDown", ...key, modifiers, text: "\r" },
sessionId,
);
await send("Input.dispatchKeyEvent", { type: "keyUp", ...key, modifiers }, sessionId);
};
await evaluate(`document.body.innerHTML = ${JSON.stringify(markup)};
${bundle}
window.steps = [];
window.capture = Recording.startRecordCapture('editor', step => window.steps.push(step), {captureNavigation:false});`);
await edit({ click, insert, enter });
return await evaluate(
`window.capture.dispose(); ({steps:window.steps,
html:document.querySelector('#editor').innerHTML,
inner:document.querySelector('#editor').innerText})`,
);
} finally {
await send("Target.closeTarget", { targetId });
}
}

const editor = '<div id="editor" contenteditable="true" aria-label="Editor"></div>';

/** The last fill step, which carries the value at the end of the session. */
function finalValue(steps: RecordStepPayload[]) {
return steps.filter((step) => step.op === "fill").at(-1)?.value;
}

describe.skipIf(!process.env.BSK_TEST_CHROME)("record contenteditable values", () => {
it("keeps the line break that Enter inserts between two typed lines", async () => {
const result = await recordEditor(editor, async (h) => {
await h.click("#editor");
await h.insert("Alpha");
await h.enter();
await h.insert("Beta");
});

expect(result.html).toBe("Alpha<div>Beta</div>");
expect(result.steps.map((step) => step.op)).toEqual(["fill", "press", "fill"]);
expect(result.steps[0]).toMatchObject({ target: { name: "Editor" }, value: "Alpha" });
expect(finalValue(result.steps)).toBe("Alpha\nBeta");
});

it("keeps the line break that Shift+Enter inserts", async () => {
const result = await recordEditor(editor, async (h) => {
await h.click("#editor");
await h.insert("Alpha");
await h.enter(true);
await h.insert("Beta");
});

expect(finalValue(result.steps)).toBe("Alpha\nBeta");
});

it("keeps an empty line between two typed lines", async () => {
const result = await recordEditor(editor, async (h) => {
await h.click("#editor");
await h.insert("Alpha");
await h.enter();
await h.enter();
await h.insert("Beta");
});

expect(finalValue(result.steps)).toBe("Alpha\n\nBeta");
});

it("keeps a trailing empty line and the whitespace a pre-wrap editor displays", async () => {
const result = await recordEditor(
'<div id="editor" contenteditable="true" aria-label="Editor" style="white-space: pre-wrap"></div>',
async (h) => {
await h.click("#editor");
await h.insert(" Alpha ");
await h.enter();
},
);

expect(finalValue(result.steps)).toBe(" Alpha \n");
});

it("reads a rich editor as rendered text", async () => {
const result = await recordEditor(
'<div id="editor" contenteditable="true" aria-label="Editor"><p>One</p><p>Two</p></div>',
async (h) => {
await h.click("#editor p:last-child");
await h.insert("!");
},
);

// Paragraphs render with a margin, so the rendered text separates them twice.
expect(result.inner).toBe("One\n\nTwo!");
expect(finalValue(result.steps)).toBe(result.inner);
});

it("keeps a typed edge space as a space instead of the non-breaking space Chrome types", async () => {
const result = await recordEditor(editor, async (h) => {
await h.click("#editor");
await h.insert(" Alpha ");
});

// Chrome types a space at a line edge as U+00A0; the trace must not carry it.
expect(result.html).toContain("&nbsp;");
expect(finalValue(result.steps)).toBe(" Alpha ");
expect([...finalValue(result.steps)!].some((char) => char.codePointAt(0) === 0xa0)).toBe(false);
});
});
49 changes: 48 additions & 1 deletion apps/extension/src/content/record-capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,58 @@ function geometryForEventTarget(
return captureGeometry(clickable ?? target);
}

// Typing in an editor stores lines as <div> blocks and <br> elements, which
// `textContent` joins without a separator, so a recorded value would read
// "AlphaBeta" for two lines. Read the value exactly the way `bsk fill` verifies
// it (`FILL_VALUE_FUNCTION` in `tools/interaction.ts`): typed layouts are read
// literally, other rich structures keep their rendered text.
function editorValue(el: HTMLElement): string {
const collapses = ["normal", "nowrap"].includes(getComputedStyle(el).whiteSpace);
const isInline = (node: Node) => node.nodeType === Node.TEXT_NODE || node.nodeName === "BR";
const isLine = (node: Node) =>
node.nodeName === "DIV" &&
getComputedStyle(node as Element).display === "block" &&
[...node.childNodes].every(isInline);
const nodes = [...el.childNodes];
let value: string;
if (nodes.every((node) => isInline(node) || isLine(node))) {
const lines: Node[][] = [];
let open: Node[] | undefined;
for (const node of nodes) {
if (isLine(node)) {
lines.push([...node.childNodes]);
open = undefined;
} else {
if (!open) lines.push((open = []));
open.push(node);
}
}
value = lines
.flatMap((line) => {
let text = line
.map((node, index) => {
if (node.nodeName !== "BR") return (node as CharacterData).data;
return index < line.length - 1 ? "\n" : "";
})
.join("");
if (collapses) text = text.replace(/^[ \t\r\n]+|[ \t\r\n]+$/g, "");
// Without a <br>, an empty line has no height.
return text === "" && !line.some((node) => node.nodeName === "BR") ? [] : [text];
})
.join("\n");
} else {
// An editor holding only an empty line, such as <p><br></p>, is empty.
const rendered = el.innerText;
value = rendered === "\n" && el.textContent === "" ? "" : rendered;
}
return collapses ? value.replace(/\u00a0/g, " ") : value;
}

function fillableValue(el: FillableElement): string {
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
return el.value;
}
return el.textContent ?? "";
return editorValue(el);
}

function hoverTriggerAttrs(el: Element): Record<string, string | undefined> {
Expand Down