From b6010d80420e7a3c1aea44e644f8e8e139055d86 Mon Sep 17 00:00:00 2001 From: "Alexandro T. Netto" Date: Fri, 16 Oct 2026 17:01:06 -0300 Subject: [PATCH 1/2] Fix the skill gaps found by the Pipedrive study (0.3.0) - pack-kb keeps sandbox/ledger.csv in the zip without object IDs, locations or URLs, so an unzipped handoff passes check-kb --closing and L-### references resolve. Test unzips the handoff and checks it. - Baseline counts cover only the container and the operator's own records, by filter or count endpoint; teardown compares those. - L3: item-level (operator-only) visibility when the container cannot be private; rule for unattended sessions that start during the team's working hours. - Stop and skip: deleting your own ledger items inside the sandbox is exempt from the "cannot be undone" stop. - mask.js: selectors become persistent CSS rules that cover elements rendered later; matching placeholders are masked, placeholders: true masks all; invalid selectors are reported. New Masking section. - tools.md: Claude in Chrome notes (long results, network list, local servers, background clicks, where save_to_disk screenshots land). --- .claude-plugin/marketplace.json | 2 +- CHANGELOG.md | 11 +++ README.md | 4 +- docs/testing.md | 4 +- package-lock.json | 4 +- package.json | 2 +- skills/reverse-engineering-saas/SKILL.md | 12 ++- .../references/extraction.md | 14 ++- .../references/knowledge-base.md | 1 + .../references/safety.md | 15 ++-- .../references/tools.md | 18 +++- .../scripts/pack-kb.mjs | 56 +++++++++++- .../scripts/page/mask.js | 87 +++++++++++++------ .../templates/kb/sandbox/environment.md | 8 +- tests/kb-scripts.test.mjs | 24 ++++- tests/page-scripts.test.mjs | 37 ++++++++ 16 files changed, 244 insertions(+), 55 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 9137a74..65cf7f6 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -15,7 +15,7 @@ "skills": [ "./skills/reverse-engineering-saas" ], - "version": "0.2.0" + "version": "0.3.0" } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 022455f..8f7a87a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 0.3.0 (2026-10-16) + +Fixes for the gaps found by the first real study (Pipedrive, L3, `examples/pipedrive/`). + +- Handoff: `pack-kb.mjs` keeps `sandbox/ledger.csv` in the zip with object IDs and locations blanked and URLs replaced by `[url]`. An unzipped handoff now passes `check-kb --closing`, and `L-###` references in the KB resolve. +- Baseline counts cover only the container and the operator's records since the first ledger row, by filter or count endpoint, never by pulling full record lists. Teardown compares those counts. `environment.md` records the filter. +- L3: when the container cannot be private, set each record's visibility to the operator only. An unattended session that starts during the team's working hours writes only when the team cannot see the records; otherwise it does the phases without writes first. +- "Stop and skip": deleting your own ledger items inside the sandbox is exempt from the "cannot be undone" stop when the dialog names only those items. +- `mask.js`: selectors become CSS rules that also blur elements rendered later (popovers, dropdowns), until `clear()`. Placeholders that match a pattern are blurred, and `placeholders: true` blurs all of them. Invalid selectors are reported. New "Masking" section in `extraction.md`. +- `tools.md`: Claude in Chrome notes. Long `javascript_tool` results are read through an offscreen `
` and `get_page_text`. Use `api-shapes.js` when `read_network_requests` misses app calls. Never load scripts from a local server in the user's tab. Check state after each click in a background window. Where `save_to_disk` screenshots land. + ## 0.2.0 (2026-10-16) - Handoff: `pack-kb.mjs` packs the knowledge base into a zip with `BUILD.md`, a build brief for any coding agent. It checks the KB first, leaves out `sandbox/` and `log/` unless `--full`, and needs no dependencies. diff --git a/README.md b/README.md index 598d98c..f78a249 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ Yes. Tell it not to ask you anything. It records what needs you under "Needs hum
How do I give the result to my developers? -Ask for the handoff, or wait for the end of the study. The agent sends a zip with the knowledge base and `BUILD.md`. Account details (`sandbox/`, `log/`) stay out of the zip. +Ask for the handoff, or wait for the end of the study. The agent sends a zip with the knowledge base and `BUILD.md`. Account details (`sandbox/`, `log/`) stay out of the zip, except a copy of the change ledger without object IDs, locations or URLs.
@@ -249,7 +249,7 @@ That depends on the product's terms and your jurisdiction. RES records the relev ## Status -Version 0.2.0. The method was tested with scenario runs on subagents, before and after each change to the skill, and the scripts with automated tests on Node.js 22 and 24. It has not yet been run end to end on a real product. Example studies of ClickUp and Pipedrive are planned. Reports from real runs are welcome: open a [study report](https://github.com/alextnetto/res/issues/new?template=study_report.yml). +Version 0.3.0. The method was tested with scenario runs on subagents, before and after each change to the skill, and the scripts with automated tests on Node.js 22 and 24. It has not yet been run end to end on a real product. Example studies of ClickUp and Pipedrive are planned. Reports from real runs are welcome: open a [study report](https://github.com/alextnetto/res/issues/new?template=study_report.yml). ## Responsible use diff --git a/docs/testing.md b/docs/testing.md index 5fe82e0..ee503ce 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -16,8 +16,8 @@ npm test | File | What it checks | |---|---| -| `tests/page-scripts.test.mjs` | Runs the five page scripts in headless Chromium against `tests/fixture.html` with mocked API and WebSocket traffic. Asserts component tags, routes, token-to-variable mapping, endpoint templates, JSON shapes, GraphQL operations, WebSocket frames, masking, annotation, and that the fake names, emails, IDs and storage values on the test page do not appear in the results | -| `tests/kb-scripts.test.mjs` | Creates a knowledge base with `init-kb.mjs` and checks the main `check-kb.mjs` rules: unresolved references, rules and barriers without evidence, malformed tags, placeholder IDs, duplicates, broken links, `[inf]` without a question, missing "Not tested" sections, ledger validation, `fixture` and `human` rows, `--closing`, evidence index, entity rows, coverage rows and counts, `--next`, `--help`, the default KB directory, and `--git` with and without a git identity, and `pack-kb.mjs` (zip validated with Python's `zipfile`, private folders left out, `--full`, `--force`) | +| `tests/page-scripts.test.mjs` | Runs the five page scripts in headless Chromium against `tests/fixture.html` with mocked API and WebSocket traffic. Asserts component tags, routes, token-to-variable mapping, endpoint templates, JSON shapes, GraphQL operations, WebSocket frames, masking (including selector rules for elements rendered later, and placeholders), annotation, and that the fake names, emails, IDs and storage values on the test page do not appear in the results | +| `tests/kb-scripts.test.mjs` | Creates a knowledge base with `init-kb.mjs` and checks the main `check-kb.mjs` rules: unresolved references, rules and barriers without evidence, malformed tags, placeholder IDs, duplicates, broken links, `[inf]` without a question, missing "Not tested" sections, ledger validation, `fixture` and `human` rows, `--closing`, evidence index, entity rows, coverage rows and counts, `--next`, `--help`, the default KB directory, and `--git` with and without a git identity, and `pack-kb.mjs` (zip validated with Python's `zipfile`, private folders left out, the ledger kept without IDs, locations or URLs, the unzipped handoff passing `check-kb --closing`, `--full`, `--force`) | To reuse an existing Playwright install instead of `npm install`, set `PLAYWRIGHT_MODULE=/path/to/node_modules/playwright`. diff --git a/package-lock.json b/package-lock.json index 17d3526..8e9ed30 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "res", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "res", - "version": "0.2.0", + "version": "0.3.0", "license": "MIT", "devDependencies": { "playwright": "1.64.0" diff --git a/package.json b/package.json index 55b64bc..40c3be4 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "engines": { "node": ">=22" }, - "version": "0.2.0", + "version": "0.3.0", "license": "MIT", "repository": { "type": "git", diff --git a/skills/reverse-engineering-saas/SKILL.md b/skills/reverse-engineering-saas/SKILL.md index 4a151d1..37ea59e 100644 --- a/skills/reverse-engineering-saas/SKILL.md +++ b/skills/reverse-engineering-saas/SKILL.md @@ -58,6 +58,8 @@ When the user asked for a test environment and no L1 exists: create L2 if free, At L3, "inside the sandbox" means the container plus records the agent created with the `[RE]` prefix that link only to sandbox records (a person created from a sandbox deal). When other members can see the container, remove all A2 records, the container included, at the end of every session, and in unattended mode finish the teardown before the team's working hours. +An unattended L3 session that starts during the team's working hours (from the charter; otherwise 08:00 to 18:00 on weekdays in the account's time zone) writes only when the team cannot see the records: a private container, or records set to operator-only visibility (`references/safety.md`, L3). The container's name may stay visible. Otherwise do the phases without writes first (recon, map) and put the writes under "Next actions" for a session outside working hours. In both cases keep the sandbox up as briefly as possible and tear it down in the same session. Record the start time and the choice in `environment.md`. + ### Action classes and what each level allows | Class | Examples | @@ -109,20 +111,22 @@ Check persistence by reloading. Never experiment on shared views: rebuild the ca Do not click through. Log the item as `blocked` and move on when you see: billing, upgrade or trial dialogs; any "send", "invite", "notify", "share", "apply to all" or "cannot be undone"; login, 2FA or CAPTCHA prompts; a location mismatch; any change that appears outside the sandbox. +Exception: deleting your own ledger items inside the sandbox. Their delete dialogs often say the action cannot be undone. Confirm when the dialog names only those items, sends or notifies nothing, and the write gate passes. A dialog that would also delete or change anything else is still a stop. + In unattended mode never ask the user. Write the item under "Needs human" in `STATUS.md` and continue. "Don't ask me anything" changes how blocks are reported. It does not widen what you may touch. After a mistake: undo it at once if the undo itself reaches nobody, then add a ledger row with class `incident`, what happened and what remains. If the undo would notify someone again (unassigning, deleting a comment), leave it and write it under "Needs human". Procedure: `references/safety.md`. ### Preflight: before the first write at L2 or L3 -Record in `environment.md`: members who could be notified; active automations, webhooks, integrations, email and calendar sync in the write scope; plan and trial end date; baseline counts of the main record types. If any automation, webhook, integration or sync would fire on records in the write scope, do not write there. Checklist: `references/safety.md`. +Record in `environment.md`: members who could be notified; active automations, webhooks, integrations, email and calendar sync in the write scope; plan and trial end date; baseline counts of the container and of the operator's own records, by filter or count endpoint, never by pulling full record lists. If any automation, webhook, integration or sync would fire on records in the write scope, do not write there. Checklist: `references/safety.md`. ### Data rules - Fake data only. Prefix every created name with `[RE]`. Emails at `example.com`. No real customer or teammate names. - KB text never contains real personal or customer data. When documenting production configuration (field options, automation conditions, filters), replace real values with placeholders such as `[customer]` or `[person]`. - Assign records to the operator or to nobody. -- At L3 and L4, run `scripts/page/mask.js` before each screenshot that can show real data. A screenshot that shows unmasked personal data is deleted at once and retaken; it never enters the KB. +- At L3 and L4, run `scripts/page/mask.js` before each screenshot that can show real data. Pass the containers of popovers, menus and lists as `selectors`: they stay masked when they render later. Company configuration (type names, reasons, field labels) matches no pattern: pass it as `names` or crop it out. A screenshot that shows unmasked personal data is deleted at once and retaken; it never enters the KB. Procedures for each level, the ledger format, teardown and incidents: `references/safety.md`. @@ -172,7 +176,7 @@ Paste the file content into the browser's JavaScript tool. Each script returns J | `scripts/page/inventory.js` | Framework, component tag names, test IDs, ARIA roles, internal route patterns, third-party SDKs | | `scripts/page/tokens.js` | CSS variables and the most frequent colors, type sizes, radii, spacing and shadows | | `scripts/page/api-shapes.js` | Installs a recorder; `__re.api.dump()` returns endpoint templates and JSON shapes; set `__re.api.enumSamples = false` to drop the short enum values too | -| `scripts/page/mask.js` | Blurs emails, phones, money, Brazilian tax IDs, small avatars, given names and given selectors; `__re.mask.clear()` removes it | +| `scripts/page/mask.js` | Blurs emails, phones, money, Brazilian tax IDs, small avatars, given names, matching placeholders, and given selectors as CSS rules that also cover elements rendered later; `__re.mask.clear()` removes it | | `scripts/page/annotate.js` | Numbered outlines on given elements for anatomy screenshots | Tool-specific notes (saving screenshots, network, profiles, drag and drop): `references/extraction.md` and `references/tools.md`. @@ -184,7 +188,7 @@ Tool-specific notes (saving screenshots, network, profiles, drag and drop): `ref 3. Write `log/-s.md`. 4. Run `node /scripts/check-kb.mjs ` and fix every error. 5. At the end of the study only: tear down from the ledger, compare baseline counts, list residuals in `STATUS.md`, then run `check-kb --closing`. -6. Hand off: run `node /scripts/pack-kb.mjs ` and send the zip to the user in the chat. Attach it with the agent's file-sending tool when there is one; otherwise give its path. The zip holds the KB and `BUILD.md`, a brief any coding agent can follow, and leaves out `sandbox/` and `log/`. With the zip, send the three-line summary from `STATUS.md`, the "Needs human" list, and one line: "Unzip it and give the folder to a coding agent, starting at BUILD.md." +6. Hand off: run `node /scripts/pack-kb.mjs ` and send the zip to the user in the chat. Attach it with the agent's file-sending tool when there is one; otherwise give its path. The zip holds the KB and `BUILD.md`, a brief any coding agent can follow, and leaves out `sandbox/` and `log/` except a copy of the ledger without object IDs, locations or URLs. With the zip, send the three-line summary from `STATUS.md`, the "Needs human" list, and one line: "Unzip it and give the folder to a coding agent, starting at BUILD.md." The study ends when coverage is complete or the user says to stop. When the user stops early, write the Phase 6 files from what exists, close, and pack: `STATUS.md` tells the reader what is missing. At L1 the trial account may keep its fixtures as `kept` ("trial kept for the developers"); ending the trial is the user's action. `pack-kb` runs the regular check, so rows marked `human` do not block the handoff; list them under "Needs human". diff --git a/skills/reverse-engineering-saas/references/extraction.md b/skills/reverse-engineering-saas/references/extraction.md index 6f00f69..b2904c6 100644 --- a/skills/reverse-engineering-saas/references/extraction.md +++ b/skills/reverse-engineering-saas/references/extraction.md @@ -4,7 +4,9 @@ Read the script file and pass its full content to the browser's JavaScript tool (`javascript_tool`, `browser_evaluate`, `evaluate_script`). Each script attaches functions to `window.__re` and returns a JSON-serializable result. Save the result as `evidence/EV-###- as text\n- Evidence: [obs EV-001]\n'); + appendFileSync(join(kb, 'clone/barriers.md'), '\n### BR-001 Example barrier\n- Severity: blocks launch (until approved)\n- Evidence: [doc SRC-001]\n'); + appendFileSync(join(kb, 'sources.md'), '| SRC-001 | Docs | https://example.com/docs | docs | 2026-10-16 | x |\n'); + const result = run('build-report.mjs', kb); + assert.equal(result.status, 0, result.stderr); + const html = readFileSync(join(kb, 'REPORT.html'), 'utf8'); + assert.match(html, /Acme CRM: reverse engineering report<\/title>/); + for (const id of ['summary', 'barriers', 'screens', 'rules', 'evidence', 'S-001', 'R-001', 'BR-001', 'EV-001', 'EV-002', 'F-001', 'SRC-001']) { + assert.match(html, new RegExp(`id="${id}"`), `anchor ${id}`); + } + assert.match(html, /<a href="#R-001">R-001<\/a>/); + assert.match(html, /<img src="data:image\/png;base64,/); + assert.equal((html.match(/data:image\/png;base64,/g) || []).length, 1, 'each image is embedded once'); + assert.match(html, /<span class="chip red">blocks launch<\/span> \(until approved\)/); + assert.match(html, /<span class="chip amber">blocked: drag<\/span>/); + assert.match(html, /<ul><li>Item<ul><li>Nested item<\/li><\/ul><\/li>/); + assert.match(html, /<script>alert\(1\)<\/script>/); + assert.doesNotMatch(html, /<script|<link|https?:\/\/[^"]*\.(js|css)/i, 'no scripts or external resources'); + assert.equal(run('build-report.mjs', '--help').status, 0); + + const out = join(dir, 'report.zip'); + assert.equal(run('pack-kb.mjs', kb, '--out', out).status, 0); + assert.ok(readZip(out).names.includes('re-acme/REPORT.html')); +}); + test('pack-kb refuses a KB with errors unless --force', () => { write('rules/a.md', '### R-001 No evidence\n- Given x\n'); const refused = run('pack-kb.mjs', kb, '--out', join(dir, 'x.zip'));