From 0da7a6f12547c7f1438a596da78a53fedf8a024b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 3 Nov 2026 15:13:44 +0000 Subject: [PATCH 1/8] feat(render): shared text snippets and code hosts shared/text/.md holds one fact in plain text. A skill inlines it with {{< name}}; code uses {{text:name}}, a quoted string literal. using-zenrows and extract-structured-data now take AUTH004, AUTH014, AUTH006 and REQS007's page_not_recognised from there; every plugin renders byte-identically. Validate fails on a snippet with markdown, more than one line, or no user. kind: code in vars.yaml makes a host whose output is code for another project: it ships only its own files, with {{...}} filled in .ts/.js too, no skills, LICENSE, assets or plugin.json, keeps its README out of dist, needs no release-please entry yet, and may keep a scripts/check.sh. --- CONTRIBUTING.md | 20 ++++++++--- scripts/lib.mjs | 40 ++++++++++++++++++---- scripts/render.mjs | 7 ++-- scripts/validate.mjs | 30 +++++++++++++--- shared/text/auth004.md | 1 + shared/text/auth006.md | 1 + shared/text/auth014.md | 1 + shared/text/reqs007-page-not-recognised.md | 1 + skills/extract-structured-data/SKILL.md | 2 +- skills/using-zenrows/SKILL.md | 4 +-- 10 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 shared/text/auth004.md create mode 100644 shared/text/auth006.md create mode 100644 shared/text/auth014.md create mode 100644 shared/text/reqs007-page-not-recognised.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a3334a2..cf664ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,20 +1,22 @@ # Contributing -Each host's plugin in `dist//` is rendered from shared skills plus that host's own files. Edit the sources, never `dist/`. +Each host's plugin in `dist//` is rendered from shared skills plus that host's own files. Code hosts, such as the Activepieces piece, are rendered the same way from their own source and the shared text. Edit the sources, never `dist/`. ## Layout ``` skills//SKILL.md shared skills; {{var}} and {{> slot}} mark what differs per host -shared/assets/ files every host ships as they are (logo) +shared/assets/ files every plugin host ships as they are (logo) +shared/text/.md one fact in plain text, used by skills ({{< name}}) and code hosts ({{text:name}}) platforms// vars.yaml values for {{var}} skills//.md the host's text for a {{> slot}} in that skill overrides.lock whole-file overrides and the shared version each was copied from (only once one exists) scripts/ the host's own tooling, never shipped: validate.mjs (run by npm run validate on - dist/) and package.sh (run by npm run package -- ), when the host needs them + dist/), package.sh (run by npm run package -- ) and check.sh (a code + host's build and test, run by its CI workflow), when the host needs them everything else the host's manifests, README, …, copied into dist// ({{version}} comes from release-please) -dist// GENERATED: the plugin the host installs +dist// GENERATED: the plugin the host installs, or a code host's code .github/plugin/marketplace.json Copilot's marketplace catalog (see below) scripts/ render, validate, override, new-host, package ``` @@ -40,7 +42,15 @@ Requires Node 22. Run `npm ci` once. - **A shared skill a host shouldn't ship:** list it under `skip_skills:` in that host's `vars.yaml` (Cursor uses a rules file instead of `using-zenrows`). A `{{skill:…}}` reference to a skipped skill fails the render. - **A reference to another skill:** write `{{skill:batch}}`, never the plain name. It renders as that skill's name on the host: `skill_prefix` plus the shared name, unless `skill_names` in `vars.yaml` renames it (OpenClaw ships `scrape-webpage` as `zenrows-scrape`). The skill's own `name:` in its frontmatter uses it too. -Unknown variables, unknown skill names, missing slots and unused slot files fail the render, and `npm run validate` fails on a reference to a skill the host doesn't ship, and on a skill name written plainly instead of as `{{skill:…}}`. +Unknown variables, unknown skill names, missing slots, unused slot files and unknown snippets fail the render, and `npm run validate` fails on a reference to a skill the host doesn't ship, and on a skill name written plainly instead of as `{{skill:…}}`. + +## Text shared with code hosts + +A fact that a skill and a code host both state (what `AUTH004` means, why Extract refused a page) lives once in `shared/text/.md`, as one line of plain text. A skill inlines it with `{{< name}}`; code uses `{{text:name}}`, which renders as a quoted string, and adds its own words around it (an agent is told to use a skill, a user to check their connection). Snippets stay plain text so they read right in both, and `npm run validate` fails on markdown in one, or on one nothing uses. + +## Code hosts + +`kind: code` in `vars.yaml` makes a host whose output is code for another project, such as `platforms/activepieces/` (the piece, as it sits in activepieces' `packages/pieces/community/zenrows`). A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped. It needs no release-please entry until it's released from this repo. Its `scripts/check.sh` builds and tests `dist//` the way the target project does, and its own workflow runs it. ## Manifest schemas diff --git a/scripts/lib.mjs b/scripts/lib.mjs index 9de5705..cf5e789 100644 --- a/scripts/lib.mjs +++ b/scripts/lib.mjs @@ -10,6 +10,7 @@ export const root = fileURLToPath(new URL("..", import.meta.url)); export const skillsDir = join(root, "skills"); export const platformsDir = join(root, "platforms"); export const distDir = join(root, "dist"); +export const sharedTextDir = join(root, "shared", "text"); // Files copied into every host's plugin as they are. const sharedFiles = { LICENSE: "LICENSE" }; @@ -19,7 +20,7 @@ const sharedDirs = { "shared/assets": "assets" }; // Everything else there (manifests, README, …) is copied into dist//. const reserved = ["vars.yaml", "overrides.lock", "skills/", "scripts/"]; // {{vars}} are filled in files with these extensions; others are copied as bytes. -const textFile = /\.(json|md|mdc|ya?ml|txt|toml)$/; +const textFile = /\.(json|md|mdc|ya?ml|txt|toml|[cm]?[jt]s)$/; // Errors here are for whoever edited a skill or platform file: print the // message, not a stack trace. @@ -54,8 +55,10 @@ function versions() { export function vars(host) { const v = parseYaml(readFileSync(join(platformsDir, host, "vars.yaml"), "utf8")) ?? {}; const version = versions()[`dist/${host}`]; - if (!version) throw new Error(`.release-please-manifest.json has no "dist/${host}" entry`); - return { ...v, version }; + // A code host keeps its version in its own package.json until it's released + // from here, so it needs no release-please entry yet. + if (!version && !isCodeHost(v)) throw new Error(`.release-please-manifest.json has no "dist/${host}" entry`); + return version ? { ...v, version } : v; } // overrides.lock: ": " per line; # starts a comment. @@ -69,6 +72,8 @@ export function readLock(host) { // the host's own skills (platforms//skills//SKILL.md with no shared // skill of that name). Returns [{ name, own }]. export function hostSkills(host, values = vars(host)) { + // A code host (kind: code, such as the Activepieces piece) ships no skills. + if (isCodeHost(values)) return []; const shared = skillNames(); const skip = values.skip_skills ?? []; for (const name of skip) { @@ -84,14 +89,34 @@ export function hostSkills(host, values = vars(host)) { ].sort((a, b) => a.name.localeCompare(b.name)); } +// kind: code in vars.yaml marks a host whose output is code (a package or a +// piece for another project), not a plugin: it ships only its own files, with +// {{…}} filled in its source, and no shared skills, LICENSE or assets. +export function isCodeHost(values) { + const kind = values.kind ?? "plugin"; + if (!["plugin", "code"].includes(kind)) throw new Error(`kind: ${kind} in vars.yaml; use plugin or code`); + return kind === "code"; +} + // A skill's name on a host: skill_prefix + the shared name, unless vars.yaml // renames it in skill_names (OpenClaw ships scrape-webpage as zenrows-scrape). export function hostSkillName(values, name) { return values.skill_names?.[name] ?? `${values.skill_prefix ?? ""}${name}`; } -// {{name}} is a variable from vars.yaml; {{skill:name}} is that skill's name on the host. +// shared/text/.md: one fact in plain text, used by skills and by code +// hosts alike, so a wording fix reaches both. +export function sharedText(name, where) { + const file = join(sharedTextDir, `${name}.md`); + if (!existsSync(file)) throw new Error(`${where}: there's no shared/text/${name}.md`); + return readFileSync(file, "utf8").replace(/\n$/, ""); +} + +// {{name}} is a variable from vars.yaml; {{skill:name}} is that skill's name on +// the host; {{< name}} inlines shared/text/.md, and {{text:name}} inserts it +// as a quoted string literal (JSON), for code. function fill(text, values, where) { + text = text.replace(/\{\{<\s*([a-z0-9-]+)\s*\}\}/g, (_, name) => sharedText(name, where)); return text.replace(/\{\{\s*([a-z_]+)(?::([a-z0-9-]+))?\s*\}\}/g, (token, name, arg) => { if (name === "skill") { if (!values.skills.includes(arg)) { @@ -100,6 +125,7 @@ function fill(text, values, where) { } return hostSkillName(values, arg); } + if (name === "text" && arg !== undefined) return JSON.stringify(sharedText(arg, where)); if (arg !== undefined || !(name in values) || typeof values[name] === "object") { throw new Error(`${where}: unknown variable ${token}`); } @@ -137,12 +163,14 @@ export function renderHost(host) { if (!usedSlots.has(path)) throw new Error(`${path}: no skill this host ships has this slot`); } - for (const [from, to] of Object.entries(sharedFiles)) out.set(to, readFileSync(join(root, from))); - for (const [from, to] of Object.entries(sharedDirs)) { + for (const [from, to] of isCodeHost(values) ? [] : Object.entries(sharedFiles)) out.set(to, readFileSync(join(root, from))); + for (const [from, to] of isCodeHost(values) ? [] : Object.entries(sharedDirs)) { for (const file of listFiles(join(root, from))) out.set(`${to}/${file}`, readFileSync(join(root, from, file))); } for (const file of listFiles(platform)) { if (reserved.some((r) => file === r || file.startsWith(r))) continue; + // A code host's README documents the integration here; it isn't part of the code. + if (isCodeHost(values) && file === "README.md") continue; out.set(file, renderFile(join(platform, file), values)); } return out; diff --git a/scripts/render.mjs b/scripts/render.mjs index 940d551..eff864e 100644 --- a/scripts/render.mjs +++ b/scripts/render.mjs @@ -4,7 +4,7 @@ // render. import { mkdirSync, readFileSync, rmSync, writeFileSync, existsSync } from "node:fs"; import { dirname, join } from "node:path"; -import { distDir, hosts, listFiles, renderHost, root, vars } from "./lib.mjs"; +import { distDir, hosts, isCodeHost, listFiles, renderHost, root, vars } from "./lib.mjs"; const check = process.argv.includes("--check"); const only = process.argv.slice(2).filter((a) => !a.startsWith("--")); @@ -26,7 +26,10 @@ function hostsList() { const rows = hosts() .map((host) => [vars(host).host_full ?? host, host]) .sort(([a], [b]) => a.localeCompare(b)) - .map(([name, host]) => `| ${name} | [dist/${host}/README.md](dist/${host}/README.md) |`); + .map(([name, host]) => { + const guide = isCodeHost(vars(host)) ? `platforms/${host}/README.md` : `dist/${host}/README.md`; + return `| ${name} | [${guide}](${guide}) |`; + }); const table = ["| Host | Setup guide |", "|---|---|", ...rows].join("\n"); const head = readme.slice(0, readme.indexOf("\n", start) + 1); return Buffer.from(`${head}${table}\n${readme.slice(end)}`); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 87a9c18..3334924 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -6,7 +6,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, dirname, join } from "node:path"; import Ajv2020 from "ajv/dist/2020.js"; import { parse as parseYaml } from "yaml"; -import { catalogs, hostSkillName, hostSkills, hosts, listFiles, platformsDir, readLock, renderHost, root, sha256, skillNames, vars } from "./lib.mjs"; +import { catalogs, hostSkillName, hostSkills, hosts, isCodeHost, listFiles, platformsDir, readLock, renderHost, root, sha256, sharedTextDir, skillNames, vars } from "./lib.mjs"; const errors = []; const report = []; @@ -167,7 +167,7 @@ for (const host of hosts()) { if (!json[key]) errors.push(`${host}: ${file} has no "${key}"`); } } - if (!manifests.length) errors.push(`${host}: no plugin.json in platforms/${host}/`); + if (!manifests.length && !isCodeHost(values)) errors.push(`${host}: no plugin.json in platforms/${host}/`); // Marketplace catalogs at the top of the repo must point at this host's // dist/ folder, with this host's version. @@ -217,10 +217,11 @@ for (const host of hosts()) { } // platforms//scripts/ is the host's tooling and never ships, so only the - // two files the repo runs belong there. + // files the repo runs belong there: validate.mjs, package.sh, and check.sh (a + // code host's build and test, run by its CI workflow). for (const file of listFiles(join(platformsDir, host, "scripts"))) { - if (!["validate.mjs", "package.sh"].includes(file)) { - errors.push(`${host}: platforms/${host}/scripts/${file} isn't shipped (scripts/ holds only validate.mjs and package.sh); move plugin files elsewhere`); + if (!["validate.mjs", "package.sh", "check.sh"].includes(file)) { + errors.push(`${host}: platforms/${host}/scripts/${file} isn't shipped (scripts/ holds only validate.mjs, package.sh and check.sh); move plugin files elsewhere`); } } @@ -240,6 +241,25 @@ for (const host of hosts()) { report.push(`${host}: ${Object.keys(lock).length} whole-file overrides, ${owned} skill lines of its own (slots and overrides)`); } +// shared/text/ snippets are plain text, so they read the same in a skill and in +// a code string, and each must be used somewhere ({{< name}} or {{text:name}}). +const snippetSources = [ + ...skillNames().map((n) => join(root, "skills", n, "SKILL.md")), + ...listFiles(platformsDir).map((f) => join(platformsDir, f)), +]; +const used = new Set(); +for (const file of snippetSources) { + if (!/\.(md|mdc|json|ya?ml|txt|toml|ts)$/.test(file)) continue; + for (const [, , name] of readFileSync(file, "utf8").matchAll(/\{\{\s*(<\s*|text:)([a-z0-9-]+)\s*\}\}/g)) used.add(name); +} +for (const file of listFiles(sharedTextDir)) { + const name = file.replace(/\.md$/, ""); + const text = readFileSync(join(sharedTextDir, file), "utf8"); + if (!used.has(name)) errors.push(`shared/text/${file} isn't used by any skill or host`); + if (/[`*_[\]#]/.test(text)) errors.push(`shared/text/${file} has markdown; keep snippets plain text so code can use them`); + if (text.trim().includes("\n")) errors.push(`shared/text/${file} has more than one line; keep one fact per snippet`); +} + console.log(report.join("\n")); if (errors.length) { console.error(`\n${errors.map((e) => `❌ ${e}`).join("\n")}`); diff --git a/shared/text/auth004.md b/shared/text/auth004.md new file mode 100644 index 0000000..09426a7 --- /dev/null +++ b/shared/text/auth004.md @@ -0,0 +1 @@ +the account's credit allowance is spent diff --git a/shared/text/auth006.md b/shared/text/auth006.md new file mode 100644 index 0000000..b5f06c3 --- /dev/null +++ b/shared/text/auth006.md @@ -0,0 +1 @@ +too many requests are running at once diff --git a/shared/text/auth014.md b/shared/text/auth014.md new file mode 100644 index 0000000..c8511e2 --- /dev/null +++ b/shared/text/auth014.md @@ -0,0 +1 @@ +this API key reached its credit cap diff --git a/shared/text/reqs007-page-not-recognised.md b/shared/text/reqs007-page-not-recognised.md new file mode 100644 index 0000000..dbc749c --- /dev/null +++ b/shared/text/reqs007-page-not-recognised.md @@ -0,0 +1 @@ +the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example) diff --git a/skills/extract-structured-data/SKILL.md b/skills/extract-structured-data/SKILL.md index 3f25523..cdb1c4c 100644 --- a/skills/extract-structured-data/SKILL.md +++ b/skills/extract-structured-data/SKILL.md @@ -24,7 +24,7 @@ Use the dedicated `extract` tool, not `scrape`. It returns structured JSON direc - **`mode` selects the extraction strategy** (`auto`, `autoparse`, `css`). It is not the adaptive request setting, which is the separate boolean `mode_auto` and is on by default. `scrape` has no `mode` parameter at all. - **`extract` with `mode='auto'` works on any domain Zenrows has prepared, with any API key.** There is no per-account gate. When it can't extract a page it returns `REQS007` (or the older `AUTH010`) with a `reason`, and nothing is charged: - `domain_not_prepared`: call `prepare_domain` with the domain and 1 to 5 example URLs, one per kind of page the user wants (for example one product page and one listing page). It costs no credits and takes a few minutes. Poll `prepared_domain_status` about every 30 seconds until `status` is `ready`, then call `extract` again. Each new domain uses one slot of the account's monthly allowance, which `prepared_domain_status` shows. If the user can't wait, use `mode='css'` meanwhile. - - `page_not_recognised`: the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example). Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. + - `page_not_recognised`: {{< reqs007-page-not-recognised}}. Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. - `domain_not_enabled`: preparing doesn't help. Use `mode='css'`, or tell the user to contact Zenrows support to have the site enabled. - **`fallback_autoparse` is deprecated and has no effect.** Extract no longer falls back to autoparse. Don't set it; if you do, the result only carries a note saying it was ignored. diff --git a/skills/using-zenrows/SKILL.md b/skills/using-zenrows/SKILL.md index f8e0a31..06f723f 100644 --- a/skills/using-zenrows/SKILL.md +++ b/skills/using-zenrows/SKILL.md @@ -78,8 +78,8 @@ Reads the connected account's plan and usage. It changes nothing. - Do not build manual retry loops around a request. Zenrows already escalates to heavier settings when a page needs it. - `401` or `AUTH003`: Zenrows is not connected, or the sign-in is no longer valid (for example the API key was rotated). Use the `{{skill:getting-started}}` skill to reconnect. Do not ask for a key. -- `AUTH004` (the account's credit allowance is spent) or `AUTH014` (this API key reached its credit cap): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. -- `AUTH006`: too many requests are running at once. Wait for running requests to finish, then retry. +- `AUTH004` ({{< auth004}}) or `AUTH014` ({{< auth014}}): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. +- `AUTH006`: {{< auth006}}. Wait for running requests to finish, then retry. - `413`: the response is too large. Narrow it with `css_extractor`. - `REQS007` (or the older `AUTH010`) from `extract`: read its `reason`. `domain_not_prepared`: prepare the domain with `prepare_domain` and wait for `prepared_domain_status` to say `ready`. `page_not_recognised`: try a product, article or listing page, or `mode='css'`; don't prepare the domain again. `domain_not_enabled`: use `mode='css'`. Not charged in any case. Details in the `{{skill:extract-structured-data}}` skill. - Look up any other error code at https://docs.zenrows.com. From 5eb4af2c036938c6cb19994de1982d6de00406cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 3 Nov 2026 15:13:44 +0000 Subject: [PATCH 2/8] feat(activepieces): Zenrows piece rendered from the shared text platforms/activepieces is the piece as it sits in activepieces' packages/pieces/community/zenrows, rendered to dist/activepieces. Its AUTH004, AUTH014, AUTH006 and page_not_recognised hints use the shared facts, and REQS007 now gives the hint for its reason (domain_not_prepared, page_not_recognised, domain_not_enabled), read from "reason" or, until the API sends it, the title. scripts/check.sh lints, builds and tests the rendered piece inside activepieces at the pinned commit, and a path-filtered workflow runs it. --- .github/workflows/activepieces.yml | 43 ++++ README.md | 1 + dist/activepieces/.eslintrc.json | 47 +++++ dist/activepieces/package.json | 23 ++ dist/activepieces/src/i18n/translation.json | 70 +++++++ dist/activepieces/src/index.ts | 37 ++++ .../lib/actions/extract-structured-data.ts | 73 +++++++ .../src/lib/actions/get-account-usage.ts | 85 ++++++++ .../src/lib/actions/scrape-url.ts | 157 ++++++++++++++ dist/activepieces/src/lib/auth.ts | 27 +++ dist/activepieces/src/lib/common/client.ts | 198 ++++++++++++++++++ dist/activepieces/src/lib/common/params.ts | 144 +++++++++++++ dist/activepieces/test/params.test.ts | 162 ++++++++++++++ dist/activepieces/tsconfig.json | 19 ++ dist/activepieces/tsconfig.lib.json | 22 ++ dist/activepieces/vitest.config.ts | 19 ++ platforms/activepieces/.eslintrc.json | 47 +++++ platforms/activepieces/README.md | 25 +++ platforms/activepieces/package.json | 23 ++ platforms/activepieces/scripts/check.sh | 60 ++++++ .../activepieces/src/i18n/translation.json | 70 +++++++ platforms/activepieces/src/index.ts | 37 ++++ .../lib/actions/extract-structured-data.ts | 73 +++++++ .../src/lib/actions/get-account-usage.ts | 85 ++++++++ .../src/lib/actions/scrape-url.ts | 157 ++++++++++++++ platforms/activepieces/src/lib/auth.ts | 27 +++ .../activepieces/src/lib/common/client.ts | 198 ++++++++++++++++++ .../activepieces/src/lib/common/params.ts | 144 +++++++++++++ platforms/activepieces/test/params.test.ts | 162 ++++++++++++++ platforms/activepieces/tsconfig.json | 19 ++ platforms/activepieces/tsconfig.lib.json | 22 ++ platforms/activepieces/vars.yaml | 8 + platforms/activepieces/vitest.config.ts | 19 ++ 33 files changed, 2303 insertions(+) create mode 100644 .github/workflows/activepieces.yml create mode 100644 dist/activepieces/.eslintrc.json create mode 100644 dist/activepieces/package.json create mode 100644 dist/activepieces/src/i18n/translation.json create mode 100644 dist/activepieces/src/index.ts create mode 100644 dist/activepieces/src/lib/actions/extract-structured-data.ts create mode 100644 dist/activepieces/src/lib/actions/get-account-usage.ts create mode 100644 dist/activepieces/src/lib/actions/scrape-url.ts create mode 100644 dist/activepieces/src/lib/auth.ts create mode 100644 dist/activepieces/src/lib/common/client.ts create mode 100644 dist/activepieces/src/lib/common/params.ts create mode 100644 dist/activepieces/test/params.test.ts create mode 100644 dist/activepieces/tsconfig.json create mode 100644 dist/activepieces/tsconfig.lib.json create mode 100644 dist/activepieces/vitest.config.ts create mode 100644 platforms/activepieces/.eslintrc.json create mode 100644 platforms/activepieces/README.md create mode 100644 platforms/activepieces/package.json create mode 100755 platforms/activepieces/scripts/check.sh create mode 100644 platforms/activepieces/src/i18n/translation.json create mode 100644 platforms/activepieces/src/index.ts create mode 100644 platforms/activepieces/src/lib/actions/extract-structured-data.ts create mode 100644 platforms/activepieces/src/lib/actions/get-account-usage.ts create mode 100644 platforms/activepieces/src/lib/actions/scrape-url.ts create mode 100644 platforms/activepieces/src/lib/auth.ts create mode 100644 platforms/activepieces/src/lib/common/client.ts create mode 100644 platforms/activepieces/src/lib/common/params.ts create mode 100644 platforms/activepieces/test/params.test.ts create mode 100644 platforms/activepieces/tsconfig.json create mode 100644 platforms/activepieces/tsconfig.lib.json create mode 100644 platforms/activepieces/vars.yaml create mode 100644 platforms/activepieces/vitest.config.ts diff --git a/.github/workflows/activepieces.yml b/.github/workflows/activepieces.yml new file mode 100644 index 0000000..873a024 --- /dev/null +++ b/.github/workflows/activepieces.yml @@ -0,0 +1,43 @@ +name: Activepieces piece + +on: + push: + branches: [main] + paths: + - platforms/activepieces/** + - dist/activepieces/** + - shared/text/** + - scripts/** + - .github/workflows/activepieces.yml + pull_request: + paths: + - platforms/activepieces/** + - dist/activepieces/** + - shared/text/** + - scripts/** + - .github/workflows/activepieces.yml + +permissions: + contents: read + +jobs: + check: + name: Lint, build and test in activepieces + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + - run: npm ci + # activepieces' packageManager at the pinned commit. If they differ, + # check.sh installs the right version instead. + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.4.0" + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.bun/install/cache + key: bun-activepieces-${{ hashFiles('platforms/activepieces/vars.yaml') }} + restore-keys: bun-activepieces- + - run: platforms/activepieces/scripts/check.sh diff --git a/README.md b/README.md index f71ab0e..fa1037a 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Each host's guide covers installing the plugin and signing in to Zenrows. You si | Host | Setup guide | |---|---| +| Activepieces | [platforms/activepieces/README.md](platforms/activepieces/README.md) | | ChatGPT and Codex | [dist/openai/README.md](dist/openai/README.md) | | GitHub Copilot | [dist/copilot/README.md](dist/copilot/README.md) | diff --git a/dist/activepieces/.eslintrc.json b/dist/activepieces/.eslintrc.json new file mode 100644 index 0000000..6f15366 --- /dev/null +++ b/dist/activepieces/.eslintrc.json @@ -0,0 +1,47 @@ +{ + "extends": [ + "../../../../.eslintrc.json" + ], + "ignorePatterns": [ + "!**/*" + ], + "overrides": [ + { + "files": [ + "*.ts", + "*.tsx", + "*.js", + "*.jsx" + ], + "rules": {} + }, + { + "files": [ + "*.ts", + "*.tsx" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + "lodash", + "lodash/*", + "@activepieces/core-*", + "@activepieces/server*", + "@activepieces/engine", + "@activepieces/shared" + ] + } + ] + } + }, + { + "files": [ + "*.js", + "*.jsx" + ], + "rules": {} + } + ] +} diff --git a/dist/activepieces/package.json b/dist/activepieces/package.json new file mode 100644 index 0000000..d118817 --- /dev/null +++ b/dist/activepieces/package.json @@ -0,0 +1,23 @@ +{ + "name": "@activepieces/piece-zenrows", + "version": "0.0.1", + "type": "commonjs", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "@activepieces/pieces-common": "workspace:*", + "@activepieces/pieces-framework": "workspace:*", + "@activepieces/core-piece-types": "workspace:*", + "@activepieces/core-utils": "workspace:*" + }, + "devDependencies": { + "tslib": "2.6.2", + "vitest": "3.2.6" + }, + "scripts": { + "build": "tsc -p tsconfig.lib.json && cp package.json dist/", + "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle", + "lint": "eslint 'src/**/*.ts'", + "test": "vitest run" + } +} diff --git a/dist/activepieces/src/i18n/translation.json b/dist/activepieces/src/i18n/translation.json new file mode 100644 index 0000000..acdef80 --- /dev/null +++ b/dist/activepieces/src/i18n/translation.json @@ -0,0 +1,70 @@ +{ + "Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.": "Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.", + "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ": "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ", + "Scrape URL": "Scrape URL", + "Extract Structured Data": "Extract Structured Data", + "Get Account Usage": "Get Account Usage", + "Custom API Call": "Custom API Call", + "Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.": "Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.", + "Get specific data from a web page as JSON, using CSS selectors or automatic extraction.": "Get specific data from a web page as JSON, using CSS selectors or automatic extraction.", + "Get your ZenRows plan, credits used this period and concurrency.": "Get your ZenRows plan, credits used this period and concurrency.", + "Make a custom API call to a specific endpoint": "Make a custom API call to a specific endpoint", + "URL": "URL", + "Output Format": "Output Format", + "Adaptive Stealth Mode": "Adaptive Stealth Mode", + "JS Rendering": "JS Rendering", + "Premium Proxy": "Premium Proxy", + "Proxy Country": "Proxy Country", + "Wait For Selector": "Wait For Selector", + "Wait (ms)": "Wait (ms)", + "JavaScript Instructions": "JavaScript Instructions", + "Custom Headers": "Custom Headers", + "Session ID": "Session ID", + "Extraction Method": "Extraction Method", + "CSS Selectors": "CSS Selectors", + "Fields": "Fields", + "Method": "Method", + "Headers": "Headers", + "Query Parameters": "Query Parameters", + "Body Type": "Body Type", + "Body": "Body", + "Response is Binary": "Response is Binary", + "Return Error as Output": "Return Error as Output", + "Timeout": "Timeout", + "Follow redirects": "Follow redirects", + "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/": "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/", + "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.": "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.", + "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.", + "Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.": "Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.", + "Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.": "Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.", + "Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.": "Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.", + "A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.": "A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.", + "Extra milliseconds to wait after the page loads. Needs JS rendering.": "Extra milliseconds to wait after the page loads. Needs JS rendering.", + "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.", + "Headers to send to the target site, such as Cookie or Referer.": "Headers to send to the target site, such as Cookie or Referer.", + "Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.": "Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.", + "The full address of the page to extract data from.": "The full address of the page to extract data from.", + "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.": "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.", + "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.": "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.", + "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.": "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.", + "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.": "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.", + "Authorization headers are injected automatically from your connection.": "Authorization headers are injected automatically from your connection.", + "Appended to the URL as ?key=value.": "Appended to the URL as ?key=value.", + "Enable for files like PDFs, images, etc.": "Enable for files like PDFs, images, etc.", + "On a failed request, output the error instead of failing the step.": "On a failed request, output the error instead of failing the step.", + "Seconds to wait for a response. Empty: up to the flow limit (10 min).": "Seconds to wait for a response. Empty: up to the flow limit (10 min).", + "Follow 3xx redirects instead of returning them as the response.": "Follow 3xx redirects instead of returning them as the response.", + "Markdown": "Markdown", + "HTML": "HTML", + "Plain Text": "Plain Text", + "Screenshot (PNG)": "Screenshot (PNG)", + "Automatic": "Automatic", + "GET": "GET", + "POST": "POST", + "PATCH": "PATCH", + "PUT": "PUT", + "DELETE": "DELETE", + "HEAD": "HEAD", + "Request": "Request", + "Data to send": "Data to send" +} \ No newline at end of file diff --git a/dist/activepieces/src/index.ts b/dist/activepieces/src/index.ts new file mode 100644 index 0000000..b187dcc --- /dev/null +++ b/dist/activepieces/src/index.ts @@ -0,0 +1,37 @@ +import { createCustomApiCallAction } from '@activepieces/pieces-common'; +import { createPiece, PieceCategory } from '@activepieces/pieces-framework'; +import { extractStructuredDataAction } from './lib/actions/extract-structured-data'; +import { getAccountUsageAction } from './lib/actions/get-account-usage'; +import { scrapeUrlAction } from './lib/actions/scrape-url'; +import { zenrowsAuth } from './lib/auth'; +import { ZENROWS_API_BASE_URL } from './lib/common/client'; + +export const zenrows = createPiece({ + displayName: 'ZenRows', + description: + 'Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.', + auth: zenrowsAuth, + minimumSupportedRelease: '0.88.2', + logoUrl: 'https://cdn.activepieces.com/pieces/zenrows.png', + categories: [PieceCategory.ARTIFICIAL_INTELLIGENCE, PieceCategory.PRODUCTIVITY], + authors: ['imoital'], + actions: [ + scrapeUrlAction, + extractStructuredDataAction, + getAccountUsageAction, + createCustomApiCallAction({ + baseUrl: () => ZENROWS_API_BASE_URL, + auth: zenrowsAuth, + authMapping: async (auth) => ({ + 'X-API-Key': auth.secret_text, + }), + props: { + url: { + description: + 'A full URL or a path relative to https://api.zenrows.com/v1, e.g. /subscriptions/self/details. The API key is sent in the X-API-Key header.', + }, + }, + }), + ], + triggers: [], +}); diff --git a/dist/activepieces/src/lib/actions/extract-structured-data.ts b/dist/activepieces/src/lib/actions/extract-structured-data.ts new file mode 100644 index 0000000..ea6f050 --- /dev/null +++ b/dist/activepieces/src/lib/actions/extract-structured-data.ts @@ -0,0 +1,73 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; +import { ExtractMethod, zenrowsParams } from '../common/params'; + +export const extractStructuredDataAction = createAction({ + auth: zenrowsAuth, + name: 'extract_structured_data', + classification: 'READ', + displayName: 'Extract Structured Data', + description: 'Get specific data from a web page as JSON, using CSS selectors or automatic extraction.', + audience: 'both', + aiMetadata: { + description: + 'Fetches one web page through ZenRows and returns structured JSON instead of raw content. Use the CSS selectors method when you know the page structure (field name → selector), or the automatic method to let ZenRows detect fields such as products, prices or titles on supported sites. Use Scrape URL instead when you need the whole page. Read-only, safe to retry, spends ZenRows credits.', + idempotent: true, + }, + props: { + url: Property.ShortText({ + displayName: 'URL', + description: 'The full address of the page to extract data from.', + required: true, + }), + method: Property.StaticDropdown({ + displayName: 'Extraction Method', + description: + 'CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.', + required: true, + defaultValue: 'css', + options: { + disabled: false, + options: [ + { label: 'CSS Selectors', value: 'css' }, + { label: 'Automatic', value: 'auto' }, + ], + }, + }), + selectors: Property.Object({ + displayName: 'CSS Selectors', + description: + 'For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.', + required: false, + }), + fields: Property.Array({ + displayName: 'Fields', + description: + 'For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.', + required: false, + }), + adaptive: Property.Checkbox({ + displayName: 'Adaptive Stealth Mode', + description: + 'Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.', + required: false, + defaultValue: true, + }), + }, + async run(context) { + const { propsValue } = context; + const params = zenrowsParams.buildExtractParams({ + url: propsValue.url, + method: propsValue.method ?? 'css', + selectors: propsValue.selectors, + fields: propsValue.fields, + adaptive: propsValue.adaptive ?? true, + }); + const response = await zenrowsClient.fetchPage({ + apiKey: context.auth.secret_text, + params, + }); + return response.body; + }, +}); diff --git a/dist/activepieces/src/lib/actions/get-account-usage.ts b/dist/activepieces/src/lib/actions/get-account-usage.ts new file mode 100644 index 0000000..618e98e --- /dev/null +++ b/dist/activepieces/src/lib/actions/get-account-usage.ts @@ -0,0 +1,85 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { createAction, OutputSchema } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; + +export const getAccountUsageAction = createAction({ + auth: zenrowsAuth, + name: 'get_account_usage', + classification: 'READ', + displayName: 'Get Account Usage', + description: 'Get your ZenRows plan, credits used this period and concurrency.', + audience: 'both', + outputSchema: usageOutputSchema(), + aiMetadata: { + description: + 'Returns the ZenRows plan, credits used and remaining in the current billing period, and concurrency in use. Use it to check the budget before a large scraping job or to alert when usage is high. Free to call and safe to retry.', + idempotent: true, + }, + props: {}, + async run(context) { + const details = await zenrowsClient.request({ + apiKey: context.auth.secret_text, + method: HttpMethod.GET, + path: '/subscriptions/self/details', + }); + const creditLimit = details.credit_limit ?? null; + const usageCredits = details.usage_credits ?? null; + return { + plan_name: details.plan?.name ?? null, + plan_recurrence: details.plan?.recurrence ?? null, + status: details.status ?? null, + period_starts_at: details.period_starts_at ?? null, + period_ends_at: details.period_ends_at ?? null, + credit_limit: creditLimit, + credits_used: usageCredits, + credits_remaining: + creditLimit === null || usageCredits === null ? null : Math.max(creditLimit - usageCredits, 0), + usage_percent: details.usage_percent ?? null, + usage_usd: details.usage ?? null, + concurrency_limit: details.plan?.products?.api?.concurrency?.limit ?? null, + concurrency_in_use: details.plan?.products?.api?.concurrency?.usage ?? null, + }; + }, +}); + +function usageOutputSchema(): OutputSchema { + return { + fields: [ + { key: 'plan_name', label: 'Plan' }, + { key: 'plan_recurrence', label: 'Billing Recurrence' }, + { key: 'status', label: 'Status' }, + { key: 'period_starts_at', label: 'Period Starts At', format: 'datetime' }, + { key: 'period_ends_at', label: 'Period Ends At', format: 'datetime' }, + { key: 'credit_limit', label: 'Credit Limit', format: 'number' }, + { key: 'credits_used', label: 'Credits Used', format: 'number' }, + { key: 'credits_remaining', label: 'Credits Remaining', format: 'number' }, + { key: 'usage_percent', label: 'Usage (%)', format: 'number' }, + { key: 'usage_usd', label: 'Spend (USD)', format: 'currency', currency: 'USD' }, + { key: 'concurrency_limit', label: 'Concurrency Limit', format: 'number' }, + { key: 'concurrency_in_use', label: 'Concurrency In Use', format: 'number' }, + ], + }; +} + +type SubscriptionDetails = { + status?: string; + period_starts_at?: string; + period_ends_at?: string; + usage?: number; + usage_credits?: number; + credit_limit?: number; + usage_percent?: number; + plan?: { + name?: string; + recurrence?: string; + products?: { + api?: { + concurrency?: { + limit?: number; + usage?: number; + }; + }; + }; + }; +}; diff --git a/dist/activepieces/src/lib/actions/scrape-url.ts b/dist/activepieces/src/lib/actions/scrape-url.ts new file mode 100644 index 0000000..c57a478 --- /dev/null +++ b/dist/activepieces/src/lib/actions/scrape-url.ts @@ -0,0 +1,157 @@ +import { randomUUID } from 'crypto'; +import { createAction, OutputSchema, Property } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; +import { ScrapeOutputFormat, zenrowsParams } from '../common/params'; + +export const scrapeUrlAction = createAction({ + auth: zenrowsAuth, + name: 'scrape_url', + classification: 'READ', + displayName: 'Scrape URL', + description: + 'Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.', + audience: 'both', + outputSchema: scrapeOutputSchema(), + aiMetadata: { + description: + 'Fetches one web page through ZenRows and returns its content as Markdown, HTML, plain text or a full-page screenshot, getting past CAPTCHAs and anti-bot systems. Pick this when you have the exact URL and need the page content; use Extract Structured Data when you need specific fields as JSON. Read-only against the target, so retries are safe, but each call spends ZenRows credits.', + idempotent: true, + }, + props: { + url: Property.ShortText({ + displayName: 'URL', + description: 'The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/', + required: true, + }), + output_format: Property.StaticDropdown({ + displayName: 'Output Format', + description: 'Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.', + required: true, + defaultValue: 'markdown', + options: { + disabled: false, + options: [ + { label: 'Markdown', value: 'markdown' }, + { label: 'HTML', value: 'html' }, + { label: 'Plain Text', value: 'plaintext' }, + { label: 'Screenshot (PNG)', value: 'screenshot' }, + ], + }, + }), + adaptive: Property.Checkbox({ + displayName: 'Adaptive Stealth Mode', + description: + 'Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.', + required: false, + defaultValue: true, + }), + js_render: Property.Checkbox({ + displayName: 'JS Rendering', + description: + 'Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.', + required: false, + defaultValue: false, + }), + premium_proxy: Property.Checkbox({ + displayName: 'Premium Proxy', + description: + 'Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.', + required: false, + defaultValue: false, + }), + proxy_country: Property.ShortText({ + displayName: 'Proxy Country', + description: + 'Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.', + required: false, + }), + wait_for: Property.ShortText({ + displayName: 'Wait For Selector', + description: 'A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.', + required: false, + }), + wait: Property.Number({ + displayName: 'Wait (ms)', + description: 'Extra milliseconds to wait after the page loads. Needs JS rendering.', + required: false, + }), + js_instructions: Property.LongText({ + displayName: 'JavaScript Instructions', + description: + 'Actions to run on the page before it is returned, as a JSON array, e.g. [{"click": ".load-more"}, {"wait": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.', + required: false, + }), + custom_headers: Property.Object({ + displayName: 'Custom Headers', + description: 'Headers to send to the target site, such as Cookie or Referer.', + required: false, + }), + session_id: Property.Number({ + displayName: 'Session ID', + description: 'Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.', + required: false, + }), + }, + async run(context) { + const { propsValue } = context; + const outputFormat = propsValue.output_format ?? 'markdown'; + const forwardedHeaders = zenrowsParams.toStringRecord({ value: propsValue.custom_headers }); + const forwardHeaders = Object.keys(forwardedHeaders).length > 0; + const isScreenshot = outputFormat === 'screenshot'; + + const params = zenrowsParams.buildScrapeParams({ + url: propsValue.url, + outputFormat, + adaptive: propsValue.adaptive ?? true, + jsRender: propsValue.js_render, + premiumProxy: propsValue.premium_proxy, + proxyCountry: propsValue.proxy_country, + waitFor: propsValue.wait_for, + wait: propsValue.wait, + jsInstructions: propsValue.js_instructions, + forwardHeaders, + sessionId: propsValue.session_id, + }); + + const response = await zenrowsClient.fetchPage({ + apiKey: context.auth.secret_text, + params, + forwardedHeaders: forwardHeaders ? forwardedHeaders : undefined, + binary: isScreenshot, + }); + + const screenshotUrl = isScreenshot + ? await context.files.write({ + fileName: `zenrows-screenshot-${randomUUID()}.png`, + data: Buffer.from(response.body), + }) + : null; + + const requestCost = zenrowsClient.readHeader({ headers: response.headers, name: 'X-Request-Cost' }); + + return { + url: propsValue.url, + output_format: outputFormat, + content: isScreenshot ? null : String(response.body), + screenshot_url: screenshotUrl, + final_url: zenrowsClient.readHeader({ headers: response.headers, name: 'Zr-Final-Url' }), + request_cost: requestCost === null ? null : Number(requestCost), + request_id: zenrowsClient.readHeader({ headers: response.headers, name: 'X-Request-Id' }), + }; + }, +}); + +function scrapeOutputSchema(): OutputSchema { + return { + fields: [ + { key: 'url', label: 'URL', format: 'url' }, + { key: 'output_format', label: 'Output Format' }, + { key: 'content', label: 'Content' }, + { key: 'screenshot_url', label: 'Screenshot', format: 'image' }, + { key: 'final_url', label: 'Final URL', format: 'url' }, + { key: 'request_cost', label: 'Request Cost', format: 'number' }, + { key: 'request_id', label: 'Request ID' }, + ], + }; +} diff --git a/dist/activepieces/src/lib/auth.ts b/dist/activepieces/src/lib/auth.ts new file mode 100644 index 0000000..4f4bc0a --- /dev/null +++ b/dist/activepieces/src/lib/auth.ts @@ -0,0 +1,27 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { PieceAuth } from '@activepieces/pieces-framework'; +import { zenrowsClient } from './common/client'; + +export const zenrowsAuth = PieceAuth.SecretText({ + displayName: 'API Key', + required: true, + description: ` +**Get your ZenRows API key:** + +1. Sign in at https://app.zenrows.com (or create a free account). +2. Open the **Builder** or **Settings** page. +3. Copy your API key and paste it here. + `, + validate: async ({ auth }) => { + try { + await zenrowsClient.request({ + apiKey: auth, + method: HttpMethod.GET, + path: '/subscriptions/self/details', + }); + return { valid: true }; + } catch (e) { + return { valid: false, error: 'Invalid API key' }; + } + }, +}); diff --git a/dist/activepieces/src/lib/common/client.ts b/dist/activepieces/src/lib/common/client.ts new file mode 100644 index 0000000..b4b6dfd --- /dev/null +++ b/dist/activepieces/src/lib/common/client.ts @@ -0,0 +1,198 @@ +import { + HttpError, + HttpHeaders, + HttpMessageBody, + HttpMethod, + HttpResponse, + httpClient, +} from '@activepieces/pieces-common'; + +async function request({ + apiKey, + method, + path, + body, +}: { + apiKey: string; + method: HttpMethod; + path: string; + body?: unknown; +}): Promise { + try { + const response = await httpClient.sendRequest({ + method, + url: `${ZENROWS_API_BASE_URL}${path}`, + headers: { + 'X-API-Key': apiKey, + 'Content-Type': 'application/json', + ...CLIENT_HEADERS, + }, + body, + }); + return response.body; + } catch (error) { + throw toZenrowsError({ error }); + } +} + +async function fetchPage({ + apiKey, + params, + forwardedHeaders, + binary, +}: { + apiKey: string; + params: Record; + forwardedHeaders?: Record; + binary?: boolean; +}): Promise { + try { + return await httpClient.sendRequest({ + method: HttpMethod.GET, + url: `${ZENROWS_API_BASE_URL}/`, + queryParams: { apikey: apiKey, ...params }, + headers: forwardedHeaders ?? CLIENT_HEADERS, + timeout: FETCH_TIMEOUT_MS, + responseType: binary ? 'arraybuffer' : undefined, + }); + } catch (error) { + throw toZenrowsError({ error }); + } +} + +function toZenrowsError({ error }: { error: unknown }): unknown { + if (!(error instanceof HttpError)) { + return error; + } + const { status, body } = error.response; + const problem = parseProblem({ body }); + if (!problem) { + return error; + } + const hint = isExtractRefusal({ code: problem.code }) + ? REQS007_HINTS[problem.extractCase ?? ''] ?? ERROR_HINTS['REQS007'] + : ERROR_HINTS[problem.code]; + return new Error( + `ZenRows error ${problem.code} (HTTP ${status}): ${problem.reason}${hint ? ` ${hint}` : ''}` + ); +} + +function readHeader({ + headers, + name, +}: { + headers: HttpHeaders | undefined; + name: string; +}): string | null { + const value = headers?.[name.toLowerCase()]; + if (Array.isArray(value)) { + return value[0] ?? null; + } + return value ?? null; +} + +function parseProblem({ body }: { body: unknown }): ZenrowsProblem | null { + if (typeof body === 'string') { + try { + return parseProblem({ body: JSON.parse(body) }); + } catch { + return null; + } + } + if (!isRecord(body) || typeof body['code'] !== 'string') { + return null; + } + const detail = body['detail']; + const title = body['title']; + return { + code: body['code'], + reason: + typeof detail === 'string' + ? detail + : typeof title === 'string' + ? title + : 'Request failed', + extractCase: readExtractCase({ body }), + }; +} + +// REQS007 (or the older AUTH010) says why Extract refused a page in its +// "reason" field. Until every API release sends it, the title tells the cases +// apart: https://docs.zenrows.com/api-error-codes#REQS007 +function readExtractCase({ body }: { body: Record }): string | undefined { + if (typeof body['reason'] === 'string') { + return body['reason']; + } + const title = body['title']; + if (typeof title !== 'string') { + return undefined; + } + return REQS007_TITLES.find(([prefix]) => title.startsWith(prefix))?.[1]; +} + +function isExtractRefusal({ code }: { code: string }): boolean { + return code === 'REQS007' || code === 'AUTH010'; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +const FETCH_TIMEOUT_MS = 180000; + +const CLIENT_HEADERS = { 'X-ZenRows-Client': 'activepieces' }; + +const AUTH_HINT = 'Check the API key in your ZenRows connection.'; + +// The facts in {{text:…}} come from zenrows-integrations' shared/text/, the same +// wording the Zenrows skills use; each hint adds what to do in Activepieces. +function sentence(fact: string, next: string): string { + return `${fact.charAt(0).toUpperCase()}${fact.slice(1)}. ${next}`; +} + +const ERROR_HINTS: Record = { + AUTH001: AUTH_HINT, + AUTH002: AUTH_HINT, + AUTH003: AUTH_HINT, + AUTH004: sentence("the account's credit allowance is spent", 'Top up or upgrade at https://app.zenrows.com.'), + AUTH014: sentence("this API key reached its credit cap", 'Raise the cap in the ZenRows dashboard.'), + REQS002: + 'This page needs JS rendering and/or premium proxies. Turn on Adaptive Stealth Mode, or enable them manually.', + REQS007: + 'Extract refused this page, and nothing was charged. Use the CSS selectors method, or see https://docs.zenrows.com/api-error-codes#REQS007.', + AUTH006: sentence("too many requests are running at once", 'Reduce parallel runs and retry.'), + AUTH008: 'Rate limit reached. Wait a moment and retry.', + RESP009: 'ZenRows is temporarily at capacity. Retry in a few seconds.', +}; + +export const ZENROWS_API_BASE_URL = 'https://api.zenrows.com/v1'; + +const REQS007_TITLES: [string, string][] = [ + ['Extract is not prepared for the requested domain', 'domain_not_prepared'], + ['Extract did not recognise this page on', 'page_not_recognised'], + ['Extract is not enabled for the requested domain', 'domain_not_enabled'], +]; + +const REQS007_HINTS: Record = { + domain_not_prepared: + 'ZenRows has not prepared this domain for automatic extraction yet. Prepare it with the Custom API Call action: POST /extract/prepared-domains with the domain and 1 to 5 example URLs. It takes a few minutes and costs no credits. Meanwhile, use the CSS selectors method.', + page_not_recognised: sentence( + "the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example)", + 'Try a product, article or listing page, or the CSS selectors method. Preparing the domain again does not change it.' + ), + domain_not_enabled: + 'Automatic extraction is not enabled for this site, and preparing it does not help. Use the CSS selectors method, or contact ZenRows support.', +}; + +export const zenrowsClient = { + request, + fetchPage, + toZenrowsError, + readHeader, +}; + +type ZenrowsProblem = { + code: string; + reason: string; + extractCase?: string; +}; diff --git a/dist/activepieces/src/lib/common/params.ts b/dist/activepieces/src/lib/common/params.ts new file mode 100644 index 0000000..9affe2f --- /dev/null +++ b/dist/activepieces/src/lib/common/params.ts @@ -0,0 +1,144 @@ +function buildScrapeParams({ + url, + outputFormat, + adaptive, + jsRender, + premiumProxy, + proxyCountry, + waitFor, + wait, + jsInstructions, + forwardHeaders, + sessionId, +}: ScrapeOptions): Record { + const isScreenshot = outputFormat === 'screenshot'; + const usesBrowser = isScreenshot || (!adaptive && jsRender === true); + const usesPremiumProxy = !adaptive && premiumProxy === true; + const needsBrowser = !isNilOrEmpty(waitFor) || !isNil(wait) || !isNilOrEmpty(jsInstructions); + + if (!adaptive && !usesBrowser && needsBrowser) { + throw new Error( + 'Wait For Selector, Wait (ms) and JavaScript Instructions need JS Rendering. Turn on JS Rendering or Adaptive Stealth Mode.' + ); + } + if (!adaptive && !usesPremiumProxy && !isNilOrEmpty(proxyCountry)) { + throw new Error( + 'Proxy Country needs Premium Proxy. Turn on Premium Proxy or Adaptive Stealth Mode.' + ); + } + + return { + url, + ...(adaptive && !isScreenshot ? { mode: 'auto' } : {}), + ...(usesBrowser ? { js_render: 'true' } : {}), + ...(usesPremiumProxy ? { premium_proxy: 'true' } : {}), + ...(!isNilOrEmpty(proxyCountry) ? { proxy_country: proxyCountry.trim().toLowerCase() } : {}), + ...(!isNilOrEmpty(waitFor) ? { wait_for: waitFor } : {}), + ...(!isNil(wait) ? { wait: String(wait) } : {}), + ...(!isNilOrEmpty(jsInstructions) ? { js_instructions: normalizeJsInstructions({ jsInstructions }) } : {}), + ...(forwardHeaders ? { custom_headers: 'true' } : {}), + ...(!isNil(sessionId) ? { session_id: String(sessionId) } : {}), + ...RESPONSE_TYPE_PARAMS[outputFormat], + }; +} + +function buildExtractParams({ + url, + method, + selectors, + fields, + adaptive, +}: ExtractOptions): Record { + const base = { url, ...(adaptive ? { mode: 'auto' } : {}) }; + if (method === 'css') { + const selectorMap = toStringRecord({ value: selectors }); + if (Object.keys(selectorMap).length === 0) { + throw new Error('Add at least one field with a CSS selector, for example title → h1.'); + } + return { ...base, css_extractor: JSON.stringify(selectorMap) }; + } + const requestedFields = (fields ?? []) + .filter((field): field is string => typeof field === 'string') + .map((field) => field.trim()) + .filter((field) => field.length > 0); + return { + ...base, + extract: 'auto', + ...(requestedFields.length > 0 ? { extract_fields: requestedFields.join(',') } : {}), + }; +} + +function toStringRecord({ + value, +}: { + value: Record | null | undefined; +}): Record { + return Object.fromEntries( + Object.entries(value ?? {}).flatMap(([key, entry]) => + typeof entry === 'string' && entry.trim().length > 0 && key.trim().length > 0 + ? [[key.trim(), entry.trim()]] + : [] + ) + ); +} + +function normalizeJsInstructions({ jsInstructions }: { jsInstructions: string }): string { + try { + const parsed: unknown = JSON.parse(jsInstructions); + if (!Array.isArray(parsed)) { + throw new Error('not an array'); + } + return JSON.stringify(parsed); + } catch { + throw new Error( + 'JavaScript Instructions must be a JSON array, for example [{"click": ".load-more"}, {"wait": 1000}].' + ); + } +} + +function isNil(value: unknown): value is null | undefined { + return value === null || value === undefined; +} + +function isNilOrEmpty(value: string | null | undefined): value is null | undefined { + return isNil(value) || value.trim().length === 0; +} + +const RESPONSE_TYPE_PARAMS: Record> = { + html: {}, + markdown: { response_type: 'markdown' }, + plaintext: { response_type: 'plaintext' }, + screenshot: { screenshot_fullpage: 'true', screenshot_format: 'png' }, +}; + +export const zenrowsParams = { + buildScrapeParams, + buildExtractParams, + toStringRecord, +}; + +export type ScrapeOutputFormat = 'html' | 'markdown' | 'plaintext' | 'screenshot'; + +export type ExtractMethod = 'css' | 'auto'; + +type ScrapeOptions = { + url: string; + outputFormat: ScrapeOutputFormat; + adaptive: boolean; + jsRender?: boolean; + premiumProxy?: boolean; + proxyCountry?: string | null; + waitFor?: string | null; + wait?: number | null; + jsInstructions?: string | null; + forwardHeaders: boolean; + sessionId?: number | null; +}; + +type ExtractOptions = { + url: string; + method: ExtractMethod; + selectors?: Record | null; + fields?: unknown[] | null; + adaptive: boolean; +}; diff --git a/dist/activepieces/test/params.test.ts b/dist/activepieces/test/params.test.ts new file mode 100644 index 0000000..f61ff03 --- /dev/null +++ b/dist/activepieces/test/params.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from 'vitest'; +import { HttpError } from '@activepieces/pieces-common'; +import { zenrowsClient } from '../src/lib/common/client'; +import { zenrowsParams } from '../src/lib/common/params'; + +const URL = 'https://www.scrapingcourse.com/ecommerce/'; + +describe('buildScrapeParams', () => { + it('uses adaptive mode and markdown by default', () => { + expect( + zenrowsParams.buildScrapeParams({ url: URL, outputFormat: 'markdown', adaptive: true, forwardHeaders: false }) + ).toEqual({ url: URL, mode: 'auto', response_type: 'markdown' }); + }); + + it('ignores manual JS rendering and premium proxy in adaptive mode', () => { + expect( + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'html', + adaptive: true, + jsRender: true, + premiumProxy: true, + proxyCountry: 'US', + forwardHeaders: false, + }) + ).toEqual({ url: URL, mode: 'auto', proxy_country: 'us' }); + }); + + it('sends manual options when adaptive mode is off', () => { + expect( + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'plaintext', + adaptive: false, + jsRender: true, + premiumProxy: true, + proxyCountry: 'de', + waitFor: '.product', + wait: 500, + jsInstructions: '[{"click": ".more"}]', + forwardHeaders: true, + sessionId: 42, + }) + ).toEqual({ + url: URL, + js_render: 'true', + premium_proxy: 'true', + proxy_country: 'de', + wait_for: '.product', + wait: '500', + js_instructions: '[{"click":".more"}]', + custom_headers: 'true', + session_id: '42', + response_type: 'plaintext', + }); + }); + + it('forces JS rendering and skips adaptive mode for screenshots', () => { + expect( + zenrowsParams.buildScrapeParams({ url: URL, outputFormat: 'screenshot', adaptive: true, forwardHeaders: false }) + ).toEqual({ url: URL, js_render: 'true', screenshot_fullpage: 'true', screenshot_format: 'png' }); + }); + + it('rejects browser options without JS rendering in manual mode', () => { + expect(() => + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'html', + adaptive: false, + waitFor: '.product', + forwardHeaders: false, + }) + ).toThrow(/need JS Rendering/); + }); + + it('rejects a proxy country without premium proxy in manual mode', () => { + expect(() => + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'html', + adaptive: false, + proxyCountry: 'us', + forwardHeaders: false, + }) + ).toThrow(/needs Premium Proxy/); + }); + + it('rejects JavaScript instructions that are not a JSON array', () => { + expect(() => + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'html', + adaptive: true, + jsInstructions: '{"click": ".more"}', + forwardHeaders: false, + }) + ).toThrow(/JSON array/); + }); +}); + +describe('buildExtractParams', () => { + it('sends CSS selectors as a JSON map', () => { + expect( + zenrowsParams.buildExtractParams({ + url: URL, + method: 'css', + selectors: { title: 'h1', price: ' .price ', empty: '', count: 3 }, + adaptive: true, + }) + ).toEqual({ url: URL, mode: 'auto', css_extractor: '{"title":"h1","price":".price"}' }); + }); + + it('requires at least one selector', () => { + expect(() => + zenrowsParams.buildExtractParams({ url: URL, method: 'css', selectors: {}, adaptive: false }) + ).toThrow(/at least one field/); + }); + + it('uses automatic extraction with optional fields', () => { + expect( + zenrowsParams.buildExtractParams({ + url: URL, + method: 'auto', + fields: ['products.name', ' products.price ', '', 7], + adaptive: false, + }) + ).toEqual({ url: URL, extract: 'auto', extract_fields: 'products.name,products.price' }); + }); +}); + +describe('toZenrowsError', () => { + it('turns a ZenRows problem response into a readable error with a hint', () => { + const error = new HttpError( + {}, + { + status: 402, + responseBody: JSON.stringify({ code: 'AUTH004', title: 'Usage exceeded (AUTH004)', detail: 'No credits left.' }), + } + ); + const mapped = zenrowsClient.toZenrowsError({ error }); + expect(mapped).toBeInstanceOf(Error); + expect(String(mapped)).toContain('ZenRows error AUTH004 (HTTP 402): No credits left.'); + expect(String(mapped)).toContain('credit allowance is spent'); + }); + + it.each([ + [{ reason: 'domain_not_prepared' }, 'POST /extract/prepared-domains'], + [{ title: 'Extract did not recognise this page on example.com' }, 'Preparing the domain again does not change it'], + [{ title: 'Extract is not enabled for the requested domain' }, 'contact ZenRows support'], + [{ title: 'Something new' }, 'api-error-codes#REQS007'], + ])('gives the REQS007 hint for its case (%o)', (fields, hint) => { + const error = new HttpError({}, { status: 403, responseBody: JSON.stringify({ code: 'REQS007', ...fields }) }); + expect(String(zenrowsClient.toZenrowsError({ error }))).toContain(hint); + }); + + it('keeps errors that are not ZenRows problems unchanged', () => { + const error = new HttpError({}, { status: 500, responseBody: 'oops' }); + expect(zenrowsClient.toZenrowsError({ error })).toBe(error); + const plain = new Error('network down'); + expect(zenrowsClient.toZenrowsError({ error: plain })).toBe(plain); + }); +}); diff --git a/dist/activepieces/tsconfig.json b/dist/activepieces/tsconfig.json new file mode 100644 index 0000000..b512ca3 --- /dev/null +++ b/dist/activepieces/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} \ No newline at end of file diff --git a/dist/activepieces/tsconfig.lib.json b/dist/activepieces/tsconfig.lib.json new file mode 100644 index 0000000..8b4345e --- /dev/null +++ b/dist/activepieces/tsconfig.lib.json @@ -0,0 +1,22 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "baseUrl": ".", + "paths": {}, + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "types": [ + "node" + ] + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "jest.config.ts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} \ No newline at end of file diff --git a/dist/activepieces/vitest.config.ts b/dist/activepieces/vitest.config.ts new file mode 100644 index 0000000..79c9726 --- /dev/null +++ b/dist/activepieces/vitest.config.ts @@ -0,0 +1,19 @@ +import path from 'path' +import { defineConfig } from 'vitest/config' + +const repoRoot = path.resolve(__dirname, '../../../..') + +export default defineConfig({ + root: __dirname, + test: { + globals: true, + environment: 'node', + include: ['test/**/*.test.ts'], + }, + resolve: { + alias: { + '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'), + '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'), + }, + }, +}) diff --git a/platforms/activepieces/.eslintrc.json b/platforms/activepieces/.eslintrc.json new file mode 100644 index 0000000..6f15366 --- /dev/null +++ b/platforms/activepieces/.eslintrc.json @@ -0,0 +1,47 @@ +{ + "extends": [ + "../../../../.eslintrc.json" + ], + "ignorePatterns": [ + "!**/*" + ], + "overrides": [ + { + "files": [ + "*.ts", + "*.tsx", + "*.js", + "*.jsx" + ], + "rules": {} + }, + { + "files": [ + "*.ts", + "*.tsx" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + "lodash", + "lodash/*", + "@activepieces/core-*", + "@activepieces/server*", + "@activepieces/engine", + "@activepieces/shared" + ] + } + ] + } + }, + { + "files": [ + "*.js", + "*.jsx" + ], + "rules": {} + } + ] +} diff --git a/platforms/activepieces/README.md b/platforms/activepieces/README.md new file mode 100644 index 0000000..f6c59a3 --- /dev/null +++ b/platforms/activepieces/README.md @@ -0,0 +1,25 @@ +# Zenrows piece for Activepieces + +The Zenrows [piece](https://www.activepieces.com/docs/build-pieces/building-pieces/overview) for Activepieces: Scrape URL, Extract structured data, Get account usage, and a custom API call. + +This is a code host (`kind: code` in `vars.yaml`). `npm run render` writes `dist/activepieces/`, which is the piece exactly as it sits in the activepieces repo (`packages/pieces/community/zenrows`): package `@activepieces/piece-zenrows`, on their workspace packages. This README stays here and isn't part of the piece. + +## Shared wording + +Facts that the Zenrows skills state too come from `shared/text/` through `{{text:name}}`, which renders as a quoted string. Today that's the meaning of `AUTH004`, `AUTH014`, `AUTH006` and REQS007's `page_not_recognised`, in `src/lib/common/client.ts`. A wording fix in `shared/text/` reaches the skills and the piece in the same render. + +## Check it + +``` +npm run render +platforms/activepieces/scripts/check.sh # clones activepieces at vars.yaml's upstream into .cache/ +platforms/activepieces/scripts/check.sh ~/path/to/activepieces # or reuse a checkout you already have +``` + +It copies `dist/activepieces/` into the activepieces checkout, registers it in `tsconfig.base.json`, installs with their pinned bun, and runs their `lint`, `build` and `test` for this piece only. CI runs the same script (`.github/workflows/activepieces.yml`). + +`upstream` in `vars.yaml` pins the activepieces commit. Moving it is its own PR, so CI shows whether their newer code breaks the piece. + +## How it ships + +Not yet. First as our own npm package; then as a PR to activepieces once they agree to take it (they close outside PRs that weren't discussed with them first; see their CONTRIBUTING.md). Both release steps come in later PRs. diff --git a/platforms/activepieces/package.json b/platforms/activepieces/package.json new file mode 100644 index 0000000..d118817 --- /dev/null +++ b/platforms/activepieces/package.json @@ -0,0 +1,23 @@ +{ + "name": "@activepieces/piece-zenrows", + "version": "0.0.1", + "type": "commonjs", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "@activepieces/pieces-common": "workspace:*", + "@activepieces/pieces-framework": "workspace:*", + "@activepieces/core-piece-types": "workspace:*", + "@activepieces/core-utils": "workspace:*" + }, + "devDependencies": { + "tslib": "2.6.2", + "vitest": "3.2.6" + }, + "scripts": { + "build": "tsc -p tsconfig.lib.json && cp package.json dist/", + "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle", + "lint": "eslint 'src/**/*.ts'", + "test": "vitest run" + } +} diff --git a/platforms/activepieces/scripts/check.sh b/platforms/activepieces/scripts/check.sh new file mode 100755 index 0000000..7acbd93 --- /dev/null +++ b/platforms/activepieces/scripts/check.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Lints, builds and tests the rendered piece (dist/activepieces/) inside an +# activepieces checkout, the way their CI does, because the piece depends on +# their workspace packages. +# +# platforms/activepieces/scripts/check.sh clones activepieces at vars.yaml's upstream into .cache/activepieces +# platforms/activepieces/scripts/check.sh uses an existing checkout instead (faster locally) +# +# dist/activepieces/ is copied to /packages/pieces/community/zenrows, so a +# checkout you pass in gets this repo's version of the piece. Run npm run render +# first; this checks the committed render is current. +set -euo pipefail + +repo="$(cd "$(dirname "$0")/../../.." && pwd)" +(cd "$repo" && node scripts/render.mjs --check >/dev/null) +sha="$(sed -n 's/^upstream: *"\{0,1\}\([0-9a-f]\{40\}\)"\{0,1\} *$/\1/p' "$repo/platforms/activepieces/vars.yaml")" +[ -n "$sha" ] || { echo "platforms/activepieces/vars.yaml has no upstream: <40-character sha>" >&2; exit 1; } +ap="${1:-$repo/.cache/activepieces}" + +if [ -z "${1:-}" ]; then + if [ "$(git -C "$ap" rev-parse HEAD 2>/dev/null)" != "$sha" ]; then + rm -rf "$ap" + git init -q "$ap" + git -C "$ap" fetch -q --depth 1 https://github.com/activepieces/activepieces.git "$sha" + git -C "$ap" checkout -q FETCH_HEAD + fi +elif [ "$(git -C "$ap" rev-parse HEAD)" != "$sha" ]; then + echo "note: $ap is at $(git -C "$ap" rev-parse --short HEAD), vars.yaml pins ${sha:0:10}" >&2 +fi + +piece="$ap/packages/pieces/community/zenrows" +rm -rf "$piece" +mkdir -p "$piece" +cp -R "$repo/dist/activepieces/." "$piece" + +# Register the piece's path, as a new piece in their repo does. +node -e ' +const fs = require("fs"); +const file = process.argv[1]; +const config = JSON.parse(fs.readFileSync(file, "utf8")); +const paths = config.compilerOptions.paths; +if (!paths["@activepieces/piece-zenrows"]) { + paths["@activepieces/piece-zenrows"] = ["packages/pieces/community/zenrows/src/index.ts"]; + config.compilerOptions.paths = Object.fromEntries(Object.entries(paths).sort(([a], [b]) => a.localeCompare(b))); + fs.writeFileSync(file, JSON.stringify(config, null, 2) + "\n"); +}' "$ap/tsconfig.base.json" + +# Their pinned bun. Turbo looks for it on PATH, so when the installed bun is +# missing or another version, install theirs into .cache/ and put it first. +bun_version="$(node -p 'require(process.argv[1]).packageManager.split("@")[1]' "$ap/package.json")" +if ! command -v bun >/dev/null || [ "$(bun --version)" != "$bun_version" ]; then + prefix="$repo/.cache/bun-$bun_version" + [ -x "$prefix/node_modules/.bin/bun" ] || npm install -s --prefix "$prefix" "bun@$bun_version" + export PATH="$prefix/node_modules/.bin:$PATH" +fi + +cd "$ap" +# Not --frozen-lockfile: the new piece adds a workspace entry to bun.lock. +bun install +npx turbo run lint build test --filter=@activepieces/piece-zenrows diff --git a/platforms/activepieces/src/i18n/translation.json b/platforms/activepieces/src/i18n/translation.json new file mode 100644 index 0000000..acdef80 --- /dev/null +++ b/platforms/activepieces/src/i18n/translation.json @@ -0,0 +1,70 @@ +{ + "Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.": "Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.", + "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ": "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ", + "Scrape URL": "Scrape URL", + "Extract Structured Data": "Extract Structured Data", + "Get Account Usage": "Get Account Usage", + "Custom API Call": "Custom API Call", + "Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.": "Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.", + "Get specific data from a web page as JSON, using CSS selectors or automatic extraction.": "Get specific data from a web page as JSON, using CSS selectors or automatic extraction.", + "Get your ZenRows plan, credits used this period and concurrency.": "Get your ZenRows plan, credits used this period and concurrency.", + "Make a custom API call to a specific endpoint": "Make a custom API call to a specific endpoint", + "URL": "URL", + "Output Format": "Output Format", + "Adaptive Stealth Mode": "Adaptive Stealth Mode", + "JS Rendering": "JS Rendering", + "Premium Proxy": "Premium Proxy", + "Proxy Country": "Proxy Country", + "Wait For Selector": "Wait For Selector", + "Wait (ms)": "Wait (ms)", + "JavaScript Instructions": "JavaScript Instructions", + "Custom Headers": "Custom Headers", + "Session ID": "Session ID", + "Extraction Method": "Extraction Method", + "CSS Selectors": "CSS Selectors", + "Fields": "Fields", + "Method": "Method", + "Headers": "Headers", + "Query Parameters": "Query Parameters", + "Body Type": "Body Type", + "Body": "Body", + "Response is Binary": "Response is Binary", + "Return Error as Output": "Return Error as Output", + "Timeout": "Timeout", + "Follow redirects": "Follow redirects", + "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/": "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/", + "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.": "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.", + "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.", + "Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.": "Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.", + "Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.": "Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.", + "Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.": "Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.", + "A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.": "A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.", + "Extra milliseconds to wait after the page loads. Needs JS rendering.": "Extra milliseconds to wait after the page loads. Needs JS rendering.", + "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.", + "Headers to send to the target site, such as Cookie or Referer.": "Headers to send to the target site, such as Cookie or Referer.", + "Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.": "Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.", + "The full address of the page to extract data from.": "The full address of the page to extract data from.", + "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.": "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.", + "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.": "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.", + "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.": "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.", + "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.": "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.", + "Authorization headers are injected automatically from your connection.": "Authorization headers are injected automatically from your connection.", + "Appended to the URL as ?key=value.": "Appended to the URL as ?key=value.", + "Enable for files like PDFs, images, etc.": "Enable for files like PDFs, images, etc.", + "On a failed request, output the error instead of failing the step.": "On a failed request, output the error instead of failing the step.", + "Seconds to wait for a response. Empty: up to the flow limit (10 min).": "Seconds to wait for a response. Empty: up to the flow limit (10 min).", + "Follow 3xx redirects instead of returning them as the response.": "Follow 3xx redirects instead of returning them as the response.", + "Markdown": "Markdown", + "HTML": "HTML", + "Plain Text": "Plain Text", + "Screenshot (PNG)": "Screenshot (PNG)", + "Automatic": "Automatic", + "GET": "GET", + "POST": "POST", + "PATCH": "PATCH", + "PUT": "PUT", + "DELETE": "DELETE", + "HEAD": "HEAD", + "Request": "Request", + "Data to send": "Data to send" +} \ No newline at end of file diff --git a/platforms/activepieces/src/index.ts b/platforms/activepieces/src/index.ts new file mode 100644 index 0000000..b187dcc --- /dev/null +++ b/platforms/activepieces/src/index.ts @@ -0,0 +1,37 @@ +import { createCustomApiCallAction } from '@activepieces/pieces-common'; +import { createPiece, PieceCategory } from '@activepieces/pieces-framework'; +import { extractStructuredDataAction } from './lib/actions/extract-structured-data'; +import { getAccountUsageAction } from './lib/actions/get-account-usage'; +import { scrapeUrlAction } from './lib/actions/scrape-url'; +import { zenrowsAuth } from './lib/auth'; +import { ZENROWS_API_BASE_URL } from './lib/common/client'; + +export const zenrows = createPiece({ + displayName: 'ZenRows', + description: + 'Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.', + auth: zenrowsAuth, + minimumSupportedRelease: '0.88.2', + logoUrl: 'https://cdn.activepieces.com/pieces/zenrows.png', + categories: [PieceCategory.ARTIFICIAL_INTELLIGENCE, PieceCategory.PRODUCTIVITY], + authors: ['imoital'], + actions: [ + scrapeUrlAction, + extractStructuredDataAction, + getAccountUsageAction, + createCustomApiCallAction({ + baseUrl: () => ZENROWS_API_BASE_URL, + auth: zenrowsAuth, + authMapping: async (auth) => ({ + 'X-API-Key': auth.secret_text, + }), + props: { + url: { + description: + 'A full URL or a path relative to https://api.zenrows.com/v1, e.g. /subscriptions/self/details. The API key is sent in the X-API-Key header.', + }, + }, + }), + ], + triggers: [], +}); diff --git a/platforms/activepieces/src/lib/actions/extract-structured-data.ts b/platforms/activepieces/src/lib/actions/extract-structured-data.ts new file mode 100644 index 0000000..ea6f050 --- /dev/null +++ b/platforms/activepieces/src/lib/actions/extract-structured-data.ts @@ -0,0 +1,73 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; +import { ExtractMethod, zenrowsParams } from '../common/params'; + +export const extractStructuredDataAction = createAction({ + auth: zenrowsAuth, + name: 'extract_structured_data', + classification: 'READ', + displayName: 'Extract Structured Data', + description: 'Get specific data from a web page as JSON, using CSS selectors or automatic extraction.', + audience: 'both', + aiMetadata: { + description: + 'Fetches one web page through ZenRows and returns structured JSON instead of raw content. Use the CSS selectors method when you know the page structure (field name → selector), or the automatic method to let ZenRows detect fields such as products, prices or titles on supported sites. Use Scrape URL instead when you need the whole page. Read-only, safe to retry, spends ZenRows credits.', + idempotent: true, + }, + props: { + url: Property.ShortText({ + displayName: 'URL', + description: 'The full address of the page to extract data from.', + required: true, + }), + method: Property.StaticDropdown({ + displayName: 'Extraction Method', + description: + 'CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.', + required: true, + defaultValue: 'css', + options: { + disabled: false, + options: [ + { label: 'CSS Selectors', value: 'css' }, + { label: 'Automatic', value: 'auto' }, + ], + }, + }), + selectors: Property.Object({ + displayName: 'CSS Selectors', + description: + 'For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.', + required: false, + }), + fields: Property.Array({ + displayName: 'Fields', + description: + 'For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.', + required: false, + }), + adaptive: Property.Checkbox({ + displayName: 'Adaptive Stealth Mode', + description: + 'Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.', + required: false, + defaultValue: true, + }), + }, + async run(context) { + const { propsValue } = context; + const params = zenrowsParams.buildExtractParams({ + url: propsValue.url, + method: propsValue.method ?? 'css', + selectors: propsValue.selectors, + fields: propsValue.fields, + adaptive: propsValue.adaptive ?? true, + }); + const response = await zenrowsClient.fetchPage({ + apiKey: context.auth.secret_text, + params, + }); + return response.body; + }, +}); diff --git a/platforms/activepieces/src/lib/actions/get-account-usage.ts b/platforms/activepieces/src/lib/actions/get-account-usage.ts new file mode 100644 index 0000000..618e98e --- /dev/null +++ b/platforms/activepieces/src/lib/actions/get-account-usage.ts @@ -0,0 +1,85 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { createAction, OutputSchema } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; + +export const getAccountUsageAction = createAction({ + auth: zenrowsAuth, + name: 'get_account_usage', + classification: 'READ', + displayName: 'Get Account Usage', + description: 'Get your ZenRows plan, credits used this period and concurrency.', + audience: 'both', + outputSchema: usageOutputSchema(), + aiMetadata: { + description: + 'Returns the ZenRows plan, credits used and remaining in the current billing period, and concurrency in use. Use it to check the budget before a large scraping job or to alert when usage is high. Free to call and safe to retry.', + idempotent: true, + }, + props: {}, + async run(context) { + const details = await zenrowsClient.request({ + apiKey: context.auth.secret_text, + method: HttpMethod.GET, + path: '/subscriptions/self/details', + }); + const creditLimit = details.credit_limit ?? null; + const usageCredits = details.usage_credits ?? null; + return { + plan_name: details.plan?.name ?? null, + plan_recurrence: details.plan?.recurrence ?? null, + status: details.status ?? null, + period_starts_at: details.period_starts_at ?? null, + period_ends_at: details.period_ends_at ?? null, + credit_limit: creditLimit, + credits_used: usageCredits, + credits_remaining: + creditLimit === null || usageCredits === null ? null : Math.max(creditLimit - usageCredits, 0), + usage_percent: details.usage_percent ?? null, + usage_usd: details.usage ?? null, + concurrency_limit: details.plan?.products?.api?.concurrency?.limit ?? null, + concurrency_in_use: details.plan?.products?.api?.concurrency?.usage ?? null, + }; + }, +}); + +function usageOutputSchema(): OutputSchema { + return { + fields: [ + { key: 'plan_name', label: 'Plan' }, + { key: 'plan_recurrence', label: 'Billing Recurrence' }, + { key: 'status', label: 'Status' }, + { key: 'period_starts_at', label: 'Period Starts At', format: 'datetime' }, + { key: 'period_ends_at', label: 'Period Ends At', format: 'datetime' }, + { key: 'credit_limit', label: 'Credit Limit', format: 'number' }, + { key: 'credits_used', label: 'Credits Used', format: 'number' }, + { key: 'credits_remaining', label: 'Credits Remaining', format: 'number' }, + { key: 'usage_percent', label: 'Usage (%)', format: 'number' }, + { key: 'usage_usd', label: 'Spend (USD)', format: 'currency', currency: 'USD' }, + { key: 'concurrency_limit', label: 'Concurrency Limit', format: 'number' }, + { key: 'concurrency_in_use', label: 'Concurrency In Use', format: 'number' }, + ], + }; +} + +type SubscriptionDetails = { + status?: string; + period_starts_at?: string; + period_ends_at?: string; + usage?: number; + usage_credits?: number; + credit_limit?: number; + usage_percent?: number; + plan?: { + name?: string; + recurrence?: string; + products?: { + api?: { + concurrency?: { + limit?: number; + usage?: number; + }; + }; + }; + }; +}; diff --git a/platforms/activepieces/src/lib/actions/scrape-url.ts b/platforms/activepieces/src/lib/actions/scrape-url.ts new file mode 100644 index 0000000..c57a478 --- /dev/null +++ b/platforms/activepieces/src/lib/actions/scrape-url.ts @@ -0,0 +1,157 @@ +import { randomUUID } from 'crypto'; +import { createAction, OutputSchema, Property } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; +import { ScrapeOutputFormat, zenrowsParams } from '../common/params'; + +export const scrapeUrlAction = createAction({ + auth: zenrowsAuth, + name: 'scrape_url', + classification: 'READ', + displayName: 'Scrape URL', + description: + 'Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.', + audience: 'both', + outputSchema: scrapeOutputSchema(), + aiMetadata: { + description: + 'Fetches one web page through ZenRows and returns its content as Markdown, HTML, plain text or a full-page screenshot, getting past CAPTCHAs and anti-bot systems. Pick this when you have the exact URL and need the page content; use Extract Structured Data when you need specific fields as JSON. Read-only against the target, so retries are safe, but each call spends ZenRows credits.', + idempotent: true, + }, + props: { + url: Property.ShortText({ + displayName: 'URL', + description: 'The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/', + required: true, + }), + output_format: Property.StaticDropdown({ + displayName: 'Output Format', + description: 'Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.', + required: true, + defaultValue: 'markdown', + options: { + disabled: false, + options: [ + { label: 'Markdown', value: 'markdown' }, + { label: 'HTML', value: 'html' }, + { label: 'Plain Text', value: 'plaintext' }, + { label: 'Screenshot (PNG)', value: 'screenshot' }, + ], + }, + }), + adaptive: Property.Checkbox({ + displayName: 'Adaptive Stealth Mode', + description: + 'Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.', + required: false, + defaultValue: true, + }), + js_render: Property.Checkbox({ + displayName: 'JS Rendering', + description: + 'Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.', + required: false, + defaultValue: false, + }), + premium_proxy: Property.Checkbox({ + displayName: 'Premium Proxy', + description: + 'Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.', + required: false, + defaultValue: false, + }), + proxy_country: Property.ShortText({ + displayName: 'Proxy Country', + description: + 'Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.', + required: false, + }), + wait_for: Property.ShortText({ + displayName: 'Wait For Selector', + description: 'A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.', + required: false, + }), + wait: Property.Number({ + displayName: 'Wait (ms)', + description: 'Extra milliseconds to wait after the page loads. Needs JS rendering.', + required: false, + }), + js_instructions: Property.LongText({ + displayName: 'JavaScript Instructions', + description: + 'Actions to run on the page before it is returned, as a JSON array, e.g. [{"click": ".load-more"}, {"wait": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.', + required: false, + }), + custom_headers: Property.Object({ + displayName: 'Custom Headers', + description: 'Headers to send to the target site, such as Cookie or Referer.', + required: false, + }), + session_id: Property.Number({ + displayName: 'Session ID', + description: 'Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.', + required: false, + }), + }, + async run(context) { + const { propsValue } = context; + const outputFormat = propsValue.output_format ?? 'markdown'; + const forwardedHeaders = zenrowsParams.toStringRecord({ value: propsValue.custom_headers }); + const forwardHeaders = Object.keys(forwardedHeaders).length > 0; + const isScreenshot = outputFormat === 'screenshot'; + + const params = zenrowsParams.buildScrapeParams({ + url: propsValue.url, + outputFormat, + adaptive: propsValue.adaptive ?? true, + jsRender: propsValue.js_render, + premiumProxy: propsValue.premium_proxy, + proxyCountry: propsValue.proxy_country, + waitFor: propsValue.wait_for, + wait: propsValue.wait, + jsInstructions: propsValue.js_instructions, + forwardHeaders, + sessionId: propsValue.session_id, + }); + + const response = await zenrowsClient.fetchPage({ + apiKey: context.auth.secret_text, + params, + forwardedHeaders: forwardHeaders ? forwardedHeaders : undefined, + binary: isScreenshot, + }); + + const screenshotUrl = isScreenshot + ? await context.files.write({ + fileName: `zenrows-screenshot-${randomUUID()}.png`, + data: Buffer.from(response.body), + }) + : null; + + const requestCost = zenrowsClient.readHeader({ headers: response.headers, name: 'X-Request-Cost' }); + + return { + url: propsValue.url, + output_format: outputFormat, + content: isScreenshot ? null : String(response.body), + screenshot_url: screenshotUrl, + final_url: zenrowsClient.readHeader({ headers: response.headers, name: 'Zr-Final-Url' }), + request_cost: requestCost === null ? null : Number(requestCost), + request_id: zenrowsClient.readHeader({ headers: response.headers, name: 'X-Request-Id' }), + }; + }, +}); + +function scrapeOutputSchema(): OutputSchema { + return { + fields: [ + { key: 'url', label: 'URL', format: 'url' }, + { key: 'output_format', label: 'Output Format' }, + { key: 'content', label: 'Content' }, + { key: 'screenshot_url', label: 'Screenshot', format: 'image' }, + { key: 'final_url', label: 'Final URL', format: 'url' }, + { key: 'request_cost', label: 'Request Cost', format: 'number' }, + { key: 'request_id', label: 'Request ID' }, + ], + }; +} diff --git a/platforms/activepieces/src/lib/auth.ts b/platforms/activepieces/src/lib/auth.ts new file mode 100644 index 0000000..4f4bc0a --- /dev/null +++ b/platforms/activepieces/src/lib/auth.ts @@ -0,0 +1,27 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { PieceAuth } from '@activepieces/pieces-framework'; +import { zenrowsClient } from './common/client'; + +export const zenrowsAuth = PieceAuth.SecretText({ + displayName: 'API Key', + required: true, + description: ` +**Get your ZenRows API key:** + +1. Sign in at https://app.zenrows.com (or create a free account). +2. Open the **Builder** or **Settings** page. +3. Copy your API key and paste it here. + `, + validate: async ({ auth }) => { + try { + await zenrowsClient.request({ + apiKey: auth, + method: HttpMethod.GET, + path: '/subscriptions/self/details', + }); + return { valid: true }; + } catch (e) { + return { valid: false, error: 'Invalid API key' }; + } + }, +}); diff --git a/platforms/activepieces/src/lib/common/client.ts b/platforms/activepieces/src/lib/common/client.ts new file mode 100644 index 0000000..eaff340 --- /dev/null +++ b/platforms/activepieces/src/lib/common/client.ts @@ -0,0 +1,198 @@ +import { + HttpError, + HttpHeaders, + HttpMessageBody, + HttpMethod, + HttpResponse, + httpClient, +} from '@activepieces/pieces-common'; + +async function request({ + apiKey, + method, + path, + body, +}: { + apiKey: string; + method: HttpMethod; + path: string; + body?: unknown; +}): Promise { + try { + const response = await httpClient.sendRequest({ + method, + url: `${ZENROWS_API_BASE_URL}${path}`, + headers: { + 'X-API-Key': apiKey, + 'Content-Type': 'application/json', + ...CLIENT_HEADERS, + }, + body, + }); + return response.body; + } catch (error) { + throw toZenrowsError({ error }); + } +} + +async function fetchPage({ + apiKey, + params, + forwardedHeaders, + binary, +}: { + apiKey: string; + params: Record; + forwardedHeaders?: Record; + binary?: boolean; +}): Promise { + try { + return await httpClient.sendRequest({ + method: HttpMethod.GET, + url: `${ZENROWS_API_BASE_URL}/`, + queryParams: { apikey: apiKey, ...params }, + headers: forwardedHeaders ?? CLIENT_HEADERS, + timeout: FETCH_TIMEOUT_MS, + responseType: binary ? 'arraybuffer' : undefined, + }); + } catch (error) { + throw toZenrowsError({ error }); + } +} + +function toZenrowsError({ error }: { error: unknown }): unknown { + if (!(error instanceof HttpError)) { + return error; + } + const { status, body } = error.response; + const problem = parseProblem({ body }); + if (!problem) { + return error; + } + const hint = isExtractRefusal({ code: problem.code }) + ? REQS007_HINTS[problem.extractCase ?? ''] ?? ERROR_HINTS['REQS007'] + : ERROR_HINTS[problem.code]; + return new Error( + `ZenRows error ${problem.code} (HTTP ${status}): ${problem.reason}${hint ? ` ${hint}` : ''}` + ); +} + +function readHeader({ + headers, + name, +}: { + headers: HttpHeaders | undefined; + name: string; +}): string | null { + const value = headers?.[name.toLowerCase()]; + if (Array.isArray(value)) { + return value[0] ?? null; + } + return value ?? null; +} + +function parseProblem({ body }: { body: unknown }): ZenrowsProblem | null { + if (typeof body === 'string') { + try { + return parseProblem({ body: JSON.parse(body) }); + } catch { + return null; + } + } + if (!isRecord(body) || typeof body['code'] !== 'string') { + return null; + } + const detail = body['detail']; + const title = body['title']; + return { + code: body['code'], + reason: + typeof detail === 'string' + ? detail + : typeof title === 'string' + ? title + : 'Request failed', + extractCase: readExtractCase({ body }), + }; +} + +// REQS007 (or the older AUTH010) says why Extract refused a page in its +// "reason" field. Until every API release sends it, the title tells the cases +// apart: https://docs.zenrows.com/api-error-codes#REQS007 +function readExtractCase({ body }: { body: Record }): string | undefined { + if (typeof body['reason'] === 'string') { + return body['reason']; + } + const title = body['title']; + if (typeof title !== 'string') { + return undefined; + } + return REQS007_TITLES.find(([prefix]) => title.startsWith(prefix))?.[1]; +} + +function isExtractRefusal({ code }: { code: string }): boolean { + return code === 'REQS007' || code === 'AUTH010'; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +const FETCH_TIMEOUT_MS = 180000; + +const CLIENT_HEADERS = { 'X-ZenRows-Client': 'activepieces' }; + +const AUTH_HINT = 'Check the API key in your ZenRows connection.'; + +// The facts in {{text:…}} come from zenrows-integrations' shared/text/, the same +// wording the Zenrows skills use; each hint adds what to do in Activepieces. +function sentence(fact: string, next: string): string { + return `${fact.charAt(0).toUpperCase()}${fact.slice(1)}. ${next}`; +} + +const ERROR_HINTS: Record = { + AUTH001: AUTH_HINT, + AUTH002: AUTH_HINT, + AUTH003: AUTH_HINT, + AUTH004: sentence({{text:auth004}}, 'Top up or upgrade at https://app.zenrows.com.'), + AUTH014: sentence({{text:auth014}}, 'Raise the cap in the ZenRows dashboard.'), + REQS002: + 'This page needs JS rendering and/or premium proxies. Turn on Adaptive Stealth Mode, or enable them manually.', + REQS007: + 'Extract refused this page, and nothing was charged. Use the CSS selectors method, or see https://docs.zenrows.com/api-error-codes#REQS007.', + AUTH006: sentence({{text:auth006}}, 'Reduce parallel runs and retry.'), + AUTH008: 'Rate limit reached. Wait a moment and retry.', + RESP009: 'ZenRows is temporarily at capacity. Retry in a few seconds.', +}; + +export const ZENROWS_API_BASE_URL = 'https://api.zenrows.com/v1'; + +const REQS007_TITLES: [string, string][] = [ + ['Extract is not prepared for the requested domain', 'domain_not_prepared'], + ['Extract did not recognise this page on', 'page_not_recognised'], + ['Extract is not enabled for the requested domain', 'domain_not_enabled'], +]; + +const REQS007_HINTS: Record = { + domain_not_prepared: + 'ZenRows has not prepared this domain for automatic extraction yet. Prepare it with the Custom API Call action: POST /extract/prepared-domains with the domain and 1 to 5 example URLs. It takes a few minutes and costs no credits. Meanwhile, use the CSS selectors method.', + page_not_recognised: sentence( + {{text:reqs007-page-not-recognised}}, + 'Try a product, article or listing page, or the CSS selectors method. Preparing the domain again does not change it.' + ), + domain_not_enabled: + 'Automatic extraction is not enabled for this site, and preparing it does not help. Use the CSS selectors method, or contact ZenRows support.', +}; + +export const zenrowsClient = { + request, + fetchPage, + toZenrowsError, + readHeader, +}; + +type ZenrowsProblem = { + code: string; + reason: string; + extractCase?: string; +}; diff --git a/platforms/activepieces/src/lib/common/params.ts b/platforms/activepieces/src/lib/common/params.ts new file mode 100644 index 0000000..9affe2f --- /dev/null +++ b/platforms/activepieces/src/lib/common/params.ts @@ -0,0 +1,144 @@ +function buildScrapeParams({ + url, + outputFormat, + adaptive, + jsRender, + premiumProxy, + proxyCountry, + waitFor, + wait, + jsInstructions, + forwardHeaders, + sessionId, +}: ScrapeOptions): Record { + const isScreenshot = outputFormat === 'screenshot'; + const usesBrowser = isScreenshot || (!adaptive && jsRender === true); + const usesPremiumProxy = !adaptive && premiumProxy === true; + const needsBrowser = !isNilOrEmpty(waitFor) || !isNil(wait) || !isNilOrEmpty(jsInstructions); + + if (!adaptive && !usesBrowser && needsBrowser) { + throw new Error( + 'Wait For Selector, Wait (ms) and JavaScript Instructions need JS Rendering. Turn on JS Rendering or Adaptive Stealth Mode.' + ); + } + if (!adaptive && !usesPremiumProxy && !isNilOrEmpty(proxyCountry)) { + throw new Error( + 'Proxy Country needs Premium Proxy. Turn on Premium Proxy or Adaptive Stealth Mode.' + ); + } + + return { + url, + ...(adaptive && !isScreenshot ? { mode: 'auto' } : {}), + ...(usesBrowser ? { js_render: 'true' } : {}), + ...(usesPremiumProxy ? { premium_proxy: 'true' } : {}), + ...(!isNilOrEmpty(proxyCountry) ? { proxy_country: proxyCountry.trim().toLowerCase() } : {}), + ...(!isNilOrEmpty(waitFor) ? { wait_for: waitFor } : {}), + ...(!isNil(wait) ? { wait: String(wait) } : {}), + ...(!isNilOrEmpty(jsInstructions) ? { js_instructions: normalizeJsInstructions({ jsInstructions }) } : {}), + ...(forwardHeaders ? { custom_headers: 'true' } : {}), + ...(!isNil(sessionId) ? { session_id: String(sessionId) } : {}), + ...RESPONSE_TYPE_PARAMS[outputFormat], + }; +} + +function buildExtractParams({ + url, + method, + selectors, + fields, + adaptive, +}: ExtractOptions): Record { + const base = { url, ...(adaptive ? { mode: 'auto' } : {}) }; + if (method === 'css') { + const selectorMap = toStringRecord({ value: selectors }); + if (Object.keys(selectorMap).length === 0) { + throw new Error('Add at least one field with a CSS selector, for example title → h1.'); + } + return { ...base, css_extractor: JSON.stringify(selectorMap) }; + } + const requestedFields = (fields ?? []) + .filter((field): field is string => typeof field === 'string') + .map((field) => field.trim()) + .filter((field) => field.length > 0); + return { + ...base, + extract: 'auto', + ...(requestedFields.length > 0 ? { extract_fields: requestedFields.join(',') } : {}), + }; +} + +function toStringRecord({ + value, +}: { + value: Record | null | undefined; +}): Record { + return Object.fromEntries( + Object.entries(value ?? {}).flatMap(([key, entry]) => + typeof entry === 'string' && entry.trim().length > 0 && key.trim().length > 0 + ? [[key.trim(), entry.trim()]] + : [] + ) + ); +} + +function normalizeJsInstructions({ jsInstructions }: { jsInstructions: string }): string { + try { + const parsed: unknown = JSON.parse(jsInstructions); + if (!Array.isArray(parsed)) { + throw new Error('not an array'); + } + return JSON.stringify(parsed); + } catch { + throw new Error( + 'JavaScript Instructions must be a JSON array, for example [{"click": ".load-more"}, {"wait": 1000}].' + ); + } +} + +function isNil(value: unknown): value is null | undefined { + return value === null || value === undefined; +} + +function isNilOrEmpty(value: string | null | undefined): value is null | undefined { + return isNil(value) || value.trim().length === 0; +} + +const RESPONSE_TYPE_PARAMS: Record> = { + html: {}, + markdown: { response_type: 'markdown' }, + plaintext: { response_type: 'plaintext' }, + screenshot: { screenshot_fullpage: 'true', screenshot_format: 'png' }, +}; + +export const zenrowsParams = { + buildScrapeParams, + buildExtractParams, + toStringRecord, +}; + +export type ScrapeOutputFormat = 'html' | 'markdown' | 'plaintext' | 'screenshot'; + +export type ExtractMethod = 'css' | 'auto'; + +type ScrapeOptions = { + url: string; + outputFormat: ScrapeOutputFormat; + adaptive: boolean; + jsRender?: boolean; + premiumProxy?: boolean; + proxyCountry?: string | null; + waitFor?: string | null; + wait?: number | null; + jsInstructions?: string | null; + forwardHeaders: boolean; + sessionId?: number | null; +}; + +type ExtractOptions = { + url: string; + method: ExtractMethod; + selectors?: Record | null; + fields?: unknown[] | null; + adaptive: boolean; +}; diff --git a/platforms/activepieces/test/params.test.ts b/platforms/activepieces/test/params.test.ts new file mode 100644 index 0000000..f61ff03 --- /dev/null +++ b/platforms/activepieces/test/params.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from 'vitest'; +import { HttpError } from '@activepieces/pieces-common'; +import { zenrowsClient } from '../src/lib/common/client'; +import { zenrowsParams } from '../src/lib/common/params'; + +const URL = 'https://www.scrapingcourse.com/ecommerce/'; + +describe('buildScrapeParams', () => { + it('uses adaptive mode and markdown by default', () => { + expect( + zenrowsParams.buildScrapeParams({ url: URL, outputFormat: 'markdown', adaptive: true, forwardHeaders: false }) + ).toEqual({ url: URL, mode: 'auto', response_type: 'markdown' }); + }); + + it('ignores manual JS rendering and premium proxy in adaptive mode', () => { + expect( + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'html', + adaptive: true, + jsRender: true, + premiumProxy: true, + proxyCountry: 'US', + forwardHeaders: false, + }) + ).toEqual({ url: URL, mode: 'auto', proxy_country: 'us' }); + }); + + it('sends manual options when adaptive mode is off', () => { + expect( + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'plaintext', + adaptive: false, + jsRender: true, + premiumProxy: true, + proxyCountry: 'de', + waitFor: '.product', + wait: 500, + jsInstructions: '[{"click": ".more"}]', + forwardHeaders: true, + sessionId: 42, + }) + ).toEqual({ + url: URL, + js_render: 'true', + premium_proxy: 'true', + proxy_country: 'de', + wait_for: '.product', + wait: '500', + js_instructions: '[{"click":".more"}]', + custom_headers: 'true', + session_id: '42', + response_type: 'plaintext', + }); + }); + + it('forces JS rendering and skips adaptive mode for screenshots', () => { + expect( + zenrowsParams.buildScrapeParams({ url: URL, outputFormat: 'screenshot', adaptive: true, forwardHeaders: false }) + ).toEqual({ url: URL, js_render: 'true', screenshot_fullpage: 'true', screenshot_format: 'png' }); + }); + + it('rejects browser options without JS rendering in manual mode', () => { + expect(() => + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'html', + adaptive: false, + waitFor: '.product', + forwardHeaders: false, + }) + ).toThrow(/need JS Rendering/); + }); + + it('rejects a proxy country without premium proxy in manual mode', () => { + expect(() => + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'html', + adaptive: false, + proxyCountry: 'us', + forwardHeaders: false, + }) + ).toThrow(/needs Premium Proxy/); + }); + + it('rejects JavaScript instructions that are not a JSON array', () => { + expect(() => + zenrowsParams.buildScrapeParams({ + url: URL, + outputFormat: 'html', + adaptive: true, + jsInstructions: '{"click": ".more"}', + forwardHeaders: false, + }) + ).toThrow(/JSON array/); + }); +}); + +describe('buildExtractParams', () => { + it('sends CSS selectors as a JSON map', () => { + expect( + zenrowsParams.buildExtractParams({ + url: URL, + method: 'css', + selectors: { title: 'h1', price: ' .price ', empty: '', count: 3 }, + adaptive: true, + }) + ).toEqual({ url: URL, mode: 'auto', css_extractor: '{"title":"h1","price":".price"}' }); + }); + + it('requires at least one selector', () => { + expect(() => + zenrowsParams.buildExtractParams({ url: URL, method: 'css', selectors: {}, adaptive: false }) + ).toThrow(/at least one field/); + }); + + it('uses automatic extraction with optional fields', () => { + expect( + zenrowsParams.buildExtractParams({ + url: URL, + method: 'auto', + fields: ['products.name', ' products.price ', '', 7], + adaptive: false, + }) + ).toEqual({ url: URL, extract: 'auto', extract_fields: 'products.name,products.price' }); + }); +}); + +describe('toZenrowsError', () => { + it('turns a ZenRows problem response into a readable error with a hint', () => { + const error = new HttpError( + {}, + { + status: 402, + responseBody: JSON.stringify({ code: 'AUTH004', title: 'Usage exceeded (AUTH004)', detail: 'No credits left.' }), + } + ); + const mapped = zenrowsClient.toZenrowsError({ error }); + expect(mapped).toBeInstanceOf(Error); + expect(String(mapped)).toContain('ZenRows error AUTH004 (HTTP 402): No credits left.'); + expect(String(mapped)).toContain('credit allowance is spent'); + }); + + it.each([ + [{ reason: 'domain_not_prepared' }, 'POST /extract/prepared-domains'], + [{ title: 'Extract did not recognise this page on example.com' }, 'Preparing the domain again does not change it'], + [{ title: 'Extract is not enabled for the requested domain' }, 'contact ZenRows support'], + [{ title: 'Something new' }, 'api-error-codes#REQS007'], + ])('gives the REQS007 hint for its case (%o)', (fields, hint) => { + const error = new HttpError({}, { status: 403, responseBody: JSON.stringify({ code: 'REQS007', ...fields }) }); + expect(String(zenrowsClient.toZenrowsError({ error }))).toContain(hint); + }); + + it('keeps errors that are not ZenRows problems unchanged', () => { + const error = new HttpError({}, { status: 500, responseBody: 'oops' }); + expect(zenrowsClient.toZenrowsError({ error })).toBe(error); + const plain = new Error('network down'); + expect(zenrowsClient.toZenrowsError({ error: plain })).toBe(plain); + }); +}); diff --git a/platforms/activepieces/tsconfig.json b/platforms/activepieces/tsconfig.json new file mode 100644 index 0000000..b512ca3 --- /dev/null +++ b/platforms/activepieces/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} \ No newline at end of file diff --git a/platforms/activepieces/tsconfig.lib.json b/platforms/activepieces/tsconfig.lib.json new file mode 100644 index 0000000..8b4345e --- /dev/null +++ b/platforms/activepieces/tsconfig.lib.json @@ -0,0 +1,22 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "baseUrl": ".", + "paths": {}, + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "types": [ + "node" + ] + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "jest.config.ts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} \ No newline at end of file diff --git a/platforms/activepieces/vars.yaml b/platforms/activepieces/vars.yaml new file mode 100644 index 0000000..eddc5cc --- /dev/null +++ b/platforms/activepieces/vars.yaml @@ -0,0 +1,8 @@ +# A code host: the Zenrows piece for Activepieces, rendered into dist/activepieces/ +# exactly as it sits in activepieces' packages/pieces/community/zenrows. +kind: code +host: "Activepieces" +host_full: "Activepieces" +# The activepieces commit the piece is built and tested against (scripts/check.sh). +# Moving it is its own PR, so CI shows whether their newer code breaks the piece. +upstream: "6f4be340d7c4bef9fb6ff874ca72224e4933d3d2" diff --git a/platforms/activepieces/vitest.config.ts b/platforms/activepieces/vitest.config.ts new file mode 100644 index 0000000..79c9726 --- /dev/null +++ b/platforms/activepieces/vitest.config.ts @@ -0,0 +1,19 @@ +import path from 'path' +import { defineConfig } from 'vitest/config' + +const repoRoot = path.resolve(__dirname, '../../../..') + +export default defineConfig({ + root: __dirname, + test: { + globals: true, + environment: 'node', + include: ['test/**/*.test.ts'], + }, + resolve: { + alias: { + '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'), + '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'), + }, + }, +}) From d7b1721bc8250537a32ecd29460d38acc66cb17d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Thu, 12 Nov 2026 13:52:05 +0000 Subject: [PATCH 3/8] fix(activepieces): word a comment so it isn't mistaken for an unfilled token --- dist/activepieces/src/lib/common/client.ts | 2 +- platforms/activepieces/src/lib/common/client.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/activepieces/src/lib/common/client.ts b/dist/activepieces/src/lib/common/client.ts index b4b6dfd..b01ec9e 100644 --- a/dist/activepieces/src/lib/common/client.ts +++ b/dist/activepieces/src/lib/common/client.ts @@ -144,7 +144,7 @@ const CLIENT_HEADERS = { 'X-ZenRows-Client': 'activepieces' }; const AUTH_HINT = 'Check the API key in your ZenRows connection.'; -// The facts in {{text:…}} come from zenrows-integrations' shared/text/, the same +// The facts below come from zenrows-integrations' shared/text/, the same // wording the Zenrows skills use; each hint adds what to do in Activepieces. function sentence(fact: string, next: string): string { return `${fact.charAt(0).toUpperCase()}${fact.slice(1)}. ${next}`; diff --git a/platforms/activepieces/src/lib/common/client.ts b/platforms/activepieces/src/lib/common/client.ts index eaff340..462a1c5 100644 --- a/platforms/activepieces/src/lib/common/client.ts +++ b/platforms/activepieces/src/lib/common/client.ts @@ -144,7 +144,7 @@ const CLIENT_HEADERS = { 'X-ZenRows-Client': 'activepieces' }; const AUTH_HINT = 'Check the API key in your ZenRows connection.'; -// The facts in {{text:…}} come from zenrows-integrations' shared/text/, the same +// The facts below come from zenrows-integrations' shared/text/, the same // wording the Zenrows skills use; each hint adds what to do in Activepieces. function sentence(fact: string, next: string): string { return `${fact.charAt(0).toUpperCase()}${fact.slice(1)}. ${next}`; From fa122c3a9bce152e145c7314cd5c0588f467fe72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Thu, 12 Nov 2026 14:27:22 +0000 Subject: [PATCH 4/8] refactor(activepieces): keep the piece to what a first version needs Drop the option-combination checks and their tests (the API validates and explains), the Extract-refusal parsing and most error hints, and six power-user options on Scrape URL (Custom API Call covers them). The error message keeps the API's code and reason, plus the credit hints from shared/text. --- dist/activepieces/package.json | 6 +- dist/activepieces/src/i18n/translation.json | 20 +-- .../lib/actions/extract-structured-data.ts | 19 +- .../src/lib/actions/scrape-url.ts | 89 ++++------ dist/activepieces/src/lib/common/client.ts | 101 ++--------- dist/activepieces/src/lib/common/params.ts | 144 ---------------- dist/activepieces/test/params.test.ts | 162 ------------------ dist/activepieces/vitest.config.ts | 19 -- platforms/activepieces/README.md | 2 +- platforms/activepieces/package.json | 6 +- .../activepieces/src/i18n/translation.json | 20 +-- .../lib/actions/extract-structured-data.ts | 19 +- .../src/lib/actions/scrape-url.ts | 89 ++++------ .../activepieces/src/lib/common/client.ts | 101 ++--------- .../activepieces/src/lib/common/params.ts | 144 ---------------- platforms/activepieces/test/params.test.ts | 162 ------------------ platforms/activepieces/vitest.config.ts | 19 -- 17 files changed, 135 insertions(+), 987 deletions(-) delete mode 100644 dist/activepieces/src/lib/common/params.ts delete mode 100644 dist/activepieces/test/params.test.ts delete mode 100644 dist/activepieces/vitest.config.ts delete mode 100644 platforms/activepieces/src/lib/common/params.ts delete mode 100644 platforms/activepieces/test/params.test.ts delete mode 100644 platforms/activepieces/vitest.config.ts diff --git a/dist/activepieces/package.json b/dist/activepieces/package.json index d118817..8c7cb33 100644 --- a/dist/activepieces/package.json +++ b/dist/activepieces/package.json @@ -11,13 +11,11 @@ "@activepieces/core-utils": "workspace:*" }, "devDependencies": { - "tslib": "2.6.2", - "vitest": "3.2.6" + "tslib": "2.6.2" }, "scripts": { "build": "tsc -p tsconfig.lib.json && cp package.json dist/", "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle", - "lint": "eslint 'src/**/*.ts'", - "test": "vitest run" + "lint": "eslint 'src/**/*.ts'" } } diff --git a/dist/activepieces/src/i18n/translation.json b/dist/activepieces/src/i18n/translation.json index acdef80..1a304f9 100644 --- a/dist/activepieces/src/i18n/translation.json +++ b/dist/activepieces/src/i18n/translation.json @@ -12,14 +12,8 @@ "URL": "URL", "Output Format": "Output Format", "Adaptive Stealth Mode": "Adaptive Stealth Mode", - "JS Rendering": "JS Rendering", - "Premium Proxy": "Premium Proxy", - "Proxy Country": "Proxy Country", - "Wait For Selector": "Wait For Selector", - "Wait (ms)": "Wait (ms)", "JavaScript Instructions": "JavaScript Instructions", "Custom Headers": "Custom Headers", - "Session ID": "Session ID", "Extraction Method": "Extraction Method", "CSS Selectors": "CSS Selectors", "Fields": "Fields", @@ -34,15 +28,7 @@ "Follow redirects": "Follow redirects", "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/": "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/", "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.": "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.", - "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.", - "Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.": "Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.", - "Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.": "Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.", - "Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.": "Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.", - "A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.": "A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.", - "Extra milliseconds to wait after the page loads. Needs JS rendering.": "Extra milliseconds to wait after the page loads. Needs JS rendering.", - "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.", "Headers to send to the target site, such as Cookie or Referer.": "Headers to send to the target site, such as Cookie or Referer.", - "Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.": "Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.", "The full address of the page to extract data from.": "The full address of the page to extract data from.", "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.": "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.", "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.": "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.", @@ -66,5 +52,7 @@ "DELETE": "DELETE", "HEAD": "HEAD", "Request": "Request", - "Data to send": "Data to send" -} \ No newline at end of file + "Data to send": "Data to send", + "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.", + "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list." +} diff --git a/dist/activepieces/src/lib/actions/extract-structured-data.ts b/dist/activepieces/src/lib/actions/extract-structured-data.ts index ea6f050..40c2e90 100644 --- a/dist/activepieces/src/lib/actions/extract-structured-data.ts +++ b/dist/activepieces/src/lib/actions/extract-structured-data.ts @@ -1,7 +1,6 @@ import { createAction, Property } from '@activepieces/pieces-framework'; import { zenrowsAuth } from '../auth'; import { zenrowsClient } from '../common/client'; -import { ExtractMethod, zenrowsParams } from '../common/params'; export const extractStructuredDataAction = createAction({ auth: zenrowsAuth, @@ -21,7 +20,7 @@ export const extractStructuredDataAction = createAction({ description: 'The full address of the page to extract data from.', required: true, }), - method: Property.StaticDropdown({ + method: Property.StaticDropdown<'css' | 'auto'>({ displayName: 'Extraction Method', description: 'CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.', @@ -57,16 +56,16 @@ export const extractStructuredDataAction = createAction({ }, async run(context) { const { propsValue } = context; - const params = zenrowsParams.buildExtractParams({ - url: propsValue.url, - method: propsValue.method ?? 'css', - selectors: propsValue.selectors, - fields: propsValue.fields, - adaptive: propsValue.adaptive ?? true, - }); + const fields = (propsValue.fields ?? []).filter((f): f is string => typeof f === 'string' && f.trim() !== ''); const response = await zenrowsClient.fetchPage({ apiKey: context.auth.secret_text, - params, + params: { + url: propsValue.url, + ...(propsValue.adaptive !== false ? { mode: 'auto' } : {}), + ...(propsValue.method === 'auto' + ? { extract: 'auto', ...(fields.length ? { extract_fields: fields.join(',') } : {}) } + : { css_extractor: JSON.stringify(propsValue.selectors ?? {}) }), + }, }); return response.body; }, diff --git a/dist/activepieces/src/lib/actions/scrape-url.ts b/dist/activepieces/src/lib/actions/scrape-url.ts index c57a478..c16516d 100644 --- a/dist/activepieces/src/lib/actions/scrape-url.ts +++ b/dist/activepieces/src/lib/actions/scrape-url.ts @@ -2,7 +2,17 @@ import { randomUUID } from 'crypto'; import { createAction, OutputSchema, Property } from '@activepieces/pieces-framework'; import { zenrowsAuth } from '../auth'; import { zenrowsClient } from '../common/client'; -import { ScrapeOutputFormat, zenrowsParams } from '../common/params'; + +type OutputFormat = 'markdown' | 'html' | 'plaintext' | 'screenshot'; + +// What each format adds to the request. A screenshot needs a real browser, so +// it sets js_render itself instead of leaving the choice to Adaptive Stealth Mode. +const FORMAT_PARAMS: Record> = { + markdown: { response_type: 'markdown' }, + html: {}, + plaintext: { response_type: 'plaintext' }, + screenshot: { js_render: 'true', screenshot_fullpage: 'true', screenshot_format: 'png' }, +}; export const scrapeUrlAction = createAction({ auth: zenrowsAuth, @@ -24,7 +34,7 @@ export const scrapeUrlAction = createAction({ description: 'The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/', required: true, }), - output_format: Property.StaticDropdown({ + output_format: Property.StaticDropdown({ displayName: 'Output Format', description: 'Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.', required: true, @@ -42,44 +52,14 @@ export const scrapeUrlAction = createAction({ adaptive: Property.Checkbox({ displayName: 'Adaptive Stealth Mode', description: - 'Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.', + 'Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.', required: false, defaultValue: true, }), - js_render: Property.Checkbox({ - displayName: 'JS Rendering', - description: - 'Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.', - required: false, - defaultValue: false, - }), - premium_proxy: Property.Checkbox({ - displayName: 'Premium Proxy', - description: - 'Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.', - required: false, - defaultValue: false, - }), - proxy_country: Property.ShortText({ - displayName: 'Proxy Country', - description: - 'Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.', - required: false, - }), - wait_for: Property.ShortText({ - displayName: 'Wait For Selector', - description: 'A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.', - required: false, - }), - wait: Property.Number({ - displayName: 'Wait (ms)', - description: 'Extra milliseconds to wait after the page loads. Needs JS rendering.', - required: false, - }), js_instructions: Property.LongText({ displayName: 'JavaScript Instructions', description: - 'Actions to run on the page before it is returned, as a JSON array, e.g. [{"click": ".load-more"}, {"wait": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.', + 'Actions to run on the page before it is returned, as a JSON array, e.g. [{"click": ".load-more"}, {"wait": 1000}]. See https://docs.zenrows.com for the full list.', required: false, }), custom_headers: Property.Object({ @@ -87,36 +67,24 @@ export const scrapeUrlAction = createAction({ description: 'Headers to send to the target site, such as Cookie or Referer.', required: false, }), - session_id: Property.Number({ - displayName: 'Session ID', - description: 'Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.', - required: false, - }), }, async run(context) { const { propsValue } = context; const outputFormat = propsValue.output_format ?? 'markdown'; - const forwardedHeaders = zenrowsParams.toStringRecord({ value: propsValue.custom_headers }); - const forwardHeaders = Object.keys(forwardedHeaders).length > 0; const isScreenshot = outputFormat === 'screenshot'; - - const params = zenrowsParams.buildScrapeParams({ - url: propsValue.url, - outputFormat, - adaptive: propsValue.adaptive ?? true, - jsRender: propsValue.js_render, - premiumProxy: propsValue.premium_proxy, - proxyCountry: propsValue.proxy_country, - waitFor: propsValue.wait_for, - wait: propsValue.wait, - jsInstructions: propsValue.js_instructions, - forwardHeaders, - sessionId: propsValue.session_id, - }); + const forwardedHeaders = stringValues(propsValue.custom_headers); + const forwardHeaders = Object.keys(forwardedHeaders).length > 0; + const jsInstructions = propsValue.js_instructions?.trim(); const response = await zenrowsClient.fetchPage({ apiKey: context.auth.secret_text, - params, + params: { + url: propsValue.url, + ...(propsValue.adaptive !== false && !isScreenshot ? { mode: 'auto' } : {}), + ...FORMAT_PARAMS[outputFormat], + ...(jsInstructions ? { js_instructions: jsInstructions } : {}), + ...(forwardHeaders ? { custom_headers: 'true' } : {}), + }, forwardedHeaders: forwardHeaders ? forwardedHeaders : undefined, binary: isScreenshot, }); @@ -142,6 +110,15 @@ export const scrapeUrlAction = createAction({ }, }); +// The headers the form holds as text, trimmed; other values are left out. +function stringValues(value: Record | null | undefined): Record { + return Object.fromEntries( + Object.entries(value ?? {}).flatMap(([key, entry]) => + typeof entry === 'string' && entry.trim() && key.trim() ? [[key.trim(), entry.trim()]] : [] + ) + ); +} + function scrapeOutputSchema(): OutputSchema { return { fields: [ diff --git a/dist/activepieces/src/lib/common/client.ts b/dist/activepieces/src/lib/common/client.ts index b01ec9e..94878cc 100644 --- a/dist/activepieces/src/lib/common/client.ts +++ b/dist/activepieces/src/lib/common/client.ts @@ -60,6 +60,9 @@ async function fetchPage({ } } +// The API answers every failure with a code and a reason +// (https://docs.zenrows.com/api-error-codes); the message keeps both, and adds +// what to do next for the codes where Activepieces changes the answer. function toZenrowsError({ error }: { error: unknown }): unknown { if (!(error instanceof HttpError)) { return error; @@ -69,29 +72,11 @@ function toZenrowsError({ error }: { error: unknown }): unknown { if (!problem) { return error; } - const hint = isExtractRefusal({ code: problem.code }) - ? REQS007_HINTS[problem.extractCase ?? ''] ?? ERROR_HINTS['REQS007'] - : ERROR_HINTS[problem.code]; - return new Error( - `ZenRows error ${problem.code} (HTTP ${status}): ${problem.reason}${hint ? ` ${hint}` : ''}` - ); + const hint = ERROR_HINTS[problem.code]; + return new Error(`ZenRows error ${problem.code} (HTTP ${status}): ${problem.reason}${hint ? ` ${hint}` : ''}`); } -function readHeader({ - headers, - name, -}: { - headers: HttpHeaders | undefined; - name: string; -}): string | null { - const value = headers?.[name.toLowerCase()]; - if (Array.isArray(value)) { - return value[0] ?? null; - } - return value ?? null; -} - -function parseProblem({ body }: { body: unknown }): ZenrowsProblem | null { +function parseProblem({ body }: { body: unknown }): { code: string; reason: string } | null { if (typeof body === 'string') { try { return parseProblem({ body: JSON.parse(body) }); @@ -99,43 +84,25 @@ function parseProblem({ body }: { body: unknown }): ZenrowsProblem | null { return null; } } - if (!isRecord(body) || typeof body['code'] !== 'string') { + if (typeof body !== 'object' || body === null || Array.isArray(body)) { + return null; + } + const { code, detail, title } = body as Record; + if (typeof code !== 'string') { return null; } - const detail = body['detail']; - const title = body['title']; return { - code: body['code'], - reason: - typeof detail === 'string' - ? detail - : typeof title === 'string' - ? title - : 'Request failed', - extractCase: readExtractCase({ body }), + code, + reason: typeof detail === 'string' ? detail : typeof title === 'string' ? title : 'Request failed', }; } -// REQS007 (or the older AUTH010) says why Extract refused a page in its -// "reason" field. Until every API release sends it, the title tells the cases -// apart: https://docs.zenrows.com/api-error-codes#REQS007 -function readExtractCase({ body }: { body: Record }): string | undefined { - if (typeof body['reason'] === 'string') { - return body['reason']; - } - const title = body['title']; - if (typeof title !== 'string') { - return undefined; +function readHeader({ headers, name }: { headers: HttpHeaders | undefined; name: string }): string | null { + const value = headers?.[name.toLowerCase()]; + if (Array.isArray(value)) { + return value[0] ?? null; } - return REQS007_TITLES.find(([prefix]) => title.startsWith(prefix))?.[1]; -} - -function isExtractRefusal({ code }: { code: string }): boolean { - return code === 'REQS007' || code === 'AUTH010'; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); + return value ?? null; } const FETCH_TIMEOUT_MS = 180000; @@ -156,43 +123,13 @@ const ERROR_HINTS: Record = { AUTH003: AUTH_HINT, AUTH004: sentence("the account's credit allowance is spent", 'Top up or upgrade at https://app.zenrows.com.'), AUTH014: sentence("this API key reached its credit cap", 'Raise the cap in the ZenRows dashboard.'), - REQS002: - 'This page needs JS rendering and/or premium proxies. Turn on Adaptive Stealth Mode, or enable them manually.', - REQS007: - 'Extract refused this page, and nothing was charged. Use the CSS selectors method, or see https://docs.zenrows.com/api-error-codes#REQS007.', - AUTH006: sentence("too many requests are running at once", 'Reduce parallel runs and retry.'), - AUTH008: 'Rate limit reached. Wait a moment and retry.', - RESP009: 'ZenRows is temporarily at capacity. Retry in a few seconds.', + REQS002: 'This page needs JS rendering or premium proxies: turn on Adaptive Stealth Mode.', }; export const ZENROWS_API_BASE_URL = 'https://api.zenrows.com/v1'; -const REQS007_TITLES: [string, string][] = [ - ['Extract is not prepared for the requested domain', 'domain_not_prepared'], - ['Extract did not recognise this page on', 'page_not_recognised'], - ['Extract is not enabled for the requested domain', 'domain_not_enabled'], -]; - -const REQS007_HINTS: Record = { - domain_not_prepared: - 'ZenRows has not prepared this domain for automatic extraction yet. Prepare it with the Custom API Call action: POST /extract/prepared-domains with the domain and 1 to 5 example URLs. It takes a few minutes and costs no credits. Meanwhile, use the CSS selectors method.', - page_not_recognised: sentence( - "the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example)", - 'Try a product, article or listing page, or the CSS selectors method. Preparing the domain again does not change it.' - ), - domain_not_enabled: - 'Automatic extraction is not enabled for this site, and preparing it does not help. Use the CSS selectors method, or contact ZenRows support.', -}; - export const zenrowsClient = { request, fetchPage, - toZenrowsError, readHeader, }; - -type ZenrowsProblem = { - code: string; - reason: string; - extractCase?: string; -}; diff --git a/dist/activepieces/src/lib/common/params.ts b/dist/activepieces/src/lib/common/params.ts deleted file mode 100644 index 9affe2f..0000000 --- a/dist/activepieces/src/lib/common/params.ts +++ /dev/null @@ -1,144 +0,0 @@ -function buildScrapeParams({ - url, - outputFormat, - adaptive, - jsRender, - premiumProxy, - proxyCountry, - waitFor, - wait, - jsInstructions, - forwardHeaders, - sessionId, -}: ScrapeOptions): Record { - const isScreenshot = outputFormat === 'screenshot'; - const usesBrowser = isScreenshot || (!adaptive && jsRender === true); - const usesPremiumProxy = !adaptive && premiumProxy === true; - const needsBrowser = !isNilOrEmpty(waitFor) || !isNil(wait) || !isNilOrEmpty(jsInstructions); - - if (!adaptive && !usesBrowser && needsBrowser) { - throw new Error( - 'Wait For Selector, Wait (ms) and JavaScript Instructions need JS Rendering. Turn on JS Rendering or Adaptive Stealth Mode.' - ); - } - if (!adaptive && !usesPremiumProxy && !isNilOrEmpty(proxyCountry)) { - throw new Error( - 'Proxy Country needs Premium Proxy. Turn on Premium Proxy or Adaptive Stealth Mode.' - ); - } - - return { - url, - ...(adaptive && !isScreenshot ? { mode: 'auto' } : {}), - ...(usesBrowser ? { js_render: 'true' } : {}), - ...(usesPremiumProxy ? { premium_proxy: 'true' } : {}), - ...(!isNilOrEmpty(proxyCountry) ? { proxy_country: proxyCountry.trim().toLowerCase() } : {}), - ...(!isNilOrEmpty(waitFor) ? { wait_for: waitFor } : {}), - ...(!isNil(wait) ? { wait: String(wait) } : {}), - ...(!isNilOrEmpty(jsInstructions) ? { js_instructions: normalizeJsInstructions({ jsInstructions }) } : {}), - ...(forwardHeaders ? { custom_headers: 'true' } : {}), - ...(!isNil(sessionId) ? { session_id: String(sessionId) } : {}), - ...RESPONSE_TYPE_PARAMS[outputFormat], - }; -} - -function buildExtractParams({ - url, - method, - selectors, - fields, - adaptive, -}: ExtractOptions): Record { - const base = { url, ...(adaptive ? { mode: 'auto' } : {}) }; - if (method === 'css') { - const selectorMap = toStringRecord({ value: selectors }); - if (Object.keys(selectorMap).length === 0) { - throw new Error('Add at least one field with a CSS selector, for example title → h1.'); - } - return { ...base, css_extractor: JSON.stringify(selectorMap) }; - } - const requestedFields = (fields ?? []) - .filter((field): field is string => typeof field === 'string') - .map((field) => field.trim()) - .filter((field) => field.length > 0); - return { - ...base, - extract: 'auto', - ...(requestedFields.length > 0 ? { extract_fields: requestedFields.join(',') } : {}), - }; -} - -function toStringRecord({ - value, -}: { - value: Record | null | undefined; -}): Record { - return Object.fromEntries( - Object.entries(value ?? {}).flatMap(([key, entry]) => - typeof entry === 'string' && entry.trim().length > 0 && key.trim().length > 0 - ? [[key.trim(), entry.trim()]] - : [] - ) - ); -} - -function normalizeJsInstructions({ jsInstructions }: { jsInstructions: string }): string { - try { - const parsed: unknown = JSON.parse(jsInstructions); - if (!Array.isArray(parsed)) { - throw new Error('not an array'); - } - return JSON.stringify(parsed); - } catch { - throw new Error( - 'JavaScript Instructions must be a JSON array, for example [{"click": ".load-more"}, {"wait": 1000}].' - ); - } -} - -function isNil(value: unknown): value is null | undefined { - return value === null || value === undefined; -} - -function isNilOrEmpty(value: string | null | undefined): value is null | undefined { - return isNil(value) || value.trim().length === 0; -} - -const RESPONSE_TYPE_PARAMS: Record> = { - html: {}, - markdown: { response_type: 'markdown' }, - plaintext: { response_type: 'plaintext' }, - screenshot: { screenshot_fullpage: 'true', screenshot_format: 'png' }, -}; - -export const zenrowsParams = { - buildScrapeParams, - buildExtractParams, - toStringRecord, -}; - -export type ScrapeOutputFormat = 'html' | 'markdown' | 'plaintext' | 'screenshot'; - -export type ExtractMethod = 'css' | 'auto'; - -type ScrapeOptions = { - url: string; - outputFormat: ScrapeOutputFormat; - adaptive: boolean; - jsRender?: boolean; - premiumProxy?: boolean; - proxyCountry?: string | null; - waitFor?: string | null; - wait?: number | null; - jsInstructions?: string | null; - forwardHeaders: boolean; - sessionId?: number | null; -}; - -type ExtractOptions = { - url: string; - method: ExtractMethod; - selectors?: Record | null; - fields?: unknown[] | null; - adaptive: boolean; -}; diff --git a/dist/activepieces/test/params.test.ts b/dist/activepieces/test/params.test.ts deleted file mode 100644 index f61ff03..0000000 --- a/dist/activepieces/test/params.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { HttpError } from '@activepieces/pieces-common'; -import { zenrowsClient } from '../src/lib/common/client'; -import { zenrowsParams } from '../src/lib/common/params'; - -const URL = 'https://www.scrapingcourse.com/ecommerce/'; - -describe('buildScrapeParams', () => { - it('uses adaptive mode and markdown by default', () => { - expect( - zenrowsParams.buildScrapeParams({ url: URL, outputFormat: 'markdown', adaptive: true, forwardHeaders: false }) - ).toEqual({ url: URL, mode: 'auto', response_type: 'markdown' }); - }); - - it('ignores manual JS rendering and premium proxy in adaptive mode', () => { - expect( - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'html', - adaptive: true, - jsRender: true, - premiumProxy: true, - proxyCountry: 'US', - forwardHeaders: false, - }) - ).toEqual({ url: URL, mode: 'auto', proxy_country: 'us' }); - }); - - it('sends manual options when adaptive mode is off', () => { - expect( - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'plaintext', - adaptive: false, - jsRender: true, - premiumProxy: true, - proxyCountry: 'de', - waitFor: '.product', - wait: 500, - jsInstructions: '[{"click": ".more"}]', - forwardHeaders: true, - sessionId: 42, - }) - ).toEqual({ - url: URL, - js_render: 'true', - premium_proxy: 'true', - proxy_country: 'de', - wait_for: '.product', - wait: '500', - js_instructions: '[{"click":".more"}]', - custom_headers: 'true', - session_id: '42', - response_type: 'plaintext', - }); - }); - - it('forces JS rendering and skips adaptive mode for screenshots', () => { - expect( - zenrowsParams.buildScrapeParams({ url: URL, outputFormat: 'screenshot', adaptive: true, forwardHeaders: false }) - ).toEqual({ url: URL, js_render: 'true', screenshot_fullpage: 'true', screenshot_format: 'png' }); - }); - - it('rejects browser options without JS rendering in manual mode', () => { - expect(() => - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'html', - adaptive: false, - waitFor: '.product', - forwardHeaders: false, - }) - ).toThrow(/need JS Rendering/); - }); - - it('rejects a proxy country without premium proxy in manual mode', () => { - expect(() => - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'html', - adaptive: false, - proxyCountry: 'us', - forwardHeaders: false, - }) - ).toThrow(/needs Premium Proxy/); - }); - - it('rejects JavaScript instructions that are not a JSON array', () => { - expect(() => - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'html', - adaptive: true, - jsInstructions: '{"click": ".more"}', - forwardHeaders: false, - }) - ).toThrow(/JSON array/); - }); -}); - -describe('buildExtractParams', () => { - it('sends CSS selectors as a JSON map', () => { - expect( - zenrowsParams.buildExtractParams({ - url: URL, - method: 'css', - selectors: { title: 'h1', price: ' .price ', empty: '', count: 3 }, - adaptive: true, - }) - ).toEqual({ url: URL, mode: 'auto', css_extractor: '{"title":"h1","price":".price"}' }); - }); - - it('requires at least one selector', () => { - expect(() => - zenrowsParams.buildExtractParams({ url: URL, method: 'css', selectors: {}, adaptive: false }) - ).toThrow(/at least one field/); - }); - - it('uses automatic extraction with optional fields', () => { - expect( - zenrowsParams.buildExtractParams({ - url: URL, - method: 'auto', - fields: ['products.name', ' products.price ', '', 7], - adaptive: false, - }) - ).toEqual({ url: URL, extract: 'auto', extract_fields: 'products.name,products.price' }); - }); -}); - -describe('toZenrowsError', () => { - it('turns a ZenRows problem response into a readable error with a hint', () => { - const error = new HttpError( - {}, - { - status: 402, - responseBody: JSON.stringify({ code: 'AUTH004', title: 'Usage exceeded (AUTH004)', detail: 'No credits left.' }), - } - ); - const mapped = zenrowsClient.toZenrowsError({ error }); - expect(mapped).toBeInstanceOf(Error); - expect(String(mapped)).toContain('ZenRows error AUTH004 (HTTP 402): No credits left.'); - expect(String(mapped)).toContain('credit allowance is spent'); - }); - - it.each([ - [{ reason: 'domain_not_prepared' }, 'POST /extract/prepared-domains'], - [{ title: 'Extract did not recognise this page on example.com' }, 'Preparing the domain again does not change it'], - [{ title: 'Extract is not enabled for the requested domain' }, 'contact ZenRows support'], - [{ title: 'Something new' }, 'api-error-codes#REQS007'], - ])('gives the REQS007 hint for its case (%o)', (fields, hint) => { - const error = new HttpError({}, { status: 403, responseBody: JSON.stringify({ code: 'REQS007', ...fields }) }); - expect(String(zenrowsClient.toZenrowsError({ error }))).toContain(hint); - }); - - it('keeps errors that are not ZenRows problems unchanged', () => { - const error = new HttpError({}, { status: 500, responseBody: 'oops' }); - expect(zenrowsClient.toZenrowsError({ error })).toBe(error); - const plain = new Error('network down'); - expect(zenrowsClient.toZenrowsError({ error: plain })).toBe(plain); - }); -}); diff --git a/dist/activepieces/vitest.config.ts b/dist/activepieces/vitest.config.ts deleted file mode 100644 index 79c9726..0000000 --- a/dist/activepieces/vitest.config.ts +++ /dev/null @@ -1,19 +0,0 @@ -import path from 'path' -import { defineConfig } from 'vitest/config' - -const repoRoot = path.resolve(__dirname, '../../../..') - -export default defineConfig({ - root: __dirname, - test: { - globals: true, - environment: 'node', - include: ['test/**/*.test.ts'], - }, - resolve: { - alias: { - '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'), - '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'), - }, - }, -}) diff --git a/platforms/activepieces/README.md b/platforms/activepieces/README.md index f6c59a3..02a2251 100644 --- a/platforms/activepieces/README.md +++ b/platforms/activepieces/README.md @@ -6,7 +6,7 @@ This is a code host (`kind: code` in `vars.yaml`). `npm run render` writes `dist ## Shared wording -Facts that the Zenrows skills state too come from `shared/text/` through `{{text:name}}`, which renders as a quoted string. Today that's the meaning of `AUTH004`, `AUTH014`, `AUTH006` and REQS007's `page_not_recognised`, in `src/lib/common/client.ts`. A wording fix in `shared/text/` reaches the skills and the piece in the same render. +Facts that the Zenrows skills state too come from `shared/text/` through `{{text:name}}`, which renders as a quoted string. Today that's the meaning of `AUTH004` and `AUTH014`, in `src/lib/common/client.ts`. The piece keeps its forms short and passes the rest through: the API validates option combinations and explains every error, so the piece doesn't repeat that. A wording fix in `shared/text/` reaches the skills and the piece in the same render. ## Check it diff --git a/platforms/activepieces/package.json b/platforms/activepieces/package.json index d118817..8c7cb33 100644 --- a/platforms/activepieces/package.json +++ b/platforms/activepieces/package.json @@ -11,13 +11,11 @@ "@activepieces/core-utils": "workspace:*" }, "devDependencies": { - "tslib": "2.6.2", - "vitest": "3.2.6" + "tslib": "2.6.2" }, "scripts": { "build": "tsc -p tsconfig.lib.json && cp package.json dist/", "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle", - "lint": "eslint 'src/**/*.ts'", - "test": "vitest run" + "lint": "eslint 'src/**/*.ts'" } } diff --git a/platforms/activepieces/src/i18n/translation.json b/platforms/activepieces/src/i18n/translation.json index acdef80..1a304f9 100644 --- a/platforms/activepieces/src/i18n/translation.json +++ b/platforms/activepieces/src/i18n/translation.json @@ -12,14 +12,8 @@ "URL": "URL", "Output Format": "Output Format", "Adaptive Stealth Mode": "Adaptive Stealth Mode", - "JS Rendering": "JS Rendering", - "Premium Proxy": "Premium Proxy", - "Proxy Country": "Proxy Country", - "Wait For Selector": "Wait For Selector", - "Wait (ms)": "Wait (ms)", "JavaScript Instructions": "JavaScript Instructions", "Custom Headers": "Custom Headers", - "Session ID": "Session ID", "Extraction Method": "Extraction Method", "CSS Selectors": "CSS Selectors", "Fields": "Fields", @@ -34,15 +28,7 @@ "Follow redirects": "Follow redirects", "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/": "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/", "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.": "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.", - "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.", - "Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.": "Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.", - "Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.": "Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.", - "Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.": "Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.", - "A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.": "A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.", - "Extra milliseconds to wait after the page loads. Needs JS rendering.": "Extra milliseconds to wait after the page loads. Needs JS rendering.", - "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.", "Headers to send to the target site, such as Cookie or Referer.": "Headers to send to the target site, such as Cookie or Referer.", - "Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.": "Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.", "The full address of the page to extract data from.": "The full address of the page to extract data from.", "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.": "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.", "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.": "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.", @@ -66,5 +52,7 @@ "DELETE": "DELETE", "HEAD": "HEAD", "Request": "Request", - "Data to send": "Data to send" -} \ No newline at end of file + "Data to send": "Data to send", + "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.", + "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list." +} diff --git a/platforms/activepieces/src/lib/actions/extract-structured-data.ts b/platforms/activepieces/src/lib/actions/extract-structured-data.ts index ea6f050..40c2e90 100644 --- a/platforms/activepieces/src/lib/actions/extract-structured-data.ts +++ b/platforms/activepieces/src/lib/actions/extract-structured-data.ts @@ -1,7 +1,6 @@ import { createAction, Property } from '@activepieces/pieces-framework'; import { zenrowsAuth } from '../auth'; import { zenrowsClient } from '../common/client'; -import { ExtractMethod, zenrowsParams } from '../common/params'; export const extractStructuredDataAction = createAction({ auth: zenrowsAuth, @@ -21,7 +20,7 @@ export const extractStructuredDataAction = createAction({ description: 'The full address of the page to extract data from.', required: true, }), - method: Property.StaticDropdown({ + method: Property.StaticDropdown<'css' | 'auto'>({ displayName: 'Extraction Method', description: 'CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.', @@ -57,16 +56,16 @@ export const extractStructuredDataAction = createAction({ }, async run(context) { const { propsValue } = context; - const params = zenrowsParams.buildExtractParams({ - url: propsValue.url, - method: propsValue.method ?? 'css', - selectors: propsValue.selectors, - fields: propsValue.fields, - adaptive: propsValue.adaptive ?? true, - }); + const fields = (propsValue.fields ?? []).filter((f): f is string => typeof f === 'string' && f.trim() !== ''); const response = await zenrowsClient.fetchPage({ apiKey: context.auth.secret_text, - params, + params: { + url: propsValue.url, + ...(propsValue.adaptive !== false ? { mode: 'auto' } : {}), + ...(propsValue.method === 'auto' + ? { extract: 'auto', ...(fields.length ? { extract_fields: fields.join(',') } : {}) } + : { css_extractor: JSON.stringify(propsValue.selectors ?? {}) }), + }, }); return response.body; }, diff --git a/platforms/activepieces/src/lib/actions/scrape-url.ts b/platforms/activepieces/src/lib/actions/scrape-url.ts index c57a478..c16516d 100644 --- a/platforms/activepieces/src/lib/actions/scrape-url.ts +++ b/platforms/activepieces/src/lib/actions/scrape-url.ts @@ -2,7 +2,17 @@ import { randomUUID } from 'crypto'; import { createAction, OutputSchema, Property } from '@activepieces/pieces-framework'; import { zenrowsAuth } from '../auth'; import { zenrowsClient } from '../common/client'; -import { ScrapeOutputFormat, zenrowsParams } from '../common/params'; + +type OutputFormat = 'markdown' | 'html' | 'plaintext' | 'screenshot'; + +// What each format adds to the request. A screenshot needs a real browser, so +// it sets js_render itself instead of leaving the choice to Adaptive Stealth Mode. +const FORMAT_PARAMS: Record> = { + markdown: { response_type: 'markdown' }, + html: {}, + plaintext: { response_type: 'plaintext' }, + screenshot: { js_render: 'true', screenshot_fullpage: 'true', screenshot_format: 'png' }, +}; export const scrapeUrlAction = createAction({ auth: zenrowsAuth, @@ -24,7 +34,7 @@ export const scrapeUrlAction = createAction({ description: 'The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/', required: true, }), - output_format: Property.StaticDropdown({ + output_format: Property.StaticDropdown({ displayName: 'Output Format', description: 'Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.', required: true, @@ -42,44 +52,14 @@ export const scrapeUrlAction = createAction({ adaptive: Property.Checkbox({ displayName: 'Adaptive Stealth Mode', description: - 'Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off to choose JS Rendering and Premium Proxy yourself.', + 'Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.', required: false, defaultValue: true, }), - js_render: Property.Checkbox({ - displayName: 'JS Rendering', - description: - 'Load the page in a real browser so content added by JavaScript is included. Only used when Adaptive Stealth Mode is off.', - required: false, - defaultValue: false, - }), - premium_proxy: Property.Checkbox({ - displayName: 'Premium Proxy', - description: - 'Route the request through residential IPs, needed by most protected sites. Only used when Adaptive Stealth Mode is off.', - required: false, - defaultValue: false, - }), - proxy_country: Property.ShortText({ - displayName: 'Proxy Country', - description: - 'Two-letter country code to scrape from, e.g. us, gb or de. Needs Premium Proxy when Adaptive Stealth Mode is off.', - required: false, - }), - wait_for: Property.ShortText({ - displayName: 'Wait For Selector', - description: 'A CSS selector to wait for before returning the page, e.g. .product-list. Needs JS rendering.', - required: false, - }), - wait: Property.Number({ - displayName: 'Wait (ms)', - description: 'Extra milliseconds to wait after the page loads. Needs JS rendering.', - required: false, - }), js_instructions: Property.LongText({ displayName: 'JavaScript Instructions', description: - 'Actions to run on the page before it is returned, as a JSON array, e.g. [{"click": ".load-more"}, {"wait": 1000}]. Needs JS rendering. See https://docs.zenrows.com for the full list.', + 'Actions to run on the page before it is returned, as a JSON array, e.g. [{"click": ".load-more"}, {"wait": 1000}]. See https://docs.zenrows.com for the full list.', required: false, }), custom_headers: Property.Object({ @@ -87,36 +67,24 @@ export const scrapeUrlAction = createAction({ description: 'Headers to send to the target site, such as Cookie or Referer.', required: false, }), - session_id: Property.Number({ - displayName: 'Session ID', - description: 'Any number you choose. Requests with the same Session ID use the same IP for up to 10 minutes.', - required: false, - }), }, async run(context) { const { propsValue } = context; const outputFormat = propsValue.output_format ?? 'markdown'; - const forwardedHeaders = zenrowsParams.toStringRecord({ value: propsValue.custom_headers }); - const forwardHeaders = Object.keys(forwardedHeaders).length > 0; const isScreenshot = outputFormat === 'screenshot'; - - const params = zenrowsParams.buildScrapeParams({ - url: propsValue.url, - outputFormat, - adaptive: propsValue.adaptive ?? true, - jsRender: propsValue.js_render, - premiumProxy: propsValue.premium_proxy, - proxyCountry: propsValue.proxy_country, - waitFor: propsValue.wait_for, - wait: propsValue.wait, - jsInstructions: propsValue.js_instructions, - forwardHeaders, - sessionId: propsValue.session_id, - }); + const forwardedHeaders = stringValues(propsValue.custom_headers); + const forwardHeaders = Object.keys(forwardedHeaders).length > 0; + const jsInstructions = propsValue.js_instructions?.trim(); const response = await zenrowsClient.fetchPage({ apiKey: context.auth.secret_text, - params, + params: { + url: propsValue.url, + ...(propsValue.adaptive !== false && !isScreenshot ? { mode: 'auto' } : {}), + ...FORMAT_PARAMS[outputFormat], + ...(jsInstructions ? { js_instructions: jsInstructions } : {}), + ...(forwardHeaders ? { custom_headers: 'true' } : {}), + }, forwardedHeaders: forwardHeaders ? forwardedHeaders : undefined, binary: isScreenshot, }); @@ -142,6 +110,15 @@ export const scrapeUrlAction = createAction({ }, }); +// The headers the form holds as text, trimmed; other values are left out. +function stringValues(value: Record | null | undefined): Record { + return Object.fromEntries( + Object.entries(value ?? {}).flatMap(([key, entry]) => + typeof entry === 'string' && entry.trim() && key.trim() ? [[key.trim(), entry.trim()]] : [] + ) + ); +} + function scrapeOutputSchema(): OutputSchema { return { fields: [ diff --git a/platforms/activepieces/src/lib/common/client.ts b/platforms/activepieces/src/lib/common/client.ts index 462a1c5..dedbd65 100644 --- a/platforms/activepieces/src/lib/common/client.ts +++ b/platforms/activepieces/src/lib/common/client.ts @@ -60,6 +60,9 @@ async function fetchPage({ } } +// The API answers every failure with a code and a reason +// (https://docs.zenrows.com/api-error-codes); the message keeps both, and adds +// what to do next for the codes where Activepieces changes the answer. function toZenrowsError({ error }: { error: unknown }): unknown { if (!(error instanceof HttpError)) { return error; @@ -69,29 +72,11 @@ function toZenrowsError({ error }: { error: unknown }): unknown { if (!problem) { return error; } - const hint = isExtractRefusal({ code: problem.code }) - ? REQS007_HINTS[problem.extractCase ?? ''] ?? ERROR_HINTS['REQS007'] - : ERROR_HINTS[problem.code]; - return new Error( - `ZenRows error ${problem.code} (HTTP ${status}): ${problem.reason}${hint ? ` ${hint}` : ''}` - ); + const hint = ERROR_HINTS[problem.code]; + return new Error(`ZenRows error ${problem.code} (HTTP ${status}): ${problem.reason}${hint ? ` ${hint}` : ''}`); } -function readHeader({ - headers, - name, -}: { - headers: HttpHeaders | undefined; - name: string; -}): string | null { - const value = headers?.[name.toLowerCase()]; - if (Array.isArray(value)) { - return value[0] ?? null; - } - return value ?? null; -} - -function parseProblem({ body }: { body: unknown }): ZenrowsProblem | null { +function parseProblem({ body }: { body: unknown }): { code: string; reason: string } | null { if (typeof body === 'string') { try { return parseProblem({ body: JSON.parse(body) }); @@ -99,43 +84,25 @@ function parseProblem({ body }: { body: unknown }): ZenrowsProblem | null { return null; } } - if (!isRecord(body) || typeof body['code'] !== 'string') { + if (typeof body !== 'object' || body === null || Array.isArray(body)) { + return null; + } + const { code, detail, title } = body as Record; + if (typeof code !== 'string') { return null; } - const detail = body['detail']; - const title = body['title']; return { - code: body['code'], - reason: - typeof detail === 'string' - ? detail - : typeof title === 'string' - ? title - : 'Request failed', - extractCase: readExtractCase({ body }), + code, + reason: typeof detail === 'string' ? detail : typeof title === 'string' ? title : 'Request failed', }; } -// REQS007 (or the older AUTH010) says why Extract refused a page in its -// "reason" field. Until every API release sends it, the title tells the cases -// apart: https://docs.zenrows.com/api-error-codes#REQS007 -function readExtractCase({ body }: { body: Record }): string | undefined { - if (typeof body['reason'] === 'string') { - return body['reason']; - } - const title = body['title']; - if (typeof title !== 'string') { - return undefined; +function readHeader({ headers, name }: { headers: HttpHeaders | undefined; name: string }): string | null { + const value = headers?.[name.toLowerCase()]; + if (Array.isArray(value)) { + return value[0] ?? null; } - return REQS007_TITLES.find(([prefix]) => title.startsWith(prefix))?.[1]; -} - -function isExtractRefusal({ code }: { code: string }): boolean { - return code === 'REQS007' || code === 'AUTH010'; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); + return value ?? null; } const FETCH_TIMEOUT_MS = 180000; @@ -156,43 +123,13 @@ const ERROR_HINTS: Record = { AUTH003: AUTH_HINT, AUTH004: sentence({{text:auth004}}, 'Top up or upgrade at https://app.zenrows.com.'), AUTH014: sentence({{text:auth014}}, 'Raise the cap in the ZenRows dashboard.'), - REQS002: - 'This page needs JS rendering and/or premium proxies. Turn on Adaptive Stealth Mode, or enable them manually.', - REQS007: - 'Extract refused this page, and nothing was charged. Use the CSS selectors method, or see https://docs.zenrows.com/api-error-codes#REQS007.', - AUTH006: sentence({{text:auth006}}, 'Reduce parallel runs and retry.'), - AUTH008: 'Rate limit reached. Wait a moment and retry.', - RESP009: 'ZenRows is temporarily at capacity. Retry in a few seconds.', + REQS002: 'This page needs JS rendering or premium proxies: turn on Adaptive Stealth Mode.', }; export const ZENROWS_API_BASE_URL = 'https://api.zenrows.com/v1'; -const REQS007_TITLES: [string, string][] = [ - ['Extract is not prepared for the requested domain', 'domain_not_prepared'], - ['Extract did not recognise this page on', 'page_not_recognised'], - ['Extract is not enabled for the requested domain', 'domain_not_enabled'], -]; - -const REQS007_HINTS: Record = { - domain_not_prepared: - 'ZenRows has not prepared this domain for automatic extraction yet. Prepare it with the Custom API Call action: POST /extract/prepared-domains with the domain and 1 to 5 example URLs. It takes a few minutes and costs no credits. Meanwhile, use the CSS selectors method.', - page_not_recognised: sentence( - {{text:reqs007-page-not-recognised}}, - 'Try a product, article or listing page, or the CSS selectors method. Preparing the domain again does not change it.' - ), - domain_not_enabled: - 'Automatic extraction is not enabled for this site, and preparing it does not help. Use the CSS selectors method, or contact ZenRows support.', -}; - export const zenrowsClient = { request, fetchPage, - toZenrowsError, readHeader, }; - -type ZenrowsProblem = { - code: string; - reason: string; - extractCase?: string; -}; diff --git a/platforms/activepieces/src/lib/common/params.ts b/platforms/activepieces/src/lib/common/params.ts deleted file mode 100644 index 9affe2f..0000000 --- a/platforms/activepieces/src/lib/common/params.ts +++ /dev/null @@ -1,144 +0,0 @@ -function buildScrapeParams({ - url, - outputFormat, - adaptive, - jsRender, - premiumProxy, - proxyCountry, - waitFor, - wait, - jsInstructions, - forwardHeaders, - sessionId, -}: ScrapeOptions): Record { - const isScreenshot = outputFormat === 'screenshot'; - const usesBrowser = isScreenshot || (!adaptive && jsRender === true); - const usesPremiumProxy = !adaptive && premiumProxy === true; - const needsBrowser = !isNilOrEmpty(waitFor) || !isNil(wait) || !isNilOrEmpty(jsInstructions); - - if (!adaptive && !usesBrowser && needsBrowser) { - throw new Error( - 'Wait For Selector, Wait (ms) and JavaScript Instructions need JS Rendering. Turn on JS Rendering or Adaptive Stealth Mode.' - ); - } - if (!adaptive && !usesPremiumProxy && !isNilOrEmpty(proxyCountry)) { - throw new Error( - 'Proxy Country needs Premium Proxy. Turn on Premium Proxy or Adaptive Stealth Mode.' - ); - } - - return { - url, - ...(adaptive && !isScreenshot ? { mode: 'auto' } : {}), - ...(usesBrowser ? { js_render: 'true' } : {}), - ...(usesPremiumProxy ? { premium_proxy: 'true' } : {}), - ...(!isNilOrEmpty(proxyCountry) ? { proxy_country: proxyCountry.trim().toLowerCase() } : {}), - ...(!isNilOrEmpty(waitFor) ? { wait_for: waitFor } : {}), - ...(!isNil(wait) ? { wait: String(wait) } : {}), - ...(!isNilOrEmpty(jsInstructions) ? { js_instructions: normalizeJsInstructions({ jsInstructions }) } : {}), - ...(forwardHeaders ? { custom_headers: 'true' } : {}), - ...(!isNil(sessionId) ? { session_id: String(sessionId) } : {}), - ...RESPONSE_TYPE_PARAMS[outputFormat], - }; -} - -function buildExtractParams({ - url, - method, - selectors, - fields, - adaptive, -}: ExtractOptions): Record { - const base = { url, ...(adaptive ? { mode: 'auto' } : {}) }; - if (method === 'css') { - const selectorMap = toStringRecord({ value: selectors }); - if (Object.keys(selectorMap).length === 0) { - throw new Error('Add at least one field with a CSS selector, for example title → h1.'); - } - return { ...base, css_extractor: JSON.stringify(selectorMap) }; - } - const requestedFields = (fields ?? []) - .filter((field): field is string => typeof field === 'string') - .map((field) => field.trim()) - .filter((field) => field.length > 0); - return { - ...base, - extract: 'auto', - ...(requestedFields.length > 0 ? { extract_fields: requestedFields.join(',') } : {}), - }; -} - -function toStringRecord({ - value, -}: { - value: Record | null | undefined; -}): Record { - return Object.fromEntries( - Object.entries(value ?? {}).flatMap(([key, entry]) => - typeof entry === 'string' && entry.trim().length > 0 && key.trim().length > 0 - ? [[key.trim(), entry.trim()]] - : [] - ) - ); -} - -function normalizeJsInstructions({ jsInstructions }: { jsInstructions: string }): string { - try { - const parsed: unknown = JSON.parse(jsInstructions); - if (!Array.isArray(parsed)) { - throw new Error('not an array'); - } - return JSON.stringify(parsed); - } catch { - throw new Error( - 'JavaScript Instructions must be a JSON array, for example [{"click": ".load-more"}, {"wait": 1000}].' - ); - } -} - -function isNil(value: unknown): value is null | undefined { - return value === null || value === undefined; -} - -function isNilOrEmpty(value: string | null | undefined): value is null | undefined { - return isNil(value) || value.trim().length === 0; -} - -const RESPONSE_TYPE_PARAMS: Record> = { - html: {}, - markdown: { response_type: 'markdown' }, - plaintext: { response_type: 'plaintext' }, - screenshot: { screenshot_fullpage: 'true', screenshot_format: 'png' }, -}; - -export const zenrowsParams = { - buildScrapeParams, - buildExtractParams, - toStringRecord, -}; - -export type ScrapeOutputFormat = 'html' | 'markdown' | 'plaintext' | 'screenshot'; - -export type ExtractMethod = 'css' | 'auto'; - -type ScrapeOptions = { - url: string; - outputFormat: ScrapeOutputFormat; - adaptive: boolean; - jsRender?: boolean; - premiumProxy?: boolean; - proxyCountry?: string | null; - waitFor?: string | null; - wait?: number | null; - jsInstructions?: string | null; - forwardHeaders: boolean; - sessionId?: number | null; -}; - -type ExtractOptions = { - url: string; - method: ExtractMethod; - selectors?: Record | null; - fields?: unknown[] | null; - adaptive: boolean; -}; diff --git a/platforms/activepieces/test/params.test.ts b/platforms/activepieces/test/params.test.ts deleted file mode 100644 index f61ff03..0000000 --- a/platforms/activepieces/test/params.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { HttpError } from '@activepieces/pieces-common'; -import { zenrowsClient } from '../src/lib/common/client'; -import { zenrowsParams } from '../src/lib/common/params'; - -const URL = 'https://www.scrapingcourse.com/ecommerce/'; - -describe('buildScrapeParams', () => { - it('uses adaptive mode and markdown by default', () => { - expect( - zenrowsParams.buildScrapeParams({ url: URL, outputFormat: 'markdown', adaptive: true, forwardHeaders: false }) - ).toEqual({ url: URL, mode: 'auto', response_type: 'markdown' }); - }); - - it('ignores manual JS rendering and premium proxy in adaptive mode', () => { - expect( - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'html', - adaptive: true, - jsRender: true, - premiumProxy: true, - proxyCountry: 'US', - forwardHeaders: false, - }) - ).toEqual({ url: URL, mode: 'auto', proxy_country: 'us' }); - }); - - it('sends manual options when adaptive mode is off', () => { - expect( - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'plaintext', - adaptive: false, - jsRender: true, - premiumProxy: true, - proxyCountry: 'de', - waitFor: '.product', - wait: 500, - jsInstructions: '[{"click": ".more"}]', - forwardHeaders: true, - sessionId: 42, - }) - ).toEqual({ - url: URL, - js_render: 'true', - premium_proxy: 'true', - proxy_country: 'de', - wait_for: '.product', - wait: '500', - js_instructions: '[{"click":".more"}]', - custom_headers: 'true', - session_id: '42', - response_type: 'plaintext', - }); - }); - - it('forces JS rendering and skips adaptive mode for screenshots', () => { - expect( - zenrowsParams.buildScrapeParams({ url: URL, outputFormat: 'screenshot', adaptive: true, forwardHeaders: false }) - ).toEqual({ url: URL, js_render: 'true', screenshot_fullpage: 'true', screenshot_format: 'png' }); - }); - - it('rejects browser options without JS rendering in manual mode', () => { - expect(() => - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'html', - adaptive: false, - waitFor: '.product', - forwardHeaders: false, - }) - ).toThrow(/need JS Rendering/); - }); - - it('rejects a proxy country without premium proxy in manual mode', () => { - expect(() => - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'html', - adaptive: false, - proxyCountry: 'us', - forwardHeaders: false, - }) - ).toThrow(/needs Premium Proxy/); - }); - - it('rejects JavaScript instructions that are not a JSON array', () => { - expect(() => - zenrowsParams.buildScrapeParams({ - url: URL, - outputFormat: 'html', - adaptive: true, - jsInstructions: '{"click": ".more"}', - forwardHeaders: false, - }) - ).toThrow(/JSON array/); - }); -}); - -describe('buildExtractParams', () => { - it('sends CSS selectors as a JSON map', () => { - expect( - zenrowsParams.buildExtractParams({ - url: URL, - method: 'css', - selectors: { title: 'h1', price: ' .price ', empty: '', count: 3 }, - adaptive: true, - }) - ).toEqual({ url: URL, mode: 'auto', css_extractor: '{"title":"h1","price":".price"}' }); - }); - - it('requires at least one selector', () => { - expect(() => - zenrowsParams.buildExtractParams({ url: URL, method: 'css', selectors: {}, adaptive: false }) - ).toThrow(/at least one field/); - }); - - it('uses automatic extraction with optional fields', () => { - expect( - zenrowsParams.buildExtractParams({ - url: URL, - method: 'auto', - fields: ['products.name', ' products.price ', '', 7], - adaptive: false, - }) - ).toEqual({ url: URL, extract: 'auto', extract_fields: 'products.name,products.price' }); - }); -}); - -describe('toZenrowsError', () => { - it('turns a ZenRows problem response into a readable error with a hint', () => { - const error = new HttpError( - {}, - { - status: 402, - responseBody: JSON.stringify({ code: 'AUTH004', title: 'Usage exceeded (AUTH004)', detail: 'No credits left.' }), - } - ); - const mapped = zenrowsClient.toZenrowsError({ error }); - expect(mapped).toBeInstanceOf(Error); - expect(String(mapped)).toContain('ZenRows error AUTH004 (HTTP 402): No credits left.'); - expect(String(mapped)).toContain('credit allowance is spent'); - }); - - it.each([ - [{ reason: 'domain_not_prepared' }, 'POST /extract/prepared-domains'], - [{ title: 'Extract did not recognise this page on example.com' }, 'Preparing the domain again does not change it'], - [{ title: 'Extract is not enabled for the requested domain' }, 'contact ZenRows support'], - [{ title: 'Something new' }, 'api-error-codes#REQS007'], - ])('gives the REQS007 hint for its case (%o)', (fields, hint) => { - const error = new HttpError({}, { status: 403, responseBody: JSON.stringify({ code: 'REQS007', ...fields }) }); - expect(String(zenrowsClient.toZenrowsError({ error }))).toContain(hint); - }); - - it('keeps errors that are not ZenRows problems unchanged', () => { - const error = new HttpError({}, { status: 500, responseBody: 'oops' }); - expect(zenrowsClient.toZenrowsError({ error })).toBe(error); - const plain = new Error('network down'); - expect(zenrowsClient.toZenrowsError({ error: plain })).toBe(plain); - }); -}); diff --git a/platforms/activepieces/vitest.config.ts b/platforms/activepieces/vitest.config.ts deleted file mode 100644 index 79c9726..0000000 --- a/platforms/activepieces/vitest.config.ts +++ /dev/null @@ -1,19 +0,0 @@ -import path from 'path' -import { defineConfig } from 'vitest/config' - -const repoRoot = path.resolve(__dirname, '../../../..') - -export default defineConfig({ - root: __dirname, - test: { - globals: true, - environment: 'node', - include: ['test/**/*.test.ts'], - }, - resolve: { - alias: { - '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'), - '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'), - }, - }, -}) From d90de37a7647394db89e94ffc96dad29a44a2863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Thu, 12 Nov 2026 14:44:43 +0000 Subject: [PATCH 5/8] fix(activepieces): return a page as text and Extract's parsed data Scrape URL asks for the page as text, so a JSON page isn't parsed into an object. Extract's automatic method returns the parsed data, not the { html, parsed } envelope. Found by a live run of each action. --- .../src/lib/actions/extract-structured-data.ts | 5 ++++- dist/activepieces/src/lib/actions/scrape-url.ts | 2 +- dist/activepieces/src/lib/common/client.ts | 8 +++++--- .../src/lib/actions/extract-structured-data.ts | 5 ++++- platforms/activepieces/src/lib/actions/scrape-url.ts | 2 +- platforms/activepieces/src/lib/common/client.ts | 8 +++++--- 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/dist/activepieces/src/lib/actions/extract-structured-data.ts b/dist/activepieces/src/lib/actions/extract-structured-data.ts index 40c2e90..3b0c098 100644 --- a/dist/activepieces/src/lib/actions/extract-structured-data.ts +++ b/dist/activepieces/src/lib/actions/extract-structured-data.ts @@ -66,7 +66,10 @@ export const extractStructuredDataAction = createAction({ ? { extract: 'auto', ...(fields.length ? { extract_fields: fields.join(',') } : {}) } : { css_extractor: JSON.stringify(propsValue.selectors ?? {}) }), }, + responseType: 'json', }); - return response.body; + // Automatic extraction answers { html, parsed }; the flow wants the data, + // not the page it came from (https://docs.zenrows.com/extract/setup#response). + return propsValue.method === 'auto' ? response.body.parsed ?? response.body : response.body; }, }); diff --git a/dist/activepieces/src/lib/actions/scrape-url.ts b/dist/activepieces/src/lib/actions/scrape-url.ts index c16516d..944b97e 100644 --- a/dist/activepieces/src/lib/actions/scrape-url.ts +++ b/dist/activepieces/src/lib/actions/scrape-url.ts @@ -86,7 +86,7 @@ export const scrapeUrlAction = createAction({ ...(forwardHeaders ? { custom_headers: 'true' } : {}), }, forwardedHeaders: forwardHeaders ? forwardedHeaders : undefined, - binary: isScreenshot, + responseType: isScreenshot ? 'arraybuffer' : 'text', }); const screenshotUrl = isScreenshot diff --git a/dist/activepieces/src/lib/common/client.ts b/dist/activepieces/src/lib/common/client.ts index 94878cc..8080628 100644 --- a/dist/activepieces/src/lib/common/client.ts +++ b/dist/activepieces/src/lib/common/client.ts @@ -35,16 +35,18 @@ async function request({ } } +// responseType: 'text' keeps a page as it is (a JSON page would otherwise come +// back parsed), 'json' is for Extract's output, 'arraybuffer' for a screenshot. async function fetchPage({ apiKey, params, forwardedHeaders, - binary, + responseType, }: { apiKey: string; params: Record; forwardedHeaders?: Record; - binary?: boolean; + responseType: 'text' | 'json' | 'arraybuffer'; }): Promise { try { return await httpClient.sendRequest({ @@ -53,7 +55,7 @@ async function fetchPage({ queryParams: { apikey: apiKey, ...params }, headers: forwardedHeaders ?? CLIENT_HEADERS, timeout: FETCH_TIMEOUT_MS, - responseType: binary ? 'arraybuffer' : undefined, + responseType, }); } catch (error) { throw toZenrowsError({ error }); diff --git a/platforms/activepieces/src/lib/actions/extract-structured-data.ts b/platforms/activepieces/src/lib/actions/extract-structured-data.ts index 40c2e90..3b0c098 100644 --- a/platforms/activepieces/src/lib/actions/extract-structured-data.ts +++ b/platforms/activepieces/src/lib/actions/extract-structured-data.ts @@ -66,7 +66,10 @@ export const extractStructuredDataAction = createAction({ ? { extract: 'auto', ...(fields.length ? { extract_fields: fields.join(',') } : {}) } : { css_extractor: JSON.stringify(propsValue.selectors ?? {}) }), }, + responseType: 'json', }); - return response.body; + // Automatic extraction answers { html, parsed }; the flow wants the data, + // not the page it came from (https://docs.zenrows.com/extract/setup#response). + return propsValue.method === 'auto' ? response.body.parsed ?? response.body : response.body; }, }); diff --git a/platforms/activepieces/src/lib/actions/scrape-url.ts b/platforms/activepieces/src/lib/actions/scrape-url.ts index c16516d..944b97e 100644 --- a/platforms/activepieces/src/lib/actions/scrape-url.ts +++ b/platforms/activepieces/src/lib/actions/scrape-url.ts @@ -86,7 +86,7 @@ export const scrapeUrlAction = createAction({ ...(forwardHeaders ? { custom_headers: 'true' } : {}), }, forwardedHeaders: forwardHeaders ? forwardedHeaders : undefined, - binary: isScreenshot, + responseType: isScreenshot ? 'arraybuffer' : 'text', }); const screenshotUrl = isScreenshot diff --git a/platforms/activepieces/src/lib/common/client.ts b/platforms/activepieces/src/lib/common/client.ts index dedbd65..1e724c2 100644 --- a/platforms/activepieces/src/lib/common/client.ts +++ b/platforms/activepieces/src/lib/common/client.ts @@ -35,16 +35,18 @@ async function request({ } } +// responseType: 'text' keeps a page as it is (a JSON page would otherwise come +// back parsed), 'json' is for Extract's output, 'arraybuffer' for a screenshot. async function fetchPage({ apiKey, params, forwardedHeaders, - binary, + responseType, }: { apiKey: string; params: Record; forwardedHeaders?: Record; - binary?: boolean; + responseType: 'text' | 'json' | 'arraybuffer'; }): Promise { try { return await httpClient.sendRequest({ @@ -53,7 +55,7 @@ async function fetchPage({ queryParams: { apikey: apiKey, ...params }, headers: forwardedHeaders ?? CLIENT_HEADERS, timeout: FETCH_TIMEOUT_MS, - responseType: binary ? 'arraybuffer' : undefined, + responseType, }); } catch (error) { throw toZenrowsError({ error }); From b4ce1b22c6810edaf8c98d4470f5ef85592086ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Thu, 12 Nov 2026 16:20:23 +0000 Subject: [PATCH 6/8] test(activepieces): run each action against the live API by hand smoke.sh asks for the API key, builds the piece in the activepieces checkout and runs smoke.ts: the four actions on real pages, plus a wrong key and a protected page that must be rejected with their code. Not in CI; it spends credits. --- platforms/activepieces/README.md | 8 +++ platforms/activepieces/scripts/smoke.sh | 38 +++++++++++ platforms/activepieces/scripts/smoke.ts | 86 +++++++++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100755 platforms/activepieces/scripts/smoke.sh create mode 100644 platforms/activepieces/scripts/smoke.ts diff --git a/platforms/activepieces/README.md b/platforms/activepieces/README.md index 02a2251..9a74241 100644 --- a/platforms/activepieces/README.md +++ b/platforms/activepieces/README.md @@ -20,6 +20,14 @@ It copies `dist/activepieces/` into the activepieces checkout, registers it in ` `upstream` in `vars.yaml` pins the activepieces commit. Moving it is its own PR, so CI shows whether their newer code breaks the piece. +## Try it live + +``` +platforms/activepieces/scripts/smoke.sh +``` + +It builds the piece as above, asks for your Zenrows API key (kept in that process only), then runs each action against the live API with a fake Activepieces context: Scrape URL as Markdown, a screenshot and a JSON page, Extract by CSS and automatic on a listing and a product page, Get Account Usage, Custom API Call, and two failures that should show the API's code with our hint. Each run spends a few credits, so it isn't in CI. It can't check the forms; for those, run Activepieces locally with `AP_DEV_PIECES=zenrows`. + ## How it ships Not yet. First as our own npm package; then as a PR to activepieces once they agree to take it (they close outside PRs that weren't discussed with them first; see their CONTRIBUTING.md). Both release steps come in later PRs. diff --git a/platforms/activepieces/scripts/smoke.sh b/platforms/activepieces/scripts/smoke.sh new file mode 100755 index 0000000..c37935e --- /dev/null +++ b/platforms/activepieces/scripts/smoke.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Runs each action of the rendered piece against the live Zenrows API, with a +# fake Activepieces context (platforms/activepieces/scripts/smoke.ts), and prints +# what each returns. It asks for your API key and keeps it in this process only. +# +# platforms/activepieces/scripts/smoke.sh builds the piece in .cache/activepieces first (check.sh) +# platforms/activepieces/scripts/smoke.sh uses an existing activepieces checkout instead +# +# Each run spends a few credits. Not for CI. +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(cd "$here/../../.." && pwd)" +ap="${1:-$repo/.cache/activepieces}" + +if [ -z "${ZENROWS_API_KEY:-}" ]; then + # A key piped in without a final newline makes read report end-of-file. + read -rsp "Zenrows API key (not shown): " ZENROWS_API_KEY || [ -n "$ZENROWS_API_KEY" ] + echo +fi +# A key is letters and digits; anything else is a paste of the wrong thing. +if ! [[ "$ZENROWS_API_KEY" =~ ^[A-Za-z0-9]{20,}$ ]]; then + echo "That isn't a Zenrows API key (expected letters and digits only, got ${#ZENROWS_API_KEY} characters). Copy it from https://app.zenrows.com and try again." >&2 + exit 1 +fi +export ZENROWS_API_KEY + +echo "Building the piece in $ap …" +"$here/check.sh" ${1:+"$1"} >/dev/null 2>&1 /dev/null || [ "$(bun --version)" != "$bun_version" ]; then + export PATH="$repo/.cache/bun-$bun_version/node_modules/.bin:$PATH" +fi + +cd "$ap" +bun run "$here/smoke.ts" "$ap" 2>/dev/null | grep -E "^(✅|❌)"; exit "${PIPESTATUS[0]}" diff --git a/platforms/activepieces/scripts/smoke.ts b/platforms/activepieces/scripts/smoke.ts new file mode 100644 index 0000000..c4ca034 --- /dev/null +++ b/platforms/activepieces/scripts/smoke.ts @@ -0,0 +1,86 @@ +// Run by smoke.sh from inside an activepieces checkout, so bun resolves the +// piece's workspace imports: calls each action's run() with a fake Activepieces +// context against the live API. The key comes in ZENROWS_API_KEY; the checkout +// is argv[2]. Screenshots are written to .cache/smoke/. +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const checkout = process.argv[2]; +const apiKey = process.env['ZENROWS_API_KEY']; +if (!checkout || !apiKey) { + console.error('usage: ZENROWS_API_KEY=… bun run smoke.ts '); + process.exit(1); +} +const piece = await import(join(checkout, 'packages/pieces/community/zenrows/src/index.ts')); +const actions = piece.zenrows.actions() as Record Promise }>; +const outDir = join(checkout, '..', 'smoke'); +mkdirSync(outDir, { recursive: true }); + +// The parts of an Activepieces action context the piece reads. +function context(name: string, propsValue: Record, key = apiKey) { + return { + auth: { secret_text: key }, + propsValue, + files: { + write: async ({ fileName, data }: { fileName: string; data: Buffer }) => { + const path = join(outDir, fileName); + writeFileSync(path, data); + return `file://${path}`; + }, + }, + store: { get: async () => null, put: async () => null, delete: async () => null }, + server: { apiUrl: 'http://localhost/', publicUrl: 'http://localhost/', token: '' }, + project: { id: 'smoke', externalId: async () => 'smoke' }, + run: { id: 'smoke', name, isTest: true, pauseId: '', stop: () => {}, pause: () => {} }, + connections: { get: async () => null }, + flows: { current: { id: 'smoke', version: { id: 'smoke' } } }, + step: { name }, + generateResumeUrl: () => '', + tags: { add: async () => {} }, + }; +} + +const LISTING = 'https://www.scrapingcourse.com/ecommerce/'; +const PRODUCT = 'https://www.scrapingcourse.com/ecommerce/product/abominable-hoodie/'; +// [label, action, form values, expected error code (the case passes only by +// failing with it), API key to use instead of the real one] +const cases: [string, string, Record, string?, string?][] = [ + ['scrape_url markdown', 'scrape_url', { url: LISTING, output_format: 'markdown', adaptive: true }], + ['scrape_url screenshot', 'scrape_url', { url: LISTING, output_format: 'screenshot', adaptive: true }], + ['scrape_url html of a JSON page, with headers and JS', 'scrape_url', { url: 'https://httpbin.io/headers', output_format: 'html', adaptive: true, custom_headers: { Referer: 'https://example.com' }, js_instructions: '[{"wait": 500}]' }], + ['extract css', 'extract_structured_data', { url: LISTING, method: 'css', selectors: { title: 'h1', products: '.product-name' }, adaptive: true }], + ['extract auto, listing page', 'extract_structured_data', { url: LISTING, method: 'auto', adaptive: true }], + ['extract auto, product page', 'extract_structured_data', { url: PRODUCT, method: 'auto', adaptive: true }], + ['get_account_usage', 'get_account_usage', {}], + // The Custom API Call's url prop is a dynamic property: { url: }. + ['custom_api_call', 'custom_api_call', { url: { url: '/subscriptions/self/details' }, method: 'GET', headers: {}, queryParams: {}, failsafe: false }], + // These pass only if the API rejects them with this code, and the message + // carries the API's reason plus our hint. + ['wrong key', 'get_account_usage', {}, 'AUTH002', 'not-a-key'], + ['protected page, adaptive off', 'scrape_url', { url: 'https://www.scrapingcourse.com/cloudflare-challenge', output_format: 'html', adaptive: false }, 'REQS002'], +]; + +let failed = 0; +for (const [label, name, props, expectError, key] of cases) { + const started = Date.now(); + const ms = () => `${Date.now() - started} ms`; + try { + const result = await actions[name].run(context(name, props, key)); + const text = JSON.stringify(result); + if (expectError) { + failed++; + console.log(`❌ ${label} (${ms()}): expected a ${expectError} error, got ${text.slice(0, 120)}`); + } else { + console.log(`✅ ${label} (${ms()}): ${text.length > 240 ? `${text.slice(0, 240)}… [${text.length} chars]` : text}`); + } + } catch (e) { + const message = (e as Error).message; + if (expectError && message.includes(expectError)) { + console.log(`✅ ${label} (${ms()}): rejected with ${expectError}, as intended: ${message}`); + } else { + failed++; + console.log(`❌ ${label} (${ms()}): ${message}`); + } + } +} +process.exit(failed ? 1 : 0); From 0d8eb50c6e09482ec10a4a9cdfb3f1d2bcd82c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Thu, 12 Nov 2026 17:51:47 +0000 Subject: [PATCH 7/8] fix(activepieces): say Automatic works on any prepared domain, and serve the logo from here --- dist/activepieces/src/i18n/translation.json | 4 ++-- dist/activepieces/src/index.ts | 3 ++- dist/activepieces/src/lib/actions/extract-structured-data.ts | 2 +- platforms/activepieces/src/i18n/translation.json | 4 ++-- platforms/activepieces/src/index.ts | 3 ++- .../activepieces/src/lib/actions/extract-structured-data.ts | 2 +- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dist/activepieces/src/i18n/translation.json b/dist/activepieces/src/i18n/translation.json index 1a304f9..c0e4b57 100644 --- a/dist/activepieces/src/i18n/translation.json +++ b/dist/activepieces/src/i18n/translation.json @@ -30,7 +30,6 @@ "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.": "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.", "Headers to send to the target site, such as Cookie or Referer.": "Headers to send to the target site, such as Cookie or Referer.", "The full address of the page to extract data from.": "The full address of the page to extract data from.", - "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.": "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.", "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.": "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.", "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.": "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.", "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.": "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.", @@ -54,5 +53,6 @@ "Request": "Request", "Data to send": "Data to send", "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.", - "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list." + "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.", + "CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains).": "CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains)." } diff --git a/dist/activepieces/src/index.ts b/dist/activepieces/src/index.ts index b187dcc..c233590 100644 --- a/dist/activepieces/src/index.ts +++ b/dist/activepieces/src/index.ts @@ -12,7 +12,8 @@ export const zenrows = createPiece({ 'Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.', auth: zenrowsAuth, minimumSupportedRelease: '0.88.2', - logoUrl: 'https://cdn.activepieces.com/pieces/zenrows.png', + // Served from this repo until Activepieces hosts it on their CDN. + logoUrl: 'https://raw.githubusercontent.com/ZenRows/zenrows-integrations/main/shared/assets/logo.png', categories: [PieceCategory.ARTIFICIAL_INTELLIGENCE, PieceCategory.PRODUCTIVITY], authors: ['imoital'], actions: [ diff --git a/dist/activepieces/src/lib/actions/extract-structured-data.ts b/dist/activepieces/src/lib/actions/extract-structured-data.ts index 3b0c098..6b6d610 100644 --- a/dist/activepieces/src/lib/actions/extract-structured-data.ts +++ b/dist/activepieces/src/lib/actions/extract-structured-data.ts @@ -23,7 +23,7 @@ export const extractStructuredDataAction = createAction({ method: Property.StaticDropdown<'css' | 'auto'>({ displayName: 'Extraction Method', description: - 'CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.', + 'CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains).', required: true, defaultValue: 'css', options: { diff --git a/platforms/activepieces/src/i18n/translation.json b/platforms/activepieces/src/i18n/translation.json index 1a304f9..c0e4b57 100644 --- a/platforms/activepieces/src/i18n/translation.json +++ b/platforms/activepieces/src/i18n/translation.json @@ -30,7 +30,6 @@ "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.": "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.", "Headers to send to the target site, such as Cookie or Referer.": "Headers to send to the target site, such as Cookie or Referer.", "The full address of the page to extract data from.": "The full address of the page to extract data from.", - "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.": "CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.", "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.": "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.", "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.": "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.", "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.": "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.", @@ -54,5 +53,6 @@ "Request": "Request", "Data to send": "Data to send", "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.", - "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list." + "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.", + "CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains).": "CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains)." } diff --git a/platforms/activepieces/src/index.ts b/platforms/activepieces/src/index.ts index b187dcc..c233590 100644 --- a/platforms/activepieces/src/index.ts +++ b/platforms/activepieces/src/index.ts @@ -12,7 +12,8 @@ export const zenrows = createPiece({ 'Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.', auth: zenrowsAuth, minimumSupportedRelease: '0.88.2', - logoUrl: 'https://cdn.activepieces.com/pieces/zenrows.png', + // Served from this repo until Activepieces hosts it on their CDN. + logoUrl: 'https://raw.githubusercontent.com/ZenRows/zenrows-integrations/main/shared/assets/logo.png', categories: [PieceCategory.ARTIFICIAL_INTELLIGENCE, PieceCategory.PRODUCTIVITY], authors: ['imoital'], actions: [ diff --git a/platforms/activepieces/src/lib/actions/extract-structured-data.ts b/platforms/activepieces/src/lib/actions/extract-structured-data.ts index 3b0c098..6b6d610 100644 --- a/platforms/activepieces/src/lib/actions/extract-structured-data.ts +++ b/platforms/activepieces/src/lib/actions/extract-structured-data.ts @@ -23,7 +23,7 @@ export const extractStructuredDataAction = createAction({ method: Property.StaticDropdown<'css' | 'auto'>({ displayName: 'Extraction Method', description: - 'CSS Selectors works on any site. Automatic needs no setup but only works on sites ZenRows supports for automatic extraction.', + 'CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains).', required: true, defaultValue: 'css', options: { From 7d97580076d6b586fffa398f40e58e936ed8bed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Fri, 13 Nov 2026 17:18:27 +0000 Subject: [PATCH 8/8] refactor(activepieces): use the shared description --- dist/activepieces/src/i18n/translation.json | 2 +- dist/activepieces/src/index.ts | 2 +- dist/openai/skills/using-zenrows/SKILL.md | 2 +- platforms/activepieces/src/i18n/translation.json | 2 +- platforms/activepieces/src/index.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dist/activepieces/src/i18n/translation.json b/dist/activepieces/src/i18n/translation.json index c0e4b57..26549c2 100644 --- a/dist/activepieces/src/i18n/translation.json +++ b/dist/activepieces/src/i18n/translation.json @@ -1,5 +1,5 @@ { - "Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.": "Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.", + "Live web data for your flows: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows.": "Live web data for your flows: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows.", "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ": "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ", "Scrape URL": "Scrape URL", "Extract Structured Data": "Extract Structured Data", diff --git a/dist/activepieces/src/index.ts b/dist/activepieces/src/index.ts index c233590..0a65608 100644 --- a/dist/activepieces/src/index.ts +++ b/dist/activepieces/src/index.ts @@ -9,7 +9,7 @@ import { ZENROWS_API_BASE_URL } from './lib/common/client'; export const zenrows = createPiece({ displayName: 'ZenRows', description: - 'Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.', + 'Live web data for your flows: ' + "fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows" + '.', auth: zenrowsAuth, minimumSupportedRelease: '0.88.2', // Served from this repo until Activepieces hosts it on their CDN. diff --git a/dist/openai/skills/using-zenrows/SKILL.md b/dist/openai/skills/using-zenrows/SKILL.md index edfd086..f8ff1ac 100644 --- a/dist/openai/skills/using-zenrows/SKILL.md +++ b/dist/openai/skills/using-zenrows/SKILL.md @@ -5,7 +5,7 @@ description: Overview of the Zenrows tools for live web data. Use when a task ne # Using Zenrows -Zenrows gives the model live web data through the Zenrows MCP server: page fetching, structured extraction, batch jobs over many URLs, crawls from a start URL, and hosted browser sessions. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. +Zenrows gives the model live web data through the Zenrows MCP server: the model can fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. The Zenrows plugin signs in with OAuth. If the Zenrows tools are missing or return `AUTH003`, use the `getting-started` skill. Never ask the user to paste an API key into the conversation. diff --git a/platforms/activepieces/src/i18n/translation.json b/platforms/activepieces/src/i18n/translation.json index c0e4b57..f1d2b83 100644 --- a/platforms/activepieces/src/i18n/translation.json +++ b/platforms/activepieces/src/i18n/translation.json @@ -1,5 +1,5 @@ { - "Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.": "Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.", + "{{< zenrows-description}}.": "{{< zenrows-description}}.", "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ": "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ", "Scrape URL": "Scrape URL", "Extract Structured Data": "Extract Structured Data", diff --git a/platforms/activepieces/src/index.ts b/platforms/activepieces/src/index.ts index c233590..e86f722 100644 --- a/platforms/activepieces/src/index.ts +++ b/platforms/activepieces/src/index.ts @@ -9,7 +9,7 @@ import { ZENROWS_API_BASE_URL } from './lib/common/client'; export const zenrows = createPiece({ displayName: 'ZenRows', description: - 'Web scraping API that gets past anti-bot systems and returns pages as Markdown, HTML, screenshots or structured JSON.', + {{text:zenrows-description}} + '.', auth: zenrowsAuth, minimumSupportedRelease: '0.88.2', // Served from this repo until Activepieces hosts it on their CDN.