diff --git a/.github/workflows/activepieces.yml b/.github/workflows/activepieces.yml new file mode 100644 index 0000000..873a024 --- /dev/null +++ b/.github/workflows/activepieces.yml @@ -0,0 +1,43 @@ +name: Activepieces piece + +on: + push: + branches: [main] + paths: + - platforms/activepieces/** + - dist/activepieces/** + - shared/text/** + - scripts/** + - .github/workflows/activepieces.yml + pull_request: + paths: + - platforms/activepieces/** + - dist/activepieces/** + - shared/text/** + - scripts/** + - .github/workflows/activepieces.yml + +permissions: + contents: read + +jobs: + check: + name: Lint, build and test in activepieces + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + - run: npm ci + # activepieces' packageManager at the pinned commit. If they differ, + # check.sh installs the right version instead. + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.4.0" + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.bun/install/cache + key: bun-activepieces-${{ hashFiles('platforms/activepieces/vars.yaml') }} + restore-keys: bun-activepieces- + - run: platforms/activepieces/scripts/check.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef4aa0c..494708f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,7 +53,7 @@ A fact that more than one place states (what an error code means, what a request ## Code hosts -`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped or rendered; `npm run validate` requires one, and the render fails if the host has a `skills/` folder. It needs no release-please entry until it's released from this repo. +`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped or rendered; `npm run validate` requires one, and the render fails if the host has a `skills/` folder. It needs no release-please entry until it's released from this repo. If it's code, its `scripts/check.sh` can build and test `dist//` the way the target project does, run by its own workflow (`platforms/activepieces/` does). ## Tests diff --git a/README.md b/README.md index f71ab0e..fa1037a 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Each host's guide covers installing the plugin and signing in to Zenrows. You si | Host | Setup guide | |---|---| +| Activepieces | [platforms/activepieces/README.md](platforms/activepieces/README.md) | | ChatGPT and Codex | [dist/openai/README.md](dist/openai/README.md) | | GitHub Copilot | [dist/copilot/README.md](dist/copilot/README.md) | diff --git a/dist/activepieces/.eslintrc.json b/dist/activepieces/.eslintrc.json new file mode 100644 index 0000000..6f15366 --- /dev/null +++ b/dist/activepieces/.eslintrc.json @@ -0,0 +1,47 @@ +{ + "extends": [ + "../../../../.eslintrc.json" + ], + "ignorePatterns": [ + "!**/*" + ], + "overrides": [ + { + "files": [ + "*.ts", + "*.tsx", + "*.js", + "*.jsx" + ], + "rules": {} + }, + { + "files": [ + "*.ts", + "*.tsx" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + "lodash", + "lodash/*", + "@activepieces/core-*", + "@activepieces/server*", + "@activepieces/engine", + "@activepieces/shared" + ] + } + ] + } + }, + { + "files": [ + "*.js", + "*.jsx" + ], + "rules": {} + } + ] +} diff --git a/dist/activepieces/package.json b/dist/activepieces/package.json new file mode 100644 index 0000000..8c7cb33 --- /dev/null +++ b/dist/activepieces/package.json @@ -0,0 +1,21 @@ +{ + "name": "@activepieces/piece-zenrows", + "version": "0.0.1", + "type": "commonjs", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "@activepieces/pieces-common": "workspace:*", + "@activepieces/pieces-framework": "workspace:*", + "@activepieces/core-piece-types": "workspace:*", + "@activepieces/core-utils": "workspace:*" + }, + "devDependencies": { + "tslib": "2.6.2" + }, + "scripts": { + "build": "tsc -p tsconfig.lib.json && cp package.json dist/", + "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle", + "lint": "eslint 'src/**/*.ts'" + } +} diff --git a/dist/activepieces/src/i18n/translation.json b/dist/activepieces/src/i18n/translation.json new file mode 100644 index 0000000..c057673 --- /dev/null +++ b/dist/activepieces/src/i18n/translation.json @@ -0,0 +1,58 @@ +{ + "Live web data: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows.": "Live web data: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows.", + "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ": "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ", + "Scrape URL": "Scrape URL", + "Extract Structured Data": "Extract Structured Data", + "Get Account Usage": "Get Account Usage", + "Custom API Call": "Custom API Call", + "Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.": "Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.", + "Get specific data from a web page as JSON, using CSS selectors or automatic extraction.": "Get specific data from a web page as JSON, using CSS selectors or automatic extraction.", + "Get your ZenRows plan, credits used this period and concurrency.": "Get your ZenRows plan, credits used this period and concurrency.", + "Make a custom API call to a specific endpoint": "Make a custom API call to a specific endpoint", + "URL": "URL", + "Output Format": "Output Format", + "Adaptive Stealth Mode": "Adaptive Stealth Mode", + "JavaScript Instructions": "JavaScript Instructions", + "Custom Headers": "Custom Headers", + "Extraction Method": "Extraction Method", + "CSS Selectors": "CSS Selectors", + "Fields": "Fields", + "Method": "Method", + "Headers": "Headers", + "Query Parameters": "Query Parameters", + "Body Type": "Body Type", + "Body": "Body", + "Response is Binary": "Response is Binary", + "Return Error as Output": "Return Error as Output", + "Timeout": "Timeout", + "Follow redirects": "Follow redirects", + "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/": "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/", + "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.": "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.", + "Headers to send to the target site, such as Cookie or Referer.": "Headers to send to the target site, such as Cookie or Referer.", + "The full address of the page to extract data from.": "The full address of the page to extract data from.", + "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.": "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.", + "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.": "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.", + "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.": "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.", + "Authorization headers are injected automatically from your connection.": "Authorization headers are injected automatically from your connection.", + "Appended to the URL as ?key=value.": "Appended to the URL as ?key=value.", + "Enable for files like PDFs, images, etc.": "Enable for files like PDFs, images, etc.", + "On a failed request, output the error instead of failing the step.": "On a failed request, output the error instead of failing the step.", + "Seconds to wait for a response. Empty: up to the flow limit (10 min).": "Seconds to wait for a response. Empty: up to the flow limit (10 min).", + "Follow 3xx redirects instead of returning them as the response.": "Follow 3xx redirects instead of returning them as the response.", + "Markdown": "Markdown", + "HTML": "HTML", + "Plain Text": "Plain Text", + "Screenshot (PNG)": "Screenshot (PNG)", + "Automatic": "Automatic", + "GET": "GET", + "POST": "POST", + "PATCH": "PATCH", + "PUT": "PUT", + "DELETE": "DELETE", + "HEAD": "HEAD", + "Request": "Request", + "Data to send": "Data to send", + "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.", + "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.", + "CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains).": "CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains)." +} diff --git a/dist/activepieces/src/index.ts b/dist/activepieces/src/index.ts new file mode 100644 index 0000000..5eba5e1 --- /dev/null +++ b/dist/activepieces/src/index.ts @@ -0,0 +1,38 @@ +import { createCustomApiCallAction } from '@activepieces/pieces-common'; +import { createPiece, PieceCategory } from '@activepieces/pieces-framework'; +import { extractStructuredDataAction } from './lib/actions/extract-structured-data'; +import { getAccountUsageAction } from './lib/actions/get-account-usage'; +import { scrapeUrlAction } from './lib/actions/scrape-url'; +import { zenrowsAuth } from './lib/auth'; +import { ZENROWS_API_BASE_URL } from './lib/common/client'; + +export const zenrows = createPiece({ + displayName: 'ZenRows', + description: + "Live web data: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows" + '.', + auth: zenrowsAuth, + minimumSupportedRelease: '0.88.2', + // Served from this repo until Activepieces hosts it on their CDN. + logoUrl: 'https://raw.githubusercontent.com/ZenRows/zenrows-integrations/main/shared/assets/logo.png', + categories: [PieceCategory.ARTIFICIAL_INTELLIGENCE, PieceCategory.PRODUCTIVITY], + authors: ['imoital'], + actions: [ + scrapeUrlAction, + extractStructuredDataAction, + getAccountUsageAction, + createCustomApiCallAction({ + baseUrl: () => ZENROWS_API_BASE_URL, + auth: zenrowsAuth, + authMapping: async (auth) => ({ + 'X-API-Key': auth.secret_text, + }), + props: { + url: { + description: + 'A full URL or a path relative to https://api.zenrows.com/v1, e.g. /subscriptions/self/details. The API key is sent in the X-API-Key header.', + }, + }, + }), + ], + triggers: [], +}); diff --git a/dist/activepieces/src/lib/actions/extract-structured-data.ts b/dist/activepieces/src/lib/actions/extract-structured-data.ts new file mode 100644 index 0000000..6b6d610 --- /dev/null +++ b/dist/activepieces/src/lib/actions/extract-structured-data.ts @@ -0,0 +1,75 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; + +export const extractStructuredDataAction = createAction({ + auth: zenrowsAuth, + name: 'extract_structured_data', + classification: 'READ', + displayName: 'Extract Structured Data', + description: 'Get specific data from a web page as JSON, using CSS selectors or automatic extraction.', + audience: 'both', + aiMetadata: { + description: + 'Fetches one web page through ZenRows and returns structured JSON instead of raw content. Use the CSS selectors method when you know the page structure (field name → selector), or the automatic method to let ZenRows detect fields such as products, prices or titles on supported sites. Use Scrape URL instead when you need the whole page. Read-only, safe to retry, spends ZenRows credits.', + idempotent: true, + }, + props: { + url: Property.ShortText({ + displayName: 'URL', + description: 'The full address of the page to extract data from.', + required: true, + }), + method: Property.StaticDropdown<'css' | 'auto'>({ + displayName: 'Extraction Method', + description: + 'CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains).', + required: true, + defaultValue: 'css', + options: { + disabled: false, + options: [ + { label: 'CSS Selectors', value: 'css' }, + { label: 'Automatic', value: 'auto' }, + ], + }, + }), + selectors: Property.Object({ + displayName: 'CSS Selectors', + description: + 'For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.', + required: false, + }), + fields: Property.Array({ + displayName: 'Fields', + description: + 'For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.', + required: false, + }), + adaptive: Property.Checkbox({ + displayName: 'Adaptive Stealth Mode', + description: + 'Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.', + required: false, + defaultValue: true, + }), + }, + async run(context) { + const { propsValue } = context; + const fields = (propsValue.fields ?? []).filter((f): f is string => typeof f === 'string' && f.trim() !== ''); + const response = await zenrowsClient.fetchPage({ + apiKey: context.auth.secret_text, + params: { + url: propsValue.url, + ...(propsValue.adaptive !== false ? { mode: 'auto' } : {}), + ...(propsValue.method === 'auto' + ? { extract: 'auto', ...(fields.length ? { extract_fields: fields.join(',') } : {}) } + : { css_extractor: JSON.stringify(propsValue.selectors ?? {}) }), + }, + responseType: 'json', + }); + // Automatic extraction answers { html, parsed }; the flow wants the data, + // not the page it came from (https://docs.zenrows.com/extract/setup#response). + return propsValue.method === 'auto' ? response.body.parsed ?? response.body : response.body; + }, +}); diff --git a/dist/activepieces/src/lib/actions/get-account-usage.ts b/dist/activepieces/src/lib/actions/get-account-usage.ts new file mode 100644 index 0000000..618e98e --- /dev/null +++ b/dist/activepieces/src/lib/actions/get-account-usage.ts @@ -0,0 +1,85 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { createAction, OutputSchema } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; + +export const getAccountUsageAction = createAction({ + auth: zenrowsAuth, + name: 'get_account_usage', + classification: 'READ', + displayName: 'Get Account Usage', + description: 'Get your ZenRows plan, credits used this period and concurrency.', + audience: 'both', + outputSchema: usageOutputSchema(), + aiMetadata: { + description: + 'Returns the ZenRows plan, credits used and remaining in the current billing period, and concurrency in use. Use it to check the budget before a large scraping job or to alert when usage is high. Free to call and safe to retry.', + idempotent: true, + }, + props: {}, + async run(context) { + const details = await zenrowsClient.request({ + apiKey: context.auth.secret_text, + method: HttpMethod.GET, + path: '/subscriptions/self/details', + }); + const creditLimit = details.credit_limit ?? null; + const usageCredits = details.usage_credits ?? null; + return { + plan_name: details.plan?.name ?? null, + plan_recurrence: details.plan?.recurrence ?? null, + status: details.status ?? null, + period_starts_at: details.period_starts_at ?? null, + period_ends_at: details.period_ends_at ?? null, + credit_limit: creditLimit, + credits_used: usageCredits, + credits_remaining: + creditLimit === null || usageCredits === null ? null : Math.max(creditLimit - usageCredits, 0), + usage_percent: details.usage_percent ?? null, + usage_usd: details.usage ?? null, + concurrency_limit: details.plan?.products?.api?.concurrency?.limit ?? null, + concurrency_in_use: details.plan?.products?.api?.concurrency?.usage ?? null, + }; + }, +}); + +function usageOutputSchema(): OutputSchema { + return { + fields: [ + { key: 'plan_name', label: 'Plan' }, + { key: 'plan_recurrence', label: 'Billing Recurrence' }, + { key: 'status', label: 'Status' }, + { key: 'period_starts_at', label: 'Period Starts At', format: 'datetime' }, + { key: 'period_ends_at', label: 'Period Ends At', format: 'datetime' }, + { key: 'credit_limit', label: 'Credit Limit', format: 'number' }, + { key: 'credits_used', label: 'Credits Used', format: 'number' }, + { key: 'credits_remaining', label: 'Credits Remaining', format: 'number' }, + { key: 'usage_percent', label: 'Usage (%)', format: 'number' }, + { key: 'usage_usd', label: 'Spend (USD)', format: 'currency', currency: 'USD' }, + { key: 'concurrency_limit', label: 'Concurrency Limit', format: 'number' }, + { key: 'concurrency_in_use', label: 'Concurrency In Use', format: 'number' }, + ], + }; +} + +type SubscriptionDetails = { + status?: string; + period_starts_at?: string; + period_ends_at?: string; + usage?: number; + usage_credits?: number; + credit_limit?: number; + usage_percent?: number; + plan?: { + name?: string; + recurrence?: string; + products?: { + api?: { + concurrency?: { + limit?: number; + usage?: number; + }; + }; + }; + }; +}; diff --git a/dist/activepieces/src/lib/actions/scrape-url.ts b/dist/activepieces/src/lib/actions/scrape-url.ts new file mode 100644 index 0000000..944b97e --- /dev/null +++ b/dist/activepieces/src/lib/actions/scrape-url.ts @@ -0,0 +1,134 @@ +import { randomUUID } from 'crypto'; +import { createAction, OutputSchema, Property } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; + +type OutputFormat = 'markdown' | 'html' | 'plaintext' | 'screenshot'; + +// What each format adds to the request. A screenshot needs a real browser, so +// it sets js_render itself instead of leaving the choice to Adaptive Stealth Mode. +const FORMAT_PARAMS: Record> = { + markdown: { response_type: 'markdown' }, + html: {}, + plaintext: { response_type: 'plaintext' }, + screenshot: { js_render: 'true', screenshot_fullpage: 'true', screenshot_format: 'png' }, +}; + +export const scrapeUrlAction = createAction({ + auth: zenrowsAuth, + name: 'scrape_url', + classification: 'READ', + displayName: 'Scrape URL', + description: + 'Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.', + audience: 'both', + outputSchema: scrapeOutputSchema(), + aiMetadata: { + description: + 'Fetches one web page through ZenRows and returns its content as Markdown, HTML, plain text or a full-page screenshot, getting past CAPTCHAs and anti-bot systems. Pick this when you have the exact URL and need the page content; use Extract Structured Data when you need specific fields as JSON. Read-only against the target, so retries are safe, but each call spends ZenRows credits.', + idempotent: true, + }, + props: { + url: Property.ShortText({ + displayName: 'URL', + description: 'The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/', + required: true, + }), + output_format: Property.StaticDropdown({ + displayName: 'Output Format', + description: 'Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.', + required: true, + defaultValue: 'markdown', + options: { + disabled: false, + options: [ + { label: 'Markdown', value: 'markdown' }, + { label: 'HTML', value: 'html' }, + { label: 'Plain Text', value: 'plaintext' }, + { label: 'Screenshot (PNG)', value: 'screenshot' }, + ], + }, + }), + adaptive: Property.Checkbox({ + displayName: 'Adaptive Stealth Mode', + description: + 'Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.', + required: false, + defaultValue: true, + }), + js_instructions: Property.LongText({ + displayName: 'JavaScript Instructions', + description: + 'Actions to run on the page before it is returned, as a JSON array, e.g. [{"click": ".load-more"}, {"wait": 1000}]. See https://docs.zenrows.com for the full list.', + required: false, + }), + custom_headers: Property.Object({ + displayName: 'Custom Headers', + description: 'Headers to send to the target site, such as Cookie or Referer.', + required: false, + }), + }, + async run(context) { + const { propsValue } = context; + const outputFormat = propsValue.output_format ?? 'markdown'; + const isScreenshot = outputFormat === 'screenshot'; + const forwardedHeaders = stringValues(propsValue.custom_headers); + const forwardHeaders = Object.keys(forwardedHeaders).length > 0; + const jsInstructions = propsValue.js_instructions?.trim(); + + const response = await zenrowsClient.fetchPage({ + apiKey: context.auth.secret_text, + params: { + url: propsValue.url, + ...(propsValue.adaptive !== false && !isScreenshot ? { mode: 'auto' } : {}), + ...FORMAT_PARAMS[outputFormat], + ...(jsInstructions ? { js_instructions: jsInstructions } : {}), + ...(forwardHeaders ? { custom_headers: 'true' } : {}), + }, + forwardedHeaders: forwardHeaders ? forwardedHeaders : undefined, + responseType: isScreenshot ? 'arraybuffer' : 'text', + }); + + const screenshotUrl = isScreenshot + ? await context.files.write({ + fileName: `zenrows-screenshot-${randomUUID()}.png`, + data: Buffer.from(response.body), + }) + : null; + + const requestCost = zenrowsClient.readHeader({ headers: response.headers, name: 'X-Request-Cost' }); + + return { + url: propsValue.url, + output_format: outputFormat, + content: isScreenshot ? null : String(response.body), + screenshot_url: screenshotUrl, + final_url: zenrowsClient.readHeader({ headers: response.headers, name: 'Zr-Final-Url' }), + request_cost: requestCost === null ? null : Number(requestCost), + request_id: zenrowsClient.readHeader({ headers: response.headers, name: 'X-Request-Id' }), + }; + }, +}); + +// The headers the form holds as text, trimmed; other values are left out. +function stringValues(value: Record | null | undefined): Record { + return Object.fromEntries( + Object.entries(value ?? {}).flatMap(([key, entry]) => + typeof entry === 'string' && entry.trim() && key.trim() ? [[key.trim(), entry.trim()]] : [] + ) + ); +} + +function scrapeOutputSchema(): OutputSchema { + return { + fields: [ + { key: 'url', label: 'URL', format: 'url' }, + { key: 'output_format', label: 'Output Format' }, + { key: 'content', label: 'Content' }, + { key: 'screenshot_url', label: 'Screenshot', format: 'image' }, + { key: 'final_url', label: 'Final URL', format: 'url' }, + { key: 'request_cost', label: 'Request Cost', format: 'number' }, + { key: 'request_id', label: 'Request ID' }, + ], + }; +} diff --git a/dist/activepieces/src/lib/auth.ts b/dist/activepieces/src/lib/auth.ts new file mode 100644 index 0000000..4f4bc0a --- /dev/null +++ b/dist/activepieces/src/lib/auth.ts @@ -0,0 +1,27 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { PieceAuth } from '@activepieces/pieces-framework'; +import { zenrowsClient } from './common/client'; + +export const zenrowsAuth = PieceAuth.SecretText({ + displayName: 'API Key', + required: true, + description: ` +**Get your ZenRows API key:** + +1. Sign in at https://app.zenrows.com (or create a free account). +2. Open the **Builder** or **Settings** page. +3. Copy your API key and paste it here. + `, + validate: async ({ auth }) => { + try { + await zenrowsClient.request({ + apiKey: auth, + method: HttpMethod.GET, + path: '/subscriptions/self/details', + }); + return { valid: true }; + } catch (e) { + return { valid: false, error: 'Invalid API key' }; + } + }, +}); diff --git a/dist/activepieces/src/lib/common/client.ts b/dist/activepieces/src/lib/common/client.ts new file mode 100644 index 0000000..8080628 --- /dev/null +++ b/dist/activepieces/src/lib/common/client.ts @@ -0,0 +1,137 @@ +import { + HttpError, + HttpHeaders, + HttpMessageBody, + HttpMethod, + HttpResponse, + httpClient, +} from '@activepieces/pieces-common'; + +async function request({ + apiKey, + method, + path, + body, +}: { + apiKey: string; + method: HttpMethod; + path: string; + body?: unknown; +}): Promise { + try { + const response = await httpClient.sendRequest({ + method, + url: `${ZENROWS_API_BASE_URL}${path}`, + headers: { + 'X-API-Key': apiKey, + 'Content-Type': 'application/json', + ...CLIENT_HEADERS, + }, + body, + }); + return response.body; + } catch (error) { + throw toZenrowsError({ error }); + } +} + +// responseType: 'text' keeps a page as it is (a JSON page would otherwise come +// back parsed), 'json' is for Extract's output, 'arraybuffer' for a screenshot. +async function fetchPage({ + apiKey, + params, + forwardedHeaders, + responseType, +}: { + apiKey: string; + params: Record; + forwardedHeaders?: Record; + responseType: 'text' | 'json' | 'arraybuffer'; +}): Promise { + try { + return await httpClient.sendRequest({ + method: HttpMethod.GET, + url: `${ZENROWS_API_BASE_URL}/`, + queryParams: { apikey: apiKey, ...params }, + headers: forwardedHeaders ?? CLIENT_HEADERS, + timeout: FETCH_TIMEOUT_MS, + responseType, + }); + } catch (error) { + throw toZenrowsError({ error }); + } +} + +// The API answers every failure with a code and a reason +// (https://docs.zenrows.com/api-error-codes); the message keeps both, and adds +// what to do next for the codes where Activepieces changes the answer. +function toZenrowsError({ error }: { error: unknown }): unknown { + if (!(error instanceof HttpError)) { + return error; + } + const { status, body } = error.response; + const problem = parseProblem({ body }); + if (!problem) { + return error; + } + const hint = ERROR_HINTS[problem.code]; + return new Error(`ZenRows error ${problem.code} (HTTP ${status}): ${problem.reason}${hint ? ` ${hint}` : ''}`); +} + +function parseProblem({ body }: { body: unknown }): { code: string; reason: string } | null { + if (typeof body === 'string') { + try { + return parseProblem({ body: JSON.parse(body) }); + } catch { + return null; + } + } + if (typeof body !== 'object' || body === null || Array.isArray(body)) { + return null; + } + const { code, detail, title } = body as Record; + if (typeof code !== 'string') { + return null; + } + return { + code, + reason: typeof detail === 'string' ? detail : typeof title === 'string' ? title : 'Request failed', + }; +} + +function readHeader({ headers, name }: { headers: HttpHeaders | undefined; name: string }): string | null { + const value = headers?.[name.toLowerCase()]; + if (Array.isArray(value)) { + return value[0] ?? null; + } + return value ?? null; +} + +const FETCH_TIMEOUT_MS = 180000; + +const CLIENT_HEADERS = { 'X-ZenRows-Client': 'activepieces' }; + +const AUTH_HINT = 'Check the API key in your ZenRows connection.'; + +// The facts below come from zenrows-integrations' shared/text/, the same +// wording the Zenrows skills use; each hint adds what to do in Activepieces. +function sentence(fact: string, next: string): string { + return `${fact.charAt(0).toUpperCase()}${fact.slice(1)}. ${next}`; +} + +const ERROR_HINTS: Record = { + AUTH001: AUTH_HINT, + AUTH002: AUTH_HINT, + AUTH003: AUTH_HINT, + AUTH004: sentence("the account's credit allowance is spent", 'Top up or upgrade at https://app.zenrows.com.'), + AUTH014: sentence("this API key reached its credit cap", 'Raise the cap in the ZenRows dashboard.'), + REQS002: 'This page needs JS rendering or premium proxies: turn on Adaptive Stealth Mode.', +}; + +export const ZENROWS_API_BASE_URL = 'https://api.zenrows.com/v1'; + +export const zenrowsClient = { + request, + fetchPage, + readHeader, +}; diff --git a/dist/activepieces/tsconfig.json b/dist/activepieces/tsconfig.json new file mode 100644 index 0000000..b512ca3 --- /dev/null +++ b/dist/activepieces/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} \ No newline at end of file diff --git a/dist/activepieces/tsconfig.lib.json b/dist/activepieces/tsconfig.lib.json new file mode 100644 index 0000000..8b4345e --- /dev/null +++ b/dist/activepieces/tsconfig.lib.json @@ -0,0 +1,22 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "baseUrl": ".", + "paths": {}, + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "types": [ + "node" + ] + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "jest.config.ts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} \ No newline at end of file diff --git a/platforms/activepieces/.eslintrc.json b/platforms/activepieces/.eslintrc.json new file mode 100644 index 0000000..6f15366 --- /dev/null +++ b/platforms/activepieces/.eslintrc.json @@ -0,0 +1,47 @@ +{ + "extends": [ + "../../../../.eslintrc.json" + ], + "ignorePatterns": [ + "!**/*" + ], + "overrides": [ + { + "files": [ + "*.ts", + "*.tsx", + "*.js", + "*.jsx" + ], + "rules": {} + }, + { + "files": [ + "*.ts", + "*.tsx" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + "lodash", + "lodash/*", + "@activepieces/core-*", + "@activepieces/server*", + "@activepieces/engine", + "@activepieces/shared" + ] + } + ] + } + }, + { + "files": [ + "*.js", + "*.jsx" + ], + "rules": {} + } + ] +} diff --git a/platforms/activepieces/README.md b/platforms/activepieces/README.md new file mode 100644 index 0000000..9a74241 --- /dev/null +++ b/platforms/activepieces/README.md @@ -0,0 +1,33 @@ +# Zenrows piece for Activepieces + +The Zenrows [piece](https://www.activepieces.com/docs/build-pieces/building-pieces/overview) for Activepieces: Scrape URL, Extract structured data, Get account usage, and a custom API call. + +This is a code host (`kind: code` in `vars.yaml`). `npm run render` writes `dist/activepieces/`, which is the piece exactly as it sits in the activepieces repo (`packages/pieces/community/zenrows`): package `@activepieces/piece-zenrows`, on their workspace packages. This README stays here and isn't part of the piece. + +## Shared wording + +Facts that the Zenrows skills state too come from `shared/text/` through `{{text:name}}`, which renders as a quoted string. Today that's the meaning of `AUTH004` and `AUTH014`, in `src/lib/common/client.ts`. The piece keeps its forms short and passes the rest through: the API validates option combinations and explains every error, so the piece doesn't repeat that. A wording fix in `shared/text/` reaches the skills and the piece in the same render. + +## Check it + +``` +npm run render +platforms/activepieces/scripts/check.sh # clones activepieces at vars.yaml's upstream into .cache/ +platforms/activepieces/scripts/check.sh ~/path/to/activepieces # or reuse a checkout you already have +``` + +It copies `dist/activepieces/` into the activepieces checkout, registers it in `tsconfig.base.json`, installs with their pinned bun, and runs their `lint`, `build` and `test` for this piece only. CI runs the same script (`.github/workflows/activepieces.yml`). + +`upstream` in `vars.yaml` pins the activepieces commit. Moving it is its own PR, so CI shows whether their newer code breaks the piece. + +## Try it live + +``` +platforms/activepieces/scripts/smoke.sh +``` + +It builds the piece as above, asks for your Zenrows API key (kept in that process only), then runs each action against the live API with a fake Activepieces context: Scrape URL as Markdown, a screenshot and a JSON page, Extract by CSS and automatic on a listing and a product page, Get Account Usage, Custom API Call, and two failures that should show the API's code with our hint. Each run spends a few credits, so it isn't in CI. It can't check the forms; for those, run Activepieces locally with `AP_DEV_PIECES=zenrows`. + +## How it ships + +Not yet. First as our own npm package; then as a PR to activepieces once they agree to take it (they close outside PRs that weren't discussed with them first; see their CONTRIBUTING.md). Both release steps come in later PRs. diff --git a/platforms/activepieces/package.json b/platforms/activepieces/package.json new file mode 100644 index 0000000..8c7cb33 --- /dev/null +++ b/platforms/activepieces/package.json @@ -0,0 +1,21 @@ +{ + "name": "@activepieces/piece-zenrows", + "version": "0.0.1", + "type": "commonjs", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "@activepieces/pieces-common": "workspace:*", + "@activepieces/pieces-framework": "workspace:*", + "@activepieces/core-piece-types": "workspace:*", + "@activepieces/core-utils": "workspace:*" + }, + "devDependencies": { + "tslib": "2.6.2" + }, + "scripts": { + "build": "tsc -p tsconfig.lib.json && cp package.json dist/", + "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle", + "lint": "eslint 'src/**/*.ts'" + } +} diff --git a/platforms/activepieces/scripts/check.sh b/platforms/activepieces/scripts/check.sh new file mode 100755 index 0000000..7acbd93 --- /dev/null +++ b/platforms/activepieces/scripts/check.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Lints, builds and tests the rendered piece (dist/activepieces/) inside an +# activepieces checkout, the way their CI does, because the piece depends on +# their workspace packages. +# +# platforms/activepieces/scripts/check.sh clones activepieces at vars.yaml's upstream into .cache/activepieces +# platforms/activepieces/scripts/check.sh uses an existing checkout instead (faster locally) +# +# dist/activepieces/ is copied to /packages/pieces/community/zenrows, so a +# checkout you pass in gets this repo's version of the piece. Run npm run render +# first; this checks the committed render is current. +set -euo pipefail + +repo="$(cd "$(dirname "$0")/../../.." && pwd)" +(cd "$repo" && node scripts/render.mjs --check >/dev/null) +sha="$(sed -n 's/^upstream: *"\{0,1\}\([0-9a-f]\{40\}\)"\{0,1\} *$/\1/p' "$repo/platforms/activepieces/vars.yaml")" +[ -n "$sha" ] || { echo "platforms/activepieces/vars.yaml has no upstream: <40-character sha>" >&2; exit 1; } +ap="${1:-$repo/.cache/activepieces}" + +if [ -z "${1:-}" ]; then + if [ "$(git -C "$ap" rev-parse HEAD 2>/dev/null)" != "$sha" ]; then + rm -rf "$ap" + git init -q "$ap" + git -C "$ap" fetch -q --depth 1 https://github.com/activepieces/activepieces.git "$sha" + git -C "$ap" checkout -q FETCH_HEAD + fi +elif [ "$(git -C "$ap" rev-parse HEAD)" != "$sha" ]; then + echo "note: $ap is at $(git -C "$ap" rev-parse --short HEAD), vars.yaml pins ${sha:0:10}" >&2 +fi + +piece="$ap/packages/pieces/community/zenrows" +rm -rf "$piece" +mkdir -p "$piece" +cp -R "$repo/dist/activepieces/." "$piece" + +# Register the piece's path, as a new piece in their repo does. +node -e ' +const fs = require("fs"); +const file = process.argv[1]; +const config = JSON.parse(fs.readFileSync(file, "utf8")); +const paths = config.compilerOptions.paths; +if (!paths["@activepieces/piece-zenrows"]) { + paths["@activepieces/piece-zenrows"] = ["packages/pieces/community/zenrows/src/index.ts"]; + config.compilerOptions.paths = Object.fromEntries(Object.entries(paths).sort(([a], [b]) => a.localeCompare(b))); + fs.writeFileSync(file, JSON.stringify(config, null, 2) + "\n"); +}' "$ap/tsconfig.base.json" + +# Their pinned bun. Turbo looks for it on PATH, so when the installed bun is +# missing or another version, install theirs into .cache/ and put it first. +bun_version="$(node -p 'require(process.argv[1]).packageManager.split("@")[1]' "$ap/package.json")" +if ! command -v bun >/dev/null || [ "$(bun --version)" != "$bun_version" ]; then + prefix="$repo/.cache/bun-$bun_version" + [ -x "$prefix/node_modules/.bin/bun" ] || npm install -s --prefix "$prefix" "bun@$bun_version" + export PATH="$prefix/node_modules/.bin:$PATH" +fi + +cd "$ap" +# Not --frozen-lockfile: the new piece adds a workspace entry to bun.lock. +bun install +npx turbo run lint build test --filter=@activepieces/piece-zenrows diff --git a/platforms/activepieces/scripts/smoke.sh b/platforms/activepieces/scripts/smoke.sh new file mode 100755 index 0000000..c37935e --- /dev/null +++ b/platforms/activepieces/scripts/smoke.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Runs each action of the rendered piece against the live Zenrows API, with a +# fake Activepieces context (platforms/activepieces/scripts/smoke.ts), and prints +# what each returns. It asks for your API key and keeps it in this process only. +# +# platforms/activepieces/scripts/smoke.sh builds the piece in .cache/activepieces first (check.sh) +# platforms/activepieces/scripts/smoke.sh uses an existing activepieces checkout instead +# +# Each run spends a few credits. Not for CI. +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(cd "$here/../../.." && pwd)" +ap="${1:-$repo/.cache/activepieces}" + +if [ -z "${ZENROWS_API_KEY:-}" ]; then + # A key piped in without a final newline makes read report end-of-file. + read -rsp "Zenrows API key (not shown): " ZENROWS_API_KEY || [ -n "$ZENROWS_API_KEY" ] + echo +fi +# A key is letters and digits; anything else is a paste of the wrong thing. +if ! [[ "$ZENROWS_API_KEY" =~ ^[A-Za-z0-9]{20,}$ ]]; then + echo "That isn't a Zenrows API key (expected letters and digits only, got ${#ZENROWS_API_KEY} characters). Copy it from https://app.zenrows.com and try again." >&2 + exit 1 +fi +export ZENROWS_API_KEY + +echo "Building the piece in $ap …" +"$here/check.sh" ${1:+"$1"} >/dev/null 2>&1 /dev/null || [ "$(bun --version)" != "$bun_version" ]; then + export PATH="$repo/.cache/bun-$bun_version/node_modules/.bin:$PATH" +fi + +cd "$ap" +bun run "$here/smoke.ts" "$ap" 2>/dev/null | grep -E "^(✅|❌)"; exit "${PIPESTATUS[0]}" diff --git a/platforms/activepieces/scripts/smoke.ts b/platforms/activepieces/scripts/smoke.ts new file mode 100644 index 0000000..c4ca034 --- /dev/null +++ b/platforms/activepieces/scripts/smoke.ts @@ -0,0 +1,86 @@ +// Run by smoke.sh from inside an activepieces checkout, so bun resolves the +// piece's workspace imports: calls each action's run() with a fake Activepieces +// context against the live API. The key comes in ZENROWS_API_KEY; the checkout +// is argv[2]. Screenshots are written to .cache/smoke/. +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const checkout = process.argv[2]; +const apiKey = process.env['ZENROWS_API_KEY']; +if (!checkout || !apiKey) { + console.error('usage: ZENROWS_API_KEY=… bun run smoke.ts '); + process.exit(1); +} +const piece = await import(join(checkout, 'packages/pieces/community/zenrows/src/index.ts')); +const actions = piece.zenrows.actions() as Record Promise }>; +const outDir = join(checkout, '..', 'smoke'); +mkdirSync(outDir, { recursive: true }); + +// The parts of an Activepieces action context the piece reads. +function context(name: string, propsValue: Record, key = apiKey) { + return { + auth: { secret_text: key }, + propsValue, + files: { + write: async ({ fileName, data }: { fileName: string; data: Buffer }) => { + const path = join(outDir, fileName); + writeFileSync(path, data); + return `file://${path}`; + }, + }, + store: { get: async () => null, put: async () => null, delete: async () => null }, + server: { apiUrl: 'http://localhost/', publicUrl: 'http://localhost/', token: '' }, + project: { id: 'smoke', externalId: async () => 'smoke' }, + run: { id: 'smoke', name, isTest: true, pauseId: '', stop: () => {}, pause: () => {} }, + connections: { get: async () => null }, + flows: { current: { id: 'smoke', version: { id: 'smoke' } } }, + step: { name }, + generateResumeUrl: () => '', + tags: { add: async () => {} }, + }; +} + +const LISTING = 'https://www.scrapingcourse.com/ecommerce/'; +const PRODUCT = 'https://www.scrapingcourse.com/ecommerce/product/abominable-hoodie/'; +// [label, action, form values, expected error code (the case passes only by +// failing with it), API key to use instead of the real one] +const cases: [string, string, Record, string?, string?][] = [ + ['scrape_url markdown', 'scrape_url', { url: LISTING, output_format: 'markdown', adaptive: true }], + ['scrape_url screenshot', 'scrape_url', { url: LISTING, output_format: 'screenshot', adaptive: true }], + ['scrape_url html of a JSON page, with headers and JS', 'scrape_url', { url: 'https://httpbin.io/headers', output_format: 'html', adaptive: true, custom_headers: { Referer: 'https://example.com' }, js_instructions: '[{"wait": 500}]' }], + ['extract css', 'extract_structured_data', { url: LISTING, method: 'css', selectors: { title: 'h1', products: '.product-name' }, adaptive: true }], + ['extract auto, listing page', 'extract_structured_data', { url: LISTING, method: 'auto', adaptive: true }], + ['extract auto, product page', 'extract_structured_data', { url: PRODUCT, method: 'auto', adaptive: true }], + ['get_account_usage', 'get_account_usage', {}], + // The Custom API Call's url prop is a dynamic property: { url: }. + ['custom_api_call', 'custom_api_call', { url: { url: '/subscriptions/self/details' }, method: 'GET', headers: {}, queryParams: {}, failsafe: false }], + // These pass only if the API rejects them with this code, and the message + // carries the API's reason plus our hint. + ['wrong key', 'get_account_usage', {}, 'AUTH002', 'not-a-key'], + ['protected page, adaptive off', 'scrape_url', { url: 'https://www.scrapingcourse.com/cloudflare-challenge', output_format: 'html', adaptive: false }, 'REQS002'], +]; + +let failed = 0; +for (const [label, name, props, expectError, key] of cases) { + const started = Date.now(); + const ms = () => `${Date.now() - started} ms`; + try { + const result = await actions[name].run(context(name, props, key)); + const text = JSON.stringify(result); + if (expectError) { + failed++; + console.log(`❌ ${label} (${ms()}): expected a ${expectError} error, got ${text.slice(0, 120)}`); + } else { + console.log(`✅ ${label} (${ms()}): ${text.length > 240 ? `${text.slice(0, 240)}… [${text.length} chars]` : text}`); + } + } catch (e) { + const message = (e as Error).message; + if (expectError && message.includes(expectError)) { + console.log(`✅ ${label} (${ms()}): rejected with ${expectError}, as intended: ${message}`); + } else { + failed++; + console.log(`❌ ${label} (${ms()}): ${message}`); + } + } +} +process.exit(failed ? 1 : 0); diff --git a/platforms/activepieces/src/i18n/translation.json b/platforms/activepieces/src/i18n/translation.json new file mode 100644 index 0000000..f1d2b83 --- /dev/null +++ b/platforms/activepieces/src/i18n/translation.json @@ -0,0 +1,58 @@ +{ + "{{< zenrows-description}}.": "{{< zenrows-description}}.", + "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ": "\n**Get your ZenRows API key:**\n\n1. Sign in at https://app.zenrows.com (or create a free account).\n2. Open the **Builder** or **Settings** page.\n3. Copy your API key and paste it here.\n ", + "Scrape URL": "Scrape URL", + "Extract Structured Data": "Extract Structured Data", + "Get Account Usage": "Get Account Usage", + "Custom API Call": "Custom API Call", + "Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.": "Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.", + "Get specific data from a web page as JSON, using CSS selectors or automatic extraction.": "Get specific data from a web page as JSON, using CSS selectors or automatic extraction.", + "Get your ZenRows plan, credits used this period and concurrency.": "Get your ZenRows plan, credits used this period and concurrency.", + "Make a custom API call to a specific endpoint": "Make a custom API call to a specific endpoint", + "URL": "URL", + "Output Format": "Output Format", + "Adaptive Stealth Mode": "Adaptive Stealth Mode", + "JavaScript Instructions": "JavaScript Instructions", + "Custom Headers": "Custom Headers", + "Extraction Method": "Extraction Method", + "CSS Selectors": "CSS Selectors", + "Fields": "Fields", + "Method": "Method", + "Headers": "Headers", + "Query Parameters": "Query Parameters", + "Body Type": "Body Type", + "Body": "Body", + "Response is Binary": "Response is Binary", + "Return Error as Output": "Return Error as Output", + "Timeout": "Timeout", + "Follow redirects": "Follow redirects", + "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/": "The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/", + "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.": "Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.", + "Headers to send to the target site, such as Cookie or Referer.": "Headers to send to the target site, such as Cookie or Referer.", + "The full address of the page to extract data from.": "The full address of the page to extract data from.", + "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.": "For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.", + "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.": "For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.", + "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.": "Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.", + "Authorization headers are injected automatically from your connection.": "Authorization headers are injected automatically from your connection.", + "Appended to the URL as ?key=value.": "Appended to the URL as ?key=value.", + "Enable for files like PDFs, images, etc.": "Enable for files like PDFs, images, etc.", + "On a failed request, output the error instead of failing the step.": "On a failed request, output the error instead of failing the step.", + "Seconds to wait for a response. Empty: up to the flow limit (10 min).": "Seconds to wait for a response. Empty: up to the flow limit (10 min).", + "Follow 3xx redirects instead of returning them as the response.": "Follow 3xx redirects instead of returning them as the response.", + "Markdown": "Markdown", + "HTML": "HTML", + "Plain Text": "Plain Text", + "Screenshot (PNG)": "Screenshot (PNG)", + "Automatic": "Automatic", + "GET": "GET", + "POST": "POST", + "PATCH": "PATCH", + "PUT": "PUT", + "DELETE": "DELETE", + "HEAD": "HEAD", + "Request": "Request", + "Data to send": "Data to send", + "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.": "Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.", + "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.": "Actions to run on the page before it is returned, as a JSON array, e.g. [{\"click\": \".load-more\"}, {\"wait\": 1000}]. See https://docs.zenrows.com for the full list.", + "CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains).": "CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains)." +} diff --git a/platforms/activepieces/src/index.ts b/platforms/activepieces/src/index.ts new file mode 100644 index 0000000..e86f722 --- /dev/null +++ b/platforms/activepieces/src/index.ts @@ -0,0 +1,38 @@ +import { createCustomApiCallAction } from '@activepieces/pieces-common'; +import { createPiece, PieceCategory } from '@activepieces/pieces-framework'; +import { extractStructuredDataAction } from './lib/actions/extract-structured-data'; +import { getAccountUsageAction } from './lib/actions/get-account-usage'; +import { scrapeUrlAction } from './lib/actions/scrape-url'; +import { zenrowsAuth } from './lib/auth'; +import { ZENROWS_API_BASE_URL } from './lib/common/client'; + +export const zenrows = createPiece({ + displayName: 'ZenRows', + description: + {{text:zenrows-description}} + '.', + auth: zenrowsAuth, + minimumSupportedRelease: '0.88.2', + // Served from this repo until Activepieces hosts it on their CDN. + logoUrl: 'https://raw.githubusercontent.com/ZenRows/zenrows-integrations/main/shared/assets/logo.png', + categories: [PieceCategory.ARTIFICIAL_INTELLIGENCE, PieceCategory.PRODUCTIVITY], + authors: ['imoital'], + actions: [ + scrapeUrlAction, + extractStructuredDataAction, + getAccountUsageAction, + createCustomApiCallAction({ + baseUrl: () => ZENROWS_API_BASE_URL, + auth: zenrowsAuth, + authMapping: async (auth) => ({ + 'X-API-Key': auth.secret_text, + }), + props: { + url: { + description: + 'A full URL or a path relative to https://api.zenrows.com/v1, e.g. /subscriptions/self/details. The API key is sent in the X-API-Key header.', + }, + }, + }), + ], + triggers: [], +}); diff --git a/platforms/activepieces/src/lib/actions/extract-structured-data.ts b/platforms/activepieces/src/lib/actions/extract-structured-data.ts new file mode 100644 index 0000000..6b6d610 --- /dev/null +++ b/platforms/activepieces/src/lib/actions/extract-structured-data.ts @@ -0,0 +1,75 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; + +export const extractStructuredDataAction = createAction({ + auth: zenrowsAuth, + name: 'extract_structured_data', + classification: 'READ', + displayName: 'Extract Structured Data', + description: 'Get specific data from a web page as JSON, using CSS selectors or automatic extraction.', + audience: 'both', + aiMetadata: { + description: + 'Fetches one web page through ZenRows and returns structured JSON instead of raw content. Use the CSS selectors method when you know the page structure (field name → selector), or the automatic method to let ZenRows detect fields such as products, prices or titles on supported sites. Use Scrape URL instead when you need the whole page. Read-only, safe to retry, spends ZenRows credits.', + idempotent: true, + }, + props: { + url: Property.ShortText({ + displayName: 'URL', + description: 'The full address of the page to extract data from.', + required: true, + }), + method: Property.StaticDropdown<'css' | 'auto'>({ + displayName: 'Extraction Method', + description: + 'CSS Selectors works on any site. Automatic needs no setup and works on any domain prepared for Extract; prepare one with the Custom API Call action (POST /extract/prepared-domains).', + required: true, + defaultValue: 'css', + options: { + disabled: false, + options: [ + { label: 'CSS Selectors', value: 'css' }, + { label: 'Automatic', value: 'auto' }, + ], + }, + }), + selectors: Property.Object({ + displayName: 'CSS Selectors', + description: + 'For the CSS Selectors method. Each key is a name for the output field and each value is the CSS selector to read, e.g. title → h1, price → .price. Add @attribute to read an attribute, e.g. link → a.product @href.', + required: false, + }), + fields: Property.Array({ + displayName: 'Fields', + description: + 'For the Automatic method. Optional list of fields to return, using dots for nested fields, e.g. products.name. Leave empty to get every field.', + required: false, + }), + adaptive: Property.Checkbox({ + displayName: 'Adaptive Stealth Mode', + description: + 'Recommended. ZenRows adds JS rendering or premium proxies only when the page needs them.', + required: false, + defaultValue: true, + }), + }, + async run(context) { + const { propsValue } = context; + const fields = (propsValue.fields ?? []).filter((f): f is string => typeof f === 'string' && f.trim() !== ''); + const response = await zenrowsClient.fetchPage({ + apiKey: context.auth.secret_text, + params: { + url: propsValue.url, + ...(propsValue.adaptive !== false ? { mode: 'auto' } : {}), + ...(propsValue.method === 'auto' + ? { extract: 'auto', ...(fields.length ? { extract_fields: fields.join(',') } : {}) } + : { css_extractor: JSON.stringify(propsValue.selectors ?? {}) }), + }, + responseType: 'json', + }); + // Automatic extraction answers { html, parsed }; the flow wants the data, + // not the page it came from (https://docs.zenrows.com/extract/setup#response). + return propsValue.method === 'auto' ? response.body.parsed ?? response.body : response.body; + }, +}); diff --git a/platforms/activepieces/src/lib/actions/get-account-usage.ts b/platforms/activepieces/src/lib/actions/get-account-usage.ts new file mode 100644 index 0000000..618e98e --- /dev/null +++ b/platforms/activepieces/src/lib/actions/get-account-usage.ts @@ -0,0 +1,85 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { createAction, OutputSchema } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; + +export const getAccountUsageAction = createAction({ + auth: zenrowsAuth, + name: 'get_account_usage', + classification: 'READ', + displayName: 'Get Account Usage', + description: 'Get your ZenRows plan, credits used this period and concurrency.', + audience: 'both', + outputSchema: usageOutputSchema(), + aiMetadata: { + description: + 'Returns the ZenRows plan, credits used and remaining in the current billing period, and concurrency in use. Use it to check the budget before a large scraping job or to alert when usage is high. Free to call and safe to retry.', + idempotent: true, + }, + props: {}, + async run(context) { + const details = await zenrowsClient.request({ + apiKey: context.auth.secret_text, + method: HttpMethod.GET, + path: '/subscriptions/self/details', + }); + const creditLimit = details.credit_limit ?? null; + const usageCredits = details.usage_credits ?? null; + return { + plan_name: details.plan?.name ?? null, + plan_recurrence: details.plan?.recurrence ?? null, + status: details.status ?? null, + period_starts_at: details.period_starts_at ?? null, + period_ends_at: details.period_ends_at ?? null, + credit_limit: creditLimit, + credits_used: usageCredits, + credits_remaining: + creditLimit === null || usageCredits === null ? null : Math.max(creditLimit - usageCredits, 0), + usage_percent: details.usage_percent ?? null, + usage_usd: details.usage ?? null, + concurrency_limit: details.plan?.products?.api?.concurrency?.limit ?? null, + concurrency_in_use: details.plan?.products?.api?.concurrency?.usage ?? null, + }; + }, +}); + +function usageOutputSchema(): OutputSchema { + return { + fields: [ + { key: 'plan_name', label: 'Plan' }, + { key: 'plan_recurrence', label: 'Billing Recurrence' }, + { key: 'status', label: 'Status' }, + { key: 'period_starts_at', label: 'Period Starts At', format: 'datetime' }, + { key: 'period_ends_at', label: 'Period Ends At', format: 'datetime' }, + { key: 'credit_limit', label: 'Credit Limit', format: 'number' }, + { key: 'credits_used', label: 'Credits Used', format: 'number' }, + { key: 'credits_remaining', label: 'Credits Remaining', format: 'number' }, + { key: 'usage_percent', label: 'Usage (%)', format: 'number' }, + { key: 'usage_usd', label: 'Spend (USD)', format: 'currency', currency: 'USD' }, + { key: 'concurrency_limit', label: 'Concurrency Limit', format: 'number' }, + { key: 'concurrency_in_use', label: 'Concurrency In Use', format: 'number' }, + ], + }; +} + +type SubscriptionDetails = { + status?: string; + period_starts_at?: string; + period_ends_at?: string; + usage?: number; + usage_credits?: number; + credit_limit?: number; + usage_percent?: number; + plan?: { + name?: string; + recurrence?: string; + products?: { + api?: { + concurrency?: { + limit?: number; + usage?: number; + }; + }; + }; + }; +}; diff --git a/platforms/activepieces/src/lib/actions/scrape-url.ts b/platforms/activepieces/src/lib/actions/scrape-url.ts new file mode 100644 index 0000000..944b97e --- /dev/null +++ b/platforms/activepieces/src/lib/actions/scrape-url.ts @@ -0,0 +1,134 @@ +import { randomUUID } from 'crypto'; +import { createAction, OutputSchema, Property } from '@activepieces/pieces-framework'; +import { zenrowsAuth } from '../auth'; +import { zenrowsClient } from '../common/client'; + +type OutputFormat = 'markdown' | 'html' | 'plaintext' | 'screenshot'; + +// What each format adds to the request. A screenshot needs a real browser, so +// it sets js_render itself instead of leaving the choice to Adaptive Stealth Mode. +const FORMAT_PARAMS: Record> = { + markdown: { response_type: 'markdown' }, + html: {}, + plaintext: { response_type: 'plaintext' }, + screenshot: { js_render: 'true', screenshot_fullpage: 'true', screenshot_format: 'png' }, +}; + +export const scrapeUrlAction = createAction({ + auth: zenrowsAuth, + name: 'scrape_url', + classification: 'READ', + displayName: 'Scrape URL', + description: + 'Get the content of any web page as Markdown, HTML, plain text or a screenshot, bypassing anti-bot protection.', + audience: 'both', + outputSchema: scrapeOutputSchema(), + aiMetadata: { + description: + 'Fetches one web page through ZenRows and returns its content as Markdown, HTML, plain text or a full-page screenshot, getting past CAPTCHAs and anti-bot systems. Pick this when you have the exact URL and need the page content; use Extract Structured Data when you need specific fields as JSON. Read-only against the target, so retries are safe, but each call spends ZenRows credits.', + idempotent: true, + }, + props: { + url: Property.ShortText({ + displayName: 'URL', + description: 'The full address of the page to scrape, e.g. https://www.scrapingcourse.com/ecommerce/', + required: true, + }), + output_format: Property.StaticDropdown({ + displayName: 'Output Format', + description: 'Markdown is the easiest format to read or pass to an AI step. Screenshots always use JS rendering.', + required: true, + defaultValue: 'markdown', + options: { + disabled: false, + options: [ + { label: 'Markdown', value: 'markdown' }, + { label: 'HTML', value: 'html' }, + { label: 'Plain Text', value: 'plaintext' }, + { label: 'Screenshot (PNG)', value: 'screenshot' }, + ], + }, + }), + adaptive: Property.Checkbox({ + displayName: 'Adaptive Stealth Mode', + description: + 'Recommended. ZenRows starts with the cheapest setup and adds JS rendering or premium proxies only when the page needs them; you pay only for what succeeds. Turn off for a basic request, or use the Custom API Call action to set each option yourself.', + required: false, + defaultValue: true, + }), + js_instructions: Property.LongText({ + displayName: 'JavaScript Instructions', + description: + 'Actions to run on the page before it is returned, as a JSON array, e.g. [{"click": ".load-more"}, {"wait": 1000}]. See https://docs.zenrows.com for the full list.', + required: false, + }), + custom_headers: Property.Object({ + displayName: 'Custom Headers', + description: 'Headers to send to the target site, such as Cookie or Referer.', + required: false, + }), + }, + async run(context) { + const { propsValue } = context; + const outputFormat = propsValue.output_format ?? 'markdown'; + const isScreenshot = outputFormat === 'screenshot'; + const forwardedHeaders = stringValues(propsValue.custom_headers); + const forwardHeaders = Object.keys(forwardedHeaders).length > 0; + const jsInstructions = propsValue.js_instructions?.trim(); + + const response = await zenrowsClient.fetchPage({ + apiKey: context.auth.secret_text, + params: { + url: propsValue.url, + ...(propsValue.adaptive !== false && !isScreenshot ? { mode: 'auto' } : {}), + ...FORMAT_PARAMS[outputFormat], + ...(jsInstructions ? { js_instructions: jsInstructions } : {}), + ...(forwardHeaders ? { custom_headers: 'true' } : {}), + }, + forwardedHeaders: forwardHeaders ? forwardedHeaders : undefined, + responseType: isScreenshot ? 'arraybuffer' : 'text', + }); + + const screenshotUrl = isScreenshot + ? await context.files.write({ + fileName: `zenrows-screenshot-${randomUUID()}.png`, + data: Buffer.from(response.body), + }) + : null; + + const requestCost = zenrowsClient.readHeader({ headers: response.headers, name: 'X-Request-Cost' }); + + return { + url: propsValue.url, + output_format: outputFormat, + content: isScreenshot ? null : String(response.body), + screenshot_url: screenshotUrl, + final_url: zenrowsClient.readHeader({ headers: response.headers, name: 'Zr-Final-Url' }), + request_cost: requestCost === null ? null : Number(requestCost), + request_id: zenrowsClient.readHeader({ headers: response.headers, name: 'X-Request-Id' }), + }; + }, +}); + +// The headers the form holds as text, trimmed; other values are left out. +function stringValues(value: Record | null | undefined): Record { + return Object.fromEntries( + Object.entries(value ?? {}).flatMap(([key, entry]) => + typeof entry === 'string' && entry.trim() && key.trim() ? [[key.trim(), entry.trim()]] : [] + ) + ); +} + +function scrapeOutputSchema(): OutputSchema { + return { + fields: [ + { key: 'url', label: 'URL', format: 'url' }, + { key: 'output_format', label: 'Output Format' }, + { key: 'content', label: 'Content' }, + { key: 'screenshot_url', label: 'Screenshot', format: 'image' }, + { key: 'final_url', label: 'Final URL', format: 'url' }, + { key: 'request_cost', label: 'Request Cost', format: 'number' }, + { key: 'request_id', label: 'Request ID' }, + ], + }; +} diff --git a/platforms/activepieces/src/lib/auth.ts b/platforms/activepieces/src/lib/auth.ts new file mode 100644 index 0000000..4f4bc0a --- /dev/null +++ b/platforms/activepieces/src/lib/auth.ts @@ -0,0 +1,27 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { PieceAuth } from '@activepieces/pieces-framework'; +import { zenrowsClient } from './common/client'; + +export const zenrowsAuth = PieceAuth.SecretText({ + displayName: 'API Key', + required: true, + description: ` +**Get your ZenRows API key:** + +1. Sign in at https://app.zenrows.com (or create a free account). +2. Open the **Builder** or **Settings** page. +3. Copy your API key and paste it here. + `, + validate: async ({ auth }) => { + try { + await zenrowsClient.request({ + apiKey: auth, + method: HttpMethod.GET, + path: '/subscriptions/self/details', + }); + return { valid: true }; + } catch (e) { + return { valid: false, error: 'Invalid API key' }; + } + }, +}); diff --git a/platforms/activepieces/src/lib/common/client.ts b/platforms/activepieces/src/lib/common/client.ts new file mode 100644 index 0000000..1e724c2 --- /dev/null +++ b/platforms/activepieces/src/lib/common/client.ts @@ -0,0 +1,137 @@ +import { + HttpError, + HttpHeaders, + HttpMessageBody, + HttpMethod, + HttpResponse, + httpClient, +} from '@activepieces/pieces-common'; + +async function request({ + apiKey, + method, + path, + body, +}: { + apiKey: string; + method: HttpMethod; + path: string; + body?: unknown; +}): Promise { + try { + const response = await httpClient.sendRequest({ + method, + url: `${ZENROWS_API_BASE_URL}${path}`, + headers: { + 'X-API-Key': apiKey, + 'Content-Type': 'application/json', + ...CLIENT_HEADERS, + }, + body, + }); + return response.body; + } catch (error) { + throw toZenrowsError({ error }); + } +} + +// responseType: 'text' keeps a page as it is (a JSON page would otherwise come +// back parsed), 'json' is for Extract's output, 'arraybuffer' for a screenshot. +async function fetchPage({ + apiKey, + params, + forwardedHeaders, + responseType, +}: { + apiKey: string; + params: Record; + forwardedHeaders?: Record; + responseType: 'text' | 'json' | 'arraybuffer'; +}): Promise { + try { + return await httpClient.sendRequest({ + method: HttpMethod.GET, + url: `${ZENROWS_API_BASE_URL}/`, + queryParams: { apikey: apiKey, ...params }, + headers: forwardedHeaders ?? CLIENT_HEADERS, + timeout: FETCH_TIMEOUT_MS, + responseType, + }); + } catch (error) { + throw toZenrowsError({ error }); + } +} + +// The API answers every failure with a code and a reason +// (https://docs.zenrows.com/api-error-codes); the message keeps both, and adds +// what to do next for the codes where Activepieces changes the answer. +function toZenrowsError({ error }: { error: unknown }): unknown { + if (!(error instanceof HttpError)) { + return error; + } + const { status, body } = error.response; + const problem = parseProblem({ body }); + if (!problem) { + return error; + } + const hint = ERROR_HINTS[problem.code]; + return new Error(`ZenRows error ${problem.code} (HTTP ${status}): ${problem.reason}${hint ? ` ${hint}` : ''}`); +} + +function parseProblem({ body }: { body: unknown }): { code: string; reason: string } | null { + if (typeof body === 'string') { + try { + return parseProblem({ body: JSON.parse(body) }); + } catch { + return null; + } + } + if (typeof body !== 'object' || body === null || Array.isArray(body)) { + return null; + } + const { code, detail, title } = body as Record; + if (typeof code !== 'string') { + return null; + } + return { + code, + reason: typeof detail === 'string' ? detail : typeof title === 'string' ? title : 'Request failed', + }; +} + +function readHeader({ headers, name }: { headers: HttpHeaders | undefined; name: string }): string | null { + const value = headers?.[name.toLowerCase()]; + if (Array.isArray(value)) { + return value[0] ?? null; + } + return value ?? null; +} + +const FETCH_TIMEOUT_MS = 180000; + +const CLIENT_HEADERS = { 'X-ZenRows-Client': 'activepieces' }; + +const AUTH_HINT = 'Check the API key in your ZenRows connection.'; + +// The facts below come from zenrows-integrations' shared/text/, the same +// wording the Zenrows skills use; each hint adds what to do in Activepieces. +function sentence(fact: string, next: string): string { + return `${fact.charAt(0).toUpperCase()}${fact.slice(1)}. ${next}`; +} + +const ERROR_HINTS: Record = { + AUTH001: AUTH_HINT, + AUTH002: AUTH_HINT, + AUTH003: AUTH_HINT, + AUTH004: sentence({{text:auth004}}, 'Top up or upgrade at https://app.zenrows.com.'), + AUTH014: sentence({{text:auth014}}, 'Raise the cap in the ZenRows dashboard.'), + REQS002: 'This page needs JS rendering or premium proxies: turn on Adaptive Stealth Mode.', +}; + +export const ZENROWS_API_BASE_URL = 'https://api.zenrows.com/v1'; + +export const zenrowsClient = { + request, + fetchPage, + readHeader, +}; diff --git a/platforms/activepieces/tsconfig.json b/platforms/activepieces/tsconfig.json new file mode 100644 index 0000000..b512ca3 --- /dev/null +++ b/platforms/activepieces/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} \ No newline at end of file diff --git a/platforms/activepieces/tsconfig.lib.json b/platforms/activepieces/tsconfig.lib.json new file mode 100644 index 0000000..8b4345e --- /dev/null +++ b/platforms/activepieces/tsconfig.lib.json @@ -0,0 +1,22 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "baseUrl": ".", + "paths": {}, + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "types": [ + "node" + ] + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "jest.config.ts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} \ No newline at end of file diff --git a/platforms/activepieces/vars.yaml b/platforms/activepieces/vars.yaml new file mode 100644 index 0000000..eddc5cc --- /dev/null +++ b/platforms/activepieces/vars.yaml @@ -0,0 +1,8 @@ +# A code host: the Zenrows piece for Activepieces, rendered into dist/activepieces/ +# exactly as it sits in activepieces' packages/pieces/community/zenrows. +kind: code +host: "Activepieces" +host_full: "Activepieces" +# The activepieces commit the piece is built and tested against (scripts/check.sh). +# Moving it is its own PR, so CI shows whether their newer code breaks the piece. +upstream: "6f4be340d7c4bef9fb6ff874ca72224e4933d3d2" diff --git a/shared/text/auth004.md b/shared/text/auth004.md new file mode 100644 index 0000000..09426a7 --- /dev/null +++ b/shared/text/auth004.md @@ -0,0 +1 @@ +the account's credit allowance is spent diff --git a/shared/text/auth006.md b/shared/text/auth006.md new file mode 100644 index 0000000..b5f06c3 --- /dev/null +++ b/shared/text/auth006.md @@ -0,0 +1 @@ +too many requests are running at once diff --git a/shared/text/auth014.md b/shared/text/auth014.md new file mode 100644 index 0000000..c8511e2 --- /dev/null +++ b/shared/text/auth014.md @@ -0,0 +1 @@ +this API key reached its credit cap diff --git a/shared/text/reqs007-page-not-recognised.md b/shared/text/reqs007-page-not-recognised.md new file mode 100644 index 0000000..dbc749c --- /dev/null +++ b/shared/text/reqs007-page-not-recognised.md @@ -0,0 +1 @@ +the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example) diff --git a/skills/extract-structured-data/SKILL.md b/skills/extract-structured-data/SKILL.md index 3f25523..cdb1c4c 100644 --- a/skills/extract-structured-data/SKILL.md +++ b/skills/extract-structured-data/SKILL.md @@ -24,7 +24,7 @@ Use the dedicated `extract` tool, not `scrape`. It returns structured JSON direc - **`mode` selects the extraction strategy** (`auto`, `autoparse`, `css`). It is not the adaptive request setting, which is the separate boolean `mode_auto` and is on by default. `scrape` has no `mode` parameter at all. - **`extract` with `mode='auto'` works on any domain Zenrows has prepared, with any API key.** There is no per-account gate. When it can't extract a page it returns `REQS007` (or the older `AUTH010`) with a `reason`, and nothing is charged: - `domain_not_prepared`: call `prepare_domain` with the domain and 1 to 5 example URLs, one per kind of page the user wants (for example one product page and one listing page). It costs no credits and takes a few minutes. Poll `prepared_domain_status` about every 30 seconds until `status` is `ready`, then call `extract` again. Each new domain uses one slot of the account's monthly allowance, which `prepared_domain_status` shows. If the user can't wait, use `mode='css'` meanwhile. - - `page_not_recognised`: the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example). Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. + - `page_not_recognised`: {{< reqs007-page-not-recognised}}. Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. - `domain_not_enabled`: preparing doesn't help. Use `mode='css'`, or tell the user to contact Zenrows support to have the site enabled. - **`fallback_autoparse` is deprecated and has no effect.** Extract no longer falls back to autoparse. Don't set it; if you do, the result only carries a note saying it was ignored. diff --git a/skills/using-zenrows/SKILL.md b/skills/using-zenrows/SKILL.md index 4bd1529..62cfdf7 100644 --- a/skills/using-zenrows/SKILL.md +++ b/skills/using-zenrows/SKILL.md @@ -78,8 +78,8 @@ Reads the connected account's plan and usage. It changes nothing. - Do not build manual retry loops around a request. Zenrows already escalates to heavier settings when a page needs it. - `401` or `AUTH003`: Zenrows is not connected, or the sign-in is no longer valid (for example the API key was rotated). Use the `{{skill:getting-started}}` skill to reconnect. Do not ask for a key. -- `AUTH004` (the account's credit allowance is spent) or `AUTH014` (this API key reached its credit cap): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. -- `AUTH006`: too many requests are running at once. Wait for running requests to finish, then retry. +- `AUTH004` ({{< auth004}}) or `AUTH014` ({{< auth014}}): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. +- `AUTH006`: {{< auth006}}. Wait for running requests to finish, then retry. - `413`: the response is too large. Narrow it with `css_extractor`. - `REQS007` (or the older `AUTH010`) from `extract`: read its `reason`. `domain_not_prepared`: prepare the domain with `prepare_domain` and wait for `prepared_domain_status` to say `ready`. `page_not_recognised`: try a product, article or listing page, or `mode='css'`; don't prepare the domain again. `domain_not_enabled`: use `mode='css'`. Not charged in any case. Details in the `{{skill:extract-structured-data}}` skill. - Look up any other error code at https://docs.zenrows.com.