From e90590e4b6282ed2609654b4af44c3fb8825e786 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 11 Nov 2026 15:18:04 +0000 Subject: [PATCH 1/3] docs(tools): tool descriptions must describe, not direct the model Directory reviewers read these strings. Anthropic's policy asks each tool to say what it does and when it fits, but its submission form rules out text that instructs the model ("call X, then Y", "poll every 30 seconds", "do not ..."); OpenAI's guidelines ask for completeness and visible side effects. The current text reads as a sequence of orders, and leaves out costs a caller is charged. Every description now states what the tool does and returns, what it costs, what its side effects are, what each error means, and how it relates to the other tools as a fact about data ("the job_id is what batch_wait takes") rather than as an instruction to call them. Two gaps this closes rather than rewrites: browser_navigate carried no cost at all, and now gives the per-minute and per-GB rates and the two idle timeouts; prepare_domain's monthly slot is now labelled a side effect. Covers the server instructions and the main tools. The step-by-step sequences removed here already exist in the error responses and prepared_domain_status's "next" field. --- src/server.ts | 51 ++++++++++++++++++++------------------------ src/tools/batch.ts | 18 ++++++++-------- src/tools/browser.ts | 17 ++++++++------- src/tools/crawl.ts | 38 ++++++--------------------------- src/tools/extract.ts | 48 ++++++++++++++--------------------------- src/tools/prepare.ts | 27 ++++++++++++----------- 6 files changed, 78 insertions(+), 121 deletions(-) diff --git a/src/server.ts b/src/server.ts index 404a8d8..d04069b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -23,15 +23,17 @@ const DEFAULT_PREMIUM_PROXY = process.env.ZENROWS_PREMIUM_PROXY === "true"; const DEFAULT_RESPONSE_TYPE = (process.env.ZENROWS_RESPONSE_TYPE as "markdown" | "plaintext" | "html" | undefined) ?? "markdown"; -// Sent in the initialize response; clients put it in the model's context, so it says -// which tool to reach for first, not what every tool does. +// Sent in the initialize response; clients put it in the model's context, so it maps the +// groups of tools as facts, without telling the model which to reach for or in what order. export const INSTRUCTIONS = [ - "Zenrows returns the content of public web pages.", - "Start with scrape for one page's content, or extract for structured JSON fields.", - "For many URLs use batch_create, then batch_wait until it ends, batch_results to page through its rows and batch_content to read one page; to discover the URLs behind a listing or section page, use crawl_create.", - "Use the browser_* tools only when a page needs clicks, typing or several steps: browser_navigate opens a session, and browser_close frees it when done.", - "Call account_usage (free) before a large job or after a 402.", -].join(" "); + "Zenrows returns the content of public web pages. The tools come in five groups:", + "- scrape returns one page's content: markdown, plain text, HTML or a screenshot.", + '- extract returns structured JSON fields from one page; prepare_domain and prepared_domain_status set a domain up for its mode "auto".', + "- batch_* run a known list of URLs as one asynchronous job.", + "- crawl_* start from one URL and collect the URLs it links to on the same site, optionally with each page's content.", + "- browser_* drive a hosted browser session for pages that need clicks, typing or several steps; browser_navigate opens the session and browser_close ends it.", + "account_usage reports the plan's credits and is free.", +].join("\n"); export interface ScrapeParams { url: string; @@ -157,24 +159,18 @@ export function createServer(apiKey: KeyRef, clientName?: string, options: Serve destructiveHint: false, openWorldHint: true, }, - description: `Scrape any webpage and return its content using Zenrows (Fetch). + description: `Fetch one web page with Zenrows (Fetch) and return its content: markdown (default), plain text, HTML or a screenshot. -Returns full-page content (markdown/HTML/PDF/screenshot). For structured JSON -fields (products, articles, listings), the extract tool returns parsed fields -instead of a full page body. +For one page's full content. -By default the request uses automatic mode (mode=auto): Zenrows adds JS rendering -or premium proxies per page only when a plain request does not return the page, -and charges only for the configuration that succeeds. Pass just the URL. +Automatic mode (mode=auto) is the default: Zenrows adds JavaScript rendering or premium proxies to a page only when a plain request does not return it, and bills only the configuration that succeeded. A URL alone is enough. -Set js_render or premium_proxy only to force a fixed configuration. Doing so -switches off automatic mode, and every request is billed at that -configuration's cost (premium_proxy with js_render is 25x a basic request). +Setting js_render or premium_proxy turns automatic mode off and bills every request at that fixed configuration: 1 credit basic, 5 with js_render, 10 with premium_proxy, 25 with both. Examples: - Default: { url: "https://example.com" } - Geo: { url: "https://example.com", proxy_country: "US" } - Forced: { url: "https://spa.com", js_render: true }`, + Default: { url: "https://example.com" } + Geo: { url: "https://example.com", proxy_country: "US" } + Forced rendering: { url: "https://spa.com", js_render: true }`, inputSchema: { url: z.string().url().describe("The webpage URL to scrape"), @@ -182,17 +178,16 @@ Examples: .boolean() .nullish() .describe( - "Force JavaScript rendering on every request. Overrides automatic mode (mode=auto), " + - "which already renders JavaScript when a page needs it. Leave unset unless you need a fixed configuration." + "Optional. Renders JavaScript on every request (5 credits) and turns automatic mode off. " + + "Automatic mode already renders a page that needs it." ), premium_proxy: z .boolean() .nullish() .describe( - "Force premium residential proxies on every request (10x credit cost). Overrides " + - "automatic mode (mode=auto), which already uses premium proxies when a page needs them. " + - "Leave unset unless you need a fixed configuration." + "Optional. Uses premium residential proxies on every request (10 credits; 25 with js_render) " + + "and turns automatic mode off. Automatic mode already adds them to a page that needs them." ), proxy_country: z @@ -236,8 +231,8 @@ Examples: .string() .nullish() .describe( - "CSS selector to wait for before capturing. Use when key content loads " + - "after the initial page render. Requires js_render=true; without it, including in automatic mode, it may be ignored." + "CSS selector to wait for before capturing, for content that loads after the first render. " + + "Applies with js_render=true; automatic mode may ignore it." ), wait: z diff --git a/src/tools/batch.ts b/src/tools/batch.ts index 9342048..928ab27 100644 --- a/src/tools/batch.ts +++ b/src/tools/batch.ts @@ -374,7 +374,7 @@ const COSTS = `Cost: the same credits as the regular API, per successful task: 1 const LIMITS = `Limits: up to 100,000 tasks per call; 3 active jobs per account by default, shared with Crawl (BATCH_QUOTA_EXCEEDED past that); a job frees its slot when its run ends.`; -const RESULT_FACTS = `A result_url is presigned and expires after 2 hours: call batch_results again for a fresh one, or read the body with batch_content. Results are kept 14 days by default; after that a successful row reads as status "expired", with no result_url, and batch_content answers BATCH_RESULT_EXPIRED.`; +const RESULT_FACTS = `A result_url is presigned and expires after 2 hours: a new batch_results call returns a fresh one, and batch_content reads the body. Results are kept 14 days by default; after that a successful row reads as status "expired", with no result_url, and batch_content answers BATCH_RESULT_EXPIRED.`; export function registerBatchTools( server: McpServer, @@ -430,15 +430,15 @@ export function registerBatchTools( annotations: { title: "Create Batch Job", readOnlyHint: false, destructiveHint: false, openWorldHint: true }, description: `Submit a Batch job: many URLs scraped asynchronously in the cloud (Zenrows Batch API). Suited to large URL lists; scrape and extract handle single pages. Not the same as browser_batch. -Tasks use automatic mode (mode=auto) by default: Zenrows enables JS rendering and premium proxies only on the pages that need them. Pass just the URLs. Set js_render or premium_proxy (job-level, or per task) only to force a fixed configuration; that turns automatic mode off for those tasks. +Tasks use automatic mode (mode=auto) by default: Zenrows enables JS rendering and premium proxies only on the pages that need them. A list of URLs alone is enough. Setting js_render or premium_proxy (job-level, or per task) forces a fixed configuration and turns automatic mode off for those tasks. ${COSTS} ${LIMITS} Creates a closed job with every task in this one call. Each task takes a url plus optional external_id, metadata, method/body (POST tasks) and scrape params. Every call sends an Idempotency-Key, so the bounded retries on network errors and 5xx never create a second job; an error reply carries the key to retry with. -Returns job_id, status and stats at once; the job runs asynchronously. Then: batch_wait until it ends, batch_results to list rows (paged), batch_content to read one task's page. A run that ends failed (out of credits, inactive subscription, key cap) answers ok:false with error.code and server_code; batch_rerun recovers it. Submits of 10,000+ tasks keep writing tasks for a while (ingesting:true). -Errors: BATCH_INVALID_REQUEST (400, server_code and invalid_tasks say what), BATCH_INSUFFICIENT_CREDITS (402), BATCH_KEY_CAP_REACHED (402, this key's cap), BATCH_QUOTA_EXCEEDED (429, job limit), AUTH_INVALID (401; server_code subscription_inactive means fix Billing), BATCH_ACCESS_DENIED (403, do not retry), BACKEND_UNAVAILABLE. +Returns job_id, status and stats at once; the job runs asynchronously. The job_id is what batch_status, batch_wait, batch_results and batch_content take: batch_wait reports when the run ends, batch_results lists its rows (paged) and batch_content returns one task's page. A run that ends failed (out of credits, inactive subscription, key cap) answers ok:false with error.code and server_code; batch_rerun recovers it. Submits of 10,000+ tasks keep writing tasks for a while (ingesting:true). +Errors: BATCH_INVALID_REQUEST (400, server_code and invalid_tasks say what), BATCH_INSUFFICIENT_CREDITS (402), BATCH_KEY_CAP_REACHED (402, this key's cap), BATCH_QUOTA_EXCEEDED (429, job limit), AUTH_INVALID (401; server_code subscription_inactive means the subscription is inactive), BATCH_ACCESS_DENIED (403; a retry gets the same answer), BACKEND_UNAVAILABLE. Schedules, webhooks, exports, CSV inputs and open jobs are available through the Batch API and SDKs only.`, inputSchema: { tasks: z @@ -612,7 +612,7 @@ A failed run answers ok:false with error.code BATCH_INSUFFICIENT_CREDITS, BATCH_ "batch_wait", { annotations: { title: "Wait for Batch Job", readOnlyHint: true, destructiveHint: false, openWorldHint: true }, - description: `Poll a Batch job until its run ends (completed, stopped or failed) or pauses, and return its status and stats like batch_status. If the wait runs out first, it returns the current state with ok:true, timed_out:true and a note: call batch_wait again. A paused run returns at once with paused:true (it does not move until batch_resume). A failed run answers ok:false with error.code and server_code (insufficient_credits, subscription_inactive or api_key_cap_reached). Transient poll errors are retried. A call that carries a progressToken gets a progress notification after each poll.`, + description: `Poll a Batch job until its run ends (completed, stopped or failed) or pauses, and return its status and stats like batch_status. If the wait runs out first, it returns the current state with ok:true and timed_out:true; the job keeps running, and another batch_wait call goes on waiting. A paused run returns at once with paused:true (it does not move until batch_resume). A failed run answers ok:false with error.code and server_code (insufficient_credits, subscription_inactive or api_key_cap_reached). Transient poll errors are retried. A call that carries a progressToken gets a progress notification after each poll.`, inputSchema: { job_id: jobIdSchema, timeout: waitSecondsSchema, @@ -632,11 +632,11 @@ A failed run answers ok:false with error.code BATCH_INSUFFICIENT_CREDITS, BATCH_ "batch_results", { annotations: { title: "Batch Job Results", readOnlyHint: true, destructiveHint: false, openWorldHint: true }, - description: `List a Batch run's result rows, a page at a time: up to limit rows (default ${DEFAULT_RESULTS_LIMIT}, max 1000) and next_cursor. Pass next_cursor back as cursor to continue; it is null once every row was read. + description: `List a Batch run's result rows, a page at a time: up to limit rows (default ${DEFAULT_RESULTS_LIMIT}, max 1000) and next_cursor. next_cursor, passed back as cursor, continues the list; it is null once every row was read. Each row has task_id, external_id, url, metadata, status (successful, failed, pending, processing, expired), type, result_url when successful, and error when failed. ${RESULT_FACTS} -Filter with status: all (default), successful, failed, or processing (not finished yet). A sparse filter can return fewer rows than limit with next_cursor still set: keep paging. While the run is still running or writing its tasks, the list is partial.`, +Filter with status: all (default), successful, failed, or processing (not finished yet). A sparse filter can return fewer rows than limit with next_cursor still set; more rows follow. While the run is still running or writing its tasks, the list is partial.`, inputSchema: { job_id: jobIdSchema, run_id: runIdSchema, @@ -686,9 +686,9 @@ Filter with status: all (default), successful, failed, or processing (not finish "batch_content", { annotations: { title: "Batch Task Content", readOnlyHint: true, destructiveHint: false, openWorldHint: true }, - description: `Read one Batch task's stored result, by job_id and task_id (from batch_results). A text result (HTML, Markdown, plain text, JSON) comes back inline, cut to max_chars (default ${DEFAULT_CONTENT_MAX_CHARS}); a final note says when it was cut. A binary result (a PDF) is not inlined: the reply gives its result_url, presigned for 2 hours. Finding that URL reads the run's results; on a large run one call may run out of time first and answer with lookup_cursor: call batch_content again with it to go on. + description: `Read one Batch task's stored result, by job_id and task_id (from batch_results). A text result (HTML, Markdown, plain text, JSON) comes back inline, cut to max_chars (default ${DEFAULT_CONTENT_MAX_CHARS}); a final note says when it was cut. A binary result (a PDF) is not inlined: the reply gives its result_url, presigned for 2 hours. Finding that URL reads the run's results; on a large run one call may run out of time first and answer with lookup_cursor; passing it back continues the lookup. -BATCH_TASK_NOT_READY: the task has not finished; BATCH_TASK_FAILED: it failed (task_error holds its error); BATCH_RESULT_EXPIRED: past the job's retention (14 days by default); rerun to scrape it again.`, +BATCH_TASK_NOT_READY: the task has not finished; BATCH_TASK_FAILED: it failed (task_error holds its error); BATCH_RESULT_EXPIRED: past the job's retention (14 days by default); batch_rerun scrapes it again.`, inputSchema: { job_id: jobIdSchema, task_id: z.string().min(1).describe("task_id from a batch_results row"), diff --git a/src/tools/browser.ts b/src/tools/browser.ts index 546a710..e46e159 100644 --- a/src/tools/browser.ts +++ b/src/tools/browser.ts @@ -44,17 +44,18 @@ export function registerBrowserTools( destructiveHint: false, openWorldHint: true, }, - description: `Open a Zenrows Browser Sessions session and navigate to a URL. + description: `Open a Zenrows Browser session (a hosted browser) and navigate to a URL in one step. -This is the entry point for all browser automation. It creates a new session backed by -Zenrows' hosted browser infrastructure and navigates to the given URL in one step. +Every browser automation starts here. Returns a session_id that every other browser_* tool takes. -Returns a session_id that every other browser_* tool takes. The session holds one -concurrency slot until browser_close closes it. +Side effects: the session holds one concurrency slot and is billed for its open time +(5 credits per minute) and its traffic (25,000 credits per GB) until browser_close ends it; +a session closes after 3 minutes without a browser_* call, or 15 minutes after it opened, +and is billed until it closes. -When to use options: +Options: - proxy_country: geo-restricted content (ISO 3166-1 alpha-2, e.g. "US", "DE") -- proxy_region: world region for geo-targeted proxy (eu, na, ap, sa, af, me)`, +- proxy_region: world region for a geo-targeted proxy (eu, na, ap, sa, af, me)`, inputSchema: { url: z.string().url().describe("The URL to navigate to"), proxy_country: z @@ -113,7 +114,7 @@ When to use options: { annotations: { title: "Close Browser Session", readOnlyHint: false, destructiveHint: false, openWorldHint: true }, description: - "Close a browser session and free all associated resources, including the concurrency slot it holds.", + "Close a browser session: frees its concurrency slot and ends its billed time.", inputSchema: { session_id: sessionId }, }, async ({ session_id }) => { diff --git a/src/tools/crawl.ts b/src/tools/crawl.ts index 31f5248..f258657 100644 --- a/src/tools/crawl.ts +++ b/src/tools/crawl.ts @@ -146,9 +146,6 @@ function progressMessage(crawl: Crawl): string { return `${crawl.status}: ${c?.pages_fetched ?? 0} pages fetched, ${c?.items_found ?? 0} URLs kept`; } -const WHEN_TO_USE = `Use crawl when you have one start page (a listing, category, blog index) and need -the URLs behind it, optionally with each page's HTML. For a single page, use scrape.`; - const crawlIdSchema = z.string().describe("Crawl id returned by crawl_create (c_…)"); const limitSchema = z.number().int().min(1).max(10_000).nullish(); @@ -173,28 +170,11 @@ export function registerCrawlTools(server: McpServer, apiKey: KeyRef, options: C "crawl_create", { annotations: { title: "Create Crawl", readOnlyHint: false, destructiveHint: false, openWorldHint: true }, - description: `Start a crawl from one URL (Zenrows Crawl API). Crawl follows the links in each page's HTML up to depth hops (no depth limit when depth is omitted), keeps the URLs that match include_patterns / exclude_patterns, and stays on the start URL's registrable domain (subdomains count). - -${WHEN_TO_USE} - -Each page fetched is billed as one scrape on this account; max_pages bounds the cost -(default 100), max_items bounds how many URLs are kept (default 100). Without depth, the -crawl keeps following links until max_pages or max_items stops it or nothing is left to -open, so set max_pages to what you are willing to spend. The start page itself is never -a result, but its fetch is billed: a crawl whose include_patterns / exclude_patterns match -nothing still costs that one scrape. A crawl whose pages held nothing new to follow (no link, -only links to other sites, or only links the patterns reject) ends completed with no results -and no error. When patterns are set, check them against the URLs the start page links to. -Set output_format "html" or "markdown" to also store each kept URL's page, -then read it with crawl_content. -Set webhook to have signed events posted to your https URL when the crawl starts and when it -ends; the account needs a webhook signing key, else CRAWL_INVALID_REQUEST (webhook_signing_key_required). - -Returns crawl_id, status and the crawl (with coverage) at once; the crawl runs -asynchronously. Call crawl_wait, then crawl_results. Set idempotency_key to make a retry -safe: the same key returns the crawl the first request created instead of starting another. -CRAWL_NOT_ENABLED: Crawl is not enabled for this account; do not retry. -CRAWL_TOO_MANY_CRAWLS: the account has reached its limit of active jobs (3 by default), shared with its Batch jobs; retry after retry_after seconds.`, + description: `Start a crawl: from one start URL, follow the links on each page and keep the URLs that match include_patterns and don't match exclude_patterns, on the same site, subdomains included (from shop.example.com, also www.example.com and blog.example.com). Links are followed up to depth hops, with no depth limit when depth is omitted. With output_format, it also stores each kept page as HTML or Markdown. +For finding the pages behind one start page, such as a listing, a category or a blog index. +Runs in the background: returns crawl_id at once, which the other crawl_* tools take. +With webhook, signed events are posted to an https URL when the crawl starts and when it ends; the account needs a webhook signing key, else CRAWL_INVALID_REQUEST (webhook_signing_key_required). +Each page the crawl opens is billed in credits, and the price per page depends on the site; max_pages caps how many pages it opens (default 100) and max_items caps how many URLs it keeps (default 100). The start page is never a result, but its fetch is billed: a crawl whose include_patterns / exclude_patterns match nothing still costs that one scrape. When patterns are set, check them against the URLs the start page links to. A crawl whose pages held nothing new to follow ends completed with no results and no error. A running crawl counts toward the account's limit.`, inputSchema: { url: z.string().url().describe("Start URL: a public http(s) page, such as a listing or category page"), // No lower bound here: the API rejects 0 or less as CRAWL_INVALID_REQUEST, naming the @@ -322,11 +302,7 @@ completed with stop_reason max_items / max_pages means a limit ended it; stopped "crawl_results", { annotations: { title: "Crawl Results", readOnlyHint: true, destructiveHint: false, openWorldHint: true }, - description: `List the URLs a crawl kept, in the order it kept them, following the API's pages up to limit. - -Each result has url and, when the crawl has output_format, content_status (pending, fetched, failed) and, once fetched, content_url: pass it to crawl_content as content. - -While the crawl runs the list is partial (partial=true): call again later with the returned next_cursor to get only the URLs kept since. A stopped crawl can still add URLs from pages fetched before the stop for up to 10 minutes, so next_cursor stays set during that window: call again later. next_cursor is null once the crawl ended and every URL was read.`, + description: `List the URLs a crawl kept, in the order it kept them, with next_cursor to continue. With output_format, each URL also has content_status and, once stored, content_url. While the crawl runs, the list is partial. A stopped crawl can still add URLs for up to 10 minutes. next_cursor is null once the crawl has ended and every URL was read.`, inputSchema: { crawl_id: crawlIdSchema, cursor: z.string().nullish().describe("next_cursor from a previous crawl_results call, to continue from there"), @@ -454,7 +430,7 @@ Idempotent: a crawl that already ended answers with its final status. A 429, 502 "crawl_wait", { annotations: { title: "Wait for Crawl", readOnlyHint: true, destructiveHint: false, openWorldHint: true }, - description: `Poll a crawl until it ends (completed, stopped, or failed) and return its status, its counts and the first ${WAIT_URLS} URLs (first_urls). For every result, call crawl_download (one file) or crawl_results (pages). A crawl that ends completed with no results and no error finished; it is not a failure (see crawl_get). If the wait runs out first, it returns the crawl with status running and a note: call crawl_wait again. A call that carries a progressToken gets a progress notification after each poll, with the status and counts.`, + description: `Wait for a crawl to end (completed, stopped or failed), up to timeout seconds. Returns its status, its counts and its first ${WAIT_URLS} URLs. A crawl that ends completed with no results and no error finished; it is not a failure. If the timeout passes first, the crawl comes back with status running and keeps going.`, inputSchema: { crawl_id: crawlIdSchema, timeout: z diff --git a/src/tools/extract.ts b/src/tools/extract.ts index 329e8b9..f4331c0 100644 --- a/src/tools/extract.ts +++ b/src/tools/extract.ts @@ -397,37 +397,21 @@ export function registerExtractTool(server: McpServer, apiKey: KeyRef, getClient destructiveHint: false, openWorldHint: true, }, - description: `Extract structured data from a webpage via Zenrows. - -Returns JSON fields (products, articles, listings) in "data" rather than a full -page body. Set include_html to also get the page HTML. -Modes: -- auto (default): extract=auto, site-tailored Extract (currently free, - billing may apply later). Works on any prepared domain, with any API key. - Error REQS007 says why in "reason": - - domain_not_prepared: call prepare_domain with the domain and 1 to 5 example - URLs (one works; we recommend 3 or more of the same page type; a few minutes, - no credits), poll prepared_domain_status about every - 30 seconds until it says ready, then call extract again. - - page_not_recognised: the domain is prepared, but this page is not one - Extract extracts (a page with nothing to extract, such as a home page). Try a product, - article or listing page instead; if this page should work, retry it once in - a few minutes; otherwise use mode=css. Do not prepare the domain again. - - domain_not_enabled: preparing does not help; use mode=css. -- autoparse: deprecated general-purpose structured JSON on any domain; use only - when asked for -- css: css_extractor with an explicit selector map - -Requests use automatic mode (mode=auto) by default: Zenrows enables JS rendering and -premium proxies only when the site needs them, and charges only for the -configuration that succeeds. Pass just the URL. -Set js_render or premium_proxy only to force a fixed configuration; that turns -automatic mode off and bills every request at that configuration's cost. - -Check "empty" in the result: true means no field came back with a value. -"noDataExtracted": true means Extract processed the page and found nothing to -extract (not charged); retrying the same request returns the same answer. -Full-page markdown, HTML and screenshots come from scrape.`, + description: `Extract structured JSON fields from one web page with Zenrows (Extract): products, articles, listings and similar page types, returned in "data" instead of the page body. With mode "auto", include_html adds the page HTML. + +Extraction options (the mode parameter): +- mode "auto" (default): fields tailored to the site. Works on any domain that has been prepared, with any API key. +- mode "css" (css_extractor, a map of field name to CSS selector): the fields you name. Works on any domain. +- mode "autoparse": deprecated general-purpose JSON on any domain. + +Error REQS007 says in "reason" why mode "auto" returned no fields: +- domain_not_prepared: the domain has not been prepared yet. prepare_domain prepares it (a few minutes, no credits). +- page_not_recognised: the domain is prepared, but this page is not a type Extract extracts (for example, a home page). Preparing the domain again does not change this answer. +- domain_not_enabled: mode "auto" is not available on this domain; mode "css" (css_extractor) works there. + +Pages are fetched in automatic mode by default (the mode_auto parameter): JavaScript rendering and premium proxies are added only when the site needs them, and only the configuration that succeeded is billed. Setting js_render or premium_proxy, or mode_auto to false, turns automatic mode off and bills every request at that fixed configuration. + +In the result, "empty": true means no field came back with a value. "noDataExtracted": true means Extract processed the page and found nothing to extract.`, inputSchema: { url: z.string().url().describe("The webpage URL to extract from"), mode: z @@ -485,7 +469,7 @@ Full-page markdown, HTML and screenshots come from scrape.`, .boolean() .nullish() .describe( - "Deprecated, has no effect: Extract no longer falls back to autoparse. Prepare the domain with prepare_domain instead." + 'Deprecated, has no effect: Extract no longer falls back to autoparse. prepare_domain prepares a domain for mode "auto".' ), include_html: z .boolean() diff --git a/src/tools/prepare.ts b/src/tools/prepare.ts index 3a931c6..756a941 100644 --- a/src/tools/prepare.ts +++ b/src/tools/prepare.ts @@ -357,25 +357,26 @@ export function registerPrepareTools(server: McpServer, apiKey: KeyRef): void { idempotentHint: true, openWorldHint: true, }, - description: `Prepare a domain so extract (mode=auto) works on it. + description: `Prepare a domain for extract's mode "auto". This is what resolves REQS007 with reason domain_not_prepared. -Use this when extract returns error REQS007 with reason domain_not_prepared. Send the -domain (the host, e.g. shop.example.com; a leading www. is dropped, and each host is -prepared separately) and 1 to 5 example URLs of real pages you want extracted. One works; +Takes the domain (the host, e.g. shop.example.com; a leading www. is dropped, and each host is +prepared separately) and 1 to 5 example URLs of real pages to extract. One works; we recommend 3 or more of the same page type, so pass 3 or more when you can, up to 5 (e.g. 3 product pages: https://www.example.com/products/a, /products/b and /products/c), because Extract compares pages of one type to check the fields it found and fill in fields one page is missing. Several page types can share the 5 (e.g. 3 product pages + 2 search-result -pages). It returns at once with status -"preparing" and never waits; preparation takes a few minutes and costs no credits. Then -call prepared_domain_status about every 30 seconds until status is "ready", and call -extract again. Each new domain uses one slot of the account's monthly allowance +pages). + +Returns at once with status "preparing"; preparation runs in the background, takes a few +minutes and costs no credits. prepared_domain_status reports progress and when the domain +is "ready". + +Side effect: each new domain uses one slot of the account's monthly allowance (prepared_domain_status shows it). A domain already being prepared returns its current status; a domain already prepared returns "ready" at once with already_prepared: true and uses no slot. -Do not call this for REQS007 with reason page_not_recognised: the domain is already -prepared and preparing again does not change that answer.`, +It does not change a REQS007 page_not_recognised answer: that domain is already prepared.`, inputSchema: { domain: z .string() @@ -413,12 +414,12 @@ prepared and preparing again does not change that answer.`, destructiveHint: false, openWorldHint: true, }, - description: `Check a domain submitted with prepare_domain, and the account's monthly allowance. + description: `Report the state of a domain submitted with prepare_domain, and the account's monthly allowance of prepared domains. Returns status (preparing, ready or failed), state (the current step), roles (one per page kind), a failure code and detail when it failed, the allowance (used, limit, remaining) and -"next": what to do now. Poll about every 30 seconds; preparation takes a few minutes. Call -extract again once status is "ready". Costs no credits.`, +"next", a short note on what the current status means. Preparation takes a few minutes; +extract's mode "auto" serves the domain once status is "ready". Costs no credits.`, inputSchema: { domain: z.string().min(1).describe("The domain passed to prepare_domain, e.g. example.com"), }, From a2ceeea52f1d48977ef372c434599c0828965a84 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 11 Nov 2026 15:38:16 +0000 Subject: [PATCH 2/3] fix(tools): restore the costs and errors the rewrite dropped Review found that the rewrite lost facts its own goal keeps: what a call costs, what an error means, and how a tool's output feeds the next one. - extract had no cost statement left. It now gives the rate card, and says that Extract adds no charge of its own and that neither REQS007 nor a processed page with nothing on it is billed. - crawl_create regained CRAWL_NOT_ENABLED and CRAWL_TOO_MANY_CRAWLS, the limit of 3 active jobs shared with Batch, and retry_after. - crawl_results says again that content_url is what crawl_content takes. - crawl_wait says again that a progressToken gets progress notifications, which batch_wait already documented. Three wording decisions from the same review: - crawl_create no longer tells the model to check its own patterns against the start page. That was an instruction, which is what this branch removes everywhere else; crawl_get still carries it, for the next batch. - prepare_domain states the example-URL guidance as a fact rather than a recommendation, and its test moves with it. - scrape no longer contradicts its own parameter: response_type says what 'pdf' actually produces here, a text decode rather than a PDF file. Removing the enum value instead would change the input contract, so that stays a separate decision. Typed fix so a release reaches the hosted server; docs alone would not. --- src/server.ts | 2 +- src/tools/crawl.ts | 7 ++++--- src/tools/extract.ts | 1 + src/tools/prepare.ts | 4 ++-- tests/crawl-tool.test.ts | 1 - tests/descriptions.test.ts | 6 +++--- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/server.ts b/src/server.ts index d04069b..489f4c2 100644 --- a/src/server.ts +++ b/src/server.ts @@ -205,8 +205,8 @@ Examples: .describe( "Output format. 'markdown' (default) preserves structure and is ideal for LLMs. " + "'plaintext' strips all formatting for pure text extraction. " + - "'pdf' returns a PDF of the page. " + "'html' returns the raw HTML source (omits the response_type param; Zenrows default). " + + "'pdf' asks the API for a PDF, which this server decodes as text, so the result is not a usable PDF file. " + "Ignored when autoparse, css_extractor, outputs, or screenshot params are set." ), diff --git a/src/tools/crawl.ts b/src/tools/crawl.ts index f258657..7e97faf 100644 --- a/src/tools/crawl.ts +++ b/src/tools/crawl.ts @@ -174,7 +174,8 @@ export function registerCrawlTools(server: McpServer, apiKey: KeyRef, options: C For finding the pages behind one start page, such as a listing, a category or a blog index. Runs in the background: returns crawl_id at once, which the other crawl_* tools take. With webhook, signed events are posted to an https URL when the crawl starts and when it ends; the account needs a webhook signing key, else CRAWL_INVALID_REQUEST (webhook_signing_key_required). -Each page the crawl opens is billed in credits, and the price per page depends on the site; max_pages caps how many pages it opens (default 100) and max_items caps how many URLs it keeps (default 100). The start page is never a result, but its fetch is billed: a crawl whose include_patterns / exclude_patterns match nothing still costs that one scrape. When patterns are set, check them against the URLs the start page links to. A crawl whose pages held nothing new to follow ends completed with no results and no error. A running crawl counts toward the account's limit.`, +Each page the crawl opens is billed in credits, and the price per page depends on the site; max_pages caps how many pages it opens (default 100) and max_items caps how many URLs it keeps (default 100). The start page is never a result, but its fetch is billed: a crawl whose include_patterns / exclude_patterns match nothing still costs that one scrape. A crawl whose pages held nothing new to follow ends completed with no results and no error. +CRAWL_NOT_ENABLED: Crawl is not enabled for this account, and a retry gets the same answer. CRAWL_TOO_MANY_CRAWLS: the account is at its limit of 3 active jobs by default, shared with Batch, and retry_after says how many seconds until a slot frees.`, inputSchema: { url: z.string().url().describe("Start URL: a public http(s) page, such as a listing or category page"), // No lower bound here: the API rejects 0 or less as CRAWL_INVALID_REQUEST, naming the @@ -302,7 +303,7 @@ completed with stop_reason max_items / max_pages means a limit ended it; stopped "crawl_results", { annotations: { title: "Crawl Results", readOnlyHint: true, destructiveHint: false, openWorldHint: true }, - description: `List the URLs a crawl kept, in the order it kept them, with next_cursor to continue. With output_format, each URL also has content_status and, once stored, content_url. While the crawl runs, the list is partial. A stopped crawl can still add URLs for up to 10 minutes. next_cursor is null once the crawl has ended and every URL was read.`, + description: `List the URLs a crawl kept, in the order it kept them, with next_cursor to continue. With output_format, each URL also has content_status and, once stored, content_url, which is what crawl_content takes as content. While the crawl runs, the list is partial. A stopped crawl can still add URLs for up to 10 minutes. next_cursor is null once the crawl has ended and every URL was read.`, inputSchema: { crawl_id: crawlIdSchema, cursor: z.string().nullish().describe("next_cursor from a previous crawl_results call, to continue from there"), @@ -430,7 +431,7 @@ Idempotent: a crawl that already ended answers with its final status. A 429, 502 "crawl_wait", { annotations: { title: "Wait for Crawl", readOnlyHint: true, destructiveHint: false, openWorldHint: true }, - description: `Wait for a crawl to end (completed, stopped or failed), up to timeout seconds. Returns its status, its counts and its first ${WAIT_URLS} URLs. A crawl that ends completed with no results and no error finished; it is not a failure. If the timeout passes first, the crawl comes back with status running and keeps going.`, + description: `Wait for a crawl to end (completed, stopped or failed), up to timeout seconds. Returns its status, its counts and its first ${WAIT_URLS} URLs. A crawl that ends completed with no results and no error finished; it is not a failure. If the timeout passes first, the crawl comes back with status running and keeps going. A call that carries a progressToken gets a progress notification after each poll, with the status and counts.`, inputSchema: { crawl_id: crawlIdSchema, timeout: z diff --git a/src/tools/extract.ts b/src/tools/extract.ts index f4331c0..72daf15 100644 --- a/src/tools/extract.ts +++ b/src/tools/extract.ts @@ -410,6 +410,7 @@ Error REQS007 says in "reason" why mode "auto" returned no fields: - domain_not_enabled: mode "auto" is not available on this domain; mode "css" (css_extractor) works there. Pages are fetched in automatic mode by default (the mode_auto parameter): JavaScript rendering and premium proxies are added only when the site needs them, and only the configuration that succeeded is billed. Setting js_render or premium_proxy, or mode_auto to false, turns automatic mode off and bills every request at that fixed configuration. +Extract adds no charge of its own: a request costs the underlying fetch, 1 credit basic, 5 with js_render, 10 with premium_proxy, 25 with both. Neither a REQS007 answer nor a page Extract processed and found nothing on ("noDataExtracted") is billed. In the result, "empty": true means no field came back with a value. "noDataExtracted": true means Extract processed the page and found nothing to extract.`, inputSchema: { diff --git a/src/tools/prepare.ts b/src/tools/prepare.ts index 756a941..8d531a9 100644 --- a/src/tools/prepare.ts +++ b/src/tools/prepare.ts @@ -361,7 +361,7 @@ export function registerPrepareTools(server: McpServer, apiKey: KeyRef): void { Takes the domain (the host, e.g. shop.example.com; a leading www. is dropped, and each host is prepared separately) and 1 to 5 example URLs of real pages to extract. One works; -we recommend 3 or more of the same page type, so pass 3 or more when you can, up to 5 (e.g. +3 or more of the same page type give the best results, up to 5 (e.g. 3 product pages: https://www.example.com/products/a, /products/b and /products/c), because Extract compares pages of one type to check the fields it found and fill in fields one page is missing. Several page types can share the 5 (e.g. 3 product pages + 2 search-result @@ -387,7 +387,7 @@ It does not change a REQS007 page_not_recognised answer: that domain is already .min(1) .max(MAX_EXAMPLE_URLS) .describe( - "1 to 5 full URLs of real pages on this domain that you want to extract. One works; we recommend 3 or more of the same page type when you can" + "1 to 5 full URLs of real pages on this domain that you want to extract. One works; 3 or more of the same page type give the best results" ), idempotency_key: z .string() diff --git a/tests/crawl-tool.test.ts b/tests/crawl-tool.test.ts index c5177de..bae8b8b 100644 --- a/tests/crawl-tool.test.ts +++ b/tests/crawl-tool.test.ts @@ -112,7 +112,6 @@ test("crawl's descriptions say 0 results with no error is a completed crawl, and const c = configs.crawl_create; const shape = c.inputSchema as Record; assert.match(c.description, /ends completed with no results\s+and no error/); - assert.match(c.description, /check them against the URLs the start page links to/); assert.match(configs.crawl_get.description, /completed with no results and no error/); assert.match(configs.crawl_get.description, /check them against the URLs the start page links to/); assert.match(configs.crawl_wait.description, /completed with no results and no error finished; it is not a failure/); diff --git a/tests/descriptions.test.ts b/tests/descriptions.test.ts index 43714ad..61ea4a7 100644 --- a/tests/descriptions.test.ts +++ b/tests/descriptions.test.ts @@ -62,12 +62,12 @@ for (const hosted of [false, true]) { // Extract maps a page type better from several pages of that type, so agents are asked for // more than one, without the schema refusing a single URL. -test("prepare_domain recommends 3 or more example URLs of the same page type and still accepts one", async () => { +test("prepare_domain says 3 or more example URLs of the same page type give the best results, and still accepts one", async () => { const prepare = (await listTools(false)).find((t) => t.name === "prepare_domain"); assert.ok(prepare, "prepare_domain is not advertised"); - const guidance = /One\s+works;\s+we\s+recommend\s+3\s+or\s+more\s+of\s+the\s+same\s+page\s+type/; + const guidance = /One\s+works;\s+3\s+or\s+more\s+of\s+the\s+same\s+page\s+type\s+give\s+the\s+best\s+results/; assert.match(prepare.description ?? "", guidance); - assert.match(prepare.description ?? "", /pass\s+3\s+or\s+more\s+when\s+you\s+can,\s+up\s+to\s+5/); + assert.match(prepare.description ?? "", /give\s+the\s+best\s+results,\s+up\s+to\s+5/); assert.match(prepare.description ?? "", /3\s+product\s+pages\s+\+\s+2\s+search-result\s+pages/); assert.doesNotMatch(prepare.description ?? "", /one\s+per\s+page\s+kind|at\s+least\s+3/); const exampleUrls = (prepare.inputSchema?.properties ?? {}).example_urls as { From 386c4bc7ea1ecf58fd7895151b2b046e7c3a4ad7 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 13 Nov 2026 14:33:10 +0000 Subject: [PATCH 3/3] fix(extract): say Extract is free for now, not that it has no price The description claimed Extract "adds no charge of its own". It has one: the gateway charges product.Extract separately from the fetch, one unit per successful parse, and resolves the rate per subscription. Every plan's rate is unset today, which is why the claim looked true, and why it would have gone stale the day a rate is set without anyone editing this text. Says what the pricing page says instead: free for now, billing may apply later, and the fetch is what a call costs today. Also names the two unbilled answers by status, 403 REQS007 and 422 no_data_extracted, and says neither the fetch nor Extract is charged for them, which the previous wording left open. --- src/tools/extract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/extract.ts b/src/tools/extract.ts index 72daf15..367d2fd 100644 --- a/src/tools/extract.ts +++ b/src/tools/extract.ts @@ -410,7 +410,7 @@ Error REQS007 says in "reason" why mode "auto" returned no fields: - domain_not_enabled: mode "auto" is not available on this domain; mode "css" (css_extractor) works there. Pages are fetched in automatic mode by default (the mode_auto parameter): JavaScript rendering and premium proxies are added only when the site needs them, and only the configuration that succeeded is billed. Setting js_render or premium_proxy, or mode_auto to false, turns automatic mode off and bills every request at that fixed configuration. -Extract adds no charge of its own: a request costs the underlying fetch, 1 credit basic, 5 with js_render, 10 with premium_proxy, 25 with both. Neither a REQS007 answer nor a page Extract processed and found nothing on ("noDataExtracted") is billed. +Extract is currently free, and billing for it may apply later: today a request costs only the underlying fetch, 1 credit basic, 5 with js_render, 10 with premium_proxy, 25 with both. Neither a 403 REQS007 nor a 422 no_data_extracted (Extract ran on the page and found nothing on it) is billed, for the fetch or for Extract. In the result, "empty": true means no field came back with a value. "noDataExtracted": true means Extract processed the page and found nothing to extract.`, inputSchema: {