From 1ec66879ad86547dbba5328716c40629d51bd740 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 13 Oct 2026 15:36:13 +0200 Subject: [PATCH 1/8] fix: list Extract in the PyPI summary and keywords The PyPI summary still read "synchronous scraping + Batch (async job) API" and the keywords left Extract out, so the package page undersold what 1.6.0 ships. Name Fetch, Extract, Batch and Crawl in the summary and add extract, structured-data, web-scraping and crawl to the keywords. Drop "domain-gated open beta" from the extract() docstring and its tests: Extract has worked on any prepared domain with any active key since the gate was removed. Point the README's Extract link at the Extract docs. The new metadata reaches PyPI on the next release. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- pyproject.toml | 4 ++-- src/zenrows/client.py | 13 ++++++++----- tests/test_fetch_extract.py | 6 +++--- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9550c8e..3275531 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ asyncio.run(main()) ### Extract -[Extract](https://docs.zenrows.com) (beta) runs a page through Zenrows' AI-powered structured extraction instead of returning raw HTML. Use `client.extract()` — it's the same request as `fetch()`, with the `extract` param set for you (defaults to `"auto"`; pass `"native"` or `"standard"` for the other contracts). +[Extract](https://docs.zenrows.com/extract/introduction) (beta) runs a page through Zenrows' AI-powered structured extraction instead of returning raw HTML. Use `client.extract()` — it's the same request as `fetch()`, with the `extract` param set for you (defaults to `"auto"`; pass `"native"` or `"standard"` for the other contracts). ```python from zenrows import ZenRowsClient diff --git a/pyproject.toml b/pyproject.toml index b33978e..7406c69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,13 +5,13 @@ build-backend = "hatchling.build" [project] name = "zenrows" version = "1.6.0" -description = "ZenRows Python SDK — synchronous scraping + Batch (async job) API" +description = "Zenrows Python SDK — Fetch, Extract (structured data), Batch and Crawl APIs" readme = "README.md" license = { text = "MIT" } authors = [ { name = "ZenRows", email = "support@zenrows.com" }, ] -keywords = ["zenrows", "scraper", "scraping", "async", "batch"] +keywords = ["zenrows", "scraper", "scraping", "web-scraping", "extract", "structured-data", "async", "batch", "crawl"] classifiers = [ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", diff --git a/src/zenrows/client.py b/src/zenrows/client.py index e9db1b8..e4b3d07 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -125,11 +125,14 @@ def extract( or "standard". Thin wrapper over `fetch()` with the `extract` param set — no separate endpoint or auth. - `mode="auto"` is a domain-gated open beta: when the target domain isn't - enabled yet, the API returns a 402 with `code: "AUTH010"`. By default - this retries once with `autoparse=True` instead of returning the error - response — pass `fallback_to_autoparse=False` to disable that and get - the raw AUTH010 response back. + Extract works on any domain once Zenrows has prepared it, with any + active API key. A domain that isn't prepared yet answers 403 with + `code: "REQS007"`, returned as is; prepare it with + `POST /v1/extract/prepared-domains`. If the API answers with + `code: "AUTH010"` (domain not enabled), this retries once with + `autoparse=True` instead of returning the error response — pass + `fallback_to_autoparse=False` to disable that and get the raw AUTH010 + response back. `adaptive_stealth=True` (default) also sends Adaptive Stealth Mode (`mode: "auto"` at the wire-param level - unrelated to this method's own diff --git a/tests/test_fetch_extract.py b/tests/test_fetch_extract.py index 4ea18ac..32102d2 100644 --- a/tests/test_fetch_extract.py +++ b/tests/test_fetch_extract.py @@ -125,8 +125,8 @@ def test_extract_does_not_mutate_caller_supplied_params_dict(self, mock_request) class TestExtractAutoparseFallback(TestCase): - """`extract(mode="auto")` is a domain-gated open beta: AUTH010 means the - target domain isn't enabled yet. By default this retries once with + """`extract(mode="auto")`: AUTH010 means the target domain isn't enabled + for Extract. By default this retries once with Autoparse instead of raising - same behavior as the CLI's extract adapter.""" @@ -173,7 +173,7 @@ def test_402_without_auth010_does_not_fall_back(self, mock_request): @mock.patch.object(Session, "request") def test_no_fallback_for_non_auto_mode(self, mock_request): """AUTH010 shouldn't trigger a fallback for native/standard modes - - only "auto" is the domain-gated beta path.""" + only "auto" takes the fallback path.""" mock_request.return_value = _fake_response(402, b'{"code": "AUTH010"}') response = self.client.extract(url, mode="native") From 41580fe9b59170acdc56d847862f2591e0b3ba80 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 13 Oct 2026 15:49:29 +0200 Subject: [PATCH 2/8] fix(extract): fall back to Autoparse on REQS007, and drop stale beta copy extract(mode="auto") only fell back to Autoparse on a 402 AUTH010, the code Extract answered while it was gated by domain. The gate is gone: a domain that isn't prepared yet now answers 403 REQS007, so in practice the fallback never fired and callers got the error. The CLI, the MCP server and the Dify plugin already fall back on both codes. Treat 403 REQS007 like the legacy 402 AUTH010, in the sync and async variants, keyed on status + code so an auth 403 or a REQS007 on another status is never mistaken for it. Bring the rest of the shipped copy in line with the docs: - docs/openapi.yaml: Extract is no longer "private beta, domains not enrolled fail with the upgrade-required error"; it works on any prepared domain and an unprepared one fails with 403 REQS007. Only that paragraph changes: the public zenrows.com/openapi.json is a summarised spec, not the Batch source this copy tracks, so a full refresh would drop most schemas. - README: Batch is available to every account with the same key and is billed per successful request, not a private beta that is free to use. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 ++-- docs/openapi.yaml | 7 +-- src/zenrows/client.py | 44 +++++++++--------- tests/test_fetch_extract.py | 90 ++++++++++++++++++++++++++++++++----- 4 files changed, 108 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 9550c8e..a3d6f34 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ This package ships three clients: - **`ZenRowsBatchClient`** — the async-job / Batch API client. Submit thousands of URLs as one job, poll for results, optionally upload a CSV of URLs in one call. Best for offline / bulk pipelines. - _(Private beta — [contact support](mailto:support@zenrows.com) for access.)_ + Available to every Zenrows account with the same API key. - **`ZenRowsCrawlClient`** — the Crawl API client. Give it one start URL; it follows links and returns the URLs (and, optionally, the pages) it finds behind it. _(New.)_ @@ -205,9 +205,9 @@ print(response.json()) ## Quickstart — Batch API (`ZenRowsBatchClient`) -> **⚠️ Beta.** The Batch API is currently in beta and not yet generally -> available — free to use while it's in beta, but the shape of the API may -> still change before GA. +> Batch is available to every Zenrows account: your existing API key works +> with no extra setup, and only successful requests cost credits. See the +> [Batch docs](https://docs.zenrows.com/batch/introduction). For workflows where you have many URLs to scrape and don't want to manage retries, concurrency, and pagination yourself, the Batch API diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 13fa3fd..4c80d2f 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1297,9 +1297,10 @@ components: `extract` selects the structured-extraction contract family (`auto`, `native`, or `standard`); `extract_fields` optionally - narrows the returned fields. Extract is in private beta on the - scraper API — tasks against domains not enrolled fail with the - gateway's upgrade-required error. + narrows the returned fields. Extract works on any domain + Zenrows has prepared, with any active API key; a task against a + domain that isn't prepared yet fails with the scraper API's + 403 `REQS007`. `custom_headers` additionally accepts an object of header name → value pairs (or that object serialised as a JSON diff --git a/src/zenrows/client.py b/src/zenrows/client.py index e9db1b8..271c918 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -32,15 +32,23 @@ _RETRY_STATUSES = [422, 429, 500, 502, 503, 504] -def _is_auth010(response: requests.Response) -> bool: - """True when a response's JSON error envelope carries the Extract - domain-not-enabled code (AUTH010).""" +# Extract answers these when it can't serve the target domain: 403 REQS007 +# (domain not prepared yet) or the legacy 402 AUTH010 (domain not enabled). +# extract(mode="auto") retries both once with Autoparse. +_EXTRACT_DOMAIN_UNAVAILABLE = {(403, "REQS007"), (402, "AUTH010")} + + +def _is_extract_domain_unavailable(response: requests.Response) -> bool: + """True when the response is Extract saying it can't serve the target + domain yet (403 REQS007, or the legacy 402 AUTH010).""" try: body = response.json() except ValueError: return False code = body.get("code") if isinstance(body, dict) else None - return isinstance(code, str) and code.upper() == "AUTH010" + if not isinstance(code, str): + return False + return (response.status_code, code.upper()) in _EXTRACT_DOMAIN_UNAVAILABLE class ZenRowsClient: @@ -125,11 +133,13 @@ def extract( or "standard". Thin wrapper over `fetch()` with the `extract` param set — no separate endpoint or auth. - `mode="auto"` is a domain-gated open beta: when the target domain isn't - enabled yet, the API returns a 402 with `code: "AUTH010"`. By default - this retries once with `autoparse=True` instead of returning the error - response — pass `fallback_to_autoparse=False` to disable that and get - the raw AUTH010 response back. + Extract works on any domain once Zenrows has prepared it, with any + active API key. A domain that isn't prepared yet answers 403 with + `code: "REQS007"` (prepare it with `POST /v1/extract/prepared-domains`); + the legacy domain-not-enabled code is a 402 with `code: "AUTH010"`. + On either, `mode="auto"` retries once with `autoparse=True` instead of + returning the error response — pass `fallback_to_autoparse=False` to + disable that and get the raw error response back. `adaptive_stealth=True` (default) also sends Adaptive Stealth Mode (`mode: "auto"` at the wire-param level - unrelated to this method's own @@ -145,12 +155,7 @@ def extract( final_params["mode"] = "auto" response = self.fetch(url, final_params, headers, **kwargs) - if ( - response.status_code == 402 - and mode == "auto" - and fallback_to_autoparse - and _is_auth010(response) - ): + if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): autoparse_params = dict(params) if params else {} autoparse_params["autoparse"] = True if adaptive_stealth: @@ -211,7 +216,7 @@ async def extract_async( **kwargs: Any, ) -> requests.Response: """Async counterpart of `extract()` - see its docstring for the - AUTH010 -> Autoparse fallback behavior, `fallback_to_autoparse`, and + REQS007 / AUTH010 -> Autoparse fallback behavior, `fallback_to_autoparse`, and `adaptive_stealth`.""" final_params = dict(params) if params else {} final_params["extract"] = mode @@ -219,12 +224,7 @@ async def extract_async( final_params["mode"] = "auto" response = await self.fetch_async(url, final_params, headers, **kwargs) - if ( - response.status_code == 402 - and mode == "auto" - and fallback_to_autoparse - and _is_auth010(response) - ): + if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): autoparse_params = dict(params) if params else {} autoparse_params["autoparse"] = True if adaptive_stealth: diff --git a/tests/test_fetch_extract.py b/tests/test_fetch_extract.py index 4ea18ac..3d4fd7f 100644 --- a/tests/test_fetch_extract.py +++ b/tests/test_fetch_extract.py @@ -125,10 +125,10 @@ def test_extract_does_not_mutate_caller_supplied_params_dict(self, mock_request) class TestExtractAutoparseFallback(TestCase): - """`extract(mode="auto")` is a domain-gated open beta: AUTH010 means the - target domain isn't enabled yet. By default this retries once with - Autoparse instead of raising - same behavior as the CLI's extract - adapter.""" + """`extract(mode="auto")`: 403 REQS007 means the target domain isn't + prepared for Extract yet, and the legacy 402 AUTH010 that it isn't + enabled. On either, by default this retries once with Autoparse instead + of returning the error - same behavior as the CLI's extract adapter.""" def setUp(self): self.client = ZenRowsClient(apikey) @@ -148,6 +148,53 @@ def test_falls_back_to_autoparse_on_auth010(self, mock_request): self.assertTrue(fallback_kwargs["params"].get("autoparse")) self.assertNotIn("extract", fallback_kwargs["params"]) + @mock.patch.object(Session, "request") + def test_falls_back_to_autoparse_on_reqs007(self, mock_request): + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007", "title": "Domain not prepared"}'), + _fake_response(200, b'[{"found": "via autoparse"}]'), + ] + + response = self.client.extract(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(mock_request.call_count, 2) + _, fallback_kwargs = mock_request.call_args + self.assertTrue(fallback_kwargs["params"].get("autoparse")) + self.assertNotIn("extract", fallback_kwargs["params"]) + + @mock.patch.object(Session, "request") + def test_reqs007_fallback_disabled_returns_error_response(self, mock_request): + mock_request.return_value = _fake_response(403, b'{"code": "REQS007"}') + + response = self.client.extract(url, fallback_to_autoparse=False) + + self.assertEqual(response.status_code, 403) + self.assertEqual(mock_request.call_count, 1) + + @mock.patch.object(Session, "request") + def test_403_without_reqs007_does_not_fall_back(self, mock_request): + """A Zenrows auth 403 must not be mistaken for a domain that isn't + prepared yet.""" + mock_request.return_value = _fake_response( + 403, b'{"code": "AUTH002", "title": "Invalid API key"}' + ) + + response = self.client.extract(url) + + self.assertEqual(response.status_code, 403) + self.assertEqual(mock_request.call_count, 1) + + @mock.patch.object(Session, "request") + def test_code_on_the_wrong_status_does_not_fall_back(self, mock_request): + """REQS007 is only the not-prepared signal on a 403.""" + mock_request.return_value = _fake_response(400, b'{"code": "REQS007"}') + + response = self.client.extract(url) + + self.assertEqual(response.status_code, 400) + self.assertEqual(mock_request.call_count, 1) + @mock.patch.object(Session, "request") def test_fallback_disabled_returns_error_response(self, mock_request): mock_request.return_value = _fake_response(402, b'{"code": "AUTH010"}') @@ -160,7 +207,7 @@ def test_fallback_disabled_returns_error_response(self, mock_request): @mock.patch.object(Session, "request") def test_402_without_auth010_does_not_fall_back(self, mock_request): """A real credits-exhausted 402 (e.g. AUTH004) must not be mistaken - for the domain-gating error.""" + for the domain-not-enabled error.""" mock_request.return_value = _fake_response( 402, b'{"code": "AUTH004", "title": "No credit available"}' ) @@ -172,14 +219,19 @@ def test_402_without_auth010_does_not_fall_back(self, mock_request): @mock.patch.object(Session, "request") def test_no_fallback_for_non_auto_mode(self, mock_request): - """AUTH010 shouldn't trigger a fallback for native/standard modes - - only "auto" is the domain-gated beta path.""" - mock_request.return_value = _fake_response(402, b'{"code": "AUTH010"}') + """Neither code triggers a fallback for native/standard modes - + only "auto" takes the fallback path.""" + mock_request.side_effect = [ + _fake_response(402, b'{"code": "AUTH010"}'), + _fake_response(403, b'{"code": "REQS007"}'), + ] - response = self.client.extract(url, mode="native") + first = self.client.extract(url, mode="native") + second = self.client.extract(url, mode="standard") - self.assertEqual(response.status_code, 402) - self.assertEqual(mock_request.call_count, 1) + self.assertEqual(first.status_code, 402) + self.assertEqual(second.status_code, 403) + self.assertEqual(mock_request.call_count, 2) @mock.patch.object(Session, "request") def test_fallback_does_not_mutate_caller_supplied_params_dict(self, mock_request): @@ -234,7 +286,7 @@ def test_fallback_request_also_carries_adaptive_stealth(self, mock_request): class TestExtractAsyncAutoparseFallback(IsolatedAsyncioTestCase): - """Async counterpart - same AUTH010 -> Autoparse behavior.""" + """Async counterpart - same REQS007 / AUTH010 -> Autoparse behavior.""" def setUp(self): self.client = ZenRowsClient(apikey, concurrency=2) @@ -251,6 +303,20 @@ async def test_falls_back_to_autoparse_on_auth010(self, mock_request): self.assertEqual(response.status_code, 200) self.assertEqual(mock_request.call_count, 2) + @mock.patch.object(Session, "request") + async def test_falls_back_to_autoparse_on_reqs007(self, mock_request): + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007"}'), + _fake_response(200, b'[{"found": "via autoparse"}]'), + ] + + response = await self.client.extract_async(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(mock_request.call_count, 2) + _, fallback_kwargs = mock_request.call_args + self.assertTrue(fallback_kwargs["params"].get("autoparse")) + class TestFetchExtractAsync(IsolatedAsyncioTestCase): def setUp(self): From 84f123e23a871959492e3ed04f7ad45e41fd2245 Mon Sep 17 00:00:00 2001 From: Ander Date: Tue, 13 Oct 2026 16:08:59 +0200 Subject: [PATCH 3/8] docs(pypi): the package summary uses a colon instead of an em dash --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7406c69..5b9273f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "zenrows" version = "1.6.0" -description = "Zenrows Python SDK — Fetch, Extract (structured data), Batch and Crawl APIs" +description = "Zenrows Python SDK: Fetch, Extract (structured data), Batch and Crawl APIs" readme = "README.md" license = { text = "MIT" } authors = [ From a5eaaae9d5d1d6589f21f600252c10a743abec6f Mon Sep 17 00:00:00 2001 From: Ander Date: Tue, 13 Oct 2026 16:09:04 +0200 Subject: [PATCH 4/8] style(extract): wrap the async docstring to the width of the lines around it --- src/zenrows/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 271c918..f72af33 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -216,8 +216,8 @@ async def extract_async( **kwargs: Any, ) -> requests.Response: """Async counterpart of `extract()` - see its docstring for the - REQS007 / AUTH010 -> Autoparse fallback behavior, `fallback_to_autoparse`, and - `adaptive_stealth`.""" + REQS007 / AUTH010 -> Autoparse fallback behavior, + `fallback_to_autoparse`, and `adaptive_stealth`.""" final_params = dict(params) if params else {} final_params["extract"] = mode if adaptive_stealth: From 11445129ace1afca03adf22baeb52ce76dc72ecb Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 13 Oct 2026 19:05:28 +0200 Subject: [PATCH 5/8] docs(extract): stop calling Extract beta Extract is generally available, but the README and the extract() docstring still labelled it beta. Drop the label. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/zenrows/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3275531..13b7208 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ asyncio.run(main()) ### Extract -[Extract](https://docs.zenrows.com/extract/introduction) (beta) runs a page through Zenrows' AI-powered structured extraction instead of returning raw HTML. Use `client.extract()` — it's the same request as `fetch()`, with the `extract` param set for you (defaults to `"auto"`; pass `"native"` or `"standard"` for the other contracts). +[Extract](https://docs.zenrows.com/extract/introduction) runs a page through Zenrows' AI-powered structured extraction instead of returning raw HTML. Use `client.extract()` — it's the same request as `fetch()`, with the `extract` param set for you (defaults to `"auto"`; pass `"native"` or `"standard"` for the other contracts). ```python from zenrows import ZenRowsClient diff --git a/src/zenrows/client.py b/src/zenrows/client.py index e4b3d07..018ecd8 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -121,7 +121,7 @@ def extract( **kwargs: Any, ) -> requests.Response: """Fetch a URL and run it through Extract — Zenrows' AI-powered structured - extraction (beta). `mode` is one of "auto" (default), "native", + extraction. `mode` is one of "auto" (default), "native", or "standard". Thin wrapper over `fetch()` with the `extract` param set — no separate endpoint or auth. From 59a31775929331e91165563f54fc68158e1f99be Mon Sep 17 00:00:00 2001 From: Ander Date: Tue, 13 Oct 2026 19:34:55 +0200 Subject: [PATCH 6/8] docs(extract): drop the em dashes left in the extract() docstring and README Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/zenrows/client.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 13b7208..ad8f988 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ asyncio.run(main()) ### Extract -[Extract](https://docs.zenrows.com/extract/introduction) runs a page through Zenrows' AI-powered structured extraction instead of returning raw HTML. Use `client.extract()` — it's the same request as `fetch()`, with the `extract` param set for you (defaults to `"auto"`; pass `"native"` or `"standard"` for the other contracts). +[Extract](https://docs.zenrows.com/extract/introduction) runs a page through Zenrows' AI-powered structured extraction instead of returning raw HTML. Use `client.extract()`: it's the same request as `fetch()`, with the `extract` param set for you (defaults to `"auto"`; pass `"native"` or `"standard"` for the other contracts). ```python from zenrows import ZenRowsClient diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 018ecd8..1a9cbc6 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -120,9 +120,9 @@ def extract( adaptive_stealth: bool = True, **kwargs: Any, ) -> requests.Response: - """Fetch a URL and run it through Extract — Zenrows' AI-powered structured + """Fetch a URL and run it through Extract, Zenrows' AI-powered structured extraction. `mode` is one of "auto" (default), "native", - or "standard". Thin wrapper over `fetch()` with the `extract` param set — + or "standard". Thin wrapper over `fetch()` with the `extract` param set: no separate endpoint or auth. Extract works on any domain once Zenrows has prepared it, with any @@ -130,7 +130,7 @@ def extract( `code: "REQS007"`, returned as is; prepare it with `POST /v1/extract/prepared-domains`. If the API answers with `code: "AUTH010"` (domain not enabled), this retries once with - `autoparse=True` instead of returning the error response — pass + `autoparse=True` instead of returning the error response. Pass `fallback_to_autoparse=False` to disable that and get the raw AUTH010 response back. From fbb827817ff5631b5dd24c71d44cf7b07a9e7127 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 14 Oct 2026 00:04:38 +0200 Subject: [PATCH 7/8] fix(extract): never retry a 422 no_data_extracted With retries enabled, every 422 was retried by urllib3. Extract answers 422 with code no_data_extracted when it knows the kind of page but finds no data on it, and that answer is final: retrying returns the same result after another full Extract call. Retry 422s in the client instead, where the body can be read: a 422 no_data_extracted is returned at once, any other 422 is still retried up to `retries` times on the same backoff schedule as before. Export is_no_data_extracted(response) so callers can branch on it. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++- src/zenrows/__init__.py | 3 +- src/zenrows/client.py | 69 +++++++++++++++++++++++++++++------ tests/test_retries.py | 80 +++++++++++++++++++++++++++++++++++++++-- 4 files changed, 141 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6ca8d31..bc7a639 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Start using the API by [creating your API Key](https://www.zenrows.com/register? The SDK uses [requests](https://docs.python-requests.org/) for HTTP requests. The client's response will be a requests `Response`. -It also uses [Retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html) to automatically retry failed requests (status codes 429, 500, 502, 503, and 504). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests. +It also uses [Retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html) to automatically retry failed requests (status codes 422, 429, 500, 502, 503, and 504, except Extract's `422 no_data_extracted`). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests. `client.fetch()` is the primary method for the main page-scraping product; `client.get()` still works and is kept as a deprecated alias. @@ -202,6 +202,8 @@ response = client.extract(url) # extract: "auto" print(response.json()) ``` +A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup). + ## Quickstart — Batch API (`ZenRowsBatchClient`) > Batch is open to every account and spends the same credits as a regular diff --git a/src/zenrows/__init__.py b/src/zenrows/__init__.py index f9e7159..8a13b06 100644 --- a/src/zenrows/__init__.py +++ b/src/zenrows/__init__.py @@ -15,7 +15,7 @@ from zenrows.__version__ import __version__ from zenrows.batch import ZenRowsBatchClient -from zenrows.client import ZenRowsClient +from zenrows.client import ZenRowsClient, is_no_data_extracted from zenrows.crawl import CrawlAPIError, ZenRowsCrawlClient __all__ = [ @@ -24,4 +24,5 @@ "ZenRowsClient", "ZenRowsCrawlClient", "__version__", + "is_no_data_extracted", ] diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 885bde4..29ccb4c 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -13,6 +13,7 @@ import asyncio import os +import time from concurrent.futures import ThreadPoolExecutor from functools import partial from typing import Any @@ -27,9 +28,32 @@ DEFAULT_SCRAPER_URL = "https://api.zenrows.com/v1/" DEFAULT_USER_AGENT = f"zenrows/{__version__} python" -# Status codes the retry layer treats as transient. The set was inherited -# from the original SDK and matches the gateway's documented retry guidance. -_RETRY_STATUSES = [422, 429, 500, 502, 503, 504] +# Status codes the urllib3 retry layer treats as transient. 422 is retried +# too, but in `_worker`, because whether it is worth retrying depends on the +# response body (see `is_no_data_extracted`). +_RETRY_STATUSES = [429, 500, 502, 503, 504] +_RETRY_BACKOFF_FACTOR = 0.5 + +# Extract's answer when it knows the kind of page but none of the fields came +# back filled. Retrying returns the same answer, so it is never retried. +_NO_DATA_EXTRACTED = "no_data_extracted" + + +def _error_code(response: requests.Response) -> str | None: + try: + body = response.json() + except ValueError: + return None + code = body.get("code") if isinstance(body, dict) else None + return code if isinstance(code, str) else None + + +def is_no_data_extracted(response: requests.Response) -> bool: + """True when the response is Extract's 422 `no_data_extracted`: Extract + knows this kind of page but found no data on it. The request is not + charged, and the client does not retry it.""" + code = _error_code(response) + return response.status_code == 422 and code is not None and code.lower() == _NO_DATA_EXTRACTED # Extract answers these when it can't serve the target domain: 403 REQS007 @@ -41,12 +65,8 @@ def _is_extract_domain_unavailable(response: requests.Response) -> bool: """True when the response is Extract saying it can't serve the target domain yet (403 REQS007, or the legacy 402 AUTH010).""" - try: - body = response.json() - except ValueError: - return False - code = body.get("code") if isinstance(body, dict) else None - if not isinstance(code, str): + code = _error_code(response) + if code is None: return False return (response.status_code, code.upper()) in _EXTRACT_DOMAIN_UNAVAILABLE @@ -81,12 +101,13 @@ def __init__( self.apikey = apikey self.api_url = base_url or os.environ.get("ZENROWS_SCRAPER_BASE_URL") or DEFAULT_SCRAPER_URL + self.retries = retries self.executor = ThreadPoolExecutor(max_workers=concurrency) self.requests_session = requests.Session() if retries > 0: max_retries = Retry( total=retries, - backoff_factor=0.5, + backoff_factor=_RETRY_BACKOFF_FACTOR, status_forcelist=_RETRY_STATUSES, raise_on_status=False, ) @@ -141,6 +162,10 @@ def extract( returning the error response. Pass `fallback_to_autoparse=False` to disable that and get the raw error response back. + A 422 with `code: "no_data_extracted"` means Extract knows this kind + of page but found no data on it. It is not charged and is never + retried; check it with `is_no_data_extracted(response)`. + `adaptive_stealth=True` (default) also sends Adaptive Stealth Mode (`mode: "auto"` at the wire-param level - unrelated to this method's own `mode` argument) on both the extract attempt and the Autoparse fallback, @@ -309,7 +334,7 @@ def _worker( final_headers["Connection"] = None final_headers.update(headers) - return self.requests_session.request( + response = self.requests_session.request( method, self.api_url, params=final_params, @@ -317,3 +342,25 @@ def _worker( data=data, **kwargs, ) + # Retry a 422 here rather than in urllib3, which can't read the body: + # Extract's `no_data_extracted` is a final answer, any other 422 is a + # failure to get the target's content and is worth another try. + attempt = 0 + while ( + response.status_code == 422 + and attempt < self.retries + and not is_no_data_extracted(response) + ): + attempt += 1 + # Same schedule as urllib3's Retry: no wait before the first retry. + if attempt > 1: + time.sleep(_RETRY_BACKOFF_FACTOR * 2 ** (attempt - 1)) + response = self.requests_session.request( + method, + self.api_url, + params=final_params, + headers=final_headers, + data=data, + **kwargs, + ) + return response diff --git a/tests/test_retries.py b/tests/test_retries.py index 3538238..0fb37d5 100644 --- a/tests/test_retries.py +++ b/tests/test_retries.py @@ -1,9 +1,10 @@ +import asyncio from unittest import TestCase from unittest.mock import patch -from requests import Session +from requests import Response, Session -from zenrows import ZenRowsClient +from zenrows import ZenRowsClient, is_no_data_extracted from zenrows.client import _RETRY_STATUSES apikey = "APIKEY" @@ -36,3 +37,78 @@ def test_adapter_is_mount(self, mock_mount): ZenRowsClient(apikey, retries=2) self.assertEqual(mock_mount.call_count, 4) + + +def _fake_response(status_code: int, body: bytes = b"") -> Response: + response = Response() + response.status_code = status_code + response._content = body + return response + + +NO_DATA = b'{"code": "no_data_extracted", "title": "No data extracted"}' +NO_CONTENT = b'{"code": "RESP001", "title": "Could not get content"}' + + +@patch("zenrows.client.time.sleep") +class TestUnprocessableRetries(TestCase): + """A 422 is retried in the client, not in urllib3, because only the body + says whether another try can help.""" + + @patch.object(Session, "request") + def test_no_data_extracted_is_not_retried(self, mock_request, mock_sleep): + mock_request.return_value = _fake_response(422, NO_DATA) + + response = ZenRowsClient(apikey, retries=3).extract(url) + + self.assertEqual(response.status_code, 422) + self.assertEqual(mock_request.call_count, 1) + mock_sleep.assert_not_called() + + @patch.object(Session, "request") + def test_other_422_is_retried_up_to_retries(self, mock_request, mock_sleep): + mock_request.return_value = _fake_response(422, NO_CONTENT) + + response = ZenRowsClient(apikey, retries=3).fetch(url) + + self.assertEqual(response.status_code, 422) + self.assertEqual(mock_request.call_count, 4) # 1 + 3 retries + self.assertEqual([c.args[0] for c in mock_sleep.call_args_list], [1.0, 2.0]) + + @patch.object(Session, "request") + def test_other_422_stops_retrying_on_success(self, mock_request, mock_sleep): + mock_request.side_effect = [_fake_response(422, NO_CONTENT), _fake_response(200, b"ok")] + + response = ZenRowsClient(apikey, retries=3).fetch(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(mock_request.call_count, 2) + + @patch.object(Session, "request") + def test_422_not_retried_without_retries(self, mock_request, mock_sleep): + mock_request.return_value = _fake_response(422, NO_CONTENT) + + ZenRowsClient(apikey).fetch(url) + + self.assertEqual(mock_request.call_count, 1) + + @patch.object(Session, "request") + def test_no_data_extracted_is_not_retried_async(self, mock_request, mock_sleep): + mock_request.return_value = _fake_response(422, NO_DATA) + + response = asyncio.run(ZenRowsClient(apikey, retries=3).extract_async(url)) + + self.assertEqual(response.status_code, 422) + self.assertEqual(mock_request.call_count, 1) + + +class TestIsNoDataExtracted(TestCase): + def test_matches_422_no_data_extracted_any_case(self): + self.assertTrue(is_no_data_extracted(_fake_response(422, NO_DATA))) + self.assertTrue(is_no_data_extracted(_fake_response(422, b'{"code": "NO_DATA_EXTRACTED"}'))) + + def test_rejects_other_codes_statuses_and_bodies(self): + self.assertFalse(is_no_data_extracted(_fake_response(422, NO_CONTENT))) + self.assertFalse(is_no_data_extracted(_fake_response(400, NO_DATA))) + self.assertFalse(is_no_data_extracted(_fake_response(422, b"not json"))) + self.assertFalse(is_no_data_extracted(_fake_response(422, b"[]"))) From 23c34c3e514447cffc89919bd40c2d20bce22d7f Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 14 Oct 2026 00:26:40 +0200 Subject: [PATCH 8/8] fix(extract): skip Adaptive Stealth Mode when js_render or premium_proxy is set extract() and extract_async() added the wire `mode=auto` (Adaptive Stealth Mode) on top of whatever the caller passed. The API rejects mode=auto combined with js_render or premium_proxy (400 REQ_INVALID_PARAMS), so a caller forcing either one, like the Extract docs' js_render + premium_proxy example, always failed unless they also knew to pass adaptive_stealth=False. Leave mode=auto out when params sets js_render or premium_proxy (true or "true"), or already carries its own wire mode, on both the Extract request and the Autoparse fallback. Document it in the docstring and README. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 ++ src/zenrows/client.py | 42 ++++++++++++++++------ tests/test_fetch_extract.py | 69 ++++++++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index bc7a639..94404fe 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,8 @@ response = client.extract(url) # extract: "auto" print(response.json()) ``` +`extract()` also sends [Adaptive Stealth Mode](https://docs.zenrows.com/fetch/features/adaptive-stealth-mode) (`mode: "auto"`) by default, except when `params` sets `js_render` or `premium_proxy`: the API rejects either combined with Adaptive Stealth Mode, so your fixed configuration is sent as is. Pass `adaptive_stealth=False` to turn it off entirely. + A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup). ## Quickstart — Batch API (`ZenRowsBatchClient`) diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 29ccb4c..2a38502 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -71,6 +71,29 @@ def _is_extract_domain_unavailable(response: requests.Response) -> bool: return (response.status_code, code.upper()) in _EXTRACT_DOMAIN_UNAVAILABLE +# Adaptive Stealth Mode (wire `mode=auto`) picks js_render / premium_proxy +# itself, so the API rejects it combined with either one set by hand. +_ADAPTIVE_STEALTH_CONFLICTS = ("js_render", "premium_proxy") + + +def _is_set(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() == "true" + return bool(value) + + +def _with_adaptive_stealth(params: dict, enabled: bool) -> dict: + """Add wire `mode=auto` to `params` unless the caller already chose a + `mode`, or forces js_render / premium_proxy, which the API won't combine + with Adaptive Stealth Mode.""" + if not enabled or "mode" in params: + return params + if any(_is_set(params.get(key)) for key in _ADAPTIVE_STEALTH_CONFLICTS): + return params + params["mode"] = "auto" + return params + + class ZenRowsClient: """Synchronous client for the Zenrows scraping API. @@ -170,21 +193,20 @@ def extract( (`mode: "auto"` at the wire-param level - unrelated to this method's own `mode` argument) on both the extract attempt and the Autoparse fallback, so a target needing `js_render`/`premium_proxy` gets escalated - automatically instead of failing with REQS002. Pass - `adaptive_stealth=False` to disable that and set `js_render`/ - `premium_proxy` yourself. + automatically instead of failing with REQS002. It is skipped when + `params` sets `js_render` or `premium_proxy` (the API rejects either + combined with Adaptive Stealth Mode) or its own wire `mode`. Pass + `adaptive_stealth=False` to turn it off entirely. """ final_params = dict(params) if params else {} final_params["extract"] = mode - if adaptive_stealth: - final_params["mode"] = "auto" + _with_adaptive_stealth(final_params, adaptive_stealth) response = self.fetch(url, final_params, headers, **kwargs) if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): autoparse_params = dict(params) if params else {} autoparse_params["autoparse"] = True - if adaptive_stealth: - autoparse_params["mode"] = "auto" + _with_adaptive_stealth(autoparse_params, adaptive_stealth) return self.fetch(url, autoparse_params, headers, **kwargs) return response @@ -245,15 +267,13 @@ async def extract_async( `fallback_to_autoparse`, and `adaptive_stealth`.""" final_params = dict(params) if params else {} final_params["extract"] = mode - if adaptive_stealth: - final_params["mode"] = "auto" + _with_adaptive_stealth(final_params, adaptive_stealth) response = await self.fetch_async(url, final_params, headers, **kwargs) if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): autoparse_params = dict(params) if params else {} autoparse_params["autoparse"] = True - if adaptive_stealth: - autoparse_params["mode"] = "auto" + _with_adaptive_stealth(autoparse_params, adaptive_stealth) return await self.fetch_async(url, autoparse_params, headers, **kwargs) return response diff --git a/tests/test_fetch_extract.py b/tests/test_fetch_extract.py index 3d4fd7f..f417928 100644 --- a/tests/test_fetch_extract.py +++ b/tests/test_fetch_extract.py @@ -107,7 +107,6 @@ def test_extract_merges_extract_param_with_other_params(self, mock_request): "apikey": apikey, "js_render": True, "extract": "native", - "mode": "auto", }, ) @@ -284,6 +283,60 @@ def test_fallback_request_also_carries_adaptive_stealth(self, mock_request): _, fallback_kwargs = mock_request.call_args self.assertEqual(fallback_kwargs["params"]["mode"], "auto") + @mock.patch.object(Session, "request") + def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): + """The API rejects mode=auto combined with js_render, so a caller's + fixed configuration is sent as is.""" + mock_request.return_value = _fake_response(200) + + self.client.extract(url, params={"js_render": True}) + + _, kwargs = mock_request.call_args + self.assertNotIn("mode", kwargs["params"]) + self.assertTrue(kwargs["params"]["js_render"]) + + @mock.patch.object(Session, "request") + def test_skips_adaptive_stealth_when_premium_proxy_is_set(self, mock_request): + mock_request.return_value = _fake_response(200) + + self.client.extract(url, params={"premium_proxy": "true"}) + + _, kwargs = mock_request.call_args + self.assertNotIn("mode", kwargs["params"]) + + @mock.patch.object(Session, "request") + def test_false_js_render_keeps_adaptive_stealth(self, mock_request): + mock_request.return_value = _fake_response(200) + + self.client.extract(url, params={"js_render": "false", "premium_proxy": False}) + + _, kwargs = mock_request.call_args + self.assertEqual(kwargs["params"]["mode"], "auto") + + @mock.patch.object(Session, "request") + def test_keeps_a_wire_mode_the_caller_set(self, mock_request): + mock_request.return_value = _fake_response(200) + + self.client.extract(url, params={"mode": "custom"}) + + _, kwargs = mock_request.call_args + self.assertEqual(kwargs["params"]["mode"], "custom") + + @mock.patch.object(Session, "request") + def test_fallback_request_skips_adaptive_stealth_with_premium_proxy(self, mock_request): + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007"}'), + _fake_response(200, b"{}"), + ] + + self.client.extract(url, params={"premium_proxy": True}) + + first_kwargs = mock_request.call_args_list[0].kwargs + _, fallback_kwargs = mock_request.call_args + self.assertNotIn("mode", first_kwargs["params"]) + self.assertNotIn("mode", fallback_kwargs["params"]) + self.assertTrue(fallback_kwargs["params"]["autoparse"]) + class TestExtractAsyncAutoparseFallback(IsolatedAsyncioTestCase): """Async counterpart - same REQS007 / AUTH010 -> Autoparse behavior.""" @@ -303,6 +356,20 @@ async def test_falls_back_to_autoparse_on_auth010(self, mock_request): self.assertEqual(response.status_code, 200) self.assertEqual(mock_request.call_count, 2) + @mock.patch.object(Session, "request") + async def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007"}'), + _fake_response(200, b"{}"), + ] + + await self.client.extract_async(url, params={"js_render": True}) + + first_kwargs = mock_request.call_args_list[0].kwargs + _, fallback_kwargs = mock_request.call_args + self.assertNotIn("mode", first_kwargs["params"]) + self.assertNotIn("mode", fallback_kwargs["params"]) + @mock.patch.object(Session, "request") async def test_falls_back_to_autoparse_on_reqs007(self, mock_request): mock_request.side_effect = [