diff --git a/README.md b/README.md index 6ca8d31..94404fe 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Start using the API by [creating your API Key](https://www.zenrows.com/register? The SDK uses [requests](https://docs.python-requests.org/) for HTTP requests. The client's response will be a requests `Response`. -It also uses [Retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html) to automatically retry failed requests (status codes 429, 500, 502, 503, and 504). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests. +It also uses [Retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html) to automatically retry failed requests (status codes 422, 429, 500, 502, 503, and 504, except Extract's `422 no_data_extracted`). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests. `client.fetch()` is the primary method for the main page-scraping product; `client.get()` still works and is kept as a deprecated alias. @@ -202,6 +202,10 @@ response = client.extract(url) # extract: "auto" print(response.json()) ``` +`extract()` also sends [Adaptive Stealth Mode](https://docs.zenrows.com/fetch/features/adaptive-stealth-mode) (`mode: "auto"`) by default, except when `params` sets `js_render` or `premium_proxy`: the API rejects either combined with Adaptive Stealth Mode, so your fixed configuration is sent as is. Pass `adaptive_stealth=False` to turn it off entirely. + +A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup). + ## Quickstart — Batch API (`ZenRowsBatchClient`) > Batch is open to every account and spends the same credits as a regular diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 13fa3fd..4c80d2f 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1297,9 +1297,10 @@ components: `extract` selects the structured-extraction contract family (`auto`, `native`, or `standard`); `extract_fields` optionally - narrows the returned fields. Extract is in private beta on the - scraper API — tasks against domains not enrolled fail with the - gateway's upgrade-required error. + narrows the returned fields. Extract works on any domain + Zenrows has prepared, with any active API key; a task against a + domain that isn't prepared yet fails with the scraper API's + 403 `REQS007`. `custom_headers` additionally accepts an object of header name → value pairs (or that object serialised as a JSON diff --git a/src/zenrows/__init__.py b/src/zenrows/__init__.py index f9e7159..8a13b06 100644 --- a/src/zenrows/__init__.py +++ b/src/zenrows/__init__.py @@ -15,7 +15,7 @@ from zenrows.__version__ import __version__ from zenrows.batch import ZenRowsBatchClient -from zenrows.client import ZenRowsClient +from zenrows.client import ZenRowsClient, is_no_data_extracted from zenrows.crawl import CrawlAPIError, ZenRowsCrawlClient __all__ = [ @@ -24,4 +24,5 @@ "ZenRowsClient", "ZenRowsCrawlClient", "__version__", + "is_no_data_extracted", ] diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 1a9cbc6..2a38502 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -13,6 +13,7 @@ import asyncio import os +import time from concurrent.futures import ThreadPoolExecutor from functools import partial from typing import Any @@ -27,20 +28,70 @@ DEFAULT_SCRAPER_URL = "https://api.zenrows.com/v1/" DEFAULT_USER_AGENT = f"zenrows/{__version__} python" -# Status codes the retry layer treats as transient. The set was inherited -# from the original SDK and matches the gateway's documented retry guidance. -_RETRY_STATUSES = [422, 429, 500, 502, 503, 504] +# Status codes the urllib3 retry layer treats as transient. 422 is retried +# too, but in `_worker`, because whether it is worth retrying depends on the +# response body (see `is_no_data_extracted`). +_RETRY_STATUSES = [429, 500, 502, 503, 504] +_RETRY_BACKOFF_FACTOR = 0.5 +# Extract's answer when it knows the kind of page but none of the fields came +# back filled. Retrying returns the same answer, so it is never retried. +_NO_DATA_EXTRACTED = "no_data_extracted" -def _is_auth010(response: requests.Response) -> bool: - """True when a response's JSON error envelope carries the Extract - domain-not-enabled code (AUTH010).""" + +def _error_code(response: requests.Response) -> str | None: try: body = response.json() except ValueError: - return False + return None code = body.get("code") if isinstance(body, dict) else None - return isinstance(code, str) and code.upper() == "AUTH010" + return code if isinstance(code, str) else None + + +def is_no_data_extracted(response: requests.Response) -> bool: + """True when the response is Extract's 422 `no_data_extracted`: Extract + knows this kind of page but found no data on it. The request is not + charged, and the client does not retry it.""" + code = _error_code(response) + return response.status_code == 422 and code is not None and code.lower() == _NO_DATA_EXTRACTED + + +# Extract answers these when it can't serve the target domain: 403 REQS007 +# (domain not prepared yet) or the legacy 402 AUTH010 (domain not enabled). +# extract(mode="auto") retries both once with Autoparse. +_EXTRACT_DOMAIN_UNAVAILABLE = {(403, "REQS007"), (402, "AUTH010")} + + +def _is_extract_domain_unavailable(response: requests.Response) -> bool: + """True when the response is Extract saying it can't serve the target + domain yet (403 REQS007, or the legacy 402 AUTH010).""" + code = _error_code(response) + if code is None: + return False + return (response.status_code, code.upper()) in _EXTRACT_DOMAIN_UNAVAILABLE + + +# Adaptive Stealth Mode (wire `mode=auto`) picks js_render / premium_proxy +# itself, so the API rejects it combined with either one set by hand. +_ADAPTIVE_STEALTH_CONFLICTS = ("js_render", "premium_proxy") + + +def _is_set(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() == "true" + return bool(value) + + +def _with_adaptive_stealth(params: dict, enabled: bool) -> dict: + """Add wire `mode=auto` to `params` unless the caller already chose a + `mode`, or forces js_render / premium_proxy, which the API won't combine + with Adaptive Stealth Mode.""" + if not enabled or "mode" in params: + return params + if any(_is_set(params.get(key)) for key in _ADAPTIVE_STEALTH_CONFLICTS): + return params + params["mode"] = "auto" + return params class ZenRowsClient: @@ -73,12 +124,13 @@ def __init__( self.apikey = apikey self.api_url = base_url or os.environ.get("ZENROWS_SCRAPER_BASE_URL") or DEFAULT_SCRAPER_URL + self.retries = retries self.executor = ThreadPoolExecutor(max_workers=concurrency) self.requests_session = requests.Session() if retries > 0: max_retries = Retry( total=retries, - backoff_factor=0.5, + backoff_factor=_RETRY_BACKOFF_FACTOR, status_forcelist=_RETRY_STATUSES, raise_on_status=False, ) @@ -127,37 +179,34 @@ def extract( Extract works on any domain once Zenrows has prepared it, with any active API key. A domain that isn't prepared yet answers 403 with - `code: "REQS007"`, returned as is; prepare it with - `POST /v1/extract/prepared-domains`. If the API answers with - `code: "AUTH010"` (domain not enabled), this retries once with - `autoparse=True` instead of returning the error response. Pass - `fallback_to_autoparse=False` to disable that and get the raw AUTH010 - response back. + `code: "REQS007"` (prepare it with `POST /v1/extract/prepared-domains`); + the legacy domain-not-enabled code is a 402 with `code: "AUTH010"`. + On either, `mode="auto"` retries once with `autoparse=True` instead of + returning the error response. Pass `fallback_to_autoparse=False` to + disable that and get the raw error response back. + + A 422 with `code: "no_data_extracted"` means Extract knows this kind + of page but found no data on it. It is not charged and is never + retried; check it with `is_no_data_extracted(response)`. `adaptive_stealth=True` (default) also sends Adaptive Stealth Mode (`mode: "auto"` at the wire-param level - unrelated to this method's own `mode` argument) on both the extract attempt and the Autoparse fallback, so a target needing `js_render`/`premium_proxy` gets escalated - automatically instead of failing with REQS002. Pass - `adaptive_stealth=False` to disable that and set `js_render`/ - `premium_proxy` yourself. + automatically instead of failing with REQS002. It is skipped when + `params` sets `js_render` or `premium_proxy` (the API rejects either + combined with Adaptive Stealth Mode) or its own wire `mode`. Pass + `adaptive_stealth=False` to turn it off entirely. """ final_params = dict(params) if params else {} final_params["extract"] = mode - if adaptive_stealth: - final_params["mode"] = "auto" + _with_adaptive_stealth(final_params, adaptive_stealth) response = self.fetch(url, final_params, headers, **kwargs) - if ( - response.status_code == 402 - and mode == "auto" - and fallback_to_autoparse - and _is_auth010(response) - ): + if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): autoparse_params = dict(params) if params else {} autoparse_params["autoparse"] = True - if adaptive_stealth: - autoparse_params["mode"] = "auto" + _with_adaptive_stealth(autoparse_params, adaptive_stealth) return self.fetch(url, autoparse_params, headers, **kwargs) return response @@ -214,24 +263,17 @@ async def extract_async( **kwargs: Any, ) -> requests.Response: """Async counterpart of `extract()` - see its docstring for the - AUTH010 -> Autoparse fallback behavior, `fallback_to_autoparse`, and - `adaptive_stealth`.""" + REQS007 / AUTH010 -> Autoparse fallback behavior, + `fallback_to_autoparse`, and `adaptive_stealth`.""" final_params = dict(params) if params else {} final_params["extract"] = mode - if adaptive_stealth: - final_params["mode"] = "auto" + _with_adaptive_stealth(final_params, adaptive_stealth) response = await self.fetch_async(url, final_params, headers, **kwargs) - if ( - response.status_code == 402 - and mode == "auto" - and fallback_to_autoparse - and _is_auth010(response) - ): + if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response): autoparse_params = dict(params) if params else {} autoparse_params["autoparse"] = True - if adaptive_stealth: - autoparse_params["mode"] = "auto" + _with_adaptive_stealth(autoparse_params, adaptive_stealth) return await self.fetch_async(url, autoparse_params, headers, **kwargs) return response @@ -312,7 +354,7 @@ def _worker( final_headers["Connection"] = None final_headers.update(headers) - return self.requests_session.request( + response = self.requests_session.request( method, self.api_url, params=final_params, @@ -320,3 +362,25 @@ def _worker( data=data, **kwargs, ) + # Retry a 422 here rather than in urllib3, which can't read the body: + # Extract's `no_data_extracted` is a final answer, any other 422 is a + # failure to get the target's content and is worth another try. + attempt = 0 + while ( + response.status_code == 422 + and attempt < self.retries + and not is_no_data_extracted(response) + ): + attempt += 1 + # Same schedule as urllib3's Retry: no wait before the first retry. + if attempt > 1: + time.sleep(_RETRY_BACKOFF_FACTOR * 2 ** (attempt - 1)) + response = self.requests_session.request( + method, + self.api_url, + params=final_params, + headers=final_headers, + data=data, + **kwargs, + ) + return response diff --git a/tests/test_fetch_extract.py b/tests/test_fetch_extract.py index 32102d2..f417928 100644 --- a/tests/test_fetch_extract.py +++ b/tests/test_fetch_extract.py @@ -107,7 +107,6 @@ def test_extract_merges_extract_param_with_other_params(self, mock_request): "apikey": apikey, "js_render": True, "extract": "native", - "mode": "auto", }, ) @@ -125,10 +124,10 @@ def test_extract_does_not_mutate_caller_supplied_params_dict(self, mock_request) class TestExtractAutoparseFallback(TestCase): - """`extract(mode="auto")`: AUTH010 means the target domain isn't enabled - for Extract. By default this retries once with - Autoparse instead of raising - same behavior as the CLI's extract - adapter.""" + """`extract(mode="auto")`: 403 REQS007 means the target domain isn't + prepared for Extract yet, and the legacy 402 AUTH010 that it isn't + enabled. On either, by default this retries once with Autoparse instead + of returning the error - same behavior as the CLI's extract adapter.""" def setUp(self): self.client = ZenRowsClient(apikey) @@ -148,6 +147,53 @@ def test_falls_back_to_autoparse_on_auth010(self, mock_request): self.assertTrue(fallback_kwargs["params"].get("autoparse")) self.assertNotIn("extract", fallback_kwargs["params"]) + @mock.patch.object(Session, "request") + def test_falls_back_to_autoparse_on_reqs007(self, mock_request): + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007", "title": "Domain not prepared"}'), + _fake_response(200, b'[{"found": "via autoparse"}]'), + ] + + response = self.client.extract(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(mock_request.call_count, 2) + _, fallback_kwargs = mock_request.call_args + self.assertTrue(fallback_kwargs["params"].get("autoparse")) + self.assertNotIn("extract", fallback_kwargs["params"]) + + @mock.patch.object(Session, "request") + def test_reqs007_fallback_disabled_returns_error_response(self, mock_request): + mock_request.return_value = _fake_response(403, b'{"code": "REQS007"}') + + response = self.client.extract(url, fallback_to_autoparse=False) + + self.assertEqual(response.status_code, 403) + self.assertEqual(mock_request.call_count, 1) + + @mock.patch.object(Session, "request") + def test_403_without_reqs007_does_not_fall_back(self, mock_request): + """A Zenrows auth 403 must not be mistaken for a domain that isn't + prepared yet.""" + mock_request.return_value = _fake_response( + 403, b'{"code": "AUTH002", "title": "Invalid API key"}' + ) + + response = self.client.extract(url) + + self.assertEqual(response.status_code, 403) + self.assertEqual(mock_request.call_count, 1) + + @mock.patch.object(Session, "request") + def test_code_on_the_wrong_status_does_not_fall_back(self, mock_request): + """REQS007 is only the not-prepared signal on a 403.""" + mock_request.return_value = _fake_response(400, b'{"code": "REQS007"}') + + response = self.client.extract(url) + + self.assertEqual(response.status_code, 400) + self.assertEqual(mock_request.call_count, 1) + @mock.patch.object(Session, "request") def test_fallback_disabled_returns_error_response(self, mock_request): mock_request.return_value = _fake_response(402, b'{"code": "AUTH010"}') @@ -160,7 +206,7 @@ def test_fallback_disabled_returns_error_response(self, mock_request): @mock.patch.object(Session, "request") def test_402_without_auth010_does_not_fall_back(self, mock_request): """A real credits-exhausted 402 (e.g. AUTH004) must not be mistaken - for the domain-gating error.""" + for the domain-not-enabled error.""" mock_request.return_value = _fake_response( 402, b'{"code": "AUTH004", "title": "No credit available"}' ) @@ -172,14 +218,19 @@ def test_402_without_auth010_does_not_fall_back(self, mock_request): @mock.patch.object(Session, "request") def test_no_fallback_for_non_auto_mode(self, mock_request): - """AUTH010 shouldn't trigger a fallback for native/standard modes - + """Neither code triggers a fallback for native/standard modes - only "auto" takes the fallback path.""" - mock_request.return_value = _fake_response(402, b'{"code": "AUTH010"}') + mock_request.side_effect = [ + _fake_response(402, b'{"code": "AUTH010"}'), + _fake_response(403, b'{"code": "REQS007"}'), + ] - response = self.client.extract(url, mode="native") + first = self.client.extract(url, mode="native") + second = self.client.extract(url, mode="standard") - self.assertEqual(response.status_code, 402) - self.assertEqual(mock_request.call_count, 1) + self.assertEqual(first.status_code, 402) + self.assertEqual(second.status_code, 403) + self.assertEqual(mock_request.call_count, 2) @mock.patch.object(Session, "request") def test_fallback_does_not_mutate_caller_supplied_params_dict(self, mock_request): @@ -232,9 +283,63 @@ def test_fallback_request_also_carries_adaptive_stealth(self, mock_request): _, fallback_kwargs = mock_request.call_args self.assertEqual(fallback_kwargs["params"]["mode"], "auto") + @mock.patch.object(Session, "request") + def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): + """The API rejects mode=auto combined with js_render, so a caller's + fixed configuration is sent as is.""" + mock_request.return_value = _fake_response(200) + + self.client.extract(url, params={"js_render": True}) + + _, kwargs = mock_request.call_args + self.assertNotIn("mode", kwargs["params"]) + self.assertTrue(kwargs["params"]["js_render"]) + + @mock.patch.object(Session, "request") + def test_skips_adaptive_stealth_when_premium_proxy_is_set(self, mock_request): + mock_request.return_value = _fake_response(200) + + self.client.extract(url, params={"premium_proxy": "true"}) + + _, kwargs = mock_request.call_args + self.assertNotIn("mode", kwargs["params"]) + + @mock.patch.object(Session, "request") + def test_false_js_render_keeps_adaptive_stealth(self, mock_request): + mock_request.return_value = _fake_response(200) + + self.client.extract(url, params={"js_render": "false", "premium_proxy": False}) + + _, kwargs = mock_request.call_args + self.assertEqual(kwargs["params"]["mode"], "auto") + + @mock.patch.object(Session, "request") + def test_keeps_a_wire_mode_the_caller_set(self, mock_request): + mock_request.return_value = _fake_response(200) + + self.client.extract(url, params={"mode": "custom"}) + + _, kwargs = mock_request.call_args + self.assertEqual(kwargs["params"]["mode"], "custom") + + @mock.patch.object(Session, "request") + def test_fallback_request_skips_adaptive_stealth_with_premium_proxy(self, mock_request): + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007"}'), + _fake_response(200, b"{}"), + ] + + self.client.extract(url, params={"premium_proxy": True}) + + first_kwargs = mock_request.call_args_list[0].kwargs + _, fallback_kwargs = mock_request.call_args + self.assertNotIn("mode", first_kwargs["params"]) + self.assertNotIn("mode", fallback_kwargs["params"]) + self.assertTrue(fallback_kwargs["params"]["autoparse"]) + class TestExtractAsyncAutoparseFallback(IsolatedAsyncioTestCase): - """Async counterpart - same AUTH010 -> Autoparse behavior.""" + """Async counterpart - same REQS007 / AUTH010 -> Autoparse behavior.""" def setUp(self): self.client = ZenRowsClient(apikey, concurrency=2) @@ -251,6 +356,34 @@ async def test_falls_back_to_autoparse_on_auth010(self, mock_request): self.assertEqual(response.status_code, 200) self.assertEqual(mock_request.call_count, 2) + @mock.patch.object(Session, "request") + async def test_skips_adaptive_stealth_when_js_render_is_set(self, mock_request): + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007"}'), + _fake_response(200, b"{}"), + ] + + await self.client.extract_async(url, params={"js_render": True}) + + first_kwargs = mock_request.call_args_list[0].kwargs + _, fallback_kwargs = mock_request.call_args + self.assertNotIn("mode", first_kwargs["params"]) + self.assertNotIn("mode", fallback_kwargs["params"]) + + @mock.patch.object(Session, "request") + async def test_falls_back_to_autoparse_on_reqs007(self, mock_request): + mock_request.side_effect = [ + _fake_response(403, b'{"code": "REQS007"}'), + _fake_response(200, b'[{"found": "via autoparse"}]'), + ] + + response = await self.client.extract_async(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(mock_request.call_count, 2) + _, fallback_kwargs = mock_request.call_args + self.assertTrue(fallback_kwargs["params"].get("autoparse")) + class TestFetchExtractAsync(IsolatedAsyncioTestCase): def setUp(self): diff --git a/tests/test_retries.py b/tests/test_retries.py index 3538238..0fb37d5 100644 --- a/tests/test_retries.py +++ b/tests/test_retries.py @@ -1,9 +1,10 @@ +import asyncio from unittest import TestCase from unittest.mock import patch -from requests import Session +from requests import Response, Session -from zenrows import ZenRowsClient +from zenrows import ZenRowsClient, is_no_data_extracted from zenrows.client import _RETRY_STATUSES apikey = "APIKEY" @@ -36,3 +37,78 @@ def test_adapter_is_mount(self, mock_mount): ZenRowsClient(apikey, retries=2) self.assertEqual(mock_mount.call_count, 4) + + +def _fake_response(status_code: int, body: bytes = b"") -> Response: + response = Response() + response.status_code = status_code + response._content = body + return response + + +NO_DATA = b'{"code": "no_data_extracted", "title": "No data extracted"}' +NO_CONTENT = b'{"code": "RESP001", "title": "Could not get content"}' + + +@patch("zenrows.client.time.sleep") +class TestUnprocessableRetries(TestCase): + """A 422 is retried in the client, not in urllib3, because only the body + says whether another try can help.""" + + @patch.object(Session, "request") + def test_no_data_extracted_is_not_retried(self, mock_request, mock_sleep): + mock_request.return_value = _fake_response(422, NO_DATA) + + response = ZenRowsClient(apikey, retries=3).extract(url) + + self.assertEqual(response.status_code, 422) + self.assertEqual(mock_request.call_count, 1) + mock_sleep.assert_not_called() + + @patch.object(Session, "request") + def test_other_422_is_retried_up_to_retries(self, mock_request, mock_sleep): + mock_request.return_value = _fake_response(422, NO_CONTENT) + + response = ZenRowsClient(apikey, retries=3).fetch(url) + + self.assertEqual(response.status_code, 422) + self.assertEqual(mock_request.call_count, 4) # 1 + 3 retries + self.assertEqual([c.args[0] for c in mock_sleep.call_args_list], [1.0, 2.0]) + + @patch.object(Session, "request") + def test_other_422_stops_retrying_on_success(self, mock_request, mock_sleep): + mock_request.side_effect = [_fake_response(422, NO_CONTENT), _fake_response(200, b"ok")] + + response = ZenRowsClient(apikey, retries=3).fetch(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(mock_request.call_count, 2) + + @patch.object(Session, "request") + def test_422_not_retried_without_retries(self, mock_request, mock_sleep): + mock_request.return_value = _fake_response(422, NO_CONTENT) + + ZenRowsClient(apikey).fetch(url) + + self.assertEqual(mock_request.call_count, 1) + + @patch.object(Session, "request") + def test_no_data_extracted_is_not_retried_async(self, mock_request, mock_sleep): + mock_request.return_value = _fake_response(422, NO_DATA) + + response = asyncio.run(ZenRowsClient(apikey, retries=3).extract_async(url)) + + self.assertEqual(response.status_code, 422) + self.assertEqual(mock_request.call_count, 1) + + +class TestIsNoDataExtracted(TestCase): + def test_matches_422_no_data_extracted_any_case(self): + self.assertTrue(is_no_data_extracted(_fake_response(422, NO_DATA))) + self.assertTrue(is_no_data_extracted(_fake_response(422, b'{"code": "NO_DATA_EXTRACTED"}'))) + + def test_rejects_other_codes_statuses_and_bodies(self): + self.assertFalse(is_no_data_extracted(_fake_response(422, NO_CONTENT))) + self.assertFalse(is_no_data_extracted(_fake_response(400, NO_DATA))) + self.assertFalse(is_no_data_extracted(_fake_response(422, b"not json"))) + self.assertFalse(is_no_data_extracted(_fake_response(422, b"[]")))