fix(extract): fall back to Autoparse on REQS007, and drop stale beta copy

README.md

@@ -57,7 +57,7 @@ Start using the API by [creating your API Key](https://www.zenrows.com/register?
 
 The SDK uses [requests](https://docs.python-requests.org/) for HTTP requests. The client's response will be a requests `Response`.
 
-It also uses [Retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html) to automatically retry failed requests (status codes 429, 500, 502, 503, and 504). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests.
+It also uses [Retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html) to automatically retry failed requests (status codes 422, 429, 500, 502, 503, and 504, except Extract's `422 no_data_extracted`). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests.
 
 `client.fetch()` is the primary method for the main page-scraping product; `client.get()` still works and is kept as a deprecated alias.
 
@@ -202,6 +202,10 @@ response = client.extract(url)  # extract: "auto"
 print(response.json())
 ```
 
+`extract()` also sends [Adaptive Stealth Mode](https://docs.zenrows.com/fetch/features/adaptive-stealth-mode) (`mode: "auto"`) by default, except when `params` sets `js_render` or `premium_proxy`: the API rejects either combined with Adaptive Stealth Mode, so your fixed configuration is sent as is. Pass `adaptive_stealth=False` to turn it off entirely.
+
+A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup).
+
 ## Quickstart — Batch API (`ZenRowsBatchClient`)
 
 > Batch is open to every account and spends the same credits as a regular

docs/openapi.yaml

@@ -1297,9 +1297,10 @@ components:
 
         `extract` selects the structured-extraction contract family
         (`auto`, `native`, or `standard`); `extract_fields` optionally
-        narrows the returned fields. Extract is in private beta on the
-        scraper API — tasks against domains not enrolled fail with the
-        gateway's upgrade-required error.
+        narrows the returned fields. Extract works on any domain
+        Zenrows has prepared, with any active API key; a task against a
+        domain that isn't prepared yet fails with the scraper API's
+        403 `REQS007`.
 
         `custom_headers` additionally accepts an object of header
         name → value pairs (or that object serialised as a JSON

src/zenrows/__init__.py

@@ -15,7 +15,7 @@
 
 from zenrows.__version__ import __version__
 from zenrows.batch import ZenRowsBatchClient
-from zenrows.client import ZenRowsClient
+from zenrows.client import ZenRowsClient, is_no_data_extracted
 from zenrows.crawl import CrawlAPIError, ZenRowsCrawlClient
 
 __all__ = [
@@ -24,4 +24,5 @@
     "ZenRowsClient",
     "ZenRowsCrawlClient",
     "__version__",
+    "is_no_data_extracted",
 ]

src/zenrows/client.py

@@ -13,6 +13,7 @@
 
 import asyncio
 import os
+import time
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from typing import Any
@@ -27,20 +28,70 @@
 DEFAULT_SCRAPER_URL = "https://api.zenrows.com/v1/"
 DEFAULT_USER_AGENT = f"zenrows/{__version__} python"
 
-# Status codes the retry layer treats as transient. The set was inherited
-# from the original SDK and matches the gateway's documented retry guidance.
-_RETRY_STATUSES = [422, 429, 500, 502, 503, 504]
+# Status codes the urllib3 retry layer treats as transient. 422 is retried
+# too, but in `_worker`, because whether it is worth retrying depends on the
+# response body (see `is_no_data_extracted`).
+_RETRY_STATUSES = [429, 500, 502, 503, 504]
+_RETRY_BACKOFF_FACTOR = 0.5
 
+# Extract's answer when it knows the kind of page but none of the fields came
+# back filled. Retrying returns the same answer, so it is never retried.
+_NO_DATA_EXTRACTED = "no_data_extracted"
 
-def _is_auth010(response: requests.Response) -> bool:
-    """True when a response's JSON error envelope carries the Extract
-    domain-not-enabled code (AUTH010)."""
+
+def _error_code(response: requests.Response) -> str | None:
     try:
         body = response.json()
     except ValueError:
-        return False
+        return None
     code = body.get("code") if isinstance(body, dict) else None
-    return isinstance(code, str) and code.upper() == "AUTH010"
+    return code if isinstance(code, str) else None
+
+
+def is_no_data_extracted(response: requests.Response) -> bool:
+    """True when the response is Extract's 422 `no_data_extracted`: Extract
+    knows this kind of page but found no data on it. The request is not
+    charged, and the client does not retry it."""
+    code = _error_code(response)
+    return response.status_code == 422 and code is not None and code.lower() == _NO_DATA_EXTRACTED
+
+
+# Extract answers these when it can't serve the target domain: 403 REQS007
+# (domain not prepared yet) or the legacy 402 AUTH010 (domain not enabled).
+# extract(mode="auto") retries both once with Autoparse.
+_EXTRACT_DOMAIN_UNAVAILABLE = {(403, "REQS007"), (402, "AUTH010")}
+
+
+def _is_extract_domain_unavailable(response: requests.Response) -> bool:
+    """True when the response is Extract saying it can't serve the target
+    domain yet (403 REQS007, or the legacy 402 AUTH010)."""
+    code = _error_code(response)
+    if code is None:
+        return False
+    return (response.status_code, code.upper()) in _EXTRACT_DOMAIN_UNAVAILABLE
+
+
+# Adaptive Stealth Mode (wire `mode=auto`) picks js_render / premium_proxy
+# itself, so the API rejects it combined with either one set by hand.
+_ADAPTIVE_STEALTH_CONFLICTS = ("js_render", "premium_proxy")
+
+
+def _is_set(value: Any) -> bool:
+    if isinstance(value, str):
+        return value.strip().lower() == "true"
+    return bool(value)
+
+
+def _with_adaptive_stealth(params: dict, enabled: bool) -> dict:
+    """Add wire `mode=auto` to `params` unless the caller already chose a
+    `mode`, or forces js_render / premium_proxy, which the API won't combine
+    with Adaptive Stealth Mode."""
+    if not enabled or "mode" in params:
+        return params
+    if any(_is_set(params.get(key)) for key in _ADAPTIVE_STEALTH_CONFLICTS):
+        return params
+    params["mode"] = "auto"
+    return params
 
 
 class ZenRowsClient:
@@ -73,12 +124,13 @@ def __init__(
         self.apikey = apikey
         self.api_url = base_url or os.environ.get("ZENROWS_SCRAPER_BASE_URL") or DEFAULT_SCRAPER_URL
 
+        self.retries = retries
         self.executor = ThreadPoolExecutor(max_workers=concurrency)
         self.requests_session = requests.Session()
         if retries > 0:
             max_retries = Retry(
                 total=retries,
-                backoff_factor=0.5,
+                backoff_factor=_RETRY_BACKOFF_FACTOR,
                 status_forcelist=_RETRY_STATUSES,
                 raise_on_status=False,
             )
@@ -127,37 +179,34 @@ def extract(
 
         Extract works on any domain once Zenrows has prepared it, with any
         active API key. A domain that isn't prepared yet answers 403 with
-        `code: "REQS007"`, returned as is; prepare it with
-        `POST /v1/extract/prepared-domains`. If the API answers with
-        `code: "AUTH010"` (domain not enabled), this retries once with
-        `autoparse=True` instead of returning the error response. Pass
-        `fallback_to_autoparse=False` to disable that and get the raw AUTH010
-        response back.
+        `code: "REQS007"` (prepare it with `POST /v1/extract/prepared-domains`);
+        the legacy domain-not-enabled code is a 402 with `code: "AUTH010"`.
+        On either, `mode="auto"` retries once with `autoparse=True` instead of
+        returning the error response. Pass `fallback_to_autoparse=False` to
+        disable that and get the raw error response back.
+
+        A 422 with `code: "no_data_extracted"` means Extract knows this kind
+        of page but found no data on it. It is not charged and is never
+        retried; check it with `is_no_data_extracted(response)`.
 
         `adaptive_stealth=True` (default) also sends Adaptive Stealth Mode
         (`mode: "auto"` at the wire-param level - unrelated to this method's own
         `mode` argument) on both the extract attempt and the Autoparse fallback,
         so a target needing `js_render`/`premium_proxy` gets escalated
-        automatically instead of failing with REQS002. Pass
-        `adaptive_stealth=False` to disable that and set `js_render`/
-        `premium_proxy` yourself.
+        automatically instead of failing with REQS002. It is skipped when
+        `params` sets `js_render` or `premium_proxy` (the API rejects either
+        combined with Adaptive Stealth Mode) or its own wire `mode`. Pass
+        `adaptive_stealth=False` to turn it off entirely.
         """
         final_params = dict(params) if params else {}
         final_params["extract"] = mode
-        if adaptive_stealth:
-            final_params["mode"] = "auto"
+        _with_adaptive_stealth(final_params, adaptive_stealth)
         response = self.fetch(url, final_params, headers, **kwargs)
 
-        if (
-            response.status_code == 402
-            and mode == "auto"
-            and fallback_to_autoparse
-            and _is_auth010(response)
-        ):
+        if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response):
             autoparse_params = dict(params) if params else {}
             autoparse_params["autoparse"] = True
-            if adaptive_stealth:
-                autoparse_params["mode"] = "auto"
+            _with_adaptive_stealth(autoparse_params, adaptive_stealth)
             return self.fetch(url, autoparse_params, headers, **kwargs)
 
         return response
@@ -214,24 +263,17 @@ async def extract_async(
         **kwargs: Any,
     ) -> requests.Response:
         """Async counterpart of `extract()` - see its docstring for the
-        AUTH010 -> Autoparse fallback behavior, `fallback_to_autoparse`, and
-        `adaptive_stealth`."""
+        REQS007 / AUTH010 -> Autoparse fallback behavior,
+        `fallback_to_autoparse`, and `adaptive_stealth`."""
         final_params = dict(params) if params else {}
         final_params["extract"] = mode
-        if adaptive_stealth:
-            final_params["mode"] = "auto"
+        _with_adaptive_stealth(final_params, adaptive_stealth)
         response = await self.fetch_async(url, final_params, headers, **kwargs)
 
-        if (
-            response.status_code == 402
-            and mode == "auto"
-            and fallback_to_autoparse
-            and _is_auth010(response)
-        ):
+        if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response):
             autoparse_params = dict(params) if params else {}
             autoparse_params["autoparse"] = True
-            if adaptive_stealth:
-                autoparse_params["mode"] = "auto"
+            _with_adaptive_stealth(autoparse_params, adaptive_stealth)
             return await self.fetch_async(url, autoparse_params, headers, **kwargs)
 
         return response
@@ -312,11 +354,33 @@ def _worker(
             final_headers["Connection"] = None
             final_headers.update(headers)
 
-        return self.requests_session.request(
+        response = self.requests_session.request(
             method,
             self.api_url,
             params=final_params,
             headers=final_headers,
             data=data,
             **kwargs,
         )
+        # Retry a 422 here rather than in urllib3, which can't read the body:
+        # Extract's `no_data_extracted` is a final answer, any other 422 is a
+        # failure to get the target's content and is worth another try.
+        attempt = 0
+        while (
+            response.status_code == 422
+            and attempt < self.retries
+            and not is_no_data_extracted(response)
+        ):
+            attempt += 1
+            # Same schedule as urllib3's Retry: no wait before the first retry.
+            if attempt > 1:
+                time.sleep(_RETRY_BACKOFF_FACTOR * 2 ** (attempt - 1))
+            response = self.requests_session.request(
+                method,
+                self.api_url,
+                params=final_params,
+                headers=final_headers,
+                data=data,
+                **kwargs,
+            )
+        return response