diff --git a/.gitignore b/.gitignore index f4ae267..68d3707 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ pformat.txt profile_slow.prof .pytest_cache/ *.zip +/_src +/.commandcode diff --git a/CLAUDE.md b/CLAUDE.md index e8d3413..14fcc1f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,7 +52,7 @@ Three-layer architecture: ``` Layer 1: Transport + Auth (newapi/api_client/) RequestsHandler (retry/CSRF/maxlag) -> WikiLoginClient (auth, cookies, pagination) - Exceptions: WikiClientError -> LoginError, CSRFError, MaxlagError, MaxRetriesExceeded, CookieError + Exceptions: WikiClientError -> LoginError, CSRFError, MaxlagError, MaxRetriesExceededError, CookieError Layer 2: Wiki Business Logic (newapi/client_wiki/) AllAPIS (main facade) -> MainPage (page ops), CategoryDepth (recursive traversal), NewApi (bulk ops) @@ -95,7 +95,7 @@ Sessions persisted as Mozilla-format cookie jar files. Auto-expire after 3 days. ### Error Handling Two parallel exception hierarchies: -1. `api_client/exceptions.py`: `WikiClientError` -> `LoginError`, `CSRFError`, `MaxlagError`, `MaxRetriesExceeded`, `CookieError` +1. `api_client/exceptions.py`: `WikiClientError` -> `LoginError`, `CSRFError`, `MaxlagError`, `MaxRetriesExceededError`, `CookieError` 2. `core/exceptions.py`: `NewApiException` -> `ApiError` -> `AbuseFilterError`, `MaxLagError`, `ArticleExistsError`, `ProtectedPageError`, etc. Includes `parse_api_error()` for mapping API error dicts. ### Patterns to Know diff --git a/PROJECT_AUDIT_REPORT.md b/PROJECT_AUDIT_REPORT.md index 649408c..4005b09 100644 --- a/PROJECT_AUDIT_REPORT.md +++ b/PROJECT_AUDIT_REPORT.md @@ -115,7 +115,7 @@ The codebase has **9 crash-level bugs** in common code paths, **1 SQL injection 1. **God classes**: `NewApi` (1300 lines), `MainPage` (1000 lines), `WikiLoginClient` (850 lines) 2. **Code duplication**: `client_request_retry` is a copy-paste of `_client_request` -3. **Unused abstractions**: `core/exceptions.py` hierarchy defined but never used; `MaxRetriesExceeded` and `CookieError` defined but never raised +3. **Unused abstractions**: `core/exceptions.py` hierarchy defined but never used; `MaxRetriesExceededError` and `CookieError` defined but never raised 4. **Hardcoded values**: `BOT_USERNAME = "Mr.Ibrahembot"`, `mdwiki.org` special case, Arabic `"قالب:"` prefix 5. **Commented-out code**: Scattered across `db_bot.py`, `bot_api.py`, `page.py` 6. **Outdated config**: `pyproject.toml` references `src_paths = "ArWikiCats"` (non-existent project) diff --git a/newapi/DB_bots/db_bot.py b/newapi/DB_bots/db_bot.py index 19df588..335ccc5 100644 --- a/newapi/DB_bots/db_bot.py +++ b/newapi/DB_bots/db_bot.py @@ -33,7 +33,7 @@ def create_table(self, table_name: str, fields: Dict[str, Any], pk: str = "id", def query(self, sql: str) -> List[tuple]: # return self.db.query(sql) - return [r for r in self.db.execute(sql).fetchall()] + return list(self.db.execute(sql).fetchall()) def update(self, sql: str) -> None: self.db.executescript(sql) diff --git a/newapi/DB_bots/pymysql_bot.py b/newapi/DB_bots/pymysql_bot.py index 4b9e8fb..4306a50 100644 --- a/newapi/DB_bots/pymysql_bot.py +++ b/newapi/DB_bots/pymysql_bot.py @@ -15,12 +15,16 @@ def sql_connect_pymysql( query, return_dict: bool = False, values=None, - main_args={}, - credentials={}, + main_args=None, + credentials=None, conversions=None, many: bool = False, **kwargs, ): + if credentials is None: + credentials = {} + if main_args is None: + main_args = {} args = copy.deepcopy(main_args) args["cursorclass"] = pymysql.cursors.DictCursor if return_dict else pymysql.cursors.Cursor if conversions: diff --git a/newapi/api_client/README.md b/newapi/api_client/README.md index 9f4a5d5..ebe9321 100644 --- a/newapi/api_client/README.md +++ b/newapi/api_client/README.md @@ -6,18 +6,18 @@ The `api_client` package provides a robust, authenticated HTTP client for the Me ### Main Modules -| Module | Purpose | -|---|---| -| `client.py` | Core client classes: `RequestsHandler` (retry/transport), `CookiesClient` (cookie I/O), `WikiLoginClient` (business layer) | -| `cookies.py` | Cookie file path resolution, staleness checks, and cleanup | -| `exceptions.py` | Custom exception hierarchy rooted at `WikiClientError` | +| Module | Purpose | +| --------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `client.py` | Core client classes: `RequestsHandler` (retry/transport), `CookiesClient` (cookie I/O), `WikiLoginClient` (business layer) | +| `cookies.py` | Cookie file path resolution, staleness checks, and cleanup | +| `exceptions.py` | Custom exception hierarchy rooted at `WikiClientError` | ### Technologies & Dependencies -- **`mwclient`** -- MediaWiki client library (provides `Site` and session management) -- **`requests`** -- HTTP transport (used internally by mwclient) -- **`http.cookiejar.LWPCookieJar`** -- Mozilla-format cookie persistence -- **Standard library:** `copy`, `logging`, `os`, `stat`, `time`, `datetime`, `pathlib` +- **`mwclient`** -- MediaWiki client library (provides `Site` and session management) +- **`requests`** -- HTTP transport (used internally by mwclient) +- **`http.cookiejar.LWPCookieJar`** -- Mozilla-format cookie persistence +- **Standard library:** `copy`, `logging`, `os`, `stat`, `time`, `datetime`, `pathlib` --- @@ -33,10 +33,10 @@ The package follows a layered architecture: ### Design Patterns -- **Template Method Pattern**: `RequestsHandler` defines the retry loop skeleton and calls abstract hooks (`_session`, `_refresh_csrf_token`, `_on_assertnameduserfailed`) that `WikiLoginClient` implements. -- **Mixin / Multiple Inheritance**: `WikiLoginClient` inherits from both `CookiesClient` and `RequestsHandler`. -- **Facade**: Wraps `mwclient.Site` and `requests.Session` behind a simplified interface. -- **Strategy-like error handling**: The retry loop dispatches on error code to distinct handlers (CSRF, maxlag, assertnameduserfailed, ratelimited). +- **Template Method Pattern**: `RequestsHandler` defines the retry loop skeleton and calls abstract hooks (`_session`, `_refresh_csrf_token`, `_on_assertnameduserfailed`) that `WikiLoginClient` implements. +- **Mixin / Multiple Inheritance**: `WikiLoginClient` inherits from both `CookiesClient` and `RequestsHandler`. +- **Facade**: Wraps `mwclient.Site` and `requests.Session` behind a simplified interface. +- **Strategy-like error handling**: The retry loop dispatches on error code to distinct handlers (CSRF, maxlag, assertnameduserfailed, ratelimited). ### Maintainability @@ -54,21 +54,21 @@ The client is single-threaded with no connection pooling configuration beyond wh ## Strengths -- **Robust retry logic**: Handles CSRF token invalidation, maxlag, rate limiting, and session expiry with configurable retry counts and exponential backoff. -- **Cookie persistence**: Sessions survive process restarts via LWPCookieJar with automatic staleness invalidation (3-day max age). -- **Write-action safety**: Automatically injects `bot=1` and `assertuser` parameters for mutating API requests. -- **Clean exception hierarchy**: Well-structured custom exceptions with a common base class. -- **Continuation pagination**: `post_continue` handles multi-page API responses transparently. +- **Robust retry logic**: Handles CSRF token invalidation, maxlag, rate limiting, and session expiry with configurable retry counts and exponential backoff. +- **Cookie persistence**: Sessions survive process restarts via LWPCookieJar with automatic staleness invalidation (3-day max age). +- **Write-action safety**: Automatically injects `bot=1` and `assertuser` parameters for mutating API requests. +- **Clean exception hierarchy**: Well-structured custom exceptions with a common base class. +- **Continuation pagination**: `post_continue` handles multi-page API responses transparently. --- ## Weaknesses -- **Code duplication**: `client_request_retry` (lines 693-766) is a copy-paste of `_client_request` logic. Any bug fix in one must be manually replicated in the other. -- **Unused exceptions**: `MaxRetriesExceeded` and `CookieError` are defined and exported but never raised anywhere in the codebase. -- **Shadowed builtins**: The `max` parameter in `post_continue` shadows Python's built-in `max()`. -- **Hardcoded domain workaround**: `mdwiki.org` special case baked into the general-purpose client (line 442). -- **Unused `**kwargs`**: All three request methods accept `**kwargs` but never pass them downstream. +- **Code duplication**: `client_request_retry` (lines 693-766) is a copy-paste of `_client_request` logic. Any bug fix in one must be manually replicated in the other. +- **Unused exceptions**: `MaxRetriesExceededError` and `CookieError` are defined and exported but never raised anywhere in the codebase. +- **Shadowed builtins**: The `max` parameter in `post_continue` shadows Python's built-in `max()`. +- **Hardcoded domain workaround**: `mdwiki.org` special case baked into the general-purpose client (line 442). +- **Unused `**kwargs`**: All three request methods accept `\*\*kwargs` but never pass them downstream. --- @@ -116,26 +116,28 @@ Uses `client_request_safe` for continuation pages. If a continuation page fails, ## Areas That Need Attention -- **Missing HTTP timeout**: Add `timeout=(connect, read)` to all HTTP requests. -- **Missing `__init__.py` exports**: `_delete_cookie_file` is a private-underscore function imported across modules -- rename or make public. -- **No context manager**: The client holds resources (`requests.Session`, cookie jar) but provides no `__enter__`/`__exit__` for clean release. -- **No abstract base class**: `RequestsHandler` uses `raise NotImplementedError` instead of `abc.ABC` with `@abstractmethod`. -- **No input validation**: `lang` and `family` parameters are not validated -- empty strings or special characters produce malformed URLs. -- **No structured error context**: Exceptions carry only a message string. Adding `error_code`, `url`, `attempt_count` fields would aid debugging. -- **f-string in logger call** (line 575): Inconsistent with the rest of the module's `%s` formatting and defeats lazy evaluation. +- **Missing HTTP timeout**: Add `timeout=(connect, read)` to all HTTP requests. +- **Missing `__init__.py` exports**: `_delete_cookie_file` is a private-underscore function imported across modules -- rename or make public. +- **No context manager**: The client holds resources (`requests.Session`, cookie jar) but provides no `__enter__`/`__exit__` for clean release. +- **No abstract base class**: `RequestsHandler` uses `raise NotImplementedError` instead of `abc.ABC` with `@abstractmethod`. +- **No input validation**: `lang` and `family` parameters are not validated -- empty strings or special characters produce malformed URLs. +- **No structured error context**: Exceptions carry only a message string. Adding `error_code`, `url`, `attempt_count` fields would aid debugging. +- **f-string in logger call** (line 575): Inconsistent with the rest of the module's `%s` formatting and defeats lazy evaluation. --- ## Improvement Plan ### Quick Wins + 1. Increment `attempt` in the `ratelimited` handler to prevent infinite loops. 2. Add `timeout` parameter to `_execute_request`. 3. Remove `client_request_retry` duplication -- delegate to `_client_request`. -4. Raise unused exceptions (`MaxRetriesExceeded`, `CookieError`) or remove them. +4. Raise unused exceptions (`MaxRetriesExceededError`, `CookieError`) or remove them. 5. Fix the f-string logger call to use `%s` formatting. ### Medium-Term Improvements + 1. Implement `_ensure_logged_in` to actually trigger login when cookies are absent. 2. Add `__enter__`/`__exit__` context manager support. 3. Use `abc.ABC` and `@abstractmethod` for `RequestsHandler`. @@ -144,6 +146,7 @@ Uses `client_request_safe` for continuation pages. If a continuation page fails, 6. Make the `mdwiki.org` workaround configurable. ### Long-Term Refactoring + 1. Split `client.py` into separate modules: `transport.py`, `auth.py`, `pagination.py`. 2. Add structured error context to all exceptions. 3. Configure `urllib3.util.retry.Retry` and `HTTPAdapter` at the transport level. @@ -154,10 +157,10 @@ Uses `client_request_safe` for continuation pages. If a continuation page fails, ## Comprehensive Review -| Metric | Score | -|---|---| -| **Overall Rating** | **6.5/10** | -| **Production Readiness** | Moderate -- functional but has infinite loop bug and missing timeouts | -| **Technical Debt** | Medium -- code duplication, unused exceptions, hardcoded workarounds | -| **Risk Assessment** | Medium-High -- the ratelimited infinite loop and missing timeouts are production risks | -| **Maintainability** | 6/10 -- clear architecture but duplicated code and 850+ line file | +| Metric | Score | +| ------------------------ | -------------------------------------------------------------------------------------- | +| **Overall Rating** | **6.5/10** | +| **Production Readiness** | Moderate -- functional but has infinite loop bug and missing timeouts | +| **Technical Debt** | Medium -- code duplication, unused exceptions, hardcoded workarounds | +| **Risk Assessment** | Medium-High -- the ratelimited infinite loop and missing timeouts are production risks | +| **Maintainability** | 6/10 -- clear architecture but duplicated code and 850+ line file | diff --git a/newapi/api_client/__init__.py b/newapi/api_client/__init__.py index 590f9e7..bae6092 100644 --- a/newapi/api_client/__init__.py +++ b/newapi/api_client/__init__.py @@ -1,24 +1,23 @@ -# api_client/__init__.py -# Public surface of the package. -# Import WikiLoginClient and all exception types from here. - from .client import WikiLoginClient +from .cookies_client import CookiesClient from .exceptions import ( CookieError, CSRFError, LoginError, MaxlagError, - MaxRetriesExceeded, + MaxRetriesExceededError, WikiClientError, ) +from .requests_handler import RequestsHandler __all__ = [ "WikiLoginClient", - # Exceptions + "RequestsHandler", + "CookiesClient", "WikiClientError", "LoginError", "CSRFError", "MaxlagError", - "MaxRetriesExceeded", + "MaxRetriesExceededError", "CookieError", ] diff --git a/newapi/api_client/client.py b/newapi/api_client/client.py index 44feac3..91184e7 100644 --- a/newapi/api_client/client.py +++ b/newapi/api_client/client.py @@ -32,30 +32,17 @@ from __future__ import annotations import copy -import http.cookiejar import logging -import time -from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, Callable, Union import mwclient import mwclient.errors -import requests - -from ..config import settings -from .cookies import ( - _delete_cookie_file, - get_cookie_path, -) -from .exceptions import ( - CSRFError, - LoginError, - MaxlagError, - WikiClientError, -) -logger = logging.getLogger(__name__) +from .cookies_client import CookiesClient +from .exceptions import LoginError, WikiClientError +from .requests_handler import RequestsHandler +logger = logging.getLogger(__name__) skip_log_params = [ "token", @@ -63,308 +50,13 @@ "lgpassword", "text", ] -# --------------------------------------------------------------------------- -# RequestsHandler — transport + retry layer -# --------------------------------------------------------------------------- - - -class RequestsHandler: - """ - Owns a ``requests.Session`` and drives every HTTP call through a unified - retry loop that handles: - - - CSRF / bad token → refresh token, reinject, retry - - maxlag → exponential back-off, retry - - assertnameduserfailed → delegate re-login hook, retry - - Subclasses must supply ``_session`` (a ``requests.Session``) and may - override ``_on_assertnameduserfailed`` to implement session recovery. - """ - - # ------------------------------------------------------------------ - # Abstract-ish contract that subclasses must satisfy - # ------------------------------------------------------------------ - - @property - def _session(self) -> requests.Session: - """The live ``requests.Session``. Subclasses must assign this.""" - raise NotImplementedError # pragma: no cover - - def _refresh_csrf_token(self) -> str: - """ - Fetch and return a fresh CSRF token. - Subclasses override this to call ``site.get_token("csrf", force=True)``. - """ - raise NotImplementedError # pragma: no cover - - def _on_assertnameduserfailed(self) -> None: - """ - Called when the API returns ``assertnameduserfailed``. - Subclasses implement session-recovery logic (re-login, cookie reset). - """ - raise NotImplementedError # pragma: no cover - - # ------------------------------------------------------------------ - # Core request execution — the only method that touches the network - # ------------------------------------------------------------------ - - def _execute_request( - self, - method: str, - url: str, - *, - params: Optional[dict] = None, - data: Optional[dict] = None, - files: Optional[Any] = None, - ) -> requests.Response: - """ - Send one HTTP request through the session with no retry logic. - Returns the raw ``requests.Response``. - """ - return self._session.request( - method, - url, - params=params, - data=data, - files=files, - ) - - # ------------------------------------------------------------------ - # Retry loop (called by WikiLoginClient.client_request) - # ------------------------------------------------------------------ - - def _request_with_retry( - self, - method: str, - url: str, - *, - params: Optional[dict] = None, - data: Optional[dict] = None, - files: Optional[Any] = None, - assertnameduser_retries: int = 1, - ) -> dict: - """ - Execute a request and automatically retry on transient API errors. - - Retry conditions (each counted against ``settings.api_client.max_retries``): - - CSRF / bad token → ``_handle_csrf`` → inject new token, retry - - maxlag → ``_handle_maxlag`` → sleep, retry - - assertnameduserfailed → ``_on_assertnameduserfailed`` → retry once - - All other errors bubble up unchanged. - - Returns: - Parsed JSON response dict. - - Raises: - CSRFError, MaxlagError: after exhausting retries. - WikiClientError: on assertnameduserfailed after recovery. - requests.HTTPError: on non-2xx HTTP status. - """ - # Mutable copies so per-retry mutations (token reinject) stay local - # to this call and don't bleed into the caller's dict. - working_params = dict(params) if params else {} - working_data = dict(data) if data else {} - - attempt = 0 - named_user_attempts = 0 - - while attempt < settings.api_client.max_retries: - response = self._execute_request( - method, - url, - params=working_params or None, - data=working_data or None, - files=files, - ) - response.raise_for_status() - - # Non-JSON responses (e.g. uploads returning HTML) go straight back - content_type = response.headers.get("Content-Type", "") - if "application/json" not in content_type: - return {} - - try: - body: dict = response.json() - except ValueError: - return {} - - error = body.get("error", {}) - if not error: - return body # ← happy path - - error_code: str = error.get("code", "") - error_info: str = error.get("info", "") - - # ── CSRF ────────────────────────────────────────────────────── - if self._is_csrf_error(error_code, error_info): - attempt += 1 - if attempt >= settings.api_client.max_retries: - raise CSRFError( - f"CSRF token remained invalid after {settings.api_client.max_retries} " - f"attempts. Last error: {error_info or error_code}" - ) - working_data, working_params = self._handle_csrf( - error_code, error_info, attempt, working_data, working_params - ) - continue - - # ── maxlag ──────────────────────────────────────────────────── - if error_code == "maxlag": - attempt += 1 - if attempt >= settings.api_client.max_retries: - raise MaxlagError(f"Server maxlag not resolved after {settings.api_client.max_retries} attempts.") - self._handle_maxlag(response, attempt) - continue - - # ── assertnameduserfailed ───────────────────────────────────── - if error_code == "assertnameduserfailed": - if named_user_attempts >= assertnameduser_retries: - raise WikiClientError("assertnameduserfailed persists after re-login") - named_user_attempts += 1 - logger.warning( - "assertnameduserfailed — attempting recovery (try %d/%d)", - named_user_attempts, - assertnameduser_retries, - ) - self._on_assertnameduserfailed() - # Reset the retry counter so maxlag/csrf budget is fresh - attempt = 0 - continue - - # ── ratelimited ─────────────────────────────────────────────── - if error_code == "ratelimited": - sleep_time = 3 - time.sleep(sleep_time) - logger.warning("ratelimited — sleeping for %d seconds before retrying", sleep_time) - continue - # ── any other error — let the caller decide ─────────────────── - raise WikiClientError(f"API error {error_code}: {error_info}") - - raise MaxlagError(f"Exceeded {settings.api_client.max_retries} retries without a successful response.") - - # ------------------------------------------------------------------ - # Protected CSRF helpers - # ------------------------------------------------------------------ - - @staticmethod - def _is_csrf_error(code: str, info: str) -> bool: - return code in ("badtoken", "notoken") or info == "Invalid CSRF token." - - def _handle_csrf( - self, - error_code: str, - error_info: str, - attempt: int, - data: dict, - params: dict, - ) -> tuple[dict, dict]: - """ - Refresh the CSRF token and reinject it into whichever dict carries it. - - Returns updated (data, params) copies — never mutates in place. - """ - - logger.debug( - "CSRF error (%s) — refreshing token (attempt %d/%d)", - error_code or error_info, - attempt, - settings.api_client.max_retries, - ) - try: - new_token = self._refresh_csrf_token() - except Exception as exc: - raise CSRFError(f"Failed to refresh CSRF token: {exc}") from exc - - # Reinject into whichever dict holds the token key - data, params = self._inject_token(new_token, data, params) - return data, params - - @staticmethod - def _inject_token(token: str, data: dict, params: dict) -> tuple[dict, dict]: - """ - Return (data, params) copies with ``token`` updated to *token*. - Only one dict should ever carry the key; we update the first match. - """ - for bucket_name, bucket in (("data", data), ("params", params)): - if "token" in bucket: - bucket = dict(bucket) - bucket["token"] = token - logger.debug("Injected new CSRF token into %s", bucket_name) - if bucket_name == "data": - return bucket, params - return data, bucket - return data, params - - # ------------------------------------------------------------------ - # Protected maxlag helper - # ------------------------------------------------------------------ - - def _handle_maxlag(self, response: requests.Response, attempt: int) -> None: - """ - Sleep for the server-requested delay (or exponential back-off). - """ - retry_after = response.headers.get(settings.api_client.maxlag_header) - try: - delay = float(retry_after) if retry_after is not None else None - except ValueError: - delay = None - - if delay is None: - delay = settings.api_client.backoff_base * (2**attempt) - - logger.debug( - "maxlag — sleeping %.1f s (attempt %d/%d)", - delay, - attempt, - settings.api_client.max_retries, - ) - time.sleep(delay) - - -# --------------------------------------------------------------------------- -# CookiesClient — isolated cookie I/O (unchanged from original) -# --------------------------------------------------------------------------- - - -class CookiesClient: - """Static helpers for loading and persisting LWP cookie jars.""" - - @staticmethod - def save_cookies(cj: http.cookiejar.LWPCookieJar) -> None: - """ - Persist the current session cookies to disk immediately. - - Called automatically after every login, but you can call this manually - to checkpoint the session after a long batch of writes. - """ - try: - # Save cookies to disk, ignoring discard and expire attributes - cj.save(ignore_discard=True, ignore_expires=True) - # Log successful cookie save operation - logger.debug("Cookies saved to _cookie_path") - except Exception: - # Log any exceptions that occur during cookie saving - logger.exception("Failed to save cookies") - - @staticmethod - def _make_cookiejar(cookie_path: Path) -> http.cookiejar.LWPCookieJar: - # Create a new LWPCookieJar instance with the specified path - cj = http.cookiejar.LWPCookieJar(cookie_path) - if cookie_path.exists(): - try: - cj.load(ignore_discard=True, ignore_expires=True) - except Exception as exc: - logger.error("Error loading cookies: %s", exc) - return cj - # --------------------------------------------------------------------------- # WikiLoginClient — business layer # --------------------------------------------------------------------------- -class WikiLoginClient(CookiesClient, RequestsHandler): +class WikiLoginClient: """ A thin wrapper around ``mwclient.Site`` that: @@ -379,8 +71,7 @@ class WikiLoginClient(CookiesClient, RequestsHandler): Usage:: - client = WikiLoginClient(lang="en", family="wikipedia", - username="MyBot", password="s3cr3t") + client = WikiLoginClient(lang="en", family="wikipedia", username="MyBot", password="s3cr3t") data = client.client_request({"action": "query", "titles": "Python"}) """ @@ -412,8 +103,11 @@ def __init__( family: str, username: str, password: str, - cookies_dir: str | None = settings.paths.cookies_dir, - use_cookies: bool = True, + cookies_dir: str | None = None, + use_cookies: None | bool = None, + max_retries: int = 5, + backoff_base: int = 1, + maxlag_header: str = "Retry-After", ) -> None: """ Initialise the client, load any saved cookies, and ensure the session @@ -432,15 +126,11 @@ def __init__( self.username = username self._password = password # kept private — never log or expose this - # ── Cookie path ──────────────────────────────────────────────────── - self._cookie_path = None - self.use_cookies = use_cookies + self.cookies_client = CookiesClient(lang, family, username, cookies_dir, use_cookies) + # ── mwclient Site ────────────────────────────────────────────────── logger.debug("Creating mwclient.Site for %s.%s.org", lang, family) - self.api_url = f"https://{self.lang}.{self.family}.org/w/api.php" - - if self.api_url == "https://www.mdwiki.org/w/api.php": - self.api_url = "https://mdwiki.org/w/api.php" + self.api_url = self._make_api_url() try: self._site = mwclient.Site(f"{self.lang}.{self.family}.org", do_init=False) @@ -449,11 +139,8 @@ def __init__( # ── Inject saved cookies ─────────────────────────────────────────── # mwclient stores its requests.Session at site.connection. - self.cj = None - if self.use_cookies: - self._cookie_path: Path = get_cookie_path(cookies_dir or settings.paths.cookies_dir, family, lang, username) - self.cj = self._make_cookiejar(self._cookie_path) - self._site.connection.cookies = self.cj + + self.cookies_client.set_site_cookies(self._site) # ── Wrap the session with retry / CSRF / maxlag logic ────────────── # wrap_session(self._site.connection, self._site) @@ -461,19 +148,25 @@ def __init__( # ── Authenticate if necessary ────────────────────────────────────── self._ensure_logged_in() + self.requests_handler = RequestsHandler( + max_retries=max_retries, + backoff_base=backoff_base, + maxlag_header=maxlag_header, + _site=self._site, + on_assertnameduserfailed=self._on_assertnameduserfailed, + ) + + def _make_api_url(self) -> str: + api_url = f"https://{self.lang}.{self.family}.org/w/api.php" + + if api_url == "https://www.mdwiki.org/w/api.php": + api_url = "https://mdwiki.org/w/api.php" + return api_url + # ------------------------------------------------------------------ # RequestsHandler contract — concrete implementations # ------------------------------------------------------------------ - @property - def _session(self) -> requests.Session: - """The mwclient-managed session.""" - return self._site.connection - - def _refresh_csrf_token(self) -> str: - """Force mwclient to fetch a fresh CSRF token from the server.""" - return self._site.get_token("csrf", force=True) - def _on_assertnameduserfailed(self) -> None: """ Session expired mid-run: nuke stale cookies and re-authenticate. @@ -485,8 +178,7 @@ def _on_assertnameduserfailed(self) -> None: self.lang, self.family, ) - if self.use_cookies: - _delete_cookie_file(self._cookie_path, reason="assertnameduserfailed") + self.cookies_client.delete_cookie_file(reason="assertnameduserfailed") self._do_login() # ------------------------------------------------------------------ @@ -506,9 +198,9 @@ def _client_request( self, params: dict, method: str = "post", - files: Optional[Any] = None, + files: Any | None = None, **kwargs, - ) -> dict: + ) -> dict[str, Any]: """ Send a GET or POST request to the wiki API and return parsed JSON. @@ -535,9 +227,12 @@ def _client_request( if method not in ("get", "post"): raise ValueError(f"method must be 'get' or 'post', got {method!r}") + method = method.upper() + # Files can only travel via multipart POST - if files is not None: - method = "post" + action = params.get("action") + if action in self._WRITE_ACTIONS or files is not None: + method = "POST" # Always request JSON and inject write-action safety params params = self._enrich_params({"format": "json", **params}) @@ -550,21 +245,26 @@ def _client_request( {k: ("***" if k in skip_log_params else v) for k, v in params.items()}, list(files.keys()) if files else None, ) - action = params.get("action") - if action in self._WRITE_ACTIONS: - method = "post" - if method == "get": - return self._request_with_retry("GET", self.api_url, params=params) - # return self._site.get(action, **params) + # Fetch a CSRF token now if the caller didn't supply one. + # The retry loop will refresh it automatically on CSRF errors. + if method == "POST" and "token" not in params: + params["token"] = self._site.get_token("csrf") + + args = {} + + if method == "GET": + args["params"] = params else: - # Fetch a CSRF token now if the caller didn't supply one. - # The retry loop will refresh it automatically on CSRF errors. - if "token" not in params: - params["token"] = self._site.get_token("csrf") + args["data"] = params + if files: + args["files"] = files - return self._request_with_retry("POST", self.api_url, data=params, files=files) - # return self._site.post(action, **params, files=files) + return self.requests_handler._request_with_retry( + method, + self.api_url, + **args, + ) def _ensure_logged_in(self) -> None: """ @@ -575,20 +275,21 @@ def _ensure_logged_in(self) -> None: logger.info(f"Session already authenticated {self._site.logged_in=}") return - if self.use_cookies: - if self._cookie_path.exists(): - try: - self._site.site_init() - if self._site.logged_in: - logger.info("Revived session via cookies as %s", self._site.username) - return - except Exception: - logger.exception("Error in site_init") + if not self.cookies_client.is_cookie_path_exists(): + return + + try: + self._site.site_init() + if self._site.logged_in: + logger.info("Revived session via cookies as %s", self._site.username) + return + except Exception: + logger.exception("Error in site_init") # if not self._site.logged_in: self._do_login() # don't login yet, user can use login() method - def _enrich_params(self, params: dict) -> dict: + def _enrich_params(self, params: dict) -> dict[str, Any]: """ Inject write-action safety parameters. @@ -635,8 +336,7 @@ def _do_login(self) -> None: self.lang, self.family, ) - if self.use_cookies: - self.save_cookies(self.cj) + self.cookies_client.save_cookies_cj() # ── Public methods ───────────────────────────────────────────────────── @@ -660,9 +360,9 @@ def client_request( self, params: dict, method: str = "post", - files: Optional[Any] = None, + files: Any | None = None, **kwargs, - ) -> dict: + ) -> dict[str, Any]: """ """ return self._client_request( params=params, @@ -675,9 +375,9 @@ def client_request_safe( self, params: dict, method: str = "post", - files: Optional[Any] = None, + files: Any | None = None, **kwargs, - ) -> dict: + ) -> dict[str, Any]: """ """ try: return self._client_request( @@ -694,89 +394,33 @@ def client_request_retry( self, params: dict, method: str = "post", - files: Optional[Any] = None, + files: Any | None = None, **kwargs, - ) -> dict: - """ - Send a GET or POST request to the wiki API and return parsed JSON. - - CSRF tokens, maxlag backoff, and ``assertnameduserfailed`` recovery are - all handled transparently by the ``RequestsHandler`` base class. - - Args: - params: MediaWiki API parameters. ``format`` defaults to ``"json"``. - method: ``"get"`` or ``"post"`` (case-insensitive). - Files automatically force POST. - files: ``{field_name: file-like}`` for multipart uploads. - - Returns: - Parsed JSON response dict. - - Raises: - ValueError: On invalid *method*. - CSRFError: CSRF token invalid after all retries. - MaxlagError: Server maxlag unresolved after all retries. - WikiClientError: On other API-level errors. - requests.HTTPError: On non-2xx HTTP responses. - """ + ) -> dict[str, Any]: + """ """ method = method.lower() if method not in ("get", "post"): raise ValueError(f"method must be 'get' or 'post', got {method!r}") - # Files can only travel via multipart POST - action = params.get("action") - if action in self._WRITE_ACTIONS or files is not None: - method = "post" - - # Always request JSON and inject write-action safety params - params = self._enrich_params({"format": "json", **params}) - - skip_log_params = [ - "token", - "password", - "lgpassword", - "text", - ] - logger.debug( - "%s %s params=%s files=%s", - method.upper(), - self.api_url, - # Never log token values - {k: ("***" if k in skip_log_params else v) for k, v in params.items()}, - list(files.keys()) if files else None, + return self._client_request( + params=params, + method=method, + files=files, + **kwargs, ) - if method == "get": - return self._request_with_retry( - "GET", - self.api_url, - params=params, - ) - else: - # Fetch a CSRF token now if the caller didn't supply one. - # The retry loop will refresh it automatically on CSRF errors. - if "token" not in params: - params["token"] = self._site.get_token("csrf") - - return self._request_with_retry( - "POST", - self.api_url, - data=params, - files=files, - ) - def post_continue( self, params: dict, action: str, - _p_: str = "pages", - p_empty: Optional[Union[list, dict]] = None, - max: int = 500_000, - first: bool = False, - _p_2: str = "", - _p_2_empty: Optional[Union[list, dict]] = None, + _p_: str | None = None, + p_empty: Union[list, dict] | None = None, + max: int | None = None, + first: int | None = None, + _p_2: str | None = None, + _p_2_empty: Union[list, dict] | None = None, **kwargs, - ) -> Union[list, dict]: + ) -> dict[str, Any]: """ Drive a MediaWiki API continuation query to completion. @@ -786,7 +430,7 @@ def post_continue( Args: params: Base API parameters. action: Top-level JSON key to extract results from - (e.g. ``"query"``, ``"wbsearchentities"``). + (e.g. ``"query"``). _p_: Sub-key inside *action* (default ``"pages"``). p_empty: Seed value for the accumulator (list or dict). max: Stop accumulating after this many results. @@ -797,7 +441,7 @@ def post_continue( Returns: Accumulated results as a list or dict, depending on *p_empty*. """ - logger.debug("post_continue start. action=%s _p_=%s", action, _p_) + logger.debug("action=%s _p_=%s", action, _p_) if isinstance(max, str) and max.isdigit(): max = int(max) @@ -819,7 +463,7 @@ def post_continue( logger.debug("Applying continue_params: %s", continue_params) page_params.update(continue_params) - body = self.client_request_safe(page_params) + body = self.client_request(page_params) if not body: logger.debug("empty response, stopping") @@ -859,11 +503,77 @@ def post_continue( logger.debug("done, %d total results", len(results)) return results + def post_continue_list( + self, + params: dict, + action: str, + _load_data: Callable, + max: int | None = None, + ) -> list[Any]: + """ + Drive a MediaWiki API continuation query to completion. + + Iterates the ``continue`` token until all pages are fetched or *max* + results have been collected. + + Args: + params: Base API parameters. + action: Top-level JSON key to extract results from + (e.g. ``"query"``). + max: Stop accumulating after this many results. + + Returns: + Accumulated results as a list + """ + logger.debug("action=%s", action) + + if isinstance(max, str) and max.isdigit(): + max = int(max) + if max == 0: + max = 500_000 + if max is None: + max = 500_000 + results = [] + continue_params: dict = {} + iterations = 0 + + while continue_params or iterations == 0: + page_params = copy.deepcopy(params) + iterations += 1 + + if continue_params: + logger.debug("Applying continue_params: %s", continue_params) + page_params.update(continue_params) + + body = self.client_request(page_params) + + if not body: + logger.debug("empty response, stopping") + break + + continue_params = body.get("continue", {}) + + data = _load_data(body) + + if not data: + logger.debug("no data in response, stopping") + break + + logger.debug("+%d items (total %d)", len(data), len(results)) + + if len(results) >= max: + logger.debug("max=%d reached, stopping", max) + break + + results.extend(data) + + logger.debug("done, %d total results", len(results)) + return results + def __repr__(self) -> str: return f"WikiLoginClient(lang={self.lang!r}, family={self.family!r}, username={self.username!r})" __all__ = [ - "RequestsHandler", "WikiLoginClient", ] diff --git a/newapi/api_client/cookies.py b/newapi/api_client/cookies.py deleted file mode 100644 index 13d68c0..0000000 --- a/newapi/api_client/cookies.py +++ /dev/null @@ -1,97 +0,0 @@ -# api_client/cookies.py -# Pure functions for loading and saving a MozillaCookieJar. -# No class, no state — compose with anything that holds a requests.Session. - -import logging -import os -import stat -from datetime import datetime, timedelta -from pathlib import Path - -logger = logging.getLogger(__name__) - -# Cookie files older than this are treated as stale and deleted before loading. -_COOKIE_MAX_AGE_DAYS = 3 - - -def get_cookie_path( - cookies_dir: str, - family: str, - lang: str, - username: str, -) -> Path: - """ - Return the cookie file path for the given site + user combination. - - Base directory resolution order (mirrors your old cookies_bot.py): - 1. *cookies_dir* if explicitly passed. - 2. $HOME/cookies/ if the HOME env var is set. - 3. A cookies/ folder next to this file as a last resort. - - Convention: {cookies_dir}/{family}_{lang}_{username}.mozilla - Example: ~/cookies/wikipedia_en_mybot.mozilla - - The directory is created if it does not already exist. - Normalisation: family, lang, and the base part of username are lowercased; - spaces replaced with underscores; bot-password suffix (@...) stripped. - """ - # ── Resolve base directory ───────────────────────────────────────────── - base = Path(cookies_dir) - - base.mkdir(parents=True, exist_ok=True) - - # Set group-readable permissions on the directory (matches old chmod logic) - try: - os.chmod(base, stat.S_IRWXU | stat.S_IRWXG) - except OSError as exc: - logger.debug("Could not chmod cookies dir %s: %s", base, exc) - - logger.info("cookie path: %s", base) - - # ── Normalise filename components ────────────────────────────────────── - family = family.lower() - lang = lang.lower() - # Strip bot-password suffix (e.g. "MyBot@BotPassword" -> "mybot") - username = username.lower().replace(" ", "_").split("@")[0] - - file_path = base / f"{family}_{lang}_{username}.mozilla" - logger.debug("resolved cookie file: %s", file_path) - - # ── Stale / empty file guard (from your check_if_file_is_old) ───────── - _delete_if_stale(file_path) - file_path.parent.mkdir(parents=True, exist_ok=True) - return file_path - - -def _delete_if_stale(path: Path) -> None: - """ - Delete the cookie file if it is zero-bytes or older than _COOKIE_MAX_AGE_DAYS. - - Silently does nothing if the file does not exist. - """ - if not path.exists(): - return - - # Zero-byte file is useless - if path.stat().st_size == 0: - _delete_cookie_file(path, reason="zero-byte file") - return - - # File too old — the session it contains has almost certainly expired - age = datetime.now() - datetime.fromtimestamp(path.stat().st_mtime) - if age > timedelta(days=_COOKIE_MAX_AGE_DAYS): - _delete_cookie_file(path, reason=f"older than {_COOKIE_MAX_AGE_DAYS} days ({age.days}d)") - - -def _delete_cookie_file(path: Path, reason: str = "") -> None: - """Delete a cookie file, logging the outcome.""" - try: - path.unlink(missing_ok=True) - logger.debug("Deleted stale cookie file %s (%s)", path, reason) - except OSError as exc: - logger.exception("Could not delete cookie file %s: %s", path, exc) - - -__all__ = [ - "get_cookie_path", -] diff --git a/newapi/api_client/cookies_client.py b/newapi/api_client/cookies_client.py new file mode 100644 index 0000000..f1a13a1 --- /dev/null +++ b/newapi/api_client/cookies_client.py @@ -0,0 +1,200 @@ +""" """ + +from __future__ import annotations + +import http.cookiejar +import logging +import os +import stat +from datetime import datetime, timedelta +from pathlib import Path + +import mwclient + +logger = logging.getLogger(__name__) + +# Cookie files older than this are treated as stale and deleted before loading. +_COOKIE_MAX_AGE_DAYS = 3 + + +def get_cookies_dir() -> str: + """Load configuration from environment variables.""" + cookies_dir = os.getenv("COOKIES_DIR") or "~/tmp/cookies" + cookies_dir = os.path.expandvars(cookies_dir) + try: + cookies_dir = Path(str(cookies_dir)).expanduser() + except Exception as e: + logger.error(f"Error expanding cookies directory: {e}") + + return str(cookies_dir) + + +def get_cookie_path( + cookies_dir: str | None, + family: str, + lang: str, + username: str, +) -> Path: + """ + Return the cookie file path for the given site + user combination. + + Base directory resolution order (mirrors your old cookies_bot.py): + 1. *cookies_dir* if explicitly passed. + 2. $HOME/cookies/ if the HOME env var is set. + 3. A cookies/ folder next to this file as a last resort. + + Convention: {cookies_dir}/{family}_{lang}_{username}.mozilla + Example: ~/cookies/wikipedia_en_mybot.mozilla + + The directory is created if it does not already exist. + Normalisation: family, lang, and the base part of username are lowercased; + spaces replaced with underscores; bot-password suffix (@...) stripped. + """ + # ── Resolve base directory ───────────────────────────────────────────── + if cookies_dir is None: + cookies_dir = get_cookies_dir() + + base = Path(cookies_dir) + + base.mkdir(parents=True, exist_ok=True) + + # Set group-readable permissions on the directory (matches old chmod logic) + try: + os.chmod(base, stat.S_IRWXU | stat.S_IRWXG) + except OSError as exc: + logger.debug("Could not chmod cookies dir %s: %s", base, exc) + + logger.info("cookie path: %s", base) + + # ── Normalise filename components ────────────────────────────────────── + family = family.lower() + lang = lang.lower() + # Strip bot-password suffix (e.g. "MyBot@BotPassword" -> "mybot") + username = username.lower().replace(" ", "_").split("@")[0] + + file_path = base / f"{family}_{lang}_{username}.mozilla" + logger.debug("resolved cookie file: %s", file_path) + + # ── Stale / empty file guard (from your check_if_file_is_old) ───────── + _delete_if_stale(file_path) + file_path.parent.mkdir(parents=True, exist_ok=True) + return file_path + + +def _delete_if_stale(path: Path) -> None: + """ + Delete the cookie file if it is zero-bytes or older than _COOKIE_MAX_AGE_DAYS. + + Silently does nothing if the file does not exist. + """ + if not path.exists(): + return + + # Zero-byte file is useless + if path.stat().st_size == 0: + _delete_cookie_file(path, reason="zero-byte file") + return + + # File too old — the session it contains has almost certainly expired + age = datetime.now() - datetime.fromtimestamp(path.stat().st_mtime) + if age > timedelta(days=_COOKIE_MAX_AGE_DAYS): + _delete_cookie_file(path, reason=f"older than {_COOKIE_MAX_AGE_DAYS} days ({age.days}d)") + + +def _delete_cookie_file(path: Path, reason: str = "") -> None: + """Delete a cookie file, logging the outcome.""" + try: + path.unlink(missing_ok=True) + logger.debug("Deleted stale cookie file %s (%s)", path, reason) + except OSError as exc: + logger.exception("Could not delete cookie file %s: %s", path, exc) + + +# --------------------------------------------------------------------------- +# CookiesClient — isolated cookie I/O (unchanged from original) +# --------------------------------------------------------------------------- + + +class CookiesClient: + """Static helpers for loading and persisting LWP cookie jars.""" + + def __init__( + self, + lang: str, + family: str, + username: str, + cookies_dir: str | None, + use_cookies: None | bool = None, + ) -> None: + self.lang = lang + self.family = family + self.username = username + self.cookies_dir = cookies_dir + self.use_cookies = use_cookies + + self.cj = None + self._cookie_path: None | Path = None + if use_cookies: + self._cookie_path = self.get_cookies_path() + self.cj = self._make_cookiejar() + + def is_cookie_path_exists(self) -> bool: + if self._cookie_path: + return self._cookie_path.exists() + + return False + + def delete_cookie_file(self, reason="") -> None: + if self.use_cookies: + self._delete_cookie_file(self._cookie_path, reason=reason) + + def set_site_cookies(self, site: mwclient.Site) -> None: + if self.use_cookies and self.cj: + site.connection.cookies = self.cj # type: ignore + + def save_cookies_cj(self) -> None: + if self.use_cookies and self.cj: + self.save_cookies(self.cj) + + @property + def cookie_path(self) -> None | Path: + return self._cookie_path + + @staticmethod + def save_cookies(cj: http.cookiejar.LWPCookieJar) -> None: + """ + Persist the current session cookies to disk immediately. + + Called automatically after every login, but you can call this manually + to checkpoint the session after a long batch of writes. + """ + try: + # Save cookies to disk, ignoring discard and expire attributes + cj.save(ignore_discard=True, ignore_expires=True) + # Log successful cookie save operation + logger.debug("Cookies saved to _cookie_path") + except Exception: + # Log any exceptions that occur during cookie saving + logger.exception("Failed to save cookies") + + def _make_cookiejar(self) -> http.cookiejar.LWPCookieJar: + # Create a new LWPCookieJar instance with the specified path + cj = http.cookiejar.LWPCookieJar(self._cookie_path) + if self.is_cookie_path_exists(): + try: + cj.load(ignore_discard=True, ignore_expires=True) + except Exception as exc: + logger.error("Error loading cookies: %s", exc) + return cj + + def _delete_cookie_file(self, path: Path, reason: str = "") -> None: + return _delete_cookie_file(path, reason) + + def get_cookies_path(self) -> Path: + return get_cookie_path(self.cookies_dir, self.family, self.lang, self.username) + + +__all__ = [ + "CookiesClient", + "get_cookie_path", +] diff --git a/newapi/api_client/exceptions.py b/newapi/api_client/exceptions.py index 05864d2..2feb8d8 100644 --- a/newapi/api_client/exceptions.py +++ b/newapi/api_client/exceptions.py @@ -18,9 +18,19 @@ class MaxlagError(WikiClientError): """Raised when server maxlag is not resolved after all attempts.""" -class MaxRetriesExceeded(WikiClientError): +class MaxRetriesExceededError(WikiClientError): """Raised when the generic retry cap is hit.""" class CookieError(WikiClientError): """Raised when the cookie file cannot be read or written.""" + + +__all__ = [ + "WikiClientError", + "LoginError", + "CSRFError", + "MaxlagError", + "MaxRetriesExceededError", + "CookieError", +] diff --git a/newapi/api_client/requests_handler.py b/newapi/api_client/requests_handler.py new file mode 100644 index 0000000..5008b3c --- /dev/null +++ b/newapi/api_client/requests_handler.py @@ -0,0 +1,285 @@ +""" """ + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable +from typing import Any + +import mwclient +import requests + +from .exceptions import ( + CSRFError, + MaxlagError, + WikiClientError, +) + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# RequestsHandler — transport + retry layer +# --------------------------------------------------------------------------- + + +class RequestsHandler: + """ + Owns a ``requests.Session`` and drives every HTTP call through a unified + retry loop that handles: + + - CSRF / bad token → refresh token, reinject, retry + - maxlag → exponential back-off, retry + - assertnameduserfailed → delegate re-login hook, retry + + The caller injects an ``on_assertnameduserfailed`` callback at construction + time for session-recovery logic (re-login, cookie reset). + """ + + def __init__( + self, + _site: mwclient.Site, + max_retries: int = 5, + backoff_base: int = 1, + maxlag_header: str = "Retry-After", + on_assertnameduserfailed: Callable[[], None] | None = None, + ) -> None: + self._site = _site + + self.max_retries = max_retries + self.backoff_base = backoff_base + self.maxlag_header = maxlag_header + self._on_assertnameduserfailed = on_assertnameduserfailed or self._on_assertnameduserfailed_default + + # ------------------------------------------------------------------ + # Abstract-ish contract that subclasses must satisfy + # ------------------------------------------------------------------ + + @property + def _session(self) -> requests.Session: + """The mwclient-managed session.""" + return self._site.connection + + def _refresh_csrf_token(self) -> str: + """Force mwclient to fetch a fresh CSRF token from the server.""" + return self._site.get_token("csrf", force=True) + + @staticmethod + def _on_assertnameduserfailed_default() -> None: + raise NotImplementedError # pragma: no cover + + # ------------------------------------------------------------------ + # Retry loop (called by WikiLoginClient.client_request) + # ------------------------------------------------------------------ + + def _request_with_retry( + self, + method: str, + url: str, + *, + params: dict | None = None, + data: dict | None = None, + files: Any | None = None, + assertnameduser_retries: int = 1, + ) -> dict[str, Any]: + """ + Execute a request and automatically retry on transient API errors. + + Retry conditions (each counted against ``settings.mediawiki.max_retries``): + - CSRF / bad token → ``_handle_csrf`` → inject new token, retry + - maxlag → ``_handle_maxlag`` → sleep, retry + - assertnameduserfailed → ``_on_assertnameduserfailed`` → retry once + + All other errors bubble up unchanged. + + Returns: + Parsed JSON response dict. + + Raises: + CSRFError, MaxlagError: after exhausting retries. + WikiClientError: on assertnameduserfailed after recovery. + requests.HTTPError: on non-2xx HTTP status. + """ + # Mutable copies so per-retry mutations (token reinject) stay local + # to this call and don't bleed into the caller's dict. + working_params = dict(params) if params else {} + working_data = dict(data) if data else {} + + attempt = 0 + named_user_attempts = 0 + + while attempt < self.max_retries: + try: + # TODO: handle HTTPError: 429 Client Error: Too Many Requests + response = self._session.request( + method, + url, + params=working_params or None, + data=working_data or None, + files=files, + ) + response.raise_for_status() + except Exception as e: + logger.error("Request failed: %s", e) + raise + + # Non-JSON responses (e.g. uploads returning HTML) go straight back + content_type = response.headers.get("Content-Type", "") + if "application/json" not in content_type: + return {} + + try: + body: dict[str, Any] = response.json() + except ValueError: + return {} + + error = body.get("error", {}) + if not error: + return body # ← happy path + + error_code: str = error.get("code", "") + error_info: str = error.get("info", "") + + # ── CSRF ────────────────────────────────────────────────────── + if self._is_csrf_error(error_code, error_info): + attempt += 1 + if attempt >= self.max_retries: + raise CSRFError( + f"CSRF token remained invalid after {self.max_retries} " + f"attempts. Last error: {error_info or error_code}" + ) + working_data, working_params = self._handle_csrf( + error_code, + error_info, + attempt, + working_data, + working_params, + ) + continue + + # ── maxlag ──────────────────────────────────────────────────── + if error_code == "maxlag": + attempt += 1 + if attempt >= self.max_retries: + raise MaxlagError(f"Server maxlag not resolved after {self.max_retries} attempts.") + self._handle_maxlag(response, attempt) + continue + + # ── assertnameduserfailed ───────────────────────────────────── + if error_code == "assertnameduserfailed": + if named_user_attempts >= assertnameduser_retries: + raise WikiClientError("assertnameduserfailed persists after re-login") + named_user_attempts += 1 + logger.warning( + "assertnameduserfailed — attempting recovery (try %d/%d)", + named_user_attempts, + assertnameduser_retries, + ) + self._on_assertnameduserfailed() + # Reset the retry counter so maxlag/csrf budget is fresh + attempt = 0 + # Refresh CSRF token — the old one is tied to the expired session + try: + new_token = self._refresh_csrf_token() + working_data, working_params = self._inject_token(new_token, working_data, working_params) + except Exception: + logger.debug("Could not refresh CSRF token after re-login — next error will retry") + continue + + # ── ratelimited ─────────────────────────────────────────────── + if error_code == "ratelimited": + attempt += 1 + if attempt >= self.max_retries: + raise WikiClientError(f"Ratelimit persists after {self.max_retries} attempts.") + sleep_time = 3 + time.sleep(sleep_time) + logger.warning("ratelimited — sleeping for %d seconds before retrying", sleep_time) + continue + + # ── any other error — let the caller decide ─────────────────── + raise WikiClientError(f"API error {error_code}: {error_info}") + + raise MaxlagError(f"Exceeded {self.max_retries} retries without a successful response.") + + # ------------------------------------------------------------------ + # Protected CSRF helpers + # ------------------------------------------------------------------ + + @staticmethod + def _is_csrf_error(code: str, info: str) -> bool: + return code in ("badtoken", "notoken") or info == "Invalid CSRF token." + + def _handle_csrf( + self, + error_code: str, + error_info: str, + attempt: int, + data: dict, + params: dict, + ) -> tuple[dict, dict]: + """ + Refresh the CSRF token and reinject it into whichever dict carries it. + + Returns updated (data, params) copies — never mutates in place. + """ + + logger.debug( + "CSRF error (%s) — refreshing token (attempt %d/%d)", + error_code or error_info, + attempt, + self.max_retries, + ) + try: + new_token = self._refresh_csrf_token() + except Exception as exc: + raise CSRFError(f"Failed to refresh CSRF token: {exc}") from exc + + # Reinject into whichever dict holds the token key + data, params = self._inject_token(new_token, data, params) + return data, params + + @staticmethod + def _inject_token(token: str, data: dict, params: dict) -> tuple[dict, dict]: + """ + Return (data, params) copies with ``token`` updated to *token*. + Only one dict should ever carry the key; we update the first match. + """ + for bucket_name, bucket in (("data", data), ("params", params)): + if "token" in bucket: + bucket = dict(bucket) + bucket["token"] = token + logger.debug("Injected new CSRF token into %s", bucket_name) + if bucket_name == "data": + return bucket, params + return data, bucket + return data, params + + # ------------------------------------------------------------------ + # Protected maxlag helper + # ------------------------------------------------------------------ + + def _handle_maxlag(self, response: requests.Response, attempt: int) -> None: + """ + Sleep for the server-requested delay (or exponential back-off). + """ + retry_after = response.headers.get(self.maxlag_header) + try: + delay = float(retry_after) if retry_after is not None else None + except ValueError: + delay = None + + if delay is None: + delay = self.backoff_base * (2**attempt) + + logger.debug( + "maxlag — sleeping %.1f s (attempt %d/%d)", + delay, + attempt, + self.max_retries, + ) + time.sleep(delay) + + +__all__ = [ + "RequestsHandler", +] diff --git a/newapi/client_wiki/all_apis.py b/newapi/client_wiki/all_apis.py index 88e75c3..48847d5 100644 --- a/newapi/client_wiki/all_apis.py +++ b/newapi/client_wiki/all_apis.py @@ -28,12 +28,14 @@ def __init__( username: str, password: str, use_cookies: bool = True, + cookies_dir: None | str = None, ) -> None: self.lang = lang self.family = family self.username = username self.password = password self.use_cookies = use_cookies + self.cookies_dir = cookies_dir self.login_bot = self._login() def MainPage(self, title: str, *args, **kwargs) -> super_page.MainPage: @@ -53,6 +55,7 @@ def _login(self) -> WikiLoginClient: username=self.username, password=self.password, use_cookies=self.use_cookies, + cookies_dir=self.cookies_dir, ) return client diff --git a/newapi/client_wiki/api_utils/bot_edit/bot_edit_by_templates.py b/newapi/client_wiki/api_utils/bot_edit/bot_edit_by_templates.py index ac9b965..83f3839 100644 --- a/newapi/client_wiki/api_utils/bot_edit/bot_edit_by_templates.py +++ b/newapi/client_wiki/api_utils/bot_edit/bot_edit_by_templates.py @@ -34,9 +34,9 @@ def _handle_nobots_template(params, title_page, botjob, _template) -> bool: Bot_Cache[botjob][title_page] = False return False elif params.get("1"): - List = [x.strip() for x in params.get("1", "").split(",")] + list_data = [x.strip() for x in params.get("1", "").split(",")] # if 'all' in List or pywikibot.calledModuleName() in List or BOT_USERNAME in List: - if "all" in List or BOT_USERNAME in List: + if "all" in list_data or BOT_USERNAME in list_data: logger.debug(f"<> botEdit.py: the page has temp:({_template}), botjob:{botjob} skipp.") logger.debug(f"bot {BOT_USERNAME} in nobots list - blocking {title_page}") # Bot_Cache[title_page] = False diff --git a/newapi/client_wiki/api_utils/txtlib.py b/newapi/client_wiki/api_utils/txtlib.py index fdec1dd..d2a2059 100644 --- a/newapi/client_wiki/api_utils/txtlib.py +++ b/newapi/client_wiki/api_utils/txtlib.py @@ -3,6 +3,7 @@ import logging from functools import lru_cache +from typing import Any import wikitextparser as wtp @@ -10,18 +11,22 @@ @lru_cache(maxsize=512) -def extract_templates_and_params(text: str): +def extract_templates_and_params(text: str) -> list[dict[str, Any]]: # --- result = [] # --- + if not text or not isinstance(text, str): + return result + # --- parsed = wtp.parse(text) - templates = parsed.templates - arguments = "arguments" # --- - for template in templates: + for template in parsed.templates: + # --- + if not template: + continue # --- params = {} - for param in getattr(template, arguments): + for param in template.arguments: value = str(param.value) # mwpfh needs upcast to str key = str(param.name) key = key.strip() @@ -29,11 +34,8 @@ def extract_templates_and_params(text: str): # --- name = template.name.strip() # --- - # print('=====') - # --- name = str(template.normal_name()).strip() pa_item = template.string - # logger.info( "<> pa_item: %s" % pa_item ) # --- namestrip = name # --- @@ -47,51 +49,3 @@ def extract_templates_and_params(text: str): result.append(ficrt) # --- return result - - -def get_one_temp_params(text: str, tempname: str = "", templates=[], lowers: bool = False, get_all_temps: bool = False): - ingr = extract_templates_and_params(text) - # --- - temps = templates - # --- - if tempname: - temps.append(tempname) - # --- - temps = [x.replace("قالب:", "").replace("Template:", "").replace("_", " ").strip() for x in temps] - # --- - if lowers: - temps = [x.lower() for x in temps] - # --- - named = {} - # --- - if get_all_temps: - named = [] - # --- - for temp in ingr: - # --- - # name, namestrip, params, template = temp['name'], temp['namestrip'], temp['params'], temp['item'] - namestrip, params = temp["namestrip"], temp["params"] - # --- - if lowers: - namestrip = namestrip.lower() - # --- - if namestrip in temps: - if not get_all_temps: - return params - # --- - # print("te:%s, namestrip:%s" % (te,namestrip) ) - # --- - tabe = {namestrip: params} - named.append(tabe) - # --- - return named - - -def get_all_temps_params(text: str, templates=None, lowers: bool = False): - # --- - if templates is None: - templates = [] - # --- - tab = get_one_temp_params(text, templates=templates, lowers=lowers, get_all_temps=True) - # --- - return tab diff --git a/newapi/client_wiki/api_utils/wd_sparql.py b/newapi/client_wiki/api_utils/wd_sparql.py index c500bfb..2ba8e99 100644 --- a/newapi/client_wiki/api_utils/wd_sparql.py +++ b/newapi/client_wiki/api_utils/wd_sparql.py @@ -60,6 +60,6 @@ def get_query_result(query): # --- data = get_query_data(query) # --- - lista = [x for x in data.get("results", {}).get("bindings", [])] + lista = list(data.get("results", {}).get("bindings", [])) # --- return lista diff --git a/newapi/client_wiki/pages/data.py b/newapi/client_wiki/pages/data.py index 6e7f932..d55c2a7 100644 --- a/newapi/client_wiki/pages/data.py +++ b/newapi/client_wiki/pages/data.py @@ -28,9 +28,9 @@ class Meta: create_data: dict = field(default_factory=dict) info: dict = field(default_factory=lambda: {"done": False}) username: str = "" - Exists: str = "" - is_redirect: str = "" - flagged: str = "" + Exists: bool = False + is_redirect: bool = False + flagged: bool = False wikibase_item: str = "" @@ -63,5 +63,5 @@ class CategoriesData: @dataclass class TemplateData: - templates: dict = field(default_factory=dict) - templates_api: dict = field(default_factory=dict) + templates: list = field(default_factory=list) + templates_api: list = field(default_factory=list) diff --git a/newapi/client_wiki/pages/super_page.py b/newapi/client_wiki/pages/super_page.py index c827da3..65529ff 100644 --- a/newapi/client_wiki/pages/super_page.py +++ b/newapi/client_wiki/pages/super_page.py @@ -30,7 +30,7 @@ def find_edit_error(old, new) -> bool: return False -class MainPage(HandleErrors, AskBot): +class MainPage(AskBot): """ Main page class for interacting with MediaWiki pages. @@ -51,6 +51,7 @@ def __init__( Sets up page attributes including title, language, family, API endpoint, and metadata fields. Normalizes the language code, loads user tables if available, and logs into the wiki if required. """ + self.error_handler = HandleErrors() self.login_bot = login_bot self.title: str = title @@ -638,7 +639,7 @@ def get_user(self): self.get_text() return self.user - def get_templates(self) -> dict: + def get_templates(self) -> list[dict[str, Any]]: if not self.text: self.text = self.get_text() self.template_data.templates = txtlib.extract_templates_and_params(self.text) @@ -660,16 +661,16 @@ def save( Prompts for confirmation and checks for invalid edits before submitting the change. Updates instance attributes with the latest revision and timestamps on success. Args: - newtext: The new wikitext to save to the page. - summary: Edit summary for the change. - nocreate: If 1 (default), prevents creating the page if it does not exist. - minor: Indicates if the edit should be marked as minor. - tags: Optional tags to associate with the edit. - nodiff: If True, skips showing a diff before saving. - ask: If True, prompts the user for confirmation before saving. + newtext: The new wikitext to save to the page. + summary: Edit summary for the change. + nocreate: If 1 (default), prevents creating the page if it does not exist. + minor: Indicates if the edit should be marked as minor. + tags: Optional tags to associate with the edit. + nodiff: If True, skips showing a diff before saving. + ask: If True, prompts the user for confirmation before saving. Returns: - True if the edit was successful, False otherwise. + True if the edit was successful, False otherwise. """ self.newtext = newtext @@ -744,7 +745,7 @@ def save( if error != {}: logger.debug(pop) - er = self.handle_err(error, function="Save", params=params) + er = self.error_handler.handle_err(error, function="Save", params=params) return er @@ -862,7 +863,7 @@ def create( if error != {}: logger.debug(pop) - er = self.handle_err(error, function="Create", params=params) + er = self.error_handler.handle_err(error, function="Create", params=params) return er return False @@ -896,7 +897,15 @@ def page_backlinks(self, ns: int = 0): # data = self.client_request_safe(params) # pages = data.get("query", {}).get("pages", []) - pages = self.post_continue(params, "query", _p_="pages", p_empty=[]) + def _load_data(body): + return body.get("query", {}).get("pages") or [] + + # --- + pages = self.login_bot.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + ) back_links = [x for x in pages if x["title"] != self.title] @@ -924,7 +933,15 @@ def page_links(self) -> list: # data = self.client_request_safe(params) # data = data.get('parse', {}).get('links', []) - data: list = self.post_continue(params, "parse", _p_="links", p_empty=[]) + def _load_data(body): + return body.get("parse", {}).get("links") or [] + + # --- + data: list = self.login_bot.post_continue_list( + params=params, + action="parse", + _load_data=_load_data, + ) # [{'ns': 14, 'title': 'تصنيف:مقالات بحاجة لشريط بوابات', 'exists': True}, {'ns': 14, 'title': 'تصنيف:مقالات بحاجة لصندوق معلومات', 'exists': False}] @@ -945,7 +962,15 @@ def page_links_query(self, plnamespace: str = "*"): # data = self.client_request_safe(params) # data = data.get('query', {}).get('links', []) - data = self.post_continue(params, "query", _p_="links", p_empty=[]) + def _load_data(body): + return body.get("query", {}).get("links") or [] + + # --- + data: list = self.login_bot.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + ) # [{'ns': 14, 'title': 'تصنيف:مقالات بحاجة لشريط بوابات', 'exists': True}, {'ns': 14, 'title': 'تصنيف:مقالات بحاجة لصندوق معلومات', 'exists': False}] @@ -982,7 +1007,15 @@ def get_revisions(self, rvprops=None) -> list: "rvprop": "|".join(rvprop), } - _revisions = self.post_continue(params, "query", _p_="pages", p_empty=[]) + def _load_data(body): + return body.get("query", {}).get("pages") or [] + + # --- + _revisions = self.login_bot.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + ) revisions = [] diff --git a/newapi/pformat.py b/newapi/pformat.py index c5c0353..19edfdc 100644 --- a/newapi/pformat.py +++ b/newapi/pformat.py @@ -2,9 +2,6 @@ python3 core8/pwb.py newapi/pformat """ -import sys -from pathlib import Path - import wikitextparser as wtp # python3 core8/pwb.py newapi/pformat -title:قالب:Cycling_race/stageclassification3 diff --git a/newapi/super/S_API/bot_api.py b/newapi/super/S_API/bot_api.py index 1faeb31..dd1f7e8 100644 --- a/newapi/super/S_API/bot_api.py +++ b/newapi/super/S_API/bot_api.py @@ -1,10 +1,12 @@ """ """ +import copy import datetime import logging import time from collections.abc import KeysView from datetime import timedelta +from typing import Any, Callable import tqdm @@ -16,18 +18,59 @@ logger = logging.getLogger(__name__) -class NewApi(HandleErrors, AskBot): +class NewApiHelpers: + def __init__(self): + pass + + def chunk_titles(self, titles, chunk_size: int = 50, noprint: bool = False): + # --- + if isinstance(titles, dict): + titles = list(titles.keys()) + + elif isinstance(titles, KeysView): + # TypeError: 'dict_keys' object is not subscriptable + titles = list(titles) + # --- + result = [titles[i : i + chunk_size] for i in range(0, len(titles), chunk_size)] + # --- + if not noprint: + result = tqdm.tqdm(result, desc=f"chunk_titles {len(titles)} split to {len(result)} chunks") + # --- + return result + + def merge_all_jsons_deep(self, all_jsons, json1): + def deep_merge(a, b): + # if both are dicts, merge keys + if isinstance(a, dict) and isinstance(b, dict): + for k, v in b.items(): + if k in a: + a[k] = deep_merge(a[k], v) + else: + a[k] = v + return a + # if both are lists, concatenate them + elif isinstance(a, list) and isinstance(b, list): + return a + b + # in case of different types, take the new one + else: + return b + + # if all_jsons is not dict, make it dict + if not isinstance(all_jsons, dict): + all_jsons = {} + + return deep_merge(all_jsons, json1) + + +class NewApi(AskBot, NewApiHelpers): def __init__(self, login_bot: WikiLoginClient, lang: str = "", family: str = "wikipedia") -> None: # --- + self.error_handler = HandleErrors() self.login_bot = login_bot # --- self.username = getattr(self, "username", "") - # self.family = family self.lang = change_codes.get(lang) or lang # --- - # self.family = family - # self.endpoint = f"https://{lang}.{family}.org/w/api.php" - # --- self.cxtoken_expiration = 0 self.cxtoken = "" # --- @@ -59,9 +102,6 @@ def Find_pages_exists_or_not(self, liste, get_redirect: bool = False, noprint: b json1 = self.login_bot.client_request_safe(params, method="post") # --- if not json1: - if not noprint: - logger.info("<> error when ") - # return table continue # --- all_jsons = self.merge_all_jsons_deep(all_jsons, json1) @@ -244,8 +284,13 @@ def Get_All_pages( # --- if start: params["apfrom"] = start + # --- - newp = self.post_continue(params, "query", _p_="allpages", p_empty=[], max=limit_all) + def _load_data(body): + return body.get("query", {}).get("allpages") or [] + + # --- + newp = self.post_continue_list(params=params, action="query", max=limit_all, _load_data=_load_data) # --- logger.debug(f"<> --- : find {len(newp)} pages.") # --- @@ -257,6 +302,66 @@ def Get_All_pages( # --- return Main_table + def Get_All_pages_generator( + self, + start: str = "", + namespace: str = "0", + limit: int = "max", + filterredir: str = "", + ppprop: str = "", + limit_all: int = 100000, + ): + # --- + logger.debug( + f"Get_All_pages_generator for start:{start}, limit:{limit},namespace:{namespace},filterredir:{filterredir}" + ) + # --- + params = { + "action": "query", + "format": "json", + "prop": "pageprops", + "generator": "allpages", + "gapnamespace": namespace, + "gaplimit": limit, + "formatversion": 2, + # "ppprop": "unlinkedwikibase_id", + "utf8": 1, + } + # --- + if str(namespace) in ["*", "", "all"]: + del params["gapnamespace"] + # --- + if ppprop: + params["ppprop"] = ppprop + # --- + if filterredir in ["redirects", "all", "nonredirects"]: + params["gapfilterredir"] = filterredir + # --- + if start: + params["gapfrom"] = start + + # --- + def _load_data(body): + return body.get("query", {}).get("pages") or [] + + # --- + newp = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + max=limit_all, + ) + # --- + logger.debug(f"<> --- Get_All_pages_generator : find {len(newp)} pages.") + # --- + Main_table = {x["title"]: x for x in newp} + # --- + logger.debug(f"len of Main_table {len(Main_table)}.") + # --- + logger.info(f"bot_api.py Get_All_pages_generator : find {len(Main_table)} pages.") + # --- + return Main_table + def PrefixSearch(self, pssearch: str = "", ns: str = "0", pslimit: str = "max", limit_all: int = 100000): """Perform a prefix search for titles in a specified namespace. @@ -303,66 +408,26 @@ def PrefixSearch(self, pssearch: str = "", ns: str = "0", pslimit: str = "max", # --- if pslimit.isdigit(): params["pslimit"] = pslimit + # --- - newp = self.post_continue(params, "query", _p_="prefixsearch", p_empty=[], max=limit_all) - # --- - logger.debug(f"<> --- : find {len(newp)} pages.") - # --- - Main_table = [x["title"] for x in newp] - # --- - logger.debug(f"len of Main_table {len(Main_table)}.") - # --- - logger.info(f"bot_api.py : find {len(Main_table)} pages.") - # --- - return Main_table + def _load_data(body): + return body.get("query", {}).get("prefixsearch") or [] - def Get_All_pages_generator( - self, - start: str = "", - namespace: str = "0", - limit: int = "max", - filterredir: str = "", - ppprop: str = "", - limit_all: int = 100000, - ): # --- - logger.debug( - f"Get_All_pages_generator for start:{start}, limit:{limit},namespace:{namespace},filterredir:{filterredir}" + newp = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + max=limit_all, ) # --- - params = { - "action": "query", - "format": "json", - "prop": "pageprops", - "generator": "allpages", - "gapnamespace": namespace, - "gaplimit": limit, - "formatversion": 2, - # "ppprop": "unlinkedwikibase_id", - "utf8": 1, - } - # --- - if str(namespace) in ["*", "", "all"]: - del params["gapnamespace"] - # --- - if ppprop: - params["ppprop"] = ppprop - # --- - if filterredir in ["redirects", "all", "nonredirects"]: - params["gapfilterredir"] = filterredir - # --- - if start: - params["gapfrom"] = start - # --- - newp = self.post_continue(params, "query", _p_="pages", p_empty=[], max=limit_all) - # --- - logger.debug(f"<> --- Get_All_pages_generator : find {len(newp)} pages.") + logger.debug(f"<> --- : find {len(newp)} pages.") # --- - Main_table = {x["title"]: x for x in newp} + Main_table = [x["title"] for x in newp] # --- logger.debug(f"len of Main_table {len(Main_table)}.") # --- - logger.info(f"bot_api.py Get_All_pages_generator : find {len(Main_table)} pages.") + logger.info(f"bot_api.py : find {len(Main_table)} pages.") # --- return Main_table @@ -400,8 +465,17 @@ def Search( if addparams: addparams = {x: v for x, v in addparams.items() if v and x not in params} params = {**params, **addparams} + # --- - search = self.post_continue(params, "query", _p_="search", p_empty=[]) + def _load_data(body): + return body.get("query", {}).get("search") or [] + + # --- + search = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + ) # --- results = [] # --- @@ -455,8 +529,16 @@ def Get_Newpages( else: limit = 5000 - json1 = self.post_continue(params, "query", _p_="recentchanges", p_empty=[], max=limit) + def _load_data(body): + return body.get("query", {}).get("recentchanges") or [] + # --- + json1 = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + max=limit, + ) Main_table = [x["title"] for x in json1] logger.debug(f'bot_api. find "{len(Main_table)}" result. s') @@ -481,28 +563,22 @@ def UserContribs(self, user, limit: int = 5000, namespace: str = "*", ucshow: st # --- if ucshow: params["ucshow"] = ucshow - # --- - results = self.post_continue(params, "query", _p_="usercontribs", p_empty=[], max=limit) - # --- - results = [x["title"] for x in results] - # --- - return results - def chunk_titles(self, titles, chunk_size: int = 50, noprint: bool = False): # --- - if isinstance(titles, dict): - titles = list(titles.keys()) + def _load_data(body): + return body.get("query", {}).get("usercontribs") or [] - elif isinstance(titles, KeysView): - # TypeError: 'dict_keys' object is not subscriptable - titles = list(titles) # --- - result = [titles[i : i + chunk_size] for i in range(0, len(titles), chunk_size)] + results = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + max=limit, + ) # --- - if not noprint: - result = tqdm.tqdm(result, desc=f"chunk_titles {len(titles)} split to {len(result)} chunks") + results = [x["title"] for x in results] # --- - return result + return results def Get_langlinks_for_list(self, titles, targtsitecode: str = "", numbes: int = 40): """Retrieve language links for a list of titles from a specified target @@ -627,8 +703,21 @@ def get_extlinks(self, title): "ellimit": "max", "formatversion": 2, } + # --- - results = self.post_continue(params, "query", "pages", [], first=True, _p_2="extlinks", _p_2_empty=[]) + def _load_data(body): + data = body.get("query", {}).get("pages") or [] + if isinstance(data, list) and data: + data = data[0] + data = data.get("extlinks", []) + return data + + # --- + results = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + ) # --- links = [x["url"] for x in results] # --- @@ -647,15 +736,16 @@ def get_pageassessments(self, titles): "ellimit": "max", "formatversion": 2, } + + # --- + def _load_data(body): + return body.get("query", {}).get("pages") or [] + # --- - results = self.post_continue( + results = self.post_continue_list( params, "query", "pages", - [], - first=False, - _p_2="pageassessments", - _p_2_empty=[], ) # --- return results @@ -678,8 +768,17 @@ def get_revisions(self, title, rvprop: str = "comment|timestamp|user|content|ids # --- if options: params.update(options) + # --- - results = self.post_continue(params, "query", _p_="pages", p_empty=[]) + def _load_data(body): + return body.get("query", {}).get("pages") or [] + + # --- + results = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + ) # --- return results @@ -742,8 +841,18 @@ def querypage_list(self, qppage: str = "Wantedcategories", qplimit=None, max=Non # --- if qppage not in qppage_values: logger.info(f"<> qppage {qppage} not in qppage_values.") + # --- - results = self.post_continue(params, "query", _p_="querypage", p_empty=[], max=max) + def _load_data(body): + return body.get("query", {}).get("querypage") or [] + + # --- + results = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + max=max, + ) # --- logger.debug(f" len(results) = {len(results)}") # --- @@ -762,8 +871,17 @@ def Get_template_pages(self, title, namespace: str = "*", max: int = 10000): "gtilimit": "max", "formatversion": "2", } + + # --- + def _load_data(body): + return body.get("query", {}).get("pages") or [] + # --- - results = self.post_continue(params, "query", _p_="pages", p_empty=[]) + results = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + ) # --- # { "pageid": 2973452, "ns": 100, "title": "بوابة:سباق الدراجات الهوائية" } pages = [x["title"] for x in results] @@ -847,8 +965,18 @@ def pageswithprop(self, pwppropname: str = "unlinkedwikibase_id", pwplimit=None, # --- if pwppropname != "": params["pwppropname"] = pwppropname + # --- - results = self.post_continue(params, "query", _p_="pageswithprop", p_empty=[], max=max) + def _load_data(body): + return body.get("query", {}).get("pageswithprop") or [] + + # --- + results = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + max=max, + ) # --- logger.debug(f" len(results) = {len(results)}") # --- @@ -869,8 +997,17 @@ def get_titles_redirects(self, titles): "utf8": 1, # "normalize": 1, } + + # --- + def _load_data(body): + return body.get("query", {}).get("redirects") or [] + # --- - json1 = self.post_continue(params, "query", _p_="redirects", p_empty=[]) + json1 = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + ) # --- lists = {x["from"]: x["to"] for x in json1} # --- @@ -879,35 +1016,6 @@ def get_titles_redirects(self, titles): # --- return redirects - def get_cxtoken(self): - # --- - if self.cxtoken and self.cxtoken_expiration: - current_time = int(time.time()) - if current_time < self.cxtoken_expiration: - return self.cxtoken - else: - self.cxtoken = "" - self.cxtoken_expiration = 0 - # --- - print("get_cxtoken") - # --- - params = {"action": "cxtoken", "format": "json"} - # --- - data = self.login_bot.client_request_safe(params, method="post") - # --- - if not data: - return "" - # --- - # { "jwt": "eyJ0eXAiOiJ.....", "exp": 1728172536, "age": 3600 } - jwt = data.get("jwt", "") - exp = data.get("exp", 0) - # --- - if jwt: - self.cxtoken = jwt - self.cxtoken_expiration = exp - # --- - return jwt - def users_infos(self, ususers=None) -> list[dict]: # --- if not isinstance(ususers, list): @@ -940,8 +1048,17 @@ def users_infos(self, ususers=None) -> list[dict]: ususers = list(set(ususers)) # --- params["ususers"] = "|".join(ususers) + # --- - results = self.post_continue(params, "query", _p_="users", p_empty=[]) + def _load_data(body): + return body.get("query", {}).get("users") or [] + + # --- + results = self.post_continue_list( + params=params, + action="query", + _load_data=_load_data, + ) # --- logger.debug(f" len(results) = {len(results)}") # --- @@ -949,59 +1066,34 @@ def users_infos(self, ususers=None) -> list[dict]: # --- return results - def post_params( - self, - params, - method: str = "get", - files=None, - **kwargs, - ): + def get_cxtoken(self): # --- - return self.login_bot.client_request_safe( - params, - method=method, - files=files, - **kwargs, - ) - - def client_request_safe( - self, - params, - method: str = "get", - files=None, - **kwargs, - ): + if self.cxtoken and self.cxtoken_expiration: + current_time = int(time.time()) + if current_time < self.cxtoken_expiration: + return self.cxtoken + else: + self.cxtoken = "" + self.cxtoken_expiration = 0 # --- - return self.login_bot.client_request_safe( - params, - method=method, - files=files, - **kwargs, - ) - - def post_continue( - self, - params, - action, - _p_: str = "pages", - p_empty=None, - max: int = 500000, - first: bool = False, - _p_2: str = "", - _p_2_empty=None, - **kwargs, - ): - return self.login_bot.post_continue( - params, - action, - _p_=_p_, - p_empty=p_empty, - max=max, - first=first, - _p_2=_p_2, - _p_2_empty=_p_2_empty, - **kwargs, - ) + print("get_cxtoken") + # --- + params = {"action": "cxtoken", "format": "json"} + # --- + data = self.login_bot.client_request_safe(params, method="post") + # --- + if not data: + return "" + # --- + # { "jwt": "eyJ0eXAiOiJ.....", "exp": 1728172536, "age": 3600 } + jwt = data.get("jwt", "") + exp = data.get("exp", 0) + # --- + if jwt: + self.cxtoken = jwt + self.cxtoken_expiration = exp + # --- + return jwt def Add_To_Bottom(self, text: str, summary, title, poss: str = "Head|Bottom"): # --- @@ -1059,7 +1151,7 @@ def Add_To_Bottom(self, text: str, summary, title, poss: str = "Head|Bottom"): # --- if error != {}: print(results) - er = self.handle_err(error, function="Add_To_Bottom", params=params) + er = self.error_handler.handle_err(error, function="Add_To_Bottom", params=params) # --- return er # --- @@ -1313,53 +1405,98 @@ def get_title_redirect_normalize(self, title, redirects, normalized): # --- return tab - def merge_all_jsons_deep(self, all_jsons, json1): - def deep_merge(a, b): - # إذا كان كلاهما dict → دمج مفاتيح - if isinstance(a, dict) and isinstance(b, dict): - for k, v in b.items(): - if k in a: - a[k] = deep_merge(a[k], v) - else: - a[k] = v - return a - # إذا كان كلاهما list → تمديد القوائم - elif isinstance(a, list) and isinstance(b, list): - return a + b - # في حالة اختلاف النوع → نأخذ الجديد - else: - return b + def post_params( + self, + params, + method: str = "get", + files=None, + **kwargs, + ): + # --- + return self.login_bot.client_request_safe( + params, + method=method, + files=files, + **kwargs, + ) - # إذا لم يكن all_jsons dict نجعله dict - if not isinstance(all_jsons, dict): - all_jsons = {} + def client_request_safe( + self, + params, + method: str = "get", + files=None, + **kwargs, + ): + # --- + return self.login_bot.client_request_safe( + params, + method=method, + files=files, + **kwargs, + ) - return deep_merge(all_jsons, json1) + def post_continue_list( + self, + params: dict, + action: str, + _load_data: Callable, + max: int | None = None, + ) -> dict[str, Any]: + """ + Drive a MediaWiki API continuation query to completion. - def merge_all_jsons(self, all_jsons, json1): - # --- إذا كان all_jsons ليس dict نحوله - if not isinstance(all_jsons, dict): - all_jsons = {} - # --- - # guard against non-dict inputs for json1 - if not isinstance(json1, dict): - return all_jsons - # --- - for x, z in json1.items(): - if x not in all_jsons: - all_jsons[x] = z - continue - # --- - tab = all_jsons[x] - # --- إذا كان كلاهما list - if isinstance(tab, list) and isinstance(z, list): - # explicit shallow copy of z to avoid surprises if z is reused - tab.extend(list(z)) - # --- إذا كان كلاهما dict - elif isinstance(tab, dict) and isinstance(z, dict): - tab.update(z) - # --- في حالة اختلاف النوع أو قيمة بسيطة - else: - all_jsons[x] = z - # --- - return all_jsons + Iterates the ``continue`` token until all pages are fetched or *max* + results have been collected. + + Args: + params: Base API parameters. + action: Top-level JSON key to extract results from + (e.g. ``"query"``). + max: Stop accumulating after this many results. + + Returns: + Accumulated results as a list + """ + logger.debug("action=%s", action) + + if isinstance(max, str) and max.isdigit(): + max = int(max) + if max == 0: + max = 500_000 + if max is None: + max = 500_000 + results = [] + continue_params: dict = {} + + while True: + page_params = copy.deepcopy(params) + + if not continue_params: + break + logger.debug("Applying continue_params: %s", continue_params) + page_params.update(continue_params) + + body = self.login_bot.client_request(page_params) + + if not body: + logger.debug("empty response, stopping") + break + + continue_params = body.get("continue", {}) + + data = _load_data(body) + + if not data: + logger.debug("no data in response, stopping") + break + + logger.debug("+%d items (total %d)", len(data), len(results)) + + if len(results) >= max: + logger.debug("max=%d reached, stopping", max) + break + + results.extend(data) + + logger.debug("done, %d total results", len(results)) + return results diff --git a/pyproject.toml b/pyproject.toml index 64d8ad1..26e62f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ exclude = ''' | buck-out | build | dist + | typings )/ ''' @@ -37,8 +38,6 @@ include_trailing_comma = true ensure_newline_before_comments = true # Project paths -src_paths = "ArWikiCats" -known_first_party = "ArWikiCats" skip = [ ".env", "env", @@ -72,6 +71,7 @@ exclude = [ ".tox", ".venv", ".vscode", + "typings", "__pypackages__", "_build", "buck-out", @@ -111,6 +111,8 @@ ignore = [ "N802", "UP007", "UP045", + "UP031", + "F841", ] select = [ "E", # pycodestyle (error) @@ -151,7 +153,6 @@ warn_return_any = true warn_unused_configs = true disallow_untyped_defs = false ignore_missing_imports = true -line-length = 120 [tool.pylint.messages_control] max-line-length = 120 @@ -160,3 +161,92 @@ disable = [ "C0103", # invalid-name "R0913", # too-many-arguments ] + + +[tool.pyright] +# Target Python version for type analysis +pythonVersion = "3.13" +stubPath = "typings" +# Paths to include and exclude from checking +include = ["src"] +exclude = ["typings", "**/node_modules", "**/__pycache__", ".venv"] + +# Diagnostic modes: "basic" or "standard" or "strict" +typeCheckingMode = "basic" + +# Automatically find virtual environments +venvPath = "." +venv = ".venv" + +# Fine-tune specific warning rules +reportMissingTypeStubs = false +reportGeneralTypeIssues = false +reportUnknownMemberType = false +enableReachabilityAnalysis = false +strictListInference = false +strictDictionaryInference = false +strictSetInference = false +deprecateTypingAliases = false +enableExperimentalFeatures = false + +analyzeUnannotatedFunctions = true +disableBytesTypePromotions = true +strictParameterNoneValue = true +enableTypeIgnoreComments = true +reportMissingModuleSource = "warning" +reportInvalidTypeForm = "warning" +reportMissingImports = "warning" +reportUndefinedVariable = "warning" + +reportReturnType = false +reportArgumentType = false +reportAttributeAccessIssue = "warning" +reportAssignmentType = "warning" +reportOptionalMemberAccess = "warning" +reportCallIssue = "warning" +reportOptionalSubscript = "warning" + + +# ----- +reportAbstractUsage = false +reportAssertAlwaysTrue = false +reportAssertTypeFailure = false +reportCallInDefaultInitializer = false +reportConstantRedefinition = false +reportDeprecated = false +reportDuplicateImport = false +reportFunctionMemberAccess = false +reportImplicitOverride = false +reportImplicitStringConcatenation = false +reportImportCycles = false +reportIncompatibleMethodOverride = false +reportIncompatibleVariableOverride = false +reportIncompleteStub = false +reportInconsistentOverload = false +reportIndexIssue = false +reportInvalidStringEscapeSequence = false +reportInvalidStubStatement = false +reportInvalidTypeArguments = false +reportInvalidTypeVarUse = false +reportMatchNotExhaustive = false +reportMissingParameterType = false +reportMissingSuperCall = false +reportMissingTypeArgument = false +reportOperatorIssue = false +reportOptionalCall = false +reportOptionalContextManager = false +reportOptionalIterable = false +reportOptionalOperand = false +reportOverlappingOverload = false +reportPossiblyUnboundVariable = false +reportPrivateImportUsage = false +reportPrivateUsage = false +reportPropertyTypeMismatch = false +reportRedeclaration = false +reportTypeCommentUsage = false +reportTypedDictNotRequiredAccess = false +reportUnboundVariable = false +reportUnhashable = false +reportUninitializedInstanceVariable = false +reportUnknownArgumentType = false +reportUnknownLambdaType = false diff --git a/tests/TestALL_APIS.py b/tests/TestALL_APIS.py index 6f5b796..e807714 100644 --- a/tests/TestALL_APIS.py +++ b/tests/TestALL_APIS.py @@ -40,6 +40,7 @@ def test_all_apis_init(mock_dependencies) -> None: username=username, password=password, use_cookies=use_cookies, + cookies_dir=None, ) diff --git a/tests/conftest.py b/tests/conftest.py index 03a79d0..2e7f9c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,14 @@ @pytest.fixture(autouse=True) -def stop_nets() -> None: +def stop_nets(request): + # Check if 'network' mark is present in the current test item + if "network" in request.node.keywords: + from pytest_socket import enable_socket + + enable_socket() + return + # Otherwise, disable the socket for all other tests disable_socket(allow_unix_socket=True) diff --git a/tests/unit/api_client/test_client.py b/tests/unit/api_client/test_client.py index ee68d16..a0cf718 100644 --- a/tests/unit/api_client/test_client.py +++ b/tests/unit/api_client/test_client.py @@ -2,7 +2,7 @@ Unit tests for src/core/api_client/client.py module. """ -from unittest.mock import MagicMock, PropertyMock, patch +from unittest.mock import MagicMock, patch import mwclient.errors import pytest @@ -13,12 +13,17 @@ def _make_client( - lang: str = "en", family: str = "wikipedia", username: str = "MyBot", password: str = "pass", cookies_dir=None + lang: str = "en", + family: str = "wikipedia", + username: str = "MyBot", + password: str = "pass", + cookies_dir=None, + use_cookies=True, ): """Create a WikiLoginClient with all external dependencies mocked.""" with ( patch("newapi.api_client.client.mwclient.Site") as mock_site, - patch("newapi.api_client.client.get_cookie_path") as mock_path, + patch("newapi.api_client.cookies_client.get_cookie_path") as mock_path, ): mock_path.return_value = MagicMock() site_instance = mock_site.return_value @@ -26,7 +31,7 @@ def _make_client( site_instance.connection = MagicMock() site_instance.api_url = "http://example.com/api" - kw = dict(lang=lang, family=family, username=username, password=password) + kw = dict(lang=lang, family=family, username=username, password=password, use_cookies=use_cookies) if cookies_dir is not None: kw["cookies_dir"] = cookies_dir client = WikiLoginClient(**kw) @@ -39,7 +44,7 @@ def _make_client( class TestEnrichParams: @patch("newapi.api_client.client.mwclient.Site") - @patch("newapi.api_client.client.get_cookie_path") + @patch("newapi.api_client.cookies_client.get_cookie_path") def test_query_action_strips_bot_and_summary(self, mock_path, mock_site) -> None: mock_path.return_value = MagicMock() mock_site.return_value.api.return_value = {"query": {"userinfo": {"id": 1}}} @@ -52,7 +57,7 @@ def test_query_action_strips_bot_and_summary(self, mock_path, mock_site) -> None assert result["titles"] == "Python" @patch("newapi.api_client.client.mwclient.Site") - @patch("newapi.api_client.client.get_cookie_path") + @patch("newapi.api_client.cookies_client.get_cookie_path") def test_write_action_injects_bot_and_assertuser(self, mock_path, mock_site) -> None: mock_path.return_value = MagicMock() mock_site.return_value.api.return_value = {"query": {"userinfo": {"id": 1}}} @@ -119,7 +124,7 @@ class TestClientRequestRetry: def test_invalid_method_raises(self) -> None: with ( patch("newapi.api_client.client.mwclient.Site") as mock_site, - patch("newapi.api_client.client.get_cookie_path"), + patch("newapi.api_client.cookies_client.get_cookie_path"), ): mock_site.return_value.api.return_value = {"query": {"userinfo": {"id": 1}}} client = WikiLoginClient("en", "wikipedia", "bot", "pass") @@ -129,7 +134,7 @@ def test_invalid_method_raises(self) -> None: def test_api_error_raises_wiki_client_error(self) -> None: with ( patch("newapi.api_client.client.mwclient.Site") as mock_site, - patch("newapi.api_client.client.get_cookie_path"), + patch("newapi.api_client.cookies_client.get_cookie_path"), ): site_instance = mock_site.return_value site_instance.api.return_value = {"query": {"userinfo": {"id": 1}}} @@ -272,7 +277,7 @@ def test_site_returns_mwclient_site(self) -> None: class TestRepr: @patch("newapi.api_client.client.mwclient.Site") - @patch("newapi.api_client.client.get_cookie_path") + @patch("newapi.api_client.cookies_client.get_cookie_path") def test_repr(self, mock_path, mock_site) -> None: mock_site.return_value.api.return_value = {"query": {"userinfo": {"id": 1}}} client = WikiLoginClient("en", "wikipedia", "MyBot", "pass") @@ -293,22 +298,22 @@ class TestInitCookiesDir: def test_passes_cookies_dir_to_get_cookie_path(self) -> None: with ( patch("newapi.api_client.client.mwclient.Site") as mock_site, - patch("newapi.api_client.client.get_cookie_path") as mock_path, + patch("newapi.api_client.cookies_client.get_cookie_path") as mock_path, ): mock_site.return_value.api.return_value = {"query": {"userinfo": {"id": 1}}} mock_path.return_value = MagicMock() - WikiLoginClient("en", "wikipedia", "bot", "pass", cookies_dir="/tmp/cookies") + WikiLoginClient("en", "wikipedia", "bot", "pass", cookies_dir="/tmp/cookies", use_cookies=True) mock_path.assert_called_once_with("/tmp/cookies", "wikipedia", "en", "bot") def test_default_cookies_dir_is_default_value(self) -> None: with ( patch("newapi.api_client.client.mwclient.Site") as mock_site, - patch("newapi.api_client.client.get_cookie_path") as mock_path, + patch("newapi.api_client.cookies_client.get_cookie_path") as mock_path, ): mock_site.return_value.api.return_value = {"query": {"userinfo": {"id": 1}}} mock_path.return_value = MagicMock() - WikiLoginClient("en", "wikipedia", "bot", "pass", "/tmp/cookies") + WikiLoginClient("en", "wikipedia", "bot", "pass", "/tmp/cookies", use_cookies=True) args = mock_path.call_args[0] assert args[0] == "/tmp/cookies" diff --git a/tests/unit/api_client/test_cookies.py b/tests/unit/api_client/test_cookies.py index 335b5ac..b8f43d9 100644 --- a/tests/unit/api_client/test_cookies.py +++ b/tests/unit/api_client/test_cookies.py @@ -4,16 +4,14 @@ import os from datetime import datetime, timedelta -from pathlib import Path -from unittest.mock import MagicMock, patch -from newapi.api_client.cookies import ( +from newapi.api_client.cookies_client import ( _COOKIE_MAX_AGE_DAYS, _delete_cookie_file, _delete_if_stale, get_cookie_path, ) -from newapi.api_client.exceptions import CookieError +from newapi.api_client.exceptions import CookieError # noqa: F401 class TestGetCookiePath: diff --git a/tests/unit/api_client/test_exceptions.py b/tests/unit/api_client/test_exceptions.py index 3d9a945..556260f 100644 --- a/tests/unit/api_client/test_exceptions.py +++ b/tests/unit/api_client/test_exceptions.py @@ -7,7 +7,7 @@ CSRFError, LoginError, MaxlagError, - MaxRetriesExceeded, + MaxRetriesExceededError, WikiClientError, ) @@ -26,7 +26,7 @@ def test_maxlag_error_is_wiki_client_error(self) -> None: assert issubclass(MaxlagError, WikiClientError) def test_max_retries_exceeded_is_wiki_client_error(self) -> None: - assert issubclass(MaxRetriesExceeded, WikiClientError) + assert issubclass(MaxRetriesExceededError, WikiClientError) def test_cookie_error_is_wiki_client_error(self) -> None: assert issubclass(CookieError, WikiClientError) diff --git a/tests/unit/api_client/test_requests_handler.py b/tests/unit/api_client/test_requests_handler.py index 7f42fbd..c442de9 100644 --- a/tests/unit/api_client/test_requests_handler.py +++ b/tests/unit/api_client/test_requests_handler.py @@ -5,16 +5,15 @@ from unittest.mock import MagicMock, patch import pytest -import requests from newapi.api_client.client import WikiLoginClient -from newapi.api_client.exceptions import CSRFError, MaxlagError, WikiClientError +from newapi.api_client.exceptions import MaxlagError def _make_client(lang: str = "en", family: str = "wikipedia", username: str = "MyBot", password: str = "pass"): """Create a WikiLoginClient with all external dependencies mocked.""" with ( patch("newapi.api_client.client.mwclient.Site") as mock_site, - patch("newapi.api_client.client.get_cookie_path") as mock_path, + patch("newapi.api_client.cookies_client.get_cookie_path") as mock_path, ): mock_path.return_value = MagicMock() site_instance = mock_site.return_value @@ -74,7 +73,7 @@ def test_maxlag_error_retries_and_succeeds(self) -> None: site.connection.request.side_effect = [maxlag_response, success_response] - with patch("newapi.api_client.client.time.sleep") as mock_sleep: + with patch("newapi.api_client.requests_handler.time.sleep") as mock_sleep: result = client.client_request_retry({"action": "query"}, method="get") assert "query" in result @@ -86,7 +85,7 @@ def test_maxlag_exhausted_retries_raises_maxlag_error(self) -> None: maxlag_response.json.return_value = {"error": {"code": "maxlag", "info": "Lag"}} site.connection.request.return_value = maxlag_response - with patch("newapi.api_client.client.time.sleep"): + with patch("newapi.api_client.requests_handler.time.sleep"): with pytest.raises(MaxlagError): client.client_request_retry({"action": "query"}, method="get") @@ -142,15 +141,27 @@ def test_assertnameduserfailed_recovery_succeeds(self) -> None: class TestOnAssertNamedUserFailed: """Tests for _on_assertnameduserfailed method.""" - @patch("newapi.api_client.client._delete_cookie_file") + @patch("newapi.api_client.cookies_client._delete_cookie_file") def test_on_assertnameduserfailed_clears_cookies_and_relogs(self, mock_delete) -> None: - client, site = _make_client() - site.login = MagicMock() - - client._on_assertnameduserfailed() - - mock_delete.assert_called_once() - site.login.assert_called_once_with("MyBot", "pass") + from newapi.api_client.client import WikiLoginClient + + with ( + patch("newapi.api_client.client.mwclient.Site") as mock_site, + patch("newapi.api_client.cookies_client.get_cookie_path") as mock_path, + ): + mock_path.return_value = MagicMock() + site_instance = mock_site.return_value + site_instance.api.return_value = {"query": {"userinfo": {"id": 1}}} + site_instance.connection = MagicMock() + site_instance.get_token = MagicMock(return_value="test_token") + site_instance.login = MagicMock() + + client = WikiLoginClient( + "en", "wikipedia", "MyBot", "pass", use_cookies=True + ) + client._on_assertnameduserfailed() + mock_delete.assert_called_once() + site_instance.login.assert_called_once_with("MyBot", "pass") class TestLoginForced: @@ -174,8 +185,8 @@ def test_handle_maxlag_with_retry_after_header(self) -> None: response = MagicMock() response.headers = {"Retry-After": "3"} - with patch("newapi.api_client.client.time.sleep") as mock_sleep: - client._handle_maxlag(response, 1) + with patch("newapi.api_client.requests_handler.time.sleep") as mock_sleep: + client.requests_handler._handle_maxlag(response, 1) mock_sleep.assert_called_with(3.0) def test_handle_maxlag_with_invalid_retry_after_uses_backoff(self) -> None: @@ -183,24 +194,18 @@ def test_handle_maxlag_with_invalid_retry_after_uses_backoff(self) -> None: response = MagicMock() response.headers = {"Retry-After": "not_a_number"} - with patch("newapi.api_client.client.time.sleep") as mock_sleep: - from newapi.api_client.client import settings - - with patch.object(settings.api_client, "backoff_base", 1): - client._handle_maxlag(response, 1) - mock_sleep.assert_called_with(2.0) # 1 * 2^1 + with patch("newapi.api_client.requests_handler.time.sleep") as mock_sleep: + client.requests_handler._handle_maxlag(response, 1) + mock_sleep.assert_called_with(2.0) # 1 * 2^1 def test_handle_maxlag_no_retry_after_uses_backoff(self) -> None: client, _ = _make_client() response = MagicMock() response.headers = {} - with patch("newapi.api_client.client.time.sleep") as mock_sleep: - from newapi.api_client.client import settings - - with patch.object(settings.api_client, "backoff_base", 1): - client._handle_maxlag(response, 2) - mock_sleep.assert_called_with(4.0) # 1 * 2^2 + with patch("newapi.api_client.requests_handler.time.sleep") as mock_sleep: + client.requests_handler._handle_maxlag(response, 2) + mock_sleep.assert_called_with(4.0) # 1 * 2^2 class TestInjectToken: @@ -264,18 +269,20 @@ def test_post_continue_with_continuation(self) -> None: class TestCookieLoading: """Tests for cookie loading error handling.""" - @patch("newapi.api_client.client.http.cookiejar.LWPCookieJar") + @patch("newapi.api_client.cookies_client.http.cookiejar.LWPCookieJar") def test_make_cookiejar_loads_existing_cookies(self, mock_jar_class) -> None: - from pathlib import Path - - from newapi.api_client.client import CookiesClient + from newapi.api_client.cookies_client import CookiesClient mock_cj = MagicMock() mock_jar_class.return_value = mock_cj - with patch("pathlib.Path.exists", return_value=True): + with ( + patch("newapi.api_client.cookies_client.get_cookie_path") as mock_path, + patch("pathlib.Path.exists", return_value=True), + ): + mock_path.return_value = MagicMock() mock_cj.load.side_effect = Exception("Parse error") - result = CookiesClient._make_cookiejar(Path("/fake/path")) + client = CookiesClient("en", "wikipedia", "MyBot", "/tmp", use_cookies=True) mock_cj.load.assert_called_once_with(ignore_discard=True, ignore_expires=True) @@ -283,9 +290,9 @@ def test_make_cookiejar_loads_existing_cookies(self, mock_jar_class) -> None: class TestCookieSaving: """Tests for cookie saving error handling.""" - @patch("newapi.api_client.client.logger") + @patch("newapi.api_client.cookies_client.logger") def test_save_cookies_failure_is_logged(self, mock_logger) -> None: - from newapi.api_client.client import CookiesClient + from newapi.api_client.cookies_client import CookiesClient mock_cj = MagicMock() mock_cj.save.side_effect = Exception("IO Error") diff --git a/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py b/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py index 3efe669..77ee913 100644 --- a/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py +++ b/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py @@ -1,3 +1,4 @@ +# ruff: noqa: F401 """ """ import sys