From cdaed9c4089ff02539fbade6327623d066bf4ce4 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Thu, 21 May 2026 07:23:47 +0300 Subject: [PATCH 1/5] Add type hints, adjust logging, and change sorting Adjustments across multiple modules: - _work_files/tree.py: change DisplayTree sortBy from 0 to 2 to alter tree output ordering. - newapi/client_wiki/categories/catdepth_new.py: refine logging (comment out an info log), and add explicit local typing for result variables (dict[str, dict] and dict[str, int]). - newapi/client_wiki/categories/category_db.py: add return type annotation for subcatquery_ (dict[str, dict]), lower some info logs to debug for progress messages, and change final log to an info with summarized result details. - newapi/super/S_API/bot_api.py: add return type annotation (-> list[str]) to the Get_All_pages-style function. These changes improve type clarity and adjust logging levels and output ordering for easier debugging and more consistent runtime information. --- _work_files/tree.py | 2 +- newapi/client_wiki/categories/catdepth_new.py | 6 +++--- newapi/client_wiki/categories/category_db.py | 8 ++++---- newapi/super/S_API/bot_api.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/_work_files/tree.py b/_work_files/tree.py index 429b4bb..7bedc6f 100644 --- a/_work_files/tree.py +++ b/_work_files/tree.py @@ -14,7 +14,7 @@ ignoreList=["__pycache__", "old", "app1.py", "example.env", "*.html"], onlyFiles=False, onlyDirs=False, - sortBy=0, + sortBy=2, raiseException=False, printErrorTraceback=False, ) diff --git a/newapi/client_wiki/categories/catdepth_new.py b/newapi/client_wiki/categories/catdepth_new.py index 20e5ccc..9c10fe0 100644 --- a/newapi/client_wiki/categories/catdepth_new.py +++ b/newapi/client_wiki/categories/catdepth_new.py @@ -58,12 +58,12 @@ def subcatquery(login_bot, title: str, sitecode: str = SITECODE, family: str = F f"<> catdepth_new.py sub cat query for {sitecode}:{title}, depth:{args2['depth']}, ns:{args2['ns']}, onlyns:{args2['onlyns']}" ) - logger.info(f"starting subcategory query: {sitecode}:{title}") + # logger.debug(f"starting subcategory query: {sitecode}:{title}") bot = CategoryDepth(login_bot, title, **kwargs) - result = bot.subcatquery_() + result: dict[str, dict] = bot.subcatquery_() if get_revids: - result = bot.get_revids() + result: dict[str, int] = bot.get_revids() if print_s: lenpages = bot.get_len_pages() diff --git a/newapi/client_wiki/categories/category_db.py b/newapi/client_wiki/categories/category_db.py index f9b09fc..5281d4b 100644 --- a/newapi/client_wiki/categories/category_db.py +++ b/newapi/client_wiki/categories/category_db.py @@ -296,8 +296,8 @@ def add_to_result_table(self, x: str, tab: dict) -> None: self.result_table[x] = tab - def subcatquery_(self) -> dict: - logger.info(f"starting subcatquery for {self.title}, depth={self.depth}") + def subcatquery_(self) -> dict[str, dict]: + logger.debug(f"starting subcatquery for {self.title}, depth={self.depth}") tablemember = self.get_cat_new(self.title) for x, zz in tablemember.items(): @@ -317,7 +317,7 @@ def subcatquery_(self) -> dict: break depth_done += 1 - logger.info(f"depth {depth_done}/{self.depth}: {len(new_list)} subcategories to process") + logger.debug(f"depth {depth_done}/{self.depth}: {len(new_list)} subcategories to process") for cat in tqdm(new_list): table2 = self.get_cat_new(cat) @@ -333,5 +333,5 @@ def subcatquery_(self) -> dict: soro = sorted(self.result_table.items(), key=lambda item: self.timestamps.get(item[0], 0), reverse=True) self.result_table = dict(soro) - logger.debug(f"subcatquery done: {len(self.result_table)} total results") + logger.info(f"{self.title=}, {self.depth}, {len(self.result_table)} total results") return self.result_table diff --git a/newapi/super/S_API/bot_api.py b/newapi/super/S_API/bot_api.py index 5cb63be..c11a46f 100644 --- a/newapi/super/S_API/bot_api.py +++ b/newapi/super/S_API/bot_api.py @@ -216,7 +216,7 @@ def Get_All_pages( apfilterredir="", ppprop="", limit_all=100000, - ): + ) -> list[str]: # --- logger.debug( f"Get_All_pages for start:{start}, limit:{limit},namespace:{namespace},apfilterredir:{apfilterredir}" From 0c1590715f53d7a6d0b7f691094b11dd019310b6 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Wed, 27 May 2026 22:24:54 +0300 Subject: [PATCH 2/5] Update client.py --- newapi/api_client/client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/newapi/api_client/client.py b/newapi/api_client/client.py index 650bc8c..44feac3 100644 --- a/newapi/api_client/client.py +++ b/newapi/api_client/client.py @@ -57,6 +57,12 @@ logger = logging.getLogger(__name__) +skip_log_params = [ + "token", + "password", + "lgpassword", + "text", +] # --------------------------------------------------------------------------- # RequestsHandler — transport + retry layer # --------------------------------------------------------------------------- @@ -541,7 +547,7 @@ def _client_request( method.upper(), self.api_url, # Never log token values - {k: ("***" if k == "token" else v) for k, v in params.items()}, + {k: ("***" if k in skip_log_params else v) for k, v in params.items()}, list(files.keys()) if files else None, ) action = params.get("action") From caa5ccf91a778cea70fc2b1275255d3943ba06b3 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Wed, 27 May 2026 23:02:09 +0300 Subject: [PATCH 3/5] . --- PROJECT_AUDIT_REPORT.md | 355 ++++++++++++++++++ newapi/DB_bots/README.md | 149 ++++++++ newapi/README.md | 219 +++++++++++ newapi/api_client/README.md | 163 ++++++++ newapi/client_wiki/README.md | 204 ++++++++++ newapi/client_wiki/api_utils/README.md | 146 +++++++ .../client_wiki/api_utils/bot_edit/README.md | 117 ++++++ newapi/client_wiki/categories/README.md | 100 +++++ newapi/client_wiki/pages/README.md | 124 ++++++ newapi/core/README.md | 118 ++++++ newapi/super/README.md | 178 +++++++++ newapi/super/S_API/README.md | 103 +++++ newapi/utils/README.md | 92 +++++ tests/README.md | 161 ++++++++ 14 files changed, 2229 insertions(+) create mode 100644 PROJECT_AUDIT_REPORT.md create mode 100644 newapi/DB_bots/README.md create mode 100644 newapi/README.md create mode 100644 newapi/api_client/README.md create mode 100644 newapi/client_wiki/README.md create mode 100644 newapi/client_wiki/api_utils/README.md create mode 100644 newapi/client_wiki/api_utils/bot_edit/README.md create mode 100644 newapi/client_wiki/categories/README.md create mode 100644 newapi/client_wiki/pages/README.md create mode 100644 newapi/core/README.md create mode 100644 newapi/super/README.md create mode 100644 newapi/super/S_API/README.md create mode 100644 newapi/utils/README.md create mode 100644 tests/README.md diff --git a/PROJECT_AUDIT_REPORT.md b/PROJECT_AUDIT_REPORT.md new file mode 100644 index 0000000..883ecbf --- /dev/null +++ b/PROJECT_AUDIT_REPORT.md @@ -0,0 +1,355 @@ +# PROJECT AUDIT REPORT + +**Project:** newapi_bot -- Wikimedia API Python Library +**Date:** 2026-05-27 +**Scope:** Full codebase audit of `newapi/` package and `tests/` suite +**Methodology:** Static analysis of all Python source files across 13 modules + +--- + +## Executive Summary + +### Purpose + +`newapi_bot` is a Python library for automated bot operations on MediaWiki projects (Wikipedia, Wikidata). It provides authenticated API access, page reading/editing/creation, recursive category traversal, Wikidata SPARQL queries, and local database storage via SQLite and MySQL. + +### Technologies + +| Layer | Technologies | +|---|---| +| Language | Python 3.13 | +| HTTP/API | `mwclient`, `requests` | +| Parsing | `wikitextparser`, `SPARQLWrapper` | +| Database | `sqlite_utils` (SQLite), `pymysql` (MySQL) | +| UI/Progress | `tqdm`, `colorlog`, `pywikibot` | +| Config | `python-dotenv`, `dataclasses` | +| Testing | `pytest`, `pytest-socket`, `unittest.mock` | +| Linting | `ruff`, `black`, `isort`, `mypy`, `flynt` | + +### Architecture Overview + +The system follows a three-layer architecture: + +``` +Layer 3: super/S_API/bot_api.py -- NewApi (30+ high-level operations) +Layer 2: client_wiki/ -- MainPage, CategoryDepth, bot edit checks +Layer 1: api_client/ -- WikiLoginClient (auth, retry, cookies) + DB_bots/ -- SQLite and MySQL access + core/ -- Exception hierarchy + config.py -- Settings singleton +``` + +The `AllAPIS` facade in `client_wiki/all_apis.py` serves as the primary entry point, composing page, category, and API access behind a single authenticated constructor. + +--- + +## Project Health Assessment + +### Overall Code Quality: 5.5/10 + +The codebase is functional and covers a wide surface area (60+ methods across all modules), but suffers from god classes, inconsistent naming, missing documentation, and several crash bugs in common code paths. + +### Maintainability: 4.5/10 + +| Factor | Assessment | +|---|---| +| Module structure | Good -- logical sub-packages at the top level | +| File sizes | Poor -- `bot_api.py` (1300+ lines), `super_page.py` (1000+ lines), `client.py` (850+ lines) | +| Naming consistency | Poor -- mix of PascalCase, snake_case, camelCase | +| Documentation | Poor -- most methods lack docstrings | +| Type annotations | Partial -- present on some methods, absent on many | +| Dead code | Moderate -- unused exceptions, commented-out code, no-op `del` statements | + +### Scalability: 5/10 + +- No MySQL connection pooling (new connection per query) +- No proactive rate limiting (reactive only) +- `LiteDbRepository.count()` fetches all rows into memory +- Category traversal can be memory-intensive for large trees +- Single-threaded HTTP client with no async support + +### Security Posture: 4/10 + +| Vulnerability | Severity | Location | +|---|---|---| +| SQL injection via raw SQL | HIGH | `db_bot.py` -- `query()`, `update()` | +| SPARQL injection risk | MEDIUM | `wd_sparql.py` -- `get_query_data()` | +| Cookie directory 0o770 permissions | MEDIUM | `cookies.py` -- group-readable/writable | +| Cookie files inherit umask | LOW | `cookies.py` -- potentially world-readable | +| Plaintext password in memory | LOW | `client.py` -- `self._password` | +| Silent exception swallowing | MEDIUM | `pymysql_bot.py`, `client_request_safe()` | + +### Production Readiness: NOT READY + +The codebase has **9 crash-level bugs** in common code paths, **1 SQL injection vulnerability**, **no HTTP timeouts**, and **20+ untested modules**. It is not suitable for production deployment in its current state. + +--- + +## Cross-Project Analysis + +### Shared Architectural Patterns + +| Pattern | Occurrences | Assessment | +|---|---|---| +| Facade | `AllAPIS`, `page.py`, `NewApi` | Good -- consistent entry-point design | +| Mixin/MI | `HandleErrors` + `AskBot` in `MainPage` and `NewApi` | Good -- reusable composition | +| Singleton | `settings` global in `config.py` | Acceptable but makes testing hard | +| Template Method | `RequestsHandler` retry loop | Good -- clean hook-based design | +| Repository | `LiteDbRepository` | Good -- clean interface | +| Decorator | `function_timer` | Good -- correct implementation | +| DTO | Dataclasses in `config.py`, `data.py` | Good -- structured data | + +### Repeated Weaknesses (Cross-Cutting) + +| Weakness | Modules Affected | +|---|---| +| **Mutable default arguments** | `pymysql_bot.py`, `txtlib.py` | +| **Shadowed builtin `max`** | `client.py`, `bot_api.py`, `super_page.py` (6+ occurrences) | +| **Silent error swallowing** | `pymysql_bot.py`, `client_request_safe()`, `LiteDbRepository` | +| **Global mutable caches** | `Bot_Cache`, `_save_or_ask`, `_created_cache` | +| **Missing type annotations** | 30+ public methods across all modules | +| **Inconsistent naming** | Every module mixes conventions | +| **No `__init__.py` re-exports** | All sub-packages have empty init files | + +### Common Technical Debt + +1. **God classes**: `NewApi` (1300 lines), `MainPage` (1000 lines), `WikiLoginClient` (850 lines) +2. **Code duplication**: `client_request_retry` is a copy-paste of `_client_request` +3. **Unused abstractions**: `core/exceptions.py` hierarchy defined but never used; `MaxRetriesExceeded` and `CookieError` defined but never raised +4. **Hardcoded values**: `BOT_USERNAME = "Mr.Ibrahembot"`, `mdwiki.org` special case, Arabic `"قالب:"` prefix +5. **Commented-out code**: Scattered across `db_bot.py`, `bot_api.py`, `page.py` +6. **Outdated config**: `pyproject.toml` references `src_paths = "ArWikiCats"` (non-existent project) + +### Dependency Issues + +- **No pinned versions**: `requirements.in` exists but no `requirements.txt` with locked versions +- **Heavy transitive dependency**: `pywikibot` pulled in just for `showDiff()` -- could be replaced with a lightweight diff library +- **`tqdm` coupled to logic**: Progress bars hardcoded in library code with no way to suppress + +### Integration Concerns + +- **Two parallel exception hierarchies**: `core/exceptions.py` and `api_client/exceptions.py` are unrelated +- **Two SQLite abstractions**: `LiteDB` and `LiteDbRepository` coexist with overlapping APIs +- **Settings side effects**: `config.py` `__post_init__` parses `sys.argv` and env vars at import time, making unit testing difficult +- **Mixed language comments**: Arabic and English throughout, hindering collaboration + +--- + +## Critical Findings + +### High-Risk Issues (9 total) + +| # | Issue | Module | Impact | +|---|---|---|---| +| 1 | **SQL injection** -- `query()` and `update()` accept raw SQL | `db_bot.py` | Data breach, data destruction | +| 2 | **Infinite loop** -- `ratelimited` handler never increments `attempt` | `client.py:236` | Process hang, resource exhaustion | +| 3 | **Infinite recursion** -- `move()` recurses on `ratelimited` with no limit | `bot_api.py:1180` | Stack overflow, process crash | +| 4 | **KeyError** -- `PrefixSearch` deletes wrong dict key | `bot_api.py:299` | Crash on common code path | +| 5 | **AttributeError** -- `querypage_list` calls `.isdigit()` on `None` | `bot_api.py:697` | Crash on common code path | +| 6 | **AttributeError** -- `upload_by_file` no None check on `data` | `bot_api.py:1271` | Crash on upload failure | +| 7 | **Mutable default** -- `main_args={}`, `credentials={}` | `pymysql_bot.py:15` | State corruption across calls | +| 8 | **Mutable default** -- `templates=[]` | `txtlib.py:52` | Corrupted template parsing | +| 9 | **Bare `raise`** -- no exception context | `super_page.py:1024` | Unhandled RuntimeError | + +### Security Vulnerabilities (3 total) + +| # | Vulnerability | Severity | Module | +|---|---|---|---| +| 1 | SQL injection via raw SQL strings | HIGH | `db_bot.py` | +| 2 | SPARQL injection (no query sanitization) | MEDIUM | `wd_sparql.py` | +| 3 | Cookie directory 0o770 + file umask | MEDIUM | `cookies.py` | + +### Performance Bottlenecks (4 total) + +| # | Bottleneck | Module | +|---|---|---| +| 1 | No MySQL connection pooling (new connection per query) | `pymysql_bot.py` | +| 2 | `count()` fetches all rows into memory | `db_bot.py` | +| 3 | No HTTP timeout (indefinite hangs possible) | `client.py` | +| 4 | `copy.deepcopy` per iteration in `post_continue` | `client.py` | + +### Stability Concerns (5 total) + +| # | Concern | Module | +|---|---|---| +| 1 | `_ensure_logged_in` does not actually log in | `client.py` | +| 2 | `client_request_safe` swallows all exceptions including `LoginError` | `client.py` | +| 3 | `post_continue` returns partial results silently on continuation failure | `client.py` | +| 4 | `settings.query.to_limit` permanently mutated at import time | `config.py` | +| 5 | `load_dotenv("$HOME/.env")` -- `$HOME` not expanded, silently fails | `config.py` | + +### Missing Infrastructure + +| Item | Status | +|---|---| +| CI/CD pipeline | Not present | +| `core/__init__.py` | Missing | +| `py.typed` marker | Missing | +| Dependency pinning | Incomplete | +| Code coverage reporting | Not configured | +| Linting in CI | Not configured | +| Integration test environment | Not present | + +--- + +## Strengths + +### Strong Engineering Decisions + +1. **Layered architecture**: Clear separation between transport (`api_client`), operations (`client_wiki`), and high-level API (`super`). The dependency direction is consistent. + +2. **Template Method pattern in `RequestsHandler`**: The retry loop with abstract hooks (`_session`, `_refresh_csrf_token`, `_on_assertnameduserfailed`) is a clean, extensible design for handling transient API errors. + +3. **Bot edit safety system**: Multi-layered permission checking (time-based + template-based) is a thoughtful approach to preventing accidental edits to protected pages. + +4. **Cookie persistence with staleness invalidation**: Sessions survive process restarts via `LWPCookieJar` with automatic 3-day expiry -- practical for long-running bots. + +5. **Write-action safety**: Automatic injection of `bot=1` and `assertuser` parameters for mutating requests prevents accidental non-bot edits. + +### Reusable Components + +- `HandleErrors` and `AskBot` mixins are cleanly composable across `MainPage` and `NewApi` +- `function_timer` decorator is correct and minimal +- `LiteDbRepository` provides a clean, typed CRUD interface +- Dataclasses in `config.py` and `data.py` provide structured, self-documenting data shapes + +### Well-Structured Modules + +- `api_client/` -- cleanest module with clear layering and good test coverage +- `core/exceptions.py` -- well-structured exception hierarchy with factory method +- `bot_edit/` -- clear separation of time-based and template-based strategies + +### Good Development Practices + +- `ruff`, `black`, `isort`, `mypy`, `flynt` configured in `pyproject.toml` +- `pytest-socket` prevents accidental network calls in tests +- `@pytest.mark.parametrize` used effectively in bot_edit tests +- `lru_cache` used appropriately for memoization + +--- + +## Improvement Roadmap + +### Immediate Fixes (Week 1) + +These are crash bugs and security vulnerabilities that must be fixed before any deployment. + +| # | Fix | File | Change | +|---|---|---|---| +| 1 | Fix SQL injection | `db_bot.py` | Remove raw `query()`/`update()` or add parameterization | +| 2 | Fix infinite loop | `client.py:236` | Add `attempt += 1` before `continue` in ratelimited handler | +| 3 | Fix infinite recursion | `bot_api.py:1180` | Add retry counter and max limit to `move()` | +| 4 | Fix KeyError | `bot_api.py:299` | Change `del params["apnamespace"]` to `del params["psnamespace"]` | +| 5 | Fix AttributeError | `bot_api.py:697` | Add `if qplimit and qplimit.isdigit()` guard | +| 6 | Fix AttributeError | `bot_api.py:1271` | Add `if not data: return {}` guard | +| 7 | Fix mutable defaults | `pymysql_bot.py:15` | Change to `main_args=None, credentials=None` with `if None` guards | +| 8 | Fix mutable default | `txtlib.py:52` | Change `templates=[]` to `templates=None` | +| 9 | Fix bare `raise` | `super_page.py:1024` | Replace with `raise RuntimeError("...")` or proper re-raise | +| 10 | Add HTTP timeout | `client.py` | Add `timeout=(10, 30)` to `_execute_request` | + +### Short-Term Improvements (Weeks 2-4) + +| # | Improvement | Impact | +|---|---|---| +| 1 | Fix `_ensure_logged_in` to actually trigger login | Prevents unauthenticated sessions | +| 2 | Add `core/__init__.py` with re-exports | Package consistency | +| 3 | Fix `get_username()` to read from `login_bot` | Fixes always-empty username | +| 4 | Use context manager for file handles in `upload_by_file` | Prevents resource leaks | +| 5 | Fix `settings.query.to_limit` mutation | Preserves original config value | +| 6 | Fix `load_dotenv` path expansion | Enables `.env` fallback | +| 7 | Remove tautological tests, fix empty test files | Accurate coverage reporting | +| 8 | Extract `_make_client()` to shared conftest | Reduces test duplication | +| 9 | Add `timeout` to all HTTP requests | Prevents indefinite hangs | +| 10 | Remove dead code (`del data`, `tracer`, unused exceptions) | Reduces confusion | + +### Medium-Term Improvements (Months 1-3) + +| # | Improvement | Module | +|---|---|---| +| 1 | Deprecate `LiteDB` in favor of `LiteDbRepository` | `db_bot.py` | +| 2 | Add MySQL connection pooling | `pymysql_bot.py` | +| 3 | Split `NewApi` into focused classes (Search, Query, Edit, Upload) | `bot_api.py` | +| 4 | Split `MainPage` into reader/writer/metadata mixins | `super_page.py` | +| 5 | Standardize all method naming to `snake_case` | All modules | +| 6 | Add `__all__` and re-exports to all `__init__.py` | All packages | +| 7 | Adopt `core/exceptions.py` hierarchy across codebase | All modules | +| 8 | Rename `handel_errors.py` to `handle_errors.py` | `api_utils/` | +| 9 | Make `BOT_USERNAME` and Arabic prefix configurable | `bot_edit/`, `txtlib.py` | +| 10 | Add type annotations to all public methods | All modules | + +### Long-Term Strategic Refactoring (Months 3-6) + +| # | Refactoring | Rationale | +|---|---|---| +| 1 | Unify exception hierarchies (`core/` + `api_client/`) | Single error handling strategy | +| 2 | Implement proactive rate limiting | Prevent API abuse | +| 3 | Add `urllib3.util.retry.Retry` + `HTTPAdapter` | Transport-level resilience | +| 4 | Split `client.py` into `transport.py`, `auth.py`, `pagination.py` | 850+ line file | +| 5 | Replace `tqdm` with callback pattern | Decouple UI from logic | +| 6 | Add `__enter__`/`__exit__` to client classes | Clean resource management | +| 7 | Add `abc.ABC` + `@abstractmethod` for abstract methods | Type safety | +| 8 | Add structured error context to exceptions | Debugging aid | + +### Security Hardening Priorities + +| Priority | Action | +|---|---| +| P0 | Remove or parameterize `LiteDB.query()` and `LiteDB.update()` | +| P0 | Add SPARQL query sanitization | +| P1 | Set cookie file permissions to 0o600 | +| P1 | Set cookie directory to 0o700 | +| P2 | Zeroize password memory after login | +| P2 | Add input validation for `lang`/`family` parameters | +| P3 | Add structured logging with credential redaction | + +### DevOps and Testing Recommendations + +| Category | Recommendation | +|---|---| +| CI/CD | Add GitHub Actions with lint (`ruff`), type check (`mypy`), and test (`pytest`) | +| Coverage | Configure `pytest-cov` with 60% minimum threshold | +| Integration | Add integration tests against a test wiki instance | +| Dependencies | Pin all dependencies in `requirements.txt` with `pip-compile` | +| Pre-commit | Add `pre-commit` hooks for `ruff`, `black`, `isort` | +| Property testing | Add `hypothesis` tests for wikitext parsing | +| Contract testing | Add API contract tests for MediaWiki response shapes | + +--- + +## Final Evaluation + +### Module Scores + +| Module | Score | Rating | +|---|---|---| +| `api_client/` | 6.5/10 | Best module -- clean architecture, good tests, fixable bugs | +| `core/` | 7.5/10 | Cleanest code -- but unused in practice | +| `utils/` | 7.0/10 | Simple and correct | +| `client_wiki/` | 6.0/10 | Good structure, messy internals | +| `client_wiki/api_utils/` | 6.0/10 | Functional but naming issues | +| `client_wiki/categories/` | 6.5/10 | Clean logic, fragile error handling | +| `client_wiki/pages/` | 6.0/10 | God class, type mismatches | +| `DB_bots/` | 5.0/10 | SQL injection, dual abstractions | +| `super/` | 5.0/10 | God class, 3 crash bugs | +| `super/S_API/` | 4.5/10 | Lowest rated -- crash bugs in common paths | +| `tests/` | 5.0/10 | Good unit tests exist but 20+ modules untested | + +### Aggregate Scores + +| Metric | Score | +|---|---| +| **Overall Project Score** | **5.5 / 10** | +| **Risk Level** | **HIGH** -- 9 crash bugs, 1 SQL injection, 3 security vulnerabilities | +| **Technical Debt Level** | **HIGH** -- god classes, inconsistent naming, unused abstractions, 20+ untested modules | +| **Production Readiness** | **NOT READY** -- requires immediate fixes + short-term improvements before deployment | +| **Estimated Effort to Production-Ready** | 4-6 weeks (1 developer) for immediate + short-term fixes | + +### Recommended Next Steps + +1. **This week**: Fix all 10 immediate issues (crash bugs + security + timeout). These are blocking. +2. **Next 2 weeks**: Address short-term improvements. Focus on error handling, resource management, and test cleanup. +3. **Month 1**: Begin medium-term refactoring. Prioritize splitting god classes and standardizing naming. +4. **Month 2**: Set up CI/CD pipeline, coverage reporting, and dependency pinning. +5. **Month 3+**: Long-term strategic refactoring -- exception unification, rate limiting, async support. + +The codebase has a solid architectural foundation and comprehensive API coverage. The issues are fixable. The priority is stabilizing the crash bugs and security vulnerabilities, then systematically reducing technical debt through the roadmap above. diff --git a/newapi/DB_bots/README.md b/newapi/DB_bots/README.md new file mode 100644 index 0000000..a4e41a7 --- /dev/null +++ b/newapi/DB_bots/README.md @@ -0,0 +1,149 @@ +# DB_bots -- Database Abstraction Layer + +## Project Overview + +The `DB_bots` package provides database access abstractions for both SQLite and MySQL, designed for use by MediaWiki bots for local data storage and querying. + +### Main Modules + +| Module | Purpose | +|---|---| +| `db_bot.py` | SQLite abstraction with two implementations: `LiteDB` (legacy) and `LiteDbRepository` (repository pattern) | +| `pymysql_bot.py` | MySQL access via PyMySQL with a single `sql_connect_pymysql` function | + +### Technologies & Dependencies + +- **`sqlite_utils`** -- SQLite database toolkit (used by `LiteDB`) +- **`pymysql`** -- Pure Python MySQL client + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Fair.** Two independent modules with no shared interface or base class. `db_bot.py` contains two unrelated implementations (`LiteDB` and `LiteDbRepository`) with significant API overlap. + +### Design Patterns + +- **Repository Pattern**: `LiteDbRepository` implements a clean repository interface with `get_by_id`, `find_by`, `insert`, `update`, `delete`. +- **Active Record-like**: `LiteDB` provides direct table operations with raw SQL access. + +### Maintainability + +**Low-Moderate.** Having two unrelated SQLite abstractions in the same file is confusing. `pymysql_bot.py` is a single function with no structure. + +### Readability + +**Fair.** Method names are descriptive but the dual-implementation approach in `db_bot.py` requires careful reading to understand which to use. + +### Scalability Considerations + +- **MySQL**: No connection pooling -- a new connection is created and destroyed per query. +- **SQLite**: `LiteDbRepository.count()` fetches all rows into memory to count them instead of using SQL `COUNT`. + +--- + +## Strengths + +- **Clean repository interface**: `LiteDbRepository` provides a well-structured CRUD API. +- **Parameterized queries**: `LiteDB.select` correctly uses `?` placeholders to prevent SQL injection. +- **Simple MySQL function**: `sql_connect_pymysql` is straightforward for one-off queries. +- **Type hints**: Both modules use type annotations on method signatures. + +--- + +## Weaknesses + +- **Two competing SQLite abstractions**: `LiteDB` and `LiteDbRepository` coexist with overlapping functionality and no shared interface. +- **Dead code**: `del data` / `del datalist` calls in `LiteDB` are no-ops. The `tracer` function is defined but commented out. +- **Silent error swallowing**: `LiteDbRepository` catches bare `Exception` and returns `None`/`False`. `pymysql_bot` returns `[]` on all errors. +- **No connection pooling**: MySQL connections are created fresh for every query call. +- **Inefficient counting**: `LiteDbRepository.count()` with criteria fetches all rows just to `len()` them. + +--- + +## Critical Issues + +### 1. SQL Injection in `LiteDB.query()` and `LiteDB.update()` (HIGH) + +```python +# db_bot.py +def query(self, sql: str) -> List[tuple]: + return self.db.query(sql) # Raw SQL, no parameterization + +def update(self, sql: str) -> None: + self.db.executescript(sql) # Raw SQL, no parameterization +``` + +These methods accept raw SQL strings with no sanitization. Any caller passing user-controlled input is fully exposed to SQL injection. + +### 2. Mutable Default Arguments in `sql_connect_pymysql()` (HIGH) + +```python +# pymysql_bot.py +def sql_connect_pymysql(query, return_dict=False, values=None, + main_args={}, # <-- Mutable default! + credentials={}, # <-- Mutable default! + ...): +``` + +Modifications to these dicts persist across calls, causing subtle state corruption. + +### 3. Silent Error Swallowing (MEDIUM) + +`pymysql_bot.py` returns `[]` on all exceptions. `LiteDbRepository` returns `None`/`False`. Callers cannot distinguish "no results" from "database error." + +### 4. `LiteDB.insert` Hardcodes Primary Key (LOW) + +```python +# db_bot.py +def insert(self, table_name, data, check=True): + # ... hardcodes pk="id" despite create_table accepting custom pk +``` + +Tables with non-`id` primary keys will fail. + +--- + +## Areas That Need Attention + +- **Unify SQLite abstractions**: Choose one API (`LiteDbRepository` is cleaner) and deprecate the other. +- **Remove dead code**: `del data`, `del datalist`, `tracer` function. +- **Add connection pooling** for MySQL. +- **Use SQL `COUNT`** instead of fetching all rows for counting. +- **Add error differentiation**: Raise or return typed errors instead of silent defaults. +- **Fix mutable default arguments** in `sql_connect_pymysql`. + +--- + +## Improvement Plan + +### Quick Wins +1. Fix mutable default arguments in `sql_connect_pymysql` (`main_args=None`, `credentials=None`). +2. Remove dead `del` statements and `tracer` function. +3. Fix `LiteDbRepository.count()` to use SQL `COUNT`. + +### Medium-Term Improvements +1. Deprecate `LiteDB` in favor of `LiteDbRepository`. +2. Add connection pooling for MySQL (e.g., `DBUtils` or `SQLAlchemy` pool). +3. Implement proper error handling with typed exceptions. +4. Add `__all__` to `__init__.py`. + +### Long-Term Refactoring +1. Define a shared abstract interface for both SQLite and MySQL. +2. Add transaction management for write operations. +3. Add retry logic for transient database errors. +4. Add comprehensive unit tests. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **5/10** | +| **Production Readiness** | Low-Moderate -- SQL injection and mutable defaults are production risks | +| **Technical Debt** | High -- dual abstractions, dead code, silent errors | +| **Risk Assessment** | High -- SQL injection vulnerability and mutable default arguments | +| **Maintainability** | 4/10 -- confusing dual APIs, no tests, silent failures | diff --git a/newapi/README.md b/newapi/README.md new file mode 100644 index 0000000..a93b16a --- /dev/null +++ b/newapi/README.md @@ -0,0 +1,219 @@ +# newapi -- Wikimedia API Python Library + +## Project Overview + +`newapi` is a Python library for interacting with the MediaWiki API, designed for automated bot operations on Wikimedia projects. It provides authenticated API access, page manipulation, category traversal, database storage, and Wikidata SPARQL queries. + +### Main Modules + +| Module | Purpose | +|---|---| +| `__init__.py` | Package entry point -- re-exports key symbols (`AllAPIS`, `WikiLoginClient`, etc.) | +| `all_apis.py` | Re-exports `AllAPIS` from `client_wiki.all_apis` | +| `config.py` | Centralized settings via dataclasses, loaded from env vars and CLI args | +| `logging_config.py` | Colored logging configuration for console and file output | +| `page.py` | Deprecated convenience wrappers (`MainPage`, `CatDepth`, `NewApi`) | +| `pformat.py` | Wikitext template reformatting utility | +| `api_client/` | MediaWiki API client with retry, authentication, and cookie persistence | +| `client_wiki/` | High-level wiki bot client (page operations, categories, bot edit checks) | +| `core/` | Exception hierarchy for MediaWiki API errors | +| `DB_bots/` | Database abstraction (SQLite and MySQL) | +| `super/` | `NewApi` class -- high-level bot API with 30+ methods | +| `utils/` | Shared utilities (function timer decorator) | + +### Architecture Diagram + +``` +newapi/ + __init__.py # Public API surface + config.py # Settings singleton + logging_config.py # Logging setup + page.py # Deprecated convenience API + all_apis.py # Re-export + pformat.py # Wikitext formatting + + api_client/ # Layer 1: Transport & Auth + client.py # WikiLoginClient (retry, CSRF, cookies) + cookies.py # Cookie file management + exceptions.py # Client exceptions + + client_wiki/ # Layer 2: Wiki Operations + all_apis.py # AllAPIS facade + pages/ # MainPage class + categories/ # CategoryDepth traversal + api_utils/ # Bot edit checks, error handling, parsing + + super/ # Layer 3: High-Level API + S_API/bot_api.py # NewApi class (search, query, edit, upload) + + core/ # Exception hierarchy + DB_bots/ # Database access (SQLite, MySQL) + utils/ # Shared utilities +``` + +### Technologies & Dependencies + +| Package | Version | Purpose | +|---|---|---| +| `mwclient` | -- | MediaWiki API client | +| `requests` | -- | HTTP transport | +| `wikitextparser` | -- | Wikitext parsing | +| `SPARQLWrapper` | -- | Wikidata SPARQL queries | +| `pywikibot` | -- | Diff display | +| `tqdm` | -- | Progress bars | +| `sqlite_utils` | -- | SQLite abstraction | +| `pymysql` | -- | MySQL client | +| `colorlog` | -- | Colored logging | +| `python-dotenv` | -- | Environment file loading | + +### Configuration + +Settings are loaded from (in order of precedence): +1. Command-line arguments (e.g., `depth:3`, `ask`, `no_cookies`) +2. Environment variables (e.g., `WIKI_LANG`, `WIKI_FAMILY`, `WIKI_USERNAME`) +3. `.env` file +4. Dataclass defaults + +The `settings` singleton is instantiated at import time in `config.py`. + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Good at the top level.** The layered architecture (transport -> wiki operations -> high-level API) is clear. Sub-packages are logically grouped. + +**Weak at the detail level.** Several modules are oversized (`bot_api.py` at 1300+ lines, `super_page.py` at 1000+ lines), naming is inconsistent, and `__init__.py` files are mostly empty. + +### Design Patterns + +| Pattern | Usage | +|---|---| +| Singleton | `settings` global in `config.py` | +| Facade | `AllAPIS`, `page.py` convenience functions | +| Template Method | `RequestsHandler` retry loop with abstract hooks | +| Mixin | `HandleErrors` and `AskBot` composed into `MainPage` and `NewApi` | +| Repository | `LiteDbRepository` in `db_bot.py` | +| Data Transfer Object | Dataclasses in `config.py` and `pages/data.py` | +| Decorator | `function_timer` in `utils/` | +| Factory Method | `parse_api_error` in `core/exceptions.py` | + +### Maintainability + +**Moderate.** The top-level structure is clear, but individual modules suffer from god classes, inconsistent naming, missing docs, and mixed languages in comments. + +### Readability + +**Mixed.** Some modules are clean (`exceptions.py`, `functions_timer.py`), while others mix Arabic/English comments and lack docstrings. + +### Scalability Considerations + +- No connection pooling for MySQL +- No proactive rate limiting (reactive only) +- Category traversal can be memory-intensive for large trees +- `LiteDbRepository.count()` fetches all rows into memory + +--- + +## Strengths + +- **Comprehensive API coverage**: 60+ methods across all modules covering page CRUD, search, categories, uploads, and more. +- **Robust authentication**: Cookie persistence, automatic retry on CSRF/maxlag/rate-limit errors, session recovery. +- **Bot safety**: Multi-layered edit permission checking (time-based + template-based). +- **Clean exception hierarchy**: Well-structured `core/exceptions.py` with factory method. +- **Configuration flexibility**: Settings from env vars, CLI args, and `.env` files. +- **Repository pattern**: `LiteDbRepository` provides a clean database interface. + +--- + +## Weaknesses + +- **God classes**: `NewApi` (1300+ lines) and `MainPage` (1000+ lines) violate single responsibility. +- **Inconsistent naming**: Mix of PascalCase, snake_case, and camelCase across the codebase. +- **Unused exception hierarchy**: `core/exceptions.py` is defined but the codebase returns mixed types instead of raising exceptions. +- **Global mutable state**: Module-level caches (`Bot_Cache`, `_save_or_ask`) never cleared. +- **Empty `__init__.py` files**: No re-exports from sub-packages. +- **Mixed language comments**: Arabic and English comments throughout. +- **No tests for most modules**: 20+ source modules have zero test coverage. + +--- + +## Critical Issues + +### 1. SQL Injection in `LiteDB.query()` and `LiteDB.update()` (HIGH) + +Raw SQL strings accepted with no parameterization. Any user-controlled input is fully exposed. + +### 2. Infinite Loop/Recursion on Rate Limiting (HIGH) + +- `api_client/client.py`: `ratelimited` handler never increments `attempt` -- infinite loop. +- `super/S_API/bot_api.py`: `move()` recurses on `ratelimited` with no limit -- stack overflow. + +### 3. Multiple Crash Bugs in `bot_api.py` (HIGH) + +- `PrefixSearch`: deletes wrong dict key (`apnamespace` instead of `psnamespace`) -- `KeyError`. +- `querypage_list`: calls `.isdigit()` on `None` -- `AttributeError`. +- `upload_by_file`: no None check on `data` before `.get()` -- `AttributeError`. + +### 4. Mutable Default Arguments in `pymysql_bot.py` (HIGH) + +`main_args={}` and `credentials={}` are shared across calls, causing state corruption. + +### 5. No HTTP Timeout (MEDIUM) + +`api_client/client.py` makes HTTP requests without timeout, risking indefinite hangs. + +### 6. Settings Mutation at Import Time (MEDIUM) + +`config.py`'s `__post_init__` permanently mutates `query.to_limit` by adding `offset`, destroying the original value. + +--- + +## Areas That Need Attention + +- **Missing `core/__init__.py`**: Inconsistent with other packages. +- **Missing `py.typed` marker**: No type-checking support declaration. +- **No CI/CD configuration visible**: No GitHub Actions or similar. +- **Outdated `pyproject.toml` references**: `src_paths = "ArWikiCats"` references a non-existent project. +- **Deprecation warnings not emitted**: `page.py` defines deprecation strings but never applies them. +- **`load_dotenv("$HOME/.env")`**: `$HOME` is not expanded, so this fallback silently fails. + +--- + +## Improvement Plan + +### Quick Wins +1. Fix the three crash bugs in `bot_api.py` (KeyError, AttributeError x2). +2. Fix the infinite loop in `api_client/client.py` ratelimited handler. +3. Fix mutable default arguments in `pymysql_bot.py`. +4. Add HTTP timeout to `_execute_request`. +5. Add `core/__init__.py`. + +### Medium-Term Improvements +1. Deprecate `LiteDB` in favor of `LiteDbRepository`. +2. Split `NewApi` and `MainPage` into focused modules. +3. Standardize naming to `snake_case` throughout. +4. Add `__all__` and re-exports to all `__init__.py` files. +5. Adopt the `core/exceptions.py` hierarchy across the codebase. +6. Add unit tests for untested modules. + +### Long-Term Refactoring +1. Unify the two exception hierarchies (`core/` and `api_client/`). +2. Implement connection pooling for MySQL. +3. Add proactive rate limiting. +4. Add comprehensive integration tests. +5. Add type stubs and `py.typed` marker. +6. Extract hardcoded values (`BOT_USERNAME`, `mdwiki.org`) to configuration. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **5.5/10** | +| **Production Readiness** | Low-Moderate -- multiple crash bugs and SQL injection risks | +| **Technical Debt** | High -- god classes, inconsistent naming, unused abstractions, no tests | +| **Risk Assessment** | High -- SQL injection, infinite loops, crash bugs in common paths | +| **Maintainability** | 5/10 -- clear top-level structure but messy internals | diff --git a/newapi/api_client/README.md b/newapi/api_client/README.md new file mode 100644 index 0000000..9f4a5d5 --- /dev/null +++ b/newapi/api_client/README.md @@ -0,0 +1,163 @@ +# api_client -- MediaWiki API Client + +## Project Overview + +The `api_client` package provides a robust, authenticated HTTP client for the MediaWiki API, built on top of `mwclient`. It handles session persistence, bot authentication, automatic retry on transient errors, and continuation pagination. + +### Main Modules + +| Module | Purpose | +|---|---| +| `client.py` | Core client classes: `RequestsHandler` (retry/transport), `CookiesClient` (cookie I/O), `WikiLoginClient` (business layer) | +| `cookies.py` | Cookie file path resolution, staleness checks, and cleanup | +| `exceptions.py` | Custom exception hierarchy rooted at `WikiClientError` | + +### Technologies & Dependencies + +- **`mwclient`** -- MediaWiki client library (provides `Site` and session management) +- **`requests`** -- HTTP transport (used internally by mwclient) +- **`http.cookiejar.LWPCookieJar`** -- Mozilla-format cookie persistence +- **Standard library:** `copy`, `logging`, `os`, `stat`, `time`, `datetime`, `pathlib` + +--- + +## Architecture & Code Quality Review + +### Code Organization + +The package follows a layered architecture: + +1. **Transport layer** (`RequestsHandler`) -- HTTP execution, retry logic, CSRF/maxlag/rate-limit handling +2. **Persistence layer** (`CookiesClient`) -- Cookie file I/O +3. **Business layer** (`WikiLoginClient`) -- Authentication, request enrichment, pagination + +### Design Patterns + +- **Template Method Pattern**: `RequestsHandler` defines the retry loop skeleton and calls abstract hooks (`_session`, `_refresh_csrf_token`, `_on_assertnameduserfailed`) that `WikiLoginClient` implements. +- **Mixin / Multiple Inheritance**: `WikiLoginClient` inherits from both `CookiesClient` and `RequestsHandler`. +- **Facade**: Wraps `mwclient.Site` and `requests.Session` behind a simplified interface. +- **Strategy-like error handling**: The retry loop dispatches on error code to distinct handlers (CSRF, maxlag, assertnameduserfailed, ratelimited). + +### Maintainability + +**Moderate.** The class hierarchy is clear, but `client_request_retry` is a near-exact copy-paste of `_client_request`, creating a maintenance hazard. The 850+ line `client.py` file would benefit from splitting. + +### Readability + +**Good.** Method names are descriptive, logging is consistent (`%s` formatting), and the retry loop is well-structured with clear error dispatch. + +### Scalability Considerations + +The client is single-threaded with no connection pooling configuration beyond what mwclient provides. No proactive rate limiting exists -- only reactive handling after receiving `ratelimited` errors. + +--- + +## Strengths + +- **Robust retry logic**: Handles CSRF token invalidation, maxlag, rate limiting, and session expiry with configurable retry counts and exponential backoff. +- **Cookie persistence**: Sessions survive process restarts via LWPCookieJar with automatic staleness invalidation (3-day max age). +- **Write-action safety**: Automatically injects `bot=1` and `assertuser` parameters for mutating API requests. +- **Clean exception hierarchy**: Well-structured custom exceptions with a common base class. +- **Continuation pagination**: `post_continue` handles multi-page API responses transparently. + +--- + +## Weaknesses + +- **Code duplication**: `client_request_retry` (lines 693-766) is a copy-paste of `_client_request` logic. Any bug fix in one must be manually replicated in the other. +- **Unused exceptions**: `MaxRetriesExceeded` and `CookieError` are defined and exported but never raised anywhere in the codebase. +- **Shadowed builtins**: The `max` parameter in `post_continue` shadows Python's built-in `max()`. +- **Hardcoded domain workaround**: `mdwiki.org` special case baked into the general-purpose client (line 442). +- **Unused `**kwargs`**: All three request methods accept `**kwargs` but never pass them downstream. + +--- + +## Critical Issues + +### 1. Infinite Loop on Persistent Rate Limiting (HIGH) + +```python +# client.py, lines 236-239 +if error_code == "ratelimited": + time.sleep(3) + continue # <-- attempt is NEVER incremented +``` + +Every other error handler increments `attempt` before `continue`. The `ratelimited` handler does not, creating an infinite loop if the server persistently returns rate-limit errors. + +### 2. `_ensure_logged_in` Does Not Actually Log In (HIGH) + +The method only checks cookie-based revival. If cookies are absent or stale, it silently does nothing. The caller `__init__` proceeds with an unauthenticated session despite the class docstring claiming authentication is ensured. + +### 3. No HTTP Timeout (MEDIUM) + +`_execute_request` calls `self._session.request(...)` without any `timeout` parameter. A stalled server will hang the client indefinitely. + +### 4. `client_request_safe` Silently Swallows All Exceptions (MEDIUM) + +```python +# client.py, lines 674-691 +except Exception: + logger.exception("...") + return {} +``` + +This catches `LoginError`, `CSRFError`, and even programming bugs. Callers cannot distinguish "API returned empty data" from "critical failure occurred." + +### 5. `post_continue` Returns Partial Results Silently (MEDIUM) + +Uses `client_request_safe` for continuation pages. If a continuation page fails, the method returns incomplete data without any indication. + +### 6. Cookie Directory Permissions (LOW) + +`os.chmod` sets `0o770` on the cookies directory, making it readable/writable by any user in the group. Cookie files themselves inherit the default umask (potentially world-readable). + +--- + +## Areas That Need Attention + +- **Missing HTTP timeout**: Add `timeout=(connect, read)` to all HTTP requests. +- **Missing `__init__.py` exports**: `_delete_cookie_file` is a private-underscore function imported across modules -- rename or make public. +- **No context manager**: The client holds resources (`requests.Session`, cookie jar) but provides no `__enter__`/`__exit__` for clean release. +- **No abstract base class**: `RequestsHandler` uses `raise NotImplementedError` instead of `abc.ABC` with `@abstractmethod`. +- **No input validation**: `lang` and `family` parameters are not validated -- empty strings or special characters produce malformed URLs. +- **No structured error context**: Exceptions carry only a message string. Adding `error_code`, `url`, `attempt_count` fields would aid debugging. +- **f-string in logger call** (line 575): Inconsistent with the rest of the module's `%s` formatting and defeats lazy evaluation. + +--- + +## Improvement Plan + +### Quick Wins +1. Increment `attempt` in the `ratelimited` handler to prevent infinite loops. +2. Add `timeout` parameter to `_execute_request`. +3. Remove `client_request_retry` duplication -- delegate to `_client_request`. +4. Raise unused exceptions (`MaxRetriesExceeded`, `CookieError`) or remove them. +5. Fix the f-string logger call to use `%s` formatting. + +### Medium-Term Improvements +1. Implement `_ensure_logged_in` to actually trigger login when cookies are absent. +2. Add `__enter__`/`__exit__` context manager support. +3. Use `abc.ABC` and `@abstractmethod` for `RequestsHandler`. +4. Add proactive rate limiting (minimum delay between requests). +5. Validate `lang` and `family` parameters in the constructor. +6. Make the `mdwiki.org` workaround configurable. + +### Long-Term Refactoring +1. Split `client.py` into separate modules: `transport.py`, `auth.py`, `pagination.py`. +2. Add structured error context to all exceptions. +3. Configure `urllib3.util.retry.Retry` and `HTTPAdapter` at the transport level. +4. Add request/response timing instrumentation. +5. Implement connection pooling configuration. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **6.5/10** | +| **Production Readiness** | Moderate -- functional but has infinite loop bug and missing timeouts | +| **Technical Debt** | Medium -- code duplication, unused exceptions, hardcoded workarounds | +| **Risk Assessment** | Medium-High -- the ratelimited infinite loop and missing timeouts are production risks | +| **Maintainability** | 6/10 -- clear architecture but duplicated code and 850+ line file | diff --git a/newapi/client_wiki/README.md b/newapi/client_wiki/README.md new file mode 100644 index 0000000..f3a7170 --- /dev/null +++ b/newapi/client_wiki/README.md @@ -0,0 +1,204 @@ +# client_wiki -- MediaWiki Bot Client Library + +## Project Overview + +The `client_wiki` package is a high-level MediaWiki bot client library that wraps API interactions for reading, editing, creating, and managing wiki pages. It provides abstractions over the raw MediaWiki API, designed for automated bot editing on Wikimedia projects (primarily Arabic and English Wikipedia). + +### Main Modules + +| Module | Purpose | +|---|---| +| `all_apis.py` | `AllAPIS` facade class -- single entry point for page, category, and API access | +| `constants.py` | Category namespace prefixes by language code | +| `pages/super_page.py` | `MainPage` class -- central abstraction for single wiki page interaction | +| `pages/data.py` | Dataclasses for structured page metadata (Content, Meta, Revisions, etc.) | +| `categories/category_db.py` | `CategoryDepth` -- recursive category tree traversal | +| `categories/catdepth_new.py` | Category depth query functions with caching | +| `api_utils/` | Utility sub-package (see below) | + +### Sub-Package: `api_utils/` + +| Module | Purpose | +|---|---| +| `botEdit.py` | Bot edit permission checks (entry point) | +| `bot_edit/bot_edit_by_time.py` | Time-based edit permission checks | +| `bot_edit/bot_edit_by_templates.py` | Template-based edit permission checks (nobots, bots, stop-edit) | +| `printe.py` | Colored console output and logging helpers | +| `ask_bot.py` | Interactive user confirmation prompts | +| `txtlib.py` | Wikitext template parsing utilities | +| `handel_errors.py` | MediaWiki API error handler | +| `user_agent.py` | User agent string generation | +| `lang_codes.py` | Legacy-to-standard language code mapping | +| `wd_sparql.py` | Wikidata SPARQL query execution | + +### Technologies & Dependencies + +- **`wikitextparser`** (`wtp`) -- Wikitext parsing for template extraction +- **`SPARQLWrapper`** -- Wikidata SPARQL queries +- **`pywikibot`** -- Diff display utilities +- **`tqdm`** -- Progress bars for category traversal +- **`mwclient`** (via `api_client`) -- MediaWiki API transport + +--- + +## Architecture & Code Quality Review + +### Code Organization + +The package is well-organized into logical sub-packages: +- `pages/` -- Page-level operations +- `categories/` -- Category traversal +- `api_utils/` -- Shared utilities +- `api_utils/bot_edit/` -- Edit permission subsystem + +### Design Patterns + +- **Facade**: `AllAPIS` unifies access to `MainPage`, `CategoryDepth`, and `NewApi` behind a single constructor. +- **Mixin/Multiple Inheritance**: `MainPage` inherits from `HandleErrors` and `AskBot`, composing error handling and user-prompt capabilities. +- **Data Transfer Objects**: Dataclasses in `pages/data.py` provide structured page metadata. +- **Lazy Loading**: Many `MainPage` methods check if data is already loaded before making API calls. +- **Caching**: `lru_cache` used in `txtlib.py`, `user_agent.py`, `catdepth_new.py`; manual dict caches in `bot_edit/`. + +### Maintainability + +**Moderate.** The sub-package structure is logical, but naming inconsistencies (PascalCase, snake_case, camelCase), a misspelled filename (`handel_errors.py`), and empty `__init__.py` files with no re-exports make navigation harder. + +### Readability + +**Mixed.** Some modules are clean and well-documented (`category_db.py`), while others mix Arabic and English comments and lack docstrings (`super_page.py` at 1000+ lines). + +### Scalability Considerations + +Category traversal uses `tqdm` for progress, which couples UI to logic. No rate limiting exists at this layer. The `MainPage` class loads data lazily but caches indefinitely within a single instance. + +--- + +## Strengths + +- **Comprehensive page abstraction**: `MainPage` provides 40+ methods covering read, edit, create, purge, metadata, and link analysis. +- **Bot edit safety system**: Multi-layered permission checking (time-based + template-based) prevents editing protected/restricted pages. +- **Facade pattern**: `AllAPIS` provides a clean single-entry-point API. +- **Lazy loading**: Data is fetched only when needed and cached within instances. +- **Wikitext parsing**: `txtlib.py` provides cached template extraction and parameter parsing. +- **Category traversal**: Recursive depth-limited traversal with configurable namespace filtering. + +--- + +## Weaknesses + +- **Empty `__init__.py` files**: Nothing is re-exported from sub-packages, requiring consumers to know internal module paths. +- **Misspelled filename**: `handel_errors.py` should be `handle_errors.py`. +- **Inconsistent naming**: Mix of `snake_case` (`get_text`), `PascalCase` (`Get_tags`, `Create`), and camelCase (`isRedirect`, `isDisambiguation`). +- **`MainPage.Create`** is just an alias for `MainPage.create` -- unnecessary duplication. +- **Global mutable state**: `Bot_Cache` in `bot_edit_by_templates.py` and `_save_or_ask` in `ask_bot.py` persist across sessions and are never cleared. +- **Duplicated data**: `change_codes` dictionary is defined identically in both `api_utils/__init__.py` and `api_utils/lang_codes.py`. + +--- + +## Critical Issues + +### 1. Mutable Default Argument Bug (HIGH) + +```python +# txtlib.py, line 52 +def get_one_temp_params(text, tempname="", templates=[], ...): + temps = templates + temps.append(tempname) # Mutates the shared default list! +``` + +The mutable default `templates=[]` is shared across all calls. Appending to it corrupts subsequent invocations. + +### 2. Bare `raise` With No Exception Context (HIGH) + +```python +# super_page.py, line 1024 +raise # noqa: PLE0704 +``` + +This raises `RuntimeError` or `TypeError` since there is no active exception. The `noqa` comment suppresses the linter warning about this known problem. + +### 3. Type Mismatch in `exists()` (MEDIUM) + +`self.meta.Exists` is typed as `str` with default `""`, but is set to `True` (a bool) when confirmed. The `if not self.meta.Exists` check works by coincidence but the type mismatch is a latent bug. + +### 4. SPARQL Injection Risk (MEDIUM) + +```python +# wd_sparql.py +def get_query_data(query): + sparql.setQuery(query) # No sanitization +``` + +If user-controlled input is interpolated into SPARQL queries upstream, this is an injection vector. + +### 5. Destructive Side Effect in `Get_tags()` (MEDIUM) + +```python +# super_page.py, line 541 +self.text = self.text.replace("", '', 1) +``` + +Permanently alters `self.text` for the instance -- a hidden side effect that can corrupt page content. + +### 6. Inconsistent Return Types from `handle_err` (MEDIUM) + +Returns `False`, `"articleexists"`, a description string, or the original error `dict` depending on the error type. Callers must handle all these types. + +### 7. Empty String Treated as "Yes" (LOW) + +```python +# ask_bot.py, line 57 +yes_list = ["y", "a", "", "Y", "A", "all", "aaa"] +``` + +Empty string `""` (user pressing Enter without intent) is treated as confirmation. + +--- + +## Areas That Need Attention + +- **Missing `__init__.py` re-exports**: Add `__all__` and re-export key classes/functions from each sub-package. +- **Missing docstrings**: Many methods in `MainPage` and `NewApi` lack docstrings. +- **Missing type annotations**: Functions like `get_text()`, `get_qid()`, `purge()` lack return types. +- **File naming**: Rename `handel_errors.py` to `handle_errors.py`; rename `botEdit.py` to `bot_edit_utils.py`. +- **`tqdm` coupling**: Progress bar is hardcoded in `category_db.py` with no way to suppress or replace. +- **Arabic template prefix hardcoded**: `"قالب:"` in `txtlib.py` will be incorrect for non-Arabic wikis. +- **No unit tests**: This entire sub-package has zero dedicated test coverage. + +--- + +## Improvement Plan + +### Quick Wins +1. Fix mutable default argument in `get_one_temp_params` (`templates=None`). +2. Fix the bare `raise` in `super_page.py`. +3. Remove empty string `""` from `ask_bot.py`'s yes_list. +4. Remove duplicated `change_codes` definition. +5. Add `__all__` to each `__init__.py`. + +### Medium-Term Improvements +1. Rename misspelled/misnamed files (`handel_errors.py`, `botEdit.py`). +2. Standardize method naming to `snake_case`. +3. Add type annotations to all public methods. +4. Make `BOT_USERNAME` configurable via `settings`. +5. Replace `tqdm` with a callback/hook pattern. +6. Fix the `Exists` type to be `bool` instead of `str`. + +### Long-Term Refactoring +1. Split `super_page.py` (1000+ lines) into focused modules. +2. Implement proper error handling with typed exceptions instead of mixed return types. +3. Add comprehensive unit tests for all modules. +4. Make the Arabic template prefix configurable. +5. Clear global caches (`Bot_Cache`, `_save_or_ask`) between runs. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **6/10** | +| **Production Readiness** | Moderate -- functional but has bugs and no tests | +| **Technical Debt** | High -- naming inconsistencies, global state, code duplication | +| **Risk Assessment** | Medium -- mutable default bug and type mismatches can cause subtle failures | +| **Maintainability** | 5/10 -- large files, inconsistent naming, empty init files | diff --git a/newapi/client_wiki/api_utils/README.md b/newapi/client_wiki/api_utils/README.md new file mode 100644 index 0000000..3d48357 --- /dev/null +++ b/newapi/client_wiki/api_utils/README.md @@ -0,0 +1,146 @@ +# api_utils -- Wiki Bot Utility Functions + +## Project Overview + +The `api_utils` package provides shared utility functions for the MediaWiki bot client, including bot edit permission checking, error handling, wikitext parsing, logging, and Wikidata SPARQL queries. + +### Main Modules + +| Module | Purpose | +|---|---| +| `botEdit.py` | Entry point for bot edit permission checks (`bot_May_Edit`) | +| `bot_edit/bot_edit_by_time.py` | Time-based edit restrictions (creation time, last edit time) | +| `bot_edit/bot_edit_by_templates.py` | Template-based edit restrictions (nobots, bots, stop-edit templates) | +| `handel_errors.py` | `HandleErrors` class for MediaWiki API error processing | +| `printe.py` | Colored console output and logging helpers (`output`, `error`, `warn`, etc.) | +| `ask_bot.py` | `AskBot` class for interactive user confirmation prompts | +| `txtlib.py` | Wikitext template extraction and parameter parsing | +| `user_agent.py` | User agent string generation | +| `lang_codes.py` | Legacy-to-standard language code mapping (`change_codes`) | +| `wd_sparql.py` | Wikidata SPARQL query execution | + +### Technologies & Dependencies + +- **`wikitextparser`** (`wtp`) -- Wikitext parsing in `txtlib.py` and `bot_edit_by_templates.py` +- **`SPARQLWrapper`** -- Wikidata SPARQL queries in `wd_sparql.py` +- **`pywikibot`** -- Diff display in `printe.py` and `ask_bot.py` + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Fair.** Related files are grouped (`bot_edit/` sub-package), but the flat structure of the parent makes it hard to see logical groupings. The misspelled filename `handel_errors.py` and camelCase `botEdit.py` break conventions. + +### Design Patterns + +- **Strategy**: Bot edit checks compose time-based and template-based strategies. +- **Caching**: `lru_cache` in `txtlib.py`, `user_agent.py`; manual dict caches in `bot_edit/`. +- **Mixin**: `HandleErrors` and `AskBot` are designed as mixins for multiple inheritance. + +### Maintainability + +**Moderate.** Individual files are focused and manageable, but naming inconsistencies and global mutable state reduce maintainability. + +### Readability + +**Mixed.** Some files are clean (`user_agent.py`), while others have inconsistent naming and missing docstrings. + +--- + +## Strengths + +- **Multi-layered bot edit safety**: Time-based + template-based permission checking prevents editing protected pages. +- **Cached parsing**: `extract_templates_and_params` uses `lru_cache` for performance. +- **Flexible error handling**: `HandleErrors.handle_err` processes diverse API error types. +- **Colored output**: Custom `<>text<>` tag system for terminal output. + +--- + +## Weaknesses + +- **Misspelled filename**: `handel_errors.py` should be `handle_errors.py`. +- **camelCase filename**: `botEdit.py` should be `bot_edit.py` or `bot_edit_utils.py`. +- **Global mutable caches**: `Bot_Cache` and `_save_or_ask` never cleared between runs. +- **Hardcoded bot username**: `BOT_USERNAME = "Mr.Ibrahembot"` in `bot_edit_by_templates.py`. +- **Hardcoded Arabic prefix**: `"قالب:"` (Template:) in `txtlib.py` breaks non-Arabic wikis. +- **Duplicated `change_codes`**: Defined identically in both `__init__.py` and `lang_codes.py`. + +--- + +## Critical Issues + +### 1. Mutable Default Argument in `get_one_temp_params` (HIGH) + +```python +# txtlib.py, line 52 +def get_one_temp_params(text, tempname="", templates=[], ...): + temps = templates + temps.append(tempname) # Mutates shared default! +``` + +### 2. Destructive Mutation in `handle_err` (MEDIUM) + +```python +# handel_errors.py, lines 108-110 +params["data"] = {} +params["text"] = {} +``` + +Mutates the caller's params dict as a side effect. + +### 3. SPARQL Injection Risk (MEDIUM) + +`wd_sparql.py` passes queries directly to `SPARQLWrapper.setQuery()` without sanitization. + +### 4. Empty String as "Yes" Confirmation (LOW) + +```python +# ask_bot.py, line 57 +yes_list = ["y", "a", "", "Y", "A", "all", "aaa"] +``` + +--- + +## Areas That Need Attention + +- **Rename files**: `handel_errors.py` -> `handle_errors.py`, `botEdit.py` -> `bot_edit_utils.py`. +- **Fix mutable default** in `txtlib.py`. +- **Make `BOT_USERNAME` configurable** via `settings`. +- **Remove duplicated `change_codes`**. +- **Add `__all__`** to all `__init__.py` files. +- **Add type annotations** to functions missing them. + +--- + +## Improvement Plan + +### Quick Wins +1. Fix mutable default argument in `get_one_temp_params`. +2. Remove empty string from `ask_bot.py`'s yes_list. +3. Remove duplicated `change_codes`. +4. Add `__all__` to `__init__.py`. + +### Medium-Term Improvements +1. Rename misspelled/misnamed files. +2. Make `BOT_USERNAME` configurable. +3. Make Arabic template prefix configurable. +4. Clear global caches between runs. + +### Long-Term Refactoring +1. Replace `handle_err` return-value pattern with exception-based error handling. +2. Replace `tqdm` coupling with callback pattern. +3. Add unit tests for all modules. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **6/10** | +| **Production Readiness** | Moderate -- functional but has naming issues and global state | +| **Technical Debt** | Medium-High -- misspellings, duplication, hardcoded values | +| **Risk Assessment** | Medium -- mutable default bug can cause subtle failures | +| **Maintainability** | 5/10 -- naming inconsistencies, no tests, global state | diff --git a/newapi/client_wiki/api_utils/bot_edit/README.md b/newapi/client_wiki/api_utils/bot_edit/README.md new file mode 100644 index 0000000..732e081 --- /dev/null +++ b/newapi/client_wiki/api_utils/bot_edit/README.md @@ -0,0 +1,117 @@ +# bot_edit -- Bot Edit Permission System + +## Project Overview + +The `bot_edit` sub-package determines whether a bot is allowed to edit a specific wiki page, based on two complementary strategies: time-based restrictions and template-based restrictions. + +### Main Modules + +| Module | Purpose | +|---|---| +| `bot_edit_by_time.py` | Checks if a page was recently created or edited (within a delay window) | +| `bot_edit_by_templates.py` | Checks for `{{nobots}}`, `{{bots}}`, and stop-edit templates in page content | + +### How It Works + +The entry point is `botEdit.py` (parent directory), which calls: + +```python +def bot_May_Edit(text, title_page, botjob, page, delay): + # 1. Check time-based restrictions + if not check_create_time(page, title_page): return False + if not check_last_edit_time(page, title_page, delay): return False + # 2. Check template-based restrictions + if not is_bot_edit_allowed(text, title_page, botjob): return False + return True +``` + +### Technologies & Dependencies + +- **`wikitextparser`** (`wtp`) -- Template extraction from page content +- **Internal**: `config.settings` for bot configuration + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Good.** Clear separation between time-based and template-based checks. + +### Design Patterns + +- **Strategy**: Two independent strategies composed in sequence. +- **Caching**: Manual dict caches (`Bot_Cache`, `_created_cache`) for repeated lookups. + +### Maintainability + +**Moderate.** Files are focused and manageable, but global mutable caches and hardcoded values reduce flexibility. + +--- + +## Strengths + +- **Multi-layered protection**: Combines time-based and template-based checks. +- **Comprehensive template detection**: Handles `{{nobots}}`, `{{bots|allow/disallow}}`, and language-specific stop-edit templates. +- **Caching**: Avoids redundant API calls for repeated page checks. + +--- + +## Weaknesses + +- **Global mutable caches**: `Bot_Cache` and `_created_cache` never expire or clear. +- **Hardcoded bot username**: `BOT_USERNAME = "Mr.Ibrahembot"` should be configurable. +- **No tests for `bot_edit_by_time.py`**: The test file exists but contains only imports. + +--- + +## Critical Issues + +### 1. Stale Cache Entries (MEDIUM) + +`_created_cache` in `bot_edit_by_time.py` never expires. If a page is deleted and re-created, the stale cache returns incorrect results. + +### 2. Hardcoded Bot Username (LOW) + +```python +# bot_edit_by_templates.py, line 23 +BOT_USERNAME = "Mr.Ibrahembot" +``` + +Any bot using this library must have this exact username, or the `{{bots}}` template check will fail. + +--- + +## Areas That Need Attention + +- **Make caches configurable** with TTL or clear-between-runs semantics. +- **Make `BOT_USERNAME` configurable** via `settings`. +- **Add tests for `bot_edit_by_time.py`**. + +--- + +## Improvement Plan + +### Quick Wins +1. Make `BOT_USERNAME` read from `settings` instead of hardcoding. +2. Add cache clearing mechanism. + +### Medium-Term Improvements +1. Add TTL to caches. +2. Add unit tests for `bot_edit_by_time.py`. + +### Long-Term Refactoring +1. Convert caches to instance-level state instead of module globals. +2. Make template names configurable per wiki language. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **6.5/10** | +| **Production Readiness** | Moderate -- works but has stale cache and hardcoded values | +| **Technical Debt** | Medium -- global state, missing tests | +| **Risk Assessment** | Low-Medium -- stale cache can cause incorrect edit decisions | +| **Maintainability** | 6/10 -- clean separation but global state | diff --git a/newapi/client_wiki/categories/README.md b/newapi/client_wiki/categories/README.md new file mode 100644 index 0000000..086a7af --- /dev/null +++ b/newapi/client_wiki/categories/README.md @@ -0,0 +1,100 @@ +# categories -- MediaWiki Category Tree Traversal + +## Project Overview + +The `categories` sub-package provides functionality for recursively traversing MediaWiki category trees to a given depth, collecting member pages with their metadata. + +### Main Modules + +| Module | Purpose | +|---|---| +| `category_db.py` | `CategoryDepth` class -- recursive category traversal with namespace filtering, template/langlink/category merging | +| `catdepth_new.py` | `subcatquery` function -- entry point with title processing and argument grouping | + +### Technologies & Dependencies + +- **`tqdm`** -- Progress bars during traversal +- **Internal**: `WikiLoginClient` for API access, `function_timer` for profiling + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Good.** Two focused files: `category_db.py` handles the traversal logic, `catdepth_new.py` provides the entry point. + +### Design Patterns + +- **Recursive Traversal**: `subcatquery_()` recurses into subcategories up to the configured depth. +- **Caching**: `lru_cache` on `title_process` for memoization. +- **Progress Reporting**: `tqdm` for visual progress during long traversals. + +### Maintainability + +**Moderate.** The `CategoryDepth` class has many methods but they are well-named and focused. + +--- + +## Strengths + +- **Depth-limited recursion**: Configurable depth prevents infinite traversal. +- **Namespace filtering**: Can filter results by namespace. +- **Metadata merging**: Can merge templates, langlinks, and categories into results. +- **Progress bars**: `tqdm` provides visual feedback for long operations. + +--- + +## Weaknesses + +- **`tqdm` coupling**: Progress bar is hardcoded with no way to suppress or replace. +- **Missing error handling**: `int(xx["ns"])` can raise `KeyError` or `ValueError` if `ns` is missing or non-numeric. +- **No return type annotations** on many methods. + +--- + +## Critical Issues + +### 1. Potential `KeyError`/`ValueError` (MEDIUM) + +```python +# category_db.py, line 306 +int(xx["ns"]) # KeyError if "ns" missing, ValueError if non-numeric +``` + +No try/except protection. + +--- + +## Areas That Need Attention + +- **Add error handling** for `ns` field access. +- **Make `tqdm` optional** via callback or flag. +- **Add return type annotations**. + +--- + +## Improvement Plan + +### Quick Wins +1. Add try/except for `int(xx["ns"])` calls. +2. Add type annotations to all methods. + +### Medium-Term Improvements +1. Replace `tqdm` with a callback pattern. +2. Add unit tests. + +### Long-Term Refactoring +1. Make traversal async-capable for large category trees. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **6.5/10** | +| **Production Readiness** | Moderate -- functional but fragile error handling | +| **Technical Debt** | Medium -- UI coupling, missing types | +| **Risk Assessment** | Medium -- KeyError on malformed API responses | +| **Maintainability** | 6/10 -- clean logic but missing tests | diff --git a/newapi/client_wiki/pages/README.md b/newapi/client_wiki/pages/README.md new file mode 100644 index 0000000..0ca2ade --- /dev/null +++ b/newapi/client_wiki/pages/README.md @@ -0,0 +1,124 @@ +# pages -- Wiki Page Abstraction + +## Project Overview + +The `pages` sub-package provides the `MainPage` class, the central abstraction for interacting with a single MediaWiki wiki page. It supports reading, editing, creating, purging, and querying page metadata. + +### Main Modules + +| Module | Purpose | +|---|---| +| `super_page.py` | `MainPage` class (1000+ lines, 40+ methods) and `find_edit_error` helper | +| `data.py` | Dataclasses for structured page metadata | + +### Dataclasses (`data.py`) + +| Class | Fields | +|---|---| +| `Content` | `text_html`, `summary`, `words`, `length` | +| `Meta` | `is_disambig`, `can_be_edit`, `userinfo`, `create_data`, `info`, `username`, `Exists`, `is_redirect`, `flagged`, `wikibase_item` | +| `RevisionsData` | `revid`, `newrevid`, `pageid`, `timestamp`, `revisions`, `touched` | +| `LinksData` | `back_links`, `extlinks`, `iwlinks`, `links_here`, `links`, `links2` | +| `CategoriesData` | `categories`, `hidden_categories`, `all_categories_with_hidden` | +| `TemplateData` | `templates`, `templates_api` | + +### Technologies & Dependencies + +- **`wikitextparser`** (`wtp`) -- Template extraction in `get_templates()` +- **Internal**: `WikiLoginClient`, `HandleErrors`, `AskBot` + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Fair.** Two files, but `super_page.py` at 1000+ lines is a god class with 40+ methods. + +### Design Patterns + +- **Lazy Loading**: Data fetched only when requested, cached within the instance. +- **Mixin Composition**: `MainPage(HandleErrors, AskBot)` composes error handling and user prompts. +- **Data Transfer Objects**: Dataclasses in `data.py` provide structured metadata. + +### Maintainability + +**Low-Moderate.** The 1000+ line class with mixed naming conventions is hard to navigate. + +--- + +## Strengths + +- **Comprehensive page API**: 40+ methods covering read, edit, create, purge, metadata, links, categories, templates. +- **Lazy loading**: Data fetched only when needed. +- **Structured metadata**: Dataclasses provide clear data shapes. + +--- + +## Weaknesses + +- **God class**: `MainPage` at 1000+ lines with 40+ methods. +- **Inconsistent naming**: `get_text` (snake_case) vs `isRedirect` (camelCase) vs `Get_tags` (PascalCase). +- **Type mismatch**: `Exists` field is `str` but set to `True` (bool). +- **Side effects**: `Get_tags()` permanently mutates `self.text`. + +--- + +## Critical Issues + +### 1. Bare `raise` With No Exception Context (HIGH) + +```python +# super_page.py, line 1024 +raise # noqa: PLE0704 +``` + +### 2. Type Mismatch in `exists()` (MEDIUM) + +`self.meta.Exists` typed as `str`, default `""`, set to `True` (bool). + +### 3. Destructive Side Effect in `Get_tags()` (MEDIUM) + +```python +self.text = self.text.replace("", '', 1) +``` + +--- + +## Areas That Need Attention + +- **Split `MainPage`** into focused mixins or separate classes. +- **Fix type mismatch** for `Exists`. +- **Standardize naming** to `snake_case`. +- **Add type annotations** to all public methods. +- **Add `__all__`** to `data.py`. + +--- + +## Improvement Plan + +### Quick Wins +1. Fix the bare `raise`. +2. Change `Exists` type to `bool`. +3. Add `__all__` to `data.py`. + +### Medium-Term Improvements +1. Standardize naming to `snake_case`. +2. Add type annotations. +3. Split read and write operations. + +### Long-Term Refactoring +1. Split `MainPage` into `PageReader`, `PageWriter`, `PageMetadata` mixins. +2. Add comprehensive unit tests. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **6/10** | +| **Production Readiness** | Moderate -- functional but has bugs and is hard to maintain | +| **Technical Debt** | High -- god class, naming issues, type mismatches | +| **Risk Assessment** | Medium -- bare raise, type mismatches, side effects | +| **Maintainability** | 5/10 -- 1000+ line class, no tests, mixed naming | diff --git a/newapi/core/README.md b/newapi/core/README.md new file mode 100644 index 0000000..7bb9689 --- /dev/null +++ b/newapi/core/README.md @@ -0,0 +1,118 @@ +# core -- Exception Hierarchy for MediaWiki API Errors + +## Project Overview + +The `core` package defines a structured exception hierarchy for MediaWiki API errors, along with a factory function that maps API error responses to typed exception instances. + +### Main Modules + +| Module | Purpose | +|---|---| +| `exceptions.py` | Exception classes and `parse_api_error` factory function | + +### Exception Hierarchy + +``` +NewApiException (base) + +-- ApiError + | +-- AbuseFilterError + | +-- MaxLagError + | +-- ArticleExistsError + | +-- NoSuchEntityError + | +-- ProtectedPageError + | +-- InvalidTokenError + +-- AuthenticationError + +-- ValidationError +``` + +### Technologies & Dependencies + +- **Standard library only**: `typing.Any`, `Dict`, `Optional` + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Good.** Single file with a clear, well-structured exception hierarchy. The `parse_api_error` factory provides a clean mapping from API error dicts to typed exceptions. + +### Design Patterns + +- **Exception Hierarchy**: All exceptions inherit from `NewApiException`, enabling catch-all handling at any level. +- **Factory Method**: `parse_api_error(error_dict)` maps raw API error responses to the appropriate exception type. + +### Maintainability + +**High.** Simple, focused module with minimal complexity. + +### Readability + +**Good.** Clear class names and inheritance structure. + +### Scalability Considerations + +Not applicable -- this is a data/exception module with no performance concerns. + +--- + +## Strengths + +- **Well-structured hierarchy**: Clear inheritance tree covering common MediaWiki API error types. +- **Factory function**: `parse_api_error` provides a single point for error-to-exception mapping. +- **Structured error data**: Each exception carries `code`, `info`, and `message` fields. +- **No external dependencies**: Pure standard library implementation. + +--- + +## Weaknesses + +- **Missing `__init__.py`**: The `core/` directory lacks an `__init__.py`, making it inconsistent with other sub-packages (works as namespace package but is unconventional). +- **No `__all__`**: All classes are implicitly public. +- **Unused in codebase**: The exception hierarchy is defined but largely unused -- the rest of the codebase returns mixed types (`bool`, `str`, `dict`) instead of raising these exceptions. +- **No `__str__` override**: Exceptions use the default `Exception.__str__`, which just returns the message. Adding structured fields to `__repr__` would aid debugging. + +--- + +## Critical Issues + +**None.** This is a simple, well-implemented module with no bugs or security concerns. + +--- + +## Areas That Need Attention + +- **Add `__init__.py`**: Create `core/__init__.py` with re-exports for consistency. +- **Adopt exceptions throughout codebase**: The rest of `newapi` should raise these exceptions instead of returning mixed error types. +- **Add `__all__`**: Explicitly declare the public API surface. +- **Add docstrings**: Classes and the factory function lack docstrings. + +--- + +## Improvement Plan + +### Quick Wins +1. Add `core/__init__.py` with `__all__` and re-exports. +2. Add docstrings to all exception classes. +3. Add `__repr__` to `NewApiException` showing `code` and `info`. + +### Medium-Term Improvements +1. Migrate `handle_err` in `handel_errors.py` to raise these exceptions instead of returning mixed types. +2. Add more exception types as needed (e.g., `RateLimitError`, `NetworkError`). +3. Add structured context fields (url, attempt_count) to exceptions. + +### Long-Term Refactoring +1. Unify the two exception hierarchies (`core/exceptions.py` and `api_client/exceptions.py`) into one. +2. Implement exception-based error handling throughout the entire codebase. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **7.5/10** | +| **Production Readiness** | High -- simple, correct, well-structured | +| **Technical Debt** | Low -- minor missing init file and docs | +| **Risk Assessment** | Low -- no bugs, no security concerns | +| **Maintainability** | 9/10 -- clean, focused, minimal complexity | diff --git a/newapi/super/README.md b/newapi/super/README.md new file mode 100644 index 0000000..0354eb3 --- /dev/null +++ b/newapi/super/README.md @@ -0,0 +1,178 @@ +# super -- High-Level MediaWiki Bot API + +## Project Overview + +The `super` package provides the `NewApi` class, the main high-level interface for MediaWiki bot operations including page queries, searches, user contributions, file uploads, and page editing. + +### Main Modules + +| Module | Purpose | +|---|---| +| `S_API/bot_api.py` | `NewApi` class -- 30+ methods for wiki operations | + +### Sub-Package Structure + +``` +super/ + __init__.py # Re-exports bot_api + S_API/ + __init__.py # Empty + bot_api.py # NewApi class (1300+ lines) +``` + +### Technologies & Dependencies + +- **`tqdm`** -- Progress bars for batch operations +- **Internal**: `WikiLoginClient`, `HandleErrors`, `AskBot`, `change_codes` + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Fair.** The `NewApi` class in `bot_api.py` is 1300+ lines with 30+ methods covering diverse functionality (search, upload, move, edit, query). This god-class would benefit from decomposition. + +### Design Patterns + +- **Facade**: Wraps `WikiLoginClient` behind a high-level API. +- **Template Method / Mixin**: Inherits from `HandleErrors` and `AskBot`. +- **Delegation**: Most methods delegate to `login_bot.client_request` or `login_bot.post_continue`. + +### Maintainability + +**Low.** A single 1300+ line class with mixed naming conventions, no docstrings on most methods, and Arabic/English comment mixing. + +### Readability + +**Low-Moderate.** Method names are inconsistent (PascalCase, snake_case, mixed). Many methods lack docstrings and type annotations. + +### Scalability Considerations + +Batch operations like `Find_pages_exists_or_not_with_qids` use chunking with `tqdm`, which is good for large datasets. However, `chunk_titles` with `tqdm` couples UI to logic. + +--- + +## Strengths + +- **Comprehensive API coverage**: 30+ methods covering search, query, edit, upload, move, and more. +- **Batch processing**: `chunk_titles` and `Find_pages_exists_or_not_with_qids` handle large title lists efficiently. +- **Continuation support**: `post_continue` handles multi-page API responses. +- **Mixin composition**: `HandleErrors` and `AskBot` provide reusable error handling and user prompts. + +--- + +## Weaknesses + +- **God class**: `NewApi` at 1300+ lines with 30+ methods violates single responsibility. +- **Inconsistent naming**: Mix of `PascalCase` (`Find_pages_exists_or_not`), `snake_case` (`get_logs`), and mixed (`Add_To_Bottom`). +- **Missing docstrings**: Most methods lack documentation. +- **Missing type annotations**: Many methods have no return type hints. +- **Shadowed builtins**: `max` parameter used in multiple methods shadows Python's built-in `max()`. +- **Arabic/English comment mixing**: Hinders maintainability for non-Arabic-speaking developers. + +--- + +## Critical Issues + +### 1. Infinite Recursion on Rate Limit in `move()` (HIGH) + +```python +# bot_api.py, line 1180 +if error_code == "ratelimited": + return self.move(old_title, to, reason, ...) # No retry limit! +``` + +If the API persistently returns `ratelimited`, this recurses infinitely until stack overflow. No delay, no retry counter, no backoff. + +### 2. `KeyError` Bug in `PrefixSearch()` (HIGH) + +```python +# bot_api.py, line 299 +del params["apnamespace"] # Bug: key is "psnamespace" +``` + +The params dict uses `"psnamespace"`, not `"apnamespace"`. This will raise `KeyError` when the code path is taken. + +### 3. `AttributeError` in `querypage_list()` (HIGH) + +```python +# bot_api.py, line 697 +if qplimit.isdigit(): # qplimit defaults to None! +``` + +`qplimit` defaults to `None`, and `.isdigit()` is called on it, raising `AttributeError`. + +### 4. File Handle Leak in `upload_by_file()` (MEDIUM) + +```python +# bot_api.py, line 1269 +file = open(file_path, "rb") # Never closed +data = self.login_bot.client_request_safe(params, files={"file": file}) +``` + +No context manager is used. If the request fails, the file handle leaks. + +### 5. `AttributeError` Risk in `upload_by_file()` (MEDIUM) + +```python +# bot_api.py, line 1271 +result = data.get("upload", {}) # data could be None from client_request_safe +``` + +`client_request_safe` can return `{}` on failure, but the preceding assignment could also yield `None` in edge cases. + +### 6. `get_username()` Always Returns `""` (MEDIUM) + +```python +# bot_api.py, __init__ +self.username = getattr(self, "username", "") # Always "" since username not set yet +``` + +The `getattr` reads `self.username` before it's ever set, so it always gets the default `""`. + +--- + +## Areas That Need Attention + +- **Split the god class**: Separate search, query, edit, upload, and utility methods into focused classes or modules. +- **Fix the three crash bugs**: `PrefixSearch` KeyError, `querypage_list` AttributeError, `move` infinite recursion. +- **Add docstrings and type annotations** to all public methods. +- **Standardize naming** to `snake_case`. +- **Use context managers** for file handles in `upload_by_file`. +- **Fix dead code**: Hardcoded `"ususers": "Mr.Ibrahembot"` in `users_infos`. + +--- + +## Improvement Plan + +### Quick Wins +1. Fix `PrefixSearch` to delete `"psnamespace"` instead of `"apnamespace"`. +2. Fix `querypage_list` to handle `None` qplimit. +3. Add retry limit to `move()` recursion. +4. Use `with open(...)` in `upload_by_file`. +5. Fix `get_username()` to read from `login_bot`. + +### Medium-Term Improvements +1. Add docstrings and return type annotations to all methods. +2. Standardize method naming to `snake_case`. +3. Remove dead code (hardcoded `ususers`, unused `_error` variables). +4. Remove `tqdm` coupling from `chunk_titles`. + +### Long-Term Refactoring +1. Split `NewApi` into focused classes: `SearchApi`, `QueryApi`, `EditApi`, `UploadApi`. +2. Implement proper error handling with the `core/exceptions.py` hierarchy. +3. Add comprehensive unit tests. +4. Make `Get_All_pages_generator` actually return a generator. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **5/10** | +| **Production Readiness** | Low -- three crash bugs and a god class | +| **Technical Debt** | High -- 1300+ line class, inconsistent naming, no docs | +| **Risk Assessment** | High -- infinite recursion, KeyError, AttributeError in common code paths | +| **Maintainability** | 4/10 -- god class, no tests, mixed naming, mixed languages | diff --git a/newapi/super/S_API/README.md b/newapi/super/S_API/README.md new file mode 100644 index 0000000..9627e3f --- /dev/null +++ b/newapi/super/S_API/README.md @@ -0,0 +1,103 @@ +# S_API -- MediaWiki Bot API Implementation + +## Project Overview + +The `S_API` package contains the `NewApi` class, the primary high-level interface for MediaWiki bot operations. This is where the bulk of the bot's wiki interaction logic lives. + +### Main Modules + +| Module | Purpose | +|---|---| +| `bot_api.py` | `NewApi` class -- 30+ methods for search, query, edit, upload, and page manipulation | + +### Technologies & Dependencies + +- **`tqdm`** -- Progress bars for batch operations +- **Internal**: `WikiLoginClient`, `HandleErrors`, `AskBot`, `change_codes` + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Fair.** Single file (`bot_api.py`) containing one large class with 30+ methods spanning 1300+ lines. + +### Design Patterns + +- **Facade**: Wraps `WikiLoginClient` behind a high-level API. +- **Mixin**: Inherits `HandleErrors` and `AskBot` for error handling and user prompts. +- **Delegation**: Most methods delegate to `login_bot.client_request` or `login_bot.post_continue`. + +### Maintainability + +**Low.** The god-class approach with 1300+ lines, mixed naming, and no docstrings makes this the hardest module to maintain. + +--- + +## Strengths + +- **Comprehensive coverage**: 30+ methods covering search, query, edit, upload, move, and more. +- **Batch processing**: Chunking with `tqdm` for large operations. +- **Delegation pattern**: Clean separation from transport layer. + +--- + +## Weaknesses + +- **God class**: 1300+ lines, 30+ methods in a single class. +- **Inconsistent naming**: PascalCase, snake_case, and mixed conventions. +- **No docstrings** on most methods. +- **No type annotations** on many methods. +- **Shadowed builtins**: `max` parameter in multiple methods. + +--- + +## Critical Issues + +See the parent `super/README.md` for the full critical issues list. Key bugs in this file: + +1. `PrefixSearch` deletes wrong dict key -- `KeyError` (line 299) +2. `querypage_list` calls `.isdigit()` on `None` -- `AttributeError` (line 697) +3. `move()` infinite recursion on `ratelimited` (line 1180) +4. `upload_by_file` file handle leak (line 1269) +5. `get_username()` always returns `""` (line 26) + +--- + +## Areas That Need Attention + +- **Split into focused modules**: Search, Query, Edit, Upload, Utilities. +- **Fix all crash bugs** listed above. +- **Add docstrings and type annotations**. +- **Standardize naming** to `snake_case`. + +--- + +## Improvement Plan + +### Quick Wins +1. Fix the three crash bugs (KeyError, AttributeError, infinite recursion). +2. Use context manager for file handles. +3. Fix `get_username()` to read from `login_bot`. + +### Medium-Term Improvements +1. Add docstrings and type annotations. +2. Standardize method naming. +3. Remove dead code. + +### Long-Term Refactoring +1. Split `NewApi` into focused classes/modules. +2. Add comprehensive unit tests. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **4.5/10** | +| **Production Readiness** | Low -- multiple crash bugs in common code paths | +| **Technical Debt** | Very High -- god class, no docs, no tests, crash bugs | +| **Risk Assessment** | High -- KeyError, AttributeError, infinite recursion | +| **Maintainability** | 3/10 -- 1300+ line god class with no tests | diff --git a/newapi/utils/README.md b/newapi/utils/README.md new file mode 100644 index 0000000..d0ffeb3 --- /dev/null +++ b/newapi/utils/README.md @@ -0,0 +1,92 @@ +# utils -- Shared Utility Functions + +## Project Overview + +The `utils` package provides shared utility functions used across the `newapi` codebase. + +### Main Modules + +| Module | Purpose | +|---|---| +| `functions_timer.py` | `function_timer` decorator for profiling function execution time | + +### Technologies & Dependencies + +- **Standard library only**: `functools`, `logging`, `time` + +--- + +## Architecture & Code Quality Review + +### Code Organization + +**Good.** Minimal, focused package with a single purpose. + +### Design Patterns + +- **Decorator Pattern**: `function_timer` wraps functions to measure and log execution time. + +### Maintainability + +**High.** Simple, self-contained utility with no external dependencies. + +### Readability + +**Good.** Clear implementation using `time.perf_counter()` and `functools.wraps`. + +### Scalability Considerations + +Not applicable -- lightweight utility with negligible overhead. + +--- + +## Strengths + +- **Correct timing**: Uses `time.perf_counter()` which is the right choice for measuring elapsed time. +- **Proper decorator**: Uses `functools.wraps` to preserve the wrapped function's metadata. +- **Minimal overhead**: Only logs at DEBUG level, so no impact in production unless debug logging is enabled. + +--- + +## Weaknesses + +- **No exception handling**: If the wrapped function raises, the timing is never recorded. +- **Fixed log level**: Only logs at DEBUG level with no option to customize. +- **No return value**: The decorator doesn't propagate the wrapped function's return value... wait, it does (`return func(...)`). This is fine. + +--- + +## Critical Issues + +**None.** This is a simple, correct utility. + +--- + +## Areas That Need Attention + +- **Add `try/finally`** to ensure timing is logged even when exceptions occur. +- **Add configurable log level** parameter. + +--- + +## Improvement Plan + +### Quick Wins +1. Wrap the function call in `try/finally` to always record timing. +2. Add `level` parameter with default `logging.DEBUG`. + +### Medium-Term Improvements +1. Add optional threshold logging (only log if execution exceeds N seconds). +2. Add cumulative statistics (min/max/avg) for repeated calls. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **7/10** | +| **Production Readiness** | High -- simple, correct, minimal | +| **Technical Debt** | Low -- minor improvement needed | +| **Risk Assessment** | Low -- no bugs, no security concerns | +| **Maintainability** | 9/10 -- clean, focused, well-implemented | diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..0bce870 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,161 @@ +# tests -- Test Suite + +## Project Overview + +The `tests/` directory contains the test suite for the `newapi` package, using **pytest** as the test framework. Tests are organized into top-level integration tests and `unit/` subdirectory tests. + +### Test Structure + +``` +tests/ + __init__.py # Package marker + conftest.py # Shared fixtures (socket disabling, mock clients) + TestAuthentication.py # WikiLoginClient login tests + TestNewAPI.py # NewApi basic tests + TestALL_APIS.py # AllAPIS facade tests + TestMainPage.py # MainPage operation tests + TestLiteDB.py # LiteDB database tests + test_mdwiki_page.py # Empty file (0 bytes) + unit/ + api_client/ + test_client.py # WikiLoginClient unit tests (17 tests) + test_cookies.py # Cookie management tests (9 tests) + test_exceptions.py # Exception hierarchy tests (9 tests) + test_requests_handler.py # RequestsHandler tests (13 tests, 1 class skipped) + api_utils/ + bot_edit/ + test_bot_edit_by_time.py # Stub -- imports only, no tests + test_bot_edit_by_templates.py # Template-based edit checks (~30 tests) + test_bot_edit_by_templates2.py # Extended template tests (~40 tests) + test_bot_edit_by_templates_pypass.py # Bypass condition tests (6 tests) +``` + +### Technologies & Dependencies + +- **`pytest`** -- Test framework +- **`pytest-socket`** -- Disables network access in unit tests +- **`unittest.mock`** -- Mocking (`MagicMock`, `patch`) + +--- + +## Architecture & Code Quality Review + +### Test Organization + +**Fair.** Clear separation between integration tests (top-level) and unit tests (`unit/`). However, naming is inconsistent (PascalCase vs snake_case filenames). + +### Test Patterns + +- **Socket disabling**: `conftest.py` autouse fixture prevents accidental network calls. +- **Mock factories**: `_make_client()` helper in unit tests creates properly configured mocks. +- **Parametrized tests**: `test_bot_edit_by_templates2.py` uses `@pytest.mark.parametrize` extensively. +- **Cache isolation**: Bot edit tests clear `Bot_Cache` before/after each test. + +### Coverage + +| Category | Status | +|---|---| +| `api_client/` | **Good** -- 48 tests across 4 files | +| `bot_edit/` | **Good** -- ~76 tests across 3 files | +| `client_wiki/` | **Poor** -- Only 4 tautological tests in `TestALL_APIS.py` | +| `super/S_API/` | **Poor** -- Only 2 tautological tests in `TestNewAPI.py` | +| `DB_bots/` | **Poor** -- 3 tests (1 skipped) in `TestLiteDB.py` | +| `core/` | **None** -- No tests | +| `config.py` | **None** -- No tests | +| `logging_config.py` | **None** -- No tests | +| `utils/` | **None** -- No tests | +| 20+ other modules | **None** -- No tests | + +--- + +## Strengths + +- **Good unit test quality**: `api_client/` and `bot_edit/` tests use proper mocking, parametrization, and edge case coverage. +- **Network isolation**: `pytest_socket.disable_socket` prevents accidental API calls. +- **Cache isolation**: Bot edit tests properly clear global state between runs. +- **Exception hierarchy testing**: `test_exceptions.py` validates inheritance chains. + +--- + +## Weaknesses + +- **Tautological assertions**: Top-level tests (`TestAuthentication`, `TestNewAPI`, `TestMainPage`) assert `result is not None` on MagicMock objects -- always passes. +- **Empty test bodies**: `test_empty_page_content` and `test_page_title_validation` have no logic. +- **Empty stub files**: `test_mdwiki_page.py` (0 bytes) and `test_bot_edit_by_time.py` (imports only). +- **Permanently skipped tests**: `TestLiteDB.test_create_table` and `TestPostContinue` class. +- **Duplicated helper**: `_make_client()` identically defined in `test_client.py` and `test_requests_handler.py`. +- **20+ untested modules**: Configuration, logging, database, text processing, category queries, and Wikidata SPARQL have zero test coverage. + +--- + +## Critical Issues + +### 1. Tautological Tests Provide False Confidence (HIGH) + +```python +# TestAuthentication.py +def test_successful_login(self, mock_login_client): + response = mock_login_client.client_request({"action": "query"}) + assert response is not None # Always True for MagicMock + assert len(response) > 0 # Always True for MagicMock +``` + +These tests can never fail and provide no safety net. + +### 2. Empty Test Files Mislead Coverage Reports (MEDIUM) + +`test_mdwiki_page.py` and `test_bot_edit_by_time.py` exist but contain no tests, giving a false impression of coverage. + +### 3. Direct `sys.argv` Mutation (MEDIUM) + +```python +# test_bot_edit_by_templates.py +sys.argv = ["script"] # Fragile -- if test fails, teardown may not run +``` + +Should use `unittest.mock.patch("sys.argv", [...])`. + +--- + +## Areas That Need Attention + +- **Fix or remove tautological tests** -- they provide no value. +- **Remove empty stub files** or implement actual tests. +- **Extract `_make_client()`** to a shared conftest or helper module. +- **Add tests for untested modules** (config, logging, DB, categories, SPARQL, etc.). +- **Fix skipped tests** or document why they remain skipped. +- **Use `patch("sys.argv")`** instead of direct mutation. + +--- + +## Improvement Plan + +### Quick Wins +1. Remove or fix tautological assertions in top-level tests. +2. Remove empty test files (`test_mdwiki_page.py`, `test_bot_edit_by_time.py`). +3. Extract `_make_client()` to `tests/unit/conftest.py`. +4. Use `patch("sys.argv")` in bot_edit tests. + +### Medium-Term Improvements +1. Add unit tests for `config.py`, `logging_config.py`, `pformat.py`. +2. Add unit tests for `db_bot.py` and `pymysql_bot.py`. +3. Add unit tests for `txtlib.py`, `wd_sparql.py`, `handel_errors.py`. +4. Fix or remove permanently skipped tests. + +### Long-Term Refactoring +1. Add integration tests with a test wiki instance. +2. Add coverage reporting to CI/CD. +3. Aim for 80%+ code coverage on core modules. +4. Add property-based testing for wikitext parsing. + +--- + +## Comprehensive Review + +| Metric | Score | +|---|---| +| **Overall Rating** | **5/10** | +| **Production Readiness** | Low -- most modules untested, tautological tests | +| **Technical Debt** | High -- empty files, duplicates, inconsistent patterns | +| **Risk Assessment** | High -- 20+ untested modules, false confidence from tautological tests | +| **Maintainability** | 5/10 -- good unit tests exist but coverage is very incomplete | From de01d1d695b626e92655249ed5001634acf053cd Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Wed, 27 May 2026 23:48:07 +0300 Subject: [PATCH 4/5] Create CLAUDE.md --- CLAUDE.md | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..e8d3413 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,117 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +A Python library for automated bot operations on MediaWiki projects (Wikipedia, Wikidata). Provides authenticated API access, page reading/editing/creation, recursive category traversal, Wikidata SPARQL queries, and local database storage via SQLite and MySQL. Used as a dependency by other bot repositories in this ecosystem. + +## Commands + +### Run Tests + +```bash +pytest # All tests (network disabled by default) +pytest -m unit # Unit tests only +pytest -m network # Network-dependent tests +pytest -k "test_name" # Single test by name +pytest --cov=newapi --cov-report=term-missing --cov-branch # With coverage +``` + +`pytest.ini` sets `pythonpath = newapi`, `testpaths = tests`, `--maxfail=25`, `--durations=10`, `--strict-markers`. Network tests excluded by default via `-m "not network"`. + +The `conftest.py` autouse fixture disables all network sockets via `pytest-socket`. + +### Lint & Format + +```bash +ruff check . # Lint +ruff check --fix . # Lint with auto-fix +ruff format . # Format +black . # Format (alternative) +isort . # Sort imports +mypy . # Type check +``` + +Line length is 120 across all tools. Target Python 3.13. + +### Install Dependencies + +```bash +pip install -r requirements.in # Runtime +pip install -r requirements-dev.txt # Testing +``` + +Runtime: `PyMySQL`, `Requests`, `sqlite_utils`, `tqdm`, `wikitextparser`, `mwclient`, `ratelimiter`, `SPARQLWrapper`, `colorlog`, `python-dotenv`, `pywikibot`. +Dev: `pytest`, `pytest-cov`, `pytest-mock`, `pytest-socket`. + +## Architecture + +Three-layer architecture: + +``` +Layer 1: Transport + Auth (newapi/api_client/) + RequestsHandler (retry/CSRF/maxlag) -> WikiLoginClient (auth, cookies, pagination) + Exceptions: WikiClientError -> LoginError, CSRFError, MaxlagError, MaxRetriesExceeded, CookieError + +Layer 2: Wiki Business Logic (newapi/client_wiki/) + AllAPIS (main facade) -> MainPage (page ops), CategoryDepth (recursive traversal), NewApi (bulk ops) + api_utils/: AskBot, HandleErrors, botEdit, txtlib, wd_sparql, lang_codes + +Layer 3: High-Level Operations (newapi/super/S_API/bot_api.py) + NewApi class: 30+ methods (search, listing, contributions, upload, move, langlinks, etc.) +``` + +### Key Abstractions + +- **`AllAPIS`** (`client_wiki/all_apis.py`) -- Main facade. `AllAPIS(lang, family, username, password)` creates authenticated client. Methods: `.MainPage(title)`, `.CatDepth(category)`, `.NewApi()`. +- **`WikiLoginClient`** (`api_client/client.py`) -- Authenticated HTTP client wrapping `mwclient.Site`. Handles CSRF tokens, maxlag backoff, session recovery, cookie persistence. +- **`MainPage`** (`client_wiki/pages/super_page.py`) -- Page-level operations: read, edit, create, get categories/templates/links, check editability. Inherits `HandleErrors` and `AskBot`. +- **`CategoryDepth`** (`client_wiki/categories/category_db.py`) -- Recursive category member retrieval with depth, namespace, template, and language-link filtering. +- **`NewApi`** (`super/S_API/bot_api.py`) -- 30+ high-level operations delegating to `WikiLoginClient`. +- **`Settings`** (`config.py`) -- Dataclass singleton loaded from env vars. Sections: `WikidataConfig`, `ApiClientConfig`, `DatabaseConfig`, `DebugConfig`, `BotConfig`, `QueryConfig`, `SiteConfig`. + +### Usage Pattern + +```python +from newapi import AllAPIS + +api = AllAPIS(lang='en', family='wikipedia', username='user', password='pass') +page = api.MainPage("Article Title") +text = page.get_text() +page.save(newtext, summary="Edit summary") +``` + +Legacy convenience (deprecated): +```python +from newapi.page import load_main_api +api = load_main_api("en", "wikipedia") +``` + +### Cookie Management + +Sessions persisted as Mozilla-format cookie jar files. Auto-expire after 3 days. Path: `{cookies_dir}/{family}_{lang}_{username}.mozilla`. `COOKIES_DIR` env var or defaults to `~/tmp/cookies`. + +### Error Handling + +Two parallel exception hierarchies: +1. `api_client/exceptions.py`: `WikiClientError` -> `LoginError`, `CSRFError`, `MaxlagError`, `MaxRetriesExceeded`, `CookieError` +2. `core/exceptions.py`: `NewApiException` -> `ApiError` -> `AbuseFilterError`, `MaxLagError`, `ArticleExistsError`, `ProtectedPageError`, etc. Includes `parse_api_error()` for mapping API error dicts. + +### Patterns to Know + +- **`{1: value}` mutable dict pattern** -- used for module-level mutable state. +- **Module-level side effects** -- importing modules can trigger login, computation, or sys.argv modification. +- **Mixed naming conventions** -- PascalCase classes, snake_case functions, some camelCase mixed in. +- **Mixed Arabic/English** in comments and string literals. + +## Environment + +Credentials from environment variables: +- `WIKIPEDIA_BOT_USERNAME` / `WIKIPEDIA_BOT_PASSWORD` -- primary bot account +- `WIKIPEDIA_HIMO_USERNAME` / `WIKIPEDIA_HIMO_PASSWORD` -- alternate account (when `workibrahem` setting active) + +## CI/CD + +- **Tests**: GitHub Actions (`pytest.yaml`) runs `pytest` on PRs to `main` (Python 3.11 in CI, 3.13 locally). +- **Publish**: `python-publish.yml` for PyPI releases. From a192f02cd79e0849d7dc6b3da6fe76d2af1f601f Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Thu, 28 May 2026 00:01:48 +0300 Subject: [PATCH 5/5] . --- {Doc => docs}/CatDepth.md | 0 {Doc => docs}/MainPage.md | 0 {Doc => docs}/NEW_API.md | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {Doc => docs}/CatDepth.md (100%) rename {Doc => docs}/MainPage.md (100%) rename {Doc => docs}/NEW_API.md (100%) diff --git a/Doc/CatDepth.md b/docs/CatDepth.md similarity index 100% rename from Doc/CatDepth.md rename to docs/CatDepth.md diff --git a/Doc/MainPage.md b/docs/MainPage.md similarity index 100% rename from Doc/MainPage.md rename to docs/MainPage.md diff --git a/Doc/NEW_API.md b/docs/NEW_API.md similarity index 100% rename from Doc/NEW_API.md rename to docs/NEW_API.md