From 9b61c1ac7a8af27fe86d99c6d29f93e857c00e99 Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 19 Oct 2025 21:58:45 -0700 Subject: [PATCH 1/3] feat: implement rate limiting in scraper to comply with API restrictions --- .github/copilot-instructions.md | 10 +++++++++- src/scraper.py | 26 +++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8397514..42e71eb 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -29,7 +29,7 @@ ### Core Components 1. **app.py** (`src/app.py` - 205 lines): FastAPI application with 30+ endpoints organized by tags (War, Planets, Statistics, Campaigns, Factions, Biomes, System) -2. **scraper.py** (`src/scraper.py` - 158 lines): HTTP client for Hell Divers 2 community API with 10 methods: `get_war_status()`, `get_planets()`, `get_statistics()`, `get_factions()`, `get_biomes()`, `get_campaign_info()`, `get_assignments()`, `get_dispatches()`, `get_planet_events()`, `get_planet_status()` +2. **scraper.py** (`src/scraper.py` - 175 lines): HTTP client for Hell Divers 2 community API with rate limiting (2 second delay between requests to stay within 5 requests/10 seconds limit). 10 methods: `get_war_status()`, `get_planets()`, `get_statistics()`, `get_factions()`, `get_biomes()`, `get_campaign_info()`, `get_assignments()`, `get_dispatches()`, `get_planet_events()`, `get_planet_status()` 3. **database.py** (`src/database.py` - 360 lines): SQLite manager with 7 tables (war_status, statistics, planet_status, campaigns, assignments, dispatches, planet_events) + indexes for fast queries 4. **collector.py** (`src/collector.py` - 109 lines): APScheduler-based background task runner that calls `collect_all_data()` every 5 minutes (300s) during app lifecycle, collecting all data types with proper error handling 5. **config.py** (`src/config.py` - 57 lines): Environment-based configuration with support for custom API URLs and headers, DevelopmentConfig (1-minute intervals) vs ProductionConfig (5-minute intervals) @@ -63,6 +63,14 @@ - **API Level**: Raise `HTTPException(status_code, detail="message")` for client errors - **Logging**: Use configured logger with module-specific `logger = logging.getLogger(__name__)` +### Rate Limiting +- **Community API Limit**: 5 requests per 10 seconds (enforced by api.helldivers2.dev) +- **Implementation**: Scraper uses 2-second delay between requests (`_rate_limit()` method) +- **Method**: Tracks `last_request_time`, sleeps if elapsed time < `request_delay` +- **Application**: All 10 scraper methods call `self._rate_limit()` before HTTP requests +- **Logging**: Debug level logs when rate limiting delay is applied +- **Graceful Degradation**: If rate limit exceeded (429 error), collector logs error and continues with next collection cycle + ### Dependencies & Tools Evolution - **Linter Shift**: Replaced `flake8` with `ruff` (faster, more rules, auto-fix support) - **Code Formatting**: Uses `black` (100 char line length) with `ruff format` diff --git a/src/scraper.py b/src/scraper.py index 43df976..3c4433e 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -1,5 +1,6 @@ import requests import logging +import time from typing import Dict, List, Optional from src.config import Config @@ -7,12 +8,19 @@ class HellDivers2Scraper: - """Scraper for Hell Divers 2 game data via community API (api.helldivers2.dev)""" + """Scraper for Hell Divers 2 game data via community API (api.helldivers2.dev) + + Rate limiting: 5 requests per 10 seconds (0.5 seconds between requests recommended) + """ def __init__(self, timeout: int = 30, base_url: Optional[str] = None): self.timeout = timeout # Use provided URL or config URL self.base_url = base_url or Config.HELLDIVERS_API_BASE + + # Rate limiting: delay between requests (2 seconds for 5 requests in 10 seconds) + self.request_delay = 2.0 + self.last_request_time = 0 # Get headers from config, use "NA" if not configured client_name = ( @@ -35,10 +43,20 @@ def __init__(self, timeout: int = 30, base_url: Optional[str] = None): } ) logger.info(f"HellDivers2Scraper initialized with base_url: {self.base_url}") + + def _rate_limit(self): + """Enforce rate limiting between requests""" + elapsed = time.time() - self.last_request_time + if elapsed < self.request_delay: + delay = self.request_delay - elapsed + logger.debug(f"Rate limiting: sleeping for {delay:.2f}s") + time.sleep(delay) + self.last_request_time = time.time() def get_war_status(self) -> Optional[Dict]: """Fetch current war status""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/war", timeout=self.timeout) response.raise_for_status() return response.json() @@ -49,6 +67,7 @@ def get_war_status(self) -> Optional[Dict]: def get_campaign_info(self) -> Optional[List[Dict]]: """Fetch active campaigns information""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/campaigns", timeout=self.timeout) response.raise_for_status() return response.json() @@ -59,6 +78,7 @@ def get_campaign_info(self) -> Optional[List[Dict]]: def get_assignments(self) -> Optional[List[Dict]]: """Fetch current assignments (Major Orders)""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/assignments", timeout=self.timeout) response.raise_for_status() return response.json() @@ -69,6 +89,7 @@ def get_assignments(self) -> Optional[List[Dict]]: def get_dispatches(self) -> Optional[List[Dict]]: """Fetch news dispatches and announcements""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/dispatches", timeout=self.timeout) response.raise_for_status() return response.json() @@ -79,6 +100,7 @@ def get_dispatches(self) -> Optional[List[Dict]]: def get_planets(self) -> Optional[List[Dict]]: """Fetch all planets information""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/planets", timeout=self.timeout) response.raise_for_status() return response.json() @@ -89,6 +111,7 @@ def get_planets(self) -> Optional[List[Dict]]: def get_planet_status(self, planet_index: int) -> Optional[Dict]: """Fetch status of a specific planet""" try: + self._rate_limit() response = self.session.get( f"{self.base_url}/planets/{planet_index}", timeout=self.timeout ) @@ -113,6 +136,7 @@ def get_statistics(self) -> Optional[Dict]: def get_planet_events(self) -> Optional[List[Dict]]: """Fetch planet events""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/planet-events", timeout=self.timeout) response.raise_for_status() return response.json() From 228798de9444b2fbcb299ea7470d8f67721b0a4c Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 19 Oct 2025 22:01:06 -0700 Subject: [PATCH 2/3] Update src/scraper.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scraper.py b/src/scraper.py index 3c4433e..3f518fa 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -10,7 +10,7 @@ class HellDivers2Scraper: """Scraper for Hell Divers 2 game data via community API (api.helldivers2.dev) - Rate limiting: 5 requests per 10 seconds (0.5 seconds between requests recommended) + Rate limiting: 5 requests per 10 seconds (2.0 seconds between requests enforced) """ def __init__(self, timeout: int = 30, base_url: Optional[str] = None): From ce14f478edb68f7739a0926a478cde58d32f8d9c Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 19 Oct 2025 22:05:46 -0700 Subject: [PATCH 3/3] fix: address rate limiting review comments --- src/scraper.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/scraper.py b/src/scraper.py index 3f518fa..f36ba55 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -1,6 +1,7 @@ import requests import logging import time +import threading from typing import Dict, List, Optional from src.config import Config @@ -20,7 +21,10 @@ def __init__(self, timeout: int = 30, base_url: Optional[str] = None): # Rate limiting: delay between requests (2 seconds for 5 requests in 10 seconds) self.request_delay = 2.0 - self.last_request_time = 0 + # Initialize to current time to allow first request immediately + self.last_request_time = time.time() + # Thread lock for rate limiting (scraper is safe for concurrent access) + self._rate_limit_lock = threading.Lock() # Get headers from config, use "NA" if not configured client_name = ( @@ -45,13 +49,14 @@ def __init__(self, timeout: int = 30, base_url: Optional[str] = None): logger.info(f"HellDivers2Scraper initialized with base_url: {self.base_url}") def _rate_limit(self): - """Enforce rate limiting between requests""" - elapsed = time.time() - self.last_request_time - if elapsed < self.request_delay: - delay = self.request_delay - elapsed - logger.debug(f"Rate limiting: sleeping for {delay:.2f}s") - time.sleep(delay) - self.last_request_time = time.time() + """Enforce rate limiting between requests (thread-safe)""" + with self._rate_limit_lock: + elapsed = time.time() - self.last_request_time + if elapsed < self.request_delay: + delay = self.request_delay - elapsed + logger.debug(f"Rate limiting: sleeping for {delay:.2f}s") + time.sleep(delay) + self.last_request_time = time.time() def get_war_status(self) -> Optional[Dict]: """Fetch current war status"""