diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 799fd98..db95edf 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -54,7 +54,7 @@ ### Core Components 1. **app.py** (`src/app.py` - 205 lines): FastAPI application with 30+ endpoints organized by tags (War, Planets, Statistics, Campaigns, Factions, Biomes, System) -2. **scraper.py** (`src/scraper.py` - 158 lines): HTTP client for Hell Divers 2 community API with 10 methods: `get_war_status()`, `get_planets()`, `get_statistics()`, `get_factions()`, `get_biomes()`, `get_campaign_info()`, `get_assignments()`, `get_dispatches()`, `get_planet_events()`, `get_planet_status()` +2. **scraper.py** (`src/scraper.py` - 175 lines): HTTP client for Hell Divers 2 community API with rate limiting (2 second delay between requests to stay within 5 requests/10 seconds limit). 10 methods: `get_war_status()`, `get_planets()`, `get_statistics()`, `get_factions()`, `get_biomes()`, `get_campaign_info()`, `get_assignments()`, `get_dispatches()`, `get_planet_events()`, `get_planet_status()` 3. **database.py** (`src/database.py` - 360 lines): SQLite manager with 7 tables (war_status, statistics, planet_status, campaigns, assignments, dispatches, planet_events) + indexes for fast queries 4. **collector.py** (`src/collector.py` - 109 lines): APScheduler-based background task runner that calls `collect_all_data()` every 5 minutes (300s) during app lifecycle, collecting all data types with proper error handling 5. **config.py** (`src/config.py` - 57 lines): Environment-based configuration with support for custom API URLs and headers, DevelopmentConfig (1-minute intervals) vs ProductionConfig (5-minute intervals) @@ -98,6 +98,14 @@ - **API Level**: Raise `HTTPException(status_code, detail="message")` for client errors - **Logging**: Use configured logger with module-specific `logger = logging.getLogger(__name__)` +### Rate Limiting +- **Community API Limit**: 5 requests per 10 seconds (enforced by api.helldivers2.dev) +- **Implementation**: Scraper uses 2-second delay between requests (`_rate_limit()` method) +- **Method**: Tracks `last_request_time`, sleeps if elapsed time < `request_delay` +- **Application**: All 10 scraper methods call `self._rate_limit()` before HTTP requests +- **Logging**: Debug level logs when rate limiting delay is applied +- **Graceful Degradation**: If rate limit exceeded (429 error), collector logs error and continues with next collection cycle + ### Dependencies & Tools Evolution - **Linter Shift**: Replaced `flake8` with `ruff` (faster, more rules, auto-fix support) - **Code Formatting**: Uses `black` (100 char line length) with `ruff format` diff --git a/src/scraper.py b/src/scraper.py index 43df976..f36ba55 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -1,5 +1,7 @@ import requests import logging +import time +import threading from typing import Dict, List, Optional from src.config import Config @@ -7,12 +9,22 @@ class HellDivers2Scraper: - """Scraper for Hell Divers 2 game data via community API (api.helldivers2.dev)""" + """Scraper for Hell Divers 2 game data via community API (api.helldivers2.dev) + + Rate limiting: 5 requests per 10 seconds (2.0 seconds between requests enforced) + """ def __init__(self, timeout: int = 30, base_url: Optional[str] = None): self.timeout = timeout # Use provided URL or config URL self.base_url = base_url or Config.HELLDIVERS_API_BASE + + # Rate limiting: delay between requests (2 seconds for 5 requests in 10 seconds) + self.request_delay = 2.0 + # Initialize to current time to allow first request immediately + self.last_request_time = time.time() + # Thread lock for rate limiting (scraper is safe for concurrent access) + self._rate_limit_lock = threading.Lock() # Get headers from config, use "NA" if not configured client_name = ( @@ -35,10 +47,21 @@ def __init__(self, timeout: int = 30, base_url: Optional[str] = None): } ) logger.info(f"HellDivers2Scraper initialized with base_url: {self.base_url}") + + def _rate_limit(self): + """Enforce rate limiting between requests (thread-safe)""" + with self._rate_limit_lock: + elapsed = time.time() - self.last_request_time + if elapsed < self.request_delay: + delay = self.request_delay - elapsed + logger.debug(f"Rate limiting: sleeping for {delay:.2f}s") + time.sleep(delay) + self.last_request_time = time.time() def get_war_status(self) -> Optional[Dict]: """Fetch current war status""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/war", timeout=self.timeout) response.raise_for_status() return response.json() @@ -49,6 +72,7 @@ def get_war_status(self) -> Optional[Dict]: def get_campaign_info(self) -> Optional[List[Dict]]: """Fetch active campaigns information""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/campaigns", timeout=self.timeout) response.raise_for_status() return response.json() @@ -59,6 +83,7 @@ def get_campaign_info(self) -> Optional[List[Dict]]: def get_assignments(self) -> Optional[List[Dict]]: """Fetch current assignments (Major Orders)""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/assignments", timeout=self.timeout) response.raise_for_status() return response.json() @@ -69,6 +94,7 @@ def get_assignments(self) -> Optional[List[Dict]]: def get_dispatches(self) -> Optional[List[Dict]]: """Fetch news dispatches and announcements""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/dispatches", timeout=self.timeout) response.raise_for_status() return response.json() @@ -79,6 +105,7 @@ def get_dispatches(self) -> Optional[List[Dict]]: def get_planets(self) -> Optional[List[Dict]]: """Fetch all planets information""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/planets", timeout=self.timeout) response.raise_for_status() return response.json() @@ -89,6 +116,7 @@ def get_planets(self) -> Optional[List[Dict]]: def get_planet_status(self, planet_index: int) -> Optional[Dict]: """Fetch status of a specific planet""" try: + self._rate_limit() response = self.session.get( f"{self.base_url}/planets/{planet_index}", timeout=self.timeout ) @@ -113,6 +141,7 @@ def get_statistics(self) -> Optional[Dict]: def get_planet_events(self) -> Optional[List[Dict]]: """Fetch planet events""" try: + self._rate_limit() response = self.session.get(f"{self.base_url}/planet-events", timeout=self.timeout) response.raise_for_status() return response.json()