feat: add rate limiting to scraper to prevent API 429 errors

.github/copilot-instructions.md

@@ -54,7 +54,7 @@
 
 ### Core Components
 1. **app.py** (`src/app.py` - 205 lines): FastAPI application with 30+ endpoints organized by tags (War, Planets, Statistics, Campaigns, Factions, Biomes, System)
-2. **scraper.py** (`src/scraper.py` - 158 lines): HTTP client for Hell Divers 2 community API with 10 methods: `get_war_status()`, `get_planets()`, `get_statistics()`, `get_factions()`, `get_biomes()`, `get_campaign_info()`, `get_assignments()`, `get_dispatches()`, `get_planet_events()`, `get_planet_status()`
+2. **scraper.py** (`src/scraper.py` - 175 lines): HTTP client for Hell Divers 2 community API with rate limiting (2 second delay between requests to stay within 5 requests/10 seconds limit). 10 methods: `get_war_status()`, `get_planets()`, `get_statistics()`, `get_factions()`, `get_biomes()`, `get_campaign_info()`, `get_assignments()`, `get_dispatches()`, `get_planet_events()`, `get_planet_status()`
 3. **database.py** (`src/database.py` - 360 lines): SQLite manager with 7 tables (war_status, statistics, planet_status, campaigns, assignments, dispatches, planet_events) + indexes for fast queries
 4. **collector.py** (`src/collector.py` - 109 lines): APScheduler-based background task runner that calls `collect_all_data()` every 5 minutes (300s) during app lifecycle, collecting all data types with proper error handling
 5. **config.py** (`src/config.py` - 57 lines): Environment-based configuration with support for custom API URLs and headers, DevelopmentConfig (1-minute intervals) vs ProductionConfig (5-minute intervals)
@@ -98,6 +98,14 @@
 - **API Level**: Raise `HTTPException(status_code, detail="message")` for client errors
 - **Logging**: Use configured logger with module-specific `logger = logging.getLogger(__name__)`
 
+### Rate Limiting
+- **Community API Limit**: 5 requests per 10 seconds (enforced by api.helldivers2.dev)
+- **Implementation**: Scraper uses 2-second delay between requests (`_rate_limit()` method)
+- **Method**: Tracks `last_request_time`, sleeps if elapsed time < `request_delay`
+- **Application**: All 10 scraper methods call `self._rate_limit()` before HTTP requests
+- **Logging**: Debug level logs when rate limiting delay is applied
+- **Graceful Degradation**: If rate limit exceeded (429 error), collector logs error and continues with next collection cycle
+
 ### Dependencies & Tools Evolution
 - **Linter Shift**: Replaced `flake8` with `ruff` (faster, more rules, auto-fix support)
 - **Code Formatting**: Uses `black` (100 char line length) with `ruff format`

src/scraper.py

@@ -1,18 +1,30 @@
 import requests
 import logging
+import time
+import threading
 from typing import Dict, List, Optional
 from src.config import Config
 
 logger = logging.getLogger(__name__)
 
 
 class HellDivers2Scraper:
-    """Scraper for Hell Divers 2 game data via community API (api.helldivers2.dev)"""
+    """Scraper for Hell Divers 2 game data via community API (api.helldivers2.dev)
+
+    Rate limiting: 5 requests per 10 seconds (2.0 seconds between requests enforced)
+    """
 
     def __init__(self, timeout: int = 30, base_url: Optional[str] = None):
         self.timeout = timeout
         # Use provided URL or config URL
         self.base_url = base_url or Config.HELLDIVERS_API_BASE
+
+        # Rate limiting: delay between requests (2 seconds for 5 requests in 10 seconds)
+        self.request_delay = 2.0
+        # Initialize to current time to allow first request immediately
+        self.last_request_time = time.time()
+        # Thread lock for rate limiting (scraper is safe for concurrent access)
+        self._rate_limit_lock = threading.Lock()
 
         # Get headers from config, use "NA" if not configured
         client_name = (
@@ -35,10 +47,21 @@ def __init__(self, timeout: int = 30, base_url: Optional[str] = None):
             }
         )
         logger.info(f"HellDivers2Scraper initialized with base_url: {self.base_url}")
+
+    def _rate_limit(self):
+        """Enforce rate limiting between requests (thread-safe)"""
+        with self._rate_limit_lock:
+            elapsed = time.time() - self.last_request_time
+            if elapsed < self.request_delay:
+                delay = self.request_delay - elapsed
+                logger.debug(f"Rate limiting: sleeping for {delay:.2f}s")
+                time.sleep(delay)
+            self.last_request_time = time.time()
 
     def get_war_status(self) -> Optional[Dict]:
         """Fetch current war status"""
         try:
+            self._rate_limit()
             response = self.session.get(f"{self.base_url}/war", timeout=self.timeout)
             response.raise_for_status()
             return response.json()
@@ -49,6 +72,7 @@ def get_war_status(self) -> Optional[Dict]:
     def get_campaign_info(self) -> Optional[List[Dict]]:
         """Fetch active campaigns information"""
         try:
+            self._rate_limit()
             response = self.session.get(f"{self.base_url}/campaigns", timeout=self.timeout)
             response.raise_for_status()
             return response.json()
@@ -59,6 +83,7 @@ def get_campaign_info(self) -> Optional[List[Dict]]:
     def get_assignments(self) -> Optional[List[Dict]]:
         """Fetch current assignments (Major Orders)"""
         try:
+            self._rate_limit()
             response = self.session.get(f"{self.base_url}/assignments", timeout=self.timeout)
             response.raise_for_status()
             return response.json()
@@ -69,6 +94,7 @@ def get_assignments(self) -> Optional[List[Dict]]:
     def get_dispatches(self) -> Optional[List[Dict]]:
         """Fetch news dispatches and announcements"""
         try:
+            self._rate_limit()
             response = self.session.get(f"{self.base_url}/dispatches", timeout=self.timeout)
             response.raise_for_status()
             return response.json()
@@ -79,6 +105,7 @@ def get_dispatches(self) -> Optional[List[Dict]]:
     def get_planets(self) -> Optional[List[Dict]]:
         """Fetch all planets information"""
         try:
+            self._rate_limit()
             response = self.session.get(f"{self.base_url}/planets", timeout=self.timeout)
             response.raise_for_status()
             return response.json()
@@ -89,6 +116,7 @@ def get_planets(self) -> Optional[List[Dict]]:
     def get_planet_status(self, planet_index: int) -> Optional[Dict]:
         """Fetch status of a specific planet"""
         try:
+            self._rate_limit()
             response = self.session.get(
                 f"{self.base_url}/planets/{planet_index}", timeout=self.timeout
             )
@@ -113,6 +141,7 @@ def get_statistics(self) -> Optional[Dict]:
     def get_planet_events(self) -> Optional[List[Dict]]:
         """Fetch planet events"""
         try:
+            self._rate_limit()
             response = self.session.get(f"{self.base_url}/planet-events", timeout=self.timeout)
             response.raise_for_status()
             return response.json()