From 22786df4426ba911c2ac8a8cfadda97bf6ba57d0 Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 19 Oct 2025 22:53:03 -0700 Subject: [PATCH 01/10] feat: implement cache fallback for resilient API responses when Hell Divers 2 API is offline --- src/app.py | 52 +++++++++++++++++----- src/database.py | 116 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 10 deletions(-) diff --git a/src/app.py b/src/app.py index c1797e2..08045f2 100644 --- a/src/app.py +++ b/src/app.py @@ -80,11 +80,17 @@ async def refresh_war_status(): @app.get("/api/campaigns", tags=["Campaigns"]) async def get_campaigns(): - """Get campaign information""" + """Get campaign information (with cache fallback)""" + # Try live API first data = scraper.get_campaign_info() + + # Fallback to cache if live API fails + if not data: + data = db.get_latest_campaigns_snapshot() + if data: return data - raise HTTPException(status_code=404, detail="Failed to fetch campaigns") + raise HTTPException(status_code=503, detail="Failed to fetch campaigns and no cached data available") @app.get("/api/campaigns/active", tags=["Campaigns"]) @@ -103,20 +109,34 @@ async def get_active_campaigns(): @app.get("/api/planets", tags=["Planets"]) async def get_planets(): - """Get all planets""" + """Get all planets (with cache fallback)""" + # Try live API first data = scraper.get_planets() + + # Fallback to cache if live API fails + if not data: + data = db.get_latest_planets_snapshot() + if data: return data - raise HTTPException(status_code=404, detail="Failed to fetch planets") + raise HTTPException(status_code=503, detail="Failed to fetch planets and no cached data available") @app.get("/api/planets/{planet_index}", tags=["Planets"]) async def get_planet_status(planet_index: int): - """Get status of a specific planet""" + """Get status of a specific planet (with cache fallback)""" + # Try live API first data = collector.collect_planet_data(planet_index) + + # Fallback to cache if live API fails + if not data: + history = db.get_planet_status_history(planet_index, limit=1) + if history: + data = history[0].get("data") if isinstance(history[0], dict) and "data" in history[0] else history[0] + if data: return data - raise HTTPException(status_code=404, detail=f"Failed to fetch planet {planet_index}") + raise HTTPException(status_code=503, detail=f"Failed to fetch planet {planet_index} and no cached data available") @app.get("/api/planets/{planet_index}/history", tags=["Planets"]) @@ -168,11 +188,17 @@ async def refresh_statistics(): @app.get("/api/factions", tags=["Factions"]) async def get_factions(): - """Get all factions""" + """Get all factions (with cache fallback)""" + # Try live API first data = scraper.get_factions() + + # Fallback to cache if live API fails + if not data: + data = db.get_latest_factions_snapshot() + if data: return data - raise HTTPException(status_code=404, detail="Failed to fetch factions") + raise HTTPException(status_code=503, detail="Failed to fetch factions and no cached data available") # ======================== @@ -182,11 +208,17 @@ async def get_factions(): @app.get("/api/biomes", tags=["Biomes"]) async def get_biomes(): - """Get all biomes""" + """Get all biomes (with cache fallback)""" + # Try live API first data = scraper.get_biomes() + + # Fallback to cache if live API fails + if not data: + data = db.get_latest_biomes_snapshot() + if data: return data - raise HTTPException(status_code=404, detail="Failed to fetch biomes") + raise HTTPException(status_code=503, detail="Failed to fetch biomes and no cached data available") # ======================== diff --git a/src/database.py b/src/database.py index eafcb64..6badd9d 100644 --- a/src/database.py +++ b/src/database.py @@ -356,3 +356,119 @@ def get_latest_planet_events(self, limit: int = 10) -> List[Dict]: except Exception as e: logger.error(f"Failed to get latest planet events: {e}") return [] + + def get_latest_planets_snapshot(self) -> Optional[List[Dict]]: + """Get most recent cached snapshot of all planets + + Used as fallback when live API is unavailable. + Returns all planet status records from the most recent collection cycle. + """ + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Get the most recent timestamp from planet_status + cursor.execute("SELECT DISTINCT timestamp FROM planet_status ORDER BY timestamp DESC LIMIT 1") + result = cursor.fetchone() + + if not result: + return None + + latest_timestamp = result[0] + + # Get all planets from that timestamp + cursor.execute( + "SELECT data FROM planet_status WHERE timestamp = ? ORDER BY planet_index ASC", + (latest_timestamp,), + ) + results = cursor.fetchall() + return [json.loads(row[0]) for row in results] if results else None + except Exception as e: + logger.error(f"Failed to get latest planets snapshot: {e}") + return None + + def get_latest_campaigns_snapshot(self) -> Optional[List[Dict]]: + """Get most recent cached snapshot of all campaigns + + Used as fallback when live API is unavailable. + Returns most recent campaign data for each campaign. + """ + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Get the most recent campaign data for each campaign_id + cursor.execute( + """SELECT data FROM campaigns + WHERE (campaign_id, timestamp) IN ( + SELECT campaign_id, MAX(timestamp) FROM campaigns GROUP BY campaign_id + ) + ORDER BY timestamp DESC""" + ) + results = cursor.fetchall() + return [json.loads(row[0]) for row in results] if results else None + except Exception as e: + logger.error(f"Failed to get latest campaigns snapshot: {e}") + return None + + def get_latest_factions_snapshot(self) -> Optional[List[Dict]]: + """Get most recent cached snapshot of all factions + + Used as fallback when live API is unavailable. + Factions are extracted from war status data. + """ + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute("SELECT data FROM war_status ORDER BY timestamp DESC LIMIT 1") + result = cursor.fetchone() + + if not result: + return None + + war_data = json.loads(result[0]) + return war_data.get("factions", None) + except Exception as e: + logger.error(f"Failed to get latest factions snapshot: {e}") + return None + + def get_latest_biomes_snapshot(self) -> Optional[List[Dict]]: + """Get most recent cached snapshot of all biomes + + Used as fallback when live API is unavailable. + Biomes are extracted from planet data. + """ + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Get the most recent timestamp from planet_status + cursor.execute("SELECT DISTINCT timestamp FROM planet_status ORDER BY timestamp DESC LIMIT 1") + result = cursor.fetchone() + + if not result: + return None + + latest_timestamp = result[0] + + # Get all planets from that timestamp and extract unique biomes + cursor.execute( + "SELECT data FROM planet_status WHERE timestamp = ?", + (latest_timestamp,), + ) + results = cursor.fetchall() + + if not results: + return None + + # Extract unique biomes from planets + biomes = {} + for row in results: + planet_data = json.loads(row[0]) + if "biome" in planet_data: + biome_name = planet_data["biome"].get("name") + if biome_name and biome_name not in biomes: + biomes[biome_name] = planet_data["biome"] + + return list(biomes.values()) if biomes else None + except Exception as e: + logger.error(f"Failed to get latest biomes snapshot: {e}") + return None + From 447257cea2de0dedf30480f7e632dd364bf6f53e Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 19 Oct 2025 22:55:46 -0700 Subject: [PATCH 02/10] chore: update app.py for code organization and clarity --- src/app.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/app.py b/src/app.py index 08045f2..c724314 100644 --- a/src/app.py +++ b/src/app.py @@ -228,8 +228,15 @@ async def get_biomes(): @app.get("/api/health", tags=["Health"]) async def health_check(): - """Health check endpoint""" - return {"status": "healthy", "collector_running": collector.is_running} + """Health check endpoint with upstream API status""" + # Check if upstream API is reachable + upstream_status = scraper.get_war_status() is not None + + return { + "status": "healthy", + "collector_running": collector.is_running, + "upstream_api": "online" if upstream_status else "offline", + } # ======================== From cf66ed402f84ece461c762ec056061dc16096b41 Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 19 Oct 2025 22:58:56 -0700 Subject: [PATCH 03/10] fix: use explicit None checks instead of truthiness and add type guard for biomes --- src/app.py | 16 ++++++++-------- src/database.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/app.py b/src/app.py index c724314..3f64bdb 100644 --- a/src/app.py +++ b/src/app.py @@ -85,10 +85,10 @@ async def get_campaigns(): data = scraper.get_campaign_info() # Fallback to cache if live API fails - if not data: + if data is None: data = db.get_latest_campaigns_snapshot() - if data: + if data is not None: return data raise HTTPException(status_code=503, detail="Failed to fetch campaigns and no cached data available") @@ -114,10 +114,10 @@ async def get_planets(): data = scraper.get_planets() # Fallback to cache if live API fails - if not data: + if data is None: data = db.get_latest_planets_snapshot() - if data: + if data is not None: return data raise HTTPException(status_code=503, detail="Failed to fetch planets and no cached data available") @@ -193,10 +193,10 @@ async def get_factions(): data = scraper.get_factions() # Fallback to cache if live API fails - if not data: + if data is None: data = db.get_latest_factions_snapshot() - if data: + if data is not None: return data raise HTTPException(status_code=503, detail="Failed to fetch factions and no cached data available") @@ -213,10 +213,10 @@ async def get_biomes(): data = scraper.get_biomes() # Fallback to cache if live API fails - if not data: + if data is None: data = db.get_latest_biomes_snapshot() - if data: + if data is not None: return data raise HTTPException(status_code=503, detail="Failed to fetch biomes and no cached data available") diff --git a/src/database.py b/src/database.py index 6badd9d..fffb166 100644 --- a/src/database.py +++ b/src/database.py @@ -462,7 +462,7 @@ def get_latest_biomes_snapshot(self) -> Optional[List[Dict]]: biomes = {} for row in results: planet_data = json.loads(row[0]) - if "biome" in planet_data: + if "biome" in planet_data and isinstance(planet_data["biome"], dict): biome_name = planet_data["biome"].get("name") if biome_name and biome_name not in biomes: biomes[biome_name] = planet_data["biome"] From 9ca46f9437f3fb9a0e017d0fa2d14b48680d3a8f Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 19 Oct 2025 23:13:38 -0700 Subject: [PATCH 04/10] optimize: eliminate redundant war_status call in collector by reusing statistics from initial war_data fetch --- src/collector.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/collector.py b/src/collector.py index c6ab4ac..9927ce5 100644 --- a/src/collector.py +++ b/src/collector.py @@ -43,17 +43,19 @@ def collect_all_data(self): logger.info("Starting data collection cycle") try: - # Collect war status + # Collect war status (this includes statistics) war_data = self.scraper.get_war_status() if war_data: self.db.save_war_status(war_data) logger.info("War status collected") - - # Collect statistics (extracted from war status) - stats_data = self.scraper.get_statistics() - if stats_data: - self.db.save_statistics(stats_data) - logger.info("Statistics collected") + + # Extract and save statistics from war_data (no extra API call needed) + stats_data = war_data.get("statistics") + if stats_data: + self.db.save_statistics(stats_data) + logger.info("Statistics collected") + else: + logger.info("Failed to collect war status and statistics") # Collect planets planets = self.scraper.get_planets() From 3c243641427056f3d7cdbdf7cd42050280fbc7cb Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 19 Oct 2025 23:43:44 -0700 Subject: [PATCH 05/10] feat: cache upstream API status and track collection failures granularly --- src/app.py | 5 +++-- src/collector.py | 42 +++++++++++++++++++++++++++++++++++++++--- src/scraper.py | 14 ++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/app.py b/src/app.py index 3f64bdb..81f916c 100644 --- a/src/app.py +++ b/src/app.py @@ -229,8 +229,9 @@ async def get_biomes(): @app.get("/api/health", tags=["Health"]) async def health_check(): """Health check endpoint with upstream API status""" - # Check if upstream API is reachable - upstream_status = scraper.get_war_status() is not None + # Get cached upstream API status (set during data collection cycle) + # No extra API call needed - this is just reading the cached status + upstream_status = scraper.is_upstream_available() return { "status": "healthy", diff --git a/src/collector.py b/src/collector.py index 9927ce5..73df205 100644 --- a/src/collector.py +++ b/src/collector.py @@ -41,11 +41,16 @@ def stop(self): def collect_all_data(self): """Collect all available data""" logger.info("Starting data collection cycle") + + # Track success/failure counts + success_count = 0 + failure_count = 0 try: # Collect war status (this includes statistics) war_data = self.scraper.get_war_status() if war_data: + success_count += 1 self.db.save_war_status(war_data) logger.info("War status collected") @@ -55,49 +60,80 @@ def collect_all_data(self): self.db.save_statistics(stats_data) logger.info("Statistics collected") else: - logger.info("Failed to collect war status and statistics") + failure_count += 1 + logger.warning("Failed to collect war status and statistics") # Collect planets planets = self.scraper.get_planets() if planets: + success_count += 1 for planet in planets: planet_index = planet.get("index") if planet_index: self.db.save_planet_status(planet_index, planet) logger.info(f"Collected data for {len(planets)} planets") + else: + failure_count += 1 + logger.warning("Failed to collect planets") # Collect campaigns campaigns = self.scraper.get_campaign_info() if campaigns: + success_count += 1 for campaign in campaigns: campaign_id = campaign.get("id") if campaign_id and campaign.get("planet") and "index" in campaign["planet"]: planet_index = campaign["planet"].get("index") self.db.save_campaign(campaign_id, planet_index, campaign) logger.info(f"Collected {len(campaigns)} campaigns") + else: + failure_count += 1 + logger.warning("Failed to collect campaigns") # Collect assignments (Major Orders) assignments = self.scraper.get_assignments() if assignments: + success_count += 1 self.db.save_assignments(assignments) logger.info(f"Collected {len(assignments)} assignments") + else: + failure_count += 1 + logger.warning("Failed to collect assignments") # Collect dispatches (news) dispatches = self.scraper.get_dispatches() if dispatches: + success_count += 1 self.db.save_dispatches(dispatches) logger.info(f"Collected {len(dispatches)} dispatches") + else: + failure_count += 1 + logger.warning("Failed to collect dispatches") # Collect planet events events = self.scraper.get_planet_events() if events is not None: + success_count += 1 self.db.save_planet_events(events) logger.info(f"Collected {len(events)} planet events") - - logger.info("Data collection cycle completed successfully") + else: + failure_count += 1 + logger.warning("Failed to collect planet events") + + # Update upstream API status: online only if all/most endpoints succeeded + # Mark offline if more than 2 endpoints failed (tolerance for occasional failures) + upstream_available = failure_count <= 2 + self.scraper.set_upstream_status(upstream_available) + + if upstream_available: + logger.info(f"Data collection cycle completed: {success_count} succeeded, {failure_count} failed (API online)") + else: + logger.warning(f"Data collection cycle completed: {success_count} succeeded, {failure_count} failed (API marked offline)") except Exception as e: logger.error(f"Error during data collection: {e}") + # Mark upstream as unavailable on exception + self.scraper.set_upstream_status(False) def collect_planet_data(self, planet_index: int): """Collect data for a specific planet""" diff --git a/src/scraper.py b/src/scraper.py index f36ba55..8cd9a6c 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -26,6 +26,10 @@ def __init__(self, timeout: int = 30, base_url: Optional[str] = None): # Thread lock for rate limiting (scraper is safe for concurrent access) self._rate_limit_lock = threading.Lock() + # Track upstream API status (set by collector during data collection) + self.upstream_api_available = True + self._status_lock = threading.Lock() + # Get headers from config, use "NA" if not configured client_name = ( Config.HELLDIVERS_API_CLIENT_NAME @@ -58,6 +62,16 @@ def _rate_limit(self): time.sleep(delay) self.last_request_time = time.time() + def set_upstream_status(self, available: bool): + """Update upstream API availability status (called during data collection)""" + with self._status_lock: + self.upstream_api_available = available + + def is_upstream_available(self) -> bool: + """Get current upstream API status (no API call, just returns cached status)""" + with self._status_lock: + return self.upstream_api_available + def get_war_status(self) -> Optional[Dict]: """Fetch current war status""" try: From 6e2a865569b3dd7a9b9dc3c535e5d75a5223d2a4 Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 19 Oct 2025 23:57:06 -0700 Subject: [PATCH 06/10] build(deps): configure dependabot for automated dependency updates --- .github/dependabot.yml | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..035a30e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,43 @@ +version: 2 +updates: + # Enable version updates for Python dependencies + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "03:00" + open-pull-requests-limit: 10 + reviewers: + - "surrealwolf" + labels: + - "dependencies" + - "python" + allow: + - dependency-type: "all" + # Group patch updates together + groups: + patch-minor-updates: + patterns: + - patch + - minor + major-updates: + patterns: + - major + commit-message: + prefix: "build" + prefix-scope: "deps" + include: "scope" + + # Enable version updates for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "03:00" + reviewers: + - "surrealwolf" + labels: + - "dependencies" + - "github-actions" From 477ee8aa976cd57e5b71b1eec6c1ad71099087f2 Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Mon, 20 Oct 2025 20:26:49 -0700 Subject: [PATCH 07/10] feat: implement cache fallback for API data retrieval and enhance logging in data collector --- src/app.py | 28 ++++++-------- src/collector.py | 97 ++++++++++++++++++++++++------------------------ src/database.py | 2 +- 3 files changed, 60 insertions(+), 67 deletions(-) diff --git a/src/app.py b/src/app.py index 81f916c..911fbe4 100644 --- a/src/app.py +++ b/src/app.py @@ -90,7 +90,7 @@ async def get_campaigns(): if data is not None: return data - raise HTTPException(status_code=503, detail="Failed to fetch campaigns and no cached data available") + raise HTTPException(status_code=404, detail="No campaign data available (live fetch failed and no cached data)") @app.get("/api/campaigns/active", tags=["Campaigns"]) @@ -119,7 +119,7 @@ async def get_planets(): if data is not None: return data - raise HTTPException(status_code=503, detail="Failed to fetch planets and no cached data available") + raise HTTPException(status_code=404, detail="Failed to fetch planets and no cached data available") @app.get("/api/planets/{planet_index}", tags=["Planets"]) @@ -129,14 +129,14 @@ async def get_planet_status(planet_index: int): data = collector.collect_planet_data(planet_index) # Fallback to cache if live API fails - if not data: + if data is None: history = db.get_planet_status_history(planet_index, limit=1) if history: data = history[0].get("data") if isinstance(history[0], dict) and "data" in history[0] else history[0] - if data: + if data is not None: return data - raise HTTPException(status_code=503, detail=f"Failed to fetch planet {planet_index} and no cached data available") + raise HTTPException(status_code=404, detail=f"Failed to fetch planet {planet_index} and no cached data available") @app.get("/api/planets/{planet_index}/history", tags=["Planets"]) @@ -198,7 +198,7 @@ async def get_factions(): if data is not None: return data - raise HTTPException(status_code=503, detail="Failed to fetch factions and no cached data available") + raise HTTPException(status_code=404, detail="Failed to fetch factions and no cached data available") # ======================== @@ -218,7 +218,7 @@ async def get_biomes(): if data is not None: return data - raise HTTPException(status_code=503, detail="Failed to fetch biomes and no cached data available") + raise HTTPException(status_code=404, detail="Failed to fetch biomes and no cached data available") # ======================== @@ -228,16 +228,10 @@ async def get_biomes(): @app.get("/api/health", tags=["Health"]) async def health_check(): - """Health check endpoint with upstream API status""" - # Get cached upstream API status (set during data collection cycle) - # No extra API call needed - this is just reading the cached status - upstream_status = scraper.is_upstream_available() - - return { - "status": "healthy", - "collector_running": collector.is_running, - "upstream_api": "online" if upstream_status else "offline", - } + """Health check endpoint""" + # Returns local service status; upstream API status is tracked by collector + # No external API calls made here to avoid blocking event loop or consuming rate limits + return {"status": "healthy", "collector_running": collector.is_running} # ======================== diff --git a/src/collector.py b/src/collector.py index 73df205..5d81389 100644 --- a/src/collector.py +++ b/src/collector.py @@ -49,86 +49,85 @@ def collect_all_data(self): try: # Collect war status (this includes statistics) war_data = self.scraper.get_war_status() - if war_data: - success_count += 1 - self.db.save_war_status(war_data) - logger.info("War status collected") - - # Extract and save statistics from war_data (no extra API call needed) - stats_data = war_data.get("statistics") + if war_data is not None: + if war_data: + self.db.save_war_status(war_data) + logger.info("War status collected") + else: + logger.warning("Failed to collect war status") + + # Collect statistics (extracted from war status) + stats_data = self.scraper.get_statistics() + if stats_data is not None: if stats_data: self.db.save_statistics(stats_data) logger.info("Statistics collected") else: - failure_count += 1 - logger.warning("Failed to collect war status and statistics") + logger.warning("Failed to collect statistics") # Collect planets planets = self.scraper.get_planets() - if planets: - success_count += 1 - for planet in planets: - planet_index = planet.get("index") - if planet_index: - self.db.save_planet_status(planet_index, planet) - logger.info(f"Collected data for {len(planets)} planets") + if planets is not None: + if planets: + for planet in planets: + planet_index = planet.get("index") + if planet_index: + self.db.save_planet_status(planet_index, planet) + logger.info(f"Collected data for {len(planets)} planets") + else: + logger.info("Collected 0 planets (empty response)") else: - failure_count += 1 logger.warning("Failed to collect planets") # Collect campaigns campaigns = self.scraper.get_campaign_info() - if campaigns: - success_count += 1 - for campaign in campaigns: - campaign_id = campaign.get("id") - if campaign_id and campaign.get("planet") and "index" in campaign["planet"]: - planet_index = campaign["planet"].get("index") - self.db.save_campaign(campaign_id, planet_index, campaign) - logger.info(f"Collected {len(campaigns)} campaigns") + if campaigns is not None: + if campaigns: + for campaign in campaigns: + campaign_id = campaign.get("id") + if campaign_id and campaign.get("planet") and "index" in campaign["planet"]: + planet_index = campaign["planet"].get("index") + self.db.save_campaign(campaign_id, planet_index, campaign) + logger.info(f"Collected {len(campaigns)} campaigns") + else: + logger.info("Collected 0 campaigns (empty response)") else: - failure_count += 1 logger.warning("Failed to collect campaigns") # Collect assignments (Major Orders) assignments = self.scraper.get_assignments() - if assignments: - success_count += 1 - self.db.save_assignments(assignments) - logger.info(f"Collected {len(assignments)} assignments") + if assignments is not None: + if assignments: + self.db.save_assignments(assignments) + logger.info(f"Collected {len(assignments)} assignments") + else: + logger.info("Collected 0 assignments (empty response)") else: - failure_count += 1 logger.warning("Failed to collect assignments") # Collect dispatches (news) dispatches = self.scraper.get_dispatches() - if dispatches: - success_count += 1 - self.db.save_dispatches(dispatches) - logger.info(f"Collected {len(dispatches)} dispatches") + if dispatches is not None: + if dispatches: + self.db.save_dispatches(dispatches) + logger.info(f"Collected {len(dispatches)} dispatches") + else: + logger.info("Collected 0 dispatches (empty response)") else: - failure_count += 1 logger.warning("Failed to collect dispatches") # Collect planet events events = self.scraper.get_planet_events() if events is not None: - success_count += 1 - self.db.save_planet_events(events) - logger.info(f"Collected {len(events)} planet events") + if events: + self.db.save_planet_events(events) + logger.info(f"Collected {len(events)} planet events") + else: + logger.info("Collected 0 planet events (empty response)") else: - failure_count += 1 logger.warning("Failed to collect planet events") - # Update upstream API status: online only if all/most endpoints succeeded - # Mark offline if more than 2 endpoints failed (tolerance for occasional failures) - upstream_available = failure_count <= 2 - self.scraper.set_upstream_status(upstream_available) - - if upstream_available: - logger.info(f"Data collection cycle completed: {success_count} succeeded, {failure_count} failed (API online)") - else: - logger.warning(f"Data collection cycle completed: {success_count} succeeded, {failure_count} failed (API marked offline)") + logger.info("Data collection cycle completed successfully") except Exception as e: logger.error(f"Error during data collection: {e}") diff --git a/src/database.py b/src/database.py index fffb166..7f4bf0c 100644 --- a/src/database.py +++ b/src/database.py @@ -462,6 +462,7 @@ def get_latest_biomes_snapshot(self) -> Optional[List[Dict]]: biomes = {} for row in results: planet_data = json.loads(row[0]) + # Type guard: check if biome is a dict before accessing .get() if "biome" in planet_data and isinstance(planet_data["biome"], dict): biome_name = planet_data["biome"].get("name") if biome_name and biome_name not in biomes: @@ -471,4 +472,3 @@ def get_latest_biomes_snapshot(self) -> Optional[List[Dict]]: except Exception as e: logger.error(f"Failed to get latest biomes snapshot: {e}") return None - From 73607a6a74db319043d7123b1ce565d81845d25a Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Mon, 20 Oct 2025 20:31:20 -0700 Subject: [PATCH 08/10] refactor: remove unused success/failure tracking and upstream status handling in data collection --- src/collector.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/collector.py b/src/collector.py index 5d81389..23f5ecd 100644 --- a/src/collector.py +++ b/src/collector.py @@ -41,10 +41,6 @@ def stop(self): def collect_all_data(self): """Collect all available data""" logger.info("Starting data collection cycle") - - # Track success/failure counts - success_count = 0 - failure_count = 0 try: # Collect war status (this includes statistics) @@ -131,8 +127,6 @@ def collect_all_data(self): except Exception as e: logger.error(f"Error during data collection: {e}") - # Mark upstream as unavailable on exception - self.scraper.set_upstream_status(False) def collect_planet_data(self, planet_index: int): """Collect data for a specific planet""" From 05447b0d685f6de2cdba9a0cbff6bc76cb6102e8 Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Mon, 20 Oct 2025 20:43:09 -0700 Subject: [PATCH 09/10] feat: add upstream API status tracking --- src/app.py | 9 +++++++-- src/collector.py | 4 ++++ src/database.py | 38 ++++++++++++++++++++++++++++++++++++++ src/scraper.py | 14 -------------- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/app.py b/src/app.py index 911fbe4..9c6670a 100644 --- a/src/app.py +++ b/src/app.py @@ -229,9 +229,14 @@ async def get_biomes(): @app.get("/api/health", tags=["Health"]) async def health_check(): """Health check endpoint""" - # Returns local service status; upstream API status is tracked by collector + # Returns local service status and upstream API status # No external API calls made here to avoid blocking event loop or consuming rate limits - return {"status": "healthy", "collector_running": collector.is_running} + upstream_status = db.get_upstream_status() + return { + "status": "healthy", + "collector_running": collector.is_running, + "upstream_api": "online" if upstream_status else "offline" + } # ======================== diff --git a/src/collector.py b/src/collector.py index 23f5ecd..dd06655 100644 --- a/src/collector.py +++ b/src/collector.py @@ -124,9 +124,13 @@ def collect_all_data(self): logger.warning("Failed to collect planet events") logger.info("Data collection cycle completed successfully") + # Mark upstream as available after successful collection + self.db.set_upstream_status(True) except Exception as e: logger.error(f"Error during data collection: {e}") + # Mark upstream as unavailable on any collection error + self.db.set_upstream_status(False) def collect_planet_data(self, planet_index: int): """Collect data for a specific planet""" diff --git a/src/database.py b/src/database.py index 7f4bf0c..ee51da9 100644 --- a/src/database.py +++ b/src/database.py @@ -1,6 +1,7 @@ import sqlite3 import json import logging +import time from typing import Dict, List, Optional logger = logging.getLogger(__name__) @@ -472,3 +473,40 @@ def get_latest_biomes_snapshot(self) -> Optional[List[Dict]]: except Exception as e: logger.error(f"Failed to get latest biomes snapshot: {e}") return None + + def set_upstream_status(self, available: bool): + """Update upstream API availability status""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Store upstream status with current timestamp + cursor.execute( + """INSERT INTO war_status (data) VALUES (?)""", + (json.dumps({"_upstream_status": available, "_timestamp": time.time()}),), + ) + conn.commit() + logger.debug(f"Upstream API status set to: {available}") + except Exception as e: + logger.error(f"Failed to set upstream status: {e}") + + def get_upstream_status(self) -> bool: + """Get last known upstream API status (defaults to True if no data)""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Get the most recent status entry + cursor.execute( + "SELECT data FROM war_status WHERE json_extract(data, '$._upstream_status') IS NOT NULL ORDER BY timestamp DESC LIMIT 1" + ) + result = cursor.fetchone() + + if result: + data = json.loads(result[0]) + if "_upstream_status" in data: + return data["_upstream_status"] + + # Default to True if no status data found + return True + except Exception as e: + logger.error(f"Failed to get upstream status: {e}") + return True diff --git a/src/scraper.py b/src/scraper.py index 8cd9a6c..f36ba55 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -26,10 +26,6 @@ def __init__(self, timeout: int = 30, base_url: Optional[str] = None): # Thread lock for rate limiting (scraper is safe for concurrent access) self._rate_limit_lock = threading.Lock() - # Track upstream API status (set by collector during data collection) - self.upstream_api_available = True - self._status_lock = threading.Lock() - # Get headers from config, use "NA" if not configured client_name = ( Config.HELLDIVERS_API_CLIENT_NAME @@ -62,16 +58,6 @@ def _rate_limit(self): time.sleep(delay) self.last_request_time = time.time() - def set_upstream_status(self, available: bool): - """Update upstream API availability status (called during data collection)""" - with self._status_lock: - self.upstream_api_available = available - - def is_upstream_available(self) -> bool: - """Get current upstream API status (no API call, just returns cached status)""" - with self._status_lock: - return self.upstream_api_available - def get_war_status(self) -> Optional[Dict]: """Fetch current war status""" try: From 7e343c0f5bfb1a0af8da66090171ef30e46574b7 Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Mon, 20 Oct 2025 20:54:03 -0700 Subject: [PATCH 10/10] fix: update HTTP status codes for error handling and improve logging in data collector --- src/app.py | 16 +++++++++------- src/collector.py | 14 ++++++-------- src/database.py | 32 ++++++++++++++++++++++---------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/app.py b/src/app.py index 9c6670a..3e7a505 100644 --- a/src/app.py +++ b/src/app.py @@ -90,7 +90,7 @@ async def get_campaigns(): if data is not None: return data - raise HTTPException(status_code=404, detail="No campaign data available (live fetch failed and no cached data)") + raise HTTPException(status_code=503, detail="No campaign data available (live fetch failed and no cached data)") @app.get("/api/campaigns/active", tags=["Campaigns"]) @@ -119,7 +119,7 @@ async def get_planets(): if data is not None: return data - raise HTTPException(status_code=404, detail="Failed to fetch planets and no cached data available") + raise HTTPException(status_code=503, detail="Failed to fetch planets and no cached data available") @app.get("/api/planets/{planet_index}", tags=["Planets"]) @@ -131,12 +131,14 @@ async def get_planet_status(planet_index: int): # Fallback to cache if live API fails if data is None: history = db.get_planet_status_history(planet_index, limit=1) - if history: - data = history[0].get("data") if isinstance(history[0], dict) and "data" in history[0] else history[0] + if history and isinstance(history[0], dict) and "data" in history[0]: + data = history[0]["data"] + elif history: + data = history[0] if data is not None: return data - raise HTTPException(status_code=404, detail=f"Failed to fetch planet {planet_index} and no cached data available") + raise HTTPException(status_code=503, detail=f"Failed to fetch planet {planet_index} and no cached data available") @app.get("/api/planets/{planet_index}/history", tags=["Planets"]) @@ -198,7 +200,7 @@ async def get_factions(): if data is not None: return data - raise HTTPException(status_code=404, detail="Failed to fetch factions and no cached data available") + raise HTTPException(status_code=503, detail="Failed to fetch factions and no cached data available") # ======================== @@ -218,7 +220,7 @@ async def get_biomes(): if data is not None: return data - raise HTTPException(status_code=404, detail="Failed to fetch biomes and no cached data available") + raise HTTPException(status_code=503, detail="Failed to fetch biomes and no cached data available") # ======================== diff --git a/src/collector.py b/src/collector.py index dd06655..3ad1069 100644 --- a/src/collector.py +++ b/src/collector.py @@ -43,21 +43,19 @@ def collect_all_data(self): logger.info("Starting data collection cycle") try: - # Collect war status (this includes statistics) + # Collect war status war_data = self.scraper.get_war_status() if war_data is not None: - if war_data: - self.db.save_war_status(war_data) - logger.info("War status collected") + self.db.save_war_status(war_data) + logger.info("War status collected") else: logger.warning("Failed to collect war status") - # Collect statistics (extracted from war status) + # Collect statistics stats_data = self.scraper.get_statistics() if stats_data is not None: - if stats_data: - self.db.save_statistics(stats_data) - logger.info("Statistics collected") + self.db.save_statistics(stats_data) + logger.info("Statistics collected") else: logger.warning("Failed to collect statistics") diff --git a/src/database.py b/src/database.py index ee51da9..5472666 100644 --- a/src/database.py +++ b/src/database.py @@ -1,7 +1,6 @@ import sqlite3 import json import logging -import time from typing import Dict, List, Optional logger = logging.getLogger(__name__) @@ -109,6 +108,18 @@ def init_db(self): """ ) + # System status table (for internal metadata like upstream API status) + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS system_status ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + status_key TEXT UNIQUE, + status_value BOOLEAN, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + # Index for faster queries cursor.execute("CREATE INDEX IF NOT EXISTS idx_war_timestamp ON war_status(timestamp)") cursor.execute( @@ -120,6 +131,7 @@ def init_db(self): cursor.execute("CREATE INDEX IF NOT EXISTS idx_assignment_timestamp ON assignments(timestamp)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_dispatch_timestamp ON dispatches(timestamp)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_event_timestamp ON planet_events(timestamp)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_system_status_key ON system_status(status_key)") conn.commit() @@ -475,14 +487,15 @@ def get_latest_biomes_snapshot(self) -> Optional[List[Dict]]: return None def set_upstream_status(self, available: bool): - """Update upstream API availability status""" + """Update upstream API availability status in system_status table""" try: with sqlite3.connect(self.db_path) as conn: cursor = conn.cursor() - # Store upstream status with current timestamp + # Use INSERT OR REPLACE to upsert the status cursor.execute( - """INSERT INTO war_status (data) VALUES (?)""", - (json.dumps({"_upstream_status": available, "_timestamp": time.time()}),), + """INSERT OR REPLACE INTO system_status (status_key, status_value) + VALUES ('upstream_api_available', ?)""", + (available,), ) conn.commit() logger.debug(f"Upstream API status set to: {available}") @@ -494,16 +507,15 @@ def get_upstream_status(self) -> bool: try: with sqlite3.connect(self.db_path) as conn: cursor = conn.cursor() - # Get the most recent status entry + # Get the upstream API status cursor.execute( - "SELECT data FROM war_status WHERE json_extract(data, '$._upstream_status') IS NOT NULL ORDER BY timestamp DESC LIMIT 1" + """SELECT status_value FROM system_status + WHERE status_key = 'upstream_api_available' LIMIT 1""" ) result = cursor.fetchone() if result: - data = json.loads(result[0]) - if "_upstream_status" in data: - return data["_upstream_status"] + return bool(result[0]) # Default to True if no status data found return True