diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..035a30e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,43 @@ +version: 2 +updates: + # Enable version updates for Python dependencies + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "03:00" + open-pull-requests-limit: 10 + reviewers: + - "surrealwolf" + labels: + - "dependencies" + - "python" + allow: + - dependency-type: "all" + # Group patch updates together + groups: + patch-minor-updates: + patterns: + - patch + - minor + major-updates: + patterns: + - major + commit-message: + prefix: "build" + prefix-scope: "deps" + include: "scope" + + # Enable version updates for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "03:00" + reviewers: + - "surrealwolf" + labels: + - "dependencies" + - "github-actions" diff --git a/src/app.py b/src/app.py index c1797e2..3e7a505 100644 --- a/src/app.py +++ b/src/app.py @@ -80,11 +80,17 @@ async def refresh_war_status(): @app.get("/api/campaigns", tags=["Campaigns"]) async def get_campaigns(): - """Get campaign information""" + """Get campaign information (with cache fallback)""" + # Try live API first data = scraper.get_campaign_info() - if data: + + # Fallback to cache if live API fails + if data is None: + data = db.get_latest_campaigns_snapshot() + + if data is not None: return data - raise HTTPException(status_code=404, detail="Failed to fetch campaigns") + raise HTTPException(status_code=503, detail="No campaign data available (live fetch failed and no cached data)") @app.get("/api/campaigns/active", tags=["Campaigns"]) @@ -103,20 +109,36 @@ async def get_active_campaigns(): @app.get("/api/planets", tags=["Planets"]) async def get_planets(): - """Get all planets""" + """Get all planets (with cache fallback)""" + # Try live API first data = scraper.get_planets() - if data: + + # Fallback to cache if live API fails + if data is None: + data = db.get_latest_planets_snapshot() + + if data is not None: return data - raise HTTPException(status_code=404, detail="Failed to fetch planets") + raise HTTPException(status_code=503, detail="Failed to fetch planets and no cached data available") @app.get("/api/planets/{planet_index}", tags=["Planets"]) async def get_planet_status(planet_index: int): - """Get status of a specific planet""" + """Get status of a specific planet (with cache fallback)""" + # Try live API first data = collector.collect_planet_data(planet_index) - if data: + + # Fallback to cache if live API fails + if data is None: + history = db.get_planet_status_history(planet_index, limit=1) + if history and isinstance(history[0], dict) and "data" in history[0]: + data = history[0]["data"] + elif history: + data = history[0] + + if data is not None: return data - raise HTTPException(status_code=404, detail=f"Failed to fetch planet {planet_index}") + raise HTTPException(status_code=503, detail=f"Failed to fetch planet {planet_index} and no cached data available") @app.get("/api/planets/{planet_index}/history", tags=["Planets"]) @@ -168,11 +190,17 @@ async def refresh_statistics(): @app.get("/api/factions", tags=["Factions"]) async def get_factions(): - """Get all factions""" + """Get all factions (with cache fallback)""" + # Try live API first data = scraper.get_factions() - if data: + + # Fallback to cache if live API fails + if data is None: + data = db.get_latest_factions_snapshot() + + if data is not None: return data - raise HTTPException(status_code=404, detail="Failed to fetch factions") + raise HTTPException(status_code=503, detail="Failed to fetch factions and no cached data available") # ======================== @@ -182,11 +210,17 @@ async def get_factions(): @app.get("/api/biomes", tags=["Biomes"]) async def get_biomes(): - """Get all biomes""" + """Get all biomes (with cache fallback)""" + # Try live API first data = scraper.get_biomes() - if data: + + # Fallback to cache if live API fails + if data is None: + data = db.get_latest_biomes_snapshot() + + if data is not None: return data - raise HTTPException(status_code=404, detail="Failed to fetch biomes") + raise HTTPException(status_code=503, detail="Failed to fetch biomes and no cached data available") # ======================== @@ -197,7 +231,14 @@ async def get_biomes(): @app.get("/api/health", tags=["Health"]) async def health_check(): """Health check endpoint""" - return {"status": "healthy", "collector_running": collector.is_running} + # Returns local service status and upstream API status + # No external API calls made here to avoid blocking event loop or consuming rate limits + upstream_status = db.get_upstream_status() + return { + "status": "healthy", + "collector_running": collector.is_running, + "upstream_api": "online" if upstream_status else "offline" + } # ======================== diff --git a/src/collector.py b/src/collector.py index c6ab4ac..3ad1069 100644 --- a/src/collector.py +++ b/src/collector.py @@ -45,57 +45,90 @@ def collect_all_data(self): try: # Collect war status war_data = self.scraper.get_war_status() - if war_data: + if war_data is not None: self.db.save_war_status(war_data) logger.info("War status collected") + else: + logger.warning("Failed to collect war status") - # Collect statistics (extracted from war status) + # Collect statistics stats_data = self.scraper.get_statistics() - if stats_data: + if stats_data is not None: self.db.save_statistics(stats_data) logger.info("Statistics collected") + else: + logger.warning("Failed to collect statistics") # Collect planets planets = self.scraper.get_planets() - if planets: - for planet in planets: - planet_index = planet.get("index") - if planet_index: - self.db.save_planet_status(planet_index, planet) - logger.info(f"Collected data for {len(planets)} planets") + if planets is not None: + if planets: + for planet in planets: + planet_index = planet.get("index") + if planet_index: + self.db.save_planet_status(planet_index, planet) + logger.info(f"Collected data for {len(planets)} planets") + else: + logger.info("Collected 0 planets (empty response)") + else: + logger.warning("Failed to collect planets") # Collect campaigns campaigns = self.scraper.get_campaign_info() - if campaigns: - for campaign in campaigns: - campaign_id = campaign.get("id") - if campaign_id and campaign.get("planet") and "index" in campaign["planet"]: - planet_index = campaign["planet"].get("index") - self.db.save_campaign(campaign_id, planet_index, campaign) - logger.info(f"Collected {len(campaigns)} campaigns") + if campaigns is not None: + if campaigns: + for campaign in campaigns: + campaign_id = campaign.get("id") + if campaign_id and campaign.get("planet") and "index" in campaign["planet"]: + planet_index = campaign["planet"].get("index") + self.db.save_campaign(campaign_id, planet_index, campaign) + logger.info(f"Collected {len(campaigns)} campaigns") + else: + logger.info("Collected 0 campaigns (empty response)") + else: + logger.warning("Failed to collect campaigns") # Collect assignments (Major Orders) assignments = self.scraper.get_assignments() - if assignments: - self.db.save_assignments(assignments) - logger.info(f"Collected {len(assignments)} assignments") + if assignments is not None: + if assignments: + self.db.save_assignments(assignments) + logger.info(f"Collected {len(assignments)} assignments") + else: + logger.info("Collected 0 assignments (empty response)") + else: + logger.warning("Failed to collect assignments") # Collect dispatches (news) dispatches = self.scraper.get_dispatches() - if dispatches: - self.db.save_dispatches(dispatches) - logger.info(f"Collected {len(dispatches)} dispatches") + if dispatches is not None: + if dispatches: + self.db.save_dispatches(dispatches) + logger.info(f"Collected {len(dispatches)} dispatches") + else: + logger.info("Collected 0 dispatches (empty response)") + else: + logger.warning("Failed to collect dispatches") # Collect planet events events = self.scraper.get_planet_events() if events is not None: - self.db.save_planet_events(events) - logger.info(f"Collected {len(events)} planet events") + if events: + self.db.save_planet_events(events) + logger.info(f"Collected {len(events)} planet events") + else: + logger.info("Collected 0 planet events (empty response)") + else: + logger.warning("Failed to collect planet events") logger.info("Data collection cycle completed successfully") + # Mark upstream as available after successful collection + self.db.set_upstream_status(True) except Exception as e: logger.error(f"Error during data collection: {e}") + # Mark upstream as unavailable on any collection error + self.db.set_upstream_status(False) def collect_planet_data(self, planet_index: int): """Collect data for a specific planet""" diff --git a/src/database.py b/src/database.py index eafcb64..5472666 100644 --- a/src/database.py +++ b/src/database.py @@ -108,6 +108,18 @@ def init_db(self): """ ) + # System status table (for internal metadata like upstream API status) + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS system_status ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + status_key TEXT UNIQUE, + status_value BOOLEAN, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + # Index for faster queries cursor.execute("CREATE INDEX IF NOT EXISTS idx_war_timestamp ON war_status(timestamp)") cursor.execute( @@ -119,6 +131,7 @@ def init_db(self): cursor.execute("CREATE INDEX IF NOT EXISTS idx_assignment_timestamp ON assignments(timestamp)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_dispatch_timestamp ON dispatches(timestamp)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_event_timestamp ON planet_events(timestamp)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_system_status_key ON system_status(status_key)") conn.commit() @@ -356,3 +369,156 @@ def get_latest_planet_events(self, limit: int = 10) -> List[Dict]: except Exception as e: logger.error(f"Failed to get latest planet events: {e}") return [] + + def get_latest_planets_snapshot(self) -> Optional[List[Dict]]: + """Get most recent cached snapshot of all planets + + Used as fallback when live API is unavailable. + Returns all planet status records from the most recent collection cycle. + """ + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Get the most recent timestamp from planet_status + cursor.execute("SELECT DISTINCT timestamp FROM planet_status ORDER BY timestamp DESC LIMIT 1") + result = cursor.fetchone() + + if not result: + return None + + latest_timestamp = result[0] + + # Get all planets from that timestamp + cursor.execute( + "SELECT data FROM planet_status WHERE timestamp = ? ORDER BY planet_index ASC", + (latest_timestamp,), + ) + results = cursor.fetchall() + return [json.loads(row[0]) for row in results] if results else None + except Exception as e: + logger.error(f"Failed to get latest planets snapshot: {e}") + return None + + def get_latest_campaigns_snapshot(self) -> Optional[List[Dict]]: + """Get most recent cached snapshot of all campaigns + + Used as fallback when live API is unavailable. + Returns most recent campaign data for each campaign. + """ + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Get the most recent campaign data for each campaign_id + cursor.execute( + """SELECT data FROM campaigns + WHERE (campaign_id, timestamp) IN ( + SELECT campaign_id, MAX(timestamp) FROM campaigns GROUP BY campaign_id + ) + ORDER BY timestamp DESC""" + ) + results = cursor.fetchall() + return [json.loads(row[0]) for row in results] if results else None + except Exception as e: + logger.error(f"Failed to get latest campaigns snapshot: {e}") + return None + + def get_latest_factions_snapshot(self) -> Optional[List[Dict]]: + """Get most recent cached snapshot of all factions + + Used as fallback when live API is unavailable. + Factions are extracted from war status data. + """ + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute("SELECT data FROM war_status ORDER BY timestamp DESC LIMIT 1") + result = cursor.fetchone() + + if not result: + return None + + war_data = json.loads(result[0]) + return war_data.get("factions", None) + except Exception as e: + logger.error(f"Failed to get latest factions snapshot: {e}") + return None + + def get_latest_biomes_snapshot(self) -> Optional[List[Dict]]: + """Get most recent cached snapshot of all biomes + + Used as fallback when live API is unavailable. + Biomes are extracted from planet data. + """ + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Get the most recent timestamp from planet_status + cursor.execute("SELECT DISTINCT timestamp FROM planet_status ORDER BY timestamp DESC LIMIT 1") + result = cursor.fetchone() + + if not result: + return None + + latest_timestamp = result[0] + + # Get all planets from that timestamp and extract unique biomes + cursor.execute( + "SELECT data FROM planet_status WHERE timestamp = ?", + (latest_timestamp,), + ) + results = cursor.fetchall() + + if not results: + return None + + # Extract unique biomes from planets + biomes = {} + for row in results: + planet_data = json.loads(row[0]) + # Type guard: check if biome is a dict before accessing .get() + if "biome" in planet_data and isinstance(planet_data["biome"], dict): + biome_name = planet_data["biome"].get("name") + if biome_name and biome_name not in biomes: + biomes[biome_name] = planet_data["biome"] + + return list(biomes.values()) if biomes else None + except Exception as e: + logger.error(f"Failed to get latest biomes snapshot: {e}") + return None + + def set_upstream_status(self, available: bool): + """Update upstream API availability status in system_status table""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Use INSERT OR REPLACE to upsert the status + cursor.execute( + """INSERT OR REPLACE INTO system_status (status_key, status_value) + VALUES ('upstream_api_available', ?)""", + (available,), + ) + conn.commit() + logger.debug(f"Upstream API status set to: {available}") + except Exception as e: + logger.error(f"Failed to set upstream status: {e}") + + def get_upstream_status(self) -> bool: + """Get last known upstream API status (defaults to True if no data)""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # Get the upstream API status + cursor.execute( + """SELECT status_value FROM system_status + WHERE status_key = 'upstream_api_available' LIMIT 1""" + ) + result = cursor.fetchone() + + if result: + return bool(result[0]) + + # Default to True if no status data found + return True + except Exception as e: + logger.error(f"Failed to get upstream status: {e}") + return True