diff --git a/backend/src/repositories/giveaway.py b/backend/src/repositories/giveaway.py index 3c06079..9a6e5b0 100644 --- a/backend/src/repositories/giveaway.py +++ b/backend/src/repositories/giveaway.py @@ -8,7 +8,7 @@ from datetime import datetime, timedelta from typing import Any -from sqlalchemy import and_, or_, select +from sqlalchemy import and_, or_, select, update from sqlalchemy.ext.asyncio import AsyncSession from core.time import utcnow @@ -399,6 +399,39 @@ async def get_wishlist( result = await self.session.execute(query) return list(result.scalars().all()) + async def unset_wishlist_except(self, codes: set[str]) -> int: + """ + Clear the wishlist flag on active giveaways not in ``codes``. + + Called after a *complete* wishlist scan: any active giveaway still + flagged wishlist but absent from the scan is no longer on the user's + Steam wishlist. Expired giveaways keep their flag as history. + + Args: + codes: Giveaway codes seen in the completed wishlist scan + + Returns: + Number of giveaways whose flag was cleared + + Note: + This method does NOT commit the transaction. The caller must + call session.commit() to persist changes to the database. + """ + now = utcnow() + stmt = ( + update(self.model) + .where( + self.model.is_wishlist == True, # noqa: E712 + self.model.end_time.isnot(None), + self.model.end_time > now, + self.model.code.notin_(codes), + ) + .values(is_wishlist=False) + ) + result = await self.session.execute(stmt) + # execute() is typed as Result, but UPDATE always yields a CursorResult. + return int(result.rowcount or 0) # type: ignore[attr-defined] + async def get_won( self, limit: int | None = None, offset: int | None = None ) -> list[Giveaway]: diff --git a/backend/src/services/giveaway_sync.py b/backend/src/services/giveaway_sync.py index eb37758..d5ecc88 100644 --- a/backend/src/services/giveaway_sync.py +++ b/backend/src/services/giveaway_sync.py @@ -7,12 +7,17 @@ from core.exceptions import SteamGiftsError from core.time import utcnow from models.giveaway import Giveaway +from repositories.activity_log import ActivityLogRepository from repositories.giveaway import GiveawayRepository from services.game_service import GameService -from utils.steamgifts_client import SteamGiftsClient +from utils.steamgifts_client import SteamGiftsClient, SteamGiftsScrapeDriftError logger = structlog.get_logger() +# SteamGifts renders 50 giveaways per listing page; a shorter page is the +# last one, which tells us a multi-page scan saw the complete list. +GIVEAWAYS_PER_PAGE = 50 + class GiveawaySyncMixin: """Giveaway Syncmethods of GiveawayService (see services.giveaway_service). @@ -62,6 +67,11 @@ async def sync_giveaways( """ new_count = 0 updated_count = 0 + scraped_codes: set[str] = set() + # True when we saw the end of the list (an empty or short page), as + # opposed to stopping at the page cap or on an error. Only a complete + # wishlist scan is allowed to un-stick stale wishlist flags below. + scan_complete = False for page in range(1, pages + 1): try: @@ -72,31 +82,64 @@ async def sync_giveaways( dlc_only=dlc_only, min_copies=min_copies, ) - - for ga_data in giveaways_data: - # Check if exists - existing = await self.giveaway_repo.get_by_code(ga_data["code"]) - - # Cache game data if we have game_id - if ga_data.get("game_id"): - try: - await self.game_service.get_or_fetch_game(ga_data["game_id"]) - except Exception as e: - logger.error("game_cache_failed", game_id=ga_data["game_id"], error=str(e)) - - if existing: - # Update existing giveaway - await self._update_giveaway(existing, ga_data) - updated_count += 1 - else: - # Create new giveaway - await self._create_giveaway(ga_data) - new_count += 1 - + except SteamGiftsScrapeDriftError as e: + # The page parsed to zero rows without a no-results marker: + # the markup likely changed. Surface it in the activity log so + # the UI shows it, and don't treat the scan as complete. + logger.error("scrape_drift_detected", page=page, error=str(e)) + await ActivityLogRepository(self.session).create( + level="warning", + event_type="error", + message=( + "Scan parsed 0 giveaways from a non-empty page - " + "SteamGifts may have changed their markup" + ), + ) + break except SteamGiftsError as e: logger.error("giveaway_page_fetch_failed", page=page, error=str(e)) break + if not giveaways_data: + # Legitimate end of the list (no-results marker present). + scan_complete = True + break + + for ga_data in giveaways_data: + scraped_codes.add(ga_data["code"]) + + # Check if exists + existing = await self.giveaway_repo.get_by_code(ga_data["code"]) + + # Cache game data if we have game_id + if ga_data.get("game_id"): + try: + await self.game_service.get_or_fetch_game(ga_data["game_id"]) + except Exception as e: + logger.error("game_cache_failed", game_id=ga_data["game_id"], error=str(e)) + + if existing: + # Update existing giveaway + await self._update_giveaway(existing, ga_data) + updated_count += 1 + else: + # Create new giveaway + await self._create_giveaway(ga_data) + new_count += 1 + + if len(giveaways_data) < GIVEAWAYS_PER_PAGE: + # Short page = last page of the list. + scan_complete = True + break + + # A complete wishlist scan is the source of truth for the wishlist + # flag: clear it on active giveaways that no longer appear (the game + # was removed from the Steam wishlist). Never after partial scans. + if giveaway_type == "wishlist" and scan_complete: + cleared = await self.giveaway_repo.unset_wishlist_except(scraped_codes) + if cleared: + logger.info("wishlist_flags_cleared", count=cleared) + await self.session.commit() return new_count, updated_count @@ -252,6 +295,8 @@ async def _update_giveaway(self, giveaway: Giveaway, ga_data: dict) -> None: if ga_data.get("game_id") and not giveaway.game_id: giveaway.game_id = ga_data["game_id"] - # Update wishlist flag (can change from False to True, but not back) + # Set the wishlist flag when this row came from a wishlist scan. + # The reverse direction (un-sticking removed games) is handled by + # unset_wishlist_except after a complete wishlist scan. if ga_data.get("is_wishlist"): giveaway.is_wishlist = True diff --git a/backend/src/utils/steamgifts_client.py b/backend/src/utils/steamgifts_client.py index 21fae93..8d41886 100644 --- a/backend/src/utils/steamgifts_client.py +++ b/backend/src/utils/steamgifts_client.py @@ -39,6 +39,15 @@ def __init__(self, message: str, safety_score: int = 0): self.safety_score = safety_score +class SteamGiftsScrapeDriftError(SteamGiftsError): + """Raised when a listing page yields zero giveaways but lacks the + 'no results' marker — the site markup has likely changed and the + scraper needs updating. Never raised for legitimately empty lists.""" + + def __init__(self, message: str, page: int = 0): + super().__init__(message, code="SG_006", details={"page": page}) + + class SteamGiftsClient: """ @@ -441,10 +450,28 @@ async def get_giveaways( details={"status_code": response.status_code}, ) - return parser.parse_giveaway_list( + giveaways = parser.parse_giveaway_list( response.text, mark_wishlist=giveaway_type == "wishlist" ) + # Zero rows without the explicit "No results were found." marker means + # the page is unrecognizable — fail loudly instead of silently + # returning an empty list (see the Featured-section regression). + if not giveaways and not parser.has_no_results_marker(response.text): + logger.warning( + "scrape_drift_suspected", + page=page, + giveaway_type=giveaway_type, + html_size=len(response.text), + ) + raise SteamGiftsScrapeDriftError( + "Giveaway page parsed to zero rows without a no-results marker " + "- SteamGifts markup may have changed", + page=page, + ) + + return giveaways + async def enter_giveaway(self, giveaway_code: str) -> bool: """ Enter a giveaway. diff --git a/backend/src/utils/steamgifts_parser.py b/backend/src/utils/steamgifts_parser.py index 68dd64b..f49d1c1 100644 --- a/backend/src/utils/steamgifts_parser.py +++ b/backend/src/utils/steamgifts_parser.py @@ -29,6 +29,17 @@ GOOD_WORDS = (" bank", " banan", " both", " band", " banner", " bang") +def has_no_results_marker(html: str) -> bool: + """True when the page explicitly says the search matched nothing. + + SteamGifts renders ``