Scan updates & robustness

backend/src/repositories/giveaway.py

@@ -8,7 +8,7 @@
 from datetime import datetime, timedelta
 from typing import Any
 
-from sqlalchemy import and_, or_, select
+from sqlalchemy import and_, or_, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from core.time import utcnow
@@ -399,6 +399,39 @@ async def get_wishlist(
         result = await self.session.execute(query)
         return list(result.scalars().all())
 
+    async def unset_wishlist_except(self, codes: set[str]) -> int:
+        """
+        Clear the wishlist flag on active giveaways not in ``codes``.
+
+        Called after a *complete* wishlist scan: any active giveaway still
+        flagged wishlist but absent from the scan is no longer on the user's
+        Steam wishlist. Expired giveaways keep their flag as history.
+
+        Args:
+            codes: Giveaway codes seen in the completed wishlist scan
+
+        Returns:
+            Number of giveaways whose flag was cleared
+
+        Note:
+            This method does NOT commit the transaction. The caller must
+            call session.commit() to persist changes to the database.
+        """
+        now = utcnow()
+        stmt = (
+            update(self.model)
+            .where(
+                self.model.is_wishlist == True,  # noqa: E712
+                self.model.end_time.isnot(None),
+                self.model.end_time > now,
+                self.model.code.notin_(codes),
+            )
+            .values(is_wishlist=False)
+        )
+        result = await self.session.execute(stmt)
+        # execute() is typed as Result, but UPDATE always yields a CursorResult.
+        return int(result.rowcount or 0)  # type: ignore[attr-defined]
+
     async def get_won(
         self, limit: int | None = None, offset: int | None = None
     ) -> list[Giveaway]:

backend/src/services/giveaway_sync.py

@@ -7,12 +7,17 @@
 from core.exceptions import SteamGiftsError
 from core.time import utcnow
 from models.giveaway import Giveaway
+from repositories.activity_log import ActivityLogRepository
 from repositories.giveaway import GiveawayRepository
 from services.game_service import GameService
-from utils.steamgifts_client import SteamGiftsClient
+from utils.steamgifts_client import SteamGiftsClient, SteamGiftsScrapeDriftError
 
 logger = structlog.get_logger()
 
+# SteamGifts renders 50 giveaways per listing page; a shorter page is the
+# last one, which tells us a multi-page scan saw the complete list.
+GIVEAWAYS_PER_PAGE = 50
+
 
 class GiveawaySyncMixin:
     """Giveaway Syncmethods of GiveawayService (see services.giveaway_service).
@@ -62,6 +67,11 @@ async def sync_giveaways(
         """
         new_count = 0
         updated_count = 0
+        scraped_codes: set[str] = set()
+        # True when we saw the end of the list (an empty or short page), as
+        # opposed to stopping at the page cap or on an error. Only a complete
+        # wishlist scan is allowed to un-stick stale wishlist flags below.
+        scan_complete = False
 
         for page in range(1, pages + 1):
             try:
@@ -72,31 +82,64 @@ async def sync_giveaways(
                     dlc_only=dlc_only,
                     min_copies=min_copies,
                 )
-
-                for ga_data in giveaways_data:
-                    # Check if exists
-                    existing = await self.giveaway_repo.get_by_code(ga_data["code"])
-
-                    # Cache game data if we have game_id
-                    if ga_data.get("game_id"):
-                        try:
-                            await self.game_service.get_or_fetch_game(ga_data["game_id"])
-                        except Exception as e:
-                            logger.error("game_cache_failed", game_id=ga_data["game_id"], error=str(e))
-
-                    if existing:
-                        # Update existing giveaway
-                        await self._update_giveaway(existing, ga_data)
-                        updated_count += 1
-                    else:
-                        # Create new giveaway
-                        await self._create_giveaway(ga_data)
-                        new_count += 1
-
+            except SteamGiftsScrapeDriftError as e:
+                # The page parsed to zero rows without a no-results marker:
+                # the markup likely changed. Surface it in the activity log so
+                # the UI shows it, and don't treat the scan as complete.
+                logger.error("scrape_drift_detected", page=page, error=str(e))
+                await ActivityLogRepository(self.session).create(
+                    level="warning",
+                    event_type="error",
+                    message=(
+                        "Scan parsed 0 giveaways from a non-empty page - "
+                        "SteamGifts may have changed their markup"
+                    ),
+                )
+                break
             except SteamGiftsError as e:
                 logger.error("giveaway_page_fetch_failed", page=page, error=str(e))
                 break
 
+            if not giveaways_data:
+                # Legitimate end of the list (no-results marker present).
+                scan_complete = True
+                break
+
+            for ga_data in giveaways_data:
+                scraped_codes.add(ga_data["code"])
+
+                # Check if exists
+                existing = await self.giveaway_repo.get_by_code(ga_data["code"])
+
+                # Cache game data if we have game_id
+                if ga_data.get("game_id"):
+                    try:
+                        await self.game_service.get_or_fetch_game(ga_data["game_id"])
+                    except Exception as e:
+                        logger.error("game_cache_failed", game_id=ga_data["game_id"], error=str(e))
+
+                if existing:
+                    # Update existing giveaway
+                    await self._update_giveaway(existing, ga_data)
+                    updated_count += 1
+                else:
+                    # Create new giveaway
+                    await self._create_giveaway(ga_data)
+                    new_count += 1
+
+            if len(giveaways_data) < GIVEAWAYS_PER_PAGE:
+                # Short page = last page of the list.
+                scan_complete = True
+                break
+
+        # A complete wishlist scan is the source of truth for the wishlist
+        # flag: clear it on active giveaways that no longer appear (the game
+        # was removed from the Steam wishlist). Never after partial scans.
+        if giveaway_type == "wishlist" and scan_complete:
+            cleared = await self.giveaway_repo.unset_wishlist_except(scraped_codes)
+            if cleared:
+                logger.info("wishlist_flags_cleared", count=cleared)
+
         await self.session.commit()
         return new_count, updated_count
 
@@ -252,6 +295,8 @@ async def _update_giveaway(self, giveaway: Giveaway, ga_data: dict) -> None:
         if ga_data.get("game_id") and not giveaway.game_id:
             giveaway.game_id = ga_data["game_id"]
 
-        # Update wishlist flag (can change from False to True, but not back)
+        # Set the wishlist flag when this row came from a wishlist scan.
+        # The reverse direction (un-sticking removed games) is handled by
+        # unset_wishlist_except after a complete wishlist scan.
         if ga_data.get("is_wishlist"):
             giveaway.is_wishlist = True

backend/src/utils/steamgifts_client.py

@@ -39,6 +39,15 @@ def __init__(self, message: str, safety_score: int = 0):
         self.safety_score = safety_score
 
 
+class SteamGiftsScrapeDriftError(SteamGiftsError):
+    """Raised when a listing page yields zero giveaways but lacks the
+    'no results' marker — the site markup has likely changed and the
+    scraper needs updating. Never raised for legitimately empty lists."""
+
+    def __init__(self, message: str, page: int = 0):
+        super().__init__(message, code="SG_006", details={"page": page})
+
+
 
 class SteamGiftsClient:
     """
@@ -441,10 +450,28 @@ async def get_giveaways(
                 details={"status_code": response.status_code},
             )
 
-        return parser.parse_giveaway_list(
+        giveaways = parser.parse_giveaway_list(
             response.text, mark_wishlist=giveaway_type == "wishlist"
         )
 
+        # Zero rows without the explicit "No results were found." marker means
+        # the page is unrecognizable — fail loudly instead of silently
+        # returning an empty list (see the Featured-section regression).
+        if not giveaways and not parser.has_no_results_marker(response.text):
+            logger.warning(
+                "scrape_drift_suspected",
+                page=page,
+                giveaway_type=giveaway_type,
+                html_size=len(response.text),
+            )
+            raise SteamGiftsScrapeDriftError(
+                "Giveaway page parsed to zero rows without a no-results marker "
+                "- SteamGifts markup may have changed",
+                page=page,
+            )
+
+        return giveaways
+
     async def enter_giveaway(self, giveaway_code: str) -> bool:
         """
         Enter a giveaway.

backend/src/utils/steamgifts_parser.py

@@ -29,6 +29,17 @@
 GOOD_WORDS = (" bank", " banan", " both", " band", " banner", " bang")
 
 
+def has_no_results_marker(html: str) -> bool:
+    """True when the page explicitly says the search matched nothing.
+
+    SteamGifts renders ``<div class="pagination pagination--no-results">``
+    ("No results were found.") on legitimately empty result pages. Zero parsed
+    giveaways WITHOUT this marker means the markup has likely changed and the
+    scraper is broken (scrape drift), not that the list is empty.
+    """
+    return "pagination--no-results" in html
+
+
 def extract_xsrf_token(html: str) -> str | None:
     """Extract the XSRF token embedded in any authenticated page, or None."""
     soup = BeautifulSoup(html, "html.parser")

backend/src/workers/automation.py

@@ -93,8 +93,10 @@ async def automation_cycle() -> dict[str, Any]:
             logger.info("automation_step", step="scan_wishlist")
 
             try:
+                # Same page cap as the regular scan; the sync stops early at
+                # the end of the list, so small wishlists cost one request.
                 wishlist_new, wishlist_updated = await giveaway_service.sync_giveaways(
-                    pages=1,
+                    pages=max_pages,
                     giveaway_type="wishlist"
                 )
                 results["wishlist"] = {

backend/src/workers/scanner.py

@@ -59,13 +59,15 @@ async def scan_giveaways() -> dict[str, Any]:
                 pages=max_pages
             )
 
-            # Also scan wishlist giveaways (single page, like the automation
-            # cycle) so the Wishlist tab is populated by manual scans too.
-            # A wishlist failure shouldn't fail the whole scan.
+            # Also scan wishlist giveaways so the Wishlist tab is populated by
+            # manual scans too. Uses the same page cap as the regular scan;
+            # the sync stops early at the end of the list, so small wishlists
+            # cost a single request. A wishlist failure shouldn't fail the
+            # whole scan.
             wishlist_new = wishlist_updated = 0
             try:
                 wishlist_new, wishlist_updated = await ctx.giveaway_service.sync_giveaways(
-                    pages=1, giveaway_type="wishlist"
+                    pages=max_pages, giveaway_type="wishlist"
                 )
             except Exception as e:
                 logger.error("scan_wishlist_failed", error=str(e))