Data collection module cleanup

PlayWright/scraper.py

@@ -9,8 +9,8 @@
 
   2. Fetch all rows from the Supabase `taxonomy` table (column: `ingredient`)
      Ingredient format — two cases:
-       "eggs"             → single word: search "eggs", keep ALL results
-       "cabbage, red"     → word + descriptor: search "cabbage, red" (full phrase),
+       "eggs"             -> single word: search "eggs", keep ALL results
+       "cabbage, red"     -> word + descriptor: search "cabbage, red" (full phrase),
                             keep only products whose name contains "red"
 
   3. For each ingredient:
@@ -53,10 +53,7 @@
 from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError
 from supabase import create_client
 
-# ---------------------------------------------------------------------------
-# Configuration
-# ---------------------------------------------------------------------------
-
+# Config
 _ENV_PATH = Path(__file__).parent.parent / ".env"
 load_dotenv(_ENV_PATH)
 
@@ -69,10 +66,8 @@
 GRID_WAIT_MS = 5_000
 
 
-# ---------------------------------------------------------------------------
-# Confirmed Instacart DOM selectors (verified against live DOM)
-# ---------------------------------------------------------------------------
 
+# Confirmed Instacart DOM selectors (verified against live DOM)
 STORE_ROW_SELECTOR    = '[data-testid="CrossRetailerResultRowWrapper"]'
 ITEM_CARD_SELECTOR    = '[data-testid^="item_list_item_"]'
 NEXT_PAGE_SELECTOR    = '[aria-label="Next page"]'
@@ -86,10 +81,7 @@
 # Maximum carousel pages to advance per store row.
 MAX_CAROUSEL_PAGES = 10
 
-# ---------------------------------------------------------------------------
 # Ingredient parsing
-# ---------------------------------------------------------------------------
-
 def parse_ingredient(ingredient: str) -> tuple[str, str | None]:
     """
     Parse a taxonomy ingredient string into (search_term, filter_word).
@@ -114,10 +106,7 @@ def apply_name_filter(products: list[dict], filter_word: str | None) -> list[dic
     fw = filter_word.lower()
     return [p for p in products if p["name"] and fw in p["name"].lower()]
 
-# ---------------------------------------------------------------------------
 # Core scraper
-# ---------------------------------------------------------------------------
-
 async def scrape_query(page, query: str) -> list[dict]:
     """
     Scrape all product cards for *query* on the Instacart cross-retailer page.
@@ -126,7 +115,7 @@ async def scrape_query(page, query: str) -> list[dict]:
     """
     url = BASE_URL.format(query=quote_plus(query))
 
-    # -- Navigate ----------------------------------------------------------
+    # Navigate
     try:
         response = await page.goto(url, wait_until="domcontentloaded", timeout=30_000)
         if response and response.status >= 400:
@@ -139,7 +128,7 @@ async def scrape_query(page, query: str) -> list[dict]:
         print(f"  [ERROR] Navigation error for query '{query}': {exc}")
         return []
 
-    # -- Block detection ---------------------------------------------------
+    # Block detection
     try:
         body_text = (await page.inner_text("body")).lower()
     except Exception:
@@ -157,16 +146,16 @@ async def scrape_query(page, query: str) -> list[dict]:
         print(f"  [WARN] Login modal detected for query '{query}' — skipping")
         return []
 
-    # -- Wait for store rows -----------------------------------------------
+    # Wait for store rows
     try:
         await page.wait_for_selector(STORE_ROW_SELECTOR, timeout=GRID_WAIT_MS)
     except PlaywrightTimeoutError:
         print(f"  [WARN] No store grid found for query '{query}' after {GRID_WAIT_MS}ms")
         return []
 
-    # -- Discovery + Scraping (merged) — process each carousel page inline --
-    # Cards are clicked immediately while their carousel page is active,
-    # so ElementHandles never become stale from carousel advancement.
+    # Discovery + Scraping (merged) — process each carousel page inline --
+    #   Cards are clicked immediately while their carousel page is active,
+    #   so ElementHandles never become stale from carousel advancement.
     store_rows = await page.query_selector_all(STORE_ROW_SELECTOR)
 
     if not store_rows:
@@ -414,10 +403,8 @@ def _empty_product(store_name: str) -> dict:
         "description": None, "out_of_stock": False,
     }
 
-# ---------------------------------------------------------------------------
-# Pipeline entry point
-# ---------------------------------------------------------------------------
 
+# Pipeline entry point
 async def run_pipeline(limit: int | None = None) -> None:
     """
     Full pipeline: fetch taxonomy → scrape Instacart → upsert to Supabase.
@@ -428,7 +415,7 @@ async def run_pipeline(limit: int | None = None) -> None:
             "SUPABASE_URL and SUPABASE_SERVICE_KEY must be set in .env"
         )
 
-    # -- Fetch taxonomy ----------------------------------------------------
+    # Fetch taxonomy
     print("Fetching taxonomy from Supabase...")
     supabase = create_client(SUPABASE_URL, SUPABASE_KEY)
     response = supabase.table("taxonomy").select("ingredient").execute()
@@ -441,7 +428,7 @@ async def run_pipeline(limit: int | None = None) -> None:
 
     total_upserted = 0
 
-    # -- Scrape ------------------------------------------------------------
+    # Scrape
     async with async_playwright() as p:
         browser = await p.chromium.launch(
             headless=False,

README.md

@@ -0,0 +1,85 @@
+# Shopwise
+
+A grocery price comparison app. The Instacart scraper collects product data from multiple retailers and stores it in Supabase, which the iOS frontend reads to display prices.
+
+## Project Structure
+
+```
+CS178-Shopwise/
+├── PlayWright/
+│   └── scraper.py          # Instacart scraper -> Supabase
+├── SWFrontUI/              # iOS app (Xcode)
+└── .env                    # Credentials (not committed)
+```
+
+## Prerequisites
+
+- Python 3.11+
+- Xcode 15+ (for the iOS app)
+- A Supabase project with the `taxonomy` and `ingredients` tables (see below)
+
+## Environment Setup
+
+Create a `.env` file at `CS178-Shopwise/.env` with the following:
+
+```
+SUPABASE_URL=your_supabase_project_url
+SUPABASE_SERVICE_KEY=your_supabase_service_role_key
+```
+
+## Supabase Tables
+
+The scraper reads from `taxonomy` and writes to `ingredients`.
+
+**`taxonomy`** — ingredient list that drives all scraping:
+```sql
+create table taxonomy (
+  ingredient text primary key
+);
+```
+
+**`ingredients`** — scraped product results:
+```sql
+create table ingredients (
+  id           text primary key,
+  taxonomy     text,
+  store        text,
+  name         text,
+  price        text,
+  price_unit   text,
+  quantity     text,
+  image_url    text,
+  description  text,
+  out_of_stock boolean
+);
+```
+
+## Running the Scraper
+
+### Install dependencies
+
+```bash
+cd CS178-Shopwise/PlayWright
+pip install playwright python-dotenv supabase
+playwright install chromium
+```
+
+### Full run
+
+Scrapes all ingredients in the taxonomy table and upserts results to Supabase:
+
+```bash
+python scraper.py
+```
+
+### Test run (first N ingredients only)
+
+```bash
+python scraper.py --limit 3
+```
+
+The browser runs in headed mode (visible) so Instacart's session/location cookies persist across queries. Do not close the browser window while the scraper is running.
+
+## iOS App
+
+Open `SWFrontUI/ShopwiseFrontEndUI.xcodeproj` in Xcode and run on a simulator or device. The app reads from the Supabase `ingredients` table.